From c97b74770c85f0b16dd57ca6bfb2a304ac1ee8b5 Mon Sep 17 00:00:00 2001
From: Abhishekpandey123-git <abhishekstock123@gmail.com>
Date: Tue, 27 Oct 2026 21:26:50 +0530
Subject: [PATCH] feat: add explicit llama.cpp provider and benchmark tooling

---
 app/api/routes/llm_config.py                  |  61 +++----
 app/api/schemas/enums.py                      |   1 +
 app/core/config.py                            |   1 +
 app/services/llm/specs.py                     |   4 +
 app/services/llm/user_config.py               |  13 +-
 benchmark/.env.example                        |  11 ++
 .../LLAMA_CPP_COMPARISON_REPORT_TEMPLATE.md   |  91 ++++++++++
 benchmark/README.md                           |  88 ++++++++--
 benchmark/compare_benchmark_runs.py           | 137 +++++++++++++++
 benchmark/profile_runtime.py                  | 160 ++++++++++++++++++
 benchmark/run.py                              |  86 +++++++---
 benchmark/run_info.py                         |  29 +++-
 benchmark/summarize_runtime.py                |  60 +++++++
 benchmark/tests/conftest.py                   |   7 +
 .../tests/test_compare_benchmark_runs.py      |  78 +++++++++
 benchmark/tests/test_profile_runtime.py       |  27 +++
 benchmark/tests/test_summarize_runtime.py     |  40 +++++
 contracts/schemas/enums.yaml                  |   3 +-
 docker/.env.example                           |  13 +-
 docs/multi-llm-support/configuration.md       |  15 +-
 tests/test_llm_registry.py                    |  12 ++
 tests/test_llm_service.py                     |   9 +
 22 files changed, 865 insertions(+), 81 deletions(-)
 create mode 100644 benchmark/LLAMA_CPP_COMPARISON_REPORT_TEMPLATE.md
 create mode 100644 benchmark/compare_benchmark_runs.py
 create mode 100644 benchmark/profile_runtime.py
 create mode 100644 benchmark/summarize_runtime.py
 create mode 100644 benchmark/tests/conftest.py
 create mode 100644 benchmark/tests/test_compare_benchmark_runs.py
 create mode 100644 benchmark/tests/test_profile_runtime.py
 create mode 100644 benchmark/tests/test_summarize_runtime.py

diff --git a/app/api/routes/llm_config.py b/app/api/routes/llm_config.py
index d4e2625..eb7b73b 100644
--- a/app/api/routes/llm_config.py
+++ b/app/api/routes/llm_config.py
@@ -62,9 +62,9 @@ def get_provider_configs(db: Session = Depends(get_db)):
         ui_key = bool(
             record and record.api_key_encrypted and has_decryptable_credential(provider.value)
         )
-        ui_custom_url = bool(provider is LLMProvider.custom and record and record.base_url)
-        source = "ui" if ui_key or ui_custom_url else "env" if env_key or env.get("BASE_URL") else "none"
-        base_url = (record.base_url if ui_custom_url else None) or env.get("BASE_URL") or spec.default_base_url
+        ui_compatible_url = bool(provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record and record.base_url)
+        source = "ui" if ui_key or ui_compatible_url else "env" if env_key or env.get("BASE_URL") else "none"
+        base_url = (record.base_url if ui_compatible_url else None) or env.get("BASE_URL") or spec.default_base_url
         key_hint = record.key_hint if ui_key else env_key[-4:] if env_key else None
         statuses.append(
             LLMProviderCredentialStatus(
@@ -77,10 +77,10 @@ def get_provider_configs(db: Session = Depends(get_db)):
                 credential_status=(
                     "available" if has_decryptable_credential(provider.value) else "unavailable"
                 ) if record and record.api_key_encrypted else None,
-                base_url=_display_base_url(base_url) if provider is LLMProvider.custom else None,
+                base_url=_display_base_url(base_url) if provider in {LLMProvider.custom, LLMProvider.llama_cpp} else None,
                 structured_output=(
                     record.structured_output
-                    if provider is LLMProvider.custom and record and record.structured_output is not None
+                    if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record and record.structured_output is not None
                     else None
                 ),
             )
@@ -100,17 +100,17 @@ def save_provider_config(
     ).first()
     env = config.LLM_PROVIDER_ENV[provider.value]
 
-    if provider is not LLMProvider.custom and body.structured_output is not None:
-        raise LLMNotAvailableError("Only the custom provider accepts a structured_output override.")
-    if provider is not LLMProvider.custom and body.base_url is not None:
-        raise LLMNotAvailableError("Only the custom provider accepts a configurable base URL.")
+    if provider not in {LLMProvider.custom, LLMProvider.llama_cpp} and body.structured_output is not None:
+        raise LLMNotAvailableError("Only OpenAI-compatible custom providers accept a structured_output override.")
+    if provider not in {LLMProvider.custom, LLMProvider.llama_cpp} and body.base_url is not None:
+        raise LLMNotAvailableError("Only OpenAI-compatible custom providers accept a configurable base URL.")
 
-    base_url = body.base_url if provider is LLMProvider.custom and body.base_url is not None else None
-    if provider is LLMProvider.custom:
-        base_url = base_url or (record.base_url if record else None) or env.get("BASE_URL") or None
+    base_url = body.base_url if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and body.base_url is not None else None
+    if provider in {LLMProvider.custom, LLMProvider.llama_cpp}:
+        base_url = base_url or (record.base_url if record else None) or env.get("BASE_URL") or SPECS[provider].default_base_url or None
         if not base_url:
-            raise LLMNotAvailableError("CUSTOM_BASE_URL is required.")
-        base_url = _validate_custom_url(base_url)
+            raise LLMNotAvailableError(f"{SPECS[provider].setting('BASE_URL')} is required.")
+        base_url = _validate_provider_url(base_url, provider)
 
     api_key_was_provided = "api_key" in body.model_fields_set and body.api_key is not None
     api_key = _provided_secret(body.api_key) if api_key_was_provided else None
@@ -129,7 +129,7 @@ def save_provider_config(
         secret = _provided_secret(body.api_key)
         record.api_key_encrypted = encrypt_api_key(secret) if secret else None
         record.key_hint = secret[-4:] if secret else None
-    if provider is LLMProvider.custom:
+    if provider in {LLMProvider.custom, LLMProvider.llama_cpp}:
         record.base_url = base_url
         if body.structured_output is not None:
             record.structured_output = body.structured_output
@@ -186,11 +186,11 @@ def validate_provider_config(
     db: Session = Depends(get_db),
 ):
     validate_provider(provider)
-    if provider is not LLMProvider.custom and body.structured_output is not None:
+    if provider not in {LLMProvider.custom, LLMProvider.llama_cpp} and body.structured_output is not None:
         return LLMProviderValidationResult(
             provider=provider,
             ok=False,
-            detail="Only the custom provider accepts a structured_output override.",
+            detail="Only OpenAI-compatible custom providers accept a structured_output override.",
             error_code="LLM_CONFIG_ERROR",
         )
     api_key = _provided_secret(body.api_key)
@@ -202,16 +202,18 @@ def validate_provider_config(
             error_code="LLM_AUTH_FAILED",
         )
     base_url = body.base_url
-    if provider is LLMProvider.custom:
+    if provider in {LLMProvider.custom, LLMProvider.llama_cpp}:
+        if not base_url:
+            base_url = config.LLM_PROVIDER_ENV[provider.value].get("BASE_URL") or SPECS[provider].default_base_url or None
         if not base_url:
             return LLMProviderValidationResult(
                 provider=provider,
                 ok=False,
-                detail="CUSTOM_BASE_URL is required.",
+                detail=f"{SPECS[provider].setting('BASE_URL')} is required.",
                 error_code="LLM_CONFIG_ERROR",
             )
         try:
-            base_url = _validate_custom_url(base_url)
+            base_url = _validate_provider_url(base_url, provider)
         except LLMError as exc:
             return LLMProviderValidationResult(provider=provider, ok=False, detail=exc.message,
                                                error_code=exc.error_code)
@@ -219,7 +221,7 @@ def validate_provider_config(
         return LLMProviderValidationResult(
             provider=provider,
             ok=False,
-            detail="Only the custom provider accepts a configurable base URL.",
+            detail="Only OpenAI-compatible custom providers accept a configurable base URL.",
             error_code="LLM_CONFIG_ERROR",
         )
 
@@ -242,7 +244,7 @@ def validate_provider_config(
         )
     except LLMError as exc:
         detail = _redact(exc.message, api_key)
-        if provider is LLMProvider.custom and "has no model list" in detail:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and "has no model list" in detail:
             detail = "This server does not expose a model list. Validate a model ID manually."
         return LLMProviderValidationResult(
             provider=provider, ok=False, detail=detail, error_code=exc.error_code
@@ -262,7 +264,7 @@ def validate_and_add_manual_model(
         ).first()
         env = config.LLM_PROVIDER_ENV[provider.value]
         base_url = (
-            existing_config.base_url if provider is LLMProvider.custom and existing_config else None
+            existing_config.base_url if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and existing_config else None
         ) or env.get("BASE_URL") or SPECS[provider].default_base_url
         _require_external_consent(db, provider, base_url)
         candidate = registry_for_candidate(
@@ -317,22 +319,23 @@ def _provided_secret(value: SecretStr | None) -> str:
     return value.get_secret_value().strip() if value else ""
 
 
-def _validate_custom_url(value: str) -> str:
+def _validate_provider_url(value: str, provider: LLMProvider) -> str:
+    setting = SPECS[provider].setting("BASE_URL")
     try:
         parsed = urlparse(value)
     except ValueError:
-        raise LLMNotAvailableError("CUSTOM_BASE_URL must be an HTTP or HTTPS URL with a hostname.") from None
+        raise LLMNotAvailableError(f"{setting} must be an HTTP or HTTPS URL with a hostname.") from None
     if parsed.username is not None or parsed.password is not None:
-        raise LLMNotAvailableError("CUSTOM_BASE_URL must not include embedded credentials.")
+        raise LLMNotAvailableError(f"{setting} must not include embedded credentials.")
     if parsed.query or parsed.fragment:
-        raise LLMNotAvailableError("CUSTOM_BASE_URL must not include query parameters or a fragment.")
+        raise LLMNotAvailableError(f"{setting} must not include query parameters or a fragment.")
     from app.services.llm.settings import _check_url
     from app.services.llm.errors import LLMConfigError
 
     try:
-        return _check_url("CUSTOM_BASE_URL", value)
+        return _check_url(setting, value)
     except LLMConfigError:
-        raise LLMNotAvailableError("CUSTOM_BASE_URL must be an HTTP or HTTPS URL with a hostname.") from None
+        raise LLMNotAvailableError(f"{setting} must be an HTTP or HTTPS URL with a hostname.") from None
 
 
 def _display_base_url(value: str) -> str:
diff --git a/app/api/schemas/enums.py b/app/api/schemas/enums.py
index 61033da..8975282 100644
--- a/app/api/schemas/enums.py
+++ b/app/api/schemas/enums.py
@@ -118,6 +118,7 @@ class OutputFormat(str, Enum):
 
 class LLMProvider(str, Enum):
     ollama = "ollama"
+    llama_cpp = "llama_cpp"
     openai = "openai"
     gemini = "gemini"
     anthropic = "anthropic"
diff --git a/app/core/config.py b/app/core/config.py
index bb85041..1750d3e 100644
--- a/app/core/config.py
+++ b/app/core/config.py
@@ -70,6 +70,7 @@ def _provider_env(prefix: str, **defaults: str) -> dict[str, str]:
     "gemini": _provider_env("GEMINI"),
     "anthropic": _provider_env("ANTHROPIC"),
     "custom": _provider_env("CUSTOM"),
+    "llama_cpp": _provider_env("LLAMA_CPP"),
 }
 
 WHISPER_HOST = os.getenv("WHISPER_HOST", "http://localhost:9000").rstrip("/")
diff --git a/app/services/llm/specs.py b/app/services/llm/specs.py
index 4bcd326..85776cf 100644
--- a/app/services/llm/specs.py
+++ b/app/services/llm/specs.py
@@ -87,6 +87,10 @@ def setting(self, key: str) -> str:
         headers=(("anthropic-version", "2023-06-01"),),
     ),
     LLMProvider.custom: ProviderSpec(LLMProvider.custom, needs_key=False),
+    # llama.cpp serves the same OpenAI-compatible API with a local default.
+    LLMProvider.llama_cpp: ProviderSpec(
+        LLMProvider.llama_cpp, "http://localhost:8080/v1", needs_key=False
+    ),
 }
 
 DEFAULT_OLLAMA_INSTALLABLE_MODELS: tuple[str, ...] = (
diff --git a/app/services/llm/user_config.py b/app/services/llm/user_config.py
index 1422117..f3d6f61 100644
--- a/app/services/llm/user_config.py
+++ b/app/services/llm/user_config.py
@@ -26,6 +26,7 @@
     LLMProvider.gemini,
     LLMProvider.anthropic,
     LLMProvider.custom,
+    LLMProvider.llama_cpp,
 })
 _cache_lock = threading.Lock()
 _credential_state_lock = threading.Lock()
@@ -149,7 +150,7 @@ def validate_provider(provider: LLMProvider) -> None:
     if provider not in EDITABLE_PROVIDERS:
         from app.services.llm.errors import LLMNotAvailableError
 
-        raise LLMNotAvailableError("Only OpenAI, Gemini, Anthropic, and custom providers can be configured here.")
+        raise LLMNotAvailableError("Only OpenAI, Gemini, Anthropic, custom, and llama.cpp providers can be configured here.")
 
 
 def registry_for_candidate(
@@ -259,11 +260,11 @@ def _build_registry(
             unavailable_credentials.add(provider.value)
         if api_key:
             provider_settings["API_KEY"] = api_key
-        if provider is LLMProvider.custom and record.base_url:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record.base_url:
             provider_settings["BASE_URL"] = record.base_url
-        if provider is LLMProvider.custom and record.structured_output is not None:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record.structured_output is not None:
             provider_settings["STRUCTURED_OUTPUT"] = str(record.structured_output).lower()
-        if not provider_settings.get("MODELS") and (api_key or (provider is LLMProvider.custom and record.base_url)):
+        if not provider_settings.get("MODELS") and (api_key or (provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record.base_url)):
             provider_settings["MODELS"] = "*"
 
     if candidate:
@@ -271,9 +272,9 @@ def _build_registry(
         provider_settings = provider_env[provider.value]
         if override["api_key"] is not None:
             provider_settings["API_KEY"] = override["api_key"]
-        if provider is LLMProvider.custom and override["base_url"] is not None:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and override["base_url"] is not None:
             provider_settings["BASE_URL"] = override["base_url"]
-        if provider is LLMProvider.custom and override["structured_output"] is not None:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and override["structured_output"] is not None:
             provider_settings["STRUCTURED_OUTPUT"] = str(override["structured_output"]).lower()
         if not provider_settings.get("MODELS"):
             provider_settings["MODELS"] = "*"
diff --git a/benchmark/.env.example b/benchmark/.env.example
index a55aa35..5fb152e 100644
--- a/benchmark/.env.example
+++ b/benchmark/.env.example
@@ -6,6 +6,17 @@ OLLAMA_HOST=http://localhost:11434
 OLLAMA_MODEL=qwen2.5:1.5b
 OLLAMA_TIMEOUT=300
 
+# --- Optional llama.cpp provider -------------------------------------------
+# The model ID must match llama-server's /v1/models response.
+# Keep these commented to leave the default provider as Ollama.
+# LLM_PROVIDER=llama_cpp
+# LLM_MODEL=your-model-id
+# LLAMA_CPP_BASE_URL=http://localhost:8080/v1
+# LLAMA_CPP_MODELS=your-model-id
+# LLAMA_CPP_API_KEY=
+# LLM_TIMEOUT=300
+# LLM_ALLOW_EXTERNAL=false
+
 # --- LLM judge ------------------------------------------------------------
 # JUDGE_HOST defaults to OLLAMA_HOST.
 # JUDGE_HOST=http://localhost:11434
diff --git a/benchmark/LLAMA_CPP_COMPARISON_REPORT_TEMPLATE.md b/benchmark/LLAMA_CPP_COMPARISON_REPORT_TEMPLATE.md
new file mode 100644
index 0000000..f44e2da
--- /dev/null
+++ b/benchmark/LLAMA_CPP_COMPARISON_REPORT_TEMPLATE.md
@@ -0,0 +1,91 @@
+# FireForm Issue #702 — Ollama vs. llama.cpp comparison
+
+> Fill this report only after the benchmark has completed. Keep raw run folders, `summary.csv`, `run_info.json`, and runtime profile CSVs as evidence. Do not infer missing measurements.
+
+## 1. Reproducibility record
+
+| Item | Value |
+|---|---|
+| Date/time and timezone | TODO |
+| FireForm commit SHA | TODO |
+| OS and kernel | TODO |
+| CPU model / cores | TODO |
+| RAM | TODO |
+| GPU / VRAM (if used) | TODO |
+| Python / dependency versions | From `run_info.json` |
+| ICS documents included | TODO |
+| Number of repetitions per provider | At least 3 |
+| Judge model and host | TODO; keep identical across runs |
+| Prompt SHA-256 | From `run_info.json` |
+
+## 2. Model equivalence and runtime configuration
+
+| Item | Ollama | llama.cpp |
+|---|---|---|
+| Model family and exact weights | TODO | TODO |
+| Artifact SHA-256 / digest | TODO | GGUF SHA-256: TODO |
+| Quantization | TODO | TODO |
+| Context window | TODO | TODO |
+| Server version/build SHA | TODO | TODO |
+| Exact server launch command | TODO | TODO |
+| API/provider config (omit secrets) | TODO | `llama_cpp` / OpenAI-compatible (or `custom` if running before the explicit provider lands) |
+| Warm-up procedure | TODO | TODO |
+
+Confirm that the model weights are equivalent. If exact equivalence is not possible, state this as a limitation and do not attribute all observed differences to the runtime alone.
+
+## 3. Per-run results
+
+Include one row per repetition and provider. Use the same documents and scoring configuration in all runs.
+
+| Provider | Run | Documents scored | Skipped | Total wall time (s) | Median doc latency (s) | Judged accuracy (%) | Populated accuracy (%) | Raw accuracy (%) | Missing fields | Unsupported fields |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| Ollama | 1 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| Ollama | 2 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| Ollama | 3 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| llama.cpp | 1 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| llama.cpp | 2 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| llama.cpp | 3 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+
+## 4. Resource usage
+
+Use the `profile_runtime.py` CSV captured on the model-server host. Record GPU measurements separately if available.
+
+| Metric | Ollama | llama.cpp | Measurement method |
+|---|---:|---:|---|
+| Mean CPU usage (% of one core) | TODO | TODO | profile CSV |
+| Peak CPU usage (% of one core) | TODO | TODO | profile CSV |
+| Peak resident RAM (MiB) | TODO | TODO | profile CSV |
+| Peak GPU VRAM (MiB) | TODO / N/A | TODO / N/A | TODO |
+
+## 5. Validity checks and limitations
+
+- Same FireForm commit and prompt: TODO
+- Same document set and narrative order: TODO
+- Same judge model/settings: TODO
+- Same weights/quantization: TODO
+- Any skipped documents or failed judge calls: TODO
+- Cold-start versus warm-run behavior: TODO
+- Remote server or unmeasured GPU resource limits: TODO
+- Other confounding factors: TODO
+
+Runs with different skipped-document sets must not be compared as if they cover the same workload. Report failed or unavailable judge results instead of treating them as correct.
+
+## 6. Summary and recommendation
+
+**Latency:** TODO (report fill-phase median and variation across repeated runs, plus end-to-end document time; do not rely only on the fastest run).
+
+**Accuracy:** TODO (state any regression or uncertainty).
+
+**Resource use:** TODO.
+
+**Integration / maintenance costs:** TODO.
+
+**Recommendation:** Adopt llama.cpp / keep Ollama / gather more evidence. Explain the trade-off and the exact scope of the recommendation. Do not recommend a runtime solely from a single run or from latency if accuracy regresses.
+
+## 7. Artifacts
+
+- Ollama run folders: TODO
+- llama.cpp run folders: TODO
+- Runtime profile CSVs: TODO
+- `run_info.json` files: TODO
+- Commands used: TODO
diff --git a/benchmark/README.md b/benchmark/README.md
index 7d54593..d3b90be 100644
--- a/benchmark/README.md
+++ b/benchmark/README.md
@@ -2,13 +2,15 @@
 
 Runs `app/services/form_filler` on the ICS narratives and scores the filled
 fields against hand-written ground truth, with an LLM judge for answers that
-are right but worded differently.
+are right but worded differently. The same runner can select any configured
+FireForm LLM provider, including an OpenAI-compatible `custom` endpoint such
+as a local llama.cpp server.
 
 ## Setup
 
 ```bash
 pip install -r requirements.txt
-cp benchmark/.env.example benchmark/.env   # then set OLLAMA_HOST etc.
+cp benchmark/.env.example benchmark/.env   # set the model and judge settings
 ```
 
 `benchmark/.env` is gitignored, so a private server address stays local.
@@ -19,23 +21,80 @@ Settings (see `config.py` and `.env.example`):
 | `OLLAMA_HOST` | `http://localhost:11434` | model under test (read by `app.core.config`) |
 | `OLLAMA_MODEL` | `qwen2.5:1.5b` | model under test |
 | `OLLAMA_TIMEOUT` | `300` | seconds per fill request |
+| `LLM_PROVIDER` | `ollama` | default provider, when `--provider` is omitted |
+| `LLM_MODEL` | provider default | default model when configured for the default provider |
+| `CUSTOM_BASE_URL` | — | OpenAI-compatible server URL, e.g. `http://localhost:8080/v1` |
+| `CUSTOM_API_KEY` | — | key expected by the custom server; use its documented local value |
+| `CUSTOM_MODELS` | — | comma-separated model IDs allowed for the custom provider |
+| `LLAMA_CPP_BASE_URL` | `http://localhost:8080/v1` | llama-server OpenAI-compatible endpoint |
+| `LLAMA_CPP_API_KEY` | empty | optional key if configured by the server |
+| `LLAMA_CPP_MODELS` | — | comma-separated llama.cpp model IDs allowed |
+| `LLM_TIMEOUT` | `OLLAMA_TIMEOUT` | request timeout used by provider adapters |
 | `JUDGE_HOST` | `OLLAMA_HOST` | LLM judge |
 | `JUDGE_MODEL` | `qwen3.5:4b` | LLM judge |
 | `JUDGE_TIMEOUT_SECONDS` | `300` | seconds per judge request |
 
-Shell environment variables override `.env`. Both models must be pulled on
-their Ollama server (`ollama pull qwen2.5:1.5b`, `ollama pull qwen3.5:4b`).
+Shell environment variables override `.env`. The judge still uses the existing
+Ollama-native endpoint; keep `JUDGE_HOST` pointed at the same separate Ollama judge
+server for both runs so scoring is consistent and judge load does not skew target-server
+CPU/RAM measurements. The Ollama model must
+be pulled on its server. For llama.cpp, start its OpenAI-compatible server first
+and set `CUSTOM_BASE_URL`, `CUSTOM_MODELS`, and (if required) `CUSTOM_API_KEY`.
 
 ## Running
 
 ```bash
 cd benchmark
-python3 run.py                          # all 56 narratives (~10 min on a GPU server)
-python3 run.py -d ics_203_1 ics_206_1   # a subset
-python3 run.py -m qwen3.5:4b            # another model under test
-python3 run.py --baseline               # full run, then publish it (see Baselines)
+python3 run.py                                      # configured provider/model, all narratives
+python3 run.py -d ics_203_1 ics_206_1              # same subset for each provider
+python3 run.py --provider ollama -m qwen2.5:1.5b   # Ollama run
+python3 run.py --provider llama_cpp -m your-model-id  # explicit llama.cpp provider
+python3 run.py --provider llama_cpp -m your-model-id -d ics_203_1 ics_206_1
+python3 run.py --baseline --provider ollama        # full run, then publish baseline
 ```
 
+Use exactly the same `-d` document list for each provider. For a fair runtime
+comparison, pin the same model weights and quantization where possible, use the
+same FireForm commit and prompt, keep the judge model/configuration fixed, and
+run multiple repetitions. llama.cpp's `/v1/models` name must match the `-m`
+argument and be allowed by `LLAMA_CPP_MODELS`. Do not compare results from runs
+that skipped different documents without reporting those skips.
+
+### CPU and memory profiling (Linux, on the model server)
+
+`run.py` records both fill-phase wall time (schema construction, inference and PDF
+writing) and total document-processing wall time in `summary.csv`. CPU/RAM belong to the
+inference server process, not the Python benchmark process, so sample the server
+on the machine where it runs. The helper writes a time-series CSV to the
+gitignored `benchmark/results/` directory:
+
+```bash
+python3 benchmark/profile_runtime.py --label ollama \
+  --process-regex 'ollama' -- \
+  python3 benchmark/run.py --provider ollama -m qwen2.5:1.5b -d ics_203_1 ics_206_1
+
+python3 benchmark/profile_runtime.py --label llama-cpp \
+  --process-regex 'llama-server' -- \
+  python3 benchmark/run.py --provider llama_cpp -m your-model-id -d ics_203_1 ics_206_1
+```
+
+Run the profiler on the same host as the server. It samples matching Linux
+processes' combined RSS and CPU usage (100% means one fully used CPU core).
+If the server is remote, collect its CPU/RAM on that machine separately. Record
+the hardware, OS, server command/build, GGUF SHA-256, context size, quantization,
+warm/cold state, and profile CSV location in the report. GPU VRAM is not exposed
+by `/proc`; collect it separately with the platform's GPU monitoring tools.
+
+Summarize profiler samples and compare provider summaries with these helpers:
+
+```bash
+python3 benchmark/summarize_runtime.py benchmark/results/runtime-profile_*.csv
+python3 benchmark/compare_benchmark_runs.py path/to/ollama/summary.csv path/to/llama_cpp/summary.csv --output benchmark/results/comparison.csv
+```
+
+`run_info.json` also records scored and attempted documents per minute. The comparison
+helper reports skipped documents and differing success sets.
+
 A document whose model call fails or times out, or whose output is cut off
 mid-JSON (runaway generation hitting `num_predict`), is skipped and listed;
 it is left out of the averages.
@@ -46,7 +105,7 @@ Each run gets its own folder, `results/<date>_<time>_<model>/` (gitignored):
 
 | File | Contents |
 |---|---|
-| `summary.csv` | one row per document: judged, populated and raw accuracy, status |
+| `summary.csv` | one row per document: provider/model, elapsed wall time, judged, populated and raw accuracy, status |
 | `wrong_fields.txt` | every wrong field, expected vs got, with the score before and after the judge |
 | `run.log` | everything printed during the run |
 | `run_info.json` | how to recreate the run (see below) |
@@ -65,8 +124,8 @@ Every run records what produced it:
 - **code**: git commit, branch, and any uncommitted changes under `app/`,
   `benchmark/` or `requirements.txt` (a run is only reproducible from its
   commit when that list is empty)
-- **model_under_test**: model name, Ollama digest (the exact weights), size and
-  quantization, the generation options, timeout, and the prompt's SHA-256
+- **model_under_test**: provider/model, Ollama digest and quantization when
+  available, generation settings, timeout, and the prompt's SHA-256
   (the prompt itself is saved next to it as `prompt.txt`)
 - **judge**: the same for the judge model, plus its threshold
 - **ollama_version**, Python and package versions, platform
@@ -85,9 +144,10 @@ prompt, template change or model), run the full benchmark with `--baseline`:
 python3 run.py --baseline
 ```
 
-It overwrites `baselines/<model>/` with that run's `summary.csv`,
-`wrong_fields.txt`, `run_info.json` and `prompt.txt`, which are small text
-files meant to be committed with the change. The diff then shows the effect
+It overwrites `baselines/<model>/` for Ollama runs and `baselines/<provider>--<model>/`
+for other providers with that run's `summary.csv`, `wrong_fields.txt`, `run_info.json`
+and `prompt.txt`. Use `benchmark/LLAMA_CPP_COMPARISON_REPORT_TEMPLATE.md` to record
+repeated results, model equivalence, hardware, and the adoption recommendation. The diff then shows the effect
 on every document, and the folder's git history is the record of how
 accuracy changed over time. The large per-document files (PDFs, raw
 responses) are not published; attach a zipped run folder to the PR if
diff --git a/benchmark/compare_benchmark_runs.py b/benchmark/compare_benchmark_runs.py
new file mode 100644
index 0000000..afc12ae
--- /dev/null
+++ b/benchmark/compare_benchmark_runs.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""Compare two FireForm benchmark summary.csv files on the same documents.
+
+Example:
+  python3 benchmark/compare_benchmark_runs.py results/ollama/summary.csv results/llama_cpp/summary.csv
+  python3 benchmark/compare_benchmark_runs.py ollama.csv llama.csv --output comparison.csv
+
+Time/throughput comparisons are only meaningful when the provider runs use the
+same FireForm commit, prompts, document set, judge settings, hardware, and
+comparable model weights/quantization. Token/s is not reported because the
+current form-filler API returns text, not provider token-usage counters.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import statistics
+from pathlib import Path
+
+NUMERIC = (
+    "fill_wall_seconds", "document_wall_seconds", "total_accuracy_percent",
+    "populated_accuracy_percent", "total_raw_accuracy_percent",
+    "populated_raw_accuracy_percent", "missing_count", "unsupported_count",
+)
+
+
+def read_run(path: Path) -> tuple[dict[str, dict], dict]:
+    with path.open(newline="", encoding="utf-8") as stream:
+        reader = csv.DictReader(stream)
+        required = {"document", "provider", "model", "status", *NUMERIC}
+        missing = required - set(reader.fieldnames or ())
+        if missing:
+            raise ValueError(f"{path}: missing columns: {', '.join(sorted(missing))}")
+        rows = {row["document"]: row for row in reader}
+    if not rows:
+        raise ValueError(f"{path}: no document rows")
+    statuses = list(rows.values())
+    scored = [row for row in statuses if row.get("status") == "ok"]
+    times = [float(row["document_wall_seconds"]) for row in scored if row.get("document_wall_seconds")]
+    fill_times = [float(row["fill_wall_seconds"]) for row in scored if row.get("fill_wall_seconds")]
+    accuracy = [float(row["total_accuracy_percent"]) for row in scored if row.get("total_accuracy_percent")]
+    summary = {
+        "provider": statuses[0].get("provider", "unknown"),
+        "model": statuses[0].get("model", "unknown"),
+        "documents_requested": len(rows),
+        "documents_scored": len(scored),
+        "documents_skipped": len(rows) - len(scored),
+        "mean_document_seconds": statistics.fmean(times) if times else None,
+        "median_document_seconds": statistics.median(times) if times else None,
+        "mean_fill_seconds": statistics.fmean(fill_times) if fill_times else None,
+        "scored_docs_per_minute": (len(scored) * 60 / sum(times)) if times and sum(times) > 0 else None,
+        "mean_judged_accuracy_percent": statistics.fmean(accuracy) if accuracy else None,
+    }
+    return rows, summary
+
+
+def compare_runs(reference_path: Path, candidate_path: Path) -> tuple[dict, list[dict]]:
+    reference_rows, reference = read_run(reference_path)
+    candidate_rows, candidate = read_run(candidate_path)
+    shared = sorted(reference_rows.keys() & candidate_rows.keys())
+    only_reference = sorted(reference_rows.keys() - candidate_rows.keys())
+    only_candidate = sorted(candidate_rows.keys() - reference_rows.keys())
+    if not shared:
+        raise ValueError("The runs have no documents in common")
+    diffs = []
+    for doc in shared:
+        old, new = reference_rows[doc], candidate_rows[doc]
+        old_ok, new_ok = old.get("status") == "ok", new.get("status") == "ok"
+        row = {
+            "document": doc,
+            "reference_provider": old.get("provider"),
+            "reference_model": old.get("model"),
+            "reference_status": old.get("status"),
+            "candidate_provider": new.get("provider"),
+            "candidate_model": new.get("model"),
+            "candidate_status": new.get("status"),
+        }
+        for field in NUMERIC:
+            a = float(old[field]) if old_ok and old.get(field) not in (None, "") else None
+            b = float(new[field]) if new_ok and new.get(field) not in (None, "") else None
+            row[f"reference_{field}"] = a
+            row[f"candidate_{field}"] = b
+            row[f"delta_{field}"] = (b - a) if a is not None and b is not None else None
+        diffs.append(row)
+    comparison = {
+        "reference": reference,
+        "candidate": candidate,
+        "common_documents": len(shared),
+        "only_reference": only_reference,
+        "only_candidate": only_candidate,
+        "same_success_set": not only_reference and not only_candidate and all(
+            (reference_rows[d].get("status") == "ok") == (candidate_rows[d].get("status") == "ok") for d in shared
+        ),
+    }
+    return comparison, diffs
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("reference", type=Path, help="baseline summary.csv, normally Ollama")
+    parser.add_argument("candidate", type=Path, help="comparison summary.csv, normally llama.cpp")
+    parser.add_argument("--output", type=Path, help="write per-document differences to this CSV")
+    args = parser.parse_args()
+    try:
+        summary, diffs = compare_runs(args.reference, args.candidate)
+    except (OSError, ValueError, KeyError) as exc:
+        parser.error(str(exc))
+    for label in ("reference", "candidate"):
+        item = summary[label]
+        print(f"\n{label.title()}: {item['provider']} / {item['model']}")
+        print(f"  scored / requested / skipped: {item['documents_scored']} / {item['documents_requested']} / {item['documents_skipped']}")
+        for key, title, unit in (
+            ("median_document_seconds", "median document time", "s"),
+            ("mean_document_seconds", "mean document time", "s"),
+            ("scored_docs_per_minute", "scored throughput", "docs/min"),
+            ("mean_judged_accuracy_percent", "mean judged accuracy", "%"),
+        ):
+            value = item[key]
+            print(f"  {title}: {value:.2f} {unit}" if value is not None else f"  {title}: unavailable")
+    print(f"\nCommon documents: {summary['common_documents']}; same successful-document set: {summary['same_success_set']}")
+    if summary["only_reference"] or summary["only_candidate"]:
+        print(f"WARNING: document lists differ. Only reference: {summary['only_reference']}; only candidate: {summary['only_candidate']}")
+    if not summary["same_success_set"]:
+        print("WARNING: at least one document succeeded in only one run; aggregate accuracy differences may be biased.")
+    if args.output:
+        args.output.parent.mkdir(parents=True, exist_ok=True)
+        fieldnames = list(diffs[0]) if diffs else ["document"]
+        with args.output.open("w", newline="", encoding="utf-8") as stream:
+            writer = csv.DictWriter(stream, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(diffs)
+        print(f"Per-document differences: {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/benchmark/profile_runtime.py b/benchmark/profile_runtime.py
new file mode 100644
index 0000000..7f72529
--- /dev/null
+++ b/benchmark/profile_runtime.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Sample matching Linux inference-server processes while a command runs.
+
+Example:
+  python3 benchmark/profile_runtime.py --label llama-cpp \\
+    --process-regex 'llama-server' -- python3 benchmark/run.py --provider custom -m MODEL
+
+Run this on the same host as the inference server. CPU is reported as percent
+of one core (so 100 means one fully utilized core); RSS is combined resident
+memory across matching processes. This intentionally uses /proc only and has
+no third-party dependencies.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import os
+import re
+import signal
+import subprocess
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+DEFAULT_RESULTS = HERE / "results"
+# os.sysconf is available on Linux; the fallback keeps pure helper tests importable on Windows.
+# Runtime profiling itself is Linux-only because process sampling reads /proc.
+CLOCK_TICKS = os.sysconf("SC_CLK_TCK") if hasattr(os, "sysconf") else 100
+
+
+def parse_proc_stat(stat_text: str) -> tuple[int, int]:
+    """Return (pid, user+system CPU ticks) from a /proc/<pid>/stat line."""
+    close = stat_text.rfind(")")
+    if close < 0:
+        raise ValueError("Malformed /proc stat line: missing command delimiter")
+    pid = int(stat_text[: stat_text.find(" ")])
+    fields_after_comm = stat_text[close + 1 :].split()
+    # fields_after_comm begins at field 3 (state); utime/stime are fields 14/15.
+    if len(fields_after_comm) <= 12:
+        raise ValueError("Malformed /proc stat line: missing CPU tick fields")
+    return pid, int(fields_after_comm[11]) + int(fields_after_comm[12])
+
+
+def cpu_percent(previous_ticks: int | None, current_ticks: int, elapsed: float) -> float:
+    """CPU usage since the prior sample, in one-core percentage units."""
+    if previous_ticks is None or elapsed <= 0:
+        return 0.0
+    return max(0.0, (current_ticks - previous_ticks) / CLOCK_TICKS / elapsed * 100.0)
+
+
+def read_process(pid_dir: Path, pattern: re.Pattern[str]) -> dict | None:
+    """Read a process if its comm name matches the requested regex."""
+    try:
+        pid = int(pid_dir.name)
+        comm = (pid_dir / "comm").read_text(encoding="utf-8").strip()
+        if not pattern.search(comm):
+            return None
+        stat_text = (pid_dir / "stat").read_text(encoding="utf-8")
+        stat_pid, ticks = parse_proc_stat(stat_text)
+        if stat_pid != pid:
+            return None
+        rss_kib = 0
+        for line in (pid_dir / "status").read_text(encoding="utf-8").splitlines():
+            if line.startswith("VmRSS:"):
+                rss_kib = int(line.split()[1])
+                break
+        return {"pid": pid, "name": comm, "ticks": ticks, "rss_kib": rss_kib}
+    except (OSError, ValueError, IndexError):
+        # A process can exit between scanning /proc and reading its files.
+        return None
+
+
+def sample_processes(pattern: re.Pattern[str], excluded_pids: set[int]) -> list[dict]:
+    matches = []
+    try:
+        entries = list(Path("/proc").iterdir())
+    except OSError as exc:
+        raise RuntimeError("This helper requires Linux /proc") from exc
+    for entry in entries:
+        if not entry.name.isdigit() or int(entry.name) in excluded_pids:
+            continue
+        process = read_process(entry, pattern)
+        if process is not None:
+            matches.append(process)
+    return matches
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--label", required=True, help="label used in the output filename")
+    parser.add_argument("--process-regex", required=True, help="regex matched against Linux process comm names")
+    parser.add_argument("--interval", type=float, default=1.0, help="sampling interval in seconds (default: 1)")
+    parser.add_argument("--output", help="CSV path (default: benchmark/results/runtime-profile_<label>_<time>.csv)")
+    parser.add_argument("command", nargs=argparse.REMAINDER, help="command to run, prefixed by --")
+    args = parser.parse_args(argv)
+    command = args.command[1:] if args.command and args.command[0] == "--" else args.command
+    if not command:
+        parser.error("provide a command after --")
+    if args.interval <= 0:
+        parser.error("--interval must be greater than zero")
+
+    if not Path("/proc").is_dir():
+        parser.error("runtime profiling requires Linux /proc; run this helper on the model-server host")
+
+    pattern = re.compile(args.process_regex)
+    stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    safe_label = re.sub(r"[^A-Za-z0-9_.-]+", "_", args.label)
+    output_path = Path(args.output) if args.output else DEFAULT_RESULTS / f"runtime-profile_{safe_label}_{stamp}.csv"
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+
+    child = subprocess.Popen(command)
+    own_pids = {os.getpid(), child.pid}
+    previous_ticks: dict[int, int] = {}
+    last_sample = time.monotonic()
+    print(f"Sampling process regex {args.process_regex!r} -> {output_path}", flush=True)
+    try:
+        with output_path.open("w", newline="", encoding="utf-8") as stream:
+            writer = csv.DictWriter(
+                stream,
+                fieldnames=("timestamp_utc", "elapsed_seconds", "matched_processes", "cpu_percent_one_core", "rss_mib", "processes"),
+            )
+            writer.writeheader()
+            started = time.monotonic()
+            while child.poll() is None:
+                now = time.monotonic()
+                processes = sample_processes(pattern, own_pids)
+                elapsed_sample = max(now - last_sample, 1e-9)
+                total_ticks = sum(
+                    max(0, proc["ticks"] - previous_ticks.get(proc["pid"], proc["ticks"]))
+                    for proc in processes
+                )
+                total_rss_kib = sum(proc["rss_kib"] for proc in processes)
+                writer.writerow({
+                    "timestamp_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+                    "elapsed_seconds": round(now - started, 3),
+                    "matched_processes": len(processes),
+                    "cpu_percent_one_core": round(total_ticks / CLOCK_TICKS / elapsed_sample * 100.0, 2),
+                    "rss_mib": round(total_rss_kib / 1024, 2),
+                    "processes": ";".join(f"{p['name']}:{p['pid']}" for p in processes),
+                })
+                stream.flush()
+                previous_ticks = {proc["pid"]: proc["ticks"] for proc in processes}
+                last_sample = now
+                time.sleep(args.interval)
+    except KeyboardInterrupt:
+        child.send_signal(signal.SIGINT)
+        child.wait()
+        raise
+    finally:
+        if child.poll() is None:
+            child.terminate()
+            child.wait()
+    print(f"Command exited with {child.returncode}; profile saved to {output_path}")
+    return child.returncode
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/benchmark/run.py b/benchmark/run.py
index 446faaa..69e3740 100644
--- a/benchmark/run.py
+++ b/benchmark/run.py
@@ -2,10 +2,11 @@
 
     python3 run.py                          # every narrative in data/narratives
     python3 run.py -d ics_203_1 ics_206_1   # a subset
-    python3 run.py -m qwen3.5:4b            # another Ollama model
+    python3 run.py --provider custom -m <model-id>  # an OpenAI-compatible server
     python3 run.py --baseline               # full run, then publish it to baselines/<model>/
 
-Servers and models come from benchmark/.env (see .env.example and config.py).
+Provider settings and models come from benchmark/.env (see .env.example and config.py).
+Use --provider to choose a provider without changing the global default.
 
 Each run gets its own folder, results/<date>_<time>_<model>/:
 
@@ -36,8 +37,10 @@
 
 import config  # noqa: F401  imported for its side effect: loads benchmark/.env before `app`
 
-from app.core.config import OLLAMA_HOST, OLLAMA_MODEL
+from app.core.config import LLM_PROVIDER
+from app.services import llm
 from app.services.form_filler import filler
+from app.services.llm.errors import LLMError
 import compare
 import report
 import review_pdf
@@ -60,7 +63,7 @@ def all_docs():
     return sorted(names, key=lambda d: (d.rsplit("_", 1)[0], int(d.rsplit("_", 1)[1])))
 
 
-def run_doc(doc, run_dir, model):
+def run_doc(doc, run_dir, model, provider, timings):
     """Fill one document, save every step, score it. Returns (result, entries)."""
     form = doc.rsplit("_", 1)[0]
     pdf_path = os.path.join(PDF_DIR, form + ".pdf")
@@ -73,7 +76,14 @@ def run_doc(doc, run_dir, model):
         narrative = f.read()
 
     # 1. Fill
-    out = filler.fill(pdf_path, narrative, os.path.join(doc_dir, "filled.pdf"), model)
+    fill_start = time.perf_counter()
+    try:
+        out = filler.fill(
+            pdf_path, narrative, os.path.join(doc_dir, "filled.pdf"), model, provider=provider
+        )
+    finally:
+        # Includes schema construction, provider inference, and PDF writing.
+        timings.setdefault(doc, {})["fill_wall_seconds"] = round(time.perf_counter() - fill_start, 3)
     save_text(doc_dir, "prompt.txt", out["prompt"])
     save_json(doc_dir, "schema.json", out["schema"])
     save_text(doc_dir, "response.json", out["response_text"])
@@ -104,7 +114,16 @@ def run_doc(doc, run_dir, model):
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument("-m", "--model", default=OLLAMA_MODEL, help="Ollama model")
+    parser.add_argument(
+        "-p", "--provider",
+        choices=("ollama", "openai", "gemini", "anthropic", "custom", "llama_cpp"),
+        default=LLM_PROVIDER,
+        help="LLM provider (default: configured LLM_PROVIDER)",
+    )
+    parser.add_argument(
+        "-m", "--model", default=None,
+        help="provider model ID (default: the provider's configured default model)",
+    )
     parser.add_argument("-d", "--docs", nargs="*", help="documents to run (default: all)")
     parser.add_argument("--baseline", action="store_true",
                         help="publish this run to baselines/<model>/ (full runs only)")
@@ -113,23 +132,33 @@ def main():
         parser.error("--baseline publishes the full benchmark; drop -d")
     docs = args.docs or all_docs()
 
+    # Resolve the exact provider/model pair once, so metadata and outputs describe
+    # the target actually used and a misconfigured provider fails before the run.
+    try:
+        choice = llm.get_registry().resolve(args.provider, args.model)
+    except LLMError as exc:
+        parser.error(exc.message)
+    provider, model = choice.provider.value, choice.model
+
     stamp = time.strftime("%Y-%m-%d_%H%M%S")
-    run_dir = os.path.join(RESULTS_DIR, f"{stamp}_{safe_name(args.model)}")
+    run_dir = os.path.join(RESULTS_DIR, f"{stamp}_{provider}_{safe_name(model)}")
     os.makedirs(run_dir, exist_ok=True)
     sys.stdout = Tee(os.path.join(run_dir, "run.log"))
-    print(f"Model: {args.model} @ {OLLAMA_HOST}   Documents: {len(docs)}   Output: {run_dir}")
+    # Do not print provider URLs; they may contain internal hostnames.
+    print(f"Provider: {provider}   Model: {model}   Documents: {len(docs)}   Output: {run_dir}")
 
-    info = run_info.collect(args.model, docs, sys.argv[1:])
+    info = run_info.collect(provider, model, docs, sys.argv[1:])
     save_text(run_dir, "prompt.txt", filler.PROMPT)
 
     scored = []    # (doc, result, entries)
     skipped = []   # (doc, reason)
+    timings = {}   # elapsed wall seconds for every document, including failures
     start = time.perf_counter()
     for doc in docs:
         print(f" --- {doc} --- ")
         doc_start = time.perf_counter()
         try:
-            result, entries = run_doc(doc, run_dir, args.model)
+            result, entries = run_doc(doc, run_dir, model, provider, timings)
         except requests.exceptions.Timeout:
             skipped.append((doc, "model call timed out"))
             print(f"SKIPPED {doc}: model call timed out")
@@ -139,28 +168,36 @@ def main():
             skipped.append((doc, f"model call failed: {e}"))
             print(f"SKIPPED {doc}: model call failed: {e}")
             continue
+        except LLMError as e:
+            # Provider adapters normalize provider failures into LLMError.
+            skipped.append((doc, f"model call failed: {e.message}"))
+            print(f"SKIPPED {doc}: model call failed: {e.message}")
+            continue
         except json.JSONDecodeError:
             # num_predict cut off a runaway generation mid-JSON
             skipped.append((doc, "output cut off (runaway generation)"))
             print(f"SKIPPED {doc}: model output was not complete JSON")
             continue
+        finally:
+            timings.setdefault(doc, {})["document_wall_seconds"] = round(time.perf_counter() - doc_start, 3)
         scored.append((doc, result, entries))
-        print(f"{time.perf_counter() - doc_start:.1f}s")
+        print(f'{timings[doc]["document_wall_seconds"]:.3f}s')
 
     seconds = time.perf_counter() - start
-    write_summary(os.path.join(run_dir, "summary.csv"), args.model, scored, skipped)
-    report.write(os.path.join(run_dir, "wrong_fields.txt"), args.model, scored, skipped)
+    write_summary(os.path.join(run_dir, "summary.csv"), provider, model, scored, skipped, timings)
+    report.write(os.path.join(run_dir, "wrong_fields.txt"), f"{provider}/{model}", scored, skipped)
     save_json(run_dir, "run_info.json", run_info.add_results(info, scored, skipped, seconds))
     print_averages(scored, skipped)
     print(f"Total time: {seconds:.0f}s   Results: {run_dir}")
 
     if args.baseline:
-        publish_baseline(run_dir, args.model)
+        publish_baseline(run_dir, provider, model)
 
 
-def publish_baseline(run_dir, model):
-    """Overwrite baselines/<model>/ with this run's text results; git shows what changed."""
-    baseline_dir = os.path.join(BASELINES_DIR, safe_name(model))
+def publish_baseline(run_dir, provider, model):
+    """Publish compact results without overwriting another provider's baseline."""
+    key = safe_name(model) if provider == "ollama" else f"{provider}--{safe_name(model)}"
+    baseline_dir = os.path.join(BASELINES_DIR, key)
     os.makedirs(baseline_dir, exist_ok=True)
     for name in BASELINE_FILES:
         shutil.copy(os.path.join(run_dir, name), os.path.join(baseline_dir, name))
@@ -171,24 +208,31 @@ def safe_name(model):
     return model.replace(":", "-").replace("/", "_")
 
 
-def write_summary(path, model, scored, skipped):
+def write_summary(path, provider, model, scored, skipped, timings):
     with open(path, "w", encoding="utf-8", newline="") as f:
         writer = csv.writer(f)
         writer.writerow([
-            "document", "model", "status",
+            "document", "provider", "model", "status", "fill_wall_seconds", "document_wall_seconds",
             "total_accuracy_percent", "populated_accuracy_percent",
             "total_raw_accuracy_percent", "populated_raw_accuracy_percent",
             "missing_count", "unsupported_count",
         ])
         for doc, r, _ in scored:
             writer.writerow([
-                doc, model, "ok",
+                doc, provider, model, "ok",
+                timings.get(doc, {}).get("fill_wall_seconds"),
+                timings.get(doc, {}).get("document_wall_seconds"),
                 round(r["accuracy"] * 100, 2), round(r["populated_accuracy"] * 100, 2),
                 round(r["raw_accuracy"] * 100, 2), round(r["populated_raw_accuracy"] * 100, 2),
                 r["missing_count"], r["unsupported_count"],
             ])
         for doc, reason in skipped:
-            writer.writerow([doc, model, f"skipped: {reason}", "", "", "", "", "", ""])
+            writer.writerow([
+                doc, provider, model, f"skipped: {reason}",
+                timings.get(doc, {}).get("fill_wall_seconds"),
+                timings.get(doc, {}).get("document_wall_seconds"),
+                "", "", "", "", "", ""
+            ])
 
 
 def print_averages(scored, skipped):
diff --git a/benchmark/run_info.py b/benchmark/run_info.py
index 4ee9b23..4162371 100644
--- a/benchmark/run_info.py
+++ b/benchmark/run_info.py
@@ -15,7 +15,8 @@
 import requests
 
 import llm_judge
-from app.core.config import OLLAMA_HOST, OLLAMA_TIMEOUT
+from app.core.config import OLLAMA_HOST
+from app.services import llm
 from app.services.form_filler import filler
 from app.services.llm.ollama import OPTIONS
 
@@ -23,17 +24,31 @@
 PACKAGES = ["pypdf", "pdfplumber", "requests"]
 
 
-def collect(model, docs, argv):
+def collect(provider, model, docs, argv):
     """Settings known before the run starts."""
+    is_ollama = provider == "ollama"
+    provider_settings = llm.get_registry().settings.providers[llm.LLMProvider(provider)]
+    model_artifact = ollama_model_info(OLLAMA_HOST, model) if is_ollama else {
+        "digest": None,
+        "details": None,
+        "note": "For a local llama.cpp run, record the GGUF SHA-256 and llama.cpp build in the report.",
+    }
     return {
         "date": time.strftime("%Y-%m-%d %H:%M:%S %z"),
         "command": "python3 run.py " + " ".join(argv),
         "code": git_state(),
         "model_under_test": {
+            "provider": provider,
             "model": model,
-            **ollama_model_info(OLLAMA_HOST, model),
-            "options": OPTIONS,
-            "timeout_seconds": OLLAMA_TIMEOUT,
+            **model_artifact,
+            "options": OPTIONS if is_ollama else {
+                "transport": "OpenAI-compatible API",
+                "temperature": provider_settings.temperature,
+                "structured_output": provider_settings.structured_output,
+                "max_tokens": provider_settings.max_tokens,
+                "settings": "record server flags in the benchmark report",
+            },
+            "timeout_seconds": provider_settings.timeout,
             "prompt_file": "prompt.txt",
             "prompt_sha256": sha256(filler.PROMPT),
         },
@@ -44,7 +59,7 @@ def collect(model, docs, argv):
             "timeout_seconds": llm_judge.JUDGE_TIMEOUT_SECONDS,
             "threshold": llm_judge.JUDGE_THRESHOLD,
         },
-        "ollama_version": ollama_version(OLLAMA_HOST),
+        "ollama_version": ollama_version(OLLAMA_HOST) if is_ollama else None,
         "environment": {
             "python": platform.python_version(),
             "platform": platform.platform(),
@@ -66,6 +81,8 @@ def add_results(info, scored, skipped, seconds):
         ),
         "average_raw_accuracy": round(sum(r["raw_accuracy"] for _, r, _ in scored) / n * 100, 2) if n else None,
         "total_seconds": round(seconds),
+        "scored_documents_per_minute": round(n * 60 / seconds, 3) if seconds > 0 else None,
+        "attempted_documents_per_minute": round((n + len(skipped)) * 60 / seconds, 3) if seconds > 0 else None,
     }
     return info
 
diff --git a/benchmark/summarize_runtime.py b/benchmark/summarize_runtime.py
new file mode 100644
index 0000000..b738fd1
--- /dev/null
+++ b/benchmark/summarize_runtime.py
@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+"""Summarize profiler CSVs written by benchmark/profile_runtime.py.
+
+Example:
+  python3 benchmark/summarize_runtime.py benchmark/results/runtime-profile_*.csv
+
+CPU percentages are fractions of one fully occupied core; values can exceed
+100% on multi-core inference servers. RSS is summed across matching processes.
+"""
+from __future__ import annotations
+
+import argparse
+import csv
+import statistics
+from pathlib import Path
+
+
+def summarize(path: Path) -> dict:
+    with path.open(newline="", encoding="utf-8") as stream:
+        rows = list(csv.DictReader(stream))
+    if not rows:
+        raise ValueError(f"{path}: no samples found (the command may have exited before the first sample)")
+    cpu = [float(row["cpu_percent_one_core"]) for row in rows if row.get("cpu_percent_one_core") not in (None, "")]
+    rss = [float(row["rss_mib"]) for row in rows if row.get("rss_mib") not in (None, "")]
+    elapsed = [float(row["elapsed_seconds"]) for row in rows if row.get("elapsed_seconds") not in (None, "")]
+    return {
+        "file": str(path),
+        "samples": len(rows),
+        "duration_seconds": max(elapsed) if elapsed else None,
+        "mean_cpu_percent_one_core": statistics.fmean(cpu) if cpu else None,
+        "peak_cpu_percent_one_core": max(cpu) if cpu else None,
+        "peak_rss_mib": max(rss) if rss else None,
+        "max_matched_processes": max((int(row.get("matched_processes") or 0) for row in rows), default=0),
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("csv", nargs="+", type=Path, help="runtime profile CSV files")
+    args = parser.parse_args()
+    failed = False
+    for path in args.csv:
+        try:
+            result = summarize(path)
+        except (OSError, ValueError, KeyError) as exc:
+            print(f"ERROR {path}: {exc}")
+            failed = True
+            continue
+        print(f"\n{result['file']}")
+        print(f"  samples: {result['samples']}")
+        print(f"  duration: {result['duration_seconds']} s")
+        print(f"  mean CPU: {result['mean_cpu_percent_one_core']:.2f}% of one core" if result['mean_cpu_percent_one_core'] is not None else "  mean CPU: unavailable")
+        print(f"  peak CPU: {result['peak_cpu_percent_one_core']:.2f}% of one core" if result['peak_cpu_percent_one_core'] is not None else "  peak CPU: unavailable")
+        print(f"  peak summed RSS: {result['peak_rss_mib']:.2f} MiB" if result['peak_rss_mib'] is not None else "  peak summed RSS: unavailable")
+        print(f"  max matched processes: {result['max_matched_processes']}")
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/benchmark/tests/conftest.py b/benchmark/tests/conftest.py
new file mode 100644
index 0000000..9f8d9d1
--- /dev/null
+++ b/benchmark/tests/conftest.py
@@ -0,0 +1,7 @@
+"""Make benchmark helper modules importable when pytest runs from repository root."""
+from pathlib import Path
+import sys
+
+BENCHMARK_ROOT = str(Path(__file__).resolve().parents[1])
+if BENCHMARK_ROOT not in sys.path:
+    sys.path.insert(0, BENCHMARK_ROOT)
diff --git a/benchmark/tests/test_compare_benchmark_runs.py b/benchmark/tests/test_compare_benchmark_runs.py
new file mode 100644
index 0000000..6999236
--- /dev/null
+++ b/benchmark/tests/test_compare_benchmark_runs.py
@@ -0,0 +1,78 @@
+"""Tests for paired FireForm benchmark summary comparison."""
+import csv
+import tempfile
+import unittest
+from pathlib import Path
+
+from compare_benchmark_runs import compare_runs, read_run
+
+FIELDS = [
+    "document", "provider", "model", "status", "fill_wall_seconds", "document_wall_seconds",
+    "total_accuracy_percent", "populated_accuracy_percent", "total_raw_accuracy_percent",
+    "populated_raw_accuracy_percent", "missing_count", "unsupported_count",
+]
+
+
+def write_summary(path: Path, provider: str, rows: list[dict]) -> None:
+    with path.open("w", newline="", encoding="utf-8") as stream:
+        writer = csv.DictWriter(stream, fieldnames=FIELDS)
+        writer.writeheader()
+        for row in rows:
+            writer.writerow({"provider": provider, "model": "same-model", **row})
+
+
+class CompareBenchmarkRunsTests(unittest.TestCase):
+    def test_paired_comparison_reports_latency_accuracy_and_same_success_set(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            reference = root / "ollama.csv"
+            candidate = root / "llama.csv"
+            common = [
+                {"document": "ics_203_1", "status": "ok", "fill_wall_seconds": "10", "document_wall_seconds": "12",
+                 "total_accuracy_percent": "80", "populated_accuracy_percent": "85", "total_raw_accuracy_percent": "75",
+                 "populated_raw_accuracy_percent": "82", "missing_count": "2", "unsupported_count": "1"},
+                {"document": "ics_206_1", "status": "ok", "fill_wall_seconds": "8", "document_wall_seconds": "10",
+                 "total_accuracy_percent": "90", "populated_accuracy_percent": "92", "total_raw_accuracy_percent": "88",
+                 "populated_raw_accuracy_percent": "91", "missing_count": "1", "unsupported_count": "0"},
+            ]
+            write_summary(reference, "ollama", common)
+            candidate_rows = [dict(row, fill_wall_seconds=str(float(row["fill_wall_seconds"]) * 0.8),
+                                   document_wall_seconds=str(float(row["document_wall_seconds"]) * 0.8)) for row in common]
+            candidate_rows[0]["total_accuracy_percent"] = "79"
+            write_summary(candidate, "llama_cpp", candidate_rows)
+            comparison, diffs = compare_runs(reference, candidate)
+        self.assertEqual(comparison["common_documents"], 2)
+        self.assertTrue(comparison["same_success_set"])
+        self.assertEqual(comparison["reference"]["provider"], "ollama")
+        self.assertEqual(comparison["candidate"]["provider"], "llama_cpp")
+        self.assertEqual(len(diffs), 2)
+        self.assertAlmostEqual(diffs[0]["delta_document_wall_seconds"], -2.4)
+        self.assertEqual(diffs[0]["delta_total_accuracy_percent"], -1)
+
+    def test_mismatched_skipped_document_is_flagged(self):
+        with tempfile.TemporaryDirectory() as temp:
+            root = Path(temp)
+            reference = root / "ollama.csv"
+            candidate = root / "llama.csv"
+            base = {"document": "ics_203_1", "status": "ok", "fill_wall_seconds": "10", "document_wall_seconds": "12",
+                    "total_accuracy_percent": "80", "populated_accuracy_percent": "85", "total_raw_accuracy_percent": "75",
+                    "populated_raw_accuracy_percent": "82", "missing_count": "2", "unsupported_count": "1"}
+            skipped = dict(base, status="skipped: timeout", fill_wall_seconds="", document_wall_seconds="",
+                           total_accuracy_percent="", populated_accuracy_percent="", total_raw_accuracy_percent="",
+                           populated_raw_accuracy_percent="", missing_count="", unsupported_count="")
+            write_summary(reference, "ollama", [base])
+            write_summary(candidate, "llama_cpp", [skipped])
+            comparison, diffs = compare_runs(reference, candidate)
+        self.assertFalse(comparison["same_success_set"])
+        self.assertIsNone(diffs[0]["candidate_document_wall_seconds"])
+
+    def test_run_without_rows_is_rejected(self):
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "empty.csv"
+            path.write_text(",".join(FIELDS) + "\n", encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "no document rows"):
+                read_run(path)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/benchmark/tests/test_profile_runtime.py b/benchmark/tests/test_profile_runtime.py
new file mode 100644
index 0000000..5878806
--- /dev/null
+++ b/benchmark/tests/test_profile_runtime.py
@@ -0,0 +1,27 @@
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+import profile_runtime
+
+
+class ProcStatTests(unittest.TestCase):
+    def test_parse_proc_stat_handles_spaces_in_command_name(self):
+        # Fields 3..13, followed by utime=123 and stime=45.
+        fields_after_comm = ["S"] + ["0"] * 10 + ["123", "45", "0"]
+        pid, ticks = profile_runtime.parse_proc_stat(
+            "321 (llama server worker) " + " ".join(fields_after_comm)
+        )
+        self.assertEqual(pid, 321)
+        self.assertEqual(ticks, 168)
+
+    def test_cpu_percent(self):
+        self.assertEqual(profile_runtime.cpu_percent(None, 100, 1.0), 0.0)
+        expected = 50.0 / profile_runtime.CLOCK_TICKS * 100.0
+        self.assertAlmostEqual(profile_runtime.cpu_percent(100, 150, 1.0), expected)
+        self.assertEqual(profile_runtime.cpu_percent(100, 120, 0), 0.0)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/benchmark/tests/test_summarize_runtime.py b/benchmark/tests/test_summarize_runtime.py
new file mode 100644
index 0000000..9740867
--- /dev/null
+++ b/benchmark/tests/test_summarize_runtime.py
@@ -0,0 +1,40 @@
+"""Tests for runtime-profile CSV summarization."""
+import csv
+import tempfile
+import unittest
+from pathlib import Path
+
+from summarize_runtime import summarize
+
+
+class SummarizeRuntimeTests(unittest.TestCase):
+    def test_summary_reports_peak_and_mean_resources(self):
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "profile.csv"
+            with path.open("w", newline="", encoding="utf-8") as stream:
+                writer = csv.DictWriter(stream, fieldnames=[
+                    "elapsed_seconds", "cpu_percent_one_core", "rss_mib", "matched_processes"
+                ])
+                writer.writeheader()
+                writer.writerows([
+                    {"elapsed_seconds": "0.5", "cpu_percent_one_core": "20", "rss_mib": "100", "matched_processes": "1"},
+                    {"elapsed_seconds": "1.5", "cpu_percent_one_core": "80", "rss_mib": "150", "matched_processes": "2"},
+                ])
+            result = summarize(path)
+        self.assertEqual(result["samples"], 2)
+        self.assertEqual(result["duration_seconds"], 1.5)
+        self.assertEqual(result["mean_cpu_percent_one_core"], 50)
+        self.assertEqual(result["peak_cpu_percent_one_core"], 80)
+        self.assertEqual(result["peak_rss_mib"], 150)
+        self.assertEqual(result["max_matched_processes"], 2)
+
+    def test_empty_csv_is_rejected(self):
+        with tempfile.TemporaryDirectory() as temp:
+            path = Path(temp) / "profile.csv"
+            path.write_text("elapsed_seconds,cpu_percent_one_core,rss_mib,matched_processes\n", encoding="utf-8")
+            with self.assertRaisesRegex(ValueError, "no samples"):
+                summarize(path)
+
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/contracts/schemas/enums.yaml b/contracts/schemas/enums.yaml
index 50f22c6..48d3020 100644
--- a/contracts/schemas/enums.yaml
+++ b/contracts/schemas/enums.yaml
@@ -88,11 +88,12 @@ LLMProvider:
   type: string
   enum:
     - ollama
+    - llama_cpp
     - openai
     - gemini
     - anthropic
     - custom
-  description: An LLM provider that can fill forms. custom is any OpenAI compatible server
+  description: An LLM provider that can fill forms. llama_cpp uses the llama.cpp OpenAI-compatible server; custom is any other OpenAI-compatible server
 
 # OutputFormat:
 #   type: string
diff --git a/docker/.env.example b/docker/.env.example
index 9f96d77..2235889 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -84,13 +84,22 @@ ANTHROPIC_API_KEY=
 ANTHROPIC_MODELS=
 # Optional: ANTHROPIC_BASE_URL, ANTHROPIC_MAX_TOKENS, ANTHROPIC_STRUCTURED_OUTPUT
 
-# Custom: any OpenAI compatible server (vLLM, LM Studio, llama.cpp, Groq,
-# OpenRouter). Needs a base URL. The key only if the server wants one.
+# Custom: any OpenAI-compatible server (vLLM, LM Studio, Groq, OpenRouter).
+# Needs a base URL. The key only if the server wants one.
 CUSTOM_BASE_URL=
 CUSTOM_API_KEY=
 CUSTOM_MODELS=
 # Optional: CUSTOM_MAX_TOKENS, CUSTOM_STRUCTURED_OUTPUT
 
+# llama.cpp: dedicated provider using llama-server's OpenAI-compatible API.
+# In Docker, use http://host.docker.internal:8080/v1 if the server runs on the host.
+# LLAMA_CPP_MODELS accepts * or a comma-separated list of IDs from /v1/models.
+# Default base URL: http://localhost:8080/v1. API key is optional for local servers.
+LLAMA_CPP_BASE_URL=
+LLAMA_CPP_API_KEY=
+LLAMA_CPP_MODELS=
+# Optional: LLAMA_CPP_MAX_TOKENS, LLAMA_CPP_STRUCTURED_OUTPUT
+
 # The optional ones, for any provider:
 #   <PROVIDER>_BASE_URL           overrides the provider's address
 #   <PROVIDER>_MAX_TOKENS         output token cap. Empty sends none.
diff --git a/docs/multi-llm-support/configuration.md b/docs/multi-llm-support/configuration.md
index 0907707..08c9677 100644
--- a/docs/multi-llm-support/configuration.md
+++ b/docs/multi-llm-support/configuration.md
@@ -13,7 +13,7 @@ Every LLM setting, as read from the env file. Empty means unset.
 
 | Setting              | Default          | Meaning                                                                       |
 | -------------------- | ---------------- | ----------------------------------------------------------------------------- |
-| `LLM_PROVIDER`       | `ollama`         | Default provider. One of `ollama`, `openai`, `gemini`, `anthropic`, `custom`. |
+| `LLM_PROVIDER`       | `ollama`         | Default provider. One of `ollama`, `openai`, `gemini`, `anthropic`, `custom`, `llama_cpp`. |
 | `LLM_MODEL`          | empty            | Default model. Must be allowed by the default provider's `_MODELS`.           |
 | `LLM_ALLOW_EXTERNAL` | `false`          | Allow providers outside localhost or a private network.                       |
 | `LLM_TIMEOUT`        | `OLLAMA_TIMEOUT` | Seconds per fill.                                                             |
@@ -22,7 +22,7 @@ Every LLM setting, as read from the env file. Empty means unset.
 
 ## Per provider settings
 
-`<PREFIX>` is `OLLAMA`, `OPENAI`, `GEMINI`, `ANTHROPIC` or `CUSTOM`.
+`<PREFIX>` is `OLLAMA`, `OPENAI`, `GEMINI`, `ANTHROPIC`, `CUSTOM` or `LLAMA_CPP`.
 
 | Setting                      | Meaning                                                                                        |
 | ---------------------------- | ---------------------------------------------------------------------------------------------- |
@@ -61,6 +61,7 @@ Every LLM setting, as read from the env file. Empty means unset.
 | Gemini    | `https://generativelanguage.googleapis.com/v1beta/openai/` | on                | none       |
 | Anthropic | `https://api.anthropic.com/v1/`                            | off               | `12288`    |
 | Custom    | `CUSTOM_BASE_URL`                                          | on                | none       |
+| llama.cpp | `LLAMA_CPP_BASE_URL` or `http://localhost:8080/v1`          | on                | none       |
 
 ## Models shown in the catalog
 
@@ -95,6 +96,16 @@ provider. The existing `probe(..., verify=False)` path used by
 escape hatch; a model added manually is included in the catalog even if the
 provider's list filter would exclude it.
 
+## llama.cpp
+
+Use `LLM_PROVIDER=llama_cpp` to make llama.cpp the default, or choose it in the provider selector.
+Set `LLAMA_CPP_MODELS` to `*` or a comma-separated list of IDs returned by `/v1/models`.
+The base URL defaults to `http://localhost:8080/v1`; from Docker, use
+`http://host.docker.internal:8080/v1` when the server runs on the host.
+API keys are optional for local servers. Remote endpoints remain subject to external-processing consent.
+If llama.cpp is the default and `LLAMA_CPP_MODELS=*`, also set `LLM_MODEL` to an actual model ID;
+an explicit comma-separated model list can supply a default from its first entry.
+
 ## What counts as local
 
 A URL is local when its host is `localhost`, a loopback or private IP, a bare name such as a Docker service (`vllm`, `ollama`), or ends in `.local`, `.internal` or `.lan`. Anything else needs `LLM_ALLOW_EXTERNAL=true`.
diff --git a/tests/test_llm_registry.py b/tests/test_llm_registry.py
index bcab69d..9292083 100644
--- a/tests/test_llm_registry.py
+++ b/tests/test_llm_registry.py
@@ -33,6 +33,7 @@
 from app.services.llm.specs import DEFAULT_OLLAMA_INSTALLABLE_MODELS, SPECS
 
 OLLAMA, GEMINI, OPENAI = LLMProvider.ollama, LLMProvider.gemini, LLMProvider.openai
+LLAMA_CPP = LLMProvider.llama_cpp
 
 
 class FakeLLM(LLM):
@@ -163,6 +164,17 @@ def test_nothing_named_is_the_default_without_asking(self, registry, ollama):
         assert registry.resolve(OLLAMA, "qwen") == ModelChoice(OLLAMA, "qwen")
         assert ollama.list_calls == 0
 
+    def test_llama_cpp_provider_can_be_selected_and_listed(self, ollama):
+        model_name = "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf"
+        llama_cpp = FakeLLM(LLAMA_CPP, [model_name])
+        settings = {
+            OLLAMA: _provider(OLLAMA, "*", "qwen"),
+            LLAMA_CPP: _provider(LLAMA_CPP, model_name, model_name),
+        }
+        registry = _registry({OLLAMA: ollama, LLAMA_CPP: llama_cpp}, settings=settings)
+        assert registry.resolve(LLAMA_CPP, model_name) == ModelChoice(LLAMA_CPP, model_name)
+        assert [m.name for m in registry.provider_catalog(LLAMA_CPP).models] == [model_name]
+
     def test_provider_alone_gets_its_default_model(self, registry):
         assert registry.resolve("gemini") == ModelChoice(GEMINI, "g1")
 
diff --git a/tests/test_llm_service.py b/tests/test_llm_service.py
index ad8043a..4a5db65 100644
--- a/tests/test_llm_service.py
+++ b/tests/test_llm_service.py
@@ -164,6 +164,15 @@ def test_mistake_in_other_provider_turns_it_off(self):
             "gemini is off: GEMINI_API_KEY is needed. Add it, or empty GEMINI_MODELS to turn gemini off.",
         )
 
+    def test_llama_cpp_uses_local_default_without_an_api_key(self):
+        settings = load_settings(_only("llama_cpp", {"MODELS": "server-model"}, LLM_MODEL="server-model"))
+        provider = settings.providers[LLMProvider.llama_cpp]
+        assert provider.base_url == "http://localhost:8080/v1"
+        assert provider.api_key == ""
+        llm = create_llm(provider)
+        assert isinstance(llm, OpenAICompatibleLLM)
+        assert llm.provider is LLMProvider.llama_cpp
+
     def test_custom_needs_base_url(self):
         with pytest.raises(LLMConfigError, match="CUSTOM_BASE_URL is needed"):
             load_settings(_only("custom", {"MODELS": "m"}))