Add explicit llama.cpp provider and benchmark tooling

app/api/routes/llm_config.py

@@ -62,9 +62,9 @@ def get_provider_configs(db: Session = Depends(get_db)):
         ui_key = bool(
             record and record.api_key_encrypted and has_decryptable_credential(provider.value)
         )
-        ui_custom_url = bool(provider is LLMProvider.custom and record and record.base_url)
-        source = "ui" if ui_key or ui_custom_url else "env" if env_key or env.get("BASE_URL") else "none"
-        base_url = (record.base_url if ui_custom_url else None) or env.get("BASE_URL") or spec.default_base_url
+        ui_compatible_url = bool(provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record and record.base_url)
+        source = "ui" if ui_key or ui_compatible_url else "env" if env_key or env.get("BASE_URL") else "none"
+        base_url = (record.base_url if ui_compatible_url else None) or env.get("BASE_URL") or spec.default_base_url
         key_hint = record.key_hint if ui_key else env_key[-4:] if env_key else None
         statuses.append(
             LLMProviderCredentialStatus(
@@ -77,10 +77,10 @@ def get_provider_configs(db: Session = Depends(get_db)):
                 credential_status=(
                     "available" if has_decryptable_credential(provider.value) else "unavailable"
                 ) if record and record.api_key_encrypted else None,
-                base_url=_display_base_url(base_url) if provider is LLMProvider.custom else None,
+                base_url=_display_base_url(base_url) if provider in {LLMProvider.custom, LLMProvider.llama_cpp} else None,
                 structured_output=(
                     record.structured_output
-                    if provider is LLMProvider.custom and record and record.structured_output is not None
+                    if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record and record.structured_output is not None
                     else None
                 ),
             )
@@ -100,17 +100,17 @@ def save_provider_config(
     ).first()
     env = config.LLM_PROVIDER_ENV[provider.value]
 
-    if provider is not LLMProvider.custom and body.structured_output is not None:
-        raise LLMNotAvailableError("Only the custom provider accepts a structured_output override.")
-    if provider is not LLMProvider.custom and body.base_url is not None:
-        raise LLMNotAvailableError("Only the custom provider accepts a configurable base URL.")
+    if provider not in {LLMProvider.custom, LLMProvider.llama_cpp} and body.structured_output is not None:
+        raise LLMNotAvailableError("Only OpenAI-compatible custom providers accept a structured_output override.")
+    if provider not in {LLMProvider.custom, LLMProvider.llama_cpp} and body.base_url is not None:
+        raise LLMNotAvailableError("Only OpenAI-compatible custom providers accept a configurable base URL.")
 
-    base_url = body.base_url if provider is LLMProvider.custom and body.base_url is not None else None
-    if provider is LLMProvider.custom:
-        base_url = base_url or (record.base_url if record else None) or env.get("BASE_URL") or None
+    base_url = body.base_url if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and body.base_url is not None else None
+    if provider in {LLMProvider.custom, LLMProvider.llama_cpp}:
+        base_url = base_url or (record.base_url if record else None) or env.get("BASE_URL") or SPECS[provider].default_base_url or None
         if not base_url:
-            raise LLMNotAvailableError("CUSTOM_BASE_URL is required.")
-        base_url = _validate_custom_url(base_url)
+            raise LLMNotAvailableError(f"{SPECS[provider].setting('BASE_URL')} is required.")
+        base_url = _validate_provider_url(base_url, provider)
 
     api_key_was_provided = "api_key" in body.model_fields_set and body.api_key is not None
     api_key = _provided_secret(body.api_key) if api_key_was_provided else None
@@ -129,7 +129,7 @@ def save_provider_config(
         secret = _provided_secret(body.api_key)
         record.api_key_encrypted = encrypt_api_key(secret) if secret else None
         record.key_hint = secret[-4:] if secret else None
-    if provider is LLMProvider.custom:
+    if provider in {LLMProvider.custom, LLMProvider.llama_cpp}:
         record.base_url = base_url
         if body.structured_output is not None:
             record.structured_output = body.structured_output
@@ -186,11 +186,11 @@ def validate_provider_config(
     db: Session = Depends(get_db),
 ):
     validate_provider(provider)
-    if provider is not LLMProvider.custom and body.structured_output is not None:
+    if provider not in {LLMProvider.custom, LLMProvider.llama_cpp} and body.structured_output is not None:
         return LLMProviderValidationResult(
             provider=provider,
             ok=False,
-            detail="Only the custom provider accepts a structured_output override.",
+            detail="Only OpenAI-compatible custom providers accept a structured_output override.",
             error_code="LLM_CONFIG_ERROR",
         )
     api_key = _provided_secret(body.api_key)
@@ -202,24 +202,26 @@ def validate_provider_config(
             error_code="LLM_AUTH_FAILED",
         )
     base_url = body.base_url
-    if provider is LLMProvider.custom:
+    if provider in {LLMProvider.custom, LLMProvider.llama_cpp}:
+        if not base_url:
+            base_url = config.LLM_PROVIDER_ENV[provider.value].get("BASE_URL") or SPECS[provider].default_base_url or None
         if not base_url:
             return LLMProviderValidationResult(
                 provider=provider,
                 ok=False,
-                detail="CUSTOM_BASE_URL is required.",
+                detail=f"{SPECS[provider].setting('BASE_URL')} is required.",
                 error_code="LLM_CONFIG_ERROR",
             )
         try:
-            base_url = _validate_custom_url(base_url)
+            base_url = _validate_provider_url(base_url, provider)
         except LLMError as exc:
             return LLMProviderValidationResult(provider=provider, ok=False, detail=exc.message,
                                                error_code=exc.error_code)
     elif base_url is not None:
         return LLMProviderValidationResult(
             provider=provider,
             ok=False,
-            detail="Only the custom provider accepts a configurable base URL.",
+            detail="Only OpenAI-compatible custom providers accept a configurable base URL.",
             error_code="LLM_CONFIG_ERROR",
         )
 
@@ -242,7 +244,7 @@ def validate_provider_config(
         )
     except LLMError as exc:
         detail = _redact(exc.message, api_key)
-        if provider is LLMProvider.custom and "has no model list" in detail:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and "has no model list" in detail:
             detail = "This server does not expose a model list. Validate a model ID manually."
         return LLMProviderValidationResult(
             provider=provider, ok=False, detail=detail, error_code=exc.error_code
@@ -262,7 +264,7 @@ def validate_and_add_manual_model(
         ).first()
         env = config.LLM_PROVIDER_ENV[provider.value]
         base_url = (
-            existing_config.base_url if provider is LLMProvider.custom and existing_config else None
+            existing_config.base_url if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and existing_config else None
         ) or env.get("BASE_URL") or SPECS[provider].default_base_url
         _require_external_consent(db, provider, base_url)
         candidate = registry_for_candidate(
@@ -317,22 +319,23 @@ def _provided_secret(value: SecretStr | None) -> str:
     return value.get_secret_value().strip() if value else ""
 
 
-def _validate_custom_url(value: str) -> str:
+def _validate_provider_url(value: str, provider: LLMProvider) -> str:
+    setting = SPECS[provider].setting("BASE_URL")
     try:
         parsed = urlparse(value)
     except ValueError:
-        raise LLMNotAvailableError("CUSTOM_BASE_URL must be an HTTP or HTTPS URL with a hostname.") from None
+        raise LLMNotAvailableError(f"{setting} must be an HTTP or HTTPS URL with a hostname.") from None
     if parsed.username is not None or parsed.password is not None:
-        raise LLMNotAvailableError("CUSTOM_BASE_URL must not include embedded credentials.")
+        raise LLMNotAvailableError(f"{setting} must not include embedded credentials.")
     if parsed.query or parsed.fragment:
-        raise LLMNotAvailableError("CUSTOM_BASE_URL must not include query parameters or a fragment.")
+        raise LLMNotAvailableError(f"{setting} must not include query parameters or a fragment.")
     from app.services.llm.settings import _check_url
     from app.services.llm.errors import LLMConfigError
 
     try:
-        return _check_url("CUSTOM_BASE_URL", value)
+        return _check_url(setting, value)
     except LLMConfigError:
-        raise LLMNotAvailableError("CUSTOM_BASE_URL must be an HTTP or HTTPS URL with a hostname.") from None
+        raise LLMNotAvailableError(f"{setting} must be an HTTP or HTTPS URL with a hostname.") from None
 
 
 def _display_base_url(value: str) -> str:

app/api/schemas/enums.py

@@ -118,6 +118,7 @@ class OutputFormat(str, Enum):
 
 class LLMProvider(str, Enum):
     ollama = "ollama"
+    llama_cpp = "llama_cpp"
     openai = "openai"
     gemini = "gemini"
     anthropic = "anthropic"

app/core/config.py

@@ -70,6 +70,7 @@ def _provider_env(prefix: str, **defaults: str) -> dict[str, str]:
     "gemini": _provider_env("GEMINI"),
     "anthropic": _provider_env("ANTHROPIC"),
     "custom": _provider_env("CUSTOM"),
+    "llama_cpp": _provider_env("LLAMA_CPP"),
 }
 
 WHISPER_HOST = os.getenv("WHISPER_HOST", "http://localhost:9000").rstrip("/")

app/services/llm/specs.py

@@ -87,6 +87,10 @@ def setting(self, key: str) -> str:
         headers=(("anthropic-version", "2023-06-01"),),
     ),
     LLMProvider.custom: ProviderSpec(LLMProvider.custom, needs_key=False),
+    # llama.cpp serves the same OpenAI-compatible API with a local default.
+    LLMProvider.llama_cpp: ProviderSpec(
+        LLMProvider.llama_cpp, "http://localhost:8080/v1", needs_key=False
+    ),
 }
 
 DEFAULT_OLLAMA_INSTALLABLE_MODELS: tuple[str, ...] = (

app/services/llm/user_config.py

@@ -26,6 +26,7 @@
     LLMProvider.gemini,
     LLMProvider.anthropic,
     LLMProvider.custom,
+    LLMProvider.llama_cpp,
 })
 _cache_lock = threading.Lock()
 _credential_state_lock = threading.Lock()
@@ -149,7 +150,7 @@ def validate_provider(provider: LLMProvider) -> None:
     if provider not in EDITABLE_PROVIDERS:
         from app.services.llm.errors import LLMNotAvailableError
 
-        raise LLMNotAvailableError("Only OpenAI, Gemini, Anthropic, and custom providers can be configured here.")
+        raise LLMNotAvailableError("Only OpenAI, Gemini, Anthropic, custom, and llama.cpp providers can be configured here.")
 
 
 def registry_for_candidate(
@@ -259,21 +260,21 @@ def _build_registry(
             unavailable_credentials.add(provider.value)
         if api_key:
             provider_settings["API_KEY"] = api_key
-        if provider is LLMProvider.custom and record.base_url:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record.base_url:
             provider_settings["BASE_URL"] = record.base_url
-        if provider is LLMProvider.custom and record.structured_output is not None:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record.structured_output is not None:
             provider_settings["STRUCTURED_OUTPUT"] = str(record.structured_output).lower()
-        if not provider_settings.get("MODELS") and (api_key or (provider is LLMProvider.custom and record.base_url)):
+        if not provider_settings.get("MODELS") and (api_key or (provider in {LLMProvider.custom, LLMProvider.llama_cpp} and record.base_url)):
             provider_settings["MODELS"] = "*"
 
     if candidate:
         provider, override, extra_model = candidate
         provider_settings = provider_env[provider.value]
         if override["api_key"] is not None:
             provider_settings["API_KEY"] = override["api_key"]
-        if provider is LLMProvider.custom and override["base_url"] is not None:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and override["base_url"] is not None:
             provider_settings["BASE_URL"] = override["base_url"]
-        if provider is LLMProvider.custom and override["structured_output"] is not None:
+        if provider in {LLMProvider.custom, LLMProvider.llama_cpp} and override["structured_output"] is not None:
             provider_settings["STRUCTURED_OUTPUT"] = str(override["structured_output"]).lower()
         if not provider_settings.get("MODELS"):
             provider_settings["MODELS"] = "*"

benchmark/.env.example

@@ -6,6 +6,17 @@ OLLAMA_HOST=http://localhost:11434
 OLLAMA_MODEL=qwen2.5:1.5b
 OLLAMA_TIMEOUT=300
 
+# --- Optional llama.cpp provider -------------------------------------------
+# The model ID must match llama-server's /v1/models response.
+# Keep these commented to leave the default provider as Ollama.
+# LLM_PROVIDER=llama_cpp
+# LLM_MODEL=your-model-id
+# LLAMA_CPP_BASE_URL=http://localhost:8080/v1
+# LLAMA_CPP_MODELS=your-model-id
+# LLAMA_CPP_API_KEY=
+# LLM_TIMEOUT=300
+# LLM_ALLOW_EXTERNAL=false
+
 # --- LLM judge ------------------------------------------------------------
 # JUDGE_HOST defaults to OLLAMA_HOST.
 # JUDGE_HOST=http://localhost:11434

benchmark/LLAMA_CPP_COMPARISON_REPORT_TEMPLATE.md

@@ -0,0 +1,91 @@
+# FireForm Issue #702 — Ollama vs. llama.cpp comparison
+
+> Fill this report only after the benchmark has completed. Keep raw run folders, `summary.csv`, `run_info.json`, and runtime profile CSVs as evidence. Do not infer missing measurements.
+
+## 1. Reproducibility record
+
+| Item | Value |
+|---|---|
+| Date/time and timezone | TODO |
+| FireForm commit SHA | TODO |
+| OS and kernel | TODO |
+| CPU model / cores | TODO |
+| RAM | TODO |
+| GPU / VRAM (if used) | TODO |
+| Python / dependency versions | From `run_info.json` |
+| ICS documents included | TODO |
+| Number of repetitions per provider | At least 3 |
+| Judge model and host | TODO; keep identical across runs |
+| Prompt SHA-256 | From `run_info.json` |
+
+## 2. Model equivalence and runtime configuration
+
+| Item | Ollama | llama.cpp |
+|---|---|---|
+| Model family and exact weights | TODO | TODO |
+| Artifact SHA-256 / digest | TODO | GGUF SHA-256: TODO |
+| Quantization | TODO | TODO |
+| Context window | TODO | TODO |
+| Server version/build SHA | TODO | TODO |
+| Exact server launch command | TODO | TODO |
+| API/provider config (omit secrets) | TODO | `llama_cpp` / OpenAI-compatible (or `custom` if running before the explicit provider lands) |
+| Warm-up procedure | TODO | TODO |
+
+Confirm that the model weights are equivalent. If exact equivalence is not possible, state this as a limitation and do not attribute all observed differences to the runtime alone.
+
+## 3. Per-run results
+
+Include one row per repetition and provider. Use the same documents and scoring configuration in all runs.
+
+| Provider | Run | Documents scored | Skipped | Total wall time (s) | Median doc latency (s) | Judged accuracy (%) | Populated accuracy (%) | Raw accuracy (%) | Missing fields | Unsupported fields |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+| Ollama | 1 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| Ollama | 2 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| Ollama | 3 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| llama.cpp | 1 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| llama.cpp | 2 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+| llama.cpp | 3 | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO | TODO |
+
+## 4. Resource usage
+
+Use the `profile_runtime.py` CSV captured on the model-server host. Record GPU measurements separately if available.
+
+| Metric | Ollama | llama.cpp | Measurement method |
+|---|---:|---:|---|
+| Mean CPU usage (% of one core) | TODO | TODO | profile CSV |
+| Peak CPU usage (% of one core) | TODO | TODO | profile CSV |
+| Peak resident RAM (MiB) | TODO | TODO | profile CSV |
+| Peak GPU VRAM (MiB) | TODO / N/A | TODO / N/A | TODO |
+
+## 5. Validity checks and limitations
+
+- Same FireForm commit and prompt: TODO
+- Same document set and narrative order: TODO
+- Same judge model/settings: TODO
+- Same weights/quantization: TODO
+- Any skipped documents or failed judge calls: TODO
+- Cold-start versus warm-run behavior: TODO
+- Remote server or unmeasured GPU resource limits: TODO
+- Other confounding factors: TODO
+
+Runs with different skipped-document sets must not be compared as if they cover the same workload. Report failed or unavailable judge results instead of treating them as correct.
+
+## 6. Summary and recommendation
+
+**Latency:** TODO (report fill-phase median and variation across repeated runs, plus end-to-end document time; do not rely only on the fastest run).
+
+**Accuracy:** TODO (state any regression or uncertainty).
+
+**Resource use:** TODO.
+
+**Integration / maintenance costs:** TODO.
+
+**Recommendation:** Adopt llama.cpp / keep Ollama / gather more evidence. Explain the trade-off and the exact scope of the recommendation. Do not recommend a runtime solely from a single run or from latency if accuracy regresses.
+
+## 7. Artifacts
+
+- Ollama run folders: TODO
+- llama.cpp run folders: TODO
+- Runtime profile CSVs: TODO
+- `run_info.json` files: TODO
+- Commands used: TODO