diff --git a/.github/instructions/ui.instructions.md b/.github/instructions/ui.instructions.md new file mode 100644 index 0000000..5eb97be --- /dev/null +++ b/.github/instructions/ui.instructions.md @@ -0,0 +1,200 @@ +# UX Design Instructions + +Follow these instructions whenever you design, build, or revise a user interface. Apply them across layouts, navigation, onboarding, forms, settings, dashboards, and interactive flows. + +Your goal is to minimize confusion, reduce effort, prevent mistakes, and help users complete their intended task as quickly as possible. + +## 1. Reduce Choices per Screen: Hick's Law + +The time required to make a decision increases with the number and complexity of available choices. + +* Give each screen one clear purpose. +* Remove irrelevant or low-priority options. +* Break complicated decisions into smaller steps. +* Recommend an option when users may struggle to choose. + +## 2. Make Targets Large: Fitts's Law + +Large, nearby targets are faster and easier to interact with. + +* Make buttons and controls easy to click or tap. +* Give interactive elements sufficient spacing. +* Avoid tiny icons as the only interaction target. +* Increase the clickable area around important controls. + +## 3. Follow Familiar Patterns: Jakob's Law + +Users expect your product to work like products they already understand. + +* Use established interface conventions. +* Place navigation, search, settings, and account controls where users expect them. +* Use familiar icons and interaction patterns. +* Do not invent a new pattern unless it provides a meaningful advantage. + +## 4. Group Related Information: Law of Proximity + +Elements positioned near one another are perceived as related. + +* Place related labels, controls, and information together. +* Use spacing to communicate relationships. +* Separate unrelated groups with additional space. +* Do not rely on borders when spacing can establish the hierarchy. + +## 5. Break Content Into Chunks: Miller's Law + +Working memory can only process a limited amount of information at once. + +* Divide long content into small, meaningful groups. +* Break complex forms and tasks into manageable steps. +* Use headings, sections, and concise labels. +* Avoid asking users to remember information between screens. + +## 6. Respond Within 400 Milliseconds: Doherty Threshold + +Interfaces feel more productive when feedback appears within approximately 400 milliseconds. + +* Acknowledge every user action immediately. +* Show loading, processing, or success states when results are not instant. +* Use optimistic updates when they are safe. +* Never leave users wondering whether their action registered. + +## 7. Highlight the Primary Action: Von Restorff Effect + +An element that visually differs from surrounding elements receives more attention. + +* Give the primary action the strongest visual emphasis. +* Use one dominant call to action per section. +* Keep secondary actions visually quieter. +* Avoid making every button compete for attention. + +## 8. Place Key Actions Nearby: Fitts's Law + +Interaction becomes faster when important targets are close to the user's current focus. + +* Place actions beside the content they affect. +* Keep form submission near the final input. +* Position frequent actions within easy reach. +* Avoid forcing unnecessary cursor or eye movement. + +## 9. Put Essentials First: Serial Position Effect + +People remember the first and last items in a sequence most clearly. + +* Put the most important information first. +* Place the final action or takeaway at the end. +* Keep lower-priority information in the middle. +* Order navigation and lists according to user importance. + +## 10. End Flows Memorably: Peak-End Rule + +Users judge an experience largely by its most intense moment and how it ends. + +* Create a clear and satisfying completion state. +* Confirm what the user accomplished. +* Explain what happens next. +* Avoid ending flows on an empty or ambiguous screen. + +## 11. Show Visible Progress: Zeigarnik Effect + +Incomplete tasks remain mentally active and encourage users to return. + +* Clearly show completed and unfinished steps. +* Save progress whenever possible. +* Make it easy to resume interrupted tasks. +* Use checklists or completion states for multi-step work. + +## 12. Simplify Complex Interfaces: Law of Prägnanz + +People interpret complex or ambiguous designs in the simplest form possible. + +* Prefer simple structures and recognizable shapes. +* Remove unnecessary decoration and visual noise. +* Create an obvious visual hierarchy. +* Make the interface understandable at a glance. + +## 13. Use Sensible Defaults: Hick's Law + +Helpful defaults reduce the number of decisions users must make. + +* Preselect the safest and most common option. +* Use existing context to reduce unnecessary input. +* Never use defaults that create unexpected commitments. +* Make every default easy to change. + +## 14. Prevent Errors Proactively: Postel's Law + +Interfaces should accept reasonable variations in user input while producing clear, predictable results. + +* Accept common input formats and variations. +* Explain requirements before submission. +* Disable impossible or unavailable actions. +* Warn users before risky or destructive actions. + +## 15. Make Errors Recoverable: Postel's Law + +The interface should handle user mistakes gracefully without creating unnecessary failure. + +* Preserve the user's work after an error. +* Explain what went wrong in plain language. +* Tell the user exactly how to fix it. +* Provide undo, retry, restore, or cancel options where appropriate. + +## 16. Maintain Pattern Consistency: Law of Similarity + +Elements that look similar are perceived as having related purposes. + +* Give similar components the same appearance and behavior. +* Use consistent colors, labels, icons, spacing, and interaction states. +* Do not use the same visual treatment for different actions. +* Reuse established components before creating new ones. + +## 17. Connect Related Elements Visually: Law of Uniform Connectedness + +Visually connected elements are perceived as more closely related. + +* Use containers, lines, backgrounds, or shared states to show relationships. +* Visually connect controls to the content they affect. +* Keep unrelated elements visually separate. +* Use connection deliberately, not decoratively. + +## 18. Reduce Task Completion Time: Parkinson's Law + +Tasks tend to expand to consume the time made available for them. + +* Minimize the number of steps required. +* Remove unnecessary confirmations and screens. +* Prefill information the user has already provided. +* Offer shortcuts for frequent or repeat actions. + +## 19. Reveal Complexity Gradually: Tesler's Law + +Every system contains some complexity that cannot be removed, only managed or transferred. + +* Show essential controls first. +* Reveal advanced options only when relevant. +* Let the system handle complexity whenever possible. +* Do not force users to understand internal technical details. + +## 20. Make Completion Feel Closer: Goal-Gradient Effect + +Motivation increases as users perceive themselves getting closer to a goal. + +* Show progress throughout multi-step flows. +* Divide long tasks into visible milestones. +* Emphasize progress already made. +* Make the remaining work feel specific and achievable. + +## Implementation Requirements + +When creating or revising an interface: + +1. Identify the user's primary goal. +2. Design the shortest clear path to that goal. +3. Make the next action visually obvious. +4. Remove anything that distracts from task completion. +5. Provide immediate feedback after every interaction. +6. Prevent errors before they occur. +7. Preserve user work when something goes wrong. +8. Confirm clearly when the goal has been completed. + +When laws appear to conflict, prioritize clarity, accessibility, user control, and successful task completion. Do not apply these laws mechanically. Use them to make deliberate decisions based on the user's context and goal. \ No newline at end of file diff --git a/README.md b/README.md index 5fa1ae7..f34c0d0 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,6 @@ The easiest way to run this is with Docker. You'll need Docker and Docker Compos git clone cd data_forecasting_agent/data_forecaster -# Copy the service-specific env examples, then edit backend/.env -# to add your LLM API key. -cp backend/.env.example backend/.env -cp frontend/.env.example frontend/.env - # Build and start everything (single-machine mode) ./scripts/build_containers.sh --single ``` @@ -43,28 +38,11 @@ That's it. Four containers come up: | `nginx-backend` | TLS termination for the API | `https://localhost:8443` | | `backend` | FastAPI + forecasting engine | internal only | -Open `https://localhost` in your browser. Log in with `admin` / `admin` (you'll be prompted to change the password). The default API credentials (`frontend` / `frontend`) are already configured, so the frontend can talk to the backend out of the box. - -> **Heads up:** The default `frontend` API key is publicly known. Rotate it before exposing this to anything beyond your local machine. See [docs/api-auth.md](docs/api-auth.md) for how. +Open `https://localhost` in your browser. On first run you'll be redirected to the **setup wizard** (`/setup`), which walks you through: backend connection → LLM provider and credentials → enabling API auth → choosing forecasting models → creating the first admin account. No `.env` secrets are needed — keys are generated and stored encrypted at setup time. ## LLM setup -The agents need an LLM to do their analysis. You can use either Google Gemini or Ollama. - -**Gemini** (easiest — just add your key): -```bash -# In backend/.env -GOOGLE_API_KEY=your_key_here -USE_OLLAMA=false -``` - -**Ollama** (runs locally or via Ollama Cloud): -```bash -# In backend/.env -USE_OLLAMA=true -OLLAMA_BASE_URL=http://localhost:11434 -OLLAMA_MODEL=llama3 -``` +The agents need an LLM to do their analysis. You can use either Google Gemini or Ollama — configured in the setup wizard or later under **Admin → LLM Config** (keys are stored encrypted in the backend database, never in the frontend). If you're running Ollama locally, pull the model first: `ollama pull llama3`. diff --git a/data_forecaster/.gitignore b/data_forecaster/.gitignore index 66fe397..eb2606d 100644 --- a/data_forecaster/.gitignore +++ b/data_forecaster/.gitignore @@ -15,12 +15,10 @@ venv/ env/ ENV/ -# Environment variables -.env - # Application runtime logs/ chroma_db/ +frontend/instance/ # Backend database (auto-generated at runtime, must not be committed) data/backend.db diff --git a/data_forecaster/backend/.env.example b/data_forecaster/backend/.env.example deleted file mode 100644 index f8bf2b5..0000000 --- a/data_forecaster/backend/.env.example +++ /dev/null @@ -1,56 +0,0 @@ -# --- LLM Configuration --- -# Set to 'true' to use local Ollama instead of Google Gemini. -USE_OLLAMA=false - -# Google Gemini Settings (required if USE_OLLAMA=false). -GOOGLE_API_KEY=your_google_api_key_here -GEMINI_MODEL=gemini-1.5-flash -GEMINI_TEMPERATURE=0.1 - -# Ollama Settings (required if USE_OLLAMA=true). -# Set USE_OLLAMA_CLOUD=true to use Ollama Cloud instead of a local daemon. -# When true, OLLAMA_API_KEY is required and OLLAMA_MODEL must be cloud-capable. -USE_OLLAMA_CLOUD=false -OLLAMA_BASE_URL=http://localhost:11434 -OLLAMA_MODEL=llama3 -OLLAMA_API_KEY= - - -# Sentence-transformers model for RAG embeddings (HuggingFace model ID). -EMBED_MODEL=sentence-transformers/all-MiniLM-L6-v2 - -# --- Backend Configuration --- -MAX_UPLOAD_MB=100 -ALLOWED_EXTENSIONS=csv,xlsx,json -CHROMA_PERSIST_DIR=./chroma_db -FILE_STORAGE_DIR=./file_store -BACKEND_DB_PATH=./data/backend.db -MAX_INMEMORY_FILES=50 -MAX_INMEMORY_JOBS=100 -# Number of forecasting pipelines permitted to run at the same time. -MAX_CONCURRENT_JOBS=2 - -# Comma-separated list of browser origins allowed to call the backend. -CORS_ALLOWED_ORIGINS=http://localhost:5000,http://frontend:5000,https://localhost,https://localhost:443 - -# Token budget configurations for LLM prompts (in tokens). -REPORT_GENERATION_TOKEN_BUDGET=800 -FORECASTING_TOKEN_BUDGET=400 -DATA_VALIDATION_TOKEN_BUDGET=300 -MODEL_SELECTION_TOKEN_BUDGET=300 -STATISTICAL_ANALYSIS_TOKEN_BUDGET=300 - -# API authentication (defaults to false: open mode). -# Enabled at runtime via the admin panel bootstrap workflow. -API_KEY_ENABLED=false - -# Deployment-time admin key protecting the one-time bootstrap endpoint. -# Generate with: python scripts/bootstrap.py -ADMIN_API_KEY=change-me-to-a-strong-random-value - -# Pre-shared frontend API credentials for Docker deployments. -# When both are set, the backend auto-creates a 'frontend' API user on first startup. -# These values must match the frontend service credentials. -# Generate matching values with: python scripts/bootstrap.py -FRONTEND_API_USERNAME=frontend -FRONTEND_API_KEY=generate-a-strong-random-key-here diff --git a/data_forecaster/backend/agents/model_selection_agent.py b/data_forecaster/backend/agents/model_selection_agent.py index 12dd685..79c385d 100644 --- a/data_forecaster/backend/agents/model_selection_agent.py +++ b/data_forecaster/backend/agents/model_selection_agent.py @@ -1,43 +1,34 @@ """Model selection agent for the Data Forecaster backend. -Python is the source of statistical decisions. When empirical metrics are -available, a deterministic selection policy selects the best model. The -LLM is used for context, critique, and explanation only — it never -decides model rankings. When no empirical metrics are available (first -run), the LLM provides a suitability-based recommendation, but the -deterministic heuristic fallback is used if the LLM is unavailable or its -output is invalid. - -All suitability-assessment, heuristic-fallback, and LLM-output parsing -logic is implemented as small, focused module-level helpers so that the -public :func:`run_model_selection_agent` stays readable and well below -the SonarQube Cognitive Complexity threshold. +This module uses an LLM to reason over statistical findings and select the +best forecasting model. All suitability-assessment, heuristic-fallback, and +LLM-output parsing logic is implemented as small, focused module-level +helpers so that the public :func:`run_model_selection_agent` stays readable +and well below the SonarQube Cognitive Complexity threshold. """ from __future__ import annotations import math -import numpy as np - from core.llm_factory import get_llm from core.logging_config import get_logger -from forecasting.selection_policy import ( - CandidateEvidence, - SelectionOutcome, - select_model_deterministic, - validate_llm_output, -) -from forecasting.contracts import ForecastAdapterResult +from forecasting.registry import MODEL_NAMES, get_enabled_models from prompts.model_selection_prompt import MODEL_SELECTION_PROMPT from schemas import ModelSelectionResult, StatisticalResult from utils.token_tracking import estimate_input_text, extract_token_usage logger = get_logger(__name__) -_MODELS = ("ARIMA", "SARIMA", "Holt-Winters", "EWMA") +# Canonical model names live in forecasting.registry.MODEL_NAMES; enabled +# state is read from the DB via get_enabled_models() at call time so that +# admin changes take effect without a restart. +_MODELS = MODEL_NAMES # Backward-compatible alias (all known models). _METRIC_PRIORITY = ("MASE", "WAPE", "RMSE", "MAE", "MAPE") +# Message shown for models disabled by the administrator. +_DISABLED_REASON = "Model disabled by administrator." + # Unicode hyphen characters that the LLM may emit instead of ASCII '-'. _UNICODE_HYPHENS = ( "\u2010", @@ -191,20 +182,70 @@ def _ewma_suitability(stat_result: StatisticalResult) -> str: return "EWMA Assessment:\n" + "\n".join(f"- {p}" for p in points) +def _prophet_suitability(stat_result: StatisticalResult) -> str: + """Build the Prophet (Meta Prophet) suitability assessment string. + + Args: + stat_result: Output of the statistical analysis agent. + + Returns: + A multi-line bullet list describing Prophet suitability. + """ + points: list[str] = [] + sp = stat_result.seasonal_period + if sp and sp > 1: + points.append( + f"Seasonal period {sp} detected — Prophet models seasonality " + "natively via Fourier terms and can combine multiple seasonalities." + ) + else: + points.append( + "No strong seasonal period — Prophet still fits a piecewise trend " + "and may capture seasonalities the statistical tests missed." + ) + if stat_result.has_trend: + points.append( + f"Trend detected (slope={stat_result.trend_slope:.4f}) — Prophet " + "models trend automatically with changepoint detection." + ) + if stat_result.outlier_ratio > 0.05: + points.append( + f"High outlier ratio ({stat_result.outlier_ratio:.1%}) — Prophet " + "is robust to outliers and missing observations." + ) + else: + points.append("Low outlier count — Prophet will be stable.") + points.append( + "Prophet needs at least 2 observations and performs best with " + "substantial history; it is heavier to fit than the other models." + ) + return "Prophet Assessment:\n" + "\n".join(f"- {p}" for p in points) + + def _build_suitability_summary(stat_result: StatisticalResult) -> str: - """Combine all four model suitability assessments into one summary. + """Combine enabled models' suitability assessments into one summary. + + Disabled models are omitted entirely so the LLM never sees (and never + selects) a model the administrator has turned off. Args: stat_result: Output of the statistical analysis agent. Returns: - A single string containing all four assessments separated by blank lines. + A single string containing the enabled models' assessments + separated by blank lines. """ + builders = { + "Holt-Winters": _hw_suitability, + "ARIMA": _arima_suitability, + "SARIMA": _sarima_suitability, + "EWMA": _ewma_suitability, + "Prophet": _prophet_suitability, + } sections = [ - _hw_suitability(stat_result), - _arima_suitability(stat_result), - _sarima_suitability(stat_result), - _ewma_suitability(stat_result), + builders[name](stat_result) + for name in get_enabled_models() + if name in builders ] if stat_result.disabled_tests: sections.append( @@ -231,11 +272,10 @@ def _heuristic_fallback( """ preference = _heuristic_preference(stat_result) fallback_model = preference[0] + enabled = set(get_enabled_models()) reasoning: dict[str, str | None] = { - "Holt-Winters": None, - "ARIMA": None, - "SARIMA": None, - "EWMA": None, + name: (None if name in enabled else _DISABLED_REASON) + for name in MODEL_NAMES } for m in preference[1:]: reasoning[m] = _heuristic_rejection_reason(stat_result, m) @@ -253,16 +293,28 @@ def _heuristic_preference(stat_result: StatisticalResult) -> list[str]: stat_result: Output of the statistical analysis agent. Returns: - A list of model names ordered by heuristic preference. + A list of enabled model names ordered by heuristic preference. + Guaranteed non-empty (the registry enforces at least one enabled + model). """ sp = stat_result.seasonal_period or 1 if sp > 1: - return ["SARIMA", "Holt-Winters", "ARIMA", "EWMA"] - if stat_result.has_trend and abs(stat_result.trend_slope) > 0.1: - return ["Holt-Winters", "ARIMA", "SARIMA", "EWMA"] - if stat_result.is_white_noise: - return ["EWMA", "ARIMA", "Holt-Winters", "SARIMA"] - return ["ARIMA", "Holt-Winters", "SARIMA", "EWMA"] + preference = ["SARIMA", "Prophet", "Holt-Winters", "ARIMA", "EWMA"] + elif stat_result.has_trend and abs(stat_result.trend_slope) > 0.1: + preference = ["Holt-Winters", "Prophet", "ARIMA", "SARIMA", "EWMA"] + elif stat_result.is_white_noise: + preference = ["EWMA", "ARIMA", "Holt-Winters", "SARIMA", "Prophet"] + else: + preference = ["ARIMA", "Prophet", "Holt-Winters", "SARIMA", "EWMA"] + enabled = set(get_enabled_models()) + filtered = [m for m in preference if m in enabled] + if not filtered: # Defensive: registry should prevent this. + logger.warning( + "Heuristic preference was empty after filtering disabled models; " + "falling back to the enabled set." + ) + return list(get_enabled_models()) + return filtered def _heuristic_rejection_reason( @@ -286,24 +338,40 @@ def _heuristic_rejection_reason( ), "ARIMA": "Seasonal pattern detected; plain ARIMA ignores seasonality.", "EWMA": "Seasonal patterns present; EWMA does not capture seasonality.", + "Prophet": ( + "Seasonality detected; SARIMA models it more explicitly and " + "is lighter to fit than Prophet for this case." + ), } elif stat_result.has_trend and abs(stat_result.trend_slope) > 0.1: reasons = { "ARIMA": "Trend present but Holt-Winters handles it more naturally.", "SARIMA": "No strong seasonality confirmed; SARIMA may overfit.", "EWMA": "Strong trend present; EWMA will lag behind trend changes.", + "Prophet": ( + "Trend present; Holt-Winters handles it more lightly than " + "Prophet for a non-seasonal series." + ), } elif stat_result.is_white_noise: reasons = { "Holt-Winters": "Series appears random; simple EWMA may suffice.", "ARIMA": "Series is random noise; complex models may overfit.", "SARIMA": "No patterns detected; SARIMA would overfit.", + "Prophet": ( + "Series is random noise; Prophet is heavier than the data " + "structure warrants and would overfit." + ), } else: reasons = { "Holt-Winters": "No clear seasonal pattern or strong trend detected.", "SARIMA": "No seasonal period confirmed; SARIMA would overfit.", "EWMA": "Series has patterns that ARIMA can better capture.", + "Prophet": ( + "No strong seasonal pattern or trend confirmed; ARIMA is " + "lighter and captures the autocorrelation more directly." + ), } return reasons.get(model, "Not selected based on heuristic reasoning.") @@ -367,9 +435,9 @@ def _match_exact(normalized_lower: str) -> str | None: normalized_lower: Lower-cased, normalized LLM output. Returns: - The matched model name, or ``None`` if no exact match is found. + The matched enabled model name, or ``None`` if no exact match. """ - for m in _MODELS: + for m in get_enabled_models(): if f"selected model: {m.lower()}" in normalized_lower: return m return None @@ -391,7 +459,7 @@ def _match_line_scan(normalized: str) -> str | None: if "selected model" not in line.lower(): continue upper_line = line.upper() - for m in sorted(_MODELS, key=len, reverse=True): + for m in sorted(get_enabled_models(), key=len, reverse=True): if m.upper() in upper_line: return m return None @@ -568,6 +636,30 @@ def _statistical_fit_reason( "EWMA is simple and stable, but it can miss autocorrelation that a " "time-series model can use for more accurate forecasts." ) + if model == "Prophet": + if has_seasonality or has_trend: + if selected: + return ( + "Prophet is a strong general-purpose choice because it " + "models trend (with changepoints) and seasonality together " + "and is robust to outliers and missing values." + ) + return ( + "Prophet can model trend and seasonality, but it is heavier to " + "fit than the selected model and offered weaker validation " + "evidence here." + ) + if selected: + return ( + "Prophet was selected for its robust trend modeling, although " + "limited seasonality or trend means simpler models may be " + "competitive." + ) + return ( + "Prophet was rejected because the series lacks strong trend or " + "seasonality, so its extra complexity and fit cost are not " + "justified for this case." + ) return ( "Selected for the best balance of validation accuracy, assumptions, " "and reliability." @@ -606,8 +698,12 @@ def _business_selection_reasons( all_metrics: dict[str, dict[str, float]] | None = None, ) -> dict[str, str | None]: """Build per-model business explanations for selection and rejection.""" + enabled = set(get_enabled_models()) reasons: dict[str, str | None] = {} - for model in _MODELS: + for model in MODEL_NAMES: + if model not in enabled: + reasons[model] = _DISABLED_REASON + continue if model == selected_model: reasons[model] = None continue @@ -625,7 +721,11 @@ def build_model_rejection_reasons( all_metrics: dict[str, dict[str, float]] | None = None, excluded_models: list[str] | None = None, ) -> dict[str, str | None]: - """Build final rejection reasons aligned to the production model.""" + """Build final rejection reasons aligned to the production selection. + + This public helper keeps retry paths consistent with the initial model + selection, including administrator-disabled and review-excluded models. + """ reasons = _business_selection_reasons(selected_model, stat_result, all_metrics) for excluded_model in excluded_models or []: if excluded_model in reasons and excluded_model != selected_model: @@ -641,22 +741,6 @@ def build_model_rejection_reasons( # ── LLM invocation ─────────────────────────────────────────────────────────── -_NOT_AVAILABLE = "not available" - - -def _format_metric_value( - value: float | None, - fmt: str, - percent: bool = False, -) -> str: - """Format a nullable metric value, returning ``_NOT_AVAILABLE`` for None/NaN.""" - if value is None or not np.isfinite(value): - return _NOT_AVAILABLE - if percent: - return format(value * 100, fmt) + "%" - return format(value, fmt) - - def _format_metrics_text( all_metrics: dict[str, dict[str, float]], ) -> str: @@ -673,17 +757,13 @@ def _format_metrics_text( return "" lines = [] for name, metrics in all_metrics.items(): - rmse_s = _format_metric_value(metrics.get("RMSE"), ".4f") - mae_s = _format_metric_value(metrics.get("MAE"), ".4f") - mape_value = _format_metric_value(metrics.get("MAPE"), ".2f") - mape_s = ( - f"{mape_value}%" if mape_value != _NOT_AVAILABLE else _NOT_AVAILABLE - ) - wape_s = _format_metric_value(metrics.get("WAPE"), ".2f", percent=True) - mase_s = _format_metric_value(metrics.get("MASE"), ".4f") + rmse = metrics.get("RMSE", float("nan")) + mae = metrics.get("MAE", float("nan")) + mape = metrics.get("MAPE", float("nan")) + wape = metrics.get("WAPE", float("nan")) * 100 + mase = metrics.get("MASE", float("nan")) lines.append( - f"- {name}: RMSE={rmse_s}, MAE={mae_s}, MAPE={mape_s}, " - f"WAPE={wape_s}, MASE={mase_s}" + f"- {name}: RMSE={rmse:.4f}, MAE={mae:.4f}, MAPE={mape:.2f}%, WAPE={wape:.2f}%, MASE={mase:.4f}" ) return ( "\n".join(lines) @@ -811,137 +891,6 @@ def _invoke_llm( return None -# ── Deterministic policy helpers ────────────────────────────────────────────── - - -def _finite_or_none(value: float | None) -> float | None: - """Return the value if finite, otherwise ``None``.""" - if value is not None and math.isfinite(value): - return value - return None - - -def _build_adapter_result( - name: str, - metrics: dict[str, float], -) -> "ForecastAdapterResult": - """Build a :class:`ForecastAdapterResult` from a metrics dict. - - Args: - name: Model name. - metrics: Dict of metric values (uppercase keys). - - Returns: - A :class:`ForecastAdapterResult` with typed metrics. - """ - from forecasting.contracts import ( - ForecastAdapterResult, - ForecastFitStatus, - ForecastMetrics, - ) - - rmse = _finite_or_none(metrics.get("RMSE")) - mae = _finite_or_none(metrics.get("MAE")) - mape = _finite_or_none(metrics.get("MAPE")) - wape = _finite_or_none(metrics.get("WAPE")) - mase = _finite_or_none(metrics.get("MASE")) - - has_finite = any(v is not None for v in (rmse, mae, mape, wape, mase)) - status = ForecastFitStatus.OK if has_finite else ForecastFitStatus.FAILED - - return ForecastAdapterResult( - status=status, - forecast=[], - lower_ci=[], - upper_ci=[], - metrics=ForecastMetrics( - rmse=rmse, - mae=mae, - mape=mape, - wape=wape, - mase=mase, - ), - fitted_configuration={"model": name}, - ) - - -def _build_candidate_evidence( - all_metrics: dict[str, dict[str, float]], -) -> list[CandidateEvidence]: - """Build :class:`CandidateEvidence` objects from the metrics dict. - - The metrics dict uses uppercase keys (``"RMSE"``, ``"MAE"``, etc.) from - the forecasting agent. This helper converts them to typed - :class:`ForecastAdapterResult`-backed evidence so the deterministic - policy can rank them. - - Args: - all_metrics: Dict mapping model names to metric dicts. - - Returns: - A list of :class:`CandidateEvidence` objects. - """ - candidates: list[CandidateEvidence] = [] - for name, metrics in all_metrics.items(): - is_baseline = name.lower().startswith( - ("naive", "seasonal naive", "mean", "drift") - ) - adapter_result = _build_adapter_result(name, metrics) - candidates.append( - CandidateEvidence( - name=name, - adapter_result=adapter_result, - is_baseline=is_baseline, - ) - ) - return candidates - - -def _build_deterministic_explanation( - outcome: SelectionOutcome, - stat_result: StatisticalResult, - all_metrics: dict[str, dict[str, float]], - review_feedback: str | None, -) -> str: - """Build a business-readable explanation for the deterministic selection. - - Args: - outcome: The deterministic selection outcome. - stat_result: Output of the statistical analysis agent. - all_metrics: Dict of all model error metrics. - review_feedback: Optional review feedback from a prior run. - - Returns: - A concise explanation string. - """ - parts = [f"Selected model: {outcome.selected_model}."] - metric = _primary_metric(all_metrics, outcome.selected_model) - if metric: - metric_name, value = metric - evidence_scope = "eligible " if outcome.exclusion_reasons else "" - parts.append( - f"It had the strongest available {evidence_scope}empirical validation metrics " - f"({_format_metric(metric_name, value)}, lower is better)." - ) - parts.append( - _statistical_fit_reason(stat_result, outcome.selected_model, selected=True) - ) - if outcome.tie_break_note: - parts.append(f"Tie-breaking: {outcome.tie_break_note}") - if outcome.exclusion_reasons: - excluded = ", ".join(outcome.exclusion_reasons.keys()) - parts.append(f"Excluded candidates: {excluded}.") - if review_feedback: - parts.append( - "The selection also accounts for statistical review feedback from " - "the prior run." - ) - metrics_text = _format_metrics_text(all_metrics) - parts.append(f"\n\nValidation metrics considered:\n{metrics_text}") - parts.append(f"\n[Statistical Review Feedback]: {review_feedback or 'N/A'}") - return " ".join(parts) - - # ── Public entry point ─────────────────────────────────────────────────────── @@ -980,49 +929,50 @@ def run_model_selection_agent( stat_result, fallback_model, exclude_model ) - # ── Deterministic policy when empirical metrics are available ──────── - # When actual error metrics are available, the deterministic selection - # policy is the source of truth. The LLM never decides model rankings. - # The policy excludes failed/degraded candidates, ranks by the - # configured loss metric, applies tie-breaking (simpler model wins - # negligible differences), and retains baselines when no complex model - # adds demonstrated value. + # ── Deterministic override when empirical metrics are available ──────── + # During a review-triggered retry, if actual error metrics are available, + # deterministically select the best-performing model rather than relying + # on the LLM. This prevents the LLM from re-selecting a suboptimal model + # based on statistical properties alone. if all_metrics: - candidates = _build_candidate_evidence(all_metrics) - outcome = select_model_deterministic( - candidates, - exclude_models=[exclude_model] if exclude_model else None, - user_loss_preference=loss_preference, - ) - if outcome.selected_model: + best_model = _select_best_metric_model(all_metrics, exclude_model) + if best_model: logger.info( - "Deterministic policy selected '%s' (method=%s, rankable=%d).", - outcome.selected_model, - outcome.method, - len(outcome.ranking), + "Deterministic override: selecting best-metric model '%s' " + "based on empirical error metrics.", + best_model, ) metrics_text = _format_metrics_text(all_metrics) - explanation = _build_deterministic_explanation( - outcome, stat_result, all_metrics, review_feedback + evidence_scope = "eligible " if exclude_model else "" + explanation = ( + "Model re-selected based on " + f"{evidence_scope}empirical validation metrics. " + + _build_selection_explanation( + best_model, stat_result, all_metrics, review_feedback + ) + + "\n\nValidation metrics considered:\n" + + metrics_text + + f"\n\n[Statistical Review Feedback]: {review_feedback or 'N/A'}" ) reasons = build_model_rejection_reasons( - outcome.selected_model, + best_model, stat_result, all_metrics, - list(outcome.exclusion_reasons), + [exclude_model] if exclude_model else None, ) return ModelSelectionResult( - selected_model=outcome.selected_model, + selected_model=best_model, explanation=explanation, holt_winters_rejected_reason=reasons["Holt-Winters"], arima_rejected_reason=reasons["ARIMA"], sarima_rejected_reason=reasons["SARIMA"], ewma_rejected_reason=reasons["EWMA"], + prophet_rejected_reason=reasons["Prophet"], reasoning_steps=[ { "thought": ( - "Deterministic selection policy applied with " - "empirical metrics." + "Review-triggered retry with empirical metrics " + "available — selecting best-performing model." ), "observation": metrics_text, }, @@ -1030,21 +980,9 @@ def run_model_selection_agent( token_usage={}, selection_method="deterministic", selection_evidence={ - "ranking": outcome.ranking, - "exclusion_reasons": outcome.exclusion_reasons, - "tie_break_note": outcome.tie_break_note, - "evidence_summary": outcome.evidence_summary, + "excluded_model": exclude_model, + "loss_preference": loss_preference, }, - narrative_claims=[ - { - "claim": f"Selected {outcome.selected_model} by deterministic ranking.", - "evidence_references": [ - "selection_evidence.ranking", - "all_metrics", - ], - "uncertainty": "empirical_backtest_evidence", - } - ], ) suitability_input = _build_suitability_input( @@ -1056,13 +994,6 @@ def run_model_selection_agent( return _build_heuristic_result(fallback_model, fallback_reasoning, stat_result) output, token_usage = llm_result - validation_warnings = validate_llm_output( - output, - list(_MODELS), - {"all_metrics": all_metrics or {}}, - ) - if validation_warnings: - logger.warning("Model-selection narrative validation: %s", validation_warnings) selected_model = _parse_selected_model(output, fallback_model) reasons = _business_selection_reasons(selected_model, stat_result) explanation = ( @@ -1080,6 +1011,7 @@ def run_model_selection_agent( arima_rejected_reason=reasons["ARIMA"], sarima_rejected_reason=reasons["SARIMA"], ewma_rejected_reason=reasons["EWMA"], + prophet_rejected_reason=reasons["Prophet"], reasoning_steps=[ { "thought": "Assessing suitability metrics for all models...", @@ -1091,15 +1023,6 @@ def run_model_selection_agent( }, ], token_usage=token_usage, - selection_method="llm", - selection_evidence={"llm_validation_warnings": validation_warnings}, - narrative_claims=[ - { - "claim": f"LLM interpreted suitability for {selected_model}.", - "evidence_references": ["selection_evidence.llm_validation_warnings"], - "uncertainty": "llm_interpretation", - } - ], ) @@ -1135,6 +1058,7 @@ def _build_heuristic_result( arima_rejected_reason=reasons["ARIMA"], sarima_rejected_reason=reasons["SARIMA"], ewma_rejected_reason=reasons["EWMA"], + prophet_rejected_reason=reasons["Prophet"], reasoning_steps=[ { "thought": "Model selection agent failed; using heuristic.", @@ -1142,13 +1066,4 @@ def _build_heuristic_result( } ], token_usage={}, - selection_method="heuristic", - selection_evidence={}, - narrative_claims=[ - { - "claim": f"Selected {fallback_model} using heuristic fallback.", - "evidence_references": ["reasoning_steps"], - "uncertainty": "heuristic", - } - ], ) diff --git a/data_forecaster/backend/agents/report_generation_agent.py b/data_forecaster/backend/agents/report_generation_agent.py index fa0d5cf..79f7181 100644 --- a/data_forecaster/backend/agents/report_generation_agent.py +++ b/data_forecaster/backend/agents/report_generation_agent.py @@ -102,8 +102,18 @@ def run_report_agent( report, token_usage = generate_narratives(report, user_prompt) reasoning_steps.append( { - "thought": "Stage 2 complete: Narratives generated", - "observation": (f"Tokens: {token_usage.get('total_tokens', 0)}"), + "thought": ( + "Stage 2 complete: Deterministic narrative fallback used" + if report.metadata.llm_narrative_fallback + else "Stage 2 complete: Narratives generated by LLM" + ), + "observation": ( + "Fallback sections: " + + ", ".join(report.metadata.llm_fallback_sections) + if report.metadata.llm_narrative_fallback + else f"Tokens: {token_usage.get('total_tokens', 0)}" + ), + "llm_fallback": report.metadata.llm_narrative_fallback, } ) except Exception as exc: @@ -123,18 +133,43 @@ def run_report_agent( "llm_fallback": True, } ) + report.metadata.llm_narrative_fallback = True + report.metadata.llm_fallback_sections = [ + "executive_summary", + "data_quality", + "historical_analysis", + "forecast_outlook", + "model_comparison", + "statistical_audit", + "explainability", + *["recommendation"] * len(report.recommendations), + ] # Ensure all narrative fields have a fallback value - report.executive_summary.narrative = _fallback_narrative("executive_summary") - report.data_quality.narrative = _fallback_narrative("data_quality") + report.executive_summary.narrative = _fallback_narrative( + report.executive_summary, "executive_summary" + ) + report.data_quality.narrative = _fallback_narrative( + report.data_quality, "data_quality" + ) report.historical_analysis.narrative = _fallback_narrative( - "historical_analysis" + report.historical_analysis, "historical_analysis" + ) + report.forecast_outlook.narrative = _fallback_narrative( + report.forecast_outlook, "forecast_outlook" + ) + report.model_comparison.narrative = _fallback_narrative( + report.model_comparison, "model_comparison" + ) + report.statistical_audit.narrative = _fallback_narrative( + report.statistical_audit, "statistical_audit" + ) + report.explainability.narrative = _fallback_narrative( + report.explainability, "explainability" ) - report.forecast_outlook.narrative = _fallback_narrative("forecast_outlook") - report.model_comparison.narrative = _fallback_narrative("model_comparison") - report.statistical_audit.narrative = _fallback_narrative("statistical_audit") - report.explainability.narrative = _fallback_narrative("explainability") report.recommendations = [ - rec.model_copy(update={"narrative": _fallback_narrative("recommendation")}) + rec.model_copy( + update={"narrative": _fallback_narrative(rec, "recommendation")} + ) for rec in report.recommendations ] diff --git a/data_forecaster/backend/agents/statistical_review_agent.py b/data_forecaster/backend/agents/statistical_review_agent.py index 4745456..386cec3 100644 --- a/data_forecaster/backend/agents/statistical_review_agent.py +++ b/data_forecaster/backend/agents/statistical_review_agent.py @@ -231,7 +231,9 @@ def _check_explanation_mismatch( """ explanation_lower = model_selection.explanation.lower() other_models = [ - m for m in ("ARIMA", "SARIMA", "Holt-Winters", "EWMA") if m != selected + m + for m in ("ARIMA", "SARIMA", "Holt-Winters", "EWMA", "Prophet") + if m != selected ] mentioned_models = [m for m in other_models if m.lower() in explanation_lower[:200]] # Only flag if another model is mentioned prominently in the first 200 diff --git a/data_forecaster/backend/core/config.py b/data_forecaster/backend/core/config.py index 85cc245..8a444b2 100644 --- a/data_forecaster/backend/core/config.py +++ b/data_forecaster/backend/core/config.py @@ -9,10 +9,6 @@ import os -from dotenv import load_dotenv - -load_dotenv() - GOOGLE_API_KEY: str | None = os.getenv("GOOGLE_API_KEY") GEMINI_MODEL: str = os.getenv("GEMINI_MODEL", "gemini-1.5-flash") GEMINI_TEMPERATURE: float = float(os.getenv("GEMINI_TEMPERATURE", "0.1")) @@ -68,12 +64,19 @@ FILE_STORAGE_DIR: str = os.getenv("FILE_STORAGE_DIR", "./file_store") BACKEND_DB_PATH: str = os.getenv("BACKEND_DB_PATH", "./data/backend.db") + +# Directory holding the backend's generated secrets (Fernet encryption +# key, ...). This MUST be a dedicated named volume in Docker deployments +# (``secret_data:/app/secrets``) — never a host bind mount, which would +# leak secrets into the working tree. The key file is created with mode +# 0600 by ``core.secret_store``. +SECRETS_DIR: str = os.getenv("SECRETS_DIR", "./secrets") + API_KEY_ENABLED: bool = os.getenv("API_KEY_ENABLED", "false").lower() == "true" -# Deployment-time secret used to protect the one-time bootstrap endpoint -# (POST /api-users/bootstrap). Set this to a strong random value in the -# backend .env and share it with the admin who will enable API authentication. -ADMIN_API_KEY: str | None = os.getenv("ADMIN_API_KEY") +# NOTE: The former ADMIN_API_KEY deployment secret was removed with the +# retirement of POST /api-users/bootstrap. First-run provisioning now +# uses the race-safe POST /setup/bootstrap guarded by "no users exist". # Pre-shared credentials for the frontend service account. When both are # set, the backend auto-creates a ``frontend`` API user on first startup @@ -116,20 +119,23 @@ def validate_llm_config() -> None: """Validate that at least one LLM provider is properly configured. Called at startup to fail fast when the backend cannot reach any LLM - provider. Raises ``RuntimeError`` with a descriptive message when - the configuration is incomplete. + provider. Reads the effective configuration (DB-backed with env + fallback) via :func:`core.llm_config_store.get_llm_config`. Raises: RuntimeError: When no LLM provider is available. """ - if USE_OLLAMA: - if USE_OLLAMA_CLOUD and not OLLAMA_API_KEY: - raise RuntimeError( - "USE_OLLAMA_CLOUD is enabled but OLLAMA_API_KEY is not set. " - "Configure it in the backend .env file." - ) - elif not GOOGLE_API_KEY: + # Imported here to avoid a circular import at module load time. + from core.llm_config_store import get_llm_config + + config = get_llm_config() + if config.provider == "ollama_cloud" and not config.api_key: + raise RuntimeError( + "Ollama Cloud is enabled but no API key is configured. " + "Set it via the admin LLM configuration page." + ) + if config.provider == "gemini" and not config.api_key: raise RuntimeError( - "No LLM provider configured. Either set USE_OLLAMA=true with a " - "running Ollama instance, or set GOOGLE_API_KEY for Gemini." + "No LLM provider configured. Either configure Ollama, or set a " + "Gemini API key via the admin LLM configuration page." ) diff --git a/data_forecaster/backend/core/database.py b/data_forecaster/backend/core/database.py index 79fac9a..89ca009 100644 --- a/data_forecaster/backend/core/database.py +++ b/data_forecaster/backend/core/database.py @@ -84,6 +84,51 @@ INSERT OR IGNORE INTO forecast_job_settings (singleton, max_running_jobs_per_user, retention_days, cleanup_enabled) VALUES (1, 1, 30, 1); + +-- Singleton LLM provider configuration. The API key is stored as a +-- Fernet ciphertext (see core/secret_store.py) and is never readable via +-- the API. ``version`` is incremented on every write so that cached +-- readers (core/llm_config_store) can invalidate cheaply. +CREATE TABLE IF NOT EXISTS llm_config ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + provider TEXT NOT NULL, + model TEXT NOT NULL, + base_url TEXT, + encrypted_api_key TEXT, + temperature REAL NOT NULL DEFAULT 0.1, + version INTEGER NOT NULL DEFAULT 1, + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- Enable/disable state for forecasting models. Seeded with the five +-- supported models, all enabled. ``forecasting/registry.py`` is the +-- canonical catalog; this table holds only mutable state. +CREATE TABLE IF NOT EXISTS model_config ( + name TEXT PRIMARY KEY, + enabled INTEGER NOT NULL DEFAULT 1, + priority INTEGER NOT NULL DEFAULT 0 +); + +INSERT OR IGNORE INTO model_config (name, enabled, priority) VALUES + ('ARIMA', 1, 10), + ('SARIMA', 1, 20), + ('Holt-Winters', 1, 30), + ('EWMA', 1, 40), + ('Prophet', 1, 50); + +-- Singleton system-wide setup and deployment state. ``setup_complete`` +-- gates the first-run wizard; once true the DB is authoritative and +-- env-based service-user reconciliation is skipped. +CREATE TABLE IF NOT EXISTS system_settings ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + setup_complete INTEGER NOT NULL DEFAULT 0, + worker_mode TEXT NOT NULL DEFAULT 'standalone', + worker_enrollment_token_hash TEXT, + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +INSERT OR IGNORE INTO system_settings (singleton, setup_complete, worker_mode) +VALUES (1, 0, 'standalone'); """ diff --git a/data_forecaster/backend/core/llm_config_store.py b/data_forecaster/backend/core/llm_config_store.py new file mode 100644 index 0000000..55c97a4 --- /dev/null +++ b/data_forecaster/backend/core/llm_config_store.py @@ -0,0 +1,212 @@ +"""DB-backed LLM provider configuration with env fallback. + +The ``llm_config`` singleton table is the authoritative source of LLM +settings once it exists (i.e. after setup). Before setup — or on installs +that have never written the row — values fall back to the environment +variables in :mod:`core.config`, preserving backward compatibility. + +The API key is stored encrypted (Fernet, :mod:`core.secret_store`) and is +only ever decrypted in-process, at call time. Reads are cached keyed on +the row's monotonically increasing ``version`` so writes invalidate +cheaply without a process restart. +""" + +from __future__ import annotations + +import sqlite3 +import threading +from dataclasses import dataclass + +import core.config as settings +from core import secret_store +from core.database import get_connection +from core.logging_config import get_logger + +logger = get_logger(__name__) + +_PROVIDER_GEMINI = "gemini" +_PROVIDER_OLLAMA = "ollama" +_PROVIDER_OLLAMA_CLOUD = "ollama_cloud" + +_cache: LLMConfig | None = None +_cache_version: int | None = None +_cache_lock = threading.Lock() + + +@dataclass(frozen=True) +class LLMConfig: + """Resolved LLM provider configuration. + + Attributes: + provider: One of ``gemini``, ``ollama``, or ``ollama_cloud``. + model: Model name (e.g. ``gemini-1.5-flash``, ``llama3``). + base_url: Provider base URL; ``None`` for Gemini. + api_key: Decrypted API key; ``None`` when not required or unset. + temperature: Default sampling temperature. + version: Row version this config was read from; ``0`` when the + values came from the environment fallback. + """ + + provider: str + model: str + base_url: str | None + api_key: str | None + temperature: float + version: int + + +def _env_fallback() -> LLMConfig: + """Build an :class:`LLMConfig` from environment variables.""" + if settings.USE_OLLAMA and settings.USE_OLLAMA_CLOUD: + return LLMConfig( + provider=_PROVIDER_OLLAMA_CLOUD, + model=settings.OLLAMA_MODEL, + base_url=settings.OLLAMA_BASE_URL, + api_key=settings.OLLAMA_API_KEY, + temperature=settings.GEMINI_TEMPERATURE, + version=0, + ) + if settings.USE_OLLAMA: + return LLMConfig( + provider=_PROVIDER_OLLAMA, + model=settings.OLLAMA_MODEL, + base_url=settings.OLLAMA_BASE_URL, + api_key=settings.OLLAMA_API_KEY, + temperature=settings.GEMINI_TEMPERATURE, + version=0, + ) + return LLMConfig( + provider=_PROVIDER_GEMINI, + model=settings.GEMINI_MODEL, + base_url=None, + api_key=settings.GOOGLE_API_KEY, + temperature=settings.GEMINI_TEMPERATURE, + version=0, + ) + + +def _read_row(connection: sqlite3.Connection) -> sqlite3.Row | None: + """Return the singleton ``llm_config`` row, or ``None`` if unset.""" + return connection.execute( + "SELECT provider, model, base_url, encrypted_api_key, temperature," + " version FROM llm_config WHERE singleton = 1" + ).fetchone() + + +def get_llm_config(db_path: str | None = None) -> LLMConfig: + """Return the effective LLM configuration, DB-first with env fallback. + + Results are cached and keyed on the row ``version``; a write via + :func:`put_llm_config` makes the next call re-read the row. + + Args: + db_path: Optional database path override (testing). + + Returns: + The resolved :class:`LLMConfig`. + """ + global _cache, _cache_version + with _cache_lock: + with get_connection(db_path) as connection: + row = _read_row(connection) + if row is None: + return _env_fallback() + version = int(row["version"]) + if _cache is not None and _cache_version == version: + return _cache + encrypted_key = row["encrypted_api_key"] + api_key = secret_store.decrypt(encrypted_key) if encrypted_key else None + config = LLMConfig( + provider=row["provider"], + model=row["model"], + base_url=row["base_url"], + api_key=api_key, + temperature=float(row["temperature"]), + version=version, + ) + _cache = config + _cache_version = version + return config + + +def put_llm_config( + provider: str, + model: str, + base_url: str | None, + api_key: str | None, + temperature: float, + db_path: str | None = None, +) -> None: + """Write the LLM configuration, encrypting the key and bumping version. + + Args: + provider: One of ``gemini``, ``ollama``, ``ollama_cloud``. + model: Model name. + base_url: Provider base URL; ``None`` for Gemini. + api_key: Plaintext key to encrypt and store; ``None`` keeps the + existing stored key (or stores none if no row exists yet). + temperature: Default sampling temperature. + db_path: Optional database path override (testing). + + Raises: + ValueError: When the provider is not recognised. + """ + if provider not in (_PROVIDER_GEMINI, _PROVIDER_OLLAMA, _PROVIDER_OLLAMA_CLOUD): + raise ValueError(f"Unknown LLM provider: {provider=}") + if api_key: + # Idempotent: keeps the existing key when already generated. + secret_store.generate_and_persist_key() + encrypted_key = secret_store.encrypt(api_key) if api_key else None + with get_connection(db_path) as connection: + if encrypted_key is None: + connection.execute( + "INSERT INTO llm_config" + " (singleton, provider, model, base_url, temperature," + " version, updated_at)" + " VALUES (1, ?, ?, ?, ?, 1, datetime('now'))" + " ON CONFLICT(singleton) DO UPDATE SET" + " provider = excluded.provider," + " model = excluded.model," + " base_url = excluded.base_url," + " temperature = excluded.temperature," + " version = llm_config.version + 1," + " updated_at = datetime('now')", + (provider, model, base_url, temperature), + ) + else: + connection.execute( + "INSERT INTO llm_config" + " (singleton, provider, model, base_url, encrypted_api_key," + " temperature, version, updated_at)" + " VALUES (1, ?, ?, ?, ?, ?, 1, datetime('now'))" + " ON CONFLICT(singleton) DO UPDATE SET" + " provider = excluded.provider," + " model = excluded.model," + " base_url = excluded.base_url," + " encrypted_api_key = excluded.encrypted_api_key," + " temperature = excluded.temperature," + " version = llm_config.version + 1," + " updated_at = datetime('now')", + (provider, model, base_url, encrypted_key, temperature), + ) + connection.commit() + logger.info( + "LLM config updated (provider=%s, model=%s, key_updated=%s)", + provider, + model, + api_key is not None, + ) + + +def is_configured(db_path: str | None = None) -> bool: + """Return ``True`` when an ``llm_config`` row exists in the DB.""" + with get_connection(db_path) as connection: + return _read_row(connection) is not None + + +def reset_cache() -> None: + """Drop the cached config (test support only).""" + global _cache, _cache_version + with _cache_lock: + _cache = None + _cache_version = None diff --git a/data_forecaster/backend/core/llm_factory.py b/data_forecaster/backend/core/llm_factory.py index febbb70..98a8e14 100644 --- a/data_forecaster/backend/core/llm_factory.py +++ b/data_forecaster/backend/core/llm_factory.py @@ -19,7 +19,7 @@ from langchain_google_genai import ChatGoogleGenerativeAI from langchain_ollama import ChatOllama -import core.config as config +from core.llm_config_store import get_llm_config from core.logging_config import get_logger from exceptions import LLMConfigError @@ -45,48 +45,55 @@ def get_llm(temperature: float = 0.0) -> BaseChatModel: A configured :class:`BaseChatModel` instance. Raises: - LLMConfigError: When Ollama Cloud is enabled but - ``OLLAMA_API_KEY`` is not set. + LLMConfigError: When Ollama Cloud is enabled but no API key is + configured. """ - if config.USE_OLLAMA and config.USE_OLLAMA_CLOUD: - if not config.OLLAMA_API_KEY: + config = get_llm_config() + if config.provider == "ollama_cloud": + if not config.api_key: raise LLMConfigError( - "USE_OLLAMA_CLOUD is enabled but OLLAMA_API_KEY is not " - "set. Create an API key at https://ollama.com/settings/keys " - "and set the OLLAMA_API_KEY environment variable." + "Ollama Cloud is enabled but no API key is configured. " + "Create an API key at https://ollama.com/settings/keys and " + "set it via the admin LLM configuration page." ) logger.info( "Using Ollama Cloud (model=%s, base_url=%s)", - config.OLLAMA_MODEL, - config.OLLAMA_BASE_URL, + config.model, + config.base_url, ) return ChatOllama( - model=config.OLLAMA_MODEL, - base_url=config.OLLAMA_BASE_URL, + model=config.model, + base_url=config.base_url, temperature=temperature, - headers={"Authorization": f"Bearer {config.OLLAMA_API_KEY}"}, + # ChatOllama passes HTTP settings through ``client_kwargs``. + # A top-level ``headers`` argument is ignored by current + # langchain-ollama releases, which results in a 401 from Ollama + # Cloud despite a successful direct connection test. + client_kwargs={ + "headers": {"Authorization": f"Bearer {config.api_key}"} + }, ) - if config.USE_OLLAMA: + if config.provider == "ollama": logger.info( "Using local Ollama (model=%s, base_url=%s)", - config.OLLAMA_MODEL, - config.OLLAMA_BASE_URL, + config.model, + config.base_url, ) return ChatOllama( - model=config.OLLAMA_MODEL, - base_url=config.OLLAMA_BASE_URL, + model=config.model, + base_url=config.base_url, temperature=temperature, - headers=( - {"Authorization": f"Bearer {config.OLLAMA_API_KEY}"} - if config.OLLAMA_API_KEY + client_kwargs=( + {"headers": {"Authorization": f"Bearer {config.api_key}"}} + if config.api_key else None ), ) - logger.info("Using Google Gemini (model=%s)", config.GEMINI_MODEL) + logger.info("Using Google Gemini (model=%s)", config.model) return ChatGoogleGenerativeAI( - model=config.GEMINI_MODEL, - google_api_key=config.GOOGLE_API_KEY, + model=config.model, + google_api_key=config.api_key, temperature=temperature, ) diff --git a/data_forecaster/backend/core/secret_store.py b/data_forecaster/backend/core/secret_store.py new file mode 100644 index 0000000..65bc353 --- /dev/null +++ b/data_forecaster/backend/core/secret_store.py @@ -0,0 +1,130 @@ +"""Backend symmetric encryption helpers for secrets stored at rest. + +Uses Fernet (AES-128-CBC + HMAC-SHA256) from the ``cryptography`` library, +mirroring ``frontend/db/crypto.py``. Unlike the frontend — which reads its +key from an environment variable — the backend GENERATES its key at setup +time and persists it to a dedicated secrets volume (``SECRETS_DIR``) with +mode 0600 so that no secret ever needs to live in a ``.env`` file. + +Losing the key file renders all stored ciphertext unrecoverable; see +``docs/`` for the backup and rotation runbook. +""" + +from __future__ import annotations + +import os +import threading + +from cryptography.fernet import Fernet + +import core.config as settings +from core.logging_config import get_logger + +logger = get_logger(__name__) + +_KEY_FILENAME = ".encryption_key" + +_fernet: Fernet | None = None +_fernet_lock = threading.Lock() + + +def _key_path() -> str: + """Return the absolute path of the persisted encryption key file.""" + return os.path.join(settings.SECRETS_DIR, _KEY_FILENAME) + + +def key_file_exists() -> bool: + """Return ``True`` when the persisted encryption key file is present.""" + return os.path.isfile(_key_path()) + + +def generate_and_persist_key() -> None: + """Generate a new Fernet key and persist it to the secrets volume. + + The key file is written with mode 0600. The write is skipped (and the + existing key kept) when the file already exists, so this function is + safe to call on every setup bootstrap. + + Raises: + OSError: When the key file cannot be written. + """ + path = _key_path() + os.makedirs(os.path.dirname(path), exist_ok=True) + if os.path.isfile(path): + logger.info("Encryption key already present at %s; keeping it.", path) + return + key = Fernet.generate_key() + # Open with O_EXCL so a concurrent first-run never truncates an + # existing key, and set 0600 before any content is written. + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + os.write(fd, key) + finally: + os.close(fd) + logger.info("Generated new encryption key at %s (mode 0600).", path) + + +def get_fernet() -> Fernet: + """Return the process-wide Fernet instance, loading the key from disk. + + Returns: + A :class:`~cryptography.fernet.Fernet` initialised from the + persisted key. + + Raises: + RuntimeError: When the key file is missing. Callers that have + stored ciphertext must treat this as fatal — the data is + unrecoverable without the key. + """ + global _fernet + if _fernet is not None: + return _fernet + with _fernet_lock: + if _fernet is not None: + return _fernet + path = _key_path() + if not os.path.isfile(path): + raise RuntimeError( + f"Backend encryption key not found at {path}. Stored " + "credentials cannot be decrypted without it. Restore the " + "key from backup, or wipe the stored configuration and run " + "setup again." + ) + with open(path, "rb") as handle: + _fernet = Fernet(handle.read().strip()) + return _fernet + + +def encrypt(plaintext: str) -> str: + """Encrypt *plaintext* and return a URL-safe base64 ciphertext string. + + Args: + plaintext: The secret value to encrypt. + + Returns: + The encrypted token, safe to store in the database. + """ + return get_fernet().encrypt(plaintext.encode()).decode() + + +def decrypt(ciphertext: str) -> str: + """Decrypt *ciphertext* and return the original plaintext. + + Args: + ciphertext: A token previously produced by :func:`encrypt`. + + Returns: + The decrypted plaintext string. + + Raises: + cryptography.fernet.InvalidToken: When the token is invalid or the + key does not match. + """ + return get_fernet().decrypt(ciphertext.encode()).decode() + + +def reset_cache() -> None: + """Drop the cached Fernet instance (test support only).""" + global _fernet + with _fernet_lock: + _fernet = None diff --git a/data_forecaster/backend/forecasting/arima_model.py b/data_forecaster/backend/forecasting/arima_model.py index 18dcb22..ffb695d 100644 --- a/data_forecaster/backend/forecasting/arima_model.py +++ b/data_forecaster/backend/forecasting/arima_model.py @@ -12,6 +12,7 @@ ForecastMetrics, ) from forecasting.evaluation import evaluate_predictions, make_terminal_holdout +from forecasting.indexing import normalize_forecast_index from forecasting.pmdarima_compat import import_pmdarima logger = get_logger(__name__) @@ -56,7 +57,7 @@ def fit_arima( :class:`ForecastAdapterResult` with status, forecast, intervals, nullable metrics, and fitted configuration provenance. """ - series = series.dropna().astype(float) + series = normalize_forecast_index(series.dropna().astype(float)) if len(series) < 3: logger.warning( diff --git a/data_forecaster/backend/forecasting/ewma_model.py b/data_forecaster/backend/forecasting/ewma_model.py index cd48949..b56f3e9 100644 --- a/data_forecaster/backend/forecasting/ewma_model.py +++ b/data_forecaster/backend/forecasting/ewma_model.py @@ -19,6 +19,7 @@ ForecastMetrics, ) from forecasting.evaluation import evaluate_predictions, make_terminal_holdout +from forecasting.indexing import normalize_forecast_index logger = get_logger(__name__) @@ -77,7 +78,7 @@ def fit_ewma( :class:`ForecastAdapterResult` with status, forecast, intervals, nullable metrics, and fitted configuration provenance. """ - series = series.dropna().astype(float) + series = normalize_forecast_index(series.dropna().astype(float)) if len(series) < 3: logger.warning( diff --git a/data_forecaster/backend/forecasting/holt_winters.py b/data_forecaster/backend/forecasting/holt_winters.py index bef5bec..22ce81d 100644 --- a/data_forecaster/backend/forecasting/holt_winters.py +++ b/data_forecaster/backend/forecasting/holt_winters.py @@ -11,6 +11,7 @@ from core.logging_config import get_logger from forecasting.contracts import ForecastAdapterResult, ForecastFitStatus, ForecastMetrics from forecasting.evaluation import evaluate_predictions, make_terminal_holdout +from forecasting.indexing import normalize_forecast_index logger = get_logger(__name__) @@ -102,7 +103,7 @@ def fit_holt_winters( mase_period: int = 1, ) -> ForecastAdapterResult: """Select the Holt-Winters form on training data and refit it on all data.""" - series = series.dropna().astype(float) + series = normalize_forecast_index(series.dropna().astype(float)) seasonal_period = max(1, int(seasonal_period)) holdout = make_terminal_holdout(series, forecast_horizon) train, test = holdout.train, holdout.test diff --git a/data_forecaster/backend/forecasting/indexing.py b/data_forecaster/backend/forecasting/indexing.py new file mode 100644 index 0000000..36aaded --- /dev/null +++ b/data_forecaster/backend/forecasting/indexing.py @@ -0,0 +1,38 @@ +"""Index normalization for forecasting libraries with strict index support.""" + +from __future__ import annotations + +import pandas as pd + +from core.logging_config import get_logger + +logger = get_logger(__name__) + + +def normalize_forecast_index(series: pd.Series) -> pd.Series: + """Return a series with an index supported by Statsmodels forecasting. + + Statsmodels cannot generate out-of-sample timestamps for a non-unique or + irregular ``DatetimeIndex``. Forecast adapters operate on ordered values, + so use a positional index in those cases while retaining regular datetime + indexes for models that can safely extend them. + + Args: + series: Clean numeric observations in chronological order. + + Returns: + The original series when its index is regular and unique; otherwise a + value-identical series using a ``RangeIndex``. + """ + index = series.index + if ( + isinstance(index, pd.DatetimeIndex) + and index.is_unique + and index.freq is not None + ): + return series + + logger.warning( + "Unsupported forecasting index detected; using a positional index." + ) + return pd.Series(series.to_numpy(dtype=float), name=series.name) diff --git a/data_forecaster/backend/forecasting/prophet_compat.py b/data_forecaster/backend/forecasting/prophet_compat.py new file mode 100644 index 0000000..2d190a6 --- /dev/null +++ b/data_forecaster/backend/forecasting/prophet_compat.py @@ -0,0 +1,41 @@ +"""Compatibility bootstrap for importing prophet (Meta Prophet). + +Prophet is a heavy optional dependency backed by the cmdstanpy toolchain. +To keep the rest of the forecasting package importable when prophet is not +installed, model adapters import it lazily through :func:`import_prophet` +rather than at module load time. This mirrors the +:mod:`forecasting.pmdarima_compat` pattern. +""" + +from __future__ import annotations + +import logging +from types import ModuleType + + +def _silence_chatty_loggers() -> None: + """Quiet the verbose cmdstanpy/prophet loggers before importing prophet. + + Prophet and its cmdstanpy backend log heavily to stdout/stderr while + fitting. Silencing them here (rather than in every caller) keeps the + forecasting pipeline output readable. Idempotent. + """ + for name in ("cmdstanpy", "prophet", "prophet.models"): + logging.getLogger(name).setLevel(logging.WARNING) + + +def import_prophet() -> ModuleType: + """Import and return the ``prophet`` module. + + Returns: + The imported ``prophet`` module (callers use ``prophet.Prophet``). + + Raises: + ImportError: If prophet is not installed. Callers in the forecasting + pipeline wrap this in try/except so the model is skipped rather + than crashing the whole run. + """ + _silence_chatty_loggers() + import prophet # pylint: disable=import-outside-toplevel + + return prophet diff --git a/data_forecaster/backend/forecasting/prophet_model.py b/data_forecaster/backend/forecasting/prophet_model.py new file mode 100644 index 0000000..d587d49 --- /dev/null +++ b/data_forecaster/backend/forecasting/prophet_model.py @@ -0,0 +1,175 @@ +"""Prophet (Meta Prophet) forecasting adapter. + +Prophet is an additive regression model that decomposes a series into trend, +seasonality (daily/weekly/yearly), and holidays. It is robust to missing +values, outliers, and irregular sampling, and produces uncertainty intervals +natively. + +Prophet is imported lazily via :func:`forecasting.prophet_compat.import_prophet` +so this module remains importable when the optional ``prophet`` dependency is +not installed. The forecasting agent wraps ``fit_prophet`` in try/except and +simply skips the model when Prophet is unavailable. +""" + +from __future__ import annotations + +import numpy as np +import pandas as pd + +from core.logging_config import get_logger +from forecasting.prophet_compat import import_prophet + +logger = get_logger(__name__) + + +def _to_history_frame(series: pd.Series) -> pd.DataFrame: + """Convert a Series to the Prophet ``ds``/``y`` history frame. + + Prophet requires a ``ds`` column of datetimes. When the input index is + not datetime-valued, a synthetic regular date range is generated so the + model can still fit. + + Args: + series: A pandas Series (any index). + + Returns: + A two-column DataFrame with ``ds`` (datetime) and ``y`` (float). + """ + if isinstance(series.index, pd.DatetimeIndex): + ds = series.index + else: + ds = pd.date_range(start="2000-01-01", periods=len(series), freq="D") + return pd.DataFrame({"ds": ds, "y": series.values}) + + +def _future_frame( + history: pd.DataFrame, periods: int, freq: str | None +) -> pd.DataFrame: + """Build a Prophet future frame of ``periods`` timestamps after history. + + Args: + history: The history frame (used to anchor the continuation). + periods: Number of future periods to generate. + freq: Optional pandas frequency string. Inferred from the history + dates when not supplied. + + Returns: + A one-column DataFrame (``ds``) covering the future periods. + """ + if freq is None: + freq = pd.infer_freq(history["ds"]) or "D" + last_ds = history["ds"].iloc[-1] + future_dates = pd.date_range(start=last_ds, periods=periods + 1, freq=freq)[1:] + return pd.DataFrame({"ds": future_dates}) + + +def _metrics_from_holdout( + train: pd.Series, + test: pd.Series, + prophet_module, + freq: str | None, +) -> tuple[float, float, float]: + """Fit Prophet on ``train`` and score RMSE/MAE/MAPE against ``test``. + + Args: + train: Training observations. + test: Holdout observations. + prophet_module: The imported ``prophet`` module. + freq: Optional pandas frequency string for the future frame. + + Returns: + ``(rmse, mae, mape)`` — zeroed on any fit/predict failure so the + caller can still produce a full-series forecast. + """ + try: + train_history = _to_history_frame(train) + m = prophet_module.Prophet() + m.fit(train_history) + future = _future_frame(train_history, periods=len(test), freq=freq) + fc = m.predict(future) + pred = fc["yhat"].to_numpy() + actual = test.to_numpy() + residuals = actual - pred + rmse = float(np.sqrt(np.mean(residuals**2))) + mae = float(np.mean(np.abs(residuals))) + mape = float(np.mean(np.abs(residuals / (actual + 1e-8))) * 100) + return rmse, mae, mape + except Exception as exc: # pylint: disable=broad-except + logger.warning("Prophet holdout metrics failed: %s", exc) + return 0.0, 0.0, 0.0 + + +def fit_prophet( + series: pd.Series, + forecast_horizon: int, + freq: str | None = None, +) -> dict: + """Fit Meta Prophet and return forecast + metrics. + + Args: + series: A pandas Series containing the time series data. + forecast_horizon: The number of periods to forecast. + freq: Optional pandas frequency string for building the future frame. + Inferred from the series index when not supplied. + + Returns: + dict with keys: forecast, lower_ci, upper_ci, rmse, mae, mape + """ + series = series.dropna().astype(float) + + if len(series) < 2: + logger.warning( + "Series too short for Prophet (%d points). Returning persistence " + "forecast.", + len(series), + ) + last_val = series.iloc[-1] if not series.empty else 0.0 + return { + "forecast": [last_val] * forecast_horizon, + "lower_ci": [last_val] * forecast_horizon, + "upper_ci": [last_val] * forecast_horizon, + "rmse": 0.0, + "mae": 0.0, + "mape": 0.0, + } + + prophet_module = import_prophet() + + # Split data into train and test sets for metrics calculation + split = max( + int(len(series) * 0.8), + len(series) - forecast_horizon, + ) + split = min(split, len(series) - 1) + train, test = series.iloc[:split], series.iloc[split:] + + rmse, mae, mape = 0.0, 0.0, 0.0 + if len(train) >= 2 and len(test) >= 1: + rmse, mae, mape = _metrics_from_holdout(train, test, prophet_module, freq) + + # Fit the model on the full series for the final forecast + history = _to_history_frame(series) + m = prophet_module.Prophet() + m.fit(history) + future = _future_frame(history, periods=forecast_horizon, freq=freq) + fc = m.predict(future) + + forecast_values = fc["yhat"].to_numpy() + lower_ci = fc["yhat_lower"].to_numpy() + upper_ci = fc["yhat_upper"].to_numpy() + + logger.info( + "Prophet fitted: series_len=%d horizon=%d freq=%s", + len(series), + forecast_horizon, + freq, + ) + + return { + "forecast": forecast_values.tolist(), + "lower_ci": lower_ci.tolist(), + "upper_ci": upper_ci.tolist(), + "rmse": rmse, + "mae": mae, + "mape": mape, + } diff --git a/data_forecaster/backend/forecasting/registry.py b/data_forecaster/backend/forecasting/registry.py new file mode 100644 index 0000000..7126b93 --- /dev/null +++ b/data_forecaster/backend/forecasting/registry.py @@ -0,0 +1,180 @@ +"""Canonical registry of forecasting models and their enable/disable state. + +The ``MODELS`` map is the single source of truth for which forecasting +models exist; the ``model_config`` database table holds the mutable +enabled/disabled state (managed via the admin panel). All agents must +consume :func:`get_enabled_models` rather than hardcoding model lists so +that disabled models are never fitted, assessed, or selected. +""" + +from __future__ import annotations + +import sqlite3 +from collections.abc import Callable +from typing import Any + +import pandas as pd + +from core.database import get_connection +from core.logging_config import get_logger +from forecasting.arima_model import fit_arima +from forecasting.ewma_model import fit_ewma +from forecasting.holt_winters import fit_holt_winters +from forecasting.prophet_model import fit_prophet +from forecasting.sarima_model import fit_sarima + +logger = get_logger(__name__) + +# Fit functions accept (series, forecast_horizon, **kwargs) and return the +# standard metrics dict (forecast, lower_ci, upper_ci, rmse, mae, mape). +FitFn = Callable[..., dict[str, Any]] + +#: Canonical model catalog. ``extra_kwargs`` maps model-specific keyword +#: arguments to keys of the fit context passed by the caller (e.g. the +#: forecasting agent supplies ``seasonal_period`` and ``freq``). +MODELS: dict[str, dict[str, Any]] = { + "Holt-Winters": { + "fit_fn": fit_holt_winters, + "display_name": "Holt-Winters", + "extra_kwargs": {}, + }, + "ARIMA": { + "fit_fn": fit_arima, + "display_name": "ARIMA", + "extra_kwargs": {}, + }, + "SARIMA": { + "fit_fn": fit_sarima, + "display_name": "SARIMA", + "extra_kwargs": {"seasonal_period": "seasonal_period"}, + }, + "EWMA": { + "fit_fn": fit_ewma, + "display_name": "EWMA", + "extra_kwargs": {}, + }, + "Prophet": { + "fit_fn": fit_prophet, + "display_name": "Prophet", + "extra_kwargs": {"freq": "freq"}, + }, +} + +#: Names in canonical (priority) order — used for heuristic fallbacks. +MODEL_NAMES: tuple[str, ...] = tuple(MODELS) + + +def _read_enabled_names(db_path: str | None = None) -> list[str]: + """Return enabled model names from ``model_config`` (canonical order).""" + with get_connection(db_path) as connection: + rows: list[sqlite3.Row] = connection.execute( + "SELECT name FROM model_config WHERE enabled = 1 ORDER BY priority" + ).fetchall() + enabled = {row["name"] for row in rows} + return [name for name in MODEL_NAMES if name in enabled] + + +def get_enabled_models(db_path: str | None = None) -> tuple[str, ...]: + """Return the names of currently enabled models. + + Defends the at-least-one-enabled invariant: if the table is missing, + unreadable, or somehow all-disabled, all models are treated as enabled + and a warning is logged. + + Args: + db_path: Optional database path override (testing). + + Returns: + A tuple of enabled model names in canonical order. + """ + try: + enabled = _read_enabled_names(db_path) + except sqlite3.Error as exc: + logger.warning( + "model_config unreadable (%s) — treating all models as enabled.", + exc, + ) + return MODEL_NAMES + if not enabled: + logger.warning( + "model_config has no enabled models — treating all as enabled." + ) + return MODEL_NAMES + return tuple(enabled) + + +def set_model_enabled( + name: str, enabled: bool, db_path: str | None = None +) -> None: + """Enable or disable a model, enforcing the at-least-one invariant. + + Args: + name: Model name (must exist in :data:`MODELS`). + enabled: ``True`` to enable, ``False`` to disable. + db_path: Optional database path override (testing). + + Raises: + ValueError: When the model name is unknown, or the change would + leave zero models enabled. + """ + if name not in MODELS: + raise ValueError(f"Unknown model: {name=}") + currently_enabled = set(get_enabled_models(db_path)) + if not enabled and currently_enabled == {name}: + raise ValueError( + "Cannot disable the last enabled model — at least one model " + "must remain enabled." + ) + with get_connection(db_path) as connection: + connection.execute( + "UPDATE model_config SET enabled = ? WHERE name = ?", + (1 if enabled else 0, name), + ) + connection.commit() + logger.info("Model '%s' %s.", name, "enabled" if enabled else "disabled") + + +def list_model_states(db_path: str | None = None) -> list[dict[str, Any]]: + """Return all models with their enabled state (for the admin UI). + + Args: + db_path: Optional database path override (testing). + + Returns: + A list of dicts with ``name``, ``display_name``, and ``enabled`` + keys, in canonical order. + """ + enabled = set(get_enabled_models(db_path)) + return [ + { + "name": name, + "display_name": MODELS[name]["display_name"], + "enabled": name in enabled, + } + for name in MODEL_NAMES + ] + + +def get_fit_functions( + context: dict[str, Any], db_path: str | None = None +) -> list[tuple[str, FitFn, dict[str, Any]]]: + """Return ``(name, fit_fn, kwargs)`` triples for enabled models. + + Args: + context: Fit context supplying values referenced by each model's + ``extra_kwargs`` mapping (e.g. ``seasonal_period``, ``freq``). + db_path: Optional database path override (testing). + + Returns: + A list of triples ready to iterate in the forecasting fit loop. + """ + triples: list[tuple[str, FitFn, dict[str, Any]]] = [] + for name in get_enabled_models(db_path): + entry = MODELS[name] + kwargs = { + kwarg: context[source] + for kwarg, source in entry["extra_kwargs"].items() + if source in context + } + triples.append((name, entry["fit_fn"], kwargs)) + return triples diff --git a/data_forecaster/backend/forecasting/sarima_model.py b/data_forecaster/backend/forecasting/sarima_model.py index f7cce97..c8490ae 100644 --- a/data_forecaster/backend/forecasting/sarima_model.py +++ b/data_forecaster/backend/forecasting/sarima_model.py @@ -12,6 +12,7 @@ ForecastMetrics, ) from forecasting.evaluation import evaluate_predictions, make_terminal_holdout +from forecasting.indexing import normalize_forecast_index from forecasting.pmdarima_compat import import_pmdarima logger = get_logger(__name__) @@ -67,7 +68,7 @@ def fit_sarima( :class:`ForecastAdapterResult` with status, forecast, intervals, nullable metrics, and fitted configuration provenance. """ - series = series.dropna().astype(float) + series = normalize_forecast_index(series.dropna().astype(float)) # Check if we have enough data for seasonal modeling if len(series) < 2 * seasonal_period: diff --git a/data_forecaster/backend/main.py b/data_forecaster/backend/main.py index 78f48ab..5493e53 100644 --- a/data_forecaster/backend/main.py +++ b/data_forecaster/backend/main.py @@ -26,11 +26,11 @@ UploadFile, ) from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse import core.config as settings from auth.api_key_db import ( create_api_user, - create_first_user, delete_api_user, has_any_users, has_bootstrap_user, @@ -41,15 +41,10 @@ set_user_enabled, ) from auth.dependency import require_admin_api_key, require_api_key -from core.config import ( - GOOGLE_API_KEY, - OLLAMA_API_KEY, - OLLAMA_MODEL, - USE_OLLAMA, - USE_OLLAMA_CLOUD, - set_api_key_enabled, -) +from core.config import set_api_key_enabled from core.database import init_database +from core.llm_config_store import get_llm_config, is_configured, put_llm_config +from forecasting import registry from core.logging_config import get_logger from schemas import ( APIKeyRotatedResponse, @@ -60,8 +55,6 @@ APIUserToggleRequest, AnalyzeRequest, AuthStatusResponse, - BootstrapRequest, - BootstrapResponse, ChatRequest, ChatResponse, DeletedJobsResponse, @@ -70,10 +63,25 @@ JobStatusResponse, JobSubmitResponse, PreflightResponse, + SetupBootstrapRequest, + SetupBootstrapResponse, + SetupStatusResponse, + ModelsResponse, + ModelUpdateRequest, + LLMConfigResponse, + LLMConfigTestResponse, + LLMConfigUpdateRequest, UploadResponse, ) from services.chat_service import chat_general, chat_with_data from services.file_service import get_file, init_storage, store_file +from services.setup_service import ( + SetupAlreadyCompleteError, + get_setup_status, + is_setup_complete, + migrate_legacy_deployment, + run_bootstrap, +) from services.job_service import ( create_job, clear_terminal_jobs, @@ -87,6 +95,7 @@ list_recent_jobs, update_job_settings, ) +from services.llm_validation_service import validate_llm_configuration from utils.data_parser import parse_upload, parse_upload_from_path from utils.preflight import run_preflight_checks @@ -131,51 +140,41 @@ def _reconcile_frontend_service_user() -> None: ) -def _create_frontend_service_user_from_env() -> None: - """Create the first API user from frontend service env vars when present.""" - if not _service_credentials_configured(): - logger.info("No API users — auth disabled (open mode).") - return - - try: - create_first_user( - username=settings.FRONTEND_API_USERNAME, - api_key=settings.FRONTEND_API_KEY, - ) - except ValueError as exc: - logger.warning("Failed to auto-create frontend API user: %s", exc) - logger.info("No API users — auth disabled (open mode).") - return - - set_api_key_enabled(True) - logger.info( - "Frontend API user '%s' auto-created from env vars — auth enabled.", - settings.FRONTEND_API_USERNAME, - ) - if settings.FRONTEND_API_KEY == "frontend": - logger.warning( - "SECURITY: The frontend API key is the default 'frontend'. " - "Rotate it via the admin panel and update the stored " - "frontend credentials before production use." - ) +def _configure_startup_auth() -> None: + """Enable or bootstrap API authentication during startup. + + Once first-run setup is complete the DB is authoritative and the + ``FRONTEND_API_USERNAME``/``FRONTEND_API_KEY`` env vars are ignored + (logged once). On a fresh install (setup incomplete) auth stays OFF + and no service user is created from env — the setup wizard generates + the service-account credentials and ``POST /setup/bootstrap`` enables + auth. Creating an env user pre-setup would 401 the wizard's + unauthenticated configuration calls and block bootstrap with a 409. + """ + if is_setup_complete(): + if has_any_users(): + set_api_key_enabled(True) + logger.info("API users found — auth enabled.") + if _service_credentials_configured(): + logger.info( + "Setup is complete — FRONTEND_API_USERNAME/FRONTEND_API_KEY " + "env vars are ignored; the database is authoritative." + ) return - logger.warning( - "The initial API key was sourced from the FRONTEND_API_KEY " - "env var. For production security, rotate this key via the " - "admin panel and update the stored frontend credentials." - ) - - -def _configure_startup_auth() -> None: - """Enable or bootstrap API authentication during startup.""" if has_any_users(): + # Pre-wizard deployment not yet migrated (defensive; the lifespan + # runs migrate_legacy_deployment() first, so this should not be + # reachable in practice). _reconcile_frontend_service_user() set_api_key_enabled(True) logger.info("API users found — auth enabled.") return - _create_frontend_service_user_from_env() + logger.info( + "Fresh install — auth disabled until the setup wizard completes " + "(POST /setup/bootstrap)." + ) @asynccontextmanager @@ -188,6 +187,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: """ logger.info("Initializing API key database…") init_database() + migrate_legacy_deployment() cleanup_terminal_jobs() init_storage() _configure_startup_auth() @@ -227,7 +227,18 @@ async def add_security_headers(request: Request, call_next: Any) -> Any: ``Strict-Transport-Security`` to mitigate MIME sniffing, clickjacking, and protocol downgrade attacks. """ - response = await call_next(request) + if request.url.path == "/api-users/bootstrap": + response = JSONResponse( + status_code=410, + content={ + "detail": ( + "This bootstrap endpoint has been retired. " + "Use POST /setup/bootstrap during initial setup." + ) + }, + ) + else: + response = await call_next(request) response.headers["X-Content-Type-Options"] = "nosniff" response.headers["X-Frame-Options"] = "DENY" response.headers["Strict-Transport-Security"] = ( @@ -297,16 +308,17 @@ async def _check_ollama_reachable() -> bool: Handles both Ollama Cloud (with optional API key) and local Ollama. """ + config = get_llm_config() try: - if USE_OLLAMA_CLOUD: - ollama_url = f"{settings.OLLAMA_BASE_URL}/api/version" + if config.provider == "ollama_cloud": + ollama_url = f"{config.base_url}/api/version" headers = {"Content-Type": _JSON_MEDIA_TYPE} - if OLLAMA_API_KEY: - headers["Authorization"] = f"Bearer {OLLAMA_API_KEY}" + if config.api_key: + headers["Authorization"] = f"Bearer {config.api_key}" async with httpx.AsyncClient() as client: response = await client.get(ollama_url, headers=headers) return response.status_code == 200 - ollama_url = f"{settings.OLLAMA_BASE_URL}/api/tags" + ollama_url = f"{config.base_url}/api/tags" async with httpx.AsyncClient() as client: response = await client.get(ollama_url) return response.status_code == 200 @@ -316,10 +328,11 @@ async def _check_ollama_reachable() -> bool: async def _check_gemini_reachable() -> bool: """Return whether the Gemini API responds to a lightweight probe.""" + config = get_llm_config() try: gemini_url = ( "https://generativelanguage.googleapis.com/v1beta/models/" - f"{settings.GEMINI_MODEL}:countTokens" + f"{config.model}:countTokens" ) headers = {"Content-Type": _JSON_MEDIA_TYPE} async with httpx.AsyncClient() as client: @@ -327,7 +340,7 @@ async def _check_gemini_reachable() -> bool: gemini_url, json={"contents": [{"parts": [{"text": "ping"}]}]}, headers=headers, - params={"key": GOOGLE_API_KEY}, + params={"key": config.api_key}, ) return response.status_code == 200 except httpx.HTTPError: @@ -351,16 +364,17 @@ def llm_health() -> dict[str, Any]: "error": None, } - if USE_OLLAMA: + config = get_llm_config() + if config.provider in ("ollama", "ollama_cloud"): result["llm_provider"] = "ollama" - if not OLLAMA_MODEL: - result["error"] = "OLLAMA_MODEL is not set." + if not config.model: + result["error"] = "Ollama model is not configured." return result result["llm_configured"] = True result["llm_reachable"] = asyncio.run(_check_ollama_reachable()) if not result["llm_reachable"]: result["error"] = "Ollama server is not reachable." - elif GOOGLE_API_KEY: + elif config.provider == "gemini" and config.api_key: result["llm_provider"] = "gemini" result["llm_configured"] = True result["llm_reachable"] = asyncio.run(_check_gemini_reachable()) @@ -368,8 +382,8 @@ def llm_health() -> dict[str, Any]: result["error"] = "Gemini API is not reachable." else: result["error"] = ( - "No LLM provider configured. Either set USE_OLLAMA=true with a " - "running Ollama instance, or set GOOGLE_API_KEY for Gemini." + "No LLM provider configured. Configure one via the admin LLM " + "configuration page." ) return result @@ -392,70 +406,53 @@ def auth_status() -> dict[str, Any]: } +@app.get("/setup/status", response_model=SetupStatusResponse) +def setup_status() -> dict[str, Any]: + """Return first-run setup state (booleans only — never secrets). + + Unauthenticated so the frontend wizard can gate the app; the payload + reveals only completion flags. + """ + return get_setup_status() + + @app.post( - "/api-users/bootstrap", - response_model=BootstrapResponse, + "/setup/bootstrap", + response_model=SetupBootstrapResponse, responses={ 400: {"description": "Invalid username or API key"}, - 403: {"description": "ADMIN_API_KEY missing or invalid"}, - 409: {"description": "API users already exist"}, + 409: {"description": "Setup already completed"}, }, ) -def api_users_bootstrap( - request: BootstrapRequest, - http_request: Request, -) -> dict[str, Any]: - """Create the first API user and enable authentication. - - This is a one-time setup endpoint protected by the ``ADMIN_API_KEY`` - deployment secret (sent via the ``X-Admin-Key`` header). It only - succeeds when: - - - ``ADMIN_API_KEY`` is set in the backend environment. - - The supplied ``X-Admin-Key`` header matches. - - No API users exist yet (bootstrap is one-time only). +def setup_bootstrap(request: SetupBootstrapRequest) -> dict[str, Any]: + """Atomically create the first admin API user and complete setup. - On success, creates the user with the admin-supplied username and - key, enables ``API_KEY_ENABLED``, and returns the user dict. + Replaces the retired ``POST /api-users/bootstrap`` (which required a + preset ``ADMIN_API_KEY``). The guard is a conditional insert inside + a ``BEGIN IMMEDIATE`` transaction, so simultaneous first-run requests + cannot both succeed — the loser receives 409. Raises: - HTTPException: 403 when the admin key is missing or mismatched. - HTTPException: 409 when users already exist (bootstrap expired). + HTTPException: 400 when the username/key is invalid. + HTTPException: 409 when setup has already completed. """ - if not settings.ADMIN_API_KEY: - raise HTTPException( - status_code=403, - detail="ADMIN_API_KEY is not set on the backend. " - "Configure it in the backend .env to use bootstrap.", - ) - supplied_key: str | None = http_request.headers.get("X-Admin-Key") - if not supplied_key or supplied_key != settings.ADMIN_API_KEY: - raise HTTPException( - status_code=403, - detail="Invalid or missing admin key.", - ) - - if has_any_users(): - raise HTTPException( - status_code=409, - detail="API users already exist — bootstrap is no longer available.", - ) - try: - user: dict[str, Any] = create_first_user( + user: dict[str, Any] = run_bootstrap( username=request.username, api_key=request.api_key, ) + except SetupAlreadyCompleteError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc set_api_key_enabled(True) logger.info( - "API auth enabled by bootstrap. User '%s' created.", + "API auth enabled by setup bootstrap. Admin user '%s' created.", request.username, ) - return {"user": user, "auth_enabled": True} + return {"user": user, "setup_complete": True} # ── File Upload & Analysis ──────────────────────────────────────────────────── @@ -958,6 +955,119 @@ def api_users_rotate( return {"user_id": user_id, "api_key": plaintext_key} +# ── Model Registry (admin) ──────────────────────────────────────────────────── + + +@app.get("/models", response_model=ModelsResponse) +def models_list( + _user: Annotated[dict, Depends(require_admin_api_key)], +) -> dict[str, Any]: + """List all forecasting models with their enabled state.""" + return {"models": registry.list_model_states()} + + +@app.put( + "/models/{name}", + response_model=ModelsResponse, + responses={ + 400: {"description": "Unknown model or last-model disable attempt"}, + }, +) +def models_update( + name: str, + request: ModelUpdateRequest, + _user: Annotated[dict, Depends(require_admin_api_key)], +) -> dict[str, Any]: + """Enable or disable a forecasting model. + + Rejects with 400 when the change would leave zero models enabled. + """ + try: + registry.set_model_enabled(name, request.enabled) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return {"models": registry.list_model_states()} + + +# ── LLM Configuration (admin) ──────────────────────────────────────────────── + + +@app.get("/config/llm", response_model=LLMConfigResponse) +def llm_config_get( + _user: Annotated[dict, Depends(require_admin_api_key)], +) -> dict[str, Any]: + """Return the LLM configuration with the API key masked. + + The response model structurally excludes the key — only + ``api_key_set`` reveals whether one is stored. + """ + config = get_llm_config() + return { + "provider": config.provider, + "model": config.model, + "base_url": config.base_url, + "temperature": config.temperature, + "api_key_set": config.api_key is not None, + "configured": is_configured(), + } + + +@app.post("/config/llm/test", response_model=LLMConfigTestResponse) +async def llm_config_test( + request: LLMConfigUpdateRequest, + _user: Annotated[dict, Depends(require_admin_api_key)], +) -> dict[str, Any]: + """Test candidate LLM settings without persisting them. + + When the write-only API key is omitted, the currently stored key is + tested. This lets an administrator retest or edit non-secret settings + without exposing or re-entering the saved credential. + """ + current = get_llm_config() + api_key = ( + request.api_key.get_secret_value() + if request.api_key is not None + else current.api_key + ) + result = await validate_llm_configuration( + provider=request.provider, + model=request.model, + base_url=request.base_url, + api_key=api_key, + ) + return result.to_dict() + + +@app.put( + "/config/llm", + response_model=LLMConfigResponse, + responses={400: {"description": "Unknown provider"}}, +) +def llm_config_put( + request: LLMConfigUpdateRequest, + _user: Annotated[dict, Depends(require_admin_api_key)], +) -> dict[str, Any]: + """Update the LLM configuration (one-way write for the API key). + + The key is accepted as a ``SecretStr``, encrypted immediately, and + never logged or returned. Omitting ``api_key`` preserves the stored + key. + """ + try: + put_llm_config( + provider=request.provider, + model=request.model, + base_url=request.base_url, + api_key=( + request.api_key.get_secret_value() if request.api_key else None + ), + temperature=request.temperature, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return llm_config_get(_user) + + @app.post( "/api-users/{user_id}/toggle", response_model=APIUserResponse, diff --git a/data_forecaster/backend/prompts/forecasting_prompt.py b/data_forecaster/backend/prompts/forecasting_prompt.py index 78d97a4..2a18d77 100644 --- a/data_forecaster/backend/prompts/forecasting_prompt.py +++ b/data_forecaster/backend/prompts/forecasting_prompt.py @@ -11,7 +11,7 @@ ( "system", "You are a Senior Forecasting Analyst specializing in " - "Holt-Winters, ARIMA, and SARIMA models. " + "Holt-Winters, ARIMA, SARIMA, EWMA, and Prophet (Meta Prophet) models. " "Your responsibility is to evaluate model performance and explain " "the rationale for model selection using evidence from the supplied results. " "Remain strictly grounded in the provided metrics and diagnostics. " diff --git a/data_forecaster/backend/prompts/general_chat_prompt.py b/data_forecaster/backend/prompts/general_chat_prompt.py index 4d5b874..2ada3e7 100644 --- a/data_forecaster/backend/prompts/general_chat_prompt.py +++ b/data_forecaster/backend/prompts/general_chat_prompt.py @@ -20,7 +20,7 @@ "You are a Specialized Time Series Forecasting Analyst. Your operational domain is " "STRICTLY LIMITED to time series forecasting and forecasting-related questions only. This " "includes: 1. Time series forecasting methodology and concepts, 2. Statistical analysis of " - "forecasting models (ARIMA, SARIMA, Holt-Winters, EWMA), 3. Interpretation of forecast results " + "forecasting models (ARIMA, SARIMA, Holt-Winters, EWMA, Prophet), 3. Interpretation of forecast results " "and metrics (RMSE, MAE, MAPE, prediction intervals), and 4. Business reporting based on " "forecast projections.\n\n" "DOMAIN RESTRICTION & OUT-OF-BOUNDS POLICY:\n" diff --git a/data_forecaster/backend/prompts/model_selection_prompt.py b/data_forecaster/backend/prompts/model_selection_prompt.py index fa15584..f9d1edb 100644 --- a/data_forecaster/backend/prompts/model_selection_prompt.py +++ b/data_forecaster/backend/prompts/model_selection_prompt.py @@ -10,7 +10,7 @@ [ ( "system", - "You are a Senior Time Series Forecasting Analyst specializing in model selection between ARIMA, SARIMA, Holt-Winters, and EWMA. " + "You are a Senior Time Series Forecasting Analyst specializing in model selection between ARIMA, SARIMA, Holt-Winters, EWMA, and Prophet (Meta Prophet). " "Your role is to select the most appropriate model strictly based on statistical evidence provided. " "You must not assume missing metrics or invent model behavior. " "When actual error metrics are provided, you MUST give strong preference to the model with the lowest MASE or RMSE. " @@ -41,12 +41,14 @@ "- ARIMA: \n" "- SARIMA: \n" "- Holt-Winters: \n" - "- EWMA: \n\n" + "- EWMA: \n" + "- Prophet: \n\n" "## Why alternatives were not selected\n" "- ARIMA: \n" "- SARIMA: \n" "- Holt-Winters: \n" - "- EWMA: \n\n" + "- EWMA: \n" + "- Prophet: \n\n" "### FINAL CONSTRAINTS ###\n" "- Every claim must be traceable to the provided evidence.\n" "- If evidence is insufficient, explicitly state uncertainty with [uncertain].\n" diff --git a/data_forecaster/backend/prompts/orchestrator_prompt.py b/data_forecaster/backend/prompts/orchestrator_prompt.py index 653058a..47e345d 100644 --- a/data_forecaster/backend/prompts/orchestrator_prompt.py +++ b/data_forecaster/backend/prompts/orchestrator_prompt.py @@ -12,7 +12,7 @@ "You are a Specialized Time Series Forecasting Analyst. Your operational domain is " "STRICTLY LIMITED to time series forecasting and forecasting-related questions about the " "provided dataset. This includes: 1. Time series forecasting methodology and concepts, " - "2. Statistical analysis of forecasting models (ARIMA, SARIMA, Holt-Winters, EWMA), " + "2. Statistical analysis of forecasting models (ARIMA, SARIMA, Holt-Winters, EWMA, Prophet), " "3. Interpretation of forecast results and metrics (RMSE, MAE, MAPE, prediction intervals), " "and 4. Business reporting based on forecast projections.\n\n" "DOMAIN RESTRICTION & OUT-OF-BOUNDS POLICY:\n" diff --git a/data_forecaster/backend/report/models.py b/data_forecaster/backend/report/models.py index eb92fb9..0015a34 100644 --- a/data_forecaster/backend/report/models.py +++ b/data_forecaster/backend/report/models.py @@ -458,6 +458,10 @@ class ReportMetadata(BaseModel): dataset_frequency: Frequency of the input dataset. data_quality_rating: Overall data quality rating. row_count: Number of rows in the input dataset. + llm_narrative_fallback: Whether deterministic narrative templates + replaced one or more LLM-generated sections. + llm_fallback_sections: Narrative sections that used deterministic + templates, if any. """ engine_version: str @@ -468,6 +472,8 @@ class ReportMetadata(BaseModel): dataset_frequency: str data_quality_rating: str row_count: int + llm_narrative_fallback: bool = False + llm_fallback_sections: list[str] = Field(default_factory=list) class Appendix(BaseModel): diff --git a/data_forecaster/backend/report/narrative.py b/data_forecaster/backend/report/narrative.py index 8c5dc81..f13ef0d 100644 --- a/data_forecaster/backend/report/narrative.py +++ b/data_forecaster/backend/report/narrative.py @@ -17,7 +17,7 @@ import re from typing import Any -from core.config import GEMINI_TEMPERATURE +from core.llm_config_store import get_llm_config from core.llm_factory import get_llm from core.logging_config import get_logger from forecasting.selection_policy import validate_llm_output @@ -50,12 +50,13 @@ def generate_narratives( Returns: A tuple of (updated :class:`ExecutiveReport`, token_usage_dict). """ - llm = get_llm(temperature=GEMINI_TEMPERATURE) + llm = get_llm(temperature=get_llm_config().temperature) total_usage: dict[str, int] = { "input_tokens": 0, "output_tokens": 0, "total_tokens": 0, } + fallback_sections: list[str] = [] extra = ( f"\n\nADDITIONAL USER INSTRUCTIONS:\n{user_prompt.strip()}" if user_prompt and user_prompt.strip() @@ -70,6 +71,7 @@ def generate_narratives( "executive_summary", total_usage, extra, + fallback_sections, ) # ── Data Quality ────────────────────────────────────────────────────── @@ -80,6 +82,7 @@ def generate_narratives( "data_quality", total_usage, extra, + fallback_sections, ) # ── Historical Analysis ─────────────────────────────────────────────── @@ -90,6 +93,7 @@ def generate_narratives( "historical_analysis", total_usage, extra, + fallback_sections, ) # ── Forecast Outlook ────────────────────────────────────────────────── @@ -100,6 +104,7 @@ def generate_narratives( "forecast_outlook", total_usage, extra, + fallback_sections, ) # ── Model Comparison ────────────────────────────────────────────────── @@ -110,6 +115,7 @@ def generate_narratives( "model_comparison", total_usage, extra, + fallback_sections, ) # ── Statistical Audit ───────────────────────────────────────────────── @@ -120,6 +126,7 @@ def generate_narratives( "statistical_audit", total_usage, extra, + fallback_sections, ) # ── Explainability ──────────────────────────────────────────────────── @@ -130,6 +137,7 @@ def generate_narratives( "explainability", total_usage, extra, + fallback_sections, ) # ── Recommendations ─────────────────────────────────────────────────── @@ -141,8 +149,11 @@ def generate_narratives( "recommendation", total_usage, extra, + fallback_sections, ) + report.metadata.llm_narrative_fallback = bool(fallback_sections) + report.metadata.llm_fallback_sections = fallback_sections logger.info("Narrative generation complete. Tokens: %s", total_usage) return report, total_usage @@ -154,6 +165,7 @@ def _generate_section( section_name: str, total_usage: dict[str, int], extra_instructions: str = "", + fallback_sections: list[str] | None = None, ) -> str: """Generate narrative for a single section via the LLM. @@ -166,6 +178,7 @@ def _generate_section( section_name: Name for logging. total_usage: Mutable token usage dict to accumulate. extra_instructions: Optional extra user instructions. + fallback_sections: Mutable list used to record fallback sections. Returns: Narrative text string. @@ -231,6 +244,8 @@ def _generate_section( section_name, "; ".join(validation_warnings), ) + if fallback_sections is not None: + fallback_sections.append(section_name) return _fallback_narrative(section, section_name) logger.debug("Narrative generated for %s", section_name) return narrative @@ -240,6 +255,8 @@ def _generate_section( section_name, exc, ) + if fallback_sections is not None: + fallback_sections.append(section_name) return _fallback_narrative(section, section_name) diff --git a/data_forecaster/backend/report/renderers/html_renderer.py b/data_forecaster/backend/report/renderers/html_renderer.py index 9176f99..0591773 100644 --- a/data_forecaster/backend/report/renderers/html_renderer.py +++ b/data_forecaster/backend/report/renderers/html_renderer.py @@ -434,6 +434,15 @@ def _priority_color(priority: str) -> str: def _render_metadata(self, report: ExecutiveReport) -> str: """Render report metadata as a small table.""" m = report.metadata + narrative_source = "LLM-generated" + if m.llm_narrative_fallback: + sections = ", ".join( + section.replace("_", " ") for section in m.llm_fallback_sections + ) + narrative_source = ( + "Deterministic fallback used" + + (f" ({sections})" if sections else "") + ) return ( '
' "
Report Metadata
" @@ -445,6 +454,7 @@ def _render_metadata(self, report: ExecutiveReport) -> str: f"Dataset Frequency{escape(m.dataset_frequency)}" f"Data Quality{escape(m.data_quality_rating)}" f"Row Count{m.row_count}" + f"Narrative Generation{escape(narrative_source)}" "" "
" ) diff --git a/data_forecaster/backend/report/renderers/markdown_renderer.py b/data_forecaster/backend/report/renderers/markdown_renderer.py index 9afd52c..59cece0 100644 --- a/data_forecaster/backend/report/renderers/markdown_renderer.py +++ b/data_forecaster/backend/report/renderers/markdown_renderer.py @@ -399,4 +399,14 @@ def _render_appendix(self, report: ExecutiveReport) -> str: lines.append(f"| Dataset Frequency | {meta.dataset_frequency} |") lines.append(f"| Data Quality Rating | {meta.data_quality_rating} |") lines.append(f"| Row Count | {meta.row_count} |") + narrative_source = "LLM-generated" + if meta.llm_narrative_fallback: + sections = ", ".join( + section.replace("_", " ") for section in meta.llm_fallback_sections + ) + narrative_source = ( + "Deterministic fallback used" + + (f" ({sections})" if sections else "") + ) + lines.append(f"| Narrative Generation | {narrative_source} |") return "\n".join(lines) diff --git a/data_forecaster/backend/requirements.txt b/data_forecaster/backend/requirements.txt index 191729c..c65c0b1 100644 --- a/data_forecaster/backend/requirements.txt +++ b/data_forecaster/backend/requirements.txt @@ -9,6 +9,7 @@ pyarrow>=23.0.1 scipy>=1.13.0 statsmodels>=0.14.0 pmdarima>=2.0.0 +prophet>=1.1.0 scikit-learn>=1.3.0 matplotlib>=3.10.9 plotly>=6.8.0 @@ -18,5 +19,4 @@ langchain>=1.3.9 langchain-google-genai>=2.0.0 langchain-ollama>=1.1.0 openpyxl>=3.1.0 -python-dotenv>=1.2.2 argon2-cffi>=23.1.0 diff --git a/data_forecaster/backend/schemas.py b/data_forecaster/backend/schemas.py index e8dd75b..94d9a46 100644 --- a/data_forecaster/backend/schemas.py +++ b/data_forecaster/backend/schemas.py @@ -9,7 +9,7 @@ from __future__ import annotations from typing import Any -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, SecretStr from forecasting.contracts import ForecastFitStatus @@ -169,6 +169,7 @@ class ModelSelectionResult(BaseModel): arima_rejected_reason: str | None = None sarima_rejected_reason: str | None = None ewma_rejected_reason: str | None = None + prophet_rejected_reason: str | None = None reasoning_steps: list[dict[str, Any]] = Field(default_factory=list) token_usage: dict[str, Any] = Field(default_factory=dict) # ── Selection policy additions ────────────────────────────────────────── @@ -442,3 +443,96 @@ class AuthStatusResponse(BaseModel): auth_enabled: bool has_users: bool + + +# ── Setup Wizard Schemas ────────────────────────────────────────────────────── + + +class SetupBootstrapRequest(BaseModel): + """Request schema for the atomic first-run setup bootstrap.""" + + username: str + api_key: str + + +class SetupBootstrapResponse(BaseModel): + """Response schema after a successful setup bootstrap.""" + + user: APIUserResponse + setup_complete: bool = True + + +class SetupStatusResponse(BaseModel): + """Response schema for setup status — booleans only, never secrets.""" + + setup_complete: bool + admin_exists: bool + llm_configured: bool + models_enabled: int + + +# ── Model Registry Schemas ─────────────────────────────────────────────────── + + +class ModelState(BaseModel): + """Enable/disable state of one forecasting model.""" + + name: str + display_name: str + enabled: bool + + +class ModelsResponse(BaseModel): + """Response schema listing all models and their states.""" + + models: list[ModelState] + + +class ModelUpdateRequest(BaseModel): + """Request schema for enabling or disabling a model.""" + + enabled: bool + + +# ── LLM Configuration Schemas ──────────────────────────────────────────────── + + +class LLMConfigResponse(BaseModel): + """Masked LLM configuration — the API key is structurally absent. + + Only ``api_key_set`` reveals whether a key is stored; the key itself + (plaintext or ciphertext) is never included in any API response. + """ + + provider: str + model: str + base_url: str | None = None + temperature: float + api_key_set: bool + configured: bool + + +class LLMConfigUpdateRequest(BaseModel): + """One-way write schema for LLM configuration. + + ``api_key`` is a :class:`SecretStr` so it cannot leak via logs or + ``repr``. Omitting it (``None``) preserves the stored key. + """ + + provider: str + model: str + base_url: str | None = None + api_key: SecretStr | None = None + temperature: float = 0.1 + + +class LLMConfigTestResponse(BaseModel): + """Result of testing candidate LLM settings without saving them.""" + + ok: bool + url_reachable: bool + credentials_valid: bool + llm_responded: bool + message: str + response: str | None = None + diagnostic: str | None = None diff --git a/data_forecaster/backend/scripts/bootstrap.py b/data_forecaster/backend/scripts/bootstrap.py deleted file mode 100644 index 286a17a..0000000 --- a/data_forecaster/backend/scripts/bootstrap.py +++ /dev/null @@ -1,407 +0,0 @@ -"""Interactive first-run bootstrap for Data Forecaster deployments. - -This script prepares the frontend and backend environment files without -requiring project dependencies on the host machine. It generates stable -runtime secrets, synchronises the pre-shared frontend/backend service -credentials, and can optionally collect LLM provider credentials. -""" - -from __future__ import annotations - -import argparse -import base64 -import getpass -import os -import secrets -import shutil -import sqlite3 -import subprocess -import sys -from pathlib import Path -from typing import Callable - -SCRIPT_PATH = Path(__file__).resolve() -BACKEND_ROOT = SCRIPT_PATH.parents[1] -PROJECT_ROOT = SCRIPT_PATH.parents[2] -RUNNING_IN_BACKEND_CONTAINER = BACKEND_ROOT == Path("/app") -sys.path.insert(0, str(BACKEND_ROOT)) - -if RUNNING_IN_BACKEND_CONTAINER: - BACKEND_ENV = BACKEND_ROOT / ".env" - BACKEND_ENV_EXAMPLE = BACKEND_ROOT / ".env.example" - FRONTEND_ENV = None - FRONTEND_ENV_EXAMPLE = None - COMPOSE_FILE = None -else: - BACKEND_ENV = BACKEND_ROOT / ".env" - BACKEND_ENV_EXAMPLE = BACKEND_ROOT / ".env.example" - FRONTEND_ENV = PROJECT_ROOT / "frontend" / ".env" - FRONTEND_ENV_EXAMPLE = PROJECT_ROOT / "frontend" / ".env.example" - COMPOSE_FILE = PROJECT_ROOT / "docker" / "docker-compose.yml" - -PLACEHOLDERS = { - "", - "admin", - "change-me-in-production", - "change-me-to-a-strong-random-value", - "frontend", - "generate-a-strong-random-key-here", - "generate-a-fernet-key-and-paste-here", - "your_google_api_key_here", -} - - -def generate_secret_urlsafe(length: int = 32) -> str: - """Return a URL-safe random secret for API keys and Flask sessions.""" - return secrets.token_urlsafe(length) - - -def generate_fernet_key() -> str: - """Return a Fernet-compatible key without importing cryptography.""" - return base64.urlsafe_b64encode(secrets.token_bytes(32)).decode("ascii") - - -def parse_env(path: Path) -> dict[str, str]: - """Parse simple KEY=VALUE lines from an env file.""" - values: dict[str, str] = {} - if not path.exists(): - return values - - for line in path.read_text(encoding="utf-8").splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#") or "=" not in stripped: - continue - key, value = stripped.split("=", 1) - values[key.strip()] = value.strip().strip('"').strip("'") - return values - - -def ensure_env_file(path: Path, example_path: Path) -> None: - """Create an env file from its example when missing.""" - if path.exists(): - return - if not example_path.exists(): - path.touch() - return - shutil.copyfile(example_path, path) - print(f"Created {display_path(path)} from example.") - - -def display_path(path: Path) -> str: - """Return a readable path for status messages.""" - try: - return str(path.relative_to(PROJECT_ROOT)) - except ValueError: - return str(path) - - -def write_env_value(path: Path, key: str, value: str) -> None: - """Update or append a KEY=VALUE line while preserving comments.""" - lines = path.read_text(encoding="utf-8").splitlines() - updated = False - for index, line in enumerate(lines): - stripped = line.strip() - if not stripped or stripped.startswith("#") or "=" not in stripped: - continue - line_key = stripped.split("=", 1)[0].strip() - if line_key == key: - lines[index] = f"{key}={value}" - updated = True - break - if not updated: - if lines and lines[-1].strip(): - lines.append("") - lines.append(f"{key}={value}") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -def is_placeholder(value: str | None) -> bool: - """Return True when a value is missing or still looks like a template.""" - if value is None: - return True - return value.strip() in PLACEHOLDERS - - -def confirm(prompt: str, default: bool = False, assume_yes: bool = False) -> bool: - """Prompt for yes/no confirmation.""" - if assume_yes: - return default - suffix = " [Y/n]: " if default else " [y/N]: " - answer = input(prompt + suffix).strip().lower() - if not answer: - return default - return answer in {"y", "yes"} - - -def maybe_set_secret( - path: Path, - values: dict[str, str], - key: str, - value_factory: Callable[[], str], - *, - assume_yes: bool, - force: bool, -) -> str: - """Generate and persist a secret when missing, placeholder, or forced.""" - current = values.get(key) - if force or is_placeholder(current): - value = value_factory() - write_env_value(path, key, value) - values[key] = value - print(f"Set {key} in {display_path(path)}.") - return value - - if confirm( - f"{key} already exists in {path.name}. Replace it?", assume_yes=assume_yes - ): - value = value_factory() - write_env_value(path, key, value) - values[key] = value - print(f"Replaced {key} in {display_path(path)}.") - return value - - return current or "" - - -def prompt_secret(label: str, current: str | None, *, force: bool) -> str | None: - """Prompt for a sensitive optional value.""" - if current and not is_placeholder(current) and not force: - replace = confirm(f"{label} is already set. Replace it?") - if not replace: - return None - - value = getpass.getpass(f"{label} (leave blank to skip): ").strip() - return value or None - - -def sync_value( - source: str, target_path: Path, target_values: dict[str, str], key: str -) -> None: - """Write a value to an env file and its in-memory mapping.""" - write_env_value(target_path, key, source) - target_values[key] = source - - -def reset_backend_service_user_if_present(username: str, api_key: str) -> None: - """Update an existing backend API user's stored hash to match api_key.""" - try: - from auth.argon2_helpers import hash_api_key - from core.config import BACKEND_DB_PATH - from core.database import get_connection - except Exception as exc: - print(f"Skipped backend API user reset; backend modules unavailable: {exc}") - return - - if not os.path.exists(BACKEND_DB_PATH): - return - - conn: sqlite3.Connection = get_connection(db_path=BACKEND_DB_PATH) - try: - row = conn.execute( - "SELECT id FROM api_users WHERE username = ?", - (username,), - ).fetchone() - if row is None: - return - - conn.execute( - "UPDATE api_users SET api_key_hash = ? WHERE id = ?", - (hash_api_key(api_key), int(row["id"])), - ) - conn.commit() - print( - f"Reset existing backend API user '{username}' to match FRONTEND_API_KEY." - ) - finally: - conn.close() - - -def collect_llm_credentials( - backend_values: dict[str, str], - *, - force: bool, - assume_yes: bool, -) -> None: - """Optionally collect provider credentials for the backend.""" - if not confirm( - "Configure LLM provider credentials now?", default=False, assume_yes=assume_yes - ): - return - - use_ollama = confirm("Use Ollama instead of Google Gemini?", default=True) - write_env_value(BACKEND_ENV, "USE_OLLAMA", str(use_ollama).lower()) - backend_values["USE_OLLAMA"] = str(use_ollama).lower() - - if use_ollama: - use_cloud = confirm("Use Ollama Cloud?", default=False) - write_env_value(BACKEND_ENV, "USE_OLLAMA_CLOUD", str(use_cloud).lower()) - backend_values["USE_OLLAMA_CLOUD"] = str(use_cloud).lower() - if use_cloud: - key = prompt_secret( - "OLLAMA_API_KEY", backend_values.get("OLLAMA_API_KEY"), force=force - ) - if key: - write_env_value(BACKEND_ENV, "OLLAMA_API_KEY", key) - model = input( - f"OLLAMA_MODEL [{backend_values.get('OLLAMA_MODEL', 'llama3')}]: " - ).strip() - if model: - write_env_value(BACKEND_ENV, "OLLAMA_MODEL", model) - base_url = input( - f"OLLAMA_BASE_URL [{backend_values.get('OLLAMA_BASE_URL', 'http://localhost:11434')}]: " - ).strip() - if base_url: - write_env_value(BACKEND_ENV, "OLLAMA_BASE_URL", base_url) - return - - key = prompt_secret( - "GOOGLE_API_KEY", backend_values.get("GOOGLE_API_KEY"), force=force - ) - if key: - write_env_value(BACKEND_ENV, "GOOGLE_API_KEY", key) - model = input( - f"GEMINI_MODEL [{backend_values.get('GEMINI_MODEL', 'gemini-1.5-flash')}]: " - ).strip() - if model: - write_env_value(BACKEND_ENV, "GEMINI_MODEL", model) - - -def restart_services() -> int: - """Restart the Docker Compose stack if Docker is available.""" - if COMPOSE_FILE is None: - print("Docker Compose restart is only available when run from the host repo.") - return 1 - command = ["docker", "compose", "-f", str(COMPOSE_FILE), "restart"] - print("Restarting services with Docker Compose...") - return subprocess.call(command, cwd=PROJECT_ROOT) - - -def bootstrap(args: argparse.Namespace) -> int: - """Run the interactive bootstrap workflow.""" - ensure_env_file(BACKEND_ENV, BACKEND_ENV_EXAMPLE) - if FRONTEND_ENV is not None and FRONTEND_ENV_EXAMPLE is not None: - ensure_env_file(FRONTEND_ENV, FRONTEND_ENV_EXAMPLE) - elif not args.backend_only: - print("Running inside the backend container; frontend .env is not available.") - print("Continuing in backend-only mode.") - - backend_values = parse_env(BACKEND_ENV) - frontend_values = parse_env(FRONTEND_ENV) if FRONTEND_ENV is not None else {} - - if FRONTEND_ENV is not None and not args.backend_only: - maybe_set_secret( - FRONTEND_ENV, - frontend_values, - "FLASK_ENCRYPTION_KEY", - generate_fernet_key, - assume_yes=args.yes, - force=args.force, - ) - maybe_set_secret( - FRONTEND_ENV, - frontend_values, - "SECRET_KEY", - generate_secret_urlsafe, - assume_yes=args.yes, - force=args.force, - ) - maybe_set_secret( - BACKEND_ENV, - backend_values, - "ADMIN_API_KEY", - generate_secret_urlsafe, - assume_yes=args.yes, - force=args.force, - ) - - service_username = backend_values.get("FRONTEND_API_USERNAME") or "frontend" - if args.force or is_placeholder(backend_values.get("FRONTEND_API_KEY")): - service_key = generate_secret_urlsafe() - sync_value( - service_username, BACKEND_ENV, backend_values, "FRONTEND_API_USERNAME" - ) - sync_value(service_key, BACKEND_ENV, backend_values, "FRONTEND_API_KEY") - reset_backend_service_user_if_present(service_username, service_key) - if FRONTEND_ENV is not None and not args.backend_only: - sync_value( - service_username, FRONTEND_ENV, frontend_values, "FRONTEND_API_USERNAME" - ) - sync_value(service_key, FRONTEND_ENV, frontend_values, "FRONTEND_API_KEY") - print("Generated matching frontend/backend service credentials.") - else: - print("Generated backend service credentials.") - else: - service_key = backend_values["FRONTEND_API_KEY"] - reset_backend_service_user_if_present(service_username, service_key) - if FRONTEND_ENV is not None and not args.backend_only: - sync_value( - service_username, FRONTEND_ENV, frontend_values, "FRONTEND_API_USERNAME" - ) - sync_value(service_key, FRONTEND_ENV, frontend_values, "FRONTEND_API_KEY") - print("Synced existing backend service credentials into frontend env.") - - if FRONTEND_ENV is not None and not args.backend_only: - admin_user = frontend_values.get("FRONTEND_ADMIN_USERNAME") or "admin" - sync_value(admin_user, FRONTEND_ENV, frontend_values, "FRONTEND_ADMIN_USERNAME") - if args.force or is_placeholder(frontend_values.get("FRONTEND_ADMIN_PASSWORD")): - if args.yes: - admin_password = generate_secret_urlsafe() - print("Generated a frontend admin password in frontend/.env.") - else: - admin_password = getpass.getpass("Frontend admin password: ").strip() - if admin_password: - sync_value( - admin_password, - FRONTEND_ENV, - frontend_values, - "FRONTEND_ADMIN_PASSWORD", - ) - else: - print( - "Skipped frontend admin password; current value was left unchanged." - ) - - if not args.skip_llm: - collect_llm_credentials(backend_values, force=args.force, assume_yes=args.yes) - - if args.restart: - return restart_services() - - print("") - print("Bootstrap complete.") - if COMPOSE_FILE is not None: - print("Restart services so env changes take effect:") - print(f" docker compose -f {COMPOSE_FILE.relative_to(PROJECT_ROOT)} restart") - else: - print("Restart the backend container so env changes take effect.") - print( - "Note: /app/.env changes are container-local unless .env is bind-mounted." - ) - return 0 - - -def parse_args(argv: list[str]) -> argparse.Namespace: - """Parse command-line arguments.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--yes", action="store_true", help="accept default yes/no prompts" - ) - parser.add_argument( - "--force", action="store_true", help="replace existing non-placeholder secrets" - ) - parser.add_argument( - "--backend-only", action="store_true", help="only update the backend .env" - ) - parser.add_argument( - "--skip-llm", action="store_true", help="do not prompt for LLM credentials" - ) - parser.add_argument( - "--restart", - action="store_true", - help="restart Docker Compose services when done", - ) - return parser.parse_args(argv) - - -if __name__ == "__main__": - raise SystemExit(bootstrap(parse_args(sys.argv[1:]))) diff --git a/data_forecaster/backend/services/llm_validation_service.py b/data_forecaster/backend/services/llm_validation_service.py new file mode 100644 index 0000000..39f8254 --- /dev/null +++ b/data_forecaster/backend/services/llm_validation_service.py @@ -0,0 +1,266 @@ +"""Validate candidate LLM settings without persisting them. + +Validation is deliberately staged so the UI can tell an administrator whether +the provider host is reachable, the credentials are accepted, and the selected +model can produce a real response. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any +from urllib.parse import quote + +import httpx + +_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com" +_PING_PROMPT = "Connection test. Reply with exactly: pong" +_RESPONSE_PREVIEW_LIMIT = 500 +_DIAGNOSTIC_PREVIEW_LIMIT = 500 +_PING_MAX_OUTPUT_TOKENS = 64 + + +@dataclass(frozen=True) +class LLMValidationResult: + """The outcome of each LLM validation stage.""" + + ok: bool = False + url_reachable: bool = False + credentials_valid: bool = False + llm_responded: bool = False + message: str = "" + response: str | None = None + diagnostic: str | None = None + + def to_dict(self) -> dict[str, Any]: + """Return a JSON-serialisable representation.""" + return asdict(self) + + +def _failed( + message: str, + *, + url_reachable: bool = False, + credentials_valid: bool = False, + diagnostic: str | None = None, +) -> LLMValidationResult: + """Build a failed validation result with a safe provider diagnostic.""" + return LLMValidationResult( + url_reachable=url_reachable, + credentials_valid=credentials_valid, + message=message, + diagnostic=diagnostic, + ) + + +def _provider_error_diagnostic(response: httpx.Response) -> str: + """Return the useful, non-sensitive portion of a provider error response. + + Providers use a few common shapes for errors. Only known diagnostic + fields are returned rather than echoing the complete body, which could + include request data or credentials on a non-conforming provider. + """ + detail = "" + try: + payload = response.json() + except ValueError: + payload = None + + if isinstance(payload, dict): + error = payload.get("error") + if isinstance(error, dict): + for field in ("message", "detail", "status", "type", "code"): + value = error.get(field) + if isinstance(value, (str, int, float)) and str(value).strip(): + detail = str(value).strip() + break + elif isinstance(error, (str, int, float)) and str(error).strip(): + detail = str(error).strip() + if not detail: + for field in ("message", "detail"): + value = payload.get(field) + if isinstance(value, (str, int, float)) and str(value).strip(): + detail = str(value).strip() + break + + status = f"HTTP {response.status_code}" + if detail: + return f"{status}: {detail[:_DIAGNOSTIC_PREVIEW_LIMIT]}" + return status + + +def _extract_gemini_text(payload: dict[str, Any]) -> str: + """Extract text from a Gemini ``generateContent`` response.""" + candidates = payload.get("candidates") + if not isinstance(candidates, list) or not candidates: + return "" + content = candidates[0].get("content", {}) + parts = content.get("parts", []) if isinstance(content, dict) else [] + if not isinstance(parts, list): + return "" + return "".join( + str(part.get("text", "")) for part in parts if isinstance(part, dict) + ).strip() + + +def _extract_ollama_text(payload: dict[str, Any]) -> str: + """Extract text from an Ollama chat response.""" + message = payload.get("message") + if not isinstance(message, dict): + return "" + return str(message.get("content", "")).strip() + + +async def validate_llm_configuration( + *, + provider: str, + model: str, + base_url: str | None, + api_key: str | None, +) -> LLMValidationResult: + """Run reachability, authentication, and generation checks in order. + + The candidate settings are used in-memory only. Failure results include a + bounded diagnostic drawn only from standard provider error fields. + """ + if provider not in {"gemini", "ollama", "ollama_cloud"}: + return _failed("The selected LLM provider is not supported.") + if not model.strip(): + return _failed("Enter a model name before testing the LLM.") + + provider_url = ( + _GEMINI_BASE_URL + if provider == "gemini" + else str(base_url or "").strip().rstrip("/") + ) + if not provider_url: + return _failed("Enter a base URL before testing the LLM.") + if provider in {"gemini", "ollama_cloud"} and not api_key: + return _failed("Enter an API key before testing the LLM.") + + headers = {"Content-Type": "application/json"} + if provider in {"ollama", "ollama_cloud"} and api_key: + headers["Authorization"] = f"Bearer {api_key}" + + timeout = httpx.Timeout(30.0, connect=5.0) + async with httpx.AsyncClient( + timeout=timeout, follow_redirects=True + ) as client: + try: + # Any HTTP response proves that the configured host is reachable. + await client.get(provider_url) + except httpx.RequestError: + return _failed("The LLM URL could not be reached.") + + try: + if provider == "gemini": + credential_response = await client.get( + f"{provider_url}/v1beta/models", + headers={"x-goog-api-key": str(api_key)}, + ) + else: + credential_response = await client.get( + f"{provider_url}/api/tags", headers=headers + ) + except httpx.RequestError: + return _failed( + "The LLM URL became unavailable while checking credentials.", + url_reachable=True, + ) + + if not credential_response.is_success: + return _failed( + "The LLM URL or API key was rejected. Check both values.", + url_reachable=True, + diagnostic=_provider_error_diagnostic(credential_response), + ) + + safe_model = quote(model.strip(), safe="") + try: + if provider == "gemini": + ping_response = await client.post( + f"{provider_url}/v1beta/models/{safe_model}:generateContent", + headers={ + "Content-Type": "application/json", + "x-goog-api-key": str(api_key), + }, + json={ + "contents": [ + {"parts": [{"text": _PING_PROMPT}]} + ], + "generationConfig": { + "maxOutputTokens": _PING_MAX_OUTPUT_TOKENS + }, + }, + ) + else: + ping_response = await client.post( + f"{provider_url}/api/chat", + headers=headers, + json={ + "model": model.strip(), + "messages": [ + {"role": "user", "content": _PING_PROMPT} + ], + "stream": False, + # Reasoning models such as gpt-oss can consume a very + # small generation budget entirely in hidden thinking, + # leaving ``message.content`` empty. A connection + # probe needs a visible response, so turn thinking off. + "think": False, + "options": {"num_predict": _PING_MAX_OUTPUT_TOKENS}, + }, + ) + except httpx.RequestError: + return _failed( + "Credentials were accepted, but the LLM did not respond to the ping.", + url_reachable=True, + credentials_valid=True, + ) + + if not ping_response.is_success: + # A 403 can mean that the otherwise valid credential is not allowed + # to use this particular model. Only 401 conclusively invalidates + # the credential-validation stage. + credentials_rejected = ping_response.status_code == 401 + return _failed( + ( + "The LLM rejected the credentials while prompting the selected " + "model." + if credentials_rejected + else "Credentials were accepted, but the selected model rejected " + "the ping." + ), + url_reachable=True, + credentials_valid=not credentials_rejected, + diagnostic=_provider_error_diagnostic(ping_response), + ) + + try: + ping_payload: dict[str, Any] = ping_response.json() + except ValueError: + return _failed( + "The LLM returned an invalid response to the ping.", + url_reachable=True, + credentials_valid=True, + ) + reply = ( + _extract_gemini_text(ping_payload) + if provider == "gemini" + else _extract_ollama_text(ping_payload) + ) + if not reply: + return _failed( + "The LLM returned an empty response to the ping.", + url_reachable=True, + credentials_valid=True, + ) + + return LLMValidationResult( + ok=True, + url_reachable=True, + credentials_valid=True, + llm_responded=True, + message="LLM connection test passed.", + response=reply[:_RESPONSE_PREVIEW_LIMIT], + ) diff --git a/data_forecaster/backend/services/pipeline_service.py b/data_forecaster/backend/services/pipeline_service.py index eeee251..05c50f2 100644 --- a/data_forecaster/backend/services/pipeline_service.py +++ b/data_forecaster/backend/services/pipeline_service.py @@ -161,7 +161,7 @@ def run_pipeline( freq: Frequency string. forecast_horizon: Number of future periods to forecast. forced_model: Optional model override (``"ARIMA"``, ``"SARIMA"``, - ``"Holt-Winters"``). + ``"Holt-Winters"``, ``"Prophet"``). user_prompt: Optional extra instructions for the report agent. preflight_options: Optional preflight configuration dict. chroma_persist_dir: Path to the ChromaDB persistence directory. @@ -548,6 +548,11 @@ def _select_model( if forced_model == "SARIMA" else "Not selected (user chose a different model)." ), + prophet_rejected_reason=( + None + if forced_model == "Prophet" + else "Not selected (user chose a different model)." + ), reasoning_steps=[ { "thought": ( diff --git a/data_forecaster/backend/services/setup_service.py b/data_forecaster/backend/services/setup_service.py new file mode 100644 index 0000000..2882714 --- /dev/null +++ b/data_forecaster/backend/services/setup_service.py @@ -0,0 +1,183 @@ +"""First-run setup service for the Data Forecaster backend. + +Owns the atomic bootstrap that creates the first admin API user, generates +the backend encryption key, and flips ``setup_complete``. Also owns the +startup migration that seeds DB-backed configuration for deployments that +pre-date the setup wizard. + +The bootstrap is race-safe: the guard is a conditional ``INSERT ... WHERE +NOT EXISTS`` inside a ``BEGIN IMMEDIATE`` transaction (the same pattern as +``services.job_service._claim_job``), so two simultaneous first-run +requests cannot both succeed. +""" + +from __future__ import annotations + +import sqlite3 +from typing import Any + +from auth.argon2_helpers import hash_api_key +from core import secret_store +from core.database import get_connection +from core.llm_config_store import is_configured +from core.logging_config import get_logger + +logger = get_logger(__name__) + +_BOOTSTRAP_DESCRIPTION: str = "Admin API user (created via setup wizard)" + + +class SetupAlreadyCompleteError(Exception): + """Raised when bootstrap is attempted after setup has completed.""" + + +def is_setup_complete(db_path: str | None = None) -> bool: + """Return whether first-run setup has completed. + + Args: + db_path: Optional database path override (testing). + + Returns: + ``True`` when the ``system_settings.setup_complete`` flag is set. + """ + with get_connection(db_path) as connection: + row: sqlite3.Row | None = connection.execute( + "SELECT setup_complete FROM system_settings WHERE singleton = 1" + ).fetchone() + return bool(row["setup_complete"]) if row else False + + +def mark_setup_complete(db_path: str | None = None) -> None: + """Set the ``setup_complete`` flag (idempotent).""" + with get_connection(db_path) as connection: + connection.execute( + "UPDATE system_settings SET setup_complete = 1," + " updated_at = datetime('now') WHERE singleton = 1" + ) + connection.commit() + + +def get_setup_status(db_path: str | None = None) -> dict[str, Any]: + """Return setup completion state without exposing any secrets. + + Args: + db_path: Optional database path override (testing). + + Returns: + A dict with ``setup_complete``, ``admin_exists``, + ``llm_configured``, and ``models_enabled`` booleans/counts. + """ + with get_connection(db_path) as connection: + user_row: sqlite3.Row | None = connection.execute( + "SELECT COUNT(*) AS cnt FROM api_users WHERE is_admin = 1" + ).fetchone() + model_row: sqlite3.Row | None = connection.execute( + "SELECT COUNT(*) AS cnt FROM model_config WHERE enabled = 1" + ).fetchone() + admin_exists = bool(user_row and int(user_row["cnt"]) > 0) + enabled_models = int(model_row["cnt"]) if model_row else 0 + return { + "setup_complete": is_setup_complete(db_path), + "admin_exists": admin_exists, + "llm_configured": is_configured(db_path), + "models_enabled": enabled_models, + } + + +def run_bootstrap( + username: str, + api_key: str, + db_path: str | None = None, +) -> dict[str, Any]: + """Atomically create the first admin API user and complete setup. + + Generates and persists the backend encryption key (skipped when already + present), inserts the admin user only when no API users exist, and sets + ``setup_complete`` — all inside a single ``BEGIN IMMEDIATE`` + transaction so concurrent first-run requests cannot both succeed. + + Args: + username: Username for the first admin API user. + api_key: Plaintext API key chosen by the admin. + db_path: Optional database path override (testing). + + Returns: + A dict with the new user's fields (no key material). + + Raises: + ValueError: When the username or key is empty. + SetupAlreadyCompleteError: When API users already exist. + """ + if not username or not username.strip(): + raise ValueError("Username is required.") + if not api_key: + raise ValueError("API key is required.") + + connection: sqlite3.Connection = get_connection(db_path) + try: + connection.execute("BEGIN IMMEDIATE") + cursor: sqlite3.Cursor = connection.execute( + """ + INSERT INTO api_users + (username, api_key_hash, description, enabled, bootstrap, + is_admin) + SELECT ?, ?, ?, 1, 1, 1 + WHERE NOT EXISTS (SELECT 1 FROM api_users) + """, + (username.strip(), hash_api_key(api_key), _BOOTSTRAP_DESCRIPTION), + ) + if cursor.rowcount == 0: + connection.rollback() + raise SetupAlreadyCompleteError( + "API users already exist — setup bootstrap is no longer " + "available." + ) + connection.execute( + "UPDATE system_settings SET setup_complete = 1," + " updated_at = datetime('now') WHERE singleton = 1" + ) + connection.commit() + except SetupAlreadyCompleteError: + raise + except Exception: + connection.rollback() + raise + finally: + connection.close() + + secret_store.generate_and_persist_key() + logger.info("Setup bootstrap complete. Admin user '%s' created.", username) + + with get_connection(db_path) as read_conn: + row: sqlite3.Row | None = read_conn.execute( + """ + SELECT id, username, description, enabled, bootstrap, is_admin, + created_at, last_used, last_used_ip + FROM api_users WHERE username = ? + """, + (username.strip(),), + ).fetchone() + return dict(row) if row else {} + + +def migrate_legacy_deployment(db_path: str | None = None) -> None: + """Seed DB-backed configuration for pre-wizard deployments. + + When API users already exist but ``setup_complete`` is false, the + deployment predates the setup wizard: mark setup complete so the + env-based service-user reconciliation is skipped from now on. The + ``model_config`` seed rows are created by ``init_database`` itself. + + Args: + db_path: Optional database path override (testing). + """ + with get_connection(db_path) as connection: + row: sqlite3.Row | None = connection.execute( + "SELECT COUNT(*) AS cnt FROM api_users" + ).fetchone() + if row and int(row["cnt"]) > 0 and not is_setup_complete(db_path): + mark_setup_complete(db_path) + logger.info( + "Existing API users found — marking setup complete (legacy " + "deployment migration)." + ) diff --git a/data_forecaster/backend/uv.txt b/data_forecaster/backend/uv.txt index a79353f..69e9898 100644 --- a/data_forecaster/backend/uv.txt +++ b/data_forecaster/backend/uv.txt @@ -17,6 +17,5 @@ matplotlib==3.10.9 python-multipart==0.0.31 aiofiles==23.2.1 openpyxl==3.1.3 -python-dotenv==1.2.2 argon2-cffi==23.1.0 pmdarima==2.0.4 diff --git a/data_forecaster/docker/docker-compose.backend.yml b/data_forecaster/docker/docker-compose.backend.yml index effa217..fec9d5f 100644 --- a/data_forecaster/docker/docker-compose.backend.yml +++ b/data_forecaster/docker/docker-compose.backend.yml @@ -11,14 +11,13 @@ services: build: context: ../backend dockerfile: ../docker/Dockerfile.backend - env_file: - - ../backend/.env volumes: - logs_data:/app/logs - chroma_data:/app/chroma_db - file_store_data:/app/file_store - ../data:/app/data - api_key_data:/app/data/api_keys + - secret_data:/app/secrets restart: unless-stopped networks: - forecasting_network @@ -66,3 +65,4 @@ volumes: chroma_data: file_store_data: api_key_data: + secret_data: diff --git a/data_forecaster/docker/docker-compose.distributed.yml b/data_forecaster/docker/docker-compose.distributed.yml index 638bbb3..ac0b0bf 100644 --- a/data_forecaster/docker/docker-compose.distributed.yml +++ b/data_forecaster/docker/docker-compose.distributed.yml @@ -31,5 +31,4 @@ services: frontend: environment: - FLASK_ENV=production - - SECRET_KEY=${SECRET_KEY:-change-me-in-production} depends_on: [] diff --git a/data_forecaster/docker/docker-compose.yml b/data_forecaster/docker/docker-compose.yml index 58363f9..bd82b16 100644 --- a/data_forecaster/docker/docker-compose.yml +++ b/data_forecaster/docker/docker-compose.yml @@ -14,14 +14,13 @@ services: build: context: ../backend dockerfile: ../docker/Dockerfile.backend - env_file: - - ../backend/.env volumes: - logs_data:/app/logs - chroma_data:/app/chroma_db - file_store_data:/app/file_store - ../data:/app/data - api_key_data:/app/data/api_keys + - secret_data:/app/secrets restart: unless-stopped networks: - forecasting_network @@ -38,11 +37,8 @@ services: build: context: ../frontend dockerfile: ../docker/Dockerfile.flask - env_file: - - ../frontend/.env environment: - FLASK_ENV=production - - SECRET_KEY=${SECRET_KEY:-change-me-in-production} volumes: - flask_instance:/app/instance depends_on: @@ -121,3 +117,4 @@ volumes: file_store_data: flask_instance: api_key_data: + secret_data: diff --git a/data_forecaster/frontend/.env.example b/data_forecaster/frontend/.env.example deleted file mode 100644 index 3ba4cf3..0000000 --- a/data_forecaster/frontend/.env.example +++ /dev/null @@ -1,12 +0,0 @@ -# --- Frontend Configuration --- - -# Flask session secret (required in production). -# Generate with: python ../backend/scripts/bootstrap.py -SECRET_KEY=change-me-in-production - -# Fernet key for encrypting stored backend API credentials. -# Generate with: python ../backend/scripts/bootstrap.py -FLASK_ENCRYPTION_KEY=generate-a-fernet-key-and-paste-here - -# Optional path to demo CSV data. -# DEMO_DATA_PATH=/absolute/path/to/demo_data.csv diff --git a/data_forecaster/frontend/app.py b/data_forecaster/frontend/app.py index daa2cad..48e7521 100644 --- a/data_forecaster/frontend/app.py +++ b/data_forecaster/frontend/app.py @@ -19,7 +19,9 @@ from blueprints.admin import admin_bp from blueprints.auth import auth_bp +from blueprints.decorators import get_backend_setup_status from blueprints.main import main_bp +from blueprints.setup import setup_bp from config import get_config from db.db import init_app as db_init_app, init_db, query_db from extensions import csrf, login_manager @@ -72,6 +74,7 @@ def create_app(config_name: str | None = None) -> Flask: _register_blueprints(app) _register_context_processors(app) _register_user_loader() + _register_setup_gate(app) _register_password_change(app) register_commands(app) @@ -159,6 +162,7 @@ def _register_blueprints(app: Flask) -> None: app.register_blueprint(auth_bp) app.register_blueprint(main_bp) app.register_blueprint(admin_bp) + app.register_blueprint(setup_bp) def _register_context_processors(app: Flask) -> None: @@ -238,6 +242,42 @@ def load_user(user_id: str) -> User | None: return None +def _register_setup_gate(app: Flask) -> None: + """Redirect all requests to the setup wizard until setup is complete. + + The backend's ``GET /setup/status`` endpoint is the source of truth. + While ``setup_complete`` is false (or the backend is unreachable), + every request outside the allowlist — setup wizard, static assets, + and the login/logout endpoints — is redirected to ``/setup`` (or + rejected with a 403 for JSON/AJAX callers). + + Args: + app: The Flask application instance. + """ + + @app.before_request + def _enforce_setup_complete() -> ( + Response | tuple[Response, int] | tuple[str, int] | None + ): + """Redirect to the setup wizard until backend setup completes.""" + endpoint = request.endpoint or "" + if ( + endpoint == "static" + or endpoint.startswith("setup.") + or endpoint in ("auth.login", "auth.logout") + ): + return None + + if get_backend_setup_status().get("setup_complete"): + return None + + # AJAX/JSON callers get a structured error instead of a redirect. + if request.path.startswith("/api/") or _wants_json(): + return jsonify({"error": "Setup required."}), 403 + + return redirect(url_for("setup.index")) + + def _register_password_change(app: Flask) -> None: """Enforce a forced password change across all blueprints. diff --git a/data_forecaster/frontend/blueprints/admin/forms.py b/data_forecaster/frontend/blueprints/admin/forms.py index e80f5d6..939da56 100644 --- a/data_forecaster/frontend/blueprints/admin/forms.py +++ b/data_forecaster/frontend/blueprints/admin/forms.py @@ -7,6 +7,7 @@ from flask_wtf import FlaskForm # type: ignore[import-untyped] from wtforms import ( BooleanField, + FloatField, IntegerField, PasswordField, SelectField, @@ -143,3 +144,61 @@ class APIKeyCreateForm(FlaskForm): # type: ignore[misc] ) is_admin = BooleanField("Administrator") submit = SubmitField("Create API User") + + +class LLMConfigForm(FlaskForm): # type: ignore[misc] + """Form for updating the backend LLM configuration. + + The API key is a write-only field: leaving it blank preserves the + key stored on the backend. The key is forwarded to the backend and + never persisted by the frontend. + + Fields: + provider: LLM provider (``gemini``, ``ollama``, ``ollama_cloud``). + model: Model name served by the provider. + base_url: Provider base URL (Ollama providers only). + api_key: New provider API key (optional, write-only). + temperature: Sampling temperature. + submit: Submission button. + """ + + provider = SelectField( + "Provider", + choices=[ + ("gemini", "Google Gemini"), + ("ollama", "Ollama (local)"), + ("ollama_cloud", "Ollama Cloud"), + ], + validators=[DataRequired()], + ) + model = StringField( + "Model", + validators=[DataRequired(), Length(min=1, max=128)], + ) + base_url = StringField( + "Base URL", + validators=[Optional(), URL(require_tld=False)], + ) + api_key = PasswordField( + "API Key", + validators=[Optional(), Length(max=256)], + ) + temperature = FloatField( + "Temperature", + validators=[DataRequired(), NumberRange(min=0.0, max=2.0)], + ) + test_llm = SubmitField("Test LLM") + submit = SubmitField("Save LLM Configuration") + + +class ModelsForm(FlaskForm): # type: ignore[misc] + """Form for model enablement toggles (CSRF token + submit only). + + The per-model checkboxes are rendered dynamically from the backend's + ``GET /models`` response as plain inputs named ``model_enabled``. + + Fields: + submit: Submission button. + """ + + submit = SubmitField("Save Model Selection") diff --git a/data_forecaster/frontend/blueprints/admin/routes.py b/data_forecaster/frontend/blueprints/admin/routes.py index 5c9d06a..be59363 100644 --- a/data_forecaster/frontend/blueprints/admin/routes.py +++ b/data_forecaster/frontend/blueprints/admin/routes.py @@ -31,6 +31,8 @@ from blueprints.admin.forms import ( APIConfigForm, APIKeyCreateForm, + LLMConfigForm, + ModelsForm, UserCreateForm, UserEditForm, ) @@ -38,6 +40,7 @@ from db.db import execute_db, query_db from services.api_client import BackendAPIClient, get_api_client from services.connection_errors import sanitize_connection_error +from services.credentials_service import save_api_credentials from services.report_service import ( delete_all_reports_for_admin, delete_report_for_admin, @@ -667,55 +670,17 @@ def _save_api_credentials( ) -> None: """Upsert the default API credential row. - When both encrypted values are supplied the row is fully updated; - otherwise only ``base_url``, ``timeout``, and ``verify_ssl`` are - touched, preserving any existing encrypted credentials. + Delegates to :func:`services.credentials_service.save_api_credentials` + so the setup wizard shares the same persistence logic. """ - if enc_user and enc_pass: - execute_db( - """ - INSERT INTO api_credentials - (label, base_url, encrypted_username, encrypted_password, - timeout, verify_ssl) - VALUES ('default', ?, ?, ?, ?, ?) - ON CONFLICT(label) DO UPDATE SET - base_url = excluded.base_url, - encrypted_username = excluded.encrypted_username, - encrypted_password = excluded.encrypted_password, - timeout = excluded.timeout, - verify_ssl = excluded.verify_ssl - """, - (base_url, enc_user, enc_pass, timeout, verify_ssl), - ) - elif enc_user and preserve_existing_key: - execute_db( - """ - INSERT INTO api_credentials - (label, base_url, encrypted_username, encrypted_password, - timeout, verify_ssl) - VALUES ('default', ?, ?, NULL, ?, ?) - ON CONFLICT(label) DO UPDATE SET - base_url = excluded.base_url, - encrypted_username = excluded.encrypted_username, - timeout = excluded.timeout, - verify_ssl = excluded.verify_ssl - """, - (base_url, enc_user, timeout, verify_ssl), - ) - else: - execute_db( - """ - INSERT INTO api_credentials - (label, base_url, encrypted_username, encrypted_password, - timeout, verify_ssl) - VALUES ('default', ?, NULL, NULL, ?, ?) - ON CONFLICT(label) DO UPDATE SET - base_url = excluded.base_url, - timeout = excluded.timeout, - verify_ssl = excluded.verify_ssl - """, - (base_url, timeout, verify_ssl), - ) + save_api_credentials( + base_url, + timeout, + verify_ssl, + enc_user, + enc_pass, + preserve_existing_key, + ) def _client_from_api_config_form() -> BackendAPIClient | None: @@ -889,80 +854,315 @@ def api_config_test() -> Response: @admin_bp.route("/api-config/enable-auth", methods=["POST"]) @admin_required def api_config_enable_auth() -> Response: - """Enable API authentication on the backend via the bootstrap endpoint. + """Route auth enablement through the first-run setup wizard. - Reads the admin key, desired username, and API key from the form, - calls the backend's ``POST /api-users/bootstrap`` endpoint, and - stores the returned credentials encrypted in the frontend database. + The backend's ``POST /api-users/bootstrap`` endpoint and the + ``ADMIN_API_KEY`` deployment secret were removed — first-run auth + enablement now happens exclusively through the setup wizard's atomic + ``POST /setup/bootstrap``. This handler redirects to the wizard when + the backend has no API users, and reports current state otherwise. Returns: - Redirect to the API config page with a flash message. + Redirect to the setup wizard or back to the API config page. """ - admin_key: str = str(request.form.get("admin_key", "")).strip() - api_username: str = str(request.form.get("api_username", "")).strip() - api_key: str = str(request.form.get("api_key", "")).strip() + status: dict[str, Any] = {} + try: + client = get_api_client() + resp = client.get_setup_status() + if resp.status_code == 200: + status = resp.json() + except (requests.RequestException, ValueError): + status = {} - if not admin_key: - flash("Admin key is required.", "danger") + if not status: + flash( + "Backend unreachable — cannot determine setup state.", + "danger", + ) return redirect(url_for(_ADMIN_API_CONFIG_ENDPOINT)) - if not api_username or not api_key: - flash("Username and API key are required.", "danger") + + if status.get("admin_exists"): + flash( + "API users already exist on the backend. Manage them under " + "API Keys; first-run bootstrap is no longer available.", + "info", + ) return redirect(url_for(_ADMIN_API_CONFIG_ENDPOINT)) - client = get_api_client() + flash( + "No API users exist yet. Complete the setup wizard to create the " + "first admin API user and enable authentication.", + "info", + ) + return redirect(url_for("setup.index")) + + +# ── LLM Configuration & Model Registry ──────────────────────────────────────── + +_ADMIN_LLM_CONFIG_ENDPOINT: str = "admin.llm_config" +_ADMIN_MODELS_ENDPOINT: str = "admin.models" + + +def _backend_error_detail(resp: requests.Response) -> str: + """Extract the ``detail`` message from a backend error response.""" try: - resp = client.bootstrap_api_user(api_username, api_key, admin_key) + detail: str = resp.json().get("detail", "Unknown error.") + return detail + except ValueError: + return "Unknown error." + + +def _llm_config_payload(form: LLMConfigForm) -> dict[str, Any]: + """Build the ``PUT /config/llm`` payload from a validated form. + + The API key is included only when a new one was entered — omitting + it preserves the key stored on the backend (one-way write). + """ + payload: dict[str, Any] = { + "provider": str(form.provider.data), + "model": str(form.model.data or "").strip(), + "temperature": float(form.temperature.data or 0.1), + } + base_url: str = str(form.base_url.data or "").strip() + if base_url: + payload["base_url"] = base_url + api_key: str = str(form.api_key.data or "").strip() + if api_key: + payload["api_key"] = api_key + return payload + + +def _submit_llm_config_update( + client: BackendAPIClient, payload: dict[str, Any] +) -> bool: + """PUT the LLM configuration to the backend, flashing the outcome.""" + try: + resp = client.put_llm_config(payload) except requests.RequestException as exc: flash( - f"Could not connect to backend: {_sanitise_connection_error(str(exc))}", + f"Could not connect to backend: " + f"{_sanitise_connection_error(str(exc))}", "danger", ) - return redirect(url_for(_ADMIN_API_CONFIG_ENDPOINT)) - - if resp.status_code == 403: + return False + if resp.status_code == 200: + flash("LLM configuration saved.", "success") + return True + else: flash( - "Invalid admin key. Verify the ADMIN_API_KEY in the backend .env.", + f"Could not save LLM configuration (HTTP {resp.status_code}): " + f"{_backend_error_detail(resp)}", "danger", ) - return redirect(url_for(_ADMIN_API_CONFIG_ENDPOINT)) - if resp.status_code == 409: + return False + + +def _test_llm_config_update( + client: BackendAPIClient, payload: dict[str, Any] +) -> dict[str, Any]: + """Test candidate LLM settings and return a safe result for the UI.""" + try: + resp = client.test_llm_config(payload) + except requests.RequestException as exc: + return { + "ok": False, + "message": ( + "Could not connect to backend: " + f"{_sanitise_connection_error(str(exc))}" + ), + } + if resp.status_code != 200: + return { + "ok": False, + "message": ( + f"Could not test LLM configuration (HTTP {resp.status_code}): " + f"{_backend_error_detail(resp)}" + ), + } + try: + result: dict[str, Any] = resp.json() + return result + except ValueError: + return { + "ok": False, + "message": "Backend returned an invalid LLM test response.", + } + + +def _fetch_llm_config(client: BackendAPIClient) -> dict[str, Any] | None: + """Fetch the masked LLM configuration, flashing on failure. + + Returns: + The configuration dict, or ``None`` when unavailable. + """ + try: + resp = client.get_llm_config() + except requests.RequestException: + flash("Backend unreachable — current values unavailable.", "warning") + return None + if resp.status_code != 200: flash( - "API users already exist on the backend. Bootstrap is no longer available.", + f"Could not load LLM configuration (HTTP {resp.status_code}).", "warning", ) - return redirect(url_for(_ADMIN_API_CONFIG_ENDPOINT)) - if resp.status_code != 200: - detail: str = "Unknown error." - try: - detail = resp.json().get("detail", detail) - except ValueError: - logger.exception("Failed to parse bootstrap error response") - flash(f"Bootstrap failed (HTTP {resp.status_code}): {detail}", "danger") - return redirect(url_for(_ADMIN_API_CONFIG_ENDPOINT)) - - # Success — store the credentials encrypted in the frontend DB + return None try: - enc_user = encrypt(api_username) - enc_pass = encrypt(api_key) - execute_db( - """ - UPDATE api_credentials - SET encrypted_username = ?, - encrypted_password = ? - WHERE label = 'default' - """, - (enc_user, enc_pass), + config: dict[str, Any] = resp.json() + return config + except ValueError: + flash("Backend returned an invalid LLM configuration.", "warning") + return None + + +@admin_bp.route("/llm-config", methods=["GET", "POST"]) +@admin_required +def llm_config() -> str | Response: + """View and update the backend LLM configuration. + + The API key is a one-way write: the backend's ``GET /config/llm`` + response structurally excludes it (only ``api_key_set`` is shown), + and leaving the form field blank preserves the stored key. + + Returns: + Rendered template on GET/validation error; redirect on success. + """ + form = LLMConfigForm() + client = get_api_client() + + if form.validate_on_submit(): + payload = _llm_config_payload(form) + test_result = _test_llm_config_update(client, payload) + if form.test_llm.data: + return render_template( + "admin/llm_config.html", + form=form, + llm_config=_fetch_llm_config(client), + llm_test_result=test_result, + ) + if not test_result.get("ok"): + flash(str(test_result.get("message", "LLM connection test failed.")), "danger") + return render_template( + "admin/llm_config.html", + form=form, + llm_config=_fetch_llm_config(client), + llm_test_result=test_result, + ) + if _submit_llm_config_update(client, payload): + return redirect(url_for(_ADMIN_LLM_CONFIG_ENDPOINT)) + return render_template( + "admin/llm_config.html", + form=form, + llm_config=_fetch_llm_config(client), + llm_test_result=test_result, ) - except RuntimeError as exc: - flash(str(exc), "danger") - return redirect(url_for(_ADMIN_API_CONFIG_ENDPOINT)) - flash( - "API authentication enabled successfully. " - "Credentials stored — the frontend can now authenticate with the backend.", - "success", + config = _fetch_llm_config(client) + if config: + form.provider.data = str(config.get("provider", "gemini")) + form.model.data = str(config.get("model", "")) + form.base_url.data = str(config.get("base_url") or "") + form.temperature.data = float(config.get("temperature", 0.1)) + + return render_template( + "admin/llm_config.html", + form=form, + llm_config=config, + llm_test_result=None, ) - return redirect(url_for(_ADMIN_API_CONFIG_ENDPOINT)) + + +@admin_bp.route("/models", methods=["GET", "POST"]) +@admin_required +def models() -> str | Response: + """Enable or disable forecasting models via the backend registry. + + At least one model must remain enabled — enforced client-side, + server-side here, and by the backend (whose 400 "last model" error + is surfaced as a flash message). + + Returns: + Rendered template on GET; redirect on POST. + """ + form = ModelsForm() + client = get_api_client() + model_list: list[dict[str, Any]] = [] + + try: + resp = client.get_models() + if resp.status_code == 200: + model_list = resp.json().get("models", []) + else: + flash( + f"Could not load models (HTTP {resp.status_code}): " + f"{_backend_error_detail(resp)}", + "danger", + ) + except (requests.RequestException, ValueError): + flash("Backend unreachable — model list unavailable.", "danger") + + if request.method == "GET" or not form.validate_on_submit(): + return render_template( + "admin/models.html", + form=form, + models=model_list, + ) + + if not model_list: + flash("No models loaded from the backend.", "danger") + return redirect(url_for(_ADMIN_MODELS_ENDPOINT)) + + selected = set(request.form.getlist("model_enabled")) + if not selected: + flash("At least one model must remain enabled.", "danger") + return redirect(url_for(_ADMIN_MODELS_ENDPOINT)) + + if not _apply_model_changes(client, model_list, selected): + return redirect(url_for(_ADMIN_MODELS_ENDPOINT)) + + flash("Model selection saved.", "success") + return redirect(url_for(_ADMIN_MODELS_ENDPOINT)) + + +def _apply_model_changes( + client: BackendAPIClient, + model_list: list[dict[str, Any]], + selected: set[str], +) -> bool: + """PUT the desired enabled state for every changed model. + + Flash messages surface connection failures and the backend's + last-model guard (HTTP 400). + + Args: + client: The configured backend API client. + model_list: Current model states from the backend. + selected: Names of the models that should be enabled. + + Returns: + ``True`` when all updates succeeded, ``False`` otherwise. + """ + for model in model_list: + desired: bool = model["name"] in selected + if desired == bool(model.get("enabled")): + continue + try: + update_resp = client.put_model(str(model["name"]), desired) + except requests.RequestException as exc: + flash( + f"Could not connect to backend: " + f"{_sanitise_connection_error(str(exc))}", + "danger", + ) + return False + if update_resp.status_code != 200: + flash( + f"Could not update " + f"{model.get('display_name', model['name'])}: " + f"{_backend_error_detail(update_resp)}", + "danger", + ) + return False + return True def _check_backend_health() -> bool: diff --git a/data_forecaster/frontend/blueprints/decorators.py b/data_forecaster/frontend/blueprints/decorators.py index 1b926b1..0e7f9b0 100644 --- a/data_forecaster/frontend/blueprints/decorators.py +++ b/data_forecaster/frontend/blueprints/decorators.py @@ -1,19 +1,59 @@ """Decorators for Flask blueprint route protection. Provides :func:`password_change_required` which redirects users to the -password-change page when their ``must_change_password`` flag is set. +password-change page when their ``must_change_password`` flag is set, and +:func:`get_backend_setup_status` which powers the first-run setup gate. """ from __future__ import annotations +import logging from functools import wraps -from typing import Callable, TypeVar +from typing import Any, Callable, TypeVar +import requests from flask import flash, redirect, request, url_for from flask_login import current_user +from services.api_client import BackendAPIClient, resolve_backend_connection + _F = TypeVar("_F", bound=Callable[..., ...]) +logger = logging.getLogger(__name__) + + +def get_backend_setup_status() -> dict[str, Any]: + """Probe the backend ``GET /setup/status`` endpoint. + + Builds a lightweight, unauthenticated client — the endpoint requires no + auth, so the probe works before setup completes and before any + credentials are stored. Connection errors are tolerated and reported + as "setup incomplete" so the wizard's backend-connection step can + handle them. + + The connection settings come from the ``api_credentials`` DB row first + (via :func:`resolve_backend_connection`) so every gunicorn worker + agrees on the backend URL, with the in-process config as fallback. + + Returns: + The parsed status payload, or ``{"setup_complete": False}`` when + the backend URL is not configured, the backend is unreachable, or + the response is unexpected. + """ + base_url, verify_ssl = resolve_backend_connection() + if not base_url: + return {"setup_complete": False} + + client = BackendAPIClient(base_url=base_url, verify=verify_ssl) + try: + resp = client.get_setup_status() + if resp.status_code == 200: + data: dict[str, Any] = resp.json() + return data + except (requests.RequestException, ValueError): + logger.debug("Setup status probe failed — treating as incomplete.") + return {"setup_complete": False} + def password_change_required(f: _F) -> _F: """Redirect to the password-change page if the user must change their password. diff --git a/data_forecaster/frontend/blueprints/main/routes.py b/data_forecaster/frontend/blueprints/main/routes.py index e4c66cd..4b594f2 100644 --- a/data_forecaster/frontend/blueprints/main/routes.py +++ b/data_forecaster/frontend/blueprints/main/routes.py @@ -312,6 +312,7 @@ def model() -> str: "ARIMA": model_sel.get("arima_rejected_reason", ""), "SARIMA": model_sel.get("sarima_rejected_reason", ""), "EWMA": model_sel.get("ewma_rejected_reason", ""), + "Prophet": model_sel.get("prophet_rejected_reason", ""), }.items() if v and k != model_sel.get("selected_model") } diff --git a/data_forecaster/frontend/blueprints/setup/__init__.py b/data_forecaster/frontend/blueprints/setup/__init__.py new file mode 100644 index 0000000..48f65a0 --- /dev/null +++ b/data_forecaster/frontend/blueprints/setup/__init__.py @@ -0,0 +1,7 @@ +"""Setup wizard blueprint package.""" + +from flask import Blueprint + +setup_bp: Blueprint = Blueprint("setup", __name__, url_prefix="/setup") + +from blueprints.setup import routes # noqa: E402, F401 diff --git a/data_forecaster/frontend/blueprints/setup/forms.py b/data_forecaster/frontend/blueprints/setup/forms.py new file mode 100644 index 0000000..3bda6b0 --- /dev/null +++ b/data_forecaster/frontend/blueprints/setup/forms.py @@ -0,0 +1,131 @@ +""" +WTForms form definitions for the first-run setup wizard blueprint. +""" + +from __future__ import annotations + +from flask_wtf import FlaskForm +from wtforms import ( + BooleanField, + FloatField, + PasswordField, + RadioField, + StringField, + SubmitField, +) +from wtforms.validators import URL, DataRequired, Length, NumberRange, Optional + +LLM_PROVIDER_CHOICES: list[tuple[str, str]] = [ + ("gemini", "Google Gemini"), + ("ollama", "Ollama (local)"), + ("ollama_cloud", "Ollama Cloud"), +] + + +class BackendConnectionForm(FlaskForm): # type: ignore[misc] + """Step 1 — backend connection settings. + + Fields: + base_url: Root URL of the FastAPI backend. + verify_ssl: Whether to verify the backend TLS certificate. + submit: Submission button. + """ + + base_url = StringField( + "Backend API Base URL", + validators=[DataRequired(), URL(require_tld=False)], + ) + verify_ssl = BooleanField( + "Verify backend SSL certificate", + default=False, + ) + submit = SubmitField("Test & Continue") + + +class LLMProviderForm(FlaskForm): # type: ignore[misc] + """Step 2 — LLM provider configuration. + + Fields: + provider: LLM provider selection. + model: Model name served by the provider. + base_url: Provider base URL (Ollama providers only). + api_key: Provider API key (optional; forwarded to the backend + only — never persisted by the frontend). + temperature: Sampling temperature. + submit: Submission button. + """ + + provider = RadioField( + "Provider", + choices=LLM_PROVIDER_CHOICES, + default="gemini", + validators=[DataRequired()], + ) + model = StringField( + "Model", + validators=[DataRequired(), Length(min=1, max=128)], + ) + base_url = StringField( + "Base URL", + validators=[Optional(), URL(require_tld=False)], + ) + api_key = PasswordField( + "API Key", + validators=[Optional(), Length(max=256)], + ) + temperature = FloatField( + "Temperature", + default=0.1, + validators=[DataRequired(), NumberRange(min=0.0, max=2.0)], + ) + submit = SubmitField("Save & Continue") + + +class EnableAuthForm(FlaskForm): # type: ignore[misc] + """Step 3 — confirmation to enable backend API authentication. + + Fields: + confirm: Acknowledgement that API auth will be enabled. + submit: Submission button. + """ + + confirm = BooleanField( + "Enable API key authentication on the backend", + default=True, + validators=[DataRequired()], + ) + submit = SubmitField("Continue") + + +class ModelsForm(FlaskForm): # type: ignore[misc] + """Step 4 — model enablement (CSRF token only). + + The model checkboxes are rendered dynamically from the backend's + ``GET /models`` response, so they are plain HTML inputs named + ``model_enabled``; this form carries only the CSRF token and submit. + + Fields: + submit: Submission button. + """ + + submit = SubmitField("Save & Continue") + + +class AdminCreateForm(FlaskForm): # type: ignore[misc] + """Step 5 — first admin API user. + + Fields: + username: Username for the first admin API user. + api_key: API key for the user (auto-generated, editable). + submit: Submission button. + """ + + username = StringField( + "Admin API Username", + validators=[DataRequired(), Length(min=1, max=64)], + ) + api_key = StringField( + "API Key", + validators=[DataRequired(), Length(min=8, max=256)], + ) + submit = SubmitField("Create Admin & Finish Setup") diff --git a/data_forecaster/frontend/blueprints/setup/routes.py b/data_forecaster/frontend/blueprints/setup/routes.py new file mode 100644 index 0000000..8e12322 --- /dev/null +++ b/data_forecaster/frontend/blueprints/setup/routes.py @@ -0,0 +1,481 @@ +""" +Route handlers for the first-run setup wizard blueprint. + +The wizard runs while the backend reports ``setup_complete = false`` and +walks the operator through: backend connection → LLM provider → enable +API auth → model selection → create first admin → done. Progress is +tracked in the Flask session; every step re-validates backend +reachability and flashes errors on failure. + +Security notes: +- The LLM API key collected in step 2 is forwarded to the backend and is + never persisted by the frontend (no DB write, no logging). +- The bootstrap admin API user doubles as the frontend↔backend service + account — after ``POST /setup/bootstrap`` succeeds, its credentials are + stored (Fernet-encrypted) in ``api_credentials`` so the frontend can + talk to the now-auth-enabled backend. This avoids a second + ``/api-users/create`` round-trip during first-run setup. +""" + +from __future__ import annotations + +import logging +import secrets +from typing import Any + +import requests +from flask import ( + current_app, + flash, + redirect, + render_template, + request, + session, + url_for, +) +from werkzeug.wrappers import Response + +from blueprints.decorators import get_backend_setup_status +from blueprints.setup import setup_bp +from blueprints.setup.forms import ( + AdminCreateForm, + BackendConnectionForm, + EnableAuthForm, + LLMProviderForm, + ModelsForm, +) +from db.crypto import encrypt +from services.api_client import ( + BackendAPIClient, + get_api_client, + resolve_backend_connection, +) +from services.connection_errors import sanitize_connection_error +from services.credentials_service import save_api_credentials + +logger = logging.getLogger(__name__) + +_DEFAULT_TIMEOUT: int = 30 +_AUTH_LOGIN_ENDPOINT: str = "auth.login" +_TEMPLATE_BACKEND: str = "setup/backend.html" +_TEMPLATE_LLM: str = "setup/llm.html" +_TEMPLATE_AUTH: str = "setup/auth.html" +_TEMPLATE_MODELS: str = "setup/models.html" +_TEMPLATE_ADMIN: str = "setup/admin.html" +_TEMPLATE_DONE: str = "setup/done.html" + + +def _setup_complete() -> bool: + """Return whether the backend reports setup as complete.""" + return bool(get_backend_setup_status().get("setup_complete")) + + +def _response_detail(resp: requests.Response) -> str: + """Extract the ``detail`` message from an error response.""" + try: + detail: str = resp.json().get("detail", "Unknown error.") + return detail + except ValueError: + return "Unknown error." + + +def _render(template: str, status_code: int = 200, **context: Any) -> str | tuple[str, int]: + """Render a wizard template with an optional non-200 status code.""" + rendered = render_template(template, **context) + return rendered if status_code == 200 else (rendered, status_code) + + +@setup_bp.route("/") +def index() -> Response: + """Redirect to the first wizard step, or to login when done.""" + if _setup_complete(): + return redirect(url_for(_AUTH_LOGIN_ENDPOINT)) + return redirect(url_for("setup.backend")) + + +@setup_bp.route("/backend", methods=["GET", "POST"]) +def backend() -> str | tuple[str, int] | Response: + """Step 1 — configure and verify the backend connection. + + Saves the verified backend URL (and TLS verification preference) to + the ``api_credentials`` table without touching stored credentials. + """ + if _setup_complete(): + return redirect(url_for(_AUTH_LOGIN_ENDPOINT)) + + form = BackendConnectionForm() + if request.method == "GET": + # Resolve from the DB first so every worker pre-fills the same + # URL (a different worker may have verified the connection). + base_url, verify_ssl = resolve_backend_connection() + form.base_url.data = base_url + form.verify_ssl.data = verify_ssl + return _render(_TEMPLATE_BACKEND, form=form) + + if not form.validate_on_submit(): + return _render(_TEMPLATE_BACKEND, 400, form=form) + + base_url: str = str(form.base_url.data or "").rstrip("/") + verify_ssl: bool = bool(form.verify_ssl.data) + + probe = BackendAPIClient(base_url=base_url, verify=verify_ssl) + try: + resp = probe.get_setup_status() + except requests.RequestException as exc: + flash( + f"Could not connect to backend: " + f"{sanitize_connection_error(str(exc))}", + "danger", + ) + return _render(_TEMPLATE_BACKEND, 200, form=form) + + if resp.status_code != 200: + flash(f"Backend responded with HTTP {resp.status_code}.", "danger") + return _render(_TEMPLATE_BACKEND, 200, form=form) + + save_api_credentials(base_url, _DEFAULT_TIMEOUT, int(verify_ssl), None, None) + current_app.config["BACKEND_URL"] = base_url + current_app.config["API_VERIFY_SSL"] = verify_ssl + session["setup_backend_ok"] = True + flash("Backend connection verified.", "success") + return redirect(url_for("setup.llm")) + + +def _prefill_llm_form(form: LLMProviderForm) -> None: + """Populate the LLM form from the backend's current configuration. + + Args: + form: The form instance to populate in place. + """ + try: + resp = get_api_client().get_llm_config() + if resp.status_code != 200: + return + config: dict[str, Any] = resp.json() + except (requests.RequestException, ValueError): + flash("Backend unreachable — verify the connection step.", "warning") + return + form.provider.data = str(config.get("provider", "gemini")) + form.model.data = str(config.get("model", "")) + form.base_url.data = str(config.get("base_url") or "") + form.temperature.data = float(config.get("temperature", 0.1)) + + +def _submit_llm_config(form: LLMProviderForm) -> str | tuple[str, int] | None: + """Validate and then forward the LLM configuration to the backend. + + Args: + form: The validated LLM provider form. + + Returns: + ``None`` on success, otherwise a rendered error response. + """ + payload: dict[str, Any] = { + "provider": str(form.provider.data), + "model": str(form.model.data or "").strip(), + "temperature": float(form.temperature.data or 0.1), + } + base_url: str = str(form.base_url.data or "").strip() + if base_url: + payload["base_url"] = base_url + api_key: str = str(form.api_key.data or "").strip() + if api_key: + payload["api_key"] = api_key + + client = get_api_client() + try: + test_resp = client.test_llm_config(payload) + except requests.RequestException as exc: + flash( + f"Could not connect to backend: " + f"{sanitize_connection_error(str(exc))}", + "danger", + ) + return _render(_TEMPLATE_LLM, 200, form=form) + + if test_resp.status_code != 200: + flash( + f"Could not test LLM configuration (HTTP {test_resp.status_code}): " + f"{_response_detail(test_resp)}", + "danger", + ) + return _render(_TEMPLATE_LLM, 200, form=form) + try: + test_result: dict[str, Any] = test_resp.json() + except ValueError: + flash("Backend returned an invalid LLM test response.", "danger") + return _render(_TEMPLATE_LLM, 200, form=form) + if not test_result.get("ok"): + flash( + str(test_result.get("message", "LLM connection test failed.")), + "danger", + ) + return _render( + _TEMPLATE_LLM, + 200, + form=form, + llm_test_result=test_result, + ) + + try: + resp = client.put_llm_config(payload) + except requests.RequestException as exc: + flash( + f"LLM test passed, but the configuration could not be saved: " + f"{sanitize_connection_error(str(exc))}", + "danger", + ) + return _render(_TEMPLATE_LLM, 200, form=form) + if resp.status_code != 200: + flash( + f"LLM test passed, but the configuration could not be saved " + f"(HTTP {resp.status_code}): {_response_detail(resp)}", + "danger", + ) + return _render(_TEMPLATE_LLM, 200, form=form) + return None + + +@setup_bp.route("/llm", methods=["GET", "POST"]) +def llm() -> str | tuple[str, int] | Response: + """Step 2 — configure the LLM provider on the backend. + + The API key is forwarded to the backend only; the frontend never + persists it. Backend auth is still off at this point, so the + unauthenticated ``PUT /config/llm`` succeeds. + """ + if _setup_complete(): + return redirect(url_for(_AUTH_LOGIN_ENDPOINT)) + if not session.get("setup_backend_ok"): + return redirect(url_for("setup.backend")) + + form = LLMProviderForm() + if request.method == "GET": + _prefill_llm_form(form) + return _render(_TEMPLATE_LLM, form=form) + + if not form.validate_on_submit(): + return _render(_TEMPLATE_LLM, 400, form=form) + + error = _submit_llm_config(form) + if error is not None: + return error + + session["setup_llm_ok"] = True + flash("LLM connection verified and configuration saved.", "success") + return redirect(url_for("setup.auth")) + + +@setup_bp.route("/auth", methods=["GET", "POST"]) +def auth() -> str | tuple[str, int] | Response: + """Step 3 — confirm enabling API authentication. + + Generates the strong service-account key (``secrets.token_urlsafe``) + that step 5 uses when creating the first admin API user. + """ + if _setup_complete(): + return redirect(url_for(_AUTH_LOGIN_ENDPOINT)) + if not session.get("setup_llm_ok"): + return redirect(url_for("setup.llm")) + + if "setup_api_key" not in session: + session["setup_api_key"] = secrets.token_urlsafe(32) + + form = EnableAuthForm() + if form.validate_on_submit(): + session["setup_auth_ok"] = True + return redirect(url_for("setup.models")) + return _render(_TEMPLATE_AUTH, form=form) + + +def _fetch_model_list(client: BackendAPIClient) -> list[dict[str, Any]]: + """Fetch the model list from the backend, flashing errors. + + Args: + client: The configured backend API client. + + Returns: + The list of model state dicts, or an empty list on failure. + """ + try: + resp = client.get_models() + except requests.RequestException as exc: + flash( + f"Could not connect to backend: " + f"{sanitize_connection_error(str(exc))}", + "danger", + ) + return [] + if resp.status_code != 200: + flash( + f"Could not load models (HTTP {resp.status_code}): " + f"{_response_detail(resp)}", + "danger", + ) + return [] + try: + model_states: list[dict[str, Any]] = resp.json().get("models", []) + return model_states + except ValueError: + flash("Backend returned an invalid model list.", "danger") + return [] + + +def _apply_model_selection( + client: BackendAPIClient, + model_list: list[dict[str, Any]], + selected: set[str], + form: ModelsForm, +) -> str | tuple[str, int] | None: + """PUT the desired enabled state for every changed model. + + Args: + client: The configured backend API client. + model_list: Current model states from the backend. + selected: Names of the models that should be enabled. + form: The form instance (for error re-rendering). + + Returns: + ``None`` on success, otherwise a rendered error response. + """ + for model in model_list: + desired: bool = model["name"] in selected + if desired == bool(model.get("enabled")): + continue + try: + update_resp = client.put_model(str(model["name"]), desired) + except requests.RequestException as exc: + flash( + f"Could not connect to backend: " + f"{sanitize_connection_error(str(exc))}", + "danger", + ) + return _render(_TEMPLATE_MODELS, 200, form=form, models=model_list) + if update_resp.status_code != 200: + flash( + f"Could not update {model.get('display_name', model['name'])}: " + f"{_response_detail(update_resp)}", + "danger", + ) + return _render(_TEMPLATE_MODELS, 200, form=form, models=model_list) + return None + + +@setup_bp.route("/models", methods=["GET", "POST"]) +def models() -> str | tuple[str, int] | Response: + """Step 4 — enable the forecasting models to use. + + At least one model must remain enabled; enforced both client-side + (template script) and server-side here, in addition to the backend's + own last-model guard. + """ + if _setup_complete(): + return redirect(url_for(_AUTH_LOGIN_ENDPOINT)) + if not session.get("setup_auth_ok"): + return redirect(url_for("setup.auth")) + + form = ModelsForm() + client = get_api_client() + model_list = _fetch_model_list(client) + + if request.method == "GET" or not model_list: + return _render(_TEMPLATE_MODELS, form=form, models=model_list) + + if not form.validate_on_submit(): + return _render(_TEMPLATE_MODELS, 400, form=form, models=model_list) + + selected = set(request.form.getlist("model_enabled")) + if not selected: + flash("At least one model must remain enabled.", "danger") + return _render(_TEMPLATE_MODELS, 200, form=form, models=model_list) + + error = _apply_model_selection(client, model_list, selected, form) + if error is not None: + return error + + session["setup_models_ok"] = True + flash("Model selection saved.", "success") + return redirect(url_for("setup.admin")) + + +@setup_bp.route("/admin", methods=["GET", "POST"]) +def admin() -> str | tuple[str, int] | Response: + """Step 5 — create the first admin API user via atomic bootstrap. + + On success the same credentials are stored (Fernet-encrypted) in the + frontend ``api_credentials`` table so the frontend can authenticate + against the now-auth-enabled backend. + """ + if _setup_complete(): + return redirect(url_for(_AUTH_LOGIN_ENDPOINT)) + if not session.get("setup_models_ok"): + return redirect(url_for("setup.models")) + + form = AdminCreateForm() + if request.method == "GET": + form.api_key.data = str(session.get("setup_api_key", "")) + return _render(_TEMPLATE_ADMIN, form=form) + + if not form.validate_on_submit(): + return _render(_TEMPLATE_ADMIN, 400, form=form) + + username: str = str(form.username.data or "").strip() + api_key: str = str(form.api_key.data or "").strip() + + client = get_api_client() + try: + resp = client.setup_bootstrap(username, api_key) + except requests.RequestException as exc: + flash( + f"Could not connect to backend: " + f"{sanitize_connection_error(str(exc))}", + "danger", + ) + return _render(_TEMPLATE_ADMIN, 200, form=form) + + if resp.status_code == 409: + flash("Setup was already completed on the backend.", "info") + session["setup_done"] = True + return redirect(url_for("setup.done")) + if resp.status_code != 200: + flash( + f"Bootstrap failed (HTTP {resp.status_code}): " + f"{_response_detail(resp)}", + "danger", + ) + return _render(_TEMPLATE_ADMIN, 200, form=form) + + try: + # Resolve from the DB (not in-process config): a different + # gunicorn worker may have handled step 1, leaving this worker's + # config empty. Using config here would wipe the stored base_url. + base_url, verify_ssl = resolve_backend_connection() + if not base_url: + flash( + "Backend URL is missing — complete the connection step " + "again before creating the admin user.", + "danger", + ) + return _render(_TEMPLATE_ADMIN, 200, form=form) + save_api_credentials( + base_url, + _DEFAULT_TIMEOUT, + int(verify_ssl), + encrypt(username), + encrypt(api_key), + ) + except RuntimeError as exc: + flash(str(exc), "danger") + return _render(_TEMPLATE_ADMIN, 200, form=form) + + logger.info("Setup bootstrap completed for admin user '%s'.", username) + session["setup_done"] = True + return redirect(url_for("setup.done")) + + +@setup_bp.route("/done") +def done() -> str | tuple[str, int] | Response: + """Step 6 — completion summary.""" + if not session.get("setup_done") and not _setup_complete(): + return redirect(url_for("setup.index")) + return _render(_TEMPLATE_DONE) diff --git a/data_forecaster/frontend/config.py b/data_forecaster/frontend/config.py index 1e71e5e..fdd00ac 100644 --- a/data_forecaster/frontend/config.py +++ b/data_forecaster/frontend/config.py @@ -14,17 +14,54 @@ from __future__ import annotations import os +import secrets from datetime import timedelta +from pathlib import Path -from dotenv import load_dotenv +_INSTANCE_DIR = Path(__file__).resolve().parent / "instance" +_SESSION_KEY_FILE = _INSTANCE_DIR / ".session_key" +_LEGACY_ENV_FILE = Path(__file__).resolve().parent / ".env" -load_dotenv() + +def _legacy_env_value(key: str) -> str | None: + """Return a value from the pre-wizard ``.env`` file, if it exists. + + This is a one-time migration path only. Runtime configuration no longer + loads dotenv files. + """ + if not _LEGACY_ENV_FILE.is_file(): + return None + for line in _LEGACY_ENV_FILE.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#") and "=" in stripped: + name, value = stripped.split("=", 1) + if name.strip() == key: + return value.strip().strip("\"'") or None + return None + + +def _read_or_create_secret(path: Path, legacy_key: str) -> str: + """Load a stable instance secret or generate it securely on first run.""" + path.parent.mkdir(parents=True, exist_ok=True) + if path.is_file(): + return path.read_text(encoding="utf-8").strip() + + value = _legacy_env_value(legacy_key) or secrets.token_urlsafe(48) + try: + descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + return path.read_text(encoding="utf-8").strip() + try: + os.write(descriptor, value.encode("utf-8")) + finally: + os.close(descriptor) + return value class BaseConfig: """Shared settings inherited by all environment configurations.""" - SECRET_KEY: str = os.environ.get("SECRET_KEY", "change-me-in-production") + SECRET_KEY: str = _read_or_create_secret(_SESSION_KEY_FILE, "SECRET_KEY") DATABASE: str = os.path.join( os.path.dirname(os.path.abspath(__file__)), "instance", "forecaster.db" ) @@ -59,8 +96,7 @@ class DevelopmentConfig(BaseConfig): class ProductionConfig(BaseConfig): """Configuration for production deployment. - Enforces a strong ``SECRET_KEY`` from the environment and disables - debug output. + Uses the generated, persistent instance secret and disables debug output. """ DEBUG: bool = False @@ -70,20 +106,6 @@ class ProductionConfig(BaseConfig): SESSION_COOKIE_HTTPONLY: bool = True SESSION_COOKIE_SAMESITE: str = "Lax" - SECRET_KEY: str = os.environ.get("SECRET_KEY", "") - - def __init__(self) -> None: - """Raise when ``SECRET_KEY`` is absent in production. - - Raises: - RuntimeError: When ``SECRET_KEY`` is not set in the environment. - """ - if not self.SECRET_KEY: - raise RuntimeError( - "SECRET_KEY environment variable must be set in production." - ) - - class TestingConfig(BaseConfig): """Configuration for automated testing.""" diff --git a/data_forecaster/frontend/db/crypto.py b/data_forecaster/frontend/db/crypto.py index a5adcd9..cb010a6 100644 --- a/data_forecaster/frontend/db/crypto.py +++ b/data_forecaster/frontend/db/crypto.py @@ -2,32 +2,62 @@ Symmetric encryption helpers for storing sensitive configuration at rest. Uses Fernet (AES-128-CBC + HMAC-SHA256) from the ``cryptography`` library. -The encryption key must be provided via the ``FLASK_ENCRYPTION_KEY`` -environment variable and must never be stored in the database. +The encryption key is generated on first run and stored in the persistent +Flask instance directory with mode 0600. It is never stored in the database. """ from __future__ import annotations import os +from pathlib import Path from cryptography.fernet import Fernet +_INSTANCE_DIR = Path(__file__).resolve().parents[1] / "instance" +_KEY_FILE = _INSTANCE_DIR / ".encryption_key" +_LEGACY_ENV_FILE = Path(__file__).resolve().parents[1] / ".env" + + +def _legacy_env_key() -> bytes | None: + """Read the legacy key once so existing installs can migrate safely.""" + if not _LEGACY_ENV_FILE.is_file(): + return None + for line in _LEGACY_ENV_FILE.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if stripped and not stripped.startswith("#") and "=" in stripped: + name, value = stripped.split("=", 1) + if name.strip() == "FLASK_ENCRYPTION_KEY" and value.strip(): + return value.strip().strip("\"'").encode() + return None + + +def _read_or_create_key() -> bytes: + """Load the persistent key, migrating legacy dotenv data if necessary.""" + _INSTANCE_DIR.mkdir(parents=True, exist_ok=True) + if _KEY_FILE.is_file(): + return _KEY_FILE.read_bytes().strip() + + key = _legacy_env_key() or Fernet.generate_key() + try: + descriptor = os.open(_KEY_FILE, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + return _KEY_FILE.read_bytes().strip() + try: + os.write(descriptor, key) + finally: + os.close(descriptor) + return key + + def get_fernet() -> Fernet: - """Return a Fernet instance initialised from the environment key. + """Return a Fernet instance initialised from the persistent key. Raises: - RuntimeError: When ``FLASK_ENCRYPTION_KEY`` is not set. ValueError: When the key is not a valid 32-byte URL-safe base-64-encoded value. """ - raw_key = os.environ.get("FLASK_ENCRYPTION_KEY", "") - if not raw_key: - raise RuntimeError( - "FLASK_ENCRYPTION_KEY environment variable is not set. " - 'Generate one with: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"' - ) - return Fernet(raw_key.encode()) + return Fernet(_read_or_create_key()) def encrypt(plaintext: str) -> str: diff --git a/data_forecaster/frontend/manage.py b/data_forecaster/frontend/manage.py index a8bdc55..0718017 100644 --- a/data_forecaster/frontend/manage.py +++ b/data_forecaster/frontend/manage.py @@ -8,7 +8,6 @@ flask user-list flask credentials-set --label default --base-url http://backend:8000 flask credentials-list - flask generate-key """ from __future__ import annotations @@ -16,7 +15,6 @@ import re import click -from cryptography.fernet import Fernet from flask import Flask from werkzeug.security import generate_password_hash @@ -166,9 +164,3 @@ def credentials_delete(label: str) -> None: with app.app_context(): execute_db("DELETE FROM api_credentials WHERE label = ?", (label,)) click.echo(f"Credential '{label}' deleted.") - - @app.cli.command("generate-key") - def generate_key() -> None: - """Generate a new Fernet encryption key for FLASK_ENCRYPTION_KEY.""" - key = Fernet.generate_key().decode() - click.echo(f"Generated key (set as FLASK_ENCRYPTION_KEY):\n{key}") diff --git a/data_forecaster/frontend/requirements.txt b/data_forecaster/frontend/requirements.txt index 789c0a9..2cdf27c 100644 --- a/data_forecaster/frontend/requirements.txt +++ b/data_forecaster/frontend/requirements.txt @@ -10,6 +10,5 @@ fpdf2>=2.8.1 markdown>=3.6 bleach>=6.1.0 pandas>=2.2.0 -python-dotenv>=1.2.2 types-requests>=2.32.0 types-bleach>=6.1.0 diff --git a/data_forecaster/frontend/services/api_client.py b/data_forecaster/frontend/services/api_client.py index e6b3bf1..e56790d 100644 --- a/data_forecaster/frontend/services/api_client.py +++ b/data_forecaster/frontend/services/api_client.py @@ -9,10 +9,11 @@ from __future__ import annotations -import base64 +import sqlite3 from typing import Any import requests +from cryptography.fernet import InvalidToken from flask import current_app from db.crypto import decrypt @@ -478,31 +479,169 @@ def get_auth_status(self) -> requests.Response: verify=self._verify, ) - def bootstrap_api_user( - self, username: str, api_key: str, admin_key: str + def get_setup_status(self) -> requests.Response: + """Return the backend's first-run setup status. + + Calls the unauthenticated ``GET /setup/status`` endpoint. The + payload contains only completion booleans — never secrets. + + Returns: + The :class:`requests.Response` from ``GET /setup/status``. + """ + return requests.get( + f"{self._base_url}/setup/status", + timeout=JOB_STATUS_TIMEOUT, + verify=self._verify, + ) + + def setup_bootstrap( + self, username: str, api_key: str ) -> requests.Response: - """Create the first API user and enable auth on the backend. + """Atomically create the first admin API user and enable auth. - Calls the ``POST /api-users/bootstrap`` endpoint, protected by - the deployment-time ``ADMIN_API_KEY`` sent via the ``X-Admin-Key`` - header. No API auth headers are required (auth is still off). + Calls the one-time ``POST /setup/bootstrap`` endpoint. No auth + headers and no admin key are required — the endpoint returns 409 + once any API user exists. Args: - username: Desired username for the first API user. - api_key: Desired plaintext API key. - admin_key: The ``ADMIN_API_KEY`` deployment secret. + username: Desired username for the first admin API user. + api_key: Desired plaintext API key. Returns: - The :class:`requests.Response` from the bootstrap endpoint. + The :class:`requests.Response` from ``POST /setup/bootstrap``. """ return requests.post( - f"{self._base_url}/api-users/bootstrap", + f"{self._base_url}/setup/bootstrap", json={"username": username, "api_key": api_key}, - headers={"X-Admin-Key": admin_key}, timeout=ANALYSIS_TIMEOUT, verify=self._verify, ) + # ── LLM Configuration & Model Registry (admin) ────────────────────── + + def get_llm_config(self) -> requests.Response: + """Return the masked LLM configuration from the backend. + + The response structurally excludes the API key — only the + ``api_key_set`` boolean reveals whether a key is stored. + + Returns: + The :class:`requests.Response` from ``GET /config/llm``. + """ + return requests.get( + f"{self._base_url}/config/llm", + headers=self._headers(), + timeout=JOB_STATUS_TIMEOUT, + verify=self._verify, + ) + + def put_llm_config(self, payload: dict[str, Any]) -> requests.Response: + """Update the backend LLM configuration. + + Args: + payload: Dict with ``provider``, ``model``, ``temperature``, + and optionally ``base_url`` and ``api_key``. Omitting + ``api_key`` preserves the stored key (one-way write). + + Returns: + The :class:`requests.Response` from ``PUT /config/llm``. + """ + return requests.put( + f"{self._base_url}/config/llm", + json=payload, + headers=self._headers(), + timeout=ANALYSIS_TIMEOUT, + verify=self._verify, + ) + + def test_llm_config(self, payload: dict[str, Any]) -> requests.Response: + """Test candidate LLM settings without saving them. + + The backend checks provider reachability, credentials, and a real + model response in sequence. + """ + return requests.post( + f"{self._base_url}/config/llm/test", + json=payload, + headers=self._headers(), + timeout=CHAT_TIMEOUT, + verify=self._verify, + ) + + def get_models(self) -> requests.Response: + """List all forecasting models with their enabled state. + + Returns: + The :class:`requests.Response` from ``GET /models``. + """ + return requests.get( + f"{self._base_url}/models", + headers=self._headers(), + timeout=JOB_STATUS_TIMEOUT, + verify=self._verify, + ) + + def put_model(self, name: str, enabled: bool) -> requests.Response: + """Enable or disable a forecasting model on the backend. + + Args: + name: Canonical model name (e.g. ``"ARIMA"``). + enabled: ``True`` to enable, ``False`` to disable. + + Returns: + The :class:`requests.Response` from ``PUT /models/{name}``. + The backend returns 400 when the change would disable the + last enabled model. + """ + return requests.put( + f"{self._base_url}/models/{name}", + json={"enabled": enabled}, + headers=self._headers(), + timeout=ANALYSIS_TIMEOUT, + verify=self._verify, + ) + + +def resolve_backend_connection() -> tuple[str, bool]: + """Resolve the backend base URL and TLS preference for this request. + + The stored ``api_credentials`` row (label ``'default'``) is the source + of truth so every gunicorn worker observes the same connection + settings. Values fall back to the application config when the row is + absent or incomplete — for example during first-run setup before the + wizard's backend step has saved anything. + + Returns: + A ``(base_url, verify_ssl)`` tuple. ``base_url`` may be an empty + string when nothing is configured anywhere. + """ + base_url: str = current_app.config.get("BACKEND_URL", "") + verify_ssl: bool = bool(current_app.config.get("API_VERIFY_SSL", False)) + + try: + row = query_db( + """ + SELECT base_url, verify_ssl + FROM api_credentials + WHERE label = 'default' + LIMIT 1 + """, + one=True, + ) + except sqlite3.Error: + # The gate may run before the DB is seeded (e.g. fresh volume); + # fall back to config-only resolution instead of failing hard. + return base_url, verify_ssl + + if row and isinstance(row, dict): + stored_url = str(row.get("base_url", "") or "") + if stored_url: + base_url = stored_url + db_verify = row.get("verify_ssl") + if db_verify is not None: + verify_ssl = bool(db_verify) + return base_url, verify_ssl + def get_api_client() -> BackendAPIClient: """Construct a :class:`BackendAPIClient` for the current request. @@ -515,8 +654,7 @@ def get_api_client() -> BackendAPIClient: Returns: A configured :class:`BackendAPIClient` instance. """ - base_url: str = current_app.config.get("BACKEND_URL", "") - verify_ssl: bool = current_app.config.get("API_VERIFY_SSL", False) + base_url, verify_ssl = resolve_backend_connection() api_username: str | None = None api_key: str | None = None @@ -545,7 +683,9 @@ def get_api_client() -> BackendAPIClient: try: api_username = decrypt(str(enc_user)) api_key = decrypt(str(enc_pass)) - except Exception: + except (InvalidToken, ValueError, TypeError): + # Corrupt credentials (e.g. key rotated) must not break the + # client — treat as unauthenticated instead. api_username = None api_key = None diff --git a/data_forecaster/frontend/services/credentials_service.py b/data_forecaster/frontend/services/credentials_service.py new file mode 100644 index 0000000..25bfef3 --- /dev/null +++ b/data_forecaster/frontend/services/credentials_service.py @@ -0,0 +1,81 @@ +"""Shared persistence helpers for backend API credentials. + +Extracted from the admin blueprint so both the admin panel and the +first-run setup wizard can upsert the ``api_credentials`` row without +importing private helpers across blueprints. +""" + +from __future__ import annotations + +from db.db import execute_db + + +def save_api_credentials( + base_url: str, + timeout: int, + verify_ssl: int, + enc_user: str | None, + enc_pass: str | None, + preserve_existing_key: bool = False, +) -> None: + """Upsert the default API credential row. + + When both encrypted values are supplied the row is fully updated; + otherwise only ``base_url``, ``timeout``, and ``verify_ssl`` are + touched, preserving any existing encrypted credentials. + + Args: + base_url: Root URL of the FastAPI backend. + timeout: Request timeout in seconds. + verify_ssl: ``1`` to verify the backend TLS certificate, + ``0`` to skip verification. + enc_user: Fernet-encrypted API username, or ``None``. + enc_pass: Fernet-encrypted API key, or ``None``. + preserve_existing_key: When ``True`` and only ``enc_user`` is + supplied, keep the stored encrypted key. + """ + if enc_user and enc_pass: + execute_db( + """ + INSERT INTO api_credentials + (label, base_url, encrypted_username, encrypted_password, + timeout, verify_ssl) + VALUES ('default', ?, ?, ?, ?, ?) + ON CONFLICT(label) DO UPDATE SET + base_url = excluded.base_url, + encrypted_username = excluded.encrypted_username, + encrypted_password = excluded.encrypted_password, + timeout = excluded.timeout, + verify_ssl = excluded.verify_ssl + """, + (base_url, enc_user, enc_pass, timeout, verify_ssl), + ) + elif enc_user and preserve_existing_key: + execute_db( + """ + INSERT INTO api_credentials + (label, base_url, encrypted_username, encrypted_password, + timeout, verify_ssl) + VALUES ('default', ?, ?, NULL, ?, ?) + ON CONFLICT(label) DO UPDATE SET + base_url = excluded.base_url, + encrypted_username = excluded.encrypted_username, + timeout = excluded.timeout, + verify_ssl = excluded.verify_ssl + """, + (base_url, enc_user, timeout, verify_ssl), + ) + else: + execute_db( + """ + INSERT INTO api_credentials + (label, base_url, encrypted_username, encrypted_password, + timeout, verify_ssl) + VALUES ('default', ?, NULL, NULL, ?, ?) + ON CONFLICT(label) DO UPDATE SET + base_url = excluded.base_url, + timeout = excluded.timeout, + verify_ssl = excluded.verify_ssl + """, + (base_url, timeout, verify_ssl), + ) diff --git a/data_forecaster/frontend/templates/admin/api_config.html b/data_forecaster/frontend/templates/admin/api_config.html index a035b83..b151415 100644 --- a/data_forecaster/frontend/templates/admin/api_config.html +++ b/data_forecaster/frontend/templates/admin/api_config.html @@ -8,24 +8,14 @@
API Authentication: Disabled
-

The backend API is currently open (no auth required). Enable authentication to secure it.

+

+ The backend API is currently open (no auth required). + Authentication is enabled by creating the first admin API + user through the setup wizard. +

{{ form.hidden_tag() }} -
- - -
-
-
- - -
-
- - -
-
- +
diff --git a/data_forecaster/frontend/templates/admin/base.html b/data_forecaster/frontend/templates/admin/base.html index 554b3b9..73cddff 100644 --- a/data_forecaster/frontend/templates/admin/base.html +++ b/data_forecaster/frontend/templates/admin/base.html @@ -19,6 +19,8 @@
  • Users
  • API Config
  • API Keys
    • +
  • LLM Config
    • +
  • Models
  • Job Queue
  • Report Management
  • Settings
    • diff --git a/data_forecaster/frontend/templates/admin/llm_config.html b/data_forecaster/frontend/templates/admin/llm_config.html new file mode 100644 index 0000000..3f9a38f --- /dev/null +++ b/data_forecaster/frontend/templates/admin/llm_config.html @@ -0,0 +1,102 @@ +{% extends "admin/base.html" %} +{% block title %}LLM Configuration — Admin{% endblock %} +{% block header %}LLM Configuration{% endblock %} +{% block content %} +
    +
    + {% if llm_config %} +
    +
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Provider{{ llm_config.provider }}
    Model{{ llm_config.model }}
    Base URL{{ llm_config.base_url or '—' }}
    Temperature{{ llm_config.temperature }}
    API Key + {% if llm_config.api_key_set %} + Set + {% else %} + Not set + {% endif %} +
    Status + {% if llm_config.configured %} + Configured + {% else %} + Incomplete + {% endif %} +
    +
    +
    +
    + {% else %} +
    Current configuration could not be loaded from the backend.
    + {% endif %} + +
    +
    + {% if llm_test_result %} + + {% endif %} +
    + {{ form.hidden_tag() }} +
    {{ form.provider.label(class="form-label") }}{{ form.provider(class="form-select") }}
    +
    {{ form.model.label(class="form-label") }}{{ form.model(class="form-control") }} + {% for error in form.model.errors %}{{ error }}{% endfor %} +
    +
    {{ form.base_url.label(class="form-label") }}{{ form.base_url(class="form-control", placeholder="http://host.docker.internal:11434") }} + Only required for Ollama providers. + {% for error in form.base_url.errors %}{{ error }}{% endfor %} +
    +
    {{ form.api_key.label(class="form-label") }}{{ form.api_key(class="form-control", value="", placeholder="•••••• (leave blank to keep current)") }} + Write-only: the stored key is never displayed. Enter a new key only when rotating credentials. + {% for error in form.api_key.errors %}{{ error }}{% endfor %} +
    +
    {{ form.temperature.label(class="form-label") }}{{ form.temperature(class="form-control", step="0.1", min="0", max="2") }} + {% for error in form.temperature.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.test_llm(class="btn btn-outline-info btn-sm") }} + {{ form.submit(class="btn btn-primary btn-sm") }} +
    +
    +
    +
    +
    +
    +{% endblock %} diff --git a/data_forecaster/frontend/templates/admin/models.html b/data_forecaster/frontend/templates/admin/models.html new file mode 100644 index 0000000..c3fdd8d --- /dev/null +++ b/data_forecaster/frontend/templates/admin/models.html @@ -0,0 +1,52 @@ +{% extends "admin/base.html" %} +{% block title %}Models — Admin{% endblock %} +{% block header %}Forecasting Models{% endblock %} +{% block content %} +
    +
    +
    +
    +

    + Enable or disable the forecasting models available to the + pipeline. At least one model must remain enabled — the + backend rejects disabling the last enabled model. +

    + {% if models %} +
    + {{ form.hidden_tag() }} + {% for model in models %} +
    + + +
    + {% endfor %} +
    + {{ form.submit(class="btn btn-primary btn-sm") }} +
    +
    + {% else %} +
    No models could be loaded from the backend.
    + {% endif %} +
    +
    +
    +
    +{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/data_forecaster/frontend/templates/main/forecast_setup.html b/data_forecaster/frontend/templates/main/forecast_setup.html index ad74a88..0e943d4 100644 --- a/data_forecaster/frontend/templates/main/forecast_setup.html +++ b/data_forecaster/frontend/templates/main/forecast_setup.html @@ -60,7 +60,7 @@

    Review data readiness

    Configure the forecast

    Choose how far ahead to predict and optionally select a model.

    -
    +

    Advanced options

    diff --git a/data_forecaster/frontend/templates/main/report.html b/data_forecaster/frontend/templates/main/report.html index 09be65e..576ab47 100644 --- a/data_forecaster/frontend/templates/main/report.html +++ b/data_forecaster/frontend/templates/main/report.html @@ -4,7 +4,10 @@ {% block content %} {% if llm_fallback %}
    - ⚠️ The LLM was not available for report generation. A local template was used as a fallback. + ⚠️ Some report narrative used deterministic fallback templates because the LLM was unavailable or its response did not pass validation. + {% if er and er.metadata.llm_fallback_sections %} + Affected sections: {{ er.metadata.llm_fallback_sections | map('replace', '_', ' ') | join(', ') }}. + {% endif %}
    {% endif %} @@ -180,7 +183,7 @@
    Report Narrative
    (function() { const llmFallback = {{ llm_fallback | tojson }}; if (llmFallback) { - console.warn("LLM fallback occurred. A local template was used for report generation."); + console.warn("LLM narrative fallback occurred. Deterministic templates were used for one or more report sections."); } })(); diff --git a/data_forecaster/frontend/templates/main/started.html b/data_forecaster/frontend/templates/main/started.html index 300e4a5..0cdd532 100644 --- a/data_forecaster/frontend/templates/main/started.html +++ b/data_forecaster/frontend/templates/main/started.html @@ -20,7 +20,7 @@
    Multi-Agent Architecture
    • Data Validation Agent — Checks data quality, missing values, duplicates
    • Statistical Analysis Agent — ADF/KPSS tests, trend detection, STL decomposition
      • -
    • Model Selection Agent — Evaluates ARIMA, SARIMA, Holt-Winters, EWMA
      • +
    • Model Selection Agent — Evaluates ARIMA, SARIMA, Holt-Winters, EWMA, Prophet
    • Forecasting Agent — Generates predictions with model-based or estimated prediction intervals when available
    • Report Generation Agent — Creates detailed reports with insights
    diff --git a/data_forecaster/frontend/templates/setup/admin.html b/data_forecaster/frontend/templates/setup/admin.html new file mode 100644 index 0000000..e751bf1 --- /dev/null +++ b/data_forecaster/frontend/templates/setup/admin.html @@ -0,0 +1,28 @@ +{% extends "setup/base.html" %} +{% block title %}Create Admin — Setup{% endblock %} +{% block content %} +
    Step 5 — Create the first admin API user
    +

    + This creates the first administrator on the backend and enables API + authentication. The same credentials are stored (encrypted) locally so + this frontend can authenticate with the backend. Store the API key + somewhere safe — it cannot be retrieved later. +

    +
    + {{ form.hidden_tag() }} +
    + {{ form.username.label(class="form-label") }} + {{ form.username(class="form-control", placeholder="e.g. frontend") }} + {% for error in form.username.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.api_key.label(class="form-label") }} + {{ form.api_key(class="form-control font-monospace") }} + Auto-generated strong key — you may replace it with your own. + {% for error in form.api_key.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.submit(class="btn btn-primary") }} +
    +
    +{% endblock %} diff --git a/data_forecaster/frontend/templates/setup/auth.html b/data_forecaster/frontend/templates/setup/auth.html new file mode 100644 index 0000000..3b64cde --- /dev/null +++ b/data_forecaster/frontend/templates/setup/auth.html @@ -0,0 +1,22 @@ +{% extends "setup/base.html" %} +{% block title %}Enable API Auth — Setup{% endblock %} +{% block content %} +
    Step 3 — Enable API authentication
    +

    + The backend API is currently open. Enabling authentication requires all + clients to present an API key. A strong service-account key has been + generated for the frontend↔backend connection and will be used when + creating the first admin user in step 5. +

    +
    + {{ form.hidden_tag() }} +
    + {{ form.confirm(class="form-check-input") }} + {{ form.confirm.label(class="form-check-label") }} + {% for error in form.confirm.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.submit(class="btn btn-primary") }} +
    +
    +{% endblock %} diff --git a/data_forecaster/frontend/templates/setup/backend.html b/data_forecaster/frontend/templates/setup/backend.html new file mode 100644 index 0000000..c385884 --- /dev/null +++ b/data_forecaster/frontend/templates/setup/backend.html @@ -0,0 +1,22 @@ +{% extends "setup/base.html" %} +{% block title %}Backend Connection — Setup{% endblock %} +{% block content %} +
    Step 1 — Backend connection
    +

    Enter the URL of the FastAPI backend. The connection is tested before continuing.

    +
    + {{ form.hidden_tag() }} +
    + {{ form.base_url.label(class="form-label") }} + {{ form.base_url(class="form-control", placeholder="http://backend:8000") }} + {% for error in form.base_url.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.verify_ssl(class="form-check-input") }} + {{ form.verify_ssl.label(class="form-check-label") }} + Uncheck when the backend uses a self-signed certificate. +
    +
    + {{ form.submit(class="btn btn-primary") }} +
    +
    +{% endblock %} diff --git a/data_forecaster/frontend/templates/setup/base.html b/data_forecaster/frontend/templates/setup/base.html new file mode 100644 index 0000000..1cec024 --- /dev/null +++ b/data_forecaster/frontend/templates/setup/base.html @@ -0,0 +1,47 @@ + + + + + + + {% block title %}Setup — Time Series Forecaster{% endblock %} + + + + + + +{% block scripts %}{% endblock %} + + diff --git a/data_forecaster/frontend/templates/setup/done.html b/data_forecaster/frontend/templates/setup/done.html new file mode 100644 index 0000000..f0cd440 --- /dev/null +++ b/data_forecaster/frontend/templates/setup/done.html @@ -0,0 +1,17 @@ +{% extends "setup/base.html" %} +{% block title %}Setup Complete — Time Series Forecaster{% endblock %} +{% block content %} +
    Step 6 — Done
    +
    + Setup is complete. The backend connection, LLM provider, API + authentication, and model selection have all been configured. +
    +

    + You can now log in with your web application account and start + forecasting. LLM and model settings can be changed later from the + admin panel. +

    +
    + Go to Login +
    +{% endblock %} diff --git a/data_forecaster/frontend/templates/setup/llm.html b/data_forecaster/frontend/templates/setup/llm.html new file mode 100644 index 0000000..8ef0e65 --- /dev/null +++ b/data_forecaster/frontend/templates/setup/llm.html @@ -0,0 +1,73 @@ +{% extends "setup/base.html" %} +{% block title %}LLM Provider — Setup{% endblock %} +{% block content %} +
    Step 2 — LLM provider
    +

    Configure the language model provider. The API key is sent to the backend and is never stored by this frontend.

    +{% if llm_test_result %} + +{% endif %} +
    + {{ form.hidden_tag() }} +
    + {{ form.provider.label.text }} + {% for subfield in form.provider %} +
    + {{ subfield(class="form-check-input") }} + {{ subfield.label(class="form-check-label") }} +
    + {% endfor %} +
    +
    + {{ form.model.label(class="form-label") }} + {{ form.model(class="form-control", placeholder="e.g. gemini-1.5-flash or llama3") }} + {% for error in form.model.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.base_url.label(class="form-label") }} + {{ form.base_url(class="form-control", placeholder="http://host.docker.internal:11434") }} + Only required for Ollama providers. + {% for error in form.base_url.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.api_key.label(class="form-label") }} + {{ form.api_key(class="form-control", placeholder="Provider API key (if required)") }} + {% for error in form.api_key.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.temperature.label(class="form-label") }} + {{ form.temperature(class="form-control", step="0.1", min="0", max="2") }} + {% for error in form.temperature.errors %}{{ error }}{% endfor %} +
    +
    + {{ form.submit(class="btn btn-primary", value="Test, Save & Continue") }} +
    +
    +{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/data_forecaster/frontend/templates/setup/models.html b/data_forecaster/frontend/templates/setup/models.html new file mode 100644 index 0000000..709719b --- /dev/null +++ b/data_forecaster/frontend/templates/setup/models.html @@ -0,0 +1,40 @@ +{% extends "setup/base.html" %} +{% block title %}Models — Setup{% endblock %} +{% block content %} +
    Step 4 — Forecasting models
    +

    Choose which forecasting models are available to the pipeline. At least one model must remain enabled.

    +{% if models %} +
    + {{ form.hidden_tag() }} + {% for model in models %} +
    + + +
    + {% endfor %} +
    + {{ form.submit(class="btn btn-primary") }} +
    +
    +{% else %} +
    No models could be loaded from the backend. Verify the backend connection and reload this page.
    +{% endif %} +{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/data_forecaster/tests/test_llm_factory.py b/data_forecaster/tests/test_llm_factory.py new file mode 100644 index 0000000..67ba566 --- /dev/null +++ b/data_forecaster/tests/test_llm_factory.py @@ -0,0 +1,43 @@ +"""Tests for provider-specific LangChain client construction.""" + +from __future__ import annotations + +from typing import Any + +import pytest + +import core.llm_factory as llm_factory +from core.llm_config_store import LLMConfig + + +@pytest.mark.parametrize("provider", ["ollama_cloud", "ollama"]) +def test_ollama_clients_pass_bearer_token_via_client_kwargs( + monkeypatch: pytest.MonkeyPatch, provider: str +) -> None: + """ChatOllama only forwards custom HTTP headers from ``client_kwargs``.""" + captured: dict[str, Any] = {} + + class FakeChatOllama: + def __init__(self, **kwargs: Any) -> None: + captured.update(kwargs) + + monkeypatch.setattr(llm_factory, "ChatOllama", FakeChatOllama) + monkeypatch.setattr( + llm_factory, + "get_llm_config", + lambda: LLMConfig( + provider=provider, + model="test-model", + base_url="https://ollama.example", + api_key="test-token", + temperature=0.1, + version=1, + ), + ) + + llm_factory.get_llm() + + assert captured["client_kwargs"] == { + "headers": {"Authorization": "Bearer test-token"} + } + assert "headers" not in captured diff --git a/data_forecaster/tests/test_model_selection_agent.py b/data_forecaster/tests/test_model_selection_agent.py index 4fec5cc..d0cd36d 100644 --- a/data_forecaster/tests/test_model_selection_agent.py +++ b/data_forecaster/tests/test_model_selection_agent.py @@ -378,3 +378,222 @@ def test_heuristic_fallback_explains_rejected_models( assert "plain ARIMA ignores seasonality" in result.arima_rejected_reason assert result.ewma_rejected_reason is not None assert "does not explicitly model seasonality" in result.ewma_rejected_reason + + +# ── Prophet awareness tests ─────────────────────────────────────────────────── + + +class TestProphetAwareness: + """Tests asserting the model-selection agent is aware of Prophet.""" + + def test_parses_plain_text_prophet( + self, + seasonal_stat_result: StatisticalResult, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Plain text 'Selected model: Prophet' should parse to Prophet.""" + response = SimpleNamespace( + content=( + "Selected model: Prophet\n\n" + "## Why this model was chosen\n" + "Prophet models trend and seasonality natively." + ), + usage_metadata={ + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + }, + ) + _patch_llm(monkeypatch, response) + + result = run_model_selection_agent(seasonal_stat_result) + assert result.selected_model == "Prophet" + + def test_parses_meta_prophet_variant( + self, + seasonal_stat_result: StatisticalResult, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """'Meta-Prophet' (unicode hyphen) normalizes to the Prophet model.""" + response = SimpleNamespace( + content=( + "**Selected model:** Meta‑Prophet\n\n" + "## Why this model was chosen\n" + "Meta-Prophet handles the seasonal cycle and changepoints." + ), + usage_metadata={ + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + }, + ) + _patch_llm(monkeypatch, response) + + result = run_model_selection_agent(seasonal_stat_result) + assert result.selected_model == "Prophet" + + def test_prophet_in_models_registry(self) -> None: + """Prophet is a candidate model in the canonical registry.""" + from forecasting import registry + + assert "Prophet" in registry.MODEL_NAMES + assert "fit_fn" in registry.MODELS["Prophet"] + + def test_deterministic_override_selects_prophet_on_best_metrics( + self, + seasonal_stat_result: StatisticalResult, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Prophet with the lowest MASE is selected by the deterministic override.""" + response = SimpleNamespace( + content="Selected model: Holt-Winters\n", + usage_metadata={ + "input_tokens": 100, + "output_tokens": 50, + "total_tokens": 150, + }, + ) + _patch_llm(monkeypatch, response) + + all_metrics = { + "Holt-Winters": {"RMSE": 0.12, "MAE": 0.10, "MAPE": 0.9, "MASE": 1.2}, + "ARIMA": {"RMSE": 0.10, "MAE": 0.08, "MAPE": 0.7, "MASE": 1.0}, + "SARIMA": {"RMSE": 0.09, "MAE": 0.07, "MAPE": 0.7, "MASE": 0.8}, + "Prophet": {"RMSE": 0.07, "MAE": 0.05, "MAPE": 0.5, "MASE": 0.5}, + } + result = run_model_selection_agent( + seasonal_stat_result, + review_feedback="Previous selection was suboptimal.", + exclude_model="Holt-Winters", + all_metrics=all_metrics, + ) + # Prophet has the lowest MASE and is not excluded + assert result.selected_model == "Prophet" + assert result.prophet_rejected_reason is None + # Other models get a rejection reason + assert result.arima_rejected_reason is not None + assert "Higher forecast error" in result.arima_rejected_reason + + def test_heuristic_fallback_provides_prophet_rejection_reason( + self, + seasonal_stat_result: StatisticalResult, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Heuristic fallback on a seasonal series rejects Prophet with a reason.""" + monkeypatch.setattr( + "agents.model_selection_agent.get_llm", + lambda temperature=0: SimpleNamespace(), + ) + monkeypatch.setattr( + "agents.model_selection_agent.MODEL_SELECTION_PROMPT", + _FailingPrompt(), + ) + + result = run_model_selection_agent(seasonal_stat_result) + + # SARIMA is the heuristic pick for a seasonal series; Prophet is not. + assert result.selected_model == "SARIMA" + assert result.prophet_rejected_reason is not None + assert "lighter to fit than Prophet" in result.prophet_rejected_reason + + def test_suitability_summary_includes_prophet_assessment( + self, + seasonal_stat_result: StatisticalResult, + ) -> None: + """The suitability summary built for the LLM includes a Prophet section.""" + from agents.model_selection_agent import _build_suitability_summary + + summary = _build_suitability_summary(seasonal_stat_result) + assert "Prophet Assessment:" in summary + assert "Prophet models seasonality" in summary + + +# ── Disabled-model (registry) Tests ────────────────────────────────────────── + + +class TestDisabledModels: + """Disabled models must be invisible to selection and heuristics.""" + + def test_suitability_summary_omits_disabled( + self, + seasonal_stat_result: StatisticalResult, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Disabled models do not appear in the LLM suitability summary.""" + monkeypatch.setattr( + "agents.model_selection_agent.get_enabled_models", + lambda: ("ARIMA", "EWMA"), + ) + from agents.model_selection_agent import _build_suitability_summary + + summary = _build_suitability_summary(seasonal_stat_result) + + assert "ARIMA Assessment:" in summary + assert "EWMA Assessment:" in summary + assert "Prophet Assessment:" not in summary + assert "SARIMA Assessment:" not in summary + + def test_heuristic_never_selects_disabled( + self, + seasonal_stat_result: StatisticalResult, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """LLM failure falls back to an enabled model, never a disabled one.""" + monkeypatch.setattr( + "agents.model_selection_agent.get_enabled_models", + lambda: ("Holt-Winters", "EWMA"), + ) + monkeypatch.setattr( + "agents.model_selection_agent.get_llm", + lambda temperature=0: SimpleNamespace(), + ) + monkeypatch.setattr( + "agents.model_selection_agent.MODEL_SELECTION_PROMPT", + _FailingPrompt(), + ) + + result = run_model_selection_agent(seasonal_stat_result) + + assert result.selected_model in ("Holt-Winters", "EWMA") + + def test_single_enabled_model_is_selected( + self, + seasonal_stat_result: StatisticalResult, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """With one enabled model, heuristics return it without error.""" + monkeypatch.setattr( + "agents.model_selection_agent.get_enabled_models", + lambda: ("EWMA",), + ) + monkeypatch.setattr( + "agents.model_selection_agent.get_llm", + lambda temperature=0: SimpleNamespace(), + ) + monkeypatch.setattr( + "agents.model_selection_agent.MODEL_SELECTION_PROMPT", + _FailingPrompt(), + ) + + result = run_model_selection_agent(seasonal_stat_result) + + assert result.selected_model == "EWMA" + + def test_llm_selecting_disabled_model_falls_back( + self, + seasonal_stat_result: StatisticalResult, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """An LLM naming a disabled model is overridden by the fallback.""" + monkeypatch.setattr( + "agents.model_selection_agent.get_enabled_models", + lambda: ("ARIMA", "EWMA"), + ) + _patch_llm( + monkeypatch, + SimpleNamespace(content="Selected model: Prophet\n\nBecause reasons."), + ) + + result = run_model_selection_agent(seasonal_stat_result) + + assert result.selected_model in ("ARIMA", "EWMA") diff --git a/data_forecaster/tests/test_report_renderers.py b/data_forecaster/tests/test_report_renderers.py index 6d47af0..e3684a6 100644 --- a/data_forecaster/tests/test_report_renderers.py +++ b/data_forecaster/tests/test_report_renderers.py @@ -9,6 +9,7 @@ from jinja2 import Environment, FileSystemLoader from forecasting.contracts import ForecastFitStatus +import report.narrative as narrative from report.builder import ExecutiveReportBuilder from report.renderers import HTMLRenderer, MarkdownRenderer from schemas import ( @@ -198,6 +199,22 @@ def test_metadata_table(self, sample_report: "object") -> None: assert "Generated At" in md assert "Forecast Horizon" in md + def test_metadata_identifies_deterministic_narrative_fallback( + self, sample_report: "object" + ) -> None: + report = sample_report.model_copy(deep=True) + report.metadata.llm_narrative_fallback = True + report.metadata.llm_fallback_sections = [ + "historical_analysis", + "forecast_outlook", + ] + + md = MarkdownRenderer().render(report) + + assert "Narrative Generation" in md + assert "Deterministic fallback used" in md + assert "historical analysis, forecast outlook" in md + def test_no_fabricated_financials(self, sample_report: "object") -> None: """Fallback narratives should not contain fabricated financials.""" renderer = MarkdownRenderer() @@ -213,6 +230,44 @@ def test_no_fabricated_financials(self, sample_report: "object") -> None: ), "Rendered markdown contains fabricated financial figures." +def test_narrative_generation_records_section_fallbacks( + sample_report: "object", monkeypatch: pytest.MonkeyPatch +) -> None: + """Section-level fallbacks must propagate to the report-level status.""" + monkeypatch.setattr(narrative, "get_llm", lambda **_kwargs: object()) + monkeypatch.setattr( + narrative, "get_llm_config", lambda: SimpleNamespace(temperature=0.0) + ) + + def always_fallback( + _llm: object, + _prompt: object, + _section: object, + section_name: str, + _total_usage: dict[str, int], + _extra_instructions: str, + fallback_sections: list[str], + ) -> str: + fallback_sections.append(section_name) + return f"Fallback for {section_name}" + + monkeypatch.setattr(narrative, "_generate_section", always_fallback) + + report, _ = narrative.generate_narratives(sample_report) + + assert report.metadata.llm_narrative_fallback is True + assert report.metadata.llm_fallback_sections == [ + "executive_summary", + "data_quality", + "historical_analysis", + "forecast_outlook", + "model_comparison", + "statistical_audit", + "explainability", + *["recommendation"] * len(report.recommendations), + ] + + # ── HTML Renderer Tests ────────────────────────────────────────────────────── @@ -370,3 +425,15 @@ def test_metadata_table(self, sample_report: "object") -> None: html = renderer.render(sample_report) assert "Report Metadata" in html assert "Engine Version" in html + + def test_metadata_identifies_deterministic_narrative_fallback( + self, sample_report: "object" + ) -> None: + report = sample_report.model_copy(deep=True) + report.metadata.llm_narrative_fallback = True + report.metadata.llm_fallback_sections = ["forecast_outlook"] + + html = HTMLRenderer().render(report) + + assert "Narrative Generation" in html + assert "Deterministic fallback used (forecast outlook)" in html diff --git a/docs/api-auth.md b/docs/api-auth.md index 79da8bb..4373f9a 100644 --- a/docs/api-auth.md +++ b/docs/api-auth.md @@ -15,7 +15,6 @@ On success, the backend updates `last_used` and `last_used_ip` on the user recor ## First setup -The frontend does not read backend API credentials from its `.env` file. After first login, configure the active backend connection in **Admin → API Config**. @@ -60,7 +59,7 @@ The admin panel lets you update these credentials under **Admin → API Config** ## Disabling auth entirely (dev only) -Set `API_KEY_ENABLED=false` in the backend `.env` to make the auth dependency a no-op. All protected endpoints become open. This is useful for local development or testing — **never do this in production**. +Set `API_KEY_ENABLED=false` in the backend process environment to make the auth dependency a no-op. All protected endpoints become open. This is useful for local development or testing — **never do this in production**. ## If a key is compromised diff --git a/docs/api-reference.md b/docs/api-reference.md index e40d13c..ca73e9e 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -118,7 +118,29 @@ These endpoints also require auth. They let you manage API users from the admin | `POST` | `/api-users/{id}/rotate` | Rotate a user's key (returns new plaintext key once) | | `POST` | `/api-users/{id}/toggle` | Enable or disable a user | | `DELETE` | `/api-users/{id}` | Delete a user | -| `POST` | `/api-users/bootstrap` | One-time bootstrap endpoint (guarded by `X-Admin-Key`) | + +## Setup wizard + +First-run provisioning. Unauthenticated; the bootstrap endpoint is guarded by an atomic "no users exist yet" check (no preset admin token). + +| Method | Path | Description | +|---|---|---| +| `GET` | `/setup/status` | Setup state flags (`setup_complete`, `admin_exists`, `llm_configured`, `models_enabled`) — never secrets | +| `POST` | `/setup/bootstrap` | One-time atomic bootstrap: creates the first admin API user, generates the backend encryption key, enables auth. `409` once any user exists | + +## LLM configuration (admin) + +| Method | Path | Description | +|---|---|---| +| `GET` | `/config/llm` | Masked LLM config (`provider`, `model`, `base_url`, `temperature`, `api_key_set`) — the key is never returned | +| `PUT` | `/config/llm` | Update LLM config; `api_key` is write-only (omit to keep the stored key) | + +## Model registry (admin) + +| Method | Path | Description | +|---|---|---| +| `GET` | `/models` | List the five forecasting models with enabled state | +| `PUT` | `/models/{name}` | Enable/disable a model; `400` when disabling the last enabled model | ## Error responses @@ -132,7 +154,7 @@ All errors return JSON with a `detail` field: |---|---| | `400` | Bad file, unsupported extension, file too large, empty file, bad preflight options | | `401` | Missing or invalid API key headers, or disabled account | -| `403` | Missing or invalid `X-Admin-Key` on the bootstrap endpoint | +| `409` | `/setup/bootstrap` called after setup completed | | `404` | Unknown `file_id` or `job_id` | | `409` | Duplicate username, or bootstrap attempted when users already exist | | `422` | Pydantic validation failure (e.g. chat query too long) | diff --git a/docs/deployment.md b/docs/deployment.md index 7ba0083..2bd8d51 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -90,7 +90,7 @@ Drop your `server.crt` and `server.key` into the `certs/frontend/` or `certs/bac ### Changing the domain -Set `SSL_DOMAIN` in your shell or Compose project `.env` to match your hostname: +Set `SSL_DOMAIN` in your shell when starting Compose to match your hostname: ```bash SSL_DOMAIN=forecaster.example.com @@ -115,10 +115,7 @@ when the backend uses a CA-signed certificate. | `FRONTEND_HTTPS_PORT` | nginx-frontend | `443` | Frontend HTTPS port | | `BACKEND_HTTPS_PORT` | nginx-backend | `8443` | Backend HTTPS port | | `CORS_ALLOWED_ORIGINS` | backend | `http://localhost:5000,...` | Comma-separated allowed origins | -| `FRONTEND_API_USERNAME` | backend | `frontend` | Optional backend bootstrap service-account username | -| `FRONTEND_API_KEY` | backend | `frontend` | Optional backend bootstrap service-account key | -| `SECRET_KEY` | frontend | (random) | Flask session secret | -| `FLASK_ENCRYPTION_KEY` | frontend | (random) | Fernet key for encrypting stored API credentials | +| `FLASK_ENV` | frontend | `production` in Compose | Flask runtime mode | The frontend stores the active backend URL, SSL verification setting, username, and encrypted key in its SQLite database via **Admin → API Config**. @@ -153,7 +150,7 @@ username, and encrypted key in its SQLite database via **Admin → API Config**. **Backend healthcheck fails on startup** — The backend takes ~25 seconds to initialize (ChromaDB + LLM setup). The healthcheck has a 40-second start period, but if your machine is slow, increase `start_period` in `docker-compose.yml`. -**Stale database after changing backend bootstrap credentials** — The backend DB lives in a Docker volume and a bind mount. If you change `FRONTEND_API_KEY` in `backend/.env` and the old user still exists, the backend won't recreate it. Rotate or create the backend API user from **Admin → API Keys**, then update **Admin → API Config**. For a full reset: +**Stale database after changing backend credentials** — Rotate or create the backend API user from **Admin → API Keys**, then update **Admin → API Config**. For a full reset: ```bash docker compose down -v diff --git a/docs/local-dev.md b/docs/local-dev.md index 3824869..c96aa35 100644 --- a/docs/local-dev.md +++ b/docs/local-dev.md @@ -20,9 +20,6 @@ pip install -r requirements.txt # Or with uv (faster): # uv pip install -r uv.txt -# Set up environment variables -cp .env.example .env - # Start the backend uvicorn main:app --reload --port 8000 ``` @@ -38,11 +35,6 @@ python -m venv .venv source .venv/bin/activate pip install -r requirements.txt -# Set up environment variables -cp .env.example .env -# Then set FLASK_ENCRYPTION_KEY in frontend/.env: -# python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" - # Initialize the database flask --app app init-db @@ -89,6 +81,6 @@ export OLLAMA_MODEL=llama3 - The backend uses `--reload` which watches for file changes and auto-restarts. Great for iterating on agents or API endpoints. - The frontend in development mode has Flask debug enabled — you get the interactive debugger in the browser on errors. -- The `FLASK_ENCRYPTION_KEY` is used to encrypt stored API credentials at rest. Generate one with `python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"`. -- Backend API credentials are entered in the frontend under Admin -> API Config, not stored in `frontend/.env`. +- The frontend generates its session and encryption keys on first startup and stores them under `frontend/instance/` (mode 0600). Back up that directory with the frontend database. +- Backend API credentials are entered in the frontend under Admin -> API Config, not stored in an environment file. - ChromaDB persists to `./chroma_db` by default. Delete that directory if you want a clean RAG knowledge base. diff --git a/tests/test_api_config_helpers.py b/tests/test_api_config_helpers.py index 45041a0..30207dd 100644 --- a/tests/test_api_config_helpers.py +++ b/tests/test_api_config_helpers.py @@ -6,7 +6,6 @@ import sys from typing import Any -from cryptography.fernet import Fernet from flask import Flask REPO_ROOT = Path(__file__).resolve().parents[1] @@ -44,7 +43,6 @@ def test_load_current_api_config_never_decrypts_key( monkeypatch: Any, ) -> None: """The summary may decrypt the username but must not decrypt the API key.""" - monkeypatch.setenv("FLASK_ENCRYPTION_KEY", Fernet.generate_key().decode()) app = _app(tmp_path) with app.app_context(): @@ -79,7 +77,6 @@ def test_save_api_credentials_preserves_existing_key( monkeypatch: Any, ) -> None: """Changing username with a blank key should keep the encrypted key.""" - monkeypatch.setenv("FLASK_ENCRYPTION_KEY", Fernet.generate_key().decode()) app = _app(tmp_path) with app.app_context(): @@ -126,7 +123,6 @@ def test_client_from_api_config_form_uses_stored_key_server_side( monkeypatch: Any, ) -> None: """Testing edited config can preserve the blank key field.""" - monkeypatch.setenv("FLASK_ENCRYPTION_KEY", Fernet.generate_key().decode()) app = _app(tmp_path) with app.app_context(): diff --git a/tests/test_api_key_roles.py b/tests/test_api_key_roles.py index 4a2bd93..d2ab522 100644 --- a/tests/test_api_key_roles.py +++ b/tests/test_api_key_roles.py @@ -19,7 +19,7 @@ from core.database import init_database from main import app -# Test API key reused from the ADMIN_API_KEY env var set in _reset_api_key_db. +# Plaintext admin API key used by the fixtures (hashed via Argon2id in DB). _ADMIN_KEY = "test-admin-key" @@ -34,13 +34,12 @@ def _reset_api_key_db(tmp_path: Any, monkeypatch: Any) -> None: db_path = str(tmp_path / "backend.db") monkeypatch.setenv("BACKEND_DB_PATH", db_path) monkeypatch.setenv("API_KEY_ENABLED", "true") - monkeypatch.setenv("ADMIN_API_KEY", _ADMIN_KEY) monkeypatch.setenv("CHROMA_PERSIST_DIR", str(tmp_path / "chroma")) monkeypatch.setenv("FILE_STORAGE_DIR", str(tmp_path / "files")) # Patch the cached module attributes (read at import time). monkeypatch.setattr(settings, "BACKEND_DB_PATH", db_path) monkeypatch.setattr(settings, "API_KEY_ENABLED", True) - monkeypatch.setattr(settings, "ADMIN_API_KEY", _ADMIN_KEY) + monkeypatch.setattr(settings, "SECRETS_DIR", str(tmp_path / "secrets")) init_database() @@ -194,27 +193,53 @@ def test_admin_can_update_forecast_job_settings( class TestBootstrap: - """Tests for the bootstrap endpoint.""" + """Tests for the setup bootstrap endpoint (replaces /api-users/bootstrap).""" - def test_bootstrap_creates_admin_user(self, client: TestClient) -> None: - """The bootstrap endpoint creates the first user as an admin.""" + def test_old_bootstrap_endpoint_removed(self, client: TestClient) -> None: + """The retired endpoint responds with an unambiguous tombstone.""" response = client.post( "/api-users/bootstrap", headers={"X-Admin-Key": "test-admin-key"}, json={"username": "bootstrap-admin", "api_key": "secret"}, ) + assert response.status_code == 410 + + def test_setup_bootstrap_creates_admin_user(self, client: TestClient) -> None: + """/setup/bootstrap creates the first user as an admin (no token).""" + response = client.post( + "/setup/bootstrap", + json={"username": "setup-admin", "api_key": "secret"}, + ) assert response.status_code == 200 data = response.json() assert data["user"]["is_admin"] is True - assert data["auth_enabled"] is True + assert data["setup_complete"] is True - def test_bootstrap_requires_admin_key(self, client: TestClient) -> None: - """The bootstrap endpoint requires the deployment admin key.""" - response = client.post( - "/api-users/bootstrap", - json={"username": "bootstrap-admin", "api_key": "secret"}, + def test_setup_bootstrap_conflict_after_first_user( + self, client: TestClient + ) -> None: + """A second /setup/bootstrap call returns 409.""" + first = client.post( + "/setup/bootstrap", + json={"username": "setup-admin", "api_key": "secret"}, ) - assert response.status_code == 403 + assert first.status_code == 200 + + second = client.post( + "/setup/bootstrap", + json={"username": "other", "api_key": "secret"}, + ) + assert second.status_code == 409 + + def test_setup_status_reports_flags(self, client: TestClient) -> None: + """/setup/status reports booleans only, no secret material.""" + response = client.get("/setup/status") + assert response.status_code == 200 + data = response.json() + assert data["setup_complete"] is False + assert data["admin_exists"] is False + assert data["models_enabled"] == 5 + assert "key" not in str(data).lower().replace("api_key_set", "") class TestSetUserAdmin: diff --git a/tests/test_frontend_db_init.py b/tests/test_frontend_db_init.py index cea5283..8cf749d 100644 --- a/tests/test_frontend_db_init.py +++ b/tests/test_frontend_db_init.py @@ -5,7 +5,6 @@ from pathlib import Path import sys -from cryptography.fernet import Fernet from flask import Flask from werkzeug.security import check_password_hash @@ -39,12 +38,8 @@ def _app(tmp_path: Path) -> Flask: return app -def test_init_db_seeds_forced_reset_admin_and_blank_api_config( - tmp_path: Path, - monkeypatch, -) -> None: +def test_init_db_seeds_forced_reset_admin_and_blank_api_config(tmp_path: Path) -> None: """First boot uses DB-owned admin/API config setup.""" - monkeypatch.setenv("FLASK_ENCRYPTION_KEY", Fernet.generate_key().decode()) app = _app(tmp_path) with app.app_context(): @@ -80,9 +75,8 @@ def test_init_db_seeds_forced_reset_admin_and_blank_api_config( assert upload_config["value"] == "100" -def test_sync_app_config_applies_upload_limit(tmp_path: Path, monkeypatch) -> None: +def test_sync_app_config_applies_upload_limit(tmp_path: Path) -> None: """The DB-owned upload limit should drive Flask MAX_CONTENT_LENGTH.""" - monkeypatch.setenv("FLASK_ENCRYPTION_KEY", Fernet.generate_key().decode()) app = _app(tmp_path) with app.app_context(): @@ -102,12 +96,8 @@ def test_sync_app_config_applies_upload_limit(tmp_path: Path, monkeypatch) -> No assert app.config["MAX_CONTENT_LENGTH"] == 42 * 1024 * 1024 -def test_init_db_preserves_existing_api_credentials( - tmp_path: Path, - monkeypatch, -) -> None: +def test_init_db_preserves_existing_api_credentials(tmp_path: Path) -> None: """Repeated initialization should not overwrite API Config.""" - monkeypatch.setenv("FLASK_ENCRYPTION_KEY", Fernet.generate_key().decode()) app = _app(tmp_path) with app.app_context(): diff --git a/tests/test_frontend_production_config.py b/tests/test_frontend_production_config.py index 26d047a..1faad69 100644 --- a/tests/test_frontend_production_config.py +++ b/tests/test_frontend_production_config.py @@ -2,36 +2,13 @@ from __future__ import annotations -import importlib +from config import get_config -import pytest +def test_production_config_uses_persistent_generated_secret() -> None: + """Production startup uses a stable non-empty instance signing secret.""" + first = get_config("production").SECRET_KEY + second = get_config("production").SECRET_KEY -def _reload_config(monkeypatch: pytest.MonkeyPatch, secret_key: str | None): - """Reload frontend config after changing SECRET_KEY.""" - if secret_key is None: - monkeypatch.delenv("SECRET_KEY", raising=False) - else: - monkeypatch.setenv("SECRET_KEY", secret_key) - - import config # pylint: disable=import-outside-toplevel - - return importlib.reload(config) - - -def test_production_config_requires_secret_key(monkeypatch: pytest.MonkeyPatch) -> None: - """Production startup must fail when no signing secret is configured.""" - config = _reload_config(monkeypatch, None) - monkeypatch.setattr(config.ProductionConfig, "SECRET_KEY", "") - - with pytest.raises(RuntimeError, match="SECRET_KEY"): - config.get_config("production") - - -def test_production_config_uses_configured_secret_key( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Production config accepts a non-empty externally supplied secret.""" - config = _reload_config(monkeypatch, "test-production-secret") - - assert config.get_config("production").SECRET_KEY == "test-production-secret" + assert first + assert first == second diff --git a/tests/test_llm_config_api.py b/tests/test_llm_config_api.py new file mode 100644 index 0000000..fd41e50 --- /dev/null +++ b/tests/test_llm_config_api.py @@ -0,0 +1,166 @@ +"""Tests for the admin LLM config endpoints (masked reads, one-way writes).""" + +from __future__ import annotations + +from typing import Any +from unittest.mock import AsyncMock + +import pytest +from fastapi.testclient import TestClient + +import core.config as settings +from core import secret_store +from core.database import init_database +import main +from main import app +from services.llm_validation_service import LLMValidationResult + + +@pytest.fixture(autouse=True) +def _isolated_backend(tmp_path: Any, monkeypatch: Any) -> None: + """Fresh DB + secrets dir per test; auth disabled for simplicity.""" + db_path = str(tmp_path / "backend.db") + monkeypatch.setattr(settings, "BACKEND_DB_PATH", db_path) + monkeypatch.setattr(settings, "API_KEY_ENABLED", False) + monkeypatch.setattr(settings, "SECRETS_DIR", str(tmp_path / "secrets")) + secret_store.reset_cache() + init_database() + yield + secret_store.reset_cache() + + +@pytest.fixture +def client() -> TestClient: + """Return a FastAPI test client.""" + return TestClient(app) + + +class TestLLMConfigRead: + """GET /config/llm never exposes the key.""" + + def test_masked_read_after_write(self, client: TestClient) -> None: + put = client.put( + "/config/llm", + json={ + "provider": "gemini", + "model": "gemini-2.0", + "api_key": "super-secret-plaintext", + "temperature": 0.2, + }, + ) + assert put.status_code == 200 + + response = client.get("/config/llm") + + assert response.status_code == 200 + data = response.json() + assert data == { + "provider": "gemini", + "model": "gemini-2.0", + "base_url": None, + "temperature": pytest.approx(0.2), + "api_key_set": True, + "configured": True, + } + assert "super-secret-plaintext" not in response.text + + def test_unconfigured_reports_false(self, client: TestClient) -> None: + data = client.get("/config/llm").json() + assert data["configured"] is False + + +class TestLLMConfigWrite: + """PUT /config/llm one-way write semantics.""" + + def test_unknown_provider_rejected(self, client: TestClient) -> None: + response = client.put( + "/config/llm", + json={"provider": "bogus", "model": "m"}, + ) + assert response.status_code == 400 + + def test_omitting_key_preserves_stored(self, client: TestClient) -> None: + client.put( + "/config/llm", + json={"provider": "gemini", "model": "m1", "api_key": "keep-me"}, + ) + response = client.put( + "/config/llm", + json={"provider": "gemini", "model": "m2"}, + ) + + assert response.status_code == 200 + data = response.json() + assert data["model"] == "m2" + assert data["api_key_set"] is True + + def test_secret_str_repr_does_not_leak(self) -> None: + from schemas import LLMConfigUpdateRequest + + request = LLMConfigUpdateRequest( + provider="gemini", model="m", api_key="top-secret" + ) + assert "top-secret" not in repr(request) + assert "top-secret" not in str(request) + + +class TestLLMConfigValidation: + """POST /config/llm/test validates candidates without saving them.""" + + def test_candidate_is_tested_without_being_persisted( + self, client: TestClient, monkeypatch: Any + ) -> None: + validator = AsyncMock( + return_value=LLMValidationResult( + ok=True, + url_reachable=True, + credentials_valid=True, + llm_responded=True, + message="LLM connection test passed.", + response="pong", + ) + ) + monkeypatch.setattr(main, "validate_llm_configuration", validator) + + response = client.post( + "/config/llm/test", + json={ + "provider": "ollama_cloud", + "model": "test-model", + "base_url": "https://ollama.example", + "api_key": "candidate-secret", + }, + ) + + assert response.status_code == 200 + assert response.json()["response"] == "pong" + validator.assert_awaited_once_with( + provider="ollama_cloud", + model="test-model", + base_url="https://ollama.example", + api_key="candidate-secret", + ) + assert client.get("/config/llm").json()["configured"] is False + + def test_omitted_key_uses_stored_secret( + self, client: TestClient, monkeypatch: Any + ) -> None: + client.put( + "/config/llm", + json={ + "provider": "gemini", + "model": "stored-model", + "api_key": "stored-secret", + }, + ) + validator = AsyncMock( + return_value=LLMValidationResult(message="failed") + ) + monkeypatch.setattr(main, "validate_llm_configuration", validator) + + client.post( + "/config/llm/test", + json={"provider": "gemini", "model": "candidate-model"}, + ) + + assert validator.await_args.kwargs["api_key"] == "stored-secret" diff --git a/tests/test_llm_config_store.py b/tests/test_llm_config_store.py new file mode 100644 index 0000000..796319d --- /dev/null +++ b/tests/test_llm_config_store.py @@ -0,0 +1,150 @@ +"""Unit tests for the DB-backed LLM configuration store.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_BACKEND = str(Path(__file__).resolve().parent.parent / "data_forecaster" / "backend") +if _BACKEND not in sys.path: + sys.path.insert(0, _BACKEND) + +from core import llm_config_store, secret_store # noqa: E402 +from core import config as settings # noqa: E402 +from core.database import init_database # noqa: E402 + + +@pytest.fixture +def db(tmp_path, monkeypatch): + """Temp backend DB + secrets dir; patch the module's own references.""" + secrets = tmp_path / "secrets" + monkeypatch.setattr(secret_store.settings, "SECRETS_DIR", str(secrets)) + secret_store.reset_cache() + secret_store.generate_and_persist_key() + llm_config_store.reset_cache() + + db_path = str(tmp_path / "backend.db") + monkeypatch.setattr(settings, "BACKEND_DB_PATH", db_path) + init_database() + yield db_path + llm_config_store.reset_cache() + + +class TestEnvFallback: + """Pre-setup behaviour when no llm_config row exists.""" + + def test_gemini_fallback(self, db, monkeypatch): + monkeypatch.setattr(settings, "USE_OLLAMA", False) + monkeypatch.setattr(settings, "GOOGLE_API_KEY", "env-key") + monkeypatch.setattr(settings, "GEMINI_MODEL", "gemini-test") + + config = llm_config_store.get_llm_config(db) + + assert config.provider == "gemini" + assert config.model == "gemini-test" + assert config.api_key == "env-key" + assert config.version == 0 + + def test_ollama_fallback(self, db, monkeypatch): + monkeypatch.setattr(settings, "USE_OLLAMA", True) + monkeypatch.setattr(settings, "USE_OLLAMA_CLOUD", False) + monkeypatch.setattr(settings, "OLLAMA_MODEL", "llama3") + monkeypatch.setattr(settings, "OLLAMA_BASE_URL", "http://ollama:11434") + + config = llm_config_store.get_llm_config(db) + + assert config.provider == "ollama" + assert config.base_url == "http://ollama:11434" + + +class TestDbBackedConfig: + """DB row is authoritative once present.""" + + def test_put_then_get_round_trip(self, db): + llm_config_store.put_llm_config( + provider="ollama_cloud", + model="gpt-oss", + base_url="https://ollama.com", + api_key="secret-key-123", + temperature=0.2, + db_path=db, + ) + + config = llm_config_store.get_llm_config(db) + + assert config.provider == "ollama_cloud" + assert config.api_key == "secret-key-123" + assert config.temperature == pytest.approx(0.2) + assert config.version == 1 + + def test_db_overrides_env(self, db, monkeypatch): + monkeypatch.setattr(settings, "USE_OLLAMA", False) + monkeypatch.setattr(settings, "GOOGLE_API_KEY", "env-key") + llm_config_store.put_llm_config( + provider="ollama", + model="llama3", + base_url="http://ollama:11434", + api_key=None, + temperature=0.1, + db_path=db, + ) + + config = llm_config_store.get_llm_config(db) + + assert config.provider == "ollama" + + def test_write_invalidates_cache_without_restart(self, db): + llm_config_store.put_llm_config( + "gemini", "gemini-1.5-flash", None, "key-a", 0.1, db_path=db + ) + assert llm_config_store.get_llm_config(db).api_key == "key-a" + + llm_config_store.put_llm_config( + "gemini", "gemini-2.0", None, "key-b", 0.1, db_path=db + ) + + config = llm_config_store.get_llm_config(db) + assert config.model == "gemini-2.0" + assert config.api_key == "key-b" + assert config.version == 2 + + def test_key_is_encrypted_at_rest(self, db): + import sqlite3 + + llm_config_store.put_llm_config( + "gemini", "gemini-1.5-flash", None, "plaintext-key", 0.1, db_path=db + ) + + with sqlite3.connect(db) as connection: + row = connection.execute( + "SELECT encrypted_api_key FROM llm_config WHERE singleton = 1" + ).fetchone() + assert row[0] != "plaintext-key" + assert "plaintext-key" not in row[0] + + def test_put_without_key_preserves_existing(self, db): + llm_config_store.put_llm_config( + "gemini", "gemini-1.5-flash", None, "keep-me", 0.1, db_path=db + ) + llm_config_store.put_llm_config( + "gemini", "gemini-2.0", None, None, 0.3, db_path=db + ) + + config = llm_config_store.get_llm_config(db) + assert config.model == "gemini-2.0" + assert config.api_key == "keep-me" + + def test_unknown_provider_rejected(self, db): + with pytest.raises(ValueError, match="Unknown LLM provider"): + llm_config_store.put_llm_config( + "bogus", "m", None, None, 0.1, db_path=db + ) + + def test_is_configured(self, db): + assert llm_config_store.is_configured(db) is False + llm_config_store.put_llm_config( + "gemini", "gemini-1.5-flash", None, "k", 0.1, db_path=db + ) + assert llm_config_store.is_configured(db) is True diff --git a/tests/test_llm_validation_service.py b/tests/test_llm_validation_service.py new file mode 100644 index 0000000..49f1d39 --- /dev/null +++ b/tests/test_llm_validation_service.py @@ -0,0 +1,165 @@ +"""Tests for staged candidate LLM validation.""" + +from __future__ import annotations + +from typing import Any + +import httpx +import pytest + +from services import llm_validation_service as service + + +class _Response: + def __init__(self, status_code: int, payload: dict[str, Any]) -> None: + self.status_code = status_code + self.is_success = 200 <= status_code < 300 + self._payload = payload + + def json(self) -> dict[str, Any]: + return self._payload + + +class _FakeClient: + def __init__(self, responses: list[_Response | httpx.RequestError]) -> None: + self.responses = responses + self.calls: list[tuple[str, str, dict[str, Any]]] = [] + + async def __aenter__(self) -> _FakeClient: + return self + + async def __aexit__(self, *args: Any) -> None: + return None + + async def _request(self, method: str, url: str, **kwargs: Any) -> _Response: + self.calls.append((method, url, kwargs)) + response = self.responses.pop(0) + if isinstance(response, httpx.RequestError): + raise response + return response + + async def get(self, url: str, **kwargs: Any) -> _Response: + return await self._request("GET", url, **kwargs) + + async def post(self, url: str, **kwargs: Any) -> _Response: + return await self._request("POST", url, **kwargs) + + +def _install_client(monkeypatch: Any, client: _FakeClient) -> None: + monkeypatch.setattr(service.httpx, "AsyncClient", lambda **_kwargs: client) + + +@pytest.mark.asyncio +async def test_ollama_validation_runs_all_three_stages(monkeypatch: Any) -> None: + client = _FakeClient( + [ + _Response(200, {}), + _Response(200, {"models": []}), + _Response(200, {"message": {"content": "pong"}}), + ] + ) + _install_client(monkeypatch, client) + + result = await service.validate_llm_configuration( + provider="ollama_cloud", + model="llama-test", + base_url="https://ollama.example/", + api_key="secret-key", + ) + + assert result.ok is True + assert result.url_reachable is True + assert result.credentials_valid is True + assert result.llm_responded is True + assert result.response == "pong" + assert [call[1] for call in client.calls] == [ + "https://ollama.example", + "https://ollama.example/api/tags", + "https://ollama.example/api/chat", + ] + assert client.calls[1][2]["headers"]["Authorization"] == "Bearer secret-key" + assert client.calls[2][2]["json"]["think"] is False + assert client.calls[2][2]["json"]["options"]["num_predict"] == 64 + + +@pytest.mark.asyncio +async def test_invalid_credentials_stop_before_ping(monkeypatch: Any) -> None: + client = _FakeClient([_Response(200, {}), _Response(401, {})]) + _install_client(monkeypatch, client) + + result = await service.validate_llm_configuration( + provider="ollama_cloud", + model="llama-test", + base_url="https://ollama.example", + api_key="invalid-key", + ) + + assert result.ok is False + assert result.url_reachable is True + assert result.credentials_valid is False + assert result.llm_responded is False + assert len(client.calls) == 2 + + +@pytest.mark.asyncio +async def test_unreachable_url_stops_before_credentials(monkeypatch: Any) -> None: + request = httpx.Request("GET", "https://offline.example") + client = _FakeClient([httpx.ConnectError("offline", request=request)]) + _install_client(monkeypatch, client) + + result = await service.validate_llm_configuration( + provider="ollama", + model="llama-test", + base_url="https://offline.example", + api_key=None, + ) + + assert result.ok is False + assert result.url_reachable is False + assert len(client.calls) == 1 + + +@pytest.mark.asyncio +async def test_model_rejection_returns_safe_provider_diagnostic( + monkeypatch: Any, +) -> None: + client = _FakeClient( + [ + _Response(200, {}), + _Response(200, {"models": []}), + _Response(404, {"error": "model 'missing-model' not found"}), + ] + ) + _install_client(monkeypatch, client) + + result = await service.validate_llm_configuration( + provider="ollama", + model="missing-model", + base_url="https://ollama.example", + api_key=None, + ) + + assert result.credentials_valid is True + assert result.diagnostic == "HTTP 404: model 'missing-model' not found" + + +@pytest.mark.asyncio +async def test_ping_auth_rejection_marks_credentials_invalid(monkeypatch: Any) -> None: + client = _FakeClient( + [ + _Response(200, {}), + _Response(200, {"models": []}), + _Response(401, {"error": {"message": "invalid API key"}}), + ] + ) + _install_client(monkeypatch, client) + + result = await service.validate_llm_configuration( + provider="ollama_cloud", + model="llama-test", + base_url="https://ollama.example", + api_key="invalid-key", + ) + + assert result.credentials_valid is False + assert result.diagnostic == "HTTP 401: invalid API key" diff --git a/tests/test_model_registry.py b/tests/test_model_registry.py new file mode 100644 index 0000000..15188d1 --- /dev/null +++ b/tests/test_model_registry.py @@ -0,0 +1,110 @@ +"""Unit tests for the forecasting model registry.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +_BACKEND = str(Path(__file__).resolve().parent.parent / "data_forecaster" / "backend") +if _BACKEND not in sys.path: + sys.path.insert(0, _BACKEND) + +from core import config as settings # noqa: E402 +from core.database import init_database # noqa: E402 +from forecasting import registry # noqa: E402 + + +@pytest.fixture +def db(tmp_path, monkeypatch): + """Temp backend DB with the model_config seed rows.""" + db_path = str(tmp_path / "backend.db") + monkeypatch.setattr(settings, "BACKEND_DB_PATH", db_path) + init_database() + return db_path + + +class TestEnabledModels: + """Registry reads of the enabled set.""" + + def test_all_enabled_by_default(self, db): + assert set(registry.get_enabled_models(db)) == set(registry.MODEL_NAMES) + + def test_disable_excludes_model(self, db): + registry.set_model_enabled("Prophet", False, db_path=db) + + enabled = registry.get_enabled_models(db) + + assert "Prophet" not in enabled + assert len(enabled) == 4 + + def test_missing_table_falls_back_to_all(self, tmp_path, monkeypatch): + db_path = str(tmp_path / "empty.db") + monkeypatch.setattr(settings, "BACKEND_DB_PATH", db_path) + # No init_database — model_config table does not exist. + assert set(registry.get_enabled_models(db_path)) == set(registry.MODEL_NAMES) + + +class TestSetModelEnabled: + """Enable/disable writes and the last-model guard.""" + + def test_unknown_model_rejected(self, db): + with pytest.raises(ValueError, match="Unknown model"): + registry.set_model_enabled("LSTM", True, db_path=db) + + def test_cannot_disable_last_enabled_model(self, db): + for name in registry.MODEL_NAMES: + if name != "EWMA": + registry.set_model_enabled(name, False, db_path=db) + + with pytest.raises(ValueError, match="last enabled model"): + registry.set_model_enabled("EWMA", False, db_path=db) + + assert registry.get_enabled_models(db) == ("EWMA",) + + def test_reenable_after_disable(self, db): + registry.set_model_enabled("ARIMA", False, db_path=db) + registry.set_model_enabled("ARIMA", True, db_path=db) + + assert "ARIMA" in registry.get_enabled_models(db) + + +class TestFitFunctions: + """Fit-loop triples honour enabled state and per-model kwargs.""" + + def test_disabled_model_not_returned(self, db): + registry.set_model_enabled("Prophet", False, db_path=db) + + triples = registry.get_fit_functions( + {"seasonal_period": 12, "freq": "MS"}, db_path=db + ) + + names = [name for name, _, _ in triples] + assert "Prophet" not in names + assert len(triples) == 4 + + def test_model_specific_kwargs(self, db): + triples = dict( + (name, kwargs) + for name, _, kwargs in registry.get_fit_functions( + {"seasonal_period": 7, "freq": "D"}, db_path=db + ) + ) + + assert triples["SARIMA"] == {"seasonal_period": 7} + assert triples["Prophet"] == {"freq": "D"} + assert triples["ARIMA"] == {} + + +class TestListModelStates: + """Admin UI state listing.""" + + def test_states_reflect_enabled(self, db): + registry.set_model_enabled("SARIMA", False, db_path=db) + + states = {m["name"]: m["enabled"] for m in registry.list_model_states(db)} + + assert states["SARIMA"] is False + assert states["ARIMA"] is True + assert len(states) == 5 diff --git a/tests/test_prophet_model.py b/tests/test_prophet_model.py new file mode 100644 index 0000000..bb6ac6e --- /dev/null +++ b/tests/test_prophet_model.py @@ -0,0 +1,246 @@ +"""Unit tests for the Prophet forecast model adapter. + +Prophet is a heavy optional dependency that is not installed in CI, so every +test patches ``forecasting.prophet_model.import_prophet`` with a lightweight +fake ``prophet`` module. The fake Prophet class records the frames it is +handed and returns deterministic ``yhat``/``yhat_lower``/``yhat_upper`` +columns so the adapter logic can be exercised end-to-end without cmdstan. +""" + +from __future__ import annotations + +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from forecasting import prophet_model + + +class _FakeProphet: + """Deterministic stand-in for ``prophet.Prophet`` used by the adapter. + + Records the last history frame it was fit on and produces forecasts whose + ``yhat`` echoes the last observed value (plus a small linear drift) and + whose intervals bracket ``yhat``. ``fit`` behaviour is configurable so + tests can simulate fit failures on the train split only. + """ + + # Class-level knobs the tests flip. + fail_first_fit: bool = False + _fit_call_count: int = 0 + + def __init__(self) -> None: + self._history: pd.DataFrame | None = None + self._make_future_freqs: list[str | None] = [] + + def fit(self, history: pd.DataFrame) -> "_FakeProphet": + """Store the history frame; optionally fail for the first fit only. + + The holdout fit is the first ``Prophet().fit(...)`` call the adapter + makes; the full-series fit is the second. Failing only the first + therefore simulates a holdout-only failure. + """ + _FakeProphet._fit_call_count += 1 + if _FakeProphet.fail_first_fit and _FakeProphet._fit_call_count == 1: + raise RuntimeError("train fit failed") + self._history = history + return self + + def make_future_dataframe( + self, + periods: int, + freq: str | None = None, + include_history: bool = True, + ) -> pd.DataFrame: + """Record the freq and build a future ``ds`` frame.""" + del include_history # Unused by the fake. + self._make_future_freqs.append(freq) + last_ds = self._history["ds"].iloc[-1] + future = pd.date_range(start=last_ds, periods=periods + 1, freq=freq or "D")[1:] + return pd.DataFrame({"ds": future}) + + def predict(self, future: pd.DataFrame) -> pd.DataFrame: + """Return deterministic yhat / yhat_lower / yhat_upper columns.""" + last_y = float(self._history["y"].iloc[-1]) + n = len(future) + drift = np.arange(1, n + 1, dtype=float) * 0.5 + yhat = np.full(n, last_y, dtype=float) + drift + return pd.DataFrame( + { + "ds": future["ds"], + "yhat": yhat, + "yhat_lower": yhat - 1.0, + "yhat_upper": yhat + 1.0, + } + ) + + +def _fake_prophet_module() -> SimpleNamespace: + """Build a fake ``prophet`` module exposing ``Prophet``.""" + return SimpleNamespace(Prophet=_FakeProphet) + + +@pytest.fixture(autouse=True) +def _reset_fake() -> None: + """Reset fake state between tests.""" + _FakeProphet.fail_first_fit = False + _FakeProphet._fit_call_count = 0 + yield + _FakeProphet.fail_first_fit = False + _FakeProphet._fit_call_count = 0 + + +def _patch_prophet(monkeypatch: pytest.MonkeyPatch) -> None: + """Patch ``import_prophet`` to return the fake prophet module.""" + monkeypatch.setattr(prophet_model, "import_prophet", lambda: _fake_prophet_module()) + + +def _monthly_series(n: int = 36) -> pd.Series: + """A deterministic monthly series with trend + seasonality.""" + idx = pd.date_range("2020-01-01", periods=n, freq="MS") + values = np.arange(n, dtype=float) + 10 * np.sin(np.arange(n) * np.pi / 6) + return pd.Series(values, index=idx) + + +class TestFitProphetHappyPath: + """Happy-path behaviour of ``fit_prophet``.""" + + def test_returns_full_contract_with_correct_lengths( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The result dict matches the standard model contract.""" + _patch_prophet(monkeypatch) + series = _monthly_series() + + result = prophet_model.fit_prophet(series, forecast_horizon=6, freq="MS") + + assert set(result) == { + "forecast", + "lower_ci", + "upper_ci", + "rmse", + "mae", + "mape", + } + assert len(result["forecast"]) == 6 + assert len(result["lower_ci"]) == 6 + assert len(result["upper_ci"]) == 6 + + def test_intervals_bracket_forecast(self, monkeypatch: pytest.MonkeyPatch) -> None: + """``lower_ci`` <= ``forecast`` <= ``upper_ci`` for every step.""" + _patch_prophet(monkeypatch) + result = prophet_model.fit_prophet(_monthly_series(), 4, freq="MS") + + for lo, fc, hi in zip( + result["lower_ci"], result["forecast"], result["upper_ci"] + ): + assert lo <= fc <= hi + + def test_metrics_are_finite(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Holdout metrics come back finite for a healthy series.""" + _patch_prophet(monkeypatch) + result = prophet_model.fit_prophet(_monthly_series(), 6, freq="MS") + + assert np.isfinite(result["rmse"]) + assert np.isfinite(result["mae"]) + assert np.isfinite(result["mape"]) + + +class TestFitProphetShortSeries: + """Short-series fallback to a persistence forecast.""" + + def test_single_point_returns_persistence_forecast( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Fewer than 2 points should never call Prophet.""" + _patch_prophet(monkeypatch) + series = pd.Series([42.0]) + + result = prophet_model.fit_prophet(series, forecast_horizon=3) + + assert result["forecast"] == [42.0, 42.0, 42.0] + assert result["lower_ci"] == [42.0, 42.0, 42.0] + assert result["upper_ci"] == [42.0, 42.0, 42.0] + assert result["rmse"] == 0.0 + assert result["mae"] == 0.0 + assert result["mape"] == 0.0 + + def test_empty_series_returns_zero_forecast( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """An empty series falls back to a zero persistence forecast.""" + _patch_prophet(monkeypatch) + result = prophet_model.fit_prophet(pd.Series(dtype=float), forecast_horizon=2) + + assert result["forecast"] == [0.0, 0.0] + assert result["rmse"] == 0.0 + + +class TestFitProphetNonDatetimeIndex: + """A non-datetime index is converted to a synthetic date range.""" + + def test_range_index_still_forecasts(self, monkeypatch: pytest.MonkeyPatch) -> None: + _patch_prophet(monkeypatch) + series = pd.Series(np.arange(20, dtype=float)) + + result = prophet_model.fit_prophet(series, forecast_horizon=3) + + assert len(result["forecast"]) == 3 + assert all(np.isfinite(result["forecast"])) + + +class TestFitProphetTrainFailure: + """Resilience when the holdout fit fails but the full fit succeeds.""" + + def test_holdout_failure_zeroes_metrics_but_keeps_forecast( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """If the train-split fit raises, metrics zero out, forecast survives.""" + _patch_prophet(monkeypatch) + # First fit (holdout) raises; the second (full-series) succeeds. + _FakeProphet.fail_first_fit = True + + result = prophet_model.fit_prophet(_monthly_series(), 4, freq="MS") + + assert result["rmse"] == 0.0 + assert result["mae"] == 0.0 + assert result["mape"] == 0.0 + assert len(result["forecast"]) == 4 + + +class TestFitProphetImportError: + """When Prophet is unavailable, the adapter surfaces the ImportError.""" + + def test_raises_when_prophet_missing(self, monkeypatch: pytest.MonkeyPatch) -> None: + def _raise() -> None: + raise ImportError("prophet not installed") + + monkeypatch.setattr(prophet_model, "import_prophet", _raise) + + with pytest.raises(ImportError): + prophet_model.fit_prophet(_monthly_series(), forecast_horizon=3) + + +class TestFitProphetFreqThreading: + """The supplied freq reaches the future-frame construction.""" + + def test_freq_is_threaded_to_future_frame( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """The freq kwarg is forwarded to ``_future_frame``.""" + _patch_prophet(monkeypatch) + seen: list[str | None] = [] + original = prophet_model._future_frame + + def _spy(history: pd.DataFrame, periods: int, freq: str | None) -> pd.DataFrame: + seen.append(freq) + return original(history, periods, freq) + + monkeypatch.setattr(prophet_model, "_future_frame", _spy) + + prophet_model.fit_prophet(_monthly_series(), forecast_horizon=3, freq="MS") + + # The holdout fit and the full-series fit each call _future_frame. + assert seen[-1] == "MS" diff --git a/tests/test_secret_store.py b/tests/test_secret_store.py new file mode 100644 index 0000000..8ef025a --- /dev/null +++ b/tests/test_secret_store.py @@ -0,0 +1,67 @@ +"""Unit tests for the backend secret store (Fernet key lifecycle).""" + +from __future__ import annotations + +import os +import stat + +import pytest + +from data_forecaster.backend.core import secret_store + + +@pytest.fixture +def secrets_dir(tmp_path, monkeypatch): + """Point SECRETS_DIR at a temp dir and reset the cached Fernet. + + Note: ``secret_store`` imports its settings as ``core.config`` (backend + import style), which is a distinct module object from + ``data_forecaster.backend.core.config``. Patch the object the module + under test actually references. + """ + directory = tmp_path / "secrets" + monkeypatch.setattr(secret_store.settings, "SECRETS_DIR", str(directory)) + secret_store.reset_cache() + yield directory + secret_store.reset_cache() + + +class TestKeyLifecycle: + """Key generation, persistence, and loading.""" + + def test_generate_persists_key_with_0600(self, secrets_dir): + secret_store.generate_and_persist_key() + + key_path = secrets_dir / ".encryption_key" + assert key_path.is_file() + mode = stat.S_IMODE(os.stat(key_path).st_mode) + assert mode == 0o600 + + def test_generate_is_idempotent(self, secrets_dir): + secret_store.generate_and_persist_key() + key_path = secrets_dir / ".encryption_key" + first = key_path.read_bytes() + + secret_store.generate_and_persist_key() + + assert key_path.read_bytes() == first + + def test_get_fernet_missing_key_raises(self, secrets_dir): + with pytest.raises(RuntimeError, match="encryption key not found"): + secret_store.get_fernet() + + def test_round_trip_encrypt_decrypt(self, secrets_dir): + secret_store.generate_and_persist_key() + + ciphertext = secret_store.encrypt("super-secret-api-key") + + assert ciphertext != "super-secret-api-key" + assert secret_store.decrypt(ciphertext) == "super-secret-api-key" + + def test_key_survives_cache_reset(self, secrets_dir): + secret_store.generate_and_persist_key() + ciphertext = secret_store.encrypt("value") + + secret_store.reset_cache() + + assert secret_store.decrypt(ciphertext) == "value" diff --git a/tests/test_setup_service.py b/tests/test_setup_service.py new file mode 100644 index 0000000..dee7130 --- /dev/null +++ b/tests/test_setup_service.py @@ -0,0 +1,124 @@ +"""Unit tests for the first-run setup service (atomic bootstrap).""" + +from __future__ import annotations + +import sys +import threading +from pathlib import Path + +import pytest + +_BACKEND = str(Path(__file__).resolve().parent.parent / "data_forecaster" / "backend") +if _BACKEND not in sys.path: + sys.path.insert(0, _BACKEND) + +from core import secret_store # noqa: E402 +from core import config as settings # noqa: E402 +from core.database import get_connection, init_database # noqa: E402 +from services import setup_service # noqa: E402 + + +@pytest.fixture +def db(tmp_path, monkeypatch): + """Temp backend DB + secrets dir; patch the modules' own references.""" + secrets = tmp_path / "secrets" + monkeypatch.setattr(secret_store.settings, "SECRETS_DIR", str(secrets)) + secret_store.reset_cache() + + db_path = str(tmp_path / "backend.db") + monkeypatch.setattr(settings, "BACKEND_DB_PATH", db_path) + init_database() + yield db_path + secret_store.reset_cache() + + +class TestBootstrap: + """Atomic first-run bootstrap.""" + + def test_first_bootstrap_succeeds(self, db): + user = setup_service.run_bootstrap("admin", "secret-key", db_path=db) + + assert user["username"] == "admin" + assert user["is_admin"] == 1 + assert "api_key_hash" not in user + assert setup_service.is_setup_complete(db) is True + assert secret_store.key_file_exists() + + def test_second_bootstrap_rejected(self, db): + setup_service.run_bootstrap("admin", "secret-key", db_path=db) + + with pytest.raises(setup_service.SetupAlreadyCompleteError): + setup_service.run_bootstrap("other", "other-key", db_path=db) + + def test_concurrent_bootstrap_exactly_one_wins(self, db): + results: list[str] = [] + errors: list[Exception] = [] + + def attempt(name: str) -> None: + try: + setup_service.run_bootstrap(name, f"key-{name}", db_path=db) + results.append(name) + except setup_service.SetupAlreadyCompleteError as exc: + errors.append(exc) + + threads = [threading.Thread(target=attempt, args=(f"user{i}",)) for i in range(4)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(results) == 1 + assert len(errors) == 3 + with get_connection(db) as connection: + count = connection.execute( + "SELECT COUNT(*) AS cnt FROM api_users" + ).fetchone()["cnt"] + assert count == 1 + + def test_empty_username_rejected(self, db): + with pytest.raises(ValueError, match="Username is required"): + setup_service.run_bootstrap(" ", "key", db_path=db) + + +class TestSetupStatus: + """Setup status reporting (no secrets).""" + + def test_fresh_install_status(self, db): + status = setup_service.get_setup_status(db) + + assert status == { + "setup_complete": False, + "admin_exists": False, + "llm_configured": False, + "models_enabled": 5, + } + + def test_status_after_bootstrap(self, db): + setup_service.run_bootstrap("admin", "secret-key", db_path=db) + + status = setup_service.get_setup_status(db) + + assert status["setup_complete"] is True + assert status["admin_exists"] is True + + +class TestLegacyMigration: + """Existing deployments migrate to setup_complete automatically.""" + + def test_marks_complete_when_users_exist(self, db): + setup_service.run_bootstrap("admin", "secret-key", db_path=db) + # Simulate a pre-wizard deployment: users exist, flag cleared. + with get_connection(db) as connection: + connection.execute( + "UPDATE system_settings SET setup_complete = 0 WHERE singleton = 1" + ) + connection.commit() + + setup_service.migrate_legacy_deployment(db) + + assert setup_service.is_setup_complete(db) is True + + def test_noop_on_fresh_install(self, db): + setup_service.migrate_legacy_deployment(db) + + assert setup_service.is_setup_complete(db) is False diff --git a/tests/test_setup_wizard_frontend.py b/tests/test_setup_wizard_frontend.py new file mode 100644 index 0000000..55eb4b1 --- /dev/null +++ b/tests/test_setup_wizard_frontend.py @@ -0,0 +1,522 @@ +"""Tests for the setup wizard blueprint, setup gating, and admin config pages. + +All HTTP calls to the FastAPI backend are mocked by patching the +``requests`` module attributes used by ``services.api_client``. +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +BACKEND_ROOT = REPO_ROOT / "data_forecaster" / "backend" +FRONTEND_ROOT = REPO_ROOT / "data_forecaster" / "frontend" +if str(FRONTEND_ROOT) in sys.path: + sys.path.remove(str(FRONTEND_ROOT)) +sys.path.insert(0, str(FRONTEND_ROOT)) +sys.modules.pop("services", None) + +from app import create_app # noqa: E402 +from db.crypto import decrypt # noqa: E402 +from db.db import execute_db, query_db # noqa: E402 +from services import api_client as frontend_api_client # noqa: E402 + +sys.modules.pop("services", None) +if str(FRONTEND_ROOT) in sys.path: + sys.path.remove(str(FRONTEND_ROOT)) +if str(BACKEND_ROOT) in sys.path: + sys.path.remove(str(BACKEND_ROOT)) +sys.path.insert(0, str(BACKEND_ROOT)) +sys.path.insert(1, str(FRONTEND_ROOT)) + +_BACKEND_URL = "http://backend:8000" + +_MODELS: list[dict[str, Any]] = [ + {"name": "Holt-Winters", "display_name": "Holt-Winters", "enabled": True}, + {"name": "ARIMA", "display_name": "ARIMA", "enabled": True}, + {"name": "SARIMA", "display_name": "SARIMA", "enabled": True}, + {"name": "EWMA", "display_name": "EWMA", "enabled": True}, + {"name": "Prophet", "display_name": "Prophet", "enabled": True}, +] + + +class _FakeResponse: + """Minimal stand-in for :class:`requests.Response`.""" + + def __init__( + self, status_code: int, payload: dict[str, Any] | None = None + ) -> None: + self.status_code = status_code + self._payload = payload or {} + + def json(self) -> dict[str, Any]: + return self._payload + + +@pytest.fixture +def backend_state() -> dict[str, Any]: + """Mutable fake-backend state shared with the request stubs.""" + return { + "setup_complete": False, + "bootstrapped": False, + "models": [dict(model) for model in _MODELS], + "llm_config": { + "provider": "gemini", + "model": "gemini-1.5-flash", + "base_url": None, + "temperature": 0.1, + "api_key_set": False, + "configured": False, + }, + "last_model_error": False, + "llm_test": { + "ok": True, + "url_reachable": True, + "credentials_valid": True, + "llm_responded": True, + "message": "LLM connection test passed.", + "response": "pong", + }, + } + + +@pytest.fixture +def mock_backend( + monkeypatch: pytest.MonkeyPatch, + backend_state: dict[str, Any], +) -> dict[str, Any]: + """Patch the requests module used by the API client with a fake backend.""" + + def fake_get(url: str, **kwargs: Any) -> _FakeResponse: + if url.endswith("/setup/status"): + return _FakeResponse( + 200, + { + "setup_complete": backend_state["setup_complete"], + "admin_exists": backend_state["bootstrapped"], + "llm_configured": backend_state["llm_config"]["configured"], + "models_enabled": sum( + 1 for m in backend_state["models"] if m["enabled"] + ), + }, + ) + if url.endswith("/models"): + return _FakeResponse(200, {"models": backend_state["models"]}) + if url.endswith("/config/llm"): + return _FakeResponse(200, backend_state["llm_config"]) + return _FakeResponse(404, {"detail": "Not found"}) + + def fake_post( + url: str, json: dict[str, Any] | None = None, **kwargs: Any + ) -> _FakeResponse: + if url.endswith("/config/llm/test"): + return _FakeResponse(200, backend_state["llm_test"]) + if url.endswith("/setup/bootstrap"): + if backend_state["bootstrapped"]: + return _FakeResponse(409, {"detail": "Setup already completed."}) + backend_state["bootstrapped"] = True + backend_state["setup_complete"] = True + return _FakeResponse( + 200, + {"user": {"username": (json or {})["username"]}, + "setup_complete": True}, + ) + return _FakeResponse(404, {"detail": "Not found"}) + + def fake_put( + url: str, json: dict[str, Any] | None = None, **kwargs: Any + ) -> _FakeResponse: + body = json or {} + if "/models/" in url: + if backend_state["last_model_error"] and body.get("enabled") is False: + return _FakeResponse( + 400, {"detail": "Cannot disable the last enabled model."} + ) + name = url.rsplit("/", 1)[-1] + for model in backend_state["models"]: + if model["name"] == name: + model["enabled"] = bool(body.get("enabled")) + return _FakeResponse(200, {"models": backend_state["models"]}) + if url.endswith("/config/llm"): + config = backend_state["llm_config"] + for key, value in body.items(): + if key != "api_key": + config[key] = value + if body.get("api_key"): + config["api_key_set"] = True + config["configured"] = True + return _FakeResponse(200, config) + return _FakeResponse(404, {"detail": "Not found"}) + + monkeypatch.setattr(frontend_api_client.requests, "get", fake_get) + monkeypatch.setattr(frontend_api_client.requests, "post", fake_post) + monkeypatch.setattr(frontend_api_client.requests, "put", fake_put) + return backend_state + + +@pytest.fixture +def app(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Create a testing Flask app with an isolated database.""" + monkeypatch.setattr( + "config.TestingConfig.DATABASE", str(tmp_path / "frontend.db") + ) + application = create_app("testing") + application.config["BACKEND_URL"] = _BACKEND_URL + return application + + +@pytest.fixture +def client(app, mock_backend: dict[str, Any]): + """Test client with the fake backend active.""" + return app.test_client() + + +@pytest.fixture +def admin_client(app, mock_backend: dict[str, Any]): + """Test client logged in as the seeded admin with setup complete.""" + mock_backend["setup_complete"] = True + with app.app_context(): + execute_db( + "UPDATE users SET must_change_password = 0 WHERE username = 'admin'" + ) + test_client = app.test_client() + with test_client.session_transaction() as sess: + sess["_user_id"] = "1" + sess["user_session_version"] = 0 + return test_client + + +class TestSetupGating: + """The app redirects to /setup until the backend reports completion.""" + + def test_redirects_to_wizard_when_incomplete(self, client) -> None: + resp = client.get("/") + assert resp.status_code == 302 + assert resp.headers["Location"].startswith("/setup") + + def test_allows_app_when_complete(self, client, backend_state) -> None: + backend_state["setup_complete"] = True + resp = client.get("/") + assert resp.status_code == 302 + assert not resp.headers["Location"].startswith("/setup") + + def test_wizard_redirects_to_login_when_complete( + self, client, backend_state + ) -> None: + backend_state["setup_complete"] = True + resp = client.get("/setup/") + assert resp.status_code == 302 + assert "/auth/login" in resp.headers["Location"] + + +class TestSetupGateWorkerConsistency: + """The gate must resolve the backend URL from the DB, not just config. + + Gunicorn runs multiple workers; only the worker that handled the + wizard's backend step has ``BACKEND_URL`` in its in-process config. + Reading config alone made workers disagree about setup state, which + caused the wizard/login redirect loop. + """ + + def test_gate_uses_db_url_when_config_is_stale( + self, app, client, backend_state + ) -> None: + """A worker with empty in-memory config must still honour the DB URL.""" + client.post("/setup/backend", data={"base_url": _BACKEND_URL}) + + # Simulate another gunicorn worker: in-process config is stale. + app.config["BACKEND_URL"] = "" + backend_state["setup_complete"] = True + + resp = client.get("/") + assert resp.status_code == 302 + assert not resp.headers["Location"].startswith("/setup") + + def test_wizard_exits_when_db_url_set_and_backend_complete( + self, app, client, backend_state + ) -> None: + """With the URL in the DB, every worker must agree setup is done.""" + client.post("/setup/backend", data={"base_url": _BACKEND_URL}) + + app.config["BACKEND_URL"] = "" + backend_state["setup_complete"] = True + + resp = client.get("/setup/") + assert resp.status_code == 302 + assert "/auth/login" in resp.headers["Location"] + + def test_wizard_backend_step_prefills_db_url_on_stale_worker( + self, app, client + ) -> None: + """Step 1 must pre-fill the URL from the DB on every worker.""" + client.post("/setup/backend", data={"base_url": _BACKEND_URL}) + + app.config["BACKEND_URL"] = "" + + resp = client.get("/setup/backend") + assert resp.status_code == 200 + assert _BACKEND_URL.encode() in resp.data + + def test_admin_step_preserves_db_base_url_on_stale_worker( + self, app, client + ) -> None: + """Step 5 must not wipe the stored base_url on a worker without config. + + Previously step 5 read ``BACKEND_URL`` from in-process config; on a + worker that had not handled step 1 that value was empty, so saving + credentials overwrote the DB's ``base_url`` with an empty string and + the wizard looped after every restart. + """ + client.post("/setup/backend", data={"base_url": _BACKEND_URL}) + client.post( + "/setup/llm", + data={ + "provider": "gemini", + "model": "gemini-1.5-flash", + "temperature": "0.1", + }, + ) + client.post("/setup/auth", data={"confirm": "y"}) + client.post( + "/setup/models", + data={"model_enabled": [m["name"] for m in _MODELS]}, + ) + + # Simulate another gunicorn worker handling the final step. + app.config["BACKEND_URL"] = "" + app.config["API_VERIFY_SSL"] = False + + resp = client.post( + "/setup/admin", + data={"username": "frontend", "api_key": "test-secret-key-123"}, + ) + assert resp.status_code == 302 + assert "/setup/done" in resp.headers["Location"] + + with app.app_context(): + row = query_db( + "SELECT base_url FROM api_credentials WHERE label = 'default'", + one=True, + ) + assert row is not None + assert row["base_url"] == _BACKEND_URL + + +class TestSetupWizard: + """Wizard step rendering and the full bootstrap flow.""" + + def test_backend_step_renders(self, client) -> None: + resp = client.get("/setup/backend") + assert resp.status_code == 200 + assert b"Backend API Base URL" in resp.data + + def test_bootstrap_submit_stores_encrypted_credentials( + self, app, client + ) -> None: + resp = client.post( + "/setup/backend", + data={"base_url": _BACKEND_URL}, + ) + assert resp.status_code == 302 + assert "/setup/llm" in resp.headers["Location"] + + resp = client.post( + "/setup/llm", + data={ + "provider": "gemini", + "model": "gemini-1.5-flash", + "temperature": "0.1", + }, + ) + assert resp.status_code == 302 + assert "/setup/auth" in resp.headers["Location"] + + resp = client.post("/setup/auth", data={"confirm": "y"}) + assert resp.status_code == 302 + assert "/setup/models" in resp.headers["Location"] + + resp = client.post( + "/setup/models", + data={"model_enabled": [m["name"] for m in _MODELS]}, + ) + assert resp.status_code == 302 + assert "/setup/admin" in resp.headers["Location"] + + resp = client.post( + "/setup/admin", + data={"username": "frontend", "api_key": "test-secret-key-123"}, + ) + assert resp.status_code == 302 + assert "/setup/done" in resp.headers["Location"] + + with app.app_context(): + row = query_db( + """ + SELECT base_url, encrypted_username, encrypted_password + FROM api_credentials + WHERE label = 'default' + """, + one=True, + ) + assert row is not None + assert row["base_url"] == _BACKEND_URL + assert decrypt(str(row["encrypted_username"])) == "frontend" + assert decrypt(str(row["encrypted_password"])) == "test-secret-key-123" + # The plaintext key must never be stored. + assert "test-secret-key-123" not in str(row["encrypted_password"]) + + def test_llm_step_never_persists_key_locally( + self, app, client, backend_state + ) -> None: + client.post("/setup/backend", data={"base_url": _BACKEND_URL}) + resp = client.post( + "/setup/llm", + data={ + "provider": "gemini", + "model": "gemini-1.5-flash", + "api_key": "super-secret-llm-key", + "temperature": "0.1", + }, + ) + assert resp.status_code == 302 + + with app.app_context(): + row = query_db( + "SELECT encrypted_username, encrypted_password " + "FROM api_credentials WHERE label = 'default'", + one=True, + ) + assert row is not None + assert row["encrypted_username"] is None + assert row["encrypted_password"] is None + # The key was forwarded to the backend instead. + assert backend_state["llm_config"]["api_key_set"] is True + + def test_llm_step_blocks_progress_when_validation_fails( + self, client, backend_state + ) -> None: + client.post("/setup/backend", data={"base_url": _BACKEND_URL}) + backend_state["llm_test"] = { + "ok": False, + "url_reachable": True, + "credentials_valid": False, + "llm_responded": False, + "message": "The LLM URL or API key was rejected. Check both values.", + "response": None, + } + + resp = client.post( + "/setup/llm", + data={ + "provider": "gemini", + "model": "gemini-1.5-flash", + "api_key": "invalid-key", + "temperature": "0.1", + }, + ) + + assert resp.status_code == 200 + assert b"URL reachable: Yes" in resp.data + assert b"URL and API key valid: No" in resp.data + assert backend_state["llm_config"]["configured"] is False + with client.session_transaction() as sess: + assert "setup_llm_ok" not in sess + + def test_models_step_rejects_unchecking_all(self, client) -> None: + client.post("/setup/backend", data={"base_url": _BACKEND_URL}) + client.post( + "/setup/llm", + data={ + "provider": "gemini", + "model": "gemini-1.5-flash", + "temperature": "0.1", + }, + ) + client.post("/setup/auth", data={"confirm": "y"}) + resp = client.post("/setup/models", data={}) + assert resp.status_code == 200 + assert b"At least one model must remain enabled." in resp.data + + +class TestAdminConfigPages: + """Admin LLM config and model registry pages.""" + + def test_llm_config_page_renders_masked(self, admin_client) -> None: + resp = admin_client.get("/admin/llm-config") + assert resp.status_code == 200 + assert b"gemini-1.5-flash" in resp.data + assert b"Not set" in resp.data + assert b"leave blank to keep current" in resp.data + assert b"Test LLM" in resp.data + + def test_llm_test_displays_response_without_saving( + self, admin_client, backend_state + ) -> None: + resp = admin_client.post( + "/admin/llm-config", + data={ + "provider": "gemini", + "model": "candidate-model", + "api_key": "candidate-key", + "temperature": "0.1", + "test_llm": "Test LLM", + }, + ) + + assert resp.status_code == 200 + assert b"LLM response:" in resp.data + assert b"pong" in resp.data + assert backend_state["llm_config"]["model"] == "gemini-1.5-flash" + + def test_llm_save_is_blocked_when_validation_fails( + self, admin_client, backend_state + ) -> None: + backend_state["llm_test"] = { + "ok": False, + "url_reachable": False, + "credentials_valid": False, + "llm_responded": False, + "message": "The LLM URL could not be reached.", + "response": None, + "diagnostic": "HTTP 404: model 'candidate-model' not found", + } + + resp = admin_client.post( + "/admin/llm-config", + data={ + "provider": "ollama_cloud", + "model": "candidate-model", + "base_url": "https://unreachable.example", + "api_key": "candidate-key", + "temperature": "0.1", + "submit": "Save LLM Configuration", + }, + ) + + assert resp.status_code == 200 + assert b"The LLM URL could not be reached." in resp.data + assert b"Provider diagnostic:" in resp.data + assert b"HTTP 404: model 'candidate-model' not found" in resp.data + assert backend_state["llm_config"]["model"] == "gemini-1.5-flash" + + def test_models_page_renders(self, admin_client) -> None: + resp = admin_client.get("/admin/models") + assert resp.status_code == 200 + assert b"Holt-Winters" in resp.data + assert b"Prophet" in resp.data + + def test_models_page_surfaces_last_model_error( + self, admin_client, backend_state + ) -> None: + backend_state["last_model_error"] = True + resp = admin_client.post( + "/admin/models", + data={"model_enabled": ["ARIMA"]}, + follow_redirects=True, + ) + assert resp.status_code == 200 + assert b"Cannot disable the last enabled model." in resp.data