diff --git a/CHANGELOG.md b/CHANGELOG.md index 779a8ae..cd76685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added +- Added `oaeval doctor` warnings for configured providers whose optional extras are not installed. - Created `scripts/changelog_validator.py` to automatically verify Keep a Changelog formatting. - Added a section to `CONTRIBUTING.md` documenting the changelog update process for future contributors. diff --git a/openagent_eval/cli/commands/doctor.py b/openagent_eval/cli/commands/doctor.py index b690318..71837ab 100644 --- a/openagent_eval/cli/commands/doctor.py +++ b/openagent_eval/cli/commands/doctor.py @@ -3,19 +3,44 @@ from __future__ import annotations import importlib +import importlib.util import os import sys -from pathlib import Path +from typing import TYPE_CHECKING import typer from rich.console import Console +from rich.markup import escape from rich.table import Table from openagent_eval import __version__ from openagent_eval.cli.context import get_context +from openagent_eval.cli.utils.discovery import find_config_file +from openagent_eval.config.loader import load_config +from openagent_eval.exceptions import ConfigurationError + +if TYPE_CHECKING: + from pathlib import Path console = Console() +_LLM_PROVIDER_EXTRAS: dict[str, tuple[tuple[str, ...], str]] = { + "anthropic": (("anthropic",), "providers"), + "gemini": (("google.genai",), "providers"), + "groq": (("groq",), "providers"), + "openai": (("openai", "tiktoken"), "providers"), +} + +_RETRIEVER_EXTRAS: dict[str, tuple[tuple[str, ...], str]] = { + "bm25": (("rank_bm25",), "bm25"), + "elasticsearch": (("elasticsearch",), "elasticsearch"), + "faiss": (("faiss",), "faiss"), + "pgvector": (("psycopg", "pgvector"), "pgvector"), + "pinecone": (("pinecone",), "pinecone"), + "qdrant": (("qdrant_client",), "qdrant"), + "weaviate": (("weaviate",), "weaviate"), +} + def doctor_command( verbose: bool = typer.Option( @@ -125,7 +150,8 @@ def doctor_command( # Configuration file check console.print("\n[bold]Configuration:[/bold]") - _check_config_files() + config_path = _check_config_files() + _check_configured_provider_extras(config_path) # Summary console.print("\n[bold]Summary:[/bold]") @@ -187,26 +213,77 @@ def _test_api_connectivity(providers: list[str]) -> None: console.print(f" [yellow]SKIP[/yellow] {provider}: {type(e).__name__}") -def _check_config_files() -> None: +def _check_config_files() -> Path | None: """Check for configuration files in current directory.""" - config_names = ["config.yaml", "config.yml", "oaeval.yaml", "oaeval.yml"] - found = False + config_path = find_config_file() - for name in config_names: - path = Path(name) - if path.exists(): - console.print(f" [green]OK[/green] Found config: {name}") - found = True - break - - if not found: + if config_path is None: console.print(" [dim]No config file in current directory[/dim]") console.print(" [dim]Run 'oaeval init' to create one[/dim]") + return None + + console.print(f" [green]OK[/green] Found config: {config_path}") + return config_path + + +def _check_configured_provider_extras(config_path: Path | None) -> None: + """Warn when the active config uses providers whose extras are not installed.""" + if config_path is None: + return + + try: + config = load_config(config_path) + except ConfigurationError as exc: + console.print( + f" [yellow]WARNING[/yellow] Could not inspect configured providers: {exc.message}" + ) + return + + missing = [ + *_missing_extra_rows("LLM", config.llm.provider, _LLM_PROVIDER_EXTRAS), + *_missing_extra_rows("Retriever", config.retriever.provider, _RETRIEVER_EXTRAS), + ] + + if not missing: + console.print(" [green]OK[/green] Configured provider extras installed") + return + + table = Table(title="Configured Provider Extras") + table.add_column("Provider", style="cyan") + table.add_column("Status", style="bold") + table.add_column("Install") + + for provider, extra in missing: + table.add_row( + provider, + "[yellow]MISSING[/yellow]", + escape(f'pip install "openagent-eval[{extra}]"'), + ) + + console.print(table) + + +def _missing_extra_rows( + kind: str, + provider: str, + extras: dict[str, tuple[tuple[str, ...], str]], +) -> list[tuple[str, str]]: + key = provider.lower().strip() + if key not in extras: + return [] + + modules, extra = extras[key] + if all(_module_available(module) for module in modules): + return [] + + return [(f"{kind}: {provider}", extra)] + - # Check environment variable - env_config = os.environ.get("OAEVAL_CONFIG") - if env_config: - console.print(f" [green]OK[/green] OAEVAL_CONFIG: {env_config}") +def _module_available(module_name: str) -> bool: + try: + return importlib.util.find_spec(module_name) is not None + except (ImportError, ModuleNotFoundError, ValueError): + return False def _print_recommendations(python_ok: bool, providers: list[str]) -> None: diff --git a/tests/integration/test_cli_integration/test_doctor.py b/tests/integration/test_cli_integration/test_doctor.py index d8f6eee..df36529 100644 --- a/tests/integration/test_cli_integration/test_doctor.py +++ b/tests/integration/test_cli_integration/test_doctor.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING +from unittest.mock import patch from openagent_eval.cli.main import app @@ -35,3 +36,42 @@ def test_doctor_shows_api_keys(runner: CliRunner) -> None: assert result.exit_code == 0 # Should show API key status table assert "API Key" in result.output or "OpenAI" in result.output + + +def test_doctor_warns_for_missing_configured_provider_extras( + runner: CliRunner, tmp_path +) -> None: + """doctor command reports install commands for configured missing extras.""" + config_path = tmp_path / "config.yaml" + config_path.write_text( + """\ +dataset: data/questions.json +llm: + provider: anthropic + model: claude-sonnet-4-20250514 +retriever: + provider: qdrant + settings: + collection_name: docs +metrics: + - latency +""", + encoding="utf-8", + ) + + def fake_find_spec(module_name: str): + if module_name in {"anthropic", "qdrant_client"}: + return None + return object() + + with patch( + "openagent_eval.cli.commands.doctor.importlib.util.find_spec", + side_effect=fake_find_spec, + ): + result = runner.invoke(app, ["doctor"], env={"OAEVAL_CONFIG": str(config_path)}) + + assert result.exit_code == 0 + assert "LLM: anthropic" in result.output + assert 'pip install "openagent-eval[providers]"' in result.output + assert "Retriever: qdrant" in result.output + assert 'pip install "openagent-eval[qdrant]"' in result.output