diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bb1eae..3d11f71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `evalshift report` and `evalshift compare` end with a one-line request to star the + project on GitHub, with the repo link. It prints on interactive terminals only, so CI + logs, the GitHub Action's output and anything piped stay free of it. + ## [1.4.3] - 2026-10-13 ### Fixed diff --git a/DOCS.md b/DOCS.md index ae692e4..0d40158 100644 --- a/DOCS.md +++ b/DOCS.md @@ -901,6 +901,7 @@ Prints a red doctor-style **broken eval harness** row when the *source* model fa `-c/--config` · `--gate ` · `--policy-gate` **`evalshift report `** — render `report.html` + `report.json`. `-c/--config` · `--open` · `--insights/--no-insights` (default on; one extra LLM call — see [Run insights](#run-insights)) +On an interactive terminal, `report` and `compare` end with one line asking for a GitHub star, with the repo link. It is not printed when output is piped or captured (CI, the GitHub Action), so logs and step summaries never carry it. **`evalshift compare`** — full pipeline under one live display: two models, **one** suite. It takes a single `--suite-name`/`--suite`, auto-selects when exactly one suite is wired, and prints a ready-to-run command per suite when several are. It does not run every suite — loop over the names, or use one CI job per suite. Formerly `all`; that name is hidden but still works, since scaffolded `EVALSHIFT.md` files reference it. All `run` flags, plus `--gate` · `--policy-gate` · `--open` · `--push` · `--insights/--no-insights` diff --git a/docs/getting-started.md b/docs/getting-started.md index ab2aafa..80b1ed7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -185,6 +185,10 @@ evalshift report --open (`--from/--to`, `--config`, `--suite`, `--suite-name`, `--yes`, `--resume`, `--gate`, `--policy-gate`, `--open`, `--push`). +When run in a terminal, `report` and `compare` finish with a one-line request +to star [the repo on GitHub](https://github.com/evalshift/evalshift-cli). It is +skipped whenever output is piped or captured, so CI logs stay clean. + One invocation compares two models on **one** suite. It picks the suite for you when `evalshift.yaml` wires exactly one; with several, name it with `--suite-name` (the error lists a ready-to-run command per suite). To cover diff --git a/llms-full.txt b/llms-full.txt index b4c1734..f71cc51 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -215,6 +215,8 @@ evalshift analyze RUN_ID [-c CONFIG] [--gate SEVS] [--policy-gate] evalshift report RUN_ID [-c CONFIG] [--open] [--insights/--no-insights (default on)] -> report.html + report.json (+ insights.json). --no-insights skips the narrative and its one extra LLM call. See "Run insights". + On an interactive terminal only, report and compare end with one GitHub-star request + line (https://github.com/evalshift/evalshift-cli); piped/CI output never carries it. report.html layout: header (run id, source->target, suite, examples/calls, spend) -> verdict / advisory-signal / economics panels -> six-cell run strip (examples, calls, failed/truncated, total cost, latency delta, mean score delta) -> recommendations + diff --git a/src/evalshift_cli/cli/commands/_star.py b/src/evalshift_cli/cli/commands/_star.py new file mode 100644 index 0000000..e5e5aba --- /dev/null +++ b/src/evalshift_cli/cli/commands/_star.py @@ -0,0 +1,37 @@ +"""The GitHub-star request printed after a report is rendered. + +``report`` and ``compare`` both end by pointing at a freshly written +``report.html``; that is the moment a person has just got value from the +tool, so it is where the request goes. It is for a person at a terminal +only: CI logs, the GitHub Action's captured output and anything piped stay +free of it. +""" + +from __future__ import annotations + +from typing import Final + +from rich.console import Console + +STAR_URL: Final = "https://github.com/evalshift/evalshift-cli" + + +def print_star_prompt(console: Console) -> None: + """Ask for a GitHub star, on interactive terminals only. + + Args: + console: The command's console. Nothing is printed unless it is a + terminal. + """ + if not console.is_terminal: + return + console.print() + # soft_wrap: a URL folded across lines stops being clickable. + console.print( + f"[bold yellow]★[/bold yellow] Find EvalShift useful? " + f"[bold]A star on GitHub[/bold] helps: [cyan]{STAR_URL}[/cyan]", + soft_wrap=True, + ) + + +__all__ = ["STAR_URL", "print_star_prompt"] diff --git a/src/evalshift_cli/cli/commands/compare.py b/src/evalshift_cli/cli/commands/compare.py index 8080fc4..cb5a8ba 100644 --- a/src/evalshift_cli/cli/commands/compare.py +++ b/src/evalshift_cli/cli/commands/compare.py @@ -33,6 +33,7 @@ from rich.text import Text from evalshift_cli.analysis.statistics import ComparisonResult +from evalshift_cli.cli.commands._star import print_star_prompt from evalshift_cli.cli.commands._suites import ( SUITE_FILENAME, AmbiguousSuiteError, @@ -748,6 +749,7 @@ def _on_progress(ev: ProgressEvent) -> None: console.print(f"[dim]run [/dim] [bold cyan]{run_result.run_id}[/bold cyan]") console.print(f"[dim]suite [/dim] {run_slug}") console.print(f"[dim]report[/dim] {report_result.html_path}") + print_star_prompt(console) if push: try: diff --git a/src/evalshift_cli/cli/commands/report.py b/src/evalshift_cli/cli/commands/report.py index ea81afa..c73045b 100644 --- a/src/evalshift_cli/cli/commands/report.py +++ b/src/evalshift_cli/cli/commands/report.py @@ -15,6 +15,7 @@ import typer from rich.console import Console +from evalshift_cli.cli.commands._star import print_star_prompt from evalshift_cli.cli.commands.doctor import CONFIG_FILENAME from evalshift_cli.config.loader import ConfigError, load_config from evalshift_cli.insights.stage import ensure_insight @@ -159,6 +160,7 @@ def report( console.print(f"[green]✓[/green] {result.html_path}") console.print(f"[green]✓[/green] {result.json_path}") + print_star_prompt(console) if open_browser: webbrowser.open(result.html_path.resolve().as_uri()) diff --git a/tests/unit/test_compare_command.py b/tests/unit/test_compare_command.py index 17f6744..3469a6f 100644 --- a/tests/unit/test_compare_command.py +++ b/tests/unit/test_compare_command.py @@ -290,6 +290,34 @@ def test_all_runs_full_pipeline( assert (runs[0] / "scores.jsonl").exists() assert (runs[0] / "analysis.json").exists() + def test_all_asks_for_a_star_on_a_terminal( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + _scaffold(tmp_path) + _patch_client(monkeypatch) + monkeypatch.chdir(tmp_path) + + result = runner.invoke(app, ["all", "--yes"], env={"TTY_COMPATIBLE": "1"}) + + assert result.exit_code == 0, result.output + assert "github.com/evalshift/evalshift-cli" in result.output + + def test_all_no_star_request_when_piped( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + _scaffold(tmp_path) + _patch_client(monkeypatch) + monkeypatch.chdir(tmp_path) + + result = runner.invoke(app, ["all", "--yes"], env={"TTY_COMPATIBLE": "0"}) + + assert result.exit_code == 0, result.output + assert "github.com/evalshift/evalshift-cli" not in result.output + def test_all_prints_warnings_under_the_pipeline_block( self, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_reports.py b/tests/unit/test_reports.py index c249c63..77e8aaf 100644 --- a/tests/unit/test_reports.py +++ b/tests/unit/test_reports.py @@ -1061,6 +1061,36 @@ def test_writes_html_and_json(self, monkeypatch: pytest.MonkeyPatch, tmp_path: P assert (run_dir / REPORT_HTML_FILENAME).exists() assert (run_dir / REPORT_JSON_FILENAME).exists() + def test_asks_for_a_star_on_a_terminal( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + cwd, run_id = _scaffold_full_run(tmp_path) + (cwd / "evalshift.yaml").write_text( + "prompts:\n - {id: greet, detection: manual, content: hi}\n", + encoding="utf-8", + ) + monkeypatch.chdir(cwd) + result = runner.invoke( + app, ["report", run_id, "--no-insights"], env={"TTY_COMPATIBLE": "1"} + ) + assert result.exit_code == 0, result.stdout + assert "github.com/evalshift/evalshift-cli" in result.stdout + + def test_no_star_request_when_piped( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + cwd, run_id = _scaffold_full_run(tmp_path) + (cwd / "evalshift.yaml").write_text( + "prompts:\n - {id: greet, detection: manual, content: hi}\n", + encoding="utf-8", + ) + monkeypatch.chdir(cwd) + result = runner.invoke( + app, ["report", run_id, "--no-insights"], env={"TTY_COMPATIBLE": "0"} + ) + assert result.exit_code == 0, result.stdout + assert "github.com/evalshift/evalshift-cli" not in result.stdout + def test_missing_analysis_exits_one_with_hint( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/unit/test_star_prompt.py b/tests/unit/test_star_prompt.py new file mode 100644 index 0000000..58af773 --- /dev/null +++ b/tests/unit/test_star_prompt.py @@ -0,0 +1,47 @@ +"""Tests for the GitHub-star request printed after a report is rendered. + +The line is for a person at a terminal. CI logs, the GitHub Action and +anything piped must not carry it, so it keys off ``Console.is_terminal``. +""" + +from __future__ import annotations + +import io + +from rich.console import Console + +from evalshift_cli.cli.commands._star import STAR_URL, print_star_prompt + + +def _console(*, terminal: bool) -> tuple[Console, io.StringIO]: + buf = io.StringIO() + return Console(file=buf, force_terminal=terminal, width=80), buf + + +class TestPrintStarPrompt: + def test_terminal_gets_the_repo_link(self) -> None: + console, buf = _console(terminal=True) + print_star_prompt(console) + assert STAR_URL in buf.getvalue() + + def test_sits_apart_from_the_lines_above(self) -> None: + # It follows the run/suite/report block; a blank line keeps it from + # reading as a fourth row of that block. + console, buf = _console(terminal=True) + print_star_prompt(console) + assert buf.getvalue().startswith("\n") + + def test_piped_output_gets_nothing(self) -> None: + console, buf = _console(terminal=False) + print_star_prompt(console) + assert buf.getvalue() == "" + + def test_link_survives_a_narrow_terminal_on_one_line(self) -> None: + # A folded URL stops being clickable; the line must soft-wrap instead. + buf = io.StringIO() + console = Console(file=buf, force_terminal=True, width=40) + print_star_prompt(console) + assert STAR_URL in buf.getvalue() + + def test_url_points_at_the_cli_repo(self) -> None: + assert STAR_URL == "https://github.com/evalshift/evalshift-cli"