From ffda5af7f09e839db69bc2b726aac17bad6c9159 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 15 Oct 2026 22:26:08 +0200 Subject: [PATCH 1/2] feat(cli): ask for a GitHub star after rendering a report `report` and `compare` end with one line asking for a star, with the repo link. It prints on interactive terminals only, so CI logs, the GitHub Action's output and anything piped stay free of it. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +++ DOCS.md | 1 + docs/getting-started.md | 4 ++ llms-full.txt | 2 + src/evalshift_cli/cli/commands/_star.py | 37 ++++++++++++++++++ src/evalshift_cli/cli/commands/compare.py | 2 + src/evalshift_cli/cli/commands/report.py | 2 + tests/unit/test_compare_command.py | 28 ++++++++++++++ tests/unit/test_reports.py | 30 +++++++++++++++ tests/unit/test_star_prompt.py | 47 +++++++++++++++++++++++ 10 files changed, 159 insertions(+) create mode 100644 src/evalshift_cli/cli/commands/_star.py create mode 100644 tests/unit/test_star_prompt.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4bb1eae..3d11f71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `evalshift report` and `evalshift compare` end with a one-line request to star the + project on GitHub, with the repo link. It prints on interactive terminals only, so CI + logs, the GitHub Action's output and anything piped stay free of it. + ## [1.4.3] - 2026-10-13 ### Fixed diff --git a/DOCS.md b/DOCS.md index ae692e4..0d40158 100644 --- a/DOCS.md +++ b/DOCS.md @@ -901,6 +901,7 @@ Prints a red doctor-style **broken eval harness** row when the *source* model fa `-c/--config` · `--gate ` · `--policy-gate` **`evalshift report `** — render `report.html` + `report.json`. `-c/--config` · `--open` · `--insights/--no-insights` (default on; one extra LLM call — see [Run insights](#run-insights)) +On an interactive terminal, `report` and `compare` end with one line asking for a GitHub star, with the repo link. It is not printed when output is piped or captured (CI, the GitHub Action), so logs and step summaries never carry it. **`evalshift compare`** — full pipeline under one live display: two models, **one** suite. It takes a single `--suite-name`/`--suite`, auto-selects when exactly one suite is wired, and prints a ready-to-run command per suite when several are. It does not run every suite — loop over the names, or use one CI job per suite. Formerly `all`; that name is hidden but still works, since scaffolded `EVALSHIFT.md` files reference it. All `run` flags, plus `--gate` · `--policy-gate` · `--open` · `--push` · `--insights/--no-insights` diff --git a/docs/getting-started.md b/docs/getting-started.md index ab2aafa..80b1ed7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -185,6 +185,10 @@ evalshift report --open (`--from/--to`, `--config`, `--suite`, `--suite-name`, `--yes`, `--resume`, `--gate`, `--policy-gate`, `--open`, `--push`). +When run in a terminal, `report` and `compare` finish with a one-line request +to star [the repo on GitHub](https://github.com/evalshift/evalshift-cli). It is +skipped whenever output is piped or captured, so CI logs stay clean. + One invocation compares two models on **one** suite. It picks the suite for you when `evalshift.yaml` wires exactly one; with several, name it with `--suite-name` (the error lists a ready-to-run command per suite). To cover diff --git a/llms-full.txt b/llms-full.txt index b4c1734..f71cc51 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -215,6 +215,8 @@ evalshift analyze RUN_ID [-c CONFIG] [--gate SEVS] [--policy-gate] evalshift report RUN_ID [-c CONFIG] [--open] [--insights/--no-insights (default on)] -> report.html + report.json (+ insights.json). --no-insights skips the narrative and its one extra LLM call. See "Run insights". + On an interactive terminal only, report and compare end with one GitHub-star request + line (https://github.com/evalshift/evalshift-cli); piped/CI output never carries it. report.html layout: header (run id, source->target, suite, examples/calls, spend) -> verdict / advisory-signal / economics panels -> six-cell run strip (examples, calls, failed/truncated, total cost, latency delta, mean score delta) -> recommendations + diff --git a/src/evalshift_cli/cli/commands/_star.py b/src/evalshift_cli/cli/commands/_star.py new file mode 100644 index 0000000..e5e5aba --- /dev/null +++ b/src/evalshift_cli/cli/commands/_star.py @@ -0,0 +1,37 @@ +"""The GitHub-star request printed after a report is rendered. + +``report`` and ``compare`` both end by pointing at a freshly written +``report.html``; that is the moment a person has just got value from the +tool, so it is where the request goes. It is for a person at a terminal +only: CI logs, the GitHub Action's captured output and anything piped stay +free of it. +""" + +from __future__ import annotations + +from typing import Final + +from rich.console import Console + +STAR_URL: Final = "https://github.com/evalshift/evalshift-cli" + + +def print_star_prompt(console: Console) -> None: + """Ask for a GitHub star, on interactive terminals only. + + Args: + console: The command's console. Nothing is printed unless it is a + terminal. + """ + if not console.is_terminal: + return + console.print() + # soft_wrap: a URL folded across lines stops being clickable. + console.print( + f"[bold yellow]★[/bold yellow] Find EvalShift useful? " + f"[bold]A star on GitHub[/bold] helps: [cyan]{STAR_URL}[/cyan]", + soft_wrap=True, + ) + + +__all__ = ["STAR_URL", "print_star_prompt"] diff --git a/src/evalshift_cli/cli/commands/compare.py b/src/evalshift_cli/cli/commands/compare.py index 8080fc4..cb5a8ba 100644 --- a/src/evalshift_cli/cli/commands/compare.py +++ b/src/evalshift_cli/cli/commands/compare.py @@ -33,6 +33,7 @@ from rich.text import Text from evalshift_cli.analysis.statistics import ComparisonResult +from evalshift_cli.cli.commands._star import print_star_prompt from evalshift_cli.cli.commands._suites import ( SUITE_FILENAME, AmbiguousSuiteError, @@ -748,6 +749,7 @@ def _on_progress(ev: ProgressEvent) -> None: console.print(f"[dim]run [/dim] [bold cyan]{run_result.run_id}[/bold cyan]") console.print(f"[dim]suite [/dim] {run_slug}") console.print(f"[dim]report[/dim] {report_result.html_path}") + print_star_prompt(console) if push: try: diff --git a/src/evalshift_cli/cli/commands/report.py b/src/evalshift_cli/cli/commands/report.py index ea81afa..c73045b 100644 --- a/src/evalshift_cli/cli/commands/report.py +++ b/src/evalshift_cli/cli/commands/report.py @@ -15,6 +15,7 @@ import typer from rich.console import Console +from evalshift_cli.cli.commands._star import print_star_prompt from evalshift_cli.cli.commands.doctor import CONFIG_FILENAME from evalshift_cli.config.loader import ConfigError, load_config from evalshift_cli.insights.stage import ensure_insight @@ -159,6 +160,7 @@ def report( console.print(f"[green]✓[/green] {result.html_path}") console.print(f"[green]✓[/green] {result.json_path}") + print_star_prompt(console) if open_browser: webbrowser.open(result.html_path.resolve().as_uri()) diff --git a/tests/unit/test_compare_command.py b/tests/unit/test_compare_command.py index 17f6744..3469a6f 100644 --- a/tests/unit/test_compare_command.py +++ b/tests/unit/test_compare_command.py @@ -290,6 +290,34 @@ def test_all_runs_full_pipeline( assert (runs[0] / "scores.jsonl").exists() assert (runs[0] / "analysis.json").exists() + def test_all_asks_for_a_star_on_a_terminal( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + _scaffold(tmp_path) + _patch_client(monkeypatch) + monkeypatch.chdir(tmp_path) + + result = runner.invoke(app, ["all", "--yes"], env={"TTY_COMPATIBLE": "1"}) + + assert result.exit_code == 0, result.output + assert "github.com/evalshift/evalshift-cli" in result.output + + def test_all_no_star_request_when_piped( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + _scaffold(tmp_path) + _patch_client(monkeypatch) + monkeypatch.chdir(tmp_path) + + result = runner.invoke(app, ["all", "--yes"], env={"TTY_COMPATIBLE": "0"}) + + assert result.exit_code == 0, result.output + assert "github.com/evalshift/evalshift-cli" not in result.output + def test_all_prints_warnings_under_the_pipeline_block( self, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_reports.py b/tests/unit/test_reports.py index c249c63..77e8aaf 100644 --- a/tests/unit/test_reports.py +++ b/tests/unit/test_reports.py @@ -1061,6 +1061,36 @@ def test_writes_html_and_json(self, monkeypatch: pytest.MonkeyPatch, tmp_path: P assert (run_dir / REPORT_HTML_FILENAME).exists() assert (run_dir / REPORT_JSON_FILENAME).exists() + def test_asks_for_a_star_on_a_terminal( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + cwd, run_id = _scaffold_full_run(tmp_path) + (cwd / "evalshift.yaml").write_text( + "prompts:\n - {id: greet, detection: manual, content: hi}\n", + encoding="utf-8", + ) + monkeypatch.chdir(cwd) + result = runner.invoke( + app, ["report", run_id, "--no-insights"], env={"TTY_COMPATIBLE": "1"} + ) + assert result.exit_code == 0, result.stdout + assert "github.com/evalshift/evalshift-cli" in result.stdout + + def test_no_star_request_when_piped( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ) -> None: + cwd, run_id = _scaffold_full_run(tmp_path) + (cwd / "evalshift.yaml").write_text( + "prompts:\n - {id: greet, detection: manual, content: hi}\n", + encoding="utf-8", + ) + monkeypatch.chdir(cwd) + result = runner.invoke( + app, ["report", run_id, "--no-insights"], env={"TTY_COMPATIBLE": "0"} + ) + assert result.exit_code == 0, result.stdout + assert "github.com/evalshift/evalshift-cli" not in result.stdout + def test_missing_analysis_exits_one_with_hint( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tests/unit/test_star_prompt.py b/tests/unit/test_star_prompt.py new file mode 100644 index 0000000..58af773 --- /dev/null +++ b/tests/unit/test_star_prompt.py @@ -0,0 +1,47 @@ +"""Tests for the GitHub-star request printed after a report is rendered. + +The line is for a person at a terminal. CI logs, the GitHub Action and +anything piped must not carry it, so it keys off ``Console.is_terminal``. +""" + +from __future__ import annotations + +import io + +from rich.console import Console + +from evalshift_cli.cli.commands._star import STAR_URL, print_star_prompt + + +def _console(*, terminal: bool) -> tuple[Console, io.StringIO]: + buf = io.StringIO() + return Console(file=buf, force_terminal=terminal, width=80), buf + + +class TestPrintStarPrompt: + def test_terminal_gets_the_repo_link(self) -> None: + console, buf = _console(terminal=True) + print_star_prompt(console) + assert STAR_URL in buf.getvalue() + + def test_sits_apart_from_the_lines_above(self) -> None: + # It follows the run/suite/report block; a blank line keeps it from + # reading as a fourth row of that block. + console, buf = _console(terminal=True) + print_star_prompt(console) + assert buf.getvalue().startswith("\n") + + def test_piped_output_gets_nothing(self) -> None: + console, buf = _console(terminal=False) + print_star_prompt(console) + assert buf.getvalue() == "" + + def test_link_survives_a_narrow_terminal_on_one_line(self) -> None: + # A folded URL stops being clickable; the line must soft-wrap instead. + buf = io.StringIO() + console = Console(file=buf, force_terminal=True, width=40) + print_star_prompt(console) + assert STAR_URL in buf.getvalue() + + def test_url_points_at_the_cli_repo(self) -> None: + assert STAR_URL == "https://github.com/evalshift/evalshift-cli" From e5d3e197f43615eeef8c33fd98583d07e06aa511 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Thu, 15 Oct 2026 22:32:20 +0200 Subject: [PATCH 2/2] chore(release): 1.5.0 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 ++ DOCS.md | 2 +- llms-full.txt | 2 +- pyproject.toml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d11f71..1e3ecfc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.5.0] - 2026-10-15 + ### Added - `evalshift report` and `evalshift compare` end with a one-line request to star the diff --git a/DOCS.md b/DOCS.md index 0d40158..4222090 100644 --- a/DOCS.md +++ b/DOCS.md @@ -12,7 +12,7 @@ hosted (opt-in) run history, diffs, PR gates The suite is the crux, so the capture SDK is the recommended way to build one: it records real production runs to disk and `evalshift capture sync` promotes them into golden suites. Hand-written suites are fully supported — see [The golden suite](#the-golden-suite). -- **Package name:** `evalshift` · **CLI entry point:** `evalshift` · **version:** 1.4.3 +- **Package name:** `evalshift` · **CLI entry point:** `evalshift` · **version:** 1.5.0 - **Python:** >= 3.11 · **License:** Apache-2.0 · **Status:** stable - **Local-first.** Runs, scores, stats, and reports all happen on your machine under `.evalshift/`. The only network calls are the model API calls you asked for — and, if you opt in, pushes to the hosted service and reads from a capture bucket you own (`captures.store`). - **Four pieces:** CLI (this doc), SDK, GitHub Action, hosted server — each with its own machine-readable reference for AI tools. See [Ecosystem and AI-tool references](#ecosystem-and-ai-tool-references). diff --git a/llms-full.txt b/llms-full.txt index f71cc51..48c440a 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1,7 +1,7 @@ # evalshift (CLI) — complete reference for AI tools Canonical hosted copy: https://www.evalshift.dev/cli-llms-full.txt -Package: evalshift (PyPI) | CLI entry point: evalshift | version: 1.4.3 +Package: evalshift (PyPI) | CLI entry point: evalshift | version: 1.5.0 Python: >=3.11 | license: Apache-2.0 | status: stable Install: pip install evalshift (or: uv pip install evalshift) diff --git a/pyproject.toml b/pyproject.toml index ce2cedd..021a822 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "evalshift" -version = "1.4.3" +version = "1.5.0" description = "Run your prompts on two LLMs and find out, with statistical confidence, what regressed." readme = "README.md" license = "Apache-2.0"