Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- `evalshift report` and `evalshift compare` end with a one-line request to star the
project on GitHub, with the repo link. It prints on interactive terminals only, so CI
logs, the GitHub Action's output and anything piped stay free of it.

## [1.4.3] - 2026-10-13

### Fixed
Expand Down
1 change: 1 addition & 0 deletions DOCS.md
Original file line number Diff line number Diff line change
Expand Up @@ -901,6 +901,7 @@ Prints a red doctor-style **broken eval harness** row when the *source* model fa
`-c/--config` · `--gate <severities>` · `--policy-gate`

**`evalshift report <run-id>`** — render `report.html` + `report.json`. `-c/--config` · `--open` · `--insights/--no-insights` (default on; one extra LLM call — see [Run insights](#run-insights))
On an interactive terminal, `report` and `compare` end with one line asking for a GitHub star, with the repo link. It is not printed when output is piped or captured (CI, the GitHub Action), so logs and step summaries never carry it.

**`evalshift compare`** — full pipeline under one live display: two models, **one** suite. It takes a single `--suite-name`/`--suite`, auto-selects when exactly one suite is wired, and prints a ready-to-run command per suite when several are. It does not run every suite — loop over the names, or use one CI job per suite. Formerly `all`; that name is hidden but still works, since scaffolded `EVALSHIFT.md` files reference it.
All `run` flags, plus `--gate` · `--policy-gate` · `--open` · `--push` · `--insights/--no-insights`
Expand Down
4 changes: 4 additions & 0 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,10 @@ evalshift report <run-id> --open
(`--from/--to`, `--config`, `--suite`, `--suite-name`, `--yes`, `--resume`,
`--gate`, `--policy-gate`, `--open`, `--push`).

When run in a terminal, `report` and `compare` finish with a one-line request
to star [the repo on GitHub](https://github.com/evalshift/evalshift-cli). It is
skipped whenever output is piped or captured, so CI logs stay clean.

One invocation compares two models on **one** suite. It picks the suite for
you when `evalshift.yaml` wires exactly one; with several, name it with
`--suite-name` (the error lists a ready-to-run command per suite). To cover
Expand Down
2 changes: 2 additions & 0 deletions llms-full.txt
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,8 @@ evalshift analyze RUN_ID [-c CONFIG] [--gate SEVS] [--policy-gate]
evalshift report RUN_ID [-c CONFIG] [--open] [--insights/--no-insights (default on)]
-> report.html + report.json (+ insights.json). --no-insights skips the narrative and its
one extra LLM call. See "Run insights".
On an interactive terminal only, report and compare end with one GitHub-star request
line (https://github.com/evalshift/evalshift-cli); piped/CI output never carries it.
report.html layout: header (run id, source->target, suite, examples/calls, spend) ->
verdict / advisory-signal / economics panels -> six-cell run strip (examples, calls,
failed/truncated, total cost, latency delta, mean score delta) -> recommendations +
Expand Down
37 changes: 37 additions & 0 deletions src/evalshift_cli/cli/commands/_star.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
"""The GitHub-star request printed after a report is rendered.

``report`` and ``compare`` both end by pointing at a freshly written
``report.html``; that is the moment a person has just got value from the
tool, so it is where the request goes. It is for a person at a terminal
only: CI logs, the GitHub Action's captured output and anything piped stay
free of it.
"""

from __future__ import annotations

from typing import Final

from rich.console import Console

STAR_URL: Final = "https://github.com/evalshift/evalshift-cli"


def print_star_prompt(console: Console) -> None:
"""Ask for a GitHub star, on interactive terminals only.

Args:
console: The command's console. Nothing is printed unless it is a
terminal.
"""
if not console.is_terminal:
return
console.print()
# soft_wrap: a URL folded across lines stops being clickable.
console.print(
f"[bold yellow]★[/bold yellow] Find EvalShift useful? "
f"[bold]A star on GitHub[/bold] helps: [cyan]{STAR_URL}[/cyan]",
soft_wrap=True,
)


__all__ = ["STAR_URL", "print_star_prompt"]
2 changes: 2 additions & 0 deletions src/evalshift_cli/cli/commands/compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from rich.text import Text

from evalshift_cli.analysis.statistics import ComparisonResult
from evalshift_cli.cli.commands._star import print_star_prompt
from evalshift_cli.cli.commands._suites import (
SUITE_FILENAME,
AmbiguousSuiteError,
Expand Down Expand Up @@ -748,6 +749,7 @@ def _on_progress(ev: ProgressEvent) -> None:
console.print(f"[dim]run [/dim] [bold cyan]{run_result.run_id}[/bold cyan]")
console.print(f"[dim]suite [/dim] {run_slug}")
console.print(f"[dim]report[/dim] {report_result.html_path}")
print_star_prompt(console)

if push:
try:
Expand Down
2 changes: 2 additions & 0 deletions src/evalshift_cli/cli/commands/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import typer
from rich.console import Console

from evalshift_cli.cli.commands._star import print_star_prompt
from evalshift_cli.cli.commands.doctor import CONFIG_FILENAME
from evalshift_cli.config.loader import ConfigError, load_config
from evalshift_cli.insights.stage import ensure_insight
Expand Down Expand Up @@ -159,6 +160,7 @@ def report(

console.print(f"[green]✓[/green] {result.html_path}")
console.print(f"[green]✓[/green] {result.json_path}")
print_star_prompt(console)

if open_browser:
webbrowser.open(result.html_path.resolve().as_uri())
Expand Down
28 changes: 28 additions & 0 deletions tests/unit/test_compare_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,34 @@ def test_all_runs_full_pipeline(
assert (runs[0] / "scores.jsonl").exists()
assert (runs[0] / "analysis.json").exists()

def test_all_asks_for_a_star_on_a_terminal(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_scaffold(tmp_path)
_patch_client(monkeypatch)
monkeypatch.chdir(tmp_path)

result = runner.invoke(app, ["all", "--yes"], env={"TTY_COMPATIBLE": "1"})

assert result.exit_code == 0, result.output
assert "github.com/evalshift/evalshift-cli" in result.output

def test_all_no_star_request_when_piped(
self,
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_scaffold(tmp_path)
_patch_client(monkeypatch)
monkeypatch.chdir(tmp_path)

result = runner.invoke(app, ["all", "--yes"], env={"TTY_COMPATIBLE": "0"})

assert result.exit_code == 0, result.output
assert "github.com/evalshift/evalshift-cli" not in result.output

def test_all_prints_warnings_under_the_pipeline_block(
self,
monkeypatch: pytest.MonkeyPatch,
Expand Down
30 changes: 30 additions & 0 deletions tests/unit/test_reports.py
Original file line number Diff line number Diff line change
Expand Up @@ -1061,6 +1061,36 @@ def test_writes_html_and_json(self, monkeypatch: pytest.MonkeyPatch, tmp_path: P
assert (run_dir / REPORT_HTML_FILENAME).exists()
assert (run_dir / REPORT_JSON_FILENAME).exists()

def test_asks_for_a_star_on_a_terminal(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
cwd, run_id = _scaffold_full_run(tmp_path)
(cwd / "evalshift.yaml").write_text(
"prompts:\n - {id: greet, detection: manual, content: hi}\n",
encoding="utf-8",
)
monkeypatch.chdir(cwd)
result = runner.invoke(
app, ["report", run_id, "--no-insights"], env={"TTY_COMPATIBLE": "1"}
)
assert result.exit_code == 0, result.stdout
assert "github.com/evalshift/evalshift-cli" in result.stdout

def test_no_star_request_when_piped(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
cwd, run_id = _scaffold_full_run(tmp_path)
(cwd / "evalshift.yaml").write_text(
"prompts:\n - {id: greet, detection: manual, content: hi}\n",
encoding="utf-8",
)
monkeypatch.chdir(cwd)
result = runner.invoke(
app, ["report", run_id, "--no-insights"], env={"TTY_COMPATIBLE": "0"}
)
assert result.exit_code == 0, result.stdout
assert "github.com/evalshift/evalshift-cli" not in result.stdout

def test_missing_analysis_exits_one_with_hint(
self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
Expand Down
47 changes: 47 additions & 0 deletions tests/unit/test_star_prompt.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
"""Tests for the GitHub-star request printed after a report is rendered.

The line is for a person at a terminal. CI logs, the GitHub Action and
anything piped must not carry it, so it keys off ``Console.is_terminal``.
"""

from __future__ import annotations

import io

from rich.console import Console

from evalshift_cli.cli.commands._star import STAR_URL, print_star_prompt


def _console(*, terminal: bool) -> tuple[Console, io.StringIO]:
buf = io.StringIO()
return Console(file=buf, force_terminal=terminal, width=80), buf


class TestPrintStarPrompt:
def test_terminal_gets_the_repo_link(self) -> None:
console, buf = _console(terminal=True)
print_star_prompt(console)
assert STAR_URL in buf.getvalue()

def test_sits_apart_from_the_lines_above(self) -> None:
# It follows the run/suite/report block; a blank line keeps it from
# reading as a fourth row of that block.
console, buf = _console(terminal=True)
print_star_prompt(console)
assert buf.getvalue().startswith("\n")

def test_piped_output_gets_nothing(self) -> None:
console, buf = _console(terminal=False)
print_star_prompt(console)
assert buf.getvalue() == ""

def test_link_survives_a_narrow_terminal_on_one_line(self) -> None:
# A folded URL stops being clickable; the line must soft-wrap instead.
buf = io.StringIO()
console = Console(file=buf, force_terminal=True, width=40)
print_star_prompt(console)
assert STAR_URL in buf.getvalue()

def test_url_points_at_the_cli_repo(self) -> None:
assert STAR_URL == "https://github.com/evalshift/evalshift-cli"
Loading