From 160d20d8074edb76e5c7c5b0637923bc7a006678 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Mon, 12 Oct 2026 22:59:10 +0200 Subject: [PATCH 1/2] chore(release): 1.4.2 Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 2 ++ DOCS.md | 2 +- llms-full.txt | 2 +- pyproject.toml | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdf2760..3766589 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.4.2] - 2026-10-12 + ### Fixed - A `semantic` evaluator set to `blocking: true` no longer fails runs whose every row diff --git a/DOCS.md b/DOCS.md index 07432e6..d0d9f41 100644 --- a/DOCS.md +++ b/DOCS.md @@ -12,7 +12,7 @@ hosted (opt-in) run history, diffs, PR gates The suite is the crux, so the capture SDK is the recommended way to build one: it records real production runs to disk and `evalshift capture sync` promotes them into golden suites. Hand-written suites are fully supported — see [The golden suite](#the-golden-suite). -- **Package name:** `evalshift` · **CLI entry point:** `evalshift` · **version:** 1.4.1 +- **Package name:** `evalshift` · **CLI entry point:** `evalshift` · **version:** 1.4.2 - **Python:** >= 3.11 · **License:** Apache-2.0 · **Status:** stable - **Local-first.** Runs, scores, stats, and reports all happen on your machine under `.evalshift/`. The only network calls are the model API calls you asked for — and, if you opt in, pushes to the hosted service and reads from a capture bucket you own (`captures.store`). - **Four pieces:** CLI (this doc), SDK, GitHub Action, hosted server — each with its own machine-readable reference for AI tools. See [Ecosystem and AI-tool references](#ecosystem-and-ai-tool-references). diff --git a/llms-full.txt b/llms-full.txt index 1a19412..708f239 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -1,7 +1,7 @@ # evalshift (CLI) — complete reference for AI tools Canonical hosted copy: https://www.evalshift.dev/cli-llms-full.txt -Package: evalshift (PyPI) | CLI entry point: evalshift | version: 1.4.1 +Package: evalshift (PyPI) | CLI entry point: evalshift | version: 1.4.2 Python: >=3.11 | license: Apache-2.0 | status: stable Install: pip install evalshift (or: uv pip install evalshift) diff --git a/pyproject.toml b/pyproject.toml index eca3bae..6385142 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "evalshift" -version = "1.4.1" +version = "1.4.2" description = "Run your prompts on two LLMs and find out, with statistical confidence, what regressed." readme = "README.md" license = "Apache-2.0" From cbd61824d3253954ea18e64a68dd71f395de296f Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Mon, 12 Oct 2026 23:01:18 +0200 Subject: [PATCH 2/2] docs(evaluators): semantic tolerance in severity, tool-free pairs skipped Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/evaluators.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/evaluators.md b/docs/evaluators.md index 43415f8..474df20 100644 --- a/docs/evaluators.md +++ b/docs/evaluators.md @@ -65,6 +65,16 @@ part of the answer and kept; an empty fenced block counts as an empty output. Whether an output is valid, unfenced JSON is a job for `structural.json_schema` or an `llm_judge` criterion. +**Drift inside `min_similarity` is no change.** Because the source is +pinned at 1.0, any rewording is a small negative delta. A row that stays +within `min_similarity` therefore counts as *equivalent* in the policy +budgets and enters the paired significance test as a delta of 0 — nine +rows at 0.90–0.98 used to test as a `critical` regression and fail a run +whose every budget passed. Only flagged rows (`SEMANTIC_REGRESSION`) carry +their full drop into severity. `delta_avg_score` still shows the raw +drift, and the comparison notes how many rows were within the threshold. +See [methodology](methodology.md#drift-inside-min_similarity-is-tested-as-no-change). + Use when: * You don't have a clean structural check. * You want to detect "wandered off" outputs that still look fine @@ -205,7 +215,10 @@ refusal changes scored separately. You rarely wire the first two by hand: `evalshift capture sync` writes `tool_selection` and `tool_arguments` into each suite's own `suites:` entry based on what that suite's captures contain, so a tool-free suite gets no -tool evaluators at all. See +tool evaluators at all. A tool evaluator it inherits from the top level — +`trace_invariants` included — skips every pair that was offered no tools: +no row, no coverage entry, no effect on the verdict, and `evaluate` (or +`compare`, under its verdict) warns that it checked nothing. See [Agent migrations](agents.md#configuration) and [Configuration → per-suite evaluators](configuration.md#per-suite-evaluators).