From 45bdfea9f75995c623ae8ec6e690a5078456c672 Mon Sep 17 00:00:00 2001 From: Jiaqi Liu <32996169+jiaq-liu@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:08:58 -0700 Subject: [PATCH] update analyzer docs --- CHANGELOG.md | 11 + CONTRIBUTING.md | 4 +- docs/analyzers.md | 347 ++++++++++++++++--- evalrx/analyzers/__init__.py | 27 +- tests/test_analyzers/test_analyzer_docs.py | 83 +++++ tools/gen_analyzer_docs.py | 368 +++++++++++++++++++++ 6 files changed, 783 insertions(+), 57 deletions(-) create mode 100644 tests/test_analyzers/test_analyzer_docs.py create mode 100644 tools/gen_analyzer_docs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bc3c190..8d6ac888 100755 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,17 @@ Versions follow [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Docs — analyzer zoo regenerated from the registries + +`docs/analyzers.md` listed 29 of the 46 registered analyzers and 16 of the 57 +model specs, and several listed rows had drifted from the code (`rollout` is +registered as `attention_rollout`; `linear_probe` is implemented, not a stub; +`mm_shap` and `relative_attention` cover more than images; two citations pointed +at the wrong paper). The page is now organised by `evalrx/analyzers/` subpackage +and by model group, and its tables are generated by `tools/gen_analyzer_docs.py` +(`--check` exits 1 when the page is stale; +`tests/test_analyzers/test_analyzer_docs.py` runs the same check). + ## [0.1.2] — 2026-09-11 First release under the `evalrx` name (formerly published as `evalvitals` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 117242f4..d6bf1a2f 100755 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,7 @@ mypy evalrx --ignore-missing-imports 2. If white-box internals are needed, add `evalrx/models/whitebox/.py` — declare `capabilities` + implement `forward(inputs, capture) -> Trace` — and register with `@register_model`. 3. The `compose()` function in `evalrx/models/compose.py` wires everything together; no changes needed there for a new spec-only model. 4. Write a unit test in `tests/test_models/` using `FakeModel` from `tests/conftest.py` (no GPU needed). +5. Run `python tools/gen_analyzer_docs.py` to add the spec to the model tables in `docs/analyzers.md`. See `docs/extending.md` for a full walkthrough. @@ -40,8 +41,9 @@ See `docs/extending.md` for a full walkthrough. 4. Decorate with `@register_analyzer("")`. 5. Import the new module in `evalrx/analyzers//__init__.py`. 6. Write a unit test using `FakeModel`. +7. Run `python tools/gen_analyzer_docs.py` and fill in the new row's name and Paper in `docs/analyzers.md`. -The new analyzer is instantly discoverable via `registry.analyzers.names_compatible_with(model)` — no other changes needed. +The new analyzer is instantly discoverable via `registry.analyzers.names_compatible_with(model)` — no other code changes needed. ## Pull requests diff --git a/docs/analyzers.md b/docs/analyzers.md index 601ffd52..6aa064d3 100644 --- a/docs/analyzers.md +++ b/docs/analyzers.md @@ -1,62 +1,319 @@ # Analyzer Zoo & Model Registry -Reference tables for what's implemented. Discover the same information at -runtime via `evalrx.list_specs()` and `evalrx.registry.analyzers` + + +Reference tables for every registered analyzer and model spec. Discover the same +information at runtime via `evalrx.list_specs()` and `evalrx.registry.analyzers` (see [Discovery](quickstart.md#discovery)). ## Analyzers -| Analyzer | Key | Capability | Modality | Paper | Status | +Analyzers are grouped the way the code is: one section per subpackage of +`evalrx/analyzers/`. Each analyzer is keyed by the capability it needs from the +model, not by a black-box / white-box label. + + +| Category | Package | Analyzers | Stubs | +|---|---|---|---| +| [Perturbation](#perturbation) | `perturbation/` | 9 | 0 | +| [Uncertainty](#uncertainty) | `uncertainty/` | 6 | 0 | +| [Hallucination](#hallucination) | `hallucination/` | 5 | 2 | +| [Attention](#attention) | `attention/` | 4 | 0 | +| [Attribution](#attribution) | `attribution/` | 2 | 2 | +| [Lens](#lens) | `lens/` | 3 | 1 | +| [Patching](#patching) | `patching/` | 1 | 1 | +| [Geometry](#geometry) | `geometry/` | 2 | 0 | +| [Reasoning](#reasoning) | `reasoning/` | 7 | 0 | +| [Agent](#agent) | `agent/` | 7 | 0 | +| **Total** | | **46** | **6** | + + +How to read the columns: + +- **Key**: the registered name, as in `evalrx.registry.analyzers.get(key)`. +- **Requires**: capabilities the model must provide (`Analyzer.requires`). + `trajectories` means the batch must carry agent trajectories + (`requires_trajectories`). A dash means nothing is required of the model: the + analyzer reads stored outputs, or re-runs cases through an injected runner or judge. +- **Modality**: `applies_to_modalities`. It is matched by intersection with the + model's modalities, so `text` covers every text-capable model, VLMs and omni + models included. **Bold** slots must also be filled in the batch + (`requires_modalities`); selection drops the analyzer when none of them is. +- **Status**: ✓ = implemented; *stub* = registered, but `_run` raises + `NotImplementedError` (Stage 2). + +### Perturbation + +`evalrx/analyzers/perturbation/`: re-query the model on masked, ablated or +rewritten inputs and measure what moves. Black-box (`GENERATE` / `LOGPROBS`), so +these run on API models too; the cost is the number of extra model calls per case. + + +| Analyzer | Key | Requires | Modality | Paper | Status | |---|---|---|---|---|---| -| Attention summary | `attention` | `ATTENTION` | text + image | — | ✓ | -| Attention rollout | `rollout` | `ATTENTION` | text + image | Abnar & Zuidema, 2020 | ✓ | -| Attention sink | `attention_sink` | `ATTENTION` | text + image | [Gu et al. 2023](https://arxiv.org/abs/2309.17453) | ✓ | -| Relative attention | `relative_attention` | `ATTENTION` | image (VLM) | [arXiv:2502.17422](https://arxiv.org/abs/2502.17422) | ✓ | | RISE | `rise` | `GENERATE` | text | [Petsiuk et al. 2018](https://arxiv.org/abs/1806.07421) | ✓ | -| MM-SHAP | `mm_shap` | `LOGPROBS` | image (VLM) | [arXiv:2212.08158](https://arxiv.org/abs/2212.08158) | ✓ | -| VL-SHAP | `vl_shap` | `LOGPROBS` | image (VLM) | [arXiv:2212.08158](https://arxiv.org/abs/2212.08158) | ✓ | -| Token entropy | `token_entropy` | `LOGITS` | text + image | — | ✓ | +| VL-SHAP | `vl_shap` | `LOGPROBS` | **image** | [Lundberg & Lee 2017](https://arxiv.org/abs/1705.07874); [Parcalabescu & Frank 2022](https://arxiv.org/abs/2212.08158) | ✓ | +| MM-SHAP | `mm_shap` | `LOGPROBS` | text + **image** + **audio** + **video** | [Parcalabescu & Frank 2022](https://arxiv.org/abs/2212.08158) | ✓ | +| Modality ablation | `modality_ablation` | `GENERATE` | text + **image** + **audio** + **video** | — | ✓ | +| Prompt contrast | `prompt_contrast` | `GENERATE` | text + image | — | ✓ | +| MC format sensitivity | `format_sensitivity` | `GENERATE` | text + image | [Zheng et al. 2024](https://arxiv.org/abs/2309.03882) | ✓ | +| CoT faithfulness | `cot_faithfulness` | `GENERATE` | text + image | [Lanham et al. 2023](https://arxiv.org/abs/2307.13702) | ✓ | +| Context Shapley | `context_shap` | `GENERATE` | text + image | [Goldshmidt & Horovicz 2024](https://arxiv.org/abs/2407.10114); [Cohen-Wang et al. 2024](https://arxiv.org/abs/2409.00729) | ✓ | +| Perturbation battery | `perturbation_battery` | `GENERATE` | text + image | [Mirzadeh et al. 2025](https://arxiv.org/abs/2410.05229) | ✓ | + + +### Uncertainty + +`evalrx/analyzers/uncertainty/`: cheap signals of how sure the model was, read +from logits or logprobs, from repeated samples, or from what the model says about +itself, and checked against PASS/FAIL labels (calibration, coverage gap). + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| +| Token entropy | `token_entropy` | `LOGITS` | text | [Malinin & Gales 2021](https://arxiv.org/abs/2002.07650) | ✓ | | Logprob entropy | `logprob_entropy` | `LOGPROBS` | text + image | [Kadavath et al. 2022](https://arxiv.org/abs/2207.05221) | ✓ | | Self-consistency | `self_consistency` | `GENERATE` | text + image | [Wang et al. 2023](https://arxiv.org/abs/2203.11171) | ✓ | -| Verbalized confidence | `verbalized_confidence` | `GENERATE` | text + image | — | ✓ | -| POPE | `pope` | `GENERATE` | image (VLM) | [arXiv:2305.10355](https://arxiv.org/abs/2305.10355) | ✓ | -| CHAIR | `chair` | `GENERATE` | image (VLM) | [arXiv:1809.02156](https://arxiv.org/abs/1809.02156) | ✓ | -| OPERA | `opera` | `ATTENTION` | image (VLM) | [arXiv:2311.17911](https://arxiv.org/abs/2311.17911) | stub | -| VCD | `vcd` | `LOGITS` | image (VLM) | [arXiv:2311.16922](https://arxiv.org/abs/2311.16922) | stub | +| Verbalized confidence | `verbalized_confidence` | `GENERATE` | text + image | [Lin et al. 2022](https://arxiv.org/abs/2205.14334) | ✓ | +| Calibration | `calibration` | `GENERATE` | text + image | [Guo et al. 2017](https://arxiv.org/abs/1706.04599) | ✓ | +| Coverage–verification gap | `coverage_verification_gap` | `GENERATE` | text + image | [Brown et al. 2024](https://arxiv.org/abs/2407.21787) | ✓ | + + +### Hallucination + +`evalrx/analyzers/hallucination/`: the dominant MLLM failure mode. Black-box +object probes (POPE, CHAIR) and sampled-passage consistency for text (SelfCheck), +plus decoding-side white-box diagnostics (OPERA, VCD). + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| +| POPE | `pope` | `GENERATE` | **image** | [Li et al. 2023](https://arxiv.org/abs/2305.10355) | ✓ | +| CHAIR | `chair` | `GENERATE` | **image** | [Rohrbach et al. 2018](https://arxiv.org/abs/1809.02156) | ✓ | +| SelfCheck consistency | `selfcheck_consistency` | `GENERATE` | text + image | [Manakul et al. 2023](https://arxiv.org/abs/2303.08896) | ✓ | +| OPERA | `opera` | `ATTENTION` | **image** | [Huang et al. 2024](https://arxiv.org/abs/2311.17911) | stub | +| VCD | `vcd` | `LOGITS` | **image** | [Leng et al. 2024](https://arxiv.org/abs/2311.16922) | stub | + + +### Attention + +`evalrx/analyzers/attention/`: read the attention maps of a white-box model +(`ATTENTION`, HF eager attention). The subpackage imports torch at load time, so +on the light (API-only) install it is skipped and these analyzers are not offered. + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| +| Attention summary | `attention` | `ATTENTION` | text | — | ✓ | +| Attention rollout | `attention_rollout` | `ATTENTION` | text + image | [Abnar & Zuidema 2020](https://arxiv.org/abs/2005.00928) | ✓ | +| Attention sink | `attention_sink` | `ATTENTION` | text + image | [Xiao et al. 2024](https://arxiv.org/abs/2309.17453); [Gu et al. 2025](https://arxiv.org/abs/2410.10781) | ✓ | +| Relative attention | `relative_attention` | `ATTENTION` | **image** + **video** | ["MLLMs Know Where to Look", 2025](https://arxiv.org/abs/2502.17422) | ✓ | + + +### Attribution + +`evalrx/analyzers/attribution/`: gradient-based saliency (`GRADIENTS`, a backward +pass through a white-box model). + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| +| Grad-CAM | `gradcam` | `GRADIENTS` | image | [Selvaraju et al. 2017](https://arxiv.org/abs/1610.02391) | stub | +| Generic attention explainability | `generic_attention` | `ATTENTION` + `GRADIENTS` | text + image | [Chefer et al. 2021](https://arxiv.org/abs/2103.15679) | stub | + + +### Lens + +`evalrx/analyzers/lens/`: project intermediate hidden states through the +unembedding (`HIDDEN_STATES`) to see at which layer a prediction forms, and where +premature layers disagree with the final one. + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| | Logit lens | `logit_lens` | `HIDDEN_STATES` | text + image | [nostalgebraist 2020](https://www.lesswrong.com/posts/AcKRB8wDpdaN6v6ru/interpreting-gpt-the-logit-lens) | ✓ | | Tuned lens | `tuned_lens` | `HIDDEN_STATES` | text + image | [Belrose et al. 2023](https://arxiv.org/abs/2303.08112) | stub | -| Grad-CAM | `gradcam` | `GRADIENTS` | image (VLM) | [Selvaraju et al. 2017](https://arxiv.org/abs/1610.02391) | stub | -| Generic attn explain | `generic_attention` | `ATTENTION + GRADIENTS` | text + image | [Chefer et al. 2021](https://arxiv.org/abs/2103.15679) | stub | -| Linear CKA | `cka` | `HIDDEN_STATES` | text + image | [Kornblith et al. 2019](https://arxiv.org/abs/1905.00414) | ✓ | -| Linear probe | `linear_probe` | `HIDDEN_STATES` | text + image | — | stub | +| Layer contrast | `layer_contrast` | `HIDDEN_STATES` | text + image | [Chuang et al. 2024](https://arxiv.org/abs/2309.03883); [Wang et al. 2025](https://arxiv.org/abs/2410.11779) | ✓ | + + +### Patching + +`evalrx/analyzers/patching/`: causal interventions. Write a clean run's +activations into a corrupted run to localise where information is used +(`HIDDEN_STATES`, read and write hooks). + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| | Causal trace | `causal_trace` | `HIDDEN_STATES` | text + image | [Meng et al. 2022](https://arxiv.org/abs/2202.05262) | stub | -| Loop detect | `loop_detect` | Trajectory | agent | — | ✓ | -| Ignored obs | `ignored_obs` | Trajectory | agent | — | ✓ | -| First-error judge | `first_error_judge` | Trajectory | agent | [Zhang et al. 2024](https://arxiv.org/abs/2406.14855) | ✓ | -| Trajectory rubric | `trajectory_rubric` | Trajectory + judge | agent | [MAST, Cemri et al. 2025](https://arxiv.org/abs/2503.13657) | ✓ | -| Counterfactual | `counterfactual` | Trajectory | agent | Pearl 2000 | ✓ | -| Reliability probe | `reliability_probe` | re-runs (injected) | agent | pass@k vs pass^k | ✓ | -| Tool Shapley | `tool_shap` | re-runs (injected) | agent | [AgentSHAP / TokenSHAP](https://arxiv.org/abs/2407.10114) | ✓ | + + +### Geometry + +`evalrx/analyzers/geometry/`: representation geometry over hidden states +(`HIDDEN_STATES`). Cross-modal CKA readings were validated on CLIP-style two-tower +encoders; on a decoder-only MLLM, read them as within-stream layer similarity. + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| +| Linear CKA | `cka` | `HIDDEN_STATES` | text + image | [Kornblith et al. 2019](https://arxiv.org/abs/1905.00414) | ✓ | +| Linear probe | `linear_probe` | `HIDDEN_STATES` | text + image | [Alain & Bengio 2016](https://arxiv.org/abs/1610.01644) | ✓ | + + +### Reasoning + +`evalrx/analyzers/reasoning/`: text-chain diagnostics, all black-box. Read the +two hygiene probes first: `answer_extraction_audit` (is the FAIL label real, or +did the grader miss the answer?) and `termination_audit` (did the generation stop +cleanly, or hit the token budget or a repetition loop?). Both produce confounds +that mimic every other probe's "bad" column. The rest decompose *how* a chain +failed. + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| +| Answer-extraction audit | `answer_extraction_audit` | — | text + image | [Lightman et al. 2024](https://arxiv.org/abs/2305.20050) | ✓ | +| Termination audit | `termination_audit` | `GENERATE` | text + image | [Holtzman et al. 2020](https://arxiv.org/abs/1904.09751) | ✓ | +| Arithmetic audit | `arith_audit` | — | text + image | [Cobbe et al. 2021](https://arxiv.org/abs/2110.14168) | ✓ | +| Self-repair | `self_repair` | `GENERATE` | text + image | [Huang et al. 2024](https://arxiv.org/abs/2310.01798) | ✓ | +| Step rollout value | `step_rollout_value` | `GENERATE` | text + image | [Wang et al. 2024 (Math-Shepherd)](https://arxiv.org/abs/2312.08935) | ✓ | +| Knowledge / reasoning split | `knowledge_reasoning_split` | `GENERATE` | text + image | [Press et al. 2023](https://arxiv.org/abs/2210.03350) | ✓ | +| Contamination | `contamination_score` | `GENERATE` | text + image | [Golchin & Surdeanu 2024](https://arxiv.org/abs/2308.08493) | ✓ | + + +### Agent + +`evalrx/analyzers/agent/`: read multi-step agent runs (`Trajectory`) rather than +single turns. Ordered cheapest first: deterministic heuristics, then LLM judges, +then probes that re-run the agent through an injected runner (`counterfactual`, +`reliability_probe`, `tool_shap`). Trajectories are usually loaded from disk with +`model=None`, which is why most of these require nothing of the model. + + +| Analyzer | Key | Requires | Modality | Paper | Status | +|---|---|---|---|---|---| +| Loop detect | `loop_detect` | trajectories | text + image | [MAST, Cemri et al. 2025](https://arxiv.org/abs/2503.13657) | ✓ | +| Ignored observation | `ignored_obs` | trajectories | text + image | [MAST, Cemri et al. 2025](https://arxiv.org/abs/2503.13657) | ✓ | +| First-error judge | `first_error_judge` | trajectories | text + image | [Who&When, Zhang et al. 2025](https://arxiv.org/abs/2505.00212) | ✓ | +| Trajectory rubric | `trajectory_rubric` | trajectories | text + image | [MAST, Cemri et al. 2025](https://arxiv.org/abs/2503.13657) | ✓ | +| Counterfactual replay | `counterfactual` | `TOOL_CALLS` + trajectories | text + image | Pearl 2000 | ✓ | +| Reliability probe | `reliability_probe` | — | text + image | pass@k vs pass^k | ✓ | +| Tool Shapley | `tool_shap` | trajectories | text + image | [AgentSHAP / TokenSHAP](https://arxiv.org/abs/2407.10114) | ✓ | + ## Model Registry -From `list_specs()`: +From `list_specs()` (`evalrx/specs.py`). **Modalities** are derived from the +components a spec declares (vision tower → image, audio tower → audio, video) and +are what analyzer selection matches on. **Traits** come from the spec flags: +`MoE` (`is_moe`), `reasoning` (`is_reasoning`), `tool calling` (`tool_calling`: the +chat template renders tools, which grants `TOOL_CALLS` on the local backend) and +`JAX` (also runnable on the `jax_local` backend). -| Key | Family | Type | Notes | -|---|---|---|---| -| `qwen2.5-7b-instruct` | Qwen2 | LLM | reference checkpoint | -| `qwen3-4b` | Qwen3 | LLM | reasoning, small smoke-test | -| `qwen3-8b` | Qwen3 | LLM | reasoning | -| `qwen3-30b-a3b` | Qwen3-MoE | LLM | MoE | -| `deepseek-v3` | DeepSeek-V3 | LLM | MoE + MLA | -| `llama-3.1-8b-instruct` | Llama | LLM | — | -| `gemma-3-1b-it` | Gemma3 | LLM | text-only | -| `qwen3-vl-4b-instruct` | Qwen3-VL | VLM | small smoke-test | -| `qwen2.5-vl-7b-instruct` | Qwen2.5-VL | VLM | reference for relative-attention | -| `qwen2-vl-7b-instruct` | Qwen2-VL | VLM | — | -| `qwen3-vl-8b-instruct` | Qwen3-VL | VLM | — | -| `glm-4.5v` | GLM-MoE | VLM | MoE + reasoning, 106B | -| `glm-4.1v-9b-thinking` | GLM | VLM | reasoning | -| `kimi-vl-a3b-thinking` | Kimi-VL | VLM | MoE + MLA + reasoning | -| `llama-4-scout` | Llama4 | VLM | MoE | -| `step-1o-vision` | Step | VLM | API-only | +### Text-only LLMs + + +| Key | Family | Modalities | Traits | Notes | +|---|---|---|---|---| +| `qwen2.5-7b-instruct` | `qwen2` | text | tool calling | reference checkpoint | +| `qwen2.5-14b-instruct` | `qwen2` | text | tool calling | — | +| `qwen2.5-32b-instruct` | `qwen2` | text | tool calling | — | +| `qwen2.5-72b-instruct` | `qwen2` | text | tool calling | — | +| `qwen3-4b` | `qwen3` | text | reasoning, tool calling | small smoke-test checkpoint; thinking off | +| `qwen3-8b` | `qwen3` | text | reasoning, tool calling | — | +| `qwen3-14b` | `qwen3` | text | reasoning, tool calling | — | +| `qwen3-32b` | `qwen3` | text | reasoning, tool calling | — | +| `qwen3-30b-a3b` | `qwen3_moe` | text | MoE, tool calling | — | +| `qwen3-235b-a22b` | `qwen3_moe` | text | MoE, reasoning, tool calling | multi-GPU / FP8 | +| `qwen3.5-2b` | `qwen3_5` | text | reasoning, tool calling | hybrid linear / full attention stack | +| `qwen3.5-4b` | `qwen3_5` | text | reasoning, tool calling | hybrid linear / full attention stack | +| `qwen3.5-9b` | `qwen3_5` | text | reasoning, tool calling | hybrid linear / full attention stack | +| `deepseek-v3` | `deepseek_v3` | text | MoE, tool calling | MLA | +| `llama-3.1-8b-instruct` | `llama` | text | tool calling | also loads in TransformerLens | +| `gemma-3-1b-it` | `gemma3` | text | — | tied embeddings | +| `nemotron-3-nano-4b` | `nemotron_h` | text | reasoning | hybrid Mamba / attention stack | +| `nemotron-3-nano-4b-fp8` | `nemotron_h` | text | reasoning | endpoint only (FP8 export) | + + +### Vision-language models + + +| Key | Family | Modalities | Traits | Notes | +|---|---|---|---|---| +| `qwen2-vl-7b-instruct` | `qwen2_vl` | text + image | tool calling | — | +| `qwen2.5-vl-3b-instruct` | `qwen2_5_vl` | text + image | tool calling | — | +| `qwen2.5-vl-7b-instruct` | `qwen2_5_vl` | text + image | tool calling | reference for relative attention | +| `qwen2.5-vl-32b-instruct` | `qwen2_5_vl` | text + image | tool calling | — | +| `qwen2.5-vl-72b-instruct` | `qwen2_5_vl` | text + image | tool calling | — | +| `qwen3-vl-2b-instruct` | `qwen3_vl` | text + image | tool calling | — | +| `qwen3-vl-4b-instruct` | `qwen3_vl` | text + image | tool calling | small smoke-test checkpoint | +| `qwen3-vl-8b-instruct` | `qwen3_vl` | text + image | tool calling | — | +| `qwen3-vl-30b-a3b-instruct` | `qwen3_vl_moe` | text + image | MoE, tool calling | multi-GPU / FP8 | +| `qwen3-vl-235b-a22b-instruct` | `qwen3_vl_moe` | text + image | MoE, tool calling | multi-GPU / FP8 | +| `qwen3.5-2b-vl` | `qwen3_5` | text + image | reasoning, tool calling | hybrid linear / full attention stack | +| `qwen3.5-4b-vl` | `qwen3_5` | text + image | reasoning, tool calling | hybrid linear / full attention stack | +| `qwen3.5-9b-vl` | `qwen3_5` | text + image | reasoning, tool calling | hybrid linear / full attention stack | +| `glm-4.1v-9b-thinking` | `glm4v` | text + image | reasoning, tool calling | — | +| `glm-4.5v` | `glm4v_moe` | text + image | MoE, reasoning, tool calling | 106B; multi-GPU / FP8 | +| `kimi-vl-a3b-thinking` | `kimi_vl` | text + image | MoE, reasoning | MLA; remote code | +| `llama-4-scout` | `llama4` | text + image | MoE | — | +| `llava-1.5-7b-hf` | `llava` | text + image | — | reference architecture for ViCrop | +| `instructblip-vicuna-7b` | `instructblip` | text + image | — | reference Q-Former architecture | + + +### Audio and omni models + + +| Key | Family | Modalities | Traits | Notes | +|---|---|---|---|---| +| `qwen2-audio-7b-instruct` | `qwen2_audio` | text + audio | — | audio in, text out | +| `qwen2.5-omni-7b` | `qwen2_5_omni` | text + image + audio + video | tool calling | — | +| `qwen3-omni-30b-a3b-instruct` | `qwen3_omni_moe` | text + image + audio + video | MoE, tool calling | — | +| `qwen3-omni-30b-a3b-thinking` | `qwen3_omni_moe` | text + image + audio + video | MoE, reasoning, tool calling | — | +| `qwen3-omni-30b-a3b-captioner` | `qwen3_omni_moe` | text + audio | MoE | audio captioning only | +| `gemma-4-e2b-it` | `gemma4` | text + image + audio | reasoning, JAX | thinking off | +| `gemma-4-e4b-it` | `gemma4` | text + image + audio | reasoning, JAX | thinking off | +| `gemma-4-12b-it` | `gemma4` | text + image + audio | reasoning | thinking off | +| `nemotron-3-nano-omni-30b-a3b-reasoning` | `nemotron_h_omni` | text + image + audio + video | MoE, reasoning | BF16 needs two 48 GB GPUs | +| `nemotron-3-nano-omni-30b-a3b-reasoning-fp8` | `nemotron_h_omni` | text + image + audio + video | MoE, reasoning | endpoint only (FP8 export) | + + +### API-only models + +Closed weights served through an API backend: no internals, so only the +black-box analyzers apply. The Gemini specs use `google-genai` with +`GEMINI_API_KEY`. + + +| Key | Family | Modalities | Traits | Notes | +|---|---|---|---|---| +| `gemini-2.5-flash-lite` | `gemini` | text + image + audio + video | reasoning | — | +| `gemini-2.5-flash` | `gemini` | text + image + audio + video | reasoning | — | +| `gemini-2.5-pro` | `gemini` | text + image + audio + video | reasoning | — | +| `gemini-3.1-flash-lite` | `gemini` | text + image + audio + video | reasoning | — | +| `gemini-3.1-pro-preview` | `gemini` | text + image + audio + video | reasoning | — | +| `gemini-3.5-flash-lite` | `gemini` | text + image + audio + video | reasoning | — | +| `gemini-3.5-flash` | `gemini` | text + image + audio + video | reasoning | — | +| `gemini-3.6-flash` | `gemini` | text + image + audio + video | reasoning | — | +| `gemini-3.7-flash` | `gemini` | text + image + audio + video | reasoning | — | +| `step-1o-vision` | `step` | text | — | spec declares no vision component, so it matches as text-only | + + +## Keeping this page current + +The tables above are generated from the analyzer registry and `evalrx/specs.py` by +`tools/gen_analyzer_docs.py`. After adding an analyzer or a model spec, run: + +```bash +python tools/gen_analyzer_docs.py # add new rows, refresh the derived columns +python tools/gen_analyzer_docs.py --check # exit 1 if the page is stale +``` + +A new key gets a placeholder row in its section; fill in its name and Paper (or +Notes) by hand. A new `analyzers/` subpackage or model group needs its own heading +and marker pair first. The test suite runs the same check +(`tests/test_analyzers/test_analyzer_docs.py`). diff --git a/evalrx/analyzers/__init__.py b/evalrx/analyzers/__init__.py index 07e464fb..3fc68bb2 100755 --- a/evalrx/analyzers/__init__.py +++ b/evalrx/analyzers/__init__.py @@ -1,19 +1,24 @@ """Analyzers — functional taxonomy keyed by required capability (not black/white-box). Every analyzer declares ``requires`` (capabilities) + ``applies_to_modalities`` and -self-registers on import. Categories: +self-registers on import. Categories (full tables: docs/analyzers.md): - perturbation/ input masking (RISE; VL-SHAP/MM-SHAP) — GENERATE/LOGPROBS - uncertainty/ entropy / self-consistency / verbalized conf — LOGITS/GENERATE - hallucination/ POPE / CHAIR / OPERA / VCD — GENERATE/ATTENTION - attention/ summary / rollout / sink / relative-attn — ATTENTION - attribution/ Grad-CAM / Chefer generic-attention — GRADIENTS - lens/ logit-lens / tuned-lens — HIDDEN_STATES - patching/ causal tracing / activation patching — HIDDEN_STATES (read+write) - geometry/ CKA / linear-probe — HIDDEN_STATES + perturbation/ RISE / VL-SHAP / MM-SHAP / modality ablation / + prompt contrast / MC format / CoT faithfulness / + context-SHAP / perturbation battery — GENERATE/LOGPROBS + uncertainty/ entropy / self-consistency / verbalized conf / + calibration / coverage-verification gap — LOGITS/LOGPROBS/GENERATE + hallucination/ POPE / CHAIR / SelfCheck / OPERA / VCD — GENERATE/ATTENTION/LOGITS + attention/ summary / rollout / sink / relative-attn — ATTENTION + attribution/ Grad-CAM / Chefer generic-attention — GRADIENTS + lens/ logit-lens / tuned-lens / layer contrast — HIDDEN_STATES + patching/ causal tracing / activation patching — HIDDEN_STATES (read+write) + geometry/ CKA / linear-probe — HIDDEN_STATES reasoning/ extraction/termination hygiene; arithmetic, - self-repair, step-value, knowledge split — GENERATE - agent/ loop / ignored-obs / first-error / counterfactual — Trajectory + self-repair, step-value, knowledge split, + contamination — GENERATE + agent/ loop / ignored-obs / first-error / rubric / + counterfactual / reliability / tool-SHAP — Trajectory Import is torch-tolerant: the ``attention`` subpackage loads torch at import, so on the light (pure-API) install it is skipped and its analyzers are simply not offered. diff --git a/tests/test_analyzers/test_analyzer_docs.py b/tests/test_analyzers/test_analyzer_docs.py new file mode 100644 index 00000000..c6862938 --- /dev/null +++ b/tests/test_analyzers/test_analyzer_docs.py @@ -0,0 +1,83 @@ +"""docs/analyzers.md must stay in step with the registries (tools/gen_analyzer_docs.py). + +The first test is the drift guard: an analyzer or model spec missing from the +page, or a registry-derived column that no longer matches the code, fails here +with the regenerate command. The rest pin the generator's own contract on small +synthetic pages. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + +pytest.importorskip("torch") # without torch the attention analyzers do not register +pytest.importorskip("numpy") + +SCRIPT = Path(__file__).resolve().parents[2] / "tools" / "gen_analyzer_docs.py" + +TABLE_HEAD = ( + "| Analyzer | Key | Requires | Modality | Paper | Status |\n" + "|---|---|---|---|---|---|\n" +) + + +@pytest.fixture(scope="module") +def gen(): + spec = importlib.util.spec_from_file_location("gen_analyzer_docs", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _patching_page(rows: str) -> str: + return ( + "### Patching\n\n" + "\n" + + TABLE_HEAD + rows + + "\n" + ) + + +def test_analyzer_doc_is_current(gen): + text = gen.DOC.read_text(encoding="utf-8") + fresh, added = gen.render(text) + assert not added, f"registered but not on the page: {added}" + assert fresh == text, f"{gen.DOC} is stale -- regenerate with: {gen.REGEN_CMD}" + + +def test_curated_cells_survive_and_derived_cells_are_rewritten(gen): + page = _patching_page("| My trace | `causal_trace` | WRONG | WRONG | [P](https://x) | ✓ |\n") + fresh, added = gen.render(page, require_all=False) + assert ( + "| My trace | `causal_trace` | `HIDDEN_STATES` | text + image | [P](https://x) | stub |" + in fresh + ) + assert added == [] + + +def test_new_key_gets_a_placeholder_row_and_a_stale_key_is_dropped(gen): + page = _patching_page("| Gone | `no_such_analyzer` | | | — | |\n") + fresh, added = gen.render(page, require_all=False) + assert "no_such_analyzer" not in fresh + assert "`causal_trace`" in fresh + assert added == ["analyzers/patching: causal_trace"] + + +def test_registered_package_without_a_section_is_an_error(gen): + with pytest.raises(ValueError, match="analyzers/perturbation"): + gen.render(_patching_page("")) + + +def test_check_mode_fails_on_a_stale_page_until_regenerated(gen, tmp_path, capsys): + doc = tmp_path / "analyzers.md" + page = gen.DOC.read_text(encoding="utf-8") + stale = page.replace("| `rise` | `GENERATE` |", "| `rise` | `LOGITS` |") + assert stale != page + doc.write_text(stale) + assert gen.main(["--doc", str(doc), "--check"]) == 1 + assert gen.REGEN_CMD in capsys.readouterr().err + assert gen.main(["--doc", str(doc)]) == 0 + assert gen.main(["--doc", str(doc), "--check"]) == 0 diff --git a/tools/gen_analyzer_docs.py b/tools/gen_analyzer_docs.py new file mode 100644 index 00000000..9ca65150 --- /dev/null +++ b/tools/gen_analyzer_docs.py @@ -0,0 +1,368 @@ +"""Generate (or check) the registry-derived tables in docs/analyzers.md. + + python tools/gen_analyzer_docs.py # rewrite the tables in place + python tools/gen_analyzer_docs.py --check # exit 1 if the doc is stale + +The page keeps its hand-written prose; only regions between marker comments are +generated:: + + category overview with counts + one table per analyzers/ subpackage + one table per model group + + +Inside a table, the columns the code knows are regenerated on every run: an +analyzer's Key, Requires, Modality and Status; a model's Key, Family, Modalities +and Traits. The columns only a person can write -- an analyzer's display name and +Paper, a model's Notes -- are carried over from the existing row with the same +key, wherever on the page it sits, so rows can be curated and reordered by hand. +A newly registered key gets a placeholder row appended to its section (the run +lists it so the placeholder gets curated); a key that is no longer registered +loses its row. A registered key whose package or group has no marked section is +an error: where it belongs on the page, and the prose around it, is a decision +this script does not make. + +Needs the full install (torch + numpy): without them the attention analyzers do +not register, and a table generated from that registry would silently drop them. +""" + +from __future__ import annotations + +import argparse +import ast +import difflib +import inspect +import re +import sys +import textwrap +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +DOC = ROOT / "docs" / "analyzers.md" +REGEN_CMD = "python tools/gen_analyzer_docs.py" + +_BLOCK = re.compile( + r"(?P\n)" + r"(?P.*?)" + r"(?P)", + re.DOTALL, +) +_HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*$", re.MULTILINE) +_KEY_CELL = re.compile(r"^`([^`]+)`$") + +_MODALITY_ORDER = ("text", "image", "audio", "video") +_NONE = "—" + +ANALYZER_HEADER = ( + "| Analyzer | Key | Requires | Modality | Paper | Status |\n" + "|---|---|---|---|---|---|\n" +) +MODEL_HEADER = ( + "| Key | Family | Modalities | Traits | Notes |\n" + "|---|---|---|---|---|\n" +) +INDEX_HEADER = ( + "| Category | Package | Analyzers | Stubs |\n" + "|---|---|---|---|\n" +) + +#: Model groups, in the order a missing-section error lists them. +MODEL_GROUPS = ("text", "vision", "omni", "api") + + +# ---------------------------------------------------------------------- +# What the code says +# ---------------------------------------------------------------------- + +def _load_registries(): + """Import evalrx (registers everything) and return (analyzers, model specs).""" + sys.path.insert(0, str(ROOT)) + import evalrx # noqa: F401 -- side effect: analyzers + models self-register + from evalrx import analyzers, registry + from evalrx.specs import REGISTRY + + if analyzers.attention is None: + raise SystemExit( + "the attention analyzers did not register (torch/numpy not importable), so the " + "generated tables would silently drop them -- run this from the full install" + ) + return registry.analyzers.all(), dict(REGISTRY) + + +def analyzer_package(cls) -> str: + """``evalrx.analyzers..`` -> ````.""" + parts = cls.__module__.split(".") + if parts[:2] != ["evalrx", "analyzers"] or len(parts) < 4: + raise SystemExit(f"{cls.__name__} lives outside evalrx/analyzers//: {cls.__module__}") + return parts[2] + + +def is_stub(cls) -> bool: + """True when ``_run`` does nothing but ``raise NotImplementedError`` (Stage-2 stubs).""" + tree = ast.parse(textwrap.dedent(inspect.getsource(cls._run))) + body = [ + stmt for stmt in tree.body[0].body + if not (isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Constant)) + ] + if len(body) != 1 or not isinstance(body[0], ast.Raise) or body[0].exc is None: + return False + exc = body[0].exc.func if isinstance(body[0].exc, ast.Call) else body[0].exc + return isinstance(exc, ast.Name) and exc.id == "NotImplementedError" + + +def _ordered(modalities) -> list[str]: + known = [m for m in _MODALITY_ORDER if m in modalities] + return known + sorted(set(modalities) - set(_MODALITY_ORDER)) + + +def requires_cell(cls) -> str: + parts = [f"`{cap.name}`" for cap in sorted(cls.requires, key=lambda c: c.name)] + if getattr(cls, "requires_trajectories", False): + parts.append("trajectories") + return " + ".join(parts) or _NONE + + +def modality_cell(cls) -> str: + """``applies_to_modalities``; slots in ``requires_modalities`` are bold.""" + required = set(getattr(cls, "requires_modalities", ()) or ()) + return " + ".join( + f"**{m}**" if m in required else m for m in _ordered(cls.applies_to_modalities) + ) + + +def default_display_name(cls) -> str: + """Placeholder name for a new row: ``CoverageVerificationGap`` -> ``Coverage verification gap``.""" + stem = re.sub(r"Analyzer$", "", cls.__name__) or cls.__name__ + words = re.findall(r"[A-Z]+(?=[A-Z][a-z])|[A-Z]?[a-z]+|[A-Z]+|\d+", stem) + if not words: + return stem + return " ".join([words[0]] + [w if w.isupper() else w.lower() for w in words[1:]]) + + +def model_group(spec) -> str: + if spec.api_only: + return "api" + if spec.modalities & {"audio", "video"}: + return "omni" + if "image" in spec.modalities: + return "vision" + return "text" + + +def model_traits(spec) -> str: + traits = [ + label for flag, label in ( + (spec.is_moe, "MoE"), + (spec.is_reasoning, "reasoning"), + (spec.tool_calling, "tool calling"), + (spec.jax is not None, "JAX"), + ) if flag + ] + return ", ".join(traits) or _NONE + + +# ---------------------------------------------------------------------- +# What the page says +# ---------------------------------------------------------------------- + +def _cells(line: str) -> list[str]: + return [c.strip() for c in line.strip().strip("|").split("|")] + + +def _rows(body: str, key_col: int) -> list[tuple[str, list[str]]]: + """``(key, cells)`` for every data row of the table in *body*.""" + rows = [] + for line in body.splitlines(): + if not line.lstrip().startswith("|"): + continue + cells = _cells(line) + match = _KEY_CELL.match(cells[key_col]) if len(cells) > key_col else None + if match: # header and separator rows have no backticked key + rows.append((match.group(1), cells)) + return rows + + +def _slug(title: str) -> str: + """Heading anchor as Python-Markdown's toc (MkDocs) and GitHub both build it for plain titles.""" + text = re.sub(r"[^\w\s-]", "", title).strip().lower() + return re.sub(r"[-\s]+", "-", text) + + +def _heading_before(text: str, pos: int) -> str | None: + headings = [m.group(1) for m in _HEADING.finditer(text, 0, pos)] + return headings[-1] if headings else None + + +# ---------------------------------------------------------------------- +# Rendering +# ---------------------------------------------------------------------- + +def render(text: str, *, require_all: bool = True) -> tuple[str, list[str]]: + """Return ``(regenerated page, keys that got a placeholder row)``. + + With *require_all*, a registered analyzer package or model group that has no + marked section on the page raises ``ValueError`` instead of being skipped. + """ + analyzers, specs = _load_registries() + blocks = list(_BLOCK.finditer(text)) + + # Curated cells by key, collected page-wide so a row moved between sections keeps them. + curated_analyzers: dict[str, list[str]] = {} + curated_models: dict[str, list[str]] = {} + for block in blocks: + name = block.group("name") + if name.startswith("analyzers/"): + curated_analyzers.update(_rows(block.group("body"), key_col=1)) + elif name.startswith("models/"): + curated_models.update(_rows(block.group("body"), key_col=0)) + + by_package: dict[str, dict[str, type]] = {} + for key, cls in analyzers.items(): + by_package.setdefault(analyzer_package(cls), {})[key] = cls + by_group: dict[str, dict[str, object]] = {} + for key, spec in specs.items(): + by_group.setdefault(model_group(spec), {})[key] = spec + + sections = {b.group("name") for b in blocks} + missing = [f"analyzers/{p}" for p in sorted(by_package) if f"analyzers/{p}" not in sections] + missing += [f"models/{g}" for g in MODEL_GROUPS if g in by_group and f"models/{g}" not in sections] + unknown = [ + s for s in sorted(sections) + if (s.startswith("analyzers/") and s.split("/", 1)[1] not in by_package) + or (s.startswith("models/") and s.split("/", 1)[1] not in by_group) + or not (s.startswith(("analyzers/", "models/")) or s == "analyzer-index") + ] + if require_all and (missing or unknown): + problems = [] + if missing: + problems.append( + "registered but no section on the page (add a heading, a sentence, and an empty " + "'' / '' pair): " + + ", ".join(missing) + ) + if unknown: + problems.append("section matches nothing registered: " + ", ".join(unknown)) + raise ValueError("; ".join(problems)) + + added: list[str] = [] + index_rows: list[str] = [] + + def analyzer_table(package: str, body: str) -> str: + registered = by_package.get(package, {}) + order = [k for k, _ in _rows(body, key_col=1) if k in registered] + order += sorted(k for k in registered if k not in order) + lines = [ANALYZER_HEADER] + for key in order: + cls = registered[key] + if key in curated_analyzers: + cells = curated_analyzers[key] + name, paper = cells[0], (cells[4] if len(cells) > 4 else _NONE) + else: + name, paper = default_display_name(cls), _NONE + added.append(f"analyzers/{package}: {key}") + status = "stub" if is_stub(cls) else "✓" + lines.append( + f"| {name} | `{key}` | {requires_cell(cls)} | {modality_cell(cls)} " + f"| {paper} | {status} |\n" + ) + return "".join(lines) + + def model_table(group: str, body: str) -> str: + registered = by_group.get(group, {}) + order = [k for k, _ in _rows(body, key_col=0) if k in registered] + order += sorted(k for k in registered if k not in order) + lines = [MODEL_HEADER] + for key in order: + spec = registered[key] + if key in curated_models: + cells = curated_models[key] + notes = cells[4] if len(cells) > 4 and cells[4] else _NONE + else: + notes = _NONE + added.append(f"models/{group}: {key}") + modalities = " + ".join(_ordered(spec.modalities)) + lines.append( + f"| `{key}` | `{spec.family}` | {modalities} | {model_traits(spec)} | {notes} |\n" + ) + return "".join(lines) + + # Pass 1: every table except the index (the index counts what pass 1 renders). + out: list[str] = [] + cursor = 0 + index_at: int | None = None + for block in blocks: + name = block.group("name") + out.append(text[cursor:block.start("body")]) + if name.startswith("analyzers/"): + package = name.split("/", 1)[1] + out.append(analyzer_table(package, block.group("body"))) + registered = by_package.get(package, {}) + title = _heading_before(text, block.start()) or package + stubs = sum(is_stub(cls) for cls in registered.values()) + index_rows.append( + f"| [{title}](#{_slug(title)}) | `{package}/` | {len(registered)} | {stubs} |\n" + ) + elif name.startswith("models/"): + out.append(model_table(name.split("/", 1)[1], block.group("body"))) + else: # analyzer-index: filled in below + index_at = len(out) + out.append("") + cursor = block.start("end") + out.append(text[cursor:]) + + if index_at is not None: + documented = [cls for pkg in by_package.values() for cls in pkg.values()] + total = ( + f"| **Total** | | **{len(documented)}** | " + f"**{sum(is_stub(cls) for cls in documented)}** |\n" + ) + out[index_at] = INDEX_HEADER + "".join(index_rows) + total + return "".join(out), added + + +# ---------------------------------------------------------------------- +# CLI +# ---------------------------------------------------------------------- + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--doc", type=Path, default=DOC, help="page to regenerate (default: %(default)s)") + ap.add_argument( + "--check", action="store_true", + help="write nothing; exit 1 if regenerating would change the page (for CI)", + ) + args = ap.parse_args(argv) + + text = args.doc.read_text(encoding="utf-8") + try: + fresh, added = render(text) + except ValueError as e: + print(f"{args.doc}: {e}", file=sys.stderr) + return 1 + + if args.check: + if fresh == text: + print(f"{args.doc}: up to date") + return 0 + diff = difflib.unified_diff( + text.splitlines(keepends=True), fresh.splitlines(keepends=True), + fromfile=f"{args.doc} (on disk)", tofile=f"{args.doc} (from registry)", + ) + sys.stderr.writelines(diff) + print(f"\n{args.doc} is out of date -- regenerate with: {REGEN_CMD}", file=sys.stderr) + return 1 + + if fresh == text: + print(f"{args.doc}: up to date") + return 0 + args.doc.write_text(fresh, encoding="utf-8") + print(f"{args.doc}: updated") + if added: + print("placeholder rows added -- fill in the hand-written cells (name/Paper, or Notes):") + for entry in added: + print(f" {entry}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())