diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dc668ab3..cc8ec748 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -65,6 +65,9 @@ jobs: # embedded taxonomy answers with no network (P. tannophilus: code 26) docker run --rm --network none --entrypoint /bin/bash matpredict:test /shell-hook.sh \ python -c "from MATPredict.db import taxonomy as t; assert t.default_genetic_code(4918) == 26; assert t.default_lineage_phylum_name(4918) == 'Ascomycota'; print('taxonomy ok')" + # report PDF with no network: WeasyPrint from the pixi environment (not a browser) + docker run --rm --network none --entrypoint /bin/bash matpredict:test /shell-hook.sh \ + python -c "from MATPredict.report.html import render_genome_report as r; from MATPredict.report.pdf import html_to_pdf as p; e = p(r({'detected': []}, print_mode=True), '/tmp/r.pdf'); assert e == 'weasyprint', e; assert open('/tmp/r.pdf', 'rb').read(5) == b'%PDF-'; print('pdf ok')" - name: Log in to GHCR if: github.event_name != 'pull_request' diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b55f939..5343ce85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,26 @@ release completed. them (2 of 2,722). Calibrated on Dothideomycetes; see `analysis/2026-10-05_dothideomycetes-full-run.md`. ### Added +- `matpredict report genome --run DIR [--pdf FILE]`: a self-contained HTML report of one `detect` run (no network, + no external files), printable to PDF from a browser or written with `--pdf` (WeasyPrint if installed, else headless + Chrome/Chromium). It leads with the result in plain words (mating type and confidence; or "no call", "not searched", + "two idiomorphs: needs review"), then one card per called locus: a gene-order figure (inline SVG; role, model status, + strand, exons, alternate model, called-locus bracket, contig end), the idiomorph evidence and the gene table; then + what was searched, withheld candidate loci, provenance and a glossary. Light and dark on screen, light in print. + Revised after an independent web/data-design review (`analysis/2026-10-07_report-design-review.md`). +- `detection_report.yaml` gains a `run` block (first key): sample, organism, taxid, phylum, MATPredict version, + database content SHA-256 and record count, taxonomy source, genome file name, SHA-256, contigs, length and N50, + parameters, start time and wall time (`detect.provenance`). Additive; reports without it still render. + `detect` gains `--sample` and `--organism` (shown in the report; routing still uses `--taxid`). +- `detect` writes `report.html` in `--out-dir` by default (curator's choice 2026-10-07: on by default, opt out). + `--no-html` or `MATPREDICT_HTML=0` turns it off; `--pdf` adds `report.pdf`. A report or PDF error is logged and + never fails the run. The batch scripts (`run_clade_panel.slurm`, `run_polish_ab.slurm`, `zygo_regression.py`, + `run_holdout_benchmark.py`) set `MATPREDICT_HTML=0`: an environment variable rather than the flag, because they may + run an older frozen worktree that would reject `--no-html`. `batch_runner` (in-process) writes no report. +- WeasyPrint (69.x, conda-forge) joins the pixi environment, `environment.yml` and so the Docker image, so `--pdf` + works everywhere MATPredict is installed: 25 packages, 10.3 MB download (Pango, Cairo, HarfBuzz, fonts). Chosen + over a headless Chromium in the image (about 300 MB). No locked version of any existing package changed. The + Docker CI smoke test renders a PDF with no network. - Report-only pheromone-receptor arrays (`detect.receptor_arrays`; study `analysis/2026-10-06_agaricomycetes-pr-arrays.md`, options 1 and 2). The STE3-like receptor hits of every family with a `pheromone_precursor_scan` (Basidiomycota PR) are merged per strand into loci and grouped into arrays (same diff --git a/README.md b/README.md index 02bebbc4..ddf99f33 100644 --- a/README.md +++ b/README.md @@ -258,6 +258,9 @@ matpredict detect --genome GENOME [--proteins PROTEINS] [--taxid TAXID] | `--no-require-core-role` | Allow polishing of clusters without a core MAT gene hit | | `--exclude-records IDS` | Withhold curated records (comma-separated) for leave-one-out tests | | `--evidence-diagnostics FILE` | Write per-hit evidence as JSON lines | +| `--html` / `--no-html` | Write `report.html` (default: on). `MATPREDICT_HTML=0` turns the default off; the batch scripts set it | +| `--pdf` | Also write `report.pdf` (WeasyPrint, in the pixi environment and the image; off by default). A report or PDF error is logged and never fails the run | +| `--sample` / `--organism` | Names shown in the report and recorded in the `run` block | | `--emit-cds-fasta` | Add CDS features with translations to the GFF3, and write `detected_loci.fasta` (full sequence of every contig with a call; can be large) | **Outputs** (in `--out-dir`): @@ -267,11 +270,26 @@ matpredict detect --genome GENOME [--proteins PROTEINS] [--taxid TAXID] | `detection_report.yaml` | Routing mode, genetic code, detected loci (family, contig, coordinates, idiomorph, confidence, `locus_class`, genes found and missing, classifier scores and margin, gene evidence), withheld loci with the reason, zygosity and two-idiomorph flags, families not detected | | `detected_loci.gff3` | One gene feature per found or missing gene at each locus | | `detected_loci.fasta` | Only with `--emit-cds-fasta` | +| `report.html` | Readable report (see `matpredict report genome`); not with `--no-html` | +| `report.pdf` | Only with `--pdf` | **Routing modes** (`routing_mode` in the report): `lineage` (taxid matched a family scope), `phylum_fallback`, `explicit_phylum` (`--phylum`), `exhaustive`, `not_searched`. +### `matpredict report genome` + +``` +matpredict report genome --run OUT_DIR [--out report.html] [--pdf report.pdf] [--sample NAME] +``` + +`detect` writes this report itself by default; use this command to re-render a run (for example one made before +the report existed, or with `--pdf`). Writes a self-contained HTML report of one `detect` run (default `OUT_DIR/report.html`): the result in plain words, +a gene-order figure and evidence table per called locus, what was searched, withheld candidates and provenance. Open it +in any browser; "Save as PDF" prints it. `--pdf` writes the PDF directly with WeasyPrint (part of the pixi environment, +`environment.yml` and the Docker image), or, where WeasyPrint is missing, a headless Chrome or Chromium on PATH. `detect --sample` and `--organism` set the names the report +shows. + ### `matpredict detect` helper commands | Command | Purpose | diff --git a/analysis/2026-10-07_report-design-review.md b/analysis/2026-10-07_report-design-review.md new file mode 100644 index 00000000..c15c1b1a --- /dev/null +++ b/analysis/2026-10-07_report-design-review.md @@ -0,0 +1,90 @@ +# Per-genome HTML/PDF report (`matpredict report genome`): build and design review +Status: open (implemented on branch `report-genome`; awaiting curator review) + +## Question +Can one `detect` run be turned into a report that a biologist reads in five seconds and a curator can audit, that +works offline, on screen (desktop, phone, light, dark) and prints to PDF? Spec: +`docs/superpowers/specs/2026-10-07-genome-report-and-reads-intake-design.md`, Part 1. + +## Data and code version +- Code: branch `report-genome`, new package `src/MATPredict/report/` (`html.py`, `svg.py`, `pdf.py`, `cli.py`) and + `src/MATPredict/detect/provenance.py` (the new `run` block of `detection_report.yaml`). +- Inputs: one real report (`results/2026-10-06_fola_50a`, old format, no `run` block) and six synthetic reports + (`tests/report/make_fixtures.py`): Mucoromycota with the HMM classifier, two idiomorphs (unlinked), not searched, + nothing called with an assembly gap and withheld loci, Basidiomycota HD with subloci plus an unconfirmed PR call in + a receptor array, and hostile strings (`\n" + ) + + +_JS = """ +(function(){ + var b=document.querySelector('[data-print]'); + document.querySelectorAll('.fig-scroll').forEach(function(box){ + var svg=box.querySelector('svg'); var cx=svg&&parseFloat(svg.getAttribute('data-core-x')); + if(box.scrollWidth>box.clientWidth&&cx){box.scrollLeft=Math.max(0,cx*svg.clientWidth/960-40);} + }); + if(b){b.addEventListener('click',function(){window.print();});} + var opened=[]; + window.addEventListener('beforeprint',function(){ + document.querySelectorAll('details:not([open])').forEach(function(d){d.open=true;opened.push(d);}); + }); + window.addEventListener('afterprint',function(){opened.forEach(function(d){d.open=false;});opened=[];}); +})(); +""" + + +def _css_string(s: str) -> str: + """A CSS string literal; `<` and `>` as CSS escapes so a name cannot close the ') + assert "<" not in s and ">" not in s + assert s.startswith('"') and s.endswith('"') + + +def test_old_report_without_run_block_says_not_recorded(): + html = render_genome_report(_doc("fola_50a_old_format"), sample="Fola 50a") + assert "not recorded" in html + assert "Fola 50a" in html + + +def test_run_block_is_shown(): + html = render_genome_report(_doc("phycomyces_classifier")) + assert "Phycomyces blakesleeanus" in html and "taxid 4837" in html + assert "N50" in html and "3f2a9c41d07be5aa" in html + + +def test_print_mode_opens_collapsed_sections(): + doc = _doc("fola_50a_old_format") + assert "
" in render_genome_report(doc) + printed = render_genome_report(doc, print_mode=True) + assert "
" not in printed and "
" in printed + + +def test_overlapping_genes_and_labels_go_to_separate_lanes(): + extents = [(100, 300), (150, 200), (310, 400), (600, 700)] + assert assign_lanes(extents, [True, False, True, True]) == [0, 1, 0, 0] + + +def test_contig_end_side_from_edge_distance(): + assert contig_end({"start": 116422, "end": 128495, "contig_edge_distance": 41212}) == ("right", 169707) + assert contig_end({"start": 1200, "end": 1980, "contig_edge_distance": 1199}) == ("left", 1) + assert contig_end({"start": 5, "end": 9}) is None + + +def test_sample_name_fallback(tmp_path): + assert sample_name({"run": {"sample": "s1"}}, tmp_path, None) == ("s1", False) + assert sample_name({"run": {"genome": {"file": "GCA_1.fna.gz"}}}, tmp_path, None) == ("GCA_1", False) + assert sample_name({}, tmp_path, "given") == ("given", False) + assert sample_name({}, tmp_path, None) == (tmp_path.name, True) + + +def test_two_idiomorphs_lead_with_review_verdict_and_do_not_repeat_causes(): + html = render_genome_report(_doc("two_idiomorphs")) + result = html[html.index('id="result-h"'):html.index("Called loci")] + assert result.index("needs review") < result.index('class="chip"') + for cause in ("a duplication or paralog", "a mixed culture or heterokaryon"): + assert result.count(cause) == 1 + + +def test_locus_without_flanks_is_not_called_complete(): + html = render_genome_report(_doc("basidio_hd_pr")) + pr = html[html.index('id="locus-1"'):] + assert "no flanking gene identified" in pr + assert "B43-like (PR family): complete locus" not in html + + +def test_cli_writes_report_next_to_the_run(tmp_path): + run = tmp_path / "run" + run.mkdir() + (run / "detection_report.yaml").write_text((FIXTURES / "phycomyces_classifier.yaml").read_text()) + (run / "detected_loci.gff3").write_text("##gff-version 3\n") + assert main(["report", "genome", "--run", str(run)]) == 0 + html = (run / "report.html").read_text() + assert "detected_loci.gff3" in html and "Phybl2" in html + + +def test_cli_rejects_a_file_that_is_not_a_detection_report(tmp_path): + bad = tmp_path / "x.yaml" + bad.write_text("a: 1\n") + assert main(["report", "genome", "--run", str(bad)]) == 1 + + +def test_no_pdf_engine_gives_a_clear_error(monkeypatch): + import builtins + real_import = builtins.__import__ + + def no_weasy(name, *a, **k): + if name == "weasyprint": + raise ImportError(name) + return real_import(name, *a, **k) + + monkeypatch.setattr(builtins, "__import__", no_weasy) + monkeypatch.setattr(pdf, "find_chrome", lambda: None) + with pytest.raises(pdf.NoPdfEngine, match="Save as PDF"): + pdf.available_engine() + + +def test_pdf_render_with_installed_engine(tmp_path): + try: + pdf.available_engine() + except pdf.NoPdfEngine: + pytest.skip("no PDF engine installed") + out = tmp_path / "r.pdf" + pdf.html_to_pdf(render_genome_report(_doc("phycomyces_classifier"), print_mode=True), out) + assert out.read_bytes()[:5] == b"%PDF-" + + +def test_element_ids_are_unique(): + import re + html = render_genome_report(_doc("two_idiomorphs")) + ids = re.findall(r'\bid="([^"]+)"', html) + assert len(ids) == len(set(ids)) + + +def test_figure_colours_do_not_depend_on_page_css(): + """WeasyPrint ignores page CSS inside SVG: role colours must be attributes.""" + html = render_genome_report(_doc("phycomyces_classifier")) + assert 'fill="#b8471a"' in html and 'fill="#2a6fb0"' in html + + +def test_single_candidate_margin_names_the_resolution_it_came_from(): + """With one idiomorph candidate the pipeline's margin is the narrowest overlap resolution (identity points); + the page must say so rather than claim no other idiomorph scored.""" + html = render_genome_report(_doc("fola_50a_old_format")) + assert "46.7" in html and "identity points" in html + assert "No other idiomorph scored" not in html + assert "No other idiomorph scored" in render_genome_report(_doc("basidio_hd_pr")) + + +def test_candidate_scores_match_the_gloss(): + for case in CASES: + for call in _doc(case).get("detected") or []: + if call.get("idiomorph_classifier"): + continue + core = [g["bitscore"] for g in call["gene_evidence"] if g.get("role") == "core_MAT" and g.get("bitscore")] + if core and call["idiomorph_candidates"]: + assert call["idiomorph_candidates"][0]["score"] == max(core), case diff --git a/tests/test_cli_smoke.py b/tests/test_cli_smoke.py index 85a7ccc4..33e6ef0b 100644 --- a/tests/test_cli_smoke.py +++ b/tests/test_cli_smoke.py @@ -387,7 +387,7 @@ def _stub_detect_cli(monkeypatch, tmp_path, recorded): detect_cli, "build_reference_fasta", lambda db_root, out, family_keys=None, **kwargs: out ) monkeypatch.setattr(detect_cli, "run_pipeline", lambda **kwargs: DetectionOutcome(results=[])) - monkeypatch.setattr(detect_cli, "write_detection_report", lambda outcome, path: None) + monkeypatch.setattr(detect_cli, "write_detection_report", lambda outcome, path, run=None: None) def fake_write_gff3(outcome, out_path, **kwargs): recorded.append(kwargs) @@ -491,7 +491,7 @@ def fake_run_pipeline(**kwargs): monkeypatch.setattr(detect_cli, "build_reference_fasta", fake_build) monkeypatch.setattr(detect_cli, "run_pipeline", fake_run_pipeline) monkeypatch.setattr(detect_cli, "write_detection_gff3", lambda *a, **k: None) - monkeypatch.setattr(detect_cli, "write_detection_report", lambda outcome, path: None) + monkeypatch.setattr(detect_cli, "write_detection_report", lambda outcome, path, run=None: None) args = SimpleNamespace( genome="g.fa", proteins=None, taxid=None, out_dir=str(tmp_path / "out"),