From 7d66ecfe9e1c82d4f1aa7c7edf6aa8fad49c6806 Mon Sep 17 00:00:00 2001 From: Jason Stajich Date: Sun, 4 Oct 2026 14:21:31 -0700 Subject: [PATCH 1/3] taxonomy: offline NCBI taxonomy table, embedded in the Docker image `curate-db build-taxonomy` writes a slim all-taxa table (taxid, parent, rank, genetic code, scientific name; merged ids) from a taxdump directory, NCBI taxdmp_*.zip or taxdump.tar.gz (plain, .gz or .zst). With $MATPREDICT_TAXONOMY set, lineage, phylum and genetic code come from it; $MATPREDICT_OFFLINE=1 never calls NCBI (a missing taxid is reported in routing_error). Reports gain `taxonomy_source`. Measured: taxdmp_2026-10-01 -> 3,016,750 taxa, 101,411 merged, 23.7 MB; load 1.96 s, 76 MB RSS (streamed into int arrays). Against 8,185 cached efetch answers: phylum and genetic code identical for all; lineage for 8,179 (6 re-parented by NCBI between fetch and snapshot); 21 deleted taxids. Offline end-to-end run (P. tannophilus, code 26): no NCBI call, calls identical to the online run. Dockerfile builds the table from the dated archive (SHA-256 pinned) and sets MATPREDICT_TAXONOMY and MATPREDICT_OFFLINE=1; CI smoke test checks a lookup with --network none. Tests: 1017 passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JkhYFEPaa1WenE5MtA8Qya --- .github/workflows/docker.yml | 3 + CHANGELOG.md | 14 + Dockerfile | 23 +- README.md | 27 +- .../README.md | 14 + .../detection_report.yaml | 1947 +++++++++++++++++ .../run.slurm | 10 + .../stderr.log.zst | Bin 0 -> 208 bytes src/MATPredict/db/cli.py | 17 + src/MATPredict/db/local_taxonomy.py | 294 +++ src/MATPredict/db/taxonomy.py | 33 + src/MATPredict/detect/report.py | 11 + tests/db/test_local_taxonomy.py | 150 ++ 13 files changed, 2540 insertions(+), 3 deletions(-) create mode 100644 results/2026-10-04_offline_taxonomy_check/README.md create mode 100644 results/2026-10-04_offline_taxonomy_check/detection_report.yaml create mode 100644 results/2026-10-04_offline_taxonomy_check/run.slurm create mode 100644 results/2026-10-04_offline_taxonomy_check/stderr.log.zst create mode 100644 src/MATPredict/db/local_taxonomy.py create mode 100644 tests/db/test_local_taxonomy.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index c6843032..dc668ab3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -62,6 +62,9 @@ jobs: docker run --rm matpredict:test detect --help > /dev/null docker run --rm --entrypoint /bin/bash matpredict:test /shell-hook.sh \ bash -c 'for t in tblastn diamond miniprot exonerate mafft augustus taxonkit; do command -v $t > /dev/null || { echo "missing $t"; exit 1; }; done; python -c "import pyhmmer, MATPredict"' + # embedded taxonomy answers with no network (P. tannophilus: code 26) + docker run --rm --network none --entrypoint /bin/bash matpredict:test /shell-hook.sh \ + python -c "from MATPredict.db import taxonomy as t; assert t.default_genetic_code(4918) == 26; assert t.default_lineage_phylum_name(4918) == 'Ascomycota'; print('taxonomy ok')" - name: Log in to GHCR if: github.event_name != 'pull_request' diff --git a/CHANGELOG.md b/CHANGELOG.md index cb43b684..bee47ba2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to MATPredict. Versions follow [semantic versioning](https:/ release tags may carry a clade suffix naming the lineage whose infrastructure that release completed. +## [Unreleased] + +### Added +- Offline NCBI taxonomy. `matpredict curate-db build-taxonomy` writes a slim + all-taxa table (taxid, parent, rank, genetic code, scientific name; merged + ids) from an NCBI taxdump directory, `taxdmp_*.zip` or `taxdump.tar.gz` + (plain, `.gz` or `.zst` files). With `$MATPREDICT_TAXONOMY` set, lineage, + phylum and genetic code come from it; `$MATPREDICT_OFFLINE=1` never calls + NCBI. Checked against 8,185 cached efetch answers: phylum and genetic code + identical for all; lineage identical for 8,179 (6 re-parented by NCBI between + fetch and snapshot). Reports gain `taxonomy_source`. +- The Docker image builds the table from the dated archive + `taxdmp_2026-10-01.zip` (SHA-256 pinned) and runs offline by default. + ## [0.6.1] — 2026-10-04 — `v0.6.1` ### Fixed diff --git a/Dockerfile b/Dockerfile index f25080e8..daf068a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,9 @@ # Runtime stage: the environment, the package source and the curated database. # # docker build -t matpredict . -# docker run --rm -v "$PWD":/data -e MATPREDICT_NCBI_EMAIL=you@example.org \ +# docker run --rm -v "$PWD":/data \ # matpredict detect --genome /data/genome.fna --taxid 4837 --out-dir /data/out +# Taxonomy lookups use the embedded NCBI snapshot (no network). FROM ghcr.io/prefix-dev/pixi:0.71.3-noble AS build WORKDIR /app @@ -14,6 +15,20 @@ RUN pixi install --locked -e default \ && pixi shell-hook -e default -s bash > /shell-hook.sh \ && echo 'exec "$@"' >> /shell-hook.sh +# Offline NCBI taxonomy (curator ruling 2026-10-04): a slim all-taxa table +# (taxid, parent, rank, genetic code, scientific name; merged ids) built from a +# DATED NCBI archive, so one image = one taxonomy snapshot and runs make no +# E-utilities call. NCBI publishes no checksum for the archive; the SHA-256 is +# the one measured when the date was pinned. Change both together. +ARG TAXDUMP_DATE=2026-10-01 +ARG TAXDUMP_SHA256=d744af371c0b9fc7269d80b49546b6ac4c9ddc52a2fd97bcbfd02783edb5c9eb +RUN /app/.pixi/envs/default/bin/python -c "import urllib.request,sys; urllib.request.urlretrieve(sys.argv[1], '/tmp/taxdmp.zip')" \ + "https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump_archive/taxdmp_${TAXDUMP_DATE}.zip" \ + && echo "${TAXDUMP_SHA256} /tmp/taxdmp.zip" | sha256sum -c - \ + && PYTHONPATH=/app/src /app/.pixi/envs/default/bin/python -m MATPredict curate-db build-taxonomy \ + --taxdump /tmp/taxdmp.zip --snapshot "${TAXDUMP_DATE}" --out /app/taxonomy/ncbi_taxonomy.tsv.zst \ + && rm /tmp/taxdmp.zip + FROM ubuntu:24.04 AS runtime ARG VERSION=dev LABEL org.opencontainers.image.title="MATPredict" \ @@ -27,7 +42,13 @@ COPY --from=build /shell-hook.sh /shell-hook.sh COPY --from=build /app/src /app/src COPY pyproject.toml README.md /app/ COPY db /app/db +COPY --from=build /app/taxonomy /app/taxonomy +# MATPREDICT_OFFLINE=1: a taxid not in the snapshot is reported (routing_error) +# instead of fetched from NCBI. Run with -e MATPREDICT_OFFLINE=0 to allow +# E-utilities for such taxids. ENV MATPREDICT_DB_ROOT=/app/db \ + MATPREDICT_TAXONOMY=/app/taxonomy/ncbi_taxonomy.tsv.zst \ + MATPREDICT_OFFLINE=1 \ MATPREDICT_CACHE_DIR=/data/.matpredict_cache WORKDIR /data ENTRYPOINT ["/bin/bash", "/shell-hook.sh", "matpredict"] diff --git a/README.md b/README.md index 7aaebe6f..6eb42ae0 100644 --- a/README.md +++ b/README.md @@ -174,11 +174,13 @@ pixi run matpredict --help ### Other ways to install **Docker.** Each release publishes an image to the GitHub Container Registry. -The image holds the environment, the package and the curated database. +The image holds the environment, the package, the curated database and an +NCBI taxonomy snapshot (see [Offline taxonomy](#offline-taxonomy)), so a run +needs no network. ```bash docker pull ghcr.io/stajichlab/matpredict:latest # or a release version, : -docker run --rm -v "$PWD":/data -e MATPREDICT_NCBI_EMAIL=you@example.org \ +docker run --rm -v "$PWD":/data \ ghcr.io/stajichlab/matpredict:latest \ detect --genome /data/genome.fna --taxid 4837 --out-dir /data/out ``` @@ -339,6 +341,27 @@ tool still runs, sends no e-mail, and logs one warning. | `MATPREDICT_CONFIG` | `~/.config/matpredict/config.toml` | Config file path | | `MATPREDICT_DB_ROOT` | `./db` | Reference database to use | | `MATPREDICT_CACHE_DIR` | `./.matpredict_cache` | Cache for NCBI responses. The e-mail and API key are not part of the cache key, so users with different settings share one cache | +| `MATPREDICT_TAXONOMY` | none (image: `/app/taxonomy/ncbi_taxonomy.tsv.zst`) | Offline NCBI taxonomy table; used before E-utilities | +| `MATPREDICT_OFFLINE` | off (image: `1`) | `1`: never call NCBI; a taxid not in the table is reported in `routing_error` | + +### Offline taxonomy + +`detect` needs the lineage, phylum and genetic code of a `--taxid`. A local +table answers these without NCBI E-utilities. Build it from a dated NCBI +archive and point `MATPREDICT_TAXONOMY` at it: + +```bash +curl -O https://ftp.ncbi.nlm.nih.gov/pub/taxonomy/taxdump_archive/taxdmp_2026-10-01.zip +matpredict curate-db build-taxonomy --taxdump taxdmp_2026-10-01.zip \ + --snapshot 2026-10-01 --out ncbi_taxonomy.tsv.zst +export MATPREDICT_TAXONOMY=$PWD/ncbi_taxonomy.tsv.zst +export MATPREDICT_OFFLINE=1 # optional: never call NCBI +``` + +All taxa, about 24 MB; loading it takes about 2 s and 80 MB of memory. Each +report records the source in `taxonomy_source` (for example +`local NCBI taxonomy snapshot 2026-10-01`). The Docker image builds this table +at image build time and sets both variables. ### Tests diff --git a/results/2026-10-04_offline_taxonomy_check/README.md b/results/2026-10-04_offline_taxonomy_check/README.md new file mode 100644 index 00000000..42739344 --- /dev/null +++ b/results/2026-10-04_offline_taxonomy_check/README.md @@ -0,0 +1,14 @@ +# Offline taxonomy check (2026-10-04) + +`detect --taxid 669874` on P. tannophilus GCA_001661245.1 with +`MATPREDICT_TAXONOMY` (table from NCBI taxdmp_2026-10-01.zip, 3,016,750 taxa, +23.7 MB), `MATPREDICT_OFFLINE=1` and an empty NCBI cache (`run.slurm`, job +29396722, 8 min 47 s, 306 MB peak). + +- NCBI cache after the run: 0 files (no E-utilities call). +- Report: `genetic_code: 26` (from the table), `routing_mode: phylum_fallback`, + `routing_error: null`, `taxonomy_source: local NCBI taxonomy snapshot 2026-10-01`. +- Calls identical to the online run of the same commit's code + (`../2026-10-04_alaninales_v061_exostring/`): MAT MAT1-1 high, MATsc MATa + medium, MATyl A medium, MTL alpha medium, same coordinates. +- Table vs 8,185 cached efetch answers: see `tests/db/test_local_taxonomy.py`. diff --git a/results/2026-10-04_offline_taxonomy_check/detection_report.yaml b/results/2026-10-04_offline_taxonomy_check/detection_report.yaml new file mode 100644 index 00000000..fa8ffe0d --- /dev/null +++ b/results/2026-10-04_offline_taxonomy_check/detection_report.yaml @@ -0,0 +1,1947 @@ +routing_mode: phylum_fallback +routing_error: null +not_searched_reason: null +genetic_code: 26 +genetic_code_error: null +taxonomy_source: local NCBI taxonomy snapshot 2026-10-01 +suppressed_unpolished: 121 +suppressed_flank_carried: 0 +suppressed_mat_gene_gate: 0 +suppressed_paralog_class: 0 +suppressed_loci: +- family: Ascomycota:PM + contig: KV454014.1 + start: 185089 + end: 282802 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454014.1 + start: 185089 + end: 282802 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat2 + contig: KV454014.1 + start: 185089 + end: 282802 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATtub + contig: KV454014.1 + start: 185089 + end: 282802 + idiomorph: MAT1-2 + polished_genes: 1 + genes_found: + - MAT1-2-1 + - SLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454014.1 + start: 345322 + end: 357829 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MAT1-1-1 + - MAT1-2-1 + - APN2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454014.1 + start: 467592 + end: 493836 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454014.1 + start: 467592 + end: 493836 + idiomorph: MAT1-1 + polished_genes: 0 + genes_found: + - MAT1-1-1 + - APN2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454014.1 + start: 467592 + end: 493836 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - PAP1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454014.1 + start: 547436 + end: 559301 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454014.1 + start: 633666 + end: 646028 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454014.1 + start: 633666 + end: 646028 + idiomorph: M + polished_genes: 0 + genes_found: + - matMc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATtub + contig: KV454014.1 + start: 633666 + end: 646028 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454014.1 + start: 633666 + end: 646028 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454014.1 + start: 633666 + end: 646028 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - PAP1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454014.1 + start: 811497 + end: 811586 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454014.1 + start: 913930 + end: 914055 + idiomorph: M + polished_genes: 0 + genes_found: + - matMc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454014.1 + start: 1130757 + end: 1130846 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454014.1 + start: 1158937 + end: 1202891 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + - APN2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454014.1 + start: 1158937 + end: 1202891 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATyl + contig: KV454014.1 + start: 1158937 + end: 1202891 + idiomorph: A + polished_genes: 0 + genes_found: + - mata2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454011.1 + start: 44148 + end: 72723 + idiomorph: M + polished_genes: 0 + genes_found: + - matMc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454011.1 + start: 397912 + end: 398142 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454011.1 + start: 635816 + end: 636124 + idiomorph: M + polished_genes: 0 + genes_found: + - matMc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454011.1 + start: 693866 + end: 714133 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat2 + contig: KV454011.1 + start: 748337 + end: 748471 + idiomorph: P + polished_genes: 0 + genes_found: + - matPc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454011.1 + start: 1104369 + end: 1130936 + idiomorph: P + polished_genes: 0 + genes_found: + - mat1-Pc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454011.1 + start: 1353063 + end: 1359606 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat2 + contig: KV454011.1 + start: 1424445 + end: 1445648 + idiomorph: P + polished_genes: 0 + genes_found: + - matPc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454011.1 + start: 1513006 + end: 1534864 + idiomorph: M + polished_genes: 0 + genes_found: + - matMc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454011.1 + start: 1513006 + end: 1534864 + idiomorph: MATa + polished_genes: 0 + genes_found: + - MATA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454011.1 + start: 1936549 + end: 1947046 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat2 + contig: KV454011.1 + start: 2045886 + end: 2061234 + idiomorph: P + polished_genes: 0 + genes_found: + - matPc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454011.1 + start: 2090901 + end: 2170604 + idiomorph: P + polished_genes: 0 + genes_found: + - mat1-Pi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454011.1 + start: 2090901 + end: 2170604 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454011.1 + start: 2090901 + end: 2170604 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + - OBP1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454011.1 + start: 2090901 + end: 2170604 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - SLA2 + - MAT1-2-10 + - MAT1-2-4 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATtub + contig: KV454011.1 + start: 2090901 + end: 2170604 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - unnamed_flanking_ORF + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454011.1 + start: 2212690 + end: 2212785 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454011.1 + start: 2372833 + end: 2401432 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454011.1 + start: 2372833 + end: 2401432 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATtub + contig: KV454011.1 + start: 2372833 + end: 2401432 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454011.1 + start: 2372833 + end: 2401432 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454011.1 + start: 2453474 + end: 2453578 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATALPHA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454018.1 + start: 94181 + end: 112133 + idiomorph: MATa + polished_genes: 0 + genes_found: + - MATA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat2 + contig: KV454018.1 + start: 146582 + end: 202008 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454018.1 + start: 146582 + end: 202008 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA2 + - PAP1 + - PIK1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454018.1 + start: 146582 + end: 202008 + idiomorph: MAT1-1 + polished_genes: 0 + genes_found: + - MAT1-1-5 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454018.1 + start: 437332 + end: 485038 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454018.1 + start: 437332 + end: 485038 + idiomorph: MAT1-1 + polished_genes: 0 + genes_found: + - MAT1-1-1 + - SLA2 + - COX13 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454018.1 + start: 437332 + end: 485038 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATtub + contig: KV454018.1 + start: 437332 + end: 485038 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - SLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454018.1 + start: 594351 + end: 594476 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454018.1 + start: 676666 + end: 676935 + idiomorph: P + polished_genes: 0 + genes_found: + - mat1-Pc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454015.1 + start: 29705 + end: 29866 + idiomorph: P + polished_genes: 0 + genes_found: + - mat1-Pc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454015.1 + start: 591978 + end: 594986 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454015.1 + start: 639057 + end: 639155 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454015.1 + start: 788678 + end: 788791 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454015.1 + start: 959947 + end: 960381 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454015.1 + start: 959947 + end: 960381 + idiomorph: M + polished_genes: 0 + genes_found: + - matMc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454015.1 + start: 959947 + end: 960381 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATtub + contig: KV454015.1 + start: 959947 + end: 960381 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454015.1 + start: 959947 + end: 960381 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454015.1 + start: 995012 + end: 995182 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454015.1 + start: 1132327 + end: 1137883 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454015.1 + start: 1242312 + end: 1242629 + idiomorph: P + polished_genes: 0 + genes_found: + - mat1-Pi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat2 + contig: KV454015.1 + start: 1242312 + end: 1242629 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454015.1 + start: 1242312 + end: 1242629 + idiomorph: MATa + polished_genes: 0 + genes_found: + - MATA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454015.1 + start: 1242312 + end: 1242629 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATyl + contig: KV454015.1 + start: 1242312 + end: 1242629 + idiomorph: A + polished_genes: 0 + genes_found: + - mata1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454016.1 + start: 3613 + end: 3726 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454016.1 + start: 30734 + end: 66537 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + - MTLA2 + - PIK1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454016.1 + start: 235814 + end: 271194 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454016.1 + start: 235814 + end: 271194 + idiomorph: MAT1-2 + polished_genes: 1 + genes_found: + - MAT1-2-1 + - COX13 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454016.1 + start: 235814 + end: 271194 + idiomorph: M + polished_genes: 0 + genes_found: + - matMc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454016.1 + start: 235814 + end: 271194 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATtub + contig: KV454016.1 + start: 235814 + end: 271194 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATyl + contig: KV454016.1 + start: 235814 + end: 271194 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - apn2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454016.1 + start: 301776 + end: 301940 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454016.1 + start: 667027 + end: 682552 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454016.1 + start: 852065 + end: 852238 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454016.1 + start: 878080 + end: 894070 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454016.1 + start: 971767 + end: 971913 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454016.1 + start: 1028201 + end: 1028317 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATALPHA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454017.1 + start: 789567 + end: 796185 + idiomorph: alpha + polished_genes: 1 + genes_found: + - MTLalpha1 + - PAP1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454017.1 + start: 971149 + end: 971244 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454012.1 + start: 27291 + end: 27488 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454012.1 + start: 324645 + end: 344191 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454012.1 + start: 472523 + end: 483990 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454012.1 + start: 472523 + end: 483990 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATALPHA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454012.1 + start: 524736 + end: 524948 + idiomorph: MATa + polished_genes: 0 + genes_found: + - MATA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454012.1 + start: 524736 + end: 524948 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATyl + contig: KV454012.1 + start: 524736 + end: 524948 + idiomorph: A + polished_genes: 0 + genes_found: + - mata1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat2 + contig: KV454012.1 + start: 602414 + end: 608262 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454012.1 + start: 602414 + end: 608262 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATALPHA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454012.1 + start: 602414 + end: 608262 + idiomorph: MAT1-1 + polished_genes: 0 + genes_found: + - MAT1-1-4 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454012.1 + start: 602414 + end: 608262 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - PIK1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454012.1 + start: 666874 + end: 666999 + idiomorph: MATa + polished_genes: 0 + genes_found: + - MATA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454012.1 + start: 1044757 + end: 1050526 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454012.1 + start: 1086975 + end: 1099512 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454012.1 + start: 1438729 + end: 1438845 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454012.1 + start: 1587973 + end: 1588290 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454012.1 + start: 1686907 + end: 1696308 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATALPHA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454012.1 + start: 1813504 + end: 1843051 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454012.1 + start: 1813504 + end: 1843051 + idiomorph: P + polished_genes: 0 + genes_found: + - mat1-Pi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat2 + contig: KV454012.1 + start: 1813504 + end: 1843051 + idiomorph: P + polished_genes: 0 + genes_found: + - matPi + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454012.1 + start: 1813504 + end: 1843051 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454012.1 + start: 1813504 + end: 1843051 + idiomorph: MAT1-1 + polished_genes: 0 + genes_found: + - MAT1-1-1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454012.1 + start: 1813504 + end: 1843051 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454012.1 + start: 2015851 + end: 2021549 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454013.1 + start: 39375 + end: 63235 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454013.1 + start: 291174 + end: 291329 + idiomorph: P + polished_genes: 0 + genes_found: + - mat1-Pc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454013.1 + start: 442369 + end: 470331 + idiomorph: MATalpha + polished_genes: 0 + genes_found: + - MATALPHA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454013.1 + start: 537589 + end: 537720 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MAT + contig: KV454013.1 + start: 578780 + end: 580606 + idiomorph: MAT1-2 + polished_genes: 0 + genes_found: + - MAT1-2-1 + - APN2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454013.1 + start: 606390 + end: 606485 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454013.1 + start: 675858 + end: 675965 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATALPHA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MATsc + contig: KV454013.1 + start: 878903 + end: 901828 + idiomorph: undetermined + polished_genes: 0 + genes_found: + - MATALPHA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:mat1 + contig: KV454013.1 + start: 964404 + end: 964667 + idiomorph: M + polished_genes: 0 + genes_found: + - mat1-Mc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454013.1 + start: 1180717 + end: 1209395 + idiomorph: alpha + polished_genes: 0 + genes_found: + - MTLalpha1 + - PIK1 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:PM + contig: KV454013.1 + start: 1388470 + end: 1396568 + idiomorph: M + polished_genes: 0 + genes_found: + - matMc + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +- family: Ascomycota:MTL + contig: KV454013.1 + start: 1636639 + end: 1636794 + idiomorph: A + polished_genes: 0 + genes_found: + - MTLA2 + withheld_reason: modelled_gene_bar + idiomorph_classifier: null +assembly_gap_at_locus: +- family: Ascomycota:MATtub + contig: KV454014.1 + start: 241410 + end: 241814 + n_bases: 405 + anchors: + - SLA2 +zygosity: null +two_idiomorphs: [] +families_attempted: +- Ascomycota:MAT +- Ascomycota:PM +- Ascomycota:mat1 +- Ascomycota:mat2 +- Ascomycota:mat3 +- Ascomycota:MATsc +- Ascomycota:MATyl +- Ascomycota:MTL +- Ascomycota:MATtub +detected: +- family: Ascomycota:MATsc + contig: KV454014.1 + start: 185089 + end: 288393 + confidence: medium + idiomorph: MATa + idiomorph_class: MAT1-2 + idiomorph_candidates: + - idiomorph: MATa + score: 79.7 + - idiomorph: MATalpha + score: 60.1 + detection_pass: strict + locus_class: mat_locus + idiomorph_unmodelled: false + verification: null + merged_from: [] + subloci: [] + polished_genes: 4 + distinct_intervals: 5 + span_exceeds_plausible_bound: false + idiomorph_margin: 19.6 + idiomorph_classifier: null + idiomorph_resolutions: + - contig: KV454014.1 + winner: MAT1-1-3 + loser: MAT1-2-1 + winner_identity: 30.0 + loser_identity: 24.242 + overlap_fraction: 0.9696969696969697 + winner_coverage: 21.0 + loser_coverage: 17.0 + basis: first_pass_identity + winner_model_score: null + loser_model_score: null + - contig: KV454014.1 + winner: MATA2 + loser: MATALPHA2 + winner_identity: 30.693 + loser_identity: 27.451 + overlap_fraction: 1.0 + winner_coverage: 55.0 + loser_coverage: 49.0 + basis: first_pass_identity + winner_model_score: null + loser_model_score: null + ambiguous_with: + - Ascomycota:PM + - Ascomycota:mat1 + - Ascomycota:MTL + - Ascomycota:MAT + - Ascomycota:mat2 + - Ascomycota:MATyl + genes_found: + - MATALPHA1 + - MATALPHA2 + - MATA1 + - MATA2 + - sla2 + genes_missing: [] + genes_not_searchable: + - cha1 + fragmented: false + reference_records: + - 28985_nrrl-y-1140_MATsc_MATa + - 4932_s288c_MATsc_HMRa + - 4956_da2_MATsc_MATalpha + segments: + - contig: KV454014.1 + start: 185089 + end: 288393 + contig_edge_distance: 185088 + gene_evidence: + - gene: MATALPHA1 + role: core_MAT + contig: KV454014.1 + start: 227925 + end: 228107 + strand: '-' + identity: 37.705 + coverage: 30.0 + reference_record: 4956_da2_MATsc_MATalpha + method: tblastn_genome + status: unpolished + alternate_model: null + exons: null + evalue: 1.4e-10 + bitscore: 60.1 + - gene: MATA2 + role: core_MAT + contig: KV454014.1 + start: 229114 + end: 229455 + strand: '-' + identity: 40.660000000000004 + coverage: null + reference_record: 4932_s288c_MATsc_HMRa + method: miniprot_refine + status: polished_single + alternate_model: null + exons: + - start: 229114 + end: 229210 + - start: 229307 + end: 229455 + evalue: 4.47e-08 + bitscore: 48.5 + - gene: MATALPHA2 + role: core_MAT + contig: KV454014.1 + start: 229306 + end: 229617 + strand: '-' + identity: 26.92 + coverage: null + reference_record: 4956_da2_MATsc_MATalpha + method: exonerate_refine + status: polished_disagree + alternate_model: + contig: KV454014.1 + start: 229114 + end: 229593 + strand: '-' + exons: + - start: 229114 + end: 229210 + - start: 229307 + end: 229593 + identity: 32.81 + method: miniprot_refine + exons: + - start: 229306 + end: 229617 + evalue: 1.6 + bitscore: 28.5 + - gene: MATA1 + role: core_MAT + contig: KV454014.1 + start: 236074 + end: 236358 + strand: + + identity: 44.21 + coverage: null + reference_record: 4932_s288c_MATsc_HMRa + method: exonerate_refine + status: polished_disagree + alternate_model: + contig: KV454014.1 + start: 236062 + end: 236361 + strand: + + exons: + - start: 236062 + end: 236361 + identity: 43.0 + method: miniprot_refine + exons: + - start: 236074 + end: 236358 + evalue: 7.79e-19 + bitscore: 79.7 + - gene: sla2 + role: flanking_conserved + contig: KV454014.1 + start: 238254 + end: 288393 + strand: '-' + identity: 45.52 + coverage: null + reference_record: 28985_nrrl-y-1140_MATsc_MATa + method: exonerate_refine + status: polished_disagree + alternate_model: + contig: KV454014.1 + start: 236553 + end: 238667 + strand: '-' + exons: + - start: 236553 + end: 236599 + - start: 237335 + end: 237805 + - start: 238154 + end: 238667 + identity: 29.509999999999998 + method: miniprot_refine + exons: + - start: 238254 + end: 238394 + - start: 238461 + end: 238667 + - start: 288340 + end: 288393 + evalue: 8.86e-18 + bitscore: 87.0 +- family: Ascomycota:MTL + contig: KV454014.1 + start: 185089 + end: 282802 + confidence: medium + idiomorph: alpha + idiomorph_class: MAT1-1 + idiomorph_candidates: + - idiomorph: alpha + score: 80.1 + - idiomorph: A + score: 69.3 + detection_pass: strict + locus_class: mat_locus + idiomorph_unmodelled: false + verification: null + merged_from: [] + subloci: [] + polished_genes: 2 + distinct_intervals: 5 + span_exceeds_plausible_bound: false + idiomorph_margin: 10.8 + idiomorph_classifier: null + idiomorph_resolutions: + - contig: KV454014.1 + winner: MAT1-1-3 + loser: MAT1-2-1 + winner_identity: 30.0 + loser_identity: 24.242 + overlap_fraction: 0.9696969696969697 + winner_coverage: 21.0 + loser_coverage: 17.0 + basis: first_pass_identity + winner_model_score: null + loser_model_score: null + - contig: KV454014.1 + winner: MATA2 + loser: MATALPHA2 + winner_identity: 30.693 + loser_identity: 27.451 + overlap_fraction: 1.0 + winner_coverage: 55.0 + loser_coverage: 49.0 + basis: first_pass_identity + winner_model_score: null + loser_model_score: null + ambiguous_with: + - Ascomycota:PM + - Ascomycota:mat1 + - Ascomycota:MATsc + - Ascomycota:MAT + - Ascomycota:mat2 + - Ascomycota:MATyl + genes_found: + - MTLA1 + - MTLA2 + - MTLalpha1 + - MTLalpha2 + - PIK1 + genes_missing: [] + genes_not_searchable: [] + fragmented: false + reference_records: + - 4959_cbs767_MTL_A + - 5476_sc5314_MTL_A + - 5476_sc5314_MTL_alpha + segments: + - contig: KV454014.1 + start: 185089 + end: 282802 + contig_edge_distance: 185088 + gene_evidence: + - gene: PIK1 + role: flanking_variable + contig: KV454014.1 + start: 219632 + end: 219763 + strand: '-' + identity: 41.304 + coverage: 5.0 + reference_record: 5476_sc5314_MTL_A + method: tblastn_genome + status: unpolished + alternate_model: null + exons: null + evalue: 9.96e-11 + bitscore: 64.7 + - gene: MTLalpha1 + role: core_MAT + contig: KV454014.1 + start: 227907 + end: 228158 + strand: '-' + identity: 44.58 + coverage: null + reference_record: 4959_cbs767_MTL_A + method: exonerate_refine + status: polished_disagree + alternate_model: + contig: KV454014.1 + start: 227907 + end: 228392 + strand: '-' + exons: + - start: 227907 + end: 228392 + identity: 29.24 + method: miniprot_refine + exons: + - start: 227907 + end: 228158 + evalue: 8.63e-18 + bitscore: 80.1 + - gene: MTLalpha2 + role: core_MAT + contig: KV454014.1 + start: 229114 + end: 229461 + strand: '-' + identity: 27.119 + coverage: 44.0 + reference_record: 5476_sc5314_MTL_alpha + method: tblastn_genome + status: unpolished + alternate_model: null + exons: null + evalue: 0.14 + bitscore: 31.2 + - gene: MTLA2 + role: core_MAT + contig: KV454014.1 + start: 234541 + end: 234768 + strand: '-' + identity: 37.037 + coverage: 31.0 + reference_record: 4959_cbs767_MTL_A + method: tblastn_genome + status: unpolished + alternate_model: null + exons: null + evalue: 1.18e-07 + bitscore: 40.4 + - gene: MTLA1 + role: core_MAT + contig: KV454014.1 + start: 236185 + end: 236358 + strand: + + identity: 53.45 + coverage: null + reference_record: 5476_sc5314_MTL_A + method: exonerate_refine + status: polished_disagree + alternate_model: + contig: KV454014.1 + start: 235825 + end: 236358 + strand: + + exons: + - start: 235825 + end: 236358 + identity: 30.0 + method: miniprot_refine + exons: + - start: 236185 + end: 236358 + evalue: 4.0e-14 + bitscore: 69.3 +- family: Ascomycota:MAT + contig: KV454014.1 + start: 185089 + end: 282802 + confidence: high + idiomorph: MAT1-1 + idiomorph_class: MAT1-1 + idiomorph_candidates: + - idiomorph: MAT1-1 + score: 35.0 + - idiomorph: MAT1-2 + score: 23.5 + detection_pass: strict + locus_class: mat_locus + idiomorph_unmodelled: false + verification: null + merged_from: [] + subloci: [] + polished_genes: 2 + distinct_intervals: 5 + span_exceeds_plausible_bound: false + idiomorph_margin: 11.5 + idiomorph_classifier: null + idiomorph_resolutions: + - contig: KV454014.1 + winner: MAT1-1-3 + loser: MAT1-2-1 + winner_identity: 30.0 + loser_identity: 24.242 + overlap_fraction: 0.9696969696969697 + winner_coverage: 21.0 + loser_coverage: 17.0 + basis: first_pass_identity + winner_model_score: null + loser_model_score: null + - contig: KV454014.1 + winner: MATA2 + loser: MATALPHA2 + winner_identity: 30.693 + loser_identity: 27.451 + overlap_fraction: 1.0 + winner_coverage: 55.0 + loser_coverage: 49.0 + basis: first_pass_identity + winner_model_score: null + loser_model_score: null + ambiguous_with: + - Ascomycota:PM + - Ascomycota:mat1 + - Ascomycota:MATsc + - Ascomycota:MTL + - Ascomycota:mat2 + - Ascomycota:MATyl + genes_found: + - MAT1-1-1 + - MAT1-2-1 + - SLA2 + - MAT1-1-3 + - MAT1-1-4 + genes_missing: + - APN2 + - COX13 + genes_not_searchable: [] + fragmented: false + reference_records: + - 115814_voucher-101_MAT_MAT1-2 + - 199306_rmscc1040_MAT_MAT1-1 + - 5141_74-ors-a_MAT_MAT1-1 + - 5518_3639_MAT_combined + - 746128_a1163_MAT_MAT1-1 + segments: + - contig: KV454014.1 + start: 185089 + end: 282802 + contig_edge_distance: 185088 + gene_evidence: + - gene: MAT1-1-4 + role: core_MAT + contig: KV454014.1 + start: 185089 + end: 185199 + strand: + + identity: 35.135 + coverage: 15.0 + reference_record: 199306_rmscc1040_MAT_MAT1-1 + method: tblastn_genome + status: unpolished + alternate_model: null + exons: null + evalue: 6.6 + bitscore: 26.9 + - gene: MAT1-1-1 + role: core_MAT + contig: KV454014.1 + start: 227961 + end: 228128 + strand: '-' + identity: 35.709999999999994 + coverage: null + reference_record: 5518_3639_MAT_combined + method: miniprot_refine + status: polished_single + alternate_model: null + exons: + - start: 227961 + end: 228128 + evalue: 0.028 + bitscore: 35.0 + - gene: MAT1-2-1 + role: core_MAT + contig: KV454014.1 + start: 234336 + end: 234443 + strand: '-' + identity: 27.778 + coverage: 10.0 + reference_record: 115814_voucher-101_MAT_MAT1-2 + method: tblastn_genome + status: unpolished + alternate_model: null + exons: null + evalue: 0.18 + bitscore: 23.5 + - gene: MAT1-1-3 + role: core_MAT + contig: KV454014.1 + start: 234553 + end: 234750 + strand: '-' + identity: 30.0 + coverage: 21.0 + reference_record: 5141_74-ors-a_MAT_MAT1-1 + method: tblastn_genome + status: unpolished + alternate_model: null + exons: null + evalue: 0.089 + bitscore: 31.6 + - gene: SLA2 + role: flanking_conserved + contig: KV454014.1 + start: 236568 + end: 238640 + strand: '-' + identity: 57.68 + coverage: null + reference_record: 746128_a1163_MAT_MAT1-1 + method: exonerate_refine + status: polished_disagree + alternate_model: + contig: KV454014.1 + start: 236568 + end: 279373 + strand: '-' + exons: + - start: 236568 + end: 238640 + - start: 279061 + end: 279373 + identity: 52.370000000000005 + method: miniprot_refine + exons: + - start: 236568 + end: 238640 + evalue: 0.0 + bitscore: 727.0 +- family: Ascomycota:MATyl + contig: KV454014.1 + start: 185089 + end: 282802 + confidence: medium + idiomorph: A + idiomorph_class: unassigned + idiomorph_candidates: + - idiomorph: A + score: 46.2 + detection_pass: strict + locus_class: partial_locus + idiomorph_unmodelled: false + verification: null + merged_from: [] + subloci: [] + polished_genes: 2 + distinct_intervals: 2 + span_exceeds_plausible_bound: false + idiomorph_margin: 3.242000000000001 + idiomorph_classifier: null + idiomorph_resolutions: + - contig: KV454014.1 + winner: MAT1-1-3 + loser: MAT1-2-1 + winner_identity: 30.0 + loser_identity: 24.242 + overlap_fraction: 0.9696969696969697 + winner_coverage: 21.0 + loser_coverage: 17.0 + basis: first_pass_identity + winner_model_score: null + loser_model_score: null + - contig: KV454014.1 + winner: MATA2 + loser: MATALPHA2 + winner_identity: 30.693 + loser_identity: 27.451 + overlap_fraction: 1.0 + winner_coverage: 55.0 + loser_coverage: 49.0 + basis: first_pass_identity + winner_model_score: null + loser_model_score: null + ambiguous_with: + - Ascomycota:PM + - Ascomycota:mat1 + - Ascomycota:MATsc + - Ascomycota:MTL + - Ascomycota:MAT + - Ascomycota:mat2 + genes_found: + - mata1 + - sla2 + genes_missing: + - mata2 + - apn2 + genes_not_searchable: [] + fragmented: false + reference_records: + - 4952_w29_MATyl_A + segments: + - contig: KV454014.1 + start: 185089 + end: 282802 + contig_edge_distance: 185088 + gene_evidence: + - gene: mata1 + role: core_MAT + contig: KV454014.1 + start: 236230 + end: 236352 + strand: + + identity: 46.34 + coverage: null + reference_record: 4952_w29_MATyl_A + method: exonerate_refine + status: polished_single + alternate_model: null + exons: + - start: 236230 + end: 236352 + evalue: 1.3e-06 + bitscore: 46.2 + - gene: sla2 + role: flanking_conserved + contig: KV454014.1 + start: 236556 + end: 237809 + strand: '-' + identity: 60.19 + coverage: null + reference_record: 4952_w29_MATyl_A + method: exonerate_refine + status: polished_disagree + alternate_model: + contig: KV454014.1 + start: 236556 + end: 238646 + strand: '-' + exons: + - start: 236556 + end: 238646 + identity: 56.220000000000006 + method: miniprot_refine + exons: + - start: 236556 + end: 237809 + evalue: 0.0 + bitscore: 694.0 +not_detected: +- family: Ascomycota:PM + reason: best cluster carried 0 modelled gene(s), below the 2 required to report + a locus; none of its genes could be modelled + best_fraction_found: 1.0 + genes_found: + - matPi + genes_missing: [] + genes_not_searchable: [] +- family: Ascomycota:mat1 + reason: best cluster carried 0 modelled gene(s), below the 2 required to report + a locus; none of its genes could be modelled + best_fraction_found: 1.0 + genes_found: + - mat1-Mc + genes_missing: [] + genes_not_searchable: + - mat1-Mi +- family: Ascomycota:mat2 + reason: best cluster carried 0 modelled gene(s), below the 2 required to report + a locus; none of its genes could be modelled + best_fraction_found: 0.5 + genes_found: + - matPi + genes_missing: + - matPc + genes_not_searchable: [] +- family: Ascomycota:mat3 + reason: no reference protein for this family is in the search set (none curated, + or all withheld) + best_fraction_found: 0.0 + genes_found: [] + genes_missing: + - matMc + - matMi + genes_not_searchable: [] +- family: Ascomycota:MATtub + reason: best cluster carried 1 modelled gene(s), below the 2 required to report + a locus; only 1 of its genes could be modelled + best_fraction_found: 0.3333333333333333 + genes_found: + - MAT1-2-1 + - SLA2 + genes_missing: + - Rsm22-like + - unnamed_flanking_ORF + genes_not_searchable: [] diff --git a/results/2026-10-04_offline_taxonomy_check/run.slurm b/results/2026-10-04_offline_taxonomy_check/run.slurm new file mode 100644 index 00000000..ddcfff09 --- /dev/null +++ b/results/2026-10-04_offline_taxonomy_check/run.slurm @@ -0,0 +1,10 @@ +#!/bin/bash +#SBATCH -p exfab -A exfab -c 4 --mem 8gb -t 2:00:00 -J offtax --output=/bigdata/stajichlab/jstajich/projects/MATPredict/results/2026-10-04_offline_taxonomy_check/slurm_%j.log +# Offline taxonomy check: detect with --taxid, MATPREDICT_OFFLINE=1, empty NCBI cache, no network use. +set -e +export PATH=/bigdata/stajichlab/jstajich/projects/MATPredict/.pixi/envs/default/bin:$PATH PYTHONPATH=/bigdata/stajichlab/jstajich/projects/MATPredict/.claude/worktrees/offline-taxonomy/src MATPREDICT_DB_ROOT=/bigdata/stajichlab/jstajich/projects/MATPredict/.claude/worktrees/offline-taxonomy/db +export MATPREDICT_TAXONOMY=/bigdata/stajichlab/jstajich/projects/MATPredict/results/ncbi_taxonomy_2026-10-01.tsv.zst MATPREDICT_OFFLINE=1 +export MATPREDICT_CACHE_DIR=${SCRATCH:?}/emptycache; mkdir -p $MATPREDICT_CACHE_DIR +zcat /bigdata/stajichlab/shared/projects/BFD/Fungi_BFD_runs/input_clean_genomes/GCA_001661245.1_Pacta1_2.fa.gz > $SCRATCH/g.fna +/usr/bin/time -f '%e s %M KB' python -m MATPredict detect --genome $SCRATCH/g.fna --taxid 669874 --out-dir /bigdata/stajichlab/jstajich/projects/MATPredict/results/2026-10-04_offline_taxonomy_check/pacta > /bigdata/stajichlab/jstajich/projects/MATPredict/results/2026-10-04_offline_taxonomy_check/stdout.log 2> /bigdata/stajichlab/jstajich/projects/MATPredict/results/2026-10-04_offline_taxonomy_check/stderr.log +echo "cache files after run: $(ls $MATPREDICT_CACHE_DIR | wc -l)" diff --git a/results/2026-10-04_offline_taxonomy_check/stderr.log.zst b/results/2026-10-04_offline_taxonomy_check/stderr.log.zst new file mode 100644 index 0000000000000000000000000000000000000000..cd16a61f70057015274144924ca2f7dacdfb09a5 GIT binary patch literal 208 zcmV;>05AV2wJ-f-0{|5U074EeA}~#?1B2B9-D!vsvkPzxtCWYE3r^GiXBkgS*e?8^WoE~8;pVkKdOe}QraXBFwB3_*=>D1 z int: return 0 +def _cmd_build_taxonomy(args: argparse.Namespace) -> int: + from MATPredict.db.local_taxonomy import build_table + counts = build_table(Path(args.taxdump), Path(args.out), snapshot=args.snapshot) + print(f"built {args.out}: {counts['taxa']} taxa, {counts['merged']} merged ids, " + f"snapshot {counts['snapshot']}") + return 0 + + def register_subcommands(subparsers: argparse._SubParsersAction) -> None: """Register `curate-db` and its actions onto the top-level parser.""" curate_db = subparsers.add_parser("curate-db", help="Curate the MAT locus reference database") @@ -426,6 +434,15 @@ def register_subcommands(subparsers: argparse._SubParsersAction) -> None: draw_synteny.add_argument("--out", required=True, help="Output clinker plot HTML path") draw_synteny.set_defaults(func=_cmd_draw_synteny) + build_tax = action.add_parser( + "build-taxonomy", + help="Build the offline NCBI taxonomy table (lineage, phylum, genetic code) from a taxdump") + build_tax.add_argument("--taxdump", required=True, + help="taxdump directory (plain/.gz/.zst files), NCBI taxdmp_*.zip or taxdump.tar.gz") + build_tax.add_argument("--out", required=True, help="output table, e.g. ncbi_taxonomy.tsv.zst") + build_tax.add_argument("--snapshot", required=True, help="taxdump date, e.g. 2026-10-01 (written to reports)") + build_tax.set_defaults(func=_cmd_build_taxonomy) + build_db = action.add_parser("build-duckdb") build_db.add_argument("--out", required=False) build_db.set_defaults(func=_cmd_build_duckdb) diff --git a/src/MATPredict/db/local_taxonomy.py b/src/MATPredict/db/local_taxonomy.py new file mode 100644 index 00000000..e1ddf5c7 --- /dev/null +++ b/src/MATPredict/db/local_taxonomy.py @@ -0,0 +1,294 @@ +"""Offline NCBI taxonomy: a slim table built from an NCBI taxdump. + +`detect` needs three facts per taxid: the ancestor lineage (for +`taxonomic_scope` routing), the phylum name (phylum fallback) and the nuclear +genetic code. All three are in the taxdump's `nodes.dmp` (parent, rank, +genetic code id) and `names.dmp` (scientific name). This module writes them, +for every taxon, to one zstd-compressed text table and answers the three +questions from it, so a run needs no NCBI E-utilities call. Measured on the +2026-05 taxdump: 2.83 M taxa, 25 MB compressed. + +Table format (UTF-8 text, zstd-compressed): + + #matpredict-taxonomyv1snapshot=source= + Ntaxidparentrankgenetic_codescientific_name + Mold_taxidnew_taxid (merged.dmp) + +Lookups mirror `NcbiClient` (efetch db=taxonomy) exactly: +- lineage: ancestor taxids, root-first, without the root (1) and without the + taxid itself (efetch `LineageEx`); +- phylum: scientific name of the phylum-rank ANCESTOR (not the taxid itself); +- genetic code: the taxid's own nuclear code (`GCId`). +A merged taxid is answered for the taxid it was merged into, as efetch does. + +Where the table is: `$MATPREDICT_TAXONOMY`. With `$MATPREDICT_OFFLINE=1` a +taxid missing from the table raises `TaxidNotInSnapshot` (the router records +it in `routing_error`); otherwise the caller falls back to E-utilities. +""" +from __future__ import annotations + +import gzip +import io +import os +import subprocess +import tarfile +import zipfile +from array import array +from pathlib import Path + +FORMAT_TAG = "#matpredict-taxonomy" +FORMAT_VERSION = "v1" + + +class TaxidNotInSnapshot(LookupError): + """The taxid is not in the local taxonomy snapshot (newer, deleted or wrong).""" + + +# --------------------------------------------------------------------- zstd I/O + +def _zstd_decompress(data: bytes) -> bytes: + try: + from compression import zstd # Python >= 3.14 + return zstd.decompress(data) + except ImportError: + pass + try: + import zstandard + return zstandard.ZstdDecompressor().decompressobj().decompress(data) + except ImportError: + pass + return subprocess.run(["zstd", "-dc"], input=data, capture_output=True, check=True).stdout + + +def _zstd_compress(data: bytes, level: int = 19) -> bytes: + try: + from compression import zstd # Python >= 3.14 + return zstd.compress(data, level=level) + except ImportError: + pass + try: + import zstandard + return zstandard.ZstdCompressor(level=level).compress(data) + except ImportError: + pass + return subprocess.run(["zstd", "-q", f"-{level}", "-c"], input=data, capture_output=True, + check=True).stdout + + +def _open_text(path: Path): + """Open a table (plain or zstd) for streamed text reading.""" + with open(path, "rb") as fh: + magic = fh.read(4) + if magic != b"\x28\xb5\x2f\xfd": + return open(path, encoding="utf-8") + try: + from compression import zstd # Python >= 3.14: streamed + return zstd.open(path, "rt", encoding="utf-8") + except ImportError: + return io.StringIO(_zstd_decompress(Path(path).read_bytes()).decode("utf-8")) + + +def _decode(name: str, data: bytes) -> str: + """Bytes of a dump file, plain or compressed (detected from the magic bytes).""" + if data[:2] == b"\x1f\x8b": + data = gzip.decompress(data) + elif data[:4] == b"\x28\xb5\x2f\xfd": + data = _zstd_decompress(data) + return data.decode("utf-8") + + +def _read_dump_files(taxdump: Path) -> dict[str, str]: + """`nodes.dmp`, `names.dmp` and `merged.dmp` (optional) from a taxdump. + + `taxdump` is a directory (files plain, `.gz` or `.zst`), an NCBI archive + `.zip` (`taxdmp_YYYY-MM-DD.zip`) or a `.tar.gz` (`taxdump.tar.gz`). + """ + wanted = ("nodes.dmp", "names.dmp", "merged.dmp") + out: dict[str, str] = {} + taxdump = Path(taxdump) + if taxdump.is_dir(): + for name in wanted: + for cand in (name, name + ".gz", name + ".zst"): + p = taxdump / cand + if p.exists(): + out[name] = _decode(cand, p.read_bytes()) + break + elif zipfile.is_zipfile(taxdump): + with zipfile.ZipFile(taxdump) as zf: + for name in wanted: + if name in zf.namelist(): + out[name] = _decode(name, zf.read(name)) + elif tarfile.is_tarfile(taxdump): + with tarfile.open(taxdump) as tf: + for name in wanted: + try: + member = tf.getmember(name) + except KeyError: + continue + out[name] = _decode(name, tf.extractfile(member).read()) + else: + raise ValueError(f"not a taxdump directory, .zip or .tar.gz: {taxdump}") + for name in ("nodes.dmp", "names.dmp"): + if name not in out: + raise FileNotFoundError(f"{name} not found in {taxdump}") + return out + + +def _dmp_rows(text: str): + """Fields of NCBI `.dmp` rows (`\\t|\\t` separated, `\\t|` at line end).""" + for line in text.splitlines(): + if line: + yield line.rstrip("\t|").split("\t|\t") + + +def build_table(taxdump: Path, out_path: Path, snapshot: str, source: str | None = None) -> dict: + """Write the slim table for every taxon in `taxdump`. Returns counts.""" + files = _read_dump_files(taxdump) + names: dict[str, str] = {} + for f in _dmp_rows(files["names.dmp"]): + if f[3] == "scientific name": + names[f[0]] = f[1] + buf = io.StringIO() + max_taxid = max(int(f[0]) for f in _dmp_rows(files["nodes.dmp"])) + buf.write(f"{FORMAT_TAG}\t{FORMAT_VERSION}\tsnapshot={snapshot}\t" + f"source={source or Path(taxdump).name}\tmax_taxid={max_taxid}\n") + n_nodes = 0 + for f in _dmp_rows(files["nodes.dmp"]): + # nodes.dmp: tax_id, parent tax_id, rank, embl code, division id, + # inherited div flag, genetic code id, ... + buf.write(f"N\t{f[0]}\t{f[1]}\t{f[2]}\t{f[6]}\t{names.get(f[0], '')}\n") + n_nodes += 1 + n_merged = 0 + for f in _dmp_rows(files.get("merged.dmp", "")): + buf.write(f"M\t{f[0]}\t{f[1]}\n") + n_merged += 1 + out_path = Path(out_path) + out_path.parent.mkdir(parents=True, exist_ok=True) + tmp = out_path.with_name(out_path.name + ".tmp") + tmp.write_bytes(_zstd_compress(buf.getvalue().encode("utf-8"))) + tmp.replace(out_path) + return {"taxa": n_nodes, "merged": n_merged, "snapshot": snapshot} + + +# ----------------------------------------------------------------------- lookups + +class LocalTaxonomy: + """Lineage, phylum name and genetic code from a slim table. + + Memory: parent and genetic code are int arrays indexed by taxid (about 4 + bytes x max taxid each); names are kept only for phylum-rank taxa. + """ + + def __init__(self, path: Path): + self.path = Path(path) + self._is_phylum: set[int] = set() + self._phylum_name: dict[int, str] = {} + self._merged: dict[int, int] = {} + self.n_taxa = 0 + with _open_text(self.path) as fh: + header = fh.readline().rstrip("\n").split("\t") + if header[0] != FORMAT_TAG or header[1] != FORMAT_VERSION: + raise ValueError(f"not a {FORMAT_TAG} {FORMAT_VERSION} table: {self.path}") + meta = dict(h.split("=", 1) for h in header[2:] if "=" in h) + self.snapshot = meta.get("snapshot", "unknown") + self.source = meta.get("source", "") + size = int(meta.get("max_taxid", 0)) + 1 + # Streamed line by line into int arrays indexed by taxid, so memory + # stays near 2 x 4 bytes x max taxid (measured below in the tests' + # docstring) instead of a list of 3 M parsed rows. + self._parent = array("i", [-1]) * size + self._code = array("i", [0]) * size + for line in fh: + kind, _, rest = line.partition("\t") + if kind == "N": + taxid, parent, rank, code, name = rest.rstrip("\n").split("\t", 4) + t = int(taxid) + if t >= len(self._parent): # table without max_taxid + grow = t + 1 - len(self._parent) + self._parent.extend([-1] * grow) + self._code.extend([0] * grow) + self._parent[t] = int(parent) + self._code[t] = int(code) if code else 0 + if rank == "phylum": + self._is_phylum.add(t) + self._phylum_name[t] = name + self.n_taxa += 1 + elif kind == "M": + old, new = rest.rstrip("\n").split("\t") + self._merged[int(old)] = int(new) + + def _resolve(self, taxid: int) -> int: + taxid = self._merged.get(int(taxid), int(taxid)) + if taxid <= 0 or taxid >= len(self._parent) or self._parent[taxid] < 0: + raise TaxidNotInSnapshot( + f"taxid {taxid} is not in the local NCBI taxonomy snapshot {self.snapshot}") + return taxid + + def __contains__(self, taxid: int) -> bool: + try: + self._resolve(taxid) + return True + except TaxidNotInSnapshot: + return False + + def lineage(self, taxid: int) -> list[int]: + """Ancestors root-first, without the root (1) and without `taxid`.""" + t = self._resolve(taxid) + out = [] + x = self._parent[t] + while x > 1: + out.append(x) + nxt = self._parent[x] + if nxt == x: + break + x = nxt + return out[::-1] + + def phylum_name(self, taxid: int) -> str | None: + for a in self.lineage(taxid): + if a in self._is_phylum: + return self._phylum_name.get(a) or None + return None + + def genetic_code(self, taxid: int) -> int | None: + code = self._code[self._resolve(taxid)] + return code or None + + +_loaded: LocalTaxonomy | None = None +_loaded_path: str | None = None + + +def configured_path() -> Path | None: + """`$MATPREDICT_TAXONOMY` when set and the file exists, else None.""" + p = os.environ.get("MATPREDICT_TAXONOMY") + if p and Path(p).is_file(): + return Path(p) + return None + + +def offline() -> bool: + """True when `$MATPREDICT_OFFLINE` is 1/true/yes: never call E-utilities.""" + return os.environ.get("MATPREDICT_OFFLINE", "").strip().lower() in ("1", "true", "yes") + + +def get_local() -> LocalTaxonomy | None: + """The configured table, loaded once per process (None if not configured).""" + global _loaded, _loaded_path + p = configured_path() + if p is None: + return None + if _loaded is None or _loaded_path != str(p): + _loaded, _loaded_path = LocalTaxonomy(p), str(p) + return _loaded + + +def source_label() -> str: + """What answers taxonomy lookups in this process, for the report.""" + local = get_local() + if local is not None: + tail = "" if offline() else "; NCBI E-utilities for taxids not in it" + return f"local NCBI taxonomy snapshot {local.snapshot}{tail}" + if offline(): + return "none (offline, no local taxonomy table)" + return "NCBI E-utilities" diff --git a/src/MATPredict/db/taxonomy.py b/src/MATPredict/db/taxonomy.py index e04f670a..d03dbd44 100644 --- a/src/MATPredict/db/taxonomy.py +++ b/src/MATPredict/db/taxonomy.py @@ -95,10 +95,37 @@ def _get_default_ncbi_client(): return _default_ncbi_client +def _local_or_none(taxid: int): + """The local taxonomy table when it can answer for `taxid`, else None. + + Order: the table in `$MATPREDICT_TAXONOMY` (no network); then NCBI + E-utilities. With `$MATPREDICT_OFFLINE=1`, a taxid the table does not hold + (or no table at all) raises `TaxidNotInSnapshot` instead of calling NCBI; + the router records the message in `routing_error`. + """ + from MATPredict.db.local_taxonomy import TaxidNotInSnapshot, get_local, offline + + local = get_local() + if local is not None and taxid in local: + return local + if offline(): + if local is None: + raise TaxidNotInSnapshot( + "offline ($MATPREDICT_OFFLINE) and no local taxonomy table " + "($MATPREDICT_TAXONOMY); give --phylum and --genetic-code") + raise TaxidNotInSnapshot( + f"taxid {taxid} is not in the local NCBI taxonomy snapshot {local.snapshot} " + "and the run is offline; give --phylum and --genetic-code") + return None + + def default_lineage_taxids(taxid: int) -> list[int]: """Fetch `taxid`'s ancestor lineage as a list of taxids using a lazily-built default `NcbiClient` (see `_get_default_ncbi_client`). This is the default `lineage_taxids_resolver` for `detect.family_registry.route`.""" + local = _local_or_none(taxid) + if local is not None: + return local.lineage(taxid) return _get_default_ncbi_client().fetch_taxonomy_lineage(taxid) @@ -113,6 +140,9 @@ def default_lineage_phylum_name(taxid: int) -> str | None: asking for the lineage of the same taxid is a cache hit, not a second network request. """ + local = _local_or_none(taxid) + if local is not None: + return local.phylum_name(taxid) return _get_default_ncbi_client().fetch_taxonomy_phylum(taxid) @@ -123,4 +153,7 @@ def default_genetic_code(taxid: int) -> int | None: `default_lineage_taxids`, reading the identical efetch document, so a run that already routed by taxid pays nothing extra for this. """ + local = _local_or_none(taxid) + if local is not None: + return local.genetic_code(taxid) return _get_default_ncbi_client().fetch_taxonomy_genetic_code(taxid) diff --git a/src/MATPredict/detect/report.py b/src/MATPredict/detect/report.py index 29b4361e..292cdc8a 100644 --- a/src/MATPredict/detect/report.py +++ b/src/MATPredict/detect/report.py @@ -415,6 +415,14 @@ def _result_doc(r: DetectionResult) -> dict: } +def _taxonomy_source() -> str: + from MATPredict.db.local_taxonomy import source_label + try: + return source_label() + except Exception as exc: # an unreadable table must not lose the report + return f"error reading local taxonomy table: {type(exc).__name__}: {str(exc)[:120]}" + + def write_detection_report(outcome: DetectionOutcome, out_path: Path) -> None: # `routing_mode` is written next to `families_attempted` because the two # are only meaningful together: the list says WHICH families were searched, @@ -434,6 +442,9 @@ def write_detection_report(outcome: DetectionOutcome, out_path: Path) -> None: "not_searched_reason": outcome.not_searched_reason, "genetic_code": outcome.genetic_code, "genetic_code_error": outcome.genetic_code_error, + # What answered taxonomy lookups: a local NCBI snapshot (with its date) + # or NCBI E-utilities. Routing and the genetic code depend on it. + "taxonomy_source": _taxonomy_source(), # Loci that were built and then withheld for carrying fewer than # `pipeline.MIN_POLISHED_GENES` polished gene models. Written # unconditionally, including as 0, so a genome that reports one locus diff --git a/tests/db/test_local_taxonomy.py b/tests/db/test_local_taxonomy.py new file mode 100644 index 00000000..cff76bf0 --- /dev/null +++ b/tests/db/test_local_taxonomy.py @@ -0,0 +1,150 @@ +"""Offline NCBI taxonomy table (`MATPredict.db.local_taxonomy`). + +Curator ruling, 2026-10-04: embed a slim all-taxa table in the container so a +run needs no NCBI E-utilities call. + +Measured on the real NCBI archive taxdmp_2026-10-01.zip: 3,016,750 taxa, +101,411 merged ids, 23.7 MB table; load 1.96 s and 76 MB RSS. Against 8,185 +taxonomy documents in the run cache (efetch answers fetched 2026-09/10): +phylum 8,185/8,185 and genetic code 8,185/8,185 identical; lineage 8,179/8,185 +(the 6 differ because NCBI re-parented the taxa between the fetch and the +snapshot, e.g. C. gattii VGV 2268382 lost the no-rank node 3111917); 21 taxids +are deleted in the snapshot (delnodes.dmp). +""" +import gzip +import zipfile + +import pytest + +from MATPredict.db import local_taxonomy as lt + + +# root(1) - cellular(131567) - Eukaryota(2759) - Fungi(4751, kingdom) +# - Ascomycota(4890, phylum) - Pachysolen(4917, genus) - P. tannophilus(4918, code 26) +# - Mucoromycota(1913637, phylum) - Mucor(4830, genus) - M. circinelloides(4837) +NODES = [ + (1, 1, "no rank", 1), (131567, 1, "cellular root", 1), (2759, 131567, "domain", 1), + (4751, 2759, "kingdom", 1), (4890, 4751, "phylum", 1), (4917, 4890, "genus", 26), + (4918, 4917, "species", 26), (1913637, 4751, "phylum", 1), (4830, 1913637, "genus", 1), + (4837, 4830, "species", 1), +] +NAMES = {1: "root", 131567: "cellular organisms", 2759: "Eukaryota", 4751: "Fungi", + 4890: "Ascomycota", 4917: "Pachysolen", 4918: "Pachysolen tannophilus", + 1913637: "Mucoromycota", 4830: "Mucor", 4837: "Mucor circinelloides"} + + +def _dmp(rows): + return "".join("\t|\t".join(str(x) for x in r) + "\t|\n" for r in rows) + + +def _dump_texts(): + nodes = _dmp([(t, p, r, "", 0, 1, c, 1, 1, 1, 0, 0, "") for t, p, r, c in NODES]) + names = _dmp([(t, n, "", "scientific name") for t, n in NAMES.items()] + + [(4837, "Mucor racemosus f. circinelloides", "", "synonym")]) + merged = _dmp([(99999, 4837)]) + return {"nodes.dmp": nodes, "names.dmp": names, "merged.dmp": merged} + + +@pytest.fixture +def table(tmp_path): + d = tmp_path / "dump" + d.mkdir() + for name, text in _dump_texts().items(): + (d / name).write_text(text) + out = tmp_path / "tax.tsv.zst" + counts = lt.build_table(d, out, snapshot="2026-10-01") + assert counts == {"taxa": len(NODES), "merged": 1, "snapshot": "2026-10-01"} + return out + + +def test_lookups_match_efetch_semantics(table): + t = lt.LocalTaxonomy(table) + assert t.snapshot == "2026-10-01" + # efetch LineageEx: root-first, no root (1), not the taxid itself + assert t.lineage(4918) == [131567, 2759, 4751, 4890, 4917] + assert t.phylum_name(4918) == "Ascomycota" + assert t.genetic_code(4918) == 26 + assert t.phylum_name(4837) == "Mucoromycota" + # a phylum's own lineage has no phylum ancestor (efetch gives none either) + assert t.phylum_name(4890) is None + + +def test_merged_taxid_answers_for_its_new_taxid(table): + t = lt.LocalTaxonomy(table) + assert t.lineage(99999) == t.lineage(4837) + assert 99999 in t + + +def test_unknown_taxid_raises(table): + t = lt.LocalTaxonomy(table) + assert 123456 not in t + with pytest.raises(lt.TaxidNotInSnapshot, match="2026-10-01"): + t.lineage(123456) + + +@pytest.mark.parametrize("form", ["gz_dir", "zip"]) +def test_builds_from_compressed_dumps(tmp_path, form): + texts = _dump_texts() + if form == "gz_dir": + src = tmp_path / "d" + src.mkdir() + for name, text in texts.items(): + (src / (name + ".gz")).write_bytes(gzip.compress(text.encode())) + else: + src = tmp_path / "taxdmp_2026-10-01.zip" + with zipfile.ZipFile(src, "w") as zf: + for name, text in texts.items(): + zf.writestr(name, text) + out = tmp_path / "t.tsv.zst" + lt.build_table(src, out, snapshot="x") + assert lt.LocalTaxonomy(out).genetic_code(4918) == 26 + + +def test_default_resolvers_use_the_table_without_network(table, monkeypatch): + from MATPredict.db import taxonomy + monkeypatch.setenv("MATPREDICT_TAXONOMY", str(table)) + monkeypatch.setattr(taxonomy, "_get_default_ncbi_client", + lambda: (_ for _ in ()).throw(AssertionError("network used"))) + assert taxonomy.default_lineage_taxids(4918)[-1] == 4917 + assert taxonomy.default_lineage_phylum_name(4918) == "Ascomycota" + assert taxonomy.default_genetic_code(4918) == 26 + + +def test_offline_unknown_taxid_raises_instead_of_calling_ncbi(table, monkeypatch): + from MATPredict.db import taxonomy + monkeypatch.setenv("MATPREDICT_TAXONOMY", str(table)) + monkeypatch.setenv("MATPREDICT_OFFLINE", "1") + monkeypatch.setattr(taxonomy, "_get_default_ncbi_client", + lambda: (_ for _ in ()).throw(AssertionError("network used"))) + with pytest.raises(lt.TaxidNotInSnapshot, match="--phylum"): + taxonomy.default_lineage_taxids(123456) + + +def test_online_unknown_taxid_falls_back_to_ncbi(table, monkeypatch): + from MATPredict.db import taxonomy + monkeypatch.setenv("MATPREDICT_TAXONOMY", str(table)) + monkeypatch.delenv("MATPREDICT_OFFLINE", raising=False) + + class Client: + def fetch_taxonomy_lineage(self, taxid): + return [42] + monkeypatch.setattr(taxonomy, "_get_default_ncbi_client", lambda: Client()) + assert taxonomy.default_lineage_taxids(123456) == [42] + + +def test_offline_without_a_table_raises(monkeypatch): + from MATPredict.db import taxonomy + monkeypatch.delenv("MATPREDICT_TAXONOMY", raising=False) + monkeypatch.setenv("MATPREDICT_OFFLINE", "1") + with pytest.raises(lt.TaxidNotInSnapshot, match="no local taxonomy table"): + taxonomy.default_genetic_code(4918) + + +def test_source_label(table, monkeypatch): + monkeypatch.delenv("MATPREDICT_TAXONOMY", raising=False) + monkeypatch.delenv("MATPREDICT_OFFLINE", raising=False) + assert lt.source_label() == "NCBI E-utilities" + monkeypatch.setenv("MATPREDICT_TAXONOMY", str(table)) + assert lt.source_label().startswith("local NCBI taxonomy snapshot 2026-10-01") + monkeypatch.setenv("MATPREDICT_OFFLINE", "1") + assert lt.source_label() == "local NCBI taxonomy snapshot 2026-10-01" From ea88b68b2f2962cb94a00cfbd11941bda516d6ed Mon Sep 17 00:00:00 2001 From: Jason Stajich Date: Sun, 4 Oct 2026 15:52:54 -0700 Subject: [PATCH 2/3] require zstd --- pixi.lock | 64 +++++++++++++++++++++++++++---------------------------- pixi.toml | 1 + 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/pixi.lock b/pixi.lock index 735d1584..e46f54b4 100644 --- a/pixi.lock +++ b/pixi.lock @@ -110,7 +110,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb03c661_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.68.1-h74cf4be_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.34-pthreads_hf13c14d_2.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenssl-static-3.6.4-h7cc23a3_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenssl-static-3.6.5-h7cc23a3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libparu-1.0.0-hc6afc67_7100101.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.59-h922cc85_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libpython-3.13.15-hdc7f604_103_cp313.conda @@ -139,7 +139,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-2.5.3-py313hb5f73ae_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.4-heb1ab33_2.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.4-h781a0a9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.5-h781a0a9_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pcre-8.45-h9c3ff4c_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.47-h8b3dc9c_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-10_h7cc23a3_perl5.conda @@ -206,19 +206,19 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/setuptools-84.0.0-pyh332efcf_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/six-1.17.0-pyhe01879c_1.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.1-pyh5ded981_0.conda - pypi: ./ - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/21/61dd2876bbaa69cf77d7b5c620e52e8b25faae7096f4d2e4a812b52095d7/duckdb-1.5.6-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/36/76fab39973ee11e7f9f357c55138197bb01c86f6502cb76487e3b4f42db0/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl test: channels: @@ -323,7 +323,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/libmpdec-4.0.0-hb03c661_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libnghttp2-1.68.1-h74cf4be_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenblas-0.3.34-pthreads_hf13c14d_2.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenssl-static-3.6.4-h7cc23a3_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/libopenssl-static-3.6.5-h7cc23a3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libparu-1.0.0-hc6afc67_7100101.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libpng-1.6.59-h922cc85_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/libpython-3.13.15-hdc7f604_103_cp313.conda @@ -352,7 +352,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/ncurses-6.6-hdb14827_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/numpy-2.5.3-py313hb5f73ae_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/openjpeg-2.5.4-heb1ab33_2.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.4-h781a0a9_0.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.5-h781a0a9_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/pcre-8.45-h9c3ff4c_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/linux-64/pcre2-10.47-h8b3dc9c_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/perl-5.32.1-10_h7cc23a3_perl5.conda @@ -427,19 +427,19 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/tomli-2.4.1-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.1-pyh5ded981_0.conda - pypi: ./ - pypi: https://files.pythonhosted.org/packages/0b/a7/71ac2cff56fec219ed242bb11b8efb69fcc4bec75db06fb7bfe35de520e6/certifi-2026.7.22-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/4b/85/41d27f234b82e47c167a5f6c0f62501dc0c640585ff4aba79e08a390336a/charset_normalizer-3.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - - pypi: https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/70/21/61dd2876bbaa69cf77d7b5c620e52e8b25faae7096f4d2e4a812b52095d7/duckdb-1.5.6-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/a0/36/76fab39973ee11e7f9f357c55138197bb01c86f6502cb76487e3b4f42db0/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl packages: - conda: https://conda.anaconda.org/bioconda/linux-64/augustus-3.5.0-pl5321h5653ebf_10.conda @@ -1958,19 +1958,19 @@ packages: - libopenblas >=0.3.34,<1.0a0 size: 6845411 timestamp: 1789141490688 -- conda: https://conda.anaconda.org/conda-forge/linux-64/libopenssl-static-3.6.4-h7cc23a3_0.conda - sha256: 332bc0cbf22ce800dbff2498217941f387bfd1f8040439e575f23f1a81febfdb - md5: 76030dff3efd7612926e51ee06a3060b +- conda: https://conda.anaconda.org/conda-forge/linux-64/libopenssl-static-3.6.5-h7cc23a3_0.conda + sha256: 33210cf94666ce96adba8c3568391d5670ce76d18412fc3c8ee35ceee8852878 + md5: bc025d2d77fc63e04fcd088cbef2127f depends: - __glibc >=2.17,<3.0.a0 - libgcc >=15 - - openssl 3.6.4 h781a0a9_0 + - openssl 3.6.5 h781a0a9_0 license: Apache-2.0 license_family: Apache purls: [] run_exports: {} - size: 2868999 - timestamp: 1787698792926 + size: 2876685 + timestamp: 1791098062503 - conda: https://conda.anaconda.org/conda-forge/linux-64/libparu-1.0.0-hc6afc67_7100101.conda sha256: 50144e87b95d1309d2043aa5bf02035b948b1ae9ec6ec44ee97b7aec1cccd70a md5: fd1d3e26c1b12c70f7449369ae3d9c1a @@ -2430,9 +2430,9 @@ packages: - openjpeg >=2.5.4,<3.0a0 size: 391242 timestamp: 1788425045145 -- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.4-h781a0a9_0.conda - sha256: 4747b2d6a8336f52343bceb8a1ebf41a9e6e665b9d2e3f989972de53a310599e - md5: 16e3034a330cc625f2c685edec60cf4e +- conda: https://conda.anaconda.org/conda-forge/linux-64/openssl-3.6.5-h781a0a9_0.conda + sha256: 9c1533f1cf784f0e250919c0c9762d0b0e6bc727a79290ffb7cea1ceca1d038c + md5: 5a51a2ea653f120240b1a9bf76e0aae9 depends: - __glibc >=2.17,<3.0.a0 - ca-certificates @@ -2442,9 +2442,9 @@ packages: purls: [] run_exports: weak: - - openssl >=3.6.4,<4.0a0 - size: 3202955 - timestamp: 1787698780103 + - openssl >=3.6.5,<4.0a0 + size: 3195299 + timestamp: 1791098052301 - conda: https://conda.anaconda.org/conda-forge/linux-64/pcre-8.45-h9c3ff4c_0.tar.bz2 sha256: 8f35c244b1631a4f31fb1d66ab6e1d9bfac0ca9b679deced1112c7225b3ad138 md5: c05d1820a6d34ff07aaaab7a9b7eddaa @@ -3517,19 +3517,19 @@ packages: run_exports: {} size: 118849 timestamp: 1784250406640 -- conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.0-pyhcf101f3_0.conda - sha256: 210bd31c22bb88f5e2a167df24c95bb5f152b2ada7502f9b8c49d1f5366db423 - md5: ba3dcdc8584155c97c648ae9c044b7a3 +- conda: https://conda.anaconda.org/conda-forge/noarch/zipp-4.1.1-pyh5ded981_0.conda + sha256: 3aa601ef121f42dcb79400375e72f833ddc68bc6243f3e5535168b2d69cc9c70 + md5: 6cad4d2cec9b7b64538c225ac91d2e8a depends: - - python >=3.10 + - python >=3.11 - python license: MIT license_family: MIT purls: - - pkg:pypi/zipp?source=hash-mapping + - pkg:pypi/zipp?source=compressed-mapping run_exports: {} - size: 24190 - timestamp: 1779159948016 + size: 24839 + timestamp: 1791091060207 - pypi: ./ name: matpredict requires_dist: @@ -3568,11 +3568,6 @@ packages: version: 3.5.2 sha256: 7218e8f32b0956cfcd048fd42d9d5779809745ca1d86113ca56f66e7ae1549c4 requires_python: '>=3.7' -- pypi: https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl - name: rpds-py - version: 2026.6.3 - sha256: acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf - requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl name: idna version: '3.20' @@ -3645,6 +3640,11 @@ packages: - pysocks>=1.5.6,!=1.5.7,<2.0 ; extra == 'socks' - backports-zstd>=1.0.0 ; python_full_version < '3.14' and extra == 'zstd' requires_python: '>=3.10' +- pypi: https://files.pythonhosted.org/packages/a0/36/76fab39973ee11e7f9f357c55138197bb01c86f6502cb76487e3b4f42db0/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl + name: rpds-py + version: 2026.9.1 + sha256: 7868b85224291c6cb6759f9b5adb9745f486d226f62b16a614dd5a2a5ab2b35b + requires_python: '>=3.11' - pypi: https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl name: requests version: 2.34.2 diff --git a/pixi.toml b/pixi.toml index bbab5c7b..9e13f2a7 100644 --- a/pixi.toml +++ b/pixi.toml @@ -5,6 +5,7 @@ platforms = ["linux-64"] [dependencies] python = ">=3.11" +zstd = "*" pip = "*" taxonkit = "*" # Search binaries `matpredict detect` shells out to (see detect/search.py): From 018a549f7cb4186e1f406c21db628841c5eccfc8 Mon Sep 17 00:00:00 2001 From: Jason Stajich Date: Sun, 4 Oct 2026 16:01:39 -0700 Subject: [PATCH 3/3] taxonomy: add zstandard so the table is read on the locked Python 3.13 CI image build failed in build-taxonomy with "No such file or directory: 'zstd'": pixi.lock pins Python 3.13, which has no compression.zstd (3.14+), and the fallback called the zstd binary, which is not on PATH in that build step. Local checks had run on a 3.14 environment. zstandard (conda-forge 0.25.0) is now a dependency for Python < 3.14 (pixi.toml, pyproject, environment.yml); the table is streamed through it. Checked on the locked default environment (Python 3.13.15) with an empty PATH: build 87 s, 2.3 GB peak; table content identical to the 3.14 build; load 1.72 s, 74 MB. Tests: 1017 passed (3.12 test env: zstd-binary fallback path, no zstandard); pytest is not in the default environment, so no test run on 3.13. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JkhYFEPaa1WenE5MtA8Qya --- environment.yml | 2 ++ pixi.lock | 55 +++++++++++++++++++++++++++++ pixi.toml | 3 ++ pyproject.toml | 2 ++ src/MATPredict/db/local_taxonomy.py | 10 +++++- 5 files changed, 71 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 336da24e..69a12c2c 100644 --- a/environment.yml +++ b/environment.yml @@ -7,6 +7,8 @@ dependencies: - python >=3.11 - pip * - taxonkit * +- zstd * +- zstandard * - diamond * - blast * - miniprot * diff --git a/pixi.lock b/pixi.lock index e46f54b4..b4f073f6 100644 --- a/pixi.lock +++ b/pixi.lock @@ -51,6 +51,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.2.0-h9908984_4.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.8-hebe6cf0_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.1.1-py313ha52865f_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.4.0-py313h2551ef2_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/curl-8.21.0-hcf29cc6_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.66.1-py313h6cd9899_0.conda @@ -178,6 +179,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-tools-5.8.3-hb03c661_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.2-h25fd6f3_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.3.3-hce19668_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.25.0-py313hd42f317_4.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argcomplete-3.7.2-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argh-0.31.3-pyhd8ed1ab_1.conda @@ -199,6 +201,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/perl-file-which-1.24-pl5321hd8ed1ab_0.tar.bz2 - conda: https://conda.anaconda.org/conda-forge/noarch/perl-parent-0.243-pl5321hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pip-26.2.1-pyh145f28c_0.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygenomeviz-1.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.3-pyh5ded981_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/python-dateutil-2.9.0.post0-pyhe01879c_2.conda @@ -264,6 +267,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/brotli-bin-1.2.0-h9908984_4.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/bzip2-1.0.8-hda65f42_10.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/c-ares-1.34.8-hebe6cf0_2.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.1.1-py313ha52865f_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.4.0-py313h2551ef2_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/curl-8.21.0-hcf29cc6_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/fonttools-4.66.1-py313h6cd9899_0.conda @@ -391,6 +395,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/xz-tools-5.8.3-hb03c661_1.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zlib-1.3.2-h25fd6f3_3.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zlib-ng-2.3.3-hce19668_1.conda + - conda: https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.25.0-py313hd42f317_4.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_7.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argcomplete-3.7.2-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/argh-0.31.3-pyhd8ed1ab_1.conda @@ -416,6 +421,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/perl-parent-0.243-pl5321hd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pip-26.2.1-pyh145f28c_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pluggy-1.6.0-pyhf9edf01_1.conda + - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-3.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygenomeviz-1.7.0-pyhd8ed1ab_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pygments-2.21.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/pyparsing-3.3.3-pyh5ded981_0.conda @@ -1040,6 +1046,23 @@ packages: - c-ares >=1.34.8,<2.0a0 size: 228700 timestamp: 1787169971173 +- conda: https://conda.anaconda.org/conda-forge/linux-64/cffi-2.1.1-py313ha52865f_3.conda + sha256: e1968f1efd8491e32a84416a4994cea98bbf51aebd8c7c04c783e121088fec63 + md5: 2b17c17965f71b59158030dce83832e1 + depends: + - __glibc >=2.17,<3.0.a0 + - libffi >=3.7.0,<3.8.0a0 + - libgcc >=15 + - pycparser + - python >=3.13,<3.14.0a0 + - python_abi 3.13.* *_cp313 + license: MIT + license_family: MIT + purls: + - pkg:pypi/cffi?source=hash-mapping + run_exports: {} + size: 304628 + timestamp: 1788485299863 - conda: https://conda.anaconda.org/conda-forge/linux-64/contourpy-1.4.0-py313h2551ef2_1.conda sha256: 078f4a0f83ee7d0f88c110e15f9f94ff4080dc925711d1d22b52779c2e48cf87 md5: 1883f23b947ffaa13f9511ea817fc4e6 @@ -3054,6 +3077,24 @@ packages: - zlib-ng >=2.3.3,<2.4.0a0 size: 123959 timestamp: 1786736890973 +- conda: https://conda.anaconda.org/conda-forge/linux-64/zstandard-0.25.0-py313hd42f317_4.conda + sha256: b183c9ff528c0454d2de07d16840e4514ecd79e404c2ebd0a3256563aab9f5da + md5: 49592828b02c30c46778c2bd6afb9710 + depends: + - python + - cffi >=1.11 + - zstd >=1.5.7,<1.5.8.0a0 + - __glibc >=2.17,<3.0.a0 + - libgcc >=15 + - python_abi 3.13.* *_cp313 + - zstd >=1.5.7,<1.6.0a0 + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/zstandard?source=hash-mapping + run_exports: {} + size: 472702 + timestamp: 1788604307981 - conda: https://conda.anaconda.org/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_7.conda sha256: 47d682b9f6d6ec9eb1a6e6c3e75ea6273e899e78fb7fc59f81d39745009fbc60 md5: aa459086047c0e5e27023ab19f8cb86a @@ -3367,6 +3408,19 @@ packages: run_exports: {} size: 25877 timestamp: 1764896838868 +- conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-3.0-pyhcf101f3_0.conda + sha256: e27e0473fc6723311a0bd48b89b616fa1b996a2f7a2b555338cbbcfb9c640568 + md5: 9c5491066224083c41b6d5635ed7107b + depends: + - python >=3.10 + - python + license: BSD-3-Clause + license_family: BSD + purls: + - pkg:pypi/pycparser?source=hash-mapping + run_exports: {} + size: 55886 + timestamp: 1779293633166 - conda: https://conda.anaconda.org/conda-forge/noarch/pygenomeviz-1.7.0-pyhd8ed1ab_0.conda sha256: 82d7c8baf4f7a0cb4c466664df10037efe4111c98114cb77454c27fda163774f md5: 22ee3f3befa9196f3255ebf3cfb33f46 @@ -3539,6 +3593,7 @@ packages: - biopython>=1.83 - requests>=2.31 - pyhmmer>=0.12 + - zstandard>=0.22 ; python_full_version < '3.14' - pygenomeviz>=1.7.0,<2 ; extra == 'viz' - clinker>=0.0.32,<0.0.33 ; extra == 'viz' requires_python: '>=3.11' diff --git a/pixi.toml b/pixi.toml index 9e13f2a7..67f04a5f 100644 --- a/pixi.toml +++ b/pixi.toml @@ -6,6 +6,9 @@ platforms = ["linux-64"] [dependencies] python = ">=3.11" zstd = "*" +# Streamed reading of the offline taxonomy table on Python < 3.14 (3.14 has +# compression.zstd); the lock pins 3.13. +zstandard = "*" pip = "*" taxonkit = "*" # Search binaries `matpredict detect` shells out to (see detect/search.py): diff --git a/pyproject.toml b/pyproject.toml index 7789d49e..6b0ca618 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,8 @@ dependencies = [ "requests>=2.31", # Profile-HMM idiomorph classifier (detect/classifier.py), curator 2026-09-26. "pyhmmer>=0.12", + # Offline taxonomy table (.zst); Python 3.14+ has compression.zstd instead. + "zstandard>=0.22; python_version < '3.14'", ] # `draw-locus`/`draw-synteny` (curate-db's visualization commands) need these, but no diff --git a/src/MATPredict/db/local_taxonomy.py b/src/MATPredict/db/local_taxonomy.py index e1ddf5c7..33a00a51 100644 --- a/src/MATPredict/db/local_taxonomy.py +++ b/src/MATPredict/db/local_taxonomy.py @@ -85,7 +85,15 @@ def _open_text(path: Path): from compression import zstd # Python >= 3.14: streamed return zstd.open(path, "rt", encoding="utf-8") except ImportError: - return io.StringIO(_zstd_decompress(Path(path).read_bytes()).decode("utf-8")) + pass + try: + import zstandard # Python < 3.14 (a dependency there): streamed + raw = open(path, "rb") + return io.TextIOWrapper(zstandard.ZstdDecompressor().stream_reader(raw, closefd=True), + encoding="utf-8") + except ImportError: + pass + return io.StringIO(_zstd_decompress(Path(path).read_bytes()).decode("utf-8")) def _decode(name: str, data: bytes) -> str: