From 54d7f3f1156ad5512b699645354652b1b61aed5d Mon Sep 17 00:00:00 2001 From: Eugenio Mattei Date: Thu, 15 Oct 2026 08:47:22 -0400 Subject: [PATCH 1/2] RENAME columns.yaml after the table it describes A folder that holds one results table also held a file called columns.yaml, so the name said nothing about what it documented and two of them could not sit side by side. Each is now named for its table: alignments/guides_alignment_metadata_columns.yaml annotations/guides_gene_annotation_columns.yaml elements/guide_element_mapping_columns.yaml igvf/guide_metadata_columns.yaml The alignment one still documents guides_mapping.bed alongside the tsv, in one document, as it did before. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 6 +++--- src/crispr_analysis_utils/cli/element_mapping.py | 2 +- src/crispr_analysis_utils/cli/guide_alignment.py | 2 +- src/crispr_analysis_utils/cli/guide_annotation.py | 2 +- src/crispr_analysis_utils/cli/igvf_export.py | 2 +- src/crispr_analysis_utils/element_mapping.py | 4 ++-- src/crispr_analysis_utils/guide_alignment/pipeline.py | 4 ++-- src/crispr_analysis_utils/guide_alignment/summary.py | 2 +- src/crispr_analysis_utils/guide_annotation/pipeline.py | 4 ++-- src/crispr_analysis_utils/guide_annotation/summary.py | 2 +- src/crispr_analysis_utils/igvf_export.py | 4 ++-- .../skills/cau-element-mapping/SKILL.md | 3 ++- .../skills/cau-element-mapping/references/outputs.md | 3 ++- .../skills/cau-guide-alignment/SKILL.md | 6 ++++-- .../skills/cau-guide-alignment/references/outputs.md | 5 +++-- .../skills/cau-guide-annotation/SKILL.md | 5 +++-- .../skills/cau-guide-annotation/references/outputs.md | 6 ++++-- src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md | 3 ++- .../skills/cau-igvf-export/references/outputs.md | 2 +- tests/test_annotation_pipeline.py | 6 ++++-- tests/test_element_mapping_pipeline.py | 4 ++-- tests/test_guide_alignment_pipeline.py | 2 +- tests/test_igvf_export_pipeline.py | 4 ++-- 23 files changed, 47 insertions(+), 36 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9415a04..368097a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -329,9 +329,9 @@ and narrowing it would buy nothing. - `alignments/` holds four files: `guides_alignment_metadata.tsv`, one row per library entry per site and the only one with the whole picture; `guides_mapping.bed`, its PAM-bearing rows for a browser, named by - spacer; `columns.yaml`, generated from the same definitions as the - table, so it always describes the folder it sits in; and `qc.tsv`, - counts; + spacer; `guides_alignment_metadata_columns.yaml`, generated from the + same definitions as the table, so it always describes the folder it + sits in; and `qc.tsv`, counts; - orientation is decided from both readings, held in the same scan: an entry is `reverse_complement` only when its reversed reading has a PAM-bearing site and the sequence as given does not. A `CCN` 5' of a diff --git a/src/crispr_analysis_utils/cli/element_mapping.py b/src/crispr_analysis_utils/cli/element_mapping.py index a38466f..356bd4c 100644 --- a/src/crispr_analysis_utils/cli/element_mapping.py +++ b/src/crispr_analysis_utils/cli/element_mapping.py @@ -17,7 +17,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: description=( "Classify every guide against the elements a screen targets. " "--outdir gets elements/ with guide_element_mapping.tsv and " - "columns.yaml, run_metadata/ and logs/." + "guide_element_mapping_columns.yaml, run_metadata/ and logs/." ), ) run_parser.add_argument( diff --git a/src/crispr_analysis_utils/cli/guide_alignment.py b/src/crispr_analysis_utils/cli/guide_alignment.py index 7589b38..ae19f7c 100644 --- a/src/crispr_analysis_utils/cli/guide_alignment.py +++ b/src/crispr_analysis_utils/cli/guide_alignment.py @@ -24,7 +24,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: description=( "Scan a reference for every guide's exact site. --outdir gets " "alignments/ with the results (guides_alignment_metadata.tsv, " - "guides_mapping.bed, columns.yaml, qc.tsv), " + "guides_mapping.bed, guides_alignment_metadata_columns.yaml, qc.tsv), " "run_metadata/ with this invocation's record, and logs/." ), ) diff --git a/src/crispr_analysis_utils/cli/guide_annotation.py b/src/crispr_analysis_utils/cli/guide_annotation.py index 19052a7..3c68fb0 100644 --- a/src/crispr_analysis_utils/cli/guide_annotation.py +++ b/src/crispr_analysis_utils/cli/guide_annotation.py @@ -42,7 +42,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help="Annotate every aligned site against the genes of a store.", description=( "Place every mapped site in a gene. --outdir gets annotations/ " - "with guides_gene_annotation.tsv and columns.yaml, run_metadata/ " + "with guides_gene_annotation.tsv and guides_gene_annotation_columns.yaml, run_metadata/ " "with this invocation's record, and logs/." ), ) diff --git a/src/crispr_analysis_utils/cli/igvf_export.py b/src/crispr_analysis_utils/cli/igvf_export.py index ec72745..b86a938 100644 --- a/src/crispr_analysis_utils/cli/igvf_export.py +++ b/src/crispr_analysis_utils/cli/igvf_export.py @@ -17,7 +17,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: description=( "Assemble the IGVF per-guide metadata table from the alignment, " "the element mapping and the annotation store. --outdir gets " - "igvf/ with guide_metadata.tsv and columns.yaml, run_metadata/ " + "igvf/ with guide_metadata.tsv and guide_metadata_columns.yaml, run_metadata/ " "and logs/." ), ) diff --git a/src/crispr_analysis_utils/element_mapping.py b/src/crispr_analysis_utils/element_mapping.py index 597a29d..238ebca 100644 --- a/src/crispr_analysis_utils/element_mapping.py +++ b/src/crispr_analysis_utils/element_mapping.py @@ -397,7 +397,7 @@ def _round(value: float) -> float | int: def write_columns_yaml(path: str | os.PathLike[str]) -> None: - """Write columns.yaml, generated from `COLUMNS` so it cannot drift.""" + """Write guide_element_mapping_columns.yaml, generated from `COLUMNS` so it cannot drift.""" lines = ["# What each column of this folder's output holds.", "", f"{TABLE}:"] lines += [f' {name}: "{_one_line(text)}"' for name, text in COLUMNS.items()] Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8") @@ -606,7 +606,7 @@ def run( distances=distances, ) write_mapping_tsv(outdir / RESULTS_DIR / TABLE, mapping) - write_columns_yaml(outdir / RESULTS_DIR / "columns.yaml") + write_columns_yaml(outdir / RESULTS_DIR / "guide_element_mapping_columns.yaml") write_qc_tsv(outdir / RESULTS_DIR / "qc.tsv", mapping, regions, sites) state["counts"] = counts(mapping) diff --git a/src/crispr_analysis_utils/guide_alignment/pipeline.py b/src/crispr_analysis_utils/guide_alignment/pipeline.py index 11769ad..94ea1fe 100644 --- a/src/crispr_analysis_utils/guide_alignment/pipeline.py +++ b/src/crispr_analysis_utils/guide_alignment/pipeline.py @@ -8,7 +8,7 @@ `outdir` gets three folders and nothing else: ``alignments/`` with the results (``guides_alignment_metadata.tsv``, ``guides_mapping.bed``, -``columns.yaml``, ``qc.tsv``), ``run_metadata/`` with +``guides_alignment_metadata_columns.yaml``, ``qc.tsv``), ``run_metadata/`` with this invocation's record, and ``logs/``. """ @@ -272,7 +272,7 @@ def _run( tables / "guides_alignment_metadata.tsv", library, summaries, scan.sites ) write_guides_bed(tables / "guides_mapping.bed", library, scan.sites) - write_columns_yaml(tables / "columns.yaml") + write_columns_yaml(tables / "guides_alignment_metadata_columns.yaml") write_qc_tsv(tables / "qc.tsv", summaries, counts=state["counts"]) state["classes"] = _ordered((summary.site_class for summary in summaries), CLASSES) diff --git a/src/crispr_analysis_utils/guide_alignment/summary.py b/src/crispr_analysis_utils/guide_alignment/summary.py index 5a4c825..84e8536 100644 --- a/src/crispr_analysis_utils/guide_alignment/summary.py +++ b/src/crispr_analysis_utils/guide_alignment/summary.py @@ -460,7 +460,7 @@ def write_guides_bed( def write_columns_yaml(path: str | os.PathLike[str]) -> None: - """Write columns.yaml: what every column of the outputs holds. + """Write guides_alignment_metadata_columns.yaml: what every column holds. Parameters ---------- diff --git a/src/crispr_analysis_utils/guide_annotation/pipeline.py b/src/crispr_analysis_utils/guide_annotation/pipeline.py index 8dadbbc..79f8953 100644 --- a/src/crispr_analysis_utils/guide_annotation/pipeline.py +++ b/src/crispr_analysis_utils/guide_annotation/pipeline.py @@ -1,7 +1,7 @@ """The gene annotation: read the aligned sites, place them in genes, write the table. `outdir` gets three folders and nothing else: ``annotations/`` with the one -results table and its ``columns.yaml``, ``run_metadata/`` with this +results table and its ``guides_gene_annotation_columns.yaml``, ``run_metadata/`` with this invocation's record, and ``logs/``. """ @@ -192,7 +192,7 @@ def _run( with _timed(state, "outputs"): tables = outdir / RESULTS_DIR write_annotation_tsv(tables / TABLE, annotated) - write_columns_yaml(tables / "columns.yaml") + write_columns_yaml(tables / "guides_gene_annotation_columns.yaml") state["counts"] = counts(annotated) state["complete"] = True diff --git a/src/crispr_analysis_utils/guide_annotation/summary.py b/src/crispr_analysis_utils/guide_annotation/summary.py index 37f96ed..e5cdedd 100644 --- a/src/crispr_analysis_utils/guide_annotation/summary.py +++ b/src/crispr_analysis_utils/guide_annotation/summary.py @@ -109,7 +109,7 @@ def write_annotation_tsv(path: str | os.PathLike[str], annotated: pd.DataFrame) def write_columns_yaml(path: str | os.PathLike[str]) -> None: - """Write columns.yaml: what every column of the table holds. + """Write guides_gene_annotation_columns.yaml: what every column holds. Parameters ---------- diff --git a/src/crispr_analysis_utils/igvf_export.py b/src/crispr_analysis_utils/igvf_export.py index 4da9c5a..9cde815 100644 --- a/src/crispr_analysis_utils/igvf_export.py +++ b/src/crispr_analysis_utils/igvf_export.py @@ -484,7 +484,7 @@ def write_table(path: str | os.PathLike[str], table: pd.DataFrame) -> None: def write_columns_yaml(path: str | os.PathLike[str]) -> None: - """Write columns.yaml, generated from `COLUMNS` so it cannot drift.""" + """Write guide_metadata_columns.yaml, generated from `COLUMNS` so it cannot drift.""" lines = ["# What each column of this folder's output holds.", "", f"{TABLE}:"] lines += [f' {name}: "{" ".join(text.split())}"' for name, text in COLUMNS.items()] Path(path).write_text("\n".join(lines) + "\n", encoding="utf-8") @@ -597,7 +597,7 @@ def run( table = build_table(align, maps, info) write_table(outdir / RESULTS_DIR / TABLE, table) - write_columns_yaml(outdir / RESULTS_DIR / "columns.yaml") + write_columns_yaml(outdir / RESULTS_DIR / "guide_metadata_columns.yaml") state["counts"] = counts(table, info) state["complete"] = True state["finished"] = _now() diff --git a/src/crispr_analysis_utils/skills/cau-element-mapping/SKILL.md b/src/crispr_analysis_utils/skills/cau-element-mapping/SKILL.md index 2fcb54f..772ccc9 100644 --- a/src/crispr_analysis_utils/skills/cau-element-mapping/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-element-mapping/SKILL.md @@ -58,7 +58,8 @@ so check it against what the user expects before believing the table. ## Reading the result `--outdir` gets `elements/`, `run_metadata/` and `logs/`. The results are one -table, `guide_element_mapping.tsv`, the `columns.yaml` that describes it, and +table, `guide_element_mapping.tsv`, the `guide_element_mapping_columns.yaml` +that describes it, and `qc.tsv`. See [`references/outputs.md`](references/outputs.md) for the columns in full. diff --git a/src/crispr_analysis_utils/skills/cau-element-mapping/references/outputs.md b/src/crispr_analysis_utils/skills/cau-element-mapping/references/outputs.md index 41ce4d1..9279433 100644 --- a/src/crispr_analysis_utils/skills/cau-element-mapping/references/outputs.md +++ b/src/crispr_analysis_utils/skills/cau-element-mapping/references/outputs.md @@ -4,7 +4,8 @@ `--outdir` gets three folders and nothing loose: ```text -elements/ guide_element_mapping.tsv columns.yaml qc.tsv +elements/ guide_element_mapping.tsv qc.tsv + guide_element_mapping_columns.yaml run_metadata/ element_mapping_.json logs/ element_mapping.log ``` diff --git a/src/crispr_analysis_utils/skills/cau-guide-alignment/SKILL.md b/src/crispr_analysis_utils/skills/cau-guide-alignment/SKILL.md index 6d42c44..ef35458 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-alignment/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-guide-alignment/SKILL.md @@ -8,7 +8,8 @@ description: >- their protospacer coordinates and cut sites, for sceptre, element annotation or IGVF-style guide metadata; whether a guide table is reverse-complemented; or whether non-targeting controls really match nothing. Also for reading - guides_alignment_metadata.tsv, guides_mapping.bed, columns.yaml or qc.tsv, + guides_alignment_metadata.tsv, guides_mapping.bed, the columns file or + qc.tsv, and for users of the earlier GEM-based workflow. --- @@ -62,7 +63,8 @@ the PAM, is the library designer's answer, not something to infer. ## Reading the result `--outdir` gets `alignments/`, `run_metadata/` and `logs/`. Start with -`alignments/columns.yaml`: it is generated from the same definitions as the +`alignments/guides_alignment_metadata_columns.yaml`: it is generated from +the same definitions as the tables, so it always describes the folder it sits in. See [`references/outputs.md`](references/outputs.md) for the columns in full. diff --git a/src/crispr_analysis_utils/skills/cau-guide-alignment/references/outputs.md b/src/crispr_analysis_utils/skills/cau-guide-alignment/references/outputs.md index cce7a71..eef64b1 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-alignment/references/outputs.md +++ b/src/crispr_analysis_utils/skills/cau-guide-alignment/references/outputs.md @@ -6,12 +6,13 @@ beside its inputs: ```text alignments/ guides_alignment_metadata.tsv guides_mapping.bed - columns.yaml qc.tsv + guides_alignment_metadata_columns.yaml qc.tsv run_metadata/ guide_alignment_.json logs/ guide_alignment.log ``` -`columns.yaml` is written from the same definitions the tables are, so it is +`guides_alignment_metadata_columns.yaml` is written from the same +definitions the tables are, so it is always an accurate data dictionary of the folder it sits in. Read it first. ## `guides_alignment_metadata.tsv` diff --git a/src/crispr_analysis_utils/skills/cau-guide-annotation/SKILL.md b/src/crispr_analysis_utils/skills/cau-guide-annotation/SKILL.md index 3e09a0d..b1d4726 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-annotation/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-guide-annotation/SKILL.md @@ -8,7 +8,7 @@ description: >- and transcript, how far it is from the nearest protein-coding TSS, whether it reads sense or antisense, where in the CDS it cuts, its GC content and its poly-T run. Also for building the GENCODE annotation store, for choosing a - promoter file, and for reading guides_gene_annotation.tsv or columns.yaml. + promoter file, and for reading guides_gene_annotation.tsv or the columns file beside it. --- # Gene annotation (`cau guide-annotation`) @@ -56,7 +56,8 @@ the store in about a second, and a 43,000-site library annotates in another. ## Reading the result `--outdir` gets `annotations/`, `run_metadata/` and `logs/`. The results are -one table, `guides_gene_annotation.tsv`, and the `columns.yaml` that +one table, `guides_gene_annotation.tsv`, and the +`guides_gene_annotation_columns.yaml` that describes it, generated from the same definitions. See [`references/outputs.md`](references/outputs.md) for the columns in full. diff --git a/src/crispr_analysis_utils/skills/cau-guide-annotation/references/outputs.md b/src/crispr_analysis_utils/skills/cau-guide-annotation/references/outputs.md index c0372f2..985b5bd 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-annotation/references/outputs.md +++ b/src/crispr_analysis_utils/skills/cau-guide-annotation/references/outputs.md @@ -4,12 +4,14 @@ `--outdir` gets three folders and nothing loose: ```text -annotations/ guides_gene_annotation.tsv columns.yaml +annotations/ guides_gene_annotation.tsv + guides_gene_annotation_columns.yaml run_metadata/ guide_annotation_.json logs/ guide_annotation.log ``` -`columns.yaml` is written from the same definitions the table is, so it is +`guides_gene_annotation_columns.yaml` is written from the same definitions +the table is, so it is always an accurate data dictionary of the folder it sits in. ## `guides_gene_annotation.tsv` diff --git a/src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md b/src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md index bd4ef3f..4c509e3 100644 --- a/src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md @@ -78,7 +78,8 @@ a promoter and the genome agrees. ## Reading the result `--outdir` gets `igvf/`, `run_metadata/` and `logs/`. The results are one -table, `guide_metadata.tsv`, and the `columns.yaml` that describes it. See +table, `guide_metadata.tsv`, and the `guide_metadata_columns.yaml` that +describes it. See [`references/outputs.md`](references/outputs.md) for the columns in full and the three checks worth running before submitting. diff --git a/src/crispr_analysis_utils/skills/cau-igvf-export/references/outputs.md b/src/crispr_analysis_utils/skills/cau-igvf-export/references/outputs.md index e32416d..596ae0a 100644 --- a/src/crispr_analysis_utils/skills/cau-igvf-export/references/outputs.md +++ b/src/crispr_analysis_utils/skills/cau-igvf-export/references/outputs.md @@ -4,7 +4,7 @@ `--outdir` gets three folders and nothing loose: ```text -igvf/ guide_metadata.tsv columns.yaml +igvf/ guide_metadata.tsv guide_metadata_columns.yaml run_metadata/ igvf_export_.json logs/ igvf_export.log ``` diff --git a/tests/test_annotation_pipeline.py b/tests/test_annotation_pipeline.py index 2dc587f..b2891d5 100644 --- a/tests/test_annotation_pipeline.py +++ b/tests/test_annotation_pipeline.py @@ -35,8 +35,8 @@ ) EXPECTED_FILES = [ - "annotations/columns.yaml", f"annotations/{TABLE}", + "annotations/guides_gene_annotation_columns.yaml", "logs/guide_annotation.log", re.compile(r"run_metadata/guide_annotation_\d{8}T\d{6}Z\.json"), ] @@ -262,7 +262,9 @@ def test_the_groups_account_for_every_column(): def test_columns_yaml_parses_and_documents_the_table(store, tmp_path): run(write_alignment(tmp_path / "aln.tsv"), store, tmp_path / "out") folder = tmp_path / "out" / "annotations" - parsed = yaml.safe_load((folder / "columns.yaml").read_text()) + parsed = yaml.safe_load( + (folder / "guides_gene_annotation_columns.yaml").read_text() + ) header = (folder / TABLE).read_text().splitlines()[0].split("\t") assert list(parsed) == [TABLE] assert list(parsed[TABLE]) == annotation_columns() == header diff --git a/tests/test_element_mapping_pipeline.py b/tests/test_element_mapping_pipeline.py index ae83dd9..5dab527 100644 --- a/tests/test_element_mapping_pipeline.py +++ b/tests/test_element_mapping_pipeline.py @@ -34,8 +34,8 @@ ) EXPECTED_FILES = [ - "elements/columns.yaml", f"elements/{TABLE}", + "elements/guide_element_mapping_columns.yaml", "elements/qc.tsv", "logs/element_mapping.log", re.compile(r"run_metadata/element_mapping_\d{8}T\d{6}Z\.json"), @@ -125,7 +125,7 @@ def test_the_table_has_every_column_of_the_data_dictionary(alignment, bed, tmp_p run(alignment, bed, tmp_path / "out") folder = tmp_path / "out" / "elements" table = pd.read_csv(folder / TABLE, sep="\t") - parsed = yaml.safe_load((folder / "columns.yaml").read_text()) + parsed = yaml.safe_load((folder / "guide_element_mapping_columns.yaml").read_text()) assert list(table.columns) == list(COLUMNS) == list(parsed[TABLE]) diff --git a/tests/test_guide_alignment_pipeline.py b/tests/test_guide_alignment_pipeline.py index 92607e9..9b50547 100644 --- a/tests/test_guide_alignment_pipeline.py +++ b/tests/test_guide_alignment_pipeline.py @@ -40,7 +40,7 @@ OUTPUTS = ( "guides_alignment_metadata.tsv", "guides_mapping.bed", - "columns.yaml", + "guides_alignment_metadata_columns.yaml", "qc.tsv", ) SITE_COLUMNS = ( diff --git a/tests/test_igvf_export_pipeline.py b/tests/test_igvf_export_pipeline.py index 24236ca..d248cf1 100644 --- a/tests/test_igvf_export_pipeline.py +++ b/tests/test_igvf_export_pipeline.py @@ -43,8 +43,8 @@ BBB = "ENSG00000000002" EXPECTED_FILES = [ - "igvf/columns.yaml", f"igvf/{TABLE}", + "igvf/guide_metadata_columns.yaml", "logs/igvf_export.log", re.compile(r"run_metadata/igvf_export_\d{8}T\d{6}Z\.json"), ] @@ -165,7 +165,7 @@ def test_the_written_table_is_the_one_the_result_carries(inputs, tmp_path): def test_the_table_has_every_column_of_the_data_dictionary(inputs, tmp_path): run(*inputs, tmp_path / "out", promoter_window=WINDOW) folder = tmp_path / "out" / "igvf" - parsed = yaml.safe_load((folder / "columns.yaml").read_text()) + parsed = yaml.safe_load((folder / "guide_metadata_columns.yaml").read_text()) assert list(table_of(tmp_path / "out").columns) == list(parsed[TABLE]) From a684095703abf533e22e43c54f8e695cd557064e Mon Sep 17 00:00:00 2001 From: Eugenio Mattei Date: Thu, 15 Oct 2026 08:51:46 -0400 Subject: [PATCH 2/2] RENAME the results tables so the step is in the name The folder names said which step ran; the file names did not, and they disagreed on plural. Each table is now singular and says what the guide was annotated against: guide_genome_annotation.tsv (was guides_alignment_metadata.tsv) guide_gene_annotation.tsv (was guides_gene_annotation.tsv) guide_element_mapping.tsv unchanged guide_mapping.bed (was guides_mapping.bed) with each columns file following its table. guide_metadata.tsv keeps its name: it is what the IGVF CRISPR Perturb-seq pipeline reads, so the name is not ours to change. This breaks every path a caller has written down, including the --alignment argument of three commands. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 8 +++--- docs/element-mapping.md | 4 +-- docs/guide-annotation.md | 2 +- docs/igvf-export.md | 2 +- docs/reference/accession.md | 2 +- .../cli/element_mapping.py | 4 +-- .../cli/guide_alignment.py | 4 +-- .../cli/guide_annotation.py | 4 +-- src/crispr_analysis_utils/cli/igvf_export.py | 2 +- src/crispr_analysis_utils/element_mapping.py | 10 +++---- .../guide_alignment/pipeline.py | 10 +++---- .../guide_alignment/summary.py | 18 ++++++------ .../guide_annotation/pipeline.py | 6 ++-- .../guide_annotation/summary.py | 6 ++-- src/crispr_analysis_utils/igvf_export.py | 4 +-- .../skills/cau-accession/SKILL.md | 2 +- .../skills/cau-element-mapping/SKILL.md | 6 ++-- .../cau-element-mapping/references/outputs.md | 2 +- .../skills/cau-guide-alignment/SKILL.md | 4 +-- .../references/migration.md | 4 +-- .../cau-guide-alignment/references/outputs.md | 10 +++---- .../skills/cau-guide-annotation/SKILL.md | 8 +++--- .../references/outputs.md | 10 +++---- .../skills/cau-igvf-export/SKILL.md | 2 +- tests/element_mapping_fixtures.py | 6 ++-- tests/igvf_export_fixtures.py | 2 +- tests/test_annotation_pipeline.py | 8 ++---- tests/test_cli_guide_alignment.py | 12 ++------ tests/test_guide_alignment_pipeline.py | 28 +++++++++---------- tests/test_guide_alignment_summary.py | 20 ++++++------- tests/test_igvf_export_pipeline.py | 4 +-- 31 files changed, 103 insertions(+), 111 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 368097a..65d9c1a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -293,7 +293,7 @@ and narrowing it would buy nothing. - overlaps are decided on the protospacer, distances and the CDS position from the cut site. `location` and `promoter` are separate columns because a cut in exon 1 is usually both; - - the table keys back to `guides_alignment_metadata.tsv` on `guide_id` + - the table keys back to `guide_genome_annotation.tsv` on `guide_id` plus the four site columns, and repeats nothing else. A guide with two sites needs all five to join unambiguously; - **pandas reads a written `TRUE`/`FALSE` column back as bool.** Both of @@ -326,10 +326,10 @@ and narrowing it would buy nothing. - `run` writes only inside `outdir`, and only three folders: the results in `alignments/`, this invocation's record in `run_metadata/` and the log in `logs/`. Nothing of the run's own sits beside an input; - - `alignments/` holds four files: `guides_alignment_metadata.tsv`, one + - `alignments/` holds four files: `guide_genome_annotation.tsv`, one row per library entry per site and the only one with the whole picture; - `guides_mapping.bed`, its PAM-bearing rows for a browser, named by - spacer; `guides_alignment_metadata_columns.yaml`, generated from the + `guide_mapping.bed`, its PAM-bearing rows for a browser, named by + spacer; `guide_genome_annotation_columns.yaml`, generated from the same definitions as the table, so it always describes the folder it sits in; and `qc.tsv`, counts; - orientation is decided from both readings, held in the same scan: an diff --git a/docs/element-mapping.md b/docs/element-mapping.md index a317e8e..f78d25b 100644 --- a/docs/element-mapping.md +++ b/docs/element-mapping.md @@ -12,8 +12,8 @@ read the output ([Agents and skills](agents-and-skills.md)). ```bash cau element-mapping run \ - --alignment results/alignments/guides_alignment_metadata.tsv \ - --annotation results/annotations/guides_gene_annotation.tsv \ + --alignment results/alignments/guide_genome_annotation.tsv \ + --annotation results/annotations/guide_gene_annotation.tsv \ --elements elements.bed \ --outdir results/ ``` diff --git a/docs/guide-annotation.md b/docs/guide-annotation.md index aab36c7..2a059b6 100644 --- a/docs/guide-annotation.md +++ b/docs/guide-annotation.md @@ -25,7 +25,7 @@ Then annotate: ```bash cau guide-annotation run \ - --alignment results/alignments/guides_alignment_metadata.tsv \ + --alignment results/alignments/guide_genome_annotation.tsv \ --store gencode.v43.protein_coding.parquet \ --promoters gencode.v43.protein_coding.TSS1bp.bed6 \ --outdir results/ diff --git a/docs/igvf-export.md b/docs/igvf-export.md index 920b1c5..1d6e7a9 100644 --- a/docs/igvf-export.md +++ b/docs/igvf-export.md @@ -13,7 +13,7 @@ the output ([Agents and skills](agents-and-skills.md)). ```bash cau igvf-export run \ - --alignment results/alignments/guides_alignment_metadata.tsv \ + --alignment results/alignments/guide_genome_annotation.tsv \ --mapping results/elements/guide_element_mapping.tsv \ --elements elements.bed \ --store gencode.v43.protein_coding.parquet \ diff --git a/docs/reference/accession.md b/docs/reference/accession.md index 582f23a..b832a01 100644 --- a/docs/reference/accession.md +++ b/docs/reference/accession.md @@ -13,7 +13,7 @@ target?"; `guide_id` answers "same oligo?". Nothing in a sequence says where the protospacer ends, so a sequence with its PAM still attached is accepted and gets a different accession, silently. One spacer in sixteen ends in `GG`. Decide which bases are the protospacer first; -`guides_alignment_metadata.tsv`'s `spacer` column already has. +`guide_genome_annotation.tsv`'s `spacer` column already has. ## The encoding diff --git a/src/crispr_analysis_utils/cli/element_mapping.py b/src/crispr_analysis_utils/cli/element_mapping.py index 356bd4c..fe6162f 100644 --- a/src/crispr_analysis_utils/cli/element_mapping.py +++ b/src/crispr_analysis_utils/cli/element_mapping.py @@ -23,7 +23,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: run_parser.add_argument( "--alignment", required=True, - help="guides_alignment_metadata.tsv from `cau guide-alignment run`.", + help="guide_genome_annotation.tsv from `cau guide-alignment run`.", ) run_parser.add_argument( "--elements", @@ -39,7 +39,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: "--annotation", default=None, help=( - "guides_gene_annotation.tsv from `cau guide-annotation run`, " + "guide_gene_annotation.tsv from `cau guide-annotation run`, " "which fills the distance_to_tss column." ), ) diff --git a/src/crispr_analysis_utils/cli/guide_alignment.py b/src/crispr_analysis_utils/cli/guide_alignment.py index ae19f7c..933ad0a 100644 --- a/src/crispr_analysis_utils/cli/guide_alignment.py +++ b/src/crispr_analysis_utils/cli/guide_alignment.py @@ -23,8 +23,8 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help="Find every guide's genomic site.", description=( "Scan a reference for every guide's exact site. --outdir gets " - "alignments/ with the results (guides_alignment_metadata.tsv, " - "guides_mapping.bed, guides_alignment_metadata_columns.yaml, qc.tsv), " + "alignments/ with the results (guide_genome_annotation.tsv, " + "guide_mapping.bed, guide_genome_annotation_columns.yaml, qc.tsv), " "run_metadata/ with this invocation's record, and logs/." ), ) diff --git a/src/crispr_analysis_utils/cli/guide_annotation.py b/src/crispr_analysis_utils/cli/guide_annotation.py index 3c68fb0..13b370e 100644 --- a/src/crispr_analysis_utils/cli/guide_annotation.py +++ b/src/crispr_analysis_utils/cli/guide_annotation.py @@ -42,14 +42,14 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: help="Annotate every aligned site against the genes of a store.", description=( "Place every mapped site in a gene. --outdir gets annotations/ " - "with guides_gene_annotation.tsv and guides_gene_annotation_columns.yaml, run_metadata/ " + "with guide_gene_annotation.tsv and guide_gene_annotation_columns.yaml, run_metadata/ " "with this invocation's record, and logs/." ), ) annotate.add_argument( "--alignment", required=True, - help="guides_alignment_metadata.tsv from `cau guide-alignment run`.", + help="guide_genome_annotation.tsv from `cau guide-alignment run`.", ) annotate.add_argument( "--store", diff --git a/src/crispr_analysis_utils/cli/igvf_export.py b/src/crispr_analysis_utils/cli/igvf_export.py index b86a938..7fd152d 100644 --- a/src/crispr_analysis_utils/cli/igvf_export.py +++ b/src/crispr_analysis_utils/cli/igvf_export.py @@ -24,7 +24,7 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: run_parser.add_argument( "--alignment", required=True, - help="guides_alignment_metadata.tsv from `cau guide-alignment run`.", + help="guide_genome_annotation.tsv from `cau guide-alignment run`.", ) run_parser.add_argument( "--mapping", diff --git a/src/crispr_analysis_utils/element_mapping.py b/src/crispr_analysis_utils/element_mapping.py index 238ebca..ea5ad97 100644 --- a/src/crispr_analysis_utils/element_mapping.py +++ b/src/crispr_analysis_utils/element_mapping.py @@ -42,7 +42,7 @@ COLUMNS = { "guide_id": ( "The sequence that was aligned. With the four site columns, the key" - " back to guides_alignment_metadata.tsv, which holds the rest." + " back to guide_genome_annotation.tsv, which holds the rest." ), "chr": "Contig of this site. Empty when the guide matches nowhere.", "start": "First base of the protospacer, 0-based.", @@ -62,7 +62,7 @@ "Bases from the cut site to the nearest protein-coding TSS, signed in" " that gene's orientation. How far a safe-targeting site has to be to" " be safe depends on the screen, so threshold this yourself rather" - " than trusting the class alone. From guides_gene_annotation.tsv, so" + " than trusting the class alone. From guide_gene_annotation.tsv, so" " empty without it." ), } @@ -150,7 +150,7 @@ def map_to_elements( Every guide in the library, so the ones that map nowhere are classified too. Defaults to the guides the sites name. distances - ``guides_gene_annotation.tsv``, for its ``distance_to_tss``. Without + ``guide_gene_annotation.tsv``, for its ``distance_to_tss``. Without it that column is empty; the classes do not depend on it. Returns @@ -521,7 +521,7 @@ def run( Parameters ---------- alignment - ``guides_alignment_metadata.tsv`` from `guide_alignment.run`, or the + ``guide_genome_annotation.tsv`` from `guide_alignment.run`, or the same table as a DataFrame. Every guide in it is classified, including the ones that match nowhere. elements @@ -530,7 +530,7 @@ def run( Where the table goes. Created if it does not exist; nothing is ever written outside it. annotation - ``guides_gene_annotation.tsv`` from `guide_annotation.run`, which + ``guide_gene_annotation.tsv`` from `guide_annotation.run`, which fills the ``distance_to_tss`` column. Returns diff --git a/src/crispr_analysis_utils/guide_alignment/pipeline.py b/src/crispr_analysis_utils/guide_alignment/pipeline.py index 94ea1fe..ab40c7f 100644 --- a/src/crispr_analysis_utils/guide_alignment/pipeline.py +++ b/src/crispr_analysis_utils/guide_alignment/pipeline.py @@ -7,8 +7,8 @@ that. `outdir` gets three folders and nothing else: ``alignments/`` with the -results (``guides_alignment_metadata.tsv``, ``guides_mapping.bed``, -``guides_alignment_metadata_columns.yaml``, ``qc.tsv``), ``run_metadata/`` with +results (``guide_genome_annotation.tsv``, ``guide_mapping.bed``, +``guide_genome_annotation_columns.yaml``, ``qc.tsv``), ``run_metadata/`` with this invocation's record, and ``logs/``. """ @@ -269,10 +269,10 @@ def _run( ) tables = outdir / RESULTS_DIR write_guides_tsv( - tables / "guides_alignment_metadata.tsv", library, summaries, scan.sites + tables / "guide_genome_annotation.tsv", library, summaries, scan.sites ) - write_guides_bed(tables / "guides_mapping.bed", library, scan.sites) - write_columns_yaml(tables / "guides_alignment_metadata_columns.yaml") + write_guides_bed(tables / "guide_mapping.bed", library, scan.sites) + write_columns_yaml(tables / "guide_genome_annotation_columns.yaml") write_qc_tsv(tables / "qc.tsv", summaries, counts=state["counts"]) state["classes"] = _ordered((summary.site_class for summary in summaries), CLASSES) diff --git a/src/crispr_analysis_utils/guide_alignment/summary.py b/src/crispr_analysis_utils/guide_alignment/summary.py index 84e8536..abe737f 100644 --- a/src/crispr_analysis_utils/guide_alignment/summary.py +++ b/src/crispr_analysis_utils/guide_alignment/summary.py @@ -319,7 +319,7 @@ def fan_out_sites( " genome has that same base." ), } -"""Every column of guides_alignment_metadata.tsv, and what it holds.""" +"""Every column of guide_genome_annotation.tsv, and what it holds.""" BED_COLUMNS = { "chr": "Contig of the match.", @@ -329,11 +329,11 @@ def fan_out_sites( "score": "Always 0; BED6 wants the field.", "strand": "Strand the protospacer reads 5' to 3' on.", } -"""The six fields of guides_mapping.bed, which has no header of its own.""" +"""The six fields of guide_mapping.bed, which has no header of its own.""" def guide_columns() -> list[str]: - """The columns of guides_alignment_metadata.tsv, in order. + """The columns of guide_genome_annotation.tsv, in order. One row per library entry per site, so the guide's own columns repeat on every site it has, and an entry with no site gets one row whose site @@ -354,7 +354,7 @@ def write_guides_tsv( summaries: Sequence[GuideSummary], sites: Mapping[int, Sequence[Site]], ) -> None: - """Write guides_alignment_metadata.tsv: every entry, a row per site. + """Write guide_genome_annotation.tsv: every entry, a row per site. Parameters ---------- @@ -419,7 +419,7 @@ def write_guides_bed( guides: Sequence[Guide], sites: Mapping[int, Sequence[Site]], ) -> None: - """Write guides_mapping.bed: every site a guide could be cut at. + """Write guide_mapping.bed: every site a guide could be cut at. Parameters ---------- @@ -438,7 +438,7 @@ def write_guides_bed( of a guide that matches more than once. Sorted by contig, then position. A match with no PAM is not somewhere anything can be cut, so it is left out; - guides_alignment_metadata.tsv has them all. + guide_genome_annotation.tsv has them all. """ # One row per distinct guide, not per entry: two entries of one sequence # would otherwise give byte-identical rows and bedtools would count twice. @@ -460,7 +460,7 @@ def write_guides_bed( def write_columns_yaml(path: str | os.PathLike[str]) -> None: - """Write guides_alignment_metadata_columns.yaml: what every column holds. + """Write guide_genome_annotation_columns.yaml: what every column holds. Parameters ---------- @@ -473,7 +473,7 @@ def write_columns_yaml(path: str | os.PathLike[str]) -> None: tables themselves. """ lines = ["# What each column of this folder's outputs holds.", ""] - lines.append("guides_alignment_metadata.tsv:") + lines.append("guide_genome_annotation.tsv:") columns = iter(COLUMNS.items()) for heading, count in GROUPS: lines.append(f" # {heading}") @@ -481,7 +481,7 @@ def write_columns_yaml(path: str | os.PathLike[str]) -> None: column, description = next(columns) lines.append(f' {column}: "{_one_line(description)}"') lines.append("") - lines.append("guides_mapping.bed:") + lines.append("guide_mapping.bed:") for column, description in BED_COLUMNS.items(): lines.append(f' {column}: "{_one_line(description)}"') lines.append("") diff --git a/src/crispr_analysis_utils/guide_annotation/pipeline.py b/src/crispr_analysis_utils/guide_annotation/pipeline.py index 79f8953..de645ea 100644 --- a/src/crispr_analysis_utils/guide_annotation/pipeline.py +++ b/src/crispr_analysis_utils/guide_annotation/pipeline.py @@ -1,7 +1,7 @@ """The gene annotation: read the aligned sites, place them in genes, write the table. `outdir` gets three folders and nothing else: ``annotations/`` with the one -results table and its ``guides_gene_annotation_columns.yaml``, ``run_metadata/`` with this +results table and its ``guide_gene_annotation_columns.yaml``, ``run_metadata/`` with this invocation's record, and ``logs/``. """ @@ -75,7 +75,7 @@ def run( Parameters ---------- alignment - ``guides_alignment_metadata.tsv`` from `guide_alignment.run`, or the + ``guide_genome_annotation.tsv`` from `guide_alignment.run`, or the same table as a DataFrame. Only the sites with a PAM are annotated, which is what it means for a guide to map. store @@ -192,7 +192,7 @@ def _run( with _timed(state, "outputs"): tables = outdir / RESULTS_DIR write_annotation_tsv(tables / TABLE, annotated) - write_columns_yaml(tables / "guides_gene_annotation_columns.yaml") + write_columns_yaml(tables / "guide_gene_annotation_columns.yaml") state["counts"] = counts(annotated) state["complete"] = True diff --git a/src/crispr_analysis_utils/guide_annotation/summary.py b/src/crispr_analysis_utils/guide_annotation/summary.py index e5cdedd..fa939b8 100644 --- a/src/crispr_analysis_utils/guide_annotation/summary.py +++ b/src/crispr_analysis_utils/guide_annotation/summary.py @@ -78,9 +78,9 @@ " so the guide expresses poorly from a U6 promoter." ), } -"""Every column of guides_gene_annotation.tsv, and what it holds.""" +"""Every column of guide_gene_annotation.tsv, and what it holds.""" -TABLE = "guides_gene_annotation.tsv" +TABLE = "guide_gene_annotation.tsv" """The one results table.""" @@ -109,7 +109,7 @@ def write_annotation_tsv(path: str | os.PathLike[str], annotated: pd.DataFrame) def write_columns_yaml(path: str | os.PathLike[str]) -> None: - """Write guides_gene_annotation_columns.yaml: what every column holds. + """Write guide_gene_annotation_columns.yaml: what every column holds. Parameters ---------- diff --git a/src/crispr_analysis_utils/igvf_export.py b/src/crispr_analysis_utils/igvf_export.py index 9cde815..32b27c8 100644 --- a/src/crispr_analysis_utils/igvf_export.py +++ b/src/crispr_analysis_utils/igvf_export.py @@ -353,7 +353,7 @@ def build_table( Parameters ---------- alignment - ``guides_alignment_metadata.tsv``. + ``guide_genome_annotation.tsv``. mapping ``guide_element_mapping.tsv``. elements @@ -523,7 +523,7 @@ def run( Parameters ---------- alignment - ``guides_alignment_metadata.tsv``. + ``guide_genome_annotation.tsv``. mapping ``guide_element_mapping.tsv``. elements diff --git a/src/crispr_analysis_utils/skills/cau-accession/SKILL.md b/src/crispr_analysis_utils/skills/cau-accession/SKILL.md index ed396be..868bd6f 100644 --- a/src/crispr_analysis_utils/skills/cau-accession/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-accession/SKILL.md @@ -52,7 +52,7 @@ One spacer in sixteen ends in `GG`, so you cannot tell by looking. designed at 20 nt and the sequences are 21, the first base is that G and is not part of the protospacer. - If the user has already run `cau guide-alignment`, the `spacer` column of - `guides_alignment_metadata.tsv` is the protospacer, and the `accession` + `guide_genome_annotation.tsv` is the protospacer, and the `accession` column is already there. Never guess which bases to strip. diff --git a/src/crispr_analysis_utils/skills/cau-element-mapping/SKILL.md b/src/crispr_analysis_utils/skills/cau-element-mapping/SKILL.md index 772ccc9..2c8da0a 100644 --- a/src/crispr_analysis_utils/skills/cau-element-mapping/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-element-mapping/SKILL.md @@ -20,8 +20,8 @@ is in, and this says which element of the screen it tests. ```bash cau element-mapping run \ - --alignment results/alignments/guides_alignment_metadata.tsv \ - --annotation results/annotations/guides_gene_annotation.tsv \ + --alignment results/alignments/guide_genome_annotation.tsv \ + --annotation results/annotations/guide_gene_annotation.tsv \ --elements elements.bed \ --outdir results/ ``` @@ -73,7 +73,7 @@ in full. - **`element_id` comes from the BED's fourth column** when it has one, so an id used elsewhere keeps its spelling. Otherwise it is `chr:start-end`, 1-based, as a locus is read aloud. -- **It keys back** to `guides_alignment_metadata.tsv` on `guide_id` and the +- **It keys back** to `guide_genome_annotation.tsv` on `guide_id` and the four site columns. ## Checking a library's own labels diff --git a/src/crispr_analysis_utils/skills/cau-element-mapping/references/outputs.md b/src/crispr_analysis_utils/skills/cau-element-mapping/references/outputs.md index 9279433..bb459f9 100644 --- a/src/crispr_analysis_utils/skills/cau-element-mapping/references/outputs.md +++ b/src/crispr_analysis_utils/skills/cau-element-mapping/references/outputs.md @@ -18,7 +18,7 @@ site and no element. | Column | Holds | | --- | --- | -| `guide_id` | The aligned sequence. With the four site columns, the key back to `guides_alignment_metadata.tsv`. | +| `guide_id` | The aligned sequence. With the four site columns, the key back to `guide_genome_annotation.tsv`. | | `chr`, `start`, `end`, `strand` | The protospacer, 0-based and half-open. Empty when the guide matches nowhere. | | `element_id` | The element this site overlaps. Empty unless targeting. | | `element_category` | The elements BED's fifth column, when it has one. | diff --git a/src/crispr_analysis_utils/skills/cau-guide-alignment/SKILL.md b/src/crispr_analysis_utils/skills/cau-guide-alignment/SKILL.md index ef35458..66a1e1f 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-alignment/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-guide-alignment/SKILL.md @@ -8,7 +8,7 @@ description: >- their protospacer coordinates and cut sites, for sceptre, element annotation or IGVF-style guide metadata; whether a guide table is reverse-complemented; or whether non-targeting controls really match nothing. Also for reading - guides_alignment_metadata.tsv, guides_mapping.bed, the columns file or + guide_genome_annotation.tsv, guide_mapping.bed, the columns file or qc.tsv, and for users of the earlier GEM-based workflow. --- @@ -63,7 +63,7 @@ the PAM, is the library designer's answer, not something to infer. ## Reading the result `--outdir` gets `alignments/`, `run_metadata/` and `logs/`. Start with -`alignments/guides_alignment_metadata_columns.yaml`: it is generated from +`alignments/guide_genome_annotation_columns.yaml`: it is generated from the same definitions as the tables, so it always describes the folder it sits in. See [`references/outputs.md`](references/outputs.md) for the columns in full. diff --git a/src/crispr_analysis_utils/skills/cau-guide-alignment/references/migration.md b/src/crispr_analysis_utils/skills/cau-guide-alignment/references/migration.md index 2351c84..088f2b9 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-alignment/references/migration.md +++ b/src/crispr_analysis_utils/skills/cau-guide-alignment/references/migration.md @@ -26,8 +26,8 @@ coordinates at contig ends — cannot arise. | Was | Now | | --- | --- | -| `guides.tsv` and `sites.tsv` | one `guides_alignment_metadata.tsv` | -| `guides.bed` | `guides_mapping.bed` | +| `guides.tsv` and `sites.tsv` | one `guide_genome_annotation.tsv` | +| `guides.bed` | `guide_mapping.bed` | | `cut_sites.bed` | gone; `cut_site` is a column | | `orientation.tsv` | gone; its verdict and site are columns | | `summary.tsv` | `qc.tsv`, counts only | diff --git a/src/crispr_analysis_utils/skills/cau-guide-alignment/references/outputs.md b/src/crispr_analysis_utils/skills/cau-guide-alignment/references/outputs.md index eef64b1..866b42a 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-alignment/references/outputs.md +++ b/src/crispr_analysis_utils/skills/cau-guide-alignment/references/outputs.md @@ -5,17 +5,17 @@ beside its inputs: ```text -alignments/ guides_alignment_metadata.tsv guides_mapping.bed - guides_alignment_metadata_columns.yaml qc.tsv +alignments/ guide_genome_annotation.tsv guide_mapping.bed + guide_genome_annotation_columns.yaml qc.tsv run_metadata/ guide_alignment_.json logs/ guide_alignment.log ``` -`guides_alignment_metadata_columns.yaml` is written from the same +`guide_genome_annotation_columns.yaml` is written from the same definitions the tables are, so it is always an accurate data dictionary of the folder it sits in. Read it first. -## `guides_alignment_metadata.tsv` +## `guide_genome_annotation.tsv` **One row per library entry per site.** A guide matching three places gets three rows; one matching nowhere gets a single row whose site columns are @@ -64,7 +64,7 @@ The protospacer occurs 89 times; one of those has a PAM. Cas9 cuts there and nowhere else. Every one of the 89 is a row in the table, 88 of them with `pam_matches` `FALSE`. -## `guides_mapping.bed` +## `guide_mapping.bed` BED6, no header, one row per **cuttable** site, so a guide that cuts twice gets two rows. `start`/`end` are the protospacer, without the PAM. The name diff --git a/src/crispr_analysis_utils/skills/cau-guide-annotation/SKILL.md b/src/crispr_analysis_utils/skills/cau-guide-annotation/SKILL.md index b1d4726..411710b 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-annotation/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-guide-annotation/SKILL.md @@ -8,7 +8,7 @@ description: >- and transcript, how far it is from the nearest protein-coding TSS, whether it reads sense or antisense, where in the CDS it cuts, its GC content and its poly-T run. Also for building the GENCODE annotation store, for choosing a - promoter file, and for reading guides_gene_annotation.tsv or the columns file beside it. + promoter file, and for reading guide_gene_annotation.tsv or the columns file beside it. --- # Gene annotation (`cau guide-annotation`) @@ -21,7 +21,7 @@ cau guide-annotation build-db gencode.v43.annotation.gtf.gz \ --out gencode.v43.protein_coding.parquet cau guide-annotation run \ - --alignment results/alignments/guides_alignment_metadata.tsv \ + --alignment results/alignments/guide_genome_annotation.tsv \ --store gencode.v43.protein_coding.parquet \ --promoters gencode.v43.protein_coding.TSS1bp.bed6 \ --outdir results/ @@ -56,8 +56,8 @@ the store in about a second, and a 43,000-site library annotates in another. ## Reading the result `--outdir` gets `annotations/`, `run_metadata/` and `logs/`. The results are -one table, `guides_gene_annotation.tsv`, and the -`guides_gene_annotation_columns.yaml` that +one table, `guide_gene_annotation.tsv`, and the +`guide_gene_annotation_columns.yaml` that describes it, generated from the same definitions. See [`references/outputs.md`](references/outputs.md) for the columns in full. diff --git a/src/crispr_analysis_utils/skills/cau-guide-annotation/references/outputs.md b/src/crispr_analysis_utils/skills/cau-guide-annotation/references/outputs.md index 985b5bd..c8d94bf 100644 --- a/src/crispr_analysis_utils/skills/cau-guide-annotation/references/outputs.md +++ b/src/crispr_analysis_utils/skills/cau-guide-annotation/references/outputs.md @@ -4,17 +4,17 @@ `--outdir` gets three folders and nothing loose: ```text -annotations/ guides_gene_annotation.tsv - guides_gene_annotation_columns.yaml +annotations/ guide_gene_annotation.tsv + guide_gene_annotation_columns.yaml run_metadata/ guide_annotation_.json logs/ guide_annotation.log ``` -`guides_gene_annotation_columns.yaml` is written from the same definitions +`guide_gene_annotation_columns.yaml` is written from the same definitions the table is, so it is always an accurate data dictionary of the folder it sits in. -## `guides_gene_annotation.tsv` +## `guide_gene_annotation.tsv` **One row per mapped site per overlapping gene.** A site in two genes gets two rows; a site in none gets one row, `intergenic`, with the gene columns @@ -22,7 +22,7 @@ empty. Only sites with a PAM are annotated. | Column | Holds | | --- | --- | -| `guide_id` | The aligned sequence. With the four site columns, the key back to `guides_alignment_metadata.tsv`. | +| `guide_id` | The aligned sequence. With the four site columns, the key back to `guide_genome_annotation.tsv`. | | `chr`, `start`, `end`, `strand` | The protospacer, 0-based and half-open. | | `ensg`, `gene_symbol` | The gene this row is about, without a version. | | `enst` | Its reference transcript, when the protospacer hits one of that transcript's exons. | diff --git a/src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md b/src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md index 4c509e3..6072310 100644 --- a/src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md +++ b/src/crispr_analysis_utils/skills/cau-igvf-export/SKILL.md @@ -19,7 +19,7 @@ everything the format asks for; this assembles it. ```bash cau igvf-export run \ - --alignment results/alignments/guides_alignment_metadata.tsv \ + --alignment results/alignments/guide_genome_annotation.tsv \ --mapping results/elements/guide_element_mapping.tsv \ --elements elements.bed \ --store gencode.v43.protein_coding.parquet \ diff --git a/tests/element_mapping_fixtures.py b/tests/element_mapping_fixtures.py index c76e884..eec4bcc 100644 --- a/tests/element_mapping_fixtures.py +++ b/tests/element_mapping_fixtures.py @@ -45,7 +45,7 @@ def write_bed(path, rows=ELEMENTS, *, columns=5, header=None): def site(guide_id, chrom, start, strand="+", *, length=LENGTH, pam_matches=True): - """One row of guides_alignment_metadata.tsv, as `guide_alignment` writes it.""" + """One row of guide_genome_annotation.tsv, as `guide_alignment` writes it.""" return { "guide_id": guide_id, "chr": chrom, @@ -69,7 +69,7 @@ def unmapped(guide_id, *, pam_matches=True): def sites_frame(*rows): - """The rows as a guides_alignment_metadata table.""" + """The rows as a guide_genome_annotation table.""" return pd.DataFrame(list(rows)) @@ -83,7 +83,7 @@ def write_alignment(path, rows, *, as_text=True): def annotation_frame(*pairs): - """guides_gene_annotation.tsv: the site columns and a distance for each.""" + """guide_gene_annotation.tsv: the site columns and a distance for each.""" frame = sites_frame(*(row for row, _ in pairs))[list(SITE_COLUMNS)].copy() frame["distance_to_tss"] = [distance for _, distance in pairs] return frame diff --git a/tests/igvf_export_fixtures.py b/tests/igvf_export_fixtures.py index ffa7d74..d2fdca7 100644 --- a/tests/igvf_export_fixtures.py +++ b/tests/igvf_export_fixtures.py @@ -94,7 +94,7 @@ def sequence_id(alias): def alignment_frame(guides=GUIDES): - """guides_alignment_metadata.tsv, one row per entry. + """guide_genome_annotation.tsv, one row per entry. Every guide here matches at most once, so the per-site columns do not repeat. diff --git a/tests/test_annotation_pipeline.py b/tests/test_annotation_pipeline.py index b2891d5..b72a2ae 100644 --- a/tests/test_annotation_pipeline.py +++ b/tests/test_annotation_pipeline.py @@ -36,7 +36,7 @@ EXPECTED_FILES = [ f"annotations/{TABLE}", - "annotations/guides_gene_annotation_columns.yaml", + "annotations/guide_gene_annotation_columns.yaml", "logs/guide_annotation.log", re.compile(r"run_metadata/guide_annotation_\d{8}T\d{6}Z\.json"), ] @@ -55,7 +55,7 @@ def store(tmp_path): def write_alignment(path, rows=SITES, *, as_text=True): - """``guides_alignment_metadata.tsv``, with pam_matches written as a table has it.""" + """``guide_genome_annotation.tsv``, with pam_matches written as a table has it.""" frame = pd.DataFrame(list(rows)) if as_text: frame["pam_matches"] = frame["pam_matches"].map({True: "TRUE", False: "FALSE"}) @@ -262,9 +262,7 @@ def test_the_groups_account_for_every_column(): def test_columns_yaml_parses_and_documents_the_table(store, tmp_path): run(write_alignment(tmp_path / "aln.tsv"), store, tmp_path / "out") folder = tmp_path / "out" / "annotations" - parsed = yaml.safe_load( - (folder / "guides_gene_annotation_columns.yaml").read_text() - ) + parsed = yaml.safe_load((folder / "guide_gene_annotation_columns.yaml").read_text()) header = (folder / TABLE).read_text().splitlines()[0].split("\t") assert list(parsed) == [TABLE] assert list(parsed[TABLE]) == annotation_columns() == header diff --git a/tests/test_cli_guide_alignment.py b/tests/test_cli_guide_alignment.py index 23f5205..c36318d 100644 --- a/tests/test_cli_guide_alignment.py +++ b/tests/test_cli_guide_alignment.py @@ -162,9 +162,7 @@ def test_run_end_to_end(tmp_path, capsys): assert captured.err == "" assert "Done in" in (tmp_path / "out" / "logs" / "guide_alignment.log").read_text() rows = ( - (tmp_path / "out" / "alignments" / "guides_mapping.bed") - .read_text() - .splitlines() + (tmp_path / "out" / "alignments" / "guide_mapping.bed").read_text().splitlines() ) # The BED's name field is the spacer, not the alias the table gives it. assert rows == [f"chr1\t{start}\t{start + 20}\t{SPACER_A}\t0\t+"] @@ -201,9 +199,7 @@ def test_column_positions_are_read_as_numbers(tmp_path, capsys): assert status == 0 assert capsys.readouterr().out.startswith("1 guides: 1 unique") rows = ( - (tmp_path / "out" / "alignments" / "guides_mapping.bed") - .read_text() - .splitlines() + (tmp_path / "out" / "alignments" / "guide_mapping.bed").read_text().splitlines() ) # The BED's name field is the spacer, not the alias the table gives it. assert rows == [f"chr1\t{start}\t{start + 20}\t{SPACER_A}\t0\t+"] @@ -332,9 +328,7 @@ def test_fix_orientation_turns_a_reversed_entry_round(tmp_path, capsys): assert status == 0 assert capsys.readouterr().out.startswith("1 guides: 1 unique") rows = ( - (tmp_path / "out" / "alignments" / "guides_mapping.bed") - .read_text() - .splitlines() + (tmp_path / "out" / "alignments" / "guide_mapping.bed").read_text().splitlines() ) # The BED's name field is the spacer, not the alias the table gives it. assert rows == [f"chr1\t{start}\t{start + 20}\t{SPACER_A}\t0\t+"] diff --git a/tests/test_guide_alignment_pipeline.py b/tests/test_guide_alignment_pipeline.py index 9b50547..5ff97a6 100644 --- a/tests/test_guide_alignment_pipeline.py +++ b/tests/test_guide_alignment_pipeline.py @@ -38,9 +38,9 @@ ("g_dup", SPACER_A), ] OUTPUTS = ( - "guides_alignment_metadata.tsv", - "guides_mapping.bed", - "guides_alignment_metadata_columns.yaml", + "guide_genome_annotation.tsv", + "guide_mapping.bed", + "guide_genome_annotation_columns.yaml", "qc.tsv", ) SITE_COLUMNS = ( @@ -148,7 +148,7 @@ def test_run_writes_every_output(setup): assert len(records) == 1 assert sorted(p.name for p in out.iterdir()) == ["alignments", "logs", METADATA_DIR] - guides = by_alias(table(out / "alignments" / "guides_alignment_metadata.tsv")) + guides = by_alias(table(out / "alignments" / "guide_genome_annotation.tsv")) chrom, a_start, _ = setup["sites"]["g_a"] (g_a,) = guides["g_a"] assert g_a["site_class"] == "unique" @@ -207,7 +207,7 @@ def test_run_writes_every_output(setup): # The BED holds every site a guide could be cut at, named by its spacer. bed = [ line.split("\t") - for line in (out / "alignments" / "guides_mapping.bed").read_text().splitlines() + for line in (out / "alignments" / "guide_mapping.bed").read_text().splitlines() ] assert [(row[0], row[1], row[3], row[5]) for row in bed] == [ # One row per distinct guide: 'g_dup' is the same sequence as 'g_a' at @@ -243,7 +243,7 @@ def test_the_scan_is_exact_so_a_mismatched_site_is_never_reported(setup): result = go(setup) _, nag_start, _ = setup["sites"]["g_c_nag"] - rows = table(setup["outdir"] / "alignments" / "guides_alignment_metadata.tsv") + rows = table(setup["outdir"] / "alignments" / "guide_genome_annotation.tsv") assert str(nag_start) not in {row["start"] for row in rows} guides = by_alias(rows) assert guides["g_c"][0]["n_cut_sites"] == "1" @@ -260,7 +260,7 @@ def test_entries_sharing_a_sequence_are_scanned_once_and_written_per_entry(setup # The sequence is searched once, however many entries carry it: four # distinct spacers, four genomic matches between them. assert result.scan.n_matches == 4 - guides = table(setup["outdir"] / "alignments" / "guides_alignment_metadata.tsv") + guides = table(setup["outdir"] / "alignments" / "guide_genome_annotation.tsv") shared = [row for row in guides if row["guide_id"] == SPACER_A + "NGG"] # One row per entry per site: both entries get the one site of the sequence. assert sorted(row["alias"] for row in shared) == ["g_a", "g_dup"] @@ -282,7 +282,7 @@ def test_add_leading_g_records_the_base_under_the_g(setup): assert bases[("G" + SPACER_A + "NGG", a_start)] == "T" assert bases[("G" + SPACER_C + "NGG", c_start)] == "A" guides = by_alias( - table(setup["outdir"] / "alignments" / "guides_alignment_metadata.tsv") + table(setup["outdir"] / "alignments" / "guide_genome_annotation.tsv") ) assert result.guides[0].extra_5p is True assert guides["g_a"][0]["guide_id"] == "G" + SPACER_A + "NGG" @@ -296,7 +296,7 @@ def test_contigs_keep_only_their_hits(setup): result = go(setup, contigs=["chr1"]) guides = by_alias( - table(setup["outdir"] / "alignments" / "guides_alignment_metadata.tsv") + table(setup["outdir"] / "alignments" / "guide_genome_annotation.tsv") ) (g_g,) = guides["g_g"] assert g_g["site_class"] == "unique" @@ -329,7 +329,7 @@ def test_a_pam_running_off_a_contig_end_is_counted_not_reported(tmp_path): 0, ) # Nothing was kept, so the guide gets its one row with no site in it. - (row,) = table(result.outdir / "alignments" / "guides_alignment_metadata.tsv") + (row,) = table(result.outdir / "alignments" / "guide_genome_annotation.tsv") assert [row[name] for name in SITE_COLUMNS] == [""] * 7 assert (row["n_matches"], row["n_cut_sites"]) == ("0", "0") assert result.summaries[0].site_class == "no_site" @@ -344,12 +344,12 @@ def test_a_match_without_a_pam_is_kept_out_of_the_bed_but_written_to_the_tsv(tmp (site,) = result.scan.sites[0] assert (site.pam, site.pam_class) == ("TTT", "none") - (row,) = table(result.outdir / "alignments" / "guides_alignment_metadata.tsv") + (row,) = table(result.outdir / "alignments" / "guide_genome_annotation.tsv") assert (row["chr"], row["start"], row["pam_matches"]) == ("c1", "4", "FALSE") assert (row["n_matches"], row["n_cut_sites"]) == ("1", "0") assert row["mapped_sequence"] == SPACER_A + "TTT" # Nothing can be cut there, so the BED is empty. - assert (result.outdir / "alignments" / "guides_mapping.bed").read_text() == "" + assert (result.outdir / "alignments" / "guide_mapping.bed").read_text() == "" assert result.scan.counts()["n_no_pam"] == 1 assert result.scan.counts()["n_sites"] == 1 assert result.summaries[0].site_class == "no_site" @@ -436,7 +436,7 @@ def reversed_rows(n): outdir=tmp_path / "tie", ) assert [s.orientation for s in tie.summaries] == ["reverse_complement", "forward"] - assert (tmp_path / "tie" / "alignments" / "guides_alignment_metadata.tsv").is_file() + assert (tmp_path / "tie" / "alignments" / "guide_genome_annotation.tsv").is_file() def test_a_reversed_library_under_error_leaves_a_failed_record(setup, tmp_path): @@ -471,7 +471,7 @@ def test_a_reversed_library_is_only_logged_when_the_check_is_warn(setup, tmp_pat assert "3 of 3 entries read 3' to 5'" in log assert {s.orientation for s in result.summaries} == {"reverse_complement"} assert {s.orientation_evidence for s in result.summaries} == {"ambiguous"} - assert (setup["outdir"] / "alignments" / "guides_alignment_metadata.tsv").is_file() + assert (setup["outdir"] / "alignments" / "guide_genome_annotation.tsv").is_file() assert result.metadata["orientation_fixed"] == 0 diff --git a/tests/test_guide_alignment_summary.py b/tests/test_guide_alignment_summary.py index c6f34e4..10b28e0 100644 --- a/tests/test_guide_alignment_summary.py +++ b/tests/test_guide_alignment_summary.py @@ -138,13 +138,13 @@ def rows_by_alias(table): def write_tables(path, guides, summaries, sites): """Write every table `run` writes into `path`, and read the two of them back.""" - write_guides_tsv(path / "guides_alignment_metadata.tsv", guides, summaries, sites) - write_guides_bed(path / "guides_mapping.bed", guides, sites) + write_guides_tsv(path / "guide_genome_annotation.tsv", guides, summaries, sites) + write_guides_bed(path / "guide_mapping.bed", guides, sites) write_columns_yaml(path / "columns.yaml") - table = read_table(path / "guides_alignment_metadata.tsv") + table = read_table(path / "guide_genome_annotation.tsv") bed = [ line.split("\t") - for line in (path / "guides_mapping.bed") + for line in (path / "guide_mapping.bed") .read_text(encoding="utf-8") .splitlines() ] @@ -334,7 +334,7 @@ def test_guides_tsv(tmp_path, guides, sites): table, _ = write_tables(tmp_path, guides, summaries, sites) lines = ( - (tmp_path / "guides_alignment_metadata.tsv") + (tmp_path / "guide_genome_annotation.tsv") .read_text(encoding="utf-8") .splitlines() ) @@ -457,14 +457,14 @@ def test_columns_yaml_documents_exactly_what_is_written(tmp_path, guides, sites) table, bed = write_tables(tmp_path, guides, summaries, sites) document = yaml.safe_load((tmp_path / "columns.yaml").read_text(encoding="utf-8")) - assert list(document) == ["guides_alignment_metadata.tsv", "guides_mapping.bed"] + assert list(document) == ["guide_genome_annotation.tsv", "guide_mapping.bed"] # The data dictionary names every column of the table, in the table's order. - assert list(document["guides_alignment_metadata.tsv"]) == list(table.columns) - assert list(document["guides_alignment_metadata.tsv"]) == guide_columns() - assert list(document["guides_mapping.bed"]) == list(BED_COLUMNS) + assert list(document["guide_genome_annotation.tsv"]) == list(table.columns) + assert list(document["guide_genome_annotation.tsv"]) == guide_columns() + assert list(document["guide_mapping.bed"]) == list(BED_COLUMNS) assert len(bed[0]) == len(BED_COLUMNS) assert sum(n for _, n in GROUPS) == len(COLUMNS) - assert all(document["guides_alignment_metadata.tsv"][name] for name in COLUMNS) + assert all(document["guide_genome_annotation.tsv"][name] for name in COLUMNS) def test_guides_tsv_rejects_summaries_that_are_not_one_per_guide( diff --git a/tests/test_igvf_export_pipeline.py b/tests/test_igvf_export_pipeline.py index d248cf1..e912369 100644 --- a/tests/test_igvf_export_pipeline.py +++ b/tests/test_igvf_export_pipeline.py @@ -64,7 +64,7 @@ def bed(tmp_path): @pytest.fixture def alignment(tmp_path): - return write_alignment(tmp_path / "guides_alignment_metadata.tsv") + return write_alignment(tmp_path / "guide_genome_annotation.tsv") @pytest.fixture @@ -141,7 +141,7 @@ def test_nothing_is_written_beside_the_inputs(inputs, genes, tmp_path): assert outside == [ "elements.bed", "guide_element_mapping.tsv", - "guides_alignment_metadata.tsv", + "guide_genome_annotation.tsv", "pairs.tsv", "store.parquet", "test.gtf",