diff --git a/.claude/skills/perturbNMF-runner/references/02-evaluation.md b/.claude/skills/perturbNMF-runner/references/02-evaluation.md index 89f8ce8..c0eef7c 100644 --- a/.claude/skills/perturbNMF-runner/references/02-evaluation.md +++ b/.claude/skills/perturbNMF-runner/references/02-evaluation.md @@ -37,7 +37,8 @@ Reconstruction error and stability are computed automatically. | `--guide_annotation_path` | None | TSV with a `targeting` column to identify non-targeting controls (alternative to `--guide_annotation_key`) | | `--gene_names_key` | `symbol` | Column in data_guide["rna"].var with gene names | | `--FDR_method` | `StoreyQ` | FDR correction: `StoreyQ` or `BH` | -| `--organism` | `human` | Species for enrichment | +| `--organism` | `human` | Species for enrichment (`human` or `mouse`; mouse genes are mapped to human orthologs) | +| `--ortholog_table` | None | Mouse-human ortholog TSV; default MGI + Ensembl, cached in `Resources/` | | `--n_top` | `300` | Top genes for enrichment tests | | `--guide_annotation_key` | `["non-targeting"]` | Non-targeting guide identifiers (accepts multiple values) | | `--use_cache` | flag | Load enrichr gene set libraries from cached JSON in `Resources/` instead of downloading; falls back to download + cache on miss | diff --git a/.claude/skills/perturbNMF-runner/references/04-visualization.md b/.claude/skills/perturbNMF-runner/references/04-visualization.md index ddc50c0..b3dd193 100644 --- a/.claude/skills/perturbNMF-runner/references/04-visualization.md +++ b/.claude/skills/perturbNMF-runner/references/04-visualization.md @@ -29,6 +29,8 @@ | `--selected_k` | None | K value to highlight with a red dashed line | | `--stability_file` | None | Pre-computed stability/error TSV/NPZ (bypasses `cnmf.consensus()`; for torch-cNMF runs) | | `--run_program_dotplot` | off | Enable per-(K, sel_thresh) program dotplots (default off; requires the inference h5mu artifact per K/threshold) | +| `--no_trait` | off | Skip trait (GWAS) enrichment inputs and plots (non-human data) | +| `--no_perturbation` | off | Skip perturbation inputs and regulator plots (non-perturbed data, no guides) | Enrichment file patterns use `{k}` placeholder (and `{sample}` for perturbation). See `references/parameter-catalog.md` Section 4 for all optional params. diff --git a/.claude/skills/perturbNMF-runner/references/05-annotation-summary.md b/.claude/skills/perturbNMF-runner/references/05-annotation-summary.md index c8c0c18..d0634ca 100644 --- a/.claude/skills/perturbNMF-runner/references/05-annotation-summary.md +++ b/.claude/skills/perturbNMF-runner/references/05-annotation-summary.md @@ -120,6 +120,7 @@ other stage (no notebook required). Submit one job per K to cover multiple K val | `--perturbation_file_name` | `perturbation_association_results` | Perturbation file stem (e.g. `CRT`) | | `--effect_size` | `log2FC` | Effect-size column (e.g. `approx_log2FC`) | | `--gene_names_key` | `symbol` | var column with gene symbols | +| `--organism` | `human` | Species of the data; `mouse` annotates genes with their human orthologs' summaries | | `--non_targeting_key` | `non-targeting` | Negative-control target label(s) | See `references/parameter-catalog.md` (Section 12) for the full flag list, including diff --git a/.claude/skills/perturbNMF-runner/references/parameter-catalog.md b/.claude/skills/perturbNMF-runner/references/parameter-catalog.md index 8d5e58d..4d8b1b0 100644 --- a/.claude/skills/perturbNMF-runner/references/parameter-catalog.md +++ b/.claude/skills/perturbNMF-runner/references/parameter-catalog.md @@ -219,7 +219,8 @@ Same as sk-cNMF except: | `--gwas_data_path` | str | None | Path to GWAS data (required only for trait enrichment) | | `--X_normalized_path` | str | None | Normalized counts h5ad (needed for explained variance). Path pattern: `//Inference/cnmf_tmp/Inference.norm_counts.h5ad` (prefix is always `Inference`, not the run_name) | | `--guide_annotation_path` | str | None | Guide annotation TSV | -| `--organism` | str | `human` | Species for enrichment | +| `--organism` | str | `human` | Species for enrichment (`human` or `mouse`; mouse genes are mapped to human orthologs) | +| `--ortholog_table` | str | None | Mouse-human ortholog TSV; default MGI + Ensembl, cached in `Resources/` | | `--FDR_method` | str | `StoreyQ` | FDR method: `StoreyQ` or `BH` | | `--n_top` | int | `300` | Top genes for enrichment tests | | `--gene_names_key` | str | `symbol` | Column in data_guide["rna"].var with gene names | @@ -271,6 +272,8 @@ Same as sk-cNMF except: | `--variance_col` | str | `Total` | No | Variance column name | | `--stability_file` | str | None | No | Pre-computed stability/error file (TSV or NPZ) | | `--run_program_dotplot` | flag | False | No | Enable per-(K, sel_thresh) program dotplots; requires `{output_dir}/{run_name}/Inference/adata/cNMF_{K}_{thresh}.h5mu` to exist for each pair | +| `--no_trait` | flag | False | No | Skip trait (GWAS) enrichment files and plots (non-human data) | +| `--no_perturbation` | flag | False | No | Skip perturbation files and regulator plots (non-perturbed data, no guides) | --- @@ -616,6 +619,8 @@ cover multiple K values. | `--save_path` | str | `{out_dir}/{run_name}/Interpretation/Summary_table/{K}_{thresh}` | Output dir for the `.xlsx`, TSVs, and specificity/correlation/KD sidecars | | `--mdata_path` | str | `{out_dir}/{run_name}/Inference/adata/cNMF_{K}_{thresh}.h5mu` | Input MuData | | `--num_gene` | int | `300` | Top genes per program / per enriched term to keep | +| `--organism` | str | `human` | Species of the data; with `mouse`, each gene gets the MyGene summaries of its human orthologs (listed in a `Human ortholog` column) | +| `--ortholog_table` | str | None | Mouse-human ortholog TSV; default MGI + Ensembl, cached in `Resources/` | | `--perturbation_file_name` | str | `perturbation_association_results` | Perturbation result file stem (between `{K}_` and `_{Condition}.txt`); e.g. `CRT` | | `--Conditions` | str (nargs=\*) | all labels found in the data | Condition labels (values of obs[categorical_key]); `--Sample` is a deprecated alias | | `--effect_size` | str | `log2FC` | Effect-size column in perturbation files (e.g. `approx_log2FC`) | diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d6c4e0..c2a2ab7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,10 +25,29 @@ Entries marked ⚠️ change pipeline output — re-run affected analyses. - Annotation runners locate their scripts via `PIPELINE_ROOT` (fixing paths into a nonexistent `AGeneTic/` folder); `gpe_export.py` reads the GeneProgramExplorer location from `GPE_DIR`. +- ⚠️ Mouse data (`--organism mouse`): genes are matched to the human gene-set + libraries (GO, Reactome) through curated mouse-human orthologs (MGI + Ensembl) + instead of upper-cased symbols. Mouse genes without a human ortholog are left + out of enrichment. Re-run gene set and GO enrichment for mouse datasets. +- `--organism` accepts only `human` or `mouse` (case-insensitive) in evaluation + and the Excel summary. Trait (GWAS) enrichment is skipped, with a message, for + mouse data. +- `ensure_umap` (program and perturbed-gene plots) reuses embeddings stored as + `obsm['UMAP']` / `obsm['PCA']` (e.g. Seurat conversions) instead of computing + a new UMAP. ### Added - `tools/check_no_lab_specific_content.py` guard (CI + pre-commit) that blocks lab/project-specific content in this public repo. +- Support for scRNA-seq data without perturbations (no guide information): + K-selection `--no_perturbation` / `--no_trait`; program plots and the Excel + summary run without `uns['guide_targets']` (the Targets Summary sheet is + skipped); evaluation stops early when `--Perform_perturbation` is requested + without guides. See the README section "Non-perturbed and mouse data". +- Mouse-human ortholog table (`src/Stage2_Evaluation/A_Metrics/src/orthologs.py`), + downloaded once and cached in `src/Stage2_Evaluation/Resources/`, or supplied + with `--ortholog_table`. With `--organism mouse`, the Excel summary annotates + genes with their human orthologs' summaries in a new `Human ortholog` column. ## [0.1.1] - 2026-08-10 diff --git a/README.md b/README.md index 8491c7a..930df55 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,29 @@ See [`src/Stage2_Evaluation/B_Calibration/README.md`](src/Stage2_Evaluation/B_Ca See [`src/Stage3_Interpretation/README.md`](src/Stage3_Interpretation/README.md) for detailed parameters and output format. +## Non-perturbed and mouse data + +The pipeline also runs on scRNA-seq data without perturbations (no guide information in the +input) and on mouse data. Per stage: + +| Stage | Non-perturbed data (no guides) | Mouse data | +|---|---|---| +| 1. Inference | Leave out the `--guide_*` options | `--species mouse` | +| 2a. Metrics | Leave out `--Perform_perturbation` and `--guide_annotation_path` | `--organism mouse`: genes are mapped to human orthologs for gene set and GO enrichment. Trait (GWAS) enrichment is human-only and is skipped with a message if requested | +| 2b. Calibration | Not applicable (needs non-targeting guides); skip | No change | +| 3a. K-selection plots | `--no_perturbation` | `--no_trait` | +| 3a. Program plots | Leave out `--perturb_path_base` and use `--output_format PDF` or `SVG` (HTML requires `--perturb_path_base`); perturbed-gene annotations are left empty | No change | +| 3a. Perturbed-gene plots | Not applicable; skip | No change | +| 3b. Excel summary | The Targets Summary sheet is skipped automatically | `--organism mouse`: genes are annotated with their human orthologs' summaries | +| 3c. Annotation | No change | `--taxid 10090` (NCBI, UniProt, OmniPath search), `--species mouse` (MyGene search), `--organism mouse` (program extraction) | + +Mouse genes are matched to human gene-set libraries and gene summaries through curated +orthologs (union of MGI and Ensembl), not by upper-casing symbols. The table is downloaded +on first use and cached in `src/Stage2_Evaluation/Resources/orthologs_mouse_human.tsv`; +pass `--ortholog_table` to use a fixed copy. Mouse genes without a human ortholog are left +out of enrichment, and a gene with several human orthologs contributes to all of them. +Custom gene sets used with mouse data must therefore use human gene symbols. + ## Claude Code Skills This repo ships four Claude Code skills under `.claude/skills/` for guided pipeline execution, validation, and maintenance. Open the linked `SKILL.md` for trigger phrases, detailed usage, and helper script references. diff --git a/src/Stage2_Evaluation/A_Metrics/README.md b/src/Stage2_Evaluation/A_Metrics/README.md index 301f502..61927cf 100644 --- a/src/Stage2_Evaluation/A_Metrics/README.md +++ b/src/Stage2_Evaluation/A_Metrics/README.md @@ -40,7 +40,8 @@ The evaluation pipeline tests cNMF programs against various criteria to assess t | X_normalized_path | str | None | Path to normalized cell x gene matrix (.h5ad), required for explained variance | | guide_annotation_path | str | None | Path to tab-separated guide annotation file with "targeting" column | | gwas_data_path | str | None | Path to GWAS data file for trait enrichment (required when --Perform_trait is set) | -| organism | str | "human" | Organism/species for enrichment analysis | +| organism | str | "human" | Organism/species for enrichment analysis (`human` or `mouse`; mouse genes are mapped to human orthologs to match the human gene-set libraries) | +| ortholog_table | str | None | For mouse: mouse-human ortholog TSV (columns `mouse_symbol`, `mouse_ensembl`, `human_symbol`, `source`). Default: MGI + Ensembl table, downloaded once and cached in `src/Stage2_Evaluation/Resources/` | | FDR_method | str | "StoreyQ" | FDR correction method for perturbation association | | n_top | int | 300 | Number of top loaded genes to use for enrichment tests | | guide_annotation_key | list of str | ["non-targeting"] | Name(s) of non-targeting guide target labels | diff --git a/src/Stage2_Evaluation/A_Metrics/Slurm_Version/cNMF_evaluation_pipeline.py b/src/Stage2_Evaluation/A_Metrics/Slurm_Version/cNMF_evaluation_pipeline.py index d41296c..43dbfc9 100644 --- a/src/Stage2_Evaluation/A_Metrics/Slurm_Version/cNMF_evaluation_pipeline.py +++ b/src/Stage2_Evaluation/A_Metrics/Slurm_Version/cNMF_evaluation_pipeline.py @@ -47,6 +47,23 @@ def _read_samples_from_h5mu(h5mu_path, data_key, categorical_key): return None +def _h5mu_has_uns_key(h5mu_path, mod_key, uns_key): + """True if mod//uns/ exists in the h5mu; None if unreadable. + + Raises ValueError if the modality itself is missing (e.g. a wrong --prog_key). + """ + try: + f = h5py.File(h5mu_path, "r") + except OSError: + return None + with f: + if f"mod/{mod_key}" not in f: + raise ValueError( + f"Modality '{mod_key}' not found in {h5mu_path}; check --prog_key." + ) + return f"mod/{mod_key}/uns/{uns_key}" in f + + # Make the repo's src/ importable regardless of the current working directory from pathlib import Path @@ -203,10 +220,20 @@ def main(): parser.add_argument( "--organism", - help="Organism/species for enrichment analysis (default: human)", - type=str, + help="Organism/species for enrichment analysis: human or mouse (default: human)", + type=str.lower, + choices=["human", "mouse"], default="human", ) + parser.add_argument( + "--ortholog_table", + type=str, + default=None, + help="For --organism mouse: TSV of mouse-human orthologs used to match genes " + "to the human gene-set libraries (columns: mouse_symbol, mouse_ensembl, " + "human_symbol, source). Default: MGI + Ensembl table, downloaded once and " + "cached in src/Stage2_Evaluation/Resources/", + ) parser.add_argument( "--FDR_method", help="Method for FDR correction in perturbation association (default: StoreyQ)", @@ -235,6 +262,15 @@ def main(): args = parser.parse_args() + # The bundled GWAS resource (OpenTargets L2G) is human-only. + if args.Perform_trait and args.organism != "human": + print( + f"--organism {args.organism}: skipping trait (GWAS) enrichment, which is " + "only available for human data. Pass --no_trait to K-selection plotting.", + flush=True, + ) + args.Perform_trait = False + if args.Perform_trait and args.gwas_data_path is None: parser.error("--gwas_data_path is required when --Perform_trait is set") @@ -284,6 +320,25 @@ def main(): else: reference_targets = args.guide_annotation_key + # Check for guide information before computing any metric. + if args.Perform_perturbation: + for sel_thresh in args.sel_threshs: + for k in args.K: + h5mu_path = ( + f"{args.out_dir}/{args.run_name}/Inference/adata/" + f"cNMF_{k}_{str(sel_thresh).replace('.', '_')}.h5mu" + ) + if ( + _h5mu_has_uns_key(h5mu_path, args.prog_key, args.guide_targets_key) + is False + ): + raise ValueError( + f"--Perform_perturbation needs guide information, but uns['" + f"{args.guide_targets_key}'] is missing from '{args.prog_key}' " + f"in {h5mu_path}. For non-perturbed data, drop " + "--Perform_perturbation." + ) + for sel_thresh in args.sel_threshs: for k in args.K: output_folder = f"{args.out_dir}/{args.run_name}/Evaluation/{k}_{str(sel_thresh).replace('.', '_')}" @@ -441,6 +496,7 @@ def main(): data_key=args.data_key, prog_name=None, organism=args.organism, + ortholog_table=args.ortholog_table, library="Reactome_2022", method="fisher", database="enrichr", @@ -461,6 +517,7 @@ def main(): data_key=args.data_key, prog_name=None, organism=args.organism, + ortholog_table=args.ortholog_table, library="GO_Biological_Process_2023", method="fisher", database="enrichr", diff --git a/src/Stage2_Evaluation/A_Metrics/src/enrichment_geneset.py b/src/Stage2_Evaluation/A_Metrics/src/enrichment_geneset.py index 209dc5c..e63a66a 100644 --- a/src/Stage2_Evaluation/A_Metrics/src/enrichment_geneset.py +++ b/src/Stage2_Evaluation/A_Metrics/src/enrichment_geneset.py @@ -12,6 +12,8 @@ from gseapy import Biomart, Msigdb from tqdm.auto import tqdm +from .orthologs import collapse_to_human, load_ortholog_table + logging.basicConfig(level=logging.INFO) _RESOURCES_DIR = os.path.join( @@ -78,6 +80,18 @@ def get_idconversion(var_names, organism="human", chunk_size=100): return gene_names +def _mouse_gene_ids(mdata, prog_key, data_key, gene_names_key): + """Mouse symbols or Ensembl IDs for the loadings columns, unchanged in case.""" + if gene_names_key is not None and gene_names_key in mdata[data_key].var.columns: + genes = mdata[data_key].var[gene_names_key].astype(str).tolist() + elif "var_names" in mdata[prog_key].uns.keys(): + genes = [str(v) for v in mdata[prog_key].uns["var_names"]] + else: + genes = [str(v) for v in mdata[data_key].var_names] + # "Symbol:ENSMUSG..." style names: keep the symbol + return [g.split(":", 1)[0] if ":ens" in g.lower() else g for g in genes] + + def get_program_gene_loadings( mdata, prog_key="prog", @@ -85,10 +99,22 @@ def get_program_gene_loadings( data_key="rna", organism="human", gene_names_key=None, + ortholog_table=None, + map_to_human=True, ): - """Get gene loadings for each program in the mudata object.""" + """Get gene loadings for each program in the mudata object. - if gene_names_key is not None and gene_names_key in mdata[data_key].var.columns: + For mouse data with map_to_human=True (gene sets in human symbols), genes are + mapped to human orthologs (see orthologs.py); ortholog_table is an optional + path to a pre-built ortholog table. With map_to_human=False, mouse symbols are + kept unchanged (gene sets in mouse symbols, e.g. MSigDB mouse). + """ + if organism not in ("human", "mouse"): + raise ValueError(f"Unsupported organism '{organism}'; use 'human' or 'mouse'.") + + if organism == "mouse": + gene_names = _mouse_gene_ids(mdata, prog_key, data_key, gene_names_key) + elif gene_names_key is not None and gene_names_key in mdata[data_key].var.columns: gene_names = ( mdata[data_key].var[gene_names_key].astype(str).str.upper().tolist() ) @@ -113,6 +139,9 @@ def get_program_gene_loadings( loadings["gene_names"] = gene_names loadings.set_index("gene_names", inplace=True) + if organism == "mouse" and map_to_human: + loadings = collapse_to_human(loadings, load_ortholog_table(ortholog_table)) + # with open('var_names.txt', 'w') as fil: # for nam in mdata[data_key].var_names: # fil.write(nam+'\n') @@ -403,6 +432,7 @@ def compute_geneset_enrichment( use_loadings_gene=False, # False = use all oncology gene, True = use interact expresseed gene gene_names_key: Optional[str] = None, use_cache: bool = False, + ortholog_table: Optional[str] = None, **kwargs, ) -> Optional[pd.DataFrame]: """ @@ -442,6 +472,9 @@ def compute_geneset_enrichment( number of threads to run processes on. inplace : bool (default: True) whether to insert the results back into the mudata object. + ortholog_table : str (default: None) + For organism='mouse': path to a mouse-human ortholog table. By default the + MGI + Ensembl table is downloaded once and cached in Resources/. Returns ------- @@ -484,6 +517,9 @@ def compute_geneset_enrichment( data_key=data_key, organism=organism, gene_names_key=gene_names_key, + ortholog_table=ortholog_table, + # Enrichr and user gene sets use human symbols; MSigDB mouse sets do not. + map_to_human=not (database == "msigdb" and user_geneset is None), ) # run enrichment diff --git a/src/Stage2_Evaluation/A_Metrics/src/orthologs.py b/src/Stage2_Evaluation/A_Metrics/src/orthologs.py new file mode 100644 index 0000000..bfedda7 --- /dev/null +++ b/src/Stage2_Evaluation/A_Metrics/src/orthologs.py @@ -0,0 +1,209 @@ +"""Mouse to human ortholog mapping. + +Human gene-set libraries (Enrichr, MSigDB human) and human gene summaries are matched +to mouse genes through curated orthologs rather than by upper-casing mouse symbols. +Upper-casing only works when the two species share a symbol, misses orthologs with +different names (e.g. Zfp451 -> ZNF451) and can pick an unrelated human gene. + +The ortholog table is the union of two sources, since each has gaps the other fills: +- MGI HOM_MouseHumanSequence.rpt (mouse-human homology classes) +- Ensembl BioMart mouse -> human homologs + +The combined table is downloaded on first use and cached as a TSV in +src/Stage2_Evaluation/Resources/, so later runs are offline and reproducible. A +pre-built table can also be passed explicitly (columns: mouse_symbol, mouse_ensembl, +human_symbol, source). +""" + +from __future__ import annotations + +import io +import logging +import os +import urllib.parse +import urllib.request +from collections.abc import Iterable + +import pandas as pd + +logger = logging.getLogger(__name__) + +RESOURCES_DIR = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), + "Resources", +) +DEFAULT_TABLE_PATH = os.path.join(RESOURCES_DIR, "orthologs_mouse_human.tsv") +TABLE_COLUMNS = ["mouse_symbol", "mouse_ensembl", "human_symbol", "source"] + +MGI_URL = "https://www.informatics.jax.org/downloads/reports/HOM_MouseHumanSequence.rpt" +# www.ensembl.org redirects BioMart requests; the mirrors answer directly. +ENSEMBL_MARTS = [ + "https://useast.ensembl.org/biomart/martservice", + "https://asia.ensembl.org/biomart/martservice", + "https://www.ensembl.org/biomart/martservice", +] +ENSEMBL_QUERY = ( + '' + '' + '' + '' + '' + '' + "" +) +# BioMart appends this line only when the full result was sent. +ENSEMBL_COMPLETION_STAMP = "[success]" +DOWNLOAD_TIMEOUT_S = 600 +MOUSE_TAXON = "10090" +HUMAN_TAXON = "9606" + + +def _download(url: str) -> str: + with urllib.request.urlopen(url, timeout=DOWNLOAD_TIMEOUT_S) as response: + return response.read().decode("utf-8") + + +def _fetch_mgi() -> pd.DataFrame: + hom = pd.read_csv(io.StringIO(_download(MGI_URL)), sep="\t", dtype=str) + key, taxon = "DB Class Key", "NCBI Taxon ID" + mouse = hom.loc[hom[taxon] == MOUSE_TAXON, [key, "Symbol"]] + human = hom.loc[hom[taxon] == HUMAN_TAXON, [key, "Symbol"]] + pairs = mouse.merge(human, on=key, suffixes=("_mouse", "_human")) + if pairs.empty: + raise RuntimeError(f"No mouse-human pairs parsed from {MGI_URL}") + return pd.DataFrame( + { + "mouse_symbol": pairs["Symbol_mouse"], + "mouse_ensembl": pd.NA, + "human_symbol": pairs["Symbol_human"], + "source": "MGI", + } + ) + + +def _fetch_ensembl() -> pd.DataFrame: + query = urllib.parse.urlencode({"query": ENSEMBL_QUERY}) + errors = [] + for mart in ENSEMBL_MARTS: + try: + text = _download(f"{mart}?{query}").rstrip("\n") + body, _, last_line = text.rpartition("\n") + if last_line.strip() != ENSEMBL_COMPLETION_STAMP: + raise RuntimeError("incomplete response (no completion stamp)") + df = pd.read_csv(io.StringIO(body), sep="\t", dtype=str) + if df.shape[1] != 3 or df.empty: + raise RuntimeError(f"unexpected response: {text[:200]!r}") + break + except (OSError, ValueError, RuntimeError) as err: # try the next mirror + errors.append(f"{mart}: {err}") + else: + raise RuntimeError("Ensembl BioMart query failed:\n" + "\n".join(errors)) + df.columns = ["mouse_ensembl", "mouse_symbol", "human_symbol"] + # Rows without a human ortholog are kept: they map Ensembl IDs to mouse symbols. + df["source"] = "Ensembl" + return df[TABLE_COLUMNS] + + +def build_ortholog_table() -> pd.DataFrame: + """Download MGI and Ensembl orthologs and return their union.""" + table = pd.concat([_fetch_mgi(), _fetch_ensembl()], ignore_index=True) + table = table.dropna(subset=["mouse_symbol"]).drop_duplicates() + return table[TABLE_COLUMNS].reset_index(drop=True) + + +def load_ortholog_table(path: str | None = None) -> pd.DataFrame: + """Load the ortholog table from `path`, the cache, or download and cache it.""" + if path is not None: + if not os.path.isfile(path): + raise FileNotFoundError(f"Ortholog table not found: {path}") + table = pd.read_csv(path, sep="\t", dtype=str) + elif os.path.isfile(DEFAULT_TABLE_PATH): + logger.info(f"Loading cached ortholog table from {DEFAULT_TABLE_PATH}") + table = pd.read_csv(DEFAULT_TABLE_PATH, sep="\t", dtype=str) + else: + logger.info("Downloading mouse-human ortholog table (MGI + Ensembl)") + try: + table = build_ortholog_table() + except (OSError, RuntimeError) as err: + raise RuntimeError( + f"Could not download the mouse-human ortholog table ({err}). On a node " + "without internet access, build it once elsewhere (it is cached at " + f"{DEFAULT_TABLE_PATH}) or pass a pre-built table with --ortholog_table." + ) from err + os.makedirs(RESOURCES_DIR, exist_ok=True) + # Write to a temporary file first so concurrent jobs never read a partial table. + tmp_path = f"{DEFAULT_TABLE_PATH}.{os.getpid()}.tmp" + try: + table.to_csv(tmp_path, sep="\t", index=False) + os.replace(tmp_path, DEFAULT_TABLE_PATH) + finally: + if os.path.exists(tmp_path): # only left over if the write failed + os.remove(tmp_path) + logger.info(f"Saved ortholog table to {DEFAULT_TABLE_PATH}") + + missing = set(TABLE_COLUMNS) - set(table.columns) + if missing: + raise ValueError(f"Ortholog table is missing columns: {sorted(missing)}") + return table + + +def map_mouse_to_human( + genes: Iterable[str], table: pd.DataFrame | None = None +) -> dict[str, list[str]]: + """Map mouse gene symbols or Ensembl IDs (ENSMUSG...) to human ortholog symbols. + + Returns a dict from each input gene to a sorted list of human symbols; genes with + no human ortholog map to an empty list. A gene with several human orthologs keeps + all of them. + """ + if table is None: + table = load_ortholog_table() + pairs = table.dropna(subset=["human_symbol"]) + symbol_to_human = pairs.groupby("mouse_symbol")["human_symbol"].apply(set) + id_to_symbol = ( + table.dropna(subset=["mouse_ensembl"]) + .groupby("mouse_ensembl")["mouse_symbol"] + .apply(set) + ) + + mapping = {} + for gene in genes: + gene = str(gene) + if gene.upper().startswith("ENSMUSG"): + symbols = id_to_symbol.get(gene.split(".", 1)[0], set()) # drop version + else: + symbols = {gene} + human = set() + for symbol in symbols: + human |= symbol_to_human.get(symbol, set()) + mapping[gene] = sorted(human) + return mapping + + +def collapse_to_human( + df: pd.DataFrame, table: pd.DataFrame | None = None +) -> pd.DataFrame: + """Re-index a (mouse genes x columns) DataFrame by human ortholog symbols. + + Each human ortholog takes the values of its mouse gene; genes without a human + ortholog are dropped, and a human gene with several mouse orthologs keeps the + per-column maximum. + """ + df = df.groupby(level=0).max() # one row per mouse gene + mapping = map_mouse_to_human(df.index, table) + rows = [(mouse, human) for mouse, hs in mapping.items() for human in hs] + n_unmapped = sum(1 for hs in mapping.values() if not hs) + logger.info( + f"Ortholog mapping: {len(mapping) - n_unmapped}/{len(mapping)} mouse genes " + f"have a human ortholog; the rest are excluded." + ) + if not rows: + raise ValueError( + "No mouse genes could be mapped to human orthologs. Check that gene " + "names are mouse symbols or Ensembl IDs (e.g. set gene_names_key)." + ) + mouse_idx, human_idx = zip(*rows) + expanded = df.loc[list(mouse_idx)].copy() + expanded.index = pd.Index(human_idx, name=df.index.name) + return expanded.groupby(level=0).max() diff --git a/src/Stage3_Interpretation/A_Plotting/README.md b/src/Stage3_Interpretation/A_Plotting/README.md index 7ce17ce..4c60668 100644 --- a/src/Stage3_Interpretation/A_Plotting/README.md +++ b/src/Stage3_Interpretation/A_Plotting/README.md @@ -64,6 +64,8 @@ Generates a multi-panel figure to guide K selection. Reads pre-computed Stage 2 | `--variance_col` | str | `Total` | Column name for variance values. Use `Total` for summary files, or a per-program column name (will be summed) | | `--stability_file` | str | `None` | Path to a pre-computed stability/error file (TSV or NPZ). Bypasses `cnmf.consensus()` — useful for torch-cNMF runs where the cnmf package isn't installed | | `--run_program_dotplot` | flag | off | If set, also generate per-(K, sel_thresh) program dotplots. Requires the inference `cNMF_{K}_{thresh}.h5mu` to exist for each pair; off by default because the loop is slow | +| `--no_trait` | flag | off | Do not require or read trait (GWAS) enrichment files and drop the trait plots. Use when evaluation ran without trait enrichment (e.g. mouse data) | +| `--no_perturbation` | flag | off | Do not require or read perturbation association files and drop the regulator plots. Use for non-perturbed data (no guides); `--Conditions` is then not needed | ### Outputs diff --git a/src/Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_k_selection.py b/src/Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_k_selection.py index dc2392a..00b1cf1 100644 --- a/src/Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_k_selection.py +++ b/src/Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_k_selection.py @@ -20,6 +20,7 @@ plot_explained_variance, programs_dotplots, plot_k_selection_panel, + plot_k_selection_panel_no_traits, read_condition_labels_from_h5mu, ) @@ -142,6 +143,20 @@ def main(): help="If set, also generate per-(K, sel_thresh) program dotplots. Requires the inference " "h5mu (cNMF_{K}_{thresh}.h5mu) to exist for each (K, sel_thresh). Default off.", ) + parser.add_argument( + "--no_trait", + action="store_true", + help="Skip trait (GWAS) enrichment: do not require or read the trait files and " + "drop the trait plots. Use when evaluation ran without --Perform_trait " + "(e.g. mouse data). Default off.", + ) + parser.add_argument( + "--no_perturbation", + action="store_true", + help="Skip perturbation results: do not require or read the perturbation " + "association files and drop the regulator plots. Use for datasets without " + "guides (non-perturbed scRNA-seq). Default off.", + ) args = parser.parse_args() @@ -154,7 +169,7 @@ def main(): yaml.dump(args_dict, f, default_flow_style=False, width=1000) # ---- Condition labels: default to the values of --groupby in the inference h5mu ---- - if args.Conditions is None: + if args.Conditions is None and not args.no_perturbation: first_h5mu = ( f"{args.output_directory}/{args.run_name}/Inference/adata/" f"cNMF_{args.K[0]}_{str(args.sel_threshs[0]).replace('.', '_')}.h5mu" @@ -175,15 +190,21 @@ def main(): ) _ev_pat = args.variance_file or "{k}_Explained_Variance_Summary.txt" + required_pats = [_go_pat, _gs_pat, _ev_pat] + if not args.no_trait: + required_pats.append(_tr_pat) + missing = [] for sel_thresh in args.sel_threshs: sel_str = str(sel_thresh).replace(".", "_") for k in args.K: k_folder = os.path.join(args.eval_folder_name, f"{k}_{sel_str}") - for pat in [_go_pat, _gs_pat, _tr_pat, _ev_pat]: + for pat in required_pats: fpath = os.path.join(k_folder, pat.format(k=k)) if not os.path.isfile(fpath): missing.append(fpath) + if args.no_perturbation: + continue for cond in args.Conditions: fpath = os.path.join(k_folder, _pt_pat.format(k=k, sample=cond)) if not os.path.isfile(fpath): @@ -218,31 +239,40 @@ def main(): trait_file=args.trait_file, term_col=args.term_col, adjpval_col=args.adjpval_col, + include_traits=not args.no_trait, ) plot_enrichment( count_df, folder_name=args.save_folder_name, file_name=f"Enrichment_{sel_thresh}", + include_traits=not args.no_trait, ) # Perturbation - test_stats_df = load_perturbation_data( - folder=args.eval_folder_name, - components=args.K, - sel_thresh=sel_thresh, - conditions=args.Conditions, - pval=args.pval, - perturbation_file=args.perturbation_file, - perturb_adjpval_col=args.perturb_adjpval_col, - perturb_target_col=args.perturb_target_col, - perturb_log2fc_col=args.perturb_log2fc_col, - ) - plot_perturbation( - test_stats_df, - folder_name=args.save_folder_name, - pval=args.pval, - file_name=f"Perturbation_{sel_thresh}", - ) + if args.no_perturbation: + print( + "--no_perturbation set: skipping perturbation results and regulator plots.", + flush=True, + ) + test_stats_df = None + else: + test_stats_df = load_perturbation_data( + folder=args.eval_folder_name, + components=args.K, + sel_thresh=sel_thresh, + conditions=args.Conditions, + pval=args.pval, + perturbation_file=args.perturbation_file, + perturb_adjpval_col=args.perturb_adjpval_col, + perturb_target_col=args.perturb_target_col, + perturb_log2fc_col=args.perturb_log2fc_col, + ) + plot_perturbation( + test_stats_df, + folder_name=args.save_folder_name, + pval=args.pval, + file_name=f"Perturbation_{sel_thresh}", + ) # Explained Variance stats_EV = load_explained_variance_data( @@ -260,8 +290,13 @@ def main(): # Motif (working in progress) - # K-selection panel - plot_k_selection_panel( + # K-selection panel (perturbation row is hidden when test_stats_df is None) + panel_fn = ( + plot_k_selection_panel_no_traits + if args.no_trait + else plot_k_selection_panel + ) + panel_fn( stats_SE, count_df, test_stats_df, diff --git a/src/Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_program_analysis.py b/src/Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_program_analysis.py index 5324231..4866e16 100644 --- a/src/Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_program_analysis.py +++ b/src/Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_program_analysis.py @@ -235,12 +235,24 @@ def main(): print(f"there are {program_len} Program found") # found detected perturbed gene (use gene symbols from var column when var_names are Ensembl IDs) - perturbed_gene = np.unique(mdata[args.prog_key].uns["guide_targets"]) - if args.gene_name_key in mdata[args.data_key].var.columns: - gene_symbols = mdata[args.data_key].var[args.gene_name_key].astype(str).tolist() + # uns['guide_targets'] only exists for perturbation data (written by inference + # when guides are present); without it there are no perturbed genes to mark. + if "guide_targets" in mdata[args.prog_key].uns: + perturbed_gene = np.unique(mdata[args.prog_key].uns["guide_targets"]) + if args.gene_name_key in mdata[args.data_key].var.columns: + gene_symbols = ( + mdata[args.data_key].var[args.gene_name_key].astype(str).tolist() + ) + else: + gene_symbols = mdata[args.data_key].var_names.tolist() + perturbed_gene_found = sorted(set(gene_symbols) & set(perturbed_gene.tolist())) else: - gene_symbols = mdata[args.data_key].var_names.tolist() - perturbed_gene_found = sorted(set(gene_symbols) & set(perturbed_gene.tolist())) + perturbed_gene_found = [] + print( + f"No uns['guide_targets'] in '{args.prog_key}': treating the data as " + "non-perturbed; perturbed-gene annotations will be empty.", + flush=True, + ) # compute correlations (both K x K; full matrices written to corr_dir every run) corr_dir = args.corr_matrix_path or args.save_path diff --git a/src/Stage3_Interpretation/A_Plotting/src/k_selection_plots.py b/src/Stage3_Interpretation/A_Plotting/src/k_selection_plots.py index 1b31c61..a235d22 100644 --- a/src/Stage3_Interpretation/A_Plotting/src/k_selection_plots.py +++ b/src/Stage3_Interpretation/A_Plotting/src/k_selection_plots.py @@ -413,7 +413,13 @@ def load_enrichment_data( trait_file=None, term_col="Term", adjpval_col="Adjusted P-value", + include_traits=True, ): + """Count unique significant terms per K for GO, gene set and trait enrichment. + + Set include_traits=False when trait enrichment was not run (non-human data); + the trait files are then not read and the 'traits' column is filled with 0. + """ # loading function def load(k, term, folder, term_col, adjpval_col): @@ -465,15 +471,16 @@ def load(k, term, folder, term_col, adjpval_col): adjpval_col, ) ) - term_df.append( - load( - k, - "traits", - "{}/{}".format(k_folder, trait_file.format(k=k)), - term_col, - adjpval_col, + if include_traits: + term_df.append( + load( + k, + "traits", + "{}/{}".format(k_folder, trait_file.format(k=k)), + term_col, + adjpval_col, + ) ) - ) term_df = pd.concat(term_df, ignore_index=True) @@ -513,7 +520,9 @@ def load(k, term, folder, term_col, adjpval_col): # plot loaded df — 3 independent square figures -def plot_enrichment(count_df, folder_name=None, file_name=None, selected_k=None): +def plot_enrichment( + count_df, folder_name=None, file_name=None, selected_k=None, include_traits=True +): enrichment_config = [ ("go_terms", "Unique GO terms", "GO term enrichment", _COLORS["go_terms"], "o"), ( @@ -526,6 +535,9 @@ def plot_enrichment(count_df, folder_name=None, file_name=None, selected_k=None) ("traits", "Unique traits", "Trait enrichment", _COLORS["traits"], "D"), ] + if not include_traits: + enrichment_config = [c for c in enrichment_config if c[0] != "traits"] + for col, ylabel, title, color, marker in enrichment_config: fig, ax = plt.subplots(figsize=(4, 3.5)) @@ -800,6 +812,88 @@ def plot_explained_variance(stats, folder_name=None, file_name=None, selected_k= plt.show() +def _plot_regulator_row(ax_all, ax_per_condition, test_stats_df, pval, _add_vline): + """Plot unique significant regulators per K, pooled and per condition.""" + # --- Row 3, Col 0: Regulators (all conditions) --- + ax = ax_all + plotting_df = ( + test_stats_df.loc[test_stats_df.adj_pval <= pval, ["K", "target_name"]] + .drop_duplicates() + .groupby(["K"]) + .count() + .reset_index() + ) + ax.plot( + plotting_df["K"], + plotting_df["target_name"], + color=_COLORS["primary"], + linewidth=1.5, + marker="o", + markersize=4, + markerfacecolor=_COLORS["perturbation"], + markeredgecolor=_COLORS["primary"], + markeredgewidth=0.8, + zorder=3, + ) + ax.fill_between( + plotting_df["K"], + plotting_df["target_name"], + alpha=0.08, + color=_COLORS["perturbation"], + ) + _add_vline(ax) + _style_ax( + ax, + xlabel="Number of components (k)", + ylabel="No. unique regulators", + title="Unique regulators for all conditions", + ) + ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True, nbins=5)) + + # --- Row 3, Col 1: Regulators (per sample) --- + ax = ax_per_condition + plotting_df_sample = ( + test_stats_df.loc[ + test_stats_df.adj_pval <= pval, ["K", "condition", "target_name"] + ] + .drop_duplicates() + .groupby(["K", "condition"]) + .count() + .reset_index() + ) + sns.lineplot( + x="K", + y="target_name", + hue="condition", + data=plotting_df_sample, + palette=_PALETTE[: plotting_df_sample["condition"].nunique()], + linewidth=1.5, + marker="o", + markersize=4, + ax=ax, + legend="brief", + ) + _add_vline(ax) + _style_ax( + ax, + xlabel="Number of components (k)", + ylabel="No. unique regulators", + title="Unique regulators per condition", + ) + ax.legend( + title="Condition", + fontsize=7, + title_fontsize=8, + frameon=True, + fancybox=False, + edgecolor="#cccccc", + bbox_to_anchor=(1.02, 1), + loc="upper left", + borderaxespad=0, + ) + ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True, nbins=5)) + + # Combined panel figure: 3 rows x 3 columns # Row 1: Stability, Error, Explained variance # Row 2: GO terms, Gene sets, Traits @@ -945,84 +1039,12 @@ def _add_vline(ax): _style_ax(ax, xlabel="Number of components (k)", ylabel=ylabel, title=title) ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True, nbins=5)) - # --- Row 3, Col 0: Regulators (all conditions) --- - ax = axes[2, 0] - plotting_df = ( - test_stats_df.loc[test_stats_df.adj_pval <= pval, ["K", "target_name"]] - .drop_duplicates() - .groupby(["K"]) - .count() - .reset_index() - ) - ax.plot( - plotting_df["K"], - plotting_df["target_name"], - color=_COLORS["primary"], - linewidth=1.5, - marker="o", - markersize=4, - markerfacecolor=_COLORS["perturbation"], - markeredgecolor=_COLORS["primary"], - markeredgewidth=0.8, - zorder=3, - ) - ax.fill_between( - plotting_df["K"], - plotting_df["target_name"], - alpha=0.08, - color=_COLORS["perturbation"], - ) - _add_vline(ax) - _style_ax( - ax, - xlabel="Number of components (k)", - ylabel="No. unique regulators", - title="Unique regulators for all conditions", - ) - ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True, nbins=5)) - - # --- Row 3, Col 1: Regulators (per sample) --- - ax = axes[2, 1] - plotting_df_sample = ( - test_stats_df.loc[ - test_stats_df.adj_pval <= pval, ["K", "condition", "target_name"] - ] - .drop_duplicates() - .groupby(["K", "condition"]) - .count() - .reset_index() - ) - sns.lineplot( - x="K", - y="target_name", - hue="condition", - data=plotting_df_sample, - palette=_PALETTE[: plotting_df_sample["condition"].nunique()], - linewidth=1.5, - marker="o", - markersize=4, - ax=ax, - legend="brief", - ) - _add_vline(ax) - _style_ax( - ax, - xlabel="Number of components (k)", - ylabel="No. unique regulators", - title="Unique regulators per condition", - ) - ax.legend( - title="Condition", - fontsize=7, - title_fontsize=8, - frameon=True, - fancybox=False, - edgecolor="#cccccc", - bbox_to_anchor=(1.02, 1), - loc="upper left", - borderaxespad=0, - ) - ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True, nbins=5)) + # --- Row 3: Regulators (hidden when there is no perturbation data) --- + if test_stats_df is None: + axes[2, 0].set_visible(False) + axes[2, 1].set_visible(False) + else: + _plot_regulator_row(axes[2, 0], axes[2, 1], test_stats_df, pval, _add_vline) # --- Row 3, Col 2: hide empty panel --- axes[2, 2].set_visible(False) @@ -1198,84 +1220,12 @@ def _add_vline(ax): # --- Row 2, Col 2: hide empty panel --- axes[1, 2].set_visible(False) - # --- Row 3, Col 0: Regulators (all conditions) --- - ax = axes[2, 0] - plotting_df = ( - test_stats_df.loc[test_stats_df.adj_pval <= pval, ["K", "target_name"]] - .drop_duplicates() - .groupby(["K"]) - .count() - .reset_index() - ) - ax.plot( - plotting_df["K"], - plotting_df["target_name"], - color=_COLORS["primary"], - linewidth=1.5, - marker="o", - markersize=4, - markerfacecolor=_COLORS["perturbation"], - markeredgecolor=_COLORS["primary"], - markeredgewidth=0.8, - zorder=3, - ) - ax.fill_between( - plotting_df["K"], - plotting_df["target_name"], - alpha=0.08, - color=_COLORS["perturbation"], - ) - _add_vline(ax) - _style_ax( - ax, - xlabel="Number of components (k)", - ylabel="No. unique regulators", - title="Unique regulators for all conditions", - ) - ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True, nbins=5)) - - # --- Row 3, Col 1: Regulators (per sample) --- - ax = axes[2, 1] - plotting_df_sample = ( - test_stats_df.loc[ - test_stats_df.adj_pval <= pval, ["K", "condition", "target_name"] - ] - .drop_duplicates() - .groupby(["K", "condition"]) - .count() - .reset_index() - ) - sns.lineplot( - x="K", - y="target_name", - hue="condition", - data=plotting_df_sample, - palette=_PALETTE[: plotting_df_sample["condition"].nunique()], - linewidth=1.5, - marker="o", - markersize=4, - ax=ax, - legend="brief", - ) - _add_vline(ax) - _style_ax( - ax, - xlabel="Number of components (k)", - ylabel="No. unique regulators", - title="Unique regulators per condition", - ) - ax.legend( - title="Condition", - fontsize=7, - title_fontsize=8, - frameon=True, - fancybox=False, - edgecolor="#cccccc", - bbox_to_anchor=(1.02, 1), - loc="upper left", - borderaxespad=0, - ) - ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True, nbins=5)) + # --- Row 3: Regulators (hidden when there is no perturbation data) --- + if test_stats_df is None: + axes[2, 0].set_visible(False) + axes[2, 1].set_visible(False) + else: + _plot_regulator_row(axes[2, 0], axes[2, 1], test_stats_df, pval, _add_vline) # --- Row 3, Col 2: hide empty panel --- axes[2, 2].set_visible(False) diff --git a/src/Stage3_Interpretation/A_Plotting/src/utilities.py b/src/Stage3_Interpretation/A_Plotting/src/utilities.py index 9fa8bf3..4258e14 100644 --- a/src/Stage3_Interpretation/A_Plotting/src/utilities.py +++ b/src/Stage3_Interpretation/A_Plotting/src/utilities.py @@ -30,7 +30,7 @@ def convert_with_mygene(dataframe, species="human", index=True): dataframe.index, scopes="ensembl.gene", fields="symbol,name", - species="human", + species=species, ) else: # Query multiple genes at once @@ -38,7 +38,7 @@ def convert_with_mygene(dataframe, species="human", index=True): dataframe.columns, scopes="ensembl.gene", fields="symbol,name", - species="human", + species=species, ) # Create mapping dictionary @@ -64,7 +64,7 @@ def convert_adata_with_mygene(adata, species="human"): mg = mygene.MyGeneInfo() gene_list = adata.var_names.tolist() annotations = mg.querymany( - gene_list, scopes="ensembl.gene", fields="symbol", species="human" + gene_list, scopes="ensembl.gene", fields="symbol", species=species ) # Process the results to create mapping @@ -225,8 +225,38 @@ def _natural_sort_key(filepath): ] +# obsm keys used by other tools (e.g. Seurat conversions) for the scanpy embeddings +ALT_EMBEDDING_KEYS = {"X_umap": ("UMAP", "umap"), "X_pca": ("PCA", "pca")} + + +def _adopt_alt_embedding_keys(adata): + """Copy an embedding stored under a non-scanpy obsm key to 'X_umap'/'X_pca'. + + Reuses the embedding the data came with instead of recomputing a different + one. Existing 'X_umap'/'X_pca' entries and the original keys are left as is. + """ + for std_key, alt_keys in ALT_EMBEDDING_KEYS.items(): + if std_key in adata.obsm: + continue + for alt_key in alt_keys: + if alt_key not in adata.obsm: + continue + emb = adata.obsm[alt_key] + # scanpy plotting needs an array; some converters store a DataFrame + emb = emb.to_numpy() if hasattr(emb, "to_numpy") else np.asarray(emb) + adata.obsm[std_key] = emb[:, :2] if std_key == "X_umap" else emb + print(f"ensure_umap: using obsm['{alt_key}'] as '{std_key}'.", flush=True) + break + + def ensure_umap(mdata, data_key, prog_key, n_top_genes=2000, n_comps=50): - """Compute PCA/UMAP from top-variance genes and write to both modalities; no-op if X_umap already exists in both.""" + """Compute PCA/UMAP from top-variance genes and write to both modalities; no-op if X_umap already exists in both. + + Embeddings stored under 'UMAP'/'PCA' (e.g. Seurat conversions) are reused first. + """ + _adopt_alt_embedding_keys(mdata[data_key]) + _adopt_alt_embedding_keys(mdata[prog_key]) + has_prog = "X_umap" in mdata[prog_key].obsm has_data = "X_umap" in mdata[data_key].obsm if has_prog and has_data: diff --git a/src/Stage3_Interpretation/B_Summarization/README.md b/src/Stage3_Interpretation/B_Summarization/README.md index 4eaf792..c707f89 100644 --- a/src/Stage3_Interpretation/B_Summarization/README.md +++ b/src/Stage3_Interpretation/B_Summarization/README.md @@ -127,7 +127,7 @@ Sidecar files (`specificity_score_*`, `corr_gene_matrix_*`, `kd_efficiency.txt`) | **Program_summary** | cNMF program | `Compile_Summary_sheet` | `manual_annotation_label`, `manual_timepoint`, `Notes`, `Automatic Timepoint`, `Total Enriched GO Terms`, `Significant regulators with positive/negative effect {Condition}`, `Top 5 specific regulators (FDR<0.1) {Condition}`, `sigfdr0.05_targets_sorted_abslog2fcd_{Condition}`, `top10_loaded_genes`, `top30_loaded_genes`, `variance_explained`, `Mean program score {Condition}`, `Fra cells above mean program score {Condition}`, `top10_enriched_genesets`, `top10_enriched_go_terms` | | **Target_summary** | perturbation target | `Compile_Target_Summary_sheet` | `mean_expression_{Condition}`, `# Cells {Condition}`, `significant programs {Condition}`, `# programs {Condition}`, `top 5 specific programs (FDR < 0.1) {Condition}`, `top 5 specificity scores (FDR < 0.1) {Condition}`, `top 5 pos/neg correls targets (program log2fc) {Condition}`, `top 5 pos/neg correls (program log2fc) {Condition}`, `kd_efficiency_{Condition}`, `Mean KD efficiency (%) across conditions` | | **Program_loadings_flat** | cNMF program | `compile_Program_loading_score_sheet_flat` | Rank `1`..`num_gene` → gene name (top genes per program, descending) | -| **Program_loadings_long** | (program, rank) | `compile_Program_loading_score_sheet_long` | `Program`, `Rank`, `Gene`, `Annotation` (mygene summary) | +| **Program_loadings_long** | (program, rank) | `compile_Program_loading_score_sheet_long` | `Program`, `Rank`, `Gene`, `Annotation` (mygene summary); with `--organism mouse` also `Human ortholog` (the human genes whose summaries are shown) | | **GO** | enriched GO term × program | `Compile_GO_sheet` | indexed by `Term`; passes through columns from `{K}_GO_term_enrichment.txt`; `Genes` truncated to top `num_gene` | | **Geneset** | enriched geneset × program | `Compile_Geneset_sheet` | same shape, from `{K}_geneset_enrichment.txt` | | **Trait** | enriched trait × program | `Compile_Trait_sheet` | same shape, from `{K}_trait_enrichment.txt` | diff --git a/src/Stage3_Interpretation/B_Summarization/Slurm_Version/cNMF_excel_summary.py b/src/Stage3_Interpretation/B_Summarization/Slurm_Version/cNMF_excel_summary.py index ef48ea4..c3c09a7 100644 --- a/src/Stage3_Interpretation/B_Summarization/Slurm_Version/cNMF_excel_summary.py +++ b/src/Stage3_Interpretation/B_Summarization/Slurm_Version/cNMF_excel_summary.py @@ -155,6 +155,23 @@ def main(): default="guide_targets", help="uns key for guide target names. Default: guide_targets", ) + parser.add_argument( + "--organism", + type=str.lower, + choices=["human", "mouse"], + default="human", + help="Species of the data, for MyGene gene summaries in the Program Loadings " + "sheet. With mouse, each gene gets the summaries of its human orthologs " + "(MGI + Ensembl), listed in a 'Human ortholog' column. Default: human", + ) + parser.add_argument( + "--ortholog_table", + type=str, + default=None, + help="For --organism mouse: TSV of mouse-human orthologs (columns: " + "mouse_symbol, mouse_ensembl, human_symbol, source). Default: the table " + "cached in src/Stage2_Evaluation/Resources/, downloaded if missing", + ) parser.add_argument( "--gene_names_key", type=str, @@ -289,6 +306,8 @@ def main(): num_gene=args.num_gene, data_key=args.data_key, gene_names_key=args.gene_names_key, + species=args.organism, + ortholog_table=args.ortholog_table, ) df_Program_loading_flat = compile_Program_loading_score_sheet_flat( mdata, @@ -370,19 +389,29 @@ def main(): ) # ── Target Summary (writes specificity / correlation / KD-efficiency sidecars to save_path) ── - df_Target_Summary = Compile_Target_Summary_sheet( - mdata, - Perturbation_path_base, - Sample=args.Conditions, - categorical_key=args.categorical_key, - prog_key=args.prog_key, - data_key=args.data_key, - guide_targets_key=args.guide_targets_key, - save_path=args.save_path, - effect_size=args.effect_size, - control_target_name=args.control_target_name, - gene_names_key=args.gene_names_key, - ) + # Needs guide information; skipped for non-perturbed data. + has_guides = args.guide_targets_key in mdata[args.prog_key].uns + if has_guides: + df_Target_Summary = Compile_Target_Summary_sheet( + mdata, + Perturbation_path_base, + Sample=args.Conditions, + categorical_key=args.categorical_key, + prog_key=args.prog_key, + data_key=args.data_key, + guide_targets_key=args.guide_targets_key, + save_path=args.save_path, + effect_size=args.effect_size, + control_target_name=args.control_target_name, + gene_names_key=args.gene_names_key, + ) + else: + print( + f"No uns['{args.guide_targets_key}'] in '{args.prog_key}': " + "skipping the Targets Summary sheet (non-perturbed data).", + flush=True, + ) + df_Target_Summary = None # ── Program Summary ── df_Summary = Compile_Summary_sheet( @@ -393,7 +422,8 @@ def main(): df_Program_loading_flat, df_Explained_Variance, Sample=args.Conditions, - specicicity_path=args.save_path, + # specificity scores are written by the Targets Summary step + specicicity_path=args.save_path if has_guides else None, categorical_key=args.categorical_key, non_tagerting_key=args.non_targeting_key, effect_size=args.effect_size, @@ -405,9 +435,10 @@ def main(): df_Program_loading_long.to_csv( f"{args.save_path}/Program_Loadings_{args.K}_{thresh_str}.tsv", sep="\t" ) - df_Target_Summary.to_csv( - f"{args.save_path}/Targets_Summary_{args.K}_{thresh_str}.tsv", sep="\t" - ) + if df_Target_Summary is not None: + df_Target_Summary.to_csv( + f"{args.save_path}/Targets_Summary_{args.K}_{thresh_str}.tsv", sep="\t" + ) print(f"Saved separate TSV files to {args.save_path}") # ── Write Excel ── @@ -418,7 +449,8 @@ def main(): df_Program_loading_long.to_excel( writer, sheet_name="Program Loadings", index=True ) - df_Target_Summary.to_excel(writer, sheet_name="Targets Summary", index=True) + if df_Target_Summary is not None: + df_Target_Summary.to_excel(writer, sheet_name="Targets Summary", index=True) if df_Association is not None: df_Association.to_excel(writer, sheet_name="Sample Association", index=True) diff --git a/src/Stage3_Interpretation/B_Summarization/src/Compile_excel_sheet.py b/src/Stage3_Interpretation/B_Summarization/src/Compile_excel_sheet.py index 924b338..bb5fe63 100644 --- a/src/Stage3_Interpretation/B_Summarization/src/Compile_excel_sheet.py +++ b/src/Stage3_Interpretation/B_Summarization/src/Compile_excel_sheet.py @@ -112,8 +112,19 @@ def add_specificity_scores_file( # -------------- helper methods for loading sheets-------------- def compile_Program_loading_score_sheet_long( - mdata, num_gene=300, data_key="rna", gene_names_key=None + mdata, + num_gene=300, + data_key="rna", + gene_names_key=None, + species="human", + ortholog_table=None, ): + """Top genes per program with MyGene summaries (long form). + + For species='mouse', each gene is annotated with the summaries of its human + orthologs (MGI + Ensembl table, see Stage2 orthologs.py), listed in a 'Human + ortholog' column; ortholog_table is an optional path to a pre-built table. + """ print("Load program loadings data in long form") @@ -124,7 +135,9 @@ def compile_Program_loading_score_sheet_long( ) # Rename columns to curated gene names - if gene_names_key is not None: + # Fall back to var_names when the symbol column is absent (e.g. var_names are + # already symbols), as compute_kd_efficiency does. + if gene_names_key is not None and gene_names_key in mdata[data_key].var.columns: rename_map = dict( zip(mdata[data_key].var_names, mdata[data_key].var[gene_names_key]) ) @@ -146,6 +159,17 @@ def compile_Program_loading_score_sheet_long( mg = MyGeneInfo() long_data = [] + use_orthologs = species == "mouse" + if use_orthologs: + from Stage2_Evaluation.A_Metrics.src.orthologs import ( + load_ortholog_table, + map_mouse_to_human, + ) + + # Map all top genes at once (building the lookup is the slow part). + all_genes = pd.unique(result_df.stack().dropna().astype(str).str.strip()) + orthologs = map_mouse_to_human(all_genes, load_ortholog_table(ortholog_table)) + for program_idx, row in result_df.iterrows(): # Query all genes for this program at once genes_list = row.dropna().astype(str).str.strip().tolist() @@ -154,13 +178,23 @@ def compile_Program_loading_score_sheet_long( if not genes_list: continue + if use_orthologs: + query_genes = sorted({h for g in genes_list for h in orthologs.get(g, [])}) + query_species = "human" + else: + query_genes, query_species = genes_list, species + # Get results for all genes in this program - results = mg.querymany( - genes_list, - scopes="symbol", - fields="summary", - species="human", - verbose=False, + results = ( + mg.querymany( + query_genes, + scopes="symbol", + fields="summary", + species=query_species, + verbose=False, + ) + if query_genes + else [] ) # Create a lookup dictionary for faster access @@ -175,16 +209,23 @@ def compile_Program_loading_score_sheet_long( # Skip NaN values if pd.notna(gene) and gene != "": gene_clean = str(gene).strip() - annotation = summary_dict.get(gene_clean, "N/A") - - long_data.append( - { - "Program": program_idx, - "Rank": rank, - "Gene": gene_clean, - "Annotation": annotation, - } - ) + record = { + "Program": program_idx, + "Rank": rank, + "Gene": gene_clean, + } + if use_orthologs: + human = orthologs.get(gene_clean, []) + summaries = [ + f"{h}: {summary_dict[h]}" + for h in human + if summary_dict.get(h, "N/A") != "N/A" + ] + record["Human ortholog"] = ";".join(human) + record["Annotation"] = " | ".join(summaries) or "N/A" + else: + record["Annotation"] = summary_dict.get(gene_clean, "N/A") + long_data.append(record) # Convert to DataFrame annotation_df = pd.DataFrame(long_data) @@ -205,7 +246,9 @@ def compile_Program_loading_score_sheet_flat( ) # Rename columns to curated gene names - if gene_names_key is not None: + # Fall back to var_names when the symbol column is absent (e.g. var_names are + # already symbols), as compute_kd_efficiency does. + if gene_names_key is not None and gene_names_key in mdata[data_key].var.columns: rename_map = dict( zip(mdata[data_key].var_names, mdata[data_key].var[gene_names_key]) ) diff --git a/tests/Script/Stage2_Evaluation/test_orthologs.py b/tests/Script/Stage2_Evaluation/test_orthologs.py new file mode 100644 index 0000000..92a73a6 --- /dev/null +++ b/tests/Script/Stage2_Evaluation/test_orthologs.py @@ -0,0 +1,227 @@ +"""Unit tests for mouse to human ortholog mapping (Stage2 orthologs.py). + +Uses a small synthetic ortholog table, so no network access is needed. + +Usage (from the repo root): + python -m pytest tests/Script/Stage2_Evaluation/test_orthologs.py -v +""" + +import anndata as ad +import mudata as md +import numpy as np +import pandas as pd +import pytest + +from Stage2_Evaluation.A_Metrics.src.enrichment_geneset import ( + get_program_gene_loadings, +) +from Stage2_Evaluation.A_Metrics.src.orthologs import ( + TABLE_COLUMNS, + collapse_to_human, + load_ortholog_table, + map_mouse_to_human, +) + +# mouse_symbol, mouse_ensembl, human_symbol, source +ORTHOLOGS = [ + ("Cdh5", "ENSMUSG00000031871", "CDH5", "Ensembl"), # same name, one-to-one + ("Cdh5", np.nan, "CDH5", "MGI"), # duplicate from the second source + ("Zfp451", np.nan, "ZNF451", "MGI"), # different name + ("H2-Ab1", np.nan, "HLA-DQB1", "Ensembl"), # one-to-many + ("H2-Ab1", np.nan, "HLA-DQB2", "Ensembl"), + ("Cd59a", np.nan, "CD59", "Ensembl"), # many-to-one + ("Cd59b", np.nan, "CD59", "Ensembl"), + ("Gm12345", "ENSMUSG00000099999", np.nan, "Ensembl"), # no human ortholog +] +# Upper-cases to an existing human symbol but is not its ortholog in the table. +CASE_DECOY = "Ccdc115" + + +@pytest.fixture +def table(): + return pd.DataFrame(ORTHOLOGS, columns=TABLE_COLUMNS) + + +@pytest.fixture +def table_path(table, tmp_path): + path = tmp_path / "orthologs.tsv" + table.to_csv(path, sep="\t", index=False) + return str(path) + + +def test_map_mouse_to_human(table): + mapping = map_mouse_to_human( + ["Cdh5", "Zfp451", "H2-Ab1", "Cd59a", "Gm12345", CASE_DECOY], table + ) + assert mapping == { + "Cdh5": ["CDH5"], + "Zfp451": ["ZNF451"], + "H2-Ab1": ["HLA-DQB1", "HLA-DQB2"], + "Cd59a": ["CD59"], + "Gm12345": [], + CASE_DECOY: [], # no fallback to upper-casing + } + + +def test_map_ensembl_ids(table): + mapping = map_mouse_to_human( + ["ENSMUSG00000031871", "ENSMUSG00000031871.7", "ENSMUSG00000099999"], table + ) + assert mapping["ENSMUSG00000031871"] == ["CDH5"] + assert mapping["ENSMUSG00000031871.7"] == ["CDH5"] # version suffix ignored + assert mapping["ENSMUSG00000099999"] == [] + + +def test_collapse_to_human(table): + loadings = pd.DataFrame( + {"p0": [5.0, 1.0, 2.0, 3.0, 4.0, 9.0], "p1": [0.5, 0.1, 0.2, 0.3, 0.6, 0.9]}, + index=["Cdh5", "Zfp451", "H2-Ab1", "Cd59a", "Cd59b", "Gm12345"], + ) + human = collapse_to_human(loadings, table) + + assert set(human.index) == {"CDH5", "ZNF451", "HLA-DQB1", "HLA-DQB2", "CD59"} + # one-to-many: both human orthologs take the mouse loading + assert human.loc["HLA-DQB1", "p0"] == human.loc["HLA-DQB2", "p0"] == 2.0 + # many-to-one: per-program maximum over the mouse orthologs + assert human.loc["CD59", "p0"] == 4.0 and human.loc["CD59", "p1"] == 0.6 + assert human.loc["ZNF451", "p1"] == 0.1 + + +def test_collapse_without_any_ortholog_fails(table): + with pytest.raises(ValueError, match="No mouse genes"): + collapse_to_human(pd.DataFrame({"p0": [1.0]}, index=["Gm12345"]), table) + + +def test_load_ortholog_table_validates_columns(tmp_path): + bad = tmp_path / "bad.tsv" + pd.DataFrame({"mouse": ["Cdh5"], "human": ["CDH5"]}).to_csv( + bad, sep="\t", index=False + ) + with pytest.raises(ValueError, match="missing columns"): + load_ortholog_table(str(bad)) + + +def _mdata(genes): + rng = np.random.default_rng(0) + rna = ad.AnnData( + X=rng.random((6, len(genes))).astype(np.float32), + var=pd.DataFrame(index=genes), + ) + prog = ad.AnnData( + X=rng.random((6, 2)).astype(np.float32), var=pd.DataFrame(index=["0", "1"]) + ) + prog.varm["loadings"] = rng.random((2, len(genes))).astype(np.float32) + prog.uns["var_names"] = np.array(genes) + return md.MuData({"rna": rna, "cNMF": prog}) + + +def test_program_loadings_mouse_uses_orthologs(table_path): + genes = ["Cdh5", "Zfp451", "H2-Ab1", "Gm12345", CASE_DECOY] + loadings = get_program_gene_loadings( + _mdata(genes), + prog_key="cNMF", + data_key="rna", + organism="mouse", + ortholog_table=table_path, + ) + assert set(loadings.index) == {"CDH5", "ZNF451", "HLA-DQB1", "HLA-DQB2"} + assert list(loadings.columns) == ["0", "1"] + + +def test_program_loadings_mouse_native_symbols(table_path): + """map_to_human=False (MSigDB mouse gene sets) keeps mouse symbols unchanged.""" + genes = ["Cdh5", "Zfp451"] + loadings = get_program_gene_loadings( + _mdata(genes), + prog_key="cNMF", + data_key="rna", + organism="mouse", + ortholog_table=table_path, + map_to_human=False, + ) + assert list(loadings.index) == genes + + +def test_program_loadings_human_unchanged(): + genes = ["CDH5", "znf451"] + loadings = get_program_gene_loadings( + _mdata(genes), prog_key="cNMF", data_key="rna", organism="human" + ) + assert list(loadings.index) == ["CDH5", "ZNF451"] + + +def test_unsupported_organism(): + with pytest.raises(ValueError, match="Unsupported organism"): + get_program_gene_loadings( + _mdata(["Cdh5"]), prog_key="cNMF", data_key="rna", organism="rat" + ) + + +# Download handling (network calls are replaced by stubs) + +ENSEMBL_HEADER = "Gene stable ID\tGene name\tHuman gene name\n" +ENSEMBL_ROW = "ENSMUSG00000031871\tCdh5\tCDH5\n" + + +def test_ensembl_requires_completion_stamp(monkeypatch): + """A response cut short by BioMart (no '[success]' line) is rejected.""" + from Stage2_Evaluation.A_Metrics.src import orthologs + + monkeypatch.setattr( + orthologs, "_download", lambda url: ENSEMBL_HEADER + ENSEMBL_ROW + ) + with pytest.raises(RuntimeError, match="completion stamp"): + orthologs._fetch_ensembl() + + monkeypatch.setattr( + orthologs, "_download", lambda url: ENSEMBL_HEADER + ENSEMBL_ROW + "[success]\n" + ) + df = orthologs._fetch_ensembl() + assert df.to_dict("records") == [ + { + "mouse_symbol": "Cdh5", + "mouse_ensembl": "ENSMUSG00000031871", + "human_symbol": "CDH5", + "source": "Ensembl", + } + ] + + +def test_download_failure_suggests_ortholog_table(monkeypatch, tmp_path): + from Stage2_Evaluation.A_Metrics.src import orthologs + + def offline(): + raise OSError("network unreachable") + + monkeypatch.setattr(orthologs, "DEFAULT_TABLE_PATH", str(tmp_path / "cache.tsv")) + monkeypatch.setattr(orthologs, "build_ortholog_table", offline) + with pytest.raises(RuntimeError, match="--ortholog_table"): + orthologs.load_ortholog_table() + + +def test_cache_is_written_atomically(monkeypatch, tmp_path, table): + """The cache appears in one step (no partial file) and is reused afterwards.""" + from Stage2_Evaluation.A_Metrics.src import orthologs + + cache = tmp_path / "cache.tsv" + monkeypatch.setattr(orthologs, "RESOURCES_DIR", str(tmp_path)) + monkeypatch.setattr(orthologs, "DEFAULT_TABLE_PATH", str(cache)) + monkeypatch.setattr(orthologs, "build_ortholog_table", lambda: table) + replaced = [] + real_replace = orthologs.os.replace + + def record_replace(src, dst): + assert not cache.exists() # nothing written to the final path before the move + replaced.append((src, dst)) + real_replace(src, dst) + + monkeypatch.setattr(orthologs.os, "replace", record_replace) + first = orthologs.load_ortholog_table() + assert replaced and replaced[0][1] == str(cache) + assert list(tmp_path.iterdir()) == [cache] # temporary file is gone + + monkeypatch.setattr( + orthologs, "build_ortholog_table", lambda: pytest.fail("rebuilt") + ) + second = orthologs.load_ortholog_table() + assert len(second) == len(first) diff --git a/tests/Script/Stage3_Interpretation/A_Plotting/Program/test_ensure_umap.py b/tests/Script/Stage3_Interpretation/A_Plotting/Program/test_ensure_umap.py new file mode 100644 index 0000000..a740d29 --- /dev/null +++ b/tests/Script/Stage3_Interpretation/A_Plotting/Program/test_ensure_umap.py @@ -0,0 +1,87 @@ +"""Tests for ensure_umap reusing embeddings stored under non-scanpy obsm keys. + +Seurat conversions store embeddings as obsm['UMAP'] / obsm['PCA'] (often as +DataFrames); ensure_umap should reuse them instead of computing a new UMAP. + +Usage (from the repo root): + python -m pytest tests/Script/Stage3_Interpretation/A_Plotting/Program/test_ensure_umap.py -v +""" + +import anndata as ad +import mudata as md +import numpy as np +import pandas as pd + +from Stage3_Interpretation.A_Plotting.src.utilities import ensure_umap + +N_CELLS = 60 +N_GENES = 40 +N_PROGRAMS = 3 +SEED = 0 + + +def _mdata(): + """MuData with 'rna' and 'cNMF' modalities and no embeddings.""" + rng = np.random.default_rng(SEED) + cells = [f"cell{i}" for i in range(N_CELLS)] + rna = ad.AnnData( + X=rng.poisson(3.0, size=(N_CELLS, N_GENES)).astype(np.float32), + obs=pd.DataFrame(index=cells), + var=pd.DataFrame(index=[f"Gene{i}" for i in range(N_GENES)]), + ) + prog = ad.AnnData( + X=rng.random((N_CELLS, N_PROGRAMS)).astype(np.float32), + obs=pd.DataFrame(index=cells), + var=pd.DataFrame(index=[str(p) for p in range(N_PROGRAMS)]), + ) + return md.MuData({"rna": rna, "cNMF": prog}) + + +def _seurat_embedding(mdata, n_cols, prefix): + rng = np.random.default_rng(SEED + n_cols) + return pd.DataFrame( + rng.normal(size=(N_CELLS, n_cols)), + index=mdata["rna"].obs_names, + columns=[f"{prefix}_{i + 1}" for i in range(n_cols)], + ) + + +def test_adopts_seurat_keys_and_mirrors_to_programs(capsys): + mdata = _mdata() + umap = _seurat_embedding(mdata, 3, "UMAP") # some converters keep 3 dims + pca = _seurat_embedding(mdata, 10, "PC") + mdata["rna"].obsm["UMAP"] = umap + mdata["rna"].obsm["PCA"] = pca + + ensure_umap(mdata, "rna", "cNMF") + + for key in ["rna", "cNMF"]: + x_umap = mdata[key].obsm["X_umap"] + assert isinstance(x_umap, np.ndarray) + np.testing.assert_array_equal(x_umap, umap.to_numpy()[:, :2]) + np.testing.assert_array_equal(mdata[key].obsm["X_pca"], pca.to_numpy()) + assert "UMAP" in mdata["rna"].obsm and "PCA" in mdata["rna"].obsm + assert "using obsm['UMAP'] as 'X_umap'" in capsys.readouterr().out + + +def test_existing_x_umap_is_not_overwritten(): + mdata = _mdata() + existing = np.zeros((N_CELLS, 2)) + for key in ["rna", "cNMF"]: + mdata[key].obsm["X_umap"] = existing + mdata["rna"].obsm["UMAP"] = _seurat_embedding(mdata, 2, "UMAP") + + ensure_umap(mdata, "rna", "cNMF") + + for key in ["rna", "cNMF"]: + np.testing.assert_array_equal(mdata[key].obsm["X_umap"], existing) + + +def test_computes_umap_when_no_embedding_exists(): + mdata = _mdata() + + ensure_umap(mdata, "rna", "cNMF", n_top_genes=N_GENES, n_comps=5) + + for key in ["rna", "cNMF"]: + assert mdata[key].obsm["X_umap"].shape == (N_CELLS, 2) + assert mdata[key].obsm["X_pca"].shape == (N_CELLS, 5) diff --git a/tests/Script/test_non_perturbed.py b/tests/Script/test_non_perturbed.py new file mode 100644 index 0000000..e6280b3 --- /dev/null +++ b/tests/Script/test_non_perturbed.py @@ -0,0 +1,407 @@ +"""Tests for running the pipeline on non-perturbed (no guide) and non-human data. + +Covers the code paths that previously assumed CRISPR guide information or human data: + +- Stage 2 evaluation skips trait (GWAS) enrichment for non-human organisms and fails + early with a clear message when perturbation association is requested without guides. +- Stage 3 K-selection runs without trait or perturbation files (--no_trait, + --no_perturbation). +- Stage 3 Excel summary runs on an h5mu without uns['guide_targets'] and, for + mouse data, annotates genes with their human orthologs' summaries. + +All inputs are small synthetic files; no network access or inference output is needed. + +Usage (from the repo root, in the pipeline environment): + python -m pytest tests/Script/test_non_perturbed.py -v +""" + +import importlib.util +import os +import subprocess +import sys +from pathlib import Path +from typing import ClassVar + +import matplotlib + +matplotlib.use("Agg") + +import anndata as ad +import mudata as md +import numpy as np +import pandas as pd +import pytest + +PIPELINE_ROOT = Path(__file__).resolve().parents[2] +SRC = PIPELINE_ROOT / "src" +sys.path.insert(0, str(SRC)) + +EVAL_SCRIPT = ( + SRC / "Stage2_Evaluation/A_Metrics/Slurm_Version/cNMF_evaluation_pipeline.py" +) +K_SELECTION_SCRIPT = ( + SRC / "Stage3_Interpretation/A_Plotting/Slurm_Version/cNMF_k_selection.py" +) +EXCEL_SCRIPT = ( + SRC / "Stage3_Interpretation/B_Summarization/Slurm_Version/cNMF_excel_summary.py" +) + +K_VALUES = [3, 5] +SEL_THRESH = 2.0 +SEL_STR = "2_0" +RUN_NAME = "non_perturbed_run" +N_CELLS = 40 +N_GENES = 25 +SEED = 0 + + +def _load_module(path, name): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _make_mdata(k, rng): + """MuData with 'rna' and 'cNMF' modalities and no guide information.""" + cells = [f"cell{i}" for i in range(N_CELLS)] + genes = [f"Gene{i}" for i in range(N_GENES)] # mouse-style capitalization + rna = ad.AnnData( + X=rng.poisson(2.0, size=(N_CELLS, N_GENES)).astype(np.float32), + obs=pd.DataFrame( + {"sample": pd.Categorical(rng.choice(["ctrl", "treated"], N_CELLS))}, + index=cells, + ), + # Symbols as var_names and no 'symbol' column, as in Seurat-converted data + var=pd.DataFrame(index=genes), + ) + prog = ad.AnnData( + X=rng.random((N_CELLS, k)).astype(np.float32), + obs=pd.DataFrame(index=cells), + var=pd.DataFrame(index=[str(p) for p in range(k)]), + ) + prog.varm["loadings"] = rng.random((k, N_GENES)).astype(np.float32) + prog.uns["var_names"] = np.array(genes) + return md.MuData({"rna": rna, "cNMF": prog}) + + +@pytest.fixture(scope="module") +def run_dir(tmp_path_factory): + """Synthetic run directory: inference h5mu files plus Stage 2 outputs without + trait or perturbation results, as produced for non-perturbed mouse data.""" + rng = np.random.default_rng(SEED) + out_dir = tmp_path_factory.mktemp("out") + run = out_dir / RUN_NAME + adata_dir = run / "Inference" / "adata" + adata_dir.mkdir(parents=True) + + stability = [] + for k in K_VALUES: + _make_mdata(k, rng).write(adata_dir / f"cNMF_{k}_{SEL_STR}.h5mu") + + eval_dir = run / "Evaluation" / f"{k}_{SEL_STR}" + eval_dir.mkdir(parents=True) + programs = [str(p) for p in range(k)] + for fname in [f"{k}_GO_term_enrichment.txt", f"{k}_geneset_enrichment.txt"]: + pd.DataFrame( + { + "Term": [f"TERM_{p}" for p in programs], + "Adjusted P-value": rng.uniform(0, 0.1, k), + "Genes": ["Gene0;Gene1"] * k, + "program_name": programs, + } + ).to_csv(eval_dir / fname, sep="\t", index=False) + pd.DataFrame({"Total": [rng.uniform(0.2, 0.6)]}, index=["all"]).to_csv( + eval_dir / f"{k}_Explained_Variance_Summary.txt", sep="\t" + ) + pd.DataFrame( + { + "VarianceExplained": rng.uniform(0, 0.1, k), + "ProgramID": programs, + "program_name": programs, + } + ).to_csv(eval_dir / f"{k}_Explained_Variance.txt", sep="\t", index=False) + stability.append( + { + "k": k, + "silhouette": rng.uniform(0.5, 1.0), + "prediction_error": rng.uniform(1, 2), + } + ) + + stability_file = run / "Inference" / "k_selection_stats.tsv" + pd.DataFrame(stability).to_csv(stability_file, sep="\t", index=False) + return {"out_dir": out_dir, "run": run, "stability_file": stability_file} + + +# Stage 2: evaluation + + +@pytest.fixture(scope="module") +def evaluation_main(): + return _load_module(EVAL_SCRIPT, "cNMF_evaluation_pipeline").main + + +def _eval_argv(run_dir, *extra): + return [ + str(EVAL_SCRIPT), + "--out_dir", + str(run_dir["out_dir"]), + "--run_name", + RUN_NAME, + "--K", + str(K_VALUES[0]), + "--sel_threshs", + str(SEL_THRESH), + *extra, + ] + + +def test_trait_enrichment_skipped_for_mouse( + run_dir, evaluation_main, monkeypatch, capsys +): + """--Perform_trait with a non-human organism is skipped with a message, + without requiring --gwas_data_path.""" + monkeypatch.setattr( + sys, "argv", _eval_argv(run_dir, "--Perform_trait", "--organism", "mouse") + ) + assert evaluation_main() == 0 + + assert "skipping trait (GWAS) enrichment" in capsys.readouterr().out + eval_dir = run_dir["run"] / "Evaluation" / f"{K_VALUES[0]}_{SEL_STR}" + assert not (eval_dir / f"{K_VALUES[0]}_trait_enrichment.txt").exists() + + +def test_trait_enrichment_still_requires_gwas_for_human( + run_dir, evaluation_main, monkeypatch +): + """Human behavior is unchanged: --Perform_trait needs --gwas_data_path.""" + monkeypatch.setattr( + sys, "argv", _eval_argv(run_dir, "--Perform_trait", "--organism", "human") + ) + with pytest.raises(SystemExit): + evaluation_main() + + +def test_perturbation_without_guides_fails_clearly( + run_dir, evaluation_main, monkeypatch +): + """Fails before any metric is computed (no categorical results written).""" + monkeypatch.setattr( + sys, + "argv", + _eval_argv( + run_dir, + "--Perform_perturbation", + "--Perform_categorical", + "--categorical_key", + "sample", + ), + ) + with pytest.raises(ValueError, match="drop --Perform_perturbation"): + evaluation_main() + eval_dir = run_dir["run"] / "Evaluation" / f"{K_VALUES[0]}_{SEL_STR}" + assert not ( + eval_dir / f"{K_VALUES[0]}_categorical_association_results.txt" + ).exists() + + +def test_perturbation_with_wrong_prog_key_names_the_modality( + run_dir, evaluation_main, monkeypatch +): + monkeypatch.setattr( + sys, "argv", _eval_argv(run_dir, "--Perform_perturbation", "--prog_key", "prog") + ) + with pytest.raises(ValueError, match="check --prog_key"): + evaluation_main() + + +def test_organism_is_case_insensitive(run_dir, evaluation_main, monkeypatch, capsys): + monkeypatch.setattr( + sys, "argv", _eval_argv(run_dir, "--Perform_trait", "--organism", "Mouse") + ) + assert evaluation_main() == 0 + assert "skipping trait (GWAS) enrichment" in capsys.readouterr().out + + +def test_unsupported_organism_is_rejected(run_dir, evaluation_main, monkeypatch): + monkeypatch.setattr(sys, "argv", _eval_argv(run_dir, "--organism", "rat")) + with pytest.raises(SystemExit): + evaluation_main() + + +# Stage 3: K-selection plotting + + +def test_k_selection_without_trait_and_perturbation(run_dir, tmp_path): + save_dir = tmp_path / "K_selection" + cmd = [ + sys.executable, + str(K_SELECTION_SCRIPT), + "--output_directory", + str(run_dir["out_dir"]), + "--run_name", + RUN_NAME, + "--save_folder_name", + str(save_dir), + "--eval_folder_name", + str(run_dir["run"] / "Evaluation"), + "--K", + *[str(k) for k in K_VALUES], + "--sel_threshs", + str(SEL_THRESH), + "--stability_file", + str(run_dir["stability_file"]), + "--no_trait", + "--no_perturbation", + ] + env = {**os.environ, "MPLBACKEND": "Agg"} + result = subprocess.run(cmd, capture_output=True, text=True, env=env, check=False) + assert result.returncode == 0, result.stdout[-3000:] + result.stderr[-3000:] + + produced = {p.name for p in save_dir.iterdir()} + assert f"K-selection_panel_{SEL_THRESH}.png" in produced + assert f"Enrichment_{SEL_THRESH}_go_terms.png" in produced + assert f"Enrichment_{SEL_THRESH}_traits.png" not in produced + assert not any(name.startswith("Perturbation_") for name in produced) + + +def test_k_selection_requires_files_by_default(run_dir, tmp_path): + """Without the new flags, missing trait/perturbation files are still an error.""" + cmd = [ + sys.executable, + str(K_SELECTION_SCRIPT), + "--output_directory", + str(run_dir["out_dir"]), + "--run_name", + RUN_NAME, + "--save_folder_name", + str(tmp_path / "K_selection"), + "--eval_folder_name", + str(run_dir["run"] / "Evaluation"), + "--K", + *[str(k) for k in K_VALUES], + "--sel_threshs", + str(SEL_THRESH), + "--Conditions", + "ctrl", + "--stability_file", + str(run_dir["stability_file"]), + ] + result = subprocess.run(cmd, capture_output=True, text=True, check=False) + assert result.returncode != 0 + assert "required evaluation file(s) not found" in result.stdout + + +def test_k_selection_panel_hides_regulator_row(tmp_path): + from Stage3_Interpretation.A_Plotting.src import k_selection_plots as ksp + + stats = pd.DataFrame( + {"k": K_VALUES, "silhouette": [0.8, 0.7], "prediction_error": [1.5, 1.2]} + ) + counts = pd.DataFrame( + {"go_terms": [3, 4], "genesets": [2, 3], "traits": [0, 0]}, index=K_VALUES + ) + variance = {3: 0.3, 5: 0.4} + figures = [] + original_subplots = ksp.plt.subplots + + def capture_subplots(*args, **kwargs): + fig, axes = original_subplots(*args, **kwargs) + figures.append(axes) + return fig, axes + + ksp.plt.subplots = capture_subplots + try: + for panel_fn in [ + ksp.plot_k_selection_panel, + ksp.plot_k_selection_panel_no_traits, + ]: + panel_fn( + stats, counts, None, variance, folder_name=str(tmp_path), file_name="p" + ) + finally: + ksp.plt.subplots = original_subplots + + for axes in figures: + assert not axes[2, 0].get_visible() + assert not axes[2, 1].get_visible() + + +# Stage 3: Excel summary + + +class _FakeMyGene: + """Offline stand-in for mygene.MyGeneInfo that records the species queried.""" + + species_seen: ClassVar[list] = [] + + def querymany(self, genes, species=None, **kwargs): + _FakeMyGene.species_seen.append(species) + return [{"query": g, "summary": f"summary of {g}"} for g in genes] + + +def test_excel_summary_without_guides(run_dir, monkeypatch, capsys, tmp_path): + """Non-perturbed mouse data: no Targets Summary sheet, and gene summaries come + from human orthologs (synthetic table: even-numbered genes have an ortholog).""" + excel = _load_module(EXCEL_SCRIPT, "cNMF_excel_summary") + from Stage3_Interpretation.B_Summarization.src import Compile_excel_sheet + + monkeypatch.setattr(Compile_excel_sheet, "MyGeneInfo", _FakeMyGene) + _FakeMyGene.species_seen.clear() + ortholog_table = tmp_path / "orthologs.tsv" + pd.DataFrame( + { + "mouse_symbol": [f"Gene{i}" for i in range(0, N_GENES, 2)], + "mouse_ensembl": pd.NA, + "human_symbol": [f"HSGENE{i}" for i in range(0, N_GENES, 2)], + "source": "test", + } + ).to_csv(ortholog_table, sep="\t", index=False) + + k = K_VALUES[1] + monkeypatch.setattr( + sys, + "argv", + [ + str(EXCEL_SCRIPT), + "--out_dir", + str(run_dir["out_dir"]), + "--run_name", + RUN_NAME, + "--K", + str(k), + "--sel_thresh", + str(SEL_THRESH), + "--organism", + "mouse", + "--categorical_key", + "sample", + "--num_gene", + "5", + "--ortholog_table", + str(ortholog_table), + ], + ) + assert excel.main() == 0 + + assert "skipping the Targets Summary sheet" in capsys.readouterr().out + assert set(_FakeMyGene.species_seen) == {"human"} + save_path = run_dir["run"] / "Interpretation" / "Summary_table" / f"{k}_{SEL_STR}" + sheets = pd.ExcelFile(save_path / f"cNMF_{k}_{SEL_STR}.xlsx").sheet_names + assert "Summary" in sheets and "Program Loadings" in sheets + assert "Targets Summary" not in sheets + assert not (save_path / f"Targets_Summary_{k}_{SEL_STR}.tsv").exists() + + loadings = pd.read_csv( + save_path / f"Program_Loadings_{k}_{SEL_STR}.tsv", + sep="\t", + keep_default_na=False, # keep the literal "N/A" + ) + for _, row in loadings.iterrows(): + n = int(row["Gene"].removeprefix("Gene")) + if n % 2 == 0: + assert row["Human ortholog"] == f"HSGENE{n}" + assert row["Annotation"] == f"HSGENE{n}: summary of HSGENE{n}" + else: + assert row["Human ortholog"] == "" and row["Annotation"] == "N/A"