diff --git a/.claude/skills/perturbNMF-runner/scripts/convert_igvf_mudata_to_h5ad.py b/.claude/skills/perturbNMF-runner/scripts/convert_igvf_mudata_to_h5ad.py index 6ad97ce..cab061a 100644 --- a/.claude/skills/perturbNMF-runner/scripts/convert_igvf_mudata_to_h5ad.py +++ b/.claude/skills/perturbNMF-runner/scripts/convert_igvf_mudata_to_h5ad.py @@ -14,7 +14,7 @@ **binary assignment in layers['guide_assignment']**. This script uses the layer; using X would treat ambient guide reads as perturbations. -Generalises the one-off ``Convert_file_adata.py`` written for IGVF_Gersbach_iHep. +Generalises a one-off ``Convert_file_adata.py`` written for a single IGVF dataset. """ import argparse diff --git a/.claude/skills/perturbNMF-runner/scripts/run_add_umap.sh b/.claude/skills/perturbNMF-runner/scripts/run_add_umap.sh index 706c09c..6ffa54c 100755 --- a/.claude/skills/perturbNMF-runner/scripts/run_add_umap.sh +++ b/.claude/skills/perturbNMF-runner/scripts/run_add_umap.sh @@ -8,40 +8,50 @@ # Run directly and it measures the h5mu, derives --mem/--time/--cpus-per-task, and # re-submits itself with those overrides. Do NOT prefix it with sbatch -- if you do, # that job just re-submits a correctly sized one and exits, costing an extra hop. -# To pin resources yourself, set ADD_UMAP_INJOB to skip the sizing path entirely: +# To pin resources yourself, set ADD_UMAP_INJOB to skip the sizing path entirely +# (Slurm does not create the log directory, so make it first): +# mkdir -p add_umap_logs # ADD_UMAP_INJOB=1 sbatch --mem=64G --time=06:00:00 run_add_umap.sh # The static #SBATCH directives below only apply on that manual path; they are -# sized for the largest jamboree input. +# sized for the largest input measured (below) and log to ./add_umap_logs. +# Edit (or export SBATCH_PARTITION) and , and set +# PIPELINE_ROOT first: +# export PIPELINE_ROOT=/path/to/PerturbNMF +# On the self-sizing path, logs go to $ADD_UMAP_LOG_DIR (default: ./add_umap_logs +# in the directory you run it from), which is created automatically. # # Sizing basis (measured 2026-09-15). Peak RSS is driven by the expression matrix # and, in the dense case, by the temporary numpy allocates for X.var(axis=0) inside -# ensure_umap -- that temporary is the full size of X again, which is why the dense -# Gersbach matrix dominates everything else: +# ensure_umap -- that temporary is the full size of X again, which is why a dense +# matrix dominates everything else: # -# dataset rna/X resident est. peak -# Gersbach 46.95 GB dense f32 1.05M x 11138 ~95-115 GB -# Hon_CM 9.91 GB sparse nnz 1.24e9 ~30-40 GB -# Huangfu DE 5.01 GB sparse nnz 627M ~15-20 GB -# Huangfu ESC 3.37 GB sparse nnz 422M ~10-14 GB +# rna/X resident est. peak +# 46.95 GB dense f32 1.05M x 11138 ~95-115 GB +# 9.91 GB sparse nnz 1.24e9 ~30-40 GB +# 5.01 GB sparse nnz 627M ~15-20 GB +# 3.37 GB sparse nnz 422M ~10-14 GB # # --in_place additionally needs free disk for a second copy of the h5mu while it -# writes (48 GB for Gersbach; Oak had ~11 TB free at time of writing). +# writes (48 GB for the largest input above). #SBATCH --job-name=add_umap -#SBATCH --output=/scratch/users/ymo/add_umap_logs/%j.out -#SBATCH --error=/scratch/users/ymo/add_umap_logs/%j.err -#SBATCH --partition=engreitz,owners +#SBATCH --output=add_umap_logs/%j.out # relative to the submit dir +#SBATCH --error=add_umap_logs/%j.err +#SBATCH --partition= #SBATCH --time=16:00:00 #SBATCH --nodes=1 #SBATCH --ntasks=1 #SBATCH --cpus-per-task=16 #SBATCH --mem=192G #SBATCH --mail-type=BEGIN,END,FAIL -#SBATCH --mail-user=ymo@stanford.edu +#SBATCH --mail-user= set -o pipefail -SCRIPT_DIR="/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/.claude/skills/perturbNMF-runner/scripts" +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF first) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/.claude/skills/perturbNMF-runner/scripts" +LOG_DIR="${ADD_UMAP_LOG_DIR:-$PWD/add_umap_logs}" MDATA_PATH="$1" if [ -z "$MDATA_PATH" ]; then @@ -57,8 +67,8 @@ fi # Self-sizing submit path. # # The guard is the ADD_UMAP_INJOB sentinel this script sets on its own sbatch -# call -- NOT $SLURM_JOB_ID. On Sherlock an OnDemand/code-server session is itself -# a Slurm allocation that exports SLURM_JOB_ID (and SLURM_CPUS_PER_TASK=2), so a +# call -- NOT $SLURM_JOB_ID. On clusters where an OnDemand/code-server session is +# itself a Slurm allocation that exports SLURM_JOB_ID (and SLURM_CPUS_PER_TASK=2), a # SLURM_JOB_ID check reads as "already in a batch job" from any interactive shell # there and silently runs the whole UMAP in that 2-CPU session instead of queuing. # --------------------------------------------------------------------------- @@ -83,14 +93,16 @@ if [ -z "$ADD_UMAP_INJOB" ]; then echo "h5mu: $MDATA_PATH" echo "size: ${SIZE_GB} GB" echo "sized: --mem=${MEM_GB}G --time=${TIME_H}:00:00 --cpus-per-task=${CPUS}" - mkdir -p /scratch/users/ymo/add_umap_logs + mkdir -p "$LOG_DIR" # Absolute self-reference so this works regardless of the caller's cwd. # ADD_UMAP_INJOB=1 is what makes the submitted copy run the workload instead of # re-submitting again; --export=ALL keeps the rest of the environment. exec sbatch --mem="${MEM_GB}G" \ --time="${TIME_H}:00:00" \ --cpus-per-task="${CPUS}" \ - --export=ALL,ADD_UMAP_INJOB=1 \ + --output="$LOG_DIR/%j.out" \ + --error="$LOG_DIR/%j.err" \ + --export=ALL,ADD_UMAP_INJOB=1,ADD_UMAP_LOG_DIR="$LOG_DIR" \ "$SCRIPT_DIR/run_add_umap.sh" "$@" fi @@ -115,8 +127,6 @@ echo "h5mu: $MDATA_PATH" echo "Size: $(du -h "$MDATA_PATH" | cut -f1)" echo "Extra args: ${EXTRA_ARGS[*]}" -mkdir -p /scratch/users/ymo/add_umap_logs - echo "Activating conda environment: NMF_Benchmarking" eval "$(conda shell.bash hook)" conda activate NMF_Benchmarking diff --git a/CHANGELOG.md b/CHANGELOG.md index aaff40a..5d6c4e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ Entries marked ⚠️ change pipeline output — re-run affected analyses. repo from their own location instead of a hardcoded checkout path. - Annotation defaults are cell-type neutral; the PubMed keyword is optional, and Vertex AI project/bucket come from env vars (`VERTEX_PROJECT_ID`, `VERTEX_BUCKET`). +- Annotation runners locate their scripts via `PIPELINE_ROOT` (fixing paths into a + nonexistent `AGeneTic/` folder); `gpe_export.py` reads the GeneProgramExplorer + location from `GPE_DIR`. ### Added - `tools/check_no_lab_specific_content.py` guard (CI + pre-commit) that blocks diff --git a/src/Stage2_Evaluation/A_Metrics/src/association_categorical.py b/src/Stage2_Evaluation/A_Metrics/src/association_categorical.py index 2dad811..ca7464d 100644 --- a/src/Stage2_Evaluation/A_Metrics/src/association_categorical.py +++ b/src/Stage2_Evaluation/A_Metrics/src/association_categorical.py @@ -365,7 +365,7 @@ def compute_categorical_association( # If test = 'pearsonr' with mode = 'one_vs_all' we are going to do something special until we can more properly integrate this if test=='pearsonr' and mode=='one_vs_all': - logging.info('Running jamboree specific version of posthoc with pearsonr, this is not yet integrated into the main pipeline') + logging.info('Running a one-vs-all pearsonr version of posthoc, this is not yet integrated into the main pipeline') res = [] # Run posthoc-tests and append results Parallel(n_jobs=n_jobs, diff --git a/src/Stage3_Interpretation/C_Annotation/0.Prepare/Extract_program_information.py b/src/Stage3_Interpretation/C_Annotation/0.Prepare/Extract_program_information.py index 7847982..228a952 100644 --- a/src/Stage3_Interpretation/C_Annotation/0.Prepare/Extract_program_information.py +++ b/src/Stage3_Interpretation/C_Annotation/0.Prepare/Extract_program_information.py @@ -284,7 +284,7 @@ def build_parser(): # context info p.add_argument("--programs", type=int, nargs="+", help="Program ids, space separated (e.g. 1 2 3). Default: all.") - p.add_argument("--cell_type", default="", help="Cell type, e.g. 'teloHAEC aortic endothelial cell'.") + p.add_argument("--cell_type", default="", help="Cell type, e.g. 'aortic endothelial cell'.") p.add_argument("--organism", default="human") # select top items of the program diff --git a/src/Stage3_Interpretation/C_Annotation/0.Prepare/run_extract_program_information.sh b/src/Stage3_Interpretation/C_Annotation/0.Prepare/run_extract_program_information.sh index 79b5eb5..364bc5a 100644 --- a/src/Stage3_Interpretation/C_Annotation/0.Prepare/run_extract_program_information.sh +++ b/src/Stage3_Interpretation/C_Annotation/0.Prepare/run_extract_program_information.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=extract_prog_info -#SBATCH -p normal +#SBATCH -p #SBATCH --time=00:30:00 #SBATCH --cpus-per-task=4 #SBATCH --mem=16GB @@ -17,11 +17,13 @@ # literature steps investigate exactly these pairs. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate NMF_Benchmarking -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/0.Prepare -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/0.Prepare" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ OUT=$RESULT/Interpretation/AGeneTic_test python $SCRIPT_DIR/Extract_program_information.py \ @@ -35,7 +37,7 @@ python $SCRIPT_DIR/Extract_program_information.py \ --gene_name_key var_names \ --categorical_key type \ --log2fc_key log2FC \ - --cell_type "teloHAEC aortic endothelial cell" \ + --cell_type "aortic endothelial cell" \ --organism human \ --top_gene 15 \ --top_unique_gene 8 \ diff --git a/src/Stage3_Interpretation/C_Annotation/1.Search/1.1.Search_literature/search_nature.py b/src/Stage3_Interpretation/C_Annotation/1.Search/1.1.Search_literature/search_nature.py index ee9f40e..2724886 100644 --- a/src/Stage3_Interpretation/C_Annotation/1.Search/1.1.Search_literature/search_nature.py +++ b/src/Stage3_Interpretation/C_Annotation/1.Search/1.1.Search_literature/search_nature.py @@ -13,11 +13,11 @@ # Nature portfolio search (Crossref, DOI prefix 10.1038) + PDF download from nature.com. # -# PDFs come from nature.com under Stanford's institutional subscription, which Nature grants -# by IP address: it works from Sherlock / campus / Stanford VPN (Sherlock egresses from a -# Stanford 171.67.x address) with no SUNet login. Off the Stanford network, only open-access -# articles download. Stanford's license covers research use, not bulk harvesting, so results -# are capped and PDF requests are spaced out. +# PDFs come from nature.com under an institutional subscription, which Nature grants by IP +# address: it works from a network covered by your institution's license (campus, VPN, or a +# cluster that egresses from an institutional address) with no login. Elsewhere, only +# open-access articles download. Institutional licenses cover research use, not bulk +# harvesting, so results are capped and PDF requests are spaced out. CROSSREF_BASE_URL = "https://api.crossref.org" NATURE_BASE_URL = "https://www.nature.com" NATURE_DOI_PREFIX = "10.1038" @@ -122,7 +122,7 @@ def search_Nature( bibliographic (title, authors, journal, year) via Crossref, restricted to DOI prefix 10.1038 2. Adding a journal name or year to the query biases toward it (e.g. "... Nature Genetics 2023") 3. Only journal articles are returned (no news, editorials split out by Crossref type) - 4. PDF download uses Stanford's subscription by IP: run on Sherlock / campus / Stanford VPN + 4. PDF download uses an institutional subscription by IP: run on a licensed network Env (optional): CROSSREF_MAILTO / PUBMED_EMAIL (Crossref polite pool). No API key needed. @@ -168,7 +168,7 @@ def search_Nature( pdf_path, msg = download_paper(out_dir, f"nature_{rec['article_id']}", doi=rec["doi"], pdf_urls=[rec["pdf_url"]]) if not pdf_path: - msg += "; is this running on the Stanford network (Sherlock / campus / VPN)?" + msg += "; is this running on a network with institutional access (campus / VPN)?" downloads.append((rec["title"], pdf_path, msg)) pdf_line = '\n' + 'PDF: ' + (pdf_path or f'Not downloaded ({msg})') @@ -177,7 +177,7 @@ def search_Nature( if out_dir: report_downloads("Nature", query, downloads, out_dir) - return "\n".join(data_list) + '\n' + "Data source: Nature (nature.com, Stanford subscription)" + return "\n".join(data_list) + '\n' + "Data source: Nature (nature.com, institutional subscription)" if __name__ == "__main__": diff --git a/src/Stage3_Interpretation/C_Annotation/1.Search/run_MyGene.sh b/src/Stage3_Interpretation/C_Annotation/1.Search/run_MyGene.sh index c5f2f42..8dc7001 100644 --- a/src/Stage3_Interpretation/C_Annotation/1.Search/run_MyGene.sh +++ b/src/Stage3_Interpretation/C_Annotation/1.Search/run_MyGene.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=mygene_extend -#SBATCH -p normal +#SBATCH -p #SBATCH --time=00:15:00 #SBATCH --cpus-per-task=1 #SBATCH --mem=4GB @@ -15,11 +15,13 @@ # --programs (required) takes space-separated ids (e.g. 1 2 3); only those P.json are read. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate NMF_Benchmarking -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/1.Search/1.0.Search_database/search_gene -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/1.Search/1.0.Search_database/search_gene" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ BUNDLE_DIR=$RESULT/Interpretation/AGeneTic_test/PerturbNMF_Info OVERWRITE="" # default off; set to --overwrite to re-query genes that already have MyGene info OUT_DIR=$RESULT/Interpretation/AGeneTic_test/Gene_info_extended_PerturbNMF_Info diff --git a/src/Stage3_Interpretation/C_Annotation/1.Search/run_NCBI.sh b/src/Stage3_Interpretation/C_Annotation/1.Search/run_NCBI.sh index 12ab892..64338d2 100644 --- a/src/Stage3_Interpretation/C_Annotation/1.Search/run_NCBI.sh +++ b/src/Stage3_Interpretation/C_Annotation/1.Search/run_NCBI.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=ncbi_extend -#SBATCH -p normal +#SBATCH -p #SBATCH --time=00:15:00 #SBATCH --cpus-per-task=1 #SBATCH --mem=4GB @@ -16,11 +16,13 @@ # Optional: export NCBI_API_KEY (10 req/s instead of 3) and NCBI_EMAIL. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate NMF_Benchmarking -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/1.Search/1.0.Search_database/search_gene -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/1.Search/1.0.Search_database/search_gene" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ BUNDLE_DIR=$RESULT/Interpretation/AGeneTic_test/PerturbNMF_Info OVERWRITE="" # default off; set to --overwrite to re-query genes that already have NCBI info OUT_DIR=$RESULT/Interpretation/AGeneTic_test/Gene_info_extended_PerturbNMF_Info diff --git a/src/Stage3_Interpretation/C_Annotation/1.Search/run_OmniPath.sh b/src/Stage3_Interpretation/C_Annotation/1.Search/run_OmniPath.sh index 1804972..58a10f0 100644 --- a/src/Stage3_Interpretation/C_Annotation/1.Search/run_OmniPath.sh +++ b/src/Stage3_Interpretation/C_Annotation/1.Search/run_OmniPath.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=omnipath_interact -#SBATCH -p normal +#SBATCH -p #SBATCH --time=00:30:00 #SBATCH --cpus-per-task=1 #SBATCH --mem=4GB @@ -18,11 +18,13 @@ # --programs (required) takes space-separated ids (e.g. 1 2 3); only those P.json are read. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate NMF_Benchmarking -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/1.Search/1.0.Search_database/search_gene_interaction -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/1.Search/1.0.Search_database/search_gene_interaction" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ BUNDLE_DIR=$RESULT/Interpretation/AGeneTic_test/PerturbNMF_Info python $SCRIPT_DIR/search_OmniPath.py \ diff --git a/src/Stage3_Interpretation/C_Annotation/1.Search/run_UniProt.sh b/src/Stage3_Interpretation/C_Annotation/1.Search/run_UniProt.sh index 4e88280..eb7641a 100644 --- a/src/Stage3_Interpretation/C_Annotation/1.Search/run_UniProt.sh +++ b/src/Stage3_Interpretation/C_Annotation/1.Search/run_UniProt.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=uniprot_extend -#SBATCH -p normal +#SBATCH -p #SBATCH --time=00:15:00 #SBATCH --cpus-per-task=1 #SBATCH --mem=4GB @@ -15,11 +15,13 @@ # --programs (required) takes space-separated ids (e.g. 1 2 3); only those P.json are read. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate NMF_Benchmarking -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/1.Search/1.0.Search_database/search_gene -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/1.Search/1.0.Search_database/search_gene" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ BUNDLE_DIR=$RESULT/Interpretation/AGeneTic_test/PerturbNMF_Info OVERWRITE="" # default off; set to --overwrite to re-query genes that already have UniProt info OUT_DIR=$RESULT/Interpretation/AGeneTic_test/Gene_info_extended_PerturbNMF_Info diff --git a/src/Stage3_Interpretation/C_Annotation/1.Search/run_literature_search.sh b/src/Stage3_Interpretation/C_Annotation/1.Search/run_literature_search.sh index 0ad979c..096b155 100755 --- a/src/Stage3_Interpretation/C_Annotation/1.Search/run_literature_search.sh +++ b/src/Stage3_Interpretation/C_Annotation/1.Search/run_literature_search.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=lit_search -#SBATCH -p normal +#SBATCH -p #SBATCH --time=04:00:00 #SBATCH --cpus-per-task=1 #SBATCH --mem=8GB @@ -14,11 +14,13 @@ # Tasks with a search_log.json are skipped; add --overwrite to re-search. # Uses the Claude Agent SDK (ANTHROPIC_API_KEY or a Claude Code login); spend is capped per task. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate geneqa -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/1.Search -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/1.Search" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ INFO_DIR=$RESULT/Interpretation/AGeneTic_test/Gene_info_extended_PerturbNMF_Info LIT_DIR=$RESULT/Interpretation/AGeneTic_test/Literature_info_extended_PerturbNMF_Info OVERWRITE="" # set to --overwrite to re-search diff --git a/src/Stage3_Interpretation/C_Annotation/1.Search/run_llm_query_agent.sh b/src/Stage3_Interpretation/C_Annotation/1.Search/run_llm_query_agent.sh index 60b70ab..99e9fc2 100755 --- a/src/Stage3_Interpretation/C_Annotation/1.Search/run_llm_query_agent.sh +++ b/src/Stage3_Interpretation/C_Annotation/1.Search/run_llm_query_agent.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=lit_query_agent -#SBATCH -p normal +#SBATCH -p #SBATCH --time=00:30:00 #SBATCH --cpus-per-task=1 #SBATCH --mem=4GB @@ -14,11 +14,13 @@ # agent sees which genes lack cell-type evidence (optional). Programs that already have the plan are # skipped; add --overwrite to re-plan. Needs ANTHROPIC_API_KEY in AGeneTic/.env. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate geneqa -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/1.Search -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/1.Search" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ INFO_DIR=$RESULT/Interpretation/AGeneTic_test/Gene_info_extended_PerturbNMF_Info OUT_DIR=$RESULT/Interpretation/AGeneTic_test/Literature_info_extended_PerturbNMF_Info QUESTION=() # e.g. QUESTION=(--question "why does VIRMA knockdown reduce translation genes" --plan_name virma) diff --git a/src/Stage3_Interpretation/C_Annotation/1.Search/run_search_generif.sh b/src/Stage3_Interpretation/C_Annotation/1.Search/run_search_generif.sh index 2aef8a0..c0706bd 100755 --- a/src/Stage3_Interpretation/C_Annotation/1.Search/run_search_generif.sh +++ b/src/Stage3_Interpretation/C_Annotation/1.Search/run_search_generif.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=generif_lit -#SBATCH -p normal +#SBATCH -p #SBATCH --time=02:00:00 #SBATCH --cpus-per-task=1 #SBATCH --mem=8GB @@ -15,11 +15,13 @@ # Per-gene results are cached in /gene_cache/; add --overwrite to re-summarize. # Needs ANTHROPIC_API_KEY (and optionally NCBI_API_KEY / NCBI_EMAIL / UNPAYWALL_EMAIL) in AGeneTic/.env. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate geneqa -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/1.Search/1.1.Search_literature -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/1.Search/1.1.Search_literature" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ INFO_DIR=$RESULT/Interpretation/AGeneTic_test/Gene_info_extended_PerturbNMF_Info OUT_DIR=$RESULT/Interpretation/AGeneTic_test/Literature_info_extended_PerturbNMF_Info DOWNLOAD_PDFS="--download_pdfs" # set to "" to skip PDF downloads diff --git a/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation/gpe_export.py b/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation/gpe_export.py index e7b859b..ee62957 100644 --- a/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation/gpe_export.py +++ b/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation/gpe_export.py @@ -1,26 +1,33 @@ """Attach cached paper-qa evidence onto an existing GPE contract, and validate it. -Imports GPE's own contract classes live from Tools/GeneProgramExplorer (``pipeline/src``) so the -emitted evidence matches the schema exactly. Attach-to-EXISTING-edges only: for each gene-gene -edge whose pair has a cached result, we append a ``literature`` entry to the edge's ``evidence[]``. -We never create new edges here. +Imports GPE's own contract classes live from a GeneProgramExplorer checkout (``pipeline/src``) so +the emitted evidence matches the schema exactly. Set the ``GPE_DIR`` environment variable to that +checkout, or pass ``src_dir`` / ``schema_path`` explicitly. Attach-to-EXISTING-edges only: for each +gene-gene edge whose pair has a cached result, we append a ``literature`` entry to the edge's +``evidence[]``. We never create new edges here. """ from __future__ import annotations import json +import os import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from evidence_cache import read_result # noqa: E402 -GPE_DIR = "/oak/stanford/groups/engreitz/Users/ymo/Tools/GeneProgramExplorer" +GPE_DIR = os.environ.get("GPE_DIR", "/path/to/GeneProgramExplorer") DEFAULT_GPE_SRC = f"{GPE_DIR}/pipeline/src" DEFAULT_SCHEMA = f"{GPE_DIR}/contract/schema.json" def _load_gpe(src_dir: str = DEFAULT_GPE_SRC): """Put the bundled GPE pipeline on sys.path and return the classes we need.""" + if not Path(src_dir).is_dir(): + raise FileNotFoundError( + f"GeneProgramExplorer source not found at {src_dir}. Set the GPE_DIR " + "environment variable to your GeneProgramExplorer checkout." + ) src_dir = str(Path(src_dir).resolve()) if src_dir not in sys.path: sys.path.insert(0, src_dir) diff --git a/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation/run_evidence_curation.sh b/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation/run_evidence_curation.sh index d02b32e..43ebf33 100644 --- a/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation/run_evidence_curation.sh +++ b/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation/run_evidence_curation.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=evidence_curation -#SBATCH -p normal +#SBATCH -p #SBATCH --time=02:00:00 #SBATCH --cpus-per-task=2 #SBATCH --mem=16GB @@ -18,11 +18,13 @@ # Leave out --cell_type to use each bundle's cell_type. set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate geneqa -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/2.Evidence_curation -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/2.Evidence_curation" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ INFO_DIR=$RESULT/Interpretation/AGeneTic_test/Gene_info_extended_PerturbNMF_Info LIT_DIR=$RESULT/Interpretation/AGeneTic_test/Literature_info_extended_PerturbNMF_Info diff --git a/src/Stage3_Interpretation/C_Annotation/3.Generate/run_html_report.sh b/src/Stage3_Interpretation/C_Annotation/3.Generate/run_html_report.sh index 68f00c3..8ad0842 100644 --- a/src/Stage3_Interpretation/C_Annotation/3.Generate/run_html_report.sh +++ b/src/Stage3_Interpretation/C_Annotation/3.Generate/run_html_report.sh @@ -1,6 +1,6 @@ #!/bin/bash #SBATCH --job-name=agenetic_report -#SBATCH -p normal +#SBATCH -p #SBATCH --time=00:10:00 #SBATCH --cpus-per-task=1 #SBATCH --mem=2GB @@ -14,16 +14,18 @@ # Add --cdn to load Cytoscape.js from the web instead of inlining it (file ~0.4 MB smaller). set -euo pipefail -source /oak/stanford/groups/engreitz/Users/ymo/miniforge3/etc/profile.d/conda.sh +eval "$(conda shell.bash hook)" conda activate NMF_Benchmarking -SCRIPT_DIR=/oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF/src/Stage3_Interpretation/C_Annotation/AGeneTic/3.Generate -RESULT=/oak/stanford/groups/engreitz/Users/ymo/Project/Helen_telohaec/Result/073126_2kG_torch_e8_64 +# Path to your PerturbNMF checkout (export PIPELINE_ROOT=/path/to/PerturbNMF before sbatch) +: "${PIPELINE_ROOT:?set PIPELINE_ROOT to the PerturbNMF repo root}" +SCRIPT_DIR="$PIPELINE_ROOT/src/Stage3_Interpretation/C_Annotation/3.Generate" +RESULT="/path/to/output_dir/" # edit: run directory with Interpretation/ INFO_DIR=$RESULT/Interpretation/AGeneTic_test/Gene_info_extended_PerturbNMF_Info python $SCRIPT_DIR/html_report.py \ --info_dir $INFO_DIR \ --out $RESULT/Interpretation/AGeneTic_test/Report/AGeneTic_report.html \ - --dataset_name "Helen teloHAEC 2kG" \ + --dataset_name "" \ --top_regulator 5 \ --top_gene 15 diff --git a/tests/Script/Stage3_Interpretation/A_Plotting/gene_html/test_html_gene_no_perturb.py b/tests/Script/Stage3_Interpretation/A_Plotting/gene_html/test_html_gene_no_perturb.py index 8c52aed..15426a6 100644 --- a/tests/Script/Stage3_Interpretation/A_Plotting/gene_html/test_html_gene_no_perturb.py +++ b/tests/Script/Stage3_Interpretation/A_Plotting/gene_html/test_html_gene_no_perturb.py @@ -8,7 +8,7 @@ Usage: eval "$(conda shell.bash hook)" && conda activate NMF_Benchmarking - cd /oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF + cd /path/to/PerturbNMF python -m pytest tests/Script/Stage3_Interpretation/A_Plotting/gene_html/test_html_gene_no_perturb.py -v """ diff --git a/tests/Script/Stage3_Interpretation/A_Plotting/program_html/test_html_program_no_perturb.py b/tests/Script/Stage3_Interpretation/A_Plotting/program_html/test_html_program_no_perturb.py index c6e801a..272a3c8 100644 --- a/tests/Script/Stage3_Interpretation/A_Plotting/program_html/test_html_program_no_perturb.py +++ b/tests/Script/Stage3_Interpretation/A_Plotting/program_html/test_html_program_no_perturb.py @@ -8,7 +8,7 @@ Usage: eval "$(conda shell.bash hook)" && conda activate NMF_Benchmarking - cd /oak/stanford/groups/engreitz/Users/ymo/Tools/PerturbNMF + cd /path/to/PerturbNMF python -m pytest tests/Script/Stage3_Interpretation/A_Plotting/program_html/test_html_program_no_perturb.py -v """