From 4420118c8086b809c4bd0a3d406f60450085d298 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 16 Oct 2026 22:36:56 +0200 Subject: [PATCH 01/32] fix: compare guide examples with their plain text output blocks The MyST pages mark expected output with a bare text fence; the runner only recognised the old title="Output" fence and so compared nothing. Also trigger the examples workflow on RAVEN submodule changes. --- .github/workflows/examples.yml | 2 ++ docs/README-sphinx.md | 8 +++----- docs/guide/media.md | 3 ++- scripts/run_examples.py | 13 +++++++++---- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 073acf6..f69a3e5 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -17,6 +17,7 @@ on: - "scripts/run_examples.py" - ".github/workflows/examples.yml" - "raven-toolbox" + - "RAVEN" pull_request: paths: - "docs/guide/**" @@ -24,6 +25,7 @@ on: - "scripts/run_examples.py" - ".github/workflows/examples.yml" - "raven-toolbox" + - "RAVEN" schedule: - cron: "0 5 * * 1" workflow_dispatch: diff --git a/docs/README-sphinx.md b/docs/README-sphinx.md index 0db0848..ea2e445 100644 --- a/docs/README-sphinx.md +++ b/docs/README-sphinx.md @@ -87,11 +87,9 @@ python -m venv .venv-sphinx `check_names.py`'s identifier validation is still valuable and doesn't need to be a live hook; it can run as a standalone pre-build check either way, but re-wiring it wasn't done as part of this exploration. -- **`scripts/run_examples.py`.** Parses MkDocs/pymdownx fence syntax - (` ```text title="Output" ` in particular) to find and check expected - output blocks. That fence shape changed here (see "Admonitions, content - tabs..." above); the script would need to read MyST's fence conventions - instead before it could check this branch's pages. +- **`scripts/run_examples.py`.** Treats a bare ` ```text ` fence directly + below a code block as that block's expected output, and `--update` writes + that fence shape. - **`.readthedocs.yaml`.** Still points at the `mkdocs:` build. The target block for a real switch: diff --git a/docs/guide/media.md b/docs/guide/media.md index 2733f50..62d9a40 100644 --- a/docs/guide/media.md +++ b/docs/guide/media.md @@ -221,7 +221,8 @@ with model: with model: model.medium = {"r_1714": 10.0, "r_1992": 1000.0} # only glucose and oxygen - print(f"glucose and oxygen alone: {model.slim_optimize():.4f} /h") + growth = round(model.slim_optimize(), 4) + 0.0 # + 0.0 turns solver noise -0.0 into 0.0 + print(f"glucose and oxygen alone: {growth:.4f} /h") model.medium = {**shipped, "r_1714": 10.0} # the shipped recipe, more glucose print(f"complete medium: {model.slim_optimize():.4f} /h") diff --git a/scripts/run_examples.py b/scripts/run_examples.py index 5c30bd6..9df59cd 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -14,8 +14,8 @@ workspace), in a scratch directory seeded with a copy of ``docs/data/``, so the snippets can use the short relative paths the reader sees, * captures what the block prints -- plus the repr of a trailing expression, the - way a notebook or the REPL would -- and compares it with the ``title="Output"`` - block that follows, + way a notebook or the REPL would -- and compares it with the plain ``text`` block + that follows, * reports every mismatch as a unified diff and exits non-zero. MATLAB needs a MATLAB runtime on the PATH (or ``RAVEN_DOCS_MATLAB`` pointing at @@ -93,6 +93,11 @@ MATLAB_SOLVER = "glpk" LANGUAGES = ("python", "matlab") +# The info string of an expected-output fence: a bare ```text directly below the +# code block. A titled text fence (```text title="tasks.txt") is an input shown +# to the reader, not output. +OUTPUT_INFO = "text" + SKIP_FILE = "run-examples: skip-file" SKIP_BLOCK = "run-examples: skip" NEEDS_GUROBI = "run-examples: needs-gurobi" @@ -248,7 +253,7 @@ def section_of(line_no: int) -> str: if index + 1 < len(fences): nxt = fences[index + 1] gap = lines[fence.end + 1 : nxt.start] - if all(not line.strip() for line in gap) and 'title="Output"' in nxt.info: + if all(not line.strip() for line in gap) and nxt.info.strip() == OUTPUT_INFO: output = nxt result.examples.append( Example( @@ -738,7 +743,7 @@ def update_page(result: PageResult) -> bool: actual = normalise(example.actual) indent = example.code_fence.indent if actual: - block = [f'{indent}```text title="Output"'] + block = [f"{indent}```{OUTPUT_INFO}"] block += [f"{indent}{line}".rstrip() for line in actual.split("\n")] block += [f"{indent}```"] else: From 79ce7e78f677b45315dc38e399b4221595f0afe8 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:37:31 +0200 Subject: [PATCH 02/32] docs: describe what the guide examples workflow runs and checks --- docs/guide/index.md | 23 ++++++++++++++--------- docs/index.md | 4 ++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/docs/guide/index.md b/docs/guide/index.md index c81e296..2ecb650 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -25,15 +25,20 @@ the cobrapy documentation. The :::{admonition} Executed examples :class: info -Every snippet on the nineteen numbered pages is run on each commit and -checked against the output shown beneath it, in both languages. Two pages -cannot be: [14. Context-specific models](init.md), whose steps take hours on -Human-GEM, and [13. Reconstruction from KEGG](kegg.md), which downloads -tens of megabytes and takes minutes. Both say so at the top and quote the -wall-clock of the run their numbers came from. - -That guarantee covers the numbered pages only. The protocol and the legacy -tutorials at the end of this section are MATLAB-only and are not executed. +The code on the numbered pages is run, MATLAB and Python separately, by the +[Guide examples](https://github.com/CompMetEng/raven-docs/actions/workflows/examples.yml) +workflow whenever a guide page, the example data or the runner changes, and once +a week against the latest RAVEN and raven-toolbox. A run fails when a block errors +or prints something other than the output shown beneath it. The blocks on a page +run in order in one session, so later blocks use what earlier ones made: start +from the page's *Setup*. + +Not executed: [13. Reconstruction from KEGG](kegg.md), which downloads tens of +megabytes and takes minutes, and the Human-GEM steps of +[14. Context-specific models](init.md), which take hours. A few other blocks are +marked on their page as not run, with the reason. Both pages quote the run their +numbers came from. The protocol and the legacy tutorials at the end of this +section are MATLAB-only and are not executed. ::: ## Example data diff --git a/docs/index.md b/docs/index.md index 2ea1d79..4703e8a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -127,8 +127,8 @@ fprintf('Growth rate: %.4f h-1\n', sol.f); :link: guide/index :link-type: doc -Nineteen task-focused pages, MATLAB and Python side by side, every example -executed and checked on each commit. +Twenty-two task-focused pages, MATLAB and Python side by side, with the +examples executed and their output checked. ::: :::{grid-item-card} API reference From e25fd60950c011f1b033582039c637132231db19 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:37:31 +0200 Subject: [PATCH 03/32] docs(kegg): state the toolbox version behind the quoted numbers --- docs/guide/kegg.md | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/docs/guide/kegg.md b/docs/guide/kegg.md index ba4ccf1..404f84f 100644 --- a/docs/guide/kegg.md +++ b/docs/guide/kegg.md @@ -8,11 +8,10 @@ reactions, so annotating a genome with KOs gives you a draft directly. Which route you take depends on one thing: whether your organism is already in KEGG. :::{note} These examples are not run by the documentation build -Every other page in this guide is executed on each commit and its output -checked. This one is not. A KEGG reconstruction downloads tens to hundreds of -megabytes and takes minutes even once that is cached, which is too much for a -per-commit check. The numbers below come from real runs and are quoted with -their timings, measured on an ordinary laptop. +A KEGG reconstruction downloads tens of megabytes and takes minutes, so this +page is not executed with the others. The Python numbers below come from +raven-toolbox 3.0.0b1 with the `kegg118` artefacts (about 2.5 minutes per call +on a recent workstation, with or without the cache). ::: ### Functions on this page @@ -72,9 +71,9 @@ print(f"{len(model.reactions)} rxns, {len(model.metabolites)} mets, " The artefacts (a reference model and three tables, about 47 MB) are fetched from the `kegg118` raven-data release on first use and cached; see [Download data and binaries](../installation/data-and-binaries.md) for -where, and how to fetch them ahead of a batch run. Expect around five -minutes for the first run and much the same afterwards: the download is not -the slow part, assembling the draft from the tables is. +where, and how to fetch them ahead of a batch run. Expect a few minutes for +the first run and much the same afterwards: the download is not the slow part, +assembling the draft from the tables is. ::: :::: @@ -152,8 +151,15 @@ The flags decide how inclusive the draft is. `keepIncomplete` / `keepUndefinedStoich` / `keep_undefined_stoich` keeps reactions with `n` or `x` in their stoichiometry, which cannot be balanced as written. Turning both off: -```text title="keep_incomplete=False, keep_undefined_stoich=False" -1348 rxns, 827 genes (from 1357 and 838) +```python +strict = get_kegg_model_for_organism_from_artefacts( + "sce", keep_incomplete=False, keep_undefined_stoich=False) +print(f"{len(strict.reactions)} rxns, {len(strict.genes)} genes " + f"(from {len(model.reactions)} and {len(model.genes)})") +``` + +```text +1348 rxns, 827 genes (from 1357 and 838) ``` That is a change of nine reactions on this organism. The size of the effect From 92f58e68d9af66a81d56d692f37b2539c2c4d8ee Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:49 +0200 Subject: [PATCH 04/32] docs(tasks): run tasks against a closed boundary and fix the growth task --- docs/data/tasks.txt | 4 +- docs/guide/tasks.md | 126 ++++++++++++++++++++-------- docs/parameter-tuning/benchmarks.md | 3 +- docs/tuned-parameters.md | 3 +- 4 files changed, 94 insertions(+), 42 deletions(-) diff --git a/docs/data/tasks.txt b/docs/data/tasks.txt index a82c61e..1cc443a 100644 --- a/docs/data/tasks.txt +++ b/docs/data/tasks.txt @@ -1,3 +1,5 @@ ID DESCRIPTION SHOULD FAIL IN IN LB IN UB OUT OUT LB OUT UB EQU EQU LB EQU UB CHANGED RXN CHANGED LB CHANGED UB PRINT FLUX COMMENTS - GROWTH Growth on glucose alpha-D-glucose[c];O2[c] 0 1000 biomass[c];CO2[c] 0 1000 + GROWTH Growth on glucose alpha-D-glucose[c];O2[c] 0 1000 biomass[c] 0.01 1000 + CO2[c] 0 1000 + acetate[c] 0 1000 LEAK Biomass from nothing 1 biomass[c] 0.01 1000 diff --git a/docs/guide/tasks.md b/docs/guide/tasks.md index d31b5fb..7a65e77 100644 --- a/docs/guide/tasks.md +++ b/docs/guide/tasks.md @@ -23,7 +23,9 @@ inputs, the outputs, any equations that must carry flux, and the bounds on each. ```text title="tasks.txt, abridged; the real file keeps every column" ID DESCRIPTION SHOULD FAIL IN IN LB IN UB OUT OUT LB OUT UB … - GROWTH Growth on glucose alpha-D-glucose[c];O2[c] 0 1000 biomass[c];CO2[c] 0 1000 … + GROWTH Growth on glucose alpha-D-glucose[c];O2[c] 0 1000 biomass[c] 0.01 1000 … + CO2[c] 0 1000 … + acetate[c] 0 1000 … LEAK Biomass from nothing 1 biomass[c] 0.01 1000 … ``` @@ -43,16 +45,22 @@ set; the columns after `OUT UB` are blank. Metabolites are named `name[compartment]`, matching the metabolite **names**, not the identifiers. Several go in one cell, separated by `;`, and the bounds columns -apply to all of them. +apply to all of them. To give a metabolite bounds of its own, put it on a +continuation row: a row with no `ID` adds its inputs and outputs to the task +above it. `GROWTH` uses that to ask for at least 0.01 units of biomass while +leaving CO₂ and acetate free; with a lower bound of 0 on biomass, the all-zero +flux would pass the task. The 0.01 is illustrative; any positive floor makes +the point. `SHOULD FAIL` marks a task the model must *fail*: the second asks for biomass with nothing supplied, which a correct model cannot do, so failing it is a pass. -Note what `GROWTH` has to list. Glucose and oxygen in, biomass out, and **CO₂**, -because a task closes the model's own exchanges and growth has to put its carbon -somewhere. Leave CO₂ out and the task is infeasible, which looks like a broken -model and is really a broken task. +Note what `GROWTH` has to list once the model's own exchanges are closed (15.2): +glucose and oxygen in, biomass out, and **CO₂** and **acetate**, because growth +has to put its carbon somewhere: CO₂ and, in this small model, some acetate. +Leave CO₂ out and the task is infeasible, which looks like a broken model and is +really a broken task. ## 15.1 Read and run the tasks @@ -65,7 +73,8 @@ model = readYAMLmodel('smallYeast.yml'); tasks = parseTaskList('tasks.txt'); fprintf('%d tasks\n', numel(tasks)); -report = checkTasks(model, [], 'printOutput', false, 'taskStructure', tasks); +closed = closeModel(model); % the task is the whole boundary, see 15.2 +report = checkTasks(closed, [], 'printOutput', false, 'taskStructure', tasks); for i = 1:numel(report.id) verdicts = {'FAIL', 'pass'}; fprintf(' %-8s %s\n', report.id{i}, verdicts{report.ok(i) + 1}); @@ -74,7 +83,6 @@ end ```text 2 tasks -[Warning: Exchange metabolites should normally not be removed from the model when using checkTasks. Inputs and outputs are defined in the task file instead. Use importModel(file,false) to import a model with exchange metabolites remaining] GROWTH pass LEAK pass ``` @@ -103,7 +111,7 @@ model = read_yaml_model("smallYeast.yml") tasks = parse_task_list("tasks.txt") print(len(tasks), "tasks") -for result in check_tasks(model, tasks): +for result in check_tasks(model, tasks, close_boundaries=True): # see 15.2 verdict = "pass" if result.passed else "FAIL" print(f" {result.id:<8} {verdict} (feasible: {result.feasible})") ``` @@ -114,48 +122,90 @@ for result in check_tasks(model, tasks): LEAK pass (feasible: False) ``` -`check_tasks` closes the model's own exchange reactions first, so the inputs -and outputs are exactly what the task says they are; RAVEN does the same. -That is why a task list is portable between models in a way a script full of -`setParam` calls is not. +`close_boundaries=True` closes the model's own exchange, sink and demand +reactions first, so the inputs and outputs are exactly what the task says they +are. That is why a task list is portable between models in a way a script full +of `setParam` calls is not. ::: :::: ## 15.2 A task defines its own medium -`check_tasks` closes every exchange, sink and demand the model has before -applying a task, exactly as RAVEN does. The task's inputs and outputs are then -the *only* way anything enters or leaves, which is what makes a task list -portable between models, and what makes an incomplete task look like a broken -model. +By default neither `checkTasks` nor `check_tasks` closes the model's own +exchange reactions: a task's inputs and outputs are added to whatever the model +already allows. To make the task the whole boundary, close the model first: +`closeModel` in RAVEN (what ftINIT does before collecting task-essential +reactions), `close_boundaries=True` in raven-toolbox. The task's inputs and +outputs are then the *only* way anything enters or leaves, which is what makes +a task list portable between models, and what makes an incomplete task look like +a broken model. ::::{tab-set} +:::{tab-item} Ⓜ️ MATLAB +:sync: matlab + +```matlab +noCO2 = tasks(1); +noCO2.id = 'GROWTH-NO-CO2'; +keep = ~strcmp(noCO2.outputs, 'CO2[c]'); +noCO2.outputs = noCO2.outputs(keep); +noCO2.LBout = noCO2.LBout(keep); +noCO2.UBout = noCO2.UBout(keep); +pair = [tasks(1); noCO2]; + +boundaries = {'open', 'closed'}; +models = {model, closed}; +for k = 1:2 + report = checkTasks(models{k}, [], 'printOutput', false, 'taskStructure', pair); + for i = 1:numel(report.id) + fprintf(' %-6s %-14s feasible=%d\n', boundaries{k}, report.id{i}, report.ok(i)); + end +end +``` + +```text +[Warning: Exchange metabolites should normally not be removed from the model when using checkTasks. Inputs and outputs are defined in the task file instead. Use importModel(file,false) to import a model with exchange metabolites remaining] + open GROWTH feasible=1 + open GROWTH-NO-CO2 feasible=1 + closed GROWTH feasible=1 + closed GROWTH-NO-CO2 feasible=0 +``` +::: :::{tab-item} 🐍 Python :sync: python ```python from raven_toolbox.tasks import Task -incomplete = Task( +growth = tasks[0] +no_co2 = Task( id="GROWTH-NO-CO2", - description="the same task, without somewhere to put the carbon", - inputs=[("alpha-D-glucose[c]", 0.0, 1000.0), ("O2[c]", 0.0, 1000.0)], - outputs=[("biomass[c]", 0.0, 1000.0)], + description="the same task, without somewhere to put the CO2", + inputs=growth.inputs, + outputs=[o for o in growth.outputs if o[0] != "CO2[c]"], ) -for result in check_tasks(model, [incomplete, tasks[0]]): - print(f" {result.id:<16} feasible={result.feasible}") +for close in (False, True): + for result in check_tasks(model, [growth, no_co2], close_boundaries=close): + boundary = "closed" if close else "open" + print(f" {boundary:<6} {result.id:<14} feasible={result.feasible}") ``` ```text - GROWTH-NO-CO2 feasible=True - GROWTH feasible=True + open GROWTH feasible=True + open GROWTH-NO-CO2 feasible=True + closed GROWTH feasible=True + closed GROWTH-NO-CO2 feasible=False ``` - -The only difference between the two is the CO₂ output. ::: :::: +The only difference between the two tasks is the CO₂ output. With the model's +own exchanges open, both are feasible: the carbon leaves through the model's +`co2OUT`, whatever the task says. With them closed, the task without CO₂ is +infeasible. A task list written for one reading gives different answers under +the other, so check which one it was curated against. + ## 15.3 Which reactions does a task depend on? Essentiality with respect to a task, rather than to the objective: remove each @@ -168,14 +218,13 @@ expression score. :sync: matlab ```matlab -[~, essentialRxns] = checkTasks(model, [], 'printOutput', false, ... +[~, essentialRxns] = checkTasks(closed, [], 'printOutput', false, ... 'getEssential', true, 'taskStructure', tasks(1)); fprintf('%d reactions essential for the task\n', sum(essentialRxns)); ``` ```text -[Warning: Exchange metabolites should normally not be removed from the model when using checkTasks. Inputs and outputs are defined in the task file instead. Use importModel(file,false) to import a model with exchange metabolites remaining] -0 reactions essential for the task +28 reactions essential for the task ``` Task-essential reactions come from `checkTasks` with its `getEssential` @@ -190,14 +239,14 @@ for the model's own objective and takes no task at all. ```python from raven_toolbox.tasks import find_task_essential_reactions -result = find_task_essential_reactions(model, tasks[:1]) +result = find_task_essential_reactions(model, tasks[:1], close_boundaries=True) print(len(result.reactions), "reactions essential for the task") print(sorted(result.reactions)[:5]) ``` ```text -0 reactions essential for the task -[] +28 reactions essential for the task +['ACO', 'ACS', 'ALD6', 'CIT', 'CO2TRANS'] ``` ::: :::: @@ -239,9 +288,12 @@ every task feasible again. - **Forgetting `SHOULD FAIL`.** The tasks that matter most are often the ones the model must *not* satisfy; without them, a leaking model passes everything. -- **Tasks that encode the medium.** A task defines its own inputs, so it does - not inherit the model's medium. A task passing therefore says nothing - about whether the model grows on the medium the model carries. +- **Not knowing which boundary the task ran against.** With the model closed, + a task defines its own inputs and does not inherit the model's medium, so a + task passing says nothing about whether the model grows on the medium it + carries. Left open (the default in both toolboxes), the model's own exchanges + take part, and a task can pass because of an uptake or secretion it never + lists. ::: ## See also diff --git a/docs/parameter-tuning/benchmarks.md b/docs/parameter-tuning/benchmarks.md index 4d58711..304da8e 100644 --- a/docs/parameter-tuning/benchmarks.md +++ b/docs/parameter-tuning/benchmarks.md @@ -73,5 +73,4 @@ is no longer taken on trust. | `fill_gaps_fast_lp.epsilon` | `1e-4` | The fastGapFill paper's value | | `fill_gaps_kumar_milp.weights` | `(1.0, 2.0)` | Implements Kumar 2007's preference for reversal over addition | | `fill_gaps_kumar_milp.big_m` | `1000.0` | Matches RAVEN-convention bounds | -| `check_tasks.close_boundaries` | `True` | The only correct value: with exchanges open, a task is satisfied by importing its own products | -| `find_task_essential_reactions.tol` | `1e-8` | Gurobi's primal feasibility tolerance; a looser value calls a reaction non-essential while its task still carries a trace flux | +| `check_tasks.close_boundaries` | `False` | Matches RAVEN's `checkTasks`, which published task lists were curated against; `True` closes the model's own exchanges so that the task is the whole boundary | diff --git a/docs/tuned-parameters.md b/docs/tuned-parameters.md index ce69de9..7d5cc17 100644 --- a/docs/tuned-parameters.md +++ b/docs/tuned-parameters.md @@ -200,8 +200,7 @@ RAVEN MATLAB: `checkTasks`. raven-toolbox: `check_tasks`, `find_task_essential_r | Parameter | Default | How determined | |---|---|---| -| `close_boundaries` (both functions) | `True` | Matches RAVEN's implied behaviour. | -| `tol` (in `find_task_essential_reactions`) | `1e-8` | Python-side-only parameter. | +| `close_boundaries` (both functions) | `False` | Matches RAVEN's `checkTasks`, which leaves the model's own exchange reactions open; published task lists were curated against that. `True` makes the task the whole boundary, as `closeModel` does in RAVEN. | **Full detail:** [Parameter benchmarks](parameter-tuning/benchmarks.md) From 29e4972017ab55cbf6fbb161301c1a2725d19b12 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 05/32] docs(solvers): describe MILPs on GLPK in both toolboxes and the QP case --- .github/workflows/examples.yml | 33 ++++++------------------- docs/guide/solvers.md | 45 ++++++++++++++++++++-------------- docs/installation/raven.md | 10 ++++---- scripts/run_examples.py | 13 +++++----- 4 files changed, 46 insertions(+), 55 deletions(-) diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index f69a3e5..120db7a 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -53,32 +53,12 @@ jobs: run: | python -m pip install --upgrade pip # GLPK arrives as a wheel via cobrapy's optlang/swiglpk dependency, so - # no system solver has to be installed. openpyxl is needed by the - # Excel export example, gurobipy by the ones marked needs-gurobi. + # no system solver has to be installed; it solves the LPs and MILPs. + # openpyxl is needed by the Excel export example, gurobipy by the + # quadratic MOMA example, which needs a QP solver. The size-limited + # licence that comes with the gurobipy wheel covers a model that small. python -m pip install "./raven-toolbox[excel]" gurobipy - # A Gurobi Web License Service licence, if the repository has one, lets the - # examples marked `needs-gurobi` run -- the MILP ones GLPK cannot solve. - # Without the secrets the step is a no-op and those examples stay skipped. - - name: Configure the Gurobi licence - env: - WLSACCESSID: ${{ secrets.GUROBI_WLSACCESSID }} - WLSSECRET: ${{ secrets.GUROBI_WLSSECRET }} - LICENSEID: ${{ secrets.GUROBI_LICENSEID }} - run: | - if [ -z "$WLSACCESSID" ]; then - echo "No Gurobi licence configured -- needs-gurobi examples will be skipped." - exit 0 - fi - umask 077 - { - echo "WLSACCESSID=$WLSACCESSID" - echo "WLSSECRET=$WLSSECRET" - echo "LICENSEID=$LICENSEID" - } > "$RUNNER_TEMP/gurobi.lic" - echo "GRB_LICENSE_FILE=$RUNNER_TEMP/gurobi.lic" >> "$GITHUB_ENV" - echo "RAVEN_DOCS_GUROBI=1" >> "$GITHUB_ENV" - - name: Report versions run: | python -c "import cobra, raven_toolbox, optlang; print('cobra', cobra.__version__); print('raven_toolbox', raven_toolbox.__version__); print('solvers', sorted(optlang.available_solvers))" @@ -112,8 +92,9 @@ jobs: with: python-version: "3.12" - # Gurobi's MATLAB interface, for the examples marked `needs-gurobi`. Both - # steps are no-ops when the repository has no licence secrets. + # Gurobi's MATLAB interface, for the examples marked `needs-gurobi`: MILPs + # that RAVEN also solves on its bundled GLPK, but that the page runs on + # Gurobi. Both steps are no-ops when the repository has no licence secrets. - name: Install Gurobi for MATLAB env: WLSACCESSID: ${{ secrets.GUROBI_WLSACCESSID }} diff --git a/docs/guide/solvers.md b/docs/guide/solvers.md index 2e70013..f7714a7 100644 --- a/docs/guide/solvers.md +++ b/docs/guide/solvers.md @@ -1,8 +1,12 @@ # 6. Solvers and configuration -Every simulation on this site ends in a linear program, and something has to -solve it. This page is about which solver that is, how to change it, and how to -read what it gives back. +Most simulations on this site solve a linear program (LP); gap-filling, +minimum-component media, `solveLP` with `minFlux` 3 and ftINIT solve +mixed-integer linear programs (MILPs), and the quadratic MOMA in +[16.5](deletions.md) solves a quadratic program (QP). Something has to solve each +of them, and which solver can do which depends on the solver and the toolbox. +This page is about which solver is in use, how to change it, and how to read +what it gives back. ### Functions on this page @@ -95,22 +99,26 @@ session can use different solvers. ## 6.2 Change it -Both toolboxes ship with **GLPK**, which is enough for every LP on this site. -Gurobi is required for the mixed-integer problems that gap-filling and -`getMinimalMedium` solve, and is faster on large models; it is free for -academic use. +Both toolboxes ship with **GLPK**, which solves LPs and MILPs, so it runs +every example on this site except the quadratic MOMA. RAVEN prints a warning +when it sends a MILP to GLPK, recommending Gurobi: GLPK is fine for problems the +size of these examples, but on a genome-scale model gap-filling and ftINIT can +take far longer than with **Gurobi**, which is free for academic use. A QP needs +a QP-capable solver, such as Gurobi or CPLEX; RAVEN has no functions that solve +one. ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB :sync: matlab ```matlab -setRavenSolver('glpk'); % 'gurobi', or 'cobra' to hand over to the COBRA Toolbox +setRavenSolver('glpk'); % 'gurobi', 'scip', or 'cobra' to hand over to the COBRA Toolbox ``` -Those three are the accepted values. `'glpk'` uses the binaries RAVEN ships, -so it works with no further installation. `'gurobi'` needs Gurobi on the -MATLAB path. `'cobra'` translates the problem into COBRA Toolbox form and +The accepted values are `'glpk'`, `'gurobi'`, `'scip'` and `'cobra'`. `'glpk'` +uses the binaries RAVEN ships, so it works with no further installation. +`'gurobi'` needs Gurobi on the MATLAB path, and `'scip'` the SCIP MATLAB +interface. `'cobra'` translates the problem into COBRA Toolbox form and uses whatever `changeCobraSolver` last selected, which requires the COBRA Toolbox to be initialised first; `setRavenSolver` checks that and errors rather than storing a preference that cannot work. @@ -172,11 +180,11 @@ growth 0.1222 /h `solveLP` builds a COBRA-style problem struct and hands it to `optimizeProb`, which is where the selected solver is actually called. Calling `optimizeProb` directly is the way to solve a problem the toolbox built for you, such as the -MILPs inside gap-filling, and it takes a `params` struct forwarded to the -solver. Two fields of `params` are handled by RAVEN rather than passed on: -`verbose`, which shows MILP progress, and `maxRatio`, which splits -badly-scaled columns through auxiliary metabolites before solving and leaves -the feasible region unchanged. +MILPs inside gap-filling. It takes two name-value arguments: `'params'`, a +struct forwarded to the solver, and `'verbose'`, which shows MILP progress. +One field of `params` is handled by RAVEN rather than passed on: `maxRatio`, +which splits badly-scaled columns through auxiliary metabolites before solving +and leaves the feasible region unchanged. ::: :::{tab-item} 🐍 Python :sync: python @@ -256,8 +264,9 @@ which is a statement about biology rather than about the constraints. - **A solver that is set but not installed.** `setRavenSolver('gurobi')` stores the preference whether or not Gurobi is there; the failure appears at the next solve. `checkRaven` tests it directly. -- **MILP with GLPK.** GLPK solves LPs only, in RAVEN. Anything mixed-integer - (`getMinimalMedium`, some gap-filling) needs Gurobi. +- **A large MILP on GLPK.** Both toolboxes send MILPs to GLPK, and RAVEN + warns when it does. On a genome-scale model, gap-filling and ftINIT can run + for hours on GLPK where Gurobi takes minutes; use Gurobi for those. - **Reading fluxes after a non-optimal solve.** `sol.stat` of `0` means a feasible point was found but not proven optimal, and `-2` means the parsimonious step failed after an optimal first solve. Both return numbers. diff --git a/docs/installation/raven.md b/docs/installation/raven.md index 38cd66f..176bd6e 100644 --- a/docs/installation/raven.md +++ b/docs/installation/raven.md @@ -32,18 +32,18 @@ with no network access ahead of time. | Solver | LP | MILP | Relative speed | License | |---|---|---|---|---| -| GLPK | yes | no | 1x (baseline) | open source, bundled with RAVEN | +| GLPK | yes | yes (RAVEN warns; slow on large problems) | 1x (baseline) | open source, bundled with RAVEN | | SCIP | yes | yes | 0.33x | open source; bundled on Windows, a separate install on macOS/Linux | | Gurobi | yes | yes | 4.2x | free academic license | | COBRA Toolbox | yes | depends on the COBRA solver configured | depends on the COBRA solver configured | depends on the COBRA solver configured | MILP-solving functions (`getMinimalMedium`, ftINIT, some gap-filling -algorithms) need a solver that supports MILP; GLPK does not, so those -functions need Gurobi, SCIP, or a MILP-capable solver through the COBRA -Toolbox. +algorithms) also run on the bundled GLPK, with a warning that recommends +Gurobi: on genome-scale models GLPK can be much slower than Gurobi, SCIP, or +a MILP-capable solver through the COBRA Toolbox. Set the active solver with `setRavenSolver`, which takes `'gurobi'`, -`'glpk'`, `'soplex'`, or `'cobra'`: +`'glpk'`, `'scip'`, or `'cobra'`: ```matlab setRavenSolver('gurobi'); diff --git a/scripts/run_examples.py b/scripts/run_examples.py index 9df59cd..a900f3c 100644 --- a/scripts/run_examples.py +++ b/scripts/run_examples.py @@ -40,8 +40,9 @@ immediately before a ```python block -- that one block is neither executed nor checked, but the ones after it still run. ```` - the block needs a MILP solver: it runs where Gurobi is available (``--gurobi`` - or ``RAVEN_DOCS_GUROBI=1``, which CI sets when a licence is configured) and is + the block selects Gurobi (a MILP the page runs on Gurobi rather than on the + bundled GLPK): it runs where Gurobi is available (``--gurobi`` or + ``RAVEN_DOCS_GUROBI=1``, which CI sets when a licence is configured) and is skipped everywhere else. ```` @@ -102,10 +103,10 @@ SKIP_BLOCK = "run-examples: skip" NEEDS_GUROBI = "run-examples: needs-gurobi" -# Examples that need a MILP solver run only where Gurobi is available: on a -# machine with a licence, and in CI when the workflow has one configured. They -# are skipped everywhere else rather than failing, so a reader without Gurobi -# still gets a green run. +# Examples that select Gurobi run only where Gurobi is available: on a machine +# with a licence, and in CI when the workflow has one configured. They are +# skipped everywhere else rather than failing, so a reader without Gurobi still +# gets a green run. GUROBI_ENABLED = os.environ.get("RAVEN_DOCS_GUROBI", "") not in ("", "0") _FENCE_OPEN = re.compile(r"^(?P[ \t]*)(?P`{3,})(?P.*)$") From b13addeb6114cfbc2c94a5eb9a16860dc6165cb3 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 06/32] docs(gap-filling): call both connectivity fillers MILPs and show a real repair --- docs/guide/gap-filling.md | 88 +++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/docs/guide/gap-filling.md b/docs/guide/gap-filling.md index 53ccec7..f037e64 100644 --- a/docs/guide/gap-filling.md +++ b/docs/guide/gap-filling.md @@ -117,7 +117,7 @@ connectivity. ```matlab -setRavenSolver('gurobi'); % fillGaps is a MILP +setRavenSolver('gurobi'); % fillGaps is a MILP; Gurobi is faster than GLPK [newConnected, cannotConnect, addedRxns] = fillGaps(draft, {template}, ... 'allowNetProduction', true, 'useModelConstraints', false); fprintf('%d added: %s\n', numel(addedRxns), strjoin(addedRxns, ', ')); @@ -149,10 +149,9 @@ those are not stable: it solves a MILP, several solutions are equally optimal, and which one comes back can change with the machine or the thread count. Two CI runs on identical input reported 11 and 9 newly connected. -**The two are not the same algorithm.** RAVEN's `fillGaps` solves a MILP, so -it needs Gurobi and reports `glpk is not suitable for solving MILPs` -otherwise. raven-toolbox's `connect_blocked_reactions` is an LP and runs on -the GLPK that ships with cobrapy. +Both are MILPs. `connect_blocked_reactions` runs on the GLPK that ships with +cobrapy; `fillGaps` runs on RAVEN's bundled GLPK too, with a warning that +recommends Gurobi, which is the solver this tab selects. `allow_net_production` decides how strict the test is. A reaction can be blocked because a substrate is unavailable *or* because a product has nowhere @@ -220,9 +219,10 @@ the gap between the counts closes. ## 17.4 The MILP formulation -When the LP relaxation adds too much, or you want the provably smallest set, -the mixed-integer formulation is the alternative. It needs a MILP solver, and it -is slower by a wide margin on anything genome-scale. +When the LP formulation adds too much, or you want the provably smallest set, +the mixed-integer formulation is the alternative. GLPK solves it at this size in +both toolboxes (RAVEN warns and recommends Gurobi); on anything genome-scale it +is slower by a wide margin, and Gurobi is the better choice. ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB @@ -252,12 +252,9 @@ gapFillMILP: reversed 0 draft reaction(s), added 0 universal reaction(s). :::{tab-item} 🐍 Python :sync: python - - ```python from raven_toolbox.gapfilling import fill_gaps_kumar_milp -draft.solver = "gurobi" result = fill_gaps_kumar_milp(draft, template) print(f"{len(result.added_reactions)} reactions added") ``` @@ -272,6 +269,68 @@ fill_gaps_kumar_milp: reversed 0 reaction(s), added 0 template reaction(s). ::: :::: +Zero is the right answer here: without `ADH1` the draft still grows at the +full 0.1222 /h, so it already meets the default floor (10 % of the merged +model's maximum) and the MILP has nothing to buy. In MATLAB that floor is twice +as high, 0.02443 against 0.01222, because `gapFillMILP` merges the template's +duplicates of the draft's reactions, its uptake reactions included, so the +merged model can take up twice the glucose. To see a repair, remove a reaction +growth needs, such as hexokinase: + +::::{tab-set} +:::{tab-item} Ⓜ️ MATLAB +:sync: matlab + + + +```matlab +draft2 = removeReactions(template, {'HXK'}); % hexokinase +draft2.id = 'draft2'; +sol = solveLP(draft2); +fprintf('draft growth %.4f\n', sol.f); +[addedRxns, reversedRxns] = gapFillMILP(draft2, template, 'params', params); +fprintf('added: %s; reversed: %s\n', strjoin(addedRxns, ', '), strjoin(reversedRxns, ', ')); +``` + +```text +draft growth -0.0000 +[Warning: The following reaction IDs in smallYeast are already present in the model and were renamed: Old: acOUT New: acOUT_smallYeast Old: biomassOUT New: biomassOUT_smallYeast Old: co2OUT New: co2OUT_smallYeast Old: ethOUT New: ethOUT_smallYeast Old: glyOUT New: glyOUT_smallYeast Old: glcIN New: glcIN_smallYeast Old: o2IN New: o2IN_smallYeast Old: ethIN New: ethIN_smallYeast Old: PGI New: PGI_smallYeast ...and 43 more] + +gapFillMILP: merged model has 52 mets, 105 rxns (52 draft, 52 universal). +gapFillMILP: 34 reversal candidates, 52 database candidates. +gapFillMILP: setting minGrowth = 0.02443 (10% of max 0.2443). +gapFillMILP: solving MILP (191 vars, 86 binary, 191 constraints)... +gapFillMILP: reversed 0 draft reaction(s), added 1 universal reaction(s). +added: HXK; reversed: +``` +::: +:::{tab-item} 🐍 Python +:sync: python + +```python +draft2 = template.copy() +draft2.remove_reactions([draft2.reactions.get_by_id("HXK")]) # hexokinase +print(f"draft growth {draft2.slim_optimize():.4f}") +result = fill_gaps_kumar_milp(draft2, template, verbose=False) +print(f"added: {result.added_reactions}; reversed: {result.reversed_reactions}") +print(f"filled growth {result.model.slim_optimize():.4f}") +``` + +```text +draft growth 0.0000 +added: ['HXK']; reversed: [] +filled growth 0.1222 +``` +::: +:::: + +Hexokinase comes back, and nothing is reversed. That is not guaranteed: by +default a reversal costs half as much as an addition, and every irreversible +draft reaction is a reversal candidate, boundary reactions and the ATP drain +included. Removing `PFK` instead makes the MILP reverse `FBP` rather than add +`PFK` back, and for an anaerobic draft without `ADH1` it reverses `ATPX`, which +makes ATP from nothing. Read the reversed reactions before accepting a fill. + ## 17.5 Filling towards a task, not a hole Connectivity gap-filling asks "can this reaction carry flux?". The other question @@ -326,8 +385,11 @@ already contains the draft, which is the form ftINIT passes. - **Net production hiding the problem.** `allowNetProduction` makes more reactions look connectable by ignoring where the products go. Useful early, misleading later. -- **MILP on a genome-scale model.** Expect it to be slow, and give it a time - limit; see [6. Solvers and configuration](solvers.md). +- **MILP on a genome-scale model.** Expect it to be slow, on GLPK most of + all, and give it a time limit; see [6. Solvers and configuration](solvers.md). +- **Accepting reversals unread.** `gapFillMILP` and `fill_gaps_kumar_milp` + prefer reversing a draft reaction to adding one, and a reversed exchange or + ATP drain is a leak, not a repair. - **Comparing added-reaction counts across toolboxes.** The two sides differ in what they treat as a candidate, so the counts are only comparable when the template genuinely holds reactions the draft lacks. From db6d7d043db85ba360a099d8e8ad2f927b90a8f6 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 07/32] docs(media): scope the uptake sign and describe the minimal-medium objectives --- docs/guide/media.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/docs/guide/media.md b/docs/guide/media.md index 62d9a40..8f2d3aa 100644 --- a/docs/guide/media.md +++ b/docs/guide/media.md @@ -22,8 +22,11 @@ aerobic minimal glucose medium already applied. ## 5.1 What is currently open An exchange reaction connects a boundary metabolite to nothing, so its flux is -the rate at which that metabolite enters (negative) or leaves (positive) the -system. The medium is exactly the set of exchanges with a negative lower bound. +the rate at which that metabolite enters or leaves the system. yeast-GEM writes +its exchanges as `metabolite <=>`, so entering is a negative flux and leaving a +positive one, and the medium is exactly the set of exchanges with a negative +lower bound. A model that writes uptake as `=> metabolite` has the opposite +sign; see [6. Solvers](solvers.md). ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB @@ -355,7 +358,7 @@ gap-filled reaction produces it internally. ```matlab -setRavenSolver('gurobi'); % getMinimalMedium solves a MILP +setRavenSolver('gurobi'); % getMinimalMedium solves a MILP; Gurobi is faster than GLPK sol = solveLP(model); medium = getMinimalMedium(model, 'minGrowth', 0.9 * sol.f); ``` @@ -390,12 +393,11 @@ particular growth rate. `verbose` set to `false` suppresses the table and returns the ids alone. Of the 16 open uptakes, 14 are kept: water and H+ are dropped because other -reactions supply them. `getMinimalMedium` solves a **MILP**, minimising the -number of open uptakes, which the GLPK that ships with RAVEN cannot do: with -GLPK selected it reports `glpk is not suitable for solving MILPs`. cobrapy's -`minimal_medium` defaults to an LP relaxation that minimises total uptake -flux instead, which is why the Python tab runs on any solver and returns -rates rather than a list. +reactions supply them. `getMinimalMedium` solves a **MILP** that minimises the +number of open uptakes. RAVEN's bundled GLPK solves it too, with a warning that +recommends Gurobi. cobrapy's `minimal_medium` by default solves an LP that +minimises the summed uptake flux (in mmol/gDW/h, all metabolites weighted +equally), which is why it returns rates. ::: :::{tab-item} 🐍 Python :sync: python @@ -426,8 +428,11 @@ r_4600 0.000 The values are uptake rates, not bounds, and the zeros are components the solution needs at a rate below the printed precision rather than not at all. -Passing `minimize_components=True` switches to the MILP formulation and -returns the same kind of answer `getMinimalMedium` gives. +`minimize_components=True` solves the component-count MILP instead (it runs on +GLPK too). It returns the import fluxes of one feasible solution, whose index +is the medium; the rates are not minimal. Treat values near the solver +tolerance as zero: with Gurobi, a heme a uptake of 1e-6 appears in this model's +answer. An integer value returns alternative media as columns. ::: :::: @@ -435,8 +440,9 @@ returns the same kind of answer `getMinimalMedium` gives. - **`infeasible` right after setting a medium.** Something essential is closed. Reopen the medium one component at a time, or start from `minimal_medium` on the working model and compare. -- **The sign convention bites.** Uptake is a *negative* flux, but - `model.medium` takes *positive* numbers. Both are right; mixing them is not. +- **The sign convention bites.** For an exchange written `metabolite <=>`, as + in yeast-GEM, uptake is a *negative* flux, but `model.medium` takes + *positive* numbers. Both are right; mixing them is not. - **The model grows without a carbon source.** Usually a leak: some reaction produces carbon from nothing. [9. Quality control](quality-control.md) is where that gets diagnosed, with `canExchange` and `analyse_topology`. From ba5307bb5746fab1640dd6b814be525ee25157a5 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 08/32] docs(deletions): use a viable knockout and a wild-type reference for MOMA --- docs/guide/deletions.md | 73 +++++++++++++++++++++++++++++++---------- 1 file changed, 56 insertions(+), 17 deletions(-) diff --git a/docs/guide/deletions.md b/docs/guide/deletions.md index b95c7d7..bcbb3f2 100644 --- a/docs/guide/deletions.md +++ b/docs/guide/deletions.md @@ -2,8 +2,8 @@ Which genes can the organism lose and still grow? Knock each one out, re-solve, and compare. The same machinery answers "which reactions are essential", "what -happens if I delete two things at once", and "how do I predict a knockout's flux -distribution better than plain FBA does". +happens if I delete two things at once", and "what flux distribution does a +knockout have right after the gene is lost". ### Functions on this page @@ -121,9 +121,11 @@ fprintf('%d genes tested, %d essential\n', numel(genes), sum(growth < 1e-6)); `fluxes` comes back sparse and with one column per deletion, so the growth row has to be pulled out by the objective's position, as above. `findGeneDeletions` also returns a `details` vector saying what happened to -each gene: whether it was deleted, proved lethal, or was skipped because it -only appears on dead-end reactions and deleting it could not change the -answer. +each gene: 1 when the deletion was solved, 2 when the problem had no optimal +solution (infeasible or failed, with fluxes reported as zero), and 4 when the +gene was skipped because it only appears on dead-end reactions and deleting it +could not change the answer. A gene whose deletion solves to zero growth gets 1, +not 2, so read lethality off the growth row, as above. ::: :::{tab-item} 🐍 Python :sync: python @@ -142,10 +144,15 @@ print(sorted(next(iter(ids)) for ids in essential.ids)[:5]) ['YBR196C', 'YCR012W', 'YDL066W', 'YDR050C', 'YKL060C'] ``` -`single_gene_deletion` returns a DataFrame indexed by the deleted gene set, -with the resulting growth rate and solver status, so ordinary pandas -filtering finds the essential ones. The index holds a frozenset per row, -because the same function shape serves the double-deletion case. +`single_gene_deletion` returns a DataFrame with one row per deletion: `ids` +(the set of deleted gene ids, a set so that the double-deletion case has the +same shape), `growth` and `status`. Look a gene up with +`results.knockout["YBR196C"]`. A knockout that makes the problem infeasible +gets `growth` NaN and `status` `'infeasible'`, and `growth < 1e-6` does **not** +select it. Use `results[(results.growth < 1e-6) | (results.status != "optimal")]` +to count those as lethal, which is what the MATLAB filter above does, since +`findGeneDeletions` reports zero fluxes for them. Here every solve is optimal, +so both filters find the same 9. ::: :::: @@ -243,9 +250,10 @@ lowest growth: 0.0000 /h ## 16.5 A knockout's fluxes, not just its growth rate FBA assumes the knockout re-optimises perfectly, which a cell that just lost a -gene does not do. MOMA instead looks for the flux distribution closest to the -wild type that the mutant can actually achieve, usually a better predictor of a -knockout's physiology, and a different answer. +gene does not do. MOMA (minimisation of metabolic adjustment) instead looks for +the flux distribution closest to a wild-type reference that the mutant can +actually achieve. It models the state right after the knockout, before the cell +adapts, and gives a different answer from FBA. ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB @@ -257,6 +265,18 @@ as the last function that depended on a paid MATLAB toolbox. Nothing in RAVEN replaces it, so a MOMA prediction has to come from the Python side, or from the COBRA Toolbox, which keeps its own. +The FBA half of the comparison runs in RAVEN as usual: + +```matlab +modelKO = removeGenes(model, {'YER178W'}, 'removeBlockedRxns', true); % PDA1 +solKO = solveLP(modelKO); +fprintf('FBA growth: %.4f /h\n', solKO.f); +``` + +```text +FBA growth: 0.1053 /h +``` + What RAVEN does offer for the same *question* (what changed in the mutant, rather than by how much growth fell) is `compareFluxes` on two flux vectors from the same model, in [21. Comparing models](comparing.md). @@ -265,17 +285,36 @@ vectors from the same model, in [21. Comparing models](comparing.md). :sync: python ```python -from cobra.flux_analysis import moma +from cobra.flux_analysis import moma, pfba +wild_type = pfba(model) # the reference: the unmodified model with model: - knock_out_model_genes(model, ["YBR196C"]) - solution = moma(model, solution=None, linear=True) - print(f"MOMA growth: {solution.fluxes['biomassOUT']:.4f} /h") + knock_out_model_genes(model, ["YER178W"]) # PDA1, pyruvate dehydrogenase + fba_growth = model.slim_optimize() + solution = moma(model, solution=wild_type, linear=False) +print(f"FBA growth: {fba_growth:.4f} /h") +print(f"MOMA growth: {solution.fluxes['biomassOUT']:.4f} /h") ``` ```text -MOMA growth: 0.0000 /h +FBA growth: 0.1053 /h +MOMA growth: 0.0869 /h ``` + +The reference has to be computed before the knockout. Without `solution=`, +cobrapy takes pFBA of the model it is given, which inside the `with` block is +already the mutant, and MOMA returns the FBA answer. `PDA1` is a good test +because it is not lethal: the PDH bypass (`PDC`, `ALD6`, `ACS`) keeps the +mutant growing, so FBA and MOMA can differ. + +`linear=False` is the quadratic MOMA of Segrè et al. (2002): it minimises the +squared distance to the reference, a quadratic program (QP). GLPK cannot solve +QPs, so cobrapy switches to a QP-capable solver for the call (Gurobi or CPLEX) +and raises `SolverNotFound` when none is installed. `pip install gurobipy` +brings a size-limited Gurobi licence that is enough for a model this small. +`linear=True` minimises the summed absolute distance instead, an LP, and is a +different method with different answers. MOMA's prediction depends on the +reference flux vector, which pFBA picks from many alternative optima. ::: :::: From 15a0347554e470335e2673b735c179b2f5d01bf5 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 09/32] docs(quality-control): scope blocked reactions to current bounds and close every boundary --- docs/guide/quality-control.md | 90 +++++++++++++++++++++-------------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/docs/guide/quality-control.md b/docs/guide/quality-control.md index 5fc2284..8a7b950 100644 --- a/docs/guide/quality-control.md +++ b/docs/guide/quality-control.md @@ -11,9 +11,9 @@ nothing. |---|---|---| | `checkModelStruct` | `check_model` | structural problems | | `getElementalBalance` | `get_elemental_balance` | mass balance, reaction by reaction | -| `haveFlux` | `find_blocked_reactions` {bdg-link-secondary}`cobrapy ` | reactions that can never carry flux | +| `haveFlux` | `find_blocked_reactions` {bdg-link-secondary}`cobrapy ` | reactions that cannot carry flux under the current bounds | | `canExchange` | `analyse_topology` | which metabolites the model can make, or consume, given its medium | -| `findLeakMetabolite` | `Model.optimize` on a demand {bdg-link-secondary}`cobrapy ` | can the model make something from **nothing** | +| `findLeakMetabolite` | `find_leak_metabolite`, `gap_report` | can the model make something from **nothing** | | `gapReport` | `check_model` + `analyse_topology` | one summary of the gaps | ## Setup @@ -176,10 +176,12 @@ one that is undecidable. `rxns` restricts the check to a subset. ::: :::: -## 9.3 What can never carry flux? +## 9.3 What cannot carry flux? -A reaction that cannot carry flux under any conditions is either a gap or a -mistake. It is also the cheapest of these checks to run. +A reaction that cannot carry flux under the model's current bounds is either a +gap, a mistake, or waiting for a medium that is not open. Both functions test +the bounds as they are; neither explores other media. It is also the cheapest +of these checks to run. `haveFlux` counts a reaction as active once it carries more than its `cutOff`, `1e-6` by default, which keeps solver noise from reading as flux. It shuffles @@ -218,6 +220,12 @@ blocked: ['ACO', 'ACS', 'ADH1', 'ADH2', 'ALD6'] ::: :::: +`smallYeastBad` has glucose and oxygen uptake open, and the same test on +`smallYeast` with that medium finds 52 of 53 reactions active: the 52 blocked +here are blocked by the errors this page goes on to find. The two that do carry +flux, `SDH` and `FRDS2`, are each other's reverse and only cycle (see +[18.3](fva.md)). + ## 9.4 Can the model make something from nothing? With every uptake closed, a correct model can produce nothing at all. If it still @@ -230,8 +238,10 @@ source. :sync: matlab ```matlab -exchangeRxns = getExchangeRxns(model); +exchangeRxns = getExchangeRxns(model); % exchanges, sinks and demands closed = setParam(model, 'eq', exchangeRxns, 0); +sol = solveLP(closed); +fprintf('closed model feasible: %d\n', ~isempty(sol.x)); % if not, it tests nothing produced = canExchange(closed, 'produce'); fprintf('%d of %d metabolites producible from nothing\n', ... @@ -242,11 +252,16 @@ fprintf('one leak: %s\n', strjoin(closed.mets(metabolite), ', ')); ``` ```text +closed model feasible: 1 33 of 52 metabolites producible from nothing one leak: F6P_c ``` -`canExchange` counts them; `findLeakMetabolite` finds one, using as few +`getExchangeRxns` returns every reaction with metabolites on one side only, +so sinks and demands are closed along with the exchanges. A model that is +infeasible once closed, for instance because of a forced maintenance flux, +cannot show a leak, so check that first. `canExchange` counts the leaks; +`findLeakMetabolite` finds one, using as few reactions as possible, so you have somewhere to start. Both take the direction as their second argument, and `'consume'` asks the mirror-image question: what this model can take up without limit. @@ -255,46 +270,47 @@ question: what this model can take up without limit. :sync: python ```python -with model: - for exchange in model.exchanges: - exchange.bounds = (0, 0) - - # a demand for every metabolite, so by-products can leave: the same - # assumption findLeakMetabolite makes with allowExcretion - demands = { - metabolite.id: model.add_boundary(metabolite, type="demand") - for metabolite in list(model.metabolites) - } - - leaks = [] - for met_id, demand in demands.items(): - model.objective = demand - if (model.slim_optimize() or 0) > 1e-6: - leaks.append(met_id) - print(f"{len(leaks)} metabolite(s) producible from nothing: {sorted(leaks)[:5]}") +from raven_toolbox.gapfilling import find_leak_metabolite, gap_report + +closed = model.copy() +for reaction in closed.boundary: # exchanges, sinks and demands + reaction.bounds = (0, 0) +closed.slim_optimize() +print("closed model:", closed.solver.status) # if not optimal, it tests nothing + +produced = gap_report(model).can_produce_without_input +print(f"{len(produced)} metabolite(s) producible from nothing: {sorted(produced)[:5]}") +print("one leak:", find_leak_metabolite(closed, "produce").metabolites) ``` ```text +closed model: optimal 33 metabolite(s) producible from nothing: ['ACA_c', 'AC_c', 'AKG_m', 'BIOMASS_c', 'CI_m'] +one leak: ['F6P_c', 'GP_c', 'PYR_c'] ``` -raven-toolbox has no single call for this. The loop is what it amounts to: -close everything, let every metabolite leave, then maximise each one in turn -and see whether anything comes out. That last assumption matters: without a -way for by-products to leave, a leaking reaction is blocked by its own -stoichiometry and the test finds nothing. `analyse_topology` answers the -connectivity half of the same question without solving an LP. +`gap_report` closes every boundary reaction itself, adds an output for every +metabolite, and lists in `can_produce_without_input` the ones whose output can +carry flux, as `canExchange` does. `find_leak_metabolite` is the port of +`findLeakMetabolite` and does not close anything, so give it the closed copy. +It returns a list: more than one id means those metabolites could only be +exchanged together. +Close `model.boundary` rather than `model.exchanges` when you do it by hand: +cobrapy does not count sinks and demands as exchanges, and an open sink is +an input. `analyse_topology` answers the connectivity half of the same question +without solving an LP. ::: :::: :::{note} Give by-products a way out, or you will find nothing -Both RAVEN functions assume every metabolite can be excreted while the test -runs: `canExchange` adds an output reaction for each metabolite it checks, -and `findLeakMetabolite` takes `allowExcretion` as true by default. The Python -loop has to do it explicitly, and the difference is not subtle: testing one -demand reaction at a time, with no outlet for the by-products, reports **zero** -leaks in this model instead of 33, because a leaking reaction is then blocked -by its own stoichiometry. +The functions above assume every metabolite can be excreted while the test +runs: `canExchange` and `gap_report` add an output reaction for each metabolite +they check, and `findLeakMetabolite` and `find_leak_metabolite` take +`allowExcretion` / `allow_excretion` as true by default. A hand-written loop +has to do it explicitly, and the difference is not subtle: testing one demand +reaction at a time, with no outlet for the by-products, reports **zero** leaks +in this model instead of 33, because a leaking reaction is then blocked by its +own stoichiometry. ::: ## 9.5 Compare against a model you trust From 2cc26ec705d1ac75b8b7c16f20c70ea4d3218841 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 10/32] docs: say what flux a model without exchanges can still carry --- docs/guide/building.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/guide/building.md b/docs/guide/building.md index 915e8da..e45f42d 100644 --- a/docs/guide/building.md +++ b/docs/guide/building.md @@ -192,10 +192,10 @@ genes are all defined. ## 7.3 Add exchanges -A network with no boundary cannot carry flux: under the steady-state assumption -every internal metabolite must be produced and consumed at the same rate, so a -metabolite that appears in only one reaction blocks it. Exchange reactions are -that boundary. +Without boundary reactions nothing can enter or leave, so there is no net +uptake, secretion or growth. Internal cycles can still carry flux (see +[18.3](fva.md)), and under the steady-state assumption a metabolite that appears +in only one reaction blocks that reaction. Exchange reactions are the boundary. ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB @@ -277,8 +277,8 @@ HEX1 flux: 1000.00 become two different things in Python and an error in MATLAB. Set `allow_new_mets=False` and `allow_new_genes=False` once everything is defined. -- **No exchange reactions.** The model then gives zero flux everywhere, with - no error to explain why. +- **No exchange reactions.** Nothing can be taken up or secreted, so growth + is zero, with no error to explain why. - **A model with no external compartment.** `add_boundary` cannot guess one, and RAVEN's `addExchangeRxns` adds exchanges for internal metabolites without warning, opening the model to uptake and secretion of From 1479b4b0876b5b07a0546fb8230f116cddbb3971 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 11/32] docs(homology): count BLAST alignments, not genes, and link the FASTA files --- docs/guide/homology.md | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/guide/homology.md b/docs/guide/homology.md index 2e6c17d..e6784e1 100644 --- a/docs/guide/homology.md +++ b/docs/guide/homology.md @@ -24,7 +24,11 @@ reliability of the draft, and they are what encode the method's assumptions. Two proteomes and a template model. The template is `smallYeast.yml` (53 reactions, 61 genes), and `sce-template.faa` holds the sequences of exactly those 61 *S. cerevisiae* genes. The organism being reconstructed is *Hansenula -polymorpha*, whose full 5177-protein proteome is in `hanpo.faa`. +polymorpha*, whose full 5177-protein proteome is in `hanpo.faa`. All three files +are in [`docs/data/`](../data/README.md): +[`smallYeast.yml`](../data/smallYeast.yml), +[`sce-template.faa`](../data/sce-template.faa) and +[`hanpo.faa`](../data/hanpo.faa). The template model's id must match the id used for the BLAST. The transfer step looks up each hit's source organism by that id to find which template model the @@ -116,10 +120,12 @@ sce hanpo 159 ::: :::: -The two directions return different counts because they ask different questions: -159 template genes found a match in *H. polymorpha*, and 178 *H. polymorpha* -genes found a match in the template. Neither number is the number of orthologs; -that is decided in the next step, from the pairs that appear in both directions. +The two directions return different counts because they ask different +questions, and neither count is of genes: each line of BLAST output is one +alignment. The 159 alignments in the template-to-*H. polymorpha* direction come +from 60 of the 61 template genes, and the 178 in the other direction from 106 +*H. polymorpha* genes. Neither number is the number of orthologs; that is decided +in the next step, from the pairs that appear in both directions. The shape of the result differs between the toolboxes. RAVEN returns a struct array with one entry per direction, each carrying `fromId`, `toId`, `fromGenes`, @@ -324,9 +330,8 @@ reactions that can carry flux: 0 of 37 :::: None of the 37 reactions can carry flux. This is not a defect in the draft: with -no exchange reactions there is no way for anything to enter or leave the system, -so every reaction is blocked by the steady-state constraint regardless of how -well connected the network is. The number says nothing yet about the quality of +no exchange reactions nothing can enter or leave, so the only flux left is +internal cycling, and this draft has none. The number says nothing yet about the quality of the reconstruction, and will only become informative once a medium is defined. A homology draft is a set of claims about which reactions the organism has. Turning From 4029adb7bf7e5d3657c52654ce699aea15ae342e Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 12/32] docs(fba): scope the uptake sign and list what the context manager undoes --- docs/guide/fba.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/guide/fba.md b/docs/guide/fba.md index 7b3a3dd..c314e90 100644 --- a/docs/guide/fba.md +++ b/docs/guide/fba.md @@ -138,8 +138,10 @@ minimising. ## 4.3 Constrain an uptake rate -Uptake is a **negative** flux through an exchange reaction, so the lower bound is -what limits it. yeast-GEM ships with glucose capped at 1 mmol/gDW/h, which is +yeast-GEM writes its exchange reactions as `metabolite <=>`, so uptake is a +**negative** flux and the lower bound is what limits it. A model that writes +uptake as `=> metabolite`, like smallYeast in [6. Solvers](solvers.md), has the +opposite sign. yeast-GEM ships with glucose capped at 1 mmol/gDW/h, which is where the growth rate above comes from; ten times the glucose gives roughly ten times the growth. @@ -238,9 +240,12 @@ back to aerobic: 0.0809 /h ``` Without `with`, the bound change would persist and every later result on this -page would be an anaerobic one. The block records every change made to the -model inside it, not only the ones written here, so added reactions and -changed objectives are undone as well. +page would be an anaerobic one. The block undoes the edits cobrapy records: +bounds, the objective, reactions and metabolites added or removed, +stoichiometry changed with `add_metabolites`, gene rules and gene knockouts. +Plain attribute edits are not recorded and survive the block: names, formulas, +charges, `subsystem`, the model id and anything written into `annotation` or +`notes`. Make those on `model.copy()`. Two things in that output need care. The anaerobic growth rate is **zero**, not a smaller positive number: closing the oxygen exchange is not enough to @@ -312,8 +317,8 @@ filtering works: `solution.fluxes[solution.fluxes.abs() > 1e-6]`. ::: :::: -The signs say which way material moves: negative through an exchange is uptake, -positive is secretion. Glucose and oxygen enter, CO2 leaves, and ethanol is zero +In yeast-GEM's orientation the signs say which way material moves: negative +through an exchange is uptake, positive is secretion. Glucose and oxygen enter, CO2 leaves, and ethanol is zero because the model has enough oxygen to respire everything it takes up. ## 4.6 Pick a parsimonious solution From c05abb7d4c82df4aac0b73a8abe86771e21c1d08 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 13/32] docs(model-structure): pair getIndexes with an id lookup and resolve name[comp] --- docs/guide/model-structure.md | 19 ++++++++++++++++--- scripts/curated_pairs.yml | 2 +- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/guide/model-structure.md b/docs/guide/model-structure.md index f48c558..0e005fe 100644 --- a/docs/guide/model-structure.md +++ b/docs/guide/model-structure.md @@ -10,7 +10,8 @@ of what you need to translate a script between the two. |---|---|---| | `checkModelStruct` | `check_model` | report problems in a model | | `sortIdentifiers` | `sort_identifiers` | sort reactions, metabolites and genes by id | -| `getIndexes` | `parse_name_comp` | split a `name[comp]` token | +| `getIndexes` | `DictList.get_by_id` {bdg-link-secondary}`cobrapy ` | look up a reaction, metabolite or gene by id | +| `getIndexes` (`'metcomps'`) | `parse_name_comp`, then a name and compartment filter | resolve a `name[comp]` token | | no equivalent | `subsystem_to_str` | one subsystem string, whatever the source stored | | `addIdentifierPrefix`, `removeIdentifierPrefix` | handled on read/write {bdg-link-secondary}`cobrapy ` | SBML identifier prefixes | | `ravenCobraWrapper` | not needed | convert between RAVEN and COBRA structs | @@ -96,7 +97,9 @@ A miss is an error rather than a zero: `getIndexes` raises `Could not find object 'X' in the model` in every mode except `'metnames'`, which returns an empty vector instead. That exception exists because a metabolite *name* can occur in several compartments, so `'metnames'` returns -every matching index rather than one. Asking for several names at once gives +every matching index rather than one. In `'metcomps'` mode a token that +matches more than one metabolite (the same name twice in one compartment) is +an error as well, naming how many it matched. Asking for several names at once gives back a cell array with one index vector per name; asking for exactly one name unwraps that to a plain vector. ::: @@ -195,16 +198,24 @@ from raven_toolbox.utils.parse import parse_name_comp print(parse_name_comp("alpha-D-glucose 6-phosphate[c]")) print(parse_name_comp("ATP")) + +name, comp = parse_name_comp("alpha-D-glucose 6-phosphate[c]") +matches = model.metabolites.query(lambda m: m.name == name and m.compartment == comp, None) +print([m.id for m in matches]) ``` ```text ('alpha-D-glucose 6-phosphate', 'c') ('ATP', None) +['G6P_c'] ``` `parse_name_comp` returns the compartment as `None` when there is no trailing bracket, so one call handles both forms and the caller decides what a missing -compartment means. Note the import path: it is in +compartment means. It only splits the string and never sees a model; the +`query` after it is the lookup that `getIndexes` performs. An empty list is a +miss, and more than one entry means the name is duplicated in that +compartment. Note the import path: it is in `raven_toolbox.utils.parse`, not in the `raven_toolbox.utils` package namespace, which re-exports only the curation helpers. ::: @@ -223,6 +234,7 @@ whichever form a model arrived with. :sync: matlab ```matlab +idx = getIndexes(model, 'PGI', 'rxns'); % nest every entry before nesting one, so the field keeps a single shape model.subSystems = cellfun(@cellstr, model.subSystems, 'UniformOutput', false); model.subSystems{idx} = {'Glycolysis', 'Pentose phosphate pathway'}; @@ -239,6 +251,7 @@ Glycolysis; Pentose phosphate pathway ```python from raven_toolbox.utils.parse import subsystem_to_str +rxn = model.reactions.get_by_id("PGI") rxn.subsystem = ["Glycolysis", "Pentose phosphate pathway"] print(subsystem_to_str(rxn.subsystem)) print(subsystem_to_str("Glycolysis")) diff --git a/scripts/curated_pairs.yml b/scripts/curated_pairs.yml index 3aedf4c..53ce8fc 100644 --- a/scripts/curated_pairs.yml +++ b/scripts/curated_pairs.yml @@ -72,7 +72,6 @@ aliases: # Comparison and queries compareMultipleModels: compare_models - getIndexes: parse_name_comp exportToExcelFormat: export_to_excel # Gap-filling and analysis @@ -92,6 +91,7 @@ aliases: cobrapy: # RAVEN function -> the cobrapy object or function that replaces it. importModel: cobra.io.read_sbml_model + getIndexes: cobra.DictList.get_by_id exportModel: cobra.io.write_sbml_model solveLP: cobra.Model.optimize setParam: cobra.Reaction.bounds From 65926eca9a7e77d6e9c732bbb3279e4c9436665a Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 14/32] docs(io): cobrapy decodes entity ids and leaves only the model id encoded --- docs/guide/io.md | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/guide/io.md b/docs/guide/io.md index 07bcadc..3e1e2d2 100644 --- a/docs/guide/io.md +++ b/docs/guide/io.md @@ -133,11 +133,10 @@ from YAML: yeastGEM_v9.1.0 same size: True ``` -cobrapy leaves the encoding in place, so the same release read from the two -formats reports two different ids while being the same model. Only the -*identifiers* are affected; `model.name` and the annotations are unchanged. -Ids have to be decoded before they can be matched against a model read from -YAML. +cobrapy decodes the reaction, metabolite and gene ids but leaves the model id +encoded, so the same release read from the two formats reports two different +model ids while being the same model. `model.name` and the annotations are +unchanged. ::: :::: @@ -332,9 +331,13 @@ repositories that pin one. letter or underscore. Both toolboxes add a prefix to a whole field when any id in it does not, which is why `importModel` and `read_sbml_model` strip prefixes on the way back. -- **Ids differ between two copies of the same release.** An id that contained - a dot comes back encoded from SBML in Python. Compare on `model.name` or on - annotations, or decode first. +- **The model id differs between two copies of the same release.** cobrapy + decodes `__46__`-style escapes in reaction, metabolite and gene ids, but not + in the model id, so yeast-GEM read from SBML reports + `yeastGEM_v9__46__1__46__0`. Decode it with + `re.sub(r"__(\d+)__", lambda m: chr(int(m.group(1))), model.id)`, or compare + `model.name`. Do not match entities on annotations: some database ids are + shared by differently named metabolites. - **Excel export fails with a missing module.** The Python side needs `openpyxl`; install `raven-toolbox[excel]`. - **A YAML round trip loses a field neither format defines.** RAVEN YAML From a988ca8c362be7f0f2ef7feffbe9d4ff297a5096 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 15/32] docs(curation): read the table by header and use a consistent formula and charge --- docs/data/curation-mets.tsv | 8 ++--- docs/guide/curation.md | 70 +++++++++++++------------------------ 2 files changed, 28 insertions(+), 50 deletions(-) diff --git a/docs/data/curation-mets.tsv b/docs/data/curation-mets.tsv index e8e633e..776e2a0 100644 --- a/docs/data/curation-mets.tsv +++ b/docs/data/curation-mets.tsv @@ -1,5 +1,5 @@ metNames comps formula charge metNotes -(S)-malate m C4H6O5 -2 charge assigned at pH 7.3 -2-oxoglutarate m C5H6O5 -2 charge assigned at pH 7.3 -2-phospho-D-glycerate c C3H7O7P -3 charge assigned at pH 7.3 -3-phospho-D-glycerate c C3H7O7P -3 charge assigned at pH 7.3 +(S)-malate m C4H4O5 -2 charge assigned at pH 7.3 +2-oxoglutarate m C5H4O5 -2 charge assigned at pH 7.3 +2-phospho-D-glycerate c C3H4O7P -3 charge assigned at pH 7.3 +3-phospho-D-glycerate c C3H4O7P -3 charge assigned at pH 7.3 diff --git a/docs/guide/curation.md b/docs/guide/curation.md index c676393..739607f 100644 --- a/docs/guide/curation.md +++ b/docs/guide/curation.md @@ -23,7 +23,9 @@ Four tables, each optional, except that the two reaction tables go together: | reactions | `rxnIdx`, `rxnNames`, `grRules`, `lb`, `ub`, `rev`, `subSystems`, `eccodes`, `rxnNotes`, `rxnReferences`, `rxnConfidenceScores`, then MIRIAM | | reaction coefficients | `rxnIdx`, `rxnNames`, `metNames`, `comps`, `coefficient`, one row per reaction and metabolite | -Entities are matched by **name**, not by identifier: metabolites on +Columns are found by their header, so their order does not matter and any of +the optional ones can be left out. Entities are matched by **name**, not by +identifier: metabolites on `metaboliteName[comp]`, genes on the gene name, reactions on the stoichiometry of their reactants and products. A row that matches nothing adds a new entity; a row that matches an existing one overwrites it. `rxnIdx` ties the two reaction tables @@ -34,14 +36,19 @@ together and is local to the files, not an identifier in the model. `smallYeast.yml` and [`curation-mets.tsv`](../data/curation-mets.tsv) from [`docs/data/`](../data/README.md). The model carries formulas but no charges, which is why the charge half of a mass balance is undecidable in -[1. Getting started](getting-started.md). The table supplies four of them. +[1. Getting started](getting-started.md). The table supplies four of them, +with the formulas of the charged species to match: a formula and a charge have +to describe the same species, or the charge balance this is for cannot work. +The rest of `smallYeast` keeps its neutral formulas, so the reactions that use +these four metabolites balance again only once their partners are curated the +same way; [9. Quality control](quality-control.md) finds them. ```text title="curation-mets.tsv" metNames comps formula charge metNotes -(S)-malate m C4H6O5 -2 charge assigned at pH 7.3 -2-oxoglutarate m C5H6O5 -2 charge assigned at pH 7.3 -2-phospho-D-glycerate c C3H7O7P -3 charge assigned at pH 7.3 -3-phospho-D-glycerate c C3H7O7P -3 charge assigned at pH 7.3 +(S)-malate m C4H4O5 -2 charge assigned at pH 7.3 +2-oxoglutarate m C5H4O5 -2 charge assigned at pH 7.3 +2-phospho-D-glycerate c C3H4O7P -3 charge assigned at pH 7.3 +3-phospho-D-glycerate c C3H4O7P -3 charge assigned at pH 7.3 ``` ## 10.1 Apply a table @@ -52,38 +59,25 @@ metNames comps formula charge metNotes ```matlab model = readYAMLmodel('smallYeast.yml'); -fprintf('before: metCharges %d, metNotes %d\n', ... - isfield(model, 'metCharges'), isfield(model, 'metNotes')); +fprintf('charges before: %d\n', isfield(model, 'metCharges')); curated = curateModelFromTables(model, 'curation-mets.tsv'); -fprintf('after: metCharges %d, metNotes %d\n', ... - isfield(curated, 'metCharges'), isfield(curated, 'metNotes')); - idx = getIndexes(curated, 'MAL_m', 'mets'); -fprintf('%s formula %s\n', curated.metNames{idx}, curated.metFormulas{idx}); +fprintf('%s charge %d, note: %s\n', curated.metNames{idx}, ... + curated.metCharges(idx), curated.metNotes{idx}); ``` ```text -before: metCharges 0, metNotes 0 -[Warning: The following metabolites are already present in the model, their annotation will be overwritten to match the metsInfo file. If you do not particularly want to curate their annotations, it would be better to removes these metabolites from metsInfo: (S)-malate[m] - 2-oxoglutarate[m] - 2-phospho-D-glycerate[c] - 3-phospho-D-glycerate[c]] -after: metCharges 0, metNotes 0 -(S)-malate formula C4H6O5 +... ``` A model with no charges has no `metCharges` field at all, rather than a field full of `NaN`. RAVEN's optional fields work this way throughout, so code that -reads one should check with `isfield` first. - -Both fields are still absent afterwards. `curateModelFromTables` writes an -optional field onto an **existing** metabolite only when the model already -carries that field. -`smallYeast.yml` has neither `metCharges` nor `metNotes`, so both columns are -dropped for these four rows, silently. `metFormulas` does exist, so the -formula column is applied. A metabolite the table *adds* is unaffected: it -arrives with every field the table supplies. +reads one should check with `isfield` first. `curateModelFromTables` creates a +field the table supplies and the model lacks, giving every metabolite the +table does not name the field's default (`NaN` for a charge, an empty string +for a note). The warning lists the four metabolites whose values were +overwritten. `curateModelFromTables` returns a new struct and leaves its input alone, in keeping with the rest of RAVEN. @@ -103,14 +97,11 @@ print("charges after: ", sum(1 for m in model.metabolites if m.charge is not Non print("updated:", result.updated_metabolites) mal = model.metabolites.get_by_id("MAL_m") -print(f"{mal.name} charge {mal.charge}") +print(f"{mal.name} charge {mal.charge}, note: {mal.notes['note']}") ``` ```text -charges before: 0 -charges after: 4 -updated: ['MAL_m', 'AKG_m', 'P2G_c', 'P3G_c'] -(S)-malate charge -2 +... ``` `batch_curate_from_tsv` edits the model **in place** and returns a @@ -124,19 +115,6 @@ builds them rather than reading them from disk. ::: :::: -:::{warning} An absent field silently swallows a column -The same table gives two different results on a model that lacks the field -being curated. raven-toolbox sets the value on every matched metabolite, -creating the attribute as needed. RAVEN sets it only on metabolites the table -*adds*, and drops the column for metabolites that already exist, because the -field it would write into is not there and it does not create it. - -So a table written to add charges to an uncharged model does nothing in -MATLAB and works in Python. Check the field afterwards rather than assuming -the table was applied in full, and note that this is a divergence between the -two toolboxes rather than a difference in the tables. -::: - ## 10.2 Adding rather than updating A row whose name matches nothing in the model adds a new entity, and the From 3e1231b26660d809432254464dbbf7eb92c5216e Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 16/32] docs(biomass): balance to 1 g/gDW, read SBO terms back, and state the SBO rules --- docs/guide/biomass.md | 46 ++++++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 11 deletions(-) diff --git a/docs/guide/biomass.md b/docs/guide/biomass.md index f7c686b..893c8f3 100644 --- a/docs/guide/biomass.md +++ b/docs/guide/biomass.md @@ -222,9 +222,13 @@ protein 0.5000, carbohydrate 0.3384, total 1.0000 the same split as in [20. Combining and simplifying](combining.md), and it is why the Python tab copies the model first. -`balanceOut` names the component that absorbs the change, so the total stays -where it was. Omit it and the component is rescaled on its own, which is correct -only if some other step restores the total. +`balanceOut` names the component that absorbs the change: it is rescaled so the +components sum to exactly 1 g/gDW, whatever the total was before (1.0051 above +becomes 1.0000). The balancing component must be large enough to absorb the +difference; if it would have to go negative, both functions raise an error +rather than turn its pseudoreaction around. Omit `balanceOut` and the component +is rescaled on its own, which is correct only if some other step restores the +total. `scaleBiomassPseudoreaction` and `rescale_pseudoreaction` work one level down, on a single pseudoreaction rather than a named component, and rebalance the protons @@ -244,12 +248,20 @@ everything that reads a model afterwards does not. ```matlab small = readYAMLmodel('smallYeast.yml'); annotated = assignSBOterms(small); -fprintf('rxnSBOs present: %d\n', isfield(annotated, 'rxnSBOs')); +idx = getIndexes(annotated, {'glcIN', 'PGI'}, 'rxns'); +sbo = extractMiriam(annotated.rxnMiriams(idx), 'sbo'); +fprintf('%s: %s\n', annotated.rxns{idx(1)}, sbo{1}); +fprintf('%s: %s\n', annotated.rxns{idx(2)}, sbo{2}); ``` ```text -rxnSBOs present: 0 +glcIN: SBO:0000628 +PGI: SBO:0000176 ``` + +RAVEN keeps the term as a MIRIAM entry with namespace `sbo` in `rxnMiriams` +and `metMiriams`, so `extractMiriam` reads it out, and `exportModel` writes it +to SBML as the `sboTerm` attribute. ::: :::{tab-item} 🐍 Python :sync: python @@ -259,19 +271,31 @@ from raven_toolbox.annotation import add_sbo_terms small = read_yaml_model("smallYeast.yml") add_sbo_terms(small) -labelled = sum(1 for r in small.reactions if "sbo" in r.annotation) -print(f"{labelled} of {len(small.reactions)} reactions labelled") +for rid in ("glcIN", "PGI"): + print(f"{rid}: {small.reactions.get_by_id(rid).annotation['sbo']}") ``` ```text -53 of 53 reactions labelled +glcIN: SBO:0000628 +PGI: SBO:0000176 ``` + +cobrapy keeps the term in each object's `annotation["sbo"]`, as a string, and +writes it to SBML as `sboTerm` too. ::: :::: -The terms are assigned from what the model already says: a reaction with one -metabolite and a boundary is an exchange, a reaction whose metabolites differ -only by compartment is a transport, and so on. Nothing is inferred that the +The terms are assigned from what the model already says. A reaction with +metabolites on one side only is an exchange (SBO:0000627) when its metabolite is +in an extracellular compartment (`e`, or one named `extracellular`), and +otherwise a sink (SBO:0000632) when it consumes the metabolite or a demand +(SBO:0000628) when it produces it; smallYeast has no extracellular compartment, +so its uptakes are demands and its outputs sinks. A reaction in which one +metabolite name occurs in more than one compartment is a transport +(SBO:0000655), which is why `GROWTH`, which uses acetyl-CoA in both `c` and +`m`, is labelled one. Everything else is a metabolic reaction (SBO:0000176), +unless its name marks it as the biomass, maintenance or another +pseudoreaction. Nothing is inferred that the stoichiometry does not already support, so running it on a model with mislabelled compartments propagates that mistake rather than catching it. From ee4e3e81e7dd31a4fbda478d5f680445edee21e9 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 17/32] docs(fva): compare ranges at the optimum and with a 90% growth floor --- docs/guide/fva.md | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/docs/guide/fva.md b/docs/guide/fva.md index de5218a..48851fc 100644 --- a/docs/guide/fva.md +++ b/docs/guide/fva.md @@ -106,42 +106,54 @@ asks a different question; see the next section. ## 18.2 The range *at* the optimum -The useful question for interpreting a result: holding growth at its maximum (or -at 90 % of it), which fluxes are still free to move? +The useful question for interpreting a result: holding growth at its maximum, +which fluxes are still free to move, and how many come free again if growth only +has to reach 90 % of it? ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB :sync: matlab ```matlab -fixed = setParam(model, 'lb', 'biomassOUT', 0.9 * sol.f); -[minFluxes, maxFluxes] = getAllowedBounds(fixed); -span = full(maxFluxes - minFluxes); -fprintf('at 90%% of optimum: %d reactions fixed\n', sum(span < 1e-9)); +for fraction = [1 0.9] + floored = setParam(model, 'lb', 'biomassOUT', fraction * sol.f); + [minFluxes, maxFluxes] = getAllowedBounds(floored); + span = full(maxFluxes - minFluxes); + fprintf('growth >= %.0f%% of optimum: %d reactions fixed\n', 100 * fraction, sum(span < 1e-9)); +end ``` ```text -at 90% of optimum: 1 reactions fixed +growth >= 100% of optimum: 51 reactions fixed +growth >= 90% of optimum: 1 reactions fixed ``` RAVEN has no `fraction_of_optimum` argument: constrain the objective -reaction yourself, then ask for the bounds. +reaction yourself, then ask for the bounds. A lower bound is a floor, not a +fixed value, so `0.9 * sol.f` lets growth sit anywhere from 90 % of the optimum +up to the optimum itself. ::: :::{tab-item} 🐍 Python :sync: python ```python -at_optimum = flux_variability_analysis(model, fraction_of_optimum=0.9) -span = at_optimum.maximum - at_optimum.minimum -print(f"at 90% of optimum: {(span < 1e-9).sum()} reactions fixed") +for fraction in (1.0, 0.9): + ranges = flux_variability_analysis(model, fraction_of_optimum=fraction) + span = ranges.maximum - ranges.minimum + print(f"growth >= {fraction:.0%} of optimum: {(span < 1e-9).sum()} reactions fixed") ``` ```text -at 90% of optimum: 1 reactions fixed +growth >= 100% of optimum: 51 reactions fixed +growth >= 90% of optimum: 1 reactions fixed ``` ::: :::: +At the optimum only `FRDS2` and `SDH` can still move (18.3 shows why); everything +else is determined. Let growth drop to 90 % and only `ethIN`, which is closed, +stays fixed: a 10 % margin frees nearly every flux. + Reactions whose span collapses to zero at the optimum are the ones the model has no choice about. Those are the predictions the model determines; a flux with a wide range at the optimum is an artefact of which vertex the solver happened to @@ -224,7 +236,9 @@ sampling over, keeping a reaction only if its **loopless** range is non-trivial. When a single flux distribution is needed (for a figure, or to compare two conditions), take a parsimonious one rather than whatever the solver returns -first. It is reproducible, and it is the natural companion to the ranges above. +first. It carries no loop flux and is less arbitrary than the first vertex +found, but it need not be unique either: where two routes cost the same total +flux, pFBA can return either. Report any single flux together with its range. ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB From 8f611a313dcd9cde736d717334528e8d03fdefa1 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 18/32] docs(sampling): attribute chain differences to the warmup LPs and fix the cost and mean bullets --- docs/guide/sampling.md | 72 +++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/docs/guide/sampling.md b/docs/guide/sampling.md index 0c6f5b0..a190625 100644 --- a/docs/guide/sampling.md +++ b/docs/guide/sampling.md @@ -58,13 +58,15 @@ model.objective = "biomassOUT" Both toolboxes take a `seed`, and you should always set one: without it a chain is different every run and nothing you report can be reproduced. -A seed does not guarantee identical numbers, though. The Python chain -here reproduces exactly across operating systems; the MATLAB one does not. Its -samplers take a nullspace basis from `null`, which comes from LAPACK, so the -same seed on Linux and on Windows explores the same *distribution* along a -slightly different walk: running this page on both put `FRDS2`'s sampled -minimum at 19.0 and at 18.8. That is why the numbers below are printed to two -decimals: the distribution is the result, the individual draws are not. +A seed does not guarantee identical numbers, though. The default `achr` +sampler starts from warmup points, and those come from LPs: two per reaction. +A different solver, solver version or platform, and with GLPK even an earlier +solve on the same model (which leaves a different starting basis), returns +different warmup vertices, so the same seed explores the same *distribution* +along a different walk. With seed 1, MATLAB puts `FRDS2`'s sampled minimum at +18.8 with GLPK and at 4.5 with Gurobi. That is why the numbers below are +printed to few decimals: the distribution is the result, the individual draws +are not. ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB @@ -229,30 +231,30 @@ its optimum, fix a measured flux) and sample the space that is left. ```matlab sol = solveLP(model); -atOptimum = setParam(model, 'lb', 'biomassOUT', 0.9 * sol.f); -constrained = randomSampling(atOptimum, 200, 'seed', 1); +floored = setParam(model, 'lb', 'biomassOUT', 0.9 * sol.f); +constrained = randomSampling(floored, 200, 'seed', 1); ethIdx = getIndexes(model, 'ethOUT', 'rxns'); -fprintf('ethanol: free %.2f, at 90%% growth %.2f\n', ... +fprintf('ethanol: free %.2f, growth >= 90%% %.2f\n', ... mean(solutions(ethIdx, :)), mean(constrained(ethIdx, :))); ``` ```text -ethanol: free 0.80, at 90% growth 0.04 +ethanol: free 0.80, growth >= 90% 0.04 ``` ::: :::{tab-item} 🐍 Python :sync: python ```python -at_optimum = model.copy() -at_optimum.reactions.get_by_id("biomassOUT").lower_bound = 0.9 * model.slim_optimize() -constrained = random_sampling(at_optimum, 200, seed=1) +floored = model.copy() +floored.reactions.get_by_id("biomassOUT").lower_bound = 0.9 * model.slim_optimize() +constrained = random_sampling(floored, 200, seed=1) print(f"ethanol: free {result.samples['ethOUT'].mean():.2f}, " - f"at 90% growth {constrained.samples['ethOUT'].mean():.2f}") + f"growth >= 90% {constrained.samples['ethOUT'].mean():.2f}") ``` ```text -ethanol: free 0.72, at 90% growth 0.04 +ethanol: free 0.72, growth >= 90% 0.04 ``` ::: :::: @@ -260,9 +262,12 @@ ethanol: free 0.72, at 90% growth 0.04 ## 19.5 Which reactions to sample over The `randomObjective` method needs reactions that can carry real flux to use as -objectives. Reactions that move only through a loop are useless for that, so -both toolboxes screen them out with a loopless FVA first, the same test -[18. Flux variability](fva.md) used by hand. +objectives. Reactions that move only through a loop are useless for that, and +the two toolboxes screen them out differently. raven-toolbox runs a loopless +FVA, the same test [18. Flux variability](fva.md) used by hand. RAVEN runs a +plain FVA (`getAllowedBounds`) and drops the reactions that reach 0.999 of the +model's largest bound, which is where loop flux ends up. Here both keep the +same 50. ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB @@ -297,21 +302,30 @@ which on a genome-scale model skips the expensive part of the run. :::: :::{warning} What can go wrong -- **No seed.** The numbers change every run. Set one, and report it, and do - not expect a seed alone to reproduce a MATLAB chain on another machine - (see 19.1). Report the distribution, not the draws. -- **Too little thinning.** Consecutive MCMC steps are correlated; the default - of 100 steps between recorded samples exists for that reason. Lowering it - gains speed and loses independence. +- **No seed.** The numbers change every run. Set one, and report it with the + solver: the warmup points come from LPs, so another solver (or, with GLPK, + an earlier solve on the same model) gives another chain (see 19.1). Report + the distribution, not the draws. +- **Too little thinning.** Consecutive MCMC steps are correlated, which is why + only every 100th is recorded. That is enough for this model, not for a + genome-scale one: on yeast-GEM, samples 100 steps apart are still 0.93 + correlated and independent chains disagree on most reactions (see the + [sampling convergence study](../parameter-tuning/studies/sampling-convergence-calibration.md)). + For results you report, run chains with different seeds and check that + they agree. - **Reading a mean as a prediction.** The mean of a sample describes the - *feasible space*, not the cell. A reaction can average 5 while no - individual feasible state puts it near 5. + feasible space and the sampler, not the cell. It is itself a feasible + state, but it sits where most of the volume is: in 19.2 that puts mean + growth below half the optimum. With `randomObjective` the samples are + vertices, so a reaction can average 5 while every sample has it at 0 or 10. - **Sampling a model that is wide open.** With an unconstrained medium the space is enormous and the distribution says nothing. Constrain first. - **Loops.** They inflate the space being sampled, and every sample drawn inside a cycle is wasted. See [18. Flux variability](fva.md). -- **Genome-scale runtime.** Sampling is many LPs per recorded sample. Start with - a few hundred samples to see the shape, not tens of thousands. +- **Genome-scale runtime.** `achr` starts with two LPs per reaction for its + warmup points and `chrr` with an FVA and a rounding step; the sampling + steps after that solve no LP. `randomObjective` solves one LP per sample. + Start with a few hundred samples to see the shape, not tens of thousands. ::: ## See also From 3ae2015d6dba33ed30108ab8be56221f36a6b3a3 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 19/32] docs(comparing): compare behaviour by growth, ranges or tasks, not one flux vector --- docs/guide/comparing.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/guide/comparing.md b/docs/guide/comparing.md index 1630479..52215f4 100644 --- a/docs/guide/comparing.md +++ b/docs/guide/comparing.md @@ -14,7 +14,7 @@ the same organism at all. |---|---|---| | `diffModels` | `diff_models` | every semantic difference between two models | | `compareMultipleModels` | `compare_models` | overlap in reactions, metabolites, genes, across two or more models | -| `compareFluxes` | `compare_fluxes` | which fluxes changed between two solutions | +| `compareFluxes` | `compare_fluxes` | which fluxes changed between two solutions of the same model | ## Setup @@ -168,8 +168,10 @@ grouping models, not for validating one. ## 21.3 Compare what they do, not what they contain Two models with the same reactions can behave differently, and two models with -different reactions can behave identically. The comparison that settles it is of -the fluxes. +different reactions can behave identically. What settles it is a simulation of +both under the same constraints: growth first, then flux ranges or tasks, not a +single flux vector, which is one optimum among many (see +[18. Flux variability](fva.md)). ::::{tab-set} :::{tab-item} Ⓜ️ MATLAB @@ -235,6 +237,11 @@ answer the same question, but you index them in opposite directions. [20. Combining and simplifying](combining.md), where merging matches on names instead. - **Reading similarity as quality.** It measures overlap, not correctness. +- **Comparing one flux vector per model.** An FBA solution is one of many + optima, so two models, or two solvers on one model, can return different + vectors with the same growth. Check that both solves succeeded and compare + growth first, then ranges or tasks. `compareFluxes` and `compare_fluxes` + compare two solutions of one model, so they need every reaction in both. - **Forgetting the medium.** A flux comparison compares conditions as much as models. Set the same bounds on both, explicitly, before drawing any conclusion. From a7146714b60ee67a15c038ae8813a13a129e69e9 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 20/32] docs(engineering): reporter metabolite scores agree between the toolboxes --- docs/guide/engineering.md | 36 +++++++++++++----------------------- 1 file changed, 13 insertions(+), 23 deletions(-) diff --git a/docs/guide/engineering.md b/docs/guide/engineering.md index 0b21cbf..92a4438 100644 --- a/docs/guide/engineering.md +++ b/docs/guide/engineering.md @@ -203,21 +203,19 @@ for i = 1:numel(model.genes) end repMets = reporterMetabolites(model, model.genes, pvalues); -[~, order] = sort(repMets(1).metPValues); -for i = 1:5 - j = order(i); - fprintf(' %-8s z %6.3f p %.4f genes %d\n', repMets(1).mets{j}, ... - repMets(1).metZScores(j), repMets(1).metPValues(j), ... - repMets(1).metNGenes(j)); +for i = 1:5 % sorted by z-score, highest first + fprintf(' %-8s z %6.3f p %.4f genes %d\n', repMets(1).mets{i}, ... + repMets(1).metZScores(i), repMets(1).metPValues(i), ... + repMets(1).metNGenes(i)); end ``` ```text - GA3P_c z 1.484 p 0.0689 genes 8 - X5P_c z 1.241 p 0.1073 genes 3 - F6P_c z 1.186 p 0.1179 genes 7 - PI_c z 1.170 p 0.1210 genes 12 - F16P_c z 1.117 p 0.1320 genes 4 + GA3P_c z 1.496 p 0.0673 genes 8 + X5P_c z 1.251 p 0.1054 genes 3 + F6P_c z 1.196 p 0.1159 genes 7 + PI_c z 1.180 p 0.1191 genes 12 + F16P_c z 1.126 p 0.1301 genes 4 ``` `printResults` prints the top 20 instead, and `outputFile` writes them. @@ -261,18 +259,10 @@ otherwise turn the whole result into `NaN`. ::: :::: -:::{note} Why the two tabs agree to three decimals and not four -The same five metabolites come out in the same order, with z-scores of -1.484 against 1.496 and 1.241 against 1.251. Both toolboxes implement the -same algorithm and the same background correction, but RAVEN estimates the -correction by Monte-Carlo sampling while raven-toolbox evaluates it in -closed form, so RAVEN's figures carry sampling noise and shift slightly -between runs. The ranking is the result; the third decimal is not. - -The two also organise the output differently. raven-toolbox reports the -one-sided enrichment p-value and sorts by z-score; RAVEN reports both tails -and sorts by p-value. With a single gene set the orderings coincide, as -above. +:::{note} Same scores, different layout +Both toolboxes compute the background correction in closed form and report +the same z-scores and one-sided p-values, sorted by z-score. RAVEN returns +them as parallel arrays in a struct, raven-toolbox as one table per gene set. ::: :::{warning} What can go wrong From be9633466649d05a77c048efdbf1218546f5e206 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 21/32] docs: list every example data file and say how to run a page --- docs/data/README.md | 24 ++++++++++++++---------- docs/guide/getting-started.md | 6 ++++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/docs/data/README.md b/docs/data/README.md index 396f01f..ee290d0 100644 --- a/docs/data/README.md +++ b/docs/data/README.md @@ -7,11 +7,15 @@ downloads the file and for the example runner in CI. | File | Size | Source | Used by | |---|---|---|---| -| `smallYeast.yml` | 45 kB | `RAVEN/tutorial/smallYeast.yml` | the default example model for most guide pages | -| `empty.xml` | 4 kB | `RAVEN/tutorial/empty.xml` | an empty SBML model with compartments defined, to build a model from scratch | -| `smallYeastBad.yml` | 45 kB | `RAVEN/tutorial/smallYeastBad.yml` | the quality-control and gap-filling pages, which need a model with known errors | -| `yeast-GEM.yml` | 3.6 MB | yeast-GEM **v9.1.0**, `model/yeast-GEM.yml` | pages that need a genome-scale model: sampling, deletions, ftINIT, gap-filling | -| `yeast-GEM.xml` | 11.6 MB | yeast-GEM **v9.1.0**, `model/yeast-GEM.xml` | the same model as SBML, for the pages about SBML I/O and annotations | +| [`smallYeast.yml`](smallYeast.yml) | 45 kB | `RAVEN/tutorials/data/models/smallYeast.yml` | the default example model for most guide pages | +| [`empty.xml`](empty.xml) | 4 kB | `RAVEN/tutorials/data/empty.xml` | [7. Building](../guide/building.md): an empty SBML model with compartments defined, to build a model from scratch | +| [`smallYeastBad.yml`](smallYeastBad.yml) | 45 kB | `RAVEN/tutorials/data/models/smallYeastBad.yml` | [9. Quality control](../guide/quality-control.md) and [21. Comparing](../guide/comparing.md), which need a model with known errors | +| [`yeast-GEM.yml`](yeast-GEM.yml) | 3.6 MB | yeast-GEM **v9.1.0**, `model/yeast-GEM.yml` | pages that need a genome-scale model: [3. I/O](../guide/io.md), [4. FBA](../guide/fba.md), [5. Media](../guide/media.md), [11. Biomass](../guide/biomass.md) | +| [`yeast-GEM.xml`](yeast-GEM.xml) | 11.6 MB | yeast-GEM **v9.1.0**, `model/yeast-GEM.xml` | the same model as SBML, for [3. I/O](../guide/io.md) | +| [`anaerobic.yml`](anaerobic.yml) | 2 kB | transcribed from yeast-GEM's `anaerobicModel.m` | [5. Media](../guide/media.md): a condition file, yeast-GEM under anaerobic conditions | +| [`tasks.txt`](tasks.txt) | 0.5 kB | written for this guide | [15. Metabolic tasks](../guide/tasks.md) and [17. Gap-filling](../guide/gap-filling.md): a task list for `smallYeast.yml` | +| [`curation-mets.tsv`](curation-mets.tsv) | 0.3 kB | written for this guide | [10. Table-driven curation](../guide/curation.md): a metabolite curation table | +| [`hpa-sample.tsv`](hpa-sample.tsv) | 1 kB | a ten-row excerpt in Human Protein Atlas format | [14. Context-specific models](../guide/init.md) | The two small yeast models are part of the RAVEN Toolbox (MIT licence). yeast-GEM is the consensus genome-scale model of *Saccharomyces cerevisiae* @@ -30,9 +34,9 @@ The two RAVEN tutorial copies are deliberate duplicates, so they do not follow t model, refresh them and re-run the examples: ```bash -cp "RAVEN/tutorial/smallYeast.yml" docs/data/smallYeast.yml -cp "RAVEN/tutorial/empty.xml" docs/data/empty.xml -cp "RAVEN/tutorial/smallYeastBad.yml" docs/data/smallYeastBad.yml +cp "RAVEN/tutorials/data/models/smallYeast.yml" docs/data/smallYeast.yml +cp "RAVEN/tutorials/data/empty.xml" docs/data/empty.xml +cp "RAVEN/tutorials/data/models/smallYeastBad.yml" docs/data/smallYeastBad.yml python scripts/run_examples.py ``` @@ -57,8 +61,8 @@ homology](../guide/homology.md). | file | what it is | |---|---| -| `hanpo.faa` | the *Hansenula polymorpha* proteome, 5177 sequences, copied unchanged from the `hanpo-GEM` submodule (`data/genomes/hanpo.faa`) | -| `sce-template.faa` | 61 *S. cerevisiae* sequences (exactly the genes of `smallYeast.yml`) extracted from the same submodule's `data/genomes/sce.faa` | +| [`hanpo.faa`](hanpo.faa) | the *Hansenula polymorpha* proteome, 5177 sequences, copied unchanged from the `hanpo-GEM` submodule (`data/genomes/hanpo.faa`) | +| [`sce-template.faa`](sce-template.faa) | 61 *S. cerevisiae* sequences (exactly the genes of `smallYeast.yml`) extracted from the same submodule's `data/genomes/sce.faa` | The template proteome is cut down to the model's own genes on purpose: BLAST against 61 subjects instead of a whole proteome is what keeps the page's example diff --git a/docs/guide/getting-started.md b/docs/guide/getting-started.md index 8550e5c..4629bc2 100644 --- a/docs/guide/getting-started.md +++ b/docs/guide/getting-started.md @@ -21,6 +21,12 @@ that ships with RAVEN. Download it from [`docs/data/smallYeast.yml`](../data/smallYeast.yml) and run everything from the directory you put it in. +Install one of the toolboxes first ([Installation](../installation/index.md)). +Every page lists the files it needs under *Setup*; they are all in +[`docs/data/`](../data/README.md), or clone the repository and work in +`raven-docs/docs/data`. The blocks on a page build on each other: run them in +order in one MATLAB or Python session. + ## 1.1 Load the model ::::{tab-set} From 6bbea6fc2ddad5b3a35bad68ec4837c7dc72cf58 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 22/32] docs: describe the Sphinx build in the README and build notes --- README.md | 39 ++++++++++++++++++++++++--------------- docs/README-sphinx.md | 11 ++++++----- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5269f0e..1999228 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,17 @@ Documentation site for **RAVEN** (the MATLAB toolbox) and **raven-toolbox** (the Python port) for genome-scale metabolic model reconstruction, analysis and -visualization. Built with [MkDocs](https://www.mkdocs.org/) + -[Material](https://squidfunk.github.io/mkdocs-material/) and published on Read -the Docs. +visualization. Built with [Sphinx](https://www.sphinx-doc.org/), +[MyST](https://myst-parser.readthedocs.io/) Markdown and the +[PyData theme](https://pydata-sphinx-theme.readthedocs.io/), and published on +Read the Docs. The site combines: -- a side-by-side **API reference** auto-generated from the function comments of - both toolboxes (via [mkdocstrings](https://mkdocstrings.github.io/) with its - Python and MATLAB handlers); +- a side-by-side **API reference** generated from the function comments of + both toolboxes; +- a **guide** of task pages with MATLAB and Python side by side, whose examples + are executed and checked against their printed output; - the **RAVEN tutorials**; and - a complete, worked **GEM reconstruction protocol** (the *Hansenula polymorpha* `hanpo-GEM` example). @@ -24,19 +26,26 @@ set in `.gitmodules`: [`RAVEN`](https://github.com/CompMetEng/RAVEN) and ```bash # clone with submodules -git clone --recurse-submodules https://github.com/edkerk/raven-docs.git +git clone --recurse-submodules https://github.com/CompMetEng/raven-docs.git cd raven-docs # (if already cloned: git submodule update --init --recursive) python -m venv .venv -source .venv/Scripts/activate # Windows; use .venv/bin/activate on macOS/Linux -pip install -r requirements.txt +source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements-sphinx.txt -mkdocs serve # live preview at http://127.0.0.1:8000 -mkdocs build # static site into ./site +sphinx-build -b html docs docs/_build/html ``` -The API reference pages are generated at build time by -[`scripts/gen_api_pages.py`](scripts/gen_api_pages.py), which walks the two -toolbox submodules and pairs their functions. No MATLAB runtime or installed -Python package is required; docstrings are collected statically from source. +The API reference pages are generated at the start of each build by +[`scripts/gen_api_pages_sphinx.py`](scripts/gen_api_pages_sphinx.py), called +from `docs/conf.py`, which walks the two toolbox submodules and pairs their +functions. No MATLAB runtime or installed Python package is required; +docstrings are collected statically from source. + +## Checking the guide examples + +[`scripts/run_examples.py`](scripts/run_examples.py) runs the code blocks on the +guide pages, MATLAB and Python, and compares what each prints with the `text` +block below it; `--update` rewrites those blocks. The +[Guide examples](.github/workflows/examples.yml) workflow runs it in CI. diff --git a/docs/README-sphinx.md b/docs/README-sphinx.md index ea2e445..fcab00b 100644 --- a/docs/README-sphinx.md +++ b/docs/README-sphinx.md @@ -1,10 +1,11 @@ # Sphinx / pydata-sphinx-theme exploration -Not part of the site. This is a working note for the `explore/pydata-sphinx-theme` -branch, read by nobody but a maintainer deciding whether to pursue the migration -for real. `mkdocs.yml` and the live MkDocs build are untouched and still what -CI and Read the Docs use; this branch adds a second, parallel toolchain over -the same `docs/` content. +Not part of the site. This note records the migration from MkDocs to Sphinx +(#78, #80). Sphinx is the live build: `.readthedocs.yaml` builds from +`docs/conf.py`. `mkdocs.yml`, `requirements.txt`, `scripts/gen_api_pages.py` +and `scripts/build_hooks.py` belong to the retired MkDocs build and are no +longer used. The sections below describe the state at the time of the +migration. ## Building it From 7728856c63cabbb04e5a412795455d4d198a39f0 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 23/32] docs(tutorials): include the scripts from RAVEN's tutorials folder --- docs/tutorials/index.md | 2 +- docs/tutorials/tutorial1.md | 2 +- docs/tutorials/tutorial2.md | 2 +- docs/tutorials/tutorial3.md | 2 +- docs/tutorials/tutorial4.md | 2 +- docs/tutorials/tutorial5.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index c8f5390..8cd5524 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -10,7 +10,7 @@ the exercises themselves are otherwise unchanged. They use the **MATLAB** toolbox and build up from running a simulation on an existing model to reconstructing a genome-scale model from sequence data. The scripts and all the data files they use live in the -[`tutorial/`](https://github.com/CompMetEng/RAVEN/tree/develop/tutorial) +[`tutorials/`](https://github.com/CompMetEng/RAVEN/tree/develop/tutorials) folder of the RAVEN repository. | # | Tutorial | What you learn | diff --git a/docs/tutorials/tutorial1.md b/docs/tutorials/tutorial1.md index 26b28ef..a5b3419 100644 --- a/docs/tutorials/tutorial1.md +++ b/docs/tutorials/tutorial1.md @@ -116,6 +116,6 @@ whereas on ethanol it involves acetate. ## Full script -```{literalinclude} ../../RAVEN/tutorial/tutorial1.m +```{literalinclude} ../../RAVEN/tutorials/tutorial1.m :language: matlab ``` diff --git a/docs/tutorials/tutorial2.md b/docs/tutorials/tutorial2.md index 6d75133..4a833d4 100644 --- a/docs/tutorials/tutorial2.md +++ b/docs/tutorials/tutorial2.md @@ -79,6 +79,6 @@ for more details and the worked answer. ## Full script -```{literalinclude} ../../RAVEN/tutorial/tutorial2.m +```{literalinclude} ../../RAVEN/tutorials/tutorial2.m :language: matlab ``` diff --git a/docs/tutorials/tutorial3.md b/docs/tutorials/tutorial3.md index 50f8563..9971b37 100644 --- a/docs/tutorials/tutorial3.md +++ b/docs/tutorials/tutorial3.md @@ -113,6 +113,6 @@ end ## Full script -```{literalinclude} ../../RAVEN/tutorial/tutorial3.m +```{literalinclude} ../../RAVEN/tutorials/tutorial3.m :language: matlab ``` diff --git a/docs/tutorials/tutorial4.md b/docs/tutorials/tutorial4.md index 8747268..a6ffde5 100644 --- a/docs/tutorials/tutorial4.md +++ b/docs/tutorials/tutorial4.md @@ -147,6 +147,6 @@ to connect the remaining gaps. ## Full script -```{literalinclude} ../../RAVEN/tutorial/tutorial4.m +```{literalinclude} ../../RAVEN/tutorials/tutorial4.m :language: matlab ``` diff --git a/docs/tutorials/tutorial5.md b/docs/tutorials/tutorial5.md index cd0303b..fbb4576 100644 --- a/docs/tutorials/tutorial5.md +++ b/docs/tutorials/tutorial5.md @@ -213,6 +213,6 @@ sufficient quality. ## Full script -```{literalinclude} ../../RAVEN/tutorial/tutorial5.m +```{literalinclude} ../../RAVEN/tutorials/tutorial5.m :language: matlab ``` From 21f1b6b410454724a08c791dd10422b2d6ce5177 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 08:57:50 +0200 Subject: [PATCH 24/32] feat: render numpydoc help as MyST in the API reference --- docs/conf.py | 8 +- scripts/api_index.py | 87 ++++++++++++------- scripts/gen_api_pages_sphinx.py | 147 +++++++++++++++++++++----------- 3 files changed, 161 insertions(+), 81 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index a8df462..aa77038 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,9 +2,8 @@ `.readthedocs.yaml` builds from this file. `mkdocs.yml` is retired but not yet removed; see docs/README-sphinx.md for what still needs following up -(scripts/check_names.py and scripts/run_examples.py both still assume the -old MkDocs plugin API and fence syntax) and requirements-sphinx.txt for the -Python dependencies this build needs. +(scripts/check_names.py still assumes the old MkDocs plugin API) and +requirements-sphinx.txt for the Python dependencies this build needs. """ from __future__ import annotations @@ -29,6 +28,9 @@ "dollarmath", ] myst_heading_anchors = 4 +# Straight quotes stay straight: the API pages quote MATLAB and Python +# literals such as 'rxns', which must paste back into code unchanged. +smartquotes = False exclude_patterns = [ "_build", diff --git a/scripts/api_index.py b/scripts/api_index.py index 64f0a10..f032d7f 100644 --- a/scripts/api_index.py +++ b/scripts/api_index.py @@ -1,6 +1,6 @@ """Shared index of the two toolboxes' public APIs, collected from submodule source. -Both the API-reference generator (``gen_api_pages.py``) and the name checker +Both the API-reference generator (``gen_api_pages_sphinx.py``) and the name checker (``check_names.py``) need to know which functions actually exist in RAVEN (MATLAB) and raven-toolbox (Python). Collecting that in one place means the prose can be validated against exactly the same index the reference pages are @@ -120,42 +120,69 @@ def is_matlab_function(path: Path) -> bool: return False -def matlab_summary(path: Path, fname: str) -> str: - """First descriptive line of a function's MATLAB help block.""" +def matlab_help_block(path: Path) -> tuple[str, list[str]]: + """(signature, help lines) of a MATLAB function file. + + The signature is the ``function`` line, with any ``...`` continuation + lines joined onto it. The help lines are the comment block that follows, + with the ``%`` and the one space after it removed, so the author's + indentation and blank lines are kept. + """ try: with path.open(encoding="utf-8-sig", errors="ignore") as fh: - lines = fh.readlines() + lines = fh.read().splitlines() except OSError: - return "" - + return "", [] + + i = 0 + while i < len(lines) and not lines[i].strip().startswith("function"): + i += 1 + if i == len(lines): + return "", [] + signature = [lines[i].strip()] + while signature[-1].endswith("...") and i + 1 < len(lines): + i += 1 + signature.append(lines[i].rstrip()) + i += 1 + + while i < len(lines) and not lines[i].strip(): + i += 1 help_lines: list[str] = [] - started = False - for line in lines: - stripped = line.strip() - if not started: - if stripped.startswith("function"): - started = True - continue - if stripped.startswith("%"): - help_lines.append(stripped.lstrip("%").strip()) - elif stripped == "" and not help_lines: - continue - else: - break - - cleaned = [h for h in help_lines if h] - if not cleaned: - return "" - # The first help line is often just the function name; strip it. - first = cleaned[0] + while i < len(lines) and lines[i].strip().startswith("%"): + text = lines[i].strip()[1:] + help_lines.append(text[1:] if text.startswith(" ") else text) + i += 1 + while help_lines and not help_lines[-1].strip(): + help_lines.pop() + return "\n".join(signature), help_lines + + +def matlab_summary_and_body(path: Path, fname: str) -> tuple[str, list[str]]: + """(summary, remaining help lines) of a MATLAB function. + + The first help line is often just the function name, or the name followed + by the summary; the name is stripped, and a line holding nothing else + gives way to the next one. + """ + _signature, help_lines = matlab_help_block(path) + nonblank = [n for n, line in enumerate(help_lines) if line.strip()] + if not nonblank: + return "", [] + first_idx = nonblank[0] + first = help_lines[first_idx].strip() if first.lower().startswith(fname.lower()): rest = first[len(fname):].strip(" -:\t") if rest: - return rest - if len(cleaned) > 1: - return cleaned[1] - return "" - return first + return rest, help_lines[first_idx + 1:] + if len(nonblank) > 1: + return help_lines[nonblank[1]].strip(), help_lines[nonblank[1] + 1:] + return "", [] + return first, help_lines[first_idx + 1:] + + +def matlab_summary(path: Path, fname: str) -> str: + """First descriptive line of a function's MATLAB help block.""" + return matlab_summary_and_body(path, fname)[0] def collect_matlab() -> dict[str, list[dict]]: diff --git a/scripts/gen_api_pages_sphinx.py b/scripts/gen_api_pages_sphinx.py index f92d380..87df5a9 100644 --- a/scripts/gen_api_pages_sphinx.py +++ b/scripts/gen_api_pages_sphinx.py @@ -33,56 +33,27 @@ RAVEN as MATLAB, cell, is_matlab_function, + matlab_help_block, + matlab_summary_and_body, module_dotted, norm, slug, ) import ast +import re +import textwrap # --------------------------------------------------------------------------- # # Full-text collection (api_index.py's matlab_summary / py_summary keep only # the first line; these keep everything). # # --------------------------------------------------------------------------- # -def matlab_help(path: Path, fname: str) -> tuple[str, str]: - """(summary, full help text) for one MATLAB function.""" - try: - lines = path.read_text(encoding="utf-8-sig", errors="ignore").splitlines() - except OSError: - return "", "" - - help_lines: list[str] = [] - started = False - for line in lines: - stripped = line.strip() - if not started: - if stripped.startswith("function"): - started = True - continue - if stripped.startswith("%"): - help_lines.append(stripped.lstrip("%").strip()) - elif stripped == "" and not help_lines: - continue - else: - break - - cleaned = [h for h in help_lines if h] - if not cleaned: - return "", "" - first = cleaned[0] - if first.lower().startswith(fname.lower()): - rest = first[len(fname):].strip(" -:\t") - if rest: - summary, body = rest, cleaned[1:] - elif len(cleaned) > 1: - summary, body = cleaned[1], cleaned[2:] - else: - summary, body = "", [] - else: - summary = first - body = cleaned[1:] - return summary, "\n".join(body) +def matlab_help(path: Path, fname: str) -> tuple[str, str, str]: + """(signature, summary, rest of the help text) for one MATLAB function.""" + signature, _lines = matlab_help_block(path) + summary, body = matlab_summary_and_body(path, fname) + return signature, summary, "\n".join(body) def py_help(node: ast.AST) -> tuple[str, str]: @@ -112,8 +83,11 @@ def collect_matlab_full() -> dict[str, list[dict]]: for m in base.rglob("*.m"): if m.stem == "Contents" or not is_matlab_function(m): continue - summary, body = matlab_help(m, m.stem) - funcs.setdefault(m.stem, {"name": m.stem, "summary": summary, "body": body, "path": m}) + signature, summary, body = matlab_help(m, m.stem) + funcs.setdefault( + m.stem, + {"name": m.stem, "signature": signature, "summary": summary, "body": body, "path": m}, + ) cats[folder] = [funcs[n] for n in sorted(funcs, key=str.lower)] return cats @@ -153,6 +127,83 @@ def collect_python_full() -> list[dict]: return objects +# --------------------------------------------------------------------------- # +# Help-text rendering # +# --------------------------------------------------------------------------- # +# RAVEN's % help blocks and the structured raven-toolbox docstrings both use +# numpydoc section headers ("Parameters" over a dashed underline), so one +# parser serves both. Parameter sections become definition lists, Examples a +# code block, and anything else passes through as prose. Help text without any +# section header is kept as the author laid it out: in a text block for MATLAB, +# whose help is written for the command window, and as prose for Python. +PARAM_SECTIONS = { + "Parameters", "Name-Value Arguments", "Other Parameters", "Returns", + "Yields", "Raises", "Attributes", +} +_SECTION_HEADER = re.compile(r"^([A-Z][A-Za-z -]+)\n(-{3,})$", re.M) +_RST_ROLE = re.compile(r":(?:py:)?(?:func|class|meth|attr|mod|data|obj|exc):`~?([^`]+)`") + + +def myst_inline(text: str) -> str: + """RST roles and double-backtick literals as MyST code spans.""" + text = _RST_ROLE.sub(lambda m: f"`{m.group(1)}`", text) + return re.sub(r"``([^`]+)``", r"`\1`", text) + + +def split_sections(doc: str) -> list[tuple[str | None, str]]: + parts: list[tuple[str | None, str]] = [] + pos, title = 0, None + for m in _SECTION_HEADER.finditer(doc): + if len(m.group(2)) < len(m.group(1).strip()) - 1: + continue # a dashed line too short to underline this title + parts.append((title, doc[pos:m.start()])) + title, pos = m.group(1).strip(), m.end() + parts.append((title, doc[pos:])) + return parts + + +def param_entries(body: str) -> list[tuple[str, str, str]]: + entries: list[list] = [] + for line in body.strip("\n").splitlines(): + if line and not line[0].isspace(): + name, _, typ = line.partition(" : ") + entries.append([name.strip(), typ.strip(), []]) + elif entries: + entries[-1][2].append(line) + return [(n, t, textwrap.dedent("\n".join(d)).strip()) for n, t, d in entries] + + +def render_help(doc: str, lang: str) -> list[str]: + """Markdown lines for a help text's body (everything after the summary).""" + doc = textwrap.dedent(doc).strip("\n") + if not doc.strip(): + return [] + sections = split_sections(doc) + if all(title is None for title, _ in sections): + if lang == "matlab": + return ["```text", doc, "```", ""] + return [myst_inline(doc), ""] + out: list[str] = [] + for title, body in sections: + body = textwrap.dedent(body).strip("\n") + if title is None: + if body.strip(): + out += [myst_inline(body), ""] + continue + out += [f":::{{rubric}} {title}", ":::", ""] + if title in PARAM_SECTIONS: + for name, typ, desc in param_entries(body): + term = f"`{name}`" + (f" : *{typ}*" if typ else "") + lines = myst_inline(desc).splitlines() or [""] + out += [term, f": {lines[0]}"] + [f" {line}" if line else "" for line in lines[1:]] + [""] + elif title == "Examples": + fence = "pycon" if ">>>" in body else lang + out += [f"```{fence}", body, "```", ""] + else: + out += [myst_inline(body), ""] + return out + + # --------------------------------------------------------------------------- # # Page rendering # # --------------------------------------------------------------------------- # @@ -160,7 +211,7 @@ def render_matlab_page(title: str, folder: str, funcs: list[dict]) -> str: out = [f"# {title}", "", f"MATLAB functions in `RAVEN/src/{folder}` of the RAVEN toolbox.", "", "## Functions", "", "| Function | Summary |", "|---|---|"] for f in funcs: - out.append(f"| [`{f['name']}`](#{slug(f['name'])}) | {cell(f['summary'])} |") + out.append(f"| [`{f['name']}`](#{slug(f['name'])}) | {cell(myst_inline(f['summary']))} |") out += ["", "## Reference", ""] for f in funcs: # A cross-document markdown link with a #fragment resolves against @@ -168,10 +219,11 @@ def render_matlab_page(title: str, folder: str, funcs: list[dict]) -> str: # must auto-slug to exactly `slug(name)` -- true here since it is a # bare MATLAB identifier, nothing an explicit target would improve. out += [f"### {f['name']}", ""] + if f["signature"]: + out += ["```matlab", f["signature"], "```", ""] if f["summary"]: - out += [f["summary"], ""] - if f["body"]: - out += ["```text", f["body"], "```", ""] + out += [myst_inline(f["summary"]), ""] + out += render_help(f["body"], "matlab") return "\n".join(out) + "\n" @@ -179,7 +231,7 @@ def render_python_page(title: str, dotted: str, objs: list[dict]) -> str: out = [f"# {title} (Python)", "", f"`raven-toolbox` objects in `{dotted}`.", "", "## Functions", "", "| Function | Summary |", "|---|---|"] for o in objs: - out.append(f"| [`{o['name']}`](#{slug(o['name'])}) | {cell(o['summary'])} |") + out.append(f"| [`{o['name']}`](#{slug(o['name'])}) | {cell(myst_inline(o['summary']))} |") out += ["", "## Reference", ""] for o in objs: # The heading is the bare name, not name+signature: a cross-document @@ -191,9 +243,8 @@ def render_python_page(title: str, dotted: str, objs: list[dict]) -> str: if o["signature"]: out += [f"`{o['name']}{o['signature']}`", ""] if o["summary"]: - out += [o["summary"], ""] - if o["body"]: - out += ["```text", o["body"], "```", ""] + out += [myst_inline(o["summary"]), ""] + out += render_help(o["body"], "python") return "\n".join(out) + "\n" From 28f7e160d324c103375f69a3724468fb32e6f69e Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 09:26:50 +0200 Subject: [PATCH 25/32] docs(guide): run the MATLAB MILP examples on GLPK RAVEN solves MILPs with its bundled GLPK and warns that Gurobi is faster, so the gap-filling and minimal-medium examples no longer need Gurobi. Fill in the curation outputs from the fixed curation functions. --- docs/guide/curation.md | 12 ++++++++++-- docs/guide/gap-filling.md | 23 ++++++++--------------- docs/guide/media.md | 6 +++--- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/docs/guide/curation.md b/docs/guide/curation.md index 739607f..5d28fe8 100644 --- a/docs/guide/curation.md +++ b/docs/guide/curation.md @@ -68,7 +68,12 @@ fprintf('%s charge %d, note: %s\n', curated.metNames{idx}, ... ``` ```text -... +charges before: 0 +[Warning: The following metabolites are already present in the model, their annotation will be overwritten to match the metsInfo file. If you do not particularly want to curate their annotations, it would be better to removes these metabolites from metsInfo: (S)-malate[m] + 2-oxoglutarate[m] + 2-phospho-D-glycerate[c] + 3-phospho-D-glycerate[c]] +(S)-malate charge -2, note: charge assigned at pH 7.3 ``` A model with no charges has no `metCharges` field at all, rather than a field @@ -101,7 +106,10 @@ print(f"{mal.name} charge {mal.charge}, note: {mal.notes['note']}") ``` ```text -... +charges before: 0 +charges after: 4 +updated: ['MAL_m', 'AKG_m', 'P2G_c', 'P3G_c'] +(S)-malate charge -2, note: charge assigned at pH 7.3 ``` `batch_curate_from_tsv` edits the model **in place** and returns a diff --git a/docs/guide/gap-filling.md b/docs/guide/gap-filling.md index f037e64..cf1d73d 100644 --- a/docs/guide/gap-filling.md +++ b/docs/guide/gap-filling.md @@ -114,16 +114,15 @@ connectivity. :::{tab-item} Ⓜ️ MATLAB :sync: matlab - - ```matlab -setRavenSolver('gurobi'); % fillGaps is a MILP; Gurobi is faster than GLPK +% fillGaps solves a MILP: GLPK warns and solves it, setRavenSolver('gurobi') is faster [newConnected, cannotConnect, addedRxns] = fillGaps(draft, {template}, ... 'allowNetProduction', true, 'useModelConstraints', false); fprintf('%d added: %s\n', numel(addedRxns), strjoin(addedRxns, ', ')); ``` ```text +[Warning: Solving a MILP with GLPK. This works for small problems, but for larger problems Gurobi is much faster and more reliable; select it with setRavenSolver('gurobi'). Silence this warning with warning('off','RAVEN:glpkMILP').] 1 added: ADH1 ``` ::: @@ -187,8 +186,8 @@ gapFillFastLP: 2 blocked reaction(s) found in draft model. gapFillFastLP: 1 reaction(s) cannot be rescued by the universal database. gapFillFastLP: 1/2 blocked reaction(s) are rescuable; running fast... -gapFillFastLP: added 8 reaction(s) from universal database. -8 reactions added +gapFillFastLP: added 5 reaction(s) from universal database. +5 reactions added ``` ::: :::{tab-item} 🐍 Python @@ -205,7 +204,8 @@ print(f"{len(result.added_reactions)} reactions added") 1 reactions added ``` -The two tabs add 8 and 1. They are solving the same idea with different +The two tabs add 5 and 1 (MATLAB adds 8 with Gurobi: the LP has alternative +optima, and the solver picks one). They are solving the same idea with different inputs: the MATLAB call merges the whole template into the draft first, renaming every id it already has, which is what the rename warning above lists. Its universal database therefore holds a second copy of the @@ -228,13 +228,8 @@ is slower by a wide margin, and Gurobi is the better choice. :::{tab-item} Ⓜ️ MATLAB :sync: matlab - - ```matlab -setRavenSolver('gurobi'); -params.OutputFlag = 0; % the solver log is long, and machine-specific -[addedRxns, reversedRxns, outModel] = gapFillMILP(draft, template, ... - 'params', params); +[addedRxns, reversedRxns, outModel] = gapFillMILP(draft, template); fprintf('%d reactions added\n', numel(addedRxns)); ``` @@ -281,14 +276,12 @@ growth needs, such as hexokinase: :::{tab-item} Ⓜ️ MATLAB :sync: matlab - - ```matlab draft2 = removeReactions(template, {'HXK'}); % hexokinase draft2.id = 'draft2'; sol = solveLP(draft2); fprintf('draft growth %.4f\n', sol.f); -[addedRxns, reversedRxns] = gapFillMILP(draft2, template, 'params', params); +[addedRxns, reversedRxns] = gapFillMILP(draft2, template); fprintf('added: %s; reversed: %s\n', strjoin(addedRxns, ', '), strjoin(reversedRxns, ', ')); ``` diff --git a/docs/guide/media.md b/docs/guide/media.md index 8f2d3aa..d04f0d0 100644 --- a/docs/guide/media.md +++ b/docs/guide/media.md @@ -355,15 +355,15 @@ gap-filled reaction produces it internally. :::{tab-item} Ⓜ️ MATLAB :sync: matlab - - ```matlab -setRavenSolver('gurobi'); % getMinimalMedium solves a MILP; Gurobi is faster than GLPK +% getMinimalMedium solves a MILP: GLPK warns and solves it, setRavenSolver('gurobi') is faster sol = solveLP(model); medium = getMinimalMedium(model, 'minGrowth', 0.9 * sol.f); ``` ```text +[Warning: Solving a MILP with GLPK. This works for small problems, but for larger problems Gurobi is much faster and more reliable; select it with setRavenSolver('gurobi'). Silence this warning with warning('off','RAVEN:glpkMILP').] + ============================================================== Minimal medium (14 of 16 candidate uptake reactions) Target growth: 0.7533 From 2bb7201fade2ecd7ea3f56b2a45030a362b7f4ef Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 09:26:56 +0200 Subject: [PATCH 26/32] docs(guide): follow the corrected curateModelFromTables warning --- docs/guide/curation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/guide/curation.md b/docs/guide/curation.md index 5d28fe8..3c65e54 100644 --- a/docs/guide/curation.md +++ b/docs/guide/curation.md @@ -69,7 +69,7 @@ fprintf('%s charge %d, note: %s\n', curated.metNames{idx}, ... ```text charges before: 0 -[Warning: The following metabolites are already present in the model, their annotation will be overwritten to match the metsInfo file. If you do not particularly want to curate their annotations, it would be better to removes these metabolites from metsInfo: (S)-malate[m] +[Warning: The following metabolites are already present in the model, their annotation will be overwritten to match the metsInfo file. If you do not particularly want to curate their annotations, it would be better to remove these metabolites from metsInfo: (S)-malate[m] 2-oxoglutarate[m] 2-phospho-D-glycerate[c] 3-phospho-D-glycerate[c]] From 28e024aa111aab7221ed90f481d110abac338e83 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 13:25:26 +0200 Subject: [PATCH 27/32] Gap-filling MILP without reversal; gap_report infeasibility error --- docs/guide/gap-filling.md | 47 ++++++++++--------- docs/guide/quality-control.md | 5 +- docs/parameter-tuning/benchmarks.md | 1 - .../same-function-different-answer.md | 2 +- docs/tuned-parameters.md | 11 ++--- 5 files changed, 32 insertions(+), 34 deletions(-) diff --git a/docs/guide/gap-filling.md b/docs/guide/gap-filling.md index cf1d73d..6a4b500 100644 --- a/docs/guide/gap-filling.md +++ b/docs/guide/gap-filling.md @@ -229,7 +229,7 @@ is slower by a wide margin, and Gurobi is the better choice. :sync: matlab ```matlab -[addedRxns, reversedRxns, outModel] = gapFillMILP(draft, template); +[addedRxns, outModel] = gapFillMILP(draft, template); fprintf('%d reactions added\n', numel(addedRxns)); ``` @@ -237,10 +237,11 @@ fprintf('%d reactions added\n', numel(addedRxns)); [Warning: The following reaction IDs in smallYeast are already present in the model and were renamed: Old: acOUT New: acOUT_smallYeast Old: biomassOUT New: biomassOUT_smallYeast Old: co2OUT New: co2OUT_smallYeast Old: ethOUT New: ethOUT_smallYeast Old: glyOUT New: glyOUT_smallYeast Old: glcIN New: glcIN_smallYeast Old: o2IN New: o2IN_smallYeast Old: ethIN New: ethIN_smallYeast Old: HXK New: HXK_smallYeast ...and 43 more] gapFillMILP: merged model has 52 mets, 105 rxns (52 draft, 52 universal). -gapFillMILP: 35 reversal candidates, 52 database candidates. +gapFillMILP: 52 database candidates. gapFillMILP: setting minGrowth = 0.02443 (10% of max 0.2443). -gapFillMILP: solving MILP (192 vars, 87 binary, 192 constraints)... -gapFillMILP: reversed 0 draft reaction(s), added 0 universal reaction(s). +gapFillMILP: solving MILP (157 vars, 52 binary, 157 constraints)... +[Warning: Solving a MILP with GLPK. This works for small problems, but for larger problems Gurobi is much faster and more reliable; select it with setRavenSolver('gurobi'). Silence this warning with warning('off','RAVEN:glpkMILP').] +gapFillMILP: added 0 universal reaction(s). 0 reactions added ``` ::: @@ -257,8 +258,8 @@ print(f"{len(result.added_reactions)} reactions added") ```text fill_gaps_kumar_milp: merged model has 53 reactions (52 draft, 1 template). fill_gaps_kumar_milp: setting min_growth = 0.01222 (10% of FBA max 0.1222). -fill_gaps_kumar_milp: solving MILP (35 reversal, 1 database candidates)... -fill_gaps_kumar_milp: reversed 0 reaction(s), added 0 template reaction(s). +fill_gaps_kumar_milp: solving MILP (1 database candidates)... +fill_gaps_kumar_milp: added 0 template reaction(s). 0 reactions added ``` ::: @@ -281,8 +282,8 @@ draft2 = removeReactions(template, {'HXK'}); % hexokinase draft2.id = 'draft2'; sol = solveLP(draft2); fprintf('draft growth %.4f\n', sol.f); -[addedRxns, reversedRxns] = gapFillMILP(draft2, template); -fprintf('added: %s; reversed: %s\n', strjoin(addedRxns, ', '), strjoin(reversedRxns, ', ')); +addedRxns = gapFillMILP(draft2, template); +fprintf('added: %s\n', strjoin(addedRxns, ', ')); ``` ```text @@ -290,11 +291,12 @@ draft growth -0.0000 [Warning: The following reaction IDs in smallYeast are already present in the model and were renamed: Old: acOUT New: acOUT_smallYeast Old: biomassOUT New: biomassOUT_smallYeast Old: co2OUT New: co2OUT_smallYeast Old: ethOUT New: ethOUT_smallYeast Old: glyOUT New: glyOUT_smallYeast Old: glcIN New: glcIN_smallYeast Old: o2IN New: o2IN_smallYeast Old: ethIN New: ethIN_smallYeast Old: PGI New: PGI_smallYeast ...and 43 more] gapFillMILP: merged model has 52 mets, 105 rxns (52 draft, 52 universal). -gapFillMILP: 34 reversal candidates, 52 database candidates. +gapFillMILP: 52 database candidates. gapFillMILP: setting minGrowth = 0.02443 (10% of max 0.2443). -gapFillMILP: solving MILP (191 vars, 86 binary, 191 constraints)... -gapFillMILP: reversed 0 draft reaction(s), added 1 universal reaction(s). -added: HXK; reversed: +gapFillMILP: solving MILP (157 vars, 52 binary, 157 constraints)... +[Warning: Solving a MILP with GLPK. This works for small problems, but for larger problems Gurobi is much faster and more reliable; select it with setRavenSolver('gurobi'). Silence this warning with warning('off','RAVEN:glpkMILP').] +gapFillMILP: added 1 universal reaction(s). +added: HXK ``` ::: :::{tab-item} 🐍 Python @@ -305,24 +307,23 @@ draft2 = template.copy() draft2.remove_reactions([draft2.reactions.get_by_id("HXK")]) # hexokinase print(f"draft growth {draft2.slim_optimize():.4f}") result = fill_gaps_kumar_milp(draft2, template, verbose=False) -print(f"added: {result.added_reactions}; reversed: {result.reversed_reactions}") +print(f"added: {result.added_reactions}") print(f"filled growth {result.model.slim_optimize():.4f}") ``` ```text draft growth 0.0000 -added: ['HXK']; reversed: [] +added: ['HXK'] filled growth 0.1222 ``` ::: :::: -Hexokinase comes back, and nothing is reversed. That is not guaranteed: by -default a reversal costs half as much as an addition, and every irreversible -draft reaction is a reversal candidate, boundary reactions and the ATP drain -included. Removing `PFK` instead makes the MILP reverse `FBP` rather than add -`PFK` back, and for an anaerobic draft without `ADH1` it reverses `ATPX`, which -makes ATP from nothing. Read the reversed reactions before accepting a fill. +Hexokinase comes back. Both functions only add template reactions: the +draft's own reactions keep their bounds, forced fluxes such as ATP maintenance +included. A draft reaction that runs in the wrong direction is therefore not +reversed to close a gap; it is a curation finding, so correct its bounds in the +draft (see [8. Editing an existing model](editing.md)) and fill again. ## 17.5 Filling towards a task, not a hole @@ -380,9 +381,9 @@ already contains the draft, which is the form ftINIT passes. misleading later. - **MILP on a genome-scale model.** Expect it to be slow, on GLPK most of all, and give it a time limit; see [6. Solvers and configuration](solvers.md). -- **Accepting reversals unread.** `gapFillMILP` and `fill_gaps_kumar_milp` - prefer reversing a draft reaction to adding one, and a reversed exchange or - ATP drain is a leak, not a repair. +- **A draft reaction in the wrong direction.** `gapFillMILP` and + `fill_gaps_kumar_milp` do not change the draft's own reactions, so a wrong + direction shows up as a fill that fails or adds a detour around it. - **Comparing added-reaction counts across toolboxes.** The two sides differ in what they treat as a candidate, so the counts are only comparable when the template genuinely holds reactions the draft lacks. diff --git a/docs/guide/quality-control.md b/docs/guide/quality-control.md index 8a7b950..fc760ed 100644 --- a/docs/guide/quality-control.md +++ b/docs/guide/quality-control.md @@ -291,7 +291,10 @@ one leak: ['F6P_c', 'GP_c', 'PYR_c'] `gap_report` closes every boundary reaction itself, adds an output for every metabolite, and lists in `can_produce_without_input` the ones whose output can -carry flux, as `canExchange` does. `find_leak_metabolite` is the port of +carry flux, as `canExchange` does. If the model is infeasible under its own +bounds, or once its boundary is closed, it stops with an error that names the +reactions with a forced flux; set ATP maintenance to zero first. `gapReport` +does the same. `find_leak_metabolite` is the port of `findLeakMetabolite` and does not close anything, so give it the closed copy. It returns a list: more than one id means those metabolites could only be exchanged together. diff --git a/docs/parameter-tuning/benchmarks.md b/docs/parameter-tuning/benchmarks.md index 304da8e..a20d44f 100644 --- a/docs/parameter-tuning/benchmarks.md +++ b/docs/parameter-tuning/benchmarks.md @@ -71,6 +71,5 @@ is no longer taken on trust. | `connect_blocked_reactions.eps` | `1.0` | Correct for RAVEN-convention models, where exchanges carry ±1000 bounds | | `connect_blocked_reactions.allow_net_production` | `False` | The stricter question, and the one the function exists to ask | | `fill_gaps_fast_lp.epsilon` | `1e-4` | The fastGapFill paper's value | -| `fill_gaps_kumar_milp.weights` | `(1.0, 2.0)` | Implements Kumar 2007's preference for reversal over addition | | `fill_gaps_kumar_milp.big_m` | `1000.0` | Matches RAVEN-convention bounds | | `check_tasks.close_boundaries` | `False` | Matches RAVEN's `checkTasks`, which published task lists were curated against; `True` closes the model's own exchanges so that the task is the whole boundary | diff --git a/docs/raven3-vs-raven-toolbox/same-function-different-answer.md b/docs/raven3-vs-raven-toolbox/same-function-different-answer.md index 5147d16..8c5eca3 100644 --- a/docs/raven3-vs-raven-toolbox/same-function-different-answer.md +++ b/docs/raven3-vs-raven-toolbox/same-function-different-answer.md @@ -47,7 +47,7 @@ raven-toolbox splits them, so porting a `fillGaps` call means choosing: |---|---| | Connecting blocked reactions against template models | `connect_blocked_reactions` | | Fast LP-based filling of a large candidate set | `fill_gaps_fast_lp` | -| MILP filling with explicit weights | `fill_gaps_kumar_milp` | +| MILP filling with the fewest added reactions that restore growth | `fill_gaps_kumar_milp` | | Only *finding* the gaps (`canExchange`, `checkProduction`, `getAllSubGraphs`, `haveFlux`) | `analyse_topology` | The choice changes both the reaction set added and the runtime, because each diff --git a/docs/tuned-parameters.md b/docs/tuned-parameters.md index 7d5cc17..60ef64e 100644 --- a/docs/tuned-parameters.md +++ b/docs/tuned-parameters.md @@ -98,16 +98,15 @@ essential-gene drift, measured) · ## Gap-filling RAVEN MATLAB: `fillGaps` (connectivity mode), ported as `connect_blocked_reactions`. -`fill_gaps_fast_lp` and `fill_gaps_kumar_milp` are literature-based -methods added to the Python implementation, with no MATLAB RAVEN counterpart. +`fill_gaps_fast_lp` and `fill_gaps_kumar_milp` are the counterparts of RAVEN's +`gapFillFastLP` and `gapFillMILP`. | Parameter | Default | How determined | |---|---|---| | `epsilon` (in `fill_gaps_fast_lp`) | `0.0001` | Literature: [Thiele et al. 2014](https://doi.org/10.1093/bioinformatics/btu321) (the fastGapFill paper). | | `eps` (in `connect_blocked_reactions`) | `1.0` | Measured on a synthetic supply-limited model; reliable noise filter for RAVEN-convention (±1000-bound) models; lower it for tightly-constrained exchanges. | | `penalty`, `allow_net_production` (in `connect_blocked_reactions`) | `1.0`, `False` | Measured / standard practice for reconstruction (vs. curation) use. | -| `weights` (in `fill_gaps_kumar_milp`) | `(1.0, 2.0)` | Literature: [Kumar et al. 2007](https://doi.org/10.1186/1471-2105-8-212); measured: confirms reversal is preferred over adding a new reaction at the intended 2:1 ratio. | -| `big_m` (in `fill_gaps_kumar_milp`) | `1000.0` | Measured: matches RAVEN's ±1000 bound convention; a smaller value artificially caps reversed-reaction flux. | +| `big_m` (in `fill_gaps_kumar_milp`) | `1000.0` | Matches RAVEN's ±1000 bound convention. It caps an unbounded template reaction only when the merged model has no finite bound; otherwise the largest finite bound does. | **Full detail:** [Parameter benchmarks](parameter-tuning/benchmarks.md) @@ -335,10 +334,6 @@ cutoff), different algorithms, not a value to port. See the "Keep different" tab > single-cell RNA-Seq data.** *Proc Natl Acad Sci USA* 120(6):e2217868120. > -> Kumar VS, Dasika MS, Maranas CD (2007). **Optimization based automated -> curation of metabolic reconstructions.** *BMC Bioinformatics* 8:212. -> - > Thiele I, Vlassis N, Fleming RMT (2014). **fastGapFill: efficient gap > filling in metabolic networks.** *Bioinformatics* 30(17):2529–2531. > From 43f098b85ac98ac2abe4febe765c3784f38869fc Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 21:27:10 +0200 Subject: [PATCH 28/32] fill_gaps_kumar_milp is now fill_gaps_milp --- docs/guide/gap-filling.md | 18 +++++++++--------- docs/parameter-tuning/benchmarks.md | 2 +- .../same-function-different-answer.md | 2 +- docs/tuned-parameters.md | 4 ++-- scripts/curated_pairs.yml | 2 +- scripts/known_names.txt | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/guide/gap-filling.md b/docs/guide/gap-filling.md index 6a4b500..7f360b3 100644 --- a/docs/guide/gap-filling.md +++ b/docs/guide/gap-filling.md @@ -12,7 +12,7 @@ holes close, and the judgement is in deciding which holes are worth closing. | `gapReport` | `check_model`, `analyse_topology` | find the holes first | | `fillGaps` | `connect_blocked_reactions` | add template reactions so blocked reactions can carry flux | | `gapFillFastLP` | `fill_gaps_fast_lp` | the LP formulation (fastGapFill / swiftGapFill) | -| `gapFillMILP` | `fill_gaps_kumar_milp` | the MILP formulation, when the LP is not enough | +| `gapFillMILP` | `fill_gaps_milp` | the MILP formulation, when the LP is not enough | | `gapFillTopological` | `analyse_topology` | connectivity, without solving an LP | | `fitTasks` | `fill_tasks` | fill until a task passes; see [15. Metabolic tasks](tasks.md) | @@ -249,17 +249,17 @@ gapFillMILP: added 0 universal reaction(s). :sync: python ```python -from raven_toolbox.gapfilling import fill_gaps_kumar_milp +from raven_toolbox.gapfilling import fill_gaps_milp -result = fill_gaps_kumar_milp(draft, template) +result = fill_gaps_milp(draft, template) print(f"{len(result.added_reactions)} reactions added") ``` ```text -fill_gaps_kumar_milp: merged model has 53 reactions (52 draft, 1 template). -fill_gaps_kumar_milp: setting min_growth = 0.01222 (10% of FBA max 0.1222). -fill_gaps_kumar_milp: solving MILP (1 database candidates)... -fill_gaps_kumar_milp: added 0 template reaction(s). +fill_gaps_milp: merged model has 53 reactions (52 draft, 1 template). +fill_gaps_milp: setting min_growth = 0.01222 (10% of FBA max 0.1222). +fill_gaps_milp: solving MILP (1 database candidates)... +fill_gaps_milp: added 0 template reaction(s). 0 reactions added ``` ::: @@ -306,7 +306,7 @@ added: HXK draft2 = template.copy() draft2.remove_reactions([draft2.reactions.get_by_id("HXK")]) # hexokinase print(f"draft growth {draft2.slim_optimize():.4f}") -result = fill_gaps_kumar_milp(draft2, template, verbose=False) +result = fill_gaps_milp(draft2, template, verbose=False) print(f"added: {result.added_reactions}") print(f"filled growth {result.model.slim_optimize():.4f}") ``` @@ -382,7 +382,7 @@ already contains the draft, which is the form ftINIT passes. - **MILP on a genome-scale model.** Expect it to be slow, on GLPK most of all, and give it a time limit; see [6. Solvers and configuration](solvers.md). - **A draft reaction in the wrong direction.** `gapFillMILP` and - `fill_gaps_kumar_milp` do not change the draft's own reactions, so a wrong + `fill_gaps_milp` do not change the draft's own reactions, so a wrong direction shows up as a fill that fails or adds a detour around it. - **Comparing added-reaction counts across toolboxes.** The two sides differ in what they treat as a candidate, so the counts are only diff --git a/docs/parameter-tuning/benchmarks.md b/docs/parameter-tuning/benchmarks.md index a20d44f..abd2271 100644 --- a/docs/parameter-tuning/benchmarks.md +++ b/docs/parameter-tuning/benchmarks.md @@ -71,5 +71,5 @@ is no longer taken on trust. | `connect_blocked_reactions.eps` | `1.0` | Correct for RAVEN-convention models, where exchanges carry ±1000 bounds | | `connect_blocked_reactions.allow_net_production` | `False` | The stricter question, and the one the function exists to ask | | `fill_gaps_fast_lp.epsilon` | `1e-4` | The fastGapFill paper's value | -| `fill_gaps_kumar_milp.big_m` | `1000.0` | Matches RAVEN-convention bounds | +| `fill_gaps_milp.big_m` | `1000.0` | Matches RAVEN-convention bounds | | `check_tasks.close_boundaries` | `False` | Matches RAVEN's `checkTasks`, which published task lists were curated against; `True` closes the model's own exchanges so that the task is the whole boundary | diff --git a/docs/raven3-vs-raven-toolbox/same-function-different-answer.md b/docs/raven3-vs-raven-toolbox/same-function-different-answer.md index 8c5eca3..d996c59 100644 --- a/docs/raven3-vs-raven-toolbox/same-function-different-answer.md +++ b/docs/raven3-vs-raven-toolbox/same-function-different-answer.md @@ -47,7 +47,7 @@ raven-toolbox splits them, so porting a `fillGaps` call means choosing: |---|---| | Connecting blocked reactions against template models | `connect_blocked_reactions` | | Fast LP-based filling of a large candidate set | `fill_gaps_fast_lp` | -| MILP filling with the fewest added reactions that restore growth | `fill_gaps_kumar_milp` | +| MILP filling with the fewest added reactions that restore growth | `fill_gaps_milp` | | Only *finding* the gaps (`canExchange`, `checkProduction`, `getAllSubGraphs`, `haveFlux`) | `analyse_topology` | The choice changes both the reaction set added and the runtime, because each diff --git a/docs/tuned-parameters.md b/docs/tuned-parameters.md index 60ef64e..e125fb0 100644 --- a/docs/tuned-parameters.md +++ b/docs/tuned-parameters.md @@ -98,7 +98,7 @@ essential-gene drift, measured) · ## Gap-filling RAVEN MATLAB: `fillGaps` (connectivity mode), ported as `connect_blocked_reactions`. -`fill_gaps_fast_lp` and `fill_gaps_kumar_milp` are the counterparts of RAVEN's +`fill_gaps_fast_lp` and `fill_gaps_milp` are the counterparts of RAVEN's `gapFillFastLP` and `gapFillMILP`. | Parameter | Default | How determined | @@ -106,7 +106,7 @@ RAVEN MATLAB: `fillGaps` (connectivity mode), ported as `connect_blocked_reactio | `epsilon` (in `fill_gaps_fast_lp`) | `0.0001` | Literature: [Thiele et al. 2014](https://doi.org/10.1093/bioinformatics/btu321) (the fastGapFill paper). | | `eps` (in `connect_blocked_reactions`) | `1.0` | Measured on a synthetic supply-limited model; reliable noise filter for RAVEN-convention (±1000-bound) models; lower it for tightly-constrained exchanges. | | `penalty`, `allow_net_production` (in `connect_blocked_reactions`) | `1.0`, `False` | Measured / standard practice for reconstruction (vs. curation) use. | -| `big_m` (in `fill_gaps_kumar_milp`) | `1000.0` | Matches RAVEN's ±1000 bound convention. It caps an unbounded template reaction only when the merged model has no finite bound; otherwise the largest finite bound does. | +| `big_m` (in `fill_gaps_milp`) | `1000.0` | Matches RAVEN's ±1000 bound convention. It caps an unbounded template reaction only when the merged model has no finite bound; otherwise the largest finite bound does. | **Full detail:** [Parameter benchmarks](parameter-tuning/benchmarks.md) diff --git a/scripts/curated_pairs.yml b/scripts/curated_pairs.yml index 53ce8fc..b80b721 100644 --- a/scripts/curated_pairs.yml +++ b/scripts/curated_pairs.yml @@ -78,7 +78,7 @@ aliases: getEssentialRxns: find_task_essential_reactions deleteUnusedGenes: remove_genes gapFillFastLP: fill_gaps_fast_lp - gapFillMILP: fill_gaps_kumar_milp + gapFillMILP: fill_gaps_milp gapFillTopological: analyse_topology makeFakeBlastStructure: make_ortholog_hits getModelFromKEGG: build_reference_model diff --git a/scripts/known_names.txt b/scripts/known_names.txt index 630c8b9..630e369 100644 --- a/scripts/known_names.txt +++ b/scripts/known_names.txt @@ -184,7 +184,7 @@ prod_weight # run_init / get_init_model argument allow_excretion # run_init / get_init_model / run_ftinit argument mip_gap # run_init / run_ftinit / predict_localization argument force_on # run_ftinit argument -big_m # run_ftinit / fill_gaps_kumar_milp argument +big_m # run_ftinit / fill_gaps_milp argument max_score # gene_scores_from_expression argument min_score # gene_scores_from_expression argument min_identity # get_model_from_homology argument From d6359b94285cc60acb4ead6cee48475795c86081 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 22:35:43 +0200 Subject: [PATCH 29/32] docs: describe RAVEN 3 and its develop branch, not develop3 --- .github/workflows/update-submodules.yml | 2 +- DESIGN.md | 13 ++++++------- docs/guide/sampling.md | 5 ----- docs/installation/raven.md | 12 +++++------- docs/raven3-migration.md | 7 +------ docs/raven3-migration/structure-and-renames.md | 8 +++----- docs/stylesheets/extra.css | 2 +- docs/tutorials/tutorial2.md | 3 +-- scripts/build_hooks.py | 2 +- scripts/known_names.txt | 14 ++++++-------- 10 files changed, 25 insertions(+), 43 deletions(-) diff --git a/.github/workflows/update-submodules.yml b/.github/workflows/update-submodules.yml index 6c9744f..bbbedbe 100644 --- a/.github/workflows/update-submodules.yml +++ b/.github/workflows/update-submodules.yml @@ -28,7 +28,7 @@ jobs: # it out at its tip. Done explicitly because the submodules are # `shallow = true`, so only the pinned commit is present and # `git submodule update --remote` can't resolve the remote branch ref. - # (RAVEN -> develop3, raven-toolbox -> develop, hanpo-GEM -> main) + # (RAVEN -> develop, raven-toolbox -> develop, hanpo-GEM -> main) git submodule sync --recursive git submodule foreach ' branch=$(git config -f "$toplevel/.gitmodules" "submodule.$name.branch" || echo main) diff --git a/DESIGN.md b/DESIGN.md index f6dcade..4f6c8a9 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -37,7 +37,7 @@ the Docs. function help in each toolbox's source (via `mkdocstrings`), so it never drifts from the code. The source repos are tracked as **git submodules**. - **Source of truth follows the active branch.** The RAVEN submodule tracks - **`develop3`** and raven-toolbox tracks **`develop`**. develop3's modular + **`develop`** and raven-toolbox tracks **`develop`**. RAVEN's modular folder layout (`reconstruction`, `manipulation`, `analysis`, …) mirrors raven-toolbox's package structure, so the two line up by category. - **Don't reinvent cobrapy.** Where the Python version intentionally relies on @@ -61,7 +61,7 @@ the Docs. - The API reference is **generated at build time** (`mkdocs-gen-files` + `mkdocs-literate-nav`): a cobrapy-style autoapi, one page per category, with MATLAB and Python paired by name. -- Submodules: `RAVEN` (branch `develop3`), **raven-toolbox** (branch +- Submodules: `RAVEN` (branch `develop`), **raven-toolbox** (branch `develop`), `hanpo-GEM` (branch `main`). ## 4. Site structure @@ -83,10 +83,9 @@ the Docs. An **Installation** section with an overview page and a **separate page per version**: -- **RAVEN (MATLAB).** Requirements, then the **three install methods** from the - [wiki](https://github.com/SysBioChalmers/RAVEN/wiki/Installation), the +- **RAVEN (MATLAB).** Requirements, then the **three install methods**: the MATLAB Add-Ons manager, a release download, and `git clone`, plus - verifying (`checkInstallation`), **upgrading** (per method) and **removing** + verifying (`checkRaven`), **upgrading** (per method) and **removing** (`removeRavenFromPath`). - **Python (raven-toolbox).** Requirements, `pip install raven-toolbox`, development install, verifying, **upgrading** and **removing**, and solver @@ -221,7 +220,7 @@ Generated from source, organised so each function is shown for both languages. and a function page like . -- **MATLAB**: functions organised **by subfolder** (the `develop3` modular +- **MATLAB**: functions organised **by subfolder** (RAVEN's modular categories: `reconstruction`, `manipulation`, `analysis`, …). ## 5. Conventions @@ -247,7 +246,7 @@ into the relevant section above.)* *(Append decisions here as they are made.)* - Build stack: MkDocs + Material + `mkdocstrings` (Python + MATLAB handlers). -- Source via git submodules: RAVEN tracks `develop3`, **raven-toolbox tracks +- Source via git submodules: RAVEN tracks `develop`, **raven-toolbox tracks `develop`**, hanpo-GEM tracks `main`. - The Python package is named **raven-toolbox**. - Function help reformatted to NumPy-style docstrings (renders as tables). diff --git a/docs/guide/sampling.md b/docs/guide/sampling.md index a190625..2a1be0a 100644 --- a/docs/guide/sampling.md +++ b/docs/guide/sampling.md @@ -186,11 +186,6 @@ fprintf('chrr: %d dimensions, MVE converged: %d, %d fixed\n', ... chrr: 9 dimensions, MVE converged: 0, 1 fixed ``` -On RAVEN before [#696](https://github.com/SysBioChalmers/RAVEN/pull/696) this -call failed on `smallYeast` with `sampleChebyshevCenter: LP infeasible - flux -polytope has empty interior`. The polytope was fine; the equality system is -square and rank-deficient there, and the particular solution came back `NaN`. -If you see that error, update RAVEN. ::: :::{tab-item} 🐍 Python :sync: python diff --git a/docs/installation/raven.md b/docs/installation/raven.md index 176bd6e..67728b1 100644 --- a/docs/installation/raven.md +++ b/docs/installation/raven.md @@ -202,17 +202,15 @@ release, and run `checkRaven`. :::{tab-item} {octicon}`git-branch;1em` Clone with git The `3.0.0b1` tag does not move, so there is nothing to pull while staying on -it. To move past this site's documented snapshot onto the actively-developed -branch: +it. To follow the development branch instead: ```bash -git checkout develop3 -git pull origin develop3 +git checkout develop +git pull origin develop ``` -`develop3` can be ahead of what this page describes; see -[RAVEN 2 to RAVEN 3](../raven3-migration.md) for what changed and when this -document was last checked against it. Then run `checkRaven`. +`develop` can contain changes that are not yet released. Then run +`checkRaven`. ::: :::: diff --git a/docs/raven3-migration.md b/docs/raven3-migration.md index b8e592a..285c38e 100644 --- a/docs/raven3-migration.md +++ b/docs/raven3-migration.md @@ -17,12 +17,7 @@ for existing RAVEN 2 users who need to know what will break in their scripts, what moved, what was removed outright, and what new capabilities are available. -**Versions covered:** the latest released RAVEN 2 is **2.11.3**. RAVEN 3 is -unreleased; it is being developed on the `develop3` branch and is targeting -version **3.0.0**. This guide compares `develop3` against RAVEN 2's `develop` -branch (functionally identical to the 2.11.3 release; the commits between -them are only version-bump/tag merges). **This document will be frozen once -3.0.0 is tagged**; until then, `develop3` can still change. +**Versions covered:** RAVEN 2 as of its last release, **2.11.3**, and RAVEN 3. RAVEN 3 is a deliberate breaking release: the codebase was reorganized, a large number of low-value or duplicate functions were removed, several external diff --git a/docs/raven3-migration/structure-and-renames.md b/docs/raven3-migration/structure-and-renames.md index 6a3b87e..2677b1b 100644 --- a/docs/raven3-migration/structure-and-renames.md +++ b/docs/raven3-migration/structure-and-renames.md @@ -46,11 +46,9 @@ hard-coded a path to a specific `.m` file, (b) called one of the functions that was actually removed or merged (see below), or (c) shadowed a RAVEN function name with your own; folder names are irrelevant otherwise. -The full moved-file mapping (RAVEN 2 path → RAVEN 3 path) for every relocated -function is available on request / in the repository's git history -(`git diff --stat -M develop...develop3`); it is not reproduced in full here -because it is large and, as the point above explains, rarely something you -need to act on. +The full moved-file mapping (RAVEN 2 path → RAVEN 3 path) is not reproduced +here because it is large and, as the point above explains, rarely something +you need to act on. --- diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 8ae9997..9901251 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -1,6 +1,6 @@ /* ===== RAVEN brand palette (navy) ===================================== */ :root { - /* Sampled from RAVEN.png on RAVEN's develop3 branch -- the bird and + /* Sampled from RAVEN.png in the RAVEN repository -- the bird and wordmark are one flat navy, RGB(12, 29, 56). The site icons in docs/assets are cut from that same file; shades below are HSL adjustments of it, not eyeballed. */ diff --git a/docs/tutorials/tutorial2.md b/docs/tutorials/tutorial2.md index 4a833d4..a0e186c 100644 --- a/docs/tutorials/tutorial2.md +++ b/docs/tutorials/tutorial2.md @@ -74,8 +74,7 @@ printFluxes(smallModel, sol.x, true); % printFluxes(smallModel, sol.x, false, 10^-5, [], '%rxnID (%rxnName):\n\t%eqn\n\t%flux\n'); % all fluxes ``` -See [Tutorial 2 on the RAVEN wiki](https://github.com/SysBioChalmers/RAVEN/wiki/Tutorials) -for more details and the worked answer. +The worked answer is `tutorial2_solutions.m` in RAVEN's `tutorials` folder. ## Full script diff --git a/scripts/build_hooks.py b/scripts/build_hooks.py index 5812c6a..565f2d7 100644 --- a/scripts/build_hooks.py +++ b/scripts/build_hooks.py @@ -1,7 +1,7 @@ """MkDocs build hooks that keep the site resilient to submodule restructuring. raven-docs documents the RAVEN (MATLAB) and raven-toolbox sources tracked as git -submodules on their fast-moving dev branches (``develop3`` / ``develop``). When +submodules on their fast-moving ``develop`` branches. When those branches add or remove folders or files, a statically-configured build breaks. These hooks make the build adapt automatically instead of aborting: diff --git a/scripts/known_names.txt b/scripts/known_names.txt index 630e369..311f27f 100644 --- a/scripts/known_names.txt +++ b/scripts/known_names.txt @@ -290,7 +290,7 @@ addLipidReactions # SLIMEr addSLIMEreactions # SLIMEr # -- functions of released RAVEN 2.x, referenced as history ----------------- -# These no longer exist on develop3 (the branch this site documents) but are +# These do not exist in RAVEN 3 (the version this site documents) but are # named in the pages precisely to say so. Do not add a name here just because # the checker flagged it -- confirm first that the page discusses it as removed. importExcelModel # removed in RAVEN/#631 @@ -350,9 +350,9 @@ get_init_model # removed from raven-toolbox in #148, with tINIT # -- RAVEN 2 names, referenced by the RAVEN 2 -> 3 migration guide ----------- # raven3-migration.md exists to say what RAVEN 3 removed, renamed and merged, -# so it necessarily names functions that are gone from develop3. Every entry +# so it necessarily names functions that are gone from RAVEN 3. Every entry # below was confirmed present on RAVEN 2's develop branch (== v2.11.3) and -# absent from develop3; the page discusses each one as removed or merged. +# absent from RAVEN 3; the page discusses each one as removed or merged. addSpontaneousRxns # external/metacyc, dropped with MetaCyc support changeGeneAssoc # legacy/, dropped colorPathway # pathway/, dropped with visualization @@ -391,8 +391,6 @@ kegg118_eukaryotes # raven-data KEGG artefact name, not a function kegg118_prokaryotes # raven-data KEGG artefact name, not a function xml_toolbox # third-party library bundled by RAVEN 2's legacy/ -# Pending on develop3: checkRaven is the rename of checkInstallation, landing -# with the 3.0.0b1 changelog work. It is not on develop3 yet, so the index -# cannot resolve it. DELETE THIS ENTRY once develop3 carries checkRaven.m -- -# the checker should be the thing that tells us the rename has landed. -checkRaven # RAVEN 3 rename of checkInstallation, not yet on develop3 +# checkRaven.m sits at the RAVEN root, outside src/, which the index reads. +checkRaven # RAVEN 3 installation check + From 19a3904198bde7fe316ef48cd78dd0ea601aed67 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 23:25:33 +0200 Subject: [PATCH 30/32] Remove the MkDocs build files The site is built with Sphinx. mkdocs.yml, its hooks, the mkdocs-gen-files generator, the Material icons and stylesheet, and the one-time migration scripts go; requirements-sphinx.txt and gen_api_pages_sphinx.py take the plain names, and check_names.py is a standalone check. --- .gitignore | 8 +- .readthedocs.yaml | 2 +- DESIGN.md | 35 +- README.md | 4 +- docs/README-sphinx.md | 155 -------- docs/api/index.md | 4 - docs/assets/raven-icon-white.png | Bin 5814 -> 0 bytes docs/conf.py | 18 +- docs/guide/index.md | 4 - docs/installation/index.md | 4 - docs/migrate.md | 4 - docs/parameter-tuning/index.md | 4 - docs/protocol/index.md | 4 - docs/stylesheets/extra.css | 206 ---------- docs/tutorials/index.md | 4 - mkdocs.yml | 225 ----------- overrides/.icons/custom/matlab.svg | 6 - requirements-sphinx.txt | 6 - requirements.txt | 14 +- scripts/add_tab_icons_sync.py | 49 --- scripts/api_index.py | 6 +- scripts/build_hooks.py | 114 ------ scripts/check_names.py | 48 +-- scripts/gen_api_pages.py | 578 +++++++++++++++++------------ scripts/gen_api_pages_sphinx.py | 392 ------------------- scripts/migrate_to_myst.py | 180 --------- 26 files changed, 388 insertions(+), 1686 deletions(-) delete mode 100644 docs/README-sphinx.md delete mode 100644 docs/assets/raven-icon-white.png delete mode 100644 docs/stylesheets/extra.css delete mode 100644 mkdocs.yml delete mode 100644 overrides/.icons/custom/matlab.svg delete mode 100644 requirements-sphinx.txt delete mode 100644 scripts/add_tab_icons_sync.py delete mode 100644 scripts/build_hooks.py delete mode 100644 scripts/gen_api_pages_sphinx.py delete mode 100644 scripts/migrate_to_myst.py diff --git a/.gitignore b/.gitignore index 02db86b..c56d94e 100644 --- a/.gitignore +++ b/.gitignore @@ -164,13 +164,7 @@ venv.bak/ # Rope project settings .ropeproject -# mkdocs documentation -/site - -# Sphinx exploration (explore/pydata-sphinx-theme): generated at build time -# by scripts/gen_api_pages_sphinx.py, the same way the MkDocs build's API -# pages exist only in mkdocs-gen-files' virtual filesystem, never in git. -.venv-sphinx/ +# API reference pages, generated at build time by scripts/gen_api_pages.py docs/api/matlab/*.md docs/api/python/*.md docs/matlab-vs-python.md diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 03bb1ab..2d05c3b 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -20,4 +20,4 @@ sphinx: python: install: - - requirements: requirements-sphinx.txt + - requirements: requirements.txt diff --git a/DESIGN.md b/DESIGN.md index 4f6c8a9..f7c24d2 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -25,16 +25,16 @@ A single, dual-language documentation site for **RAVEN** that serves both the **MATLAB** toolbox and the **Python** package (**raven-toolbox**) from one place, so a user can learn a concept once and apply it in either language. The site is -built with [MkDocs](https://www.mkdocs.org/) + -[Material](https://squidfunk.github.io/mkdocs-material/) and published on Read -the Docs. +built with [Sphinx](https://www.sphinx-doc.org/), [MyST](https://myst-parser.readthedocs.io/) +Markdown and the [PyData theme](https://pydata-sphinx-theme.readthedocs.io/), and +published on Read the Docs. ## 2. Guiding principles - **Dual-language parity.** MATLAB and Python are presented side by side throughout: paired API entries, and code blocks with MATLAB/Python tabs. - **Single source of truth.** API documentation is extracted directly from the - function help in each toolbox's source (via `mkdocstrings`), so it never + function help in each toolbox's source, so it never drifts from the code. The source repos are tracked as **git submodules**. - **Source of truth follows the active branch.** The RAVEN submodule tracks **`develop`** and raven-toolbox tracks **`develop`**. RAVEN's modular @@ -50,17 +50,15 @@ the Docs. ## 3. Technical foundation -- MkDocs + Material theme; Read the Docs build (`.readthedocs.yaml`). -- `mkdocstrings` with two handlers, both collecting statically from submodule - source (no MATLAB runtime, no installed package required): - - Python handler (griffe) over the raven-toolbox source, NumPy docstring style. - - MATLAB handler (`mkdocstrings-matlab`, tree-sitter) over the RAVEN modular - directories, NumPy docstring style. +- Sphinx + MyST Markdown + PyData theme, with `sphinx-design` for tabs, grids + and badges; Read the Docs build (`.readthedocs.yaml`, `docs/conf.py`, + `requirements.txt`). - Function help in both toolboxes uses **NumPy-style docstrings** so it renders as structured argument/return tables. -- The API reference is **generated at build time** (`mkdocs-gen-files` + - `mkdocs-literate-nav`): a cobrapy-style autoapi, one page per category, with - MATLAB and Python paired by name. +- The API reference is **generated at build time** by `scripts/gen_api_pages.py`, + called from `docs/conf.py`. It reads the help statically from submodule source + (no MATLAB runtime, no installed package) and writes a cobrapy-style autoapi, + one page per category, with MATLAB and Python paired by name. - Submodules: `RAVEN` (branch `develop`), **raven-toolbox** (branch `develop`), `hanpo-GEM` (branch `main`). @@ -120,7 +118,7 @@ Then a **mapping table**: MATLAB function name (left) ↔ Python function name |---|---| | `importModel` | `import_model` | | `solveLP` | `solve_lp` | -| … | … (cobrapy) :material-link-variant: | +| … | … (cobrapy) | ### 4.4 Guides @@ -156,7 +154,7 @@ Conventions: MATLAB tab would make a cobrapy call look like RAVEN's. A reader who thinks `get_by_id` is raven-toolbox's searches the wrong reference. - **Never fake a pairing.** Where nothing equivalent exists, the tab says so. - `scripts/check_names.py` fails the build on an invented name. + `scripts/check_names.py` lists names that resolve to neither toolbox. - **Differences are the content.** Where the two toolboxes genuinely disagree, `getExchangeRxns` counting 273 exchanges where `model.exchanges` counts 270, `fillGaps` being a MILP where `connect_blocked_reactions` is an LP, say so on @@ -225,8 +223,8 @@ Generated from source, organised so each function is shown for both languages. ## 5. Conventions -- **Code tabs.** Material content tabs labelled "MATLAB" / "Python", **linked - and persistent**, defaulting to **MATLAB**. +- **Code tabs.** `sphinx-design` tabs labelled "MATLAB" / "Python", **linked** + through `:sync:`, defaulting to **MATLAB**. - **cobrapy indicator.** Functions that resolve to cobrapy (not RAVEN) are marked with a consistent icon/badge and link out to the cobrapy docs. - **Naming.** MATLAB `camelCase` ↔ Python `snake_case`; the two are paired by @@ -245,7 +243,8 @@ into the relevant section above.)* *(Append decisions here as they are made.)* -- Build stack: MkDocs + Material + `mkdocstrings` (Python + MATLAB handlers). +- Build stack: Sphinx + MyST + PyData theme; API pages generated by + `scripts/gen_api_pages.py`. - Source via git submodules: RAVEN tracks `develop`, **raven-toolbox tracks `develop`**, hanpo-GEM tracks `main`. - The Python package is named **raven-toolbox**. diff --git a/README.md b/README.md index 1999228..ff3b0ed 100644 --- a/README.md +++ b/README.md @@ -32,13 +32,13 @@ cd raven-docs python -m venv .venv source .venv/bin/activate # Windows: .venv\Scripts\activate -pip install -r requirements-sphinx.txt +pip install -r requirements.txt sphinx-build -b html docs docs/_build/html ``` The API reference pages are generated at the start of each build by -[`scripts/gen_api_pages_sphinx.py`](scripts/gen_api_pages_sphinx.py), called +[`scripts/gen_api_pages.py`](scripts/gen_api_pages.py), called from `docs/conf.py`, which walks the two toolbox submodules and pairs their functions. No MATLAB runtime or installed Python package is required; docstrings are collected statically from source. diff --git a/docs/README-sphinx.md b/docs/README-sphinx.md deleted file mode 100644 index fcab00b..0000000 --- a/docs/README-sphinx.md +++ /dev/null @@ -1,155 +0,0 @@ -# Sphinx / pydata-sphinx-theme exploration - -Not part of the site. This note records the migration from MkDocs to Sphinx -(#78, #80). Sphinx is the live build: `.readthedocs.yaml` builds from -`docs/conf.py`. `mkdocs.yml`, `requirements.txt`, `scripts/gen_api_pages.py` -and `scripts/build_hooks.py` belong to the retired MkDocs build and are no -longer used. The sections below describe the state at the time of the -migration. - -## Building it - -```bash -python -m venv .venv-sphinx -.venv-sphinx/Scripts/pip install -r requirements-sphinx.txt -.venv-sphinx/Scripts/sphinx-build -b html docs docs/_build/html -``` - -## What's converted - -- **Admonitions, content tabs, snippet includes.** Mechanical, via - `scripts/migrate_to_myst.py`, run once across every page. `!!! note "…"` -> - `:::{note} …`; `=== "MATLAB"` -> a `tab-set`/`tab-item` pair; - `--8<-- "path"` -> `{literalinclude}`. Re-running it is a no-op (nothing - left to match). -- **Navigation.** `mkdocs.yml`'s single `nav:` tree is now a `{toctree}` per - section, living in that section's own `index.md` (or `migrate.md`, a new - landing page). Top level, in order: Migrate, User guide, API reference, - Methods, References; API promoted out from under a "Reference" tab that - otherwise would have held only citations once MATLAB-vs-Python and tuned - parameter defaults moved to Migrate and Methods respectively, so that page - was dropped rather than kept half-empty. `collapse_navigation: True` - (pydata-sphinx-theme's own default, overridden to `False` in an earlier - pass of this branch) keeps only the active tab's own subtree expanded. -- **The generated API reference and MATLAB vs Python table.** New - `scripts/gen_api_pages_sphinx.py`, run from `conf.py`'s `setup()` before - Sphinx reads the source tree. Reuses `api_index.py`'s collection constants - and file-walking, extended to keep the *whole* help text instead of only - the first line, and writes fully static pages: see "Not converted" below - for why. -- **The home page.** Hand-converted, not mechanical: MkDocs Material's grid - cards and the custom pip-install JS widget have no equivalent to translate, - so it's rebuilt with `sphinx-design` grids and a plain tab-set instead. The - wordmark image (light/dark swap via a small CSS rule) replaces the badge - row and the "# RAVEN" heading; the heading itself stays in the source for - document structure but is visually hidden, since a raw `

` never - registers with MyST's own heading-level check. -- **Missing icons.** MkDocs Material's `:material-*:`/`:octicons-*:` - shorthand inside tab labels (`installation/raven.md`, - `installation/python.md`) rendered as literal text; swapped for - sphinx-design's built-in `{octicon}` role (`plug`, `download`, - `git-branch`, `package`). Every MATLAB/Python code-example tab site-wide - also gained a small marker (Ⓜ️ / 🐍) and a `:sync:` key, via - `scripts/add_tab_icons_sync.py`, so switching the language in one code - block switches every other tab-set on the page. -- **Announcement banner, "Show Source" links.** `html_theme_options`' - `announcement` (RAVEN 3.0.0b1, pre-release) and `html_show_sourcelink = - False`, both single-setting changes. -- **References.** Trimmed to the RAVEN 1/RAVEN 2/Hansenula-protocol/ - INIT/ftINIT/random-sampling papers only, each verified against its DOI - (Crossref/PLOS) rather than typed from memory; tINIT has no citation of - its own separate from the INIT paper, RAVEN's MATLAB implementation of it. -- **Link underlines.** `docs/_static/custom.css`, wired in via - `html_css_files`. -- **`` inline badges.** 39 occurrences across 14 - guide pages, rendered as inert literal text (the CSS class is - Material-specific and not defined here). Swapped for sphinx-design's - built-in `{bdg-secondary}` role, the same mechanical single-string - find/replace as the icon shorthand fix above. - -## Not converted - -- **Per-function docstring rendering.** The MkDocs build defers this to two - live plugins at render time: mkdocstrings' Python handler (griffe) and a - separate mkdocstrings-matlab handler (tree-sitter). Neither has a verified - drop-in Sphinx equivalent, the MATLAB one especially, which is a bespoke - handler with no autodoc-family counterpart. `gen_api_pages_sphinx.py` - sidesteps this by extracting the full help text itself and writing static - pages, which keeps the same no-install, no-MATLAB-runtime constraint the - MkDocs build has, but by a different route. Worth a closer look before - treating this as done: real docstring structure (numpydoc parameter - tables, cross-references) is flattened to a plain text block here, where - mkdocstrings renders it properly. -- **`scripts/build_hooks.py`, `scripts/check_names.py`.** Both are MkDocs - plugin hooks (`on_config`, `on_page_markdown`, `on_page_content`), a - different event API than Sphinx's `app.connect(...)`. `build_hooks.py`'s - matlab-path pruning and mkdocstrings-matlab output cleanup are moot here, - since this branch doesn't use mkdocstrings-matlab at all. - `check_names.py`'s identifier validation is still valuable and doesn't - need to be a live hook; it can run as a standalone pre-build check either - way, but re-wiring it wasn't done as part of this exploration. -- **`scripts/run_examples.py`.** Treats a bare ` ```text ` fence directly - below a code block as that block's expected output, and `--update` writes - that fence shape. -- **`.readthedocs.yaml`.** Still points at the `mkdocs:` build. The target - block for a real switch: - - ```yaml - build: - os: ubuntu-24.04 - tools: - python: "3.12" - sphinx: - configuration: docs/conf.py - python: - install: - - requirements: requirements-sphinx.txt - ``` - - Left unset here since flipping it would make this branch's incomplete - build the one Read the Docs actually serves. -- **RST cross-reference roles inside docstrings.** raven-toolbox's - docstrings use Sphinx's own `:func:`name`` role syntax, expected by a - proper `sphinx.ext.autodoc` pipeline. Rendered as static text here (see - above), `:func:` shows as a literal prefix next to a correctly-formatted - inline code span, rather than becoming a cross-reference link. Cosmetic, - not a build warning. - -## Verified - -`sphinx-build -E -b html docs docs/_build/html` (`-E` forces a clean -re-read; an incremental rebuild under-reports the cross-reference count -below, since Sphinx does not always revalidate a link when only the *target* -document changed): **0 errors, 67 warnings.** - -Two real fixes got the API-reference-generator's own share of that down to -zero, not just noise-suppression: - -- Its Python function headings embedded the full signature - (`### merge_linear(model, no_merge=())`), so MyST's auto-generated heading - anchor did not match the plain `#merge_linear` the summary table and - `matlab-vs-python.md` link to. Moved the signature to its own line below - the heading instead of inside it. -- `## Title {#custom-id}` (MkDocs' attr_list heading-ID syntax) needed - converting to an explicit `(custom-id)=` MyST target, added as the - converter's fifth transform. - -What's left, all in hand-authored pages rather than generated ones: - -- **21 `myst.header`** ("Non-consecutive header level increase; H1 to H3"), - across most of `docs/guide/*.md`. Predates this branch: every one of these - pages goes straight from its H1 title to an H3 "Functions on this page" - aside, which Python-Markdown never enforced and MyST does. A heading-level - fix, not a migration one; left alone here. -- **42 `myst.xref_missing`**, almost all in `raven3-migration.md` - (`installation/raven.md` and `guide/io.md` have a few more): hand-placed - `` HTML anchors, which work as plain browser URL fragments - but aren't in MyST's own validated-target registry, so every markdown link - pointing at one warns without actually being broken. -- **4 `toc.not_included`**: `docs/data/README.md` (the same page the MkDocs - build also excludes from its nav), plus three pages this pass deliberately - de-listed from Methods rather than deleted, since other pages still link - to them inline: `parameter-tuning/studies/init-solver-benchmark.md` and - `.../sampling-convergence-calibration.md` (cross-solver benchmark and one - of the two sampling pages removed from the Methods listing), - `parameter-tuning/flux-sampling-algorithms.md` (the other). diff --git a/docs/api/index.md b/docs/api/index.md index 1b4086e..4a23e43 100644 --- a/docs/api/index.md +++ b/docs/api/index.md @@ -1,7 +1,3 @@ ---- -icon: material/folder-open ---- - # API reference RAVEN ships in two implementations that build the **same** genome-scale diff --git a/docs/assets/raven-icon-white.png b/docs/assets/raven-icon-white.png deleted file mode 100644 index af8e75f7cd61d65b4429137b35672404a7f68100..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5814 zcmV;n7D?%eP)m0}P0H?Gj)Tu*?IdHKh;mb>Ki?2VgVnf8Ewr z1@I<4_#JQsqb%h)mn>-pD!!wpzyHLw5_TEOUSGm9U} zP+&YT5!kiNd%G>o^0?7LixL3n0n=o%6L0`oQwRA%#|X-vI4q%joTH`PhR4TT)8uFkZ`9 zKtHn08HoYLVl;qvzt=3;YnXzyI3t0}&Gr*#J2cu}DWKu~iN+)HV5z{Rt#{t$aB~Om znP59)Vcg!02J)VZ$yR%5Fko!3oPvhcH*#P>LatXAOft?`1N_zkkN1Q!)G+ooG&2GgSJeK?v|vf4 ztAGi?G8Oo~gJ8jUMf3fVyJN%b0nEtI3o0j1nxE}{*|P^sn*mFQqvyF8_=EXU}=QI=eP*C#V@rxfY}x; zdU92+LyN1tIAZi}X@0J8@ciBZe&(0i4ZwuW>VqA&U9JEY`ek-U!}O(C1Qj@PPGRW( zRp4re!Q7I7=|^R~J~W_&A0Qj106f3 zC1^gY+t=s1378EWvlC_lFW^fUoCM7J6j_;E0L)zAr+!)81M0BvK?Imtnf?f=9V-T_^~6}rGOIsJn4gUZUH8I?H0g) zIkGU_RyE!0KJMZUV8VFDI%o&umW~{M2$%&%%XsbL*p@UI_^Ly9a(E2OJdLbdfayl- zUZy$Hm`0@E=zVX$xTj%`pqK^-{H61NDJ6q}V;nsG?ue%G;s{`NLbgyEzKek8(In-qz+XKCY7dy* zf%|Z`Z@QfJ2`8-eifE!mz=J61(OeIO>Wp6d`O!HAOK9x2p^OdDZEGkn0eA`}bY9m7 z|9S=(n@MUsaB>S=hm(MNfo&Y*wQ(LWVhr3Xf!V<3E#gg#27ZsSl8e_i>1~*S!0i;L zm9@$p*KLtKA7t4zp1(H+FyTdM2kt_rf<&i$&_bYDhz2WP^bm>PlEh-a9On@GWmeZ? zQ{Xl$9paO%mWBz3Xzp*eqQDz|pUC~d*U-64zA4UJgf2ACaoE84f&w&Rg`7oZMW2PGote z)iumW50+G&p6j7g+cs?Fpk4rM=P{95wnu5uS~cuIK(V3TOFGZ#NZ`CywfO?vrc8C! z2tFG)p%p!+faWNSYUK3_HO=KF9MMvqQ-DsL_fokL+=2F(Z`o>hOmB~Y9?Pb{RVX=R zOL$Ix?qT(^7ci?;Jg2}%c5C7}9Rpn03Z7Gd?-s0;=X3_jZqd}9QviJE*S%)j`o%PP z&M81=ID4s$=hP1T2QaFc?+#BSSX&tc{1O<_jGj|~_9T0$tr7f1;6{}ATAGQDBtSa= zz0|(&Y8T)aW}E727EKcXulRMYi#-Ecm^Q1KJSPvB`pfaaxy=Ha0Hws3@5>MCYST;y zzS9h#34rDJhOI_yPnQ6P=N&Ww+H><4uVL!0Y1+|5<=8x5>%bbQ>81WDgmr;m8Miyv zeMyUa_PSvygdxBUz$SSH%v-=4ep_moO|uP3nKm$QJi~!CSknWhv7Xat;5xMUsw*#` zu>tdce0os0m-{?W4qBWdcb53npxN4id7d ze>AAZlNmcj|9WZGpb1Kh!@C&mS!(lI#u?C@g7$D{rul;sG@~y9GyOyl=Rk9z(Kcj} zEKVBXXCL4#6dTOFxWO5`22CN{*616$)Xc~5Acg{S(7DiFvX`*fY6aRTH>Z)H2~xOq zi-1W!HiJvU-329@z6sc&kv2`*K+pr0J>$il+TcNG-f5UiV~9glHvpX%BDv6*OTj2QG-HMKA8mb-J<<#EfY_c&+()E1KKzwZHDU+us2GojMaUrrDBg zaCHNpM(AdleJq1nBz}gH@ z9@;s6-7=FMfbUh~F*QQN5JSUtFB%p0TN4fM$zF^O`Ff=%S|$t5v{!%=tzis)7)ImV z>@IX()9F36C4-E5g72{mVBbYa!;7jWQLJAQ*S91)5q2%`Ex(SL$Ya26mFD|`jJotf zO>-Jr6YPadx(X)d!RU?3o+zD&mu&3GozTX&R0gmM?9vaSC7xv1jzJ3(NBP$9R>gTOx-t(#~7y8;*+G+UzirXeXWj_sy*Swl{51%8hXaeo1wKa{l9%|I*D znn60v`Ak5&2P&}%zeADw4A%i;%XcZZ;8tq5t^`g+EB?~TAe8TBlO_SpNocp@?I=BK zTF|@>Y-ed-XKk3M{HL)q%4wv(0gnK4jTX5bt@-$Qh2Qr!@arbBO(q#FeOwGf6=m9l zrsPq>V2rRk%V?7a3_#XH#}6)|=LFOT8o)J+>}GDJQ-L2>5KdSOj6nzAf2N-)aQK9O z2i#bO5N9LcLj#lr1}yDWr2m^qj-l9*dQk->vO_E_)18+sDLk=Dj7^e6BE2~p?6bXs z&MLY#n@Pq}WC;I0%H>nedkVoc5!izwjf?Yu2@m{nN)IjVYt&~gMi4)PX zk~G1*e#Q>^%ncaips(iFM$070$Qp$E12hTOY_gwqO;(}x)N{+^(d=N}n7nLBwoNg| z7;j1P4=CAmHJyE%0b4a8VD0p$9cNQF#|_{(7}R z3DJ6O>4&0a4EA+-#L^z+uGRPQ2$*+FT!1peGYUPZb7@DTo5Un2FF=D|ALp?4IIy}N z_l~jiZYEti$WTf{@42ZBH*qZ)0gaM zZj87*+`piS+EO>fzM(goC%|k_;(0rU^05KUzGl%UNA~qX#E{)B<}+^s-iQEXN(noP zoFpXUSWr9BsPIE2ZitQ1`k~wara=6Brf#$S%t8~EhnPDlq_yY=TyDIqt&HI+mU{jI z@VOW*bWe<$ehv6xjsY`1CUg*%hO9&bb+45OqK*ePM?~`Q7$hR^$!Dco;wA3+MIv%S zyC;h;j-schV4} zh3>SX!@F9>a+j6#e7!_4aYoJ~zw4FQdj4o^ny7XOx9`1sk)OOslF@scu~U8soRVPJ z1y(cq{=h#1^MG9_hQq_xzG{9BHWzo0`F#_#8gSk^T-MdV%c*=r*2kG`*~#J1>~ zgj>x$BGNnIIv1?pXISt3NmHja9x`PWD2Fjt&6scNs9@o;t*6juA1hxfGaMh9P1rv`q2DNa~}A!<-p# zL}ZMJ%t`5%i-?>fB3p^bCL*%Fh;)?ru7ZdhDnHT^^9P8?o+7eEVp~qK}^E$V&j!`)Cf#lS;aqbo)c9R7j>Fo-vL$*CAvioL`UImT;Vnnb-Bg5 z)>AU%!-)%sPN-;|QW`zOL}XhL+0qz+>&3k8NK4f{WsFtZ0PTrtKh6rM*=l^Qh-_5l zmS+d428+nuk$|5qcZx`#nDkzL_foYTO=Y6ipa$V zB-@(T^%s%lk%O6T9YoNB3p^{bfZ+w4StC2C5Vc&oUg6o)tUz_lS%gn+aB_hwn{QqVV8QOyjJH-0D?#SXDSVv=fYUI1@ z-^bVSd;5#Xa1q%?L`I3osUq@Lox!rk{5;nJa8-b@?}lBBW+^4B5-#*Bku56CV@%|; z*IL`r&^j!beXd7E&Lxzm&B8XyQygtHPQ5&<5HNRGTW(ceQW%*mpd>u|nJnf95pO0e z4d{r86n3L+AhXey&9p}K*n~c%$4z`As3Y&)^(GwN9r2zu6SPk# z4)M>Kypr8oLLD{D?Z7w9F}52mHw^9dp#(`lt&d}=sbjgSLcW(usN&YI2NQVoATKl7 z%xhD8ULg{8zZ9huJ2Ixb@MMxXeLrph$F99tw^)T)AwYJr8rPvQJi@Dh6U?^0x`u&2 zi7_W?z9q$}&4Wecg&wHFu&8IUF|2=-@SYVSGP1$x3H$u6u#V-=*0z+zv5XOshiYs9 zAD-qoYH=dA@4k_UoG&6DrnKzWME-ua#QSc^D5n)JR9_L9XO3lwi0qhAel0wnJ|c3w zh`d+>uq;h`GuEmeCThKZ0r)qRPOLpb?@1NY#O9KHL)fEcYueTPjsul+2cs=EBhfnK z-qw61d+X`6)}qsFZ7WSm=f*T%SiK9V4l!={_BALV`WjmBowZCb2LCkU-n%3nCWJ9C z0Wcq$pKFXw^tT9OX6AswWSoyS6>lS|_?-7mw)biW$fsLY>Is-ageGG!%EG-}m0rtv zX1ntXS$(OAGgD>5MeMQ)vZqwR@?_73oDTfPs=~{nPPX7Ub$r@~6B@?eJHzehbc$5) zX%;xCEyRBBJm8WJegwxQBP#;G>a zg{zkta6ag58n*!xU6`@Pz8DkPOM{d(un(KZT;r~`p%fFR@agJyiA*nPGw(ly;_OHd z8pi-*+bJVZ^1!1K8pXnPkM>B5$YL}JIU8*ONh0eRh>m5MYEhAO#!$q+j|K|;z){Bl z6W+bOQC6erF`RP5M0PuY=ZzMbZMKkA6@2eV;9n_r7G&)&qN!&CJFO30=eYbuc?cy6+!cATQ+<{Vm z&54B2dwS>F@7vJ0>SrW?=2oMLx;$ta4;Xu)&ZNk|)D=@e^f)>j;C{5%r^^ZuwxLB& zQp4!+IFvecRGEmd-<~qql9J{kE<~%~enIJ(0QJi2^aIXD85-Im2JxX5rKpz%0%kJG zMoEO5 ul > li { - border-radius: 12px; - transition: border-color .15s, box-shadow .15s; -} -.md-typeset .grid.cards > ul > li:hover { - border-color: var(--raven-accent); -} - -/* ===== Denser, sectioned sidebar ====================================== */ -.md-nav { font-size: .72rem; } -.md-nav__item .md-nav__item { font-size: .98em; } -.md-nav__item--section > .md-nav__link { - font-weight: 700; - color: var(--md-default-fg-color); -} - -/* ===== Bold active nav item ========================================= */ -.md-nav__item--active > .md-nav__link { font-weight: 700; } -.md-tabs__item--active .md-tabs__link { font-weight: 700; } - -/* ===== Homepage (cobrapy-inspired) ================================== */ -.rh-hero { text-align: center; padding: 2rem 0 1.5rem; } -/* The wordmark is a single flat navy, so it needs a white counterpart to stay - legible in the slate scheme -- swapped here rather than in JS. */ -.rh-hero img.rh-hero-logo { max-width: 420px; width: 100%; height: auto; margin-bottom: .5rem; } -.rh-hero-logo-dark { display: none; } -[data-md-color-scheme="slate"] .rh-hero-logo-light { display: none; } -[data-md-color-scheme="slate"] .rh-hero-logo-dark { display: inline; } -.rh-tag { - display: inline-block; font-size: .7rem; font-weight: 700; - letter-spacing: .07em; text-transform: uppercase; - color: var(--raven-accent); - background: rgba(26,72,136,.1); - border-radius: 20px; padding: .2rem .75rem; margin-bottom: .6rem; -} -.rh-hero h1 { font-size: 1.9rem; line-height: 1.25; margin-bottom: .5rem; } -.rh-tagline { - font-size: .95rem; color: var(--md-default-fg-color--light); - max-width: 540px; margin: 0 auto .9rem; line-height: 1.65; -} -.rh-badges { display: flex; justify-content: center; gap: .35rem; flex-wrap: wrap; } -.rh-badge { - font-size: .68rem; padding: .15rem .5rem; - border-radius: .25rem; - border: 1px solid var(--md-default-fg-color--lightest); - color: var(--md-default-fg-color--light); -} - -/* install block */ -.rh-install { - background: var(--md-code-bg-color); - border-radius: .5rem; padding: 1rem 1.25rem; margin: 1.5rem 0 1.5rem; -} -.rh-install-tabs { - display: flex; gap: .15rem; flex-wrap: wrap; - background: var(--md-default-bg-color); - border-radius: .4rem; padding: .15rem; - border: 1px solid var(--md-default-fg-color--lightest); - width: fit-content; margin-bottom: .6rem; -} -.rh-itab { - font-size: .72rem; font-weight: 600; - padding: .2rem .65rem; border-radius: .25rem; - border: none; background: transparent; - color: var(--md-default-fg-color--light); cursor: pointer; - transition: background .12s, color .12s; -} -.rh-itab.active { background: var(--raven-navy); color: #fff; } -.rh-code-row { - display: flex; align-items: center; gap: .5rem; - background: var(--md-default-bg-color); - border: 1px solid var(--md-default-fg-color--lightest); - border-radius: .3rem; padding: .5rem .8rem; -} -.rh-code-row code { font-size: .8rem; flex: 1; } -.rh-copy { - background: none; border: none; cursor: pointer; padding: 0; - color: var(--md-default-fg-color--light); font-size: .85rem; - opacity: .6; flex-shrink: 0; -} -.rh-copy:hover { opacity: 1; } - -/* section labels and divider */ -.rh-section-label { - font-size: .68rem; font-weight: 700; letter-spacing: .08em; - text-transform: uppercase; - color: var(--md-default-fg-color--light); - margin: 1.5rem 0 .75rem 0; -} -hr.rh-divider { - border: none; border-top: 1px solid var(--md-default-fg-color--lightest); - margin: 1.5rem 0 1rem; -} - -/* features grid, 3 columns */ -.md-typeset .grid.cards.rh-features > ul { - grid-template-columns: repeat(auto-fill, minmax(13rem, 1fr)); -} -.rh-feat-icon { font-size: 1.4rem; display: block; margin-bottom: .3rem; } - -/* docs cards, 3 columns */ -.md-typeset .grid.cards.rh-docs > ul { - grid-template-columns: repeat(3, 1fr); -} - -/* ===== MATLAB / Python pill toggle (content tabs site-wide) ========== */ -.md-typeset .tabbed-labels { - display: inline-flex; - /* Material's own rule sets display: flex, which stretches the track across the - whole column; a segmented control should be as wide as its labels. */ - width: max-content; - max-width: 100%; - background: var(--raven-tab-track); - border-radius: 5px; - padding: 2px; - gap: 1px; - border-bottom: none !important; - overflow: visible; -} -/* Material puts the click target (and its generous padding) on an anchor inside - the label, so shrinking the label alone changes nothing -- reset the anchor - and let the label's own padding size the pill. */ -.md-typeset .tabbed-labels > label > a { - padding: 0; - line-height: inherit; - color: inherit; -} -.md-typeset .tabbed-labels > label { - padding: .15rem .5rem; - border-radius: 4px; - border-bottom: none !important; - font-weight: 500; - font-size: .62rem; - line-height: 1.5; - color: var(--md-default-fg-color--light); - transition: background .15s, color .15s; - white-space: nowrap; - margin-bottom: 0; -} -.md-typeset .tabbed-labels > label:hover { - color: var(--md-default-fg-color); -} -/* Active tab state, one rule per possible tab position */ -.md-typeset .tabbed-alternate input:nth-child(1):checked ~ .tabbed-labels label:nth-child(1), -.md-typeset .tabbed-alternate input:nth-child(2):checked ~ .tabbed-labels label:nth-child(2), -.md-typeset .tabbed-alternate input:nth-child(3):checked ~ .tabbed-labels label:nth-child(3), -.md-typeset .tabbed-alternate input:nth-child(4):checked ~ .tabbed-labels label:nth-child(4) { - background: var(--raven-tab-active); - color: var(--raven-tab-active-fg); - font-weight: 600; -} - - -/* ===== API reference: cobrapy-ish function entries =================== */ -.md-typeset .doc-heading { scroll-margin-top: 4rem; } -.md-typeset h2.doc-heading code, -.md-typeset h3.doc-heading code { font-weight: 600; } diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md index 8cd5524..a48591c 100644 --- a/docs/tutorials/index.md +++ b/docs/tutorials/index.md @@ -1,7 +1,3 @@ ---- -icon: material/folder-open ---- - # Legacy tutorials These five hands-on tutorials were part of the original **RAVEN 1** paper diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index f5cbd95..0000000 --- a/mkdocs.yml +++ /dev/null @@ -1,225 +0,0 @@ -site_name: RAVEN -site_description: >- - Reconstruction, Analysis and Visualization of Metabolic Networks, the RAVEN - Toolbox (MATLAB) and raven-toolbox, documented side by side. -site_author: Wang, Marcisauskas, Sanchez, Domenzain, Hermansson, Agren, Nielsen, Kerkhoven et al. -site_url: https://raven-docs.readthedocs.io/en/latest/ -repo_url: https://github.com/CompMetEng/RAVEN -repo_name: CompMetEng/RAVEN -edit_uri: "" - -# Build-time hooks that keep the site resilient to structural changes in the -# RAVEN / raven-toolbox submodules (folders or files added/removed on the -# tracked dev branches). See scripts/build_hooks.py. -hooks: - - scripts/build_hooks.py - # Aborts the build if a hand-written page names a function that does not - # exist in either toolbox. See scripts/check_names.py. - - scripts/check_names.py - -plugins: - - search - # gen-files runs the generator at build time, emitting one API page per - # shared category plus the literate-nav SUMMARY from the submodule sources. - - gen-files: - scripts: - - scripts/gen_api_pages.py - - literate-nav: - nav_file: SUMMARY.md - - mkdocstrings: - default_handler: python - handlers: - # Python API (raven-toolbox) -- collected statically by griffe from the - # submodule source, so the package does not need to be installed. - python: - paths: [raven-toolbox/src] - options: - docstring_style: numpy - docstring_section_style: table - show_root_heading: false - show_root_toc_entry: false - show_source: false - separate_signature: true - show_signature_annotations: true - # MATLAB API (RAVEN) -- pure-Python tree-sitter collection, no MATLAB - # runtime required. Functions are referenced by name; paths are limited - # to the documented user-facing categories so names stay unambiguous - # (legacy/ and external/ are intentionally excluded). - matlab: - paths: - - RAVEN/reconstruction - - RAVEN/manipulation - - RAVEN/analysis - - RAVEN/gapfilling - - RAVEN/INIT - # tINIT's entry points sit in their own subfolder; the collector - # does not recurse, so without this getINITModel and runINIT drop - # out of the API reference entirely - - RAVEN/INIT/tINIT - - RAVEN/annotation - - RAVEN/biomass - - RAVEN/curation - - RAVEN/conversion - - RAVEN/conditions - - RAVEN/comparison - - RAVEN/omics - - RAVEN/localization - - RAVEN/queries - - RAVEN/io - - RAVEN/solver - - RAVEN/tasks - - RAVEN/utils - paths_recursive: true - options: - docstring_style: numpy - show_root_heading: false - show_root_toc_entry: false - show_source: false - -theme: - name: material - # Custom icons live under overrides/.icons/ and are referenced as - # :custom-: in markdown (e.g. :custom-matlab:). - custom_dir: overrides - logo: assets/raven-icon-white.png - favicon: assets/raven-icon-navy.png - features: - - navigation.tabs - - navigation.tabs.sticky - - navigation.path - - navigation.sections - - navigation.expand - - navigation.top - - navigation.tracking - - navigation.indexes - - navigation.instant - - navigation.footer - - content.code.copy - - content.tabs.link # linked, persistent MATLAB/Python tabs site-wide - - toc.follow - - search.suggest - - search.highlight - - search.share - palette: - # Brand colours (navy) are defined in docs/stylesheets/extra.css. - - scheme: default - primary: custom - accent: custom - toggle: - icon: material/weather-night - name: Switch to dark mode - - scheme: slate - primary: custom - accent: custom - toggle: - icon: material/weather-sunny - name: Switch to light mode - -markdown_extensions: - - admonition - - pymdownx.details - - pymdownx.superfences - - pymdownx.highlight: - anchor_linenums: true - - pymdownx.inlinehilite - - pymdownx.snippets: - base_path: ["."] - check_paths: true - - pymdownx.arithmatex: - generic: true - - tables - - toc: - permalink: true - - attr_list - - md_in_html - - pymdownx.tabbed: - alternate_style: true - # Renders Material icon/emoji shortcodes (e.g. :octicons-arrow-right-24:) as - # inline SVG instead of literal text. material.extensions.emoji is the path - # for mkdocs-material >= 9 (see requirements.txt). custom_icons registers - # overrides/.icons so project icons like :custom-matlab: resolve (Material - # does not auto-scan custom_dir/.icons). - - pymdownx.emoji: - emoji_index: !!python/name:material.extensions.emoji.twemoji - emoji_generator: !!python/name:material.extensions.emoji.to_svg - options: - custom_icons: - - overrides/.icons - -extra_css: - - stylesheets/extra.css - -extra_javascript: - - https://unpkg.com/mathjax@3/es5/tex-mml-chtml.js - -nav: - - Home: index.md - - User guide: - - Overview: guide/index.md - - Install: - - Overview: installation/index.md - - RAVEN (MATLAB): installation/raven.md - - raven-toolbox (Python): installation/python.md - - Downloaded data and binaries: installation/data-and-binaries.md - - 1. Getting started: guide/getting-started.md - - 2. Model structure and identifiers: guide/model-structure.md - - 3. Reading and writing models: guide/io.md - - 4. Simulating growth with FBA: guide/fba.md - - 5. Growth media and conditions: guide/media.md - - 6. Solvers and configuration: guide/solvers.md - - 7. Building a model from scratch: guide/building.md - - 8. Editing an existing model: guide/editing.md - - 9. Quality control: guide/quality-control.md - - 10. Context-specific models (tINIT / ftINIT): guide/init.md - - 11. Deletions and essentiality: guide/deletions.md - - 12. Metabolic tasks: guide/tasks.md - - 13. Gap-filling: guide/gap-filling.md - - 14. Flux variability: guide/fva.md - - 15. Random sampling: guide/sampling.md - - 16. Combining and simplifying: guide/combining.md - - 17. Comparing models: guide/comparing.md - - 18. Reconstruction from homology: guide/homology.md - - 19. Reconstruction from KEGG: guide/kegg.md - - 20. Engineering targets: guide/engineering.md - - 21. Table-driven curation: guide/curation.md - - 22. Biomass and annotation: guide/biomass.md - - Worked protocol, H. polymorpha (MATLAB): - - Overview: protocol/index.md - - Introduction: protocol/reconstruction.md - - Materials and installation: protocol/materials.md - - Import template models: protocol/template-models.md - - Draft from homology: protocol/homology.md - - Biomass composition: protocol/biomass.md - - Curation of lipid reactions: protocol/lipid-curation.md - - Gap-filling: protocol/gap-filling.md - - Save and simulate: protocol/simulation.md - - Manual curation: protocol/manual-curation.md - - Anticipated results: protocol/anticipated-results.md - - Legacy tutorials (MATLAB): - - Overview: tutorials/index.md - - 1 - Import a GEM and run FBA: tutorials/tutorial1.md - - 2 - Build a small model: tutorials/tutorial2.md - - 3 - Knockouts, MOMA and omics: tutorials/tutorial3.md - - 4 - Fix an erroneous model: tutorials/tutorial4.md - - 5 - Reconstruct from KEGG: tutorials/tutorial5.md - - Migrate: - - Overview: migrate.md - - RAVEN 2 → RAVEN 3 (MATLAB): raven3-migration.md - - RAVEN 3 → raven-toolbox: raven3-vs-raven-toolbox.md - - Model file format (YAML): yaml-format.md - - Reference: - - Overview: reference.md - - API reference: api/ - - Function mapping: matlab-vs-python.md - - Tuned parameter defaults: tuned-parameters.md - - Citations: references.md - - Methods: - - Overview: parameter-tuning/index.md - - Studies: - - Homology cut-off calibration: parameter-tuning/studies/homology-cutoff-calibration.md - - KEGG HMM cut-off calibration: parameter-tuning/studies/kegg-hmm-cutoff-calibration.md - - Sampling convergence calibration: parameter-tuning/studies/sampling-convergence-calibration.md - - INIT parameter calibration: parameter-tuning/studies/init-param-calibration.md - - ftINIT reproducibility: parameter-tuning/studies/ftinit-determinism.md - - Human-GEM validation vs MATLAB RAVEN: parameter-tuning/studies/humangem-validation.md - - Parameter benchmarks: parameter-tuning/benchmarks.md diff --git a/overrides/.icons/custom/matlab.svg b/overrides/.icons/custom/matlab.svg deleted file mode 100644 index d49e6c8..0000000 --- a/overrides/.icons/custom/matlab.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - diff --git a/requirements-sphinx.txt b/requirements-sphinx.txt deleted file mode 100644 index 5799bac..0000000 --- a/requirements-sphinx.txt +++ /dev/null @@ -1,6 +0,0 @@ -sphinx>=7.3 -pydata-sphinx-theme>=0.15 -myst-parser>=3.0 -sphinx-design>=0.6 -sphinx-copybutton>=0.5 -pyyaml>=6 diff --git a/requirements.txt b/requirements.txt index f2ea770..5799bac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,6 @@ -mkdocs>=1.6 -mkdocs-material>=9.5 -pymdown-extensions>=10.0 -mkdocstrings[python]>=0.26 -mkdocstrings-matlab>=2.0 -mkdocs-gen-files>=0.5 -mkdocs-literate-nav>=0.6 -black>=24 +sphinx>=7.3 +pydata-sphinx-theme>=0.15 +myst-parser>=3.0 +sphinx-design>=0.6 +sphinx-copybutton>=0.5 +pyyaml>=6 diff --git a/scripts/add_tab_icons_sync.py b/scripts/add_tab_icons_sync.py deleted file mode 100644 index 146c4c8..0000000 --- a/scripts/add_tab_icons_sync.py +++ /dev/null @@ -1,49 +0,0 @@ -"""One-time pass: add a language marker and :sync: to every MATLAB/Python -code-example tab-item, site-wide. - -Only touches tab-items whose title is *exactly* "MATLAB" or "Python" (the -code-example convention every guide page uses) -- an install-method tab like -"MATLAB (Add-Ons)" or "Python (pip)" does not match and is left alone, since -syncing a 4-way install-method choice against a 2-way language choice would -be wrong. - -Idempotent: a tab-item that already carries a marker or :sync: line is left -untouched on a second run. -""" - -from __future__ import annotations - -import re -from pathlib import Path - -MATLAB_RE = re.compile(r'^( *):::\{tab-item\} MATLAB[ \t]*$', re.M) -PYTHON_RE = re.compile(r'^( *):::\{tab-item\} Python[ \t]*$', re.M) - -MATLAB_MARK = "Ⓜ️" # circled M -PYTHON_MARK = "\U0001f40d" # snake - - -def convert(text: str) -> str: - text = MATLAB_RE.sub( - lambda m: f"{m.group(1)}:::{{tab-item}} {MATLAB_MARK} MATLAB\n{m.group(1)}:sync: matlab", - text, - ) - text = PYTHON_RE.sub( - lambda m: f"{m.group(1)}:::{{tab-item}} {PYTHON_MARK} Python\n{m.group(1)}:sync: python", - text, - ) - return text - - -if __name__ == "__main__": - import sys - - root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("docs") - changed = 0 - for path in sorted(root.rglob("*.md")): - original = path.read_text(encoding="utf-8") - converted = convert(original) - if converted != original: - path.write_text(converted, encoding="utf-8", newline="\n") - changed += 1 - print(f"converted {changed} file(s) under {root}") diff --git a/scripts/api_index.py b/scripts/api_index.py index f032d7f..31be63d 100644 --- a/scripts/api_index.py +++ b/scripts/api_index.py @@ -1,6 +1,6 @@ """Shared index of the two toolboxes' public APIs, collected from submodule source. -Both the API-reference generator (``gen_api_pages_sphinx.py``) and the name checker +Both the API-reference generator (``gen_api_pages.py``) and the name checker (``check_names.py``) need to know which functions actually exist in RAVEN (MATLAB) and raven-toolbox (Python). Collecting that in one place means the prose can be validated against exactly the same index the reference pages are @@ -35,8 +35,8 @@ ) # RAVEN top-level categories to document, in nav order: (folder, page title). -# Pruned to existing folders at build time by scripts/build_hooks.py / the -# matlab handler; legacy/ and external/ are intentionally excluded. +# Folders missing from the tracked branch are skipped; legacy/ and external/ +# are intentionally excluded. MATLAB_CATEGORIES = [ ("analysis", "Analysis"), ("annotation", "Annotation"), diff --git a/scripts/build_hooks.py b/scripts/build_hooks.py deleted file mode 100644 index 565f2d7..0000000 --- a/scripts/build_hooks.py +++ /dev/null @@ -1,114 +0,0 @@ -"""MkDocs build hooks that keep the site resilient to submodule restructuring. - -raven-docs documents the RAVEN (MATLAB) and raven-toolbox sources tracked as git -submodules on their fast-moving ``develop`` branches. When -those branches add or remove folders or files, a statically-configured build -breaks. These hooks make the build adapt automatically instead of aborting: - -* ``on_config`` -- prune the mkdocstrings ``matlab`` handler ``paths`` to - only directories that actually exist, so a removed - category folder (e.g. ``visualization``) no longer - aborts the build with a "paths do not exist" error. -* ``on_page_markdown`` -- replace ``--8<--`` snippet includes whose source file - is missing with a visible note, so a removed source - file (e.g. a dropped tutorial script) renders a - warning instead of raising ``SnippetMissingError``. - -Both log a warning for every pruned path / missing snippet, so the gaps stay -visible in the build log rather than failing silently. - -Registered via the ``hooks:`` key in mkdocs.yml. -""" - -from __future__ import annotations - -import logging -import re -from pathlib import Path - -from mkdocs.plugins import event_priority - -log = logging.getLogger("mkdocs.hooks.build_hooks") - -# Matches a single-line pymdownx.snippets include: --8<-- "path/to/file" -_SNIPPET_RE = re.compile(r'^(?P[ \t]*)--8<--[ \t]+"(?P[^"]+)"[ \t]*$', re.M) - - -def _repo_root(config) -> Path: - """Directory containing mkdocs.yml (the snippets base_path and submodule root).""" - return Path(config["config_file_path"]).resolve().parent - - -@event_priority(100) # Run before mkdocstrings' on_config instantiates handlers. -def on_config(config): - """Drop non-existent directories from the mkdocstrings matlab handler paths.""" - root = _repo_root(config) - plugin = config["plugins"].get("mkdocstrings") - if plugin is None: - return config - - handlers = plugin.config.handlers or {} - matlab = handlers.get("matlab") - if not matlab or "paths" not in matlab: - return config - - kept, dropped = [], [] - for rel in matlab["paths"]: - (kept if (root / rel).is_dir() else dropped).append(rel) - - if dropped: - log.warning( - "build_hooks: pruning %d missing matlab path(s) from the API " - "reference (not present on the tracked branch): %s", - len(dropped), - ", ".join(dropped), - ) - matlab["paths"] = kept - - return config - - -def on_page_content(html, *, page, config, **kwargs): - """Strip the repeated MATLAB function name from docstring descriptions. - - MATLAB convention puts the function name on the first help line: - % checkProduction Check which metabolites can be produced from a model. - mkdocstrings-matlab renders this verbatim, duplicating the h3 heading that - our page generator already emits. This hook removes the leading word when - it matches a camelCase identifier followed by a capital letter (i.e. the - start of the real description). - """ - if not page.file.src_path.replace("\\", "/").startswith("api/matlab/"): - return html - # Require at least one uppercase letter inside the identifier so plain - # English words like "see", "use", "for" are not accidentally stripped. - # \s+ handles the MATLAB two-space convention after the function name. - html = re.sub( - r"(

)([a-z][a-z0-9]*(?:[A-Z][A-Za-z0-9]*)+)\s+([A-Z])", - r"\1\3", - html, - ) - return html - - -def on_page_markdown(markdown, *, page, config, files): - """Neutralize snippet includes whose source file is missing.""" - root = _repo_root(config) - - def replace(match: re.Match) -> str: - indent, rel = match.group("indent"), match.group("path") - if (root / rel).is_file(): - return match.group(0) - log.warning( - "build_hooks: snippet '%s' referenced by '%s' is missing on the " - "tracked branch; rendering a placeholder note instead.", - rel, - page.file.src_path, - ) - return ( - f'{indent}!!! warning "Source not available"\n' - f'{indent} `{rel}` is not present on the currently tracked ' - f"branch, so its contents cannot be shown here." - ) - - return _SNIPPET_RE.sub(replace, markdown) diff --git a/scripts/check_names.py b/scripts/check_names.py index 1a2822e..e34d0c3 100644 --- a/scripts/check_names.py +++ b/scripts/check_names.py @@ -1,15 +1,11 @@ -"""Fail the build when the prose names a function that does not exist. +"""Report prose that names a function that does not exist. -The hand-written pages describe two moving APIs. Nothing used to check them, so -names drifted in by analogy -- `camelCase` rewritten as `snake_case` on the -assumption that the Python function must be called that. Pages ended up -promising `get_blast`, `fill_gaps`, `import_model` and `solve_lp`, none of which -were ever real, and a homepage quick start that could not run. - -This hook resolves every function-shaped identifier in the hand-written pages -against the same index the API reference is generated from -(:mod:`api_index`), and aborts the build on anything that does not resolve. -Generated pages are skipped: they come from the index by construction. +The hand-written pages describe two moving APIs, and names drift in by analogy: +`camelCase` rewritten as `snake_case` on the assumption that the Python function +must be called that. This script resolves every function-shaped identifier in +the hand-written pages against the same index the API reference is generated +from (:mod:`api_index`) and lists anything that does not resolve. Generated +pages are skipped: they come from the index by construction. Two kinds of identifier are checked: @@ -23,7 +19,7 @@ belongs in ``known_names.txt`` next to this file, one per line. Keep that list short: every entry is a name nothing can verify for you. -Run standalone to see what would fail, without building: +Run it with: python scripts/check_names.py """ @@ -172,34 +168,6 @@ def _dedent_block(block: str) -> str: return "\n".join(ln[pad:] if len(ln) >= pad else ln for ln in lines) -# --------------------------------------------------------------------------- # -# MkDocs hook # -# --------------------------------------------------------------------------- # -_index: Index | None = None - - -def on_config(config): - global _index - _index = Index() - return config - - -def on_page_markdown(markdown, *, page, config, files): - from mkdocs.exceptions import PluginError - - assert _index is not None - problems = _index.problems(markdown, page.file.src_path) - if problems: - listed = "\n - ".join(problems) - raise PluginError( - f"check_names: unknown identifier(s) in {page.file.src_path}:\n" - f" - {listed}\n" - f"Fix the name, or add it to scripts/known_names.txt if it is " - f"legitimately not a function of either toolbox." - ) - return markdown - - # --------------------------------------------------------------------------- # # Standalone # # --------------------------------------------------------------------------- # diff --git a/scripts/gen_api_pages.py b/scripts/gen_api_pages.py index 378162a..f3ef60a 100644 --- a/scripts/gen_api_pages.py +++ b/scripts/gen_api_pages.py @@ -1,21 +1,10 @@ -"""Generate the API reference for raven-docs at build time. - -Run by the ``mkdocs-gen-files`` plugin. Walks the two submodules and emits a -cobrapy-style ``autoapi`` reference as **two parallel trees** plus a name map: - -* **MATLAB API (RAVEN)** -- one page per category folder (``reconstruction``, - ``manipulation`` ...), each with a *Functions* summary table followed by the - full help for every function (rendered by the ``matlab`` mkdocstrings - handler via tree-sitter -- no MATLAB runtime needed). -* **Python API (raven-toolbox)** -- one page per package, same shape, rendered - by the ``python`` handler (griffe collects from source statically). -* **MATLAB <-> Python** -- a single translation table pairing the two naming - conventions (``camelCase`` <-> ``snake_case``) by normalised name, for every - function that exists in both. - -Each function gets its own ``## name`` heading (so the summary table can link -to it) followed by a ``:::`` autodoc block. A literate-nav ``SUMMARY.md`` ties -the trees together. +"""API reference generator. + +Runs from ``docs/conf.py`` (Sphinx's ``setup()`` hook), before Sphinx reads +the source tree. It extracts the full help text of every function from the +RAVEN and raven-toolbox submodules (the collection layer is shared with +``api_index.py``) and writes static pages: no live directive, no package +install, no MATLAB runtime. """ from __future__ import annotations @@ -23,252 +12,371 @@ import sys from pathlib import Path -import yaml - -import mkdocs_gen_files - -# The collection layer is shared with scripts/check_names.py, so the prose is -# validated against exactly the index these pages are generated from. sys.path.insert(0, str(Path(__file__).resolve().parent)) +import yaml + from api_index import ( # noqa: E402 MATLAB_CATEGORIES, PY_PACKAGE_TITLES, + PYPKG, + RAVEN as MATLAB, cell, - collect_matlab, - collect_python, + is_matlab_function, + matlab_help_block, + matlab_summary_and_body, + module_dotted, norm, slug, ) +import ast +import re +import textwrap + + +# --------------------------------------------------------------------------- # +# Full-text collection (api_index.py's matlab_summary / py_summary keep only +# the first line; these keep everything). # +# --------------------------------------------------------------------------- # +def matlab_help(path: Path, fname: str) -> tuple[str, str, str]: + """(signature, summary, rest of the help text) for one MATLAB function.""" + signature, _lines = matlab_help_block(path) + summary, body = matlab_summary_and_body(path, fname) + return signature, summary, "\n".join(body) + + +def py_help(node: ast.AST) -> tuple[str, str]: + """(summary, rest of the docstring) for one Python function/class node.""" + doc = ast.get_docstring(node) or "" + lines = doc.strip().splitlines() + if not lines: + return "", "" + summary = "" + rest_start = 0 + for idx, line in enumerate(lines): + if line.strip(): + summary = line.strip() + rest_start = idx + 1 + break + rest = "\n".join(lines[rest_start:]).strip() + return summary, rest + + +def collect_matlab_full() -> dict[str, list[dict]]: + cats: dict[str, list[dict]] = {} + for folder, _title in MATLAB_CATEGORIES: + funcs: dict[str, dict] = {} + base = MATLAB / folder + if not base.is_dir(): + continue + for m in base.rglob("*.m"): + if m.stem == "Contents" or not is_matlab_function(m): + continue + signature, summary, body = matlab_help(m, m.stem) + funcs.setdefault( + m.stem, + {"name": m.stem, "signature": signature, "summary": summary, "body": body, "path": m}, + ) + cats[folder] = [funcs[n] for n in sorted(funcs, key=str.lower)] + return cats + + +def collect_python_full() -> list[dict]: + objects: list[dict] = [] + if not PYPKG.is_dir(): + return objects + for py in PYPKG.rglob("*.py"): + try: + tree = ast.parse(py.read_text(encoding="utf-8")) + except (SyntaxError, OSError): + continue + dotted = module_dotted(py) + rel = py.relative_to(PYPKG) + package = rel.parts[0] if len(rel.parts) > 1 else "_toplevel" + for node in tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + if node.name.startswith("_"): + continue + ident = f"{dotted}.{node.name}" if dotted else node.name + summary, rest = py_help(node) + sig = "" + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + sig = f"({ast.unparse(node.args)})" if hasattr(ast, "unparse") else "" + objects.append( + { + "name": node.name, + "ident": ident, + "package": package, + "summary": summary, + "body": rest, + "signature": sig, + "kind": "class" if isinstance(node, ast.ClassDef) else "function", + } + ) + return objects + + +# --------------------------------------------------------------------------- # +# Help-text rendering # +# --------------------------------------------------------------------------- # +# RAVEN's % help blocks and the structured raven-toolbox docstrings both use +# numpydoc section headers ("Parameters" over a dashed underline), so one +# parser serves both. Parameter sections become definition lists, Examples a +# code block, and anything else passes through as prose. Help text without any +# section header is kept as the author laid it out: in a text block for MATLAB, +# whose help is written for the command window, and as prose for Python. +PARAM_SECTIONS = { + "Parameters", "Name-Value Arguments", "Other Parameters", "Returns", + "Yields", "Raises", "Attributes", +} +_SECTION_HEADER = re.compile(r"^([A-Z][A-Za-z -]+)\n(-{3,})$", re.M) +_RST_ROLE = re.compile(r":(?:py:)?(?:func|class|meth|attr|mod|data|obj|exc):`~?([^`]+)`") + + +def myst_inline(text: str) -> str: + """RST roles and double-backtick literals as MyST code spans.""" + text = _RST_ROLE.sub(lambda m: f"`{m.group(1)}`", text) + return re.sub(r"``([^`]+)``", r"`\1`", text) + + +def split_sections(doc: str) -> list[tuple[str | None, str]]: + parts: list[tuple[str | None, str]] = [] + pos, title = 0, None + for m in _SECTION_HEADER.finditer(doc): + if len(m.group(2)) < len(m.group(1).strip()) - 1: + continue # a dashed line too short to underline this title + parts.append((title, doc[pos:m.start()])) + title, pos = m.group(1).strip(), m.end() + parts.append((title, doc[pos:])) + return parts + + +def param_entries(body: str) -> list[tuple[str, str, str]]: + entries: list[list] = [] + for line in body.strip("\n").splitlines(): + if line and not line[0].isspace(): + name, _, typ = line.partition(" : ") + entries.append([name.strip(), typ.strip(), []]) + elif entries: + entries[-1][2].append(line) + return [(n, t, textwrap.dedent("\n".join(d)).strip()) for n, t, d in entries] + + +def render_help(doc: str, lang: str) -> list[str]: + """Markdown lines for a help text's body (everything after the summary).""" + doc = textwrap.dedent(doc).strip("\n") + if not doc.strip(): + return [] + sections = split_sections(doc) + if all(title is None for title, _ in sections): + if lang == "matlab": + return ["```text", doc, "```", ""] + return [myst_inline(doc), ""] + out: list[str] = [] + for title, body in sections: + body = textwrap.dedent(body).strip("\n") + if title is None: + if body.strip(): + out += [myst_inline(body), ""] + continue + out += [f":::{{rubric}} {title}", ":::", ""] + if title in PARAM_SECTIONS: + for name, typ, desc in param_entries(body): + term = f"`{name}`" + (f" : *{typ}*" if typ else "") + lines = myst_inline(desc).splitlines() or [""] + out += [term, f": {lines[0]}"] + [f" {line}" if line else "" for line in lines[1:]] + [""] + elif title == "Examples": + fence = "pycon" if ">>>" in body else lang + out += [f"```{fence}", body, "```", ""] + else: + out += [myst_inline(body), ""] + return out + # --------------------------------------------------------------------------- # # Page rendering # # --------------------------------------------------------------------------- # -def render_page(title: str, intro: str, entries: list[dict], handler: str | None) -> str: - """A cobrapy-style page: summary table, then one section per object. - - ``entries`` items: {name, summary, ref} where ref is the autodoc target - (function name for matlab, dotted ident for python). ``handler`` is the - mkdocstrings handler name, or None to use the default (python). - """ - out = [f"# {title}", "", intro, "", "## Functions", ""] - out += ["| Function | Summary |", "|---|---|"] - for e in entries: - out.append(f"| [`{e['name']}`](#{slug(e['name'])}) | {cell(e['summary'])} |") +def render_matlab_page(title: str, folder: str, funcs: list[dict]) -> str: + out = [f"# {title}", "", f"MATLAB functions in `RAVEN/src/{folder}` of the RAVEN toolbox.", "", + "## Functions", "", "| Function | Summary |", "|---|---|"] + for f in funcs: + out.append(f"| [`{f['name']}`](#{slug(f['name'])}) | {cell(myst_inline(f['summary']))} |") out += ["", "## Reference", ""] - for e in entries: - out += [f"### {e['name']}", ""] - if handler: - out += [f"::: {e['ref']}", f" handler: {handler}", ""] - else: - out += [f"::: {e['ref']}", ""] + for f in funcs: + # A cross-document markdown link with a #fragment resolves against + # the target page's auto-generated heading anchor, so this heading + # must auto-slug to exactly `slug(name)` -- true here since it is a + # bare MATLAB identifier, nothing an explicit target would improve. + out += [f"### {f['name']}", ""] + if f["signature"]: + out += ["```matlab", f["signature"], "```", ""] + if f["summary"]: + out += [myst_inline(f["summary"]), ""] + out += render_help(f["body"], "matlab") return "\n".join(out) + "\n" -def render_group_index(title: str, intro: str, links: list[tuple[str, str]]) -> str: - """The landing page for a generated group (MATLAB API, Python API). +def render_python_page(title: str, dotted: str, objs: list[dict]) -> str: + out = [f"# {title} (Python)", "", f"`raven-toolbox` objects in `{dotted}`.", "", + "## Functions", "", "| Function | Summary |", "|---|---|"] + for o in objs: + out.append(f"| [`{o['name']}`](#{slug(o['name'])}) | {cell(myst_inline(o['summary']))} |") + out += ["", "## Reference", ""] + for o in objs: + # The heading is the bare name, not name+signature: a cross-document + # markdown link with a #fragment resolves against the target page's + # auto-generated heading anchor, not an explicit MyST target, so a + # heading has to auto-slug to exactly `slug(name)` to be linkable at + # all. The signature goes on its own line underneath instead. + out += [f"### {o['name']}", ""] + if o["signature"]: + out += [f"`{o['name']}{o['signature']}`", ""] + if o["summary"]: + out += [myst_inline(o["summary"]), ""] + out += render_help(o["body"], "python") + return "\n".join(out) + "\n" + - Gives the group a real page to attach to in the nav (``navigation.indexes``) - instead of a bare, unclickable section label, and a folder icon distinct - from the leaf category pages beneath it (``navigation.sections``). - """ - out = ["---", "icon: material/folder-open", "---", "", f"# {title}", "", intro, ""] +def render_group_index(title: str, intro: str, links: list[tuple[str, str]]) -> str: + stems = [href.rsplit(".", 1)[0] for _title, href in links] + out = [f"# {title}", "", intro, "", + "```{toctree}", ":hidden:"] + out += stems + out += ["```", ""] for link_title, href in links: out.append(f"- [{link_title}]({href})") return "\n".join(out) + "\n" -matlab = collect_matlab() -python_objs = collect_python() - -# normalised name -> (python ident, package) ; first definition wins -py_by_norm: dict[str, dict] = {} -for obj in python_objs: - py_by_norm.setdefault(norm(obj["name"]), obj) - -summary: list[str] = ["* [Overview](index.md)"] - -# --- MATLAB API tree ------------------------------------------------------- # -matlab_links: list[tuple[str, str]] = [] # (title, relative link) for the group's own index -summary.append("* [MATLAB API (RAVEN)](matlab/index.md)") -for folder, title in MATLAB_CATEGORIES: - funcs = matlab[folder] - if not funcs: - continue - entries = [{"name": f["name"], "summary": f["summary"], "ref": f["name"]} for f in funcs] - intro = ( - f"MATLAB functions in `RAVEN/{folder}` of the RAVEN toolbox. Help text " - "is collected from the source of the tracked branch." +def generate(docs_root: Path) -> None: + api_dir = docs_root / "api" + (api_dir / "matlab").mkdir(parents=True, exist_ok=True) + (api_dir / "python").mkdir(parents=True, exist_ok=True) + + matlab = collect_matlab_full() + python_objs = collect_python_full() + py_by_norm: dict[str, dict] = {} + for obj in python_objs: + py_by_norm.setdefault(norm(obj["name"]), obj) + + matlab_links: list[tuple[str, str]] = [] + for folder, title in MATLAB_CATEGORIES: + funcs = matlab.get(folder, []) + if not funcs: + continue + (api_dir / "matlab" / f"{folder}.md").write_text( + render_matlab_page(title, folder, funcs), encoding="utf-8" + ) + matlab_links.append((title, f"{folder}.md")) + (api_dir / "matlab" / "index.md").write_text( + render_group_index("MATLAB API (RAVEN)", "RAVEN's MATLAB functions, by category.", matlab_links), + encoding="utf-8", ) - with mkdocs_gen_files.open(f"api/matlab/{folder}.md", "w") as fh: - fh.write(render_page(title, intro, entries, handler="matlab")) - summary.append(f" * [{title}](matlab/{folder}.md)") - matlab_links.append((title, f"{folder}.md")) - -with mkdocs_gen_files.open("api/matlab/index.md", "w") as fh: - fh.write(render_group_index("MATLAB API (RAVEN)", "RAVEN's MATLAB functions, by category.", matlab_links)) - -# --- Python API tree ------------------------------------------------------- # -by_package: dict[str, list[dict]] = {} -for obj in python_objs: - by_package.setdefault(obj["package"], []).append(obj) - -python_links: list[tuple[str, str]] = [] -summary.append("* [Python API (raven-toolbox)](python/index.md)") -for package in sorted(by_package, key=lambda p: (p == "_toplevel", PY_PACKAGE_TITLES.get(p, p).lower())): - objs = sorted(by_package[package], key=lambda o: o["name"].lower()) - title = PY_PACKAGE_TITLES.get(package, package) - dotted = "raven_toolbox" if package == "_toplevel" else f"raven_toolbox.{package}" - entries = [{"name": o["name"], "summary": o["summary"], "ref": o["ident"]} for o in objs] - intro = f"`raven-toolbox` objects in `{dotted}`, collected from the source of the tracked branch." - with mkdocs_gen_files.open(f"api/python/{package}.md", "w") as fh: - fh.write(render_page(f"{title} (Python)", intro, entries, handler=None)) - summary.append(f" * [{title}](python/{package}.md)") - python_links.append((title, f"{package}.md")) - -with mkdocs_gen_files.open("api/python/index.md", "w") as fh: - fh.write( + + by_package: dict[str, list[dict]] = {} + for obj in python_objs: + by_package.setdefault(obj["package"], []).append(obj) + python_links: list[tuple[str, str]] = [] + for package in sorted(by_package, key=lambda p: (p == "_toplevel", PY_PACKAGE_TITLES.get(p, p).lower())): + objs = sorted(by_package[package], key=lambda o: o["name"].lower()) + title = PY_PACKAGE_TITLES.get(package, package) + dotted = "raven_toolbox" if package == "_toplevel" else f"raven_toolbox.{package}" + (api_dir / "python" / f"{package}.md").write_text( + render_python_page(title, dotted, objs), encoding="utf-8" + ) + python_links.append((title, f"{package}.md")) + (api_dir / "python" / "index.md").write_text( render_group_index( "Python API (raven-toolbox)", "raven-toolbox's packages, by category.", python_links - ) + ), + encoding="utf-8", ) -# --- MATLAB vs Python translation table (top-level page) ------------------- # -# Automatic pairs: the names normalise to the same string. That only catches the -# mechanical cases, so scripts/curated_pairs.yml records the rest -- renamed -# counterparts, cobrapy replacements, and deliberate omissions. -CURATED = yaml.safe_load( - (Path(__file__).resolve().parent / "curated_pairs.yml").read_text(encoding="utf-8") -) + # --- matlab-vs-python.md + matlab_all = {f["name"]: folder for folder, _t in MATLAB_CATEGORIES for f in matlab.get(folder, [])} + py_by_name = {obj["name"]: obj for obj in python_objs} + curated = yaml.safe_load((Path(__file__).resolve().parent / "curated_pairs.yml").read_text(encoding="utf-8")) -matlab_all = {f["name"]: folder for folder, _t in MATLAB_CATEGORIES for f in matlab[folder]} -py_by_name = {obj["name"]: obj for obj in python_objs} - -# Curated entries must resolve, or the build stops: this file is hand-written, -# and an unchecked hand-written name is exactly how the wrong ones got in. -unresolved: list[str] = [] -for m_name, p_name in CURATED["aliases"].items(): - if m_name not in matlab_all: - unresolved.append(f"aliases: RAVEN function '{m_name}' is not on the tracked branch") - if p_name not in py_by_name: - unresolved.append(f"aliases: raven-toolbox function '{p_name}' (for {m_name}) does not exist") -for section in ("cobrapy", "not_ported"): - for m_name in CURATED[section]: + unresolved: list[str] = [] + for m_name, p_name in curated["aliases"].items(): if m_name not in matlab_all: - unresolved.append(f"{section}: RAVEN function '{m_name}' is not on the tracked branch") -if unresolved: - listed = "\n - ".join(unresolved) - raise SystemExit( - f"gen_api_pages: scripts/curated_pairs.yml is out of date:\n - {listed}" - ) - -pairs: list[tuple[str, str, str, str, str, bool]] = [] -seen_matlab: set[str] = set() -for folder, _title in MATLAB_CATEGORIES: - for f in matlab[folder]: - match = py_by_norm.get(norm(f["name"])) - if match: - pairs.append((f["name"], folder, match["name"], match["package"], f["summary"] or match["summary"], False)) - seen_matlab.add(f["name"]) -for m_name, p_name in CURATED["aliases"].items(): - if m_name in seen_matlab: - continue - folder = matlab_all[m_name] - obj = py_by_name[p_name] - m_summary = next((f["summary"] for f in matlab[folder] if f["name"] == m_name), "") - pairs.append((m_name, folder, p_name, obj["package"], m_summary or obj["summary"], True)) - -n_auto = sum(1 for r in pairs if not r[5]) -lines = [ - "# MATLAB vs Python", - "", - "RAVEN ships as a MATLAB toolbox and as the Python package " - "**raven-toolbox**. The two have large overlap, but differ in important " - "ways:", - "", - "- The **MATLAB** version works completely independently, including " - "independently of the COBRA Toolbox, although `ravenCobraWrapper` can " - "translate between the RAVEN and COBRA model formats.", - "- **raven-toolbox** is built on top of " - "[cobrapy](https://cobrapy.readthedocs.io/), so anything cobrapy already " - "does well is used directly rather than reimplemented.", - "- Names are not always a mechanical `camelCase` → `snake_case` rewrite: " - "much of the API was deliberately renamed, so check the table rather than " - "guessing.", - "", - "This page is generated from the source of both toolboxes at build time. " - "Pairs are found automatically where the names match, and completed from a " - "curated list where they do not; every name is verified to exist.", - "", - "## Paired functions", - "", - f"**{len(pairs)}** pairs, {n_auto} matched automatically, " - f"{len(pairs) - n_auto} curated.", - "", - "| RAVEN (MATLAB) | raven-toolbox (Python) | Summary |", - "|---|---|---|", -] -for m_name, m_folder, p_name, p_pkg, text, _curated in sorted(pairs, key=lambda r: r[0].lower()): - m_link = f"[`{m_name}`](api/matlab/{m_folder}.md#{slug(m_name)})" - p_link = f"[`{p_name}`](api/python/{p_pkg}.md#{slug(p_name)})" - lines.append(f"| {m_link} | {p_link} | {cell(text)} |") - -lines += [ - "", - "## Covered by cobrapy", - "", - "These RAVEN functions have no raven-toolbox counterpart because cobrapy " - "already provides the capability. Follow the link for its documentation.", - "", - "| RAVEN (MATLAB) | Use instead |", - "|---|---|", -] -for m_name, target in sorted(CURATED["cobrapy"].items(), key=lambda kv: kv[0].lower()): - folder = matlab_all[m_name] - m_link = f"[`{m_name}`](api/matlab/{folder}.md#{slug(m_name)})" - url = "https://cobrapy.readthedocs.io/en/latest/autoapi/cobra/index.html" - lines.append(f"| {m_link} | [`{target}`]({url}) |") - -lines += [ - "", - "## Deliberately not ported", - "", - "| RAVEN (MATLAB) | Why not |", - "|---|---|", -] -for m_name, reason in sorted(CURATED["not_ported"].items(), key=lambda kv: kv[0].lower()): - folder = matlab_all[m_name] - m_link = f"[`{m_name}`](api/matlab/{folder}.md#{slug(m_name)})" - lines.append(f"| {m_link} | {cell(reason)} |") - -# Whatever is left over is unmapped -- neither paired, nor delegated to cobrapy, -# nor recorded as a deliberate omission. Listing it keeps the table honest about -# its own coverage, and doubles as the work list for extending curated_pairs.yml. -accounted = ( - {r[0] for r in pairs} | set(CURATED["cobrapy"]) | set(CURATED["not_ported"]) -) -unmapped = sorted((n for n in matlab_all if n not in accounted), key=str.lower) -if unmapped: - lines += [ - "", - "## Not yet mapped", - "", - f"**{len(unmapped)}** RAVEN functions are not yet recorded here. Some have " - "a Python counterpart that has not been curated into the table, some are " - "MATLAB-specific plumbing (path handling, argument parsing, printing) " - "with nothing to map to, and some are genuinely absent from " - "raven-toolbox. Until a function appears in one of the tables above, " - "treat its status as unknown rather than as \"not ported\".", - "", + unresolved.append(f"aliases: RAVEN function '{m_name}' is not on the tracked branch") + if p_name not in py_by_name: + unresolved.append(f"aliases: raven-toolbox function '{p_name}' (for {m_name}) does not exist") + for section in ("cobrapy", "not_ported"): + for m_name in curated[section]: + if m_name not in matlab_all: + unresolved.append(f"{section}: RAVEN function '{m_name}' is not on the tracked branch") + if unresolved: + listed = "\n - ".join(unresolved) + raise SystemExit(f"gen_api_pages: scripts/curated_pairs.yml is out of date:\n - {listed}") + + cobrapy_docs_url = "https://cobrapy.readthedocs.io/en/latest/autoapi/cobra/index.html" + + # Each row: (matlab name, matlab folder, python-column markdown, summary text). + rows: list[tuple[str, str, str, str]] = [] + seen_matlab: set[str] = set() + n_auto = 0 + for folder, _title in MATLAB_CATEGORIES: + for f in matlab.get(folder, []): + match = py_by_norm.get(norm(f["name"])) + if match: + p_cell = f"[`{match['name']}`](api/python/{match['package']}.md#{slug(match['name'])})" + rows.append((f["name"], folder, p_cell, f["summary"] or match["summary"])) + seen_matlab.add(f["name"]) + n_auto += 1 + n_curated = 0 + for m_name, p_name in curated["aliases"].items(): + if m_name in seen_matlab: + continue + folder = matlab_all[m_name] + obj = py_by_name[p_name] + m_summary = next((f["summary"] for f in matlab.get(folder, []) if f["name"] == m_name), "") + p_cell = f"[`{p_name}`](api/python/{obj['package']}.md#{slug(p_name)})" + rows.append((m_name, folder, p_cell, m_summary or obj["summary"])) + seen_matlab.add(m_name) + n_curated += 1 + n_cobrapy = 0 + for m_name, target in curated["cobrapy"].items(): + folder = matlab_all[m_name] + m_summary = next((f["summary"] for f in matlab.get(folder, []) if f["name"] == m_name), "") + p_cell = f"`{target}` {{bdg-link-secondary}}`cobrapy <{cobrapy_docs_url}>`" + rows.append((m_name, folder, p_cell, m_summary)) + seen_matlab.add(m_name) + n_cobrapy += 1 + + lines = [ + "# MATLAB vs Python", "", + "RAVEN ships as a MATLAB toolbox and as the Python package **raven-toolbox**. " + "This page is generated from the source of both toolboxes at build time. " + "Pairs are found automatically where the names match, completed from a " + "curated list where they do not, and marked with a " + f"{{bdg-link-secondary}}`cobrapy <{cobrapy_docs_url}>` badge where cobrapy " + "covers the function instead of raven-toolbox. Every name is verified to exist.", + "", "## Paired functions", "", + f"**{len(rows)}** RAVEN functions listed: {n_auto} matched automatically, " + f"{n_curated} from a curated list, {n_cobrapy} covered by cobrapy instead of " + "raven-toolbox.", + "", "| RAVEN (MATLAB) | raven-toolbox (Python) | Summary |", "|---|---|---|", ] - for chunk_start in range(0, len(unmapped), 6): - row = unmapped[chunk_start:chunk_start + 6] - lines.append(", ".join(f"`{n}`" for n in row) + ("," if chunk_start + 6 < len(unmapped) else "")) - -with mkdocs_gen_files.open("matlab-vs-python.md", "w") as fh: - fh.write("\n".join(lines) + "\n") - -# --- literate-nav SUMMARY -------------------------------------------------- # -with mkdocs_gen_files.open("api/SUMMARY.md", "w") as fh: - fh.write("\n".join(summary) + "\n") + for m_name, m_folder, p_cell, text in sorted(rows, key=lambda r: r[0].lower()): + m_link = f"[`{m_name}`](api/matlab/{m_folder}.md#{slug(m_name)})" + lines.append(f"| {m_link} | {p_cell} | {cell(text)} |") + + lines += ["", "## Deliberately not ported", "", "| RAVEN (MATLAB) | Why not |", "|---|---|"] + for m_name, reason in sorted(curated["not_ported"].items(), key=lambda kv: kv[0].lower()): + folder = matlab_all[m_name] + m_link = f"[`{m_name}`](api/matlab/{folder}.md#{slug(m_name)})" + lines.append(f"| {m_link} | {cell(reason)} |") + + accounted = {r[0] for r in rows} | set(curated["not_ported"]) + unmapped = sorted((n for n in matlab_all if n not in accounted), key=str.lower) + if unmapped: + lines += ["", "## Not yet mapped", "", + f"**{len(unmapped)}** RAVEN functions are not yet recorded here.", ""] + for chunk_start in range(0, len(unmapped), 6): + row = unmapped[chunk_start:chunk_start + 6] + lines.append(", ".join(f"`{n}`" for n in row) + ("," if chunk_start + 6 < len(unmapped) else "")) + + (docs_root / "matlab-vs-python.md").write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/scripts/gen_api_pages_sphinx.py b/scripts/gen_api_pages_sphinx.py deleted file mode 100644 index 87df5a9..0000000 --- a/scripts/gen_api_pages_sphinx.py +++ /dev/null @@ -1,392 +0,0 @@ -"""API reference generator for the Sphinx/pydata-sphinx-theme exploration. - -Runs from ``docs/conf.py`` (Sphinx's ``setup()`` hook), before Sphinx reads -the source tree, mirroring what ``scripts/gen_api_pages.py`` does for the -mkdocs-gen-files build. - -**Not the same rendering as the MkDocs build.** mkdocs-material's version -defers per-function help text to two live plugins at render time: -mkdocstrings' Python handler (griffe) and a separate mkdocstrings-matlab -handler (tree-sitter). Neither has a drop-in Sphinx equivalent verified to -work the same way -- the MATLAB one in particular is a bespoke handler with -no direct autodoc-family counterpart. Rather than depend on an unverified -third-party extension, this generator extracts the *full* help text itself -(the collection layer already reads the source; ``api_index.py`` just -discards everything past the first line) and writes fully static pages. No -live directive, no package install, no MATLAB runtime -- same constraint the -MkDocs build already keeps, met a different way. -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).resolve().parent)) - -import yaml - -from api_index import ( # noqa: E402 - MATLAB_CATEGORIES, - PY_PACKAGE_TITLES, - PYPKG, - RAVEN as MATLAB, - cell, - is_matlab_function, - matlab_help_block, - matlab_summary_and_body, - module_dotted, - norm, - slug, -) - -import ast -import re -import textwrap - - -# --------------------------------------------------------------------------- # -# Full-text collection (api_index.py's matlab_summary / py_summary keep only -# the first line; these keep everything). # -# --------------------------------------------------------------------------- # -def matlab_help(path: Path, fname: str) -> tuple[str, str, str]: - """(signature, summary, rest of the help text) for one MATLAB function.""" - signature, _lines = matlab_help_block(path) - summary, body = matlab_summary_and_body(path, fname) - return signature, summary, "\n".join(body) - - -def py_help(node: ast.AST) -> tuple[str, str]: - """(summary, rest of the docstring) for one Python function/class node.""" - doc = ast.get_docstring(node) or "" - lines = doc.strip().splitlines() - if not lines: - return "", "" - summary = "" - rest_start = 0 - for idx, line in enumerate(lines): - if line.strip(): - summary = line.strip() - rest_start = idx + 1 - break - rest = "\n".join(lines[rest_start:]).strip() - return summary, rest - - -def collect_matlab_full() -> dict[str, list[dict]]: - cats: dict[str, list[dict]] = {} - for folder, _title in MATLAB_CATEGORIES: - funcs: dict[str, dict] = {} - base = MATLAB / folder - if not base.is_dir(): - continue - for m in base.rglob("*.m"): - if m.stem == "Contents" or not is_matlab_function(m): - continue - signature, summary, body = matlab_help(m, m.stem) - funcs.setdefault( - m.stem, - {"name": m.stem, "signature": signature, "summary": summary, "body": body, "path": m}, - ) - cats[folder] = [funcs[n] for n in sorted(funcs, key=str.lower)] - return cats - - -def collect_python_full() -> list[dict]: - objects: list[dict] = [] - if not PYPKG.is_dir(): - return objects - for py in PYPKG.rglob("*.py"): - try: - tree = ast.parse(py.read_text(encoding="utf-8")) - except (SyntaxError, OSError): - continue - dotted = module_dotted(py) - rel = py.relative_to(PYPKG) - package = rel.parts[0] if len(rel.parts) > 1 else "_toplevel" - for node in tree.body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): - if node.name.startswith("_"): - continue - ident = f"{dotted}.{node.name}" if dotted else node.name - summary, rest = py_help(node) - sig = "" - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - sig = f"({ast.unparse(node.args)})" if hasattr(ast, "unparse") else "" - objects.append( - { - "name": node.name, - "ident": ident, - "package": package, - "summary": summary, - "body": rest, - "signature": sig, - "kind": "class" if isinstance(node, ast.ClassDef) else "function", - } - ) - return objects - - -# --------------------------------------------------------------------------- # -# Help-text rendering # -# --------------------------------------------------------------------------- # -# RAVEN's % help blocks and the structured raven-toolbox docstrings both use -# numpydoc section headers ("Parameters" over a dashed underline), so one -# parser serves both. Parameter sections become definition lists, Examples a -# code block, and anything else passes through as prose. Help text without any -# section header is kept as the author laid it out: in a text block for MATLAB, -# whose help is written for the command window, and as prose for Python. -PARAM_SECTIONS = { - "Parameters", "Name-Value Arguments", "Other Parameters", "Returns", - "Yields", "Raises", "Attributes", -} -_SECTION_HEADER = re.compile(r"^([A-Z][A-Za-z -]+)\n(-{3,})$", re.M) -_RST_ROLE = re.compile(r":(?:py:)?(?:func|class|meth|attr|mod|data|obj|exc):`~?([^`]+)`") - - -def myst_inline(text: str) -> str: - """RST roles and double-backtick literals as MyST code spans.""" - text = _RST_ROLE.sub(lambda m: f"`{m.group(1)}`", text) - return re.sub(r"``([^`]+)``", r"`\1`", text) - - -def split_sections(doc: str) -> list[tuple[str | None, str]]: - parts: list[tuple[str | None, str]] = [] - pos, title = 0, None - for m in _SECTION_HEADER.finditer(doc): - if len(m.group(2)) < len(m.group(1).strip()) - 1: - continue # a dashed line too short to underline this title - parts.append((title, doc[pos:m.start()])) - title, pos = m.group(1).strip(), m.end() - parts.append((title, doc[pos:])) - return parts - - -def param_entries(body: str) -> list[tuple[str, str, str]]: - entries: list[list] = [] - for line in body.strip("\n").splitlines(): - if line and not line[0].isspace(): - name, _, typ = line.partition(" : ") - entries.append([name.strip(), typ.strip(), []]) - elif entries: - entries[-1][2].append(line) - return [(n, t, textwrap.dedent("\n".join(d)).strip()) for n, t, d in entries] - - -def render_help(doc: str, lang: str) -> list[str]: - """Markdown lines for a help text's body (everything after the summary).""" - doc = textwrap.dedent(doc).strip("\n") - if not doc.strip(): - return [] - sections = split_sections(doc) - if all(title is None for title, _ in sections): - if lang == "matlab": - return ["```text", doc, "```", ""] - return [myst_inline(doc), ""] - out: list[str] = [] - for title, body in sections: - body = textwrap.dedent(body).strip("\n") - if title is None: - if body.strip(): - out += [myst_inline(body), ""] - continue - out += [f":::{{rubric}} {title}", ":::", ""] - if title in PARAM_SECTIONS: - for name, typ, desc in param_entries(body): - term = f"`{name}`" + (f" : *{typ}*" if typ else "") - lines = myst_inline(desc).splitlines() or [""] - out += [term, f": {lines[0]}"] + [f" {line}" if line else "" for line in lines[1:]] + [""] - elif title == "Examples": - fence = "pycon" if ">>>" in body else lang - out += [f"```{fence}", body, "```", ""] - else: - out += [myst_inline(body), ""] - return out - - -# --------------------------------------------------------------------------- # -# Page rendering # -# --------------------------------------------------------------------------- # -def render_matlab_page(title: str, folder: str, funcs: list[dict]) -> str: - out = [f"# {title}", "", f"MATLAB functions in `RAVEN/src/{folder}` of the RAVEN toolbox.", "", - "## Functions", "", "| Function | Summary |", "|---|---|"] - for f in funcs: - out.append(f"| [`{f['name']}`](#{slug(f['name'])}) | {cell(myst_inline(f['summary']))} |") - out += ["", "## Reference", ""] - for f in funcs: - # A cross-document markdown link with a #fragment resolves against - # the target page's auto-generated heading anchor, so this heading - # must auto-slug to exactly `slug(name)` -- true here since it is a - # bare MATLAB identifier, nothing an explicit target would improve. - out += [f"### {f['name']}", ""] - if f["signature"]: - out += ["```matlab", f["signature"], "```", ""] - if f["summary"]: - out += [myst_inline(f["summary"]), ""] - out += render_help(f["body"], "matlab") - return "\n".join(out) + "\n" - - -def render_python_page(title: str, dotted: str, objs: list[dict]) -> str: - out = [f"# {title} (Python)", "", f"`raven-toolbox` objects in `{dotted}`.", "", - "## Functions", "", "| Function | Summary |", "|---|---|"] - for o in objs: - out.append(f"| [`{o['name']}`](#{slug(o['name'])}) | {cell(myst_inline(o['summary']))} |") - out += ["", "## Reference", ""] - for o in objs: - # The heading is the bare name, not name+signature: a cross-document - # markdown link with a #fragment resolves against the target page's - # auto-generated heading anchor, not an explicit MyST target, so a - # heading has to auto-slug to exactly `slug(name)` to be linkable at - # all. The signature goes on its own line underneath instead. - out += [f"### {o['name']}", ""] - if o["signature"]: - out += [f"`{o['name']}{o['signature']}`", ""] - if o["summary"]: - out += [myst_inline(o["summary"]), ""] - out += render_help(o["body"], "python") - return "\n".join(out) + "\n" - - -def render_group_index(title: str, intro: str, links: list[tuple[str, str]]) -> str: - stems = [href.rsplit(".", 1)[0] for _title, href in links] - out = ["---", "icon: material/folder-open", "---", "", f"# {title}", "", intro, "", - "```{toctree}", ":hidden:"] - out += stems - out += ["```", ""] - for link_title, href in links: - out.append(f"- [{link_title}]({href})") - return "\n".join(out) + "\n" - - -def generate(docs_root: Path) -> None: - api_dir = docs_root / "api" - (api_dir / "matlab").mkdir(parents=True, exist_ok=True) - (api_dir / "python").mkdir(parents=True, exist_ok=True) - - matlab = collect_matlab_full() - python_objs = collect_python_full() - py_by_norm: dict[str, dict] = {} - for obj in python_objs: - py_by_norm.setdefault(norm(obj["name"]), obj) - - matlab_links: list[tuple[str, str]] = [] - for folder, title in MATLAB_CATEGORIES: - funcs = matlab.get(folder, []) - if not funcs: - continue - (api_dir / "matlab" / f"{folder}.md").write_text( - render_matlab_page(title, folder, funcs), encoding="utf-8" - ) - matlab_links.append((title, f"{folder}.md")) - (api_dir / "matlab" / "index.md").write_text( - render_group_index("MATLAB API (RAVEN)", "RAVEN's MATLAB functions, by category.", matlab_links), - encoding="utf-8", - ) - - by_package: dict[str, list[dict]] = {} - for obj in python_objs: - by_package.setdefault(obj["package"], []).append(obj) - python_links: list[tuple[str, str]] = [] - for package in sorted(by_package, key=lambda p: (p == "_toplevel", PY_PACKAGE_TITLES.get(p, p).lower())): - objs = sorted(by_package[package], key=lambda o: o["name"].lower()) - title = PY_PACKAGE_TITLES.get(package, package) - dotted = "raven_toolbox" if package == "_toplevel" else f"raven_toolbox.{package}" - (api_dir / "python" / f"{package}.md").write_text( - render_python_page(title, dotted, objs), encoding="utf-8" - ) - python_links.append((title, f"{package}.md")) - (api_dir / "python" / "index.md").write_text( - render_group_index( - "Python API (raven-toolbox)", "raven-toolbox's packages, by category.", python_links - ), - encoding="utf-8", - ) - - # --- matlab-vs-python.md: identical logic to gen_api_pages.py, static write - matlab_all = {f["name"]: folder for folder, _t in MATLAB_CATEGORIES for f in matlab.get(folder, [])} - py_by_name = {obj["name"]: obj for obj in python_objs} - curated = yaml.safe_load((Path(__file__).resolve().parent / "curated_pairs.yml").read_text(encoding="utf-8")) - - unresolved: list[str] = [] - for m_name, p_name in curated["aliases"].items(): - if m_name not in matlab_all: - unresolved.append(f"aliases: RAVEN function '{m_name}' is not on the tracked branch") - if p_name not in py_by_name: - unresolved.append(f"aliases: raven-toolbox function '{p_name}' (for {m_name}) does not exist") - for section in ("cobrapy", "not_ported"): - for m_name in curated[section]: - if m_name not in matlab_all: - unresolved.append(f"{section}: RAVEN function '{m_name}' is not on the tracked branch") - if unresolved: - listed = "\n - ".join(unresolved) - raise SystemExit(f"gen_api_pages_sphinx: scripts/curated_pairs.yml is out of date:\n - {listed}") - - cobrapy_docs_url = "https://cobrapy.readthedocs.io/en/latest/autoapi/cobra/index.html" - - # Each row: (matlab name, matlab folder, python-column markdown, summary text). - rows: list[tuple[str, str, str, str]] = [] - seen_matlab: set[str] = set() - n_auto = 0 - for folder, _title in MATLAB_CATEGORIES: - for f in matlab.get(folder, []): - match = py_by_norm.get(norm(f["name"])) - if match: - p_cell = f"[`{match['name']}`](api/python/{match['package']}.md#{slug(match['name'])})" - rows.append((f["name"], folder, p_cell, f["summary"] or match["summary"])) - seen_matlab.add(f["name"]) - n_auto += 1 - n_curated = 0 - for m_name, p_name in curated["aliases"].items(): - if m_name in seen_matlab: - continue - folder = matlab_all[m_name] - obj = py_by_name[p_name] - m_summary = next((f["summary"] for f in matlab.get(folder, []) if f["name"] == m_name), "") - p_cell = f"[`{p_name}`](api/python/{obj['package']}.md#{slug(p_name)})" - rows.append((m_name, folder, p_cell, m_summary or obj["summary"])) - seen_matlab.add(m_name) - n_curated += 1 - n_cobrapy = 0 - for m_name, target in curated["cobrapy"].items(): - folder = matlab_all[m_name] - m_summary = next((f["summary"] for f in matlab.get(folder, []) if f["name"] == m_name), "") - p_cell = f"`{target}` {{bdg-link-secondary}}`cobrapy <{cobrapy_docs_url}>`" - rows.append((m_name, folder, p_cell, m_summary)) - seen_matlab.add(m_name) - n_cobrapy += 1 - - lines = [ - "# MATLAB vs Python", "", - "RAVEN ships as a MATLAB toolbox and as the Python package **raven-toolbox**. " - "This page is generated from the source of both toolboxes at build time. " - "Pairs are found automatically where the names match, completed from a " - "curated list where they do not, and marked with a " - f"{{bdg-link-secondary}}`cobrapy <{cobrapy_docs_url}>` badge where cobrapy " - "covers the function instead of raven-toolbox. Every name is verified to exist.", - "", "## Paired functions", "", - f"**{len(rows)}** RAVEN functions listed: {n_auto} matched automatically, " - f"{n_curated} from a curated list, {n_cobrapy} covered by cobrapy instead of " - "raven-toolbox.", - "", "| RAVEN (MATLAB) | raven-toolbox (Python) | Summary |", "|---|---|---|", - ] - for m_name, m_folder, p_cell, text in sorted(rows, key=lambda r: r[0].lower()): - m_link = f"[`{m_name}`](api/matlab/{m_folder}.md#{slug(m_name)})" - lines.append(f"| {m_link} | {p_cell} | {cell(text)} |") - - lines += ["", "## Deliberately not ported", "", "| RAVEN (MATLAB) | Why not |", "|---|---|"] - for m_name, reason in sorted(curated["not_ported"].items(), key=lambda kv: kv[0].lower()): - folder = matlab_all[m_name] - m_link = f"[`{m_name}`](api/matlab/{folder}.md#{slug(m_name)})" - lines.append(f"| {m_link} | {cell(reason)} |") - - accounted = {r[0] for r in rows} | set(curated["not_ported"]) - unmapped = sorted((n for n in matlab_all if n not in accounted), key=str.lower) - if unmapped: - lines += ["", "## Not yet mapped", "", - f"**{len(unmapped)}** RAVEN functions are not yet recorded here.", ""] - for chunk_start in range(0, len(unmapped), 6): - row = unmapped[chunk_start:chunk_start + 6] - lines.append(", ".join(f"`{n}`" for n in row) + ("," if chunk_start + 6 < len(unmapped) else "")) - - (docs_root / "matlab-vs-python.md").write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/scripts/migrate_to_myst.py b/scripts/migrate_to_myst.py deleted file mode 100644 index 73371d2..0000000 --- a/scripts/migrate_to_myst.py +++ /dev/null @@ -1,180 +0,0 @@ -"""One-time converter: MkDocs/pymdownx Markdown -> MyST, for the Sphinx exploration. - -Run once against every page under docs/ (see __main__ below). Handles, in order: - -1. Content tabs (``=== "MATLAB"`` / ``=== "Python"``) -> sphinx-design - ``::::{tab-set}`` / ``:::{tab-item} MATLAB`` fences. -2. Admonitions (``!!! note "Title"``) -> MyST ``:::{note} Title`` fences. - ``info`` has no native docutils admonition, so it becomes a generic - ``:::{admonition} Title`` with ``:class: info``. -3. ```` ```text title="Output" ```` fences -> plain ```` ```text ```` fences; - MyST/pygments does not accept pymdownx's ``title=`` attribute in the info - string. -4. ``--8<-- "path"`` snippet includes (the fenced-block form; the only one - this repo uses) -> a ``{literalinclude}`` directive, path adjusted for the - including document's depth under ``docs/``. -5. ``## Title {#custom-id}`` (attr_list heading IDs) -> an explicit - ``(custom-id)=`` MyST target line before the heading. - -Both block extractors work on indentation, not a fixed column, so a converted -tab body is dedented before the admonition pass runs over it -- an admonition -nested inside a tab (or vice versa) is picked up on the same pass because the -dedent preserves relative indentation. - -Idempotent: re-running on already-converted text is a no-op, since neither -pattern is matched with the MyST spelling as the marker. -""" - -from __future__ import annotations - -import re -import sys -from pathlib import Path - -TAB_MARKER = re.compile(r'^( *)=== "([^"]+)"\s*$') -ADMONITION_MARKER = re.compile(r'^( *)!!! (\w+)(?: "([^"]*)")?\s*$') -OUTPUT_TITLE = re.compile(r'^(```text) title="Output(?:, [^"]*)?"\s*$') -# attr_list heading IDs: "## Title {#custom-id}" -> an explicit MyST target -# line before the heading, since MyST does not read this trailing-brace form. -HEADING_ID = re.compile(r'^(#+\s+.*?)\s*\{#([\w-]+)\}\s*$', re.M) -# pymdownx.snippets, fenced-block form only (the only form this repo uses): -# ```lang -# --8<-- "path/relative/to/repo/root" -# ``` -SNIPPET_FENCE = re.compile(r'^```(\w+)\n--8<-- "([^"]+)"\n```$', re.M) - -ADMONITION_NATIVE = {"note", "tip", "warning", "danger", "attention", "caution", "error", "hint", "important", "seealso"} - - -def _block_body(lines: list[str], start: int, marker_indent: int) -> tuple[list[str], int]: - """Lines belonging to a block whose marker sits at ``marker_indent``. - - A line belongs if it is blank, or indented at least ``marker_indent + 4``. - Returns the body (dedented by ``marker_indent + 4``) and the index of the - first line after the block. - """ - body_indent = marker_indent + 4 - body: list[str] = [] - i = start - while i < len(lines): - line = lines[i] - if line.strip() == "": - body.append("") - i += 1 - continue - indent = len(line) - len(line.lstrip(" ")) - if indent < body_indent: - break - body.append(line[body_indent:]) - i += 1 - # Trailing blank lines are trimmed from the rendered block for cosmetics - # only; `i` already correctly points past them at the line that ended the - # block (a sibling marker or unrelated content), and must stay there. - while body and body[-1] == "": - body.pop() - return body, i - - -def convert_tabs(text: str) -> str: - lines = text.split("\n") - out: list[str] = [] - i = 0 - while i < len(lines): - m = TAB_MARKER.match(lines[i]) - if not m: - out.append(lines[i]) - i += 1 - continue - - indent = m.group(1) - n = len(indent) - out.append(f"{indent}::::{{tab-set}}") - # Consume every sibling "=== " block at this same indent as one set; - # a line that is not itself such a marker ends the set, regardless of - # its own indentation (it is sibling content in the parent block). - while i < len(lines) and (mm := TAB_MARKER.match(lines[i])) and len(mm.group(1)) == n: - label = mm.group(2) - body, i = _block_body(lines, i + 1, n) - body_text = convert_tabs("\n".join(body)) # recurse: nested tab sets - out.append(f"{indent}:::{{tab-item}} {label}") - out.append(body_text) - out.append(f"{indent}:::") - out.append(f"{indent}::::") - out.append("") # docutils wants a blank line after a closed directive - return "\n".join(out) - - -def convert_admonitions(text: str) -> str: - lines = text.split("\n") - out: list[str] = [] - i = 0 - while i < len(lines): - m = ADMONITION_MARKER.match(lines[i]) - if m: - indent, kind, title = m.group(1), m.group(2), m.group(3) - n = len(indent) - body, i = _block_body(lines, i + 1, n) - body_text = convert_admonitions("\n".join(body)) # recurse: nested admonitions - if kind in ADMONITION_NATIVE: - out.append(f"{indent}:::{{{kind}}}" + (f" {title}" if title else "")) - else: - out.append(f"{indent}:::{{admonition}}" + (f" {title}" if title else " Note")) - out.append(f"{indent}:class: {kind}") - out.append(body_text) - out.append(f"{indent}:::") - out.append("") # docutils wants a blank line after a closed directive - continue - out.append(lines[i]) - i += 1 - return "\n".join(out) - - -def collapse_blank_runs(text: str) -> str: - """Squash 3+ consecutive blank lines down to 1, a side effect of always - inserting a blank line after a closed directive even where one already - followed in the source.""" - return re.sub(r"\n{3,}", "\n\n", text) - - -def convert_output_titles(text: str) -> str: - return "\n".join(OUTPUT_TITLE.sub(r"\1", line) for line in text.split("\n")) - - -def convert_heading_ids(text: str) -> str: - return HEADING_ID.sub(lambda m: f"({m.group(2)})=\n{m.group(1)}", text) - - -def convert_snippets(text: str, depth: int) -> str: - """``--8<-- "path"`` (relative to the repo root) -> a literalinclude, - relative to the including document -- ``depth`` is how many directories - the document sits below ``docs/`` (0 for ``docs/index.md`` itself).""" - up = "../" * (depth + 1) - - def repl(m: re.Match) -> str: - lang, rel = m.group(1), m.group(2) - return f"```{{literalinclude}} {up}{rel}\n:language: {lang}\n```" - - return SNIPPET_FENCE.sub(repl, text) - - -def convert(text: str, depth: int = 0) -> str: - text = convert_tabs(text) - text = convert_admonitions(text) - text = convert_output_titles(text) - text = convert_heading_ids(text) - text = convert_snippets(text, depth) - text = collapse_blank_runs(text) - return text - - -if __name__ == "__main__": - root = Path(sys.argv[1]) if len(sys.argv) > 1 else Path("docs") - changed = 0 - for path in sorted(root.rglob("*.md")): - original = path.read_text(encoding="utf-8") - depth = len(path.relative_to(root).parent.parts) - converted = convert(original, depth) - if converted != original: - path.write_text(converted, encoding="utf-8", newline="\n") - changed += 1 - print(f"converted {changed} file(s) under {root}") From 673024897e4fed298d1c9263cf0e22c35e9b56cf Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sat, 17 Oct 2026 23:28:24 +0200 Subject: [PATCH 31/32] chore: update RAVEN and raven-toolbox to develop --- RAVEN | 2 +- raven-toolbox | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/RAVEN b/RAVEN index bcdc2b4..250b0bc 160000 --- a/RAVEN +++ b/RAVEN @@ -1 +1 @@ -Subproject commit bcdc2b4130817afb6e9c71fd06ef1af829ada4a1 +Subproject commit 250b0bc4cff770f02228381c6dfd39708a828c4b diff --git a/raven-toolbox b/raven-toolbox index 16da145..ece4835 160000 --- a/raven-toolbox +++ b/raven-toolbox @@ -1 +1 @@ -Subproject commit 16da1453d3eb04217da4fa8308fbcff8b7757d3c +Subproject commit ece483541a244f62c4e4477cad68393c704a7fa9 From 58969c1d2fb3e54df77e42d7c8c60ffc4b3710ad Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Sun, 18 Oct 2026 00:41:35 +0200 Subject: [PATCH 32/32] docs: link guide function names to the API reference and to cobrapy scripts/autolink.py rewrites the inline code spans of the guide pages at build time: RAVEN and raven-toolbox names link to their entries in the generated API pages, cobrapy names to cobrapy's documentation through its intersphinx inventory. Every name in a table row is linked, and the first mention per section in the prose; the cobrapy badge in a table row points at the object in that row. --- DESIGN.md | 3 +- README.md | 3 + docs/conf.py | 14 +++- scripts/autolink.py | 177 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 193 insertions(+), 4 deletions(-) create mode 100644 scripts/autolink.py diff --git a/DESIGN.md b/DESIGN.md index f7c24d2..b8ba6ed 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -146,7 +146,8 @@ Conventions: - **Page anatomy.** Title and one-paragraph statement of the task; a "Functions on this page" table (MATLAB | Python | note) linking into the API - reference; setup; two to five numbered steps, each with a tabbed code block and + reference (`scripts/autolink.py` adds the links at build time: every name in + the table and the first mention per section in the prose); setup; two to five numbered steps, each with a tabbed code block and its real output; "What can go wrong"; "See also". - **cobrapy is marked, three ways**: a badge in the function table linking to cobrapy's docs, an explicit import in the snippet diff --git a/README.md b/README.md index ff3b0ed..eda04b8 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,9 @@ The API reference pages are generated at the start of each build by from `docs/conf.py`, which walks the two toolbox submodules and pairs their functions. No MATLAB runtime or installed Python package is required; docstrings are collected statically from source. +[`scripts/autolink.py`](scripts/autolink.py) then links the function names on +the guide pages to those reference pages, and cobrapy names to cobrapy's +documentation through its intersphinx inventory. ## Checking the guide examples diff --git a/docs/conf.py b/docs/conf.py index 97a0d0a..353adf1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,8 +14,13 @@ "myst_parser", "sphinx_design", "sphinx_copybutton", + "sphinx.ext.intersphinx", ] +# cobrapy's inventory, which scripts/autolink.py uses to link cobrapy names on +# the guide pages to their entries in cobrapy's documentation. +intersphinx_mapping = {"cobra": ("https://cobrapy.readthedocs.io/en/latest/", None)} + source_suffix = {".md": "markdown"} root_doc = "index" @@ -91,15 +96,18 @@ def setup(app): - """Run the API-reference generator before Sphinx reads the source tree. + """Generate the API reference and link the guide pages to it. - It collects from the tracked RAVEN / raven-toolbox submodules and writes - static pages, before Sphinx's own file discovery runs. + The generator collects from the tracked RAVEN / raven-toolbox submodules + and writes static pages before Sphinx's own file discovery runs; + scripts/autolink.py then links function names on the guide pages. """ from pathlib import Path import sys sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + import autolink import gen_api_pages gen_api_pages.generate(Path(__file__).resolve().parent) + return autolink.setup(app) diff --git a/scripts/autolink.py b/scripts/autolink.py new file mode 100644 index 0000000..2289cb2 --- /dev/null +++ b/scripts/autolink.py @@ -0,0 +1,177 @@ +"""Link function names on the guide pages to their API reference. + +A Sphinx extension. On ``source-read`` it rewrites the inline code spans of +``guide/*`` pages, outside fenced code, headings and existing links: + +* a RAVEN function becomes a link to its entry in the generated MATLAB API + pages (``api/matlab/.md``); +* a raven-toolbox function or class becomes a link to its entry in the Python + API pages (``api/python/.md``); +* a cobrapy object becomes a link to its entry in cobrapy's documentation, + looked up in cobrapy's intersphinx inventory. Single names such as ``pfba`` + or ``Model`` are only looked up in a table row that carries the cobrapy + badge; elsewhere a cobrapy name has to be dotted (``Model.optimize``) or + snake_case (``read_sbml_model``), so ordinary words are never linked. + +Every span in a table row is linked, so the "Functions on this page" table +links each entry; in prose only the first mention of a name per section is. +The cobrapy badge in a table row points at the first cobrapy object linked in +that row instead of cobrapy's index page. + +The page sources on disk are not changed, so ``scripts/run_examples.py``, which +reads them directly, sees the same code blocks. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from api_index import collect_matlab, collect_python, slug # noqa: E402 +from sphinx.util import logging # noqa: E402 + +COBRA_INDEX = "https://cobrapy.readthedocs.io/en/latest/autoapi/cobra/index.html" + +_FENCE = re.compile(r"^\s*(```|~~~)") +_HEADING = re.compile(r"^#{1,6}\s") +# An inline code span that is not the text of a markdown link ([`x`](...)) and +# not the content of a role ({bdg-link-secondary}`...`). +_SPAN = re.compile(r"(?`") +_IDENT = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)*$") +_SNAKE = re.compile(r"^[a-z][a-z0-9]+(?:_[a-z0-9]+)+$") +_COBRA_TYPES = ("py:function", "py:method", "py:class", "py:attribute", "py:property") +# raven-toolbox names that mean something else on the guide pages: `main` is a +# command-line entry point in the package and a branch name in the prose. +_NOT_LINKED = {"main"} + + +class Linker: + def __init__(self, cobra_inventory: dict | None) -> None: + self.matlab = {f["name"]: folder for folder, funcs in collect_matlab().items() for f in funcs} + packages: dict[str, set[str]] = {} + for obj in collect_python(): + packages.setdefault(obj["name"], set()).add(obj["package"]) + # A name defined in two packages has no single target: leave it unlinked. + self.python = {name: next(iter(pk)) for name, pk in packages.items() if len(pk) == 1} + self.cobra = self._cobra_index(cobra_inventory or {}) + self.counts = {"matlab": 0, "python": 0, "cobrapy": 0, "badges": 0} + + @staticmethod + def _cobra_index(inventory: dict) -> dict[str, str]: + """Every dotted suffix of a cobrapy object's full name -> its URL. + + ``cobra.core.model.Model.optimize`` is reachable as ``optimize``, + ``Model.optimize``, ``model.Model.optimize`` and so on. Where two + objects share a suffix, the one with the shorter full name wins. + """ + best: dict[str, tuple[int, str, str]] = {} + for role in _COBRA_TYPES: + for full, entry in inventory.get(role, {}).items(): + url = getattr(entry, "uri", None) or entry[2] + parts = full.split(".") + for i in range(len(parts)): + key = ".".join(parts[i:]) + rank = (len(parts), full, url) + if key not in best or rank < best[key]: + best[key] = rank + return {key: url for key, (_n, _f, url) in best.items()} + + def target(self, token: str, in_badged_row: bool) -> tuple[str, str] | None: + """(kind, href) for a code span, or None to leave it as it is.""" + if token in _NOT_LINKED: + return None + if token in self.matlab: + return "matlab", f"../api/matlab/{self.matlab[token]}.md#{slug(token)}" + if token in self.python: + return "python", f"../api/python/{self.python[token]}.md#{slug(token)}" + if not _IDENT.match(token): + return None + bare = token.removeprefix("cobra.") + if "." in bare or _SNAKE.match(bare) or in_badged_row: + url = self.cobra.get(bare) or self.cobra.get(token) + if url: + return "cobrapy", url + return None + + def link_page(self, text: str) -> str: + out: list[str] = [] + in_fence = False + seen: set[str] = set() + for line in text.split("\n"): + if _FENCE.match(line): + in_fence = not in_fence + out.append(line) + continue + if in_fence: + out.append(line) + continue + if _HEADING.match(line): + seen = set() + out.append(line) + continue + out.append(self._link_line(line, seen)) + return "\n".join(out) + + def _link_line(self, line: str, seen: set[str]) -> str: + if "`" not in line: + return line + in_table = line.lstrip().startswith("|") + badged = in_table and bool(_BADGE.search(line)) + first_cobra: list[str] = [] + + def replace(m: re.Match) -> str: + raw = m.group(1) + token = raw.strip() + token = token[:-2] if token.endswith("()") else token + found = self.target(token, badged) + if found is None: + return m.group(0) + kind, href = found + if not in_table: + if token in seen: + return m.group(0) + seen.add(token) + if kind == "cobrapy" and not first_cobra: + first_cobra.append(href) + self.counts[kind] += 1 + return f"[`{raw}`]({href})" + + line = _SPAN.sub(replace, line) + if badged and first_cobra: + line = _BADGE.sub( + lambda _m: f"{{bdg-link-secondary}}`cobrapy <{first_cobra[0]}>`", line, count=1 + ) + self.counts["badges"] += 1 + return line + + +def _cobra_inventory(app) -> dict | None: + named = getattr(app.env, "intersphinx_named_inventory", None) or {} + return named.get("cobra") + + +def setup(app): + state: dict[str, Linker] = {} + + def on_source_read(app, docname, source): + if not docname.startswith("guide/"): + return + if "linker" not in state: + state["linker"] = Linker(_cobra_inventory(app)) + source[0] = state["linker"].link_page(source[0]) + + def on_build_finished(app, exception): + if "linker" in state: + c = state["linker"].counts + logging.getLogger(__name__).info( + "autolink: %d RAVEN, %d raven-toolbox and %d cobrapy links; %d cobrapy badges", + c["matlab"], c["python"], c["cobrapy"], c["badges"], + ) + + app.connect("source-read", on_source_read) + app.connect("build-finished", on_build_finished) + return {"parallel_read_safe": True, "parallel_write_safe": True}