Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .github/scripts/assemble_offline_software.py
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
#!/usr/bin/env python
"""Populate RAVEN's software/ with every platform's external binaries.

Downloads the per-platform binary ZIPs from raven-data and lays them out under
Downloads the per-platform binary ZIPs from toolbox-assets and lays them out under
``<raven>/software/<tool>/`` using RAVEN's suffix scheme — bare name = Linux,
``.mac`` = macOS, ``.exe`` = Windows. With all platforms present in one tree, the
whole RAVEN checkout can be zipped into a single **OS-independent** offline
distribution (see the release-bundle workflow).

Run from the RAVEN root, or pass ``--raven <dir>``. Requires network access to
https://github.com/SysBioChalmers/raven-data .
https://github.com/CompMetEng/toolbox-assets .
"""
from __future__ import annotations

Expand All @@ -18,11 +18,11 @@
import zipfile
from pathlib import Path

RAVEN_DATA = "https://github.com/SysBioChalmers/raven-data/releases/download"
# raven-data platform key -> RAVEN binary suffix.
TOOLBOX_ASSETS = "https://github.com/CompMetEng/toolbox-assets/releases/download"
# toolbox-assets platform key -> RAVEN binary suffix.
PLATFORMS = {"linux-x86_64": "", "macos-arm64": ".mac", "windows-x86_64": ".exe"}

# (software subdir, raven-data bundle, {platform|"_": version}, {bare executable names})
# (software subdir, toolbox-assets bundle, {platform|"_": version}, {bare executable names})
CLI_TOOLS = [
("blast+", "blast", {"_": "2.17.0"}, {"blastp", "makeblastdb"}),
("diamond", "diamond", {"_": "2.1.17"}, {"diamond"}),
Expand All @@ -31,7 +31,7 @@


def _fetch(url: str) -> bytes:
with urllib.request.urlopen(url, timeout=120) as resp: # noqa: S310 (pinned raven-data URLs)
with urllib.request.urlopen(url, timeout=120) as resp: # noqa: S310 (pinned toolbox-assets URLs)
return resp.read()


Expand Down Expand Up @@ -60,7 +60,7 @@ def assemble(raven: Path) -> None:
# macOS ships bare names; rename them to <name>.mac. Linux keeps bare names,
# Windows already carries .exe — so only the macOS members need renaming.
rename = {e: e + ".mac" for e in exes} if suffix == ".mac" else {}
_extract(_fetch(f"{RAVEN_DATA}/{bundle}-{version}/{asset}"), dest, rename)
_extract(_fetch(f"{TOOLBOX_ASSETS}/{bundle}-{version}/{asset}"), dest, rename)


def main(argv: list[str] | None = None) -> None:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ jobs:

binary-tests:
name: Binary download (${{ matrix.os }})
# Validate the on-demand fetch of the external binaries from raven-data on
# Validate the on-demand fetch of the external binaries from toolbox-assets on
# Linux and macOS. Running the downloaded binaries also confirms the macOS
# Gatekeeper quarantine is cleared so they can execute.
strategy:
Expand Down
6 changes: 3 additions & 3 deletions checkRaven.m
Original file line number Diff line number Diff line change
Expand Up @@ -251,13 +251,13 @@
printOrange(' Skipping check of binary executables\n')
else
% The BLAST+/DIAMOND/HMMER binaries are downloaded on demand from
% raven-data; fetch any that are missing for this platform before
% toolbox-assets; fetch any that are missing for this platform before
% checking. This doubles as the up-front "prefetch" for users who want
% everything ready after installation.
try
downloadRavenBinaries({'blast+','diamond','hmmer'});
catch ME
printOrange([' Could not download binaries from raven-data: ' ME.message '\n' ...
printOrange([' Could not download binaries from toolbox-assets: ' ME.message '\n' ...
' (offline? fetch the "*-binaries" RAVEN release for an offline install)\n']);
end
% Run each bundled binary with a version/help flag to confirm it
Expand Down Expand Up @@ -294,7 +294,7 @@ function testSolver(solver, model)
function testBinary(ravenDir, tool, binName, versionArg)
% Run a bundled binary with versionArg to confirm it executes; error if not.
if ispc
binEnd = '.exe'; % native Windows builds (raven-data windows-x86_64 ZIPs)
binEnd = '.exe'; % native Windows builds (toolbox-assets windows-x86_64 ZIPs)
elseif ismac
binEnd = '.mac';
else
Expand Down
6 changes: 3 additions & 3 deletions docs/v3_review.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ So the review does not repeat completed work. v3 already:
and `combnk`.
- Dropped the Statistics, Bioinformatics and Text Analytics toolbox dependencies.
- Moved KEGG HMM libraries and external binaries out of git to on-demand fetch from the
`raven-data` release.
`toolbox-assets` release.
- Reorganized the repo into functional folders; reformatted docstrings to NumPy style.
- Added `progressReport`, `parseRAVENargs` (positional-or-named arguments), and a
`matlab.unittest` suite.
Expand Down Expand Up @@ -147,7 +147,7 @@ Plus a standalone `mapIDsViaMNXref(type, ids, fromDB, toDB)` ID translator.
1. Relocate to the develop3 layout (e.g. `reconstruction/metanetx/` or `annotation/metanetx/`),
drop the duplicated `struct_conversion/*`, rebase on develop3's `convertMiriams`.
2. **Delete the committed data**; host MNXref 4.5 (gzipped TSVs, and ideally a prebuilt `.mat`)
as a `raven-data` release asset and add a `downloadMNXref.m` modeled exactly on the
as a `toolbox-assets` release asset and add a `downloadMNXref.m` modeled exactly on the
develop3 KEGG-HMM / `downloadRavenBinaries` on-demand pattern (sentinel-file check, cache
under a gitignored dir).
3. Update parsing to the documented **4.5** columns (header-driven `readtable`, handle the
Expand Down Expand Up @@ -379,6 +379,6 @@ tracked as a low-priority item in the Tier-3 table.
2. **Where should new strain-design / community code that needs metaheuristics live**, given
the dependency question — lean on the Global Optimization Toolbox (`gamultiobj`), or keep
v3's "minimize toolbox dependencies" stance and ship a self-contained EA?
4. **MNXref data hosting** — confirm the `raven-data` release is the right home for the MNXref
4. **MNXref data hosting** — confirm the `toolbox-assets` release is the right home for the MNXref
4.5 assets (mirroring KEGG HMMs/binaries), and whether to ship a prebuilt `.mat` (7 MB,
fast) or only the gzipped TSVs (slower first build, always current).
10 changes: 5 additions & 5 deletions installation/downloadRavenBinaries.m
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
function downloadRavenBinaries(tools)
% downloadRavenBinaries
% Download RAVEN's external command-line binaries on demand from the
% raven-data repository into RAVEN's software/ directory. Only the binary
% toolbox-assets repository into RAVEN's software/ directory. Only the binary
% matching the current platform is fetched, and only when needed.
%
% Input:
Expand All @@ -14,7 +14,7 @@ function downloadRavenBinaries(tools)
% unzip it into the RAVEN software/ directory instead.
%
% The binaries are hosted as release assets in
% https://github.com/SysBioChalmers/raven-data.
% https://github.com/CompMetEng/toolbox-assets.
%
% Usage: downloadRavenBinaries({'blast+','diamond'})
%
Expand All @@ -29,7 +29,7 @@ function downloadRavenBinaries(tools)

ravenDir = findRAVENroot();

% raven-data platform key for the per-platform binary ZIPs.
% toolbox-assets platform key for the per-platform binary ZIPs.
if ispc
plat = 'windows-x86_64';
elseif ismac
Expand Down Expand Up @@ -78,11 +78,11 @@ function downloadRavenBinaries(tools)
continue; % already provisioned
end

zipPath = fetchRavenDataAsset(fullfile(ravenDir,'software'),tag,asset);
zipPath = fetchToolboxAsset(fullfile(ravenDir,'software'),tag,asset);
unzip(zipPath,destDir);
delete(zipPath);

% macOS uses the .mac suffix throughout RAVEN, but the raven-data macOS ZIP
% macOS uses the .mac suffix throughout RAVEN, but the toolbox-assets macOS ZIP
% ships bare names, so rename the extracted executables to match.
if ismac
for k = 1:numel(execs)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
function localPath=fetchRavenDataAsset(destDir,releaseTag,assetName)
% fetchRavenDataAsset Download (if needed) a raven-data release asset.
function localPath=fetchToolboxAsset(destDir,releaseTag,assetName)
% fetchToolboxAsset Download (if needed) a toolbox-assets release asset.
%
% Downloads assetName from the given raven-data release tag into destDir
% Downloads assetName from the given release tag of
% https://github.com/CompMetEng/toolbox-assets into destDir
% and returns its local path. An already-downloaded copy is reused as-is.
% Generic across raven-data's release families (KEGG artefacts, HMM
% Generic across toolbox-assets' release families (KEGG artefacts, HMM
% libraries, binaries, ...): nothing here is specific to any one of
% them, so callers supply the release tag and asset name rather than
% this function hardcoding either.
Expand All @@ -13,7 +14,7 @@
% destDir : char
% directory to download into (created if it does not exist yet).
% releaseTag : char
% the raven-data release tag, e.g. 'kegg118'.
% the toolbox-assets release tag, e.g. 'kegg118'.
% assetName : char
% the release asset file name, e.g. 'kegg118_core.tar.gz'.
%
Expand All @@ -31,7 +32,7 @@
end
fprintf(['Downloading ' assetName '... ']);
try
websave(localPath,['https://github.com/SysBioChalmers/raven-data/releases/download/' releaseTag '/' assetName]);
websave(localPath,['https://github.com/CompMetEng/toolbox-assets/releases/download/' releaseTag '/' assetName]);
catch ME
if strcmp(ME.identifier,'MATLAB:webservices:HTTP404StatusCodeError')
error('Failed to download %s, the server returned a 404 error, try again later. If the problem persists please report it on the RAVEN GitHub Issues page: https://github.com/SysBioChalmers/RAVEN/issues',assetName)
Expand Down
4 changes: 2 additions & 2 deletions reconstruction/kegg/KEGG_VERSION.md
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
kegg118

This is the raven-data release tag that every KEGG-artefact download in
This is the toolbox-assets release tag that every KEGG-artefact download in
this folder reads from --- getModelFromKEGG's core reference-model/table
bundle, and getKEGGModelForOrganism's HMM libraries. It is the single
place this version is recorded; every function that downloads a KEGG
artefact reads it from here via keggDataVersion.m, instead of
hardcoding a "kegg###" string of its own. Bump this file, not the
calling code, when raven-data publishes a new KEGG release.
calling code, when toolbox-assets publishes a new KEGG release.

The version is read as the file's first line; everything below this
point is only for a human reader.
4 changes: 2 additions & 2 deletions reconstruction/kegg/buildGlobalKEGGModel.m
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
function [model,KOModel,isSpontaneous,isUndefinedStoich,isIncomplete,isGeneral]=buildGlobalKEGGModel(ravenPath)
% buildGlobalKEGGModel Download and assemble the global KEGG model.
%
% Fetches the raven-data <kegg version>_core.tar.gz bundle (the
% Fetches the toolbox-assets <kegg version>_core.tar.gz bundle (the
% gene-free reference model plus the ko_reaction / organism_gene_ko /
% rxn_flags relational tables), then joins every organism's
% genes onto the reference reactions through their shared KO ids. The
Expand All @@ -27,7 +27,7 @@

kver=keggDataVersion();
keggDir=fullfile(ravenPath,'reconstruction','kegg');
archive=fetchRavenDataAsset(keggDir,kver,[kver '_core.tar.gz']);
archive=fetchToolboxAsset(keggDir,kver,[kver '_core.tar.gz']);

coreDir=fullfile(keggDir,[kver '_core']);
if ~isfolder(coreDir)
Expand Down
8 changes: 4 additions & 4 deletions reconstruction/kegg/getKEGGModelForOrganism.m
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
% and whether the HMMs were trained on pro- or eukaryotic sequences. The
% directory name matches the published HMM library it is paired with. The
% prebuilt concatenated KO HMM library (dataDir.hmm) is downloaded here
% from the corresponding raven-data release if not already present. This
% from the corresponding toolbox-assets release if not already present. This
% parameter should ALWAYS be provided.
% outDir : char
% directory to save the results from the quering of the hidden Markov
Expand Down Expand Up @@ -197,8 +197,8 @@
%pre-trained HMM set, compatible with the current RAVEN version. The
%reconstruction uses the concatenated KO HMM library (a single
%gzip-compressed flatfile, queried in one hmmsearch); if it is not already
%present it is downloaded from the corresponding raven-data release
%(https://github.com/SysBioChalmers/raven-data).
%present it is downloaded from the corresponding toolbox-assets release
%(https://github.com/CompMetEng/toolbox-assets).
%Only needed for the protein-homology path (fastaFile supplied): the
%annotation-only path never touches libraryFile, so skip the (100+ MB)
%download/extraction entirely when there is no FASTA file to search.
Expand All @@ -219,7 +219,7 @@
gunzip([libraryFile '.gz']);
fprintf('COMPLETE\n');
else
fetchRavenDataAsset(fileparts(libraryFile),kver,[hmmName '.hmm.gz']);
fetchToolboxAsset(fileparts(libraryFile),kver,[hmmName '.hmm.gz']);
fprintf('Extracting the HMM library file... ');
gunzip([libraryFile '.gz']);
fprintf('COMPLETE\n');
Expand Down
2 changes: 1 addition & 1 deletion reconstruction/kegg/getModelFromKEGG.m
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
% Loads the global KEGG reaction/gene model from keggModel.mat. On first
% use --- when no keggModel.mat is present yet --- the underlying
% artefacts (the gene-free reference model plus the KO/reaction/organism-
% gene relational tables, published as a raven-data release) are
% gene relational tables, published as a toolbox-assets release) are
% downloaded and assembled instead, and the result is cached to
% keggModel.mat so later calls load instantly. The first build can take
% a while (the organism-gene table covers every KEGG organism) and needs
Expand Down
3 changes: 2 additions & 1 deletion reconstruction/kegg/getPhylDist.m
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
% getPhylDist Load the pre-built KEGG phylogenetic distance matrix.
%
% Loads the pre-built phylogenetic distance matrix from keggPhylDist.mat.
% The artefact is distributed as a raven-data release asset.
% The matrix ships with RAVEN; it is derived from the KEGG taxonomy, which
% is a toolbox-assets release asset.
%
% Name-Value Arguments
% --------------------
Expand Down
6 changes: 3 additions & 3 deletions reconstruction/kegg/keggDataVersion.m
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
function version=keggDataVersion()
% keggDataVersion The raven-data release tag for KEGG artefacts.
% keggDataVersion The toolbox-assets release tag for KEGG artefacts.
%
% Reads reconstruction/kegg/KEGG_VERSION.md --- the single place this
% version is recorded --- so every function that downloads a KEGG
% artefact (getModelFromKEGG, getKEGGModelForOrganism) stays in sync
% when raven-data publishes a new KEGG release. The version is the
% when toolbox-assets publishes a new KEGG release. The version is the
% file's first line.
%
% Returns
% -------
% version : char
% the raven-data release tag, e.g. 'kegg118'.
% the toolbox-assets release tag, e.g. 'kegg118'.

ravenPath=findRAVENroot();
versionFile=fullfile(ravenPath,'reconstruction','kegg','KEGG_VERSION.md');
Expand Down
4 changes: 2 additions & 2 deletions testing/function_tests/tBinaries.m
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
classdef tBinaries < matlab.unittest.TestCase
% tBinaries On-demand download of RAVEN's external binaries from raven-data.
% tBinaries On-demand download of RAVEN's external binaries from toolbox-assets.
%
% Validates downloadRavenBinaries on the current platform: each tool is
% fetched into software/<tool>/, the executable is present, and (for the CLI
% tools) it actually runs. On macOS, running the downloaded binary also
% confirms the Gatekeeper quarantine attribute was cleared.
%
% Exercised on Linux and macOS by the "binary-tests" CI job. Requires network
% access to https://github.com/SysBioChalmers/raven-data .
% access to https://github.com/CompMetEng/toolbox-assets .

methods (Test)
function blast(testCase)
Expand Down
4 changes: 2 additions & 2 deletions testing/function_tests/tReconstruction.m
Original file line number Diff line number Diff line change
Expand Up @@ -112,14 +112,14 @@ function getDiamondRunsWhenAvailable(testCase)
end

function getKEGGModelForOrganismNeedsData(testCase)
testCase.assumeFail('Requires keggModel.mat and an HMM library from raven-data.');
testCase.assumeFail('Requires keggModel.mat and an HMM library from toolbox-assets.');
end

function getModelFromKEGGNeedsData(testCase)
matFile = fullfile(testCase.ravenRoot, 'reconstruction', 'kegg', 'keggModel.mat');
testCase.assumeFalse(exist(matFile, 'file') == 2, ...
'keggModel.mat is present; skipping the build-from-artefacts path.');
testCase.assumeFail('Downloads and assembles the full KEGG artefact set from raven-data; not run automatically.');
testCase.assumeFail('Downloads and assembles the full KEGG artefact set from toolbox-assets; not run automatically.');
end

function buildGlobalGPRJoinsGenesThroughSharedKO(testCase)
Expand Down
Loading