From f50baf2ab2079d4338010dcd6d4094cf40d373bd Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 16 Oct 2026 00:07:03 +0200 Subject: [PATCH] Download assets from CompMetEng/toolbox-assets with fetchToolboxAsset raven-data moved to CompMetEng/toolbox-assets, which holds the downloadable assets of RAVEN, raven-toolbox, GECKO and geckopy. fetchRavenDataAsset is renamed fetchToolboxAsset and downloads from the new repository; every caller and reference is updated. --- .github/scripts/assemble_offline_software.py | 14 +++++++------- .github/workflows/tests.yml | 2 +- checkRaven.m | 6 +++--- docs/v3_review.md | 6 +++--- installation/downloadRavenBinaries.m | 10 +++++----- .../{fetchRavenDataAsset.m => fetchToolboxAsset.m} | 13 +++++++------ reconstruction/kegg/KEGG_VERSION.md | 4 ++-- reconstruction/kegg/buildGlobalKEGGModel.m | 4 ++-- reconstruction/kegg/getKEGGModelForOrganism.m | 8 ++++---- reconstruction/kegg/getModelFromKEGG.m | 2 +- reconstruction/kegg/getPhylDist.m | 3 ++- reconstruction/kegg/keggDataVersion.m | 6 +++--- testing/function_tests/tBinaries.m | 4 ++-- testing/function_tests/tReconstruction.m | 4 ++-- 14 files changed, 44 insertions(+), 42 deletions(-) rename installation/{fetchRavenDataAsset.m => fetchToolboxAsset.m} (68%) diff --git a/.github/scripts/assemble_offline_software.py b/.github/scripts/assemble_offline_software.py index 0d0384d1..511337cc 100644 --- a/.github/scripts/assemble_offline_software.py +++ b/.github/scripts/assemble_offline_software.py @@ -1,14 +1,14 @@ #!/usr/bin/env python """Populate RAVEN's software/ with every platform's external binaries. -Downloads the per-platform binary ZIPs from raven-data and lays them out under +Downloads the per-platform binary ZIPs from toolbox-assets and lays them out under ``/software//`` using RAVEN's suffix scheme — bare name = Linux, ``.mac`` = macOS, ``.exe`` = Windows. With all platforms present in one tree, the whole RAVEN checkout can be zipped into a single **OS-independent** offline distribution (see the release-bundle workflow). Run from the RAVEN root, or pass ``--raven ``. Requires network access to -https://github.com/SysBioChalmers/raven-data . +https://github.com/CompMetEng/toolbox-assets . """ from __future__ import annotations @@ -18,11 +18,11 @@ import zipfile from pathlib import Path -RAVEN_DATA = "https://github.com/SysBioChalmers/raven-data/releases/download" -# raven-data platform key -> RAVEN binary suffix. +TOOLBOX_ASSETS = "https://github.com/CompMetEng/toolbox-assets/releases/download" +# toolbox-assets platform key -> RAVEN binary suffix. PLATFORMS = {"linux-x86_64": "", "macos-arm64": ".mac", "windows-x86_64": ".exe"} -# (software subdir, raven-data bundle, {platform|"_": version}, {bare executable names}) +# (software subdir, toolbox-assets bundle, {platform|"_": version}, {bare executable names}) CLI_TOOLS = [ ("blast+", "blast", {"_": "2.17.0"}, {"blastp", "makeblastdb"}), ("diamond", "diamond", {"_": "2.1.17"}, {"diamond"}), @@ -31,7 +31,7 @@ def _fetch(url: str) -> bytes: - with urllib.request.urlopen(url, timeout=120) as resp: # noqa: S310 (pinned raven-data URLs) + with urllib.request.urlopen(url, timeout=120) as resp: # noqa: S310 (pinned toolbox-assets URLs) return resp.read() @@ -60,7 +60,7 @@ def assemble(raven: Path) -> None: # macOS ships bare names; rename them to .mac. Linux keeps bare names, # Windows already carries .exe — so only the macOS members need renaming. rename = {e: e + ".mac" for e in exes} if suffix == ".mac" else {} - _extract(_fetch(f"{RAVEN_DATA}/{bundle}-{version}/{asset}"), dest, rename) + _extract(_fetch(f"{TOOLBOX_ASSETS}/{bundle}-{version}/{asset}"), dest, rename) def main(argv: list[str] | None = None) -> None: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bef69b5b..235e8582 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -72,7 +72,7 @@ jobs: binary-tests: name: Binary download (${{ matrix.os }}) - # Validate the on-demand fetch of the external binaries from raven-data on + # Validate the on-demand fetch of the external binaries from toolbox-assets on # Linux and macOS. Running the downloaded binaries also confirms the macOS # Gatekeeper quarantine is cleared so they can execute. strategy: diff --git a/checkRaven.m b/checkRaven.m index aa462a86..469f2318 100755 --- a/checkRaven.m +++ b/checkRaven.m @@ -251,13 +251,13 @@ printOrange(' Skipping check of binary executables\n') else % The BLAST+/DIAMOND/HMMER binaries are downloaded on demand from - % raven-data; fetch any that are missing for this platform before + % toolbox-assets; fetch any that are missing for this platform before % checking. This doubles as the up-front "prefetch" for users who want % everything ready after installation. try downloadRavenBinaries({'blast+','diamond','hmmer'}); catch ME - printOrange([' Could not download binaries from raven-data: ' ME.message '\n' ... + printOrange([' Could not download binaries from toolbox-assets: ' ME.message '\n' ... ' (offline? fetch the "*-binaries" RAVEN release for an offline install)\n']); end % Run each bundled binary with a version/help flag to confirm it @@ -294,7 +294,7 @@ function testSolver(solver, model) function testBinary(ravenDir, tool, binName, versionArg) % Run a bundled binary with versionArg to confirm it executes; error if not. if ispc - binEnd = '.exe'; % native Windows builds (raven-data windows-x86_64 ZIPs) + binEnd = '.exe'; % native Windows builds (toolbox-assets windows-x86_64 ZIPs) elseif ismac binEnd = '.mac'; else diff --git a/docs/v3_review.md b/docs/v3_review.md index 691c4568..4768562c 100644 --- a/docs/v3_review.md +++ b/docs/v3_review.md @@ -30,7 +30,7 @@ So the review does not repeat completed work. v3 already: and `combnk`. - Dropped the Statistics, Bioinformatics and Text Analytics toolbox dependencies. - Moved KEGG HMM libraries and external binaries out of git to on-demand fetch from the - `raven-data` release. + `toolbox-assets` release. - Reorganized the repo into functional folders; reformatted docstrings to NumPy style. - Added `progressReport`, `parseRAVENargs` (positional-or-named arguments), and a `matlab.unittest` suite. @@ -147,7 +147,7 @@ Plus a standalone `mapIDsViaMNXref(type, ids, fromDB, toDB)` ID translator. 1. Relocate to the develop3 layout (e.g. `reconstruction/metanetx/` or `annotation/metanetx/`), drop the duplicated `struct_conversion/*`, rebase on develop3's `convertMiriams`. 2. **Delete the committed data**; host MNXref 4.5 (gzipped TSVs, and ideally a prebuilt `.mat`) - as a `raven-data` release asset and add a `downloadMNXref.m` modeled exactly on the + as a `toolbox-assets` release asset and add a `downloadMNXref.m` modeled exactly on the develop3 KEGG-HMM / `downloadRavenBinaries` on-demand pattern (sentinel-file check, cache under a gitignored dir). 3. Update parsing to the documented **4.5** columns (header-driven `readtable`, handle the @@ -379,6 +379,6 @@ tracked as a low-priority item in the Tier-3 table. 2. **Where should new strain-design / community code that needs metaheuristics live**, given the dependency question — lean on the Global Optimization Toolbox (`gamultiobj`), or keep v3's "minimize toolbox dependencies" stance and ship a self-contained EA? -4. **MNXref data hosting** — confirm the `raven-data` release is the right home for the MNXref +4. **MNXref data hosting** — confirm the `toolbox-assets` release is the right home for the MNXref 4.5 assets (mirroring KEGG HMMs/binaries), and whether to ship a prebuilt `.mat` (7 MB, fast) or only the gzipped TSVs (slower first build, always current). diff --git a/installation/downloadRavenBinaries.m b/installation/downloadRavenBinaries.m index 16c8ebf9..0ed6b3b9 100644 --- a/installation/downloadRavenBinaries.m +++ b/installation/downloadRavenBinaries.m @@ -1,7 +1,7 @@ function downloadRavenBinaries(tools) % downloadRavenBinaries % Download RAVEN's external command-line binaries on demand from the -% raven-data repository into RAVEN's software/ directory. Only the binary +% toolbox-assets repository into RAVEN's software/ directory. Only the binary % matching the current platform is fetched, and only when needed. % % Input: @@ -14,7 +14,7 @@ function downloadRavenBinaries(tools) % unzip it into the RAVEN software/ directory instead. % % The binaries are hosted as release assets in -% https://github.com/SysBioChalmers/raven-data. +% https://github.com/CompMetEng/toolbox-assets. % % Usage: downloadRavenBinaries({'blast+','diamond'}) % @@ -29,7 +29,7 @@ function downloadRavenBinaries(tools) ravenDir = findRAVENroot(); -% raven-data platform key for the per-platform binary ZIPs. +% toolbox-assets platform key for the per-platform binary ZIPs. if ispc plat = 'windows-x86_64'; elseif ismac @@ -78,11 +78,11 @@ function downloadRavenBinaries(tools) continue; % already provisioned end - zipPath = fetchRavenDataAsset(fullfile(ravenDir,'software'),tag,asset); + zipPath = fetchToolboxAsset(fullfile(ravenDir,'software'),tag,asset); unzip(zipPath,destDir); delete(zipPath); - % macOS uses the .mac suffix throughout RAVEN, but the raven-data macOS ZIP + % macOS uses the .mac suffix throughout RAVEN, but the toolbox-assets macOS ZIP % ships bare names, so rename the extracted executables to match. if ismac for k = 1:numel(execs) diff --git a/installation/fetchRavenDataAsset.m b/installation/fetchToolboxAsset.m similarity index 68% rename from installation/fetchRavenDataAsset.m rename to installation/fetchToolboxAsset.m index 5e5266eb..1942a794 100644 --- a/installation/fetchRavenDataAsset.m +++ b/installation/fetchToolboxAsset.m @@ -1,9 +1,10 @@ -function localPath=fetchRavenDataAsset(destDir,releaseTag,assetName) -% fetchRavenDataAsset Download (if needed) a raven-data release asset. +function localPath=fetchToolboxAsset(destDir,releaseTag,assetName) +% fetchToolboxAsset Download (if needed) a toolbox-assets release asset. % -% Downloads assetName from the given raven-data release tag into destDir +% Downloads assetName from the given release tag of +% https://github.com/CompMetEng/toolbox-assets into destDir % and returns its local path. An already-downloaded copy is reused as-is. -% Generic across raven-data's release families (KEGG artefacts, HMM +% Generic across toolbox-assets' release families (KEGG artefacts, HMM % libraries, binaries, ...): nothing here is specific to any one of % them, so callers supply the release tag and asset name rather than % this function hardcoding either. @@ -13,7 +14,7 @@ % destDir : char % directory to download into (created if it does not exist yet). % releaseTag : char -% the raven-data release tag, e.g. 'kegg118'. +% the toolbox-assets release tag, e.g. 'kegg118'. % assetName : char % the release asset file name, e.g. 'kegg118_core.tar.gz'. % @@ -31,7 +32,7 @@ end fprintf(['Downloading ' assetName '... ']); try - websave(localPath,['https://github.com/SysBioChalmers/raven-data/releases/download/' releaseTag '/' assetName]); + websave(localPath,['https://github.com/CompMetEng/toolbox-assets/releases/download/' releaseTag '/' assetName]); catch ME if strcmp(ME.identifier,'MATLAB:webservices:HTTP404StatusCodeError') error('Failed to download %s, the server returned a 404 error, try again later. If the problem persists please report it on the RAVEN GitHub Issues page: https://github.com/SysBioChalmers/RAVEN/issues',assetName) diff --git a/reconstruction/kegg/KEGG_VERSION.md b/reconstruction/kegg/KEGG_VERSION.md index 5095c18c..e43f3b80 100644 --- a/reconstruction/kegg/KEGG_VERSION.md +++ b/reconstruction/kegg/KEGG_VERSION.md @@ -1,12 +1,12 @@ kegg118 -This is the raven-data release tag that every KEGG-artefact download in +This is the toolbox-assets release tag that every KEGG-artefact download in this folder reads from --- getModelFromKEGG's core reference-model/table bundle, and getKEGGModelForOrganism's HMM libraries. It is the single place this version is recorded; every function that downloads a KEGG artefact reads it from here via keggDataVersion.m, instead of hardcoding a "kegg###" string of its own. Bump this file, not the -calling code, when raven-data publishes a new KEGG release. +calling code, when toolbox-assets publishes a new KEGG release. The version is read as the file's first line; everything below this point is only for a human reader. diff --git a/reconstruction/kegg/buildGlobalKEGGModel.m b/reconstruction/kegg/buildGlobalKEGGModel.m index da6b0a65..ca5201cd 100644 --- a/reconstruction/kegg/buildGlobalKEGGModel.m +++ b/reconstruction/kegg/buildGlobalKEGGModel.m @@ -1,7 +1,7 @@ function [model,KOModel,isSpontaneous,isUndefinedStoich,isIncomplete,isGeneral]=buildGlobalKEGGModel(ravenPath) % buildGlobalKEGGModel Download and assemble the global KEGG model. % -% Fetches the raven-data _core.tar.gz bundle (the +% Fetches the toolbox-assets _core.tar.gz bundle (the % gene-free reference model plus the ko_reaction / organism_gene_ko / % rxn_flags relational tables), then joins every organism's % genes onto the reference reactions through their shared KO ids. The @@ -27,7 +27,7 @@ kver=keggDataVersion(); keggDir=fullfile(ravenPath,'reconstruction','kegg'); -archive=fetchRavenDataAsset(keggDir,kver,[kver '_core.tar.gz']); +archive=fetchToolboxAsset(keggDir,kver,[kver '_core.tar.gz']); coreDir=fullfile(keggDir,[kver '_core']); if ~isfolder(coreDir) diff --git a/reconstruction/kegg/getKEGGModelForOrganism.m b/reconstruction/kegg/getKEGGModelForOrganism.m index 6aa08f9e..08303253 100755 --- a/reconstruction/kegg/getKEGGModelForOrganism.m +++ b/reconstruction/kegg/getKEGGModelForOrganism.m @@ -30,7 +30,7 @@ % and whether the HMMs were trained on pro- or eukaryotic sequences. The % directory name matches the published HMM library it is paired with. The % prebuilt concatenated KO HMM library (dataDir.hmm) is downloaded here -% from the corresponding raven-data release if not already present. This +% from the corresponding toolbox-assets release if not already present. This % parameter should ALWAYS be provided. % outDir : char % directory to save the results from the quering of the hidden Markov @@ -197,8 +197,8 @@ %pre-trained HMM set, compatible with the current RAVEN version. The %reconstruction uses the concatenated KO HMM library (a single %gzip-compressed flatfile, queried in one hmmsearch); if it is not already -%present it is downloaded from the corresponding raven-data release -%(https://github.com/SysBioChalmers/raven-data). +%present it is downloaded from the corresponding toolbox-assets release +%(https://github.com/CompMetEng/toolbox-assets). %Only needed for the protein-homology path (fastaFile supplied): the %annotation-only path never touches libraryFile, so skip the (100+ MB) %download/extraction entirely when there is no FASTA file to search. @@ -219,7 +219,7 @@ gunzip([libraryFile '.gz']); fprintf('COMPLETE\n'); else - fetchRavenDataAsset(fileparts(libraryFile),kver,[hmmName '.hmm.gz']); + fetchToolboxAsset(fileparts(libraryFile),kver,[hmmName '.hmm.gz']); fprintf('Extracting the HMM library file... '); gunzip([libraryFile '.gz']); fprintf('COMPLETE\n'); diff --git a/reconstruction/kegg/getModelFromKEGG.m b/reconstruction/kegg/getModelFromKEGG.m index ca8a3780..242078ce 100755 --- a/reconstruction/kegg/getModelFromKEGG.m +++ b/reconstruction/kegg/getModelFromKEGG.m @@ -4,7 +4,7 @@ % Loads the global KEGG reaction/gene model from keggModel.mat. On first % use --- when no keggModel.mat is present yet --- the underlying % artefacts (the gene-free reference model plus the KO/reaction/organism- -% gene relational tables, published as a raven-data release) are +% gene relational tables, published as a toolbox-assets release) are % downloaded and assembled instead, and the result is cached to % keggModel.mat so later calls load instantly. The first build can take % a while (the organism-gene table covers every KEGG organism) and needs diff --git a/reconstruction/kegg/getPhylDist.m b/reconstruction/kegg/getPhylDist.m index 1a8c7244..f2b5ca34 100755 --- a/reconstruction/kegg/getPhylDist.m +++ b/reconstruction/kegg/getPhylDist.m @@ -2,7 +2,8 @@ % getPhylDist Load the pre-built KEGG phylogenetic distance matrix. % % Loads the pre-built phylogenetic distance matrix from keggPhylDist.mat. -% The artefact is distributed as a raven-data release asset. +% The matrix ships with RAVEN; it is derived from the KEGG taxonomy, which +% is a toolbox-assets release asset. % % Name-Value Arguments % -------------------- diff --git a/reconstruction/kegg/keggDataVersion.m b/reconstruction/kegg/keggDataVersion.m index f8a8e9d0..3c9c5666 100644 --- a/reconstruction/kegg/keggDataVersion.m +++ b/reconstruction/kegg/keggDataVersion.m @@ -1,16 +1,16 @@ function version=keggDataVersion() -% keggDataVersion The raven-data release tag for KEGG artefacts. +% keggDataVersion The toolbox-assets release tag for KEGG artefacts. % % Reads reconstruction/kegg/KEGG_VERSION.md --- the single place this % version is recorded --- so every function that downloads a KEGG % artefact (getModelFromKEGG, getKEGGModelForOrganism) stays in sync -% when raven-data publishes a new KEGG release. The version is the +% when toolbox-assets publishes a new KEGG release. The version is the % file's first line. % % Returns % ------- % version : char -% the raven-data release tag, e.g. 'kegg118'. +% the toolbox-assets release tag, e.g. 'kegg118'. ravenPath=findRAVENroot(); versionFile=fullfile(ravenPath,'reconstruction','kegg','KEGG_VERSION.md'); diff --git a/testing/function_tests/tBinaries.m b/testing/function_tests/tBinaries.m index 2630b0df..8514d27d 100644 --- a/testing/function_tests/tBinaries.m +++ b/testing/function_tests/tBinaries.m @@ -1,5 +1,5 @@ classdef tBinaries < matlab.unittest.TestCase -% tBinaries On-demand download of RAVEN's external binaries from raven-data. +% tBinaries On-demand download of RAVEN's external binaries from toolbox-assets. % % Validates downloadRavenBinaries on the current platform: each tool is % fetched into software//, the executable is present, and (for the CLI @@ -7,7 +7,7 @@ % confirms the Gatekeeper quarantine attribute was cleared. % % Exercised on Linux and macOS by the "binary-tests" CI job. Requires network -% access to https://github.com/SysBioChalmers/raven-data . +% access to https://github.com/CompMetEng/toolbox-assets . methods (Test) function blast(testCase) diff --git a/testing/function_tests/tReconstruction.m b/testing/function_tests/tReconstruction.m index 96e3b277..d8b89382 100644 --- a/testing/function_tests/tReconstruction.m +++ b/testing/function_tests/tReconstruction.m @@ -112,14 +112,14 @@ function getDiamondRunsWhenAvailable(testCase) end function getKEGGModelForOrganismNeedsData(testCase) - testCase.assumeFail('Requires keggModel.mat and an HMM library from raven-data.'); + testCase.assumeFail('Requires keggModel.mat and an HMM library from toolbox-assets.'); end function getModelFromKEGGNeedsData(testCase) matFile = fullfile(testCase.ravenRoot, 'reconstruction', 'kegg', 'keggModel.mat'); testCase.assumeFalse(exist(matFile, 'file') == 2, ... 'keggModel.mat is present; skipping the build-from-artefacts path.'); - testCase.assumeFail('Downloads and assembles the full KEGG artefact set from raven-data; not run automatically.'); + testCase.assumeFail('Downloads and assembles the full KEGG artefact set from toolbox-assets; not run automatically.'); end function buildGlobalGPRJoinsGenesThroughSharedKO(testCase)