feat: session and subject types, file adapters for the EBRAINS dataset formats, and a source for bucket files

+nansen/+module/+sharebrain/+dataio/EbrainsBucketSource.m

@@ -0,0 +1,65 @@
+classdef EbrainsBucketSource < nansen.dataio.RemoteFileSource
+%EbrainsBucketSource - Source of the files of a dataset cloned from an EBRAINS bucket
+%
+%   A dataset cloned with sharebrain.cloneKGDataset has an empty file for
+%   every object of its EBRAINS Data Proxy bucket, and names the bucket in
+%   the file .ebrains-bucket.json in the root folder of the clone. This
+%   source counts an empty file below such a folder as stored online, and
+%   downloads it with ebrains.bucket.downloadFile. The name of the object
+%   is the path of the file relative to the root folder of the clone,
+%   because the clone keeps the folder of a shared bucket in its paths.
+%
+%   Make it the source of online files of the current project:
+%       project = nansen.getCurrentProject();
+%       project.setRemoteFileSource("nansen.module.sharebrain.dataio.EbrainsBucketSource")
+%
+%   source = EbrainsBucketSource(rootPath, Downloader=FCN) downloads with
+%   FCN(bucketName, objectName, targetFile) instead.
+%
+%   See also nansen.dataio.RemoteFileSource,
+%   nansen.module.sharebrain.ebrains.writeBucketInfo
+
+    properties (SetAccess = private)
+        % Downloader - Function that downloads an object, called as
+        % Downloader(bucketName, objectName, targetFile)
+        Downloader function_handle
+    end
+
+    methods
+        function obj = EbrainsBucketSource(rootPath, options)
+            arguments
+                rootPath (1,1) string
+                options.Downloader (1,1) function_handle = @downloadWithProgressInCommandWindow
+            end
+            obj@nansen.dataio.RemoteFileSource(rootPath)
+            obj.Downloader = options.Downloader;
+        end
+
+        function tf = isOnlineOnly(~, filePath)
+            fileInfo = dir(filePath);
+            tf = isscalar(fileInfo) && fileInfo.bytes == 0 ...
+                && ~isempty(nansen.module.sharebrain.ebrains.findBucketInfo(filePath));
+        end
+
+        function download(obj, filePath)
+            [bucketInfo, cloneFolder] = nansen.module.sharebrain.ebrains.findBucketInfo(filePath);
+            if isempty(bucketInfo)
+                error('SHAREbrain:EbrainsBucketSource:NotInClone', ...
+                    ['No folder above "%s" has the file %s, so the bucket of the file is not known. ', ...
+                     'Clone the dataset with sharebrain.cloneKGDataset, or record its bucket with ', ...
+                     'nansen.module.sharebrain.ebrains.writeBucketInfo.'], ...
+                    filePath, nansen.module.sharebrain.constants.BucketInfoFilename)
+            end
+
+            relativePath = extractAfter(string(filePath), strlength(cloneFolder) + 1);
+            objectName = replace(relativePath, filesep, "/");
+            obj.Downloader(string(bucketInfo.bucket), objectName, string(filePath))
+        end
+    end
+end
+
+function downloadWithProgressInCommandWindow(bucketName, objectName, targetFile)
+% A dialog box would need a display, which a batch session or an agent
+% does not have
+    ebrains.bucket.downloadFile(bucketName, objectName, targetFile, DisplayMode="Command Window")
+end

+nansen/+module/+sharebrain/+ebrains/findBucketInfo.m

@@ -0,0 +1,35 @@
+function [bucketInfo, cloneFolder] = findBucketInfo(filePath)
+%findBucketInfo - The EBRAINS bucket of a cloned dataset that contains a file
+%   [bucketInfo, cloneFolder] = nansen.module.sharebrain.ebrains.findBucketInfo(filePath)
+%   looks for .ebrains-bucket.json in the folder of filePath and in each
+%   folder above it, and reads the first one it finds. bucketInfo is a
+%   struct with the fields bucket, prefix and datasetVersionId, and
+%   cloneFolder is the folder that holds the file.
+%
+%   Both outputs are empty when no folder above filePath has the file.
+%
+%   See also nansen.module.sharebrain.ebrains.writeBucketInfo
+
+    arguments
+        filePath (1,1) string
+    end
+
+    bucketInfo = struct.empty;
+    cloneFolder = "";
+
+    folderPath = fileparts(filePath);
+    while strlength(folderPath) > 0
+        infoFilePath = fullfile(folderPath, nansen.module.sharebrain.constants.BucketInfoFilename);
+        if isfile(infoFilePath)
+            bucketInfo = jsondecode(fileread(infoFilePath));
+            cloneFolder = folderPath;
+            return
+        end
+
+        parentFolder = fileparts(folderPath);
+        if parentFolder == folderPath % The root of the file system
+            return
+        end
+        folderPath = parentFolder;
+    end
+end

+nansen/+module/+sharebrain/+ebrains/writeBucketInfo.m

@@ -0,0 +1,34 @@
+function writeBucketInfo(cloneFolder, bucketName, options)
+%writeBucketInfo - Record which EBRAINS bucket a cloned dataset comes from
+%   nansen.module.sharebrain.ebrains.writeBucketInfo(cloneFolder, bucketName)
+%   writes the file .ebrains-bucket.json in cloneFolder. The file names the
+%   Data Proxy bucket whose objects the empty files below cloneFolder stand
+%   for, so that EbrainsBucketSource can download them.
+%
+%   writeBucketInfo(..., Prefix=PREFIX) records the folder of a bucket that
+%   is shared with other datasets.
+%
+%   writeBucketInfo(..., DatasetVersionId=UUID) records the dataset version
+%   in the EBRAINS Knowledge Graph that the files belong to.
+%
+%   The file holds a JSON object with the fields bucket, prefix and
+%   datasetVersionId.
+%
+%   See also nansen.module.sharebrain.ebrains.findBucketInfo,
+%   nansen.module.sharebrain.dataio.EbrainsBucketSource
+
+    arguments
+        cloneFolder (1,1) string {mustBeFolder}
+        bucketName (1,1) string {mustBeNonzeroLengthText}
+        options.Prefix (1,1) string = ""
+        options.DatasetVersionId (1,1) string = ""
+    end
+
+    bucketInfo = struct( ...
+        'bucket', bucketName, ...
+        'prefix', options.Prefix, ...
+        'datasetVersionId', options.DatasetVersionId);
+
+    filePath = fullfile(cloneFolder, nansen.module.sharebrain.constants.BucketInfoFilename);
+    writelines(jsonencode(bucketInfo, PrettyPrint=true), filePath)
+end

+nansen/+module/+sharebrain/+fileadapter/ABF.m

@@ -0,0 +1,42 @@
+classdef ABF < nansen.dataio.FileAdapter
+%ABF - File adapter for Axon Binary Files (ABF versions 1 and 2) recorded with pCLAMP
+%
+%   sessionObject.loadData(variableName) reads every channel and sweep of
+%   the file and returns a struct with the fields
+%       Data         - The samples. Episodic recordings give a
+%                      samples x channels x sweeps array, gap-free
+%                      recordings a samples x channels array, and
+%                      event-driven variable-length recordings a cell array
+%                      with one element per sweep.
+%       SamplingRate - Samples per second of each channel
+%       ChannelNames - 1 x channels string with the names of the channels
+%       ChannelUnits - 1 x channels string with the units of the channels
+%       Header       - The header struct returned by abfload
+%
+%   The file is read by abfload, written by Harald Hentschke and Forrest
+%   Collman, which is in the private folder with its BSD licence.
+
+    properties (Constant)
+        DataType = 'struct'
+    end
+
+    properties (Constant, Hidden, Access = protected)
+        SUPPORTED_FILE_TYPES = {'abf'}
+    end
+
+    methods (Access = protected)
+        function data = readData(obj, varargin)
+            [samples, samplingIntervalMicroseconds, header] = abfload(obj.Filename, ...
+                'channels', 'a', 'sweeps', 'a', 'doDispInfo', false);
+
+            % abfload gives channel names and units as a cell array for
+            % ABF 2 files and as a padded char matrix for ABF 1 files
+            data = struct( ...
+                'Data', samples, ...
+                'SamplingRate', 1e6 / samplingIntervalMicroseconds, ...
+                'ChannelNames', reshape(strtrim(string(header.recChNames)), 1, []), ...
+                'ChannelUnits', reshape(strtrim(string(header.recChUnits)), 1, []), ...
+                'Header', header);
+        end
+    end
+end

+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m

@@ -0,0 +1,44 @@
+classdef DelimitedText < nansen.dataio.FileAdapter
+%DelimitedText - File adapter for text files with columns separated by a delimiter
+%
+%   sessionObject.loadData(variableName) returns the file as a table, read
+%   with readtable. readtable detects the delimiter, the number of header
+%   lines and whether the first line holds column names.
+%
+%   Lines that start with # are comments. The comment lines at the top of
+%   a file, such as the protocol, stimulus and units that the derived
+%   traces of several HBP datasets start with, are joined in the table's
+%   Properties.Description.
+
+    properties (Constant)
+        DataType = 'table'
+    end
+
+    properties (Constant, Hidden, Access = protected)
+        % Not .dat, which is binary in some datasets; a project assigns
+        % this adapter to .dat files of text by name
+        SUPPORTED_FILE_TYPES = {'txt', 'csv', 'tsv'}
+    end
+
+    methods (Access = protected)
+        function data = readData(obj, varargin)
+            data = readtable(obj.Filename, 'CommentStyle', '#');
+            header = readCommentHeader(obj.Filename);
+            if ~isempty(header)
+                data.Properties.Description = strjoin(header, newline);
+            end
+        end
+    end
+end
+
+function header = readCommentHeader(filePath)
+%readCommentHeader - The lines that start with # at the top of a file
+    header = strings(0, 1);
+    fileID = fopen(filePath, 'r');
+    closeFile = onCleanup(@() fclose(fileID));
+    line = fgetl(fileID);
+    while ischar(line) && startsWith(strtrim(line), "#")
+        header(end+1, 1) = string(line); %#ok<AGROW>
+        line = fgetl(fileID);
+    end
+end

+nansen/+module/+sharebrain/+fileadapter/IgorPro.m

@@ -0,0 +1,24 @@
+classdef IgorPro < nansen.dataio.FileAdapter
+%IgorPro - File adapter for Igor Pro binary waves (.ibw) and packed experiments (.pxp)
+%
+%   sessionObject.loadData(variableName) returns the file as a Neo block
+%   struct, in the form described in NeoPickle; the waves are the analog
+%   signals of its segment.
+%
+%   The file is read by Python with the packages Neo and igor2 (see
+%   readWithNeo).
+
+    properties (Constant)
+        DataType = 'struct'
+    end
+
+    properties (Constant, Hidden, Access = protected)
+        SUPPORTED_FILE_TYPES = {'ibw', 'pxp'}
+    end
+
+    methods (Access = protected)
+        function data = readData(obj, varargin)
+            data = readWithNeo(obj.Filename, "igor");
+        end
+    end
+end

+nansen/+module/+sharebrain/+fileadapter/JSON.m

@@ -0,0 +1,21 @@
+classdef JSON < nansen.dataio.FileAdapter
+%JSON - File adapter for JSON files
+%
+%   sessionObject.loadData(variableName) returns the content of the file
+%   decoded with jsondecode: objects become structs and arrays become
+%   arrays or cell arrays.
+
+    properties (Constant)
+        DataType = 'struct'
+    end
+
+    properties (Constant, Hidden, Access = protected)
+        SUPPORTED_FILE_TYPES = {'json'}
+    end
+
+    methods (Access = protected)
+        function data = readData(obj, varargin)
+            data = jsondecode(fileread(obj.Filename));
+        end
+    end
+end

+nansen/+module/+sharebrain/+fileadapter/NIfTI.m

@@ -0,0 +1,27 @@
+classdef NIfTI < nansen.dataio.FileAdapter
+%NIfTI - File adapter for NIfTI images (.nii and .nii.gz)
+%
+%   sessionObject.loadData(variableName) returns a struct with the fields
+%       Data   - The image, as niftiread reads it: x-y-z for a volume and
+%                x-y-z-t for a series of volumes
+%       Header - The header, as niftiinfo reads it, with the voxel size
+%                (PixelDimensions), units (SpaceUnits, TimeUnits) and the
+%                transform to scanner space (Transform)
+%
+%   Reading needs the Image Processing Toolbox (niftiread, niftiinfo).
+
+    properties (Constant)
+        DataType = 'struct'
+    end
+
+    properties (Constant, Hidden, Access = protected)
+        SUPPORTED_FILE_TYPES = {'nii', 'nii.gz'}
+    end
+
+    methods (Access = protected)
+        function data = readData(obj, varargin)
+            header = niftiinfo(obj.Filename);
+            data = struct('Data', niftiread(header), 'Header', header);
+        end
+    end
+end

+nansen/+module/+sharebrain/+fileadapter/NWB.m

@@ -0,0 +1,26 @@
+classdef NWB < nansen.dataio.FileAdapter
+%NWB - File adapter for Neurodata Without Borders (NWB) files
+%
+%   sessionObject.loadData(variableName) returns the NwbFile object that
+%   nwbRead of MatNWB creates for the file. The object reads datasets
+%   lazily, so loading is fast and the data of a dataset is read when it
+%   is indexed or loaded with its load method.
+%
+%   MatNWB must be on the MATLAB path. nwbRead generates the type classes
+%   from the schema stored in the file, so files written with different
+%   NWB versions can be read.
+
+    properties (Constant)
+        DataType = 'NwbFile'
+    end
+
+    properties (Constant, Hidden, Access = protected)
+        SUPPORTED_FILE_TYPES = {'nwb'}
+    end
+
+    methods (Access = protected)
+        function data = readData(obj, varargin)
+            data = nwbRead(obj.Filename);
+        end
+    end
+end

+nansen/+module/+sharebrain/+fileadapter/NeoNix.m

@@ -0,0 +1,23 @@
+classdef NeoNix < nansen.dataio.FileAdapter
+%NeoNix - File adapter for NIX files written by Neo (.nio, .nix)
+%
+%   sessionObject.loadData(variableName) returns the first block of the
+%   file as a struct, in the form described in NeoPickle.
+%
+%   The file is read by Python with the packages Neo and nixio (see
+%   readWithNeo).
+
+    properties (Constant)
+        DataType = 'struct'
+    end
+
+    properties (Constant, Hidden, Access = protected)
+        SUPPORTED_FILE_TYPES = {'nio', 'nix'}
+    end
+
+    methods (Access = protected)
+        function data = readData(obj, varargin)
+            data = readWithNeo(obj.Filename, "nix");
+        end
+    end
+end

+nansen/+module/+sharebrain/+fileadapter/NeoPickle.m

@@ -0,0 +1,28 @@
+classdef NeoPickle < nansen.dataio.FileAdapter
+%NeoPickle - File adapter for Python pickle files that hold a Neo block
+%
+%   sessionObject.loadData(variableName) returns the block as a struct with
+%   the fields name, description, file_origin, rec_datetime, annotations
+%   and segments. Each segment holds its analogsignals,
+%   irregularlysampledsignals, spiketrains, events and epochs; times are in
+%   seconds, sampling rates in Hz, and each signal gives its units as text.
+%
+%   The file is unpickled by Python with the package Neo (see
+%   readWithNeo). A pickle runs code of the classes it holds when it is
+%   read, so load only files from a source you trust, such as a published
+%   dataset.
+
+    properties (Constant)
+        DataType = 'struct'
+    end
+
+    properties (Constant, Hidden, Access = protected)
+        SUPPORTED_FILE_TYPES = {'pkl'}
+    end
+
+    methods (Access = protected)
+        function data = readData(obj, varargin)
+            data = readWithNeo(obj.Filename, "pickle");
+        end
+    end
+end