diff --git a/+nansen/+module/+sharebrain/+constant/DataProxyBaseUrl.m b/+nansen/+module/+sharebrain/+constant/DataProxyBaseUrl.m deleted file mode 100644 index 6d20084..0000000 --- a/+nansen/+module/+sharebrain/+constant/DataProxyBaseUrl.m +++ /dev/null @@ -1,5 +0,0 @@ -function url = DataProxyBaseUrl() -%DataProxyBaseUrl Base URL for data proxy api - url = "https://data-proxy.ebrains.eu/api/v1/"; -end - diff --git a/+nansen/+module/+sharebrain/+constant/NeuroSiftBaseUrl.m b/+nansen/+module/+sharebrain/+constant/NeuroSiftBaseUrl.m deleted file mode 100644 index 0f7c87e..0000000 --- a/+nansen/+module/+sharebrain/+constant/NeuroSiftBaseUrl.m +++ /dev/null @@ -1,8 +0,0 @@ -function url = NeuroSiftBaseUrl() -%NeuroSiftBaseUrl Base URL for neurosift app - url = "https://neurosift.app/?p=/nwb"; -end - - - - diff --git a/+nansen/+module/+sharebrain/+dataio/EbrainsBucketSource.m b/+nansen/+module/+sharebrain/+dataio/EbrainsBucketSource.m new file mode 100644 index 0000000..b953713 --- /dev/null +++ b/+nansen/+module/+sharebrain/+dataio/EbrainsBucketSource.m @@ -0,0 +1,65 @@ +classdef EbrainsBucketSource < nansen.dataio.RemoteFileSource +%EbrainsBucketSource - Source of the files of a dataset cloned from an EBRAINS bucket +% +% A dataset cloned with sharebrain.cloneKGDataset has an empty file for +% every object of its EBRAINS Data Proxy bucket, and names the bucket in +% the file .ebrains-bucket.json in the root folder of the clone. This +% source counts an empty file below such a folder as stored online, and +% downloads it with ebrains.bucket.downloadFile. The name of the object +% is the path of the file relative to the root folder of the clone, +% because the clone keeps the folder of a shared bucket in its paths. +% +% Make it the source of online files of the current project: +% project = nansen.getCurrentProject(); +% project.setRemoteFileSource("nansen.module.sharebrain.dataio.EbrainsBucketSource") +% +% source = EbrainsBucketSource(rootPath, Downloader=FCN) downloads with +% FCN(bucketName, objectName, targetFile) instead. +% +% See also nansen.dataio.RemoteFileSource, +% nansen.module.sharebrain.ebrains.writeBucketInfo + + properties (SetAccess = private) + % Downloader - Function that downloads an object, called as + % Downloader(bucketName, objectName, targetFile) + Downloader function_handle + end + + methods + function obj = EbrainsBucketSource(rootPath, options) + arguments + rootPath (1,1) string + options.Downloader (1,1) function_handle = @downloadWithProgressInCommandWindow + end + obj@nansen.dataio.RemoteFileSource(rootPath) + obj.Downloader = options.Downloader; + end + + function tf = isOnlineOnly(~, filePath) + fileInfo = dir(filePath); + tf = isscalar(fileInfo) && fileInfo.bytes == 0 ... + && ~isempty(nansen.module.sharebrain.ebrains.findBucketInfo(filePath)); + end + + function download(obj, filePath) + [bucketInfo, cloneFolder] = nansen.module.sharebrain.ebrains.findBucketInfo(filePath); + if isempty(bucketInfo) + error('SHAREbrain:EbrainsBucketSource:NotInClone', ... + ['No folder above "%s" has the file %s, so the bucket of the file is not known. ', ... + 'Clone the dataset with sharebrain.cloneKGDataset, or record its bucket with ', ... + 'nansen.module.sharebrain.ebrains.writeBucketInfo.'], ... + filePath, nansen.module.sharebrain.constants.BucketInfoFilename) + end + + relativePath = extractAfter(string(filePath), strlength(cloneFolder) + 1); + objectName = replace(relativePath, filesep, "/"); + obj.Downloader(string(bucketInfo.bucket), objectName, string(filePath)) + end + end +end + +function downloadWithProgressInCommandWindow(bucketName, objectName, targetFile) +% A dialog box would need a display, which a batch session or an agent +% does not have + ebrains.bucket.downloadFile(bucketName, objectName, targetFile, DisplayMode="Command Window") +end diff --git a/+nansen/+module/+sharebrain/+ebrains/findBucketInfo.m b/+nansen/+module/+sharebrain/+ebrains/findBucketInfo.m new file mode 100644 index 0000000..8f30505 --- /dev/null +++ b/+nansen/+module/+sharebrain/+ebrains/findBucketInfo.m @@ -0,0 +1,35 @@ +function [bucketInfo, cloneFolder] = findBucketInfo(filePath) +%findBucketInfo - The EBRAINS bucket of a cloned dataset that contains a file +% [bucketInfo, cloneFolder] = nansen.module.sharebrain.ebrains.findBucketInfo(filePath) +% looks for .ebrains-bucket.json in the folder of filePath and in each +% folder above it, and reads the first one it finds. bucketInfo is a +% struct with the fields bucket, prefix and datasetVersionId, and +% cloneFolder is the folder that holds the file. +% +% Both outputs are empty when no folder above filePath has the file. +% +% See also nansen.module.sharebrain.ebrains.writeBucketInfo + + arguments + filePath (1,1) string + end + + bucketInfo = struct.empty; + cloneFolder = ""; + + folderPath = fileparts(filePath); + while strlength(folderPath) > 0 + infoFilePath = fullfile(folderPath, nansen.module.sharebrain.constants.BucketInfoFilename); + if isfile(infoFilePath) + bucketInfo = jsondecode(fileread(infoFilePath)); + cloneFolder = folderPath; + return + end + + parentFolder = fileparts(folderPath); + if parentFolder == folderPath % The root of the file system + return + end + folderPath = parentFolder; + end +end diff --git a/+nansen/+module/+sharebrain/+ebrains/writeBucketInfo.m b/+nansen/+module/+sharebrain/+ebrains/writeBucketInfo.m new file mode 100644 index 0000000..6efd7b5 --- /dev/null +++ b/+nansen/+module/+sharebrain/+ebrains/writeBucketInfo.m @@ -0,0 +1,34 @@ +function writeBucketInfo(cloneFolder, bucketName, options) +%writeBucketInfo - Record which EBRAINS bucket a cloned dataset comes from +% nansen.module.sharebrain.ebrains.writeBucketInfo(cloneFolder, bucketName) +% writes the file .ebrains-bucket.json in cloneFolder. The file names the +% Data Proxy bucket whose objects the empty files below cloneFolder stand +% for, so that EbrainsBucketSource can download them. +% +% writeBucketInfo(..., Prefix=PREFIX) records the folder of a bucket that +% is shared with other datasets. +% +% writeBucketInfo(..., DatasetVersionId=UUID) records the dataset version +% in the EBRAINS Knowledge Graph that the files belong to. +% +% The file holds a JSON object with the fields bucket, prefix and +% datasetVersionId. +% +% See also nansen.module.sharebrain.ebrains.findBucketInfo, +% nansen.module.sharebrain.dataio.EbrainsBucketSource + + arguments + cloneFolder (1,1) string {mustBeFolder} + bucketName (1,1) string {mustBeNonzeroLengthText} + options.Prefix (1,1) string = "" + options.DatasetVersionId (1,1) string = "" + end + + bucketInfo = struct( ... + 'bucket', bucketName, ... + 'prefix', options.Prefix, ... + 'datasetVersionId', options.DatasetVersionId); + + filePath = fullfile(cloneFolder, nansen.module.sharebrain.constants.BucketInfoFilename); + writelines(jsonencode(bucketInfo, PrettyPrint=true), filePath) +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/ABF.m b/+nansen/+module/+sharebrain/+fileadapter/ABF.m new file mode 100644 index 0000000..265c49b --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/ABF.m @@ -0,0 +1,42 @@ +classdef ABF < nansen.dataio.FileAdapter +%ABF - File adapter for Axon Binary Files (ABF versions 1 and 2) recorded with pCLAMP +% +% sessionObject.loadData(variableName) reads every channel and sweep of +% the file and returns a struct with the fields +% Data - The samples. Episodic recordings give a +% samples x channels x sweeps array, gap-free +% recordings a samples x channels array, and +% event-driven variable-length recordings a cell array +% with one element per sweep. +% SamplingRate - Samples per second of each channel +% ChannelNames - 1 x channels string with the names of the channels +% ChannelUnits - 1 x channels string with the units of the channels +% Header - The header struct returned by abfload +% +% The file is read by abfload, written by Harald Hentschke and Forrest +% Collman, which is in the private folder with its BSD licence. + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'abf'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + [samples, samplingIntervalMicroseconds, header] = abfload(obj.Filename, ... + 'channels', 'a', 'sweeps', 'a', 'doDispInfo', false); + + % abfload gives channel names and units as a cell array for + % ABF 2 files and as a padded char matrix for ABF 1 files + data = struct( ... + 'Data', samples, ... + 'SamplingRate', 1e6 / samplingIntervalMicroseconds, ... + 'ChannelNames', reshape(strtrim(string(header.recChNames)), 1, []), ... + 'ChannelUnits', reshape(strtrim(string(header.recChUnits)), 1, []), ... + 'Header', header); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m b/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m new file mode 100644 index 0000000..2078940 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m @@ -0,0 +1,44 @@ +classdef DelimitedText < nansen.dataio.FileAdapter +%DelimitedText - File adapter for text files with columns separated by a delimiter +% +% sessionObject.loadData(variableName) returns the file as a table, read +% with readtable. readtable detects the delimiter, the number of header +% lines and whether the first line holds column names. +% +% Lines that start with # are comments. The comment lines at the top of +% a file, such as the protocol, stimulus and units that the derived +% traces of several HBP datasets start with, are joined in the table's +% Properties.Description. + + properties (Constant) + DataType = 'table' + end + + properties (Constant, Hidden, Access = protected) + % Not .dat, which is binary in some datasets; a project assigns + % this adapter to .dat files of text by name + SUPPORTED_FILE_TYPES = {'txt', 'csv', 'tsv'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readtable(obj.Filename, 'CommentStyle', '#'); + header = readCommentHeader(obj.Filename); + if ~isempty(header) + data.Properties.Description = strjoin(header, newline); + end + end + end +end + +function header = readCommentHeader(filePath) +%readCommentHeader - The lines that start with # at the top of a file + header = strings(0, 1); + fileID = fopen(filePath, 'r'); + closeFile = onCleanup(@() fclose(fileID)); + line = fgetl(fileID); + while ischar(line) && startsWith(strtrim(line), "#") + header(end+1, 1) = string(line); %#ok + line = fgetl(fileID); + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/IgorPro.m b/+nansen/+module/+sharebrain/+fileadapter/IgorPro.m new file mode 100644 index 0000000..ca049a7 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/IgorPro.m @@ -0,0 +1,24 @@ +classdef IgorPro < nansen.dataio.FileAdapter +%IgorPro - File adapter for Igor Pro binary waves (.ibw) and packed experiments (.pxp) +% +% sessionObject.loadData(variableName) returns the file as a Neo block +% struct, in the form described in NeoPickle; the waves are the analog +% signals of its segment. +% +% The file is read by Python with the packages Neo and igor2 (see +% readWithNeo). + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'ibw', 'pxp'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readWithNeo(obj.Filename, "igor"); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/JSON.m b/+nansen/+module/+sharebrain/+fileadapter/JSON.m new file mode 100644 index 0000000..68bbabb --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/JSON.m @@ -0,0 +1,21 @@ +classdef JSON < nansen.dataio.FileAdapter +%JSON - File adapter for JSON files +% +% sessionObject.loadData(variableName) returns the content of the file +% decoded with jsondecode: objects become structs and arrays become +% arrays or cell arrays. + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'json'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = jsondecode(fileread(obj.Filename)); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/NIfTI.m b/+nansen/+module/+sharebrain/+fileadapter/NIfTI.m new file mode 100644 index 0000000..05f55f2 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/NIfTI.m @@ -0,0 +1,27 @@ +classdef NIfTI < nansen.dataio.FileAdapter +%NIfTI - File adapter for NIfTI images (.nii and .nii.gz) +% +% sessionObject.loadData(variableName) returns a struct with the fields +% Data - The image, as niftiread reads it: x-y-z for a volume and +% x-y-z-t for a series of volumes +% Header - The header, as niftiinfo reads it, with the voxel size +% (PixelDimensions), units (SpaceUnits, TimeUnits) and the +% transform to scanner space (Transform) +% +% Reading needs the Image Processing Toolbox (niftiread, niftiinfo). + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'nii', 'nii.gz'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + header = niftiinfo(obj.Filename); + data = struct('Data', niftiread(header), 'Header', header); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/NWB.m b/+nansen/+module/+sharebrain/+fileadapter/NWB.m new file mode 100644 index 0000000..606ebaf --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/NWB.m @@ -0,0 +1,26 @@ +classdef NWB < nansen.dataio.FileAdapter +%NWB - File adapter for Neurodata Without Borders (NWB) files +% +% sessionObject.loadData(variableName) returns the NwbFile object that +% nwbRead of MatNWB creates for the file. The object reads datasets +% lazily, so loading is fast and the data of a dataset is read when it +% is indexed or loaded with its load method. +% +% MatNWB must be on the MATLAB path. nwbRead generates the type classes +% from the schema stored in the file, so files written with different +% NWB versions can be read. + + properties (Constant) + DataType = 'NwbFile' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'nwb'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = nwbRead(obj.Filename); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/NeoNix.m b/+nansen/+module/+sharebrain/+fileadapter/NeoNix.m new file mode 100644 index 0000000..7d26083 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/NeoNix.m @@ -0,0 +1,23 @@ +classdef NeoNix < nansen.dataio.FileAdapter +%NeoNix - File adapter for NIX files written by Neo (.nio, .nix) +% +% sessionObject.loadData(variableName) returns the first block of the +% file as a struct, in the form described in NeoPickle. +% +% The file is read by Python with the packages Neo and nixio (see +% readWithNeo). + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'nio', 'nix'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readWithNeo(obj.Filename, "nix"); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/NeoPickle.m b/+nansen/+module/+sharebrain/+fileadapter/NeoPickle.m new file mode 100644 index 0000000..dc395b2 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/NeoPickle.m @@ -0,0 +1,28 @@ +classdef NeoPickle < nansen.dataio.FileAdapter +%NeoPickle - File adapter for Python pickle files that hold a Neo block +% +% sessionObject.loadData(variableName) returns the block as a struct with +% the fields name, description, file_origin, rec_datetime, annotations +% and segments. Each segment holds its analogsignals, +% irregularlysampledsignals, spiketrains, events and epochs; times are in +% seconds, sampling rates in Hz, and each signal gives its units as text. +% +% The file is unpickled by Python with the package Neo (see +% readWithNeo). A pickle runs code of the classes it holds when it is +% read, so load only files from a source you trust, such as a published +% dataset. + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'pkl'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readWithNeo(obj.Filename, "pickle"); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/XML.m b/+nansen/+module/+sharebrain/+fileadapter/XML.m new file mode 100644 index 0000000..5e840bf --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/XML.m @@ -0,0 +1,21 @@ +classdef XML < nansen.dataio.FileAdapter +%XML - File adapter for XML files +% +% sessionObject.loadData(variableName) returns the content of the file +% as a struct, read with readstruct: elements become fields, and the +% attributes of an element become fields whose names end in Attribute. + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'xml'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readstruct(obj.Filename, 'FileType', 'xml'); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m b/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m new file mode 100644 index 0000000..56f4b35 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m @@ -0,0 +1,1578 @@ +function [d,si,h]=abfload(fn,varargin) +% ** function [d,si,h]=abfload(fn,varargin) +% loads and returns data in ABF (Axon Binary File) format. +% Data may have been acquired in the following modes: +% (1) event-driven variable-length (currently only abf versions < 2.0) +% (2) event-driven fixed-length or waveform-fixed length +% (3) gap-free +% Information about scaling, the time base and the number of channels and +% episodes is extracted from the header of the abf file. +% +% OPERATION +% If the second input variable is the char array 'info' as in +% [d,si,h]=abfload('d:\data01.abf','info') +% abfload will not load any data but return detailed information (header +% parameters) on the file in output variable h. d and si will be empty. +% In all other cases abfload will load data. Optional input parameters +% listed below (= all except the file name) must be specified as +% parameter/value pairs, e.g. as in +% d=abfload('d:\data01.abf','start',100,'stop','e'); +% +% >>> INPUT VARIABLES >>> +% NAME TYPE, DEFAULT DESCRIPTION +% fn char array abf data file name +% start scalar, 0 only gap-free-data: start of cutout to be +% read (unit: s) +% stop scalar or char, only gap-free-data: end of cutout to be +% 'e' read (unit: sec). May be set to 'e' (end +% of file). +% sweeps 1d-array or char, only episodic data: sweep numbers to be +% 'a' read. By default, all sweeps will be read +% ('a'). +% channels cell array names of channels to be read, like +% or char, 'a' {'IN 0','IN 8'} (make sure spelling is +% 100% correct, including blanks). If set +% to 'a', all channels will be read. +% ***************************************** +% NOTE: channel order in output variable d +% ignores the order in 'channels', and +% instead always matches the order inherent +% to the abf file, to be retrieved in +% output variable h! +% ***************************************** +% chunk scalar, 0.05 only gap-free-data: the elementary chunk +% size (megabytes) to be used for the +% 'discontinuous' mode of reading data +% (fewer channels to be read than exist) +% machineF char array, the 'machineformat' input parameter of the +% 'ieee-le' matlab fopen function. 'ieee-le' is the +% correct option for windows; depending on +% the platform the data were recorded/shall +% be read by abfload 'ieee-be' is the +% alternative. +% doDispInfo logical, true if true, information on the loaded file +% will be put out to console (if false, +% only ínformation on erroneous input will +% be displayed) +% << OUTPUT VARIABLES <<< +% NAME TYPE DESCRIPTION +% d the data read, the format depending on the record- +% ing mode +% 1. GAP-FREE: +% 2d array 2d array of size +% by +% Examples of access: +% d(:,2) data from channel 2 at full length +% d(1:100,:) first 100 data points from all channels +% 2. EPISODIC FIXED-LENGTH/WAVEFORM FIXED-LENGTH/HIGH-SPEED OSCILLOSCOPE: +% 3d array 3d array of size +% by by . +% Examples of access: +% d(:,2,:) a matrix containing all episodes +% (at full length) of the second +% channel in its columns +% d(1:200,:,[1 11]) contains first 200 data points of +% episodes 1 and 11 of all channels +% 3. EPISODIC VARIABLE-LENGTH: +% cell array cell array whose elements correspond to single sweeps. +% Each element is a (regular) array of size +% by +% Examples of access: +% d{1} a 2d-array which contains episode 1 +% (all of it, all channels) +% d{2}(1:100,2) a 1d-array containing the first 100 +% data points of channel 2 in episode 1 +% si scalar the sampling interval in us +% h struct information on file (selected header parameters) +% +% CONTRIBUTORS +% Original version by Harald Hentschke (harald.hentschke@uni-tuebingen.de) +% Extended to abf version 2.0 by Forrest Collman (fcollman@Princeton.edu) +% pvpmod.m by Ulrich Egert (egert@bccn.uni-freiburg.de) +% Small patch to avoid error caused by activated HumSilencer by Michael Rabenstein (michael.rabenstein@ukbonn.de) + +% ------------------------------------------------------------------------- +% PART 1: check of input vars +% ------------------------------------------------------------------------- +% --- defaults +% gap-free +start=0.0; +stop='e'; +% episodic +sweeps='a'; +% general +channels='a'; +% the size of data chunks (see above) in Mb. 0.05 Mb is an empirical value +% which works well for abf with 6-16 channels and recording durations of +% 5-30 min +chunk=0.05; +machineF='ieee-le'; +doDispInfo=true; +% if first and only optional input argument is string 'info' the user's +% request is to obtain information on the file (header parameters), so set +% flag accordingly +if nargin==2 && ischar(varargin{1}) && strcmp('info',varargin{1}) + doLoadData=false; + % if no output argument is requested assume that the user would like to + % obtain some basic information on the file on the command window, so + % leave doDispInfo at its default (true value). Do the same if only one + % output arg is specified (which is most certainly done inadvertently + % because in 'info' mode this will be an empty array). In all other cases + % assume that text output is not required so suppress it + if nargout>1 + doDispInfo=false; + end +else + doLoadData=true; + % assign values of optional input parameters if any were given + pvpmod(varargin); +end + +% some constants +BLOCKSIZE=512; +% output variables +d=[]; +si=[]; +h=[]; +if ischar(stop) + if ~strcmpi(stop,'e') + error('input parameter ''stop'' must be specified as ''e'' (=end of recording) or as a scalar'); + end +end +% check existence of file +if ~isfile(fn) + error(['could not find file ' fn]); +end + +% ------------------------------------------------------------------------- +% PART 2a: determine abf version +% ------------------------------------------------------------------------- +dispif(doDispInfo,['opening ' fn '..']); +[fid,messg]=fopen(fn,'r',machineF); +if fid == -1 + error(messg); +end +% on the occasion, determine absolute file size +fileListing = dir(fn); +fileSz = fileListing.bytes; + +% *** read value of parameter 'fFileSignature' (i.e. abf version) from header *** +sz=4; +[fFileSignature,n]=fread(fid,sz,'uchar=>char'); +if n~=sz + fclose(fid); + error('something went wrong reading value(s) for fFileSignature'); +end +% rewind +fseek(fid,0,'bof'); +% transpose +fFileSignature=fFileSignature'; + +% one of the first checks must be whether file signature is valid +switch fFileSignature + case 'ABF ' % ** note the blank + % ************************ + % abf version < 2.0 + % ************************ + case 'ABF2' + % ************************ + % abf version >= 2.0 + % ************************ + otherwise + error(['unknown or incompatible file signature: ' fFileSignature]); +end + +% ------------------------------------------------------------------------- +% PART 2b: define file information ('header' parameters) of interest +% ------------------------------------------------------------------------- +% The list of header parameters created below (variable 'headPar') is +% derived from the abf version 1.8 header section. It is by no means +% exhaustive (i.e. there are many more parameters in abf files) but +% sufficient for proper upload, scaling and arrangement of data acquired +% under many conditons. Further below, these parameters will be made fields +% of struct h. h, which is also an output variable, is then used in PART 3, +% which does the actual job of uploading, scaling and rearranging the data. +% That part of the code relies on h having a certain set of fields +% irrespective of ABF version. +% Unfortunately, in the transition to ABF version 2.0 many of the header +% parameters were moved to different places within the abf file and/or +% given other names or completely restructured. In order for the code to +% work with pre- and post-2.0 data files, all parameters missing in the +% header must be gotten into h. This is accomplished in lines ~288 and +% following: +% if h.fFileVersionNumber>=2 +% ... +% Furthermore, +% - h as an output from an ABF version < 2.0 file will not contain new +% parameters introduced into the header like 'nCRCEnable' +% - h will in any case contain a few 'home-made' fields that have +% proven to be useful. Some of them depend on the recording mode. Among +% the more or less self-explanatory ones are +% -- si sampling interval +% -- recChNames the names of all channels, e.g. 'IN 8',... +% -- dataPtsPerChan sample points per channel +% -- dataPts sample points in file +% -- recTime recording start and stop time in seconds from +% midnight (millisecond resolution) +% -- sweepLengthInPts sample points per sweep (one channel) +% -- sweepStartInPts the start times of sweeps in sample points +% (from beginning of recording) + + +% define header proper depending on ABF version by call to local function +headPar=define_header(fFileSignature); +% define all sections that there are +Sections=define_Sections; +% define a few of these (currently, only the TagInfo section is used for +% all versions of ABF, but that may change in the future) +ProtocolInfo=define_ProtocolInfo; +ADCInfo=define_ADCInfo; +TagInfo=define_TagInfo; +UserListInfo=define_UserListInfo; +EpochPerDACInfo=define_EpochPerDACInfo; + +% ------------------------------------------------------------------------- +% PART 2c: read parameters of interest +% ------------------------------------------------------------------------- +% convert headPar to struct +s=cell2struct(headPar,{'name','offs','numType','value'},2); +numOfParams=size(s,1); +clear tmp headPar; + +% convert names in structure to variables and read value from header +for g=1:numOfParams + if fseek(fid, s(g).offs,'bof')~=0 + fclose(fid); + error(['something went wrong locating ' s(g).name]); + end + sz=length(s(g).value); + % use dynamic field names + [h.(s(g).name),n]=fread(fid,sz,s(g).numType); + if n~=sz + fclose(fid); + error(['something went wrong reading value(s) for ' s(g).name]); + end +end +% file signature needs to be transposed +h.fFileSignature=h.fFileSignature'; +% several header parameters need a fix or version-specific refinement: +if strcmp(h.fFileSignature,'ABF2') + % h.fFileVersionNumber needs to be converted from an array of integers to + % a float + h.fFileVersionNumber=h.fFileVersionNumber(4)+h.fFileVersionNumber(3)*.1... + +h.fFileVersionNumber(2)*.001+h.fFileVersionNumber(1)*.0001; + % convert ms to s + h.lFileStartTime=h.uFileStartTimeMS*.001; +else + % h.fFileVersionNumber is a float32 the value of which is sometimes a + % little less than what it should be (e.g. 1.6499999 instead of 1.65) + h.fFileVersionNumber=.001*round(h.fFileVersionNumber*1000); + % in abf < 2.0 two parameters are needed to obtain the file start time + % with millisecond precision - let's integrate both into parameter + % lFileStartTime (unit: s) so that nFileStartMillisecs will not be needed + h.lFileStartTime=h.lFileStartTime+h.nFileStartMillisecs*.001; +end + +if h.fFileVersionNumber>=2 + % ----------------------------------------------------------------------- + % *** read file information that has moved from the header section to + % other sections in ABF version >= 2.0 and assign selected values to + % fields of 'generic' header variable h *** + % ----------------------------------------------------------------------- + % --- read in the Sections + Sects=cell2struct(Sections,{'name'},2); + numOfSections=length(Sections); + offset=76; + % this creates all sections (ADCSection, ProtocolSection, etc.) + for i=1:numOfSections + eval([Sects(i).name '=ReadSectionInfo(fid,offset);']); + offset=offset+4+4+8; + end + % --- read in the StringsSection and use some fields (to retrieve + % information on the names of recorded channels and the units as well as + % the recording protocol used) + fseek(fid,StringsSection.uBlockIndex*BLOCKSIZE,'bof'); + BigString=fread(fid,StringsSection.uBytes,'char'); + % extract path and name of protocol file (excluding drive letter): + % - find the first occurrence of a backslash in BigString + tmpIx1=strfind(lower(char(BigString)'),'\'); + % - find '.pro' as this is the file extension of protocol files + tmpIx2=strfind(lower(char(BigString)'),'.pro'); + % - extract everything in between and place in field of header struct h + if ~isempty(tmpIx1) && ~isempty(tmpIx2) + h.protocolName=char(BigString(tmpIx1(1):tmpIx2(1)+3))'; + else + h.protocolName='protocol name could not be identified'; + end + % this is a hack: determine where either of strings 'clampex', + % 'clampfit', 'axoscope' or patchxpress' begin + progString={'clampex','clampfit','axoscope','patchxpress'}; + goodstart=[]; + for i=1:numel(progString) + goodstart=cat(1,goodstart,strfind(lower(char(BigString)'),progString{i})); + end + if isempty(goodstart) + error('problems in StringsSection: unrecognized acquisition program'); + end + BigString=BigString(goodstart(1):end)'; + stringends=find(BigString==0); + stringends=[0 stringends]; + Strings=cell(1,length(stringends)-1); + for i=1:length(stringends)-1 + Strings{i}=char(BigString(stringends(i)+1:stringends(i+1)-1)); + end + h.recChNames=cell(ADCSection.llNumEntries,1); + h.recChUnits=cell(ADCSection.llNumEntries,1); + + % --- read in the ADCSection & copy some values to header h + for i=1:ADCSection.llNumEntries + ADCsec(i)=ReadSection(fid,ADCSection.uBlockIndex*BLOCKSIZE+ADCSection.uBytes*(i-1),ADCInfo); + ii=ADCsec(i).nADCNum+1; + h.nADCSamplingSeq(i)=ADCsec(i).nADCNum; + h.recChNames(i)=Strings(ADCsec(i).lADCChannelNameIndex); + unitsIndex=ADCsec(i).lADCUnitsIndex; + if unitsIndex>0 + h.recChUnits(i)=Strings(ADCsec(i).lADCUnitsIndex); + else + h.recChUnits(i)={'nil'}; + end + h.nTelegraphEnable(ii)=ADCsec(i).nTelegraphEnable; + h.fTelegraphAdditGain(ii)=ADCsec(i).fTelegraphAdditGain; + h.fInstrumentScaleFactor(ii)=ADCsec(i).fInstrumentScaleFactor; + h.fSignalGain(ii)=ADCsec(i).fSignalGain; + h.fADCProgrammableGain(ii)=ADCsec(i).fADCProgrammableGain; + h.fInstrumentOffset(ii)=ADCsec(i).fInstrumentOffset; + h.fSignalOffset(ii)=ADCsec(i).fSignalOffset; + end + % --- read in the protocol section & copy some values to header h + ProtocolSec=ReadSection(fid,ProtocolSection.uBlockIndex*BLOCKSIZE,ProtocolInfo); + h.nOperationMode=ProtocolSec.nOperationMode; + h.fSynchTimeUnit=ProtocolSec.fSynchTimeUnit; + h.lNumSamplesPerEpisode=ProtocolSec.lNumSamplesPerEpisode; + + h.nADCNumChannels=ADCSection.llNumEntries; + h.lActualAcqLength=DataSection.llNumEntries; + h.lDataSectionPtr=DataSection.uBlockIndex; + h.nNumPointsIgnored=0; + % in ABF version < 2.0 h.fADCSampleInterval is the sampling interval + % defined as + % 1/(sampling freq*number_of_channels) + % so divide ProtocolSec.fADCSequenceInterval by the number of + % channels + h.fADCSampleInterval=ProtocolSec.fADCSequenceInterval/h.nADCNumChannels; + h.fADCRange=ProtocolSec.fADCRange; + h.lADCResolution=ProtocolSec.lADCResolution; + + % --- read in the Epoch section, rearrange values and place in header + % struct h + for i=1:EpochPerDACSection.llNumEntries + EPDsec(i)=ReadSection(fid,EpochPerDACSection.uBlockIndex*BLOCKSIZE+EpochPerDACSection.uBytes*(i-1),EpochPerDACInfo); + end + if EpochPerDACSection.llNumEntries>0 + % number of analog output (AO) channels + uniqueAO=unique([EPDsec.nDACNum]); + for k=1:numel(uniqueAO) + % index to elements of EDPsec dealing with current AO channel + ix=[EPDsec.nDACNum]==uniqueAO(k); + % struct DACEpoch contains one element per AO channel; the values of + % its fields represent the values of the different epochs (columns + % labeled A, B and so on in the waveform tab in clampex), the only + % exception being the numeric index of the channel + h.DACEpoch(k).nDACNum=uniqueAO(k); + % compute the 'first/last holding' values, the small additional delay + % after which stimulation waveforms are put out. The unit is sample + % points. ** NOTE: the formula below is based on + % i) code in function ABFH_GetHoldingDuration in abfhwave.cpp, + % presumably a code file belonging to an early version of program + % 'stimfit', and presumably derived from code available back then + % from the company formerly known as Axon Instruments + % ii) heuristics, determined from a number of abf V. 2.00 files from + % different labs in all of which 16/8-channel ADC/DAC systems were + % used. + % The formula may be hardware dependent; it may change (=be + % corrected, should it not be general) in the future. + h.DACEpoch(k).firstHolding=floor(... + 8*(h.lNumSamplesPerEpisode/h.nADCNumChannels)/512); + fieldNm=setdiff(fieldnames(EPDsec),'nDACNum','stable'); + for fIx=1:numel(fieldNm) + h.DACEpoch(k).(fieldNm{fIx})=[EPDsec(ix).(fieldNm{fIx})]; + end + end + else + h.DACEpoch=[]; + end + % --- read in the user list section - unclear how to get the values + % (commented code lines below are a first guess) + UserListSec=ReadSection(fid,UserListSection.uBlockIndex*BLOCKSIZE,UserListInfo); + % fseek(fid,UserListSec.lULParamValueListIndex*BLOCKSIZE,'bof'); + % nada=fread(fid,100,'uchar=>char') + + % --- in contrast to procedures with all other sections do not read the + % sync array section but rather copy the values of its fields to the + % corresponding fields of h + h.lSynchArrayPtr=SynchArraySection.uBlockIndex; + h.lSynchArraySize=SynchArraySection.llNumEntries; +else + % ------------------------------------------------------------------------- + % *** here, do the inverse: in ABF version<2 files extract information + % from header variable h and place it in corresponding new section + % variable(s) + % ------------------------------------------------------------------------- + TagSection.llNumEntries=h.lNumTagEntries; + TagSection.uBlockIndex=h.lTagSectionPtr; + TagSection.uBytes=64; +end +% ------------------------------------------------------------------------- +% PART 2d: groom parameters & perform some plausibility checks +% ------------------------------------------------------------------------- +if h.lActualAcqLength=1.65 + addGain=h.nTelegraphEnable.*h.fTelegraphAdditGain; + addGain(addGain==0)=1; +else + addGain=ones(size(h.fTelegraphAdditGain)); +end + +% determine offset at which data start +switch h.nDataFormat + case 0 + dataSz=2; % bytes/point + precision='int16'; + case 1 + dataSz=4; % bytes/point + precision='float32'; + otherwise + fclose(fid); + error('invalid number format'); +end +headOffset=h.lDataSectionPtr*BLOCKSIZE+h.nNumPointsIgnored*dataSz; +% h.fADCSampleInterval is the TOTAL sampling interval +h.si=h.fADCSampleInterval*h.nADCNumChannels; +% assign same value to si, which is an output variable +si=h.si; +if ischar(sweeps) && sweeps=='a' + nSweeps=h.lActualEpisodes; + sweeps=1:h.lActualEpisodes; +else + nSweeps=length(sweeps); +end + +% determine time unit in synch array section +switch h.fSynchTimeUnit + case 0 + % time information in synch array section is in terms of ticks + h.synchArrTimeBase=1; + otherwise + % time information in synch array section is in terms of usec + h.synchArrTimeBase=h.fSynchTimeUnit; +end + +% read in the TagSection, do a few computations & write to h.tags +h.tags=[]; +for i=1:TagSection.llNumEntries + tmp=ReadSection(fid,TagSection.uBlockIndex*BLOCKSIZE+TagSection.uBytes*(i-1),TagInfo); + % time of tag entry from start of experiment in s (corresponding expisode + % number, if applicable, will be determined later) + h.tags(i).timeSinceRecStart=tmp.lTagTime*h.synchArrTimeBase/1e6; + h.tags(i).comment=char(tmp.sComment)'; +end + +% ------------------------------------------------------------------------- +% PART 3: read data (note: from here on code is generic and abf version +% should not matter) +% ------------------------------------------------------------------------- +switch h.nOperationMode + case 1 + dispif(doDispInfo,'data were acquired in event-driven variable-length mode'); + if h.fFileVersionNumber>=2.0 + errordlg('abfload currently does not work with data acquired in event-driven variable-length mode and ABF version 2.0','ABF version issue'); + else + if (h.lSynchArrayPtr<=0 || h.lSynchArraySize<=0) + fclose(fid); + error('internal variables ''lSynchArraynnn'' are zero or negative'); + end + % the byte offset at which the SynchArraySection starts + h.lSynchArrayPtrByte=BLOCKSIZE*h.lSynchArrayPtr; + % before reading Synch Arr parameters check if file is big enough to hold them + % 4 bytes/long, 2 values per episode (start and length) + if h.lSynchArrayPtrByte+2*4*h.lSynchArraySize0 + fclose(fid); + error('number of data points in episode not OK'); + end + % separate channels.. + tmpd=reshape(tmpd,h.nADCNumChannels,h.dataPtsPerChan); + % retain only requested channels + tmpd=tmpd(chInd,:); + tmpd=tmpd'; + % if data format is integer, scale appropriately; if it's float, tmpd is fine + if ~h.nDataFormat + for j=1:length(chInd) + ch=recChIdx(chInd(j))+1; + tmpd(:,j)=tmpd(:,j)/(h.fInstrumentScaleFactor(ch)*h.fSignalGain(ch)*h.fADCProgrammableGain(ch)*addGain(ch))... + *h.fADCRange/h.lADCResolution+h.fInstrumentOffset(ch)-h.fSignalOffset(ch); + end + end + % now place in cell array, an element consisting of one sweep with channels in columns + d{i}=tmpd; + end + end + end + + case {2,4,5} + if h.nOperationMode==2 + dispif(doDispInfo,'data were acquired in event-driven fixed-length mode'); + elseif h.nOperationMode==4 + dispif(doDispInfo,'data were acquired in high-speed oscilloscope mode'); + else + dispif(doDispInfo,'data were acquired in waveform fixed-length mode'); + end + % extract timing information on sweeps + if (h.lSynchArrayPtr<=0 || h.lSynchArraySize<=0) + % Modified for NANSEN-SHAREbrain: an ABF 1 file recorded in a + % fixed-length mode can lack the synch array, where the original + % stops with an error. The sweeps are read one after the other + % below in either case, so their length is the number of points + % per episode and channel. Only the start times of the sweeps are + % not known, so h.sweepStartInPts is not set and the stop time in + % h.recTime is NaN. + if h.lActualEpisodes<=0 + fclose(fid); + error('file has no synch array and records no episodes, so the sweep length is not known'); + end + if rem(h.lActualAcqLength,h.lActualEpisodes*h.nADCNumChannels)>0 + fclose(fid); + error('file has no synch array, and its data points do not divide into sweeps of equal length'); + end + h.sweepLengthInPts=h.lActualAcqLength/h.lActualEpisodes/h.nADCNumChannels; + h.recTime=[h.lFileStartTime NaN]; + else + % the byte offset at which the SynchArraySection starts + h.lSynchArrayPtrByte=BLOCKSIZE*h.lSynchArrayPtr; + % before reading Synch Arr parameters check if file is big enough to hold them + % 4 bytes/long, 2 values per episode (start and length) + if h.lSynchArrayPtrByte+2*4*h.lSynchArraySize>fileSz + fclose(fid); + error('file seems not to contain complete Synch Array Section'); + end + if fseek(fid,h.lSynchArrayPtrByte,'bof')~=0 + fclose(fid); + error('something went wrong positioning file pointer to Synch Array Section'); + end + [synchArr,n]=fread(fid,h.lSynchArraySize*2,'int32'); + if n~=h.lSynchArraySize*2 + fclose(fid); + error('something went wrong reading synch array section'); + end + % make synchArr a h.lSynchArraySize x 2 matrix + synchArr=permute(reshape(synchArr',2,h.lSynchArraySize),[2 1]); + if numel(unique(synchArr(:,2)))>1 + fclose(fid); + error('sweeps of unequal length in file recorded in fixed-length mode'); + end + % the length of sweeps in sample points (**note: parameter lLength of + % the ABF synch section is expressed in samples (ticks) whereas + % parameter lStart is given in synchArrTimeBase units) + h.sweepLengthInPts=synchArr(1,2)/h.nADCNumChannels; + % the starting ticks of episodes in sample points (t0=1=beginning of + % recording) + h.sweepStartInPts=synchArr(:,1)*(h.synchArrTimeBase/h.fADCSampleInterval/h.nADCNumChannels); + % recording start and stop times in seconds from midnight + h.recTime=h.lFileStartTime; + h.recTime=h.recTime+[0 (1e-6*(h.sweepStartInPts(end)+h.sweepLengthInPts))*h.fADCSampleInterval*h.nADCNumChannels]; + end + % determine first point and number of points to be read + startPt=0; + h.dataPts=h.lActualAcqLength; + h.dataPtsPerChan=h.dataPts/h.nADCNumChannels; + if rem(h.dataPts,h.nADCNumChannels)>0 || rem(h.dataPtsPerChan,h.lActualEpisodes)>0 + fclose(fid); + error('number of data points not OK'); + end + % temporary helper var + dataPtsPerSweep=h.sweepLengthInPts*h.nADCNumChannels; + if fseek(fid,startPt*dataSz+headOffset,'bof')~=0 + fclose(fid); + error('something went wrong positioning file pointer (too few data points ?)'); + end + % the starting ticks of episodes in sample points WITHIN THE DATA FILE + selectedSegStartInPts=((sweeps-1)*dataPtsPerSweep)*dataSz+headOffset; + % ** load data if requested + if doLoadData + % preallocate d + d=zeros(h.sweepLengthInPts,length(chInd),nSweeps); + for i=1:nSweeps + status=fseek(fid,selectedSegStartInPts(i),'bof'); + if status==-1 + fclose(fid); + error(['something went wrong reading episode ' int2str(sweeps(i)) '; file pointer beyond file limits (check sweeps)']); + end + [tmpd,n]=fread(fid,dataPtsPerSweep,precision); + if n~=dataPtsPerSweep + fclose(fid); + error(['something went wrong reading episode ' int2str(sweeps(i)) ': ' dataPtsPerSweep ' points should have been read, ' int2str(n) ' points actually read']); + end + h.dataPtsPerChan=n/h.nADCNumChannels; + if rem(n,h.nADCNumChannels)>0 + fclose(fid); + error('number of data points in episode not OK'); + end + % separate channels.. + tmpd=reshape(tmpd,h.nADCNumChannels,h.dataPtsPerChan); + % retain only requested channels + tmpd=tmpd(chInd,:); + tmpd=tmpd'; + % if data format is integer, scale appropriately; if it's float, d is fine + if ~h.nDataFormat + for j=1:length(chInd) + ch=recChIdx(chInd(j))+1; + tmpd(:,j)=tmpd(:,j)/(h.fInstrumentScaleFactor(ch)*h.fSignalGain(ch)*h.fADCProgrammableGain(ch)*addGain(ch))... + *h.fADCRange/h.lADCResolution+h.fInstrumentOffset(ch)-h.fSignalOffset(ch); + end + end + % now fill 3d array + d(:,:,i)=tmpd; + end + end + + case 3 + dispif(doDispInfo,'data were acquired in gap-free mode'); + % from start, stop, headOffset and h.fADCSampleInterval calculate first point to be read + % and - unless stop is given as 'e' - number of points + startPt=floor(1e6*start*(1/h.fADCSampleInterval)); + % this corrects undesired shifts in the reading frame due to rounding errors in the previous calculation + startPt=floor(startPt/h.nADCNumChannels)*h.nADCNumChannels; + % if stop is a char array, it can only be 'e' at this point (other values would have + % been caught above) + if ischar(stop) + h.dataPtsPerChan=h.lActualAcqLength/h.nADCNumChannels-floor(1e6*start/h.si); + h.dataPts=h.dataPtsPerChan*h.nADCNumChannels; + else + h.dataPtsPerChan=floor(1e6*(stop-start)*(1/h.si)); + h.dataPts=h.dataPtsPerChan*h.nADCNumChannels; + if h.dataPts<=0 + fclose(fid); + error('start is larger than or equal to stop'); + end + end + if rem(h.dataPts,h.nADCNumChannels)>0 + fclose(fid); + error('number of data points not OK'); + end + tmp=1e-6*h.lActualAcqLength*h.fADCSampleInterval; + dispif(doDispInfo,['total length of recording: ' num2str(tmp,'%5.1f') ' s ~ ' num2str(tmp/60,'%3.0f') ' min']); + dispif(doDispInfo,['sampling interval: ' num2str(h.si,'%5.0f') ' µs']); + % 8 bytes per data point expressed in Mb + dispif(doDispInfo,['memory requirement for complete upload in matlab: '... + num2str(round(8*h.lActualAcqLength/2^20)) ' MB']); + % recording start and stop times in seconds from midnight + h.recTime=h.lFileStartTime; + h.recTime=[h.recTime h.recTime+tmp]; + if fseek(fid,startPt*dataSz+headOffset,'bof')~=0 + fclose(fid); + error('something went wrong positioning file pointer (too few data points ?)'); + end + if doLoadData + % *** decide on the most efficient way to read data: + % (i) all (of one or several) channels requested: read, done + % (ii) one (of many) channels requested: use the 'skip' feature of + % fread (which for up to 16 channels is slower than scenario iii below) + % (iii) more than one but not all (of several) channels requested: + % 'discontinuous' mode of reading data. Read a reasonable chunk of data + % (all channels), separate channels, discard non-requested ones (if + % any), place data in preallocated array, repeat until done. This is + % faster than reading the data in one big lump, separating channels and + % discarding the ones not requested + if length(chInd)==1 && h.nADCNumChannels>16 + % --- situation (ii) + % jump to proper reading frame position in file + if fseek(fid,(chInd-1)*dataSz,'cof')~=0 + fclose(fid); + error('something went wrong positioning file pointer (too few data points ?)'); + end + % read, skipping h.nADCNumChannels-1 data points after each read + [d,n]=fread(fid,h.dataPtsPerChan,precision,dataSz*(h.nADCNumChannels-1)); + if n~=h.dataPtsPerChan + fclose(fid); + error(['something went wrong reading file (' int2str(h.dataPtsPerChan) ' points should have been read, ' int2str(n) ' points actually read']); + end + elseif length(chInd)/h.nADCNumChannels<1 + % --- situation (iii) + % prepare chunkwise upload: + % preallocate d + d=nan(h.dataPtsPerChan,length(chInd)); + % the number of data points corresponding to the maximal chunk size, + % rounded off such that from each channel the same number of points is + % read (do not forget that each data point will by default be made a + % double of 8 bytes, no matter what the original data format is) + chunkPtsPerChan=floor(chunk*2^20/8/h.nADCNumChannels); + chunkPts=chunkPtsPerChan*h.nADCNumChannels; + % the number of those chunks.. + nChunk=floor(h.dataPts/chunkPts); + % ..and the remainder + restPts=h.dataPts-nChunk*chunkPts; + restPtsPerChan=restPts/h.nADCNumChannels; + % chunkwise row indices into d + dix=(1:chunkPtsPerChan:h.dataPtsPerChan)'; + dix(:,2)=dix(:,1)+chunkPtsPerChan-1; + dix(end,2)=h.dataPtsPerChan; + if nChunk + dispif(doDispInfo,['reading file in ' int2str(nChunk) ' chunks of ~' num2str(chunk) ' Mb']); + end + % do it: if no remainder exists loop through all rows of dix, + % otherwise spare last row for the lines below (starting with + % 'if restPts') + for ci=1:size(dix,1)-(restPts>0) + [tmpd,n]=fread(fid,chunkPts,precision); + if n~=chunkPts + fclose(fid); + error(['something went wrong reading chunk #' int2str(ci) ' (' ... + int2str(chunkPts) ' points should have been read, ' int2str(n) ' points actually read']); + end + % separate channels.. + tmpd=reshape(tmpd,h.nADCNumChannels,chunkPtsPerChan); + d(dix(ci,1):dix(ci,2),:)=tmpd(chInd,:)'; + end + % collect the rest, if any + if restPts + [tmpd,n]=fread(fid,restPts,precision); + if n~=restPts + fclose(fid); + error(['something went wrong reading last chunk (' ... + int2str(restPts) ' points should have been read, ' int2str(n) ' points actually read']); + end + % separate channels.. + tmpd=reshape(tmpd,h.nADCNumChannels,restPtsPerChan); + d(dix(end,1):dix(end,2),:)=tmpd(chInd,:)'; + end + else + % --- situation (i) + [d,n]=fread(fid,h.dataPts,precision); + if n~=h.dataPts + fclose(fid); + error(['something went wrong reading file (' int2str(h.dataPts) ' points should have been read, ' int2str(n) ' points actually read']); + end + % separate channels.. + d=reshape(d,h.nADCNumChannels,h.dataPtsPerChan); + d=d'; + end + % if data format is integer, scale appropriately; if it's float, d is fine + if ~h.nDataFormat + for j=1:length(chInd) + ch=recChIdx(chInd(j))+1; + d(:,j)=d(:,j)/(h.fInstrumentScaleFactor(ch)*h.fSignalGain(ch)*h.fADCProgrammableGain(ch)*addGain(ch))... + *h.fADCRange/h.lADCResolution+h.fInstrumentOffset(ch)-h.fSignalOffset(ch); + end + end + end + otherwise + warning('unknown recording mode -- returning empty matrix'); + d=[]; + h.si=[]; +end +fclose(fid); + +% finally, possibly add information on episode number to tags +%Modified by MR to avoid error caused by activated HumSilencer +if ~isempty(h.tags) && isfield(h,'sweepStartInPts') + for i=1:numel(h.tags) + if h.tags(i).timeSinceRecStart > 0 %Comments added during recording + tmp=find(h.tags(i).timeSinceRecStart>=h.sweepStartInPts/1e6*h.si); + h.tags(i).episodeIndex=tmp(end); + elseif h.tags(i).timeSinceRecStart == 0 %HumSilencer comment at timeSinceRecStart = 0 + h.tags(i).episodeIndex = 0; %Will be added to episode 0 + end + end +end + + +% ######################################################################## +% LOCAL FUNCTIONS +% ######################################################################## + +function headPar=define_header(fileSig) +switch fileSig + case 'ABF ' % ** note the blank + % ************************ + % abf version < 2.0 + % ************************ + % + % temporary initializing var + tmp=repmat(-1,1,16); + % define vital header parameters and initialize them with -1: set up a + % cell array (and convert it to a struct later on, which is more + % convenient) + % column order is + % name, position in header in bytes, type, value) + headPar={ + 'fFileSignature',0,'*char',[-1 -1 -1 -1]; + 'fFileVersionNumber',4,'float32',-1; + 'nOperationMode',8,'int16',-1; + 'lActualAcqLength',10,'int32',-1; + 'nNumPointsIgnored',14,'int16',-1; + 'lActualEpisodes',16,'int32',-1; + 'lFileStartTime',24,'int32',-1; + 'lDataSectionPtr',40,'int32',-1; + 'lTagSectionPtr',44,'int32',-1; + 'lNumTagEntries',48,'int32',-1; + 'lSynchArrayPtr',92,'int32',-1; + 'lSynchArraySize',96,'int32',-1; + 'nDataFormat',100,'int16',-1; + 'nADCNumChannels', 120, 'int16', -1; + 'fADCSampleInterval',122,'float', -1; + 'fSynchTimeUnit',130,'float',-1; + 'lNumSamplesPerEpisode',138,'int32',-1; + 'lPreTriggerSamples',142,'int32',-1; + 'lEpisodesPerRun',146,'int32',-1; + 'fADCRange', 244, 'float', -1; + 'lADCResolution', 252, 'int32', -1; + 'nFileStartMillisecs', 366, 'int16', -1; + 'nADCPtoLChannelMap', 378, 'int16', tmp; + 'nADCSamplingSeq', 410, 'int16', tmp; + 'sADCChannelName',442, 'uchar', repmat(tmp,1,10); + 'sADCUnits',602, 'uchar', repmat(tmp,1,8); + 'fADCProgrammableGain', 730, 'float', tmp; + 'fInstrumentScaleFactor', 922, 'float', tmp; + 'fInstrumentOffset', 986, 'float', tmp; + 'fSignalGain', 1050, 'float', tmp; + 'fSignalOffset', 1114, 'float', tmp; + 'nTelegraphEnable',4512,'int16',tmp; + 'fTelegraphAdditGain',4576,'float',tmp + }; + case 'ABF2' + % ************************ + % abf version >= 2.0 + % ************************ + headPar={ + 'fFileSignature',0,'*char',[-1 -1 -1 -1]; + 'fFileVersionNumber',4,'bit8=>int',[-1 -1 -1 -1]; + 'uFileInfoSize',8,'uint32',-1; + 'lActualEpisodes',12,'uint32',-1; + 'uFileStartDate',16','uint32',-1; + 'uFileStartTimeMS',20,'uint32',-1; + 'uStopwatchTime',24,'uint32',-1; + 'nFileType',28,'int16',-1; + 'nDataFormat',30,'int16',-1; + 'nSimultaneousScan',32,'int16',-1; + 'nCRCEnable',34,'int16',-1; + 'uFileCRC',36,'uint32',-1; + 'FileGUID',40,'uint32',-1; + 'uCreatorVersion',56,'uint32',-1; + 'uCreatorNameIndex',60,'uint32',-1; + 'uModifierVersion',64,'uint32',-1; + 'uModifierNameIndex',68,'uint32',-1; + 'uProtocolPathIndex',72,'uint32',-1; + }; +end + +function Sections=define_Sections +Sections={'ProtocolSection'; + 'ADCSection'; + 'DACSection'; + 'EpochSection'; + 'ADCPerDACSection'; + 'EpochPerDACSection'; + 'UserListSection'; + 'StatsRegionSection'; + 'MathSection'; + 'StringsSection'; + 'DataSection'; + 'TagSection'; + 'ScopeSection'; + 'DeltaSection'; + 'VoiceTagSection'; + 'SynchArraySection'; + 'AnnotationSection'; + 'StatsSection'; + }; + +function ProtocolInfo=define_ProtocolInfo +ProtocolInfo={ + 'nOperationMode','int16',1; + 'fADCSequenceInterval','float',1; + 'bEnableFileCompression','bit1',1; + 'sUnused1','char',3; + 'uFileCompressionRatio','uint32',1; + 'fSynchTimeUnit','float',1; + 'fSecondsPerRun','float',1; + 'lNumSamplesPerEpisode','int32',1; + 'lPreTriggerSamples','int32',1; + 'lEpisodesPerRun','int32',1; + 'lRunsPerTrial','int32',1; + 'lNumberOfTrials','int32',1; + 'nAveragingMode','int16',1; + 'nUndoRunCount','int16',1; + 'nFirstEpisodeInRun','int16',1; + 'fTriggerThreshold','float',1; + 'nTriggerSource','int16',1; + 'nTriggerAction','int16',1; + 'nTriggerPolarity','int16',1; + 'fScopeOutputInterval','float',1; + 'fEpisodeStartToStart','float',1; + 'fRunStartToStart','float',1; + 'lAverageCount','int32',1; + 'fTrialStartToStart','float',1; + 'nAutoTriggerStrategy','int16',1; + 'fFirstRunDelayS','float',1; + 'nChannelStatsStrategy','int16',1; + 'lSamplesPerTrace','int32',1; + 'lStartDisplayNum','int32',1; + 'lFinishDisplayNum','int32',1; + 'nShowPNRawData','int16',1; + 'fStatisticsPeriod','float',1; + 'lStatisticsMeasurements','int32',1; + 'nStatisticsSaveStrategy','int16',1; + 'fADCRange','float',1; + 'fDACRange','float',1; + 'lADCResolution','int32',1; + 'lDACResolution','int32',1; + 'nExperimentType','int16',1; + 'nManualInfoStrategy','int16',1; + 'nCommentsEnable','int16',1; + 'lFileCommentIndex','int32',1; + 'nAutoAnalyseEnable','int16',1; + 'nSignalType','int16',1; + 'nDigitalEnable','int16',1; + 'nActiveDACChannel','int16',1; + 'nDigitalHolding','int16',1; + 'nDigitalInterEpisode','int16',1; + 'nDigitalDACChannel','int16',1; + 'nDigitalTrainActiveLogic','int16',1; + 'nStatsEnable','int16',1; + 'nStatisticsClearStrategy','int16',1; + 'nLevelHysteresis','int16',1; + 'lTimeHysteresis','int32',1; + 'nAllowExternalTags','int16',1; + 'nAverageAlgorithm','int16',1; + 'fAverageWeighting','float',1; + 'nUndoPromptStrategy','int16',1; + 'nTrialTriggerSource','int16',1; + 'nStatisticsDisplayStrategy','int16',1; + 'nExternalTagType','int16',1; + 'nScopeTriggerOut','int16',1; + 'nLTPType','int16',1; + 'nAlternateDACOutputState','int16',1; + 'nAlternateDigitalOutputState','int16',1; + 'fCellID','float',3; + 'nDigitizerADCs','int16',1; + 'nDigitizerDACs','int16',1; + 'nDigitizerTotalDigitalOuts','int16',1; + 'nDigitizerSynchDigitalOuts','int16',1; + 'nDigitizerType','int16',1; + }; + +function ADCInfo=define_ADCInfo +ADCInfo={ + 'nADCNum','int16',1; + 'nTelegraphEnable','int16',1; + 'nTelegraphInstrument','int16',1; + 'fTelegraphAdditGain','float',1; + 'fTelegraphFilter','float',1; + 'fTelegraphMembraneCap','float',1; + 'nTelegraphMode','int16',1; + 'fTelegraphAccessResistance','float',1; + 'nADCPtoLChannelMap','int16',1; + 'nADCSamplingSeq','int16',1; + 'fADCProgrammableGain','float',1; + 'fADCDisplayAmplification','float',1; + 'fADCDisplayOffset','float',1; + 'fInstrumentScaleFactor','float',1; + 'fInstrumentOffset','float',1; + 'fSignalGain','float',1; + 'fSignalOffset','float',1; + 'fSignalLowpassFilter','float',1; + 'fSignalHighpassFilter','float',1; + 'nLowpassFilterType','char',1; + 'nHighpassFilterType','char',1; + 'fPostProcessLowpassFilter','float',1; + 'nPostProcessLowpassFilterType','char',1; + 'bEnabledDuringPN','bit1',1; + 'nStatsChannelPolarity','int16',1; + 'lADCChannelNameIndex','int32',1; + 'lADCUnitsIndex','int32',1; + }; + +function EpochPerDACInfo=define_EpochPerDACInfo +EpochPerDACInfo={ + 'nEpochNum','int16',1; + 'nDACNum','int16',1; + 'nEpochType','int16',1; + 'fEpochInitLevel','float',1; + 'fEpochLevelInc','float',1; + 'lEpochInitDuration','int32',1; + 'lEpochDurationInc','int32',1; + 'lEpochPulsePeriod','int32',1; + 'lEpochPulseWidth','int32',1; +}; + +function TagInfo=define_TagInfo +TagInfo={ + 'lTagTime','int32',1; + 'sComment','char',56; + 'nTagType','int16',1; + 'nVoiceTagNumber_or_AnnotationIndex','int16',1; +}; + +function UserListInfo=define_UserListInfo +UserListInfo={ + 'nListNum','int16',1; + 'nULEnable','int16',1; + 'nULParamToVary','int16',1; + 'nULRepeat','int16',1; + 'lULParamValueListIndex','int32',1; + 'sUnused','uchar',52; +}; + +% // FUNCTION: ReadUserList +% // PURPOSE: Reads the user list from the data file. +% // +% BOOL CABF2ProtocolReader::ReadUserList() +% { +% MEMBERASSERT(); +% +% BOOL bOK = TRUE; +% if( m_FileInfo.UserListSection.uBlockIndex ) +% { +% ABF_UserListInfo UserList; +% ASSERT( m_FileInfo.UserListSection.uBytes == sizeof( UserList ) ); +% ASSERT( m_FileInfo.UserListSection.llNumEntries ); +% bOK &= m_pFI->Seek( LONGLONG(m_FileInfo.UserListSection.uBlockIndex) * ABF_BLOCKSIZE, FILE_BEGIN ); +% if( !bOK ) +% return FALSE; +% +% for( long i=0; iRead( &UserList, sizeof( UserList ) ); +% short u = UserList.nListNum; +% +% m_pFH->nULEnable[u] = 1; +% m_pFH->nULParamToVary[u] = UserList.nULParamToVary; +% m_pFH->nULRepeat[u] = UserList.nULRepeat; +% +% bOK &= GetString( UserList.lULParamValueListIndex, m_pFH->sULParamValueList[u], ABF_USERLISTLEN ); +% } +% } +% return bOK; +% } + +function Section=ReadSection(fid,offset,Format) +s=cell2struct(Format,{'name','numType','number'},2); +fseek(fid,offset,'bof'); +for i=1:length(s) + [Section.(s(i).name)]=fread(fid,s(i).number,s(i).numType); +end + +function SectionInfo=ReadSectionInfo(fid,offset) +fseek(fid,offset,'bof'); +SectionInfo.uBlockIndex=fread(fid,1,'uint32'); +fseek(fid,offset+4,'bof'); +SectionInfo.uBytes=fread(fid,1,'uint32'); +fseek(fid,offset+8,'bof'); +SectionInfo.llNumEntries=fread(fid,1,'int64'); + +function pvpmod(x) +% PVPMOD - evaluate parameter/value pairs +% pvpmod(x) assigns the value x(i+1) to the parameter defined by the +% string x(i) in the calling workspace. This is useful to evaluate +% contents in an mfile, e.g. to change default settings +% of any variable initialized before pvpmod(x) is called. +% +% (c) U. Egert 1998 + +% this loop is assigns the parameter/value pairs in x to the calling +% workspace. +if ~isempty(x) + for i = 1:2:size(x,2) + assignin('caller', x{i}, x{i+1}); + end +end + +function dispif(doDispInfo, msg) +if doDispInfo + disp(msg) +end + + +% +% struct ABF_FileInfo +% { +% UINT uFileSignature; +% UINT uFileVersionNumber; +% +% // After this point there is no need to be the same as the ABF 1 equivalent. +% UINT uFileInfoSize; +% +% UINT uActualEpisodes; +% UINT uFileStartDate; +% UINT uFileStartTimeMS; +% UINT uStopwatchTime; +% short nFileType; +% short nDataFormat; +% short nSimultaneousScan; +% short nCRCEnable; +% UINT uFileCRC; +% GUID FileGUID; +% UINT uCreatorVersion; +% UINT uCreatorNameIndex; +% UINT uModifierVersion; +% UINT uModifierNameIndex; +% UINT uProtocolPathIndex; +% +% // New sections in ABF 2 - protocol stuff ... +% ABF_Section ProtocolSection; // the protocol +% ABF_Section ADCSection; // one for each ADC channel +% ABF_Section DACSection; // one for each DAC channel +% ABF_Section EpochSection; // one for each epoch +% ABF_Section ADCPerDACSection; // one for each ADC for each DAC +% ABF_Section EpochPerDACSection; // one for each epoch for each DAC +% ABF_Section UserListSection; // one for each user list +% ABF_Section StatsRegionSection; // one for each stats region +% ABF_Section MathSection; +% ABF_Section StringsSection; +% +% // ABF 1 sections ... +% ABF_Section DataSection; // Data +% ABF_Section TagSection; // Tags +% ABF_Section ScopeSection; // Scope config +% ABF_Section DeltaSection; // Deltas +% ABF_Section VoiceTagSection; // Voice Tags +% ABF_Section SynchArraySection; // Synch Array +% ABF_Section AnnotationSection; // Annotations +% ABF_Section StatsSection; // Stats config +% +% char sUnused[148]; // size = 512 bytes +% +% ABF_FileInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_FileInfo ) == 512 ); +% +% uFileSignature = ABF_FILESIGNATURE; +% uFileInfoSize = sizeof( ABF_FileInfo); +% } +% +% }; +% +% struct ABF_ProtocolInfo +% { +% short nOperationMode; +% float fADCSequenceInterval; +% bool bEnableFileCompression; +% char sUnused1[3]; +% UINT uFileCompressionRatio; +% +% float fSynchTimeUnit; +% float fSecondsPerRun; +% long lNumSamplesPerEpisode; +% long lPreTriggerSamples; +% long lEpisodesPerRun; +% long lRunsPerTrial; +% long lNumberOfTrials; +% short nAveragingMode; +% short nUndoRunCount; +% short nFirstEpisodeInRun; +% float fTriggerThreshold; +% short nTriggerSource; +% short nTriggerAction; +% short nTriggerPolarity; +% float fScopeOutputInterval; +% float fEpisodeStartToStart; +% float fRunStartToStart; +% long lAverageCount; +% float fTrialStartToStart; +% short nAutoTriggerStrategy; +% float fFirstRunDelayS; +% +% short nChannelStatsStrategy; +% long lSamplesPerTrace; +% long lStartDisplayNum; +% long lFinishDisplayNum; +% short nShowPNRawData; +% float fStatisticsPeriod; +% long lStatisticsMeasurements; +% short nStatisticsSaveStrategy; +% +% float fADCRange; +% float fDACRange; +% long lADCResolution; +% long lDACResolution; +% +% short nExperimentType; +% short nManualInfoStrategy; +% short nCommentsEnable; +% long lFileCommentIndex; +% short nAutoAnalyseEnable; +% short nSignalType; +% +% short nDigitalEnable; +% short nActiveDACChannel; +% short nDigitalHolding; +% short nDigitalInterEpisode; +% short nDigitalDACChannel; +% short nDigitalTrainActiveLogic; +% +% short nStatsEnable; +% short nStatisticsClearStrategy; +% +% short nLevelHysteresis; +% long lTimeHysteresis; +% short nAllowExternalTags; +% short nAverageAlgorithm; +% float fAverageWeighting; +% short nUndoPromptStrategy; +% short nTrialTriggerSource; +% short nStatisticsDisplayStrategy; +% short nExternalTagType; +% short nScopeTriggerOut; +% +% short nLTPType; +% short nAlternateDACOutputState; +% short nAlternateDigitalOutputState; +% +% float fCellID[3]; +% +% short nDigitizerADCs; +% short nDigitizerDACs; +% short nDigitizerTotalDigitalOuts; +% short nDigitizerSynchDigitalOuts; +% short nDigitizerType; +% +% char sUnused[304]; // size = 512 bytes +% +% ABF_ProtocolInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_ProtocolInfo ) == 512 ); +% } +% }; +% +% struct ABF_MathInfo +% { +% short nMathEnable; +% short nMathExpression; +% UINT uMathOperatorIndex; +% UINT uMathUnitsIndex; +% float fMathUpperLimit; +% float fMathLowerLimit; +% short nMathADCNum[2]; +% char sUnused[16]; +% float fMathK[6]; +% +% char sUnused2[64]; // size = 128 bytes +% +% ABF_MathInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_MathInfo ) == 128 ); +% } +% }; +% +% struct ABF_ADCInfo +% { +% // The ADC this struct is describing. +% short nADCNum; +% +% short nTelegraphEnable; +% short nTelegraphInstrument; +% float fTelegraphAdditGain; +% float fTelegraphFilter; +% float fTelegraphMembraneCap; +% short nTelegraphMode; +% float fTelegraphAccessResistance; +% +% short nADCPtoLChannelMap; +% short nADCSamplingSeq; +% +% float fADCProgrammableGain; +% float fADCDisplayAmplification; +% float fADCDisplayOffset; +% float fInstrumentScaleFactor; +% float fInstrumentOffset; +% float fSignalGain; +% float fSignalOffset; +% float fSignalLowpassFilter; +% float fSignalHighpassFilter; +% +% char nLowpassFilterType; +% char nHighpassFilterType; +% float fPostProcessLowpassFilter; +% char nPostProcessLowpassFilterType; +% bool bEnabledDuringPN; +% +% short nStatsChannelPolarity; +% +% long lADCChannelNameIndex; +% long lADCUnitsIndex; +% +% char sUnused[46]; // size = 128 bytes +% +% ABF_ADCInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_ADCInfo ) == 128 ); +% } +% }; +% +% struct ABF_DACInfo +% { +% // The DAC this struct is describing. +% short nDACNum; +% +% short nTelegraphDACScaleFactorEnable; +% float fInstrumentHoldingLevel; +% +% float fDACScaleFactor; +% float fDACHoldingLevel; +% float fDACCalibrationFactor; +% float fDACCalibrationOffset; +% +% long lDACChannelNameIndex; +% long lDACChannelUnitsIndex; +% +% long lDACFilePtr; +% long lDACFileNumEpisodes; +% +% short nWaveformEnable; +% short nWaveformSource; +% short nInterEpisodeLevel; +% +% float fDACFileScale; +% float fDACFileOffset; +% long lDACFileEpisodeNum; +% short nDACFileADCNum; +% +% short nConditEnable; +% long lConditNumPulses; +% float fBaselineDuration; +% float fBaselineLevel; +% float fStepDuration; +% float fStepLevel; +% float fPostTrainPeriod; +% float fPostTrainLevel; +% short nMembTestEnable; +% +% short nLeakSubtractType; +% short nPNPolarity; +% float fPNHoldingLevel; +% short nPNNumADCChannels; +% short nPNPosition; +% short nPNNumPulses; +% float fPNSettlingTime; +% float fPNInterpulse; +% +% short nLTPUsageOfDAC; +% short nLTPPresynapticPulses; +% +% long lDACFilePathIndex; +% +% float fMembTestPreSettlingTimeMS; +% float fMembTestPostSettlingTimeMS; +% +% short nLeakSubtractADCIndex; +% +% char sUnused[124]; // size = 256 bytes +% +% ABF_DACInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_DACInfo ) == 256 ); +% } +% }; +% +% struct ABF_EpochInfoPerDAC +% { +% // The Epoch / DAC this struct is describing. +% short nEpochNum; +% short nDACNum; +% +% // One full set of epochs (ABF_EPOCHCOUNT) for each DAC channel ... +% short nEpochType; +% float fEpochInitLevel; +% float fEpochLevelInc; +% long lEpochInitDuration; +% long lEpochDurationInc; +% long lEpochPulsePeriod; +% long lEpochPulseWidth; +% +% char sUnused[18]; // size = 48 bytes +% +% ABF_EpochInfoPerDAC() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_EpochInfoPerDAC ) == 48 ); +% } +% }; +% +% struct ABF_EpochInfo +% { +% // The Epoch this struct is describing. +% short nEpochNum; +% +% // Describes one epoch +% short nDigitalValue; +% short nDigitalTrainValue; +% short nAlternateDigitalValue; +% short nAlternateDigitalTrainValue; +% bool bEpochCompression; // Compress the data from this epoch using uFileCompressionRatio +% +% char sUnused[21]; // size = 32 bytes +% +% ABF_EpochInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_EpochInfo ) == 32 ); +% } +% }; +% +% struct ABF_StatsRegionInfo +% { +% // The stats region this struct is describing. +% short nRegionNum; +% short nADCNum; +% +% short nStatsActiveChannels; +% short nStatsSearchRegionFlags; +% short nStatsSelectedRegion; +% short nStatsSmoothing; +% short nStatsSmoothingEnable; +% short nStatsBaseline; +% long lStatsBaselineStart; +% long lStatsBaselineEnd; +% +% // Describes one stats region +% long lStatsMeasurements; +% long lStatsStart; +% long lStatsEnd; +% short nRiseBottomPercentile; +% short nRiseTopPercentile; +% short nDecayBottomPercentile; +% short nDecayTopPercentile; +% short nStatsSearchMode; +% short nStatsSearchDAC; +% short nStatsBaselineDAC; +% +% char sUnused[78]; // size = 128 bytes +% +% ABF_StatsRegionInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_StatsRegionInfo ) == 128 ); +% } +% }; +% +% struct ABF_UserListInfo +% { +% // The user list this struct is describing. +% short nListNum; +% +% // Describes one user list +% short nULEnable; +% short nULParamToVary; +% short nULRepeat; +% long lULParamValueListIndex; +% +% char sUnused[52]; // size = 64 bytes +% +% ABF_UserListInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_UserListInfo ) == 64 ); +% } +% };*/= diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/abfload_license.txt b/+nansen/+module/+sharebrain/+fileadapter/private/abfload_license.txt new file mode 100644 index 0000000..ff102c6 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/private/abfload_license.txt @@ -0,0 +1,25 @@ +Copyright (c) 2009, Forrest Collman +Copyright (c) 2004, Harald Hentschke +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/neo_to_mat.py b/+nansen/+module/+sharebrain/+fileadapter/private/neo_to_mat.py new file mode 100644 index 0000000..5b5f8ed --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/private/neo_to_mat.py @@ -0,0 +1,208 @@ +"""Read a file with Neo and write its block to a MATLAB .mat file. + +Used by the NANSEN-SHAREbrain file adapters for formats that have no +MATLAB reader: pickled Neo objects, NIX files and Igor Pro files. + + python neo_to_mat.py + + is "pickle", "nix" or "igor". The .mat file holds one struct, +block, with the block's name, description, annotations and segments. Each +segment holds its analog signals, irregularly sampled signals, spike +trains, events and epochs, with times in seconds, sampling rates in Hz and +the units of each signal as text. Lists (segments, signals, labels) are +cell arrays, also when they are empty. +""" + +import pickle +import re +import sys + +import numpy as np +import scipy.io + + +def read_block(file_path, file_format): + """The first Neo block in a file.""" + if file_format == "pickle": + with open(file_path, "rb") as file: + content = pickle.load(file) + elif file_format == "nix": + import neo + reader = neo.io.NixIO(file_path, mode="ro") + try: + content = reader.read_block() + finally: + reader.close() + elif file_format == "igor": + import neo + content = neo.io.IgorIO(file_path).read_block() + else: + raise ValueError(f"Unknown format {file_format!r}; use pickle, nix or igor.") + + # A pickle can hold a list of blocks + if isinstance(content, (list, tuple)): + content = content[0] + return content + + +def cell(items): + """A 1-by-n object array, which a .mat file stores as a cell array. + + scipy.io.savemat writes an empty list as a 0-by-0 double and a list of + strings as a padded char matrix; an object array keeps both a cell. + """ + array = np.empty((1, len(items)), dtype=object) + for index, item in enumerate(items): + array[0, index] = item + return array + + +def field_name(key, taken): + """A valid MATLAB field name for an annotation key, not already in taken. + + Keys that differ only in characters MATLAB does not allow ("a-b", "a_b") + get the suffixes _2, _3, ... instead of overwriting each other. + """ + name = re.sub(r"\W", "_", str(key)) + if not re.match(r"[A-Za-z]", name): + name = "x" + name + name = name[:63] + candidate, number = name, 1 + while candidate in taken: + number += 1 + suffix = f"_{number}" + candidate = name[:63 - len(suffix)] + suffix + return candidate + + +def simple_value(value): + """An annotation value that a .mat file can hold, or None to leave it out.""" + if isinstance(value, (str, bool, int, float, np.integer, np.floating, np.bool_)): + return value + if isinstance(value, np.ndarray) and value.dtype.kind in "biufU": + return value + if hasattr(value, "magnitude"): # A quantity + return np.asarray(value.magnitude) + if isinstance(value, (list, tuple)) and all( + isinstance(item, (str, int, float, bool)) for item in value): + if value and all(isinstance(item, (int, float, bool)) for item in value): + return np.asarray(value) + return cell(list(value)) + if value is None: + return None + return str(value) + + +def annotations(obj): + """The annotations of a Neo object that a .mat file can hold.""" + result = {} + for key, value in getattr(obj, "annotations", {}).items(): + converted = simple_value(value) + if converted is not None: + result[field_name(key, result)] = converted + return result + + +def text(value): + return "" if value is None else str(value) + + +def seconds(quantity): + return np.asarray(quantity.rescale("s").magnitude, dtype=float) + + +def array_annotations(obj): + """One value per channel or event; values that are not numbers or text, + such as None, are written as text in a cell array.""" + result = {} + for key, value in getattr(obj, "array_annotations", {}).items(): + values = np.asarray(value) + if values.dtype.kind not in "biufU": + values = cell([text(item) for item in values.ravel()]) + result[field_name(key, result)] = values + return result + + +def analog_signal(signal): + return { + "name": text(signal.name), + "data": np.asarray(signal.magnitude), + "units": str(signal.units.dimensionality), + "sampling_rate": float(signal.sampling_rate.rescale("Hz").magnitude), + "t_start": float(seconds(signal.t_start)), + "array_annotations": array_annotations(signal), + "annotations": annotations(signal), + } + + +def irregular_signal(signal): + return { + "name": text(signal.name), + "times": seconds(signal.times), + "data": np.asarray(signal.magnitude), + "units": str(signal.units.dimensionality), + "array_annotations": array_annotations(signal), + "annotations": annotations(signal), + } + + +def spike_train(train): + return { + "name": text(train.name), + "times": seconds(train.times), + "t_start": float(seconds(train.t_start)), + "t_stop": float(seconds(train.t_stop)), + "annotations": annotations(train), + } + + +def event(item): + return { + "name": text(item.name), + "times": seconds(item.times), + "labels": cell([str(label) for label in item.labels]), + "annotations": annotations(item), + } + + +def epoch(item): + return { + "name": text(item.name), + "times": seconds(item.times), + "durations": seconds(item.durations), + "labels": cell([str(label) for label in item.labels]), + "annotations": annotations(item), + } + + +def segment(item): + return { + "name": text(item.name), + "annotations": annotations(item), + "analogsignals": cell([analog_signal(s) for s in item.analogsignals]), + "irregularlysampledsignals": cell([irregular_signal(s) for s in item.irregularlysampledsignals]), + "spiketrains": cell([spike_train(s) for s in item.spiketrains]), + "events": cell([event(e) for e in item.events]), + "epochs": cell([epoch(e) for e in item.epochs]), + } + + +def block_to_dict(block): + return { + "name": text(block.name), + "description": text(block.description), + "file_origin": text(block.file_origin), + "rec_datetime": text(block.rec_datetime), + "annotations": annotations(block), + "segments": cell([segment(s) for s in block.segments]), + } + + +def main(file_path, file_format, mat_path): + block = read_block(file_path, file_format) + scipy.io.savemat(mat_path, {"block": block_to_dict(block)}, + long_field_names=True, do_compression=True) + + +if __name__ == "__main__": + main(*sys.argv[1:4]) diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/readWithNeo.m b/+nansen/+module/+sharebrain/+fileadapter/private/readWithNeo.m new file mode 100644 index 0000000..fbf53ee --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/private/readWithNeo.m @@ -0,0 +1,46 @@ +function block = readWithNeo(filePath, fileFormat) +%readWithNeo - Read a file with the Python package Neo and return its block as a struct +% block = readWithNeo(filePath, fileFormat) runs neo_to_mat.py, in this +% folder, with the Python interpreter that pyenv names, and loads the +% .mat file that it writes. fileFormat is "pickle", "nix" or "igor". +% +% Python runs as a separate process rather than inside MATLAB: reading +% NIX files loads h5py, whose HDF5 library can conflict with the one +% MATLAB uses. +% +% The Python environment needs neo, quantities and scipy, and nixio for +% NIX files and igor2 for Igor files. + + arguments + filePath (1,1) string {mustBeFile} + fileFormat (1,1) string {mustBeMember(fileFormat, ["pickle", "nix", "igor"])} + end + + pythonExecutable = string(pyenv().Executable); + if strlength(pythonExecutable) == 0 + error('SHAREbrain:ReadWithNeo:NoPython', ... + ['No Python interpreter is set up for MATLAB. Set one with pyenv, for example ', ... + 'pyenv(Version="/path/to/python3"), with the packages neo, quantities and scipy.']) + end + + helperScript = fullfile(fileparts(mfilename('fullpath')), 'neo_to_mat.py'); + matFile = string(tempname) + ".mat"; + cleanup = onCleanup(@() deleteIfFile(matFile)); + + command = sprintf('"%s" "%s" "%s" %s "%s"', pythonExecutable, helperScript, ... + filePath, fileFormat, matFile); + [status, output] = system(command); + if status ~= 0 + error('SHAREbrain:ReadWithNeo:ReadFailed', ... + 'Neo could not read "%s" as a %s file:\n%s', filePath, fileFormat, strtrim(output)) + end + + S = load(matFile); + block = S.block; +end + +function deleteIfFile(filePath) + if isfile(filePath) + delete(filePath) + end +end diff --git a/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m b/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m new file mode 100644 index 0000000..7945b85 --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m @@ -0,0 +1,15 @@ +function collabInfo = fetchCollabInfo(collabId) +% fetchCollabInfo - Fetch info / metadata for a Data Sharing Collab + arguments + collabId (1,1) string {nansen.module.sharebrain.internal.validator.mustBeDataSharingCollabName} + end + + collabClient = ebrains.collaboratory.api.Collab(); + [status, collabInfo] = collabClient.getCollab(collabId); + if status ~= matlab.net.http.StatusCode.OK + errorInfo = jsondecode(collabInfo); + error('SHAREBRAIN:FetchCollab:FailedToGetCollab', ... + 'Something went wrong when fetching collab. Reason: \n%s', ... + sprintf("%s: %s", status, errorInfo.description)) + end +end diff --git a/+nansen/+module/+sharebrain/+internal/+dsc/listDatasetCollabs.m b/+nansen/+module/+sharebrain/+internal/+dsc/listDatasetCollabs.m new file mode 100644 index 0000000..441aa57 --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/+dsc/listDatasetCollabs.m @@ -0,0 +1,29 @@ +function collab_info = listDatasetCollabs(options) +% listDatasetCollabs - List dataset collabs + + arguments + options.Verbose (1,1) logical = true + end + + SEARCH_TERM = 'Dataset working title'; + + if options.Verbose + fprintf('Please wait, fetching dataset collabs...\n') + end + + collabClient = ebrains.collaboratory.api.Collab(); + [~, result, ~] = collabClient.searchCollab(... + 'search', SEARCH_TERM, ... + 'limit', 1000, ... + 'visibility', 'private'); + + [~, isSelection] = sort([result.createDate], 'descend'); + collab_info = result(isSelection); + + if isempty(collab_info) + error('SHAREbrain:UISelectDatasetCollab:NoCollabsFound', ... + ['Could not find any data sharing collabs. You might not be a ', ... + 'member of any data sharing collabs. Please verify access ', ... + 'permissions with your curator']) + end +end diff --git a/+nansen/+module/+sharebrain/+internal/+fileio/getLocalFileSize.m b/+nansen/+module/+sharebrain/+internal/+fileio/getLocalFileSize.m index 870857f..365a00a 100644 --- a/+nansen/+module/+sharebrain/+internal/+fileio/getLocalFileSize.m +++ b/+nansen/+module/+sharebrain/+internal/+fileio/getLocalFileSize.m @@ -6,7 +6,8 @@ L = dir(filepath); if isempty(L) - fileSizeBytes = []; % TODO: Consider whether to throw error instead. + error('SHAREBRAIN:FileNotFound', ... + 'File not found: %s', filepath); elseif numel(L) > 1 error('Multiple files/folders matched the given filepath') else diff --git a/+nansen/+module/+sharebrain/+internal/+validator/mustBeDataSharingCollabName.m b/+nansen/+module/+sharebrain/+internal/+validator/mustBeDataSharingCollabName.m new file mode 100644 index 0000000..bd182da --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/+validator/mustBeDataSharingCollabName.m @@ -0,0 +1,13 @@ +function mustBeDataSharingCollabName(value) +% mustBeDataSharingCollabName - Ensure a name is formatted as a valid DSC name + arguments + value (1,1) string + end + + dscPrefix = nansen.module.sharebrain.constants.DataSharingCollabPrefix; + assert( startsWith(value, dscPrefix), ... + 'Data Sharing Collab name must start with "%s"', dscPrefix) + + uuid = extractAfter(value, dscPrefix); + nansen.module.sharebrain.internal.validator.mustBeValidUUID(uuid) +end diff --git a/+nansen/+module/+sharebrain/+internal/+validator/mustBeValidUUID.m b/+nansen/+module/+sharebrain/+internal/+validator/mustBeValidUUID.m new file mode 100644 index 0000000..03a679f --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/+validator/mustBeValidUUID.m @@ -0,0 +1,13 @@ +function mustBeValidUUID(identifier) + % UUID regex: 8-4-4-4-12 hex digits (RFC 4122 format) + uuidPattern = "^[0-9a-fA-F]{8}-" + ... + "[0-9a-fA-F]{4}-" + ... + "[0-9a-fA-F]{4}-" + ... + "[0-9a-fA-F]{4}-" + ... + "[0-9a-fA-F]{12}$"; + + if ~contains(identifier, regexpPattern(uuidPattern)) + error("SHAREBRAIN:validator:InvalidUUID", ... + "Identifier must be a valid UUID. Got: %s", identifier); + end +end diff --git a/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m b/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m index 6c798ab..7ea0b92 100644 --- a/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m +++ b/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m @@ -1,10 +1,11 @@ function collabId = getCollabIdFromProject(project) - configFolder = project.getConfigurationFolder(); - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(project); + if ~isfile(filePath) - nansen.module.sharebrain.internal.uiGetBucketId() + nansen.module.sharebrain.internal.uiSelectDatasetCollab() end + collabInfo = jsondecode( fileread(filePath)); collabId = collabInfo.name; end \ No newline at end of file diff --git a/+nansen/+module/+sharebrain/+internal/getCollabInfoFilepath.m b/+nansen/+module/+sharebrain/+internal/getCollabInfoFilepath.m new file mode 100644 index 0000000..5d64d4c --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/getCollabInfoFilepath.m @@ -0,0 +1,23 @@ +function filePath = getCollabInfoFilepath(project) +% getCollabInfoFilepath - Retrieve the file path for the collab information. +% +% Syntax: +% filePath = getCollabInfoFilepath(project) retrieves the path to the +% file containing collab metadata associated with the specified project. +% +% Input Arguments: +% project (optional) - An instance of nansen.config.project.Project. +% If not provided, the current project is used. +% +% Output Arguments: +% filePath - A string representing the file path to the collab metadata file. + + arguments + project (1,1) nansen.config.project.Project = nansen.getCurrentProject() + end + + parentFolder = project.getConfigurationFolder(); + + fileName = nansen.module.sharebrain.constants.CollabInfoFilename; + filePath = fullfile(parentFolder, fileName); +end diff --git a/+nansen/+module/+sharebrain/+internal/linkCollabToProject.m b/+nansen/+module/+sharebrain/+internal/linkCollabToProject.m new file mode 100644 index 0000000..3301629 --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/linkCollabToProject.m @@ -0,0 +1,25 @@ +function linkCollabToProject(collabName) +% linkCollabToProject - Saves the metadata of a collab to current NANSEN project +% +% Syntax: +% nansen.module.sharebrain.internal.linkCollabToProject(collabName) +% +% Input Arguments: +% collabName (1,1) string - The name of the collab to be fetched and saved. + + arguments + collabName (1,1) string {nansen.module.sharebrain.internal.validator.mustBeDataSharingCollabName} + end + + % Fetch collab metadata from EBRAINS Collaboratory + collabInfo = nansen.module.sharebrain.internal.dsc.fetchCollabInfo(collabName); + + % Get filepath in current project to save collab metadata to + project = nansen.getCurrentProject(); + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(project); + + % Save collab metadata to current project + fid = fopen(filePath, "wt"); + fileCleanup = onCleanup(@() fclose(fid)); + fwrite(fid, jsonencode(collabInfo)) +end diff --git a/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m b/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m deleted file mode 100644 index de43d62..0000000 --- a/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m +++ /dev/null @@ -1,47 +0,0 @@ -function bucketId = uiGetBucketId() - - myClient = ebrains.collaboratory.api.Collab(); - [~,result,~] = myClient.searchCollab('search', 'Dataset working title', 'limit', 1000, 'visibility', 'private'); - - [~, isSelection] = sort([result.createDate], 'descend'); - collab_info = result(isSelection); - - % parse description: - expression = '(?<=Dataset working title: ).+?(?=\n)'; - datasetTitles = regexp([collab_info.description], expression, 'match', 'once')'; - - selection = uim.dialog.searchSelectDlg(cellstr(datasetTitles), 'Select a Dataset'); - - if isempty(selection) - error('Aborted') - end - - isSelection = strcmp(datasetTitles, selection); - bucketId = collab_info(isSelection).title; - - project = nansen.getCurrentProject(); - configFolder = project.getConfigurationFolder(); - - % Save to project configuration - S = collab_info(isSelection); - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); - utility.filewrite(filePath, jsonencode(S)) -end - -% % BASE_URL = "https://wiki.ebrains.eu/rest/"; -% % -% % bucketId = inputdlg('Please enter your bucket ID'); -% % -% % webOpts = weboptions; -% % webOpts.HeaderFields = ["Authorization", sprintf("Bearer %s", myClient.bearerToken)]; -% % webOpts.HeaderFields = ["Authorization", sprintf("Bearer %s", token)]; -% % collabResp = webread(BASE_URL+"v1/collabs", 'limit', 300, 'search', 'd-','visibility', 'private', webOpts); -% % -% % [~, idx] = sort([collabResp.createDate]); -% % collab_info = collabResp(idx); -% % -% % timestamp = [collab_info.createDate] / 1000; % Convert milliseconds to seconds -% % datetimeObj = num2cell( datetime(timestamp, 'ConvertFrom', 'posixtime') ); -% % [collab_info(:).createDate] = deal(datetimeObj{:}); -% % struct2table(collab_info) -% % end \ No newline at end of file diff --git a/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m b/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m deleted file mode 100644 index 4ecaf8a..0000000 --- a/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m +++ /dev/null @@ -1,38 +0,0 @@ -function collabInfo = uiSelectDataSharingCollabInfo(options) - arguments - options.TargetFolder (1,:) string {mustBeFolder} - %options.Logger % Todo - end - - myClient = ebrains.collaboratory.api.Collab(); - fprintf('Please wait, fetching datasets...\n') - [~,result,~] = myClient.searchCollab('search', 'Dataset working title', 'limit', 1000, 'visibility', 'private'); - - [~, isSelection] = sort([result.createDate], 'descend'); - collab_info = result(isSelection); - - if isempty(collab_info) - error('SHAREbrain:UISelectDatasetCollab:NoCollabsFound', ... - ['Could not find any data sharing collabs. You might not be a ', ... - 'member of any data sharing collabs. Please verify access ', ... - 'permissions with your curator']) - end - - % parse description: - expression = '(?<=Dataset working title: ).+?(?=\n)'; - datasetTitles = regexp([collab_info.description], expression, 'match', 'once')'; - - selection = uim.dialog.searchSelectDlg(cellstr(datasetTitles), 'Select a Dataset'); - - if isempty(selection) - error('Aborted') - end - - isSelection = strcmp(datasetTitles, selection); - collabInfo = collab_info(isSelection); - - if isfield(options, "TargetFolder") && ~isempty(options.TargetFolder) - filePath = fullfile(options.TargetFolder, 'ebrains_collab_info.json'); - utility.filewrite(filePath, jsonencode(collabInfo)) - end -end diff --git a/+nansen/+module/+sharebrain/+internal/uiSelectDatasetCollab.m b/+nansen/+module/+sharebrain/+internal/uiSelectDatasetCollab.m new file mode 100644 index 0000000..c383c41 --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/uiSelectDatasetCollab.m @@ -0,0 +1,23 @@ +function collabInfo = uiSelectDatasetCollab() + arguments + %options.Logger % Todo + end + + collab_info = nansen.module.sharebrain.internal.dsc.listDatasetCollabs(); + + % Parse description: + expression = '(?<=Dataset working title: ).+?(?=\n)'; + datasetTitles = regexp([collab_info.description], expression, 'match', 'once')'; + + selection = uim.dialog.searchSelectDlg(cellstr(datasetTitles), 'Select a Dataset'); + + if isempty(selection) + error('Aborted') % Todo: Add error id and detailed error message + end + + isSelection = strcmp(datasetTitles, selection); + collabInfo = collab_info(isSelection); + + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); + utility.filewrite(filePath, jsonencode(collabInfo)) +end diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m index 0a5fbff..ffb5210 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m @@ -1,10 +1,11 @@ function openDataSharingCollab() - project = nansen.getCurrentProject(); - configFolder = project.getConfigurationFolder(); - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); + if ~isfile(filePath) - nansen.module.sharebrain.internal.uiGetBucketId() + error(... + ['This project is not associated with any dataset collabs. ', ... + 'Please link to a dataset collab using the "Set Dataset" method']) % Todo: add error id, improve error message end collabInfo = jsondecode( fileread(filePath)); diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m index fe20389..049aacb 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m @@ -1,6 +1,6 @@ function openNwbFileInNeurosift() - baseUrl = nansen.module.sharebrain.constant.NeuroSiftBaseUrl(); + baseUrl = nansen.module.sharebrain.constants.NeurosiftBaseUrl; fileUrl = inputdlg('Please enter dataproxy api url:'); diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m index 5245d2a..ec18150 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m @@ -9,11 +9,7 @@ function setDataset() % selection among accessible Dataset Collabs. The selected collab’s name % and metadata are then saved to the project’s configuration folder. - project = nansen.getCurrentProject(); - configFolder = project.getConfigurationFolder(); - - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); - + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); if isfile(filePath) title = 'Reset dataset?'; question = 'EBRAINS Dataset is already configured. Do you want to change this dataset?'; @@ -27,6 +23,5 @@ function setDataset() end end - nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo(... - "TargetFolder", configFolder) + nansen.module.sharebrain.internal.uiSelectDatasetCollab() end diff --git a/+nansen/+module/+sharebrain/+services/+ebrains/downloadFileFromBucket.m b/+nansen/+module/+sharebrain/+services/+ebrains/downloadFileFromBucket.m deleted file mode 100644 index 80b4084..0000000 --- a/+nansen/+module/+sharebrain/+services/+ebrains/downloadFileFromBucket.m +++ /dev/null @@ -1,80 +0,0 @@ -function downloadFileFromBucket(filePath, relativeFilePath, bucketName, progressDisplay) -% downloadFileFromBucket - Download a file from an EBRAINS bucket (swift object storage) - - if strncmp(char(relativeFilePath), filesep, 1) - relativeFilePath = char(relativeFilePath); - relativeFilePath = relativeFilePath(2:end); - end - - BASE_API_URL = nansen.module.sharebrain.constant.DataProxyBaseUrl; - endpointPath = sprintf("buckets/%s/%s", bucketName, relativeFilePath); - - apiURL = BASE_API_URL + endpointPath; - - disp('starting download') - - % finished = false; - % while ~finished - %try - %downloadURL = getDownloadUrl(apiURL, useToken=false); - - try - webFileSize = nansen.module.sharebrain.internal.fileio.getWebFileSize(apiURL); - [filePath] = downloadFile(filePath, apiURL, ShowFilename=true); - catch ME - fileSize = nansen.module.sharebrain.internal.fileio.getLocalFileSize(filePath); - if fileSize ~= webFileSize - delete(filePath) - filePath = strrep(filePath, ' ', '\ '); - [status, msg] = system( sprintf('touch %s', filePath )); - end - rethrow(ME) - end - - - %task.concrete.downloadFile(filePath, apiURL, 'ProgressDisplay', progressDisplay) - finished = true; - %catch ME - % Reset access token and try again - setenv('EBRAINS_ACCESS_TOKEN', ''); - %end - % end - - %[filePath] = downloadFile(filePath, downloadURL, ShowFilename=true); -end - -function downloadURL = getDownloadUrl(apiURL, options) - arguments - apiURL (1,1) string - options.useToken (1,1) logical = false - end - - if options.useToken - accessToken = getToken(); - - header = matlab.net.http.HeaderField(... - "accept", "application/json", ... - "Authorization", "Bearer " + accessToken); - else - header=[]; - end - - warning('off', 'MATLAB:http:BodyExpectedFor') - method = matlab.net.http.RequestMethod.GET; - req = matlab.net.http.RequestMessage(method, header, []); - [resp, ~, ~] = req.send(apiURL); - warning('on', 'MATLAB:http:BodyExpectedFor') - - if resp.StatusCode ~= matlab.net.http.StatusCode.OK - error(string(resp.StatusLine)) - else - downloadURL = resp.Body.Data.url; - end -end - -function accessToken = getToken() - accessToken = getenv('EBRAINS_ACCESS_TOKEN'); - if isempty(accessToken) - accessToken = nansen.module.sharebrain.services.ebrains.uiGetToken(); - end -end \ No newline at end of file diff --git a/+nansen/+module/+sharebrain/+services/+ebrains/uploadFileToBucket.m b/+nansen/+module/+sharebrain/+services/+ebrains/uploadFileToBucket.m deleted file mode 100644 index f5ffd9e..0000000 --- a/+nansen/+module/+sharebrain/+services/+ebrains/uploadFileToBucket.m +++ /dev/null @@ -1,57 +0,0 @@ -function uploadFileToBucket(filePath, relativeFilePath, bucketName) - - if strncmp(char(relativeFilePath), filesep, 1) - relativeFilePath = char(relativeFilePath); - relativeFilePath = relativeFilePath(2:end); - end - - BASE_API_URL = nansen.module.sharebrain.constant.DataProxyBaseUrl; - endpointPath = sprintf("buckets/%s/data/%s", bucketName, relativeFilePath); - - apiURL = BASE_API_URL + endpointPath; - - finished = false; - while ~finished - try - uploadURL = getUploadUrl(apiURL); - finished = true; - catch ME - % Reset access token and try again - setenv('EBRAINS_ACCESS_TOKEN', ''); - end - end - - [wasSuccess, response] = uploadFile(filePath, uploadURL, ShowFilename=true); -end - -function uploadURL = getUploadUrl(apiURL) - - accessToken = getToken(); - - header = matlab.net.http.HeaderField(... - "accept", "application/json", ... - "Authorization", "Bearer " + accessToken); - - warning('off', 'MATLAB:http:BodyExpectedFor') - method = matlab.net.http.RequestMethod.PUT; - req = matlab.net.http.RequestMessage(method, header, []); - [resp, ~, ~] = req.send(apiURL); - warning('on', 'MATLAB:http:BodyExpectedFor') - - if resp.StatusCode ~= matlab.net.http.StatusCode.OK - error(string(resp.StatusLine)) - else - uploadURL = resp.Body.Data.url; - end -end - -function accessToken = getToken() - authClient = ebrains.iam.AuthenticationClient.instance(); - accessToken = authClient.AccessToken; - return - - accessToken = getenv('EBRAINS_ACCESS_TOKEN'); - if isempty(accessToken) - accessToken = nansen.module.sharebrain.services.ebrains.uiGetToken(); - end -end \ No newline at end of file diff --git a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m index 05e64f7..4ca00ec 100644 --- a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m +++ b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m @@ -2,23 +2,6 @@ %CHANGERAWDATAROOT Summary of this function goes here % Detailed explanation goes here - -% % % % % % % % % % % % % % % INSTRUCTIONS % % % % % % % % % % % % % % % -% - - - - - - - - - - You can remove this part - - - - - - - - - - - -% Instructions on how to use this template: -% 1) If the session method should have parameters, these should be -% defined in the local function getDefaultParameters at the bottom of -% this script. -% 2) Scroll down to the custom code block below and write code to do -% operations on the sessionObjects and it's data. -% 3) Add documentation (summary and explanation) for the session method -% above. PS: Don't change the function definition (inputs/outputs) -% -% For examples: Press e on the keyboard while browsing the session -% methods. (e) should appear after the name in the menu, and when you -% select a session method, the m-file will open. - - % % % % % % % % % % % % CONFIGURATION CODE BLOCK % % % % % % % % % % % % % Create a struct of default parameters (if applicable) and specify one or % more attributes (see nansen.session.SessionMethod.setAttributes) for @@ -56,13 +39,11 @@ % Get data locations, make a list of available roots, let user select % which one to use and then apply that data location to all sessions - import nansen.module.sharebrain.services.ebrains.uploadFileToBucket + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); - project = nansen.getCurrentProject(); - configFolder = project.getConfigurationFolder(); - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); if ~isfile(filePath) - nansen.module.sharebrain.internal.uiGetBucketId() + % Todo: Consider throwing error instead. + nansen.module.sharebrain.internal.uiSelectDatasetCollab() end collabInfo = jsondecode( fileread(filePath)); bucketName = collabInfo.title; @@ -88,7 +69,7 @@ if contains(filePath, rootDataFolder) relativeFilePath = strrep(filePath, rootDataFolder, ''); relativeFilePath = strrep(relativeFilePath, ' ', '_'); - uploadFileToBucket(filePath, relativeFilePath, bucketName) + ebrains.bucket.uploadFile(filePath, relativeFilePath, bucketName) end end diff --git a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadMetadata.m b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadMetadata.m index 0ba9bdd..9c26e5a 100644 --- a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadMetadata.m +++ b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadMetadata.m @@ -2,23 +2,6 @@ %CHANGERAWDATAROOT Summary of this function goes here % Detailed explanation goes here - -% % % % % % % % % % % % % % % INSTRUCTIONS % % % % % % % % % % % % % % % -% - - - - - - - - - - You can remove this part - - - - - - - - - - - -% Instructions on how to use this template: -% 1) If the session method should have parameters, these should be -% defined in the local function getDefaultParameters at the bottom of -% this script. -% 2) Scroll down to the custom code block below and write code to do -% operations on the sessionObjects and it's data. -% 3) Add documentation (summary and explanation) for the session method -% above. PS: Don't change the function definition (inputs/outputs) -% -% For examples: Press e on the keyboard while browsing the session -% methods. (e) should appear after the name in the menu, and when you -% select a session method, the m-file will open. - - % % % % % % % % % % % % CONFIGURATION CODE BLOCK % % % % % % % % % % % % % Create a struct of default parameters (if applicable) and specify one or % more attributes (see nansen.session.SessionMethod.setAttributes) for diff --git a/+nansen/+module/+sharebrain/+type/Session.m b/+nansen/+module/+sharebrain/+type/Session.m new file mode 100644 index 0000000..9c92147 --- /dev/null +++ b/+nansen/+module/+sharebrain/+type/Session.m @@ -0,0 +1,30 @@ +classdef Session < nansen.metadata.type.Session +%Session - A NANSEN session with the state of its subject from the EBRAINS Knowledge Graph +% +% The Knowledge Graph records values that change over time, such as age +% and weight, on subject states (openMINDS SubjectState) rather than on +% the subject, and a state describes the subject during the sessions it +% applies to. This class adds the values of that state to the session. +% sharebrain.importKGSubjects fills them. +% +% Create the session table of a project with this class to have the +% columns from the start: +% nansen.config.initializeSessionTable(dataLocationModel, ... +% @nansen.module.sharebrain.type.Session, SkipInteractiveSteps=true) +% +% See also sharebrain.importKGSubjects, nansen.module.sharebrain.type.Subject + + properties (SetObservable) + SubjectAge char % Age of the subject, value and unit, e.g. '3 month' + SubjectAgeCategory char % Age category, e.g. 'adult' + SubjectWeight char % Weight of the subject, value or range and unit, e.g. '400-500 gram' + SubjectAttributes char % Attributes of the subject state, e.g. 'alive, awake' + SubjectStateKGId char % UUID of the openMINDS SubjectState in the Knowledge Graph + end + + methods + function obj = Session(varargin) + obj@nansen.metadata.type.Session(varargin{:}) + end + end +end diff --git a/+nansen/+module/+sharebrain/+type/Subject.m b/+nansen/+module/+sharebrain/+type/Subject.m new file mode 100644 index 0000000..cc43e09 --- /dev/null +++ b/+nansen/+module/+sharebrain/+type/Subject.m @@ -0,0 +1,19 @@ +classdef Subject < nansen.metadata.type.Subject +%Subject - A NANSEN subject linked to its subject in the EBRAINS Knowledge Graph +% +% Adds to nansen.metadata.type.Subject the identifiers of the openMINDS +% Subject that the Knowledge Graph lists for the dataset version, so a +% row of a project's subject table can be traced back to it. +% sharebrain.importKGSubjects fills these properties together with +% BiologicalSex, Species and Strain. +% +% The properties that change over time (age, weight) belong to subject +% states and are held by nansen.module.sharebrain.type.Session. +% +% See also sharebrain.importKGSubjects, nansen.module.sharebrain.type.Session + + properties (SetObservable) + KGInstanceId char % UUID of the openMINDS Subject in the Knowledge Graph + KGLookupLabel char % lookupLabel of the Subject in the Knowledge Graph + end +end diff --git a/+nansen/+module/+sharebrain/constants.m b/+nansen/+module/+sharebrain/constants.m new file mode 100644 index 0000000..cf553b6 --- /dev/null +++ b/+nansen/+module/+sharebrain/constants.m @@ -0,0 +1,13 @@ +classdef constants + properties (Constant) + DataSharingCollabPrefix = "d-" + CollabInfoFilename = "ebrains_collab_info.json" + + % Names the bucket of a dataset cloned from EBRAINS, in the root + % folder of the clone + BucketInfoFilename = ".ebrains-bucket.json" + + % URL Constants + NeurosiftBaseUrl = "https://neurosift.app/?p=/nwb" + end +end diff --git a/tests/EbrainsBucketSourceTest.m b/tests/EbrainsBucketSourceTest.m new file mode 100644 index 0000000..db94d14 --- /dev/null +++ b/tests/EbrainsBucketSourceTest.m @@ -0,0 +1,106 @@ +classdef EbrainsBucketSourceTest < matlab.unittest.TestCase +%EbrainsBucketSourceTest - Tests for EbrainsBucketSource and the bucket info file of a clone +% +% Each test makes a clone folder with a .ebrains-bucket.json and empty +% placeholder files, as sharebrain.cloneKGDataset leaves them. The +% downloader is a fake that writes the bucket and object name it is +% given into the target file, so no test connects to EBRAINS. +% +% Run tests: +% runtests('EbrainsBucketSourceTest') + + properties + CloneFolder string + OutsideFolder string + end + + methods (TestMethodSetup) + function createClone(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + testCase.CloneFolder = string(testCase.applyFixture(TemporaryFolderFixture).Folder); + testCase.OutsideFolder = string(testCase.applyFixture(TemporaryFolderFixture).Folder); + + nansen.module.sharebrain.ebrains.writeBucketInfo(testCase.CloneFolder, "my-bucket", ... + Prefix="shared-folder/", DatasetVersionId="0b4a1f5e-0000-4000-8000-000000000000"); + + mkdir(fullfile(testCase.CloneFolder, "shared-folder", "sub")) + createFile(fullfile(testCase.CloneFolder, "shared-folder", "sub", "cell_1a.abf"), "") + createFile(fullfile(testCase.CloneFolder, "shared-folder", "downloaded.txt"), "content") + createFile(fullfile(testCase.OutsideFolder, "empty.abf"), "") + end + end + + methods (Test) + function testFindBucketInfoReadsFileAboveFile(testCase) + [bucketInfo, cloneFolder] = nansen.module.sharebrain.ebrains.findBucketInfo( ... + fullfile(testCase.CloneFolder, "shared-folder", "sub", "cell_1a.abf")); + + testCase.verifyEqual(string(bucketInfo.bucket), "my-bucket") + testCase.verifyEqual(string(bucketInfo.prefix), "shared-folder/") + testCase.verifyEqual(cloneFolder, testCase.CloneFolder) + end + + function testFindBucketInfoOutsideCloneIsEmpty(testCase) + bucketInfo = nansen.module.sharebrain.ebrains.findBucketInfo( ... + fullfile(testCase.OutsideFolder, "empty.abf")); + + testCase.verifyEmpty(bucketInfo) + end + + function testEmptyFileInCloneIsOnlineOnly(testCase) + source = testCase.createSource(); + + testCase.verifyTrue(source.isOnlineOnly( ... + fullfile(testCase.CloneFolder, "shared-folder", "sub", "cell_1a.abf"))) + end + + function testFileWithContentIsNotOnlineOnly(testCase) + source = testCase.createSource(); + + testCase.verifyFalse(source.isOnlineOnly( ... + fullfile(testCase.CloneFolder, "shared-folder", "downloaded.txt"))) + end + + function testEmptyFileOutsideCloneIsNotOnlineOnly(testCase) + source = testCase.createSource(); + + testCase.verifyFalse(source.isOnlineOnly(fullfile(testCase.OutsideFolder, "empty.abf"))) + end + + function testDownloadRequestsObjectAtPathInClone(testCase) + % The clone keeps the folder of a shared bucket in its paths, so + % the object name is the path relative to the clone folder + source = testCase.createSource(); + filePath = fullfile(testCase.CloneFolder, "shared-folder", "sub", "cell_1a.abf"); + + source.download(filePath) + + testCase.verifyEqual(strtrim(string(fileread(filePath))), ... + "my-bucket shared-folder/sub/cell_1a.abf") + end + + function testDownloadOutsideCloneErrors(testCase) + source = testCase.createSource(); + + testCase.verifyError(@() source.download(fullfile(testCase.OutsideFolder, "empty.abf")), ... + 'SHAREbrain:EbrainsBucketSource:NotInClone') + end + end + + methods (Access = private) + function source = createSource(testCase) + source = nansen.module.sharebrain.dataio.EbrainsBucketSource(testCase.CloneFolder, ... + Downloader=@writeRequestIntoTarget); + end + end +end + +function writeRequestIntoTarget(bucketName, objectName, targetFile) + writelines(bucketName + " " + objectName, targetFile) +end + +function createFile(filePath, content) + fileID = fopen(filePath, "w"); + fprintf(fileID, "%s", content); + fclose(fileID); +end diff --git a/tests/FileAdapterTest.m b/tests/FileAdapterTest.m new file mode 100644 index 0000000..15e64a0 --- /dev/null +++ b/tests/FileAdapterTest.m @@ -0,0 +1,199 @@ +classdef FileAdapterTest < matlab.unittest.TestCase +%FileAdapterTest - Tests for the file adapters of the module +% +% Each test writes a small file of the format and loads it with the +% adapter: DelimitedText, JSON, XML, NIfTI, NWB, and NeoPickle and +% NeoNix, whose files are written with Neo in Python. ABF and IgorPro +% are not tested here: MATLAB cannot write either format. +% +% Run tests: +% runtests('FileAdapterTest') + + properties + Folder string + end + + methods (TestMethodSetup) + function createFolder(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + testCase.Folder = string(testCase.applyFixture(TemporaryFolderFixture).Folder); + end + end + + methods (Test) + function testDelimitedTextReadsTabSeparatedColumns(testCase) + filePath = fullfile(testCase.Folder, "trace.txt"); + writelines(["time" + char(9) + "voltage"; "0" + char(9) + "-65.5"; "0.1" + char(9) + "-64.25"], filePath) + + data = nansen.module.sharebrain.fileadapter.DelimitedText(char(filePath)).load(); + + testCase.verifyClass(data, 'table') + testCase.verifyEqual(data.Properties.VariableNames, {'time', 'voltage'}) + testCase.verifyEqual(data.voltage, [-65.5; -64.25]) + end + + function testDelimitedTextKeepsCommentHeader(testCase) + filePath = fullfile(testCase.Folder, "trace_ch5_578.dat"); + writelines(["#"; "# DATE#17/09/2015 (5)"; "# STIMULUSDEF##" + char(9) + "Name" + char(9) + "Units"; ... + "0" + char(9) + "2.25e-08"; "0.2" + char(9) + "2.875e-08"], filePath) + + data = nansen.module.sharebrain.fileadapter.DelimitedText(char(filePath)).load(); + + testCase.verifyEqual(width(data), 2) + testCase.verifyEqual(data{:, 2}, [2.25e-08; 2.875e-08]) + testCase.verifyTrue(contains(data.Properties.Description, "DATE#17/09/2015")) + end + + function testDelimitedTextSkipsCommentLinesAfterData(testCase) + filePath = fullfile(testCase.Folder, "positions.csv"); + writelines(["x,y"; "1,2"; "# pause"; "3,4"], filePath) + + data = nansen.module.sharebrain.fileadapter.DelimitedText(char(filePath)).load(); + + testCase.verifyEqual(data.Properties.VariableNames, {'x', 'y'}) + testCase.verifyEqual(data.y, [2; 4]) + end + + function testDelimitedTextReadsCsv(testCase) + filePath = fullfile(testCase.Folder, "positions.csv"); + writelines(["x,y"; "1,2"; "3,4"], filePath) + + data = nansen.module.sharebrain.fileadapter.DelimitedText(char(filePath)).load(); + + testCase.verifyEqual(data.y, [2; 4]) + end + + function testJSONReturnsDecodedContent(testCase) + filePath = fullfile(testCase.Folder, "sidecar.json"); + writelines('{"RepetitionTime": 2.0, "SliceTiming": [0, 1]}', filePath) + + data = nansen.module.sharebrain.fileadapter.JSON(char(filePath)).load(); + + testCase.verifyEqual(data.RepetitionTime, 2) + testCase.verifyEqual(data.SliceTiming, [0; 1]) + end + + function testXMLReturnsElementsAndAttributes(testCase) + filePath = fullfile(testCase.Folder, "spatial_data.xml"); + writelines('0.5', filePath) + + data = nansen.module.sharebrain.fileadapter.XML(char(filePath)).load(); + + testCase.verifyEqual(data.versionAttribute, 2) + testCase.verifyEqual(data.scale.Text, 0.5) + testCase.verifyEqual(data.scale.unitAttribute, "um") + end + + function testNIfTIReadsCompressedVolume(testCase) + testCase.assumeEqual(exist('niftiwrite', 'file'), 2, 'The Image Processing Toolbox is not installed.') + volume = reshape(single(1:24), 2, 3, 4); + niftiwrite(volume, char(fullfile(testCase.Folder, "run.nii")), 'Compressed', true) + + data = nansen.module.sharebrain.fileadapter.NIfTI( ... + char(fullfile(testCase.Folder, "run.nii.gz"))).load(); + + testCase.verifyEqual(data.Data, volume) + testCase.verifyEqual(data.Header.ImageSize, [2, 3, 4]) + end + + function testNeoPickleReturnsSignalsAndSpikes(testCase) + filePath = testCase.writeNeoBlock("block.pkl", "pickle"); + + block = nansen.module.sharebrain.fileadapter.NeoPickle(char(filePath)).load(); + + testCase.verifyNeoBlock(block) + end + + function testNeoPickleKeepsListsAsCellArrays(testCase) + % An empty segment, event labels, a channel annotation with a + % missing value, and two annotation keys that give the same + % MATLAB field name + filePath = testCase.writeNeoBlock("lists.pkl", "pickle", [ + "block = neo.Block(name='lists', **{'a-b': 1, 'a_b': 2})" + "segment = neo.Segment(name='trial 1')" + "segment.analogsignals.append(neo.AnalogSignal(np.zeros((3, 2)), units='mV', sampling_rate=1 * pq.kHz," + " array_annotations={'note': np.array([None, 'x'], dtype=object)}))" + "segment.events.append(neo.Event([1, 2] * pq.s, labels=np.array(['start', 'stop'])))" + "block.segments.extend([segment, neo.Segment(name='empty')])"]); + + block = nansen.module.sharebrain.fileadapter.NeoPickle(char(filePath)).load(); + + testCase.verifySize(block.segments, [1, 2]) + testCase.verifyClass(block.segments{2}.analogsignals, 'cell') + testCase.verifyEmpty(block.segments{2}.analogsignals) + testCase.verifyEqual(block.segments{1}.events{1}.labels, {'start', 'stop'}) + testCase.verifyEqual(block.segments{1}.analogsignals{1}.array_annotations.note, {'', 'x'}) + % Python integers arrive as int64 + testCase.verifyEqual(double([block.annotations.a_b, block.annotations.a_b_2]), [1, 2]) + end + + function testNeoNixReturnsSignalsAndSpikes(testCase) + filePath = testCase.writeNeoBlock("block.nio", "nix"); + + block = nansen.module.sharebrain.fileadapter.NeoNix(char(filePath)).load(); + + testCase.verifyNeoBlock(block) + end + + function testNWBReturnsNwbFile(testCase) + testCase.assumeEqual(exist('nwbRead', 'file'), 2, 'MatNWB is not on the path.') + filePath = fullfile(testCase.Folder, "session.nwb"); + nwb = NwbFile( ... + 'session_description', 'A test session', ... + 'identifier', 'test-session', ... + 'session_start_time', datetime(2024, 1, 2, 3, 4, 5, 'TimeZone', 'UTC')); + nwbExport(nwb, char(filePath)) + + data = nansen.module.sharebrain.fileadapter.NWB(char(filePath)).load(); + + testCase.verifyClass(data, 'NwbFile') + testCase.verifyEqual(string(data.identifier), "test-session") + end + end + + methods (Access = private) + function filePath = writeNeoBlock(testCase, fileName, fileFormat, blockLines) + % Writes the block that blockLines build in Python. The default + % is a block with one segment that holds a 2-channel signal of + % 100 samples at 1 kHz in mV, and a spike train at 0.1 and 0.5 s + if nargin < 4 + blockLines = [ + "block = neo.Block(name='test block')" + "segment = neo.Segment(name='trial 1')" + "segment.analogsignals.append(neo.AnalogSignal(np.arange(200).reshape(100, 2) * 1.0," + " units='mV', sampling_rate=1 * pq.kHz, name='vm'))" + "segment.spiketrains.append(neo.SpikeTrain([0.1, 0.5] * pq.s, t_stop=1 * pq.s, name='unit 1'))" + "block.segments.append(segment)"]; + end + python = string(pyenv().Executable); + testCase.assumeNotEmpty(char(python), 'No Python is set up for MATLAB.') + [status, ~] = system(sprintf('"%s" -c "import neo, quantities"', python)); + testCase.assumeEqual(status, 0, 'Neo is not installed in the Python for MATLAB.') + + filePath = fullfile(testCase.Folder, fileName); + scriptPath = fullfile(testCase.Folder, "write_block.py"); + writelines([ + "import pickle, sys" + "import numpy as np, quantities as pq, neo" + blockLines + "path, file_format = sys.argv[1], sys.argv[2]" + "if file_format == 'pickle':" + " pickle.dump(block, open(path, 'wb'))" + "else:" + " writer = neo.io.NixIO(path, mode='ow')" + " writer.write_block(block)" + " writer.close()"], scriptPath) + [status, output] = system(sprintf('"%s" "%s" "%s" %s', python, scriptPath, filePath, fileFormat)); + testCase.assumeEqual(status, 0, "Writing the test block failed: " + output) + end + + function verifyNeoBlock(testCase, block) + segment = block.segments{1}; + signal = segment.analogsignals{1}; + testCase.verifyEqual(size(signal.data), [100, 2]) + testCase.verifyEqual(signal.sampling_rate, 1000) + testCase.verifyEqual(strtrim(string(signal.units)), "mV") + testCase.verifyEqual(double(segment.spiketrains{1}.times(:))', [0.1, 0.5], 'AbsTol', 1e-12) + end + end +end