From 0ad00a97217f4308d5e812feceafb57c12d0868f Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 10:46:12 +0200 Subject: [PATCH 01/18] Add getCollabInfoFilepath function to make single source for this filepath --- .../+internal/getCollabIdFromProject.m | 4 ++-- .../+internal/getCollabInfoFilepath.m | 23 +++++++++++++++++++ .../+sharebrain/+internal/uiGetBucketId.m | 5 +--- .../+internal/uiSelectDataSharingCollabInfo.m | 2 +- .../+tool/+sharebrain/openDataSharingCollab.m | 5 ++-- .../+mixin/+tool/+sharebrain/setDataset.m | 6 +---- .../+data/+SHAREBrain/uploadData.m | 5 ++-- +nansen/+module/+sharebrain/constants.m | 5 ++++ 8 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 +nansen/+module/+sharebrain/+internal/getCollabInfoFilepath.m create mode 100644 +nansen/+module/+sharebrain/constants.m diff --git a/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m b/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m index 6c798ab..9cab676 100644 --- a/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m +++ b/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m @@ -1,7 +1,7 @@ function collabId = getCollabIdFromProject(project) - configFolder = project.getConfigurationFolder(); - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(project); + if ~isfile(filePath) nansen.module.sharebrain.internal.uiGetBucketId() end diff --git a/+nansen/+module/+sharebrain/+internal/getCollabInfoFilepath.m b/+nansen/+module/+sharebrain/+internal/getCollabInfoFilepath.m new file mode 100644 index 0000000..5d64d4c --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/getCollabInfoFilepath.m @@ -0,0 +1,23 @@ +function filePath = getCollabInfoFilepath(project) +% getCollabInfoFilepath - Retrieve the file path for the collab information. +% +% Syntax: +% filePath = getCollabInfoFilepath(project) retrieves the path to the +% file containing collab metadata associated with the specified project. +% +% Input Arguments: +% project (optional) - An instance of nansen.config.project.Project. +% If not provided, the current project is used. +% +% Output Arguments: +% filePath - A string representing the file path to the collab metadata file. + + arguments + project (1,1) nansen.config.project.Project = nansen.getCurrentProject() + end + + parentFolder = project.getConfigurationFolder(); + + fileName = nansen.module.sharebrain.constants.CollabInfoFilename; + filePath = fullfile(parentFolder, fileName); +end diff --git a/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m b/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m index de43d62..2102fcf 100644 --- a/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m +++ b/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m @@ -18,13 +18,10 @@ isSelection = strcmp(datasetTitles, selection); bucketId = collab_info(isSelection).title; - - project = nansen.getCurrentProject(); - configFolder = project.getConfigurationFolder(); % Save to project configuration S = collab_info(isSelection); - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); utility.filewrite(filePath, jsonencode(S)) end diff --git a/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m b/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m index 4ecaf8a..2289c82 100644 --- a/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m +++ b/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m @@ -32,7 +32,7 @@ collabInfo = collab_info(isSelection); if isfield(options, "TargetFolder") && ~isempty(options.TargetFolder) - filePath = fullfile(options.TargetFolder, 'ebrains_collab_info.json'); + filePath = fullfile(options.TargetFolder, nansen.module.sharebrain.constants.CollabInfoFilename); utility.filewrite(filePath, jsonencode(collabInfo)) end end diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m index 0a5fbff..9ba6463 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m @@ -1,8 +1,7 @@ function openDataSharingCollab() - project = nansen.getCurrentProject(); - configFolder = project.getConfigurationFolder(); - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); + if ~isfile(filePath) nansen.module.sharebrain.internal.uiGetBucketId() end diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m index 5245d2a..99d2f78 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m @@ -9,11 +9,7 @@ function setDataset() % selection among accessible Dataset Collabs. The selected collab’s name % and metadata are then saved to the project’s configuration folder. - project = nansen.getCurrentProject(); - configFolder = project.getConfigurationFolder(); - - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); - + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); if isfile(filePath) title = 'Reset dataset?'; question = 'EBRAINS Dataset is already configured. Do you want to change this dataset?'; diff --git a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m index 05e64f7..3fed05d 100644 --- a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m +++ b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m @@ -58,9 +58,8 @@ import nansen.module.sharebrain.services.ebrains.uploadFileToBucket - project = nansen.getCurrentProject(); - configFolder = project.getConfigurationFolder(); - filePath = fullfile(configFolder, 'ebrains_collab_info.json'); + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); + if ~isfile(filePath) nansen.module.sharebrain.internal.uiGetBucketId() end diff --git a/+nansen/+module/+sharebrain/constants.m b/+nansen/+module/+sharebrain/constants.m new file mode 100644 index 0000000..d407219 --- /dev/null +++ b/+nansen/+module/+sharebrain/constants.m @@ -0,0 +1,5 @@ +classdef constants + properties (Constant) + CollabInfoFilename = "ebrains_collab_info.json" + end +end From d9aa349d82f92bd0cc6563b11b0730dbd8e0851d Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 14:16:52 +0200 Subject: [PATCH 02/18] Add validators for DataSharingCollab name and UUID Introduces mustBeDataSharingCollabName to validate Data Sharing Collab names by prefix and UUID, and mustBeValidUUID to ensure identifiers match the RFC 4122 UUID format. --- .../+validator/mustBeDataSharingCollabName.m | 13 +++++++++++++ .../+internal/+validator/mustBeValidUUID.m | 13 +++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 +nansen/+module/+sharebrain/+internal/+validator/mustBeDataSharingCollabName.m create mode 100644 +nansen/+module/+sharebrain/+internal/+validator/mustBeValidUUID.m diff --git a/+nansen/+module/+sharebrain/+internal/+validator/mustBeDataSharingCollabName.m b/+nansen/+module/+sharebrain/+internal/+validator/mustBeDataSharingCollabName.m new file mode 100644 index 0000000..bd182da --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/+validator/mustBeDataSharingCollabName.m @@ -0,0 +1,13 @@ +function mustBeDataSharingCollabName(value) +% mustBeDataSharingCollabName - Ensure a name is formatted as a valid DSC name + arguments + value (1,1) string + end + + dscPrefix = nansen.module.sharebrain.constants.DataSharingCollabPrefix; + assert( startsWith(value, dscPrefix), ... + 'Data Sharing Collab name must start with "%s"', dscPrefix) + + uuid = extractAfter(value, dscPrefix); + nansen.module.sharebrain.internal.validator.mustBeValidUUID(uuid) +end diff --git a/+nansen/+module/+sharebrain/+internal/+validator/mustBeValidUUID.m b/+nansen/+module/+sharebrain/+internal/+validator/mustBeValidUUID.m new file mode 100644 index 0000000..03a679f --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/+validator/mustBeValidUUID.m @@ -0,0 +1,13 @@ +function mustBeValidUUID(identifier) + % UUID regex: 8-4-4-4-12 hex digits (RFC 4122 format) + uuidPattern = "^[0-9a-fA-F]{8}-" + ... + "[0-9a-fA-F]{4}-" + ... + "[0-9a-fA-F]{4}-" + ... + "[0-9a-fA-F]{4}-" + ... + "[0-9a-fA-F]{12}$"; + + if ~contains(identifier, regexpPattern(uuidPattern)) + error("SHAREBRAIN:validator:InvalidUUID", ... + "Identifier must be a valid UUID. Got: %s", identifier); + end +end From 021eff651ce68eb2c907e6ad8c443802b976a6a4 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 14:17:10 +0200 Subject: [PATCH 03/18] Add fetchCollabInfo function for collab metadata Introduces fetchCollabInfo.m to retrieve metadata for a Data Sharing Collab using the ebrains.collaboratory.api.Collab client. --- .../+sharebrain/+internal/+dsc/fetchCollabInfo.m | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 +nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m diff --git a/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m b/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m new file mode 100644 index 0000000..6f58a08 --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m @@ -0,0 +1,15 @@ +function fetchCollabInfo(collabId) +% fetchCollabInfo - Fetch info / metadata for a Data Sharing Collab + arguments + collabId (1,1) string {nansen.module.sharebrain.internal.validator.mustBeDataSharingCollabName} + end + + collabClient = ebrains.collaboratory.api.Collab(); + [status, collabInfo] = collabClient.getCollab(collabId); + if status ~= matlab.net.http.StatusCode.OK + errorInfo = jsondecode(collabInfo); + error('SHAREBRAIN:FetchCollab:FailedToGetCollab', ... + 'Something went wrong when fetching collab. Reason: \n%s', ... + sprintf("%s: %s", status, errorInfo.description)) + end +end From dd7d6772378264b44e76eb9ff89cadc1024a9dd4 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 14:17:23 +0200 Subject: [PATCH 04/18] Update constants.m --- +nansen/+module/+sharebrain/constants.m | 1 + 1 file changed, 1 insertion(+) diff --git a/+nansen/+module/+sharebrain/constants.m b/+nansen/+module/+sharebrain/constants.m index d407219..1e5bed2 100644 --- a/+nansen/+module/+sharebrain/constants.m +++ b/+nansen/+module/+sharebrain/constants.m @@ -1,5 +1,6 @@ classdef constants properties (Constant) + DataSharingCollabPrefix = "d-" CollabInfoFilename = "ebrains_collab_info.json" end end From 98fbf9a57dad290b7aa2193fb6ee65952ebc6865 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 14:40:38 +0200 Subject: [PATCH 05/18] delete uiGetBucketId Replace with uiSelectDataSharingCollabInfo as this function has an almost identical implementation --- .../+internal/+dsc/listDatasetCollabs.m | 29 ++++++++++++ .../+internal/getCollabIdFromProject.m | 3 +- .../+sharebrain/+internal/uiGetBucketId.m | 44 ------------------- .../+internal/uiSelectDataSharingCollabInfo.m | 27 +++--------- .../+tool/+sharebrain/openDataSharingCollab.m | 4 +- .../+mixin/+tool/+sharebrain/setDataset.m | 3 +- .../+data/+SHAREBrain/uploadData.m | 3 +- 7 files changed, 43 insertions(+), 70 deletions(-) create mode 100644 +nansen/+module/+sharebrain/+internal/+dsc/listDatasetCollabs.m delete mode 100644 +nansen/+module/+sharebrain/+internal/uiGetBucketId.m diff --git a/+nansen/+module/+sharebrain/+internal/+dsc/listDatasetCollabs.m b/+nansen/+module/+sharebrain/+internal/+dsc/listDatasetCollabs.m new file mode 100644 index 0000000..441aa57 --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/+dsc/listDatasetCollabs.m @@ -0,0 +1,29 @@ +function collab_info = listDatasetCollabs(options) +% listDatasetCollabs - List dataset collabs + + arguments + options.Verbose (1,1) logical = true + end + + SEARCH_TERM = 'Dataset working title'; + + if options.Verbose + fprintf('Please wait, fetching dataset collabs...\n') + end + + collabClient = ebrains.collaboratory.api.Collab(); + [~, result, ~] = collabClient.searchCollab(... + 'search', SEARCH_TERM, ... + 'limit', 1000, ... + 'visibility', 'private'); + + [~, isSelection] = sort([result.createDate], 'descend'); + collab_info = result(isSelection); + + if isempty(collab_info) + error('SHAREbrain:UISelectDatasetCollab:NoCollabsFound', ... + ['Could not find any data sharing collabs. You might not be a ', ... + 'member of any data sharing collabs. Please verify access ', ... + 'permissions with your curator']) + end +end diff --git a/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m b/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m index 9cab676..b89e2f8 100644 --- a/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m +++ b/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m @@ -3,8 +3,9 @@ filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(project); if ~isfile(filePath) - nansen.module.sharebrain.internal.uiGetBucketId() + nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo() end + collabInfo = jsondecode( fileread(filePath)); collabId = collabInfo.name; end \ No newline at end of file diff --git a/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m b/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m deleted file mode 100644 index 2102fcf..0000000 --- a/+nansen/+module/+sharebrain/+internal/uiGetBucketId.m +++ /dev/null @@ -1,44 +0,0 @@ -function bucketId = uiGetBucketId() - - myClient = ebrains.collaboratory.api.Collab(); - [~,result,~] = myClient.searchCollab('search', 'Dataset working title', 'limit', 1000, 'visibility', 'private'); - - [~, isSelection] = sort([result.createDate], 'descend'); - collab_info = result(isSelection); - - % parse description: - expression = '(?<=Dataset working title: ).+?(?=\n)'; - datasetTitles = regexp([collab_info.description], expression, 'match', 'once')'; - - selection = uim.dialog.searchSelectDlg(cellstr(datasetTitles), 'Select a Dataset'); - - if isempty(selection) - error('Aborted') - end - - isSelection = strcmp(datasetTitles, selection); - bucketId = collab_info(isSelection).title; - - % Save to project configuration - S = collab_info(isSelection); - filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); - utility.filewrite(filePath, jsonencode(S)) -end - -% % BASE_URL = "https://wiki.ebrains.eu/rest/"; -% % -% % bucketId = inputdlg('Please enter your bucket ID'); -% % -% % webOpts = weboptions; -% % webOpts.HeaderFields = ["Authorization", sprintf("Bearer %s", myClient.bearerToken)]; -% % webOpts.HeaderFields = ["Authorization", sprintf("Bearer %s", token)]; -% % collabResp = webread(BASE_URL+"v1/collabs", 'limit', 300, 'search', 'd-','visibility', 'private', webOpts); -% % -% % [~, idx] = sort([collabResp.createDate]); -% % collab_info = collabResp(idx); -% % -% % timestamp = [collab_info.createDate] / 1000; % Convert milliseconds to seconds -% % datetimeObj = num2cell( datetime(timestamp, 'ConvertFrom', 'posixtime') ); -% % [collab_info(:).createDate] = deal(datetimeObj{:}); -% % struct2table(collab_info) -% % end \ No newline at end of file diff --git a/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m b/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m index 2289c82..66149c3 100644 --- a/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m +++ b/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m @@ -1,38 +1,23 @@ -function collabInfo = uiSelectDataSharingCollabInfo(options) +function collabInfo = uiSelectDataSharingCollabInfo() arguments - options.TargetFolder (1,:) string {mustBeFolder} %options.Logger % Todo end - myClient = ebrains.collaboratory.api.Collab(); - fprintf('Please wait, fetching datasets...\n') - [~,result,~] = myClient.searchCollab('search', 'Dataset working title', 'limit', 1000, 'visibility', 'private'); - - [~, isSelection] = sort([result.createDate], 'descend'); - collab_info = result(isSelection); - - if isempty(collab_info) - error('SHAREbrain:UISelectDatasetCollab:NoCollabsFound', ... - ['Could not find any data sharing collabs. You might not be a ', ... - 'member of any data sharing collabs. Please verify access ', ... - 'permissions with your curator']) - end + collab_info = nansen.module.sharebrain.internal.dsc.listDatasetCollabs(); - % parse description: + % Parse description: expression = '(?<=Dataset working title: ).+?(?=\n)'; datasetTitles = regexp([collab_info.description], expression, 'match', 'once')'; selection = uim.dialog.searchSelectDlg(cellstr(datasetTitles), 'Select a Dataset'); if isempty(selection) - error('Aborted') + error('Aborted') % Todo: Add error id and detailed error message end isSelection = strcmp(datasetTitles, selection); collabInfo = collab_info(isSelection); - if isfield(options, "TargetFolder") && ~isempty(options.TargetFolder) - filePath = fullfile(options.TargetFolder, nansen.module.sharebrain.constants.CollabInfoFilename); - utility.filewrite(filePath, jsonencode(collabInfo)) - end + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); + utility.filewrite(filePath, jsonencode(collabInfo)) end diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m index 9ba6463..ffb5210 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openDataSharingCollab.m @@ -3,7 +3,9 @@ function openDataSharingCollab() filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); if ~isfile(filePath) - nansen.module.sharebrain.internal.uiGetBucketId() + error(... + ['This project is not associated with any dataset collabs. ', ... + 'Please link to a dataset collab using the "Set Dataset" method']) % Todo: add error id, improve error message end collabInfo = jsondecode( fileread(filePath)); diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m index 99d2f78..6a97131 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m @@ -23,6 +23,5 @@ function setDataset() end end - nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo(... - "TargetFolder", configFolder) + nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo() end diff --git a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m index 3fed05d..6614210 100644 --- a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m +++ b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m @@ -61,7 +61,8 @@ filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); if ~isfile(filePath) - nansen.module.sharebrain.internal.uiGetBucketId() + % Todo: Consider throwing error instead. + nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo() end collabInfo = jsondecode( fileread(filePath)); bucketName = collabInfo.title; From 70828af6c24b9a867427a660dbfb3b58a37e3c70 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 14:43:31 +0200 Subject: [PATCH 06/18] Renamed "nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo" to "nansen.module.sharebrain.internal.uiSelectDatasetCollab". --- +nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m | 2 +- ...{uiSelectDataSharingCollabInfo.m => uiSelectDatasetCollab.m} | 2 +- .../+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m | 2 +- .../+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) rename +nansen/+module/+sharebrain/+internal/{uiSelectDataSharingCollabInfo.m => uiSelectDatasetCollab.m} (93%) diff --git a/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m b/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m index b89e2f8..7ea0b92 100644 --- a/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m +++ b/+nansen/+module/+sharebrain/+internal/getCollabIdFromProject.m @@ -3,7 +3,7 @@ filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(project); if ~isfile(filePath) - nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo() + nansen.module.sharebrain.internal.uiSelectDatasetCollab() end collabInfo = jsondecode( fileread(filePath)); diff --git a/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m b/+nansen/+module/+sharebrain/+internal/uiSelectDatasetCollab.m similarity index 93% rename from +nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m rename to +nansen/+module/+sharebrain/+internal/uiSelectDatasetCollab.m index 66149c3..c383c41 100644 --- a/+nansen/+module/+sharebrain/+internal/uiSelectDataSharingCollabInfo.m +++ b/+nansen/+module/+sharebrain/+internal/uiSelectDatasetCollab.m @@ -1,4 +1,4 @@ -function collabInfo = uiSelectDataSharingCollabInfo() +function collabInfo = uiSelectDatasetCollab() arguments %options.Logger % Todo end diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m index 6a97131..ec18150 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/setDataset.m @@ -23,5 +23,5 @@ function setDataset() end end - nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo() + nansen.module.sharebrain.internal.uiSelectDatasetCollab() end diff --git a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m index 6614210..c2670c9 100644 --- a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m +++ b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m @@ -62,7 +62,7 @@ if ~isfile(filePath) % Todo: Consider throwing error instead. - nansen.module.sharebrain.internal.uiSelectDataSharingCollabInfo() + nansen.module.sharebrain.internal.uiSelectDatasetCollab() end collabInfo = jsondecode( fileread(filePath)); bucketName = collabInfo.title; From f59bb0926c62702f7c82d9d6d1eba16ea64cbc26 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 17:41:49 +0200 Subject: [PATCH 07/18] Deleted functions for uploading and downloading files to/from bucket These functions will be part of the EBRAINS Services Toolbox --- .../+ebrains/downloadFileFromBucket.m | 80 ------------------- .../+services/+ebrains/uploadFileToBucket.m | 57 ------------- .../+data/+SHAREBrain/uploadData.m | 4 +- 3 files changed, 1 insertion(+), 140 deletions(-) delete mode 100644 +nansen/+module/+sharebrain/+services/+ebrains/downloadFileFromBucket.m delete mode 100644 +nansen/+module/+sharebrain/+services/+ebrains/uploadFileToBucket.m diff --git a/+nansen/+module/+sharebrain/+services/+ebrains/downloadFileFromBucket.m b/+nansen/+module/+sharebrain/+services/+ebrains/downloadFileFromBucket.m deleted file mode 100644 index 80b4084..0000000 --- a/+nansen/+module/+sharebrain/+services/+ebrains/downloadFileFromBucket.m +++ /dev/null @@ -1,80 +0,0 @@ -function downloadFileFromBucket(filePath, relativeFilePath, bucketName, progressDisplay) -% downloadFileFromBucket - Download a file from an EBRAINS bucket (swift object storage) - - if strncmp(char(relativeFilePath), filesep, 1) - relativeFilePath = char(relativeFilePath); - relativeFilePath = relativeFilePath(2:end); - end - - BASE_API_URL = nansen.module.sharebrain.constant.DataProxyBaseUrl; - endpointPath = sprintf("buckets/%s/%s", bucketName, relativeFilePath); - - apiURL = BASE_API_URL + endpointPath; - - disp('starting download') - - % finished = false; - % while ~finished - %try - %downloadURL = getDownloadUrl(apiURL, useToken=false); - - try - webFileSize = nansen.module.sharebrain.internal.fileio.getWebFileSize(apiURL); - [filePath] = downloadFile(filePath, apiURL, ShowFilename=true); - catch ME - fileSize = nansen.module.sharebrain.internal.fileio.getLocalFileSize(filePath); - if fileSize ~= webFileSize - delete(filePath) - filePath = strrep(filePath, ' ', '\ '); - [status, msg] = system( sprintf('touch %s', filePath )); - end - rethrow(ME) - end - - - %task.concrete.downloadFile(filePath, apiURL, 'ProgressDisplay', progressDisplay) - finished = true; - %catch ME - % Reset access token and try again - setenv('EBRAINS_ACCESS_TOKEN', ''); - %end - % end - - %[filePath] = downloadFile(filePath, downloadURL, ShowFilename=true); -end - -function downloadURL = getDownloadUrl(apiURL, options) - arguments - apiURL (1,1) string - options.useToken (1,1) logical = false - end - - if options.useToken - accessToken = getToken(); - - header = matlab.net.http.HeaderField(... - "accept", "application/json", ... - "Authorization", "Bearer " + accessToken); - else - header=[]; - end - - warning('off', 'MATLAB:http:BodyExpectedFor') - method = matlab.net.http.RequestMethod.GET; - req = matlab.net.http.RequestMessage(method, header, []); - [resp, ~, ~] = req.send(apiURL); - warning('on', 'MATLAB:http:BodyExpectedFor') - - if resp.StatusCode ~= matlab.net.http.StatusCode.OK - error(string(resp.StatusLine)) - else - downloadURL = resp.Body.Data.url; - end -end - -function accessToken = getToken() - accessToken = getenv('EBRAINS_ACCESS_TOKEN'); - if isempty(accessToken) - accessToken = nansen.module.sharebrain.services.ebrains.uiGetToken(); - end -end \ No newline at end of file diff --git a/+nansen/+module/+sharebrain/+services/+ebrains/uploadFileToBucket.m b/+nansen/+module/+sharebrain/+services/+ebrains/uploadFileToBucket.m deleted file mode 100644 index f5ffd9e..0000000 --- a/+nansen/+module/+sharebrain/+services/+ebrains/uploadFileToBucket.m +++ /dev/null @@ -1,57 +0,0 @@ -function uploadFileToBucket(filePath, relativeFilePath, bucketName) - - if strncmp(char(relativeFilePath), filesep, 1) - relativeFilePath = char(relativeFilePath); - relativeFilePath = relativeFilePath(2:end); - end - - BASE_API_URL = nansen.module.sharebrain.constant.DataProxyBaseUrl; - endpointPath = sprintf("buckets/%s/data/%s", bucketName, relativeFilePath); - - apiURL = BASE_API_URL + endpointPath; - - finished = false; - while ~finished - try - uploadURL = getUploadUrl(apiURL); - finished = true; - catch ME - % Reset access token and try again - setenv('EBRAINS_ACCESS_TOKEN', ''); - end - end - - [wasSuccess, response] = uploadFile(filePath, uploadURL, ShowFilename=true); -end - -function uploadURL = getUploadUrl(apiURL) - - accessToken = getToken(); - - header = matlab.net.http.HeaderField(... - "accept", "application/json", ... - "Authorization", "Bearer " + accessToken); - - warning('off', 'MATLAB:http:BodyExpectedFor') - method = matlab.net.http.RequestMethod.PUT; - req = matlab.net.http.RequestMessage(method, header, []); - [resp, ~, ~] = req.send(apiURL); - warning('on', 'MATLAB:http:BodyExpectedFor') - - if resp.StatusCode ~= matlab.net.http.StatusCode.OK - error(string(resp.StatusLine)) - else - uploadURL = resp.Body.Data.url; - end -end - -function accessToken = getToken() - authClient = ebrains.iam.AuthenticationClient.instance(); - accessToken = authClient.AccessToken; - return - - accessToken = getenv('EBRAINS_ACCESS_TOKEN'); - if isempty(accessToken) - accessToken = nansen.module.sharebrain.services.ebrains.uiGetToken(); - end -end \ No newline at end of file diff --git a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m index c2670c9..ff5f6a7 100644 --- a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m +++ b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m @@ -56,8 +56,6 @@ % Get data locations, make a list of available roots, let user select % which one to use and then apply that data location to all sessions - import nansen.module.sharebrain.services.ebrains.uploadFileToBucket - filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(); if ~isfile(filePath) @@ -88,7 +86,7 @@ if contains(filePath, rootDataFolder) relativeFilePath = strrep(filePath, rootDataFolder, ''); relativeFilePath = strrep(relativeFilePath, ' ', '_'); - uploadFileToBucket(filePath, relativeFilePath, bucketName) + ebrains.bucket.uploadFile(filePath, relativeFilePath, bucketName) end end From 3d6fab96d98648ba99c72b4834dd54772c529053 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 17:43:59 +0200 Subject: [PATCH 08/18] Add neurosift base url to constants class --- +nansen/+module/+sharebrain/+constant/NeuroSiftBaseUrl.m | 8 -------- .../+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m | 2 +- +nansen/+module/+sharebrain/constants.m | 3 +++ 3 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 +nansen/+module/+sharebrain/+constant/NeuroSiftBaseUrl.m diff --git a/+nansen/+module/+sharebrain/+constant/NeuroSiftBaseUrl.m b/+nansen/+module/+sharebrain/+constant/NeuroSiftBaseUrl.m deleted file mode 100644 index 0f7c87e..0000000 --- a/+nansen/+module/+sharebrain/+constant/NeuroSiftBaseUrl.m +++ /dev/null @@ -1,8 +0,0 @@ -function url = NeuroSiftBaseUrl() -%NeuroSiftBaseUrl Base URL for neurosift app - url = "https://neurosift.app/?p=/nwb"; -end - - - - diff --git a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m index fe20389..049aacb 100644 --- a/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m +++ b/+nansen/+module/+sharebrain/+mixin/+tool/+sharebrain/openNwbFileInNeurosift.m @@ -1,6 +1,6 @@ function openNwbFileInNeurosift() - baseUrl = nansen.module.sharebrain.constant.NeuroSiftBaseUrl(); + baseUrl = nansen.module.sharebrain.constants.NeurosiftBaseUrl; fileUrl = inputdlg('Please enter dataproxy api url:'); diff --git a/+nansen/+module/+sharebrain/constants.m b/+nansen/+module/+sharebrain/constants.m index 1e5bed2..e51c72c 100644 --- a/+nansen/+module/+sharebrain/constants.m +++ b/+nansen/+module/+sharebrain/constants.m @@ -2,5 +2,8 @@ properties (Constant) DataSharingCollabPrefix = "d-" CollabInfoFilename = "ebrains_collab_info.json" + + % URL Constants + NeurosiftBaseUrl = "https://neurosift.app/?p=/nwb" end end From c05273e133eec852d7ffd6a86a39261f4b2f4179 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 17:44:27 +0200 Subject: [PATCH 09/18] Delete DataProxyBaseUrl.m --- +nansen/+module/+sharebrain/+constant/DataProxyBaseUrl.m | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 +nansen/+module/+sharebrain/+constant/DataProxyBaseUrl.m diff --git a/+nansen/+module/+sharebrain/+constant/DataProxyBaseUrl.m b/+nansen/+module/+sharebrain/+constant/DataProxyBaseUrl.m deleted file mode 100644 index 6d20084..0000000 --- a/+nansen/+module/+sharebrain/+constant/DataProxyBaseUrl.m +++ /dev/null @@ -1,5 +0,0 @@ -function url = DataProxyBaseUrl() -%DataProxyBaseUrl Base URL for data proxy api - url = "https://data-proxy.ebrains.eu/api/v1/"; -end - From e8fd58232a8a3d9077075149d9feb4e26c445c85 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 17:53:45 +0200 Subject: [PATCH 10/18] Add linkCollabToProject function --- .../+internal/+dsc/fetchCollabInfo.m | 2 +- .../+internal/linkCollabToProject.m | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 +nansen/+module/+sharebrain/+internal/linkCollabToProject.m diff --git a/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m b/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m index 6f58a08..7945b85 100644 --- a/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m +++ b/+nansen/+module/+sharebrain/+internal/+dsc/fetchCollabInfo.m @@ -1,4 +1,4 @@ -function fetchCollabInfo(collabId) +function collabInfo = fetchCollabInfo(collabId) % fetchCollabInfo - Fetch info / metadata for a Data Sharing Collab arguments collabId (1,1) string {nansen.module.sharebrain.internal.validator.mustBeDataSharingCollabName} diff --git a/+nansen/+module/+sharebrain/+internal/linkCollabToProject.m b/+nansen/+module/+sharebrain/+internal/linkCollabToProject.m new file mode 100644 index 0000000..3301629 --- /dev/null +++ b/+nansen/+module/+sharebrain/+internal/linkCollabToProject.m @@ -0,0 +1,25 @@ +function linkCollabToProject(collabName) +% linkCollabToProject - Saves the metadata of a collab to current NANSEN project +% +% Syntax: +% nansen.module.sharebrain.internal.linkCollabToProject(collabName) +% +% Input Arguments: +% collabName (1,1) string - The name of the collab to be fetched and saved. + + arguments + collabName (1,1) string {nansen.module.sharebrain.internal.validator.mustBeDataSharingCollabName} + end + + % Fetch collab metadata from EBRAINS Collaboratory + collabInfo = nansen.module.sharebrain.internal.dsc.fetchCollabInfo(collabName); + + % Get filepath in current project to save collab metadata to + project = nansen.getCurrentProject(); + filePath = nansen.module.sharebrain.internal.getCollabInfoFilepath(project); + + % Save collab metadata to current project + fid = fopen(filePath, "wt"); + fileCleanup = onCleanup(@() fclose(fid)); + fwrite(fid, jsonencode(collabInfo)) +end From 4c927d300d511804493ea5baa03381af1da05523 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 20:14:22 +0200 Subject: [PATCH 11/18] Throw error if file not found in getLocalFileSize Replaces returning an empty array with an error when the specified file does not exist, improving error handling and making missing files explicit. --- .../+module/+sharebrain/+internal/+fileio/getLocalFileSize.m | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/+nansen/+module/+sharebrain/+internal/+fileio/getLocalFileSize.m b/+nansen/+module/+sharebrain/+internal/+fileio/getLocalFileSize.m index 870857f..365a00a 100644 --- a/+nansen/+module/+sharebrain/+internal/+fileio/getLocalFileSize.m +++ b/+nansen/+module/+sharebrain/+internal/+fileio/getLocalFileSize.m @@ -6,7 +6,8 @@ L = dir(filepath); if isempty(L) - fileSizeBytes = []; % TODO: Consider whether to throw error instead. + error('SHAREBRAIN:FileNotFound', ... + 'File not found: %s', filepath); elseif numel(L) > 1 error('Multiple files/folders matched the given filepath') else From 4617617eb64a37edf373857ea113c74932ad3454 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Thu, 16 Oct 2025 20:29:11 +0200 Subject: [PATCH 12/18] Remove session method instructions --- .../+data/+SHAREBrain/uploadData.m | 17 ----------------- .../+data/+SHAREBrain/uploadMetadata.m | 17 ----------------- 2 files changed, 34 deletions(-) diff --git a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m index ff5f6a7..4ca00ec 100644 --- a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m +++ b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadData.m @@ -2,23 +2,6 @@ %CHANGERAWDATAROOT Summary of this function goes here % Detailed explanation goes here - -% % % % % % % % % % % % % % % INSTRUCTIONS % % % % % % % % % % % % % % % -% - - - - - - - - - - You can remove this part - - - - - - - - - - - -% Instructions on how to use this template: -% 1) If the session method should have parameters, these should be -% defined in the local function getDefaultParameters at the bottom of -% this script. -% 2) Scroll down to the custom code block below and write code to do -% operations on the sessionObjects and it's data. -% 3) Add documentation (summary and explanation) for the session method -% above. PS: Don't change the function definition (inputs/outputs) -% -% For examples: Press e on the keyboard while browsing the session -% methods. (e) should appear after the name in the menu, and when you -% select a session method, the m-file will open. - - % % % % % % % % % % % % CONFIGURATION CODE BLOCK % % % % % % % % % % % % % Create a struct of default parameters (if applicable) and specify one or % more attributes (see nansen.session.SessionMethod.setAttributes) for diff --git a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadMetadata.m b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadMetadata.m index 0ba9bdd..9c26e5a 100644 --- a/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadMetadata.m +++ b/+nansen/+module/+sharebrain/+sessionmethod/+data/+SHAREBrain/uploadMetadata.m @@ -2,23 +2,6 @@ %CHANGERAWDATAROOT Summary of this function goes here % Detailed explanation goes here - -% % % % % % % % % % % % % % % INSTRUCTIONS % % % % % % % % % % % % % % % -% - - - - - - - - - - You can remove this part - - - - - - - - - - - -% Instructions on how to use this template: -% 1) If the session method should have parameters, these should be -% defined in the local function getDefaultParameters at the bottom of -% this script. -% 2) Scroll down to the custom code block below and write code to do -% operations on the sessionObjects and it's data. -% 3) Add documentation (summary and explanation) for the session method -% above. PS: Don't change the function definition (inputs/outputs) -% -% For examples: Press e on the keyboard while browsing the session -% methods. (e) should appear after the name in the menu, and when you -% select a session method, the m-file will open. - - % % % % % % % % % % % % CONFIGURATION CODE BLOCK % % % % % % % % % % % % % Create a struct of default parameters (if applicable) and specify one or % more attributes (see nansen.session.SessionMethod.setAttributes) for From 40d9495eb2f21f28bff14e850ad54d3be8acbf42 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 18 Sep 2026 22:16:35 +0200 Subject: [PATCH 13/18] feat: add session and subject types and a source for EBRAINS bucket files nansen.module.sharebrain.type.Session and .Subject extend the NANSEN types with the values that sharebrain.importKGSubjects takes from the EBRAINS Knowledge Graph: the subject's identifiers in the Knowledge Graph, and the age, weight and attributes of its subject state on each session. They were in the sharebrain package of the SHAREbrain repository, which env sharebrain does not add to the path; tables of these classes do not open without them. A dataset cloned from an EBRAINS bucket holds empty files. The file .ebrains-bucket.json at the root of the clone names the bucket (writeBucketInfo, findBucketInfo). EbrainsBucketSource implements nansen.dataio.RemoteFileSource: it counts an empty file below such a folder as stored online and downloads it with ebrains.bucket.downloadFile, using the path relative to the clone as the object name. Co-Authored-By: Claude Opus 5 --- .../+sharebrain/+dataio/EbrainsBucketSource.m | 65 +++++++++++ .../+sharebrain/+ebrains/findBucketInfo.m | 35 ++++++ .../+sharebrain/+ebrains/writeBucketInfo.m | 34 ++++++ +nansen/+module/+sharebrain/+type/Session.m | 30 +++++ +nansen/+module/+sharebrain/+type/Subject.m | 19 ++++ +nansen/+module/+sharebrain/constants.m | 4 + tests/EbrainsBucketSourceTest.m | 106 ++++++++++++++++++ 7 files changed, 293 insertions(+) create mode 100644 +nansen/+module/+sharebrain/+dataio/EbrainsBucketSource.m create mode 100644 +nansen/+module/+sharebrain/+ebrains/findBucketInfo.m create mode 100644 +nansen/+module/+sharebrain/+ebrains/writeBucketInfo.m create mode 100644 +nansen/+module/+sharebrain/+type/Session.m create mode 100644 +nansen/+module/+sharebrain/+type/Subject.m create mode 100644 tests/EbrainsBucketSourceTest.m diff --git a/+nansen/+module/+sharebrain/+dataio/EbrainsBucketSource.m b/+nansen/+module/+sharebrain/+dataio/EbrainsBucketSource.m new file mode 100644 index 0000000..b953713 --- /dev/null +++ b/+nansen/+module/+sharebrain/+dataio/EbrainsBucketSource.m @@ -0,0 +1,65 @@ +classdef EbrainsBucketSource < nansen.dataio.RemoteFileSource +%EbrainsBucketSource - Source of the files of a dataset cloned from an EBRAINS bucket +% +% A dataset cloned with sharebrain.cloneKGDataset has an empty file for +% every object of its EBRAINS Data Proxy bucket, and names the bucket in +% the file .ebrains-bucket.json in the root folder of the clone. This +% source counts an empty file below such a folder as stored online, and +% downloads it with ebrains.bucket.downloadFile. The name of the object +% is the path of the file relative to the root folder of the clone, +% because the clone keeps the folder of a shared bucket in its paths. +% +% Make it the source of online files of the current project: +% project = nansen.getCurrentProject(); +% project.setRemoteFileSource("nansen.module.sharebrain.dataio.EbrainsBucketSource") +% +% source = EbrainsBucketSource(rootPath, Downloader=FCN) downloads with +% FCN(bucketName, objectName, targetFile) instead. +% +% See also nansen.dataio.RemoteFileSource, +% nansen.module.sharebrain.ebrains.writeBucketInfo + + properties (SetAccess = private) + % Downloader - Function that downloads an object, called as + % Downloader(bucketName, objectName, targetFile) + Downloader function_handle + end + + methods + function obj = EbrainsBucketSource(rootPath, options) + arguments + rootPath (1,1) string + options.Downloader (1,1) function_handle = @downloadWithProgressInCommandWindow + end + obj@nansen.dataio.RemoteFileSource(rootPath) + obj.Downloader = options.Downloader; + end + + function tf = isOnlineOnly(~, filePath) + fileInfo = dir(filePath); + tf = isscalar(fileInfo) && fileInfo.bytes == 0 ... + && ~isempty(nansen.module.sharebrain.ebrains.findBucketInfo(filePath)); + end + + function download(obj, filePath) + [bucketInfo, cloneFolder] = nansen.module.sharebrain.ebrains.findBucketInfo(filePath); + if isempty(bucketInfo) + error('SHAREbrain:EbrainsBucketSource:NotInClone', ... + ['No folder above "%s" has the file %s, so the bucket of the file is not known. ', ... + 'Clone the dataset with sharebrain.cloneKGDataset, or record its bucket with ', ... + 'nansen.module.sharebrain.ebrains.writeBucketInfo.'], ... + filePath, nansen.module.sharebrain.constants.BucketInfoFilename) + end + + relativePath = extractAfter(string(filePath), strlength(cloneFolder) + 1); + objectName = replace(relativePath, filesep, "/"); + obj.Downloader(string(bucketInfo.bucket), objectName, string(filePath)) + end + end +end + +function downloadWithProgressInCommandWindow(bucketName, objectName, targetFile) +% A dialog box would need a display, which a batch session or an agent +% does not have + ebrains.bucket.downloadFile(bucketName, objectName, targetFile, DisplayMode="Command Window") +end diff --git a/+nansen/+module/+sharebrain/+ebrains/findBucketInfo.m b/+nansen/+module/+sharebrain/+ebrains/findBucketInfo.m new file mode 100644 index 0000000..8f30505 --- /dev/null +++ b/+nansen/+module/+sharebrain/+ebrains/findBucketInfo.m @@ -0,0 +1,35 @@ +function [bucketInfo, cloneFolder] = findBucketInfo(filePath) +%findBucketInfo - The EBRAINS bucket of a cloned dataset that contains a file +% [bucketInfo, cloneFolder] = nansen.module.sharebrain.ebrains.findBucketInfo(filePath) +% looks for .ebrains-bucket.json in the folder of filePath and in each +% folder above it, and reads the first one it finds. bucketInfo is a +% struct with the fields bucket, prefix and datasetVersionId, and +% cloneFolder is the folder that holds the file. +% +% Both outputs are empty when no folder above filePath has the file. +% +% See also nansen.module.sharebrain.ebrains.writeBucketInfo + + arguments + filePath (1,1) string + end + + bucketInfo = struct.empty; + cloneFolder = ""; + + folderPath = fileparts(filePath); + while strlength(folderPath) > 0 + infoFilePath = fullfile(folderPath, nansen.module.sharebrain.constants.BucketInfoFilename); + if isfile(infoFilePath) + bucketInfo = jsondecode(fileread(infoFilePath)); + cloneFolder = folderPath; + return + end + + parentFolder = fileparts(folderPath); + if parentFolder == folderPath % The root of the file system + return + end + folderPath = parentFolder; + end +end diff --git a/+nansen/+module/+sharebrain/+ebrains/writeBucketInfo.m b/+nansen/+module/+sharebrain/+ebrains/writeBucketInfo.m new file mode 100644 index 0000000..6efd7b5 --- /dev/null +++ b/+nansen/+module/+sharebrain/+ebrains/writeBucketInfo.m @@ -0,0 +1,34 @@ +function writeBucketInfo(cloneFolder, bucketName, options) +%writeBucketInfo - Record which EBRAINS bucket a cloned dataset comes from +% nansen.module.sharebrain.ebrains.writeBucketInfo(cloneFolder, bucketName) +% writes the file .ebrains-bucket.json in cloneFolder. The file names the +% Data Proxy bucket whose objects the empty files below cloneFolder stand +% for, so that EbrainsBucketSource can download them. +% +% writeBucketInfo(..., Prefix=PREFIX) records the folder of a bucket that +% is shared with other datasets. +% +% writeBucketInfo(..., DatasetVersionId=UUID) records the dataset version +% in the EBRAINS Knowledge Graph that the files belong to. +% +% The file holds a JSON object with the fields bucket, prefix and +% datasetVersionId. +% +% See also nansen.module.sharebrain.ebrains.findBucketInfo, +% nansen.module.sharebrain.dataio.EbrainsBucketSource + + arguments + cloneFolder (1,1) string {mustBeFolder} + bucketName (1,1) string {mustBeNonzeroLengthText} + options.Prefix (1,1) string = "" + options.DatasetVersionId (1,1) string = "" + end + + bucketInfo = struct( ... + 'bucket', bucketName, ... + 'prefix', options.Prefix, ... + 'datasetVersionId', options.DatasetVersionId); + + filePath = fullfile(cloneFolder, nansen.module.sharebrain.constants.BucketInfoFilename); + writelines(jsonencode(bucketInfo, PrettyPrint=true), filePath) +end diff --git a/+nansen/+module/+sharebrain/+type/Session.m b/+nansen/+module/+sharebrain/+type/Session.m new file mode 100644 index 0000000..9c92147 --- /dev/null +++ b/+nansen/+module/+sharebrain/+type/Session.m @@ -0,0 +1,30 @@ +classdef Session < nansen.metadata.type.Session +%Session - A NANSEN session with the state of its subject from the EBRAINS Knowledge Graph +% +% The Knowledge Graph records values that change over time, such as age +% and weight, on subject states (openMINDS SubjectState) rather than on +% the subject, and a state describes the subject during the sessions it +% applies to. This class adds the values of that state to the session. +% sharebrain.importKGSubjects fills them. +% +% Create the session table of a project with this class to have the +% columns from the start: +% nansen.config.initializeSessionTable(dataLocationModel, ... +% @nansen.module.sharebrain.type.Session, SkipInteractiveSteps=true) +% +% See also sharebrain.importKGSubjects, nansen.module.sharebrain.type.Subject + + properties (SetObservable) + SubjectAge char % Age of the subject, value and unit, e.g. '3 month' + SubjectAgeCategory char % Age category, e.g. 'adult' + SubjectWeight char % Weight of the subject, value or range and unit, e.g. '400-500 gram' + SubjectAttributes char % Attributes of the subject state, e.g. 'alive, awake' + SubjectStateKGId char % UUID of the openMINDS SubjectState in the Knowledge Graph + end + + methods + function obj = Session(varargin) + obj@nansen.metadata.type.Session(varargin{:}) + end + end +end diff --git a/+nansen/+module/+sharebrain/+type/Subject.m b/+nansen/+module/+sharebrain/+type/Subject.m new file mode 100644 index 0000000..cc43e09 --- /dev/null +++ b/+nansen/+module/+sharebrain/+type/Subject.m @@ -0,0 +1,19 @@ +classdef Subject < nansen.metadata.type.Subject +%Subject - A NANSEN subject linked to its subject in the EBRAINS Knowledge Graph +% +% Adds to nansen.metadata.type.Subject the identifiers of the openMINDS +% Subject that the Knowledge Graph lists for the dataset version, so a +% row of a project's subject table can be traced back to it. +% sharebrain.importKGSubjects fills these properties together with +% BiologicalSex, Species and Strain. +% +% The properties that change over time (age, weight) belong to subject +% states and are held by nansen.module.sharebrain.type.Session. +% +% See also sharebrain.importKGSubjects, nansen.module.sharebrain.type.Session + + properties (SetObservable) + KGInstanceId char % UUID of the openMINDS Subject in the Knowledge Graph + KGLookupLabel char % lookupLabel of the Subject in the Knowledge Graph + end +end diff --git a/+nansen/+module/+sharebrain/constants.m b/+nansen/+module/+sharebrain/constants.m index e51c72c..cf553b6 100644 --- a/+nansen/+module/+sharebrain/constants.m +++ b/+nansen/+module/+sharebrain/constants.m @@ -3,6 +3,10 @@ DataSharingCollabPrefix = "d-" CollabInfoFilename = "ebrains_collab_info.json" + % Names the bucket of a dataset cloned from EBRAINS, in the root + % folder of the clone + BucketInfoFilename = ".ebrains-bucket.json" + % URL Constants NeurosiftBaseUrl = "https://neurosift.app/?p=/nwb" end diff --git a/tests/EbrainsBucketSourceTest.m b/tests/EbrainsBucketSourceTest.m new file mode 100644 index 0000000..db94d14 --- /dev/null +++ b/tests/EbrainsBucketSourceTest.m @@ -0,0 +1,106 @@ +classdef EbrainsBucketSourceTest < matlab.unittest.TestCase +%EbrainsBucketSourceTest - Tests for EbrainsBucketSource and the bucket info file of a clone +% +% Each test makes a clone folder with a .ebrains-bucket.json and empty +% placeholder files, as sharebrain.cloneKGDataset leaves them. The +% downloader is a fake that writes the bucket and object name it is +% given into the target file, so no test connects to EBRAINS. +% +% Run tests: +% runtests('EbrainsBucketSourceTest') + + properties + CloneFolder string + OutsideFolder string + end + + methods (TestMethodSetup) + function createClone(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + testCase.CloneFolder = string(testCase.applyFixture(TemporaryFolderFixture).Folder); + testCase.OutsideFolder = string(testCase.applyFixture(TemporaryFolderFixture).Folder); + + nansen.module.sharebrain.ebrains.writeBucketInfo(testCase.CloneFolder, "my-bucket", ... + Prefix="shared-folder/", DatasetVersionId="0b4a1f5e-0000-4000-8000-000000000000"); + + mkdir(fullfile(testCase.CloneFolder, "shared-folder", "sub")) + createFile(fullfile(testCase.CloneFolder, "shared-folder", "sub", "cell_1a.abf"), "") + createFile(fullfile(testCase.CloneFolder, "shared-folder", "downloaded.txt"), "content") + createFile(fullfile(testCase.OutsideFolder, "empty.abf"), "") + end + end + + methods (Test) + function testFindBucketInfoReadsFileAboveFile(testCase) + [bucketInfo, cloneFolder] = nansen.module.sharebrain.ebrains.findBucketInfo( ... + fullfile(testCase.CloneFolder, "shared-folder", "sub", "cell_1a.abf")); + + testCase.verifyEqual(string(bucketInfo.bucket), "my-bucket") + testCase.verifyEqual(string(bucketInfo.prefix), "shared-folder/") + testCase.verifyEqual(cloneFolder, testCase.CloneFolder) + end + + function testFindBucketInfoOutsideCloneIsEmpty(testCase) + bucketInfo = nansen.module.sharebrain.ebrains.findBucketInfo( ... + fullfile(testCase.OutsideFolder, "empty.abf")); + + testCase.verifyEmpty(bucketInfo) + end + + function testEmptyFileInCloneIsOnlineOnly(testCase) + source = testCase.createSource(); + + testCase.verifyTrue(source.isOnlineOnly( ... + fullfile(testCase.CloneFolder, "shared-folder", "sub", "cell_1a.abf"))) + end + + function testFileWithContentIsNotOnlineOnly(testCase) + source = testCase.createSource(); + + testCase.verifyFalse(source.isOnlineOnly( ... + fullfile(testCase.CloneFolder, "shared-folder", "downloaded.txt"))) + end + + function testEmptyFileOutsideCloneIsNotOnlineOnly(testCase) + source = testCase.createSource(); + + testCase.verifyFalse(source.isOnlineOnly(fullfile(testCase.OutsideFolder, "empty.abf"))) + end + + function testDownloadRequestsObjectAtPathInClone(testCase) + % The clone keeps the folder of a shared bucket in its paths, so + % the object name is the path relative to the clone folder + source = testCase.createSource(); + filePath = fullfile(testCase.CloneFolder, "shared-folder", "sub", "cell_1a.abf"); + + source.download(filePath) + + testCase.verifyEqual(strtrim(string(fileread(filePath))), ... + "my-bucket shared-folder/sub/cell_1a.abf") + end + + function testDownloadOutsideCloneErrors(testCase) + source = testCase.createSource(); + + testCase.verifyError(@() source.download(fullfile(testCase.OutsideFolder, "empty.abf")), ... + 'SHAREbrain:EbrainsBucketSource:NotInClone') + end + end + + methods (Access = private) + function source = createSource(testCase) + source = nansen.module.sharebrain.dataio.EbrainsBucketSource(testCase.CloneFolder, ... + Downloader=@writeRequestIntoTarget); + end + end +end + +function writeRequestIntoTarget(bucketName, objectName, targetFile) + writelines(bucketName + " " + objectName, targetFile) +end + +function createFile(filePath, content) + fileID = fopen(filePath, "w"); + fprintf(fileID, "%s", content); + fclose(fileID); +end From 0984ebef0e5294e8ca643d1c42fa02a93228b616 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 18 Sep 2026 22:16:35 +0200 Subject: [PATCH 14/18] feat: add file adapters for ABF, NWB and delimited text files The DSM converter of NANSEN assigns a variable the first file adapter of the project that supports its file extension. With this module in a project, .abf files get ABF, .nwb files NWB, and .txt, .csv and .tsv files DelimitedText. ABF reads the file with abfload (Harald Hentschke and Forrest Collman, BSD licence, in the private folder) and returns the samples with the sampling rate, channel names and units, and the header. NWB returns the NwbFile from nwbRead. DelimitedText returns a table from readtable. Co-Authored-By: Claude Opus 5 --- .../+module/+sharebrain/+fileadapter/ABF.m | 42 + .../+sharebrain/+fileadapter/DelimitedText.m | 21 + .../+module/+sharebrain/+fileadapter/NWB.m | 26 + .../+fileadapter/private/abfload.m | 1562 +++++++++++++++++ .../+fileadapter/private/abfload_license.txt | 25 + tests/FileAdapterTest.m | 58 + 6 files changed, 1734 insertions(+) create mode 100644 +nansen/+module/+sharebrain/+fileadapter/ABF.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/DelimitedText.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/NWB.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/private/abfload.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/private/abfload_license.txt create mode 100644 tests/FileAdapterTest.m diff --git a/+nansen/+module/+sharebrain/+fileadapter/ABF.m b/+nansen/+module/+sharebrain/+fileadapter/ABF.m new file mode 100644 index 0000000..265c49b --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/ABF.m @@ -0,0 +1,42 @@ +classdef ABF < nansen.dataio.FileAdapter +%ABF - File adapter for Axon Binary Files (ABF versions 1 and 2) recorded with pCLAMP +% +% sessionObject.loadData(variableName) reads every channel and sweep of +% the file and returns a struct with the fields +% Data - The samples. Episodic recordings give a +% samples x channels x sweeps array, gap-free +% recordings a samples x channels array, and +% event-driven variable-length recordings a cell array +% with one element per sweep. +% SamplingRate - Samples per second of each channel +% ChannelNames - 1 x channels string with the names of the channels +% ChannelUnits - 1 x channels string with the units of the channels +% Header - The header struct returned by abfload +% +% The file is read by abfload, written by Harald Hentschke and Forrest +% Collman, which is in the private folder with its BSD licence. + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'abf'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + [samples, samplingIntervalMicroseconds, header] = abfload(obj.Filename, ... + 'channels', 'a', 'sweeps', 'a', 'doDispInfo', false); + + % abfload gives channel names and units as a cell array for + % ABF 2 files and as a padded char matrix for ABF 1 files + data = struct( ... + 'Data', samples, ... + 'SamplingRate', 1e6 / samplingIntervalMicroseconds, ... + 'ChannelNames', reshape(strtrim(string(header.recChNames)), 1, []), ... + 'ChannelUnits', reshape(strtrim(string(header.recChUnits)), 1, []), ... + 'Header', header); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m b/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m new file mode 100644 index 0000000..52f1554 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m @@ -0,0 +1,21 @@ +classdef DelimitedText < nansen.dataio.FileAdapter +%DelimitedText - File adapter for text files with columns separated by a delimiter +% +% sessionObject.loadData(variableName) returns the file as a table, read +% with readtable. readtable detects the delimiter, the number of header +% lines and whether the first line holds column names. + + properties (Constant) + DataType = 'table' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'txt', 'csv', 'tsv'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readtable(obj.Filename); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/NWB.m b/+nansen/+module/+sharebrain/+fileadapter/NWB.m new file mode 100644 index 0000000..606ebaf --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/NWB.m @@ -0,0 +1,26 @@ +classdef NWB < nansen.dataio.FileAdapter +%NWB - File adapter for Neurodata Without Borders (NWB) files +% +% sessionObject.loadData(variableName) returns the NwbFile object that +% nwbRead of MatNWB creates for the file. The object reads datasets +% lazily, so loading is fast and the data of a dataset is read when it +% is indexed or loaded with its load method. +% +% MatNWB must be on the MATLAB path. nwbRead generates the type classes +% from the schema stored in the file, so files written with different +% NWB versions can be read. + + properties (Constant) + DataType = 'NwbFile' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'nwb'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = nwbRead(obj.Filename); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m b/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m new file mode 100644 index 0000000..a8e6836 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m @@ -0,0 +1,1562 @@ +function [d,si,h]=abfload(fn,varargin) +% ** function [d,si,h]=abfload(fn,varargin) +% loads and returns data in ABF (Axon Binary File) format. +% Data may have been acquired in the following modes: +% (1) event-driven variable-length (currently only abf versions < 2.0) +% (2) event-driven fixed-length or waveform-fixed length +% (3) gap-free +% Information about scaling, the time base and the number of channels and +% episodes is extracted from the header of the abf file. +% +% OPERATION +% If the second input variable is the char array 'info' as in +% [d,si,h]=abfload('d:\data01.abf','info') +% abfload will not load any data but return detailed information (header +% parameters) on the file in output variable h. d and si will be empty. +% In all other cases abfload will load data. Optional input parameters +% listed below (= all except the file name) must be specified as +% parameter/value pairs, e.g. as in +% d=abfload('d:\data01.abf','start',100,'stop','e'); +% +% >>> INPUT VARIABLES >>> +% NAME TYPE, DEFAULT DESCRIPTION +% fn char array abf data file name +% start scalar, 0 only gap-free-data: start of cutout to be +% read (unit: s) +% stop scalar or char, only gap-free-data: end of cutout to be +% 'e' read (unit: sec). May be set to 'e' (end +% of file). +% sweeps 1d-array or char, only episodic data: sweep numbers to be +% 'a' read. By default, all sweeps will be read +% ('a'). +% channels cell array names of channels to be read, like +% or char, 'a' {'IN 0','IN 8'} (make sure spelling is +% 100% correct, including blanks). If set +% to 'a', all channels will be read. +% ***************************************** +% NOTE: channel order in output variable d +% ignores the order in 'channels', and +% instead always matches the order inherent +% to the abf file, to be retrieved in +% output variable h! +% ***************************************** +% chunk scalar, 0.05 only gap-free-data: the elementary chunk +% size (megabytes) to be used for the +% 'discontinuous' mode of reading data +% (fewer channels to be read than exist) +% machineF char array, the 'machineformat' input parameter of the +% 'ieee-le' matlab fopen function. 'ieee-le' is the +% correct option for windows; depending on +% the platform the data were recorded/shall +% be read by abfload 'ieee-be' is the +% alternative. +% doDispInfo logical, true if true, information on the loaded file +% will be put out to console (if false, +% only ínformation on erroneous input will +% be displayed) +% << OUTPUT VARIABLES <<< +% NAME TYPE DESCRIPTION +% d the data read, the format depending on the record- +% ing mode +% 1. GAP-FREE: +% 2d array 2d array of size +% by +% Examples of access: +% d(:,2) data from channel 2 at full length +% d(1:100,:) first 100 data points from all channels +% 2. EPISODIC FIXED-LENGTH/WAVEFORM FIXED-LENGTH/HIGH-SPEED OSCILLOSCOPE: +% 3d array 3d array of size +% by by . +% Examples of access: +% d(:,2,:) a matrix containing all episodes +% (at full length) of the second +% channel in its columns +% d(1:200,:,[1 11]) contains first 200 data points of +% episodes 1 and 11 of all channels +% 3. EPISODIC VARIABLE-LENGTH: +% cell array cell array whose elements correspond to single sweeps. +% Each element is a (regular) array of size +% by +% Examples of access: +% d{1} a 2d-array which contains episode 1 +% (all of it, all channels) +% d{2}(1:100,2) a 1d-array containing the first 100 +% data points of channel 2 in episode 1 +% si scalar the sampling interval in us +% h struct information on file (selected header parameters) +% +% CONTRIBUTORS +% Original version by Harald Hentschke (harald.hentschke@uni-tuebingen.de) +% Extended to abf version 2.0 by Forrest Collman (fcollman@Princeton.edu) +% pvpmod.m by Ulrich Egert (egert@bccn.uni-freiburg.de) +% Small patch to avoid error caused by activated HumSilencer by Michael Rabenstein (michael.rabenstein@ukbonn.de) + +% ------------------------------------------------------------------------- +% PART 1: check of input vars +% ------------------------------------------------------------------------- +% --- defaults +% gap-free +start=0.0; +stop='e'; +% episodic +sweeps='a'; +% general +channels='a'; +% the size of data chunks (see above) in Mb. 0.05 Mb is an empirical value +% which works well for abf with 6-16 channels and recording durations of +% 5-30 min +chunk=0.05; +machineF='ieee-le'; +doDispInfo=true; +% if first and only optional input argument is string 'info' the user's +% request is to obtain information on the file (header parameters), so set +% flag accordingly +if nargin==2 && ischar(varargin{1}) && strcmp('info',varargin{1}) + doLoadData=false; + % if no output argument is requested assume that the user would like to + % obtain some basic information on the file on the command window, so + % leave doDispInfo at its default (true value). Do the same if only one + % output arg is specified (which is most certainly done inadvertently + % because in 'info' mode this will be an empty array). In all other cases + % assume that text output is not required so suppress it + if nargout>1 + doDispInfo=false; + end +else + doLoadData=true; + % assign values of optional input parameters if any were given + pvpmod(varargin); +end + +% some constants +BLOCKSIZE=512; +% output variables +d=[]; +si=[]; +h=[]; +if ischar(stop) + if ~strcmpi(stop,'e') + error('input parameter ''stop'' must be specified as ''e'' (=end of recording) or as a scalar'); + end +end +% check existence of file +if ~isfile(fn) + error(['could not find file ' fn]); +end + +% ------------------------------------------------------------------------- +% PART 2a: determine abf version +% ------------------------------------------------------------------------- +dispif(doDispInfo,['opening ' fn '..']); +[fid,messg]=fopen(fn,'r',machineF); +if fid == -1 + error(messg); +end +% on the occasion, determine absolute file size +fileListing = dir(fn); +fileSz = fileListing.bytes; + +% *** read value of parameter 'fFileSignature' (i.e. abf version) from header *** +sz=4; +[fFileSignature,n]=fread(fid,sz,'uchar=>char'); +if n~=sz + fclose(fid); + error('something went wrong reading value(s) for fFileSignature'); +end +% rewind +fseek(fid,0,'bof'); +% transpose +fFileSignature=fFileSignature'; + +% one of the first checks must be whether file signature is valid +switch fFileSignature + case 'ABF ' % ** note the blank + % ************************ + % abf version < 2.0 + % ************************ + case 'ABF2' + % ************************ + % abf version >= 2.0 + % ************************ + otherwise + error(['unknown or incompatible file signature: ' fFileSignature]); +end + +% ------------------------------------------------------------------------- +% PART 2b: define file information ('header' parameters) of interest +% ------------------------------------------------------------------------- +% The list of header parameters created below (variable 'headPar') is +% derived from the abf version 1.8 header section. It is by no means +% exhaustive (i.e. there are many more parameters in abf files) but +% sufficient for proper upload, scaling and arrangement of data acquired +% under many conditons. Further below, these parameters will be made fields +% of struct h. h, which is also an output variable, is then used in PART 3, +% which does the actual job of uploading, scaling and rearranging the data. +% That part of the code relies on h having a certain set of fields +% irrespective of ABF version. +% Unfortunately, in the transition to ABF version 2.0 many of the header +% parameters were moved to different places within the abf file and/or +% given other names or completely restructured. In order for the code to +% work with pre- and post-2.0 data files, all parameters missing in the +% header must be gotten into h. This is accomplished in lines ~288 and +% following: +% if h.fFileVersionNumber>=2 +% ... +% Furthermore, +% - h as an output from an ABF version < 2.0 file will not contain new +% parameters introduced into the header like 'nCRCEnable' +% - h will in any case contain a few 'home-made' fields that have +% proven to be useful. Some of them depend on the recording mode. Among +% the more or less self-explanatory ones are +% -- si sampling interval +% -- recChNames the names of all channels, e.g. 'IN 8',... +% -- dataPtsPerChan sample points per channel +% -- dataPts sample points in file +% -- recTime recording start and stop time in seconds from +% midnight (millisecond resolution) +% -- sweepLengthInPts sample points per sweep (one channel) +% -- sweepStartInPts the start times of sweeps in sample points +% (from beginning of recording) + + +% define header proper depending on ABF version by call to local function +headPar=define_header(fFileSignature); +% define all sections that there are +Sections=define_Sections; +% define a few of these (currently, only the TagInfo section is used for +% all versions of ABF, but that may change in the future) +ProtocolInfo=define_ProtocolInfo; +ADCInfo=define_ADCInfo; +TagInfo=define_TagInfo; +UserListInfo=define_UserListInfo; +EpochPerDACInfo=define_EpochPerDACInfo; + +% ------------------------------------------------------------------------- +% PART 2c: read parameters of interest +% ------------------------------------------------------------------------- +% convert headPar to struct +s=cell2struct(headPar,{'name','offs','numType','value'},2); +numOfParams=size(s,1); +clear tmp headPar; + +% convert names in structure to variables and read value from header +for g=1:numOfParams + if fseek(fid, s(g).offs,'bof')~=0 + fclose(fid); + error(['something went wrong locating ' s(g).name]); + end + sz=length(s(g).value); + % use dynamic field names + [h.(s(g).name),n]=fread(fid,sz,s(g).numType); + if n~=sz + fclose(fid); + error(['something went wrong reading value(s) for ' s(g).name]); + end +end +% file signature needs to be transposed +h.fFileSignature=h.fFileSignature'; +% several header parameters need a fix or version-specific refinement: +if strcmp(h.fFileSignature,'ABF2') + % h.fFileVersionNumber needs to be converted from an array of integers to + % a float + h.fFileVersionNumber=h.fFileVersionNumber(4)+h.fFileVersionNumber(3)*.1... + +h.fFileVersionNumber(2)*.001+h.fFileVersionNumber(1)*.0001; + % convert ms to s + h.lFileStartTime=h.uFileStartTimeMS*.001; +else + % h.fFileVersionNumber is a float32 the value of which is sometimes a + % little less than what it should be (e.g. 1.6499999 instead of 1.65) + h.fFileVersionNumber=.001*round(h.fFileVersionNumber*1000); + % in abf < 2.0 two parameters are needed to obtain the file start time + % with millisecond precision - let's integrate both into parameter + % lFileStartTime (unit: s) so that nFileStartMillisecs will not be needed + h.lFileStartTime=h.lFileStartTime+h.nFileStartMillisecs*.001; +end + +if h.fFileVersionNumber>=2 + % ----------------------------------------------------------------------- + % *** read file information that has moved from the header section to + % other sections in ABF version >= 2.0 and assign selected values to + % fields of 'generic' header variable h *** + % ----------------------------------------------------------------------- + % --- read in the Sections + Sects=cell2struct(Sections,{'name'},2); + numOfSections=length(Sections); + offset=76; + % this creates all sections (ADCSection, ProtocolSection, etc.) + for i=1:numOfSections + eval([Sects(i).name '=ReadSectionInfo(fid,offset);']); + offset=offset+4+4+8; + end + % --- read in the StringsSection and use some fields (to retrieve + % information on the names of recorded channels and the units as well as + % the recording protocol used) + fseek(fid,StringsSection.uBlockIndex*BLOCKSIZE,'bof'); + BigString=fread(fid,StringsSection.uBytes,'char'); + % extract path and name of protocol file (excluding drive letter): + % - find the first occurrence of a backslash in BigString + tmpIx1=strfind(lower(char(BigString)'),'\'); + % - find '.pro' as this is the file extension of protocol files + tmpIx2=strfind(lower(char(BigString)'),'.pro'); + % - extract everything in between and place in field of header struct h + if ~isempty(tmpIx1) && ~isempty(tmpIx2) + h.protocolName=char(BigString(tmpIx1(1):tmpIx2(1)+3))'; + else + h.protocolName='protocol name could not be identified'; + end + % this is a hack: determine where either of strings 'clampex', + % 'clampfit', 'axoscope' or patchxpress' begin + progString={'clampex','clampfit','axoscope','patchxpress'}; + goodstart=[]; + for i=1:numel(progString) + goodstart=cat(1,goodstart,strfind(lower(char(BigString)'),progString{i})); + end + if isempty(goodstart) + error('problems in StringsSection: unrecognized acquisition program'); + end + BigString=BigString(goodstart(1):end)'; + stringends=find(BigString==0); + stringends=[0 stringends]; + Strings=cell(1,length(stringends)-1); + for i=1:length(stringends)-1 + Strings{i}=char(BigString(stringends(i)+1:stringends(i+1)-1)); + end + h.recChNames=cell(ADCSection.llNumEntries,1); + h.recChUnits=cell(ADCSection.llNumEntries,1); + + % --- read in the ADCSection & copy some values to header h + for i=1:ADCSection.llNumEntries + ADCsec(i)=ReadSection(fid,ADCSection.uBlockIndex*BLOCKSIZE+ADCSection.uBytes*(i-1),ADCInfo); + ii=ADCsec(i).nADCNum+1; + h.nADCSamplingSeq(i)=ADCsec(i).nADCNum; + h.recChNames(i)=Strings(ADCsec(i).lADCChannelNameIndex); + unitsIndex=ADCsec(i).lADCUnitsIndex; + if unitsIndex>0 + h.recChUnits(i)=Strings(ADCsec(i).lADCUnitsIndex); + else + h.recChUnits(i)={'nil'}; + end + h.nTelegraphEnable(ii)=ADCsec(i).nTelegraphEnable; + h.fTelegraphAdditGain(ii)=ADCsec(i).fTelegraphAdditGain; + h.fInstrumentScaleFactor(ii)=ADCsec(i).fInstrumentScaleFactor; + h.fSignalGain(ii)=ADCsec(i).fSignalGain; + h.fADCProgrammableGain(ii)=ADCsec(i).fADCProgrammableGain; + h.fInstrumentOffset(ii)=ADCsec(i).fInstrumentOffset; + h.fSignalOffset(ii)=ADCsec(i).fSignalOffset; + end + % --- read in the protocol section & copy some values to header h + ProtocolSec=ReadSection(fid,ProtocolSection.uBlockIndex*BLOCKSIZE,ProtocolInfo); + h.nOperationMode=ProtocolSec.nOperationMode; + h.fSynchTimeUnit=ProtocolSec.fSynchTimeUnit; + h.lNumSamplesPerEpisode=ProtocolSec.lNumSamplesPerEpisode; + + h.nADCNumChannels=ADCSection.llNumEntries; + h.lActualAcqLength=DataSection.llNumEntries; + h.lDataSectionPtr=DataSection.uBlockIndex; + h.nNumPointsIgnored=0; + % in ABF version < 2.0 h.fADCSampleInterval is the sampling interval + % defined as + % 1/(sampling freq*number_of_channels) + % so divide ProtocolSec.fADCSequenceInterval by the number of + % channels + h.fADCSampleInterval=ProtocolSec.fADCSequenceInterval/h.nADCNumChannels; + h.fADCRange=ProtocolSec.fADCRange; + h.lADCResolution=ProtocolSec.lADCResolution; + + % --- read in the Epoch section, rearrange values and place in header + % struct h + for i=1:EpochPerDACSection.llNumEntries + EPDsec(i)=ReadSection(fid,EpochPerDACSection.uBlockIndex*BLOCKSIZE+EpochPerDACSection.uBytes*(i-1),EpochPerDACInfo); + end + if EpochPerDACSection.llNumEntries>0 + % number of analog output (AO) channels + uniqueAO=unique([EPDsec.nDACNum]); + for k=1:numel(uniqueAO) + % index to elements of EDPsec dealing with current AO channel + ix=[EPDsec.nDACNum]==uniqueAO(k); + % struct DACEpoch contains one element per AO channel; the values of + % its fields represent the values of the different epochs (columns + % labeled A, B and so on in the waveform tab in clampex), the only + % exception being the numeric index of the channel + h.DACEpoch(k).nDACNum=uniqueAO(k); + % compute the 'first/last holding' values, the small additional delay + % after which stimulation waveforms are put out. The unit is sample + % points. ** NOTE: the formula below is based on + % i) code in function ABFH_GetHoldingDuration in abfhwave.cpp, + % presumably a code file belonging to an early version of program + % 'stimfit', and presumably derived from code available back then + % from the company formerly known as Axon Instruments + % ii) heuristics, determined from a number of abf V. 2.00 files from + % different labs in all of which 16/8-channel ADC/DAC systems were + % used. + % The formula may be hardware dependent; it may change (=be + % corrected, should it not be general) in the future. + h.DACEpoch(k).firstHolding=floor(... + 8*(h.lNumSamplesPerEpisode/h.nADCNumChannels)/512); + fieldNm=setdiff(fieldnames(EPDsec),'nDACNum','stable'); + for fIx=1:numel(fieldNm) + h.DACEpoch(k).(fieldNm{fIx})=[EPDsec(ix).(fieldNm{fIx})]; + end + end + else + h.DACEpoch=[]; + end + % --- read in the user list section - unclear how to get the values + % (commented code lines below are a first guess) + UserListSec=ReadSection(fid,UserListSection.uBlockIndex*BLOCKSIZE,UserListInfo); + % fseek(fid,UserListSec.lULParamValueListIndex*BLOCKSIZE,'bof'); + % nada=fread(fid,100,'uchar=>char') + + % --- in contrast to procedures with all other sections do not read the + % sync array section but rather copy the values of its fields to the + % corresponding fields of h + h.lSynchArrayPtr=SynchArraySection.uBlockIndex; + h.lSynchArraySize=SynchArraySection.llNumEntries; +else + % ------------------------------------------------------------------------- + % *** here, do the inverse: in ABF version<2 files extract information + % from header variable h and place it in corresponding new section + % variable(s) + % ------------------------------------------------------------------------- + TagSection.llNumEntries=h.lNumTagEntries; + TagSection.uBlockIndex=h.lTagSectionPtr; + TagSection.uBytes=64; +end +% ------------------------------------------------------------------------- +% PART 2d: groom parameters & perform some plausibility checks +% ------------------------------------------------------------------------- +if h.lActualAcqLength=1.65 + addGain=h.nTelegraphEnable.*h.fTelegraphAdditGain; + addGain(addGain==0)=1; +else + addGain=ones(size(h.fTelegraphAdditGain)); +end + +% determine offset at which data start +switch h.nDataFormat + case 0 + dataSz=2; % bytes/point + precision='int16'; + case 1 + dataSz=4; % bytes/point + precision='float32'; + otherwise + fclose(fid); + error('invalid number format'); +end +headOffset=h.lDataSectionPtr*BLOCKSIZE+h.nNumPointsIgnored*dataSz; +% h.fADCSampleInterval is the TOTAL sampling interval +h.si=h.fADCSampleInterval*h.nADCNumChannels; +% assign same value to si, which is an output variable +si=h.si; +if ischar(sweeps) && sweeps=='a' + nSweeps=h.lActualEpisodes; + sweeps=1:h.lActualEpisodes; +else + nSweeps=length(sweeps); +end + +% determine time unit in synch array section +switch h.fSynchTimeUnit + case 0 + % time information in synch array section is in terms of ticks + h.synchArrTimeBase=1; + otherwise + % time information in synch array section is in terms of usec + h.synchArrTimeBase=h.fSynchTimeUnit; +end + +% read in the TagSection, do a few computations & write to h.tags +h.tags=[]; +for i=1:TagSection.llNumEntries + tmp=ReadSection(fid,TagSection.uBlockIndex*BLOCKSIZE+TagSection.uBytes*(i-1),TagInfo); + % time of tag entry from start of experiment in s (corresponding expisode + % number, if applicable, will be determined later) + h.tags(i).timeSinceRecStart=tmp.lTagTime*h.synchArrTimeBase/1e6; + h.tags(i).comment=char(tmp.sComment)'; +end + +% ------------------------------------------------------------------------- +% PART 3: read data (note: from here on code is generic and abf version +% should not matter) +% ------------------------------------------------------------------------- +switch h.nOperationMode + case 1 + dispif(doDispInfo,'data were acquired in event-driven variable-length mode'); + if h.fFileVersionNumber>=2.0 + errordlg('abfload currently does not work with data acquired in event-driven variable-length mode and ABF version 2.0','ABF version issue'); + else + if (h.lSynchArrayPtr<=0 || h.lSynchArraySize<=0) + fclose(fid); + error('internal variables ''lSynchArraynnn'' are zero or negative'); + end + % the byte offset at which the SynchArraySection starts + h.lSynchArrayPtrByte=BLOCKSIZE*h.lSynchArrayPtr; + % before reading Synch Arr parameters check if file is big enough to hold them + % 4 bytes/long, 2 values per episode (start and length) + if h.lSynchArrayPtrByte+2*4*h.lSynchArraySize0 + fclose(fid); + error('number of data points in episode not OK'); + end + % separate channels.. + tmpd=reshape(tmpd,h.nADCNumChannels,h.dataPtsPerChan); + % retain only requested channels + tmpd=tmpd(chInd,:); + tmpd=tmpd'; + % if data format is integer, scale appropriately; if it's float, tmpd is fine + if ~h.nDataFormat + for j=1:length(chInd) + ch=recChIdx(chInd(j))+1; + tmpd(:,j)=tmpd(:,j)/(h.fInstrumentScaleFactor(ch)*h.fSignalGain(ch)*h.fADCProgrammableGain(ch)*addGain(ch))... + *h.fADCRange/h.lADCResolution+h.fInstrumentOffset(ch)-h.fSignalOffset(ch); + end + end + % now place in cell array, an element consisting of one sweep with channels in columns + d{i}=tmpd; + end + end + end + + case {2,4,5} + if h.nOperationMode==2 + dispif(doDispInfo,'data were acquired in event-driven fixed-length mode'); + elseif h.nOperationMode==4 + dispif(doDispInfo,'data were acquired in high-speed oscilloscope mode'); + else + dispif(doDispInfo,'data were acquired in waveform fixed-length mode'); + end + % extract timing information on sweeps + if (h.lSynchArrayPtr<=0 || h.lSynchArraySize<=0) + fclose(fid); + error('internal variables ''lSynchArraynnn'' are zero or negative'); + end + % the byte offset at which the SynchArraySection starts + h.lSynchArrayPtrByte=BLOCKSIZE*h.lSynchArrayPtr; + % before reading Synch Arr parameters check if file is big enough to hold them + % 4 bytes/long, 2 values per episode (start and length) + if h.lSynchArrayPtrByte+2*4*h.lSynchArraySize>fileSz + fclose(fid); + error('file seems not to contain complete Synch Array Section'); + end + if fseek(fid,h.lSynchArrayPtrByte,'bof')~=0 + fclose(fid); + error('something went wrong positioning file pointer to Synch Array Section'); + end + [synchArr,n]=fread(fid,h.lSynchArraySize*2,'int32'); + if n~=h.lSynchArraySize*2 + fclose(fid); + error('something went wrong reading synch array section'); + end + % make synchArr a h.lSynchArraySize x 2 matrix + synchArr=permute(reshape(synchArr',2,h.lSynchArraySize),[2 1]); + if numel(unique(synchArr(:,2)))>1 + fclose(fid); + error('sweeps of unequal length in file recorded in fixed-length mode'); + end + % the length of sweeps in sample points (**note: parameter lLength of + % the ABF synch section is expressed in samples (ticks) whereas + % parameter lStart is given in synchArrTimeBase units) + h.sweepLengthInPts=synchArr(1,2)/h.nADCNumChannels; + % the starting ticks of episodes in sample points (t0=1=beginning of + % recording) + h.sweepStartInPts=synchArr(:,1)*(h.synchArrTimeBase/h.fADCSampleInterval/h.nADCNumChannels); + % recording start and stop times in seconds from midnight + h.recTime=h.lFileStartTime; + h.recTime=h.recTime+[0 (1e-6*(h.sweepStartInPts(end)+h.sweepLengthInPts))*h.fADCSampleInterval*h.nADCNumChannels]; + % determine first point and number of points to be read + startPt=0; + h.dataPts=h.lActualAcqLength; + h.dataPtsPerChan=h.dataPts/h.nADCNumChannels; + if rem(h.dataPts,h.nADCNumChannels)>0 || rem(h.dataPtsPerChan,h.lActualEpisodes)>0 + fclose(fid); + error('number of data points not OK'); + end + % temporary helper var + dataPtsPerSweep=h.sweepLengthInPts*h.nADCNumChannels; + if fseek(fid,startPt*dataSz+headOffset,'bof')~=0 + fclose(fid); + error('something went wrong positioning file pointer (too few data points ?)'); + end + % the starting ticks of episodes in sample points WITHIN THE DATA FILE + selectedSegStartInPts=((sweeps-1)*dataPtsPerSweep)*dataSz+headOffset; + % ** load data if requested + if doLoadData + % preallocate d + d=zeros(h.sweepLengthInPts,length(chInd),nSweeps); + for i=1:nSweeps + status=fseek(fid,selectedSegStartInPts(i),'bof'); + if status==-1 + fclose(fid); + error(['something went wrong reading episode ' int2str(sweeps(i)) '; file pointer beyond file limits (check sweeps)']); + end + [tmpd,n]=fread(fid,dataPtsPerSweep,precision); + if n~=dataPtsPerSweep + fclose(fid); + error(['something went wrong reading episode ' int2str(sweeps(i)) ': ' dataPtsPerSweep ' points should have been read, ' int2str(n) ' points actually read']); + end + h.dataPtsPerChan=n/h.nADCNumChannels; + if rem(n,h.nADCNumChannels)>0 + fclose(fid); + error('number of data points in episode not OK'); + end + % separate channels.. + tmpd=reshape(tmpd,h.nADCNumChannels,h.dataPtsPerChan); + % retain only requested channels + tmpd=tmpd(chInd,:); + tmpd=tmpd'; + % if data format is integer, scale appropriately; if it's float, d is fine + if ~h.nDataFormat + for j=1:length(chInd) + ch=recChIdx(chInd(j))+1; + tmpd(:,j)=tmpd(:,j)/(h.fInstrumentScaleFactor(ch)*h.fSignalGain(ch)*h.fADCProgrammableGain(ch)*addGain(ch))... + *h.fADCRange/h.lADCResolution+h.fInstrumentOffset(ch)-h.fSignalOffset(ch); + end + end + % now fill 3d array + d(:,:,i)=tmpd; + end + end + + case 3 + dispif(doDispInfo,'data were acquired in gap-free mode'); + % from start, stop, headOffset and h.fADCSampleInterval calculate first point to be read + % and - unless stop is given as 'e' - number of points + startPt=floor(1e6*start*(1/h.fADCSampleInterval)); + % this corrects undesired shifts in the reading frame due to rounding errors in the previous calculation + startPt=floor(startPt/h.nADCNumChannels)*h.nADCNumChannels; + % if stop is a char array, it can only be 'e' at this point (other values would have + % been caught above) + if ischar(stop) + h.dataPtsPerChan=h.lActualAcqLength/h.nADCNumChannels-floor(1e6*start/h.si); + h.dataPts=h.dataPtsPerChan*h.nADCNumChannels; + else + h.dataPtsPerChan=floor(1e6*(stop-start)*(1/h.si)); + h.dataPts=h.dataPtsPerChan*h.nADCNumChannels; + if h.dataPts<=0 + fclose(fid); + error('start is larger than or equal to stop'); + end + end + if rem(h.dataPts,h.nADCNumChannels)>0 + fclose(fid); + error('number of data points not OK'); + end + tmp=1e-6*h.lActualAcqLength*h.fADCSampleInterval; + dispif(doDispInfo,['total length of recording: ' num2str(tmp,'%5.1f') ' s ~ ' num2str(tmp/60,'%3.0f') ' min']); + dispif(doDispInfo,['sampling interval: ' num2str(h.si,'%5.0f') ' µs']); + % 8 bytes per data point expressed in Mb + dispif(doDispInfo,['memory requirement for complete upload in matlab: '... + num2str(round(8*h.lActualAcqLength/2^20)) ' MB']); + % recording start and stop times in seconds from midnight + h.recTime=h.lFileStartTime; + h.recTime=[h.recTime h.recTime+tmp]; + if fseek(fid,startPt*dataSz+headOffset,'bof')~=0 + fclose(fid); + error('something went wrong positioning file pointer (too few data points ?)'); + end + if doLoadData + % *** decide on the most efficient way to read data: + % (i) all (of one or several) channels requested: read, done + % (ii) one (of many) channels requested: use the 'skip' feature of + % fread (which for up to 16 channels is slower than scenario iii below) + % (iii) more than one but not all (of several) channels requested: + % 'discontinuous' mode of reading data. Read a reasonable chunk of data + % (all channels), separate channels, discard non-requested ones (if + % any), place data in preallocated array, repeat until done. This is + % faster than reading the data in one big lump, separating channels and + % discarding the ones not requested + if length(chInd)==1 && h.nADCNumChannels>16 + % --- situation (ii) + % jump to proper reading frame position in file + if fseek(fid,(chInd-1)*dataSz,'cof')~=0 + fclose(fid); + error('something went wrong positioning file pointer (too few data points ?)'); + end + % read, skipping h.nADCNumChannels-1 data points after each read + [d,n]=fread(fid,h.dataPtsPerChan,precision,dataSz*(h.nADCNumChannels-1)); + if n~=h.dataPtsPerChan + fclose(fid); + error(['something went wrong reading file (' int2str(h.dataPtsPerChan) ' points should have been read, ' int2str(n) ' points actually read']); + end + elseif length(chInd)/h.nADCNumChannels<1 + % --- situation (iii) + % prepare chunkwise upload: + % preallocate d + d=nan(h.dataPtsPerChan,length(chInd)); + % the number of data points corresponding to the maximal chunk size, + % rounded off such that from each channel the same number of points is + % read (do not forget that each data point will by default be made a + % double of 8 bytes, no matter what the original data format is) + chunkPtsPerChan=floor(chunk*2^20/8/h.nADCNumChannels); + chunkPts=chunkPtsPerChan*h.nADCNumChannels; + % the number of those chunks.. + nChunk=floor(h.dataPts/chunkPts); + % ..and the remainder + restPts=h.dataPts-nChunk*chunkPts; + restPtsPerChan=restPts/h.nADCNumChannels; + % chunkwise row indices into d + dix=(1:chunkPtsPerChan:h.dataPtsPerChan)'; + dix(:,2)=dix(:,1)+chunkPtsPerChan-1; + dix(end,2)=h.dataPtsPerChan; + if nChunk + dispif(doDispInfo,['reading file in ' int2str(nChunk) ' chunks of ~' num2str(chunk) ' Mb']); + end + % do it: if no remainder exists loop through all rows of dix, + % otherwise spare last row for the lines below (starting with + % 'if restPts') + for ci=1:size(dix,1)-(restPts>0) + [tmpd,n]=fread(fid,chunkPts,precision); + if n~=chunkPts + fclose(fid); + error(['something went wrong reading chunk #' int2str(ci) ' (' ... + int2str(chunkPts) ' points should have been read, ' int2str(n) ' points actually read']); + end + % separate channels.. + tmpd=reshape(tmpd,h.nADCNumChannels,chunkPtsPerChan); + d(dix(ci,1):dix(ci,2),:)=tmpd(chInd,:)'; + end + % collect the rest, if any + if restPts + [tmpd,n]=fread(fid,restPts,precision); + if n~=restPts + fclose(fid); + error(['something went wrong reading last chunk (' ... + int2str(restPts) ' points should have been read, ' int2str(n) ' points actually read']); + end + % separate channels.. + tmpd=reshape(tmpd,h.nADCNumChannels,restPtsPerChan); + d(dix(end,1):dix(end,2),:)=tmpd(chInd,:)'; + end + else + % --- situation (i) + [d,n]=fread(fid,h.dataPts,precision); + if n~=h.dataPts + fclose(fid); + error(['something went wrong reading file (' int2str(h.dataPts) ' points should have been read, ' int2str(n) ' points actually read']); + end + % separate channels.. + d=reshape(d,h.nADCNumChannels,h.dataPtsPerChan); + d=d'; + end + % if data format is integer, scale appropriately; if it's float, d is fine + if ~h.nDataFormat + for j=1:length(chInd) + ch=recChIdx(chInd(j))+1; + d(:,j)=d(:,j)/(h.fInstrumentScaleFactor(ch)*h.fSignalGain(ch)*h.fADCProgrammableGain(ch)*addGain(ch))... + *h.fADCRange/h.lADCResolution+h.fInstrumentOffset(ch)-h.fSignalOffset(ch); + end + end + end + otherwise + warning('unknown recording mode -- returning empty matrix'); + d=[]; + h.si=[]; +end +fclose(fid); + +% finally, possibly add information on episode number to tags +%Modified by MR to avoid error caused by activated HumSilencer +if ~isempty(h.tags) && isfield(h,'sweepStartInPts') + for i=1:numel(h.tags) + if h.tags(i).timeSinceRecStart > 0 %Comments added during recording + tmp=find(h.tags(i).timeSinceRecStart>=h.sweepStartInPts/1e6*h.si); + h.tags(i).episodeIndex=tmp(end); + elseif h.tags(i).timeSinceRecStart == 0 %HumSilencer comment at timeSinceRecStart = 0 + h.tags(i).episodeIndex = 0; %Will be added to episode 0 + end + end +end + + +% ######################################################################## +% LOCAL FUNCTIONS +% ######################################################################## + +function headPar=define_header(fileSig) +switch fileSig + case 'ABF ' % ** note the blank + % ************************ + % abf version < 2.0 + % ************************ + % + % temporary initializing var + tmp=repmat(-1,1,16); + % define vital header parameters and initialize them with -1: set up a + % cell array (and convert it to a struct later on, which is more + % convenient) + % column order is + % name, position in header in bytes, type, value) + headPar={ + 'fFileSignature',0,'*char',[-1 -1 -1 -1]; + 'fFileVersionNumber',4,'float32',-1; + 'nOperationMode',8,'int16',-1; + 'lActualAcqLength',10,'int32',-1; + 'nNumPointsIgnored',14,'int16',-1; + 'lActualEpisodes',16,'int32',-1; + 'lFileStartTime',24,'int32',-1; + 'lDataSectionPtr',40,'int32',-1; + 'lTagSectionPtr',44,'int32',-1; + 'lNumTagEntries',48,'int32',-1; + 'lSynchArrayPtr',92,'int32',-1; + 'lSynchArraySize',96,'int32',-1; + 'nDataFormat',100,'int16',-1; + 'nADCNumChannels', 120, 'int16', -1; + 'fADCSampleInterval',122,'float', -1; + 'fSynchTimeUnit',130,'float',-1; + 'lNumSamplesPerEpisode',138,'int32',-1; + 'lPreTriggerSamples',142,'int32',-1; + 'lEpisodesPerRun',146,'int32',-1; + 'fADCRange', 244, 'float', -1; + 'lADCResolution', 252, 'int32', -1; + 'nFileStartMillisecs', 366, 'int16', -1; + 'nADCPtoLChannelMap', 378, 'int16', tmp; + 'nADCSamplingSeq', 410, 'int16', tmp; + 'sADCChannelName',442, 'uchar', repmat(tmp,1,10); + 'sADCUnits',602, 'uchar', repmat(tmp,1,8); + 'fADCProgrammableGain', 730, 'float', tmp; + 'fInstrumentScaleFactor', 922, 'float', tmp; + 'fInstrumentOffset', 986, 'float', tmp; + 'fSignalGain', 1050, 'float', tmp; + 'fSignalOffset', 1114, 'float', tmp; + 'nTelegraphEnable',4512,'int16',tmp; + 'fTelegraphAdditGain',4576,'float',tmp + }; + case 'ABF2' + % ************************ + % abf version >= 2.0 + % ************************ + headPar={ + 'fFileSignature',0,'*char',[-1 -1 -1 -1]; + 'fFileVersionNumber',4,'bit8=>int',[-1 -1 -1 -1]; + 'uFileInfoSize',8,'uint32',-1; + 'lActualEpisodes',12,'uint32',-1; + 'uFileStartDate',16','uint32',-1; + 'uFileStartTimeMS',20,'uint32',-1; + 'uStopwatchTime',24,'uint32',-1; + 'nFileType',28,'int16',-1; + 'nDataFormat',30,'int16',-1; + 'nSimultaneousScan',32,'int16',-1; + 'nCRCEnable',34,'int16',-1; + 'uFileCRC',36,'uint32',-1; + 'FileGUID',40,'uint32',-1; + 'uCreatorVersion',56,'uint32',-1; + 'uCreatorNameIndex',60,'uint32',-1; + 'uModifierVersion',64,'uint32',-1; + 'uModifierNameIndex',68,'uint32',-1; + 'uProtocolPathIndex',72,'uint32',-1; + }; +end + +function Sections=define_Sections +Sections={'ProtocolSection'; + 'ADCSection'; + 'DACSection'; + 'EpochSection'; + 'ADCPerDACSection'; + 'EpochPerDACSection'; + 'UserListSection'; + 'StatsRegionSection'; + 'MathSection'; + 'StringsSection'; + 'DataSection'; + 'TagSection'; + 'ScopeSection'; + 'DeltaSection'; + 'VoiceTagSection'; + 'SynchArraySection'; + 'AnnotationSection'; + 'StatsSection'; + }; + +function ProtocolInfo=define_ProtocolInfo +ProtocolInfo={ + 'nOperationMode','int16',1; + 'fADCSequenceInterval','float',1; + 'bEnableFileCompression','bit1',1; + 'sUnused1','char',3; + 'uFileCompressionRatio','uint32',1; + 'fSynchTimeUnit','float',1; + 'fSecondsPerRun','float',1; + 'lNumSamplesPerEpisode','int32',1; + 'lPreTriggerSamples','int32',1; + 'lEpisodesPerRun','int32',1; + 'lRunsPerTrial','int32',1; + 'lNumberOfTrials','int32',1; + 'nAveragingMode','int16',1; + 'nUndoRunCount','int16',1; + 'nFirstEpisodeInRun','int16',1; + 'fTriggerThreshold','float',1; + 'nTriggerSource','int16',1; + 'nTriggerAction','int16',1; + 'nTriggerPolarity','int16',1; + 'fScopeOutputInterval','float',1; + 'fEpisodeStartToStart','float',1; + 'fRunStartToStart','float',1; + 'lAverageCount','int32',1; + 'fTrialStartToStart','float',1; + 'nAutoTriggerStrategy','int16',1; + 'fFirstRunDelayS','float',1; + 'nChannelStatsStrategy','int16',1; + 'lSamplesPerTrace','int32',1; + 'lStartDisplayNum','int32',1; + 'lFinishDisplayNum','int32',1; + 'nShowPNRawData','int16',1; + 'fStatisticsPeriod','float',1; + 'lStatisticsMeasurements','int32',1; + 'nStatisticsSaveStrategy','int16',1; + 'fADCRange','float',1; + 'fDACRange','float',1; + 'lADCResolution','int32',1; + 'lDACResolution','int32',1; + 'nExperimentType','int16',1; + 'nManualInfoStrategy','int16',1; + 'nCommentsEnable','int16',1; + 'lFileCommentIndex','int32',1; + 'nAutoAnalyseEnable','int16',1; + 'nSignalType','int16',1; + 'nDigitalEnable','int16',1; + 'nActiveDACChannel','int16',1; + 'nDigitalHolding','int16',1; + 'nDigitalInterEpisode','int16',1; + 'nDigitalDACChannel','int16',1; + 'nDigitalTrainActiveLogic','int16',1; + 'nStatsEnable','int16',1; + 'nStatisticsClearStrategy','int16',1; + 'nLevelHysteresis','int16',1; + 'lTimeHysteresis','int32',1; + 'nAllowExternalTags','int16',1; + 'nAverageAlgorithm','int16',1; + 'fAverageWeighting','float',1; + 'nUndoPromptStrategy','int16',1; + 'nTrialTriggerSource','int16',1; + 'nStatisticsDisplayStrategy','int16',1; + 'nExternalTagType','int16',1; + 'nScopeTriggerOut','int16',1; + 'nLTPType','int16',1; + 'nAlternateDACOutputState','int16',1; + 'nAlternateDigitalOutputState','int16',1; + 'fCellID','float',3; + 'nDigitizerADCs','int16',1; + 'nDigitizerDACs','int16',1; + 'nDigitizerTotalDigitalOuts','int16',1; + 'nDigitizerSynchDigitalOuts','int16',1; + 'nDigitizerType','int16',1; + }; + +function ADCInfo=define_ADCInfo +ADCInfo={ + 'nADCNum','int16',1; + 'nTelegraphEnable','int16',1; + 'nTelegraphInstrument','int16',1; + 'fTelegraphAdditGain','float',1; + 'fTelegraphFilter','float',1; + 'fTelegraphMembraneCap','float',1; + 'nTelegraphMode','int16',1; + 'fTelegraphAccessResistance','float',1; + 'nADCPtoLChannelMap','int16',1; + 'nADCSamplingSeq','int16',1; + 'fADCProgrammableGain','float',1; + 'fADCDisplayAmplification','float',1; + 'fADCDisplayOffset','float',1; + 'fInstrumentScaleFactor','float',1; + 'fInstrumentOffset','float',1; + 'fSignalGain','float',1; + 'fSignalOffset','float',1; + 'fSignalLowpassFilter','float',1; + 'fSignalHighpassFilter','float',1; + 'nLowpassFilterType','char',1; + 'nHighpassFilterType','char',1; + 'fPostProcessLowpassFilter','float',1; + 'nPostProcessLowpassFilterType','char',1; + 'bEnabledDuringPN','bit1',1; + 'nStatsChannelPolarity','int16',1; + 'lADCChannelNameIndex','int32',1; + 'lADCUnitsIndex','int32',1; + }; + +function EpochPerDACInfo=define_EpochPerDACInfo +EpochPerDACInfo={ + 'nEpochNum','int16',1; + 'nDACNum','int16',1; + 'nEpochType','int16',1; + 'fEpochInitLevel','float',1; + 'fEpochLevelInc','float',1; + 'lEpochInitDuration','int32',1; + 'lEpochDurationInc','int32',1; + 'lEpochPulsePeriod','int32',1; + 'lEpochPulseWidth','int32',1; +}; + +function TagInfo=define_TagInfo +TagInfo={ + 'lTagTime','int32',1; + 'sComment','char',56; + 'nTagType','int16',1; + 'nVoiceTagNumber_or_AnnotationIndex','int16',1; +}; + +function UserListInfo=define_UserListInfo +UserListInfo={ + 'nListNum','int16',1; + 'nULEnable','int16',1; + 'nULParamToVary','int16',1; + 'nULRepeat','int16',1; + 'lULParamValueListIndex','int32',1; + 'sUnused','uchar',52; +}; + +% // FUNCTION: ReadUserList +% // PURPOSE: Reads the user list from the data file. +% // +% BOOL CABF2ProtocolReader::ReadUserList() +% { +% MEMBERASSERT(); +% +% BOOL bOK = TRUE; +% if( m_FileInfo.UserListSection.uBlockIndex ) +% { +% ABF_UserListInfo UserList; +% ASSERT( m_FileInfo.UserListSection.uBytes == sizeof( UserList ) ); +% ASSERT( m_FileInfo.UserListSection.llNumEntries ); +% bOK &= m_pFI->Seek( LONGLONG(m_FileInfo.UserListSection.uBlockIndex) * ABF_BLOCKSIZE, FILE_BEGIN ); +% if( !bOK ) +% return FALSE; +% +% for( long i=0; iRead( &UserList, sizeof( UserList ) ); +% short u = UserList.nListNum; +% +% m_pFH->nULEnable[u] = 1; +% m_pFH->nULParamToVary[u] = UserList.nULParamToVary; +% m_pFH->nULRepeat[u] = UserList.nULRepeat; +% +% bOK &= GetString( UserList.lULParamValueListIndex, m_pFH->sULParamValueList[u], ABF_USERLISTLEN ); +% } +% } +% return bOK; +% } + +function Section=ReadSection(fid,offset,Format) +s=cell2struct(Format,{'name','numType','number'},2); +fseek(fid,offset,'bof'); +for i=1:length(s) + [Section.(s(i).name)]=fread(fid,s(i).number,s(i).numType); +end + +function SectionInfo=ReadSectionInfo(fid,offset) +fseek(fid,offset,'bof'); +SectionInfo.uBlockIndex=fread(fid,1,'uint32'); +fseek(fid,offset+4,'bof'); +SectionInfo.uBytes=fread(fid,1,'uint32'); +fseek(fid,offset+8,'bof'); +SectionInfo.llNumEntries=fread(fid,1,'int64'); + +function pvpmod(x) +% PVPMOD - evaluate parameter/value pairs +% pvpmod(x) assigns the value x(i+1) to the parameter defined by the +% string x(i) in the calling workspace. This is useful to evaluate +% contents in an mfile, e.g. to change default settings +% of any variable initialized before pvpmod(x) is called. +% +% (c) U. Egert 1998 + +% this loop is assigns the parameter/value pairs in x to the calling +% workspace. +if ~isempty(x) + for i = 1:2:size(x,2) + assignin('caller', x{i}, x{i+1}); + end +end + +function dispif(doDispInfo, msg) +if doDispInfo + disp(msg) +end + + +% +% struct ABF_FileInfo +% { +% UINT uFileSignature; +% UINT uFileVersionNumber; +% +% // After this point there is no need to be the same as the ABF 1 equivalent. +% UINT uFileInfoSize; +% +% UINT uActualEpisodes; +% UINT uFileStartDate; +% UINT uFileStartTimeMS; +% UINT uStopwatchTime; +% short nFileType; +% short nDataFormat; +% short nSimultaneousScan; +% short nCRCEnable; +% UINT uFileCRC; +% GUID FileGUID; +% UINT uCreatorVersion; +% UINT uCreatorNameIndex; +% UINT uModifierVersion; +% UINT uModifierNameIndex; +% UINT uProtocolPathIndex; +% +% // New sections in ABF 2 - protocol stuff ... +% ABF_Section ProtocolSection; // the protocol +% ABF_Section ADCSection; // one for each ADC channel +% ABF_Section DACSection; // one for each DAC channel +% ABF_Section EpochSection; // one for each epoch +% ABF_Section ADCPerDACSection; // one for each ADC for each DAC +% ABF_Section EpochPerDACSection; // one for each epoch for each DAC +% ABF_Section UserListSection; // one for each user list +% ABF_Section StatsRegionSection; // one for each stats region +% ABF_Section MathSection; +% ABF_Section StringsSection; +% +% // ABF 1 sections ... +% ABF_Section DataSection; // Data +% ABF_Section TagSection; // Tags +% ABF_Section ScopeSection; // Scope config +% ABF_Section DeltaSection; // Deltas +% ABF_Section VoiceTagSection; // Voice Tags +% ABF_Section SynchArraySection; // Synch Array +% ABF_Section AnnotationSection; // Annotations +% ABF_Section StatsSection; // Stats config +% +% char sUnused[148]; // size = 512 bytes +% +% ABF_FileInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_FileInfo ) == 512 ); +% +% uFileSignature = ABF_FILESIGNATURE; +% uFileInfoSize = sizeof( ABF_FileInfo); +% } +% +% }; +% +% struct ABF_ProtocolInfo +% { +% short nOperationMode; +% float fADCSequenceInterval; +% bool bEnableFileCompression; +% char sUnused1[3]; +% UINT uFileCompressionRatio; +% +% float fSynchTimeUnit; +% float fSecondsPerRun; +% long lNumSamplesPerEpisode; +% long lPreTriggerSamples; +% long lEpisodesPerRun; +% long lRunsPerTrial; +% long lNumberOfTrials; +% short nAveragingMode; +% short nUndoRunCount; +% short nFirstEpisodeInRun; +% float fTriggerThreshold; +% short nTriggerSource; +% short nTriggerAction; +% short nTriggerPolarity; +% float fScopeOutputInterval; +% float fEpisodeStartToStart; +% float fRunStartToStart; +% long lAverageCount; +% float fTrialStartToStart; +% short nAutoTriggerStrategy; +% float fFirstRunDelayS; +% +% short nChannelStatsStrategy; +% long lSamplesPerTrace; +% long lStartDisplayNum; +% long lFinishDisplayNum; +% short nShowPNRawData; +% float fStatisticsPeriod; +% long lStatisticsMeasurements; +% short nStatisticsSaveStrategy; +% +% float fADCRange; +% float fDACRange; +% long lADCResolution; +% long lDACResolution; +% +% short nExperimentType; +% short nManualInfoStrategy; +% short nCommentsEnable; +% long lFileCommentIndex; +% short nAutoAnalyseEnable; +% short nSignalType; +% +% short nDigitalEnable; +% short nActiveDACChannel; +% short nDigitalHolding; +% short nDigitalInterEpisode; +% short nDigitalDACChannel; +% short nDigitalTrainActiveLogic; +% +% short nStatsEnable; +% short nStatisticsClearStrategy; +% +% short nLevelHysteresis; +% long lTimeHysteresis; +% short nAllowExternalTags; +% short nAverageAlgorithm; +% float fAverageWeighting; +% short nUndoPromptStrategy; +% short nTrialTriggerSource; +% short nStatisticsDisplayStrategy; +% short nExternalTagType; +% short nScopeTriggerOut; +% +% short nLTPType; +% short nAlternateDACOutputState; +% short nAlternateDigitalOutputState; +% +% float fCellID[3]; +% +% short nDigitizerADCs; +% short nDigitizerDACs; +% short nDigitizerTotalDigitalOuts; +% short nDigitizerSynchDigitalOuts; +% short nDigitizerType; +% +% char sUnused[304]; // size = 512 bytes +% +% ABF_ProtocolInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_ProtocolInfo ) == 512 ); +% } +% }; +% +% struct ABF_MathInfo +% { +% short nMathEnable; +% short nMathExpression; +% UINT uMathOperatorIndex; +% UINT uMathUnitsIndex; +% float fMathUpperLimit; +% float fMathLowerLimit; +% short nMathADCNum[2]; +% char sUnused[16]; +% float fMathK[6]; +% +% char sUnused2[64]; // size = 128 bytes +% +% ABF_MathInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_MathInfo ) == 128 ); +% } +% }; +% +% struct ABF_ADCInfo +% { +% // The ADC this struct is describing. +% short nADCNum; +% +% short nTelegraphEnable; +% short nTelegraphInstrument; +% float fTelegraphAdditGain; +% float fTelegraphFilter; +% float fTelegraphMembraneCap; +% short nTelegraphMode; +% float fTelegraphAccessResistance; +% +% short nADCPtoLChannelMap; +% short nADCSamplingSeq; +% +% float fADCProgrammableGain; +% float fADCDisplayAmplification; +% float fADCDisplayOffset; +% float fInstrumentScaleFactor; +% float fInstrumentOffset; +% float fSignalGain; +% float fSignalOffset; +% float fSignalLowpassFilter; +% float fSignalHighpassFilter; +% +% char nLowpassFilterType; +% char nHighpassFilterType; +% float fPostProcessLowpassFilter; +% char nPostProcessLowpassFilterType; +% bool bEnabledDuringPN; +% +% short nStatsChannelPolarity; +% +% long lADCChannelNameIndex; +% long lADCUnitsIndex; +% +% char sUnused[46]; // size = 128 bytes +% +% ABF_ADCInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_ADCInfo ) == 128 ); +% } +% }; +% +% struct ABF_DACInfo +% { +% // The DAC this struct is describing. +% short nDACNum; +% +% short nTelegraphDACScaleFactorEnable; +% float fInstrumentHoldingLevel; +% +% float fDACScaleFactor; +% float fDACHoldingLevel; +% float fDACCalibrationFactor; +% float fDACCalibrationOffset; +% +% long lDACChannelNameIndex; +% long lDACChannelUnitsIndex; +% +% long lDACFilePtr; +% long lDACFileNumEpisodes; +% +% short nWaveformEnable; +% short nWaveformSource; +% short nInterEpisodeLevel; +% +% float fDACFileScale; +% float fDACFileOffset; +% long lDACFileEpisodeNum; +% short nDACFileADCNum; +% +% short nConditEnable; +% long lConditNumPulses; +% float fBaselineDuration; +% float fBaselineLevel; +% float fStepDuration; +% float fStepLevel; +% float fPostTrainPeriod; +% float fPostTrainLevel; +% short nMembTestEnable; +% +% short nLeakSubtractType; +% short nPNPolarity; +% float fPNHoldingLevel; +% short nPNNumADCChannels; +% short nPNPosition; +% short nPNNumPulses; +% float fPNSettlingTime; +% float fPNInterpulse; +% +% short nLTPUsageOfDAC; +% short nLTPPresynapticPulses; +% +% long lDACFilePathIndex; +% +% float fMembTestPreSettlingTimeMS; +% float fMembTestPostSettlingTimeMS; +% +% short nLeakSubtractADCIndex; +% +% char sUnused[124]; // size = 256 bytes +% +% ABF_DACInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_DACInfo ) == 256 ); +% } +% }; +% +% struct ABF_EpochInfoPerDAC +% { +% // The Epoch / DAC this struct is describing. +% short nEpochNum; +% short nDACNum; +% +% // One full set of epochs (ABF_EPOCHCOUNT) for each DAC channel ... +% short nEpochType; +% float fEpochInitLevel; +% float fEpochLevelInc; +% long lEpochInitDuration; +% long lEpochDurationInc; +% long lEpochPulsePeriod; +% long lEpochPulseWidth; +% +% char sUnused[18]; // size = 48 bytes +% +% ABF_EpochInfoPerDAC() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_EpochInfoPerDAC ) == 48 ); +% } +% }; +% +% struct ABF_EpochInfo +% { +% // The Epoch this struct is describing. +% short nEpochNum; +% +% // Describes one epoch +% short nDigitalValue; +% short nDigitalTrainValue; +% short nAlternateDigitalValue; +% short nAlternateDigitalTrainValue; +% bool bEpochCompression; // Compress the data from this epoch using uFileCompressionRatio +% +% char sUnused[21]; // size = 32 bytes +% +% ABF_EpochInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_EpochInfo ) == 32 ); +% } +% }; +% +% struct ABF_StatsRegionInfo +% { +% // The stats region this struct is describing. +% short nRegionNum; +% short nADCNum; +% +% short nStatsActiveChannels; +% short nStatsSearchRegionFlags; +% short nStatsSelectedRegion; +% short nStatsSmoothing; +% short nStatsSmoothingEnable; +% short nStatsBaseline; +% long lStatsBaselineStart; +% long lStatsBaselineEnd; +% +% // Describes one stats region +% long lStatsMeasurements; +% long lStatsStart; +% long lStatsEnd; +% short nRiseBottomPercentile; +% short nRiseTopPercentile; +% short nDecayBottomPercentile; +% short nDecayTopPercentile; +% short nStatsSearchMode; +% short nStatsSearchDAC; +% short nStatsBaselineDAC; +% +% char sUnused[78]; // size = 128 bytes +% +% ABF_StatsRegionInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_StatsRegionInfo ) == 128 ); +% } +% }; +% +% struct ABF_UserListInfo +% { +% // The user list this struct is describing. +% short nListNum; +% +% // Describes one user list +% short nULEnable; +% short nULParamToVary; +% short nULRepeat; +% long lULParamValueListIndex; +% +% char sUnused[52]; // size = 64 bytes +% +% ABF_UserListInfo() +% { +% MEMSET_CTOR; +% STATIC_ASSERT( sizeof( ABF_UserListInfo ) == 64 ); +% } +% };*/= diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/abfload_license.txt b/+nansen/+module/+sharebrain/+fileadapter/private/abfload_license.txt new file mode 100644 index 0000000..ff102c6 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/private/abfload_license.txt @@ -0,0 +1,25 @@ +Copyright (c) 2009, Forrest Collman +Copyright (c) 2004, Harald Hentschke +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the distribution + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/tests/FileAdapterTest.m b/tests/FileAdapterTest.m new file mode 100644 index 0000000..fcfee3b --- /dev/null +++ b/tests/FileAdapterTest.m @@ -0,0 +1,58 @@ +classdef FileAdapterTest < matlab.unittest.TestCase +%FileAdapterTest - Tests for the NWB and DelimitedText file adapters of the module +% +% Each test writes a small file of the format and loads it with the +% adapter. The ABF adapter needs a recorded ABF file, which there is no +% way to write, and is tested on files downloaded from EBRAINS instead. +% +% Run tests: +% runtests('FileAdapterTest') + + properties + Folder string + end + + methods (TestMethodSetup) + function createFolder(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + testCase.Folder = string(testCase.applyFixture(TemporaryFolderFixture).Folder); + end + end + + methods (Test) + function testDelimitedTextReadsTabSeparatedColumns(testCase) + filePath = fullfile(testCase.Folder, "trace.txt"); + writelines(["time" + char(9) + "voltage"; "0" + char(9) + "-65.5"; "0.1" + char(9) + "-64.25"], filePath) + + data = nansen.module.sharebrain.fileadapter.DelimitedText(char(filePath)).load(); + + testCase.verifyClass(data, 'table') + testCase.verifyEqual(data.Properties.VariableNames, {'time', 'voltage'}) + testCase.verifyEqual(data.voltage, [-65.5; -64.25]) + end + + function testDelimitedTextReadsCsv(testCase) + filePath = fullfile(testCase.Folder, "positions.csv"); + writelines(["x,y"; "1,2"; "3,4"], filePath) + + data = nansen.module.sharebrain.fileadapter.DelimitedText(char(filePath)).load(); + + testCase.verifyEqual(data.y, [2; 4]) + end + + function testNWBReturnsNwbFile(testCase) + testCase.assumeEqual(exist('nwbRead', 'file'), 2, 'MatNWB is not on the path.') + filePath = fullfile(testCase.Folder, "session.nwb"); + nwb = NwbFile( ... + 'session_description', 'A test session', ... + 'identifier', 'test-session', ... + 'session_start_time', datetime(2024, 1, 2, 3, 4, 5, 'TimeZone', 'UTC')); + nwbExport(nwb, char(filePath)) + + data = nansen.module.sharebrain.fileadapter.NWB(char(filePath)).load(); + + testCase.verifyClass(data, 'NwbFile') + testCase.verifyEqual(string(data.identifier), "test-session") + end + end +end From d23b04cba620211de9edd50c7a088bb2b4e723b9 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sat, 19 Sep 2026 08:28:41 +0200 Subject: [PATCH 15/18] fix(abf): read ABF 1 fixed-length files that have no synch array abfload stopped with "internal variables 'lSynchArraynnn' are zero or negative" for ABF 1 files recorded in a fixed-length mode without a synch array, such as the files of garad-2022. It reads the sweeps one after the other whether or not there is a synch array, and takes only the sweep length and the sweep start times from it. Without the array, the sweep length is now the number of data points divided by episodes and channels, which abfload already requires to divide evenly, and the start times are left unknown. For 170518_1a.ABF of garad-2022 the file size equals the data offset plus two bytes per sample, and the ten 700 ms sweeps range from -71 mV to +38 mV. Co-Authored-By: Claude Opus 5 --- .../+fileadapter/private/abfload.m | 86 +++++++++++-------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m b/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m index a8e6836..56f4b35 100644 --- a/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m +++ b/+nansen/+module/+sharebrain/+fileadapter/private/abfload.m @@ -626,42 +626,58 @@ end % extract timing information on sweeps if (h.lSynchArrayPtr<=0 || h.lSynchArraySize<=0) - fclose(fid); - error('internal variables ''lSynchArraynnn'' are zero or negative'); - end - % the byte offset at which the SynchArraySection starts - h.lSynchArrayPtrByte=BLOCKSIZE*h.lSynchArrayPtr; - % before reading Synch Arr parameters check if file is big enough to hold them - % 4 bytes/long, 2 values per episode (start and length) - if h.lSynchArrayPtrByte+2*4*h.lSynchArraySize>fileSz - fclose(fid); - error('file seems not to contain complete Synch Array Section'); - end - if fseek(fid,h.lSynchArrayPtrByte,'bof')~=0 - fclose(fid); - error('something went wrong positioning file pointer to Synch Array Section'); - end - [synchArr,n]=fread(fid,h.lSynchArraySize*2,'int32'); - if n~=h.lSynchArraySize*2 - fclose(fid); - error('something went wrong reading synch array section'); - end - % make synchArr a h.lSynchArraySize x 2 matrix - synchArr=permute(reshape(synchArr',2,h.lSynchArraySize),[2 1]); - if numel(unique(synchArr(:,2)))>1 - fclose(fid); - error('sweeps of unequal length in file recorded in fixed-length mode'); + % Modified for NANSEN-SHAREbrain: an ABF 1 file recorded in a + % fixed-length mode can lack the synch array, where the original + % stops with an error. The sweeps are read one after the other + % below in either case, so their length is the number of points + % per episode and channel. Only the start times of the sweeps are + % not known, so h.sweepStartInPts is not set and the stop time in + % h.recTime is NaN. + if h.lActualEpisodes<=0 + fclose(fid); + error('file has no synch array and records no episodes, so the sweep length is not known'); + end + if rem(h.lActualAcqLength,h.lActualEpisodes*h.nADCNumChannels)>0 + fclose(fid); + error('file has no synch array, and its data points do not divide into sweeps of equal length'); + end + h.sweepLengthInPts=h.lActualAcqLength/h.lActualEpisodes/h.nADCNumChannels; + h.recTime=[h.lFileStartTime NaN]; + else + % the byte offset at which the SynchArraySection starts + h.lSynchArrayPtrByte=BLOCKSIZE*h.lSynchArrayPtr; + % before reading Synch Arr parameters check if file is big enough to hold them + % 4 bytes/long, 2 values per episode (start and length) + if h.lSynchArrayPtrByte+2*4*h.lSynchArraySize>fileSz + fclose(fid); + error('file seems not to contain complete Synch Array Section'); + end + if fseek(fid,h.lSynchArrayPtrByte,'bof')~=0 + fclose(fid); + error('something went wrong positioning file pointer to Synch Array Section'); + end + [synchArr,n]=fread(fid,h.lSynchArraySize*2,'int32'); + if n~=h.lSynchArraySize*2 + fclose(fid); + error('something went wrong reading synch array section'); + end + % make synchArr a h.lSynchArraySize x 2 matrix + synchArr=permute(reshape(synchArr',2,h.lSynchArraySize),[2 1]); + if numel(unique(synchArr(:,2)))>1 + fclose(fid); + error('sweeps of unequal length in file recorded in fixed-length mode'); + end + % the length of sweeps in sample points (**note: parameter lLength of + % the ABF synch section is expressed in samples (ticks) whereas + % parameter lStart is given in synchArrTimeBase units) + h.sweepLengthInPts=synchArr(1,2)/h.nADCNumChannels; + % the starting ticks of episodes in sample points (t0=1=beginning of + % recording) + h.sweepStartInPts=synchArr(:,1)*(h.synchArrTimeBase/h.fADCSampleInterval/h.nADCNumChannels); + % recording start and stop times in seconds from midnight + h.recTime=h.lFileStartTime; + h.recTime=h.recTime+[0 (1e-6*(h.sweepStartInPts(end)+h.sweepLengthInPts))*h.fADCSampleInterval*h.nADCNumChannels]; end - % the length of sweeps in sample points (**note: parameter lLength of - % the ABF synch section is expressed in samples (ticks) whereas - % parameter lStart is given in synchArrTimeBase units) - h.sweepLengthInPts=synchArr(1,2)/h.nADCNumChannels; - % the starting ticks of episodes in sample points (t0=1=beginning of - % recording) - h.sweepStartInPts=synchArr(:,1)*(h.synchArrTimeBase/h.fADCSampleInterval/h.nADCNumChannels); - % recording start and stop times in seconds from midnight - h.recTime=h.lFileStartTime; - h.recTime=h.recTime+[0 (1e-6*(h.sweepStartInPts(end)+h.sweepLengthInPts))*h.fADCSampleInterval*h.nADCNumChannels]; % determine first point and number of points to be read startPt=0; h.dataPts=h.lActualAcqLength; From 83b4b2172945fa28a193bc32f46ae9c76dfb7e87 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sat, 19 Sep 2026 13:00:10 +0200 Subject: [PATCH 16/18] feat: add file adapters for JSON, NIfTI, pickled Neo, NIX and Igor files JSON returns the decoded content. NIfTI returns the image and header from niftiread and niftiinfo, for .nii and .nii.gz. NeoPickle (.pkl), NeoNix (.nio, .nix) and IgorPro (.ibw, .pxp) read formats with no MATLAB reader through the Python package Neo. readWithNeo runs neo_to_mat.py with the Python that pyenv names, as a separate process because h5py, which nixio loads, can conflict with the HDF5 library of MATLAB. The script writes the block to a .mat file: segments with analog and irregular signals, spike trains, events and epochs, times in seconds, rates in Hz and units as text. Lists, such as the segments and the labels of events, are cell arrays, also when they are empty. Co-Authored-By: Claude Opus 5 --- .../+sharebrain/+fileadapter/IgorPro.m | 24 ++ .../+module/+sharebrain/+fileadapter/JSON.m | 21 ++ .../+module/+sharebrain/+fileadapter/NIfTI.m | 27 +++ .../+module/+sharebrain/+fileadapter/NeoNix.m | 23 ++ .../+sharebrain/+fileadapter/NeoPickle.m | 28 +++ .../+fileadapter/private/neo_to_mat.py | 208 ++++++++++++++++++ .../+fileadapter/private/readWithNeo.m | 46 ++++ tests/FileAdapterTest.m | 107 +++++++++ 8 files changed, 484 insertions(+) create mode 100644 +nansen/+module/+sharebrain/+fileadapter/IgorPro.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/JSON.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/NIfTI.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/NeoNix.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/NeoPickle.m create mode 100644 +nansen/+module/+sharebrain/+fileadapter/private/neo_to_mat.py create mode 100644 +nansen/+module/+sharebrain/+fileadapter/private/readWithNeo.m diff --git a/+nansen/+module/+sharebrain/+fileadapter/IgorPro.m b/+nansen/+module/+sharebrain/+fileadapter/IgorPro.m new file mode 100644 index 0000000..ca049a7 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/IgorPro.m @@ -0,0 +1,24 @@ +classdef IgorPro < nansen.dataio.FileAdapter +%IgorPro - File adapter for Igor Pro binary waves (.ibw) and packed experiments (.pxp) +% +% sessionObject.loadData(variableName) returns the file as a Neo block +% struct, in the form described in NeoPickle; the waves are the analog +% signals of its segment. +% +% The file is read by Python with the packages Neo and igor2 (see +% readWithNeo). + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'ibw', 'pxp'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readWithNeo(obj.Filename, "igor"); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/JSON.m b/+nansen/+module/+sharebrain/+fileadapter/JSON.m new file mode 100644 index 0000000..68bbabb --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/JSON.m @@ -0,0 +1,21 @@ +classdef JSON < nansen.dataio.FileAdapter +%JSON - File adapter for JSON files +% +% sessionObject.loadData(variableName) returns the content of the file +% decoded with jsondecode: objects become structs and arrays become +% arrays or cell arrays. + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'json'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = jsondecode(fileread(obj.Filename)); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/NIfTI.m b/+nansen/+module/+sharebrain/+fileadapter/NIfTI.m new file mode 100644 index 0000000..05f55f2 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/NIfTI.m @@ -0,0 +1,27 @@ +classdef NIfTI < nansen.dataio.FileAdapter +%NIfTI - File adapter for NIfTI images (.nii and .nii.gz) +% +% sessionObject.loadData(variableName) returns a struct with the fields +% Data - The image, as niftiread reads it: x-y-z for a volume and +% x-y-z-t for a series of volumes +% Header - The header, as niftiinfo reads it, with the voxel size +% (PixelDimensions), units (SpaceUnits, TimeUnits) and the +% transform to scanner space (Transform) +% +% Reading needs the Image Processing Toolbox (niftiread, niftiinfo). + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'nii', 'nii.gz'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + header = niftiinfo(obj.Filename); + data = struct('Data', niftiread(header), 'Header', header); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/NeoNix.m b/+nansen/+module/+sharebrain/+fileadapter/NeoNix.m new file mode 100644 index 0000000..7d26083 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/NeoNix.m @@ -0,0 +1,23 @@ +classdef NeoNix < nansen.dataio.FileAdapter +%NeoNix - File adapter for NIX files written by Neo (.nio, .nix) +% +% sessionObject.loadData(variableName) returns the first block of the +% file as a struct, in the form described in NeoPickle. +% +% The file is read by Python with the packages Neo and nixio (see +% readWithNeo). + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'nio', 'nix'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readWithNeo(obj.Filename, "nix"); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/NeoPickle.m b/+nansen/+module/+sharebrain/+fileadapter/NeoPickle.m new file mode 100644 index 0000000..dc395b2 --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/NeoPickle.m @@ -0,0 +1,28 @@ +classdef NeoPickle < nansen.dataio.FileAdapter +%NeoPickle - File adapter for Python pickle files that hold a Neo block +% +% sessionObject.loadData(variableName) returns the block as a struct with +% the fields name, description, file_origin, rec_datetime, annotations +% and segments. Each segment holds its analogsignals, +% irregularlysampledsignals, spiketrains, events and epochs; times are in +% seconds, sampling rates in Hz, and each signal gives its units as text. +% +% The file is unpickled by Python with the package Neo (see +% readWithNeo). A pickle runs code of the classes it holds when it is +% read, so load only files from a source you trust, such as a published +% dataset. + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'pkl'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readWithNeo(obj.Filename, "pickle"); + end + end +end diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/neo_to_mat.py b/+nansen/+module/+sharebrain/+fileadapter/private/neo_to_mat.py new file mode 100644 index 0000000..5b5f8ed --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/private/neo_to_mat.py @@ -0,0 +1,208 @@ +"""Read a file with Neo and write its block to a MATLAB .mat file. + +Used by the NANSEN-SHAREbrain file adapters for formats that have no +MATLAB reader: pickled Neo objects, NIX files and Igor Pro files. + + python neo_to_mat.py + + is "pickle", "nix" or "igor". The .mat file holds one struct, +block, with the block's name, description, annotations and segments. Each +segment holds its analog signals, irregularly sampled signals, spike +trains, events and epochs, with times in seconds, sampling rates in Hz and +the units of each signal as text. Lists (segments, signals, labels) are +cell arrays, also when they are empty. +""" + +import pickle +import re +import sys + +import numpy as np +import scipy.io + + +def read_block(file_path, file_format): + """The first Neo block in a file.""" + if file_format == "pickle": + with open(file_path, "rb") as file: + content = pickle.load(file) + elif file_format == "nix": + import neo + reader = neo.io.NixIO(file_path, mode="ro") + try: + content = reader.read_block() + finally: + reader.close() + elif file_format == "igor": + import neo + content = neo.io.IgorIO(file_path).read_block() + else: + raise ValueError(f"Unknown format {file_format!r}; use pickle, nix or igor.") + + # A pickle can hold a list of blocks + if isinstance(content, (list, tuple)): + content = content[0] + return content + + +def cell(items): + """A 1-by-n object array, which a .mat file stores as a cell array. + + scipy.io.savemat writes an empty list as a 0-by-0 double and a list of + strings as a padded char matrix; an object array keeps both a cell. + """ + array = np.empty((1, len(items)), dtype=object) + for index, item in enumerate(items): + array[0, index] = item + return array + + +def field_name(key, taken): + """A valid MATLAB field name for an annotation key, not already in taken. + + Keys that differ only in characters MATLAB does not allow ("a-b", "a_b") + get the suffixes _2, _3, ... instead of overwriting each other. + """ + name = re.sub(r"\W", "_", str(key)) + if not re.match(r"[A-Za-z]", name): + name = "x" + name + name = name[:63] + candidate, number = name, 1 + while candidate in taken: + number += 1 + suffix = f"_{number}" + candidate = name[:63 - len(suffix)] + suffix + return candidate + + +def simple_value(value): + """An annotation value that a .mat file can hold, or None to leave it out.""" + if isinstance(value, (str, bool, int, float, np.integer, np.floating, np.bool_)): + return value + if isinstance(value, np.ndarray) and value.dtype.kind in "biufU": + return value + if hasattr(value, "magnitude"): # A quantity + return np.asarray(value.magnitude) + if isinstance(value, (list, tuple)) and all( + isinstance(item, (str, int, float, bool)) for item in value): + if value and all(isinstance(item, (int, float, bool)) for item in value): + return np.asarray(value) + return cell(list(value)) + if value is None: + return None + return str(value) + + +def annotations(obj): + """The annotations of a Neo object that a .mat file can hold.""" + result = {} + for key, value in getattr(obj, "annotations", {}).items(): + converted = simple_value(value) + if converted is not None: + result[field_name(key, result)] = converted + return result + + +def text(value): + return "" if value is None else str(value) + + +def seconds(quantity): + return np.asarray(quantity.rescale("s").magnitude, dtype=float) + + +def array_annotations(obj): + """One value per channel or event; values that are not numbers or text, + such as None, are written as text in a cell array.""" + result = {} + for key, value in getattr(obj, "array_annotations", {}).items(): + values = np.asarray(value) + if values.dtype.kind not in "biufU": + values = cell([text(item) for item in values.ravel()]) + result[field_name(key, result)] = values + return result + + +def analog_signal(signal): + return { + "name": text(signal.name), + "data": np.asarray(signal.magnitude), + "units": str(signal.units.dimensionality), + "sampling_rate": float(signal.sampling_rate.rescale("Hz").magnitude), + "t_start": float(seconds(signal.t_start)), + "array_annotations": array_annotations(signal), + "annotations": annotations(signal), + } + + +def irregular_signal(signal): + return { + "name": text(signal.name), + "times": seconds(signal.times), + "data": np.asarray(signal.magnitude), + "units": str(signal.units.dimensionality), + "array_annotations": array_annotations(signal), + "annotations": annotations(signal), + } + + +def spike_train(train): + return { + "name": text(train.name), + "times": seconds(train.times), + "t_start": float(seconds(train.t_start)), + "t_stop": float(seconds(train.t_stop)), + "annotations": annotations(train), + } + + +def event(item): + return { + "name": text(item.name), + "times": seconds(item.times), + "labels": cell([str(label) for label in item.labels]), + "annotations": annotations(item), + } + + +def epoch(item): + return { + "name": text(item.name), + "times": seconds(item.times), + "durations": seconds(item.durations), + "labels": cell([str(label) for label in item.labels]), + "annotations": annotations(item), + } + + +def segment(item): + return { + "name": text(item.name), + "annotations": annotations(item), + "analogsignals": cell([analog_signal(s) for s in item.analogsignals]), + "irregularlysampledsignals": cell([irregular_signal(s) for s in item.irregularlysampledsignals]), + "spiketrains": cell([spike_train(s) for s in item.spiketrains]), + "events": cell([event(e) for e in item.events]), + "epochs": cell([epoch(e) for e in item.epochs]), + } + + +def block_to_dict(block): + return { + "name": text(block.name), + "description": text(block.description), + "file_origin": text(block.file_origin), + "rec_datetime": text(block.rec_datetime), + "annotations": annotations(block), + "segments": cell([segment(s) for s in block.segments]), + } + + +def main(file_path, file_format, mat_path): + block = read_block(file_path, file_format) + scipy.io.savemat(mat_path, {"block": block_to_dict(block)}, + long_field_names=True, do_compression=True) + + +if __name__ == "__main__": + main(*sys.argv[1:4]) diff --git a/+nansen/+module/+sharebrain/+fileadapter/private/readWithNeo.m b/+nansen/+module/+sharebrain/+fileadapter/private/readWithNeo.m new file mode 100644 index 0000000..fbf53ee --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/private/readWithNeo.m @@ -0,0 +1,46 @@ +function block = readWithNeo(filePath, fileFormat) +%readWithNeo - Read a file with the Python package Neo and return its block as a struct +% block = readWithNeo(filePath, fileFormat) runs neo_to_mat.py, in this +% folder, with the Python interpreter that pyenv names, and loads the +% .mat file that it writes. fileFormat is "pickle", "nix" or "igor". +% +% Python runs as a separate process rather than inside MATLAB: reading +% NIX files loads h5py, whose HDF5 library can conflict with the one +% MATLAB uses. +% +% The Python environment needs neo, quantities and scipy, and nixio for +% NIX files and igor2 for Igor files. + + arguments + filePath (1,1) string {mustBeFile} + fileFormat (1,1) string {mustBeMember(fileFormat, ["pickle", "nix", "igor"])} + end + + pythonExecutable = string(pyenv().Executable); + if strlength(pythonExecutable) == 0 + error('SHAREbrain:ReadWithNeo:NoPython', ... + ['No Python interpreter is set up for MATLAB. Set one with pyenv, for example ', ... + 'pyenv(Version="/path/to/python3"), with the packages neo, quantities and scipy.']) + end + + helperScript = fullfile(fileparts(mfilename('fullpath')), 'neo_to_mat.py'); + matFile = string(tempname) + ".mat"; + cleanup = onCleanup(@() deleteIfFile(matFile)); + + command = sprintf('"%s" "%s" "%s" %s "%s"', pythonExecutable, helperScript, ... + filePath, fileFormat, matFile); + [status, output] = system(command); + if status ~= 0 + error('SHAREbrain:ReadWithNeo:ReadFailed', ... + 'Neo could not read "%s" as a %s file:\n%s', filePath, fileFormat, strtrim(output)) + end + + S = load(matFile); + block = S.block; +end + +function deleteIfFile(filePath) + if isfile(filePath) + delete(filePath) + end +end diff --git a/tests/FileAdapterTest.m b/tests/FileAdapterTest.m index fcfee3b..e0a0548 100644 --- a/tests/FileAdapterTest.m +++ b/tests/FileAdapterTest.m @@ -40,6 +40,67 @@ function testDelimitedTextReadsCsv(testCase) testCase.verifyEqual(data.y, [2; 4]) end + function testJSONReturnsDecodedContent(testCase) + filePath = fullfile(testCase.Folder, "sidecar.json"); + writelines('{"RepetitionTime": 2.0, "SliceTiming": [0, 1]}', filePath) + + data = nansen.module.sharebrain.fileadapter.JSON(char(filePath)).load(); + + testCase.verifyEqual(data.RepetitionTime, 2) + testCase.verifyEqual(data.SliceTiming, [0; 1]) + end + + function testNIfTIReadsCompressedVolume(testCase) + testCase.assumeEqual(exist('niftiwrite', 'file'), 2, 'The Image Processing Toolbox is not installed.') + volume = reshape(single(1:24), 2, 3, 4); + niftiwrite(volume, char(fullfile(testCase.Folder, "run.nii")), 'Compressed', true) + + data = nansen.module.sharebrain.fileadapter.NIfTI( ... + char(fullfile(testCase.Folder, "run.nii.gz"))).load(); + + testCase.verifyEqual(data.Data, volume) + testCase.verifyEqual(data.Header.ImageSize, [2, 3, 4]) + end + + function testNeoPickleReturnsSignalsAndSpikes(testCase) + filePath = testCase.writeNeoBlock("block.pkl", "pickle"); + + block = nansen.module.sharebrain.fileadapter.NeoPickle(char(filePath)).load(); + + testCase.verifyNeoBlock(block) + end + + function testNeoPickleKeepsListsAsCellArrays(testCase) + % An empty segment, event labels, a channel annotation with a + % missing value, and two annotation keys that give the same + % MATLAB field name + filePath = testCase.writeNeoBlock("lists.pkl", "pickle", [ + "block = neo.Block(name='lists', **{'a-b': 1, 'a_b': 2})" + "segment = neo.Segment(name='trial 1')" + "segment.analogsignals.append(neo.AnalogSignal(np.zeros((3, 2)), units='mV', sampling_rate=1 * pq.kHz," + " array_annotations={'note': np.array([None, 'x'], dtype=object)}))" + "segment.events.append(neo.Event([1, 2] * pq.s, labels=np.array(['start', 'stop'])))" + "block.segments.extend([segment, neo.Segment(name='empty')])"]); + + block = nansen.module.sharebrain.fileadapter.NeoPickle(char(filePath)).load(); + + testCase.verifySize(block.segments, [1, 2]) + testCase.verifyClass(block.segments{2}.analogsignals, 'cell') + testCase.verifyEmpty(block.segments{2}.analogsignals) + testCase.verifyEqual(block.segments{1}.events{1}.labels, {'start', 'stop'}) + testCase.verifyEqual(block.segments{1}.analogsignals{1}.array_annotations.note, {'', 'x'}) + % Python integers arrive as int64 + testCase.verifyEqual(double([block.annotations.a_b, block.annotations.a_b_2]), [1, 2]) + end + + function testNeoNixReturnsSignalsAndSpikes(testCase) + filePath = testCase.writeNeoBlock("block.nio", "nix"); + + block = nansen.module.sharebrain.fileadapter.NeoNix(char(filePath)).load(); + + testCase.verifyNeoBlock(block) + end + function testNWBReturnsNwbFile(testCase) testCase.assumeEqual(exist('nwbRead', 'file'), 2, 'MatNWB is not on the path.') filePath = fullfile(testCase.Folder, "session.nwb"); @@ -55,4 +116,50 @@ function testNWBReturnsNwbFile(testCase) testCase.verifyEqual(string(data.identifier), "test-session") end end + + methods (Access = private) + function filePath = writeNeoBlock(testCase, fileName, fileFormat, blockLines) + % Writes the block that blockLines build in Python. The default + % is a block with one segment that holds a 2-channel signal of + % 100 samples at 1 kHz in mV, and a spike train at 0.1 and 0.5 s + if nargin < 4 + blockLines = [ + "block = neo.Block(name='test block')" + "segment = neo.Segment(name='trial 1')" + "segment.analogsignals.append(neo.AnalogSignal(np.arange(200).reshape(100, 2) * 1.0," + " units='mV', sampling_rate=1 * pq.kHz, name='vm'))" + "segment.spiketrains.append(neo.SpikeTrain([0.1, 0.5] * pq.s, t_stop=1 * pq.s, name='unit 1'))" + "block.segments.append(segment)"]; + end + python = string(pyenv().Executable); + testCase.assumeNotEmpty(char(python), 'No Python is set up for MATLAB.') + [status, ~] = system(sprintf('"%s" -c "import neo, quantities"', python)); + testCase.assumeEqual(status, 0, 'Neo is not installed in the Python for MATLAB.') + + filePath = fullfile(testCase.Folder, fileName); + scriptPath = fullfile(testCase.Folder, "write_block.py"); + writelines([ + "import pickle, sys" + "import numpy as np, quantities as pq, neo" + blockLines + "path, file_format = sys.argv[1], sys.argv[2]" + "if file_format == 'pickle':" + " pickle.dump(block, open(path, 'wb'))" + "else:" + " writer = neo.io.NixIO(path, mode='ow')" + " writer.write_block(block)" + " writer.close()"], scriptPath) + [status, output] = system(sprintf('"%s" "%s" "%s" %s', python, scriptPath, filePath, fileFormat)); + testCase.assumeEqual(status, 0, "Writing the test block failed: " + output) + end + + function verifyNeoBlock(testCase, block) + segment = block.segments{1}; + signal = segment.analogsignals{1}; + testCase.verifyEqual(size(signal.data), [100, 2]) + testCase.verifyEqual(signal.sampling_rate, 1000) + testCase.verifyEqual(strtrim(string(signal.units)), "mV") + testCase.verifyEqual(double(segment.spiketrains{1}.times(:))', [0.1, 0.5], 'AbsTol', 1e-12) + end + end end From c27cf69df0afee66e63eb81a56f1edd0220e92df Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sat, 19 Sep 2026 13:02:01 +0200 Subject: [PATCH 17/18] feat: add a file adapter for XML files XML returns the content read with readstruct: elements become fields, and an element's attributes become fields ending in Attribute. Co-Authored-By: Claude Opus 5 --- .../+module/+sharebrain/+fileadapter/XML.m | 21 +++++++++++++++++++ tests/FileAdapterTest.m | 11 ++++++++++ 2 files changed, 32 insertions(+) create mode 100644 +nansen/+module/+sharebrain/+fileadapter/XML.m diff --git a/+nansen/+module/+sharebrain/+fileadapter/XML.m b/+nansen/+module/+sharebrain/+fileadapter/XML.m new file mode 100644 index 0000000..5e840bf --- /dev/null +++ b/+nansen/+module/+sharebrain/+fileadapter/XML.m @@ -0,0 +1,21 @@ +classdef XML < nansen.dataio.FileAdapter +%XML - File adapter for XML files +% +% sessionObject.loadData(variableName) returns the content of the file +% as a struct, read with readstruct: elements become fields, and the +% attributes of an element become fields whose names end in Attribute. + + properties (Constant) + DataType = 'struct' + end + + properties (Constant, Hidden, Access = protected) + SUPPORTED_FILE_TYPES = {'xml'} + end + + methods (Access = protected) + function data = readData(obj, varargin) + data = readstruct(obj.Filename, 'FileType', 'xml'); + end + end +end diff --git a/tests/FileAdapterTest.m b/tests/FileAdapterTest.m index e0a0548..25d766d 100644 --- a/tests/FileAdapterTest.m +++ b/tests/FileAdapterTest.m @@ -50,6 +50,17 @@ function testJSONReturnsDecodedContent(testCase) testCase.verifyEqual(data.SliceTiming, [0; 1]) end + function testXMLReturnsElementsAndAttributes(testCase) + filePath = fullfile(testCase.Folder, "spatial_data.xml"); + writelines('0.5', filePath) + + data = nansen.module.sharebrain.fileadapter.XML(char(filePath)).load(); + + testCase.verifyEqual(data.versionAttribute, 2) + testCase.verifyEqual(data.scale.Text, 0.5) + testCase.verifyEqual(data.scale.unitAttribute, "um") + end + function testNIfTIReadsCompressedVolume(testCase) testCase.assumeEqual(exist('niftiwrite', 'file'), 2, 'The Image Processing Toolbox is not installed.') volume = reshape(single(1:24), 2, 3, 4); From 625c62e6f09cc430044f49bedb99d872a136a31a Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sat, 19 Sep 2026 13:41:33 +0200 Subject: [PATCH 18/18] fix: read text files whose header lines start with # The derived traces of several HBP datasets (.dat, e.g. in kozlov-2019-projection-neurons) start with 149 lines that begin with # and hold the protocol, stimulus and units, followed by two columns of numbers. readtable took parts of the header for column names. DelimitedText now reads lines that start with # as comments and joins the comment lines at the top of a file in the table's Properties.Description. Co-Authored-By: Claude Opus 5 --- .../+sharebrain/+fileadapter/DelimitedText.m | 25 +++++++++++++++- tests/FileAdapterTest.m | 29 +++++++++++++++++-- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m b/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m index 52f1554..2078940 100644 --- a/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m +++ b/+nansen/+module/+sharebrain/+fileadapter/DelimitedText.m @@ -4,18 +4,41 @@ % sessionObject.loadData(variableName) returns the file as a table, read % with readtable. readtable detects the delimiter, the number of header % lines and whether the first line holds column names. +% +% Lines that start with # are comments. The comment lines at the top of +% a file, such as the protocol, stimulus and units that the derived +% traces of several HBP datasets start with, are joined in the table's +% Properties.Description. properties (Constant) DataType = 'table' end properties (Constant, Hidden, Access = protected) + % Not .dat, which is binary in some datasets; a project assigns + % this adapter to .dat files of text by name SUPPORTED_FILE_TYPES = {'txt', 'csv', 'tsv'} end methods (Access = protected) function data = readData(obj, varargin) - data = readtable(obj.Filename); + data = readtable(obj.Filename, 'CommentStyle', '#'); + header = readCommentHeader(obj.Filename); + if ~isempty(header) + data.Properties.Description = strjoin(header, newline); + end end end end + +function header = readCommentHeader(filePath) +%readCommentHeader - The lines that start with # at the top of a file + header = strings(0, 1); + fileID = fopen(filePath, 'r'); + closeFile = onCleanup(@() fclose(fileID)); + line = fgetl(fileID); + while ischar(line) && startsWith(strtrim(line), "#") + header(end+1, 1) = string(line); %#ok + line = fgetl(fileID); + end +end diff --git a/tests/FileAdapterTest.m b/tests/FileAdapterTest.m index 25d766d..15e64a0 100644 --- a/tests/FileAdapterTest.m +++ b/tests/FileAdapterTest.m @@ -1,9 +1,10 @@ classdef FileAdapterTest < matlab.unittest.TestCase -%FileAdapterTest - Tests for the NWB and DelimitedText file adapters of the module +%FileAdapterTest - Tests for the file adapters of the module % % Each test writes a small file of the format and loads it with the -% adapter. The ABF adapter needs a recorded ABF file, which there is no -% way to write, and is tested on files downloaded from EBRAINS instead. +% adapter: DelimitedText, JSON, XML, NIfTI, NWB, and NeoPickle and +% NeoNix, whose files are written with Neo in Python. ABF and IgorPro +% are not tested here: MATLAB cannot write either format. % % Run tests: % runtests('FileAdapterTest') @@ -31,6 +32,28 @@ function testDelimitedTextReadsTabSeparatedColumns(testCase) testCase.verifyEqual(data.voltage, [-65.5; -64.25]) end + function testDelimitedTextKeepsCommentHeader(testCase) + filePath = fullfile(testCase.Folder, "trace_ch5_578.dat"); + writelines(["#"; "# DATE#17/09/2015 (5)"; "# STIMULUSDEF##" + char(9) + "Name" + char(9) + "Units"; ... + "0" + char(9) + "2.25e-08"; "0.2" + char(9) + "2.875e-08"], filePath) + + data = nansen.module.sharebrain.fileadapter.DelimitedText(char(filePath)).load(); + + testCase.verifyEqual(width(data), 2) + testCase.verifyEqual(data{:, 2}, [2.25e-08; 2.875e-08]) + testCase.verifyTrue(contains(data.Properties.Description, "DATE#17/09/2015")) + end + + function testDelimitedTextSkipsCommentLinesAfterData(testCase) + filePath = fullfile(testCase.Folder, "positions.csv"); + writelines(["x,y"; "1,2"; "# pause"; "3,4"], filePath) + + data = nansen.module.sharebrain.fileadapter.DelimitedText(char(filePath)).load(); + + testCase.verifyEqual(data.Properties.VariableNames, {'x', 'y'}) + testCase.verifyEqual(data.y, [2; 4]) + end + function testDelimitedTextReadsCsv(testCase) filePath = fullfile(testCase.Folder, "positions.csv"); writelines(["x,y"; "1,2"; "3,4"], filePath)