From 47f23bedeaa51d5cf5c095d016bdac6db9b55982 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 11 Sep 2026 17:36:27 +0200 Subject: [PATCH] feat(dloc): convert a Dataset Structure Model to data locations and variables Imports a dataset described by a Dataset Structure Model into a NANSEN project, mapping what NANSEN can represent and reporting the rest. nansen.config.dloc.dsm2DataLocationModel returns data location items, variable items and a table of the model elements that were not mapped, each with the reason. nansen.config.dloc.importDatasetStructureModel adds them to a project's models. Adding a data location assigns it a new uuid, so the import relinks each variable to the uuid its location was given. Choosing what a NANSEN session is: an entity type named session with a layout level, otherwise the entity type of the deepest level, because a session has to be found on disk. The subject is an entity type named subject, otherwise the session entity's parent. Both can be overridden. Three places where the two models differ needed a conversion rather than a copy, each checked against NANSEN's own extraction: - A model's regex rule yields its first capture group; NANSEN's expr mode yields the whole match. The text around the group moves into lookbehind and lookahead, so the whole match is the value. - A model's substring rule is a 0-based Python slice. It becomes an ind rule where both bounds are known positions, and an expr rule otherwise, because NANSEN's ind mode accepts end only as 1:end. - File patterns become wildcard file name expressions. Exclude patterns are not mapped: NANSEN's ignore list matches literal substrings, so a pattern such as ^\..* would become "." and exclude every file. Tested with the model written for Garad and Lessmann (2022), a flat folder of ABF recordings: each file becomes a session, the cell becomes the subject, and NANSEN's own session detection finds the ten sessions of a synthetic folder laid out the same way, with their ids and dates. Co-Authored-By: Claude Opus 5 --- .../+config/+dloc/+dsmconversion/Report.m | 40 ++ .../convertFilePatternToWildcard.m | 73 +++ .../+dsmconversion/convertRegexToWholeMatch.m | 119 ++++ .../+dsmconversion/convertSliceToExtraction.m | 84 +++ .../+config/+dloc/dsm2DataLocationModel.m | 512 ++++++++++++++++++ .../+dloc/importDatasetStructureModel.m | 57 ++ .../+fixture/datasetstructure/garad-2022.json | 153 ++++++ .../ImportDatasetStructureModelTest.m | 150 +++++ .../+config/Dsm2DataLocationModelTest.m | 197 +++++++ 9 files changed, 1385 insertions(+) create mode 100644 code/+nansen/+config/+dloc/+dsmconversion/Report.m create mode 100644 code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m create mode 100644 code/+nansen/+config/+dloc/+dsmconversion/convertRegexToWholeMatch.m create mode 100644 code/+nansen/+config/+dloc/+dsmconversion/convertSliceToExtraction.m create mode 100644 code/+nansen/+config/+dloc/dsm2DataLocationModel.m create mode 100644 code/+nansen/+config/+dloc/importDatasetStructureModel.m create mode 100644 tests/+nansen/+fixture/datasetstructure/garad-2022.json create mode 100644 tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m create mode 100644 tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m diff --git a/code/+nansen/+config/+dloc/+dsmconversion/Report.m b/code/+nansen/+config/+dloc/+dsmconversion/Report.m new file mode 100644 index 00000000..ba268e47 --- /dev/null +++ b/code/+nansen/+config/+dloc/+dsmconversion/Report.m @@ -0,0 +1,40 @@ +classdef (Sealed) Report < handle +%Report Collects the model elements a conversion did not map, and why +% +% A handle, so that the helpers of a conversion can add to one report +% without passing it back. + + properties (SetAccess = private) + Element (:,1) string = string.empty(0, 1) + Reason (:,1) string = string.empty(0, 1) + end + + methods + function add(obj, element, reason, condition) + %add Record an element and the reason it was not mapped + % + % report.add(element, reason, condition) records only when + % condition is true, so that callers can report conditionally + % without wrapping each call in an if block. + + arguments + obj + element (1,1) string + reason (1,1) string + condition (1,1) logical = true + end + + if ~condition + return + end + obj.Element(end+1, 1) = element; + obj.Reason(end+1, 1) = reason; + end + + function reportTable = toTable(obj) + %toTable The report as a table with Element and Reason columns + reportTable = table(obj.Element, obj.Reason, ... + 'VariableNames', {'Element', 'Reason'}); + end + end +end diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m new file mode 100644 index 00000000..246f6f44 --- /dev/null +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m @@ -0,0 +1,73 @@ +function [fileNameExpression, fileType, isConverted] = convertFilePatternToWildcard(dsmPattern, sessionTokens) +%convertFilePatternToWildcard Convert a file pattern regex to a NANSEN file name expression +% +% Syntax: +% [fileNameExpression, fileType, isConverted] = ... +% convertFilePatternToWildcard(dsmPattern, sessionTokens) +% +% A Dataset Structure Model file pattern is a regular expression that may +% contain {token} references to an entity's metadata. NANSEN's +% FileNameExpression is a wildcard pattern anchored with ^ and $, applied +% in the session folder. NANSEN already restricts a virtual session +% folder to the files that contain the session id, so tokens naming the +% session's identity become *: +% +% ^{recording_id}\.ABF$ -> ^*.ABF$ with file type .ABF +% +% Escaped punctuation becomes literal and .* becomes *. Any other regex +% construct, or a token that does not name the session identity, has no +% wildcard equivalent; isConverted is then false. +% +% Input arguments: +% dsmPattern - The file pattern from the Dataset Structure Model. +% sessionTokens - Names of the metadata fields that identify a session. + + arguments + dsmPattern (1,1) string + sessionTokens (1,:) string + end + + fileNameExpression = ""; + fileType = ""; + isConverted = false; + + extension = regexp(dsmPattern, "\\\.([A-Za-z0-9]+)\$$", "tokens", "once"); + if ~isempty(extension) + fileType = "." + extension{1}; + end + + tokens = regexp(dsmPattern, "\{([A-Za-z_][A-Za-z0-9_]*)\}", "tokens"); + pattern = dsmPattern; + for i = 1:numel(tokens) + if ~ismember(tokens{i}{1}, sessionTokens) + return + end + pattern = replace(pattern, "{" + tokens{i}{1} + "}", "*"); + end + + pattern = char(pattern); + wildcard = ''; + i = 1; + while i <= numel(pattern) + c = pattern(i); + if c == '\' + if i == numel(pattern) || isstrprop(pattern(i+1), 'alphanum') + return % \d, \w and the like have no wildcard equivalent + end + wildcard(end+1) = pattern(i+1); %#ok + i = i + 2; + elseif c == '.' && i < numel(pattern) && pattern(i+1) == '*' + wildcard(end+1) = '*'; %#ok + i = i + 2; + elseif c == '*' || (c == '^' && i == 1) || (c == '$' && i == numel(pattern)) ... + || ~any(c == '.[](){}+?|^$') + wildcard(end+1) = c; %#ok + i = i + 1; + else + return + end + end + + fileNameExpression = string(wildcard); + isConverted = true; +end diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertRegexToWholeMatch.m b/code/+nansen/+config/+dloc/+dsmconversion/convertRegexToWholeMatch.m new file mode 100644 index 00000000..5f981b26 --- /dev/null +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertRegexToWholeMatch.m @@ -0,0 +1,119 @@ +function [nansenPattern, isConverted] = convertRegexToWholeMatch(dsmPattern) +%convertRegexToWholeMatch Rewrite a capture group regex so its whole match is the value +% +% Syntax: +% [nansenPattern, isConverted] = convertRegexToWholeMatch(dsmPattern) +% +% A Dataset Structure Model extracts the first capture group of a regular +% expression. NANSEN's expr mode returns the whole match instead. The +% text before and after the first capture group is moved into lookbehind +% and lookahead assertions, so that the whole match is the group: +% +% ^(\d{6}_\d+)[a-z]\.ABF$ -> (?<=^)\d{6}_\d+(?=[a-z]\.ABF$) +% +% A pattern without a capture group is returned unchanged. A pattern +% whose group is quantified, or whose surrounding text has a top level +% alternation, can not be rewritten this way; isConverted is then false +% and nansenPattern is empty. + + arguments + dsmPattern (1,1) string + end + + nansenPattern = ""; + isConverted = false; + + [groupStart, groupEnd] = findFirstCaptureGroup(char(dsmPattern)); + + if isempty(groupStart) + nansenPattern = dsmPattern; + isConverted = true; + return + end + + pattern = char(dsmPattern); + prefix = pattern(1:groupStart-1); + group = pattern(groupStart+1:groupEnd-1); + suffix = pattern(groupEnd+1:end); + + isQuantified = ~isempty(suffix) && any(suffix(1) == '*+?{'); + if isQuantified || hasTopLevelAlternation(prefix) || hasTopLevelAlternation(suffix) + return + end + + if hasTopLevelAlternation(group) + group = ['(?:', group, ')']; + end + + nansenPattern = string(group); + if ~isempty(prefix) + nansenPattern = "(?<=" + prefix + ")" + nansenPattern; + end + if ~isempty(suffix) + nansenPattern = nansenPattern + "(?=" + suffix + ")"; + end + isConverted = true; +end + +function [groupStart, groupEnd] = findFirstCaptureGroup(pattern) +%findFirstCaptureGroup Indices of the parentheses of the first capturing group + groupStart = []; + groupEnd = []; + depth = 0; + inClass = false; + i = 1; + while i <= numel(pattern) + c = pattern(i); + if c == '\' + i = i + 2; + continue + elseif inClass + inClass = c ~= ']'; + elseif c == '[' + inClass = true; + elseif c == '(' + isCapturing = ~(i < numel(pattern) && pattern(i+1) == '?'); + if isempty(groupStart) && isCapturing + groupStart = i; + depth = 1; + elseif ~isempty(groupStart) + depth = depth + 1; + end + elseif c == ')' && ~isempty(groupStart) + depth = depth - 1; + if depth == 0 + groupEnd = i; + return + end + end + i = i + 1; + end + groupStart = []; +end + +function tf = hasTopLevelAlternation(pattern) +%hasTopLevelAlternation True when | appears outside groups and character classes + tf = false; + depth = 0; + inClass = false; + i = 1; + while i <= numel(pattern) + c = pattern(i); + if c == '\' + i = i + 2; + continue + elseif inClass + inClass = c ~= ']'; + elseif c == '[' + inClass = true; + elseif c == '(' + depth = depth + 1; + elseif c == ')' + depth = depth - 1; + elseif c == '|' && depth == 0 + tf = true; + return + end + i = i + 1; + end +end diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertSliceToExtraction.m b/code/+nansen/+config/+dloc/+dsmconversion/convertSliceToExtraction.m new file mode 100644 index 00000000..cea8aee9 --- /dev/null +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertSliceToExtraction.m @@ -0,0 +1,84 @@ +function [mode, input, isConverted] = convertSliceToExtraction(slice) +%convertSliceToExtraction Convert a 0-based slice to a NANSEN ind or expr rule +% +% Syntax: +% [mode, input, isConverted] = convertSliceToExtraction(slice) +% +% A Dataset Structure Model substring rule is a Python style slice, +% "start:stop", 0-based and half-open, where either bound may be omitted +% and a negative bound counts from the end. NANSEN's ind mode takes a +% 1-based MATLAB index expression. +% +% Slices whose bounds are both known positions become ind rules: +% 0:6 -> ind 1:6 +% : -> ind 1:end +% Slices that depend on the length of the text become expr rules, so +% that they do not rely on end appearing anywhere but in 1:end: +% 9: -> expr (?<=^.{9}).* +% :-4 -> expr ^.*(?=.{4}$) +% +% A slice with a negative start and a non-negative stop depends on the +% length of the text in a way neither mode can express; isConverted is +% then false and mode and input are empty. + + arguments + slice (1,1) string + end + + mode = ""; + input = ""; + isConverted = false; + + bounds = split(slice, ":"); + if numel(bounds) ~= 2 + return + end + + start = parseBound(bounds(1)); + stop = parseBound(bounds(2)); + + if isnan(start); start = 0; end + + if start >= 0 && ~isnan(stop) && stop >= 0 + mode = "ind"; + input = sprintf("%d:%d", start + 1, stop); + + elseif start == 0 && isnan(stop) + mode = "ind"; + input = "1:end"; + + elseif start > 0 && isnan(stop) + mode = "expr"; + input = sprintf("(?<=^.{%d}).*", start); + + elseif start >= 0 && stop < 0 + mode = "expr"; + if start == 0 + input = sprintf("^.*(?=.{%d}$)", -stop); + else + input = sprintf("(?<=^.{%d}).*(?=.{%d}$)", start, -stop); + end + + elseif start < 0 && isnan(stop) + mode = "expr"; + input = sprintf(".{%d}$", -start); + + elseif start < 0 && stop < 0 && stop > start + mode = "expr"; + input = sprintf(".{%d}(?=.{%d}$)", stop - start, -stop); + + else + return + end + + isConverted = true; +end + +function value = parseBound(bound) +%parseBound An integer bound, or NaN when the bound is omitted + if strlength(strtrim(bound)) == 0 + value = NaN; + else + value = str2double(bound); + end +end diff --git a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m new file mode 100644 index 00000000..c36976fe --- /dev/null +++ b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m @@ -0,0 +1,512 @@ +function [dataLocations, variables, report] = dsm2DataLocationModel(dsmConfig, options) +%dsm2DataLocationModel Convert a Dataset Structure Model to NANSEN data locations and variables +% +% Syntax: +% [dataLocations, variables, report] = ... +% nansen.config.dloc.dsm2DataLocationModel(dsmConfig) +% +% [...] = nansen.config.dloc.dsm2DataLocationModel(dsmConfig, ... +% SessionEntity=name, SubjectEntity=name) +% +% Converts what a NANSEN project can represent and reports the rest. A +% Dataset Structure Model describes any number of entity types with any +% metadata. NANSEN has sessions and subjects, and four metadata +% variables: Subject ID, Session ID, Experiment Date and Experiment Time. +% +% Which entity becomes a NANSEN session: +% an entity type named session with a layout level, otherwise the +% entity type of the deepest layout level. A NANSEN session has to be +% found on disk, so it must be an entity with a folder or files. +% Which entity becomes a NANSEN subject: +% an entity type named subject, otherwise the source of a +% relationship whose target is the session entity. +% +% Input arguments: +% dsmConfig - Path to a Dataset Structure Model json file, or its +% decoded struct. Validate it first (dsm validate). +% options.SessionEntity - Entity type to use as the NANSEN session. +% options.SubjectEntity - Entity type to use as the NANSEN subject. +% +% Output arguments: +% dataLocations - Struct array of data location model items. +% variables - Struct array of variable model items. +% report - Table of the model elements that were not mapped, +% or were mapped approximately, with the reason. +% +% See also nansen.config.dloc.DataLocationModel, +% nansen.config.varmodel.VariableModel + + arguments + dsmConfig + options.SessionEntity (1,1) string = missing + options.SubjectEntity (1,1) string = missing + end + + import nansen.config.dloc.dsmconversion.* + + if ischar(dsmConfig) || isstring(dsmConfig) + dsmConfig = jsondecode(fileread(dsmConfig)); + end + + report = Report(); + + entityTypes = asList(dsmConfig, 'entityTypes'); + definitions = getOr(dsmConfig, 'metadataDefinitions', struct()); + locations = asList(dsmConfig, 'dataLocations'); + + sessionEntity = options.SessionEntity; + if ismissing(sessionEntity) + sessionEntity = chooseSessionEntity(entityTypes, locations); + end + subjectEntity = options.SubjectEntity; + if ismissing(subjectEntity) + subjectEntity = chooseSubjectEntity(entityTypes, asList(dsmConfig, 'entityRelationships'), sessionEntity); + end + + sessionKeys = identityKeys(entityTypes, sessionEntity); + subjectKeys = identityKeys(entityTypes, subjectEntity); + usedFields = string.empty(1, 0); + + report.add("entityRelationships", "NANSEN has no relationships between entity types beyond subject and session.", ... + ~isempty(asList(dsmConfig, 'entityRelationships'))) + for i = 1:numel(entityTypes) + name = string(entityTypes{i}.name); + if ~ismember(name, [sessionEntity, subjectEntity]) + report.add("entityTypes[" + name + "]", "Only the session and subject entity types have a NANSEN counterpart.") + end + end + report.add("preferences", "Preferences belong to the data location model, not to its items; set DefaultDataLocation after importing.", ... + isfield(dsmConfig, 'preferences')) + + dataLocations = struct.empty; + variables = struct.empty; + + for i = 1:numel(locations) + location = locations{i}; + where = "dataLocations[" + string(location.identifier) + "]"; + + if string(getOr(location, 'sourceType', 'filesystem')) ~= "filesystem" + report.add(where, "Only filesystem data locations can be converted.") + continue + end + + [item, sessionLevel, usedHere] = convertLocation(location, where, definitions, ... + sessionEntity, subjectEntity, sessionKeys, subjectKeys, report); + if isempty(item) + continue + end + usedFields = [usedFields, usedHere]; %#ok + + dataLocations = appendItems(dataLocations, item); + variables = appendItems(variables, convertFilePatterns(sessionLevel, item, where, sessionKeys, report)); + end + + for fieldName = string(fieldnames(definitions))' + if ~ismember(fieldName, usedFields) + report.add("metadataDefinitions[" + fieldName + "]", ... + "NANSEN extracts only Subject ID, Session ID, Experiment Date and Experiment Time.") + end + end + + report = report.toTable(); +end + +% ------------------------------------------------------------------------- + +function [item, sessionLevel, usedFields] = convertLocation(location, where, definitions, ... + sessionEntity, subjectEntity, sessionKeys, subjectKeys, report) +%convertLocation Convert one filesystem data location + + item = []; + sessionLevel = []; + usedFields = string.empty(1, 0); + source = location.filesystemSource; + layout = asList(source, 'entityLayout'); + + levelTypes = fieldValues(layout, 'entityType'); + sessionIndex = find(levelTypes == sessionEntity, 1); + if isempty(sessionIndex) + report.add(where, "No layout level holds the session entity '" + sessionEntity + "', so NANSEN could not find sessions here.") + return + end + for j = sessionIndex+1:numel(layout) + report.add(where + ".entityLayout[" + string(layout{j}.name) + "]", ... + "NANSEN treats the deepest level it walks as the session; levels below the session level are dropped.") + end + layout = layout(1:sessionIndex); + sessionLevel = layout{end}; + + item = nansen.config.dloc.DataLocationModel.getBlankItem(); + item.Name = char(matlab.lang.makeValidName(string(location.identifier))); + report.add(where + ".identifier", "Renamed to '" + item.Name + "' to be a valid MATLAB name.", ... + ~strcmp(item.Name, location.identifier)) + % A placeholder that links variables to this item until it is inserted; + % inserting an item into a catalog assigns it a new uuid. + item.Uuid = nansen.util.getuuid(); + report.add(where + ".uuid", "NANSEN assigns its own uuid when a data location is added; the model's uuid is not kept.", ... + isfield(location, 'uuid')) + item = orderfields(item, ['Uuid'; setdiff(fieldnames(item), 'Uuid', 'stable')]); + item.Type = convertType(location, where, report); + item.RootPath = convertRootPaths(asList(source, 'rootStoragePaths'), where, report); + item.SubfolderStructure = convertLayout(layout, where, definitions, sessionEntity, subjectEntity, report); + + levelNames = fieldValues(asList(source, 'entityLayout'), 'name'); + [item.MetaDataDef, usedFields] = convertMetadata(asList(source, 'metadataMapping'), ... + definitions, levelNames, sessionIndex, where, sessionEntity, subjectEntity, ... + sessionKeys, subjectKeys, report); +end + +function type = convertType(location, where, report) +%convertType Map access and data category to a NANSEN data location type + category = string(getOr(location, 'dataCategory', 'raw')); + access = string(getOr(location, 'access', 'read')); + + if access == "read" + typeName = "recorded"; + isExact = category == "raw"; + elseif category == "temporary" + typeName = "temporary"; + isExact = true; + else + typeName = "processed"; + isExact = ismember(category, ["processed", "derived"]); + end + report.add(where + ".dataCategory", "NANSEN has no type for '" + category + "' with " + access + ... + " access; mapped to " + typeName + ".", ~isExact) + type = nansen.config.dloc.DataLocationType(char(typeName)); +end + +function rootPaths = convertRootPaths(storagePaths, where, report) +%convertRootPaths Map root storage paths to NANSEN root paths + rootPaths = struct('Key', {}, 'Value', {}, 'DiskName', {}, 'DiskType', {}); + for j = 1:numel(storagePaths) + storagePath = storagePaths{j}; + here = where + ".rootStoragePaths[" + string(storagePath.identifier) + "]"; + + key = char(getOr(storagePath, 'uuid', '')); + if isempty(key); key = nansen.util.getuuid(); end + + storageType = string(getOr(storagePath, 'storageType', 'local')); + diskType = 'External'; + if storageType == "local"; diskType = 'Local'; end + + rootPaths(j) = struct('Key', key, 'Value', char(storagePath.path), ... + 'DiskName', char(getOr(storagePath, 'volumeName', '')), 'DiskType', diskType); + + report.add(here + ".storageType", "NANSEN distinguishes only local and external disks; mapped to External.", ... + ~ismember(storageType, ["local", "external"])) + report.add(here + ".environment", "NANSEN keeps per-machine root paths in local settings rather than by environment.", ... + isfield(storagePath, 'environment')) + end +end + +function structure = convertLayout(layout, where, definitions, sessionEntity, subjectEntity, report) +%convertLayout Map layout levels down to the session level to a subfolder structure + import nansen.config.dloc.dsmconversion.* + + structure = repmat(nansen.config.dloc.DataLocationModel.getDefaultSubfolderStructure(), 1, numel(layout)); + for j = 1:numel(layout) + level = layout{j}; + here = where + ".entityLayout[" + string(level.name) + "]"; + entityType = string(getOr(level, 'entityType', "")); + + if entityType == sessionEntity + structure(j).Type = 'Session'; + elseif entityType == subjectEntity + structure(j).Type = 'Subject'; + else + structure(j).Type = ''; + report.add(here, "Entity type '" + entityType + "' has no NANSEN folder type; kept as an untyped level.", entityType ~= "") + end + + if ~getOr(level, 'isVariable', true) + structure(j).Name = char(level.fixedName); + structure(j).Expression = ['^', regexptranslate('escape', char(level.fixedName)), '$']; + elseif isfield(level, 'matchPattern') + structure(j).Expression = char(level.matchPattern); + else + structure(j).Expression = char(templateToPattern(string(level.pathComponentTemplate), definitions)); + end + structure(j).IsFolder = string(getOr(level, 'fileSystemType', 'folder')) ~= "file"; + structure(j).IgnoreList = {}; + + report.add(here + ".excludePatterns", "Not mapped: NANSEN's ignore list matches literal substrings, not patterns. " + ... + "The level's expression already restricts the names, and hidden files are skipped.", ... + ~isempty(asList(level, 'excludePatterns'))) + report.add(here + ".pathComponentTemplate", "NANSEN generates folder names itself; the template is not used for writing.", ... + isfield(level, 'pathComponentTemplate')) + end +end + +function [metaDataDef, usedFields] = convertMetadata(mapping, definitions, levelNames, sessionIndex, ... + where, sessionEntity, subjectEntity, sessionKeys, subjectKeys, report) +%convertMetadata Map extraction rules onto NANSEN's four metadata variables + import nansen.config.dloc.dsmconversion.* + + metaDataDef = nansen.config.dloc.DataLocationModel.getDefaultMetadataStructure(); + usedFields = string.empty(1, 0); + + targets = struct( ... + 'Variable', {'Subject ID', 'Session ID', 'Experiment Date', 'Experiment Time'}, ... + 'Field', {identityField(subjectKeys, "subject", where, report), ... + identityField(sessionKeys, "session", where, report), ... + temporalField(definitions, ["date", "datetime"], sessionEntity, subjectEntity), ... + temporalField(definitions, "time", sessionEntity, subjectEntity)}); + + for k = 1:numel(targets) + field = targets(k).Field; + if field == ""; continue; end + usedFields(end+1) = field; %#ok + + rule = findRule(mapping, field); + here = where + ".metadataMapping[" + field + "]"; + if isempty(rule) + report.add(here, "No extraction rule for this field in this data location; " + targets(k).Variable + " is left unset.") + continue + end + extraction = rule.extraction; + + levelRef = getOr(extraction, 'entityLayoutLevel', []); + if isnumeric(levelRef) && ~isempty(levelRef) + levelIndex = levelRef + 1; + elseif ~isempty(levelRef) + levelIndex = find(levelNames == string(levelRef), 1); + else + levelIndex = []; + end + if isempty(levelIndex) || levelIndex > sessionIndex + report.add(here, "The rule does not read a single level at or above the session level; " + targets(k).Variable + " is left unset.") + continue + end + + method = string(extraction.method); + if method == "regex" + [input, isConverted] = convertRegexToWholeMatch(string(extraction.pattern)); + mode = "expr"; + elseif method == "substring" + [mode, input, isConverted] = convertSliceToExtraction(string(extraction.pattern)); + else + isConverted = false; + end + if ~isConverted + report.add(here, "The " + method + " rule can not be expressed as a NANSEN ind or expr rule; " + targets(k).Variable + " is left unset.") + continue + end + report.add(here + ".normalize", "Normalisation is not applied by NANSEN; extracted values may differ.", ... + string(getOr(extraction, 'normalize', 'none')) ~= "none") + + idx = strcmp({metaDataDef.VariableName}, targets(k).Variable); + metaDataDef(idx).SubfolderLevel = levelIndex; + metaDataDef(idx).StringDetectMode = char(mode); + metaDataDef(idx).StringDetectInput = char(input); + metaDataDef(idx).StringFormat = char(getOr(extraction, 'valueFormat', '')); + end +end + +function variables = convertFilePatterns(sessionLevel, item, where, sessionKeys, report) +%convertFilePatterns Map the session level's file patterns to variable model items + import nansen.config.dloc.dsmconversion.* + + variables = struct.empty; + here = where + ".entityLayout[" + string(sessionLevel.name) + "].filePatterns"; + patterns = asList(sessionLevel, 'filePatterns'); + + for j = 1:numel(patterns) + filePattern = patterns{j}; + if ~isfield(filePattern, 'name') + report.add(here, "An unnamed file pattern has no variable name to map to.") + continue + end + patternWhere = here + "[" + string(filePattern.name) + "]"; + + [expression, fileType, isConverted] = convertFilePatternToWildcard(string(filePattern.pattern), sessionKeys); + if ~isConverted + report.add(patternWhere, "The pattern has no wildcard equivalent.") + continue + end + + variable = nansen.config.varmodel.VariableModel.getBlankItem(); + variable.VariableName = char(matlab.lang.makeValidName(string(filePattern.name))); + variable.DataLocation = item.Name; + variable.DataLocationUuid = item.Uuid; + variable.FileNameExpression = char(expression); + variable.FileType = char(fileType); + variable.FileAdapter = findFileAdapter(fileType, patternWhere, report); + variable.IsCustom = true; + + report.add(patternWhere + ".isRequired", "NANSEN does not check that required files are present.", ... + isfield(filePattern, 'isRequired')) + variables = appendItems(variables, variable); + end +end + +function adapterName = findFileAdapter(fileType, where, report) +%findFileAdapter Name of a file adapter for a file type, or Default + adapterName = 'Default'; + adapters = struct.empty; + if fileType ~= "" + try + adapters = nansen.dataio.listFileAdapters(char(fileType)); + catch exception + % File adapters are listed from the current project, which + % needs a user session. Converting a model does not, so an + % absent session only means the adapter can not be looked up. + if ~strcmp(exception.identifier, 'NANSEN:NoActiveUserSession') + rethrow(exception) + end + end + end + if isempty(adapters) || strcmp(adapters(1).FileAdapterName, 'N/A') + report.add(where + ".fileAdapter", "No file adapter supports '" + fileType + ... + "' (or no project is active); set to Default, so the file can be found but not loaded.") + else + adapterName = adapters(1).FileAdapterName; + end +end + +% ------------------------------------------------------------------------- + +function name = chooseSessionEntity(entityTypes, locations) +%chooseSessionEntity An entity type named session with a level, else the deepest level's + names = fieldValues(entityTypes, 'name'); + levelTypes = string.empty(1, 0); + deepest = ""; + for i = 1:numel(locations) + if ~isfield(locations{i}, 'filesystemSource'); continue; end + layout = asList(locations{i}.filesystemSource, 'entityLayout'); + types = fieldValues(layout, 'entityType'); + levelTypes = [levelTypes, types]; %#ok + typed = types(types ~= ""); + if ~isempty(typed) && deepest == "" + deepest = typed(end); + end + end + if ismember("session", names) && ismember("session", levelTypes) + name = "session"; + else + name = deepest; + end +end + +function name = chooseSubjectEntity(entityTypes, relationships, sessionEntity) +%chooseSubjectEntity An entity type named subject, else the session entity's parent + names = fieldValues(entityTypes, 'name'); + name = ""; + if ismember("subject", names) && sessionEntity ~= "subject" + name = "subject"; + return + end + for i = 1:numel(relationships) + if string(relationships{i}.targetEntity) == sessionEntity ... + && ismember(string(relationships{i}.relationType), ["oneToMany", "oneToOne"]) + name = string(relationships{i}.sourceEntity); + return + end + end +end + +function keys = identityKeys(entityTypes, entityName) +%identityKeys Identity fields of an entity type + keys = string.empty(1, 0); + for i = 1:numel(entityTypes) + if string(entityTypes{i}.name) == entityName + if isfield(entityTypes{i}, 'identifierRef') + keys = string(entityTypes{i}.identifierRef); + else + keys = reshape(string(entityTypes{i}.identifierRefs), 1, []); + end + end + end +end + +function field = identityField(keys, role, where, report) +%identityField The single identity field, or "" with a report for a composite one + field = ""; + if isscalar(keys) + field = keys; + elseif numel(keys) > 1 + report.add(where, "The " + role + " identity is composite (" + strjoin(keys, ", ") + "); NANSEN needs a single id field.") + end +end + +function field = temporalField(definitions, dataTypes, sessionEntity, subjectEntity) +%temporalField First field of a temporal data type, preferring the session entity + field = ""; + for owner = [sessionEntity, subjectEntity] + for name = string(fieldnames(definitions))' + definition = definitions.(name); + if ismember(string(definition.dataType), dataTypes) && string(definition.ofEntity) == owner + field = name; + return + end + end + end +end + +function rule = findRule(mapping, field) +%findRule The extraction rule for a field, or [] + rule = []; + for i = 1:numel(mapping) + if string(mapping{i}.metadataRef) == field + rule = mapping{i}; + return + end + end +end + +function pattern = templateToPattern(template, definitions) +%templateToPattern Derive a name pattern from a path component template + [tokens, literals] = regexp(char(template), '\{([A-Za-z_][A-Za-z0-9_]*)\}', 'tokens', 'split'); + pattern = "^" + regexptranslate('escape', literals{1}); + for i = 1:numel(tokens) + tokenPattern = "[^/\\]+"; + if isfield(definitions, tokens{i}{1}) && isfield(definitions.(tokens{i}{1}), 'validation') ... + && isfield(definitions.(tokens{i}{1}).validation, 'pattern') + tokenPattern = "(?:" + regexprep(string(definitions.(tokens{i}{1}).validation.pattern), '^\^|\$$', '') + ")"; + end + pattern = pattern + tokenPattern + regexptranslate('escape', literals{i+1}); + end + pattern = pattern + "$"; +end + +function values = fieldValues(items, fieldName) +%fieldValues One field of each item in a list, as a string row; "" where absent + values = strings(1, numel(items)); + for i = 1:numel(items) + values(i) = string(getOr(items{i}, fieldName, "")); + end +end + +function list = appendItems(list, items) +%appendItems Append struct items to a struct array that may still be empty + if isempty(items) + return + elseif isempty(list) + list = items; + else + list = [list, items]; + end +end + +function items = asList(s, fieldName) +%asList A json array field as a row cell array of scalar structs or values + items = {}; + if ~isstruct(s) || ~isfield(s, fieldName) || isempty(s.(fieldName)); return; end + value = s.(fieldName); + if iscell(value) + items = reshape(value, 1, []); + elseif isstruct(value) + items = num2cell(reshape(value, 1, [])); + else + items = {value}; + end +end + +function value = getOr(s, fieldName, default) +%getOr A struct field, or a default when it is absent or empty + if isstruct(s) && isfield(s, fieldName) && ~isempty(s.(fieldName)) + value = s.(fieldName); + else + value = default; + end +end diff --git a/code/+nansen/+config/+dloc/importDatasetStructureModel.m b/code/+nansen/+config/+dloc/importDatasetStructureModel.m new file mode 100644 index 00000000..7b0f61da --- /dev/null +++ b/code/+nansen/+config/+dloc/importDatasetStructureModel.m @@ -0,0 +1,57 @@ +function report = importDatasetStructureModel(dsmConfig, dataLocationModel, variableModel, options) +%importDatasetStructureModel Add a Dataset Structure Model's data locations and variables to a project +% +% Syntax: +% report = nansen.config.dloc.importDatasetStructureModel( ... +% dsmConfig, dataLocationModel, variableModel) +% +% report = nansen.config.dloc.importDatasetStructureModel(..., ... +% SessionEntity=name, SubjectEntity=name) +% +% Converts the model with dsm2DataLocationModel, adds the data locations +% to dataLocationModel and the variables to variableModel, and returns +% the report of what was not mapped. +% +% Adding a data location assigns it a new uuid, so each variable is +% relinked to the uuid its data location was given. +% +% Adding an item whose name is already in a model raises an error, so +% importing the same model twice into one project fails on the second +% attempt rather than creating duplicates. +% +% Example: +% project = nansen.getCurrentProject(); +% report = nansen.config.dloc.importDatasetStructureModel( ... +% "garad-2022.json", project.DataLocationModel, project.VariableModel); +% disp(report) +% +% See also nansen.config.dloc.dsm2DataLocationModel + + arguments + dsmConfig + dataLocationModel (1,1) nansen.config.dloc.DataLocationModel + variableModel (1,1) nansen.config.varmodel.VariableModel + options.SessionEntity (1,1) string = missing + options.SubjectEntity (1,1) string = missing + end + + nameValueArgs = namedargs2cell(options); + [dataLocations, variables, report] = ... + nansen.config.dloc.dsm2DataLocationModel(dsmConfig, nameValueArgs{:}); + + for i = 1:numel(dataLocations) + placeholderUuid = dataLocations(i).Uuid; + dataLocationModel.addDataLocation(dataLocations(i)); + assignedUuid = dataLocationModel.getDataLocation(dataLocations(i).Name).Uuid; + + for j = 1:numel(variables) + if strcmp(variables(j).DataLocationUuid, placeholderUuid) + variables(j).DataLocationUuid = assignedUuid; + end + end + end + + for j = 1:numel(variables) + variableModel.insertItem(variables(j)); + end +end diff --git a/tests/+nansen/+fixture/datasetstructure/garad-2022.json b/tests/+nansen/+fixture/datasetstructure/garad-2022.json new file mode 100644 index 00000000..ffb7decb --- /dev/null +++ b/tests/+nansen/+fixture/datasetstructure/garad-2022.json @@ -0,0 +1,153 @@ +{ + "schemaVersion": "1.0.0", + "entityTypes": [ + { + "name": "cell", + "isPrimary": true, + "identifierRef": "cell_id", + "description": "One whole-cell patched CA1 pyramidal neuron; one neuron per slice per day. The descriptor reports 7 cells from 2 mice on 2 days. Cells have no folder; identity is read from the recording file names." + }, + { + "name": "recording", + "identifierRef": "recording_id", + "description": "One ABF file: one repetition of the excitability protocol (0-450 pA, 50 pA steps, 500 ms) for a cell. Five repetitions (a-e) per cell." + } + ], + "metadataDefinitions": { + "cell_id": { + "name": "cell_id", + "title": "Cell ID", + "description": "Recording date (yyMMdd) and slice number on that day, e.g. 170518_1. One neuron was recorded per slice.", + "dataType": "string", + "ofEntity": "cell", + "validation": { + "pattern": "^\\d{6}_\\d+$" + } + }, + "recording_date": { + "name": "recording_date", + "title": "Recording date", + "dataType": "date", + "ofEntity": "cell", + "description": "Day of recording, from the yyMMdd prefix of the file name" + }, + "recording_id": { + "name": "recording_id", + "title": "Recording ID", + "description": "File name without extension, e.g. 170518_1a", + "dataType": "string", + "ofEntity": "recording", + "validation": { + "pattern": "^\\d{6}_\\d+[a-z]$" + } + }, + "slice_number": { + "name": "slice_number", + "title": "Slice number", + "description": "Number of the slice recorded on that day from the same animal (data descriptor naming convention)", + "dataType": "integer", + "ofEntity": "cell" + }, + "protocol_repeat": { + "name": "protocol_repeat", + "title": "Protocol repeat", + "description": "Letter a-e: which of the five repetitions of the excitability protocol this file holds", + "dataType": "string", + "ofEntity": "recording", + "validation": { + "pattern": "^[a-e]$" + } + } + }, + "dataLocations": [ + { + "identifier": "garad", + "displayName": "Garad & Lessmann 2022, CA1 excitability ABF recordings", + "description": "Flat folder of Axon Binary Format files named _.ABF, plus the EBRAINS data descriptor and licence PDFs. Category 'raw' per the descriptor (original Patchmaster recordings). Every entry in the folder was listed; nothing was sampled.", + "dataCategory": "raw", + "access": "read", + "sourceType": "filesystem", + "filesystemSource": { + "rootStoragePaths": [ + { + "identifier": "main", + "path": "/Users/eivihe/Data/ShareBrain/InDepthCuration/garad-2022", + "storageType": "local" + } + ], + "entityLayout": [ + { + "name": "recording-files", + "entityType": "recording", + "fileSystemType": "file", + "matchPattern": "^\\d{6}_\\d+[a-z]\\.ABF$", + "excludePatterns": [ + "^\\..*", + "^EBRAINS-datadescriptor_.*", + "^Licence-CC-BY\\.pdf$" + ], + "filePatterns": [ + { + "name": "abf", + "pattern": "^{recording_id}\\.ABF$", + "isRequired": true, + "cardinality": "one" + } + ] + } + ], + "metadataMapping": [ + { + "metadataRef": "recording_id", + "extraction": { + "method": "regex", + "pattern": "^(\\d{6}_\\d+[a-z])\\.ABF$", + "entityLayoutLevel": "recording-files" + } + }, + { + "metadataRef": "protocol_repeat", + "extraction": { + "method": "regex", + "pattern": "^\\d{6}_\\d+([a-z])\\.ABF$", + "entityLayoutLevel": "recording-files" + } + }, + { + "metadataRef": "cell_id", + "extraction": { + "method": "regex", + "pattern": "^(\\d{6}_\\d+)[a-z]\\.ABF$", + "entityLayoutLevel": "recording-files" + } + }, + { + "metadataRef": "slice_number", + "extraction": { + "method": "regex", + "pattern": "^\\d{6}_(\\d+)[a-z]\\.ABF$", + "entityLayoutLevel": "recording-files" + } + }, + { + "metadataRef": "recording_date", + "extraction": { + "method": "substring", + "pattern": "0:6", + "valueFormat": "yyMMdd", + "entityLayoutLevel": "recording-files" + } + } + ] + } + } + ], + "entityRelationships": [ + { + "sourceEntity": "cell", + "targetEntity": "recording", + "relationType": "oneToMany", + "relationName": "hasRecordings" + } + ] +} diff --git a/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m b/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m new file mode 100644 index 00000000..1664fda9 --- /dev/null +++ b/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m @@ -0,0 +1,150 @@ +classdef ImportDatasetStructureModelTest < matlab.unittest.TestCase + %ImportDatasetStructureModelTest Import a Dataset Structure Model and detect sessions with NANSEN + % + % Imports the model written for Garad and Lessmann (2022) into data + % location and variable models, then asks NANSEN's own session + % detection what it finds in a folder laid out like that dataset. The + % folder is synthetic: it follows the naming convention in the model, + % _.ABF, with two cells of five repeats, plus + % the descriptor, licence and a hidden file that must be skipped. + % + % Run tests: + % runtests('nansen.integrationtest.ImportDatasetStructureModelTest') + + properties + DataFolder char + DataLocationModel + VariableModel + Report + end + + methods (TestClassSetup) + function setupProject(testCase) + testCase.applyFixture(nansen.fixture.ProjectFixture) + end + end + + methods (TestMethodSetup) + function importGarad(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + root = testCase.applyFixture(TemporaryFolderFixture).Folder; + + testCase.DataFolder = fullfile(root, 'garad-2022'); + mkdir(testCase.DataFolder) + fileNames = [ ... + compose("170518_1%s.ABF", ["a", "b", "c", "d", "e"]), ... + compose("170519_2%s.ABF", ["a", "b", "c", "d", "e"]), ... + "EBRAINS-datadescriptor_garad.pdf", "Licence-CC-BY.pdf", ".DS_Store"]; + for fileName = fileNames + fclose(fopen(fullfile(testCase.DataFolder, fileName), 'w')); + end + + fixture = fullfile(fileparts(fileparts(mfilename('fullpath'))), ... + '+fixture', 'datasetstructure', 'garad-2022.json'); + dsm = jsondecode(fileread(fixture)); + dsm.dataLocations.filesystemSource.rootStoragePaths.path = testCase.DataFolder; + + configFolder = fullfile(root, 'configurations'); + mkdir(configFolder) + dataLocationPath = fullfile(configFolder, 'datalocation_settings.json'); + variablePath = fullfile(configFolder, 'filepath_settings.json'); + + testCase.Report = nansen.config.dloc.importDatasetStructureModel(dsm, ... + nansen.config.dloc.DataLocationModel(dataLocationPath), ... + nansen.config.varmodel.VariableModel(variablePath)); + + % Read both back, as a project would on its next start + testCase.DataLocationModel = nansen.config.dloc.DataLocationModel(dataLocationPath); + testCase.VariableModel = nansen.config.varmodel.VariableModel(variablePath); + end + end + + methods (Access = private) + function paths = sessionPaths(testCase) + sessionFolders = nansen.dataio.session.listSessionFolders( ... + testCase.DataLocationModel, 'garad'); + paths = reshape(sessionFolders.garad, 1, []); + end + end + + methods (Test) + + function testImportedModelsLoadFromDisk(testCase) + % The only data location is read-only, so there is no default. + % Loading such a model used to raise. + testCase.verifyEqual(testCase.DataLocationModel.DataLocationNames, {'garad'}) + testCase.verifyEmpty(testCase.DataLocationModel.DefaultDataLocation) + testCase.verifyEqual(testCase.VariableModel.getItem('abf').FileType, '.ABF') + end + + function testEachAbfFileIsASession(testCase) + paths = testCase.sessionPaths(); + + testCase.verifyNumElements(paths, 10, ... + 'The descriptor, licence and hidden file must not be sessions.') + + [~, names, extensions] = fileparts(paths); + testCase.verifyEqual(unique(extensions), {'.ABF'}) + testCase.verifyEqual(sort(names), ... + sort(cellstr([compose("170518_1%s", ["a" "b" "c" "d" "e"]), ... + compose("170519_2%s", ["a" "b" "c" "d" "e"])]))) + end + + function testSessionAndSubjectIdsComeFromTheFileNames(testCase) + model = testCase.DataLocationModel; + index = model.getItemIndex('garad'); + paths = testCase.sessionPaths(); + + sessionIds = cellfun(@(p) model.getSessionID(p, index), paths, 'UniformOutput', false); + subjectIds = cellfun(@(p) model.getSubjectID(p, index), paths, 'UniformOutput', false); + + [~, names] = fileparts(paths); + testCase.verifyEqual(sessionIds, names, ... + 'The session id is the file name without its extension.') + testCase.verifyEqual(unique(subjectIds), {'170518_1', '170519_2'}, ... + 'The subject is the cell: the date and slice number.') + end + + function testExperimentDateIsParsed(testCase) + model = testCase.DataLocationModel; + paths = testCase.sessionPaths(); + firstPath = paths{contains(paths, '170518_1a')}; + + experimentDate = model.getDate(firstPath, model.getItemIndex('garad')); + + testCase.verifyEqual([year(experimentDate), month(experimentDate), day(experimentDate)], ... + [2017, 5, 18]) + end + + function testSessionsAreMatchedOnePerFile(testCase) + sessionFolders = nansen.dataio.session.listSessionFolders(testCase.DataLocationModel, 'all'); + matched = nansen.dataio.session.matchSessionFolders(testCase.DataLocationModel, sessionFolders); + + testCase.verifyNumElements(matched, 10, ... + 'Grouping by session id must not merge the repeats of a cell.') + end + + function testVariableIsLinkedToItsDataLocation(testCase) + variable = testCase.VariableModel.getItem('abf'); + dataLocation = testCase.DataLocationModel.getDataLocation('garad'); + + testCase.verifyEqual(variable.DataLocationUuid, dataLocation.Uuid, ... + 'Adding the data location assigns a new uuid; the variable must follow it.') + end + + function testVariableFindsTheSessionsFile(testCase) + % A session stored as a file shares its folder with every other + % session. NANSEN narrows the folder to the files containing + % the session id, as done here. + fileName = testCase.VariableModel.lookForFile(testCase.DataFolder, 'abf', ... + 'FilterFcn', @(names) contains(names, '170519_2c')); + + testCase.verifyEqual(fileName, '170519_2c.ABF') + end + + function testReportIsReturned(testCase) + testCase.verifyClass(testCase.Report, 'table') + testCase.verifyTrue(any(contains(testCase.Report.Element, "slice_number"))) + end + end +end diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m new file mode 100644 index 00000000..36f13e01 --- /dev/null +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m @@ -0,0 +1,197 @@ +classdef Dsm2DataLocationModelTest < matlab.unittest.TestCase + %Dsm2DataLocationModelTest Converting a Dataset Structure Model to NANSEN items + % + % Uses the model written for Garad and Lessmann (2022): one flat folder + % of ABF files named _.ABF, a cell entity with no + % folder of its own, and a recording entity per file. + % + % Extraction rules are checked by running them through NANSEN's own + % extraction, so the tests assert the values NANSEN will produce rather + % than the text of the rules. + % + % Run tests: + % runtests('nansen.unittest.config.Dsm2DataLocationModelTest') + + properties (Constant) + FileName = '170518_1a.ABF' + end + + properties + DataLocations + Variables + Report + end + + properties (TestParameter) + regexCase = { ... + {'^(\d{6}_\d+[a-z])\.ABF$', '170518_1a'}, ... + {'^(\d{6}_\d+)[a-z]\.ABF$', '170518_1'}, ... + {'^\d{6}_(\d+)[a-z]\.ABF$', '1'}, ... + {'^.*_(\d+)[a-z]\.ABF$', '1'}, ... + {'\.(ABF|abf)$', 'ABF'}, ... + {'^\d{6}_\d+[a-z]', '170518_1a'}} + sliceCase = { ... + {'0:6', '170518'}, {':', '170518_1a.ABF'}, {'9:', '.ABF'}, ... + {':-4', '170518_1a'}, {'7:-4', '1a'}, {'-3:', 'ABF'}, {'-4:-3', '.'}} + end + + methods (TestClassSetup) + function convertGarad(testCase) + fixture = fullfile(fileparts(fileparts(fileparts(mfilename('fullpath')))), ... + '+fixture', 'datasetstructure', 'garad-2022.json'); + [testCase.DataLocations, testCase.Variables, testCase.Report] = ... + nansen.config.dloc.dsm2DataLocationModel(fixture); + end + end + + methods (Access = private) + function value = extract(testCase, variableName) + %extract Run a converted metadata rule on the example file name + rule = testCase.DataLocations.MetaDataDef( ... + strcmp({testCase.DataLocations.MetaDataDef.VariableName}, variableName)); + value = nansen.config.dloc.DataLocationModel.applyExtractionPattern( ... + testCase.FileName, rule.StringDetectMode, rule.StringDetectInput); + end + + function verifyReported(testCase, elementPart) + testCase.verifyTrue(any(contains(testCase.Report.Element, elementPart)), ... + sprintf('Expected the report to mention %s.', elementPart)) + end + + function verifyNotReported(testCase, elementPart) + testCase.verifyFalse(any(contains(testCase.Report.Element, elementPart)), ... + sprintf('%s was mapped and should not be in the report.', elementPart)) + end + end + + methods (Test) + + function testOneReadOnlyLocationOfFiles(testCase) + dataLocation = testCase.DataLocations; + testCase.verifyNumElements(dataLocation, 1) + testCase.verifyEqual(dataLocation.Name, 'garad') + testCase.verifyEqual(dataLocation.Type.Name, 'recorded') + + level = dataLocation.SubfolderStructure; + testCase.verifyNumElements(level, 1) + testCase.verifyEqual(level.Type, 'Session') + testCase.verifyFalse(level.IsFolder, 'Each recording is a file, not a folder.') + testCase.verifyEqual(level.Expression, '^\d{6}_\d+[a-z]\.ABF$') + testCase.verifyEqual(level.IgnoreList, {}, ... + 'Exclude patterns must not become substring ignores; "." would drop every file.') + + testCase.verifyEqual(dataLocation.RootPath.Value, ... + '/Users/eivihe/Data/ShareBrain/InDepthCuration/garad-2022') + testCase.verifyEqual(dataLocation.RootPath.DiskType, 'Local') + end + + function testRecordingIsTheSessionAndCellTheSubject(testCase) + testCase.verifyEqual(testCase.extract('Session ID'), '170518_1a') + testCase.verifyEqual(testCase.extract('Subject ID'), '170518_1') + end + + function testRecordingDateUsesItsFormat(testCase) + rule = testCase.DataLocations.MetaDataDef( ... + strcmp({testCase.DataLocations.MetaDataDef.VariableName}, 'Experiment Date')); + testCase.verifyEqual(rule.StringDetectMode, 'ind') + testCase.verifyEqual(rule.StringFormat, 'yyMMdd') + testCase.verifyEqual(testCase.extract('Experiment Date'), '170518') + end + + function testAbsentTimeIsLeftUnset(testCase) + rule = testCase.DataLocations.MetaDataDef( ... + strcmp({testCase.DataLocations.MetaDataDef.VariableName}, 'Experiment Time')); + testCase.verifyEmpty(rule.SubfolderLevel) + end + + function testAbfFileBecomesAVariable(testCase) + variable = testCase.Variables; + testCase.verifyNumElements(variable, 1) + testCase.verifyEqual(variable.VariableName, 'abf') + testCase.verifyEqual(variable.FileNameExpression, '^*.ABF$') + testCase.verifyEqual(variable.FileType, '.ABF') + testCase.verifyEqual(variable.DataLocation, 'garad') + testCase.verifyEqual(variable.DataLocationUuid, testCase.DataLocations.Uuid) + testCase.verifyEqual(variable.FileAdapter, 'Default', ... + 'NANSEN has no ABF file adapter.') + end + + function testReportListsWhatWasNotMapped(testCase) + testCase.verifyReported("metadataDefinitions[slice_number]") + testCase.verifyReported("metadataDefinitions[protocol_repeat]") + testCase.verifyReported("entityRelationships") + testCase.verifyReported("excludePatterns") + testCase.verifyReported("filePatterns[abf].isRequired") + testCase.verifyReported("filePatterns[abf].fileAdapter") + end + + function testReportLeavesOutWhatWasMapped(testCase) + testCase.verifyNotReported("metadataDefinitions[cell_id]") + testCase.verifyNotReported("metadataDefinitions[recording_id]") + testCase.verifyNotReported("metadataDefinitions[recording_date]") + testCase.verifyNotReported("dataCategory") + testCase.verifyNotReported("entityTypes[") + end + + function testSessionEntityWithoutALevelIsReported(testCase) + % A cell has no folder or file of its own, so NANSEN could not + % find it on disk as a session. + fixture = fullfile(fileparts(fileparts(fileparts(mfilename('fullpath')))), ... + '+fixture', 'datasetstructure', 'garad-2022.json'); + [dataLocations, ~, report] = nansen.config.dloc.dsm2DataLocationModel( ... + fixture, SessionEntity="cell"); + + testCase.verifyEmpty(dataLocations) + testCase.verifyTrue(any(contains(report.Reason, "session entity 'cell'"))) + end + + function testRegexBecomesWholeMatch(testCase, regexCase) + [pattern, isConverted] = ... + nansen.config.dloc.dsmconversion.convertRegexToWholeMatch(regexCase{1}); + testCase.assertTrue(isConverted) + value = nansen.config.dloc.DataLocationModel.applyExtractionPattern( ... + testCase.FileName, 'expr', char(pattern)); + testCase.verifyEqual(value, regexCase{2}) + end + + function testQuantifiedGroupIsNotConverted(testCase) + [~, isConverted] = ... + nansen.config.dloc.dsmconversion.convertRegexToWholeMatch('^(\d)+_'); + testCase.verifyFalse(isConverted) + end + + function testSliceGivesThePythonValue(testCase, sliceCase) + [mode, input, isConverted] = ... + nansen.config.dloc.dsmconversion.convertSliceToExtraction(sliceCase{1}); + testCase.assertTrue(isConverted) + value = nansen.config.dloc.DataLocationModel.applyExtractionPattern( ... + testCase.FileName, char(mode), char(input)); + testCase.verifyEqual(value, sliceCase{2}) + end + + function testSliceNeverUsesEndOutsideTheWholeRange(testCase) + % NANSEN's ind mode accepts end only as 1:end. + for slice = ["0:6", ":", "9:", ":-4", "7:-4", "-3:"] + [mode, input] = nansen.config.dloc.dsmconversion.convertSliceToExtraction(slice); + if mode == "ind" + testCase.verifyTrue(~contains(input, "end") || input == "1:end", slice) + end + end + end + + function testFilePatternsBecomeWildcards(testCase) + convert = @(p) nansen.config.dloc.dsmconversion.convertFilePatternToWildcard(p, "recording_id"); + + [expression, fileType, isConverted] = convert("^{recording_id}_meta\.json$"); + testCase.verifyTrue(isConverted) + testCase.verifyEqual(expression, "^*_meta.json$") + testCase.verifyEqual(fileType, ".json") + + [~, ~, isConverted] = convert("^{other_id}\.ABF$"); + testCase.verifyFalse(isConverted, 'Only tokens naming the session identity become *.') + + [~, ~, isConverted] = convert("^\d+\.ABF$"); + testCase.verifyFalse(isConverted, '\d has no wildcard equivalent.') + end + end +end