diff --git a/code/+nansen/+config/+dloc/+dsmconversion/Report.m b/code/+nansen/+config/+dloc/+dsmconversion/Report.m new file mode 100644 index 00000000..ba268e47 --- /dev/null +++ b/code/+nansen/+config/+dloc/+dsmconversion/Report.m @@ -0,0 +1,40 @@ +classdef (Sealed) Report < handle +%Report Collects the model elements a conversion did not map, and why +% +% A handle, so that the helpers of a conversion can add to one report +% without passing it back. + + properties (SetAccess = private) + Element (:,1) string = string.empty(0, 1) + Reason (:,1) string = string.empty(0, 1) + end + + methods + function add(obj, element, reason, condition) + %add Record an element and the reason it was not mapped + % + % report.add(element, reason, condition) records only when + % condition is true, so that callers can report conditionally + % without wrapping each call in an if block. + + arguments + obj + element (1,1) string + reason (1,1) string + condition (1,1) logical = true + end + + if ~condition + return + end + obj.Element(end+1, 1) = element; + obj.Reason(end+1, 1) = reason; + end + + function reportTable = toTable(obj) + %toTable The report as a table with Element and Reason columns + reportTable = table(obj.Element, obj.Reason, ... + 'VariableNames', {'Element', 'Reason'}); + end + end +end diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m new file mode 100644 index 00000000..246f6f44 --- /dev/null +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m @@ -0,0 +1,73 @@ +function [fileNameExpression, fileType, isConverted] = convertFilePatternToWildcard(dsmPattern, sessionTokens) +%convertFilePatternToWildcard Convert a file pattern regex to a NANSEN file name expression +% +% Syntax: +% [fileNameExpression, fileType, isConverted] = ... +% convertFilePatternToWildcard(dsmPattern, sessionTokens) +% +% A Dataset Structure Model file pattern is a regular expression that may +% contain {token} references to an entity's metadata. NANSEN's +% FileNameExpression is a wildcard pattern anchored with ^ and $, applied +% in the session folder. NANSEN already restricts a virtual session +% folder to the files that contain the session id, so tokens naming the +% session's identity become *: +% +% ^{recording_id}\.ABF$ -> ^*.ABF$ with file type .ABF +% +% Escaped punctuation becomes literal and .* becomes *. Any other regex +% construct, or a token that does not name the session identity, has no +% wildcard equivalent; isConverted is then false. +% +% Input arguments: +% dsmPattern - The file pattern from the Dataset Structure Model. +% sessionTokens - Names of the metadata fields that identify a session. + + arguments + dsmPattern (1,1) string + sessionTokens (1,:) string + end + + fileNameExpression = ""; + fileType = ""; + isConverted = false; + + extension = regexp(dsmPattern, "\\\.([A-Za-z0-9]+)\$$", "tokens", "once"); + if ~isempty(extension) + fileType = "." + extension{1}; + end + + tokens = regexp(dsmPattern, "\{([A-Za-z_][A-Za-z0-9_]*)\}", "tokens"); + pattern = dsmPattern; + for i = 1:numel(tokens) + if ~ismember(tokens{i}{1}, sessionTokens) + return + end + pattern = replace(pattern, "{" + tokens{i}{1} + "}", "*"); + end + + pattern = char(pattern); + wildcard = ''; + i = 1; + while i <= numel(pattern) + c = pattern(i); + if c == '\' + if i == numel(pattern) || isstrprop(pattern(i+1), 'alphanum') + return % \d, \w and the like have no wildcard equivalent + end + wildcard(end+1) = pattern(i+1); %#ok + i = i + 2; + elseif c == '.' && i < numel(pattern) && pattern(i+1) == '*' + wildcard(end+1) = '*'; %#ok + i = i + 2; + elseif c == '*' || (c == '^' && i == 1) || (c == '$' && i == numel(pattern)) ... + || ~any(c == '.[](){}+?|^$') + wildcard(end+1) = c; %#ok + i = i + 1; + else + return + end + end + + fileNameExpression = string(wildcard); + isConverted = true; +end diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertRegexToWholeMatch.m b/code/+nansen/+config/+dloc/+dsmconversion/convertRegexToWholeMatch.m new file mode 100644 index 00000000..5f981b26 --- /dev/null +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertRegexToWholeMatch.m @@ -0,0 +1,119 @@ +function [nansenPattern, isConverted] = convertRegexToWholeMatch(dsmPattern) +%convertRegexToWholeMatch Rewrite a capture group regex so its whole match is the value +% +% Syntax: +% [nansenPattern, isConverted] = convertRegexToWholeMatch(dsmPattern) +% +% A Dataset Structure Model extracts the first capture group of a regular +% expression. NANSEN's expr mode returns the whole match instead. The +% text before and after the first capture group is moved into lookbehind +% and lookahead assertions, so that the whole match is the group: +% +% ^(\d{6}_\d+)[a-z]\.ABF$ -> (?<=^)\d{6}_\d+(?=[a-z]\.ABF$) +% +% A pattern without a capture group is returned unchanged. A pattern +% whose group is quantified, or whose surrounding text has a top level +% alternation, can not be rewritten this way; isConverted is then false +% and nansenPattern is empty. + + arguments + dsmPattern (1,1) string + end + + nansenPattern = ""; + isConverted = false; + + [groupStart, groupEnd] = findFirstCaptureGroup(char(dsmPattern)); + + if isempty(groupStart) + nansenPattern = dsmPattern; + isConverted = true; + return + end + + pattern = char(dsmPattern); + prefix = pattern(1:groupStart-1); + group = pattern(groupStart+1:groupEnd-1); + suffix = pattern(groupEnd+1:end); + + isQuantified = ~isempty(suffix) && any(suffix(1) == '*+?{'); + if isQuantified || hasTopLevelAlternation(prefix) || hasTopLevelAlternation(suffix) + return + end + + if hasTopLevelAlternation(group) + group = ['(?:', group, ')']; + end + + nansenPattern = string(group); + if ~isempty(prefix) + nansenPattern = "(?<=" + prefix + ")" + nansenPattern; + end + if ~isempty(suffix) + nansenPattern = nansenPattern + "(?=" + suffix + ")"; + end + isConverted = true; +end + +function [groupStart, groupEnd] = findFirstCaptureGroup(pattern) +%findFirstCaptureGroup Indices of the parentheses of the first capturing group + groupStart = []; + groupEnd = []; + depth = 0; + inClass = false; + i = 1; + while i <= numel(pattern) + c = pattern(i); + if c == '\' + i = i + 2; + continue + elseif inClass + inClass = c ~= ']'; + elseif c == '[' + inClass = true; + elseif c == '(' + isCapturing = ~(i < numel(pattern) && pattern(i+1) == '?'); + if isempty(groupStart) && isCapturing + groupStart = i; + depth = 1; + elseif ~isempty(groupStart) + depth = depth + 1; + end + elseif c == ')' && ~isempty(groupStart) + depth = depth - 1; + if depth == 0 + groupEnd = i; + return + end + end + i = i + 1; + end + groupStart = []; +end + +function tf = hasTopLevelAlternation(pattern) +%hasTopLevelAlternation True when | appears outside groups and character classes + tf = false; + depth = 0; + inClass = false; + i = 1; + while i <= numel(pattern) + c = pattern(i); + if c == '\' + i = i + 2; + continue + elseif inClass + inClass = c ~= ']'; + elseif c == '[' + inClass = true; + elseif c == '(' + depth = depth + 1; + elseif c == ')' + depth = depth - 1; + elseif c == '|' && depth == 0 + tf = true; + return + end + i = i + 1; + end +end diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertSliceToExtraction.m b/code/+nansen/+config/+dloc/+dsmconversion/convertSliceToExtraction.m new file mode 100644 index 00000000..cea8aee9 --- /dev/null +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertSliceToExtraction.m @@ -0,0 +1,84 @@ +function [mode, input, isConverted] = convertSliceToExtraction(slice) +%convertSliceToExtraction Convert a 0-based slice to a NANSEN ind or expr rule +% +% Syntax: +% [mode, input, isConverted] = convertSliceToExtraction(slice) +% +% A Dataset Structure Model substring rule is a Python style slice, +% "start:stop", 0-based and half-open, where either bound may be omitted +% and a negative bound counts from the end. NANSEN's ind mode takes a +% 1-based MATLAB index expression. +% +% Slices whose bounds are both known positions become ind rules: +% 0:6 -> ind 1:6 +% : -> ind 1:end +% Slices that depend on the length of the text become expr rules, so +% that they do not rely on end appearing anywhere but in 1:end: +% 9: -> expr (?<=^.{9}).* +% :-4 -> expr ^.*(?=.{4}$) +% +% A slice with a negative start and a non-negative stop depends on the +% length of the text in a way neither mode can express; isConverted is +% then false and mode and input are empty. + + arguments + slice (1,1) string + end + + mode = ""; + input = ""; + isConverted = false; + + bounds = split(slice, ":"); + if numel(bounds) ~= 2 + return + end + + start = parseBound(bounds(1)); + stop = parseBound(bounds(2)); + + if isnan(start); start = 0; end + + if start >= 0 && ~isnan(stop) && stop >= 0 + mode = "ind"; + input = sprintf("%d:%d", start + 1, stop); + + elseif start == 0 && isnan(stop) + mode = "ind"; + input = "1:end"; + + elseif start > 0 && isnan(stop) + mode = "expr"; + input = sprintf("(?<=^.{%d}).*", start); + + elseif start >= 0 && stop < 0 + mode = "expr"; + if start == 0 + input = sprintf("^.*(?=.{%d}$)", -stop); + else + input = sprintf("(?<=^.{%d}).*(?=.{%d}$)", start, -stop); + end + + elseif start < 0 && isnan(stop) + mode = "expr"; + input = sprintf(".{%d}$", -start); + + elseif start < 0 && stop < 0 && stop > start + mode = "expr"; + input = sprintf(".{%d}(?=.{%d}$)", stop - start, -stop); + + else + return + end + + isConverted = true; +end + +function value = parseBound(bound) +%parseBound An integer bound, or NaN when the bound is omitted + if strlength(strtrim(bound)) == 0 + value = NaN; + else + value = str2double(bound); + end +end diff --git a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m new file mode 100644 index 00000000..c36976fe --- /dev/null +++ b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m @@ -0,0 +1,512 @@ +function [dataLocations, variables, report] = dsm2DataLocationModel(dsmConfig, options) +%dsm2DataLocationModel Convert a Dataset Structure Model to NANSEN data locations and variables +% +% Syntax: +% [dataLocations, variables, report] = ... +% nansen.config.dloc.dsm2DataLocationModel(dsmConfig) +% +% [...] = nansen.config.dloc.dsm2DataLocationModel(dsmConfig, ... +% SessionEntity=name, SubjectEntity=name) +% +% Converts what a NANSEN project can represent and reports the rest. A +% Dataset Structure Model describes any number of entity types with any +% metadata. NANSEN has sessions and subjects, and four metadata +% variables: Subject ID, Session ID, Experiment Date and Experiment Time. +% +% Which entity becomes a NANSEN session: +% an entity type named session with a layout level, otherwise the +% entity type of the deepest layout level. A NANSEN session has to be +% found on disk, so it must be an entity with a folder or files. +% Which entity becomes a NANSEN subject: +% an entity type named subject, otherwise the source of a +% relationship whose target is the session entity. +% +% Input arguments: +% dsmConfig - Path to a Dataset Structure Model json file, or its +% decoded struct. Validate it first (dsm validate). +% options.SessionEntity - Entity type to use as the NANSEN session. +% options.SubjectEntity - Entity type to use as the NANSEN subject. +% +% Output arguments: +% dataLocations - Struct array of data location model items. +% variables - Struct array of variable model items. +% report - Table of the model elements that were not mapped, +% or were mapped approximately, with the reason. +% +% See also nansen.config.dloc.DataLocationModel, +% nansen.config.varmodel.VariableModel + + arguments + dsmConfig + options.SessionEntity (1,1) string = missing + options.SubjectEntity (1,1) string = missing + end + + import nansen.config.dloc.dsmconversion.* + + if ischar(dsmConfig) || isstring(dsmConfig) + dsmConfig = jsondecode(fileread(dsmConfig)); + end + + report = Report(); + + entityTypes = asList(dsmConfig, 'entityTypes'); + definitions = getOr(dsmConfig, 'metadataDefinitions', struct()); + locations = asList(dsmConfig, 'dataLocations'); + + sessionEntity = options.SessionEntity; + if ismissing(sessionEntity) + sessionEntity = chooseSessionEntity(entityTypes, locations); + end + subjectEntity = options.SubjectEntity; + if ismissing(subjectEntity) + subjectEntity = chooseSubjectEntity(entityTypes, asList(dsmConfig, 'entityRelationships'), sessionEntity); + end + + sessionKeys = identityKeys(entityTypes, sessionEntity); + subjectKeys = identityKeys(entityTypes, subjectEntity); + usedFields = string.empty(1, 0); + + report.add("entityRelationships", "NANSEN has no relationships between entity types beyond subject and session.", ... + ~isempty(asList(dsmConfig, 'entityRelationships'))) + for i = 1:numel(entityTypes) + name = string(entityTypes{i}.name); + if ~ismember(name, [sessionEntity, subjectEntity]) + report.add("entityTypes[" + name + "]", "Only the session and subject entity types have a NANSEN counterpart.") + end + end + report.add("preferences", "Preferences belong to the data location model, not to its items; set DefaultDataLocation after importing.", ... + isfield(dsmConfig, 'preferences')) + + dataLocations = struct.empty; + variables = struct.empty; + + for i = 1:numel(locations) + location = locations{i}; + where = "dataLocations[" + string(location.identifier) + "]"; + + if string(getOr(location, 'sourceType', 'filesystem')) ~= "filesystem" + report.add(where, "Only filesystem data locations can be converted.") + continue + end + + [item, sessionLevel, usedHere] = convertLocation(location, where, definitions, ... + sessionEntity, subjectEntity, sessionKeys, subjectKeys, report); + if isempty(item) + continue + end + usedFields = [usedFields, usedHere]; %#ok + + dataLocations = appendItems(dataLocations, item); + variables = appendItems(variables, convertFilePatterns(sessionLevel, item, where, sessionKeys, report)); + end + + for fieldName = string(fieldnames(definitions))' + if ~ismember(fieldName, usedFields) + report.add("metadataDefinitions[" + fieldName + "]", ... + "NANSEN extracts only Subject ID, Session ID, Experiment Date and Experiment Time.") + end + end + + report = report.toTable(); +end + +% ------------------------------------------------------------------------- + +function [item, sessionLevel, usedFields] = convertLocation(location, where, definitions, ... + sessionEntity, subjectEntity, sessionKeys, subjectKeys, report) +%convertLocation Convert one filesystem data location + + item = []; + sessionLevel = []; + usedFields = string.empty(1, 0); + source = location.filesystemSource; + layout = asList(source, 'entityLayout'); + + levelTypes = fieldValues(layout, 'entityType'); + sessionIndex = find(levelTypes == sessionEntity, 1); + if isempty(sessionIndex) + report.add(where, "No layout level holds the session entity '" + sessionEntity + "', so NANSEN could not find sessions here.") + return + end + for j = sessionIndex+1:numel(layout) + report.add(where + ".entityLayout[" + string(layout{j}.name) + "]", ... + "NANSEN treats the deepest level it walks as the session; levels below the session level are dropped.") + end + layout = layout(1:sessionIndex); + sessionLevel = layout{end}; + + item = nansen.config.dloc.DataLocationModel.getBlankItem(); + item.Name = char(matlab.lang.makeValidName(string(location.identifier))); + report.add(where + ".identifier", "Renamed to '" + item.Name + "' to be a valid MATLAB name.", ... + ~strcmp(item.Name, location.identifier)) + % A placeholder that links variables to this item until it is inserted; + % inserting an item into a catalog assigns it a new uuid. + item.Uuid = nansen.util.getuuid(); + report.add(where + ".uuid", "NANSEN assigns its own uuid when a data location is added; the model's uuid is not kept.", ... + isfield(location, 'uuid')) + item = orderfields(item, ['Uuid'; setdiff(fieldnames(item), 'Uuid', 'stable')]); + item.Type = convertType(location, where, report); + item.RootPath = convertRootPaths(asList(source, 'rootStoragePaths'), where, report); + item.SubfolderStructure = convertLayout(layout, where, definitions, sessionEntity, subjectEntity, report); + + levelNames = fieldValues(asList(source, 'entityLayout'), 'name'); + [item.MetaDataDef, usedFields] = convertMetadata(asList(source, 'metadataMapping'), ... + definitions, levelNames, sessionIndex, where, sessionEntity, subjectEntity, ... + sessionKeys, subjectKeys, report); +end + +function type = convertType(location, where, report) +%convertType Map access and data category to a NANSEN data location type + category = string(getOr(location, 'dataCategory', 'raw')); + access = string(getOr(location, 'access', 'read')); + + if access == "read" + typeName = "recorded"; + isExact = category == "raw"; + elseif category == "temporary" + typeName = "temporary"; + isExact = true; + else + typeName = "processed"; + isExact = ismember(category, ["processed", "derived"]); + end + report.add(where + ".dataCategory", "NANSEN has no type for '" + category + "' with " + access + ... + " access; mapped to " + typeName + ".", ~isExact) + type = nansen.config.dloc.DataLocationType(char(typeName)); +end + +function rootPaths = convertRootPaths(storagePaths, where, report) +%convertRootPaths Map root storage paths to NANSEN root paths + rootPaths = struct('Key', {}, 'Value', {}, 'DiskName', {}, 'DiskType', {}); + for j = 1:numel(storagePaths) + storagePath = storagePaths{j}; + here = where + ".rootStoragePaths[" + string(storagePath.identifier) + "]"; + + key = char(getOr(storagePath, 'uuid', '')); + if isempty(key); key = nansen.util.getuuid(); end + + storageType = string(getOr(storagePath, 'storageType', 'local')); + diskType = 'External'; + if storageType == "local"; diskType = 'Local'; end + + rootPaths(j) = struct('Key', key, 'Value', char(storagePath.path), ... + 'DiskName', char(getOr(storagePath, 'volumeName', '')), 'DiskType', diskType); + + report.add(here + ".storageType", "NANSEN distinguishes only local and external disks; mapped to External.", ... + ~ismember(storageType, ["local", "external"])) + report.add(here + ".environment", "NANSEN keeps per-machine root paths in local settings rather than by environment.", ... + isfield(storagePath, 'environment')) + end +end + +function structure = convertLayout(layout, where, definitions, sessionEntity, subjectEntity, report) +%convertLayout Map layout levels down to the session level to a subfolder structure + import nansen.config.dloc.dsmconversion.* + + structure = repmat(nansen.config.dloc.DataLocationModel.getDefaultSubfolderStructure(), 1, numel(layout)); + for j = 1:numel(layout) + level = layout{j}; + here = where + ".entityLayout[" + string(level.name) + "]"; + entityType = string(getOr(level, 'entityType', "")); + + if entityType == sessionEntity + structure(j).Type = 'Session'; + elseif entityType == subjectEntity + structure(j).Type = 'Subject'; + else + structure(j).Type = ''; + report.add(here, "Entity type '" + entityType + "' has no NANSEN folder type; kept as an untyped level.", entityType ~= "") + end + + if ~getOr(level, 'isVariable', true) + structure(j).Name = char(level.fixedName); + structure(j).Expression = ['^', regexptranslate('escape', char(level.fixedName)), '$']; + elseif isfield(level, 'matchPattern') + structure(j).Expression = char(level.matchPattern); + else + structure(j).Expression = char(templateToPattern(string(level.pathComponentTemplate), definitions)); + end + structure(j).IsFolder = string(getOr(level, 'fileSystemType', 'folder')) ~= "file"; + structure(j).IgnoreList = {}; + + report.add(here + ".excludePatterns", "Not mapped: NANSEN's ignore list matches literal substrings, not patterns. " + ... + "The level's expression already restricts the names, and hidden files are skipped.", ... + ~isempty(asList(level, 'excludePatterns'))) + report.add(here + ".pathComponentTemplate", "NANSEN generates folder names itself; the template is not used for writing.", ... + isfield(level, 'pathComponentTemplate')) + end +end + +function [metaDataDef, usedFields] = convertMetadata(mapping, definitions, levelNames, sessionIndex, ... + where, sessionEntity, subjectEntity, sessionKeys, subjectKeys, report) +%convertMetadata Map extraction rules onto NANSEN's four metadata variables + import nansen.config.dloc.dsmconversion.* + + metaDataDef = nansen.config.dloc.DataLocationModel.getDefaultMetadataStructure(); + usedFields = string.empty(1, 0); + + targets = struct( ... + 'Variable', {'Subject ID', 'Session ID', 'Experiment Date', 'Experiment Time'}, ... + 'Field', {identityField(subjectKeys, "subject", where, report), ... + identityField(sessionKeys, "session", where, report), ... + temporalField(definitions, ["date", "datetime"], sessionEntity, subjectEntity), ... + temporalField(definitions, "time", sessionEntity, subjectEntity)}); + + for k = 1:numel(targets) + field = targets(k).Field; + if field == ""; continue; end + usedFields(end+1) = field; %#ok + + rule = findRule(mapping, field); + here = where + ".metadataMapping[" + field + "]"; + if isempty(rule) + report.add(here, "No extraction rule for this field in this data location; " + targets(k).Variable + " is left unset.") + continue + end + extraction = rule.extraction; + + levelRef = getOr(extraction, 'entityLayoutLevel', []); + if isnumeric(levelRef) && ~isempty(levelRef) + levelIndex = levelRef + 1; + elseif ~isempty(levelRef) + levelIndex = find(levelNames == string(levelRef), 1); + else + levelIndex = []; + end + if isempty(levelIndex) || levelIndex > sessionIndex + report.add(here, "The rule does not read a single level at or above the session level; " + targets(k).Variable + " is left unset.") + continue + end + + method = string(extraction.method); + if method == "regex" + [input, isConverted] = convertRegexToWholeMatch(string(extraction.pattern)); + mode = "expr"; + elseif method == "substring" + [mode, input, isConverted] = convertSliceToExtraction(string(extraction.pattern)); + else + isConverted = false; + end + if ~isConverted + report.add(here, "The " + method + " rule can not be expressed as a NANSEN ind or expr rule; " + targets(k).Variable + " is left unset.") + continue + end + report.add(here + ".normalize", "Normalisation is not applied by NANSEN; extracted values may differ.", ... + string(getOr(extraction, 'normalize', 'none')) ~= "none") + + idx = strcmp({metaDataDef.VariableName}, targets(k).Variable); + metaDataDef(idx).SubfolderLevel = levelIndex; + metaDataDef(idx).StringDetectMode = char(mode); + metaDataDef(idx).StringDetectInput = char(input); + metaDataDef(idx).StringFormat = char(getOr(extraction, 'valueFormat', '')); + end +end + +function variables = convertFilePatterns(sessionLevel, item, where, sessionKeys, report) +%convertFilePatterns Map the session level's file patterns to variable model items + import nansen.config.dloc.dsmconversion.* + + variables = struct.empty; + here = where + ".entityLayout[" + string(sessionLevel.name) + "].filePatterns"; + patterns = asList(sessionLevel, 'filePatterns'); + + for j = 1:numel(patterns) + filePattern = patterns{j}; + if ~isfield(filePattern, 'name') + report.add(here, "An unnamed file pattern has no variable name to map to.") + continue + end + patternWhere = here + "[" + string(filePattern.name) + "]"; + + [expression, fileType, isConverted] = convertFilePatternToWildcard(string(filePattern.pattern), sessionKeys); + if ~isConverted + report.add(patternWhere, "The pattern has no wildcard equivalent.") + continue + end + + variable = nansen.config.varmodel.VariableModel.getBlankItem(); + variable.VariableName = char(matlab.lang.makeValidName(string(filePattern.name))); + variable.DataLocation = item.Name; + variable.DataLocationUuid = item.Uuid; + variable.FileNameExpression = char(expression); + variable.FileType = char(fileType); + variable.FileAdapter = findFileAdapter(fileType, patternWhere, report); + variable.IsCustom = true; + + report.add(patternWhere + ".isRequired", "NANSEN does not check that required files are present.", ... + isfield(filePattern, 'isRequired')) + variables = appendItems(variables, variable); + end +end + +function adapterName = findFileAdapter(fileType, where, report) +%findFileAdapter Name of a file adapter for a file type, or Default + adapterName = 'Default'; + adapters = struct.empty; + if fileType ~= "" + try + adapters = nansen.dataio.listFileAdapters(char(fileType)); + catch exception + % File adapters are listed from the current project, which + % needs a user session. Converting a model does not, so an + % absent session only means the adapter can not be looked up. + if ~strcmp(exception.identifier, 'NANSEN:NoActiveUserSession') + rethrow(exception) + end + end + end + if isempty(adapters) || strcmp(adapters(1).FileAdapterName, 'N/A') + report.add(where + ".fileAdapter", "No file adapter supports '" + fileType + ... + "' (or no project is active); set to Default, so the file can be found but not loaded.") + else + adapterName = adapters(1).FileAdapterName; + end +end + +% ------------------------------------------------------------------------- + +function name = chooseSessionEntity(entityTypes, locations) +%chooseSessionEntity An entity type named session with a level, else the deepest level's + names = fieldValues(entityTypes, 'name'); + levelTypes = string.empty(1, 0); + deepest = ""; + for i = 1:numel(locations) + if ~isfield(locations{i}, 'filesystemSource'); continue; end + layout = asList(locations{i}.filesystemSource, 'entityLayout'); + types = fieldValues(layout, 'entityType'); + levelTypes = [levelTypes, types]; %#ok + typed = types(types ~= ""); + if ~isempty(typed) && deepest == "" + deepest = typed(end); + end + end + if ismember("session", names) && ismember("session", levelTypes) + name = "session"; + else + name = deepest; + end +end + +function name = chooseSubjectEntity(entityTypes, relationships, sessionEntity) +%chooseSubjectEntity An entity type named subject, else the session entity's parent + names = fieldValues(entityTypes, 'name'); + name = ""; + if ismember("subject", names) && sessionEntity ~= "subject" + name = "subject"; + return + end + for i = 1:numel(relationships) + if string(relationships{i}.targetEntity) == sessionEntity ... + && ismember(string(relationships{i}.relationType), ["oneToMany", "oneToOne"]) + name = string(relationships{i}.sourceEntity); + return + end + end +end + +function keys = identityKeys(entityTypes, entityName) +%identityKeys Identity fields of an entity type + keys = string.empty(1, 0); + for i = 1:numel(entityTypes) + if string(entityTypes{i}.name) == entityName + if isfield(entityTypes{i}, 'identifierRef') + keys = string(entityTypes{i}.identifierRef); + else + keys = reshape(string(entityTypes{i}.identifierRefs), 1, []); + end + end + end +end + +function field = identityField(keys, role, where, report) +%identityField The single identity field, or "" with a report for a composite one + field = ""; + if isscalar(keys) + field = keys; + elseif numel(keys) > 1 + report.add(where, "The " + role + " identity is composite (" + strjoin(keys, ", ") + "); NANSEN needs a single id field.") + end +end + +function field = temporalField(definitions, dataTypes, sessionEntity, subjectEntity) +%temporalField First field of a temporal data type, preferring the session entity + field = ""; + for owner = [sessionEntity, subjectEntity] + for name = string(fieldnames(definitions))' + definition = definitions.(name); + if ismember(string(definition.dataType), dataTypes) && string(definition.ofEntity) == owner + field = name; + return + end + end + end +end + +function rule = findRule(mapping, field) +%findRule The extraction rule for a field, or [] + rule = []; + for i = 1:numel(mapping) + if string(mapping{i}.metadataRef) == field + rule = mapping{i}; + return + end + end +end + +function pattern = templateToPattern(template, definitions) +%templateToPattern Derive a name pattern from a path component template + [tokens, literals] = regexp(char(template), '\{([A-Za-z_][A-Za-z0-9_]*)\}', 'tokens', 'split'); + pattern = "^" + regexptranslate('escape', literals{1}); + for i = 1:numel(tokens) + tokenPattern = "[^/\\]+"; + if isfield(definitions, tokens{i}{1}) && isfield(definitions.(tokens{i}{1}), 'validation') ... + && isfield(definitions.(tokens{i}{1}).validation, 'pattern') + tokenPattern = "(?:" + regexprep(string(definitions.(tokens{i}{1}).validation.pattern), '^\^|\$$', '') + ")"; + end + pattern = pattern + tokenPattern + regexptranslate('escape', literals{i+1}); + end + pattern = pattern + "$"; +end + +function values = fieldValues(items, fieldName) +%fieldValues One field of each item in a list, as a string row; "" where absent + values = strings(1, numel(items)); + for i = 1:numel(items) + values(i) = string(getOr(items{i}, fieldName, "")); + end +end + +function list = appendItems(list, items) +%appendItems Append struct items to a struct array that may still be empty + if isempty(items) + return + elseif isempty(list) + list = items; + else + list = [list, items]; + end +end + +function items = asList(s, fieldName) +%asList A json array field as a row cell array of scalar structs or values + items = {}; + if ~isstruct(s) || ~isfield(s, fieldName) || isempty(s.(fieldName)); return; end + value = s.(fieldName); + if iscell(value) + items = reshape(value, 1, []); + elseif isstruct(value) + items = num2cell(reshape(value, 1, [])); + else + items = {value}; + end +end + +function value = getOr(s, fieldName, default) +%getOr A struct field, or a default when it is absent or empty + if isstruct(s) && isfield(s, fieldName) && ~isempty(s.(fieldName)) + value = s.(fieldName); + else + value = default; + end +end diff --git a/code/+nansen/+config/+dloc/importDatasetStructureModel.m b/code/+nansen/+config/+dloc/importDatasetStructureModel.m new file mode 100644 index 00000000..7b0f61da --- /dev/null +++ b/code/+nansen/+config/+dloc/importDatasetStructureModel.m @@ -0,0 +1,57 @@ +function report = importDatasetStructureModel(dsmConfig, dataLocationModel, variableModel, options) +%importDatasetStructureModel Add a Dataset Structure Model's data locations and variables to a project +% +% Syntax: +% report = nansen.config.dloc.importDatasetStructureModel( ... +% dsmConfig, dataLocationModel, variableModel) +% +% report = nansen.config.dloc.importDatasetStructureModel(..., ... +% SessionEntity=name, SubjectEntity=name) +% +% Converts the model with dsm2DataLocationModel, adds the data locations +% to dataLocationModel and the variables to variableModel, and returns +% the report of what was not mapped. +% +% Adding a data location assigns it a new uuid, so each variable is +% relinked to the uuid its data location was given. +% +% Adding an item whose name is already in a model raises an error, so +% importing the same model twice into one project fails on the second +% attempt rather than creating duplicates. +% +% Example: +% project = nansen.getCurrentProject(); +% report = nansen.config.dloc.importDatasetStructureModel( ... +% "garad-2022.json", project.DataLocationModel, project.VariableModel); +% disp(report) +% +% See also nansen.config.dloc.dsm2DataLocationModel + + arguments + dsmConfig + dataLocationModel (1,1) nansen.config.dloc.DataLocationModel + variableModel (1,1) nansen.config.varmodel.VariableModel + options.SessionEntity (1,1) string = missing + options.SubjectEntity (1,1) string = missing + end + + nameValueArgs = namedargs2cell(options); + [dataLocations, variables, report] = ... + nansen.config.dloc.dsm2DataLocationModel(dsmConfig, nameValueArgs{:}); + + for i = 1:numel(dataLocations) + placeholderUuid = dataLocations(i).Uuid; + dataLocationModel.addDataLocation(dataLocations(i)); + assignedUuid = dataLocationModel.getDataLocation(dataLocations(i).Name).Uuid; + + for j = 1:numel(variables) + if strcmp(variables(j).DataLocationUuid, placeholderUuid) + variables(j).DataLocationUuid = assignedUuid; + end + end + end + + for j = 1:numel(variables) + variableModel.insertItem(variables(j)); + end +end diff --git a/tests/+nansen/+fixture/datasetstructure/garad-2022.json b/tests/+nansen/+fixture/datasetstructure/garad-2022.json new file mode 100644 index 00000000..ffb7decb --- /dev/null +++ b/tests/+nansen/+fixture/datasetstructure/garad-2022.json @@ -0,0 +1,153 @@ +{ + "schemaVersion": "1.0.0", + "entityTypes": [ + { + "name": "cell", + "isPrimary": true, + "identifierRef": "cell_id", + "description": "One whole-cell patched CA1 pyramidal neuron; one neuron per slice per day. The descriptor reports 7 cells from 2 mice on 2 days. Cells have no folder; identity is read from the recording file names." + }, + { + "name": "recording", + "identifierRef": "recording_id", + "description": "One ABF file: one repetition of the excitability protocol (0-450 pA, 50 pA steps, 500 ms) for a cell. Five repetitions (a-e) per cell." + } + ], + "metadataDefinitions": { + "cell_id": { + "name": "cell_id", + "title": "Cell ID", + "description": "Recording date (yyMMdd) and slice number on that day, e.g. 170518_1. One neuron was recorded per slice.", + "dataType": "string", + "ofEntity": "cell", + "validation": { + "pattern": "^\\d{6}_\\d+$" + } + }, + "recording_date": { + "name": "recording_date", + "title": "Recording date", + "dataType": "date", + "ofEntity": "cell", + "description": "Day of recording, from the yyMMdd prefix of the file name" + }, + "recording_id": { + "name": "recording_id", + "title": "Recording ID", + "description": "File name without extension, e.g. 170518_1a", + "dataType": "string", + "ofEntity": "recording", + "validation": { + "pattern": "^\\d{6}_\\d+[a-z]$" + } + }, + "slice_number": { + "name": "slice_number", + "title": "Slice number", + "description": "Number of the slice recorded on that day from the same animal (data descriptor naming convention)", + "dataType": "integer", + "ofEntity": "cell" + }, + "protocol_repeat": { + "name": "protocol_repeat", + "title": "Protocol repeat", + "description": "Letter a-e: which of the five repetitions of the excitability protocol this file holds", + "dataType": "string", + "ofEntity": "recording", + "validation": { + "pattern": "^[a-e]$" + } + } + }, + "dataLocations": [ + { + "identifier": "garad", + "displayName": "Garad & Lessmann 2022, CA1 excitability ABF recordings", + "description": "Flat folder of Axon Binary Format files named _.ABF, plus the EBRAINS data descriptor and licence PDFs. Category 'raw' per the descriptor (original Patchmaster recordings). Every entry in the folder was listed; nothing was sampled.", + "dataCategory": "raw", + "access": "read", + "sourceType": "filesystem", + "filesystemSource": { + "rootStoragePaths": [ + { + "identifier": "main", + "path": "/Users/eivihe/Data/ShareBrain/InDepthCuration/garad-2022", + "storageType": "local" + } + ], + "entityLayout": [ + { + "name": "recording-files", + "entityType": "recording", + "fileSystemType": "file", + "matchPattern": "^\\d{6}_\\d+[a-z]\\.ABF$", + "excludePatterns": [ + "^\\..*", + "^EBRAINS-datadescriptor_.*", + "^Licence-CC-BY\\.pdf$" + ], + "filePatterns": [ + { + "name": "abf", + "pattern": "^{recording_id}\\.ABF$", + "isRequired": true, + "cardinality": "one" + } + ] + } + ], + "metadataMapping": [ + { + "metadataRef": "recording_id", + "extraction": { + "method": "regex", + "pattern": "^(\\d{6}_\\d+[a-z])\\.ABF$", + "entityLayoutLevel": "recording-files" + } + }, + { + "metadataRef": "protocol_repeat", + "extraction": { + "method": "regex", + "pattern": "^\\d{6}_\\d+([a-z])\\.ABF$", + "entityLayoutLevel": "recording-files" + } + }, + { + "metadataRef": "cell_id", + "extraction": { + "method": "regex", + "pattern": "^(\\d{6}_\\d+)[a-z]\\.ABF$", + "entityLayoutLevel": "recording-files" + } + }, + { + "metadataRef": "slice_number", + "extraction": { + "method": "regex", + "pattern": "^\\d{6}_(\\d+)[a-z]\\.ABF$", + "entityLayoutLevel": "recording-files" + } + }, + { + "metadataRef": "recording_date", + "extraction": { + "method": "substring", + "pattern": "0:6", + "valueFormat": "yyMMdd", + "entityLayoutLevel": "recording-files" + } + } + ] + } + } + ], + "entityRelationships": [ + { + "sourceEntity": "cell", + "targetEntity": "recording", + "relationType": "oneToMany", + "relationName": "hasRecordings" + } + ] +} diff --git a/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m b/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m new file mode 100644 index 00000000..1664fda9 --- /dev/null +++ b/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m @@ -0,0 +1,150 @@ +classdef ImportDatasetStructureModelTest < matlab.unittest.TestCase + %ImportDatasetStructureModelTest Import a Dataset Structure Model and detect sessions with NANSEN + % + % Imports the model written for Garad and Lessmann (2022) into data + % location and variable models, then asks NANSEN's own session + % detection what it finds in a folder laid out like that dataset. The + % folder is synthetic: it follows the naming convention in the model, + % _.ABF, with two cells of five repeats, plus + % the descriptor, licence and a hidden file that must be skipped. + % + % Run tests: + % runtests('nansen.integrationtest.ImportDatasetStructureModelTest') + + properties + DataFolder char + DataLocationModel + VariableModel + Report + end + + methods (TestClassSetup) + function setupProject(testCase) + testCase.applyFixture(nansen.fixture.ProjectFixture) + end + end + + methods (TestMethodSetup) + function importGarad(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + root = testCase.applyFixture(TemporaryFolderFixture).Folder; + + testCase.DataFolder = fullfile(root, 'garad-2022'); + mkdir(testCase.DataFolder) + fileNames = [ ... + compose("170518_1%s.ABF", ["a", "b", "c", "d", "e"]), ... + compose("170519_2%s.ABF", ["a", "b", "c", "d", "e"]), ... + "EBRAINS-datadescriptor_garad.pdf", "Licence-CC-BY.pdf", ".DS_Store"]; + for fileName = fileNames + fclose(fopen(fullfile(testCase.DataFolder, fileName), 'w')); + end + + fixture = fullfile(fileparts(fileparts(mfilename('fullpath'))), ... + '+fixture', 'datasetstructure', 'garad-2022.json'); + dsm = jsondecode(fileread(fixture)); + dsm.dataLocations.filesystemSource.rootStoragePaths.path = testCase.DataFolder; + + configFolder = fullfile(root, 'configurations'); + mkdir(configFolder) + dataLocationPath = fullfile(configFolder, 'datalocation_settings.json'); + variablePath = fullfile(configFolder, 'filepath_settings.json'); + + testCase.Report = nansen.config.dloc.importDatasetStructureModel(dsm, ... + nansen.config.dloc.DataLocationModel(dataLocationPath), ... + nansen.config.varmodel.VariableModel(variablePath)); + + % Read both back, as a project would on its next start + testCase.DataLocationModel = nansen.config.dloc.DataLocationModel(dataLocationPath); + testCase.VariableModel = nansen.config.varmodel.VariableModel(variablePath); + end + end + + methods (Access = private) + function paths = sessionPaths(testCase) + sessionFolders = nansen.dataio.session.listSessionFolders( ... + testCase.DataLocationModel, 'garad'); + paths = reshape(sessionFolders.garad, 1, []); + end + end + + methods (Test) + + function testImportedModelsLoadFromDisk(testCase) + % The only data location is read-only, so there is no default. + % Loading such a model used to raise. + testCase.verifyEqual(testCase.DataLocationModel.DataLocationNames, {'garad'}) + testCase.verifyEmpty(testCase.DataLocationModel.DefaultDataLocation) + testCase.verifyEqual(testCase.VariableModel.getItem('abf').FileType, '.ABF') + end + + function testEachAbfFileIsASession(testCase) + paths = testCase.sessionPaths(); + + testCase.verifyNumElements(paths, 10, ... + 'The descriptor, licence and hidden file must not be sessions.') + + [~, names, extensions] = fileparts(paths); + testCase.verifyEqual(unique(extensions), {'.ABF'}) + testCase.verifyEqual(sort(names), ... + sort(cellstr([compose("170518_1%s", ["a" "b" "c" "d" "e"]), ... + compose("170519_2%s", ["a" "b" "c" "d" "e"])]))) + end + + function testSessionAndSubjectIdsComeFromTheFileNames(testCase) + model = testCase.DataLocationModel; + index = model.getItemIndex('garad'); + paths = testCase.sessionPaths(); + + sessionIds = cellfun(@(p) model.getSessionID(p, index), paths, 'UniformOutput', false); + subjectIds = cellfun(@(p) model.getSubjectID(p, index), paths, 'UniformOutput', false); + + [~, names] = fileparts(paths); + testCase.verifyEqual(sessionIds, names, ... + 'The session id is the file name without its extension.') + testCase.verifyEqual(unique(subjectIds), {'170518_1', '170519_2'}, ... + 'The subject is the cell: the date and slice number.') + end + + function testExperimentDateIsParsed(testCase) + model = testCase.DataLocationModel; + paths = testCase.sessionPaths(); + firstPath = paths{contains(paths, '170518_1a')}; + + experimentDate = model.getDate(firstPath, model.getItemIndex('garad')); + + testCase.verifyEqual([year(experimentDate), month(experimentDate), day(experimentDate)], ... + [2017, 5, 18]) + end + + function testSessionsAreMatchedOnePerFile(testCase) + sessionFolders = nansen.dataio.session.listSessionFolders(testCase.DataLocationModel, 'all'); + matched = nansen.dataio.session.matchSessionFolders(testCase.DataLocationModel, sessionFolders); + + testCase.verifyNumElements(matched, 10, ... + 'Grouping by session id must not merge the repeats of a cell.') + end + + function testVariableIsLinkedToItsDataLocation(testCase) + variable = testCase.VariableModel.getItem('abf'); + dataLocation = testCase.DataLocationModel.getDataLocation('garad'); + + testCase.verifyEqual(variable.DataLocationUuid, dataLocation.Uuid, ... + 'Adding the data location assigns a new uuid; the variable must follow it.') + end + + function testVariableFindsTheSessionsFile(testCase) + % A session stored as a file shares its folder with every other + % session. NANSEN narrows the folder to the files containing + % the session id, as done here. + fileName = testCase.VariableModel.lookForFile(testCase.DataFolder, 'abf', ... + 'FilterFcn', @(names) contains(names, '170519_2c')); + + testCase.verifyEqual(fileName, '170519_2c.ABF') + end + + function testReportIsReturned(testCase) + testCase.verifyClass(testCase.Report, 'table') + testCase.verifyTrue(any(contains(testCase.Report.Element, "slice_number"))) + end + end +end diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m new file mode 100644 index 00000000..36f13e01 --- /dev/null +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m @@ -0,0 +1,197 @@ +classdef Dsm2DataLocationModelTest < matlab.unittest.TestCase + %Dsm2DataLocationModelTest Converting a Dataset Structure Model to NANSEN items + % + % Uses the model written for Garad and Lessmann (2022): one flat folder + % of ABF files named _.ABF, a cell entity with no + % folder of its own, and a recording entity per file. + % + % Extraction rules are checked by running them through NANSEN's own + % extraction, so the tests assert the values NANSEN will produce rather + % than the text of the rules. + % + % Run tests: + % runtests('nansen.unittest.config.Dsm2DataLocationModelTest') + + properties (Constant) + FileName = '170518_1a.ABF' + end + + properties + DataLocations + Variables + Report + end + + properties (TestParameter) + regexCase = { ... + {'^(\d{6}_\d+[a-z])\.ABF$', '170518_1a'}, ... + {'^(\d{6}_\d+)[a-z]\.ABF$', '170518_1'}, ... + {'^\d{6}_(\d+)[a-z]\.ABF$', '1'}, ... + {'^.*_(\d+)[a-z]\.ABF$', '1'}, ... + {'\.(ABF|abf)$', 'ABF'}, ... + {'^\d{6}_\d+[a-z]', '170518_1a'}} + sliceCase = { ... + {'0:6', '170518'}, {':', '170518_1a.ABF'}, {'9:', '.ABF'}, ... + {':-4', '170518_1a'}, {'7:-4', '1a'}, {'-3:', 'ABF'}, {'-4:-3', '.'}} + end + + methods (TestClassSetup) + function convertGarad(testCase) + fixture = fullfile(fileparts(fileparts(fileparts(mfilename('fullpath')))), ... + '+fixture', 'datasetstructure', 'garad-2022.json'); + [testCase.DataLocations, testCase.Variables, testCase.Report] = ... + nansen.config.dloc.dsm2DataLocationModel(fixture); + end + end + + methods (Access = private) + function value = extract(testCase, variableName) + %extract Run a converted metadata rule on the example file name + rule = testCase.DataLocations.MetaDataDef( ... + strcmp({testCase.DataLocations.MetaDataDef.VariableName}, variableName)); + value = nansen.config.dloc.DataLocationModel.applyExtractionPattern( ... + testCase.FileName, rule.StringDetectMode, rule.StringDetectInput); + end + + function verifyReported(testCase, elementPart) + testCase.verifyTrue(any(contains(testCase.Report.Element, elementPart)), ... + sprintf('Expected the report to mention %s.', elementPart)) + end + + function verifyNotReported(testCase, elementPart) + testCase.verifyFalse(any(contains(testCase.Report.Element, elementPart)), ... + sprintf('%s was mapped and should not be in the report.', elementPart)) + end + end + + methods (Test) + + function testOneReadOnlyLocationOfFiles(testCase) + dataLocation = testCase.DataLocations; + testCase.verifyNumElements(dataLocation, 1) + testCase.verifyEqual(dataLocation.Name, 'garad') + testCase.verifyEqual(dataLocation.Type.Name, 'recorded') + + level = dataLocation.SubfolderStructure; + testCase.verifyNumElements(level, 1) + testCase.verifyEqual(level.Type, 'Session') + testCase.verifyFalse(level.IsFolder, 'Each recording is a file, not a folder.') + testCase.verifyEqual(level.Expression, '^\d{6}_\d+[a-z]\.ABF$') + testCase.verifyEqual(level.IgnoreList, {}, ... + 'Exclude patterns must not become substring ignores; "." would drop every file.') + + testCase.verifyEqual(dataLocation.RootPath.Value, ... + '/Users/eivihe/Data/ShareBrain/InDepthCuration/garad-2022') + testCase.verifyEqual(dataLocation.RootPath.DiskType, 'Local') + end + + function testRecordingIsTheSessionAndCellTheSubject(testCase) + testCase.verifyEqual(testCase.extract('Session ID'), '170518_1a') + testCase.verifyEqual(testCase.extract('Subject ID'), '170518_1') + end + + function testRecordingDateUsesItsFormat(testCase) + rule = testCase.DataLocations.MetaDataDef( ... + strcmp({testCase.DataLocations.MetaDataDef.VariableName}, 'Experiment Date')); + testCase.verifyEqual(rule.StringDetectMode, 'ind') + testCase.verifyEqual(rule.StringFormat, 'yyMMdd') + testCase.verifyEqual(testCase.extract('Experiment Date'), '170518') + end + + function testAbsentTimeIsLeftUnset(testCase) + rule = testCase.DataLocations.MetaDataDef( ... + strcmp({testCase.DataLocations.MetaDataDef.VariableName}, 'Experiment Time')); + testCase.verifyEmpty(rule.SubfolderLevel) + end + + function testAbfFileBecomesAVariable(testCase) + variable = testCase.Variables; + testCase.verifyNumElements(variable, 1) + testCase.verifyEqual(variable.VariableName, 'abf') + testCase.verifyEqual(variable.FileNameExpression, '^*.ABF$') + testCase.verifyEqual(variable.FileType, '.ABF') + testCase.verifyEqual(variable.DataLocation, 'garad') + testCase.verifyEqual(variable.DataLocationUuid, testCase.DataLocations.Uuid) + testCase.verifyEqual(variable.FileAdapter, 'Default', ... + 'NANSEN has no ABF file adapter.') + end + + function testReportListsWhatWasNotMapped(testCase) + testCase.verifyReported("metadataDefinitions[slice_number]") + testCase.verifyReported("metadataDefinitions[protocol_repeat]") + testCase.verifyReported("entityRelationships") + testCase.verifyReported("excludePatterns") + testCase.verifyReported("filePatterns[abf].isRequired") + testCase.verifyReported("filePatterns[abf].fileAdapter") + end + + function testReportLeavesOutWhatWasMapped(testCase) + testCase.verifyNotReported("metadataDefinitions[cell_id]") + testCase.verifyNotReported("metadataDefinitions[recording_id]") + testCase.verifyNotReported("metadataDefinitions[recording_date]") + testCase.verifyNotReported("dataCategory") + testCase.verifyNotReported("entityTypes[") + end + + function testSessionEntityWithoutALevelIsReported(testCase) + % A cell has no folder or file of its own, so NANSEN could not + % find it on disk as a session. + fixture = fullfile(fileparts(fileparts(fileparts(mfilename('fullpath')))), ... + '+fixture', 'datasetstructure', 'garad-2022.json'); + [dataLocations, ~, report] = nansen.config.dloc.dsm2DataLocationModel( ... + fixture, SessionEntity="cell"); + + testCase.verifyEmpty(dataLocations) + testCase.verifyTrue(any(contains(report.Reason, "session entity 'cell'"))) + end + + function testRegexBecomesWholeMatch(testCase, regexCase) + [pattern, isConverted] = ... + nansen.config.dloc.dsmconversion.convertRegexToWholeMatch(regexCase{1}); + testCase.assertTrue(isConverted) + value = nansen.config.dloc.DataLocationModel.applyExtractionPattern( ... + testCase.FileName, 'expr', char(pattern)); + testCase.verifyEqual(value, regexCase{2}) + end + + function testQuantifiedGroupIsNotConverted(testCase) + [~, isConverted] = ... + nansen.config.dloc.dsmconversion.convertRegexToWholeMatch('^(\d)+_'); + testCase.verifyFalse(isConverted) + end + + function testSliceGivesThePythonValue(testCase, sliceCase) + [mode, input, isConverted] = ... + nansen.config.dloc.dsmconversion.convertSliceToExtraction(sliceCase{1}); + testCase.assertTrue(isConverted) + value = nansen.config.dloc.DataLocationModel.applyExtractionPattern( ... + testCase.FileName, char(mode), char(input)); + testCase.verifyEqual(value, sliceCase{2}) + end + + function testSliceNeverUsesEndOutsideTheWholeRange(testCase) + % NANSEN's ind mode accepts end only as 1:end. + for slice = ["0:6", ":", "9:", ":-4", "7:-4", "-3:"] + [mode, input] = nansen.config.dloc.dsmconversion.convertSliceToExtraction(slice); + if mode == "ind" + testCase.verifyTrue(~contains(input, "end") || input == "1:end", slice) + end + end + end + + function testFilePatternsBecomeWildcards(testCase) + convert = @(p) nansen.config.dloc.dsmconversion.convertFilePatternToWildcard(p, "recording_id"); + + [expression, fileType, isConverted] = convert("^{recording_id}_meta\.json$"); + testCase.verifyTrue(isConverted) + testCase.verifyEqual(expression, "^*_meta.json$") + testCase.verifyEqual(fileType, ".json") + + [~, ~, isConverted] = convert("^{other_id}\.ABF$"); + testCase.verifyFalse(isConverted, 'Only tokens naming the session identity become *.') + + [~, ~, isConverted] = convert("^\d+\.ABF$"); + testCase.verifyFalse(isConverted, '\d has no wildcard equivalent.') + end + end +end