diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m index 246f6f44..a77db4b2 100644 --- a/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m @@ -1,45 +1,63 @@ -function [fileNameExpression, fileType, isConverted] = convertFilePatternToWildcard(dsmPattern, sessionTokens) +function [fileNameExpression, fileType, isConverted, isApproximate] = convertFilePatternToWildcard(dsmPattern, identityTokens) %convertFilePatternToWildcard Convert a file pattern regex to a NANSEN file name expression % % Syntax: -% [fileNameExpression, fileType, isConverted] = ... -% convertFilePatternToWildcard(dsmPattern, sessionTokens) +% [fileNameExpression, fileType, isConverted, isApproximate] = ... +% convertFilePatternToWildcard(dsmPattern, identityTokens) % % A Dataset Structure Model file pattern is a regular expression that may % contain {token} references to an entity's metadata. NANSEN's -% FileNameExpression is a wildcard pattern anchored with ^ and $, applied -% in the session folder. NANSEN already restricts a virtual session -% folder to the files that contain the session id, so tokens naming the -% session's identity become *: +% FileNameExpression is a wildcard pattern anchored with ^ and $, which +% NANSEN passes to dir, so * is its only wildcard. NANSEN already +% restricts a virtual session folder to the files that contain the +% session id, and the files of a session belong to one subject, so tokens +% naming the identity of the session or of an ancestor become *: % % ^{recording_id}\.ABF$ -> ^*.ABF$ with file type .ABF % -% Escaped punctuation becomes literal and .* becomes *. Any other regex -% construct, or a token that does not name the session identity, has no -% wildcard equivalent; isConverted is then false. +% An extension of several parts gives a file type of several parts: +% \.nii\.gz$ has the file type .nii.gz. +% +% Escaped punctuation becomes literal and .* becomes *. A character class +% (\d, \w, \s, [...]) or a . with its quantifier also becomes *. The +% wildcard then matches more names than the pattern, and isApproximate +% is true: +% +% ^TT\d+\.ntt$ -> ^TT*.ntt$ +% +% An approximation that keeps no literal text but dots would match every +% file in the folder (^{session_id}\.\d$ would become ^*.*$), so it is +% not converted. Alternation, groups, quantified literals and tokens that +% name other fields have no wildcard equivalent either; isConverted is +% then false. % % Input arguments: -% dsmPattern - The file pattern from the Dataset Structure Model. -% sessionTokens - Names of the metadata fields that identify a session. +% dsmPattern - The file pattern from the Dataset Structure Model. +% identityTokens - Names of the metadata fields that identify the +% session or one of its ancestors. arguments dsmPattern (1,1) string - sessionTokens (1,:) string + identityTokens (1,:) string end fileNameExpression = ""; fileType = ""; isConverted = false; + isApproximate = false; - extension = regexp(dsmPattern, "\\\.([A-Za-z0-9]+)\$$", "tokens", "once"); + % An extension may have several parts, as .nii.gz. Each part starts + % with a letter, so that a version number before the extension, as in + % v1\.2\.txt, is not taken for part of it. + extension = regexp(dsmPattern, "((?:\\\.[A-Za-z][A-Za-z0-9]*)+)\$$", "tokens", "once"); if ~isempty(extension) - fileType = "." + extension{1}; + fileType = string(erase(extension{1}, "\")); end tokens = regexp(dsmPattern, "\{([A-Za-z_][A-Za-z0-9_]*)\}", "tokens"); pattern = dsmPattern; for i = 1:numel(tokens) - if ~ismember(tokens{i}{1}, sessionTokens) + if ~ismember(tokens{i}{1}, identityTokens) return end pattern = replace(pattern, "{" + tokens{i}{1} + "}", "*"); @@ -51,16 +69,32 @@ while i <= numel(pattern) c = pattern(i); if c == '\' - if i == numel(pattern) || isstrprop(pattern(i+1), 'alphanum') - return % \d, \w and the like have no wildcard equivalent + if i == numel(pattern) + return + end + escaped = pattern(i+1); + if any(escaped == 'dws') + [wildcard, i] = appendAnyText(wildcard, pattern, i + 2); + isApproximate = true; + elseif isstrprop(escaped, 'alphanum') + return % \b, \D and the like have no wildcard equivalent + else + wildcard(end+1) = escaped; %#ok + i = i + 2; end - wildcard(end+1) = pattern(i+1); %#ok - i = i + 2; - elseif c == '.' && i < numel(pattern) && pattern(i+1) == '*' - wildcard(end+1) = '*'; %#ok - i = i + 2; + elseif c == '[' + closing = find(pattern(i+1:end) == ']', 1); + if isempty(closing) + return + end + [wildcard, i] = appendAnyText(wildcard, pattern, i + closing + 1); + isApproximate = true; + elseif c == '.' + % .* is any text, as * is; any other . is approximated + isApproximate = isApproximate || i == numel(pattern) || pattern(i+1) ~= '*'; + [wildcard, i] = appendAnyText(wildcard, pattern, i + 1); elseif c == '*' || (c == '^' && i == 1) || (c == '$' && i == numel(pattern)) ... - || ~any(c == '.[](){}+?|^$') + || ~any(c == '[](){}+?|^$') wildcard(end+1) = c; %#ok i = i + 1; else @@ -68,6 +102,28 @@ end end + if isApproximate && all(ismember(wildcard, '^$*.')) + isApproximate = false; + return % the wildcard would match every file in the folder + end + fileNameExpression = string(wildcard); isConverted = true; end + +function [wildcard, next] = appendAnyText(wildcard, pattern, next) +%appendAnyText Append * for one pattern element, skipping the quantifier that follows it + if next <= numel(pattern) + if any(pattern(next) == '*+?') + next = next + 1; + elseif pattern(next) == '{' + closing = find(pattern(next+1:end) == '}', 1); + if ~isempty(closing) + next = next + closing + 1; + end + end + end + if isempty(wildcard) || wildcard(end) ~= '*' + wildcard(end+1) = '*'; + end +end diff --git a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m index c36976fe..b76dec2f 100644 --- a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m +++ b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m @@ -90,7 +90,7 @@ continue end - [item, sessionLevel, usedHere] = convertLocation(location, where, definitions, ... + [item, sessionLevel, usedHere, levelsBelowSession] = convertLocation(location, where, definitions, ... sessionEntity, subjectEntity, sessionKeys, subjectKeys, report); if isempty(item) continue @@ -98,8 +98,12 @@ usedFields = [usedFields, usedHere]; %#ok dataLocations = appendItems(dataLocations, item); - variables = appendItems(variables, convertFilePatterns(sessionLevel, item, where, sessionKeys, report)); + variables = appendItems(variables, convertFilePatterns(sessionLevel, item, where, ... + [sessionKeys, subjectKeys], report)); + variables = appendItems(variables, convertFilePatternsBelowSession(levelsBelowSession, ... + item, where, [sessionKeys, subjectKeys], report)); end + variables = qualifyRepeatedVariableNames(variables, report); for fieldName = string(fieldnames(definitions))' if ~ismember(fieldName, usedFields) @@ -113,13 +117,18 @@ % ------------------------------------------------------------------------- -function [item, sessionLevel, usedFields] = convertLocation(location, where, definitions, ... +function [item, sessionLevel, usedFields, levelsBelowSession] = convertLocation(location, where, definitions, ... sessionEntity, subjectEntity, sessionKeys, subjectKeys, report) %convertLocation Convert one filesystem data location +% +% levelsBelowSession are the layout levels under the session level. NANSEN +% walks folders down to the session only; convertFilePatternsBelowSession +% finds the files below it that a variable can still point to. item = []; sessionLevel = []; usedFields = string.empty(1, 0); + levelsBelowSession = {}; source = location.filesystemSource; layout = asList(source, 'entityLayout'); @@ -129,10 +138,7 @@ report.add(where, "No layout level holds the session entity '" + sessionEntity + "', so NANSEN could not find sessions here.") return end - for j = sessionIndex+1:numel(layout) - report.add(where + ".entityLayout[" + string(layout{j}.name) + "]", ... - "NANSEN treats the deepest level it walks as the session; levels below the session level are dropped.") - end + levelsBelowSession = layout(sessionIndex+1:end); layout = layout(1:sessionIndex); sessionLevel = layout{end}; @@ -150,9 +156,8 @@ item.RootPath = convertRootPaths(asList(source, 'rootStoragePaths'), where, report); item.SubfolderStructure = convertLayout(layout, where, definitions, sessionEntity, subjectEntity, report); - levelNames = fieldValues(asList(source, 'entityLayout'), 'name'); [item.MetaDataDef, usedFields] = convertMetadata(asList(source, 'metadataMapping'), ... - definitions, levelNames, sessionIndex, where, sessionEntity, subjectEntity, ... + definitions, asList(source, 'entityLayout'), sessionIndex, where, sessionEntity, subjectEntity, ... sessionKeys, subjectKeys, report); end @@ -238,18 +243,32 @@ end end -function [metaDataDef, usedFields] = convertMetadata(mapping, definitions, levelNames, sessionIndex, ... +function [metaDataDef, usedFields] = convertMetadata(mapping, definitions, layout, sessionIndex, ... where, sessionEntity, subjectEntity, sessionKeys, subjectKeys, report) %convertMetadata Map extraction rules onto NANSEN's four metadata variables import nansen.config.dloc.dsmconversion.* metaDataDef = nansen.config.dloc.DataLocationModel.getDefaultMetadataStructure(); usedFields = string.empty(1, 0); + levelNames = fieldValues(layout, 'name'); + + % A composite session identity becomes a Session ID read from the + % joined names of several levels; a single identity field is mapped + % by the loop below like the other variables. + % compositeIdentityLevels reports why when the parts cannot be joined. + compositeLevels = []; + sessionField = ""; + if numel(sessionKeys) > 1 + compositeLevels = compositeIdentityLevels(mapping, sessionKeys, layout, sessionIndex, ... + sessionEntity, where, report); + else + sessionField = identityField(sessionKeys, "session", where, report); + end targets = struct( ... 'Variable', {'Subject ID', 'Session ID', 'Experiment Date', 'Experiment Time'}, ... 'Field', {identityField(subjectKeys, "subject", where, report), ... - identityField(sessionKeys, "session", where, report), ... + sessionField, ... temporalField(definitions, ["date", "datetime"], sessionEntity, subjectEntity), ... temporalField(definitions, "time", sessionEntity, subjectEntity)}); @@ -266,14 +285,7 @@ end extraction = rule.extraction; - levelRef = getOr(extraction, 'entityLayoutLevel', []); - if isnumeric(levelRef) && ~isempty(levelRef) - levelIndex = levelRef + 1; - elseif ~isempty(levelRef) - levelIndex = find(levelNames == string(levelRef), 1); - else - levelIndex = []; - end + levelIndex = levelIndexOf(extraction, levelNames); if isempty(levelIndex) || levelIndex > sessionIndex report.add(here, "The rule does not read a single level at or above the session level; " + targets(k).Variable + " is left unset.") continue @@ -301,12 +313,140 @@ metaDataDef(idx).StringDetectInput = char(input); metaDataDef(idx).StringFormat = char(getOr(extraction, 'valueFormat', '')); end + + if ~isempty(compositeLevels) + idx = strcmp({metaDataDef.VariableName}, 'Session ID'); + metaDataDef(idx).SubfolderLevel = compositeLevels; + metaDataDef(idx).StringDetectMode = 'expr'; + metaDataDef(idx).StringDetectInput = '.+'; + metaDataDef(idx).Separator = '_'; + usedFields = [usedFields, sessionKeys]; + report.add(where + ".metadataMapping[" + strjoin(sessionKeys, ", ") + "]", ... + "The session identity is composite; the Session ID joins the names of levels " + ... + strjoin(levelNames(compositeLevels), ", ") + " with '_', so it holds whole folder names rather than the extracted values.") + end +end + +function levels = compositeIdentityLevels(mapping, keys, layout, sessionIndex, sessionEntity, where, report) +%compositeIdentityLevels Levels whose joined names identify a session with a composite identity +% +% NANSEN reads a Session ID from one string. It can join the names of +% several levels into that string, but it applies one rule to the joined +% string, so the parts of a composite identity cannot each be extracted. +% The whole names of the levels the parts are read from are used +% instead: every part is read from the name of its level, so sessions +% with the same level names have the same identity. +% +% The levels of the session's ancestor entities are joined too. A model's +% identity is unique within its parent entities, while a NANSEN Session ID +% is unique within the project; without the ancestors, day 1 of two +% subjects would be one session. +% +% A part given by a fixed rule is not in the path and is left out. +% Returns empty, with a report, when a part is read otherwise than from +% one level at or above the session level. + + levels = []; + levelNames = fieldValues(layout, 'name'); + levelTypes = fieldValues(layout, 'entityType'); + + for key = keys + here = where + ".metadataMapping[" + key + "]"; + rule = findRule(mapping, key); + if isempty(rule) + report.add(here, "No extraction rule for this part of the composite session identity; the Session ID is left unset.") + levels = []; + return + end + method = string(rule.extraction.method); + if method == "fixed" + report.add(here, "A fixed part of the session identity is not in the path; the Session ID leaves it out.") + continue + end + levelIndex = levelIndexOf(rule.extraction, levelNames); + if ~ismember(method, ["substring", "regex"]) || isempty(levelIndex) || levelIndex > sessionIndex + report.add(here, "This part of the composite session identity is not read from one level at or above the session level; the Session ID is left unset.") + levels = []; + return + end + levels(end+1) = levelIndex; %#ok + end + + ancestorLevels = find(levelTypes(1:sessionIndex-1) ~= "" & levelTypes(1:sessionIndex-1) ~= sessionEntity); + levels = unique([levels, ancestorLevels]); +end + +function levelIndex = levelIndexOf(extraction, levelNames) +%levelIndexOf 1-based index of the level an extraction reads, or [] for the whole path + levelRef = getOr(extraction, 'entityLayoutLevel', []); + if isnumeric(levelRef) && ~isempty(levelRef) + levelIndex = levelRef + 1; + elseif ~isempty(levelRef) + levelIndex = find(levelNames == string(levelRef), 1); + else + levelIndex = []; + end +end + +function variables = convertFilePatternsBelowSession(levels, item, where, identityKeys, report) +%convertFilePatternsBelowSession Map file patterns of levels below the session to variables +% +% A NANSEN variable is a file in the session folder or in a subfolder of +% it with a fixed name. Folder levels with a fixed name therefore become +% the variable's Subfolder, and the file patterns of the levels reached +% through them become variables. The first level that is neither a +% fixed folder nor a level of files ends the walk; it and the levels +% below it are dropped. + variables = struct.empty; + subfolders = strings(1, 0); + for j = 1:numel(levels) + level = levels{j}; + levelWhere = where + ".entityLayout[" + string(level.name) + "]"; + isFile = string(getOr(level, 'fileSystemType', 'folder')) == "file"; + isFixedFolder = ~isFile && isfield(level, 'fixedName') && ~getOr(level, 'isVariable', true); + + if isFixedFolder + subfolders(end+1) = string(level.fixedName); %#ok + variables = appendItems(variables, convertFilePatterns(level, item, where, ... + identityKeys, report, strjoin(subfolders, "/"))); + elseif isFile + variables = appendItems(variables, convertFilePatterns(level, item, where, ... + identityKeys, report, strjoin(subfolders, "/"))); + report.add(levelWhere, "The files of this level are found in the session folder" + ... + subfolderText(subfolders) + ". Where a session has several of them, " + ... + "NANSEN uses the first.", isfield(level, 'entityType')) + else + for k = j:numel(levels) + report.add(where + ".entityLayout[" + string(levels{k}.name) + "]", ... + "NANSEN finds files below the session only through folders with a fixed name; " + ... + "this level and the levels below it are dropped.") + end + return + end + end end -function variables = convertFilePatterns(sessionLevel, item, where, sessionKeys, report) -%convertFilePatterns Map the session level's file patterns to variable model items +function text = subfolderText(subfolders) + if isempty(subfolders) + text = ""; + else + text = "'s subfolder " + strjoin(subfolders, "/"); + end +end + +function variables = convertFilePatterns(level, item, where, identityKeys, report, subfolder) +%convertFilePatterns Map a layout level's file patterns to variable model items +% +% identityKeys are the identity fields of the session and the subject, +% the {tokens} a file pattern may use in place of a wildcard. subfolder +% is the folder below the session folder where the files are, "" for +% files in the session folder. import nansen.config.dloc.dsmconversion.* + if nargin < 6 + subfolder = ""; + end + sessionLevel = level; variables = struct.empty; here = where + ".entityLayout[" + string(sessionLevel.name) + "].filePatterns"; patterns = asList(sessionLevel, 'filePatterns'); @@ -319,16 +459,22 @@ end patternWhere = here + "[" + string(filePattern.name) + "]"; - [expression, fileType, isConverted] = convertFilePatternToWildcard(string(filePattern.pattern), sessionKeys); + [expression, fileType, isConverted, isApproximate] = ... + convertFilePatternToWildcard(string(filePattern.pattern), identityKeys); if ~isConverted report.add(patternWhere, "The pattern has no wildcard equivalent.") continue end + report.add(patternWhere, "Approximated as '" + expression + "': a character class became *, " + ... + "so NANSEN may find files the pattern does not match.", isApproximate) + report.add(patternWhere + ".cardinality", "A NANSEN variable is one file per session; " + ... + "where several files match, NANSEN uses the first.", string(getOr(filePattern, 'cardinality', 'one')) == "many") variable = nansen.config.varmodel.VariableModel.getBlankItem(); variable.VariableName = char(matlab.lang.makeValidName(string(filePattern.name))); variable.DataLocation = item.Name; variable.DataLocationUuid = item.Uuid; + variable.Subfolder = char(subfolder); variable.FileNameExpression = char(expression); variable.FileType = char(fileType); variable.FileAdapter = findFileAdapter(fileType, patternWhere, report); @@ -340,6 +486,58 @@ end end +function variables = qualifyRepeatedVariableNames(variables, report) +%qualifyRepeatedVariableNames Qualify a name that several file patterns share +% +% A file pattern name is unique within one layout level, but a NANSEN +% variable name is unique within a project, and adding a second variable +% with the same name fails. A name that occurs in more than one data +% location is prefixed with the data location, _, +% so no location keeps the bare name only because it was converted +% first. A name that occurs more than once within one data location, on +% levels in different folders of the session, is also prefixed with the +% variable's subfolder, _; the variable of the session +% folder itself has no subfolder to add. Names that are still equal +% after this, from two levels in the same folder, are numbered. + + if isempty(variables) + return + end + + names = string({variables.VariableName}); + locations = string({variables.DataLocation}); + subfolders = string({variables.Subfolder}); + + newNames = names; + for i = 1:numel(variables) + isSameName = names == names(i); + parts = strings(1, 0); + if numel(unique(locations(isSameName))) > 1 + parts(end+1) = locations(i); %#ok + end + if sum(isSameName & locations == locations(i)) > 1 && strlength(subfolders(i)) > 0 + parts(end+1) = replace(subfolders(i), "/", "_"); %#ok + end + newNames(i) = matlab.lang.makeValidName(strjoin([parts, names(i)], "_")); + end + newNames = matlab.lang.makeUniqueStrings(newNames); + + for i = find(newNames ~= names) + report.add("variables[" + names(i) + "]", "The file pattern name is used by several file patterns; " + ... + "renamed to '" + newNames(i) + "' in data location " + locations(i) + folderText(subfolders(i)) + ".") + variables(i).VariableName = char(newNames(i)); + end +end + +function text = folderText(subfolder) +%folderText The subfolder of a variable for a report, "" for the session folder + if strlength(subfolder) == 0 + text = ""; + else + text = ", subfolder " + subfolder; + end +end + function adapterName = findFileAdapter(fileType, where, report) %findFileAdapter Name of a file adapter for a file type, or Default adapterName = 'Default'; diff --git a/code/+nansen/+dataio/listFileAdapters.m b/code/+nansen/+dataio/listFileAdapters.m index 9c4be633..53519045 100644 --- a/code/+nansen/+dataio/listFileAdapters.m +++ b/code/+nansen/+dataio/listFileAdapters.m @@ -21,7 +21,9 @@ fileAdapterList = table2struct(project.getTable('FileAdapter', refresh)); if nargin < 1; fileExtension = ''; end - if ~isempty(fileExtension); fileExtension = strrep(fileExtension, '.', ''); end + % Only the leading dot is removed, so that an extension of several + % parts, such as .nii.gz, keeps its inner dot and matches nii.gz + if ~isempty(fileExtension); fileExtension = regexprep(fileExtension, '^\.', ''); end if ~isempty(fileExtension) validationFcn = @(extList) any(contains(extList, fileExtension, "IgnoreCase", true)); diff --git a/code/+nansen/+plugin/+fileadapter/+internal/detectFileAdapterForFilepath.m b/code/+nansen/+plugin/+fileadapter/+internal/detectFileAdapterForFilepath.m index 6a17646a..4d9f8343 100644 --- a/code/+nansen/+plugin/+fileadapter/+internal/detectFileAdapterForFilepath.m +++ b/code/+nansen/+plugin/+fileadapter/+internal/detectFileAdapterForFilepath.m @@ -1,21 +1,34 @@ -function [fileAdapterName, isDynamic] = detectFileAdapterForFilepath(filePath) +function [fileAdapterName, isDynamic] = detectFileAdapterForFilepath(filePath, fileAdapterList) % detectFileAdapterForFilepath - Locate file adapter for a given filename and extension +% +% [fileAdapterName, isDynamic] = detectFileAdapterForFilepath(filePath) +% returns the first file adapter of nansen.dataio.listFileAdapters() +% that supports the file. An adapter supports a file whose name ends +% with one of its SupportedFileTypes, so a type of several parts, such +% as nii.gz, is matched as well. Where several adapters support the +% file, one with the longest matching type is used (nii.gz before gz). +% Case is ignored, as in listFileAdapters. +% +% ... = detectFileAdapterForFilepath(filePath, fileAdapterList) looks in +% the given list, a struct array with the fields of listFileAdapters. - fileAdapterList = nansen.dataio.listFileAdapters(); + if nargin < 2 + fileAdapterList = nansen.dataio.listFileAdapters(); + end - [~, ~, fileExtension] = fileparts(filePath); + [~, fileName, fileExtension] = fileparts(filePath); + fileName = string(fileName) + string(fileExtension); - supportsFile = false(1, numel(fileAdapterList)); - % Find matching file type + % Length of each adapter's longest supported type that the name ends with + matchLength = zeros(1, numel(fileAdapterList)); for i = 1:numel(fileAdapterList) - currentFileAdapterInfo = fileAdapterList(i); - - if ismember(fileExtension, currentFileAdapterInfo.SupportedFileTypes) || ... - ismember(extractAfter(fileExtension, '.'), currentFileAdapterInfo.SupportedFileTypes) - supportsFile(i) = true; + fileTypes = "." + regexprep(string(fileAdapterList(i).SupportedFileTypes), '^\.', ''); + isMatch = endsWith(fileName, fileTypes, 'IgnoreCase', true); + if any(isMatch) + matchLength(i) = max(strlength(fileTypes(isMatch))); end end - fileAdapterList = fileAdapterList(supportsFile); + fileAdapterList = fileAdapterList(matchLength > 0 & matchLength == max(matchLength)); if isempty(fileAdapterList) error('No file adapters exist that can open files of type "%s"', fileExtension) diff --git a/tests/+nansen/+fixture/datasetstructure/runs-below-session.json b/tests/+nansen/+fixture/datasetstructure/runs-below-session.json new file mode 100644 index 00000000..d34a98a1 --- /dev/null +++ b/tests/+nansen/+fixture/datasetstructure/runs-below-session.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": "0.1.0", + "entityTypes": [ + {"name": "subject", "description": "One monkey.", "isPrimary": true, "identifierRef": "subject_id"}, + {"name": "session", "description": "One scanning day of one monkey.", "identifierRef": "session_id"}, + {"name": "run", "description": "One functional run of a scanning day, in the day's funct folder.", "identifierRef": "run_number"} + ], + "entityRelationships": [ + {"sourceEntity": "subject", "targetEntity": "session", "relationType": "oneToMany", "relationName": "hasSessions"}, + {"sourceEntity": "session", "targetEntity": "run", "relationType": "oneToMany", "relationName": "hasRuns"} + ], + "metadataDefinitions": { + "subject_id": {"name": "subject_id", "title": "Monkey", "dataType": "string", "ofEntity": "subject"}, + "session_id": {"name": "session_id", "title": "Day", "dataType": "string", "ofEntity": "session"}, + "run_number": {"name": "run_number", "title": "Run", "dataType": "integer", "ofEntity": "run"} + }, + "dataLocations": [ + { + "identifier": "fmri", + "displayName": "fMRI", + "dataCategory": "raw", + "access": "read", + "sourceType": "filesystem", + "filesystemSource": { + "rootStoragePaths": [{"identifier": "local-copy", "path": "/data/fmri"}], + "entityLayout": [ + {"name": "monkeys", "entityType": "subject", "matchPattern": "^m\\d+$"}, + {"name": "days", "entityType": "session", "matchPattern": "^d\\d+$"}, + {"name": "funct-folder", "isVariable": false, "fixedName": "funct"}, + { + "name": "run-files", + "entityType": "run", + "fileSystemType": "file", + "matchPattern": "_run_\\d+\\.(nii\\.gz|json)$", + "filePatterns": [ + {"name": "bold", "pattern": "\\.nii\\.gz$", "isRequired": true, "cardinality": "one", "description": "Functional run, NIfTI"}, + {"name": "sidecar", "pattern": "\\.json$", "cardinality": "one", "description": "Acquisition parameters of the run"} + ] + } + ], + "metadataMapping": [ + {"metadataRef": "subject_id", "extraction": {"method": "substring", "pattern": ":", "entityLayoutLevel": "monkeys"}}, + {"metadataRef": "session_id", "extraction": {"method": "substring", "pattern": ":", "entityLayoutLevel": "days"}}, + {"metadataRef": "run_number", "extraction": {"method": "regex", "pattern": "_run_(\\d+)\\.(nii\\.gz|json)$", "entityLayoutLevel": "run-files"}} + ] + } + } + ] +} diff --git a/tests/+nansen/+fixture/datasetstructure/scanning-days.json b/tests/+nansen/+fixture/datasetstructure/scanning-days.json new file mode 100644 index 00000000..df504963 --- /dev/null +++ b/tests/+nansen/+fixture/datasetstructure/scanning-days.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": "0.1.0", + "entityTypes": [ + { "name": "subject", "description": "One monkey.", "isPrimary": true, "identifierRef": "subject_id" }, + { "name": "session", "description": "One scanning day. Day numbers restart in each experiment.", "identifierRefs": ["experiment", "day_number"] } + ], + "entityRelationships": [ + { "sourceEntity": "subject", "targetEntity": "session", "relationType": "oneToMany", "relationName": "hasSessions" } + ], + "metadataDefinitions": { + "subject_id": { "name": "subject_id", "dataType": "string", "ofEntity": "subject", "validation": { "pattern": "^m\\d+$" } }, + "experiment": { "name": "experiment", "dataType": "string", "ofEntity": "session", "validation": { "enum": ["PRE_stim", "POST_stim"] } }, + "day_number": { "name": "day_number", "dataType": "integer", "ofEntity": "session" } + }, + "dataLocations": [ + { + "identifier": "scanning", + "displayName": "Scanning days", + "description": "//d/, laid out like the raw fMRI of the EBRAINS dataset of Murris et al. (2021).", + "dataCategory": "raw", + "access": "read", + "sourceType": "filesystem", + "filesystemSource": { + "rootStoragePaths": [ { "identifier": "main", "path": "/data/scanning" } ], + "entityLayout": [ + { "name": "experiments", "matchPattern": "^(PRE|POST)_stim$" }, + { "name": "monkeys", "entityType": "subject", "matchPattern": "^m\\d+$" }, + { "name": "days", "entityType": "session", "matchPattern": "^d\\d+$" } + ], + "metadataMapping": [ + { "metadataRef": "experiment", "extraction": { "method": "substring", "pattern": ":", "entityLayoutLevel": "experiments" } }, + { "metadataRef": "subject_id", "extraction": { "method": "substring", "pattern": ":", "entityLayoutLevel": "monkeys" } }, + { "metadataRef": "day_number", "extraction": { "method": "regex", "pattern": "^d(\\d+)$", "entityLayoutLevel": "days" } } + ] + } + } + ] +} diff --git a/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m b/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m index 1664fda9..a9d9ff22 100644 --- a/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m +++ b/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m @@ -146,5 +146,39 @@ function testReportIsReturned(testCase) testCase.verifyClass(testCase.Report, 'table') testCase.verifyTrue(any(contains(testCase.Report.Element, "slice_number"))) end + + function testCompositeIdentityKeepsSessionsOfSubjectsApart(testCase) + % Day numbers restart in each experiment and each monkey has a + % day 1, so a Session ID made of the day alone would merge + % four scanning days into one session. + import matlab.unittest.fixtures.TemporaryFolderFixture + root = testCase.applyFixture(TemporaryFolderFixture).Folder; + dataFolder = fullfile(root, 'scanning'); + for experiment = ["PRE_stim", "POST_stim"] + for monkey = ["m1", "m2"] + dayFolder = fullfile(dataFolder, experiment, monkey, 'd1'); + mkdir(dayFolder) + fclose(fopen(fullfile(dayFolder, 'run_1.nii'), 'w')); + end + end + + dsm = jsondecode(fileread(fullfile(fileparts(fileparts(mfilename('fullpath'))), ... + '+fixture', 'datasetstructure', 'scanning-days.json'))); + dsm.dataLocations.filesystemSource.rootStoragePaths.path = dataFolder; + + configFolder = fullfile(root, 'configurations'); + mkdir(configFolder) + dataLocationPath = fullfile(configFolder, 'datalocation_settings.json'); + nansen.config.dloc.importDatasetStructureModel(dsm, ... + nansen.config.dloc.DataLocationModel(dataLocationPath), ... + nansen.config.varmodel.VariableModel(fullfile(configFolder, 'filepath_settings.json'))); + model = nansen.config.dloc.DataLocationModel(dataLocationPath); + + sessionFolders = nansen.dataio.session.listSessionFolders(model, 'all'); + [~, sessionIds] = nansen.dataio.session.matchSessionFolders(model, sessionFolders); + + testCase.verifyEqual(sort(string(sessionIds(:)')), ... + ["POST_stim_m1_d1", "POST_stim_m2_d1", "PRE_stim_m1_d1", "PRE_stim_m2_d1"]) + end end end diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m new file mode 100644 index 00000000..8d47cfe4 --- /dev/null +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m @@ -0,0 +1,89 @@ +classdef Dsm2DataLocationFilesBelowSessionTest < matlab.unittest.TestCase +%Dsm2DataLocationFilesBelowSessionTest - File patterns on layout levels below the session +% +% Uses a model of monkey folders with a folder per scanning day (the +% session), a folder named funct in each day, and run files in it with +% the file patterns bold (.nii.gz) and sidecar (.json). +% +% Run tests: +% runtests('nansen.unittest.config.Dsm2DataLocationFilesBelowSessionTest') + + properties + Model + end + + methods (TestClassSetup) + function loadModel(testCase) + fixture = fullfile(fileparts(fileparts(fileparts(mfilename('fullpath')))), ... + '+fixture', 'datasetstructure', 'runs-below-session.json'); + testCase.Model = jsondecode(fileread(fixture)); + end + end + + methods (Test) + function testFilePatternsInFixedFolderBecomeVariables(testCase) + [~, variables] = nansen.config.dloc.dsm2DataLocationModel(testCase.Model); + + testCase.verifyEqual(sort(string({variables.VariableName})), ["bold", "sidecar"]) + testCase.verifyEqual(string({variables.Subfolder}), ["funct", "funct"]) + end + + function testExtensionOfSeveralPartsIsFileType(testCase) + [~, variables] = nansen.config.dloc.dsm2DataLocationModel(testCase.Model); + + bold = variables(strcmp({variables.VariableName}, 'bold')); + testCase.verifyEqual(bold.FileType, '.nii.gz') + end + + function testNameInSessionAndFixedFolderIsQualifiedBySubfolder(testCase) + % A sidecar file in the day folder and one in its funct folder + model = testCase.Model; + model.dataLocations.filesystemSource.entityLayout{2}.filePatterns = ... + {struct('name', 'sidecar', 'pattern', '_day\.json$')}; + + [~, variables, report] = nansen.config.dloc.dsm2DataLocationModel(model); + + names = string({variables.VariableName}); + testCase.verifyEqual(sort(names), ["bold", "funct_sidecar", "sidecar"]) + testCase.verifyEqual(variables(names == "funct_sidecar").Subfolder, 'funct') + testCase.verifyEmpty(variables(names == "sidecar").Subfolder) + testCase.verifyTrue(any(contains(report.Element, "variables[sidecar]"))) + end + + function testNameInNestedFixedFoldersIsQualifiedBySubfolder(testCase) + % funct/ and funct/raw/ each hold a NIfTI file + model = testCase.Model; + layout = model.dataLocations.filesystemSource.entityLayout(:); + nifti = {struct('name', 'nifti', 'pattern', '\.nii$')}; + layout{3}.filePatterns = nifti; + raw = struct('name', 'raw-folder', 'isVariable', false, 'fixedName', 'raw', 'filePatterns', {nifti}); + model.dataLocations.filesystemSource.entityLayout = [layout(1:3); {raw}; layout(4:end)]; + + [~, variables] = nansen.config.dloc.dsm2DataLocationModel(model); + + names = string({variables.VariableName}); + testCase.verifyEqual(sort(names(contains(names, "nifti"))), ["funct_nifti", "funct_raw_nifti"]) + testCase.verifyEqual(variables(names == "funct_raw_nifti").Subfolder, 'funct/raw') + end + + function testSeveralFilesPerSessionIsReported(testCase) + [~, ~, report] = nansen.config.dloc.dsm2DataLocationModel(testCase.Model); + + isRunLevel = endsWith(report.Element, "entityLayout[run-files]"); + testCase.verifyTrue(any(isRunLevel & contains(report.Reason, "uses the first"))) + end + + function testLevelsBelowVariableFolderAreDropped(testCase) + model = testCase.Model; + layout = model.dataLocations.filesystemSource.entityLayout; + layout{3} = struct('name', 'protocol-folders', 'matchPattern', '^[a-z]+$'); + model.dataLocations.filesystemSource.entityLayout = layout; + + [~, variables, report] = nansen.config.dloc.dsm2DataLocationModel(model); + + testCase.verifyEmpty(variables) + testCase.verifyTrue(any(endsWith(report.Element, "entityLayout[run-files]") ... + & contains(report.Reason, "dropped"))) + end + end +end diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m index 36f13e01..5e2cfcc7 100644 --- a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m @@ -58,6 +58,14 @@ function verifyReported(testCase, elementPart) sprintf('Expected the report to mention %s.', elementPart)) end + function sessionId = sessionIdOf(~, rule, sessionFolder, numLevels) + %sessionIdOf Run a converted Session ID rule the way NANSEN does + joinedNames = nansen.config.dloc.DataLocationModel.combineFolderNamesFromPath( ... + sessionFolder, rule.SubfolderLevel, numLevels, rule.Separator); + sessionId = nansen.config.dloc.DataLocationModel.applyExtractionPattern( ... + joinedNames, rule.StringDetectMode, rule.StringDetectInput); + end + function verifyNotReported(testCase, elementPart) testCase.verifyFalse(any(contains(testCase.Report.Element, elementPart)), ... sprintf('%s was mapped and should not be in the report.', elementPart)) @@ -180,18 +188,160 @@ function testSliceNeverUsesEndOutsideTheWholeRange(testCase) end function testFilePatternsBecomeWildcards(testCase) - convert = @(p) nansen.config.dloc.dsmconversion.convertFilePatternToWildcard(p, "recording_id"); + convert = @(p) nansen.config.dloc.dsmconversion.convertFilePatternToWildcard( ... + p, ["recording_id", "cell_id"]); - [expression, fileType, isConverted] = convert("^{recording_id}_meta\.json$"); + [expression, fileType, isConverted, isApproximate] = convert("^{recording_id}_meta\.json$"); testCase.verifyTrue(isConverted) + testCase.verifyFalse(isApproximate) testCase.verifyEqual(expression, "^*_meta.json$") testCase.verifyEqual(fileType, ".json") + [expression, ~, isConverted] = convert("^{cell_id}_{recording_id}\.ABF$"); + testCase.verifyTrue(isConverted, 'A token naming an ancestor identity becomes * too.') + testCase.verifyEqual(expression, "^*_*.ABF$") + [~, ~, isConverted] = convert("^{other_id}\.ABF$"); - testCase.verifyFalse(isConverted, 'Only tokens naming the session identity become *.') + testCase.verifyFalse(isConverted, 'Tokens naming other fields have no wildcard equivalent.') + + [expression, ~, ~, isApproximate] = convert("^.*\.ABF$"); + testCase.verifyEqual(expression, "^*.ABF$") + testCase.verifyFalse(isApproximate, '.* is any text, as * is.') + end + + function testCharacterClassesAreApproximated(testCase) + % NANSEN passes the expression to dir, whose only wildcard is *. + convert = @(p) nansen.config.dloc.dsmconversion.convertFilePatternToWildcard(p, "session_id"); + cases = { ... + "^TT\d+\.ntt$", "^TT*.ntt$"; ... + "_ch\d+_\d+\.dat$", "_ch*_*.dat$"; ... + "_feature_[A-Za-z0-9]+\.fd$", "_feature_*.fd$"; ... + "_{session_id}__\d{4}-\d{2}-\d{2}\.pkl$", "_*__*-*-*.pkl$"; ... + "^{session_id}\.eeg\d?$", "^*.eeg*$"}; + for i = 1:height(cases) + [expression, ~, isConverted, isApproximate] = convert(cases{i, 1}); + testCase.verifyTrue(isConverted, cases{i, 1}) + testCase.verifyTrue(isApproximate, cases{i, 1}) + testCase.verifyEqual(expression, cases{i, 2}, cases{i, 1}) + end + end + + function testApproximationMatchingEveryFileIsRefused(testCase) + [expression, ~, isConverted] = nansen.config.dloc.dsmconversion.convertFilePatternToWildcard( ... + "^{session_id}\.\d$", "session_id"); + testCase.verifyFalse(isConverted, '^*.*$ would match every file in the folder.') + testCase.verifyEqual(expression, "") + end + + function testRepeatedPatternNamesAreQualifiedByLocation(testCase) + % A file pattern name is unique within a level, but a NANSEN + % variable name is unique within a project. Two locations that + % hold the same recording in two formats both name their + % pattern "original". + [~, variables, report] = nansen.config.dloc.dsm2DataLocationModel( ... + jsondecode(twoFormatModel())); + + names = string({variables.VariableName}); + testCase.verifyEqual(sort(names), ["igor_original", "notes", "nwb_original"]) + testCase.verifyEqual(variables(names == "igor_original").DataLocation, 'igor') + testCase.verifyEqual(variables(names == "nwb_original").FileNameExpression, '.nwb$') + testCase.verifyTrue(any(contains(report.Element, "variables[original]"))) + testCase.verifyFalse(any(contains(report.Element, "variables[notes]")), ... + 'A name used in one location keeps its name.') + end + + function testCompositeSessionIdJoinsLevelNames(testCase) + % Day numbers restart in each experiment, so a scanning day is + % identified by the experiment and the day, within its monkey. + [dataLocations, ~, report] = nansen.config.dloc.dsm2DataLocationModel( ... + jsondecode(scanningDayModel()), SessionEntity="scanning_day"); + association = dataLocations(strcmp({dataLocations.Name}, 'association')); + + rule = association.MetaDataDef(strcmp({association.MetaDataDef.VariableName}, 'Session ID')); + testCase.verifyEqual(rule.SubfolderLevel, [1 2 3], ... + 'The experiment level, the monkey level (an ancestor) and the day level.') + testCase.verifyEqual(rule.Separator, '_') + + sessionId = testCase.sessionIdOf(rule, '/data/association/PRE_stim/m1/d1', 3); + testCase.verifyEqual(sessionId, 'PRE_stim_m1_d1') + testCase.verifyTrue(any(contains(report.Reason, "the Session ID joins the names of levels"))) + testCase.verifyFalse(any(contains(report.Reason, "NANSEN needs a single id field"))) + end + + function testFixedPartOfCompositeIdentityIsLeftOut(testCase) + % The localizer has no experiment level; a fixed rule gives the + % experiment, and the Session ID is made of the other levels. + [dataLocations, ~, report] = nansen.config.dloc.dsm2DataLocationModel( ... + jsondecode(scanningDayModel()), SessionEntity="scanning_day"); + localizer = dataLocations(strcmp({dataLocations.Name}, 'localizer')); + + rule = localizer.MetaDataDef(strcmp({localizer.MetaDataDef.VariableName}, 'Session ID')); + testCase.verifyEqual(rule.SubfolderLevel, [1 2]) + testCase.verifyEqual(testCase.sessionIdOf(rule, '/data/localizer/m2/d1', 2), 'm2_d1') + testCase.verifyTrue(any(contains(report.Element, "dataLocations[localizer].metadataMapping[experiment]"))) + end + + function testUnreadablePartOfCompositeIdentityIsReportedOnce(testCase) + % Without a rule for the experiment, the Session ID is left + % unset, and the report gives that reason only + model = replace(scanningDayModel(), '"metadataRef":"experiment"', '"metadataRef":"protocol"'); - [~, ~, isConverted] = convert("^\d+\.ABF$"); - testCase.verifyFalse(isConverted, '\d has no wildcard equivalent.') + [~, ~, report] = nansen.config.dloc.dsm2DataLocationModel(jsondecode(model), ... + SessionEntity="scanning_day"); + + isAssociation = startsWith(report.Element, "dataLocations[association]"); + testCase.verifyTrue(any(isAssociation & contains(report.Reason, "No extraction rule for this part"))) + testCase.verifyFalse(any(contains(report.Reason, "NANSEN needs a single id field"))) end end end + +function text = scanningDayModel() +%scanningDayModel Scanning days of monkeys in two experiments, numbered from 1 in each + mapping = @(key, extraction) struct('metadataRef', key, 'extraction', extraction); + fromLevel = @(method, pattern, level) struct('method', method, 'pattern', pattern, 'entityLayoutLevel', level); + level = @(name, entityType, pattern) struct('name', name, 'entityType', entityType, 'matchPattern', pattern); + location = @(name, layout, rules) struct( ... + 'identifier', name, 'displayName', name, 'dataCategory', 'raw', 'sourceType', 'filesystem', ... + 'filesystemSource', struct( ... + 'rootStoragePaths', {{struct('identifier', 'main', 'path', ['/data/' name])}}, ... + 'entityLayout', {layout}, 'metadataMapping', {rules})); + dayRules = {mapping('subject_id', fromLevel('substring', ':', 'monkeys')), ... + mapping('day_number', fromLevel('regex', '^d(\d+)$', 'days'))}; + association = location('association', ... + {struct('name', 'experiments', 'matchPattern', '^(PRE|POST)_stim$'), ... + level('monkeys', 'subject', '^m\d+$'), level('days', 'scanning_day', '^d\d+$')}, ... + [{mapping('experiment', fromLevel('substring', ':', 'experiments'))}, dayRules]); + localizer = location('localizer', ... + {level('monkeys', 'subject', '^m\d+$'), level('days', 'scanning_day', '^d\d+$')}, ... + [{mapping('experiment', struct('method', 'fixed', 'value', 'localizer'))}, dayRules]); + model = struct( ... + 'schemaVersion', '0.1.0', ... + 'entityTypes', {{struct('name', 'subject', 'identifierRef', 'subject_id'), ... + struct('name', 'scanning_day', 'identifierRefs', {{'experiment'; 'day_number'}})}}, ... + 'metadataDefinitions', struct( ... + 'subject_id', struct('name', 'subject_id', 'dataType', 'string', 'ofEntity', 'subject'), ... + 'experiment', struct('name', 'experiment', 'dataType', 'string', 'ofEntity', 'scanning_day'), ... + 'day_number', struct('name', 'day_number', 'dataType', 'integer', 'ofEntity', 'scanning_day')), ... + 'dataLocations', {{association, localizer}}); + text = jsonencode(model); +end + +function text = twoFormatModel() +%twoFormatModel A cell folder per location, the recording as Igor Pro in one and NWB in the other + location = @(name, extension, extraPattern) struct( ... + 'identifier', name, 'displayName', name, 'dataCategory', 'raw', 'sourceType', 'filesystem', ... + 'filesystemSource', struct( ... + 'rootStoragePaths', {{struct('identifier', 'main', 'path', ['/data/' name])}}, ... + 'entityLayout', {{struct('name', 'cells', 'entityType', 'cell', 'matchPattern', '^cell\d+$', ... + 'filePatterns', {[{struct('name', 'original', 'pattern', ['\.' extension '$'])}, extraPattern]})}}, ... + 'metadataMapping', {{struct('metadataRef', 'cell_id', ... + 'extraction', struct('method', 'substring', 'pattern', ':', 'entityLayoutLevel', 'cells'))}})); + model = struct( ... + 'schemaVersion', '0.1.0', ... + 'entityTypes', {{struct('name', 'cell', 'identifierRef', 'cell_id')}}, ... + 'metadataDefinitions', struct('cell_id', struct('name', 'cell_id', 'dataType', 'string', 'ofEntity', 'cell')), ... + 'dataLocations', {{location('igor', 'pxp', {}), ... + location('nwb', 'nwb', {struct('name', 'notes', 'pattern', '\.txt$')})}}); + text = jsonencode(model); +end diff --git a/tests/+nansen/+unittest/+plugin/DetectFileAdapterForFilepathTest.m b/tests/+nansen/+unittest/+plugin/DetectFileAdapterForFilepathTest.m new file mode 100644 index 00000000..7e3f9629 --- /dev/null +++ b/tests/+nansen/+unittest/+plugin/DetectFileAdapterForFilepathTest.m @@ -0,0 +1,50 @@ +classdef DetectFileAdapterForFilepathTest < matlab.unittest.TestCase +%DetectFileAdapterForFilepathTest - Choose the file adapter that nansen.load and nansen.save use +% +% The adapter list is given to the function, so no project is needed. +% +% Run tests: +% runtests('nansen.unittest.plugin.DetectFileAdapterForFilepathTest') + + properties + AdapterList + end + + methods (TestMethodSetup) + function createAdapterList(testCase) + testCase.AdapterList = struct( ... + 'FunctionName', {'test.Gzip', 'test.NIfTI', 'test.MatFile'}, ... + 'SupportedFileTypes', {{'gz'}, {'nii', 'nii.gz'}, {'.mat'}}, ... + 'IsDynamic', {false, false, true}); + end + end + + methods (Test) + function testSingleExtension(testCase) + [name, isDynamic] = nansen.plugin.fileadapter.internal.detectFileAdapterForFilepath( ... + fullfile('data', 'values.mat'), testCase.AdapterList); + + testCase.verifyEqual(name, 'test.MatFile') + testCase.verifyTrue(isDynamic) + end + + function testExtensionOfSeveralPartsPrefersLongestType(testCase) + name = nansen.plugin.fileadapter.internal.detectFileAdapterForFilepath( ... + fullfile('data', 'run.nii.gz'), testCase.AdapterList); + + testCase.verifyEqual(name, 'test.NIfTI') + end + + function testCaseIsIgnored(testCase) + name = nansen.plugin.fileadapter.internal.detectFileAdapterForFilepath( ... + fullfile('data', 'RUN.NII'), testCase.AdapterList); + + testCase.verifyEqual(name, 'test.NIfTI') + end + + function testUnsupportedTypeErrors(testCase) + testCase.verifyError(@() nansen.plugin.fileadapter.internal.detectFileAdapterForFilepath( ... + fullfile('data', 'trace.abf'), testCase.AdapterList), ?MException) + end + end +end