From 444f1f89b4960e657cff5370ea951e4ca6a0c852 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 18 Sep 2026 18:45:42 +0200 Subject: [PATCH 1/5] fix(dloc): qualify file pattern names used in several data locations A Dataset Structure Model names file patterns per layout level, so two data locations may both have a pattern called "original" (the same recording as Igor Pro in one location and NWB in the other, as in the EBRAINS datasets of Dorst et al. and Carannante et al.). A NANSEN variable name is unique within a project, and importing such a model failed when the second variable was inserted. dsm2DataLocationModel now renames every variable whose name occurs in more than one data location to _, and reports each rename. Names used in one location are unchanged. Co-Authored-By: Claude Opus 5 --- .../+config/+dloc/dsm2DataLocationModel.m | 26 ++++++++++++++ .../+config/Dsm2DataLocationModelTest.m | 36 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m index c36976fe..eef6b622 100644 --- a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m +++ b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m @@ -100,6 +100,7 @@ dataLocations = appendItems(dataLocations, item); variables = appendItems(variables, convertFilePatterns(sessionLevel, item, where, sessionKeys, report)); end + variables = qualifyRepeatedVariableNames(variables, report); for fieldName = string(fieldnames(definitions))' if ~ismember(fieldName, usedFields) @@ -340,6 +341,31 @@ end end +function variables = qualifyRepeatedVariableNames(variables, report) +%qualifyRepeatedVariableNames Prefix a name used in several data locations with the location +% +% A file pattern name is unique within one layout level, but a NANSEN +% variable name is unique within a project, and adding a second variable +% with the same name fails. Every variable whose name occurs in more than +% one data location is renamed to _, so no location +% keeps the bare name only because it was converted first. + + if isempty(variables) + return + end + + names = string({variables.VariableName}); + [~, ~, groupIndex] = unique(names); + occurrences = accumarray(groupIndex(:), 1); + + for i = find(occurrences(groupIndex)' > 1) + newName = matlab.lang.makeValidName(string(variables(i).DataLocation) + "_" + names(i)); + report.add("variables[" + names(i) + "]", "The file pattern name is used in several data locations; " + ... + "renamed to '" + newName + "' in data location " + string(variables(i).DataLocation) + ".") + variables(i).VariableName = char(newName); + end +end + function adapterName = findFileAdapter(fileType, where, report) %findFileAdapter Name of a file adapter for a file type, or Default adapterName = 'Default'; diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m index 36f13e01..e1590647 100644 --- a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m @@ -193,5 +193,41 @@ function testFilePatternsBecomeWildcards(testCase) [~, ~, isConverted] = convert("^\d+\.ABF$"); testCase.verifyFalse(isConverted, '\d has no wildcard equivalent.') end + + function testRepeatedPatternNamesAreQualifiedByLocation(testCase) + % A file pattern name is unique within a level, but a NANSEN + % variable name is unique within a project. Two locations that + % hold the same recording in two formats both name their + % pattern "original". + [~, variables, report] = nansen.config.dloc.dsm2DataLocationModel( ... + jsondecode(twoFormatModel())); + + names = string({variables.VariableName}); + testCase.verifyEqual(sort(names), ["igor_original", "notes", "nwb_original"]) + testCase.verifyEqual(variables(names == "igor_original").DataLocation, 'igor') + testCase.verifyEqual(variables(names == "nwb_original").FileNameExpression, '.nwb$') + testCase.verifyTrue(any(contains(report.Element, "variables[original]"))) + testCase.verifyFalse(any(contains(report.Element, "variables[notes]")), ... + 'A name used in one location keeps its name.') + end end end + +function text = twoFormatModel() +%twoFormatModel A cell folder per location, the recording as Igor Pro in one and NWB in the other + location = @(name, extension, extraPattern) struct( ... + 'identifier', name, 'displayName', name, 'dataCategory', 'raw', 'sourceType', 'filesystem', ... + 'filesystemSource', struct( ... + 'rootStoragePaths', {{struct('identifier', 'main', 'path', ['/data/' name])}}, ... + 'entityLayout', {{struct('name', 'cells', 'entityType', 'cell', 'matchPattern', '^cell\d+$', ... + 'filePatterns', {[{struct('name', 'original', 'pattern', ['\.' extension '$'])}, extraPattern]})}}, ... + 'metadataMapping', {{struct('metadataRef', 'cell_id', ... + 'extraction', struct('method', 'substring', 'pattern', ':', 'entityLayoutLevel', 'cells'))}})); + model = struct( ... + 'schemaVersion', '0.1.0', ... + 'entityTypes', {{struct('name', 'cell', 'identifierRef', 'cell_id')}}, ... + 'metadataDefinitions', struct('cell_id', struct('name', 'cell_id', 'dataType', 'string', 'ofEntity', 'cell')), ... + 'dataLocations', {{location('igor', 'pxp', {}), ... + location('nwb', 'nwb', {struct('name', 'notes', 'pattern', '\.txt$')})}}); + text = jsonencode(model); +end From 062e9df3d867e91bda297157200fe2f0615f62ef Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 18 Sep 2026 18:48:44 +0200 Subject: [PATCH 2/5] feat(dloc): read a composite session identity from joined level names A Dataset Structure Model may identify a session by several fields, for example the experiment and the day number when day numbers restart in each experiment (the raw fMRI of the EBRAINS dataset of Murris et al.). The converter left the Session ID unset for such a model, and NANSEN then found no sessions to match. NANSEN can join the names of several folder levels before it applies one extraction rule. For a composite identity whose parts are each read from one level at or above the session level, the Session ID now joins the whole names of those levels with '_'. The levels of the session's ancestor entities are joined too: a model's identity is unique within its parents, a NANSEN Session ID within the project, and without the subject day 1 of two monkeys would be one session. A part given by a fixed rule is not in the path and is left out; both cases are reported. Single-field identities are mapped as before. convertMetadata receives the layout instead of its level names, and the level lookup it already did moved into levelIndexOf so the new function can use it. Co-Authored-By: Claude Opus 5 --- .../+config/+dloc/dsm2DataLocationModel.m | 103 ++++++++++++++++-- .../datasetstructure/scanning-days.json | 38 +++++++ .../ImportDatasetStructureModelTest.m | 34 ++++++ .../+config/Dsm2DataLocationModelTest.m | 83 ++++++++++++++ 4 files changed, 246 insertions(+), 12 deletions(-) create mode 100644 tests/+nansen/+fixture/datasetstructure/scanning-days.json diff --git a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m index eef6b622..59b19c1c 100644 --- a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m +++ b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m @@ -151,9 +151,8 @@ item.RootPath = convertRootPaths(asList(source, 'rootStoragePaths'), where, report); item.SubfolderStructure = convertLayout(layout, where, definitions, sessionEntity, subjectEntity, report); - levelNames = fieldValues(asList(source, 'entityLayout'), 'name'); [item.MetaDataDef, usedFields] = convertMetadata(asList(source, 'metadataMapping'), ... - definitions, levelNames, sessionIndex, where, sessionEntity, subjectEntity, ... + definitions, asList(source, 'entityLayout'), sessionIndex, where, sessionEntity, subjectEntity, ... sessionKeys, subjectKeys, report); end @@ -239,18 +238,32 @@ end end -function [metaDataDef, usedFields] = convertMetadata(mapping, definitions, levelNames, sessionIndex, ... +function [metaDataDef, usedFields] = convertMetadata(mapping, definitions, layout, sessionIndex, ... where, sessionEntity, subjectEntity, sessionKeys, subjectKeys, report) %convertMetadata Map extraction rules onto NANSEN's four metadata variables import nansen.config.dloc.dsmconversion.* metaDataDef = nansen.config.dloc.DataLocationModel.getDefaultMetadataStructure(); usedFields = string.empty(1, 0); + levelNames = fieldValues(layout, 'name'); + + % A composite session identity becomes a Session ID read from the + % joined names of several levels; a single identity field is mapped + % by the loop below like the other variables. + % compositeIdentityLevels reports why when the parts cannot be joined. + compositeLevels = []; + sessionField = ""; + if numel(sessionKeys) > 1 + compositeLevels = compositeIdentityLevels(mapping, sessionKeys, layout, sessionIndex, ... + sessionEntity, where, report); + else + sessionField = identityField(sessionKeys, "session", where, report); + end targets = struct( ... 'Variable', {'Subject ID', 'Session ID', 'Experiment Date', 'Experiment Time'}, ... 'Field', {identityField(subjectKeys, "subject", where, report), ... - identityField(sessionKeys, "session", where, report), ... + sessionField, ... temporalField(definitions, ["date", "datetime"], sessionEntity, subjectEntity), ... temporalField(definitions, "time", sessionEntity, subjectEntity)}); @@ -267,14 +280,7 @@ end extraction = rule.extraction; - levelRef = getOr(extraction, 'entityLayoutLevel', []); - if isnumeric(levelRef) && ~isempty(levelRef) - levelIndex = levelRef + 1; - elseif ~isempty(levelRef) - levelIndex = find(levelNames == string(levelRef), 1); - else - levelIndex = []; - end + levelIndex = levelIndexOf(extraction, levelNames); if isempty(levelIndex) || levelIndex > sessionIndex report.add(here, "The rule does not read a single level at or above the session level; " + targets(k).Variable + " is left unset.") continue @@ -302,6 +308,79 @@ metaDataDef(idx).StringDetectInput = char(input); metaDataDef(idx).StringFormat = char(getOr(extraction, 'valueFormat', '')); end + + if ~isempty(compositeLevels) + idx = strcmp({metaDataDef.VariableName}, 'Session ID'); + metaDataDef(idx).SubfolderLevel = compositeLevels; + metaDataDef(idx).StringDetectMode = 'expr'; + metaDataDef(idx).StringDetectInput = '.+'; + metaDataDef(idx).Separator = '_'; + usedFields = [usedFields, sessionKeys]; + report.add(where + ".metadataMapping[" + strjoin(sessionKeys, ", ") + "]", ... + "The session identity is composite; the Session ID joins the names of levels " + ... + strjoin(levelNames(compositeLevels), ", ") + " with '_', so it holds whole folder names rather than the extracted values.") + end +end + +function levels = compositeIdentityLevels(mapping, keys, layout, sessionIndex, sessionEntity, where, report) +%compositeIdentityLevels Levels whose joined names identify a session with a composite identity +% +% NANSEN reads a Session ID from one string. It can join the names of +% several levels into that string, but it applies one rule to the joined +% string, so the parts of a composite identity cannot each be extracted. +% The whole names of the levels the parts are read from are used +% instead: every part is read from the name of its level, so sessions +% with the same level names have the same identity. +% +% The levels of the session's ancestor entities are joined too. A model's +% identity is unique within its parent entities, while a NANSEN Session ID +% is unique within the project; without the ancestors, day 1 of two +% subjects would be one session. +% +% A part given by a fixed rule is not in the path and is left out. +% Returns empty, with a report, when a part is read otherwise than from +% one level at or above the session level. + + levels = []; + levelNames = fieldValues(layout, 'name'); + levelTypes = fieldValues(layout, 'entityType'); + + for key = keys + here = where + ".metadataMapping[" + key + "]"; + rule = findRule(mapping, key); + if isempty(rule) + report.add(here, "No extraction rule for this part of the composite session identity; the Session ID is left unset.") + levels = []; + return + end + method = string(rule.extraction.method); + if method == "fixed" + report.add(here, "A fixed part of the session identity is not in the path; the Session ID leaves it out.") + continue + end + levelIndex = levelIndexOf(rule.extraction, levelNames); + if ~ismember(method, ["substring", "regex"]) || isempty(levelIndex) || levelIndex > sessionIndex + report.add(here, "This part of the composite session identity is not read from one level at or above the session level; the Session ID is left unset.") + levels = []; + return + end + levels(end+1) = levelIndex; %#ok + end + + ancestorLevels = find(levelTypes(1:sessionIndex-1) ~= "" & levelTypes(1:sessionIndex-1) ~= sessionEntity); + levels = unique([levels, ancestorLevels]); +end + +function levelIndex = levelIndexOf(extraction, levelNames) +%levelIndexOf 1-based index of the level an extraction reads, or [] for the whole path + levelRef = getOr(extraction, 'entityLayoutLevel', []); + if isnumeric(levelRef) && ~isempty(levelRef) + levelIndex = levelRef + 1; + elseif ~isempty(levelRef) + levelIndex = find(levelNames == string(levelRef), 1); + else + levelIndex = []; + end end function variables = convertFilePatterns(sessionLevel, item, where, sessionKeys, report) diff --git a/tests/+nansen/+fixture/datasetstructure/scanning-days.json b/tests/+nansen/+fixture/datasetstructure/scanning-days.json new file mode 100644 index 00000000..df504963 --- /dev/null +++ b/tests/+nansen/+fixture/datasetstructure/scanning-days.json @@ -0,0 +1,38 @@ +{ + "schemaVersion": "0.1.0", + "entityTypes": [ + { "name": "subject", "description": "One monkey.", "isPrimary": true, "identifierRef": "subject_id" }, + { "name": "session", "description": "One scanning day. Day numbers restart in each experiment.", "identifierRefs": ["experiment", "day_number"] } + ], + "entityRelationships": [ + { "sourceEntity": "subject", "targetEntity": "session", "relationType": "oneToMany", "relationName": "hasSessions" } + ], + "metadataDefinitions": { + "subject_id": { "name": "subject_id", "dataType": "string", "ofEntity": "subject", "validation": { "pattern": "^m\\d+$" } }, + "experiment": { "name": "experiment", "dataType": "string", "ofEntity": "session", "validation": { "enum": ["PRE_stim", "POST_stim"] } }, + "day_number": { "name": "day_number", "dataType": "integer", "ofEntity": "session" } + }, + "dataLocations": [ + { + "identifier": "scanning", + "displayName": "Scanning days", + "description": "//d/, laid out like the raw fMRI of the EBRAINS dataset of Murris et al. (2021).", + "dataCategory": "raw", + "access": "read", + "sourceType": "filesystem", + "filesystemSource": { + "rootStoragePaths": [ { "identifier": "main", "path": "/data/scanning" } ], + "entityLayout": [ + { "name": "experiments", "matchPattern": "^(PRE|POST)_stim$" }, + { "name": "monkeys", "entityType": "subject", "matchPattern": "^m\\d+$" }, + { "name": "days", "entityType": "session", "matchPattern": "^d\\d+$" } + ], + "metadataMapping": [ + { "metadataRef": "experiment", "extraction": { "method": "substring", "pattern": ":", "entityLayoutLevel": "experiments" } }, + { "metadataRef": "subject_id", "extraction": { "method": "substring", "pattern": ":", "entityLayoutLevel": "monkeys" } }, + { "metadataRef": "day_number", "extraction": { "method": "regex", "pattern": "^d(\\d+)$", "entityLayoutLevel": "days" } } + ] + } + } + ] +} diff --git a/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m b/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m index 1664fda9..a9d9ff22 100644 --- a/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m +++ b/tests/+nansen/+integrationtest/ImportDatasetStructureModelTest.m @@ -146,5 +146,39 @@ function testReportIsReturned(testCase) testCase.verifyClass(testCase.Report, 'table') testCase.verifyTrue(any(contains(testCase.Report.Element, "slice_number"))) end + + function testCompositeIdentityKeepsSessionsOfSubjectsApart(testCase) + % Day numbers restart in each experiment and each monkey has a + % day 1, so a Session ID made of the day alone would merge + % four scanning days into one session. + import matlab.unittest.fixtures.TemporaryFolderFixture + root = testCase.applyFixture(TemporaryFolderFixture).Folder; + dataFolder = fullfile(root, 'scanning'); + for experiment = ["PRE_stim", "POST_stim"] + for monkey = ["m1", "m2"] + dayFolder = fullfile(dataFolder, experiment, monkey, 'd1'); + mkdir(dayFolder) + fclose(fopen(fullfile(dayFolder, 'run_1.nii'), 'w')); + end + end + + dsm = jsondecode(fileread(fullfile(fileparts(fileparts(mfilename('fullpath'))), ... + '+fixture', 'datasetstructure', 'scanning-days.json'))); + dsm.dataLocations.filesystemSource.rootStoragePaths.path = dataFolder; + + configFolder = fullfile(root, 'configurations'); + mkdir(configFolder) + dataLocationPath = fullfile(configFolder, 'datalocation_settings.json'); + nansen.config.dloc.importDatasetStructureModel(dsm, ... + nansen.config.dloc.DataLocationModel(dataLocationPath), ... + nansen.config.varmodel.VariableModel(fullfile(configFolder, 'filepath_settings.json'))); + model = nansen.config.dloc.DataLocationModel(dataLocationPath); + + sessionFolders = nansen.dataio.session.listSessionFolders(model, 'all'); + [~, sessionIds] = nansen.dataio.session.matchSessionFolders(model, sessionFolders); + + testCase.verifyEqual(sort(string(sessionIds(:)')), ... + ["POST_stim_m1_d1", "POST_stim_m2_d1", "PRE_stim_m1_d1", "PRE_stim_m2_d1"]) + end end end diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m index e1590647..8c4d8e20 100644 --- a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m @@ -58,6 +58,14 @@ function verifyReported(testCase, elementPart) sprintf('Expected the report to mention %s.', elementPart)) end + function sessionId = sessionIdOf(~, rule, sessionFolder, numLevels) + %sessionIdOf Run a converted Session ID rule the way NANSEN does + joinedNames = nansen.config.dloc.DataLocationModel.combineFolderNamesFromPath( ... + sessionFolder, rule.SubfolderLevel, numLevels, rule.Separator); + sessionId = nansen.config.dloc.DataLocationModel.applyExtractionPattern( ... + joinedNames, rule.StringDetectMode, rule.StringDetectInput); + end + function verifyNotReported(testCase, elementPart) testCase.verifyFalse(any(contains(testCase.Report.Element, elementPart)), ... sprintf('%s was mapped and should not be in the report.', elementPart)) @@ -210,9 +218,84 @@ function testRepeatedPatternNamesAreQualifiedByLocation(testCase) testCase.verifyFalse(any(contains(report.Element, "variables[notes]")), ... 'A name used in one location keeps its name.') end + + function testCompositeSessionIdJoinsLevelNames(testCase) + % Day numbers restart in each experiment, so a scanning day is + % identified by the experiment and the day, within its monkey. + [dataLocations, ~, report] = nansen.config.dloc.dsm2DataLocationModel( ... + jsondecode(scanningDayModel()), SessionEntity="scanning_day"); + association = dataLocations(strcmp({dataLocations.Name}, 'association')); + + rule = association.MetaDataDef(strcmp({association.MetaDataDef.VariableName}, 'Session ID')); + testCase.verifyEqual(rule.SubfolderLevel, [1 2 3], ... + 'The experiment level, the monkey level (an ancestor) and the day level.') + testCase.verifyEqual(rule.Separator, '_') + + sessionId = testCase.sessionIdOf(rule, '/data/association/PRE_stim/m1/d1', 3); + testCase.verifyEqual(sessionId, 'PRE_stim_m1_d1') + testCase.verifyTrue(any(contains(report.Reason, "the Session ID joins the names of levels"))) + testCase.verifyFalse(any(contains(report.Reason, "NANSEN needs a single id field"))) + end + + function testFixedPartOfCompositeIdentityIsLeftOut(testCase) + % The localizer has no experiment level; a fixed rule gives the + % experiment, and the Session ID is made of the other levels. + [dataLocations, ~, report] = nansen.config.dloc.dsm2DataLocationModel( ... + jsondecode(scanningDayModel()), SessionEntity="scanning_day"); + localizer = dataLocations(strcmp({dataLocations.Name}, 'localizer')); + + rule = localizer.MetaDataDef(strcmp({localizer.MetaDataDef.VariableName}, 'Session ID')); + testCase.verifyEqual(rule.SubfolderLevel, [1 2]) + testCase.verifyEqual(testCase.sessionIdOf(rule, '/data/localizer/m2/d1', 2), 'm2_d1') + testCase.verifyTrue(any(contains(report.Element, "dataLocations[localizer].metadataMapping[experiment]"))) + end + + function testUnreadablePartOfCompositeIdentityIsReportedOnce(testCase) + % Without a rule for the experiment, the Session ID is left + % unset, and the report gives that reason only + model = replace(scanningDayModel(), '"metadataRef":"experiment"', '"metadataRef":"protocol"'); + + [~, ~, report] = nansen.config.dloc.dsm2DataLocationModel(jsondecode(model), ... + SessionEntity="scanning_day"); + + isAssociation = startsWith(report.Element, "dataLocations[association]"); + testCase.verifyTrue(any(isAssociation & contains(report.Reason, "No extraction rule for this part"))) + testCase.verifyFalse(any(contains(report.Reason, "NANSEN needs a single id field"))) + end end end +function text = scanningDayModel() +%scanningDayModel Scanning days of monkeys in two experiments, numbered from 1 in each + mapping = @(key, extraction) struct('metadataRef', key, 'extraction', extraction); + fromLevel = @(method, pattern, level) struct('method', method, 'pattern', pattern, 'entityLayoutLevel', level); + level = @(name, entityType, pattern) struct('name', name, 'entityType', entityType, 'matchPattern', pattern); + location = @(name, layout, rules) struct( ... + 'identifier', name, 'displayName', name, 'dataCategory', 'raw', 'sourceType', 'filesystem', ... + 'filesystemSource', struct( ... + 'rootStoragePaths', {{struct('identifier', 'main', 'path', ['/data/' name])}}, ... + 'entityLayout', {layout}, 'metadataMapping', {rules})); + dayRules = {mapping('subject_id', fromLevel('substring', ':', 'monkeys')), ... + mapping('day_number', fromLevel('regex', '^d(\d+)$', 'days'))}; + association = location('association', ... + {struct('name', 'experiments', 'matchPattern', '^(PRE|POST)_stim$'), ... + level('monkeys', 'subject', '^m\d+$'), level('days', 'scanning_day', '^d\d+$')}, ... + [{mapping('experiment', fromLevel('substring', ':', 'experiments'))}, dayRules]); + localizer = location('localizer', ... + {level('monkeys', 'subject', '^m\d+$'), level('days', 'scanning_day', '^d\d+$')}, ... + [{mapping('experiment', struct('method', 'fixed', 'value', 'localizer'))}, dayRules]); + model = struct( ... + 'schemaVersion', '0.1.0', ... + 'entityTypes', {{struct('name', 'subject', 'identifierRef', 'subject_id'), ... + struct('name', 'scanning_day', 'identifierRefs', {{'experiment'; 'day_number'}})}}, ... + 'metadataDefinitions', struct( ... + 'subject_id', struct('name', 'subject_id', 'dataType', 'string', 'ofEntity', 'subject'), ... + 'experiment', struct('name', 'experiment', 'dataType', 'string', 'ofEntity', 'scanning_day'), ... + 'day_number', struct('name', 'day_number', 'dataType', 'integer', 'ofEntity', 'scanning_day')), ... + 'dataLocations', {{association, localizer}}); + text = jsonencode(model); +end + function text = twoFormatModel() %twoFormatModel A cell folder per location, the recording as Igor Pro in one and NWB in the other location = @(name, extension, extraPattern) struct( ... From 337b0c6a1a4dd0e7656f44b4cd162c5a73f69043 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Fri, 18 Sep 2026 18:52:07 +0200 Subject: [PATCH 3/5] feat(dloc): approximate character classes when converting file patterns NANSEN resolves a variable's FileNameExpression with dir, so * is its only wildcard. The converter refused every file pattern with a character class, which dropped the only data file of some EBRAINS datasets (Bos et al.: _{session_id}__\d{4}-\d{2}-\d{2}\.pkl$) and most of the files of others (Blackstad et al.: ^TT\d+\.ntt$, ^CSC\d+\.ncs$). A character class (\d, \w, \s, [...]) or a . with its quantifier now becomes *, and the report says the variable was approximated. An approximation that keeps no literal text but dots would match every file in the folder and is still refused. Tokens naming the subject's identity become * like the session's, since the files of a session belong to one subject (Kanter et al.: ^{subject_id}_{session_id}\.nwb$). A cardinality 'many' pattern is reported, because a NANSEN variable holds one file and NANSEN uses the first match. Co-Authored-By: Claude Opus 5 --- .../convertFilePatternToWildcard.m | 94 ++++++++++++++----- .../+config/+dloc/dsm2DataLocationModel.m | 15 ++- .../+config/Dsm2DataLocationModelTest.m | 41 +++++++- 3 files changed, 120 insertions(+), 30 deletions(-) diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m index 246f6f44..b3ef7e10 100644 --- a/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m @@ -1,35 +1,47 @@ -function [fileNameExpression, fileType, isConverted] = convertFilePatternToWildcard(dsmPattern, sessionTokens) +function [fileNameExpression, fileType, isConverted, isApproximate] = convertFilePatternToWildcard(dsmPattern, identityTokens) %convertFilePatternToWildcard Convert a file pattern regex to a NANSEN file name expression % % Syntax: -% [fileNameExpression, fileType, isConverted] = ... -% convertFilePatternToWildcard(dsmPattern, sessionTokens) +% [fileNameExpression, fileType, isConverted, isApproximate] = ... +% convertFilePatternToWildcard(dsmPattern, identityTokens) % % A Dataset Structure Model file pattern is a regular expression that may % contain {token} references to an entity's metadata. NANSEN's -% FileNameExpression is a wildcard pattern anchored with ^ and $, applied -% in the session folder. NANSEN already restricts a virtual session -% folder to the files that contain the session id, so tokens naming the -% session's identity become *: +% FileNameExpression is a wildcard pattern anchored with ^ and $, which +% NANSEN passes to dir, so * is its only wildcard. NANSEN already +% restricts a virtual session folder to the files that contain the +% session id, and the files of a session belong to one subject, so tokens +% naming the identity of the session or of an ancestor become *: % % ^{recording_id}\.ABF$ -> ^*.ABF$ with file type .ABF % -% Escaped punctuation becomes literal and .* becomes *. Any other regex -% construct, or a token that does not name the session identity, has no -% wildcard equivalent; isConverted is then false. +% Escaped punctuation becomes literal and .* becomes *. A character class +% (\d, \w, \s, [...]) or a . with its quantifier also becomes *. The +% wildcard then matches more names than the pattern, and isApproximate +% is true: +% +% ^TT\d+\.ntt$ -> ^TT*.ntt$ +% +% An approximation that keeps no literal text but dots would match every +% file in the folder (^{session_id}\.\d$ would become ^*.*$), so it is +% not converted. Alternation, groups, quantified literals and tokens that +% name other fields have no wildcard equivalent either; isConverted is +% then false. % % Input arguments: -% dsmPattern - The file pattern from the Dataset Structure Model. -% sessionTokens - Names of the metadata fields that identify a session. +% dsmPattern - The file pattern from the Dataset Structure Model. +% identityTokens - Names of the metadata fields that identify the +% session or one of its ancestors. arguments dsmPattern (1,1) string - sessionTokens (1,:) string + identityTokens (1,:) string end fileNameExpression = ""; fileType = ""; isConverted = false; + isApproximate = false; extension = regexp(dsmPattern, "\\\.([A-Za-z0-9]+)\$$", "tokens", "once"); if ~isempty(extension) @@ -39,7 +51,7 @@ tokens = regexp(dsmPattern, "\{([A-Za-z_][A-Za-z0-9_]*)\}", "tokens"); pattern = dsmPattern; for i = 1:numel(tokens) - if ~ismember(tokens{i}{1}, sessionTokens) + if ~ismember(tokens{i}{1}, identityTokens) return end pattern = replace(pattern, "{" + tokens{i}{1} + "}", "*"); @@ -51,16 +63,32 @@ while i <= numel(pattern) c = pattern(i); if c == '\' - if i == numel(pattern) || isstrprop(pattern(i+1), 'alphanum') - return % \d, \w and the like have no wildcard equivalent + if i == numel(pattern) + return + end + escaped = pattern(i+1); + if any(escaped == 'dws') + [wildcard, i] = appendAnyText(wildcard, pattern, i + 2); + isApproximate = true; + elseif isstrprop(escaped, 'alphanum') + return % \b, \D and the like have no wildcard equivalent + else + wildcard(end+1) = escaped; %#ok + i = i + 2; end - wildcard(end+1) = pattern(i+1); %#ok - i = i + 2; - elseif c == '.' && i < numel(pattern) && pattern(i+1) == '*' - wildcard(end+1) = '*'; %#ok - i = i + 2; + elseif c == '[' + closing = find(pattern(i+1:end) == ']', 1); + if isempty(closing) + return + end + [wildcard, i] = appendAnyText(wildcard, pattern, i + closing + 1); + isApproximate = true; + elseif c == '.' + % .* is any text, as * is; any other . is approximated + isApproximate = isApproximate || i == numel(pattern) || pattern(i+1) ~= '*'; + [wildcard, i] = appendAnyText(wildcard, pattern, i + 1); elseif c == '*' || (c == '^' && i == 1) || (c == '$' && i == numel(pattern)) ... - || ~any(c == '.[](){}+?|^$') + || ~any(c == '[](){}+?|^$') wildcard(end+1) = c; %#ok i = i + 1; else @@ -68,6 +96,28 @@ end end + if isApproximate && all(ismember(wildcard, '^$*.')) + isApproximate = false; + return % the wildcard would match every file in the folder + end + fileNameExpression = string(wildcard); isConverted = true; end + +function [wildcard, next] = appendAnyText(wildcard, pattern, next) +%appendAnyText Append * for one pattern element, skipping the quantifier that follows it + if next <= numel(pattern) + if any(pattern(next) == '*+?') + next = next + 1; + elseif pattern(next) == '{' + closing = find(pattern(next+1:end) == '}', 1); + if ~isempty(closing) + next = next + closing + 1; + end + end + end + if isempty(wildcard) || wildcard(end) ~= '*' + wildcard(end+1) = '*'; + end +end diff --git a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m index 59b19c1c..114b3d83 100644 --- a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m +++ b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m @@ -98,7 +98,8 @@ usedFields = [usedFields, usedHere]; %#ok dataLocations = appendItems(dataLocations, item); - variables = appendItems(variables, convertFilePatterns(sessionLevel, item, where, sessionKeys, report)); + variables = appendItems(variables, convertFilePatterns(sessionLevel, item, where, ... + [sessionKeys, subjectKeys], report)); end variables = qualifyRepeatedVariableNames(variables, report); @@ -383,8 +384,11 @@ end end -function variables = convertFilePatterns(sessionLevel, item, where, sessionKeys, report) +function variables = convertFilePatterns(sessionLevel, item, where, identityKeys, report) %convertFilePatterns Map the session level's file patterns to variable model items +% +% identityKeys are the identity fields of the session and the subject, +% the {tokens} a file pattern may use in place of a wildcard. import nansen.config.dloc.dsmconversion.* variables = struct.empty; @@ -399,11 +403,16 @@ end patternWhere = here + "[" + string(filePattern.name) + "]"; - [expression, fileType, isConverted] = convertFilePatternToWildcard(string(filePattern.pattern), sessionKeys); + [expression, fileType, isConverted, isApproximate] = ... + convertFilePatternToWildcard(string(filePattern.pattern), identityKeys); if ~isConverted report.add(patternWhere, "The pattern has no wildcard equivalent.") continue end + report.add(patternWhere, "Approximated as '" + expression + "': a character class became *, " + ... + "so NANSEN may find files the pattern does not match.", isApproximate) + report.add(patternWhere + ".cardinality", "A NANSEN variable is one file per session; " + ... + "where several files match, NANSEN uses the first.", string(getOr(filePattern, 'cardinality', 'one')) == "many") variable = nansen.config.varmodel.VariableModel.getBlankItem(); variable.VariableName = char(matlab.lang.makeValidName(string(filePattern.name))); diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m index 8c4d8e20..5e2cfcc7 100644 --- a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m @@ -188,18 +188,49 @@ function testSliceNeverUsesEndOutsideTheWholeRange(testCase) end function testFilePatternsBecomeWildcards(testCase) - convert = @(p) nansen.config.dloc.dsmconversion.convertFilePatternToWildcard(p, "recording_id"); + convert = @(p) nansen.config.dloc.dsmconversion.convertFilePatternToWildcard( ... + p, ["recording_id", "cell_id"]); - [expression, fileType, isConverted] = convert("^{recording_id}_meta\.json$"); + [expression, fileType, isConverted, isApproximate] = convert("^{recording_id}_meta\.json$"); testCase.verifyTrue(isConverted) + testCase.verifyFalse(isApproximate) testCase.verifyEqual(expression, "^*_meta.json$") testCase.verifyEqual(fileType, ".json") + [expression, ~, isConverted] = convert("^{cell_id}_{recording_id}\.ABF$"); + testCase.verifyTrue(isConverted, 'A token naming an ancestor identity becomes * too.') + testCase.verifyEqual(expression, "^*_*.ABF$") + [~, ~, isConverted] = convert("^{other_id}\.ABF$"); - testCase.verifyFalse(isConverted, 'Only tokens naming the session identity become *.') + testCase.verifyFalse(isConverted, 'Tokens naming other fields have no wildcard equivalent.') + + [expression, ~, ~, isApproximate] = convert("^.*\.ABF$"); + testCase.verifyEqual(expression, "^*.ABF$") + testCase.verifyFalse(isApproximate, '.* is any text, as * is.') + end + + function testCharacterClassesAreApproximated(testCase) + % NANSEN passes the expression to dir, whose only wildcard is *. + convert = @(p) nansen.config.dloc.dsmconversion.convertFilePatternToWildcard(p, "session_id"); + cases = { ... + "^TT\d+\.ntt$", "^TT*.ntt$"; ... + "_ch\d+_\d+\.dat$", "_ch*_*.dat$"; ... + "_feature_[A-Za-z0-9]+\.fd$", "_feature_*.fd$"; ... + "_{session_id}__\d{4}-\d{2}-\d{2}\.pkl$", "_*__*-*-*.pkl$"; ... + "^{session_id}\.eeg\d?$", "^*.eeg*$"}; + for i = 1:height(cases) + [expression, ~, isConverted, isApproximate] = convert(cases{i, 1}); + testCase.verifyTrue(isConverted, cases{i, 1}) + testCase.verifyTrue(isApproximate, cases{i, 1}) + testCase.verifyEqual(expression, cases{i, 2}, cases{i, 1}) + end + end - [~, ~, isConverted] = convert("^\d+\.ABF$"); - testCase.verifyFalse(isConverted, '\d has no wildcard equivalent.') + function testApproximationMatchingEveryFileIsRefused(testCase) + [expression, ~, isConverted] = nansen.config.dloc.dsmconversion.convertFilePatternToWildcard( ... + "^{session_id}\.\d$", "session_id"); + testCase.verifyFalse(isConverted, '^*.*$ would match every file in the folder.') + testCase.verifyEqual(expression, "") end function testRepeatedPatternNamesAreQualifiedByLocation(testCase) From 2400e2a5027b3b07f138186f014898e35dbda661 Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sat, 19 Sep 2026 12:58:40 +0200 Subject: [PATCH 4/5] feat(dloc): convert file patterns of layout levels below the session The DSM converter dropped every layout level below the session level, so a dataset whose files sit in a folder of the session, such as the runs in each scanning day's funct folder in murris-2021, got no variables. A NANSEN variable can name a subfolder of the session folder. Folder levels with a fixed name below the session now become the Subfolder of the variables made from the file patterns reached through them. The first level that is neither a fixed folder nor a level of files, and the levels below it, are still dropped and reported. Where a session has several files of a level, the report says that NANSEN uses the first. With levels below the session, one data location can have two file patterns of the same name, in the session folder and in a subfolder. Such a name is now also prefixed with the variable's subfolder (funct_sidecar), where before both became _ and the second insert into the variable model failed. Names still equal after that, from two levels in the same folder, are numbered. An extension of several parts, such as \.nii\.gz$, now gives the file type .nii.gz, and listFileAdapters removes only the leading dot of an extension, so such a type can match an adapter that lists nii.gz. Co-Authored-By: Claude Opus 5 --- .../convertFilePatternToWildcard.m | 10 +- .../+config/+dloc/dsm2DataLocationModel.m | 126 +++++++++++++++--- code/+nansen/+dataio/listFileAdapters.m | 4 +- .../datasetstructure/runs-below-session.json | 49 +++++++ .../Dsm2DataLocationFilesBelowSessionTest.m | 89 +++++++++++++ 5 files changed, 254 insertions(+), 24 deletions(-) create mode 100644 tests/+nansen/+fixture/datasetstructure/runs-below-session.json create mode 100644 tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m diff --git a/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m index b3ef7e10..a77db4b2 100644 --- a/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m +++ b/code/+nansen/+config/+dloc/+dsmconversion/convertFilePatternToWildcard.m @@ -15,6 +15,9 @@ % % ^{recording_id}\.ABF$ -> ^*.ABF$ with file type .ABF % +% An extension of several parts gives a file type of several parts: +% \.nii\.gz$ has the file type .nii.gz. +% % Escaped punctuation becomes literal and .* becomes *. A character class % (\d, \w, \s, [...]) or a . with its quantifier also becomes *. The % wildcard then matches more names than the pattern, and isApproximate @@ -43,9 +46,12 @@ isConverted = false; isApproximate = false; - extension = regexp(dsmPattern, "\\\.([A-Za-z0-9]+)\$$", "tokens", "once"); + % An extension may have several parts, as .nii.gz. Each part starts + % with a letter, so that a version number before the extension, as in + % v1\.2\.txt, is not taken for part of it. + extension = regexp(dsmPattern, "((?:\\\.[A-Za-z][A-Za-z0-9]*)+)\$$", "tokens", "once"); if ~isempty(extension) - fileType = "." + extension{1}; + fileType = string(erase(extension{1}, "\")); end tokens = regexp(dsmPattern, "\{([A-Za-z_][A-Za-z0-9_]*)\}", "tokens"); diff --git a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m index 114b3d83..b76dec2f 100644 --- a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m +++ b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m @@ -90,7 +90,7 @@ continue end - [item, sessionLevel, usedHere] = convertLocation(location, where, definitions, ... + [item, sessionLevel, usedHere, levelsBelowSession] = convertLocation(location, where, definitions, ... sessionEntity, subjectEntity, sessionKeys, subjectKeys, report); if isempty(item) continue @@ -100,6 +100,8 @@ dataLocations = appendItems(dataLocations, item); variables = appendItems(variables, convertFilePatterns(sessionLevel, item, where, ... [sessionKeys, subjectKeys], report)); + variables = appendItems(variables, convertFilePatternsBelowSession(levelsBelowSession, ... + item, where, [sessionKeys, subjectKeys], report)); end variables = qualifyRepeatedVariableNames(variables, report); @@ -115,13 +117,18 @@ % ------------------------------------------------------------------------- -function [item, sessionLevel, usedFields] = convertLocation(location, where, definitions, ... +function [item, sessionLevel, usedFields, levelsBelowSession] = convertLocation(location, where, definitions, ... sessionEntity, subjectEntity, sessionKeys, subjectKeys, report) %convertLocation Convert one filesystem data location +% +% levelsBelowSession are the layout levels under the session level. NANSEN +% walks folders down to the session only; convertFilePatternsBelowSession +% finds the files below it that a variable can still point to. item = []; sessionLevel = []; usedFields = string.empty(1, 0); + levelsBelowSession = {}; source = location.filesystemSource; layout = asList(source, 'entityLayout'); @@ -131,10 +138,7 @@ report.add(where, "No layout level holds the session entity '" + sessionEntity + "', so NANSEN could not find sessions here.") return end - for j = sessionIndex+1:numel(layout) - report.add(where + ".entityLayout[" + string(layout{j}.name) + "]", ... - "NANSEN treats the deepest level it walks as the session; levels below the session level are dropped.") - end + levelsBelowSession = layout(sessionIndex+1:end); layout = layout(1:sessionIndex); sessionLevel = layout{end}; @@ -384,13 +388,65 @@ end end -function variables = convertFilePatterns(sessionLevel, item, where, identityKeys, report) -%convertFilePatterns Map the session level's file patterns to variable model items +function variables = convertFilePatternsBelowSession(levels, item, where, identityKeys, report) +%convertFilePatternsBelowSession Map file patterns of levels below the session to variables +% +% A NANSEN variable is a file in the session folder or in a subfolder of +% it with a fixed name. Folder levels with a fixed name therefore become +% the variable's Subfolder, and the file patterns of the levels reached +% through them become variables. The first level that is neither a +% fixed folder nor a level of files ends the walk; it and the levels +% below it are dropped. + variables = struct.empty; + subfolders = strings(1, 0); + for j = 1:numel(levels) + level = levels{j}; + levelWhere = where + ".entityLayout[" + string(level.name) + "]"; + isFile = string(getOr(level, 'fileSystemType', 'folder')) == "file"; + isFixedFolder = ~isFile && isfield(level, 'fixedName') && ~getOr(level, 'isVariable', true); + + if isFixedFolder + subfolders(end+1) = string(level.fixedName); %#ok + variables = appendItems(variables, convertFilePatterns(level, item, where, ... + identityKeys, report, strjoin(subfolders, "/"))); + elseif isFile + variables = appendItems(variables, convertFilePatterns(level, item, where, ... + identityKeys, report, strjoin(subfolders, "/"))); + report.add(levelWhere, "The files of this level are found in the session folder" + ... + subfolderText(subfolders) + ". Where a session has several of them, " + ... + "NANSEN uses the first.", isfield(level, 'entityType')) + else + for k = j:numel(levels) + report.add(where + ".entityLayout[" + string(levels{k}.name) + "]", ... + "NANSEN finds files below the session only through folders with a fixed name; " + ... + "this level and the levels below it are dropped.") + end + return + end + end +end + +function text = subfolderText(subfolders) + if isempty(subfolders) + text = ""; + else + text = "'s subfolder " + strjoin(subfolders, "/"); + end +end + +function variables = convertFilePatterns(level, item, where, identityKeys, report, subfolder) +%convertFilePatterns Map a layout level's file patterns to variable model items % % identityKeys are the identity fields of the session and the subject, -% the {tokens} a file pattern may use in place of a wildcard. +% the {tokens} a file pattern may use in place of a wildcard. subfolder +% is the folder below the session folder where the files are, "" for +% files in the session folder. import nansen.config.dloc.dsmconversion.* + if nargin < 6 + subfolder = ""; + end + sessionLevel = level; variables = struct.empty; here = where + ".entityLayout[" + string(sessionLevel.name) + "].filePatterns"; patterns = asList(sessionLevel, 'filePatterns'); @@ -418,6 +474,7 @@ variable.VariableName = char(matlab.lang.makeValidName(string(filePattern.name))); variable.DataLocation = item.Name; variable.DataLocationUuid = item.Uuid; + variable.Subfolder = char(subfolder); variable.FileNameExpression = char(expression); variable.FileType = char(fileType); variable.FileAdapter = findFileAdapter(fileType, patternWhere, report); @@ -430,27 +487,54 @@ end function variables = qualifyRepeatedVariableNames(variables, report) -%qualifyRepeatedVariableNames Prefix a name used in several data locations with the location +%qualifyRepeatedVariableNames Qualify a name that several file patterns share % % A file pattern name is unique within one layout level, but a NANSEN % variable name is unique within a project, and adding a second variable -% with the same name fails. Every variable whose name occurs in more than -% one data location is renamed to _, so no location -% keeps the bare name only because it was converted first. +% with the same name fails. A name that occurs in more than one data +% location is prefixed with the data location, _, +% so no location keeps the bare name only because it was converted +% first. A name that occurs more than once within one data location, on +% levels in different folders of the session, is also prefixed with the +% variable's subfolder, _; the variable of the session +% folder itself has no subfolder to add. Names that are still equal +% after this, from two levels in the same folder, are numbered. if isempty(variables) return end names = string({variables.VariableName}); - [~, ~, groupIndex] = unique(names); - occurrences = accumarray(groupIndex(:), 1); - - for i = find(occurrences(groupIndex)' > 1) - newName = matlab.lang.makeValidName(string(variables(i).DataLocation) + "_" + names(i)); - report.add("variables[" + names(i) + "]", "The file pattern name is used in several data locations; " + ... - "renamed to '" + newName + "' in data location " + string(variables(i).DataLocation) + ".") - variables(i).VariableName = char(newName); + locations = string({variables.DataLocation}); + subfolders = string({variables.Subfolder}); + + newNames = names; + for i = 1:numel(variables) + isSameName = names == names(i); + parts = strings(1, 0); + if numel(unique(locations(isSameName))) > 1 + parts(end+1) = locations(i); %#ok + end + if sum(isSameName & locations == locations(i)) > 1 && strlength(subfolders(i)) > 0 + parts(end+1) = replace(subfolders(i), "/", "_"); %#ok + end + newNames(i) = matlab.lang.makeValidName(strjoin([parts, names(i)], "_")); + end + newNames = matlab.lang.makeUniqueStrings(newNames); + + for i = find(newNames ~= names) + report.add("variables[" + names(i) + "]", "The file pattern name is used by several file patterns; " + ... + "renamed to '" + newNames(i) + "' in data location " + locations(i) + folderText(subfolders(i)) + ".") + variables(i).VariableName = char(newNames(i)); + end +end + +function text = folderText(subfolder) +%folderText The subfolder of a variable for a report, "" for the session folder + if strlength(subfolder) == 0 + text = ""; + else + text = ", subfolder " + subfolder; end end diff --git a/code/+nansen/+dataio/listFileAdapters.m b/code/+nansen/+dataio/listFileAdapters.m index 9c4be633..53519045 100644 --- a/code/+nansen/+dataio/listFileAdapters.m +++ b/code/+nansen/+dataio/listFileAdapters.m @@ -21,7 +21,9 @@ fileAdapterList = table2struct(project.getTable('FileAdapter', refresh)); if nargin < 1; fileExtension = ''; end - if ~isempty(fileExtension); fileExtension = strrep(fileExtension, '.', ''); end + % Only the leading dot is removed, so that an extension of several + % parts, such as .nii.gz, keeps its inner dot and matches nii.gz + if ~isempty(fileExtension); fileExtension = regexprep(fileExtension, '^\.', ''); end if ~isempty(fileExtension) validationFcn = @(extList) any(contains(extList, fileExtension, "IgnoreCase", true)); diff --git a/tests/+nansen/+fixture/datasetstructure/runs-below-session.json b/tests/+nansen/+fixture/datasetstructure/runs-below-session.json new file mode 100644 index 00000000..d34a98a1 --- /dev/null +++ b/tests/+nansen/+fixture/datasetstructure/runs-below-session.json @@ -0,0 +1,49 @@ +{ + "schemaVersion": "0.1.0", + "entityTypes": [ + {"name": "subject", "description": "One monkey.", "isPrimary": true, "identifierRef": "subject_id"}, + {"name": "session", "description": "One scanning day of one monkey.", "identifierRef": "session_id"}, + {"name": "run", "description": "One functional run of a scanning day, in the day's funct folder.", "identifierRef": "run_number"} + ], + "entityRelationships": [ + {"sourceEntity": "subject", "targetEntity": "session", "relationType": "oneToMany", "relationName": "hasSessions"}, + {"sourceEntity": "session", "targetEntity": "run", "relationType": "oneToMany", "relationName": "hasRuns"} + ], + "metadataDefinitions": { + "subject_id": {"name": "subject_id", "title": "Monkey", "dataType": "string", "ofEntity": "subject"}, + "session_id": {"name": "session_id", "title": "Day", "dataType": "string", "ofEntity": "session"}, + "run_number": {"name": "run_number", "title": "Run", "dataType": "integer", "ofEntity": "run"} + }, + "dataLocations": [ + { + "identifier": "fmri", + "displayName": "fMRI", + "dataCategory": "raw", + "access": "read", + "sourceType": "filesystem", + "filesystemSource": { + "rootStoragePaths": [{"identifier": "local-copy", "path": "/data/fmri"}], + "entityLayout": [ + {"name": "monkeys", "entityType": "subject", "matchPattern": "^m\\d+$"}, + {"name": "days", "entityType": "session", "matchPattern": "^d\\d+$"}, + {"name": "funct-folder", "isVariable": false, "fixedName": "funct"}, + { + "name": "run-files", + "entityType": "run", + "fileSystemType": "file", + "matchPattern": "_run_\\d+\\.(nii\\.gz|json)$", + "filePatterns": [ + {"name": "bold", "pattern": "\\.nii\\.gz$", "isRequired": true, "cardinality": "one", "description": "Functional run, NIfTI"}, + {"name": "sidecar", "pattern": "\\.json$", "cardinality": "one", "description": "Acquisition parameters of the run"} + ] + } + ], + "metadataMapping": [ + {"metadataRef": "subject_id", "extraction": {"method": "substring", "pattern": ":", "entityLayoutLevel": "monkeys"}}, + {"metadataRef": "session_id", "extraction": {"method": "substring", "pattern": ":", "entityLayoutLevel": "days"}}, + {"metadataRef": "run_number", "extraction": {"method": "regex", "pattern": "_run_(\\d+)\\.(nii\\.gz|json)$", "entityLayoutLevel": "run-files"}} + ] + } + } + ] +} diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m new file mode 100644 index 00000000..8d47cfe4 --- /dev/null +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m @@ -0,0 +1,89 @@ +classdef Dsm2DataLocationFilesBelowSessionTest < matlab.unittest.TestCase +%Dsm2DataLocationFilesBelowSessionTest - File patterns on layout levels below the session +% +% Uses a model of monkey folders with a folder per scanning day (the +% session), a folder named funct in each day, and run files in it with +% the file patterns bold (.nii.gz) and sidecar (.json). +% +% Run tests: +% runtests('nansen.unittest.config.Dsm2DataLocationFilesBelowSessionTest') + + properties + Model + end + + methods (TestClassSetup) + function loadModel(testCase) + fixture = fullfile(fileparts(fileparts(fileparts(mfilename('fullpath')))), ... + '+fixture', 'datasetstructure', 'runs-below-session.json'); + testCase.Model = jsondecode(fileread(fixture)); + end + end + + methods (Test) + function testFilePatternsInFixedFolderBecomeVariables(testCase) + [~, variables] = nansen.config.dloc.dsm2DataLocationModel(testCase.Model); + + testCase.verifyEqual(sort(string({variables.VariableName})), ["bold", "sidecar"]) + testCase.verifyEqual(string({variables.Subfolder}), ["funct", "funct"]) + end + + function testExtensionOfSeveralPartsIsFileType(testCase) + [~, variables] = nansen.config.dloc.dsm2DataLocationModel(testCase.Model); + + bold = variables(strcmp({variables.VariableName}, 'bold')); + testCase.verifyEqual(bold.FileType, '.nii.gz') + end + + function testNameInSessionAndFixedFolderIsQualifiedBySubfolder(testCase) + % A sidecar file in the day folder and one in its funct folder + model = testCase.Model; + model.dataLocations.filesystemSource.entityLayout{2}.filePatterns = ... + {struct('name', 'sidecar', 'pattern', '_day\.json$')}; + + [~, variables, report] = nansen.config.dloc.dsm2DataLocationModel(model); + + names = string({variables.VariableName}); + testCase.verifyEqual(sort(names), ["bold", "funct_sidecar", "sidecar"]) + testCase.verifyEqual(variables(names == "funct_sidecar").Subfolder, 'funct') + testCase.verifyEmpty(variables(names == "sidecar").Subfolder) + testCase.verifyTrue(any(contains(report.Element, "variables[sidecar]"))) + end + + function testNameInNestedFixedFoldersIsQualifiedBySubfolder(testCase) + % funct/ and funct/raw/ each hold a NIfTI file + model = testCase.Model; + layout = model.dataLocations.filesystemSource.entityLayout(:); + nifti = {struct('name', 'nifti', 'pattern', '\.nii$')}; + layout{3}.filePatterns = nifti; + raw = struct('name', 'raw-folder', 'isVariable', false, 'fixedName', 'raw', 'filePatterns', {nifti}); + model.dataLocations.filesystemSource.entityLayout = [layout(1:3); {raw}; layout(4:end)]; + + [~, variables] = nansen.config.dloc.dsm2DataLocationModel(model); + + names = string({variables.VariableName}); + testCase.verifyEqual(sort(names(contains(names, "nifti"))), ["funct_nifti", "funct_raw_nifti"]) + testCase.verifyEqual(variables(names == "funct_raw_nifti").Subfolder, 'funct/raw') + end + + function testSeveralFilesPerSessionIsReported(testCase) + [~, ~, report] = nansen.config.dloc.dsm2DataLocationModel(testCase.Model); + + isRunLevel = endsWith(report.Element, "entityLayout[run-files]"); + testCase.verifyTrue(any(isRunLevel & contains(report.Reason, "uses the first"))) + end + + function testLevelsBelowVariableFolderAreDropped(testCase) + model = testCase.Model; + layout = model.dataLocations.filesystemSource.entityLayout; + layout{3} = struct('name', 'protocol-folders', 'matchPattern', '^[a-z]+$'); + model.dataLocations.filesystemSource.entityLayout = layout; + + [~, variables, report] = nansen.config.dloc.dsm2DataLocationModel(model); + + testCase.verifyEmpty(variables) + testCase.verifyTrue(any(endsWith(report.Element, "entityLayout[run-files]") ... + & contains(report.Reason, "dropped"))) + end + end +end From 556df193b7c91a0d515196e0438744c0e0aeaaef Mon Sep 17 00:00:00 2001 From: ehennestad Date: Sat, 19 Sep 2026 18:13:52 +0200 Subject: [PATCH 5/5] fix(fileadapter): find an adapter for an extension of several parts nansen.load and nansen.save choose an adapter with detectFileAdapterForFilepath, which took the extension from fileparts (.gz for run.nii.gz) and compared it with ismember, so an adapter that lists nii.gz was never found. The DSM converter, which asks listFileAdapters, already found it. An adapter now supports a file whose name ends with one of its SupportedFileTypes. Where several adapters support the file, one with the longest matching type is used, so nii.gz wins over gz. Case is ignored, as in listFileAdapters. The adapter list can be passed in, so the choice can be tested without a project. Co-Authored-By: Claude Opus 5 --- .../+internal/detectFileAdapterForFilepath.m | 35 +++++++++---- .../DetectFileAdapterForFilepathTest.m | 50 +++++++++++++++++++ 2 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 tests/+nansen/+unittest/+plugin/DetectFileAdapterForFilepathTest.m diff --git a/code/+nansen/+plugin/+fileadapter/+internal/detectFileAdapterForFilepath.m b/code/+nansen/+plugin/+fileadapter/+internal/detectFileAdapterForFilepath.m index 6a17646a..4d9f8343 100644 --- a/code/+nansen/+plugin/+fileadapter/+internal/detectFileAdapterForFilepath.m +++ b/code/+nansen/+plugin/+fileadapter/+internal/detectFileAdapterForFilepath.m @@ -1,21 +1,34 @@ -function [fileAdapterName, isDynamic] = detectFileAdapterForFilepath(filePath) +function [fileAdapterName, isDynamic] = detectFileAdapterForFilepath(filePath, fileAdapterList) % detectFileAdapterForFilepath - Locate file adapter for a given filename and extension +% +% [fileAdapterName, isDynamic] = detectFileAdapterForFilepath(filePath) +% returns the first file adapter of nansen.dataio.listFileAdapters() +% that supports the file. An adapter supports a file whose name ends +% with one of its SupportedFileTypes, so a type of several parts, such +% as nii.gz, is matched as well. Where several adapters support the +% file, one with the longest matching type is used (nii.gz before gz). +% Case is ignored, as in listFileAdapters. +% +% ... = detectFileAdapterForFilepath(filePath, fileAdapterList) looks in +% the given list, a struct array with the fields of listFileAdapters. - fileAdapterList = nansen.dataio.listFileAdapters(); + if nargin < 2 + fileAdapterList = nansen.dataio.listFileAdapters(); + end - [~, ~, fileExtension] = fileparts(filePath); + [~, fileName, fileExtension] = fileparts(filePath); + fileName = string(fileName) + string(fileExtension); - supportsFile = false(1, numel(fileAdapterList)); - % Find matching file type + % Length of each adapter's longest supported type that the name ends with + matchLength = zeros(1, numel(fileAdapterList)); for i = 1:numel(fileAdapterList) - currentFileAdapterInfo = fileAdapterList(i); - - if ismember(fileExtension, currentFileAdapterInfo.SupportedFileTypes) || ... - ismember(extractAfter(fileExtension, '.'), currentFileAdapterInfo.SupportedFileTypes) - supportsFile(i) = true; + fileTypes = "." + regexprep(string(fileAdapterList(i).SupportedFileTypes), '^\.', ''); + isMatch = endsWith(fileName, fileTypes, 'IgnoreCase', true); + if any(isMatch) + matchLength(i) = max(strlength(fileTypes(isMatch))); end end - fileAdapterList = fileAdapterList(supportsFile); + fileAdapterList = fileAdapterList(matchLength > 0 & matchLength == max(matchLength)); if isempty(fileAdapterList) error('No file adapters exist that can open files of type "%s"', fileExtension) diff --git a/tests/+nansen/+unittest/+plugin/DetectFileAdapterForFilepathTest.m b/tests/+nansen/+unittest/+plugin/DetectFileAdapterForFilepathTest.m new file mode 100644 index 00000000..7e3f9629 --- /dev/null +++ b/tests/+nansen/+unittest/+plugin/DetectFileAdapterForFilepathTest.m @@ -0,0 +1,50 @@ +classdef DetectFileAdapterForFilepathTest < matlab.unittest.TestCase +%DetectFileAdapterForFilepathTest - Choose the file adapter that nansen.load and nansen.save use +% +% The adapter list is given to the function, so no project is needed. +% +% Run tests: +% runtests('nansen.unittest.plugin.DetectFileAdapterForFilepathTest') + + properties + AdapterList + end + + methods (TestMethodSetup) + function createAdapterList(testCase) + testCase.AdapterList = struct( ... + 'FunctionName', {'test.Gzip', 'test.NIfTI', 'test.MatFile'}, ... + 'SupportedFileTypes', {{'gz'}, {'nii', 'nii.gz'}, {'.mat'}}, ... + 'IsDynamic', {false, false, true}); + end + end + + methods (Test) + function testSingleExtension(testCase) + [name, isDynamic] = nansen.plugin.fileadapter.internal.detectFileAdapterForFilepath( ... + fullfile('data', 'values.mat'), testCase.AdapterList); + + testCase.verifyEqual(name, 'test.MatFile') + testCase.verifyTrue(isDynamic) + end + + function testExtensionOfSeveralPartsPrefersLongestType(testCase) + name = nansen.plugin.fileadapter.internal.detectFileAdapterForFilepath( ... + fullfile('data', 'run.nii.gz'), testCase.AdapterList); + + testCase.verifyEqual(name, 'test.NIfTI') + end + + function testCaseIsIgnored(testCase) + name = nansen.plugin.fileadapter.internal.detectFileAdapterForFilepath( ... + fullfile('data', 'RUN.NII'), testCase.AdapterList); + + testCase.verifyEqual(name, 'test.NIfTI') + end + + function testUnsupportedTypeErrors(testCase) + testCase.verifyError(@() nansen.plugin.fileadapter.internal.detectFileAdapterForFilepath( ... + fullfile('data', 'trace.abf'), testCase.AdapterList), ?MException) + end + end +end