diff --git a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m index b76dec2f..31f32d72 100644 --- a/code/+nansen/+config/+dloc/dsm2DataLocationModel.m +++ b/code/+nansen/+config/+dloc/dsm2DataLocationModel.m @@ -472,6 +472,7 @@ variable = nansen.config.varmodel.VariableModel.getBlankItem(); variable.VariableName = char(matlab.lang.makeValidName(string(filePattern.name))); + variable.Description = char(getOr(filePattern, 'description', '')); variable.DataLocation = item.Name; variable.DataLocationUuid = item.Uuid; variable.Subfolder = char(subfolder); diff --git a/code/+nansen/+config/+project/@Project/Project.m b/code/+nansen/+config/+project/@Project/Project.m index 724383a4..80d7426a 100644 --- a/code/+nansen/+config/+project/@Project/Project.m +++ b/code/+nansen/+config/+project/@Project/Project.m @@ -227,6 +227,67 @@ function setAutoDownloadRemoteFiles(obj, isAutoDownload) obj.Preferences.AutoDownloadRemoteFiles = isAutoDownload; end + function sessions = getSessionObjects(obj, sessionIDs) + %getSessionObjects Session objects of the project that can load and save data + % + % sessions = project.getSessionObjects() returns an object for + % every session in the project's master session table. + % + % sessions = project.getSessionObjects(sessionIDs) returns the + % sessions with the given IDs, in the given order. + % + % Each object has the project's data location and variable models, + % which Session.loadData and Session.saveData need; the NANSEN app + % gives them to the sessions it creates in the same way. The + % project must be the current project, because the session table + % completes its data location paths from the current project. + arguments + obj (1,1) nansen.config.project.Project + sessionIDs (1,:) string = string.empty(1, 0) + end + + currentProject = nansen.getCurrentProject(); + if isempty(currentProject) || ~strcmp(currentProject.Name, obj.Name) + error('NANSEN:Project:NotCurrentProject', ... + ['Sessions can only be created for the current project. ', ... + 'Make "%s" the current project with nansen.ProjectManager().changeProject("%s").'], ... + obj.Name, obj.Name) + end + + sessionTable = obj.MetaTableCatalog.getMasterMetaTable('session'); + tableSessionIDs = string(sessionTable.entries.(sessionTable.SchemaIdName)); + if isempty(sessionIDs) + rowIndices = 1:numel(tableSessionIDs); + else + [isFound, rowIndices] = ismember(sessionIDs, tableSessionIDs); + if ~all(isFound) + error('NANSEN:Project:SessionNotFound', ... + 'The session table of "%s" has no session with the ID: %s.', ... + obj.Name, strjoin(sessionIDs(~isFound), ', ')) + end + end + + sessions = sessionTable.getMetaObjects(rowIndices, ... + 'DataLocationModel', obj.DataLocationModel, ... + 'VariableModel', obj.VariableModel); + end + + function setWriteMetaTableCsvCopies(obj, isEnabled) + %setWriteMetaTableCsvCopies Set whether saving a metatable also writes a CSV copy + % + % project.setWriteMetaTableCsvCopies(true) makes MetaTable.save + % write a CSV copy of each of the project's tables next to its + % .mat file, so that the tables can be read without MATLAB. The + % default is false. + % + % See also nansen.metadata.MetaTable/writeCsvCopy + arguments + obj (1,1) nansen.config.project.Project + isEnabled (1,1) logical + end + obj.Preferences.WriteMetaTableCsvCopies = isEnabled; + end + function initializeProjectFolder(obj) % Todo: implement? I.e if a project object is created % programmatically diff --git a/code/+nansen/+config/+varmodel/VariableModel.m b/code/+nansen/+config/+varmodel/VariableModel.m index f8b79249..eabc1490 100644 --- a/code/+nansen/+config/+varmodel/VariableModel.m +++ b/code/+nansen/+config/+varmodel/VariableModel.m @@ -45,6 +45,7 @@ S = struct(... 'VariableName', '', ... % Name of variable + 'Description', '', ... % What the file holds, e.g. its channels, columns or units (optional) 'DataLocation', '', ... % todo: rename DataLocationName? Name of datalocation where variable is stored. 'DataLocationUuid', '', ... % uuid of datalocation variable belongs to (internal) 'Subfolder', '', ... % Subfolder within sessionfolder where variable is saved to file (optional) @@ -220,6 +221,10 @@ function updateDefaultValues(obj) if ~isfield(obj.Data, 'PathInFile') [obj.Data(:).PathInFile] = deal(''); end + + if ~isfield(obj.Data, 'Description') + [obj.Data(:).Description] = deal(''); + end end % % Variable interaction and utility methods @@ -499,6 +504,11 @@ function removeDataVariableSet(obj, variableList) methods % Override superclass methods function newItem = insertItem(obj, newItem) import nansen.config.varmodel.event.VariableAddedEventData + + % An item made before a field was added to the blank item, such + % as a variable template of a module, gets the field with its + % blank value, because the catalog requires every field + newItem = addMissingFields(newItem); newItem = obj.updateVariableDataType(newItem); newItem = insertItem@utility.data.StorableCatalog(obj, newItem); @@ -668,3 +678,12 @@ function disableNotifications(obj) function tf = useDefaultDataLocation(dataLocationName) tf = strcmp(dataLocationName, 'DEFAULT') || isempty(dataLocationName); end + +function item = addMissingFields(item) +%addMissingFields Give an item every field of the blank item that it lacks + blankItem = nansen.config.varmodel.VariableModel.getBlankItem(); + missingFields = setdiff(fieldnames(blankItem), fieldnames(item)); + for i = 1:numel(missingFields) + item.(missingFields{i}) = blankItem.(missingFields{i}); + end +end diff --git a/code/+nansen/+metadata/@MetaTable/MetaTable.m b/code/+nansen/+metadata/@MetaTable/MetaTable.m index c0da8081..12133c22 100644 --- a/code/+nansen/+metadata/@MetaTable/MetaTable.m +++ b/code/+nansen/+metadata/@MetaTable/MetaTable.m @@ -236,11 +236,45 @@ function setAsDummy(obj) wasSaved = save@nansen.metadata.mixin.VersionedFile(obj, force); if wasSaved fprintf('MetaTable saved to %s\n', obj.filepath) + if obj.isCsvCopyEnabled() + % The table is saved; a copy that cannot be written + % is reported without failing the save + try + obj.writeCsvCopy() + catch exception + warning('NANSEN:MetaTable:CsvCopyFailed', ... + 'The table was saved, but its CSV copy could not be written: %s', ... + exception.message) + end + end end if ~nargout; clear wasSaved; end end + function csvFilePath = writeCsvCopy(obj) + %writeCsvCopy Write the table's entries to a CSV file next to its .mat file + % + % csvFilePath = metaTable.writeCsvCopy() writes the entries to a + % CSV file with the name of the table's file, for reading the + % table without MATLAB. Columns of text, numbers, logical values + % and dates are copied. The DataLocation column becomes one column + % per data location, DataLocation_, with the path of the + % row's folder or file there, empty where the row has none. + % Columns of other values, such as Progress and Notebook, are left + % out. + % + % save writes the copy after each save when the table belongs to + % the current project and its preference WriteMetaTableCsvCopies + % is true (Project.setWriteMetaTableCsvCopies). + + [folderPath, fileName] = fileparts(obj.filepath); + csvFilePath = fullfile(folderPath, fileName + ".csv"); + writetable(tableForCsv(obj.entries), csvFilePath) + + if ~nargout; clear csvFilePath; end + end + function load(obj) %load Load a MetaTable from file % @@ -1161,6 +1195,34 @@ function resetMetaObjectCache(obj) end end + methods (Access = private) % CSV copy + + function tf = isCsvCopyEnabled(obj) + %isCsvCopyEnabled Whether the table's project asks for a CSV copy + % + % The preference belongs to a project, and a table can be saved + % while another project is current, so only a table in the folder + % of the current project uses the current project's preference. + tf = false; + try + project = nansen.getCurrentProject(); + catch exception + % Without a user session there is no project to ask + if strcmp(exception.identifier, 'NANSEN:NoActiveUserSession') + return + end + rethrow(exception) + end + if isempty(project) || ~startsWith(string(obj.filepath), string(project.FolderPath)) + return + end + + preferences = project.Preferences; + tf = isfield(preferences, 'WriteMetaTableCsvCopies') ... + && preferences.WriteMetaTableCsvCopies; + end + end + methods (Access = protected) % Data location column function tableEntries = reduceDataLocationColumn(obj, tableEntries) @@ -1853,3 +1915,67 @@ function removeDuplicates(obj) value = fcn(value); end + +function csvTable = tableForCsv(entries) +%tableForCsv The columns of a metatable that a CSV file can hold +% Columns of text, numbers, logical values and dates are kept. The +% DataLocation column becomes one text column per data location with the +% path of each row's folder or file there. Other columns are left out. + + csvTable = table('Size', [height(entries), 0]); + for variableName = string(entries.Properties.VariableNames) + values = entries.(variableName); + if variableName == "DataLocation" && (isstruct(values) || iscell(values)) + csvTable = [csvTable, dataLocationPaths(values)]; %#ok + elseif isnumeric(values) || islogical(values) || isdatetime(values) ... + || isstring(values) || iscategorical(values) + csvTable.(variableName) = values; + elseif iscell(values) && all(cellfun(@isScalarValue, values)) + % Empty values are written as empty text, so that the column + % keeps one type + values(cellfun(@isempty, values)) = {''}; + csvTable.(variableName) = values; + end + end +end + +function pathTable = dataLocationPaths(dataLocations) +%dataLocationPaths One column per data location with the path of each row there +% dataLocations is a struct array with one row per table row, or a cell +% array with one struct array per table row when rows have different +% numbers of data locations. The columns are named by the data +% locations' Name, in the order they first occur; a row without a data +% location of that name has an empty path. + numRows = size(dataLocations, 1); + if isstruct(dataLocations) + rows = num2cell(dataLocations, 2); + else + rows = dataLocations(:); + end + + names = strings(1, 0); + paths = strings(numRows, 0); + for i = 1:numRows + for location = reshape(rows{i}, 1, []) + columnIndex = find(names == string(location.Name), 1); + if isempty(columnIndex) + names(end+1) = string(location.Name); %#ok + paths(:, end+1) = ""; %#ok + columnIndex = numel(names); + end + if isfield(location, 'RootPath') && ~isempty(location.Subfolders) + paths(i, columnIndex) = fullfile(location.RootPath, location.Subfolders); + end + end + end + pathTable = array2table(paths, 'VariableNames', "DataLocation_" + names); +end + +function tf = isScalarValue(value) +%isScalarValue Whether a cell of a table column holds one text, number, logical value or date, or nothing +% An empty struct, as in Progress and Notebook, is still a structure and +% does not count as nothing. + isEmptyTextOrNumber = isempty(value) && (isnumeric(value) || isstring(value)); + tf = ischar(value) || isEmptyTextOrNumber || (isscalar(value) && (isstring(value) ... + || isnumeric(value) || islogical(value) || isdatetime(value))); +end diff --git a/tests/+nansen/+unittest/+config/+project/GetSessionObjectsTest.m b/tests/+nansen/+unittest/+config/+project/GetSessionObjectsTest.m new file mode 100644 index 00000000..b46a4e0b --- /dev/null +++ b/tests/+nansen/+unittest/+config/+project/GetSessionObjectsTest.m @@ -0,0 +1,69 @@ +classdef GetSessionObjectsTest < matlab.unittest.TestCase +%GetSessionObjectsTest - Tests for Project.getSessionObjects +% +% Uses the sessions of the mock project, whose variable NeuralData is a +% .mat file in each session folder. +% +% Run tests: +% runtests('nansen.unittest.config.project.GetSessionObjectsTest') + + properties + Project + SessionIDs string + end + + methods (TestClassSetup) + function setupProject(testCase) + testCase.applyFixture(nansen.fixture.ProjectFixture) + testCase.Project = nansen.getCurrentProject(); + sessionTable = testCase.Project.MetaTableCatalog.getMasterMetaTable('session'); + testCase.SessionIDs = string(sessionTable.entries.sessionID); + end + end + + methods (Test) + function testReturnsEverySessionWithoutIDs(testCase) + sessions = testCase.Project.getSessionObjects(); + + testCase.verifyEqual(string({sessions.sessionID})', testCase.SessionIDs) + end + + function testReturnsSessionsInGivenOrder(testCase) + sessionIDs = testCase.SessionIDs([2, 1]); + + sessions = testCase.Project.getSessionObjects(sessionIDs); + + testCase.verifyEqual(string({sessions.sessionID}), sessionIDs') + end + + function testSessionCanLoadData(testCase) + % The mock project's own variable is found through a field the + % variable template lacks, so a variable of its own is added, + % with the fields of the mock project's item + variableModel = testCase.Project.VariableModel; + item = variableModel.Data(1); + item.VariableName = 'Values'; + item.Uuid = nansen.util.getuuid(); + item.FilePathPattern = ''; + item.FileNameExpression = 'values'; + item.FileType = '.mat'; + item.FileAdapter = 'Default'; + variableModel.insertItem(item); + variableModel.save() + testCase.addTeardown(@() variableModel.removeItem('Values')) + + session = testCase.Project.getSessionObjects(testCase.SessionIDs(1)); + Values = magic(3); + save(fullfile(session.getSessionFolder('MockData'), 'values.mat'), 'Values') + + data = session.loadData('Values'); + + testCase.verifyEqual(data, magic(3)) + end + + function testUnknownSessionIDErrors(testCase) + testCase.verifyError(@() testCase.Project.getSessionObjects("no-such-session"), ... + 'NANSEN:Project:SessionNotFound') + end + end +end diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m index 8d47cfe4..0115d97b 100644 --- a/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m @@ -33,6 +33,7 @@ function testExtensionOfSeveralPartsIsFileType(testCase) bold = variables(strcmp({variables.VariableName}, 'bold')); testCase.verifyEqual(bold.FileType, '.nii.gz') + testCase.verifyEqual(bold.Description, 'Functional run, NIfTI') end function testNameInSessionAndFixedFolderIsQualifiedBySubfolder(testCase) diff --git a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m index 5e2cfcc7..4e6cd0a5 100644 --- a/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m +++ b/tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m @@ -124,6 +124,22 @@ function testAbfFileBecomesAVariable(testCase) 'NANSEN has no ABF file adapter.') end + function testVariableWithoutPatternDescriptionHasEmptyDescription(testCase) + testCase.verifyEqual(testCase.Variables.Description, '') + end + + function testPatternDescriptionBecomesVariableDescription(testCase) + fixture = fullfile(fileparts(fileparts(fileparts(mfilename('fullpath')))), ... + '+fixture', 'datasetstructure', 'garad-2022.json'); + dsm = jsondecode(fileread(fixture)); + description = "One repetition of the excitability protocol, membrane potential in mV"; + dsm.dataLocations.filesystemSource.entityLayout(1).filePatterns.description = description; + + [~, variables] = nansen.config.dloc.dsm2DataLocationModel(dsm); + + testCase.verifyEqual(variables.Description, char(description)) + end + function testReportListsWhatWasNotMapped(testCase) testCase.verifyReported("metadataDefinitions[slice_number]") testCase.verifyReported("metadataDefinitions[protocol_repeat]") diff --git a/tests/+nansen/+unittest/+config/VariableDescriptionTest.m b/tests/+nansen/+unittest/+config/VariableDescriptionTest.m new file mode 100644 index 00000000..0399c596 --- /dev/null +++ b/tests/+nansen/+unittest/+config/VariableDescriptionTest.m @@ -0,0 +1,62 @@ +classdef VariableDescriptionTest < matlab.unittest.TestCase +%VariableDescriptionTest - The Description field of data variables +% +% Variables got a Description field after models and module templates +% were written without one. A model saved without it and an item made +% without it must still load and insert. +% +% Run tests: +% runtests('nansen.unittest.config.VariableDescriptionTest') + + properties + FilePath char + end + + methods (TestClassSetup) + function setupProject(testCase) + % The model constructor resolves the current project. + testCase.applyFixture(nansen.fixture.ProjectFixture) + end + end + + methods (TestMethodSetup) + function createModelFile(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + folder = testCase.applyFixture(TemporaryFolderFixture).Folder; + testCase.FilePath = fullfile(folder, 'filepath_settings.json'); + end + end + + methods (Test) + function testBlankItemHasEmptyDescription(testCase) + item = nansen.config.varmodel.VariableModel.getBlankItem(); + testCase.verifyEqual(item.Description, '') + end + + function testItemWithoutDescriptionIsInserted(testCase) + % Module templates are JSON files written without the field + model = nansen.config.varmodel.VariableModel(testCase.FilePath); + item = rmfield(model.getDefaultItem('Template'), 'Description'); + + model.insertItem(item); + + testCase.verifyEqual(model.getItem('Template').Description, '') + end + + function testModelSavedWithoutDescriptionLoadsWithIt(testCase) + model = nansen.config.varmodel.VariableModel(testCase.FilePath); + item = model.getDefaultItem('Saved'); + item.Description = 'Membrane potential in mV'; + model.insertItem(item); + model.save() + + S = jsondecode(fileread(testCase.FilePath)); + S.Data = rmfield(S.Data, 'Description'); + writelines(jsonencode(S), testCase.FilePath) + + reloaded = nansen.config.varmodel.VariableModel(testCase.FilePath); + + testCase.verifyEqual(reloaded.getItem('Saved').Description, '') + end + end +end diff --git a/tests/+nansen/+unittest/+metadata/MetaTableCsvCopyTest.m b/tests/+nansen/+unittest/+metadata/MetaTableCsvCopyTest.m new file mode 100644 index 00000000..ed43c317 --- /dev/null +++ b/tests/+nansen/+unittest/+metadata/MetaTableCsvCopyTest.m @@ -0,0 +1,141 @@ +classdef MetaTableCsvCopyTest < matlab.unittest.TestCase +%MetaTableCsvCopyTest - Tests for the CSV copy that MetaTable.save writes +% +% The mock project's session table is saved with the project +% preference WriteMetaTableCsvCopies on and off. +% +% Run tests: +% runtests('nansen.unittest.metadata.MetaTableCsvCopyTest') + + properties + Project + SessionTable + CsvFilePath char + end + + methods (TestClassSetup) + function setupProject(testCase) + testCase.applyFixture(nansen.fixture.ProjectFixture) + testCase.Project = nansen.getCurrentProject(); + testCase.SessionTable = testCase.Project.MetaTableCatalog.getMasterMetaTable('session'); + [folderPath, fileName] = fileparts(testCase.SessionTable.filepath); + testCase.CsvFilePath = fullfile(folderPath, fileName + ".csv"); + end + end + + methods (TestMethodSetup) + function removeCopy(testCase) + if isfile(testCase.CsvFilePath) + delete(testCase.CsvFilePath) + end + testCase.addTeardown(@() testCase.Project.setWriteMetaTableCsvCopies(false)) + end + end + + methods (Test) + function testSaveWritesCopyWhenPreferenceIsOn(testCase) + testCase.Project.setWriteMetaTableCsvCopies(true) + + testCase.SessionTable.save(true); + + copy = readtable(testCase.CsvFilePath, 'TextType', 'string'); + testCase.verifyEqual(height(copy), height(testCase.SessionTable.entries)) + testCase.verifyEqual(copy.sessionID, string(testCase.SessionTable.entries.sessionID)) + end + + function testSaveWritesNoCopyWhenPreferenceIsOff(testCase) + testCase.SessionTable.save(true); + + testCase.verifyFalse(isfile(testCase.CsvFilePath)) + end + + function testDataLocationBecomesPathColumn(testCase) + testCase.Project.setWriteMetaTableCsvCopies(true) + + testCase.SessionTable.save(true); + + copy = readtable(testCase.CsvFilePath, 'TextType', 'string'); + testCase.verifyTrue(ismember("DataLocation_MockData", copy.Properties.VariableNames)) + testCase.verifyTrue(all(isfolder(copy.DataLocation_MockData))) + end + + function testNestedColumnsAreLeftOut(testCase) + testCase.Project.setWriteMetaTableCsvCopies(true) + + testCase.SessionTable.save(true); + + copy = readtable(testCase.CsvFilePath, 'TextType', 'string'); + testCase.verifyEmpty(intersect(copy.Properties.VariableNames, ... + {'DataLocation', 'Progress', 'Notebook'})) + end + + function testTableWithoutRowsGetsCopy(testCase) + % The data-location column of a table with no rows has no + % names to make columns from + metaTable = testCase.createTableInTemporaryFolder(testCase.SessionTable.entries([], :)); + + csvFilePath = metaTable.writeCsvCopy(); + + testCase.verifyTrue(isfile(csvFilePath)) + end + + function testRowsWithDifferentDataLocationsGetPathColumns(testCase) + % Rows with different numbers of data locations make the + % column a cell array of struct arrays + import matlab.unittest.fixtures.TemporaryFolderFixture + rootPath = testCase.applyFixture(TemporaryFolderFixture).Folder; + raw1 = struct('Name', 'Raw', 'RootPath', rootPath, 'Subfolders', 's1'); + raw2 = struct('Name', 'Raw', 'RootPath', rootPath, 'Subfolders', 's2'); + processed2 = struct('Name', 'Processed', 'RootPath', rootPath, 'Subfolders', fullfile('processed', 's2')); + entries = table({'s1'; 's2'}, {raw1; [raw2, processed2]}, 'VariableNames', {'sessionID', 'DataLocation'}); + metaTable = testCase.createTableInTemporaryFolder(entries); + + % Every column is text and mostly paths, so readtable would + % take / for the delimiter and the header row for data + copy = readtable(metaTable.writeCsvCopy(), 'TextType', 'string', ... + 'Delimiter', ',', 'ReadVariableNames', true); + + testCase.assertEqual(string(copy.Properties.VariableNames), ... + ["sessionID", "DataLocation_Raw", "DataLocation_Processed"]) + testCase.verifyEqual(copy.DataLocation_Raw, string(fullfile(rootPath, {'s1'; 's2'}))) + testCase.verifyEqual(copy.DataLocation_Processed(2), string(fullfile(rootPath, 'processed', 's2'))) + testCase.verifyTrue(ismissing(copy.DataLocation_Processed(1)) || copy.DataLocation_Processed(1) == "") + end + + function testCopyThatCannotBeWrittenWarns(testCase) + % A folder in place of the CSV file makes writetable fail + testCase.Project.setWriteMetaTableCsvCopies(true) + mkdir(testCase.CsvFilePath) + testCase.addTeardown(@() rmdir(testCase.CsvFilePath)) + + wasSaved = testCase.verifyWarning(@() testCase.SessionTable.save(true), ... + 'NANSEN:MetaTable:CsvCopyFailed'); + + testCase.verifyTrue(wasSaved) + end + + function testTableOutsideCurrentProjectGetsNoCopy(testCase) + import matlab.unittest.fixtures.TemporaryFolderFixture + testCase.Project.setWriteMetaTableCsvCopies(true) + folder = testCase.applyFixture(TemporaryFolderFixture).Folder; + entries = table({'s1'; 's2'}, [1; 2], 'VariableNames', {'sessionID', 'Value'}); + metaTable = nansen.unittest.metadata.helper.MetaTableFake(entries, ... + 'MetaTableClass', 'table', 'ItemClassName', '', 'MetaTableIdVarname', 'sessionID'); + metaTable.setFilepath(fullfile(folder, 'outside.mat')); + + metaTable.save(true); + + testCase.verifyFalse(isfile(fullfile(folder, 'outside.csv'))) + end + end + + methods (Access = private) + function metaTable = createTableInTemporaryFolder(testCase, entries) + import matlab.unittest.fixtures.TemporaryFolderFixture + folder = testCase.applyFixture(TemporaryFolderFixture).Folder; + metaTable = nansen.unittest.metadata.helper.MetaTableFake(entries, ... + 'MetaTableClass', 'table', 'ItemClassName', '', 'MetaTableIdVarname', 'sessionID'); + metaTable.setFilepath(fullfile(folder, 'table.mat')); + end + end +end