feat: describe data variables, write CSV copies of tables, get session objects from a project

code/+nansen/+config/+dloc/dsm2DataLocationModel.m

@@ -472,6 +472,7 @@
 
         variable = nansen.config.varmodel.VariableModel.getBlankItem();
         variable.VariableName = char(matlab.lang.makeValidName(string(filePattern.name)));
+        variable.Description = char(getOr(filePattern, 'description', ''));
         variable.DataLocation = item.Name;
         variable.DataLocationUuid = item.Uuid;
         variable.Subfolder = char(subfolder);

code/+nansen/+config/+project/@Project/Project.m

@@ -227,6 +227,67 @@ function setAutoDownloadRemoteFiles(obj, isAutoDownload)
             obj.Preferences.AutoDownloadRemoteFiles = isAutoDownload;
         end
 
+        function sessions = getSessionObjects(obj, sessionIDs)
+        %getSessionObjects Session objects of the project that can load and save data
+        %
+        %   sessions = project.getSessionObjects() returns an object for
+        %   every session in the project's master session table.
+        %
+        %   sessions = project.getSessionObjects(sessionIDs) returns the
+        %   sessions with the given IDs, in the given order.
+        %
+        %   Each object has the project's data location and variable models,
+        %   which Session.loadData and Session.saveData need; the NANSEN app
+        %   gives them to the sessions it creates in the same way. The
+        %   project must be the current project, because the session table
+        %   completes its data location paths from the current project.
+            arguments
+                obj (1,1) nansen.config.project.Project
+                sessionIDs (1,:) string = string.empty(1, 0)
+            end
+
+            currentProject = nansen.getCurrentProject();
+            if isempty(currentProject) || ~strcmp(currentProject.Name, obj.Name)
+                error('NANSEN:Project:NotCurrentProject', ...
+                    ['Sessions can only be created for the current project. ', ...
+                     'Make "%s" the current project with nansen.ProjectManager().changeProject("%s").'], ...
+                    obj.Name, obj.Name)
+            end
+
+            sessionTable = obj.MetaTableCatalog.getMasterMetaTable('session');
+            tableSessionIDs = string(sessionTable.entries.(sessionTable.SchemaIdName));
+            if isempty(sessionIDs)
+                rowIndices = 1:numel(tableSessionIDs);
+            else
+                [isFound, rowIndices] = ismember(sessionIDs, tableSessionIDs);
+                if ~all(isFound)
+                    error('NANSEN:Project:SessionNotFound', ...
+                        'The session table of "%s" has no session with the ID: %s.', ...
+                        obj.Name, strjoin(sessionIDs(~isFound), ', '))
+                end
+            end
+
+            sessions = sessionTable.getMetaObjects(rowIndices, ...
+                'DataLocationModel', obj.DataLocationModel, ...
+                'VariableModel', obj.VariableModel);
+        end
+
+        function setWriteMetaTableCsvCopies(obj, isEnabled)
+        %setWriteMetaTableCsvCopies Set whether saving a metatable also writes a CSV copy
+        %
+        %   project.setWriteMetaTableCsvCopies(true) makes MetaTable.save
+        %   write a CSV copy of each of the project's tables next to its
+        %   .mat file, so that the tables can be read without MATLAB. The
+        %   default is false.
+        %
+        %   See also nansen.metadata.MetaTable/writeCsvCopy
+            arguments
+                obj (1,1) nansen.config.project.Project
+                isEnabled (1,1) logical
+            end
+            obj.Preferences.WriteMetaTableCsvCopies = isEnabled;
+        end
+
         function initializeProjectFolder(obj)
             % Todo: implement? I.e if a project object is created
             % programmatically

code/+nansen/+config/+varmodel/VariableModel.m

@@ -45,6 +45,7 @@
 
             S = struct(...
                 'VariableName', '', ...         % Name of variable
+                'Description', '', ...          % What the file holds, e.g. its channels, columns or units (optional)
                 'DataLocation', '', ...         % todo: rename DataLocationName? Name of datalocation where variable is stored.
                 'DataLocationUuid', '', ...     % uuid of datalocation variable belongs to (internal)
                 'Subfolder', '', ...            % Subfolder within sessionfolder where variable is saved to file (optional)
@@ -220,6 +221,10 @@ function updateDefaultValues(obj)
             if ~isfield(obj.Data, 'PathInFile')
                 [obj.Data(:).PathInFile] = deal('');
             end
+
+            if ~isfield(obj.Data, 'Description')
+                [obj.Data(:).Description] = deal('');
+            end
         end
 
         % % Variable interaction and utility methods
@@ -499,6 +504,11 @@ function removeDataVariableSet(obj, variableList)
     methods % Override superclass methods
         function newItem = insertItem(obj, newItem)
             import nansen.config.varmodel.event.VariableAddedEventData
+
+            % An item made before a field was added to the blank item, such
+            % as a variable template of a module, gets the field with its
+            % blank value, because the catalog requires every field
+            newItem = addMissingFields(newItem);
             newItem = obj.updateVariableDataType(newItem);
             newItem = insertItem@utility.data.StorableCatalog(obj, newItem);
 
@@ -668,3 +678,12 @@ function disableNotifications(obj)
 function tf = useDefaultDataLocation(dataLocationName)
     tf = strcmp(dataLocationName, 'DEFAULT') || isempty(dataLocationName);
 end
+
+function item = addMissingFields(item)
+%addMissingFields Give an item every field of the blank item that it lacks
+    blankItem = nansen.config.varmodel.VariableModel.getBlankItem();
+    missingFields = setdiff(fieldnames(blankItem), fieldnames(item));
+    for i = 1:numel(missingFields)
+        item.(missingFields{i}) = blankItem.(missingFields{i});
+    end
+end

code/+nansen/+metadata/@MetaTable/MetaTable.m

@@ -236,11 +236,45 @@ function setAsDummy(obj)
             wasSaved = save@nansen.metadata.mixin.VersionedFile(obj, force);
             if wasSaved
                 fprintf('MetaTable saved to %s\n', obj.filepath)
+                if obj.isCsvCopyEnabled()
+                    % The table is saved; a copy that cannot be written
+                    % is reported without failing the save
+                    try
+                        obj.writeCsvCopy()
+                    catch exception
+                        warning('NANSEN:MetaTable:CsvCopyFailed', ...
+                            'The table was saved, but its CSV copy could not be written: %s', ...
+                            exception.message)
+                    end
+                end
             end
 
             if ~nargout; clear wasSaved; end
         end
 
+        function csvFilePath = writeCsvCopy(obj)
+        %writeCsvCopy Write the table's entries to a CSV file next to its .mat file
+        %
+        %   csvFilePath = metaTable.writeCsvCopy() writes the entries to a
+        %   CSV file with the name of the table's file, for reading the
+        %   table without MATLAB. Columns of text, numbers, logical values
+        %   and dates are copied. The DataLocation column becomes one column
+        %   per data location, DataLocation_<name>, with the path of the
+        %   row's folder or file there, empty where the row has none.
+        %   Columns of other values, such as Progress and Notebook, are left
+        %   out.
+        %
+        %   save writes the copy after each save when the table belongs to
+        %   the current project and its preference WriteMetaTableCsvCopies
+        %   is true (Project.setWriteMetaTableCsvCopies).
+
+            [folderPath, fileName] = fileparts(obj.filepath);
+            csvFilePath = fullfile(folderPath, fileName + ".csv");
+            writetable(tableForCsv(obj.entries), csvFilePath)
+
+            if ~nargout; clear csvFilePath; end
+        end
+
         function load(obj)
         %load Load a MetaTable from file
         %
@@ -1161,6 +1195,34 @@ function resetMetaObjectCache(obj)
         end
     end
 
+    methods (Access = private) % CSV copy
+
+        function tf = isCsvCopyEnabled(obj)
+        %isCsvCopyEnabled Whether the table's project asks for a CSV copy
+        %
+        %   The preference belongs to a project, and a table can be saved
+        %   while another project is current, so only a table in the folder
+        %   of the current project uses the current project's preference.
+            tf = false;
+            try
+                project = nansen.getCurrentProject();
+            catch exception
+                % Without a user session there is no project to ask
+                if strcmp(exception.identifier, 'NANSEN:NoActiveUserSession')
+                    return
+                end
+                rethrow(exception)
+            end
+            if isempty(project) || ~startsWith(string(obj.filepath), string(project.FolderPath))
+                return
+            end
+
+            preferences = project.Preferences;
+            tf = isfield(preferences, 'WriteMetaTableCsvCopies') ...
+                && preferences.WriteMetaTableCsvCopies;
+        end
+    end
+
     methods (Access = protected) % Data location column
 
         function tableEntries = reduceDataLocationColumn(obj, tableEntries)
@@ -1853,3 +1915,67 @@ function removeDuplicates(obj)
 
     value = fcn(value);
 end
+
+function csvTable = tableForCsv(entries)
+%tableForCsv The columns of a metatable that a CSV file can hold
+%   Columns of text, numbers, logical values and dates are kept. The
+%   DataLocation column becomes one text column per data location with the
+%   path of each row's folder or file there. Other columns are left out.
+
+    csvTable = table('Size', [height(entries), 0]);
+    for variableName = string(entries.Properties.VariableNames)
+        values = entries.(variableName);
+        if variableName == "DataLocation" && (isstruct(values) || iscell(values))
+            csvTable = [csvTable, dataLocationPaths(values)]; %#ok<AGROW>
+        elseif isnumeric(values) || islogical(values) || isdatetime(values) ...
+                || isstring(values) || iscategorical(values)
+            csvTable.(variableName) = values;
+        elseif iscell(values) && all(cellfun(@isScalarValue, values))
+            % Empty values are written as empty text, so that the column
+            % keeps one type
+            values(cellfun(@isempty, values)) = {''};
+            csvTable.(variableName) = values;
+        end
+    end
+end
+
+function pathTable = dataLocationPaths(dataLocations)
+%dataLocationPaths One column per data location with the path of each row there
+%   dataLocations is a struct array with one row per table row, or a cell
+%   array with one struct array per table row when rows have different
+%   numbers of data locations. The columns are named by the data
+%   locations' Name, in the order they first occur; a row without a data
+%   location of that name has an empty path.
+    numRows = size(dataLocations, 1);
+    if isstruct(dataLocations)
+        rows = num2cell(dataLocations, 2);
+    else
+        rows = dataLocations(:);
+    end
+
+    names = strings(1, 0);
+    paths = strings(numRows, 0);
+    for i = 1:numRows
+        for location = reshape(rows{i}, 1, [])
+            columnIndex = find(names == string(location.Name), 1);
+            if isempty(columnIndex)
+                names(end+1) = string(location.Name); %#ok<AGROW>
+                paths(:, end+1) = ""; %#ok<AGROW>
+                columnIndex = numel(names);
+            end
+            if isfield(location, 'RootPath') && ~isempty(location.Subfolders)
+                paths(i, columnIndex) = fullfile(location.RootPath, location.Subfolders);
+            end
+        end
+    end
+    pathTable = array2table(paths, 'VariableNames', "DataLocation_" + names);
+end
+
+function tf = isScalarValue(value)
+%isScalarValue Whether a cell of a table column holds one text, number, logical value or date, or nothing
+%   An empty struct, as in Progress and Notebook, is still a structure and
+%   does not count as nothing.
+    isEmptyTextOrNumber = isempty(value) && (isnumeric(value) || isstring(value));
+    tf = ischar(value) || isEmptyTextOrNumber || (isscalar(value) && (isstring(value) ...
+        || isnumeric(value) || islogical(value) || isdatetime(value)));
+end

tests/+nansen/+unittest/+config/+project/GetSessionObjectsTest.m

@@ -0,0 +1,69 @@
+classdef GetSessionObjectsTest < matlab.unittest.TestCase
+%GetSessionObjectsTest - Tests for Project.getSessionObjects
+%
+%   Uses the sessions of the mock project, whose variable NeuralData is a
+%   .mat file in each session folder.
+%
+%   Run tests:
+%       runtests('nansen.unittest.config.project.GetSessionObjectsTest')
+
+    properties
+        Project
+        SessionIDs string
+    end
+
+    methods (TestClassSetup)
+        function setupProject(testCase)
+            testCase.applyFixture(nansen.fixture.ProjectFixture)
+            testCase.Project = nansen.getCurrentProject();
+            sessionTable = testCase.Project.MetaTableCatalog.getMasterMetaTable('session');
+            testCase.SessionIDs = string(sessionTable.entries.sessionID);
+        end
+    end
+
+    methods (Test)
+        function testReturnsEverySessionWithoutIDs(testCase)
+            sessions = testCase.Project.getSessionObjects();
+
+            testCase.verifyEqual(string({sessions.sessionID})', testCase.SessionIDs)
+        end
+
+        function testReturnsSessionsInGivenOrder(testCase)
+            sessionIDs = testCase.SessionIDs([2, 1]);
+
+            sessions = testCase.Project.getSessionObjects(sessionIDs);
+
+            testCase.verifyEqual(string({sessions.sessionID}), sessionIDs')
+        end
+
+        function testSessionCanLoadData(testCase)
+            % The mock project's own variable is found through a field the
+            % variable template lacks, so a variable of its own is added,
+            % with the fields of the mock project's item
+            variableModel = testCase.Project.VariableModel;
+            item = variableModel.Data(1);
+            item.VariableName = 'Values';
+            item.Uuid = nansen.util.getuuid();
+            item.FilePathPattern = '';
+            item.FileNameExpression = 'values';
+            item.FileType = '.mat';
+            item.FileAdapter = 'Default';
+            variableModel.insertItem(item);
+            variableModel.save()
+            testCase.addTeardown(@() variableModel.removeItem('Values'))
+
+            session = testCase.Project.getSessionObjects(testCase.SessionIDs(1));
+            Values = magic(3);
+            save(fullfile(session.getSessionFolder('MockData'), 'values.mat'), 'Values')
+
+            data = session.loadData('Values');
+
+            testCase.verifyEqual(data, magic(3))
+        end
+
+        function testUnknownSessionIDErrors(testCase)
+            testCase.verifyError(@() testCase.Project.getSessionObjects("no-such-session"), ...
+                'NANSEN:Project:SessionNotFound')
+        end
+    end
+end

tests/+nansen/+unittest/+config/Dsm2DataLocationFilesBelowSessionTest.m

@@ -33,6 +33,7 @@ function testExtensionOfSeveralPartsIsFileType(testCase)
 
             bold = variables(strcmp({variables.VariableName}, 'bold'));
             testCase.verifyEqual(bold.FileType, '.nii.gz')
+            testCase.verifyEqual(bold.Description, 'Functional run, NIfTI')
         end
 
         function testNameInSessionAndFixedFolderIsQualifiedBySubfolder(testCase)

tests/+nansen/+unittest/+config/Dsm2DataLocationModelTest.m

@@ -124,6 +124,22 @@ function testAbfFileBecomesAVariable(testCase)
                 'NANSEN has no ABF file adapter.')
         end
 
+        function testVariableWithoutPatternDescriptionHasEmptyDescription(testCase)
+            testCase.verifyEqual(testCase.Variables.Description, '')
+        end
+
+        function testPatternDescriptionBecomesVariableDescription(testCase)
+            fixture = fullfile(fileparts(fileparts(fileparts(mfilename('fullpath')))), ...
+                '+fixture', 'datasetstructure', 'garad-2022.json');
+            dsm = jsondecode(fileread(fixture));
+            description = "One repetition of the excitability protocol, membrane potential in mV";
+            dsm.dataLocations.filesystemSource.entityLayout(1).filePatterns.description = description;
+
+            [~, variables] = nansen.config.dloc.dsm2DataLocationModel(dsm);
+
+            testCase.verifyEqual(variables.Description, char(description))
+        end
+
         function testReportListsWhatWasNotMapped(testCase)
             testCase.verifyReported("metadataDefinitions[slice_number]")
             testCase.verifyReported("metadataDefinitions[protocol_repeat]")