diff --git a/code/+nansen/+config/+project/@Project/Project.m b/code/+nansen/+config/+project/@Project/Project.m index 122989ce..724383a4 100644 --- a/code/+nansen/+config/+project/@Project/Project.m +++ b/code/+nansen/+config/+project/@Project/Project.m @@ -184,6 +184,49 @@ function setOptionalModules(obj, optionalModulesNames) obj.updateModules() end + function setRemoteFileSource(obj, className) + %setRemoteFileSource Name the source of the project's files that are stored online + % + % project.setRemoteFileSource(className) names a subclass of + % nansen.dataio.RemoteFileSource. Session.loadData and + % Session.downloadDataFile use it to find and download data files + % that are placeholders for online files. + % + % project.setRemoteFileSource("") removes the source. + % + % See also nansen.dataio.RemoteFileSource, setAutoDownloadRemoteFiles + arguments + obj (1,1) nansen.config.project.Project + className (1,1) string + end + + if strlength(className) > 0 + sourceClass = meta.class.fromName(className); + if isempty(sourceClass) || ~(sourceClass <= ?nansen.dataio.RemoteFileSource) + error('NANSEN:Project:InvalidRemoteFileSource', ... + ['"%s" is not a subclass of nansen.dataio.RemoteFileSource on the MATLAB path. ', ... + 'Name a class that inherits from nansen.dataio.RemoteFileSource.'], className) + end + end + obj.Preferences.RemoteFileSource = char(className); + end + + function setAutoDownloadRemoteFiles(obj, isAutoDownload) + %setAutoDownloadRemoteFiles Set whether loading data downloads files that are stored online + % + % project.setAutoDownloadRemoteFiles(true) makes Session.loadData + % download a placeholder file from the project's remote file source + % before loading it. With false, the default, loadData raises an + % error for such a file and Session.downloadDataFile downloads it. + % + % See also setRemoteFileSource + arguments + obj (1,1) nansen.config.project.Project + isAutoDownload (1,1) logical + end + obj.Preferences.AutoDownloadRemoteFiles = isAutoDownload; + end + function initializeProjectFolder(obj) % Todo: implement? I.e if a project object is created % programmatically diff --git a/code/+nansen/+dataio/RemoteFileSource.m b/code/+nansen/+dataio/RemoteFileSource.m new file mode 100644 index 00000000..8ef56cba --- /dev/null +++ b/code/+nansen/+dataio/RemoteFileSource.m @@ -0,0 +1,43 @@ +classdef (Abstract) RemoteFileSource < handle +%RemoteFileSource - Source of data files that are stored online +% +% A data location can hold placeholders for files that are stored +% online, such as the empty files of a cloned EBRAINS bucket. A subclass +% of RemoteFileSource tells which local files are placeholders and +% replaces a placeholder with the online file. +% +% A project names its subclass in the project preference +% RemoteFileSource. Session.loadData then checks every file it loads: +% when the file is a placeholder, it downloads it if the project +% preference AutoDownloadRemoteFiles is true, and raises an error that +% names Session.downloadDataFile otherwise. +% +% preferences = project.Preferences; +% preferences.RemoteFileSource = 'mymodule.dataio.MyFileSource'; +% preferences.AutoDownloadRemoteFiles = true; +% project.Preferences = preferences; +% +% A subclass constructor takes one input, the root folder of the data +% location that holds the file, and passes it to this constructor. +% +% See also nansen.metadata.type.Session/loadData, +% nansen.metadata.type.Session/downloadDataFile + + properties (SetAccess = immutable) + RootPath (1,1) string % Root folder of the data location that holds the files + end + + methods + function obj = RemoteFileSource(rootPath) + obj.RootPath = rootPath; + end + end + + methods (Abstract) + %isOnlineOnly Whether a local file is a placeholder for a file stored online + tf = isOnlineOnly(obj, filePath) + + %download Replace the placeholder at filePath with the file stored online + download(obj, filePath) + end +end diff --git a/code/+nansen/+metadata/+type/Session.m b/code/+nansen/+metadata/+type/Session.m index d916b052..4f104ba7 100644 --- a/code/+nansen/+metadata/+type/Session.m +++ b/code/+nansen/+metadata/+type/Session.m @@ -780,6 +780,8 @@ function viewDataVariable(obj, varName) end if isfile(filePath) + obj.downloadIfOnlineOnly(filePath, variableInfo, varName) + L = dir(filePath); if L.bytes==0 error('NANSEN:Session:EmptyFile', ... @@ -820,6 +822,34 @@ function viewDataVariable(obj, varName) end end + function downloadDataFile(obj, varName) + %downloadDataFile Download the file of a variable that is stored online + % + % sessionObj.downloadDataFile(varName) downloads the file that holds + % the variable varName when the file in the session folder is a + % placeholder for a file stored online. A file that is already + % local is left as it is. The project names the source of online + % files in its preference RemoteFileSource. + % + % See also nansen.dataio.RemoteFileSource + + [filePath, variableInfo] = obj.getDataFilePath(varName, '-r'); + if ~isfile(filePath) + error('NANSEN:Session:VariableNotFound', 'Variable ''%s'' was not found.', varName) + end + + source = obj.getRemoteFileSource(variableInfo.DataLocation); + if isempty(source) + error('NANSEN:Session:NoRemoteFileSource', ... + ['The current project has no source of online files. Name a subclass of ', ... + 'nansen.dataio.RemoteFileSource in the project preference RemoteFileSource.']) + end + + if source.isOnlineOnly(filePath) + source.download(filePath) + end + end + function saveData(obj, varName, data, varargin) %saveData Saves data for given variable % @@ -1366,6 +1396,67 @@ function removeSessionFolder(obj, dataLocationName) end end + methods (Access = private) % Files stored online + function downloadIfOnlineOnly(obj, filePath, variableInfo, varName) + %downloadIfOnlineOnly Make sure a file is local before it is loaded + % + % A placeholder for an online file is downloaded when the project + % preference AutoDownloadRemoteFiles is true. Otherwise loading + % stops with an error that says how to download the file, since a + % download can be large and take long. + + source = obj.getRemoteFileSource(variableInfo.DataLocation); + if isempty(source) || ~source.isOnlineOnly(filePath) + return + end + + preferences = nansen.getCurrentProject().Preferences; + isAutoDownload = isfield(preferences, 'AutoDownloadRemoteFiles') ... + && preferences.AutoDownloadRemoteFiles; + + if isAutoDownload + source.download(filePath) + else + error('NANSEN:Session:FileIsOnlineOnly', ... + ['The file of variable "%s" is stored online and has not been downloaded: %s\n', ... + 'Download it with sessionObject.downloadDataFile("%s"), or set the project ', ... + 'preference AutoDownloadRemoteFiles to true to download files when they are loaded.'], ... + varName, filePath, varName) + end + end + + function source = getRemoteFileSource(obj, dataLocationName) + %getRemoteFileSource The project's source of online files for a data location + % + % Returns [] when no user session is active, when there is no + % current project, or when the project names no source in its + % preference RemoteFileSource. + + source = []; + try + project = nansen.getCurrentProject(); + catch exception + % A session object can load data without a user session, + % for example in a parallel worker; files are then local + if ~strcmp(exception.identifier, 'NANSEN:NoActiveUserSession') + rethrow(exception) + end + project = []; + end + if isempty(project) + return + end + + preferences = project.Preferences; + if ~isfield(preferences, 'RemoteFileSource') || isempty(preferences.RemoteFileSource) + return + end + + sourceConstructor = str2func(preferences.RemoteFileSource); + source = sourceConstructor(obj.getDataLocationRootDir(dataLocationName)); + end + end + methods (Static) function value = extractSubjectId(sessionObj) diff --git a/tests/+nansen/+unittest/+session/+helper/RemoteFileSourceFake.m b/tests/+nansen/+unittest/+session/+helper/RemoteFileSourceFake.m new file mode 100644 index 00000000..53211cef --- /dev/null +++ b/tests/+nansen/+unittest/+session/+helper/RemoteFileSourceFake.m @@ -0,0 +1,27 @@ +classdef RemoteFileSourceFake < nansen.dataio.RemoteFileSource +%RemoteFileSourceFake - A source of online files that writes a known file instead of downloading +% +% An empty file counts as online only. download writes a .mat file that +% holds the variable OnlineData with the value DownloadedValue, so a test +% can check that loading returns the downloaded content. + + properties (Constant) + DownloadedValue = magic(3) + end + + methods + function obj = RemoteFileSourceFake(rootPath) + obj@nansen.dataio.RemoteFileSource(rootPath) + end + + function tf = isOnlineOnly(~, filePath) + fileInfo = dir(filePath); + tf = fileInfo.bytes == 0; + end + + function download(~, filePath) + OnlineData = nansen.unittest.session.helper.RemoteFileSourceFake.DownloadedValue; + save(filePath, 'OnlineData') + end + end +end diff --git a/tests/+nansen/+unittest/+session/LoadDataWithoutUserSessionTest.m b/tests/+nansen/+unittest/+session/LoadDataWithoutUserSessionTest.m new file mode 100644 index 00000000..d3437d93 --- /dev/null +++ b/tests/+nansen/+unittest/+session/LoadDataWithoutUserSessionTest.m @@ -0,0 +1,69 @@ +classdef LoadDataWithoutUserSessionTest < matlab.unittest.TestCase +%LoadDataWithoutUserSessionTest - Load data from a session object when no user session is active +% +% A session object holds its data-location and variable models, so it +% can load data where no NANSEN user session is active, such as in a +% parallel worker. The mock project's first session gets a .mat file for +% the variable LocalData; the test ends the user session and loads it. +% The user session is ended in this class only, because the other +% session tests hold the project object of the session. +% +% Run tests: +% runtests('nansen.unittest.session.LoadDataWithoutUserSessionTest') + + properties (Constant) + VariableName = 'LocalData' + StoredValue = magic(4) + end + + properties + Session + end + + methods (TestClassSetup) + function setupProject(testCase) + testCase.applyFixture(nansen.fixture.ProjectFixture) + project = nansen.getCurrentProject(); + + % The new item copies the fields of the mock project's item, + % which has fields that a default item lacks + variableModel = project.VariableModel; + item = variableModel.Data(1); + item.VariableName = testCase.VariableName; + item.Uuid = nansen.util.getuuid(); + item.DataLocation = 'MockData'; + item.FilePathPattern = ''; + item.FileNameExpression = 'local_data'; + item.FileType = '.mat'; + item.FileAdapter = 'Default'; + variableModel.insertItem(item); + variableModel.save() + + sessionTable = project.MetaTableCatalog.getMasterMetaTable('session'); + testCase.Session = sessionTable.getMetaObjects(1, ... + 'DataLocationModel', project.DataLocationModel, ... + 'VariableModel', variableModel); + + LocalData = testCase.StoredValue; %#ok Saved by name + save(fullfile(testCase.Session.getSessionFolder('MockData'), 'local_data.mat'), 'LocalData') + end + end + + methods (TestMethodSetup) + function endUserSession(testCase) + % The fixture's teardown starts the test profile's user + % session again to remove the mock project + userName = nansen.internal.user.NansenUserSession.instance('', 'nocreate').CurrentUserName; + nansen.internal.user.NansenUserSession.instance(userName, 'reset'); + testCase.assertEmpty(nansen.internal.user.NansenUserSession.instance('', 'nocreate')) + end + end + + methods (Test) + function testLoadDataReadsLocalFile(testCase) + data = testCase.Session.loadData(testCase.VariableName); + + testCase.verifyEqual(data, testCase.StoredValue) + end + end +end diff --git a/tests/+nansen/+unittest/+session/RemoteFileDownloadTest.m b/tests/+nansen/+unittest/+session/RemoteFileDownloadTest.m new file mode 100644 index 00000000..f9b6ed6c --- /dev/null +++ b/tests/+nansen/+unittest/+session/RemoteFileDownloadTest.m @@ -0,0 +1,114 @@ +classdef RemoteFileDownloadTest < matlab.unittest.TestCase +%RemoteFileDownloadTest - Tests for loading data files that are stored online +% +% The mock project gets the variable OnlineData, whose file in the first +% session is an empty placeholder. The project preference +% RemoteFileSource names RemoteFileSourceFake, which counts empty files +% as online and downloads by writing a known .mat file. +% +% Run tests: +% runtests('nansen.unittest.session.RemoteFileDownloadTest') + + properties (Constant) + VariableName = 'OnlineData' + SourceClassName = 'nansen.unittest.session.helper.RemoteFileSourceFake' + end + + properties + Project + Session + FilePath char + end + + methods (TestClassSetup) + function setupProject(testCase) + testCase.applyFixture(nansen.fixture.ProjectFixture) + testCase.Project = nansen.getCurrentProject(); + + % The new item copies the fields of the mock project's item, + % which has fields that a default item lacks, and the model + % holds its items in one struct array + variableModel = testCase.Project.VariableModel; + item = variableModel.Data(1); + item.VariableName = testCase.VariableName; + item.Uuid = nansen.util.getuuid(); + item.DataLocation = 'MockData'; + item.FilePathPattern = ''; + item.FileNameExpression = 'online_data'; + item.FileType = '.mat'; + item.FileAdapter = 'Default'; + variableModel.insertItem(item); + variableModel.save() + + sessionTable = testCase.Project.MetaTableCatalog.getMasterMetaTable('session'); + % The models are passed as the app passes them + testCase.Session = sessionTable.getMetaObjects(1, ... + 'DataLocationModel', testCase.Project.DataLocationModel, ... + 'VariableModel', variableModel); + testCase.FilePath = fullfile(testCase.Session.getSessionFolder('MockData'), ... + 'online_data.mat'); + end + end + + methods (TestMethodSetup) + function createPlaceholder(testCase) + fclose(fopen(testCase.FilePath, 'w')); + end + + function removeSourceAfterTest(testCase) + testCase.addTeardown(@() testCase.Project.setRemoteFileSource("")) + testCase.addTeardown(@() testCase.Project.setAutoDownloadRemoteFiles(false)) + end + end + + methods (Test) + function testLoadDataDownloadsWhenAutoDownloadIsOn(testCase) + testCase.setSourcePreferences(true) + + data = testCase.Session.loadData(testCase.VariableName); + + testCase.verifyEqual(data, nansen.unittest.session.helper.RemoteFileSourceFake.DownloadedValue) + end + + function testLoadDataErrorsWhenAutoDownloadIsOff(testCase) + testCase.setSourcePreferences(false) + + testCase.verifyError(@() testCase.Session.loadData(testCase.VariableName), ... + 'NANSEN:Session:FileIsOnlineOnly') + fileInfo = dir(testCase.FilePath); + testCase.verifyEqual(fileInfo.bytes, 0) + end + + function testDownloadDataFileDownloadsPlaceholder(testCase) + testCase.setSourcePreferences(false) + + testCase.Session.downloadDataFile(testCase.VariableName) + + data = testCase.Session.loadData(testCase.VariableName); + testCase.verifyEqual(data, nansen.unittest.session.helper.RemoteFileSourceFake.DownloadedValue) + end + + function testLoadDataWithoutSourceReportsEmptyFile(testCase) + % A project without a source treats an empty file as before + testCase.verifyError(@() testCase.Session.loadData(testCase.VariableName), ... + 'NANSEN:Session:EmptyFile') + end + + function testDownloadDataFileWithoutSourceErrors(testCase) + testCase.verifyError(@() testCase.Session.downloadDataFile(testCase.VariableName), ... + 'NANSEN:Session:NoRemoteFileSource') + end + + function testSetRemoteFileSourceRejectsOtherClass(testCase) + testCase.verifyError(@() testCase.Project.setRemoteFileSource("nansen.metadata.type.Session"), ... + 'NANSEN:Project:InvalidRemoteFileSource') + end + end + + methods (Access = private) + function setSourcePreferences(testCase, isAutoDownload) + testCase.Project.setRemoteFileSource(testCase.SourceClassName) + testCase.Project.setAutoDownloadRemoteFiles(isAutoDownload) + end + end +end