Skip to content

feat(session): download data files that are stored online #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ehennestad wants to merge 1 commit into from
Draft

feat(session): download data files that are stored online #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions code/+nansen/+config/+project/@Project/Project.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,49 @@ function setOptionalModules(obj, optionalModulesNames)
obj.updateModules()
end

function setRemoteFileSource(obj, className)
%setRemoteFileSource Name the source of the project's files that are stored online
%
% project.setRemoteFileSource(className) names a subclass of
% nansen.dataio.RemoteFileSource. Session.loadData and
% Session.downloadDataFile use it to find and download data files
% that are placeholders for online files.
%
% project.setRemoteFileSource("") removes the source.
%
% See also nansen.dataio.RemoteFileSource, setAutoDownloadRemoteFiles
arguments
obj (1,1) nansen.config.project.Project
className (1,1) string
end

if strlength(className) > 0
sourceClass = meta.class.fromName(className);
if isempty(sourceClass) || ~(sourceClass <= ?nansen.dataio.RemoteFileSource)
error('NANSEN:Project:InvalidRemoteFileSource', ...
['"%s" is not a subclass of nansen.dataio.RemoteFileSource on the MATLAB path. ', ...
'Name a class that inherits from nansen.dataio.RemoteFileSource.'], className)
end
end
obj.Preferences.RemoteFileSource = char(className);
end

function setAutoDownloadRemoteFiles(obj, isAutoDownload)
%setAutoDownloadRemoteFiles Set whether loading data downloads files that are stored online
%
% project.setAutoDownloadRemoteFiles(true) makes Session.loadData
% download a placeholder file from the project's remote file source
% before loading it. With false, the default, loadData raises an
% error for such a file and Session.downloadDataFile downloads it.
%
% See also setRemoteFileSource
arguments
obj (1,1) nansen.config.project.Project
isAutoDownload (1,1) logical
end
obj.Preferences.AutoDownloadRemoteFiles = isAutoDownload;
end

function initializeProjectFolder(obj)
% Todo: implement? I.e if a project object is created
% programmatically
Expand Down
43 changes: 43 additions & 0 deletions code/+nansen/+dataio/RemoteFileSource.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
classdef (Abstract) RemoteFileSource < handle
%RemoteFileSource - Source of data files that are stored online
%
% A data location can hold placeholders for files that are stored
% online, such as the empty files of a cloned EBRAINS bucket. A subclass
% of RemoteFileSource tells which local files are placeholders and
% replaces a placeholder with the online file.
%
% A project names its subclass in the project preference
% RemoteFileSource. Session.loadData then checks every file it loads:
% when the file is a placeholder, it downloads it if the project
% preference AutoDownloadRemoteFiles is true, and raises an error that
% names Session.downloadDataFile otherwise.
%
% preferences = project.Preferences;
% preferences.RemoteFileSource = 'mymodule.dataio.MyFileSource';
% preferences.AutoDownloadRemoteFiles = true;
% project.Preferences = preferences;
%
% A subclass constructor takes one input, the root folder of the data
% location that holds the file, and passes it to this constructor.
%
% See also nansen.metadata.type.Session/loadData,
% nansen.metadata.type.Session/downloadDataFile

properties (SetAccess = immutable)
RootPath (1,1) string % Root folder of the data location that holds the files
end

methods
function obj = RemoteFileSource(rootPath)
obj.RootPath = rootPath;
end
end

methods (Abstract)
%isOnlineOnly Whether a local file is a placeholder for a file stored online
tf = isOnlineOnly(obj, filePath)

%download Replace the placeholder at filePath with the file stored online
download(obj, filePath)
end
end
91 changes: 91 additions & 0 deletions code/+nansen/+metadata/+type/Session.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -780,6 +780,8 @@ function viewDataVariable(obj, varName)
end

if isfile(filePath)
obj.downloadIfOnlineOnly(filePath, variableInfo, varName)

L = dir(filePath);
if L.bytes==0
error('NANSEN:Session:EmptyFile', ...
Expand Down Expand Up @@ -820,6 +822,34 @@ function viewDataVariable(obj, varName)
end
end

function downloadDataFile(obj, varName)
%downloadDataFile Download the file of a variable that is stored online
%
% sessionObj.downloadDataFile(varName) downloads the file that holds
% the variable varName when the file in the session folder is a
% placeholder for a file stored online. A file that is already
% local is left as it is. The project names the source of online
% files in its preference RemoteFileSource.
%
% See also nansen.dataio.RemoteFileSource

[filePath, variableInfo] = obj.getDataFilePath(varName, '-r');
if ~isfile(filePath)
error('NANSEN:Session:VariableNotFound', 'Variable ''%s'' was not found.', varName)
end

source = obj.getRemoteFileSource(variableInfo.DataLocation);
if isempty(source)
error('NANSEN:Session:NoRemoteFileSource', ...
['The current project has no source of online files. Name a subclass of ', ...
'nansen.dataio.RemoteFileSource in the project preference RemoteFileSource.'])
end

if source.isOnlineOnly(filePath)
source.download(filePath)
end
end

function saveData(obj, varName, data, varargin)
%saveData Saves data for given variable
%
Expand Down Expand Up @@ -1366,6 +1396,67 @@ function removeSessionFolder(obj, dataLocationName)
end
end

methods (Access = private) % Files stored online
function downloadIfOnlineOnly(obj, filePath, variableInfo, varName)
%downloadIfOnlineOnly Make sure a file is local before it is loaded
%
% A placeholder for an online file is downloaded when the project
% preference AutoDownloadRemoteFiles is true. Otherwise loading
% stops with an error that says how to download the file, since a
% download can be large and take long.

source = obj.getRemoteFileSource(variableInfo.DataLocation);
if isempty(source) || ~source.isOnlineOnly(filePath)
return
end

preferences = nansen.getCurrentProject().Preferences;
isAutoDownload = isfield(preferences, 'AutoDownloadRemoteFiles') ...
&& preferences.AutoDownloadRemoteFiles;

if isAutoDownload
source.download(filePath)
else
error('NANSEN:Session:FileIsOnlineOnly', ...
['The file of variable "%s" is stored online and has not been downloaded: %s\n', ...
'Download it with sessionObject.downloadDataFile("%s"), or set the project ', ...
'preference AutoDownloadRemoteFiles to true to download files when they are loaded.'], ...
varName, filePath, varName)
end
end

function source = getRemoteFileSource(obj, dataLocationName)
%getRemoteFileSource The project's source of online files for a data location
%
% Returns [] when no user session is active, when there is no
% current project, or when the project names no source in its
% preference RemoteFileSource.

source = [];
try
project = nansen.getCurrentProject();
catch exception
% A session object can load data without a user session,
% for example in a parallel worker; files are then local
if ~strcmp(exception.identifier, 'NANSEN:NoActiveUserSession')
rethrow(exception)
end
project = [];
end
if isempty(project)
return
end

preferences = project.Preferences;
if ~isfield(preferences, 'RemoteFileSource') || isempty(preferences.RemoteFileSource)
return
end

sourceConstructor = str2func(preferences.RemoteFileSource);
source = sourceConstructor(obj.getDataLocationRootDir(dataLocationName));
end
end

methods (Static)

function value = extractSubjectId(sessionObj)
Expand Down
27 changes: 27 additions & 0 deletions tests/+nansen/+unittest/+session/+helper/RemoteFileSourceFake.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
classdef RemoteFileSourceFake < nansen.dataio.RemoteFileSource
%RemoteFileSourceFake - A source of online files that writes a known file instead of downloading
%
% An empty file counts as online only. download writes a .mat file that
% holds the variable OnlineData with the value DownloadedValue, so a test
% can check that loading returns the downloaded content.

properties (Constant)
DownloadedValue = magic(3)
end

methods
function obj = RemoteFileSourceFake(rootPath)
obj@nansen.dataio.RemoteFileSource(rootPath)
end

function tf = isOnlineOnly(~, filePath)
fileInfo = dir(filePath);
tf = fileInfo.bytes == 0;
end

function download(~, filePath)
OnlineData = nansen.unittest.session.helper.RemoteFileSourceFake.DownloadedValue;
save(filePath, 'OnlineData')
end
end
end
69 changes: 69 additions & 0 deletions tests/+nansen/+unittest/+session/LoadDataWithoutUserSessionTest.m
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
classdef LoadDataWithoutUserSessionTest < matlab.unittest.TestCase
%LoadDataWithoutUserSessionTest - Load data from a session object when no user session is active
%
% A session object holds its data-location and variable models, so it
% can load data where no NANSEN user session is active, such as in a
% parallel worker. The mock project's first session gets a .mat file for
% the variable LocalData; the test ends the user session and loads it.
% The user session is ended in this class only, because the other
% session tests hold the project object of the session.
%
% Run tests:
% runtests('nansen.unittest.session.LoadDataWithoutUserSessionTest')

properties (Constant)
VariableName = 'LocalData'
StoredValue = magic(4)
end

properties
Session
end

methods (TestClassSetup)
function setupProject(testCase)
testCase.applyFixture(nansen.fixture.ProjectFixture)
project = nansen.getCurrentProject();

% The new item copies the fields of the mock project's item,
% which has fields that a default item lacks
variableModel = project.VariableModel;
item = variableModel.Data(1);
item.VariableName = testCase.VariableName;
item.Uuid = nansen.util.getuuid();
item.DataLocation = 'MockData';
item.FilePathPattern = '';
item.FileNameExpression = 'local_data';
item.FileType = '.mat';
item.FileAdapter = 'Default';
variableModel.insertItem(item);
variableModel.save()

sessionTable = project.MetaTableCatalog.getMasterMetaTable('session');
testCase.Session = sessionTable.getMetaObjects(1, ...
'DataLocationModel', project.DataLocationModel, ...
'VariableModel', variableModel);

LocalData = testCase.StoredValue; %#ok<NASGU> Saved by name
save(fullfile(testCase.Session.getSessionFolder('MockData'), 'local_data.mat'), 'LocalData')
end
end

methods (TestMethodSetup)
function endUserSession(testCase)
% The fixture's teardown starts the test profile's user
% session again to remove the mock project
userName = nansen.internal.user.NansenUserSession.instance('', 'nocreate').CurrentUserName;
nansen.internal.user.NansenUserSession.instance(userName, 'reset');
testCase.assertEmpty(nansen.internal.user.NansenUserSession.instance('', 'nocreate'))
end
end

methods (Test)
function testLoadDataReadsLocalFile(testCase)
data = testCase.Session.loadData(testCase.VariableName);

testCase.verifyEqual(data, testCase.StoredValue)
end
end
end
Loading
Loading