From c22f88ce16659c02fb6c9ef33c68d55ae0bd85e4 Mon Sep 17 00:00:00 2001 From: Hugo Karas Date: Mon, 31 Aug 2026 16:00:22 +0200 Subject: [PATCH 1/4] Added BatchUpload --- src/deeranalysis/components/batch_import.py | 444 ++++++++++++++++++ src/deeranalysis/components/dataset_form.py | 52 +- .../components/logs_import_modal.py | 4 +- src/deeranalysis/components/metadata_table.py | 27 +- src/deeranalysis/pages/dataset_detail.py | 4 +- src/deeranalysis/pages/datasets.py | 4 +- src/deeranalysis/pages/upload.py | 179 +++---- src/deeranalysis/utils/file_parser.py | 130 +++++ 8 files changed, 703 insertions(+), 141 deletions(-) create mode 100644 src/deeranalysis/components/batch_import.py create mode 100644 src/deeranalysis/utils/file_parser.py diff --git a/src/deeranalysis/components/batch_import.py b/src/deeranalysis/components/batch_import.py new file mode 100644 index 0000000..95129e3 --- /dev/null +++ b/src/deeranalysis/components/batch_import.py @@ -0,0 +1,444 @@ +"""Batch File Import — upload many datasets at once, review them as cards +with a sparkline preview, set Project/Sample/Experiment globally or per +card, then expand any card to edit it with the same form used on the +Single Import tab. + +Each card gets its own hidden per-dataset form (delays grid, tmin, data +viewer, save button) tagged with a unique ``page`` id (``batch-``). +Because every callback in ``dataset_form.py`` and ``data_viewer.py`` is +registered with Dash's ``MATCH`` wildcard on ``page``, those callbacks bind +automatically to these dynamically-created components — no new callback +logic is needed for delays syncing, phase correction, masking, autocomplete +population, or saving to the database. Saving happens only via "Save All", +which fires the shared per-item save callback for every dataset at once. +""" +import json + +import dash +from dash import html, dcc, callback, Input, Output, State, ALL, MATCH +import dash_ag_grid as dag +import dash_mantine_components as dmc +from dash_iconify import DashIconify +import numpy as np + +from deeranalysis.components.data_viewer import data_viewer_layout, plot_upload +from deeranalysis.components.metadata_table import build_metadata_section_datarray, metadata_long_values_model +from deeranalysis.utils.deerlab_options import experiment_type_options +from deeranalysis.utils.file_parser import group_uploaded_files, parse_file_group +import deeranalysis.components.dataset_form as df # registers shared MATCH callbacks + + +def batch_import_layout(): + return html.Div([ + dcc.Store(id='batch-next-index', data=0), + + dcc.Upload( + id='batch-upload-data', + children=dmc.Group([ + DashIconify(icon="material-symbols:upload-file-outline", width=36, color="var(--mantine-color-blue-6)"), + dmc.Stack([ + dmc.Text("Drag and drop many files or click to select", size="sm", fw=500), + dmc.Text(".DSC/.DTA (Bruker BES3T) or .h5 (HDF5) — select all files at once", size="xs", c="dimmed"), + ], gap=2), + ], px="md", py="sm"), + style={ + 'width': '100%', + 'borderWidth': '2px', + 'borderStyle': 'dashed', + 'borderRadius': 'var(--mantine-radius-sm)', + 'borderColor': 'var(--mantine-color-blue-3)', + 'backgroundColor': 'var(--mantine-color-blue-light)', + 'cursor': 'pointer', + }, + multiple=True, className="mb-2", + ), + + dmc.Group([ + dmc.Autocomplete(id='batch-global-project', label="Set Project for all", w=220), + dmc.Autocomplete(id='batch-global-sample', label="Set Sample for all", w=220), + dmc.Select(id='batch-global-exp', label="Set Experiment Type for all", data=experiment_type_options, w=200), + dmc.Button("Apply to All", id='batch-apply-all-btn', variant='light', mt=22), + ], align='flex-end', mt='md', mb='lg'), + + html.Div( + id='batch-cards-grid', + children=[], + style={ + 'display': 'grid', + # 'gridTemplateColumns': 'repeat(auto-fit, minmax(280px, 1fr))', + 'gap': '16px', + 'width': '100%', + }, + ), + + dmc.Group([ + dmc.Button("Save All", id='batch-save-all-btn', color='blue', size='lg', mt='lg', + leftSection=DashIconify(icon="mdi:content-save-all-outline", width=18)), + ], justify='flex-end'), + + html.Div(id='batch-modals-container', children=[]), + ]) + + +def _card(index, title, dataset_store): + page = f'batch-{index}' + V = np.array(dataset_store['RealData']) + V = (V / np.max(np.abs(V))).tolist() if np.max(np.abs(V)) > 0 else V.tolist() + return dmc.Paper( + [ + dcc.ConfirmDialog( + id={'type': 'batch-remove-confirm', 'page': page}, + message="Remove this dataset from the batch?", + ), + dmc.Group([ + dmc.Box( + dmc.Sparkline(data=V, color="blue", h=60, w="100%", curveType="linear", strokeWidth=1.5), + style={'width': '200px', 'flexShrink': 0}, + ), + dmc.Stack([ + dmc.Group([ + dmc.Text(title, size="sm", fw=500, truncate=True, style={'flex': 1}), + dmc.ActionIcon( + DashIconify(icon="mdi:arrow-expand", width=16), + id={'type': 'batch-expand-btn', 'page': page}, + variant="subtle", color="gray", size="sm", + ), + dmc.ActionIcon( + DashIconify(icon="mdi:trash-can-outline", width=16), + id={'type': 'batch-remove-btn', 'page': page}, + variant="subtle", color="red", size="sm", + ), + ], justify="space-between", wrap="nowrap", gap="xs"), + dmc.Group([ + dmc.Autocomplete(id={'type': 'card-project-name', 'page': page}, placeholder="Project", size="xs", style={'flex': 1}), + dmc.Autocomplete(id={'type': 'card-sample-name', 'page': page}, placeholder="Sample", size="xs", style={'flex': 1}), + dmc.Select( + id={'type': 'card-exp-type', 'page': page}, + placeholder="Experiment Type", value='4pDEER', + data=experiment_type_options, size="xs", style={'flex': 1}, + ), + ], gap="xs", grow=True, wrap="nowrap"), + dmc.Text(id={'type': 'batch-tmin-warning', 'page': page}, size="xs", c="yellow.9"), + ], gap="xs", style={'flex': 1}), + ], align="center", wrap="nowrap", gap="md"), + ], + id={'type': 'batch-card', 'page': page}, + withBorder=True, p="xs", radius="md", + ) + + +def _error_card(index, title, message): + page = f'batch-{index}' + return dmc.Paper( + [ + dcc.ConfirmDialog( + id={'type': 'batch-remove-confirm', 'page': page}, + message="Remove this dataset from the batch?", + ), + dmc.Group([ + dmc.Text(title, size="sm", fw=500, truncate=True, style={'flex': 1}), + dmc.ActionIcon( + DashIconify(icon="mdi:trash-can-outline", width=16), + id={'type': 'batch-remove-btn', 'page': page}, + variant="subtle", color="red", size="sm", + ), + ], justify="space-between", wrap="nowrap", gap="xs", mb="xs"), + dmc.Alert(message, color="red", variant="light"), + ], + id={'type': 'batch-card', 'page': page}, + withBorder=True, p="xs", radius="md", + ) + + +def _item_modal(index, title, dataset_store, delays_data, tmin): + page = f'batch-{index}' + metadata_children, long_values_store = build_metadata_section_datarray(dataset_store['attrs'], page=page) + return dmc.Modal( + id={'type': 'batch-modal', 'page': page}, + title=f"Edit Dataset — {title}", + size="90%", + opened=False, + children=dmc.Grid([ + dmc.GridCol([ + metadata_long_values_model(page), + dcc.Store(id={'type': 'dataset-store', 'page': page}, data=dataset_store), + dcc.Store(id={'type': 'metadata-modal-store', 'page': page}, data=long_values_store), + dmc.Autocomplete(id={'type': 'project-name', 'page': page}, label="Project Name", mb="sm"), + dmc.Autocomplete(id={'type': 'sample-name', 'page': page}, label="Sample Name", mb="sm"), + dmc.TextInput(id={'type': 'dataset-name', 'page': page}, label="Dataset Name", value=title, mb="sm"), + dmc.Select( + label='Experiment Type:', + id={'type': 'experiment-type-dropdown', 'page': page}, + value='4pDEER', + data=experiment_type_options, + ), + dmc.Title("Delays", order=4, mt="md", mb="sm"), + html.Div(id={'type': 'tmin-warning-div', 'page': page}), + dmc.NumberInput( + id={'type': 'tmin', 'page': page}, + label='tmin (ns)', allowNegative=True, value=tmin, step=4, + stepHoldDelay=500, stepHoldInterval=100, decimalScale=2, mb="sm", + ), + dag.AgGrid( + id={'type': 'delays-grid', 'page': page}, + columnDefs=[ + {'field': 'parameter', 'headerName': 'Parameter'}, + {'field': 'value', 'headerName': 'Value (ns)', 'editable': True}, + ], + rowData=delays_data, + className="ag-theme-alpine", + style={'height': '160px', 'width': '100%'}, + ), + dmc.Accordion( + children=[ + dmc.AccordionItem( + value="parameters", + children=[ + dmc.AccordionControl("Metadata"), + dmc.AccordionPanel( + html.Div( + id={'type': "metadata-content", 'page': page}, + children=metadata_children, + style={"maxHeight": "24vh", "overflow": "auto"}, + ) + ), + ], + ) + ], + mt="md", + ), + ], span=4), + dmc.GridCol([ + data_viewer_layout(page_id=page, masking_enabled=True, inital_figure=plot_upload(dataset_store)), + ], span=8), + # Hidden trigger — actual saving is done in bulk via the "Save All" button, + # which fires this shared save-dataset-btn pattern for every item. + dmc.Button(id={'type': 'save-dataset-btn', 'page': page}, style={'display': 'none'}), + ]), + ) + + +def _error_message(message): + return [dict( + title='Error', message=message, + icon=DashIconify(icon="material-symbols:warning"), + color='red', duration=5000, position="top-center", + )] + + +@callback( + Output('batch-cards-grid', 'children'), + Output('batch-modals-container', 'children'), + Output('batch-next-index', 'data'), + Output('notification-container', 'sendNotifications', allow_duplicate=True), + Input('batch-upload-data', 'contents'), + State('batch-upload-data', 'filename'), + State('batch-cards-grid', 'children'), + State('batch-modals-container', 'children'), + State('batch-next-index', 'data'), + prevent_initial_call=True, +) +def handle_batch_upload(contents_list, filenames_list, cards, modals, next_index): + if not contents_list: + return dash.no_update, dash.no_update, dash.no_update, dash.no_update + + cards = list(cards or []) + modals = list(modals or []) + errors = [] + + for group in group_uploaded_files(filenames_list): + group_contents = [contents_list[i] for i in group] + group_filenames = [filenames_list[i] for i in group] + try: + store_data, delays_data, title, tmin = parse_file_group(group_contents, group_filenames) + except Exception as e: + errors.append(f"{' + '.join(group_filenames)}: {e}") + cards.append(_error_card(next_index, ' + '.join(group_filenames), str(e))) + next_index += 1 + continue + + delays_data, store_data = df.check_delays('4pDEER', delays_data, store_data) + cards.append(_card(next_index, title, store_data)) + modals.append(_item_modal(next_index, title, store_data, delays_data, tmin)) + next_index += 1 + + alert = _error_message("Some files could not be imported:\n" + "\n".join(errors)) if errors else dash.no_update + return cards, modals, next_index, alert + + +@callback( + Output({'type': 'batch-modal', 'page': MATCH}, 'opened'), + Input({'type': 'batch-expand-btn', 'page': MATCH}, 'n_clicks'), + prevent_initial_call=True, +) +def open_batch_item_modal(n_clicks): + return bool(n_clicks) + + +@callback( + Output({'type': 'metadata-value-modal', 'page': MATCH}, 'opened', allow_duplicate=True), + Output({'type': 'metadata-value-modal-text', 'page': MATCH}, 'value', allow_duplicate=True), + Input({'type': 'metadata-show-btn', 'page': MATCH, 'key': ALL}, 'n_clicks'), + State({'type': 'metadata-modal-store', 'page': MATCH}, 'data'), + prevent_initial_call=True, +) +def open_batch_metadata_modal(n_clicks_list, store_data): + if not any(n_clicks_list or []): + return dash.no_update, dash.no_update + triggered_id = dash.callback_context.triggered[0]['prop_id'].split('.')[0] + key = json.loads(triggered_id)['key'] + full_value = (store_data or {}).get(key, "Value not found.") + return True, full_value + + +@callback( + Output({'type': 'batch-card', 'page': MATCH}, 'style'), + Output({'type': 'batch-tmin-warning', 'page': MATCH}, 'children'), + Input({'type': 'dataset-store', 'page': MATCH}, 'data'), + Input({'type': 'card-exp-type', 'page': MATCH}, 'value'), +) +def update_card_style(dataset_store, exp_type): + if dataset_store is None: + return {'opacity': 0.4}, None + warning = df.check_tmin_warning(dataset_store, exp_type) + if warning: + style = { + 'backgroundColor': 'var(--mantine-color-yellow-0)', + 'borderColor': 'var(--mantine-color-yellow-6)', + } + return style, f"⚠ {warning}" + return {}, None + + +# --------------------------------------------------------------------------- +# Keep the compact card fields and the modal's full-size fields in sync. +# --------------------------------------------------------------------------- +@callback( + Output({'type': 'project-name', 'page': MATCH}, 'value', allow_duplicate=True), + Input({'type': 'card-project-name', 'page': MATCH}, 'value'), + prevent_initial_call=True, +) +def sync_project_card_to_modal(value): + return value + + +@callback( + Output({'type': 'card-project-name', 'page': MATCH}, 'value', allow_duplicate=True), + Input({'type': 'project-name', 'page': MATCH}, 'value'), + prevent_initial_call=True, +) +def sync_project_modal_to_card(value): + return value + + +@callback( + Output({'type': 'sample-name', 'page': MATCH}, 'value', allow_duplicate=True), + Input({'type': 'card-sample-name', 'page': MATCH}, 'value'), + prevent_initial_call=True, +) +def sync_sample_card_to_modal(value): + return value + + +@callback( + Output({'type': 'card-sample-name', 'page': MATCH}, 'value', allow_duplicate=True), + Input({'type': 'sample-name', 'page': MATCH}, 'value'), + prevent_initial_call=True, +) +def sync_sample_modal_to_card(value): + return value + + +@callback( + Output({'type': 'experiment-type-dropdown', 'page': MATCH}, 'value', allow_duplicate=True), + Input({'type': 'card-exp-type', 'page': MATCH}, 'value'), + prevent_initial_call=True, +) +def sync_exp_card_to_modal(value): + return value + + +@callback( + Output({'type': 'card-exp-type', 'page': MATCH}, 'value', allow_duplicate=True), + Input({'type': 'experiment-type-dropdown', 'page': MATCH}, 'value'), + prevent_initial_call=True, +) +def sync_exp_modal_to_card(value): + return value + + +@callback( + Output({'type': 'card-project-name', 'page': MATCH}, 'data'), + Output({'type': 'card-sample-name', 'page': MATCH}, 'data'), + Input({'type': 'card-project-name', 'page': MATCH}, 'n_clicks'), + prevent_initial_call=False, +) +def populate_card_autocomplete_options(_n_clicks): + return df.get_projects_and_samples() + + +# --------------------------------------------------------------------------- +# Apply the global Project / Sample / Experiment fields to every dataset +# --------------------------------------------------------------------------- +@callback( + Output({'type': 'project-name', 'page': ALL}, 'value'), + Output({'type': 'sample-name', 'page': ALL}, 'value'), + Output({'type': 'experiment-type-dropdown', 'page': ALL}, 'value'), + Input('batch-apply-all-btn', 'n_clicks'), + State('batch-global-project', 'value'), + State('batch-global-sample', 'value'), + State('batch-global-exp', 'value'), + State({'type': 'project-name', 'page': ALL}, 'value'), + prevent_initial_call=True, +) +def apply_to_all(_, global_project, global_sample, global_exp, projects): + n = len(projects) + new_projects = [global_project] * n if global_project else dash.no_update + new_samples = [global_sample] * n if global_sample else dash.no_update + new_exps = [global_exp] * n if global_exp else dash.no_update + return new_projects, new_samples, new_exps + + +# --------------------------------------------------------------------------- +# Save All — fires the shared per-item save-dataset-btn callback for +# every dataset currently in the batch. +# --------------------------------------------------------------------------- +@callback( + Output({'type': 'save-dataset-btn', 'page': ALL}, 'n_clicks'), + Input('batch-save-all-btn', 'n_clicks'), + State({'type': 'save-dataset-btn', 'page': ALL}, 'n_clicks'), + prevent_initial_call=True, +) +def save_all(_, current_clicks): + return [(c or 0) + 1 for c in current_clicks] + + +# --------------------------------------------------------------------------- +# Remove a dataset from the batch (with confirmation) +# --------------------------------------------------------------------------- +@callback( + Output({'type': 'batch-remove-confirm', 'page': MATCH}, 'displayed'), + Input({'type': 'batch-remove-btn', 'page': MATCH}, 'n_clicks'), + prevent_initial_call=True, +) +def ask_remove_confirmation(n_clicks): + return bool(n_clicks) + + +@callback( + Output('batch-cards-grid', 'children', allow_duplicate=True), + Output('batch-modals-container', 'children', allow_duplicate=True), + Input({'type': 'batch-remove-confirm', 'page': ALL}, 'submit_n_clicks'), + State('batch-cards-grid', 'children'), + State('batch-modals-container', 'children'), + prevent_initial_call=True, +) +def remove_item(_submit_clicks, cards, modals): + ctx = dash.callback_context + if not ctx.triggered or not ctx.triggered[0]['value']: + return dash.no_update, dash.no_update + triggered_id = ctx.triggered[0]['prop_id'].split('.')[0] + page = json.loads(triggered_id)['page'] + cards = [c for c in cards if c['props']['id'].get('page') != page] + modals = [m for m in modals if m['props']['id'].get('page') != page] + return cards, modals diff --git a/src/deeranalysis/components/dataset_form.py b/src/deeranalysis/components/dataset_form.py index dd88d0e..325c2ac 100644 --- a/src/deeranalysis/components/dataset_form.py +++ b/src/deeranalysis/components/dataset_form.py @@ -72,13 +72,9 @@ def update_delays(_, delays_row_data, dataset_store): # --------------------------------------------------------------------------- # Store / experiment type change → check tmin plausibility # --------------------------------------------------------------------------- -@callback( - Output({'type': 'tmin-warning-div', 'page': MATCH}, 'children'), - Input({'type': 'dataset-store', 'page': MATCH}, 'data'), - Input({'type': 'experiment-type-dropdown', 'page': MATCH}, 'value'), - prevent_initial_call=True, -) -def check_tmin(store, exp_type): +def check_tmin_warning(store, exp_type): + """Returns a warning message string if tmin looks wrong for the given + experiment type, or None if the delays/data are plausible.""" if store is None: return None delays = store.get('delays', {}) @@ -98,22 +94,39 @@ def check_tmin(store, exp_type): threshold = 50 / 1e3 if abs(peak_time - data_max_time) > threshold: - return dmc.Alert( - title="Warning: tmin may be incorrect", - children=( - f"The expected peak time based on delays is {peak_time * 1e3:.0f} ns, " - f"but the data maximum is at {data_max_time * 1e3:.0f} ns. " - "Please check that tmin is set correctly." - ), - color="yellow", - mb="md", + return ( + f"The expected peak time based on delays is {peak_time * 1e3:.0f} ns, " + f"but the data maximum is at {data_max_time * 1e3:.0f} ns. " + "Please check that tmin is set correctly." ) return None +@callback( + Output({'type': 'tmin-warning-div', 'page': MATCH}, 'children'), + Input({'type': 'dataset-store', 'page': MATCH}, 'data'), + Input({'type': 'experiment-type-dropdown', 'page': MATCH}, 'value'), + prevent_initial_call=True, +) +def check_tmin(store, exp_type): + message = check_tmin_warning(store, exp_type) + if message: + return dmc.Alert(title="Warning: tmin may be incorrect", children=message, color="yellow", mb="md") + return None + + # --------------------------------------------------------------------------- # Populate project / sample autocomplete options from the database # --------------------------------------------------------------------------- +def get_projects_and_samples(): + session = get_session() + datasets = session.query(Dataset).all() + session.close() + projects = list(set(ds.project for ds in datasets)) + samples = list(set(ds.sample for ds in datasets)) + return projects, samples + + @callback( Output({'type': 'project-name', 'page': MATCH}, 'data'), Output({'type': 'sample-name', 'page': MATCH}, 'data'), @@ -121,12 +134,7 @@ def check_tmin(store, exp_type): prevent_initial_call=False, ) def update_projects_and_samples(_n_clicks): - session = get_session() - datasets = session.query(Dataset).all() - session.close() - projects = list(set(ds.project for ds in datasets)) - samples = list(set(ds.sample for ds in datasets)) - return projects, samples + return get_projects_and_samples() # --------------------------------------------------------------------------- diff --git a/src/deeranalysis/components/logs_import_modal.py b/src/deeranalysis/components/logs_import_modal.py index 2070bc4..3e72513 100644 --- a/src/deeranalysis/components/logs_import_modal.py +++ b/src/deeranalysis/components/logs_import_modal.py @@ -130,8 +130,8 @@ def _build_signal_figure(dataset_store, correct_phase, masking_enabled): @callback( - Output({"type": "metadata-value-modal", 'page': page_id}, "opened"), - Output({"type": "metadata-value-modal-text", 'page': page_id}, "value"), + Output({"type": "metadata-value-modal", 'page': page_id}, "opened", allow_duplicate=True), + Output({"type": "metadata-value-modal-text", 'page': page_id}, "value", allow_duplicate=True), Input({"type": "metadata-show-btn", "key": dash.ALL}, "n_clicks"), State({"type": "metadata-modal-store", "page": page_id}, "data"), prevent_initial_call=True, diff --git a/src/deeranalysis/components/metadata_table.py b/src/deeranalysis/components/metadata_table.py index add999d..a46acc3 100644 --- a/src/deeranalysis/components/metadata_table.py +++ b/src/deeranalysis/components/metadata_table.py @@ -26,10 +26,13 @@ def metadata_long_values_model(page_id): )]) -def make_value_cell(key, value,long_values_store): +def make_value_cell(key, value, long_values_store, page=None): val_str = str(value) if len(val_str) > MAX_LEN: long_values_store[key] = val_str + show_btn_id = {"type": "metadata-show-btn", "key": key} + if page is not None: + show_btn_id["page"] = page return html.Td([ html.Span( val_str[:MAX_LEN] + "…", @@ -37,7 +40,7 @@ def make_value_cell(key, value,long_values_store): ), dmc.Button( "Show", - id={"type": "metadata-show-btn", "key": key}, + id=show_btn_id, size="compact-xs", variant="subtle", n_clicks=0, @@ -45,16 +48,16 @@ def make_value_cell(key, value,long_values_store): ]) return html.Td(val_str, style={"fontFamily": "monospace"}) -def build_table_rows(items_dict,long_values_store): +def build_table_rows(items_dict, long_values_store, page=None): rows = [] for key, value in items_dict.items(): rows.append(html.Tr([ html.Td(html.Strong(str(key)), style={"whiteSpace": "nowrap", "paddingRight": "16px"}), - make_value_cell(key, value,long_values_store), + make_value_cell(key, value, long_values_store, page=page), ])) return rows -def build_metadata_section(dataset,delays=True): +def build_metadata_section(dataset,delays=True,page=None): """Build the metadata & delays section children for a dataset database element. Returns a tuple of (children, long_values_store) where: @@ -67,7 +70,7 @@ def build_metadata_section(dataset,delays=True): if dataset.meta: try: - rows = build_table_rows(dataset.meta,long_values_store) + rows = build_table_rows(dataset.meta,long_values_store,page=page) # metadata_sections.append(dmc.Title("Metadata", order=5, mb="xs")) metadata_sections.append( dmc.Table( @@ -86,7 +89,7 @@ def build_metadata_section(dataset,delays=True): if dataset.delays and delays: try: - rows = build_table_rows(dataset.delays,long_values_store) + rows = build_table_rows(dataset.delays,long_values_store,page=page) metadata_sections.append(dmc.Divider(my="sm")) metadata_sections.append(dmc.Title("Delays", order=5, mb="xs")) metadata_sections.append( @@ -105,8 +108,8 @@ def build_metadata_section(dataset,delays=True): return html.Div(metadata_sections, style={"padding": "8px"}), long_values_store -def build_metadata_section_datarray(datarray): - """Build the metadata section children for a datarray. +def build_metadata_section_datarray(datarray, page=None): + """Build the metadata section children for a datarray (or a plain attrs dict). Returns a tuple of (children, long_values_store) where: - children is a list of Dash components to render @@ -117,9 +120,11 @@ def build_metadata_section_datarray(datarray): metadata_sections = [] - if datarray.attrs: + attrs = datarray.attrs if hasattr(datarray, 'attrs') else datarray + + if attrs: try: - rows = build_table_rows(datarray.attrs,long_values_store) + rows = build_table_rows(attrs,long_values_store,page=page) metadata_sections.append( dmc.Table( children=[html.Tbody(rows)], diff --git a/src/deeranalysis/pages/dataset_detail.py b/src/deeranalysis/pages/dataset_detail.py index 89e5a7f..ad05285 100644 --- a/src/deeranalysis/pages/dataset_detail.py +++ b/src/deeranalysis/pages/dataset_detail.py @@ -542,8 +542,8 @@ def save_delays(n_clicks,dataset_id,rowData,tmin): return _alert(f"Error saving delays: {exc}", "red", "Save Error")#,dash.no_update @callback( - Output({"type": "metadata-value-modal",'page': page_id}, "opened"), - Output({"type": "metadata-value-modal-text",'page': page_id}, "value"), + Output({"type": "metadata-value-modal",'page': page_id}, "opened", allow_duplicate=True), + Output({"type": "metadata-value-modal-text",'page': page_id}, "value", allow_duplicate=True), Input({"type": "metadata-show-btn", "key": dash.ALL}, "n_clicks"), State({"type": "metadata-modal-store",'page': page_id}, "data"), prevent_initial_call=True, diff --git a/src/deeranalysis/pages/datasets.py b/src/deeranalysis/pages/datasets.py index 632fb97..7adb835 100644 --- a/src/deeranalysis/pages/datasets.py +++ b/src/deeranalysis/pages/datasets.py @@ -270,8 +270,8 @@ def populate_details(selected_rows): @callback( - Output({"type": "metadata-value-modal",'page': page_id}, "opened"), - Output({"type":"metadata-value-modal-text",'page': page_id}, "value"), + Output({"type": "metadata-value-modal",'page': page_id}, "opened", allow_duplicate=True), + Output({"type":"metadata-value-modal-text",'page': page_id}, "value", allow_duplicate=True), Input({"type": "metadata-show-btn", "key": dash.ALL}, "n_clicks"), State("metadata-modal-store", "data"), prevent_initial_call=True, diff --git a/src/deeranalysis/pages/upload.py b/src/deeranalysis/pages/upload.py index ff5fc51..b370dde 100644 --- a/src/deeranalysis/pages/upload.py +++ b/src/deeranalysis/pages/upload.py @@ -1,81 +1,23 @@ import dash from dash import html, dcc, callback, Input, Output, State -import base64 -import io import json -from deeranalysis.utils.eprload import bes3t_eprload -from deeranalysis.utils.pulsespel_parser import parse_PulseSpel -import numpy as np -import xarray as xr import dash_ag_grid as dag import dash_mantine_components as dmc from dash_iconify import DashIconify -import pyepr as pyepr from deeranalysis.components.metadata_table import build_metadata_section_datarray, metadata_long_values_model from deeranalysis.components.data_viewer import data_viewer_layout, plot_upload from deeranalysis.utils.deerlab_options import experiment_type_options from deeranalysis.utils.csv_loader import parse_csv_raw, build_csv_store +from deeranalysis.utils.file_parser import load_bes3t, enrich_bes3t, load_hdf5, enrich_hdf5, dataarray_to_store +from deeranalysis.components.batch_import import batch_import_layout import deeranalysis.components.dataset_form as df # registers shared MATCH callbacks dash.register_page(__name__) page_id = 'upload' -layout = html.Div([ - metadata_long_values_model(page_id), - dcc.Store(id={"type": "metadata-modal-store", "page": page_id}, data=""), - dcc.Store(id='csv-raw-store'), - # ---- CSV import modal -------------------------------------------------- - dmc.Modal( - id='csv-import-modal', - title=dmc.Title("Import CSV File", order=3), - size='xl', - opened=False, - children=dmc.Stack([ - html.Div(id='csv-preview'), - dmc.Group([ - dmc.NumberInput( - id='csv-skiprows', label='Skip rows', value=0, min=0, step=1, w=140, - ), - dmc.Select( - id='csv-separator', label='Separator', value=',', w=180, - data=[ - {'label': 'Comma ( , )', 'value': ','}, - {'label': 'Semicolon ( ; )', 'value': ';'}, - {'label': 'Tab', 'value': '\t'}, - {'label': 'Space', 'value': ' '}, - ], - ), - dmc.Select( - id='csv-time-unit', label='Time unit', value='us', w=140, - data=[ - {'label': 'ns', 'value': 'ns'}, - {'label': 'µs', 'value': 'us'}, - {'label': 'ms', 'value': 'ms'}, - {'label': 's', 'value': 's'}, - ], - ), - dmc.Switch( - id='csv-has-header', label='Has header row', checked=True, mt='xl', - ), - ]), - dmc.Group([ - dmc.Select(id='csv-t-col', label='Time column (t)', placeholder='Select column…', style={'flex': 1}), - dmc.Select(id='csv-vre-col', label='Real signal (V_re)', placeholder='Select column…', style={'flex': 1}), - dmc.Select(id='csv-vim-col', label='Imaginary signal (V_im)', placeholder='None', style={'flex': 1}, clearable=True), - ], grow=True), - dmc.Group([ - dmc.Button('Cancel', id='csv-cancel-btn', color='gray', variant='subtle'), - dmc.Button('Import', id='csv-import-btn', color='blue', - leftSection=DashIconify(icon='mdi:file-import-outline', width=16)), - ], justify='flex-end'), - ], gap='md'), - ), - - dmc.Title("Import Dataset from File", order=1, mb="md"), - dmc.Divider(mb="lg"), - - dmc.Grid([ +def _single_import_layout(): + return dmc.Grid([ dmc.GridCol([ dcc.Upload( id='upload-data', @@ -156,7 +98,72 @@ dmc.Button("Add Dataset to Library", id={'type': 'save-dataset-btn', 'page': page_id}, color="blue", mb="lg", size="lg"), ], span=8), - ]), + ]) + + +layout = html.Div([ + metadata_long_values_model(page_id), + dcc.Store(id={"type": "metadata-modal-store", "page": page_id}, data=""), + dcc.Store(id='csv-raw-store'), + + # ---- CSV import modal -------------------------------------------------- + dmc.Modal( + id='csv-import-modal', + title=dmc.Title("Import CSV File", order=3), + size='xl', + opened=False, + children=dmc.Stack([ + html.Div(id='csv-preview'), + dmc.Group([ + dmc.NumberInput( + id='csv-skiprows', label='Skip rows', value=0, min=0, step=1, w=140, + ), + dmc.Select( + id='csv-separator', label='Separator', value=',', w=180, + data=[ + {'label': 'Comma ( , )', 'value': ','}, + {'label': 'Semicolon ( ; )', 'value': ';'}, + {'label': 'Tab', 'value': '\t'}, + {'label': 'Space', 'value': ' '}, + ], + ), + dmc.Select( + id='csv-time-unit', label='Time unit', value='us', w=140, + data=[ + {'label': 'ns', 'value': 'ns'}, + {'label': 'µs', 'value': 'us'}, + {'label': 'ms', 'value': 'ms'}, + {'label': 's', 'value': 's'}, + ], + ), + dmc.Switch( + id='csv-has-header', label='Has header row', checked=True, mt='xl', + ), + ]), + dmc.Group([ + dmc.Select(id='csv-t-col', label='Time column (t)', placeholder='Select column…', style={'flex': 1}), + dmc.Select(id='csv-vre-col', label='Real signal (V_re)', placeholder='Select column…', style={'flex': 1}), + dmc.Select(id='csv-vim-col', label='Imaginary signal (V_im)', placeholder='None', style={'flex': 1}, clearable=True), + ], grow=True), + dmc.Group([ + dmc.Button('Cancel', id='csv-cancel-btn', color='gray', variant='subtle'), + dmc.Button('Import', id='csv-import-btn', color='blue', + leftSection=DashIconify(icon='mdi:file-import-outline', width=16)), + ], justify='flex-end'), + ], gap='md'), + ), + + dmc.Title("Import Dataset from File", order=1, mb="md"), + dmc.Divider(mb="lg"), + + dmc.Tabs([ + dmc.TabsList([ + dmc.TabsTab("Single Import", value="single", leftSection=DashIconify(icon="mdi:file-upload-outline", width=16)), + dmc.TabsTab("Batch Import", value="batch", leftSection=DashIconify(icon="mdi:file-multiple-outline", width=16)), + ]), + dmc.TabsPanel(value="single", pt="lg", children=_single_import_layout()), + dmc.TabsPanel(value="batch", pt="lg", children=batch_import_layout()), + ], value="single"), ]) @@ -226,25 +233,14 @@ def handle_file_upload(contents_list, filenames_list): dsc_content, dta_content = contents_list[0], contents_list[1] else: dsc_content, dta_content = contents_list[1], contents_list[0] - dsc_decoded = base64.b64decode(dsc_content.split(',')[1]) - dta_decoded = base64.b64decode(dta_content.split(',')[1]) - dataarray = bes3t_eprload(DSC=dsc_decoded, DTA=dta_decoded) + dataarray = load_bes3t(dsc_content, dta_content) metadata_children, long_values_store = build_metadata_section_datarray(dataarray) - dataarray.attrs.update({'file_format': 'BES3T'}) - dataarray.attrs.update(parse_PulseSpel(dataarray.attrs.get('PlsSPELGlbTxt', ''))) - delays = get_delays_dict(dataarray) - tmin = dataarray.attrs.get('deadtime', 0) - - if dataarray.ndim ==2: - dataarray = dataarray.sum('Y') + dataarray, delays, tmin = enrich_bes3t(dataarray) elif file_format == 'hdf5': - decoded = base64.b64decode(contents_list[0].split(',')[1]) - dataarray = pyepr.eprload(io.BytesIO(decoded), type='HDF5') + dataarray = load_hdf5(contents_list[0]) metadata_children, long_values_store = build_metadata_section_datarray(dataarray) - delays = get_delays_dict(dataarray) - tmin = dataarray.t.values.min() * 1e3 - dataarray.attrs['title'] = filenames_list[0].split('/')[-1].split('.')[0] + delays, tmin = enrich_hdf5(dataarray, filenames_list[0]) except Exception as e: import traceback @@ -252,18 +248,7 @@ def handle_file_upload(contents_list, filenames_list): print(traceback.format_exc()) return *no_update_9[:6], _error_message(f"Error processing files: {str(e)}"), dash.no_update, dash.no_update - t = dataarray.t.values if 't' in dataarray.coords else dataarray.X.values - t = t - t[0] - - store_data = { - 'RealData': dataarray.real.values.tolist(), - 'ImagData': dataarray.imag.values.tolist(), - 't': t.tolist(), - 'attrs': dataarray.attrs, - 'delays': delays, - 'tmin': tmin, - 'masked_indices': [], - } + store_data = dataarray_to_store(dataarray, delays, tmin) delays_data = [{'parameter': k, 'value': v} for k, v in delays.items()] delays_data, store_data = df.check_delays('4pDEER',delays_data,store_data) return store_data, metadata_children, long_values_store, delays_data, dataarray.attrs.get('title', ''), tmin, alert, dash.no_update, dash.no_update @@ -283,8 +268,8 @@ def _build_signal_figure(dataset_store, correct_phase, masking_enabled): @callback( - Output({"type": "metadata-value-modal", 'page': page_id}, "opened"), - Output({"type": "metadata-value-modal-text", 'page': page_id}, "value"), + Output({"type": "metadata-value-modal", 'page': page_id}, "opened", allow_duplicate=True), + Output({"type": "metadata-value-modal-text", 'page': page_id}, "value", allow_duplicate=True), Input({"type": "metadata-show-btn", "key": dash.ALL}, "n_clicks"), State({"type": "metadata-modal-store", "page": page_id}, "data"), prevent_initial_call=True, @@ -299,16 +284,6 @@ def open_metadata_modal(n_clicks_list, store_data): return True, full_value -def get_delays_dict(dataarray: xr.DataArray): - """Extracts delay parameters from the dataset attributes.""" - delay_keys = ['tau1', 'tau2', 'tau3', 'tau4', 'tau5', 'tau6'] - delays = {} - for key in delay_keys: - if key in dataarray.attrs: - delays[key] = dataarray.attrs[key] - return delays - - # --------------------------------------------------------------------------- # CSV import modal callbacks # --------------------------------------------------------------------------- diff --git a/src/deeranalysis/utils/file_parser.py b/src/deeranalysis/utils/file_parser.py new file mode 100644 index 0000000..6ede2c9 --- /dev/null +++ b/src/deeranalysis/utils/file_parser.py @@ -0,0 +1,130 @@ +"""Shared dataset-file parsing logic used by both the single and batch +File Import flows. Kept separate from the Dash callbacks so the same +parsing code can be reused without duplicating it per page. +""" +import base64 +import io + +import pyepr as pyepr +from deeranalysis.utils.eprload import bes3t_eprload +from deeranalysis.utils.pulsespel_parser import parse_PulseSpel + + +def get_delays_dict(dataarray): + """Extracts delay parameters from the dataset attributes.""" + delay_keys = ['tau1', 'tau2', 'tau3', 'tau4', 'tau5', 'tau6'] + return {k: dataarray.attrs[k] for k in delay_keys if k in dataarray.attrs} + + +def load_bes3t(dsc_content, dta_content): + """Decodes and loads a Bruker BES3T (.DSC/.DTA) pair into a raw DataArray.""" + dsc_decoded = base64.b64decode(dsc_content.split(',')[1]) + dta_decoded = base64.b64decode(dta_content.split(',')[1]) + return bes3t_eprload(DSC=dsc_decoded, DTA=dta_decoded) + + +def enrich_bes3t(dataarray): + """Adds derived attrs/delays to a raw BES3T DataArray and collapses the Y axis if present. + + Returns (dataarray, delays, tmin). + """ + dataarray.attrs.update({'file_format': 'BES3T'}) + dataarray.attrs.update(parse_PulseSpel(dataarray.attrs.get('PlsSPELGlbTxt', ''))) + delays = get_delays_dict(dataarray) + tmin = dataarray.attrs.get('deadtime', 0) + if dataarray.ndim == 2: + dataarray = dataarray.sum('Y') + return dataarray, delays, tmin + + +def load_hdf5(content): + """Decodes and loads an HDF5 file into a raw DataArray.""" + decoded = base64.b64decode(content.split(',')[1]) + return pyepr.eprload(io.BytesIO(decoded), type='HDF5') + + +def enrich_hdf5(dataarray, filename): + """Adds derived title/delays/tmin to a raw HDF5 DataArray. Returns (delays, tmin).""" + delays = get_delays_dict(dataarray) + tmin = dataarray.t.values.min() * 1e3 + dataarray.attrs['title'] = filename.split('/')[-1].split('.')[0] + return delays, tmin + + +def dataarray_to_store(dataarray, delays, tmin): + """Converts a parsed DataArray into the dataset-store dict used across the app. + + *tmin* is given in ns (matching the delay parameters/the tmin NumberInput), + but 't' is in µs, and every consumer of the dataset-store dict (plotting, + tmin-plausibility checks, saving) adds 'tmin' directly to 't' with no + further conversion — so it must be stored here in µs already. + """ + t = dataarray.t.values if 't' in dataarray.coords else dataarray.X.values + t = t - t[0] + return { + 'RealData': dataarray.real.values.tolist(), + 'ImagData': dataarray.imag.values.tolist(), + 't': t.tolist(), + 'attrs': dataarray.attrs, + 'delays': delays, + 'tmin': tmin / 1e3, + 'masked_indices': [], + } + + +def group_uploaded_files(filenames): + """Groups a flat list of uploaded filenames into per-dataset groups. + + .DSC/.DTA files sharing a basename are paired together; every other file + forms its own single-file group. Returns a list of lists of indices into + *filenames*. + """ + groups = [] + used = set() + for i, name in enumerate(filenames): + if i in used: + continue + if name.endswith('.DSC') or name.endswith('.DTA'): + base = name.rsplit('.', 1)[0] + pair_ext = '.DTA' if name.endswith('.DSC') else '.DSC' + partner = next( + (j for j, other in enumerate(filenames) + if j not in used and j != i and other == base + pair_ext), + None, + ) + if partner is not None: + groups.append(sorted([i, partner])) + used.update([i, partner]) + continue + groups.append([i]) + used.add(i) + return groups + + +def parse_file_group(contents, filenames): + """Parses one dataset group (1 or 2 files) into a dataset-store dict. + + Returns (store_data, delays_data, title, tmin). Raises ValueError on + unsupported/invalid input. + """ + if len(filenames) == 2: + if {filenames[0].rsplit('.', 1)[-1], filenames[1].rsplit('.', 1)[-1]} != {'DSC', 'DTA'}: + raise ValueError("Expected a matching .DSC and .DTA file pair.") + if filenames[0].endswith('.DSC'): + dsc_content, dta_content = contents[0], contents[1] + else: + dsc_content, dta_content = contents[1], contents[0] + dataarray = load_bes3t(dsc_content, dta_content) + dataarray, delays, tmin = enrich_bes3t(dataarray) + elif filenames[0].endswith('.h5'): + dataarray = load_hdf5(contents[0]) + delays, tmin = enrich_hdf5(dataarray, filenames[0]) + elif filenames[0].endswith('.DSC') or filenames[0].endswith('.DTA'): + raise ValueError("Missing matching .DSC/.DTA file — upload both files together.") + else: + raise ValueError("Unsupported file type. Only .DSC/.DTA (Bruker BES3T) and .h5 (HDF5) are supported here.") + + store_data = dataarray_to_store(dataarray, delays, tmin) + delays_data = [{'parameter': k, 'value': v} for k, v in delays.items()] + title = dataarray.attrs.get('title', filenames[0].rsplit('.', 1)[0].split('/')[-1]) + return store_data, delays_data, title, tmin From c1247e5a3ebdf71a1ba593a0bfe1830c8383a203 Mon Sep 17 00:00:00 2001 From: Hugo Karas Date: Thu, 10 Sep 2026 14:42:56 +0200 Subject: [PATCH 2/4] Move LOGS-API page to be an Import Tab --- src/deeranalysis/app.py | 3 +- src/deeranalysis/components/batch_import.py | 40 ++- src/deeranalysis/components/dataset_form.py | 5 +- .../logs_import_tab.py} | 319 ++++++++---------- src/deeranalysis/pages/upload.py | 135 ++++---- 5 files changed, 254 insertions(+), 248 deletions(-) rename src/deeranalysis/{pages/logs_upload.py => components/logs_import_tab.py} (57%) diff --git a/src/deeranalysis/app.py b/src/deeranalysis/app.py index d9e0c68..41ea88b 100644 --- a/src/deeranalysis/app.py +++ b/src/deeranalysis/app.py @@ -11,7 +11,7 @@ from deeranalysis.components.setup_modal_desktop import create_setup_modal, get_DeerAnalysis_directory from deeranalysis.components.new_version_modal import new_version_modal, update_button from deeranalysis.components.dmc_theme import da_dmctheme -from deeranalysis.utils.logs_plugin import initialize_logs_api,check_logs_api_key +from deeranalysis.utils.logs_plugin import initialize_logs_api from deeranalysis.utils.database import get_appearance_settings from deeranalysis.utils.deerlab_options import resolve_plot_template @@ -80,7 +80,6 @@ def create_nav_link(label, href, icon): [ dmc.Text("Datasets", size="sm", c="dimmed", fw=500), create_nav_link("File Import", "/upload", "mdi:upload"), - *( [create_nav_link("Logs Import", "/logs_upload", "mdi:file-document-outline")] if check_logs_api_key() else [] ), create_nav_link("Datasets", "/", "mdi:database"), create_nav_link("Comparison", "/comparison", "mdi:compare"), diff --git a/src/deeranalysis/components/batch_import.py b/src/deeranalysis/components/batch_import.py index 95129e3..a2607df 100644 --- a/src/deeranalysis/components/batch_import.py +++ b/src/deeranalysis/components/batch_import.py @@ -370,10 +370,20 @@ def sync_exp_modal_to_card(value): @callback( Output({'type': 'card-project-name', 'page': MATCH}, 'data'), Output({'type': 'card-sample-name', 'page': MATCH}, 'data'), - Input({'type': 'card-project-name', 'page': MATCH}, 'n_clicks'), + Input({'type': 'card-project-name', 'page': MATCH}, 'n_blur'), + Input({'type': 'card-sample-name', 'page': MATCH}, 'n_blur'), prevent_initial_call=False, ) -def populate_card_autocomplete_options(_n_clicks): +def populate_card_autocomplete_options(_project_blur, _sample_blur): + return df.get_projects_and_samples() + +@callback( + Output('batch-global-project', 'data'), + Output('batch-global-sample', 'data'), + Input('batch-upload-data', 'contents'), + prevent_initial_call=False, +) +def populate_global_autocomplete_options(_contents): return df.get_projects_and_samples() @@ -388,15 +398,27 @@ def populate_card_autocomplete_options(_n_clicks): State('batch-global-project', 'value'), State('batch-global-sample', 'value'), State('batch-global-exp', 'value'), - State({'type': 'project-name', 'page': ALL}, 'value'), prevent_initial_call=True, ) -def apply_to_all(_, global_project, global_sample, global_exp, projects): - n = len(projects) - new_projects = [global_project] * n if global_project else dash.no_update - new_samples = [global_sample] * n if global_sample else dash.no_update - new_exps = [global_exp] * n if global_exp else dash.no_update - return new_projects, new_samples, new_exps +def apply_to_all(_, global_project, global_sample, global_exp): + # Each ALL output must be given a list exactly as long as the number of + # components it matched — a bare dash.no_update is rejected by Dash. The + # patterns also match the Single Import tab (page 'upload'), so leave any + # non-batch page untouched. + def _column(outputs, value): + if not value: + return [dash.no_update] * len(outputs) + return [ + value if str(o['id']['page']).startswith('batch-') else dash.no_update + for o in outputs + ] + + projects_out, samples_out, exps_out = dash.callback_context.outputs_list + return ( + _column(projects_out, global_project), + _column(samples_out, global_sample), + _column(exps_out, global_exp), + ) # --------------------------------------------------------------------------- diff --git a/src/deeranalysis/components/dataset_form.py b/src/deeranalysis/components/dataset_form.py index 325c2ac..919c3b8 100644 --- a/src/deeranalysis/components/dataset_form.py +++ b/src/deeranalysis/components/dataset_form.py @@ -130,10 +130,11 @@ def get_projects_and_samples(): @callback( Output({'type': 'project-name', 'page': MATCH}, 'data'), Output({'type': 'sample-name', 'page': MATCH}, 'data'), - Input({'type': 'project-name', 'page': MATCH}, 'n_clicks'), + Input({'type': 'project-name', 'page': MATCH}, 'n_blur'), + Input({'type': 'sample-name', 'page': MATCH}, 'n_blur'), prevent_initial_call=False, ) -def update_projects_and_samples(_n_clicks): +def update_projects_and_samples(_project_blur, _sample_blur): return get_projects_and_samples() diff --git a/src/deeranalysis/pages/logs_upload.py b/src/deeranalysis/components/logs_import_tab.py similarity index 57% rename from src/deeranalysis/pages/logs_upload.py rename to src/deeranalysis/components/logs_import_tab.py index a2b711c..91181a1 100644 --- a/src/deeranalysis/pages/logs_upload.py +++ b/src/deeranalysis/components/logs_import_tab.py @@ -1,6 +1,15 @@ +"""LOGS Data Import — browse a connected LOGS server and import datasets +from it. Shown as a tab on the Import page (``pages/upload.py``) alongside +Single Import and Batch Import; only rendered when a LOGS server/API key is +configured (``deeranalysis.utils.logs_plugin.check_logs_api_key``). + +Picking a dataset opens either a read-only preview modal or the shared +``logs_import_modal`` (the same dataset-entry form used elsewhere) to review +and save it — that flow is unchanged from when this lived at its own +``/logs_upload`` route. +""" import dash from dash import html, dcc, callback, Input, Output, State -import dash_bootstrap_components as dbc import dash_mantine_components as dmc from dash_iconify import DashIconify import dash_ag_grid as dag @@ -11,8 +20,6 @@ peek_page_id = 'logs-peek' -dash.register_page(__name__, path='/logs_upload') - logsTable_column_defs = [ {"field": "id", "headerName": "ID", "filter": False, "sortable": False,"width": 75}, {"field": "name", "headerName": "Name", "filter": True, "sortable": True, "flex": 1}, @@ -24,149 +31,148 @@ {"field": "actions", "headerName": "Actions", "filter": False, "sortable": False,"width":80, "cellRenderer": "DMC_DualIconButton", "cellRendererParams": {"label": "View","leftIcon":"ph:eye","rightIcon": "ph:download-simple", "variant": "outline", "size": "xs"}}, ] -# Layout -layout = dmc.Container([ - dcc.Store(id='logs-connected', data=False), - dcc.Store(id='logs-datasets-store', data=[]), - dmc.Group([ + +def logs_import_tab_layout(): + return html.Div([ + dcc.Store(id='logs-connected', data=False), + dcc.Store(id='logs-datasets-store', data=[]), dmc.Group([ - dmc.Title("LOGS Data Import", order=1), - dmc.Badge("Beta", color="orange", variant="light", size="lg"), - ], gap="xs"), - dmc.ActionIcon( - DashIconify(icon="mdi:refresh", width=20), - id="logs-refresh-btn", - variant="subtle", - size="lg", - ), - ], mb="md"), - dmc.Modal(id='connection-error', title="Connection Error!",withCloseButton=False,children=[ - dmc.Text("Unable to connect to the LOGs server. Please check your connection and credentials."), - dmc.Code(id='connection-error-msg') - ]), + dmc.Badge("Beta", color="orange", variant="light", size="sm"), + dmc.ActionIcon( + DashIconify(icon="mdi:refresh", width=20), + id="logs-refresh-btn", + variant="subtle", + size="lg", + ), + ], justify="space-between", mb="md"), + dmc.Modal(id='connection-error', title="Connection Error!",withCloseButton=False,children=[ + dmc.Text("Unable to connect to the LOGs server. Please check your connection and credentials."), + dmc.Code(id='connection-error-msg') + ]), + + # Filter Section with Multi-Select Boxes + dmc.Paper([ + dmc.Stack([ + dmc.Grid([ + # Persons Multi-Select + dmc.GridCol([ + dmc.MultiSelect( + id="persons-multiselect", + label="Persons", + placeholder="Select persons", + data=[], + searchable=True, + clearable=True, + leftSection=DashIconify(icon="mdi:account-multiple"), + ) + ], span={"base": 12, "sm": 6, "md": 4}), - # Filter Section with Multi-Select Boxes - dmc.Paper([ - dmc.Stack([ - dmc.Grid([ - # Persons Multi-Select - dmc.GridCol([ - dmc.MultiSelect( - id="persons-multiselect", - label="Persons", - placeholder="Select persons", - data=[], - searchable=True, - clearable=True, - leftSection=DashIconify(icon="mdi:account-multiple"), - ) - ], span={"base": 12, "sm": 6, "md": 4}), - - # Project Multi-Select - dmc.GridCol([ - dmc.MultiSelect( - id="project-multiselect", - label="Projects", - placeholder="Select projects", - data=[], - searchable=True, - clearable=True, - leftSection=DashIconify(icon="mdi:folder-multiple"), - ) - ], span={"base": 12, "sm": 6, "md": 4}), - - # Samples Multi-Select - dmc.GridCol([ - dmc.MultiSelect( - id="samples-multiselect", - label="Samples", - placeholder="Select samples", - data=[], - searchable=True, - clearable=True, - leftSection=DashIconify(icon="mdi:test-tube"), - ) - ], span={"base": 12, "sm": 6, "md": 4}), - - # Experiment Multi-Select - dmc.GridCol([ - dmc.MultiSelect( - id="experiment-multiselect", - label="Experiments", - placeholder="Select experiments", - data=[], - searchable=True, - clearable=True, - leftSection=DashIconify(icon="mdi:flask"), - ) - ], span={"base": 12, "sm": 6, "md": 4}), - - # Date Range - dmc.GridCol([ - dmc.DatePickerInput( - id="date-range-picker", - label="Date Range", - type="range", - placeholder="Select date range", - clearable=True, - ) - ], span={"base": 12, "sm": 6, "md": 4}), - ], gutter="md"), - ], gap="sm"), - ], p="md", mb="lg", shadow="sm", withBorder=True), - - # AG Grid Section - dmc.Paper([ - dmc.Title("Datasets", order=4, mb="md"), - dmc.Box( - [dmc.LoadingOverlay( - id="datasets-loading-overlay", - visible=False, - overlayProps={"radius": "sm", "blur": 2}, - zIndex=10, + # Project Multi-Select + dmc.GridCol([ + dmc.MultiSelect( + id="project-multiselect", + label="Projects", + placeholder="Select projects", + data=[], + searchable=True, + clearable=True, + leftSection=DashIconify(icon="mdi:folder-multiple"), + ) + ], span={"base": 12, "sm": 6, "md": 4}), + # Samples Multi-Select + dmc.GridCol([ + dmc.MultiSelect( + id="samples-multiselect", + label="Samples", + placeholder="Select samples", + data=[], + searchable=True, + clearable=True, + leftSection=DashIconify(icon="mdi:test-tube"), + ) + ], span={"base": 12, "sm": 6, "md": 4}), + + # Experiment Multi-Select + dmc.GridCol([ + dmc.MultiSelect( + id="experiment-multiselect", + label="Experiments", + placeholder="Select experiments", + data=[], + searchable=True, + clearable=True, + leftSection=DashIconify(icon="mdi:flask"), + ) + ], span={"base": 12, "sm": 6, "md": 4}), + + # Date Range + dmc.GridCol([ + dmc.DatePickerInput( + id="date-range-picker", + label="Date Range", + type="range", + placeholder="Select date range", + clearable=True, + ) + ], span={"base": 12, "sm": 6, "md": 4}), + ], gutter="md"), + ], gap="sm"), + ], p="md", mb="lg", shadow="sm", withBorder=True), + + # AG Grid Section + dmc.Paper([ + dmc.Title("Datasets", order=4, mb="md"), + dmc.Box( + [dmc.LoadingOverlay( + id="datasets-loading-overlay", + visible=False, + overlayProps={"radius": "sm", "blur": 2}, + zIndex=10, + + ), + dag.AgGrid( + id="datasets-grid", + columnDefs=logsTable_column_defs, + rowData=[], + defaultColDef={ + "resizable": True, + "sortable": True, + "filter": True, + }, + dashGridOptions={ + "pagination": True, + "paginationPageSize": 20, + "suppressPaginationPanel": True, + }, + style={"height": "600px"}, + className="ag-theme-alpine", + ), + ],pos="relative", ), - dag.AgGrid( - id="datasets-grid", - columnDefs=logsTable_column_defs, - rowData=[], - defaultColDef={ - "resizable": True, - "sortable": True, - "filter": True, - }, - dashGridOptions={ - "pagination": True, - "paginationPageSize": 20, - "suppressPaginationPanel": True, - }, - style={"height": "600px"}, - className="ag-theme-alpine", + dmc.Group([ + dmc.Pagination(id="datasets-pagination", total=1, value=1, siblings=1), + ], justify="center", mt="sm"), + ], p="md", shadow="sm", withBorder=True), + dmc.Modal( + id='logs-peek-modal', + title="Preview Dataset", + size="80%", + opened=False, + children=[ + dcc.Store(id={'type': 'dataset-store', 'page': peek_page_id}), + data_viewer_layout(page_id=peek_page_id, correct_phase=True), + dmc.Group( + [dmc.Button("Close", id="close-logs-peek-btn", variant="subtle", color="gray")], + justify="flex-end", + mt="md" ), - ],pos="relative", + ], + overlayProps={"color": "black", "opacity": 0.5, "blur": 0.5}, ), - dmc.Group([ - dmc.Pagination(id="datasets-pagination", total=1, value=1, siblings=1), - ], justify="center", mt="sm"), - ], p="md", shadow="sm", withBorder=True), - dmc.Modal( - id='logs-peek-modal', - title="Preview Dataset", - size="80%", - opened=False, - children=[ - dcc.Store(id={'type': 'dataset-store', 'page': peek_page_id}), - data_viewer_layout(page_id=peek_page_id, correct_phase=True), - dmc.Group( - [dmc.Button("Close", id="close-logs-peek-btn", variant="subtle", color="gray")], - justify="flex-end", - mt="md" - ), - ], - overlayProps={"color": "black", "opacity": 0.5, "blur": 0.5}, - ), - create_logs_import_modal() -],fluid=True, size="xl") + create_logs_import_modal() + ]) + @callback( Output("persons-multiselect", "data"), @@ -216,37 +222,6 @@ def update_datasets_store(connected, persons, projects): return logs.get_datasets_rowdata(person_ids=persons, project_ids=projects) -# @callback( -# Output("datasets-grid", "rowData"), -# Output("experiment-multiselect", "data"), -# Input("logs-datasets-store", "data"), -# Input("samples-multiselect", "value"), -# Input("date-range-picker", "value"), -# Input("experiment-multiselect", "value"), -# ) -# def filter_datasets_grid(all_data, samples, date_range, experiments): -# if not all_data: -# experiment_options = [] -# return [], experiment_options - -# # Derive experiment options from current store data -# experiment_options = sorted(set(r["experiment"] for r in all_data if r.get("experiment"))) -# experiment_options = [{"value": e, "label": e} for e in experiment_options] - -# filtered = all_data - -# if samples: -# filtered = [r for r in filtered if r.get("sample") in samples] - -# if date_range and len(date_range) == 2 and date_range[0] and date_range[1]: -# start, end = date_range[0], date_range[1] -# filtered = [r for r in filtered if start <= (r.get("date") or "") <= end] - -# if experiments: -# filtered = [r for r in filtered if r.get("experiment") in experiments] - -# return filtered, experiment_options - @callback( Output("datasets-grid", "rowData"), Output("datasets-pagination", "total"), @@ -370,4 +345,4 @@ def check_logs_connection(url, n_clicks): disabled_grid_style = {**grid_style, "pointerEvents": "none", "opacity": 0.5} return True, e.__str__(), True, True, True, True, True, disabled_grid_style, False else: - return False, "", False, False, False, False, False, grid_style, True \ No newline at end of file + return False, "", False, False, False, False, False, grid_style, True diff --git a/src/deeranalysis/pages/upload.py b/src/deeranalysis/pages/upload.py index b370dde..4e6a82d 100644 --- a/src/deeranalysis/pages/upload.py +++ b/src/deeranalysis/pages/upload.py @@ -10,6 +10,8 @@ from deeranalysis.utils.csv_loader import parse_csv_raw, build_csv_store from deeranalysis.utils.file_parser import load_bes3t, enrich_bes3t, load_hdf5, enrich_hdf5, dataarray_to_store from deeranalysis.components.batch_import import batch_import_layout +from deeranalysis.components.logs_import_tab import logs_import_tab_layout +from deeranalysis.utils.logs_plugin import check_logs_api_key import deeranalysis.components.dataset_form as df # registers shared MATCH callbacks dash.register_page(__name__) @@ -101,70 +103,77 @@ def _single_import_layout(): ]) -layout = html.Div([ - metadata_long_values_model(page_id), - dcc.Store(id={"type": "metadata-modal-store", "page": page_id}, data=""), - dcc.Store(id='csv-raw-store'), - - # ---- CSV import modal -------------------------------------------------- - dmc.Modal( - id='csv-import-modal', - title=dmc.Title("Import CSV File", order=3), - size='xl', - opened=False, - children=dmc.Stack([ - html.Div(id='csv-preview'), - dmc.Group([ - dmc.NumberInput( - id='csv-skiprows', label='Skip rows', value=0, min=0, step=1, w=140, - ), - dmc.Select( - id='csv-separator', label='Separator', value=',', w=180, - data=[ - {'label': 'Comma ( , )', 'value': ','}, - {'label': 'Semicolon ( ; )', 'value': ';'}, - {'label': 'Tab', 'value': '\t'}, - {'label': 'Space', 'value': ' '}, - ], - ), - dmc.Select( - id='csv-time-unit', label='Time unit', value='us', w=140, - data=[ - {'label': 'ns', 'value': 'ns'}, - {'label': 'µs', 'value': 'us'}, - {'label': 'ms', 'value': 'ms'}, - {'label': 's', 'value': 's'}, - ], - ), - dmc.Switch( - id='csv-has-header', label='Has header row', checked=True, mt='xl', - ), +def layout(): + return html.Div([ + metadata_long_values_model(page_id), + dcc.Store(id={"type": "metadata-modal-store", "page": page_id}, data=""), + dcc.Store(id='csv-raw-store'), + + # ---- CSV import modal -------------------------------------------------- + dmc.Modal( + id='csv-import-modal', + title=dmc.Title("Import CSV File", order=3), + size='xl', + opened=False, + children=dmc.Stack([ + html.Div(id='csv-preview'), + dmc.Group([ + dmc.NumberInput( + id='csv-skiprows', label='Skip rows', value=0, min=0, step=1, w=140, + ), + dmc.Select( + id='csv-separator', label='Separator', value=',', w=180, + data=[ + {'label': 'Comma ( , )', 'value': ','}, + {'label': 'Semicolon ( ; )', 'value': ';'}, + {'label': 'Tab', 'value': '\t'}, + {'label': 'Space', 'value': ' '}, + ], + ), + dmc.Select( + id='csv-time-unit', label='Time unit', value='us', w=140, + data=[ + {'label': 'ns', 'value': 'ns'}, + {'label': 'µs', 'value': 'us'}, + {'label': 'ms', 'value': 'ms'}, + {'label': 's', 'value': 's'}, + ], + ), + dmc.Switch( + id='csv-has-header', label='Has header row', checked=True, mt='xl', + ), + ]), + dmc.Group([ + dmc.Select(id='csv-t-col', label='Time column (t)', placeholder='Select column…', style={'flex': 1}), + dmc.Select(id='csv-vre-col', label='Real signal (V_re)', placeholder='Select column…', style={'flex': 1}), + dmc.Select(id='csv-vim-col', label='Imaginary signal (V_im)', placeholder='None', style={'flex': 1}, clearable=True), + ], grow=True), + dmc.Group([ + dmc.Button('Cancel', id='csv-cancel-btn', color='gray', variant='subtle'), + dmc.Button('Import', id='csv-import-btn', color='blue', + leftSection=DashIconify(icon='mdi:file-import-outline', width=16)), + ], justify='flex-end'), + ], gap='md'), + ), + + dmc.Title("Import Dataset from File", order=1, mb="md"), + dmc.Divider(mb="lg"), + + dmc.Tabs([ + dmc.TabsList([ + dmc.TabsTab("Single Import", value="single", leftSection=DashIconify(icon="mdi:file-upload-outline", width=16)), + dmc.TabsTab("Batch Import", value="batch", leftSection=DashIconify(icon="mdi:file-multiple-outline", width=16)), + *([dmc.TabsTab( + "LOGS Import", value="logs", + leftSection=DashIconify(icon="mdi:file-document-outline", width=16), + rightSection=dmc.Badge("Beta", color="orange", variant="light", size="xs"), + )] if check_logs_api_key() else []), ]), - dmc.Group([ - dmc.Select(id='csv-t-col', label='Time column (t)', placeholder='Select column…', style={'flex': 1}), - dmc.Select(id='csv-vre-col', label='Real signal (V_re)', placeholder='Select column…', style={'flex': 1}), - dmc.Select(id='csv-vim-col', label='Imaginary signal (V_im)', placeholder='None', style={'flex': 1}, clearable=True), - ], grow=True), - dmc.Group([ - dmc.Button('Cancel', id='csv-cancel-btn', color='gray', variant='subtle'), - dmc.Button('Import', id='csv-import-btn', color='blue', - leftSection=DashIconify(icon='mdi:file-import-outline', width=16)), - ], justify='flex-end'), - ], gap='md'), - ), - - dmc.Title("Import Dataset from File", order=1, mb="md"), - dmc.Divider(mb="lg"), - - dmc.Tabs([ - dmc.TabsList([ - dmc.TabsTab("Single Import", value="single", leftSection=DashIconify(icon="mdi:file-upload-outline", width=16)), - dmc.TabsTab("Batch Import", value="batch", leftSection=DashIconify(icon="mdi:file-multiple-outline", width=16)), - ]), - dmc.TabsPanel(value="single", pt="lg", children=_single_import_layout()), - dmc.TabsPanel(value="batch", pt="lg", children=batch_import_layout()), - ], value="single"), -]) + dmc.TabsPanel(value="single", pt="lg", children=_single_import_layout()), + dmc.TabsPanel(value="batch", pt="lg", children=batch_import_layout()), + *([dmc.TabsPanel(value="logs", pt="lg", children=logs_import_tab_layout())] if check_logs_api_key() else []), + ], value="single"), + ]) def _error_message(message): From 382ef2681c606688cf3a95d20237a2a605fd82d4 Mon Sep 17 00:00:00 2001 From: Hugo Karas Date: Wed, 30 Sep 2026 09:22:15 +0200 Subject: [PATCH 3/4] Removed some extra docstrings --- src/deeranalysis/components/batch_import.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/deeranalysis/components/batch_import.py b/src/deeranalysis/components/batch_import.py index a2607df..d6708bf 100644 --- a/src/deeranalysis/components/batch_import.py +++ b/src/deeranalysis/components/batch_import.py @@ -1,17 +1,3 @@ -"""Batch File Import — upload many datasets at once, review them as cards -with a sparkline preview, set Project/Sample/Experiment globally or per -card, then expand any card to edit it with the same form used on the -Single Import tab. - -Each card gets its own hidden per-dataset form (delays grid, tmin, data -viewer, save button) tagged with a unique ``page`` id (``batch-``). -Because every callback in ``dataset_form.py`` and ``data_viewer.py`` is -registered with Dash's ``MATCH`` wildcard on ``page``, those callbacks bind -automatically to these dynamically-created components — no new callback -logic is needed for delays syncing, phase correction, masking, autocomplete -population, or saving to the database. Saving happens only via "Save All", -which fires the shared per-item save callback for every dataset at once. -""" import json import dash From 137dfad5c92cf19e5dd9d8768c2c0d9e24ec3592 Mon Sep 17 00:00:00 2001 From: Hugo Karas Date: Fri, 2 Oct 2026 12:37:23 +0200 Subject: [PATCH 4/4] Better SparkLine --- src/deeranalysis/components/batch_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deeranalysis/components/batch_import.py b/src/deeranalysis/components/batch_import.py index d6708bf..e4177b3 100644 --- a/src/deeranalysis/components/batch_import.py +++ b/src/deeranalysis/components/batch_import.py @@ -78,7 +78,7 @@ def _card(index, title, dataset_store): ), dmc.Group([ dmc.Box( - dmc.Sparkline(data=V, color="blue", h=60, w="100%", curveType="linear", strokeWidth=1.5), + dmc.Sparkline(data=V, color="blue", h=60, w="100%", curveType="linear", strokeWidth=1.5, fillOpacity=0), style={'width': '200px', 'flexShrink': 0}, ), dmc.Stack([