diff --git a/frontend/eslint.config.js b/frontend/eslint.config.js index 531e375..4516b58 100644 --- a/frontend/eslint.config.js +++ b/frontend/eslint.config.js @@ -15,7 +15,9 @@ const filenamesPlugin = { Program(node) { const filename = context.filename; if (!filename) return; - const stem = filename.split("/").pop().replace(/\.js$/, ""); + // Split on both separators: on Windows, ESLint hands us + // backslash-joined absolute paths, not just "/"-joined ones. + const stem = filename.split(/[/\\]/).pop().replace(/\.js$/, ""); if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(stem)) { context.report({ node, diff --git a/frontend/src/api/binary-payloads.js b/frontend/src/api/binary-payloads.js index 128e8c1..5c650be 100644 --- a/frontend/src/api/binary-payloads.js +++ b/frontend/src/api/binary-payloads.js @@ -1,13 +1,18 @@ /** - * Decoders for the compact binary transport the API uses for large float32 + * Decoders for the compact binary transport the API uses for large float * matrices (pulse trains). Each payload is either plain JSON or a binary frame: * * magic(4 bytes) | uint32 version | uint32 metaLen | (uint32 rows, uint32 cols) - * per array | JSON metadata (metaLen bytes) | float32 array data + * per array | JSON metadata (metaLen bytes) | array data * - * All integers/floats are little-endian. When the magic prefix is absent the - * server fell back to JSON, so the bytes are parsed as text instead. Mirrors the - * Python `pack_json_f32_payload` packer in `api/binary.py`. + * All integers/floats are little-endian. The array element width depends on + * the payload: the edit-load frame (MELD) carries float64 — a display copy of + * one file's worth of pulse trains costs nothing extra to widen, and it means + * a value shown to the user never differs from what is on disk. QC/preview + * frames stay float32, sized for many more samples at once. When the magic + * prefix is absent the server fell back to JSON, so the bytes are parsed as + * text instead. Mirrors the Python `pack_json_f32_payload` packer in + * `api/binary.py`. */ /** @typedef {import("../app/context.js").JsonObject} JsonObject */ @@ -42,7 +47,22 @@ function readFloat32Values(view, offset, count) { } /** - * @param {Float32Array} raw + * @param {DataView} view + * @param {number} offset + * @param {number} count + */ +function readFloat64Values(view, offset, count) { + const out = new Float64Array(count); + let cursor = offset; + for (let i = 0; i < count; i++) { + out[i] = view.getFloat64(cursor, true); + cursor += 8; + } + return out; +} + +/** + * @param {Float32Array | Float64Array} raw * @param {number} rows * @param {number} cols * @returns {number[][]} @@ -131,8 +151,8 @@ export function decodeQcRawF32(buffer) { * @param {ArrayBuffer} buffer * @param {string | null} [formatHeader] */ -function isEditLoadF32Payload(buffer, formatHeader = "") { - return formatHeader === "edit-load-f32-v1" || hasMagic(buffer, "MELD"); +function isEditLoadF64Payload(buffer, formatHeader = "") { + return formatHeader === "edit-load-f64-v1" || hasMagic(buffer, "MELD"); } /** @@ -141,7 +161,7 @@ function isEditLoadF32Payload(buffer, formatHeader = "") { * @returns {JsonObject} */ export function decodeEditLoadPayload(buffer, formatHeader = "") { - if (!isEditLoadF32Payload(buffer, formatHeader)) { + if (!isEditLoadF64Payload(buffer, formatHeader)) { const text = textDecoder.decode(new Uint8Array(buffer)); return JSON.parse(text); } @@ -163,7 +183,7 @@ export function decodeEditLoadPayload(buffer, formatHeader = "") { textDecoder.decode(new Uint8Array(buffer, offset, metaLen)), ); offset += metaLen; - const pulse = to2d(readFloat32Values(view, offset, rows * cols), rows, cols); + const pulse = to2d(readFloat64Values(view, offset, rows * cols), rows, cols); return { ...meta, pulse_trains_full: pulse }; } diff --git a/frontend/src/api/client.js b/frontend/src/api/client.js index 09e57fa..ed1ce76 100644 --- a/frontend/src/api/client.js +++ b/frontend/src/api/client.js @@ -9,7 +9,6 @@ import { decodeQcJsonPayload, decodeQcRawF32, decodeDecomposePreviewPayload, - decodeEditLoadPayload, } from "./binary-payloads.js"; import { normalizePreviewPayload } from "./payloads.js"; @@ -150,19 +149,21 @@ export function createApiClient({ apiFetch, apiJson, API_BASE }) { * @param {string} filepath */ async editLoadByPath(filepath) { - const res = await apiFetch( + // Light load: metadata + session token, rows are fetched one at a time + // (editSessionRow). When no session could be opened the server sends the + // full matrix instead (as JSON), which the editor then holds as before. + return apiJson( `${API_BASE}${routes.editLoadByPath}`, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { + "Content-Type": "application/json", + "X-Muedit-Lazy-Rows": "1", + }, body: JSON.stringify({ path: filepath }), }, 120000, ); - return decodeEditLoadPayload( - await res.arrayBuffer(), - res.headers.get("x-muedit-format"), - ); }, /** @@ -172,6 +173,37 @@ export function createApiClient({ apiFetch, apiJson, API_BASE }) { return postJson(`${API_BASE}${routes.editSave}`, payload, 120000); }, + /** + * @param {JsonObject} payload + */ + editSessionDuplicate(payload) { + return postJson( + `${API_BASE}${routes.editSessionDuplicate}`, + payload, + 30000, + ); + }, + + /** + * One motor unit's row (float64 on the wire) from the server-side session. + * @param {JsonObject} payload + * @returns {Promise} + */ + async editSessionRow(payload) { + const res = await apiFetch( + `${API_BASE}${routes.editSessionRow}`, + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }, + 60000, + ); + // Sent at the session's own precision (float64), not narrowed to + // float32: see the row rationale in the backend route. + return Array.from(new Float64Array(await res.arrayBuffer())); + }, + healthUrl() { return `${API_BASE}${routes.health}`; }, diff --git a/frontend/src/api/routes.js b/frontend/src/api/routes.js index 9b5896a..1843536 100644 --- a/frontend/src/api/routes.js +++ b/frontend/src/api/routes.js @@ -11,6 +11,8 @@ export const routes = { decomposePreview: (/** @type {string} */ token) => `/decompose_preview/${encodeURIComponent(token)}`, editSave: "/edit/save", + editSessionDuplicate: "/edit/session/duplicate", + editSessionRow: "/edit/session/row", editAction: (/** @type {string} */ action) => `/edit/${action}`, editMode: (/** @type {string} */ mode) => `/edit/${mode}`, editRemoveOutliers: "/edit/remove-outliers", diff --git a/frontend/src/app/context.js b/frontend/src/app/context.js index 0ad42c4..b891b62 100644 --- a/frontend/src/app/context.js +++ b/frontend/src/app/context.js @@ -156,7 +156,7 @@ * @property {() => Promise} removeOutliers * @property {() => Promise} flagMuForDeletion * @property {() => void} resetCurrentMuEdits - * @property {() => void} duplicateMu + * @property {() => void | Promise} duplicateMu * @property {() => Promise} removeDuplicateMus * @property {() => Promise} saveEditedFile * @property {(file: FileRef, path: string) => Promise} loadDecompositionForEdit diff --git a/frontend/src/app/services/editing-service.js b/frontend/src/app/services/editing-service.js index 49e7b02..dbc1d55 100644 --- a/frontend/src/app/services/editing-service.js +++ b/frontend/src/app/services/editing-service.js @@ -8,7 +8,7 @@ import { handleError } from "./error-service.js"; import { normalizeEditLoadPayload } from "../../api/payloads.js"; import { inferGridCount, normalizeGridNames } from "../../io/grid.js"; import { getSuggestedNpzName } from "../../io/bids.js"; -import { muUidFor } from "../../state/selectors.js"; +import { muUidFor, pulseSource } from "../../state/selectors.js"; import { spikesDiff } from "../../editing/operations.js"; import { clearAllEditSelections, @@ -29,6 +29,7 @@ import { setEditFlagForMu, setEditFlaggedArray, setEditFsamp, + setEditLazyRows, setEditGridNames, setEditHistory, keepEditMus, @@ -39,6 +40,8 @@ import { setEditParameters, setEditPulseTrainForMu, setEditPulseTrains, + setEditRowRev, + setEditSessionToken, setEditSignalToken, setEditTotalSamples, setEditView, @@ -106,7 +109,7 @@ export async function requestRoiEdit(app, action, payload) { const data = await api.editAction(action, { distimes: state.edit.distimes, mu_index: payload.muIdx, - pulse_train: payload.pulse, + ...pulseSource(state, payload.muIdx, payload.pulse), fsamp: payload.fs, x_start: payload.xStart, x_end: payload.xEnd, @@ -248,7 +251,10 @@ export async function requestFilterUpdate(app, mode) { mu_index: muIdx, distimes: state.edit.distimes, mu_grid_index: state.edit.muGridIndex || [], - pulse_train: pulse, + ...pulseSource(state, muIdx, pulse), + // The row lives on the server: the editor refetches it instead of taking + // a 700k-value JSON array back. + omit_pulse_train: !!state.edit.sessionToken, view_start: start, view_end: end, use_peeloff: els.editPeelOffToggle?.dataset.state === "on", @@ -260,6 +266,9 @@ export async function requestFilterUpdate(app, mode) { if (data.pulse_train && Array.isArray(data.pulse_train)) { setEditPulseTrainForMu(state, muIdx, data.pulse_train); } + if (state.edit.sessionToken && typeof data.row_rev === "number") { + setEditRowRev(state, muUidFor(state, muIdx), data.row_rev); + } ensureEditFlagged(); setEditFlagForMu(state, muIdx, false); const muUid = muUidFor(state, muIdx); @@ -321,7 +330,7 @@ export async function removeOutliers(app) { const data = await api.editRemoveOutliers({ distimes: state.edit.distimes, mu_index: muIdx, - pulse_train: pulse, + ...pulseSource(state, muIdx, pulse), fsamp: state.edit.fsamp || 0, }); setEditDistimesForMu(state, muIdx, data.distimes || []); @@ -484,12 +493,15 @@ export async function saveEditedFile(app) { return; } const muscleNames = getBidsMuscleNames(); - const maxSpike = Math.max( - 0, - ...distimes - .flatMap((d) => d || []) - .map((v) => (Number.isFinite(v) ? v : 0)), - ); + // A plain loop, not Math.max(0, ...spikes): spreading every spike index of a + // large file as call arguments exceeds the engine's argument limit and throws + // before the "Saving..." status is shown (this sits above the try block). + let maxSpike = 0; + for (const d of distimes) { + for (const v of d || []) { + if (Number.isFinite(v) && v > maxSpike) maxSpike = v; + } + } const totalSamples = state.edit.totalSamples || (state.edit.pulseTrains?.[0]?.length ?? 0) || @@ -502,7 +514,14 @@ export async function saveEditedFile(app) { const payload = { distimes, flagged: state.edit.flagged || [], - pulse_trains: state.edit.pulseTrains || [], + // With a server session the file is written from the reference (float64) + // rows held there; the browser only names which revision of each row to use. + ...(state.edit.sessionToken + ? { + session_token: state.edit.sessionToken, + row_revs: state.edit.rowRevs || {}, + } + : { pulse_trains: state.edit.pulseTrains || [] }), total_samples: totalSamples, fsamp: state.edit.fsamp, grid_names: state.edit.gridNames, @@ -597,16 +616,22 @@ export async function loadDecompositionForEdit(app, file, filepath) { if (els.bidsProject) els.bidsProject.value = loadedProject; setEditProject(state, loadedProject); setEditSignalToken(state, data.edit_signal_token || ""); + setEditSessionToken(state, data.session_token || ""); setEditFile(state, file); setEditFilename(state, file.name || data.file_label || "decomposition"); - setEditPulseTrains( - state, - data.pulse_trains_full || data.pulse_trains || [], - ); - setEditOriginalPulseTrains( - state, - (state.edit.pulseTrains || []).map((row) => (row ? [...row] : row)), - ); + if (data.lazy_rows) { + // Pulse trains stay on the server; the editor fetches the row it shows. + setEditLazyRows(state, (data.distime_all || data.distime || []).length); + } else { + setEditPulseTrains( + state, + data.pulse_trains_full || data.pulse_trains || [], + ); + setEditOriginalPulseTrains( + state, + (state.edit.pulseTrains || []).map((row) => (row ? [...row] : row)), + ); + } const dist = data.distime_all || data.distime || []; setEditDistimes( state, diff --git a/frontend/src/app/state.js b/frontend/src/app/state.js index 537bc63..207e19d 100644 --- a/frontend/src/app/state.js +++ b/frontend/src/app/state.js @@ -20,6 +20,7 @@ import { GRID_COLORS } from "../config.js"; * @property {number[] | null} pulseTrain * @property {number[]} artifactTimes * @property {number} historyLength Edit-log length when the backup was taken. + * @property {number} rowRev Server-side revision of the row when the backup was taken. */ /** @@ -72,6 +73,11 @@ import { GRID_COLORS } from "../config.js"; * @property {string} bidsRoot * @property {string} project * @property {string} editSignalToken + * @property {string} sessionToken Server-side edit session holding the reference pulse trains. + * @property {Record} rowRevs Row revision in force per MU id (missing = as read from the file). + * @property {boolean} lazyRows Rows are fetched one at a time from the session (pulseTrains holds only a few). + * @property {Record} rowCacheRevs Revision of each row currently held in pulseTrains, by MU id. + * @property {string[]} rowCacheOrder MU ids of the rows held, oldest first (bounds browser memory). * @property {string[]} muUids * @property {EditHistoryEntry[]} editHistory * @property {Bookmark | null} bookmarkPosition @@ -149,6 +155,11 @@ export function createEditSlice() { bidsRoot: "", project: "", editSignalToken: "", + sessionToken: "", + rowRevs: {}, + lazyRows: false, + rowCacheRevs: {}, + rowCacheOrder: [], muUids: [], editHistory: [], bookmarkPosition: null, diff --git a/frontend/src/decomp/run.js b/frontend/src/decomp/run.js index 4bb6909..25af04c 100644 --- a/frontend/src/decomp/run.js +++ b/frontend/src/decomp/run.js @@ -46,13 +46,19 @@ export async function autoSaveRunDecomposition(app) { if (state.lastRunDownloadKey === key) return; const muscleNames = getBidsMuscleNames(); + // A plain loop, not Math.max(0, ...spikes): see saveEditedFile in + // app/services/editing-service.js. + let maxSpike = 0; + for (const d of state.muDistimes) { + for (const v of d || []) { + const n = Number(v) || 0; + if (n > maxSpike) maxSpike = n; + } + } const totalSamples = state.seriesLength || (state.muPulseTrains?.[0]?.length ?? 0) || - Math.max( - 0, - ...state.muDistimes.flatMap((d) => (d || []).map((v) => Number(v) || 0)), - ) + 1; + maxSpike + 1; const fs = state.fsamp; const payload = { distimes: state.muDistimes || [], diff --git a/frontend/src/editing/operations.js b/frontend/src/editing/operations.js index 2f3c7a4..0ce7c8b 100644 --- a/frontend/src/editing/operations.js +++ b/frontend/src/editing/operations.js @@ -11,10 +11,13 @@ import { setEditFlagForMu, setEditFlaggedArray, setEditPulseTrainForMu, + setEditCachedRow, + setEditRowRev, setEditTotalSamples, dropEditHistoryForMuSince, } from "../state/actions.js"; -import { muUidFor } from "../state/selectors.js"; +import { muUidFor, rowIsCurrent, rowRevOf } from "../state/selectors.js"; +import { handleError } from "../app/services/error-service.js"; /** @typedef {import("../app/state.js").State} State */ /** @typedef {import("../app/state.js").Selection} Selection */ @@ -69,14 +72,54 @@ export function backupEditMu(state) { muIdx, distimes: [...(state.edit.distimes?.[muIdx] || [])], flagged: !!state.edit.flagged?.[muIdx], - pulseTrain: state.edit.pulseTrains?.[muIdx] - ? [...state.edit.pulseTrains[muIdx]] - : null, + // With a session the previous row is still on the server (see rowRev): no copy. + pulseTrain: + !state.edit.sessionToken && state.edit.pulseTrains?.[muIdx] + ? [...state.edit.pulseTrains[muIdx]] + : null, artifactTimes: [...(state.edit.artifactTimes?.[muIdx] || [])], historyLength: state.edit.editHistory?.length ?? 0, + rowRev: rowRevOf(state, muIdx), }); } +// Rows being fetched, so that two renders asking for the same row share one request. +/** @type {Map>} */ +const rowLoads = new Map(); + +/** + * Lazy mode: make sure the editor holds the current revision of a MU's row, + * fetching it from the server session if not. Resolves once it is held. + * @param {App} app + * @param {number} muIdx + * @returns {Promise} + */ +export function ensureEditRow(app, muIdx) { + const { state, api } = app; + if (rowIsCurrent(state, muIdx)) return Promise.resolve(); + const token = state.edit.sessionToken; + const uid = muUidFor(state, muIdx); + const rev = rowRevOf(state, muIdx); + const key = `${token}|${uid}|${rev}`; + let pending = rowLoads.get(key); + if (!pending) { + pending = api + .editSessionRow({ session_token: token, mu_uid: uid, row_rev: rev }) + .then((row) => { + // The user may have moved on (reopened a file, undone the edit) meanwhile. + const idx = (state.edit.muUids || []).indexOf(uid); + if (state.edit.sessionToken === token && idx !== -1) { + if (rowRevOf(state, idx) === rev) { + setEditCachedRow(state, idx, row, rev); + } + } + }) + .finally(() => rowLoads.delete(key)); + rowLoads.set(key, pending); + } + return pending; +} + /** @param {App} app */ export function restoreEditBackup(app) { const { @@ -102,6 +145,10 @@ export function restoreEditBackup(app) { setEditPulseTrainForMu(state, muIdx, backup.pulseTrain); } const muUid = muUidFor(state, muIdx); + // Undo is a revision number too: the server still holds the previous row. + if (state.edit.sessionToken && typeof backup.rowRev === "number") { + setEditRowRev(state, muUid, backup.rowRev); + } dropEditHistoryForMuSince(state, muUid, backup.historyLength); setEditBackup(state, null); clearAllEditSelections(state); @@ -416,9 +463,10 @@ export function computeInstantaneousDr(spikes, fsamp, totalSamples) { } /** @param {App} app */ -export function duplicateMu(app) { +export async function duplicateMu(app) { const { state, + api, setEditStatus, ensureEditFlagged, recomputeEditDirty, @@ -444,11 +492,33 @@ export function duplicateMu(app) { existingCounts.length > 0 ? Math.max(...existingCounts) + 1 : 0; const newUid = `${prefix}${newCount}`; + const sourceUid = muUidFor(state, muIdx); + // The copy must exist on the server too, or it could not be saved from there. + if (state.edit.sessionToken) { + try { + await api.editSessionDuplicate({ + session_token: state.edit.sessionToken, + source_uid: sourceUid, + source_rev: rowRevOf(state, muIdx), + new_uid: newUid, + }); + } catch (err) { + handleError(err, setEditStatus, "Duplicate failed"); + return; + } + } + const newIdx = state.edit.distimes.length; - appendEditMu(state, { distimes, pulseTrain: pulse, gridIdx, uid: newUid }); + // In lazy mode the copy's row is fetched from the session when it is shown. + const copiedPulse = state.edit.lazyRows ? [] : pulse; + appendEditMu(state, { + distimes, + pulseTrain: copiedPulse, + gridIdx, + uid: newUid, + }); ensureEditFlagged(); - const sourceUid = muUidFor(state, muIdx); app.appendEditHistory({ type: "duplicate_mu", mu_uid: newUid, @@ -478,9 +548,13 @@ export function resetCurrentMuEdits(app) { const wasFlagged = !!state.edit.flagged?.[muIdx]; setEditDistimesForMu(state, muIdx, baseline); setEditArtifactTimesForMu(state, muIdx, []); - if (state.edit.originalPulseTrains?.[muIdx]) { + // An empty entry is a lazy-mode placeholder, not a real row to restore: leave + // pulseTrains alone and let the rowRev reset below pull the real one back. + if (state.edit.originalPulseTrains?.[muIdx]?.length) { setEditPulseTrainForMu(state, muIdx, state.edit.originalPulseTrains[muIdx]); } + // Revision 0 is the row as read from the file, always kept by the server. + if (state.edit.sessionToken) setEditRowRev(state, muUidFor(state, muIdx), 0); ensureEditFlagged(); setEditFlagForMu(state, muIdx, false); /** @type {EditHistoryEntry} */ diff --git a/frontend/src/state/actions.js b/frontend/src/state/actions.js index 27aa890..d99ed4b 100644 --- a/frontend/src/state/actions.js +++ b/frontend/src/state/actions.js @@ -104,6 +104,25 @@ export function setEditProject(state, value) { state.edit.project = String(value || "").trim(); } +/** + * @param {State} state + * @param {string | null | undefined} value + */ +export function setEditSessionToken(state, value) { + state.edit.sessionToken = String(value || "").trim(); + state.edit.rowRevs = {}; +} + +/** + * Record the server-side revision now in force for one motor unit. + * @param {State} state + * @param {string} muUid + * @param {number} rev + */ +export function setEditRowRev(state, muUid, rev) { + state.edit.rowRevs[muUid] = rev; +} + /** * @param {State} state * @param {string | null | undefined} value @@ -401,9 +420,53 @@ export function setEditFilename(state, filename) { * @param {number[][] | null | undefined} pulseTrains */ export function setEditPulseTrains(state, pulseTrains) { + state.edit.lazyRows = false; state.edit.pulseTrains = Array.isArray(pulseTrains) ? pulseTrains : []; } +// How many pulse-train rows the editor keeps at once in lazy mode: the one on +// screen plus the ones just visited, so going back to a MU is instant. +const EDIT_ROW_CACHE_SIZE = 4; + +/** + * Lazy mode: the pulse trains stay on the server. `pulseTrains` becomes one + * empty placeholder per motor unit, filled in by `setEditCachedRow`. + * @param {State} state + * @param {number} muCount + */ +export function setEditLazyRows(state, muCount) { + state.edit.lazyRows = true; + state.edit.pulseTrains = Array.from({ length: muCount }, () => []); + // One empty placeholder per MU, not an empty array: appendEditMu (duplicate) + // pushes onto the end and must land on the new MU's own index, not MU 0's. + state.edit.originalPulseTrains = Array.from({ length: muCount }, () => []); + state.edit.rowCacheRevs = {}; + state.edit.rowCacheOrder = []; +} + +/** + * Hold a row fetched from the server, dropping the oldest held row(s) beyond the cache size. + * @param {State} state + * @param {number} muIdx + * @param {number[]} row + * @param {number} rev Revision of the row on the server. + */ +export function setEditCachedRow(state, muIdx, row, rev) { + const e = state.edit; + const uid = e.muUids?.[muIdx] ?? `mu${muIdx}`; + e.pulseTrains[muIdx] = row; + e.rowCacheRevs[uid] = rev; + e.rowCacheOrder = e.rowCacheOrder.filter((u) => u !== uid); + e.rowCacheOrder.push(uid); + while (e.rowCacheOrder.length > EDIT_ROW_CACHE_SIZE) { + const stale = e.rowCacheOrder.shift(); + if (stale === undefined) break; + delete e.rowCacheRevs[stale]; + const idx = (e.muUids || []).indexOf(stale); + if (idx !== -1) e.pulseTrains[idx] = []; + } +} + /** * @param {State} state * @param {number[][] | null | undefined} pulseTrains diff --git a/frontend/src/state/selectors.js b/frontend/src/state/selectors.js index 8425ba1..e321290 100644 --- a/frontend/src/state/selectors.js +++ b/frontend/src/state/selectors.js @@ -78,3 +78,43 @@ export function getEditMuIndicesForGrid(state, gridIdx) { gridIdx, ); } + +/** + * Server-side revision in force for a motor unit (0 = as read from the file). + * @param {State} state + * @param {number} muIdx + * @returns {number} + */ +export function rowRevOf(state, muIdx) { + return state.edit.rowRevs?.[muUidFor(state, muIdx)] ?? 0; +} + +/** + * How a request names the pulse train it acts on: the server-side session row + * (id + revision) when there is a session, otherwise the uploaded values. + * @param {State} state + * @param {number} muIdx + * @param {number[]} pulse Values held by the editor, used only without a session. + * @returns {Record} + */ +export function pulseSource(state, muIdx, pulse) { + if (!state.edit.sessionToken) return { pulse_train: pulse }; + return { + session_token: state.edit.sessionToken, + mu_uid: muUidFor(state, muIdx), + row_rev: rowRevOf(state, muIdx), + }; +} + +/** + * Whether the row held by the editor for this MU is the one the server has in + * force (always true outside lazy mode, where every row is held). + * @param {State} state + * @param {number} muIdx + * @returns {boolean} + */ +export function rowIsCurrent(state, muIdx) { + if (!state.edit.lazyRows) return true; + const held = state.edit.rowCacheRevs[muUidFor(state, muIdx)]; + return held !== undefined && held === rowRevOf(state, muIdx); +} diff --git a/frontend/src/view/edit-canvas.js b/frontend/src/view/edit-canvas.js index 3113d7f..9772c10 100644 --- a/frontend/src/view/edit-canvas.js +++ b/frontend/src/view/edit-canvas.js @@ -16,7 +16,12 @@ import { setEditView, setShowBookmark, } from "../state/actions.js"; -import { computeInstantaneousDr } from "../editing/operations.js"; +import { + computeInstantaneousDr, + ensureEditRow, +} from "../editing/operations.js"; +import { handleError } from "../app/services/error-service.js"; +import { rowIsCurrent } from "../state/selectors.js"; import { renderSelectPair } from "./select-renderers.js"; /** @typedef {import("../app/context.js").App} App */ @@ -169,6 +174,30 @@ export function renderEditExplorer(app) { renderEditDropdowns(); const muIdx = state.edit.currentMu ?? 0; + if (!rowIsCurrent(state, muIdx)) { + // Lazy mode: the row is on the server. Ask for it and draw again on arrival; + // meanwhile a previous version of the row (if any) stays on screen. + ensureEditRow(app, muIdx) + .then(() => app.renderEditExplorer()) + .catch((err) => + handleError(err, app.setEditStatus, "Loading the pulse train failed"), + ); + if (!state.edit.pulseTrains?.[muIdx]?.length) { + drawSeries( + els.editPulseCanvas, + [], + UNIFORM_PULSE_COLOR, + [], + [], + 0, + null, + null, + true, + { noDataText: "Loading pulse train..." }, + ); + return; + } + } const pulse = getDisplayPulse(muIdx); const spikes = state.edit.distimes?.[muIdx] || []; if (!state.edit.view || (pulse && state.edit.view.end > pulse.length)) { diff --git a/frontend/src/view/plots.js b/frontend/src/view/plots.js index 107bacc..e517c2a 100644 --- a/frontend/src/view/plots.js +++ b/frontend/src/view/plots.js @@ -281,12 +281,35 @@ export function drawSeries( ctx.strokeStyle = color; ctx.lineWidth = 1.4; ctx.beginPath(); - sliced.forEach((v, idx) => { - const x = toCanvasX(idx); - const y = toCanvasY(v); - if (idx === 0) ctx.moveTo(x, y); - else ctx.lineTo(x, y); - }); + const columns = Math.max(1, Math.floor(plotWidth)); + if (sliced.length > columns * 4) { + // Many more samples than pixels: draw, for each pixel column, the line from + // its minimum to its maximum. Every peak stays visible (unlike subsampling) + // and the cost follows the width of the plot, not the length of the recording. + const n = sliced.length; + for (let c = 0; c < columns; c++) { + const from = Math.floor((c * n) / columns); + const to = Math.max(from + 1, Math.floor(((c + 1) * n) / columns)); + let lo = Infinity; + let hi = -Infinity; + for (let i = from; i < to; i++) { + const v = sliced[i]; + if (v < lo) lo = v; + if (v > hi) hi = v; + } + const x = padding.left + ((c + 0.5) * plotWidth) / columns; + ctx.moveTo(x, toCanvasY(lo)); + // A flat column still needs some width to show up (e.g. a flagged MU). + ctx.lineTo(lo === hi ? x + 1 : x, toCanvasY(hi)); + } + } else { + sliced.forEach((v, idx) => { + const x = toCanvasX(idx); + const y = toCanvasY(v); + if (idx === 0) ctx.moveTo(x, y); + else ctx.lineTo(x, y); + }); + } ctx.stroke(); } diff --git a/frontend/tests/binary-payloads.test.js b/frontend/tests/binary-payloads.test.js index e395402..fac44e7 100644 --- a/frontend/tests/binary-payloads.test.js +++ b/frontend/tests/binary-payloads.test.js @@ -1,5 +1,6 @@ // Binary payload decoders, fed frames laid out like the Python packers -// (`pack_json_f32_payload` in api/binary.py, `_encode_qc_raw_f32` in +// (`pack_json_f32_payload` in api/binary.py — float32 for MDPV/decompose-preview, +// float64 for MELD/edit-load — and `_encode_qc_raw_f32` in // api/services/preview_service.py). import { test, describe } from "node:test"; import assert from "node:assert/strict"; @@ -45,23 +46,45 @@ function f32s(values) { return b; } -/** Mirror of `pack_json_f32_payload`: magic | v | metaLen | shapes | meta | data. */ -function packJsonF32(magic, meta, matrices, { version = 1 } = {}) { +function f64s(values) { + const b = new Uint8Array(values.length * 8); + const view = new DataView(b.buffer); + values.forEach((v, i) => view.setFloat64(i * 8, v, true)); + return b; +} + +/** + * Mirror of `pack_json_f32_payload`: magic | v | metaLen | shapes | meta | data. + * `elementBytes` picks the element width: 4 (float32, the default — matches + * MDPV/decompose-preview) or 8 (float64 — matches MELD/edit-load). + */ +function packJsonF32( + magic, + meta, + matrices, + { version = 1, elementBytes = 4 } = {}, +) { const metaBytes = encoder.encode(JSON.stringify(meta)); const shapes = matrices.flatMap((m) => [ u32(m.length), u32(m.length ? m[0].length : 0), ]); + const pack = elementBytes === 8 ? f64s : f32s; return concat([ encoder.encode(magic), u32(version), u32(metaBytes.byteLength), ...shapes, metaBytes, - ...matrices.map((m) => f32s(m.flat())), + ...matrices.map((m) => pack(m.flat())), ]); } +/** MELD (edit-load) frames are float64 on the wire; a small alias keeps call sites readable. */ +function packMeld(meta, matrices, options = {}) { + return packJsonF32("MELD", meta, matrices, { ...options, elementBytes: 8 }); +} + /** Mirror of the MQCR packer: fixed header, then (index, n, samples) per channel. */ function packQcRaw(header, channels, { magic = "MQCR", version = 1 } = {}) { return concat([ @@ -94,25 +117,25 @@ describe("decodeEditLoadPayload", () => { test("decodes a MELD frame into metadata plus the pulse matrix", () => { const meta = { fsamp: 2048, distimes: [[1, 2], [3]] }; - const out = decodeEditLoadPayload(packJsonF32("MELD", meta, [pulse])); + const out = decodeEditLoadPayload(packMeld(meta, [pulse])); assert.deepEqual(out, { ...meta, pulse_trains_full: pulse }); }); test("the format header alone selects the binary path", () => { - const buf = packJsonF32("MELD", { a: 1 }, [pulse]); - const out = decodeEditLoadPayload(buf, "edit-load-f32-v1"); + const buf = packMeld({ a: 1 }, [pulse]); + const out = decodeEditLoadPayload(buf, "edit-load-f64-v1"); assert.deepEqual(out.pulse_trains_full, pulse); }); test("odd-length metadata leaves the float data unaligned", () => { const meta = { name: "xy" }; - assert.equal(encoder.encode(JSON.stringify(meta)).byteLength % 4, 1); - const out = decodeEditLoadPayload(packJsonF32("MELD", meta, [pulse])); + assert.notEqual(encoder.encode(JSON.stringify(meta)).byteLength % 8, 0); + const out = decodeEditLoadPayload(packMeld(meta, [pulse])); assert.deepEqual(out.pulse_trains_full, pulse); }); test("an empty matrix decodes to no rows", () => { - const out = decodeEditLoadPayload(packJsonF32("MELD", {}, [[]])); + const out = decodeEditLoadPayload(packMeld({}, [[]])); assert.deepEqual(out.pulse_trains_full, []); }); @@ -122,14 +145,14 @@ describe("decodeEditLoadPayload", () => { }); test("rejects an unsupported version", () => { - const buf = packJsonF32("MELD", {}, [pulse], { version: 2 }); + const buf = packMeld({}, [pulse], { version: 2 }); assert.throws(() => decodeEditLoadPayload(buf), /version: 2/); }); test("rejects a binary header on a frame without the magic", () => { - const buf = packJsonF32("XXXX", {}, [pulse]); + const buf = packJsonF32("XXXX", {}, [pulse], { elementBytes: 8 }); assert.throws( - () => decodeEditLoadPayload(buf, "edit-load-f32-v1"), + () => decodeEditLoadPayload(buf, "edit-load-f64-v1"), /Invalid edit-load/, ); }); diff --git a/frontend/tests/editing-operations.test.js b/frontend/tests/editing-operations.test.js index 8d1bc97..0ea5563 100644 --- a/frontend/tests/editing-operations.test.js +++ b/frontend/tests/editing-operations.test.js @@ -500,6 +500,31 @@ describe("resetCurrentMuEdits", () => { ); }); + test("resetting the source MU after duplicating it does not empty its row (lazy/session mode)", async () => { + // In lazy mode `originalPulseTrains` holds a placeholder, not real data; + // appendEditMu (duplicate) must not push that placeholder onto MU 0's + // slot, and reset must never mistake an empty placeholder for real data + // to restore (regression: both did, wiping MU 0's row to `[]`). + state.edit.sessionToken = "tok1"; + state.edit.lazyRows = true; + state.edit.rowRevs = { g0_mu0: 0, g0_mu1: 0 }; + state.edit.rowCacheRevs = { g0_mu0: 0, g0_mu1: 0 }; + state.edit.rowCacheOrder = ["g0_mu0", "g0_mu1"]; + state.edit.originalPulseTrains = [[], []]; + app.api = { editSessionDuplicate: async () => ({ duplicated: "g0_mu2" }) }; + + await app.duplicateMu(); + // The new MU's placeholder landed at its own index, not MU 0's. + assert.equal(state.edit.originalPulseTrains.length, 3); + assert.deepEqual(state.edit.originalPulseTrains[0], []); + + state.edit.currentMu = 0; + const before = [...state.edit.pulseTrains[0]]; + app.resetCurrentMuEdits(); + assert.deepEqual(state.edit.pulseTrains[0], before); + assert.notDeepEqual(state.edit.pulseTrains[0], []); + }); + test("an MU with no baseline is left alone", () => { state.edit.originalDistimes = []; state.edit.distimes[0] = [7]; diff --git a/python/src/muedit/api/binary.py b/python/src/muedit/api/binary.py index ed67133..a6926be 100644 --- a/python/src/muedit/api/binary.py +++ b/python/src/muedit/api/binary.py @@ -11,13 +11,15 @@ from muedit.api.common import make_json_safe -def pack_json_f32_payload(magic: bytes, meta: dict[str, Any], *arrays: np.ndarray) -> bytes: - """Pack versioned binary payload: magic(4) + v1 + JSON-meta + float32 arrays.""" +def pack_json_f32_payload( + magic: bytes, meta: dict[str, Any], *arrays: np.ndarray, dtype: str = " bytes: + """Pack versioned binary payload: magic(4) + v1 + JSON-meta + arrays in ``dtype``.""" meta_bytes = json.dumps(make_json_safe(meta), separators=(",", ":")).encode("utf-8") parts: list[bytes] = [magic, struct.pack(" int: return self.context.nbytes +@dataclass +class _EditSessionEntry(_Entry): + session: EditSession + + @property + def nbytes(self) -> int: + return self.session.nbytes + + _E = TypeVar("_E", bound=_Entry) _UPLOAD_SESSION_CACHE: dict[str, _UploadEntry] = {} _DECOMP_PREVIEW_BINARY_CACHE: dict[str, _PreviewBinaryEntry] = {} _EDIT_SIGNAL_CONTEXT_CACHE: dict[str, _EditContextEntry] = {} _EDIT_SIGNAL_LABEL_INDEX: dict[str, str] = {} +_EDIT_SESSION_CACHE: dict[str, _EditSessionEntry] = {} def _purge_expired_caches_locked() -> None: @@ -104,6 +118,9 @@ def _purge_expired_caches_locked() -> None: expired_edit = { token for token, entry in _EDIT_SIGNAL_CONTEXT_CACHE.items() if entry.expires_at <= now } + for token, session in list(_EDIT_SESSION_CACHE.items()): + if session.expires_at <= now: + _EDIT_SESSION_CACHE.pop(token, None) for token in expired_edit: _EDIT_SIGNAL_CONTEXT_CACHE.pop(token, None) if expired_edit: @@ -321,3 +338,27 @@ def _get_edit_signal_context_by_label(file_label: str | None) -> EditSignalConte if _EDIT_SIGNAL_LABEL_INDEX.get(label) == token: _EDIT_SIGNAL_LABEL_INDEX.pop(label, None) return result + + +def _store_edit_session(session: EditSession) -> str: + """Keep ``session`` (the reference copy of an open decomposition) and return its token.""" + token = uuid.uuid4().hex + entry = _EditSessionEntry(expires_at=time.time() + EDIT_SESSION_TTL_SEC, session=session) + with _CACHE_LOCK: + _purge_expired_caches_locked() + _EDIT_SESSION_CACHE[token] = entry + _evict_to_budget_locked(_EDIT_SESSION_CACHE, EDIT_SESSION_MAX_ITEMS, protect=token) + return token + + +def _get_edit_session(token: str | None) -> EditSession | None: + """Resolve an edit-session token (not a copy: the session is shared), refreshing its TTL.""" + if not token: + return None + with _CACHE_LOCK: + _purge_expired_caches_locked() + entry = _EDIT_SESSION_CACHE.get(token) + if entry is None: + return None + entry.expires_at = time.time() + EDIT_SESSION_TTL_SEC + return entry.session diff --git a/python/src/muedit/api/edit_session.py b/python/src/muedit/api/edit_session.py new file mode 100644 index 0000000..f7d410d --- /dev/null +++ b/python/src/muedit/api/edit_session.py @@ -0,0 +1,122 @@ +"""Server-side edit session: the reference copy of a loaded decomposition. + +The browser only ever holds a display copy of the pulse trains. Everything that +is written to disk comes from here, so untouched motor units come out exactly as +they were read and recomputed ones keep the precision they were computed in. + +Motor units are addressed by their stable id (``g_mu``), never by +position: positions change when duplicates or flagged units are removed. +""" + +from __future__ import annotations + +import threading +from dataclasses import dataclass, field + +import numpy as np + +from muedit.models import FloatArray + +# Newest recomputed versions kept per motor unit, besides revision 0. Undo only +# ever needs the previous version, reset only needs revision 0. +KEEP_VERSIONS = 3 + + +class UnknownMuError(KeyError): + """The session has no motor unit with this id.""" + + +class RowUnavailableError(LookupError): + """The requested revision of a motor unit's row has been discarded.""" + + +@dataclass +class EditSession: + """Reference pulse trains of one open decomposition, addressed by motor-unit id.""" + + source_path: str + n_samples: int + # uid -> {revision: row}. Revision 0 is the row as read from the file (a + # read-only view of the loaded matrix, no copy); each recomputation adds one. + _versions: dict[str, dict[int, FloatArray]] + _latest: dict[str, int] + _nbytes_base: int + _extra_bytes: int = 0 # recomputed rows only; rows shared by a duplicate are not recounted + _lock: threading.Lock = field(default_factory=threading.Lock, repr=False, compare=False) + + @classmethod + def from_matrix(cls, source_path: str, matrix: FloatArray, mu_uids: list[str]) -> EditSession: + """Wrap ``matrix`` (one row per motor unit, in ``mu_uids`` order) without copying it.""" + if matrix.ndim != 2 or matrix.shape[0] != len(mu_uids): + raise ValueError( + f"matrix has shape {matrix.shape} but {len(mu_uids)} motor-unit ids were given" + ) + if len(set(mu_uids)) != len(mu_uids): + raise ValueError("motor-unit ids must be unique") + frozen = np.ascontiguousarray(matrix).view() + frozen.flags.writeable = False + return cls( + source_path=source_path, + n_samples=int(frozen.shape[1]), + _versions={uid: {0: frozen[i]} for i, uid in enumerate(mu_uids)}, + _latest=dict.fromkeys(mu_uids, 0), + _nbytes_base=int(frozen.nbytes), + ) + + @property + def mu_uids(self) -> list[str]: + with self._lock: + return list(self._versions) + + @property + def nbytes(self) -> int: + """Resident size: the loaded matrix plus the recomputed rows kept on top of it.""" + with self._lock: + return self._nbytes_base + self._extra_bytes + + def row(self, uid: str, revision: int = 0) -> FloatArray: + """The row of ``uid`` at ``revision`` (read-only).""" + with self._lock: + versions = self._versions.get(uid) + if versions is None: + raise UnknownMuError(uid) + row = versions.get(revision) + if row is None: + raise RowUnavailableError(f"{uid} revision {revision}") + return row + + def add_version(self, uid: str, values: FloatArray) -> int: + """Store a recomputed row for ``uid`` and return its revision number.""" + if values.shape != (self.n_samples,): + raise ValueError(f"row has shape {values.shape}, expected ({self.n_samples},)") + frozen = np.array(values, dtype=np.float64, copy=True) + frozen.flags.writeable = False + with self._lock: + versions = self._versions.get(uid) + if versions is None: + raise UnknownMuError(uid) + revision = self._latest[uid] + 1 + versions[revision] = frozen + self._latest[uid] = revision + self._extra_bytes += int(frozen.nbytes) + recomputed = sorted(rev for rev in versions if rev != 0) + for stale in recomputed[:-KEEP_VERSIONS]: + self._extra_bytes -= int(versions.pop(stale).nbytes) + return revision + + def duplicate(self, source_uid: str, source_revision: int, new_uid: str) -> None: + """Register ``new_uid`` as a copy of a row of ``source_uid`` (shared, not copied).""" + row = self.row(source_uid, source_revision) + with self._lock: + if new_uid in self._versions: + raise ValueError(f"motor-unit id already exists: {new_uid}") + self._versions[new_uid] = {0: row} + self._latest[new_uid] = 0 + + def assemble(self, uids: list[str], revisions: dict[str, int] | None = None) -> FloatArray: + """Rows for ``uids`` (at the given revisions, default 0) as one float64 matrix.""" + revisions = revisions or {} + out = np.empty((len(uids), self.n_samples), dtype=np.float64) + for i, uid in enumerate(uids): + out[i, :] = self.row(uid, int(revisions.get(uid, 0))) + return out diff --git a/python/src/muedit/api/routes/editing.py b/python/src/muedit/api/routes/editing.py index 9d72ee1..6e589d2 100644 --- a/python/src/muedit/api/routes/editing.py +++ b/python/src/muedit/api/routes/editing.py @@ -15,6 +15,8 @@ EditOutliersPayload, EditRoiPayload, EditSavePayload, + EditSessionDuplicatePayload, + EditSessionRowPayload, PathPayload, ) from muedit.api.services.editing_service import ( @@ -22,9 +24,12 @@ add_spikes, delete_dr, delete_spikes, + duplicate_session_row, flag_mu, + get_session_row, load_decomposition_binary_from_path, load_decomposition_from_path, + load_decomposition_lazy_from_path, remove_duplicates_service, remove_outliers, save_edits, @@ -42,6 +47,8 @@ async def load_decomposition_by_path_endpoint( path = payload.path if not path: raise HTTPException(status_code=400, detail="path is required") + if request.headers.get("x-muedit-lazy-rows") == "1": + return success_payload(load_decomposition_lazy_from_path(path)) wants_binary = request.headers.get("x-muedit-binary", "1") != "0" if wants_binary: return load_decomposition_binary_from_path(path) @@ -55,6 +62,18 @@ async def save_edits_endpoint(payload: EditSavePayload) -> dict[str, Any]: return success_payload(result) +@router.post("/edit/session/duplicate") +async def duplicate_session_row_endpoint(payload: EditSessionDuplicatePayload) -> dict[str, Any]: + """Register a duplicated motor unit in the edit session so it can be saved from there.""" + return success_payload(duplicate_session_row(payload)) + + +@router.post("/edit/session/row", response_model=None) +async def session_row_endpoint(payload: EditSessionRowPayload) -> Response: + """One motor unit's row from the edit session, as raw float32 for display.""" + return get_session_row(payload) + + @router.post("/edit/update-filter") async def update_filter_endpoint(payload: EditFilterPayload) -> dict[str, Any]: """Recompute MU filter segment from raw BIDS EMG for one motor unit.""" diff --git a/python/src/muedit/api/schemas.py b/python/src/muedit/api/schemas.py index 9ef9c78..63c8dfc 100644 --- a/python/src/muedit/api/schemas.py +++ b/python/src/muedit/api/schemas.py @@ -15,6 +15,15 @@ class PathPayload(BaseModel): path: str +class EditSessionDuplicatePayload(BaseModel): + """Register a duplicated motor unit as a copy of a row of the edit session.""" + + session_token: str + source_uid: str + source_rev: int = 0 + new_uid: str + + class QcWindowPayload(BaseModel): """Typed request body for QC channel-window retrieval.""" @@ -56,6 +65,11 @@ class EditSavePayload(BaseModel): artifact_times: list[list[int]] | None = None artifact_regions: list[Any] | None = None edit_signal_token: str | None = None + # Save from the server-side session instead of an uploaded ``pulse_trains`` + # matrix: ``row_revs`` says which recomputed version of a motor unit's row is + # in force (missing id = the row as read from the file). + session_token: str | None = None + row_revs: dict[str, int] | None = None participant_meta: dict[str, Any] | None = None powerline_freq: float | None = None manufacturer: str | None = None @@ -78,6 +92,12 @@ class EditFilterPayload(BaseModel): distimes: list[list[int]] mu_grid_index: list[int] | None = None pulse_train: list[float] | None = None + # With a session, the server merges the recomputed window into its own copy of + # the row (``row_rev`` = the version the editor is currently showing). + session_token: str | None = None + mu_uid: str | None = None + row_rev: int | None = None + omit_pulse_train: bool = False # the editor refetches the row instead of taking it inline view_start: int = 0 view_end: int = 0 nbextchan: int = DEFAULT_NBEXTCHAN @@ -94,6 +114,10 @@ class EditRoiPayload(BaseModel): distimes: list[list[int]] mu_index: int = 0 pulse_train: list[float] | None = None + # With a session the pulse train is read server-side (row ``mu_uid`` at ``row_rev``). + session_token: str | None = None + mu_uid: str | None = None + row_rev: int | None = None fsamp: float | None = None x_start: int = 0 x_end: int = 0 @@ -108,6 +132,10 @@ class EditOutliersPayload(BaseModel): distimes: list[list[int]] mu_index: int = 0 pulse_train: list[float] | None = None + # With a session the pulse train is read server-side (row ``mu_uid`` at ``row_rev``). + session_token: str | None = None + mu_uid: str | None = None + row_rev: int | None = None fsamp: float | None = None @@ -128,3 +156,11 @@ class EditFlagPayload(BaseModel): distimes: list[list[int]] mu_index: int = 0 flag: bool | None = None + + +class EditSessionRowPayload(BaseModel): + """Ask the edit session for one motor unit's row (float32, for display).""" + + session_token: str + mu_uid: str + row_rev: int = 0 diff --git a/python/src/muedit/api/services/editing_service.py b/python/src/muedit/api/services/editing_service.py index 3f0c55b..6fde81f 100644 --- a/python/src/muedit/api/services/editing_service.py +++ b/python/src/muedit/api/services/editing_service.py @@ -14,8 +14,10 @@ from muedit.api.binary import pack_json_f32_payload from muedit.api.cache import ( + _get_edit_session, _get_edit_signal_context, _get_edit_signal_context_by_label, + _store_edit_session, _store_edit_signal_context, ) from muedit.api.common import ( @@ -24,6 +26,7 @@ require_existing_path, ) from muedit.api.config import DATA_ROOT, resolve_bids_root +from muedit.api.edit_session import EditSession, RowUnavailableError, UnknownMuError from muedit.api.schemas import ( EditDeduplicatePayload, EditFilterPayload, @@ -31,6 +34,8 @@ EditOutliersPayload, EditRoiPayload, EditSavePayload, + EditSessionDuplicatePayload, + EditSessionRowPayload, ) from muedit.api.services.bids_helpers import ( _infer_bids_root_from_decomp_path, @@ -88,6 +93,7 @@ class EditLoadResult: decomposition: LoadedDecomposition file_label: str edit_signal_token: str | None = None # set when the file embeds the raw EMG + session_token: str | None = None # server-side reference copy of the pulse trains project: str | None = None # BIDS project folder, when the file sits in one # Restored from the ``.json`` edit log saved next to the decomposition. mu_uids: list[Any] | None = None @@ -102,6 +108,7 @@ def to_dict(self) -> dict[str, Any]: out["file_label"] = self.file_label session = { "edit_signal_token": self.edit_signal_token, + "session_token": self.session_token, "project": self.project, "mu_uids": self.mu_uids, "edit_history": self.edit_history, @@ -112,35 +119,43 @@ def to_dict(self) -> dict[str, Any]: return out -def _encode_edit_load_f32(loaded: dict[str, Any]) -> bytes | None: - """Custom MELD v1 binary: JSON metadata header + float32 pulse matrix.""" +def _encode_edit_load_f64(loaded: dict[str, Any]) -> bytes | None: + """Custom MELD v1 binary: JSON metadata header + float64 pulse matrix. + + Only the legacy fallback path (no session could be opened) ever sends the + full matrix this way; the display copy is exactly what is on disk, so this + costs nothing in fidelity. It costs little in size either: at 8 bytes per + sample instead of 4, this only doubles what a modern connection moves in a + handful of milliseconds — see the /edit/session/row rationale below. + """ pulse_raw = loaded.get("pulse_trains_full") if pulse_raw is None: return None - pulse = np.asarray(pulse_raw, dtype=np.float32) + pulse = np.asarray(pulse_raw, dtype=np.float64) if pulse.ndim != 2: return None metadata = dict(loaded) metadata.pop("pulse_trains_full", None) metadata["pulse_shape"] = [int(pulse.shape[0]), int(pulse.shape[1])] - metadata["pulse_dtype"] = "float32" + metadata["pulse_dtype"] = "float64" metadata["pulse_binary"] = True - return pack_json_f32_payload(b"MELD", metadata, pulse) + return pack_json_f32_payload(b"MELD", metadata, pulse, dtype=" Response | dict[str, Any]: - """Encode a loaded decomposition as f32 binary, falling back to JSON when not encodable.""" - blob = _encode_edit_load_f32(loaded) + """Encode a loaded decomposition as f64 binary, falling back to JSON when not encodable.""" + blob = _encode_edit_load_f64(loaded) if blob is None: - return loaded + return make_json_safe(loaded) return Response( content=blob, media_type="application/octet-stream", - headers={"x-muedit-format": "edit-load-f32-v1"}, + headers={"x-muedit-format": "edit-load-f64-v1"}, ) -def load_decomposition_from_path(filepath: str) -> dict[str, Any]: +def _build_edit_load_result(filepath: str) -> EditLoadResult: + """Load a decomposition and its edit-session fields, arrays left as NumPy arrays.""" if require_existing_path(filepath).suffix.lower() not in {".npz", ".mat"}: raise HTTPException( status_code=400, @@ -204,12 +219,99 @@ def load_decomposition_from_path(filepath: str) -> dict[str, Any]: except (OSError, ValueError, KeyError): pass # best-effort; missing or corrupt editlog is non-fatal - return make_json_safe(result.to_dict()) + _open_edit_session(result, filepath) + return result + + +def _open_edit_session(result: EditLoadResult, filepath: str) -> None: + """Keep the loaded matrix server-side and give the editor a token to refer to it. + + Motor units are identified by id, so the ids are settled here (from the edit log + when it matches the file, otherwise generated) and returned to the editor. + """ + decomp = result.decomposition + matrix = decomp.pulse_trains_full + n_mu = len(decomp.distime_all) + if matrix.ndim != 2 or matrix.size == 0 or matrix.shape[0] != n_mu: + return # nothing to keep: saving falls back to the matrix the editor sends + uids = [str(uid) for uid in result.mu_uids or []] + if len(uids) != n_mu or len(set(uids)) != n_mu: + uids = _generate_mu_uids(decomp.mu_grid_index) + result.mu_uids = uids + result.session_token = _store_edit_session(EditSession.from_matrix(filepath, matrix, uids)) + + +def load_decomposition_from_path(filepath: str) -> dict[str, Any]: + """JSON flavour of the edit load: the matrix becomes nested lists.""" + return make_json_safe(_build_edit_load_result(filepath).to_dict()) + + +def load_decomposition_lazy_from_path(filepath: str) -> dict[str, Any]: + """Light flavour of the edit load: metadata and session token, no pulse-train matrix. + + The editor then asks for one motor unit's row at a time (``get_session_row``), so the + size of the file no longer limits what the browser has to hold. When no session could + be opened the full matrix is still sent, since the editor has nothing to ask for. + """ + result = _build_edit_load_result(filepath) + out = result.to_dict() + if result.session_token: + out.pop("pulse_trains_full", None) + out["lazy_rows"] = True + return make_json_safe(out) + + +def get_session_row(payload: EditSessionRowPayload) -> Response: + """One motor unit's row at a given revision as raw little-endian float64. + + Sent at the session's own precision, not narrowed to float32: one row is at + most a few million samples (tens of MB at 8 bytes each), so the display copy + costs the browser nothing extra to hold or draw, and this removes the last + place where a value shown to the user could differ from what is on disk. + """ + session = _require_edit_session(payload.session_token) + row = _session_row(session, payload.mu_uid, payload.row_rev) + return Response( + content=np.asarray(row, dtype=" np.ndarray: + """A row of the session (read-only float64), with HTTP errors for unknown ids/revisions.""" + try: + return session.row(mu_uid, row_rev) + except UnknownMuError as exc: + raise HTTPException( + status_code=400, + detail={"field": "mu_uid", "reason": f"Unknown motor unit id: {exc.args[0]}"}, + ) from exc + except RowUnavailableError as exc: + raise HTTPException( + status_code=409, + detail={"field": "row_rev", "reason": f"Row version unavailable ({exc})."}, + ) from exc + + +def _edit_pulse( + pulse_train: list[float] | None, + session_token: str | None, + mu_uid: str | None, + row_rev: int | None, +) -> np.ndarray: + """The pulse train an edit acts on: the session's row when given, else the uploaded one.""" + if session_token and mu_uid: + session = _require_edit_session(session_token) + return np.asarray(_session_row(session, mu_uid, row_rev or 0), dtype=float) + if pulse_train is None: + raise HTTPException(status_code=400, detail="pulse_train is required") + return np.array(pulse_train, dtype=float) def load_decomposition_binary_from_path(filepath: str) -> Response | dict[str, Any]: - loaded = load_decomposition_from_path(filepath) - return _wrap_edit_load_binary(loaded) + """Binary flavour: the matrix goes straight from the array to the wire format.""" + return _wrap_edit_load_binary(_build_edit_load_result(filepath).to_dict()) def _dedup( @@ -239,6 +341,60 @@ def _clean_distimes(spikes: list[int]) -> list[int]: return sorted({int(v) for v in spikes if int(v) >= 0}) +def _require_edit_session(token: str | None) -> EditSession: + """The live edit session for ``token``, or a 404 telling the user to reopen the file.""" + session = _get_edit_session(token) + if session is None: + raise HTTPException( + status_code=404, + detail={ + "field": "session_token", + "reason": "Edit session expired or unknown; reopen the file to continue.", + }, + ) + return session + + +def _resolve_save_session(payload: EditSavePayload, total_samples: int) -> EditSession | None: + """The session to save from: a token without an uploaded matrix means "use the server's".""" + if payload.pulse_trains is not None or not payload.session_token: + return None + session = _require_edit_session(payload.session_token) + if session.n_samples != total_samples: + raise HTTPException( + status_code=400, + detail={ + "field": "total_samples", + "reason": ( + f"The session holds {session.n_samples} samples per motor unit, " + f"the request says {total_samples}." + ), + }, + ) + return session + + +def _assemble_session_rows( + session: EditSession, mu_uids: list[str], row_revs: dict[str, int] | None +) -> np.ndarray: + """The saved matrix: one float64 row per surviving motor unit, in saved order.""" + try: + return session.assemble(mu_uids, row_revs) + except UnknownMuError as exc: + raise HTTPException( + status_code=400, + detail={"field": "mu_uids", "reason": f"Unknown motor unit id: {exc.args[0]}"}, + ) from exc + except RowUnavailableError as exc: + raise HTTPException( + status_code=409, + detail={ + "field": "row_revs", + "reason": f"A recomputed row is no longer available ({exc}); reopen the file.", + }, + ) from exc + + def _export_bids_from_mat_context( bids_root: Path, entity_label: str, @@ -343,20 +499,26 @@ def save_edits(payload: EditSavePayload) -> dict[str, Any]: if muscle_names and not parameters.get("target_muscle"): parameters["target_muscle"] = muscle_names if len(muscle_names) > 1 else muscle_names[0] - pulse_trains = None - if pulse_trains_raw is not None: - try: - pulse_trains = np.array(pulse_trains_raw, dtype=float) - except (TypeError, ValueError): - pulse_trains = None - if ( - pulse_trains is None - or pulse_trains.size == 0 - or pulse_trains.ndim != 2 - or pulse_trains.shape[0] != len(distimes) - or pulse_trains.shape[1] != total_samples - ): - pulse_trains = build_pulse_trains_from_distimes(distimes, total_samples) + # With an edit session the matrix is not uploaded: the rows are assembled from + # the server's reference copy once the surviving motor units are known. + edit_session = _resolve_save_session(payload, total_samples) + pulse_trains = np.zeros((0, total_samples)) + if edit_session is None: + candidate = None + if pulse_trains_raw is not None: + try: + candidate = np.array(pulse_trains_raw, dtype=float) + except (TypeError, ValueError): + candidate = None + if ( + candidate is None + or candidate.size == 0 + or candidate.ndim != 2 + or candidate.shape[0] != len(distimes) + or candidate.shape[1] != total_samples + ): + candidate = build_pulse_trains_from_distimes(distimes, total_samples) + pulse_trains = candidate mu_uids_raw = payload.mu_uids mu_uids: list[str] = ( @@ -385,7 +547,8 @@ def save_edits(payload: EditSavePayload) -> dict[str, Any]: distimes = [distimes[i] for i in keep_idx] mu_grid_index = [mu_grid_index[i] for i in keep_idx] mu_uids = [mu_uids[i] for i in keep_idx] - pulse_trains = pulse_trains[keep_idx, :] if pulse_trains.size else pulse_trains + if edit_session is None: + pulse_trains = pulse_trains[keep_idx, :] if pulse_trains.size else pulse_trains artifact_times_all = [artifact_times_all[i] for i in keep_idx] kept_mus = [kept_mus[i] for i in keep_idx] if removed_uids: @@ -395,7 +558,8 @@ def save_edits(payload: EditSavePayload) -> dict[str, Any]: kept_idx = _dedup(distimes, mu_grid_index, parameters, fsamp, total_samples) kept_set = set(kept_idx) removed_uids = [uid for i, uid in enumerate(mu_uids) if i not in kept_set] - pulse_trains = pulse_trains[kept_idx, :] if kept_idx else np.zeros((0, total_samples)) + if edit_session is None: + pulse_trains = pulse_trains[kept_idx, :] if kept_idx else np.zeros((0, total_samples)) distimes = [_clean_distimes(distimes[i]) for i in kept_idx] mu_grid_index = [mu_grid_index[i] for i in kept_idx] mu_uids = [mu_uids[i] for i in kept_idx] @@ -404,6 +568,9 @@ def save_edits(payload: EditSavePayload) -> dict[str, Any]: if removed_uids: edit_history.append(_save_removal_entry("remove_duplicates", removed_uids)) + if edit_session is not None: + pulse_trains = _assemble_session_rows(edit_session, mu_uids, payload.row_revs) + bids_root = resolve_bids_root(payload.project) file_label = payload.file_label or "" entity_label = payload.entity_label or parse_entity_label(file_label) @@ -644,35 +811,72 @@ def update_filter(payload: EditFilterPayload) -> dict[str, Any]: pulse_train = payload.pulse_train updated_pulse = None - if pulse_train is not None: + row_rev: int | None = None + edit_session = _require_edit_session(payload.session_token) if payload.session_token else None + pulse_arr: np.ndarray | None = None + if edit_session is not None and payload.mu_uid: + # The server's copy is authoritative: merge into the row version the editor shows. + row_rev = payload.row_rev or 0 + pulse_arr = np.array(_session_row(edit_session, payload.mu_uid, row_rev), dtype=float) + elif pulse_train is not None: try: pulse_arr = np.array(pulse_train, dtype=float) except (TypeError, ValueError): pulse_arr = None - if pulse_arr is not None and pt is not None: - edge = int(round(0.1 * fsamp)) - seg_start = view_start + edge - seg_end = min(view_start + len(pt) - edge, pulse_arr.shape[0]) - if seg_end > seg_start and len(pt) > 2 * edge: - pulse_arr[seg_start:seg_end] = pt[edge : edge + (seg_end - seg_start)] - updated_pulse = pulse_arr - - return make_json_safe( - { - "fsamp": fsamp, - "distimes": updated, - "pulse_train": ( - updated_pulse.tolist() if isinstance(updated_pulse, np.ndarray) else pulse_train - ), - } - ) + if pulse_arr is not None and pt is not None: + changed = _merge_filter_window(pulse_arr, pt, view_start, fsamp) + updated_pulse = pulse_arr + if changed and edit_session is not None and payload.mu_uid: + row_rev = edit_session.add_version(payload.mu_uid, pulse_arr) + + response: dict[str, Any] = {"fsamp": fsamp, "distimes": updated} + if not (payload.omit_pulse_train and edit_session is not None): + response["pulse_train"] = ( + updated_pulse.tolist() if isinstance(updated_pulse, np.ndarray) else pulse_train + ) + if row_rev is not None: + response["row_rev"] = row_rev + return make_json_safe(response) + + +def _merge_filter_window( + pulse_arr: np.ndarray, pt: np.ndarray, view_start: int, fsamp: float +) -> bool: + """Write the recomputed window ``pt`` into ``pulse_arr`` (minus its edge samples).""" + edge = int(round(0.1 * fsamp)) + seg_start = view_start + edge + seg_end = min(view_start + len(pt) - edge, pulse_arr.shape[0]) + if seg_end > seg_start and len(pt) > 2 * edge: + pulse_arr[seg_start:seg_end] = pt[edge : edge + (seg_end - seg_start)] + return True + return False + + +def duplicate_session_row(payload: EditSessionDuplicatePayload) -> dict[str, Any]: + """Register a duplicated motor unit in the edit session (its row is shared, not copied).""" + session = _require_edit_session(payload.session_token) + try: + session.duplicate(payload.source_uid, payload.source_rev, payload.new_uid) + except UnknownMuError as exc: + raise HTTPException( + status_code=400, + detail={"field": "source_uid", "reason": f"Unknown motor unit id: {exc.args[0]}"}, + ) from exc + except RowUnavailableError as exc: + raise HTTPException( + status_code=409, + detail={"field": "source_rev", "reason": f"Row version unavailable ({exc})."}, + ) from exc + except ValueError as exc: + raise HTTPException( + status_code=400, detail={"field": "new_uid", "reason": str(exc)} + ) from exc + return {"duplicated": payload.new_uid} def add_spikes(payload: EditRoiPayload) -> dict[str, Any]: """Add spikes in ROI for selected motor unit.""" - pulse_train = payload.pulse_train - if pulse_train is None: - raise HTTPException(status_code=400, detail="pulse_train is required") + pulse = _edit_pulse(payload.pulse_train, payload.session_token, payload.mu_uid, payload.row_rev) distimes = normalize_distimes(payload.distimes or []) fsamp = payload.fsamp or 0.0 if fsamp <= 0: @@ -684,16 +888,13 @@ def add_spikes(payload: EditRoiPayload) -> dict[str, Any]: if mu_index < 0 or mu_index >= len(distimes): raise HTTPException(status_code=400, detail="mu_index out of range") - pulse = np.array(pulse_train, dtype=float) updated = add_spikes_in_roi(pulse, distimes[mu_index], fsamp, x_start, x_end, y_min) return make_json_safe({"distimes": updated}) def add_artifact(payload: EditRoiPayload) -> dict[str, Any]: """Mark a peak in the ROI as an artifact for the selected motor unit.""" - pulse_train = payload.pulse_train - if pulse_train is None: - raise HTTPException(status_code=400, detail="pulse_train is required") + pulse = _edit_pulse(payload.pulse_train, payload.session_token, payload.mu_uid, payload.row_rev) fsamp = payload.fsamp or 0.0 if fsamp <= 0: raise HTTPException(status_code=400, detail="fsamp is required") @@ -704,16 +905,13 @@ def add_artifact(payload: EditRoiPayload) -> dict[str, Any]: artifact_times_raw = payload.artifact_times or [] artifact_times = list(artifact_times_raw) - pulse = np.array(pulse_train, dtype=float) updated = add_artifact_in_roi(pulse, artifact_times, fsamp, x_start, x_end, y_min) return make_json_safe({"artifact_times": updated}) def delete_spikes(payload: EditRoiPayload) -> dict[str, Any]: """Delete spikes and artifacts in ROI for selected motor unit.""" - pulse_train = payload.pulse_train - if pulse_train is None: - raise HTTPException(status_code=400, detail="pulse_train is required") + pulse = _edit_pulse(payload.pulse_train, payload.session_token, payload.mu_uid, payload.row_rev) distimes = normalize_distimes(payload.distimes or []) x_start = payload.x_start x_end = payload.x_end @@ -723,7 +921,6 @@ def delete_spikes(payload: EditRoiPayload) -> dict[str, Any]: if mu_index < 0 or mu_index >= len(distimes): raise HTTPException(status_code=400, detail="mu_index out of range") - pulse = np.array(pulse_train, dtype=float) updated_distimes = delete_spikes_in_roi(pulse, distimes[mu_index], x_start, x_end, y_min, y_max) # Also delete artifacts in the same ROI @@ -744,9 +941,7 @@ def delete_spikes(payload: EditRoiPayload) -> dict[str, Any]: def delete_dr(payload: EditRoiPayload) -> dict[str, Any]: """Delete spikes with high discharge rates inside ROI for selected MU.""" - pulse_train = payload.pulse_train - if pulse_train is None: - raise HTTPException(status_code=400, detail="pulse_train is required") + pulse = _edit_pulse(payload.pulse_train, payload.session_token, payload.mu_uid, payload.row_rev) distimes = normalize_distimes(payload.distimes or []) fsamp = payload.fsamp or 0.0 if fsamp <= 0: @@ -758,7 +953,6 @@ def delete_dr(payload: EditRoiPayload) -> dict[str, Any]: if mu_index < 0 or mu_index >= len(distimes): raise HTTPException(status_code=400, detail="mu_index out of range") - pulse = np.array(pulse_train, dtype=float) updated = delete_high_discharge_rate_spikes_in_roi( pulse, distimes[mu_index], fsamp, x_start, x_end, y_min ) @@ -767,9 +961,7 @@ def delete_dr(payload: EditRoiPayload) -> dict[str, Any]: def remove_outliers(payload: EditOutliersPayload) -> dict[str, Any]: """Remove discharge-rate outlier spikes and return removal count.""" - pulse_train = payload.pulse_train - if pulse_train is None: - raise HTTPException(status_code=400, detail="pulse_train is required") + pulse = _edit_pulse(payload.pulse_train, payload.session_token, payload.mu_uid, payload.row_rev) distimes = normalize_distimes(payload.distimes or []) fsamp = payload.fsamp or 0.0 if fsamp <= 0: @@ -778,7 +970,6 @@ def remove_outliers(payload: EditOutliersPayload) -> dict[str, Any]: if mu_index < 0 or mu_index >= len(distimes): raise HTTPException(status_code=400, detail="mu_index out of range") - pulse = np.array(pulse_train, dtype=float) source = sorted({int(x) for x in distimes[mu_index]}) updated = remove_discharge_rate_outliers(pulse, source, fsamp) removed = max(0, len(source) - len(updated)) diff --git a/python/src/muedit/decomp/decomposition_file.py b/python/src/muedit/decomp/decomposition_file.py index 9333aeb..28bce9a 100644 --- a/python/src/muedit/decomp/decomposition_file.py +++ b/python/src/muedit/decomp/decomposition_file.py @@ -565,10 +565,12 @@ def load_decomposition_file(filepath: str) -> LoadedDecomposition: if not mu_grid_index or len(mu_grid_index) != len(distimes): mu_grid_index = [0] * len(distimes) - pulse_trains_full = [list(map(float, row)) for row in pulse_matrix.tolist()] - + # Keep the matrix as the NumPy array: turning it into nested Python lists here + # multiplied its size several times over (~9 GB for a 378x727,400 matrix) and + # was the main cause of out-of-memory failures on large files. Callers that + # need JSON convert it with make_json_safe() at the point they need it. return LoadedDecomposition( - pulse_trains_full=pulse_trains_full, + pulse_trains_full=pulse_matrix, distime_all=distimes, fsamp=d.fsamp, grid_names=grid_names, diff --git a/python/src/muedit/models.py b/python/src/muedit/models.py index 61a0f3a..d45ba8b 100644 --- a/python/src/muedit/models.py +++ b/python/src/muedit/models.py @@ -204,7 +204,7 @@ def nbytes(self) -> int: class LoadedDecomposition: """Decomposition state loaded from a .npz or .mat file for the interactive editing stage.""" - pulse_trains_full: list[list[float]] = field(default_factory=list) + pulse_trains_full: np.ndarray = field(default_factory=lambda: np.zeros((0, 0))) distime_all: list[list[int]] = field(default_factory=list) fsamp: float | None = None grid_names: list[str] = field(default_factory=list) @@ -216,7 +216,7 @@ class LoadedDecomposition: sil: list[float] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: - """Serialise to a JSON-safe dictionary.""" + """Serialise to a dict; ``pulse_trains_full`` stays a NumPy array (not JSON-safe).""" return { "pulse_trains_full": self.pulse_trains_full, "distime_all": self.distime_all, diff --git a/tests/test_api_http.py b/tests/test_api_http.py index b2166d1..a40e2da 100644 --- a/tests/test_api_http.py +++ b/tests/test_api_http.py @@ -35,6 +35,8 @@ ("GET", "/decompose_preview/{token}"), ("POST", "/edit/load-by-path"), ("POST", "/edit/save"), + ("POST", "/edit/session/duplicate"), + ("POST", "/edit/session/row"), ("POST", "/edit/update-filter"), ("POST", "/edit/add-spikes"), ("POST", "/edit/add-artifact"), @@ -187,7 +189,7 @@ def _err(resp: Response, status: int, code: str | None = None) -> dict[str, Any] def _unpack_json_f32( - blob: bytes, magic: bytes, n_arrays: int + blob: bytes, magic: bytes, n_arrays: int, bytes_per_value: int = 4 ) -> tuple[dict, list[tuple[int, int]]]: """Decode ``pack_json_f32_payload`` output; check the byte length adds up.""" assert blob[:4] == magic @@ -196,7 +198,7 @@ def _unpack_json_f32( shapes = [struct.unpack(" None: def _check_meld(resp: Response) -> dict[str, Any]: assert resp.status_code == 200 - assert resp.headers["x-muedit-format"] == "edit-load-f32-v1" - meta, shapes = _unpack_json_f32(resp.content, b"MELD", 1) + assert resp.headers["x-muedit-format"] == "edit-load-f64-v1" + meta, shapes = _unpack_json_f32(resp.content, b"MELD", 1, bytes_per_value=8) assert set(meta) >= EDIT_LOAD_KEYS assert meta["pulse_binary"] is True assert shapes == [(2, N_SAMPLES)] == [tuple(meta["pulse_shape"])] diff --git a/tests/test_edit_fidelity.py b/tests/test_edit_fidelity.py new file mode 100644 index 0000000..ebaff2c --- /dev/null +++ b/tests/test_edit_fidelity.py @@ -0,0 +1,227 @@ +"""Edit round-trip fidelity: load -> edit -> save must not alter untouched data. + +The editor works on a display copy of the pulse trains. Whatever the browser +holds, the saved ``pulse_trains`` of every motor unit the user did not +recompute must be bit-identical to the source file. +""" + +from __future__ import annotations + +import json +import struct +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import numpy as np +import pytest +from starlette.testclient import TestClient + +API = "/api/v1" +FSAMP = 2000.0 +N_SAMPLES = 8000 +GRIDS = ["GR08MM1305", "GR08MM1305"] +MU_GRID_INDEX = [0, 0, 0, 1, 1, 1] +FILE_LABEL = "sub-01_task-fidelity_decomp.npz" +ENTITY = "sub-01_task-fidelity" + + +# ── Fixtures ───────────────────────────────────────────────────────────────── + + +@pytest.fixture(scope="module") +def workspace(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Path]: + import muedit.api.config as config + import muedit.api.services.editing_service as editing_service + + root = tmp_path_factory.mktemp("edit_fidelity") + with pytest.MonkeyPatch.context() as mp: + mp.setattr(config, "DATA_ROOT", root) + mp.setattr(editing_service, "DATA_ROOT", root) + mp.chdir(root) + yield root + + +@pytest.fixture(scope="module") +def client(workspace: Path) -> Iterator[TestClient]: + from muedit.api.app_factory import create_app + from muedit.api.routes import include_routers + + app = create_app() + include_routers(app) + with TestClient(app, raise_server_exceptions=False) as c: + yield c + + +@pytest.fixture(scope="module") +def source_pulse() -> np.ndarray: + """Full-precision float64 pulse trains, none of them exactly float32-representable.""" + rng = np.random.default_rng(7) + return rng.standard_normal((len(MU_GRID_INDEX), N_SAMPLES)) * rng.uniform(1.0, 500.0, (6, 1)) + + +@pytest.fixture(scope="module") +def source_distimes() -> list[list[int]]: + return [[400 + 350 * k + 40 * mu for k in range(20)] for mu in range(len(MU_GRID_INDEX))] + + +@pytest.fixture(scope="module") +def decomp_npz(workspace: Path, source_pulse: np.ndarray, source_distimes: list[list[int]]) -> Path: + from muedit.decomp.decomposition_file import save_decomposition_npz + + path = workspace / FILE_LABEL + save_decomposition_npz( + path, + pulse_trains=source_pulse, + distimes=source_distimes, + fsamp=FSAMP, + grid_names=GRIDS, + mu_grid_index=MU_GRID_INDEX, + muscles=["ta"], + parameters={}, + total_samples=N_SAMPLES, + ) + return path + + +# ── A minimal stand-in for the browser ─────────────────────────────────────── + + +def _decode_meld(blob: bytes) -> tuple[dict[str, Any], np.ndarray]: + """Decode the MELD edit-load payload the way the frontend does (float64 matrix).""" + assert blob[:4] == b"MELD" + version, meta_len, rows, cols = struct.unpack(" list[str]: + counts: dict[int, int] = {} + out = [] + for g in mu_grid_index: + out.append(f"g{g}_mu{counts.get(g, 0)}") + counts[g] = counts.get(g, 0) + 1 + return out + + +def _load_like_browser(client: TestClient, path: Path) -> tuple[dict[str, Any], np.ndarray]: + resp = client.post(f"{API}/edit/load-by-path", json={"path": str(path)}) + assert resp.status_code == 200, resp.text + return _decode_meld(resp.content) + + +def _save_like_browser( + client: TestClient, + meta: dict[str, Any], + browser_pulse: np.ndarray, + distimes: list[list[int]], + **overrides: Any, +) -> np.ndarray: + """Send what ``saveEditedFile`` sends, then read the written ``pulse_trains`` back.""" + payload: dict[str, Any] = { + "distimes": distimes, + "flagged": [False] * len(distimes), + "total_samples": meta["total_samples"], + "fsamp": meta["fsamp"], + "grid_names": meta["grid_names"], + "mu_grid_index": meta["mu_grid_index"], + "mu_uids": _uids(meta["mu_grid_index"]), + "parameters": {}, + "muscle": [], + "edit_history": [], + "artifact_times": [[] for _ in distimes], + "file_label": FILE_LABEL, + "entity_label": ENTITY, + "project": "fidelity", + "remove_flagged": False, + "remove_duplicates": False, + } + # The browser only uploads the matrix when the server did not hand it a session. + if meta.get("session_token"): + payload["session_token"] = meta["session_token"] + else: + payload["pulse_trains"] = browser_pulse.tolist() + payload.update(overrides) + resp = client.post(f"{API}/edit/save", json=payload) + assert resp.status_code == 200, resp.text + saved = np.load(resp.json()["data"]["path"], allow_pickle=True) + return np.asarray(saved["pulse_trains"]) + + +# ── The contract ───────────────────────────────────────────────────────────── + + +def test_save_without_edits_is_bit_identical( + client: TestClient, decomp_npz: Path, source_pulse: np.ndarray +) -> None: + meta, browser_pulse = _load_like_browser(client, decomp_npz) + out = _save_like_browser(client, meta, browser_pulse, meta["distime_all"]) + assert out.shape == source_pulse.shape + assert np.array_equal(out, source_pulse) + + +def test_editing_one_mu_leaves_every_other_mu_bit_identical( + client: TestClient, decomp_npz: Path, source_pulse: np.ndarray, source_distimes: list[list[int]] +) -> None: + meta, browser_pulse = _load_like_browser(client, decomp_npz) + edited = [list(t) for t in meta["distime_all"]] + edited[2] = sorted(edited[2] + [1234]) # a spike added to MU 2: integers only + out = _save_like_browser(client, meta, browser_pulse, edited) + others = [i for i in range(len(edited)) if i != 2] + assert np.array_equal(out[others], source_pulse[others]) + + +def test_removing_a_flagged_mu_keeps_the_others_bit_identical( + client: TestClient, decomp_npz: Path, source_pulse: np.ndarray +) -> None: + meta, browser_pulse = _load_like_browser(client, decomp_npz) + flagged = [False] * len(meta["distime_all"]) + flagged[1] = True + out = _save_like_browser( + client, meta, browser_pulse, meta["distime_all"], flagged=flagged, remove_flagged=True + ) + kept = [i for i in range(len(flagged)) if i != 1] + assert out.shape[0] == len(kept) + assert np.array_equal(out, source_pulse[kept]) + + +def test_discharge_times_and_metadata_survive_the_round_trip( + client: TestClient, decomp_npz: Path, source_distimes: list[list[int]] +) -> None: + """Integers and labels are exact today: this must keep holding at every step.""" + meta, browser_pulse = _load_like_browser(client, decomp_npz) + edited = [list(t) for t in source_distimes] + edited[0] = sorted(edited[0] + [999]) + payload_path = client.post( + f"{API}/edit/save", + json={ + "distimes": edited, + "flagged": [False] * len(edited), + "total_samples": meta["total_samples"], + "fsamp": meta["fsamp"], + "grid_names": meta["grid_names"], + "mu_grid_index": meta["mu_grid_index"], + "mu_uids": _uids(meta["mu_grid_index"]), + "parameters": {}, + "muscle": [], + "edit_history": [], + "artifact_times": [[] for _ in edited], + "file_label": FILE_LABEL, + "entity_label": ENTITY, + "project": "fidelity", + "remove_flagged": False, + "remove_duplicates": False, + **( + {"session_token": meta["session_token"]} + if meta.get("session_token") + else {"pulse_trains": browser_pulse.tolist()} + ), + }, + ).json()["data"]["path"] + saved = np.load(payload_path, allow_pickle=True) + assert [list(map(int, d)) for d in saved["discharge_times"]] == edited + assert float(saved["fsamp"]) == FSAMP + assert list(saved["mu_grid_index"]) == MU_GRID_INDEX + assert int(saved["total_samples"]) == N_SAMPLES diff --git a/tests/test_edit_lazy_rows.py b/tests/test_edit_lazy_rows.py new file mode 100644 index 0000000..1ca5952 --- /dev/null +++ b/tests/test_edit_lazy_rows.py @@ -0,0 +1,204 @@ +"""Lazy edit load: the pulse-train matrix stays on the server, rows are fetched one at a time.""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import numpy as np +import pytest +from starlette.testclient import TestClient + +API = "/api/v1" +FSAMP = 2000.0 +N_SAMPLES = 6000 +N_CHANNELS = 64 +GRID = "GR08MM1305" +FILE_LABEL = "sub-04_task-lazy_decomp.npz" +ENTITY = "sub-04_task-lazy" +LAZY = {"x-muedit-lazy-rows": "1"} + + +@pytest.fixture(scope="module") +def workspace(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Path]: + import muedit.api.config as config + import muedit.api.services.editing_service as editing_service + + root = tmp_path_factory.mktemp("edit_lazy") + with pytest.MonkeyPatch.context() as mp: + mp.setattr(config, "DATA_ROOT", root) + mp.setattr(editing_service, "DATA_ROOT", root) + mp.chdir(root) + yield root + + +@pytest.fixture(scope="module") +def client(workspace: Path) -> Iterator[TestClient]: + from muedit.api.app_factory import create_app + from muedit.api.routes import include_routers + + app = create_app() + include_routers(app) + with TestClient(app, raise_server_exceptions=False) as c: + yield c + + +@pytest.fixture(scope="module") +def source_pulse() -> np.ndarray: + rng = np.random.default_rng(23) + return rng.standard_normal((2, N_SAMPLES)) * rng.uniform(1.0, 300.0, (2, 1)) + + +@pytest.fixture(scope="module") +def decomp_npz(workspace: Path, source_pulse: np.ndarray) -> Path: + from muedit.decomp.decomposition_file import pack_object_array, save_decomposition_npz + + path = workspace / FILE_LABEL + save_decomposition_npz( + path, + pulse_trains=source_pulse, + distimes=[[1000, 1400, 1800, 2200, 2600, 3000], [1200, 2000, 2800]], + fsamp=FSAMP, + grid_names=[GRID], + mu_grid_index=[0, 0], + muscles=["ta"], + parameters={}, + total_samples=N_SAMPLES, + extras={ + "emg_data": np.random.default_rng(0).normal(0, 1, (N_CHANNELS, N_SAMPLES)), + "discard_channels": pack_object_array([np.zeros(N_CHANNELS, dtype=int)]), + "coordinates": pack_object_array([np.zeros((N_CHANNELS, 2))]), + }, + ) + return path + + +def _lazy_load(client: TestClient, path: Path) -> dict[str, Any]: + resp = client.post(f"{API}/edit/load-by-path", json={"path": str(path)}, headers=LAZY) + assert resp.status_code == 200, resp.text + data: dict[str, Any] = resp.json()["data"] + return data + + +def _row(client: TestClient, meta: dict[str, Any], uid: str, rev: int = 0) -> Any: + return client.post( + f"{API}/edit/session/row", + json={"session_token": meta["session_token"], "mu_uid": uid, "row_rev": rev}, + ) + + +class TestLazyLoad: + def test_no_matrix_is_sent_but_the_metadata_and_token_are( + self, client: TestClient, decomp_npz: Path + ) -> None: + meta = _lazy_load(client, decomp_npz) + assert meta["lazy_rows"] is True + assert "pulse_trains_full" not in meta + assert meta["session_token"] and meta["mu_uids"] == ["g0_mu0", "g0_mu1"] + assert meta["total_samples"] == N_SAMPLES + assert len(meta["distime_all"]) == 2 + + def test_without_the_header_the_matrix_is_still_sent( + self, client: TestClient, decomp_npz: Path + ) -> None: + resp = client.post(f"{API}/edit/load-by-path", json={"path": str(decomp_npz)}) + assert resp.content[:4] == b"MELD" + + +class TestRowEndpoint: + def test_a_row_is_bit_identical_to_the_float64_source( + self, client: TestClient, decomp_npz: Path, source_pulse: np.ndarray + ) -> None: + meta = _lazy_load(client, decomp_npz) + resp = _row(client, meta, "g0_mu1") + assert resp.status_code == 200 + assert resp.headers["x-muedit-format"] == "edit-row-f64-v1" + row = np.frombuffer(resp.content, dtype=" None: + meta = _lazy_load(client, decomp_npz) + resp = client.post( + f"{API}/edit/update-filter", + json={ + "project": "lazy", + "edit_signal_token": meta["edit_signal_token"], + "file_label": FILE_LABEL, + "distimes": meta["distime_all"], + "mu_index": 0, + "view_start": 500, + "view_end": 4500, + "nbextchan": 200, + "session_token": meta["session_token"], + "mu_uid": "g0_mu0", + "omit_pulse_train": True, + }, + ) + assert resp.status_code == 200, resp.text + data = resp.json()["data"] + assert "pulse_train" not in data # the editor fetches the row instead + assert data["row_rev"] == 1 + new = np.frombuffer(_row(client, meta, "g0_mu0", 1).content, dtype=" None: + meta = _lazy_load(client, decomp_npz) + assert _row(client, meta, "g9_mu9").status_code == 400 + assert _row(client, meta, "g0_mu0", 7).status_code == 409 + bad = client.post( + f"{API}/edit/session/row", + json={"session_token": "nope", "mu_uid": "g0_mu0", "row_rev": 0}, + ) + assert bad.status_code == 404 + assert "reopen the file" in bad.text + + +class TestEditsUseTheSessionRow: + def test_delete_spikes_reads_the_row_from_the_session( + self, client: TestClient, decomp_npz: Path, source_pulse: np.ndarray + ) -> None: + meta = _lazy_load(client, decomp_npz) + via_session = client.post( + f"{API}/edit/delete-spikes", + json={ + "distimes": meta["distime_all"], + "mu_index": 0, + "session_token": meta["session_token"], + "mu_uid": "g0_mu0", + "row_rev": 0, + "x_start": 900, + "x_end": 2000, + "y_min": -1e9, + "y_max": 1e9, + }, + ) + uploaded = client.post( + f"{API}/edit/delete-spikes", + json={ + "distimes": meta["distime_all"], + "mu_index": 0, + "pulse_train": source_pulse[0].tolist(), + "x_start": 900, + "x_end": 2000, + "y_min": -1e9, + "y_max": 1e9, + }, + ) + assert via_session.status_code == 200, via_session.text + assert via_session.json()["data"] == uploaded.json()["data"] + + def test_an_edit_without_any_pulse_train_is_a_400( + self, client: TestClient, decomp_npz: Path + ) -> None: + meta = _lazy_load(client, decomp_npz) + resp = client.post( + f"{API}/edit/remove-outliers", + json={"distimes": meta["distime_all"], "mu_index": 0, "fsamp": FSAMP}, + ) + assert resp.status_code == 400 diff --git a/tests/test_edit_session.py b/tests/test_edit_session.py new file mode 100644 index 0000000..3960b37 --- /dev/null +++ b/tests/test_edit_session.py @@ -0,0 +1,452 @@ +"""Server-side edit session: row versions, cache lifecycle and the save/update-filter flow.""" + +from __future__ import annotations + +import json +import struct +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import numpy as np +import pytest +from starlette.testclient import TestClient + +from muedit.api import cache +from muedit.api.edit_session import ( + KEEP_VERSIONS, + EditSession, + RowUnavailableError, + UnknownMuError, +) + +API = "/api/v1" +FSAMP = 2000.0 +N_SAMPLES = 6000 +N_CHANNELS = 64 +GRID = "GR08MM1305" +FILE_LABEL = "sub-02_task-session_decomp.npz" +ENTITY = "sub-02_task-session" +UIDS = ["g0_mu0", "g0_mu1", "g1_mu0"] + + +def _matrix(n: int = 3, cols: int = 500, dtype: type = np.float64) -> np.ndarray: + rng = np.random.default_rng(3) + return (rng.standard_normal((n, cols)) * 100).astype(dtype) + + +# ── EditSession ────────────────────────────────────────────────────────────── + + +class TestEditSession: + def test_rows_are_read_only_views_of_the_matrix(self) -> None: + matrix = _matrix() + session = EditSession.from_matrix("f.npz", matrix, UIDS) + row = session.row("g0_mu1") + assert np.shares_memory(row, matrix) + assert np.array_equal(row, matrix[1]) + assert not row.flags.writeable + assert session.n_samples == matrix.shape[1] + assert session.nbytes == matrix.nbytes # nothing copied yet + + def test_from_matrix_rejects_inconsistent_input(self) -> None: + with pytest.raises(ValueError, match="shape"): + EditSession.from_matrix("f.npz", _matrix(3), UIDS[:2]) + with pytest.raises(ValueError, match="unique"): + EditSession.from_matrix("f.npz", _matrix(3), ["a", "a", "b"]) + + def test_add_version_keeps_the_original_row_untouched(self) -> None: + matrix = _matrix() + session = EditSession.from_matrix("f.npz", matrix, UIDS) + new = matrix[0] * 2.0 + rev = session.add_version("g0_mu0", new) + assert rev == 1 + assert np.array_equal(session.row("g0_mu0", 1), new) + assert np.array_equal(session.row("g0_mu0", 0), matrix[0]) + assert np.array_equal(matrix, _matrix()) # the loaded matrix itself never changes + assert session.nbytes == matrix.nbytes + new.nbytes + + def test_old_versions_are_pruned_but_revision_zero_never_is(self) -> None: + session = EditSession.from_matrix("f.npz", _matrix(), UIDS) + revs = [session.add_version("g0_mu0", np.full(500, float(i))) for i in range(6)] + assert revs == [1, 2, 3, 4, 5, 6] + assert session.row("g0_mu0", 0) is not None + for kept in revs[-KEEP_VERSIONS:]: + assert session.row("g0_mu0", kept) is not None + with pytest.raises(RowUnavailableError): + session.row("g0_mu0", revs[0]) + # memory accounting follows the pruning + assert session.nbytes == 3 * 500 * 8 + KEEP_VERSIONS * 500 * 8 + + def test_versions_are_kept_per_motor_unit(self) -> None: + session = EditSession.from_matrix("f.npz", _matrix(), UIDS) + session.add_version("g0_mu0", np.zeros(500)) + assert session.add_version("g0_mu1", np.ones(500)) == 1 + with pytest.raises(RowUnavailableError): + session.row("g1_mu0", 1) + + def test_unknown_motor_unit(self) -> None: + session = EditSession.from_matrix("f.npz", _matrix(), UIDS) + with pytest.raises(UnknownMuError): + session.row("nope") + with pytest.raises(UnknownMuError): + session.add_version("nope", np.zeros(500)) + + def test_add_version_rejects_wrong_length(self) -> None: + session = EditSession.from_matrix("f.npz", _matrix(), UIDS) + with pytest.raises(ValueError, match="shape"): + session.add_version("g0_mu0", np.zeros(10)) + + def test_duplicate_shares_the_row_it_was_made_from(self) -> None: + matrix = _matrix() + session = EditSession.from_matrix("f.npz", matrix, UIDS) + session.add_version("g0_mu0", matrix[0] + 1.0) + session.duplicate("g0_mu0", 1, "g0_mu2") + assert np.array_equal(session.row("g0_mu2"), matrix[0] + 1.0) + # editing the duplicate leaves its source alone, and the other way round + session.add_version("g0_mu2", np.zeros(500)) + assert np.array_equal(session.row("g0_mu0", 1), matrix[0] + 1.0) + assert np.array_equal(session.row("g0_mu2", 0), matrix[0] + 1.0) + + def test_duplicate_errors(self) -> None: + session = EditSession.from_matrix("f.npz", _matrix(), UIDS) + with pytest.raises(UnknownMuError): + session.duplicate("nope", 0, "x") + with pytest.raises(RowUnavailableError): + session.duplicate("g0_mu0", 9, "x") + with pytest.raises(ValueError, match="already exists"): + session.duplicate("g0_mu0", 0, "g0_mu1") + + def test_assemble_follows_the_requested_order_and_revisions(self) -> None: + matrix = _matrix() + session = EditSession.from_matrix("f.npz", matrix, UIDS) + session.add_version("g0_mu1", np.full(500, 7.0)) + out = session.assemble(["g1_mu0", "g0_mu1", "g0_mu0"], {"g0_mu1": 1}) + assert out.dtype == np.float64 + assert np.array_equal(out[0], matrix[2]) + assert np.array_equal(out[1], np.full(500, 7.0)) + assert np.array_equal(out[2], matrix[0]) + assert np.array_equal(session.assemble(["g0_mu1"])[0], matrix[1]) # default = row as read + + def test_float32_source_is_widened_exactly(self) -> None: + matrix = _matrix(dtype=np.float32) + out = EditSession.from_matrix("f.npz", matrix, UIDS).assemble(UIDS) + assert out.dtype == np.float64 + assert np.array_equal(out, matrix.astype(np.float64)) + + def test_non_contiguous_matrix_is_made_contiguous(self) -> None: + transposed = _matrix(500, 3).T # a (3, 500) view with strided rows + assert not transposed.flags.c_contiguous + session = EditSession.from_matrix("f.npz", transposed, UIDS) + assert np.array_equal(session.row("g0_mu1"), transposed[1]) + assert session.row("g0_mu1").flags.c_contiguous + + +# ── Cache lifecycle ────────────────────────────────────────────────────────── + + +class TestSessionCache: + def test_store_and_get(self) -> None: + session = EditSession.from_matrix("f.npz", _matrix(), UIDS) + token = cache._store_edit_session(session) + assert cache._get_edit_session(token) is session + assert cache._get_edit_session("unknown") is None + assert cache._get_edit_session(None) is None + + def test_opening_a_second_file_releases_the_first_session(self) -> None: + first = cache._store_edit_session(EditSession.from_matrix("a.npz", _matrix(), UIDS)) + second = cache._store_edit_session(EditSession.from_matrix("b.npz", _matrix(), UIDS)) + assert cache._get_edit_session(first) is None + assert cache._get_edit_session(second) is not None + + def test_expired_sessions_are_purged(self, monkeypatch: pytest.MonkeyPatch) -> None: + token = cache._store_edit_session(EditSession.from_matrix("f.npz", _matrix(), UIDS)) + monkeypatch.setattr(cache.time, "time", lambda: 1e12) + assert cache._get_edit_session(token) is None + + def test_a_hit_refreshes_the_expiry(self) -> None: + token = cache._store_edit_session(EditSession.from_matrix("f.npz", _matrix(), UIDS)) + entry = cache._EDIT_SESSION_CACHE[token] + entry.expires_at = cache.time.time() + 1.0 # about to expire + assert cache._get_edit_session(token) is not None + assert entry.expires_at > cache.time.time() + 3600 + + +# ── API flow ───────────────────────────────────────────────────────────────── + + +@pytest.fixture(scope="module") +def workspace(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Path]: + import muedit.api.config as config + import muedit.api.services.editing_service as editing_service + + root = tmp_path_factory.mktemp("edit_session") + with pytest.MonkeyPatch.context() as mp: + mp.setattr(config, "DATA_ROOT", root) + mp.setattr(editing_service, "DATA_ROOT", root) + mp.chdir(root) + yield root + + +@pytest.fixture(scope="module") +def client(workspace: Path) -> Iterator[TestClient]: + from muedit.api.app_factory import create_app + from muedit.api.routes import include_routers + + app = create_app() + include_routers(app) + with TestClient(app, raise_server_exceptions=False) as c: + yield c + + +@pytest.fixture(scope="module") +def source_pulse() -> np.ndarray: + rng = np.random.default_rng(11) + return rng.standard_normal((2, N_SAMPLES)) * rng.uniform(1.0, 300.0, (2, 1)) + + +@pytest.fixture(scope="module") +def decomp_npz(workspace: Path, source_pulse: np.ndarray) -> Path: + """Decomposition with its raw EMG embedded, so ``update-filter`` has a signal to work on.""" + from muedit.decomp.decomposition_file import pack_object_array, save_decomposition_npz + + path = workspace / FILE_LABEL + save_decomposition_npz( + path, + pulse_trains=source_pulse, + distimes=[[1000, 1400, 1800, 2200, 2600, 3000], [1200, 2000, 2800]], + fsamp=FSAMP, + grid_names=[GRID], + mu_grid_index=[0, 0], + muscles=["ta"], + parameters={}, + total_samples=N_SAMPLES, + extras={ + "emg_data": np.random.default_rng(0).normal(0, 1, (N_CHANNELS, N_SAMPLES)), + "discard_channels": pack_object_array([np.zeros(N_CHANNELS, dtype=int)]), + "coordinates": pack_object_array([np.zeros((N_CHANNELS, 2))]), + }, + ) + return path + + +def _load(client: TestClient, path: Path) -> dict[str, Any]: + resp = client.post( + f"{API}/edit/load-by-path", json={"path": str(path)}, headers={"x-muedit-binary": "0"} + ) + assert resp.status_code == 200, resp.text + data: dict[str, Any] = resp.json()["data"] + return data + + +def _save(client: TestClient, meta: dict[str, Any], distimes: list[list[int]], **extra: Any) -> Any: + payload: dict[str, Any] = { + "distimes": distimes, + "flagged": [False] * len(distimes), + "total_samples": N_SAMPLES, + "fsamp": FSAMP, + "grid_names": [GRID], + "mu_grid_index": [0] * len(distimes), + "mu_uids": extra.pop("mu_uids", meta["mu_uids"]), + "parameters": {}, + "muscle": [], + "edit_history": [], + "artifact_times": [[] for _ in distimes], + "file_label": FILE_LABEL, + "entity_label": ENTITY, + "project": "session", + "remove_flagged": False, + "remove_duplicates": False, + "session_token": meta["session_token"], + **extra, + } + return client.post(f"{API}/edit/save", json=payload) + + +def _saved_pulse(resp: Any) -> np.ndarray: + assert resp.status_code == 200, resp.text + return np.asarray(np.load(resp.json()["data"]["path"], allow_pickle=True)["pulse_trains"]) + + +def _update_filter(client: TestClient, meta: dict[str, Any], **extra: Any) -> Any: + return client.post( + f"{API}/edit/update-filter", + json={ + "project": "session", + "edit_signal_token": meta["edit_signal_token"], + "file_label": FILE_LABEL, + "distimes": meta["distime_all"], + "mu_index": 0, + "view_start": 500, + "view_end": 4500, + "nbextchan": 200, + "session_token": meta["session_token"], + "mu_uid": "g0_mu0", + **extra, + }, + ) + + +class TestSessionApi: + def test_load_hands_out_a_session_and_the_motor_unit_ids( + self, client: TestClient, decomp_npz: Path + ) -> None: + meta = _load(client, decomp_npz) + assert isinstance(meta["session_token"], str) and meta["session_token"] + assert meta["mu_uids"] == ["g0_mu0", "g0_mu1"] + + def test_binary_load_carries_the_token_too(self, client: TestClient, decomp_npz: Path) -> None: + resp = client.post(f"{API}/edit/load-by-path", json={"path": str(decomp_npz)}) + meta_len = struct.unpack(" None: + from muedit.decomp.decomposition_file import save_decomposition_npz + + path = workspace / "sub-03_task-log_decomp.npz" + save_decomposition_npz( + path, + pulse_trains=source_pulse, + distimes=[[10, 20], [30, 40]], + fsamp=FSAMP, + grid_names=[GRID], + mu_grid_index=[0, 0], + muscles=["ta"], + parameters={}, + total_samples=N_SAMPLES, + ) + path.with_suffix(".json").write_text( + json.dumps({"mu_uids": ["g0_mu4", "g0_mu7"], "history": []}), encoding="utf-8" + ) + assert _load(client, path)["mu_uids"] == ["g0_mu4", "g0_mu7"] + + def test_saving_from_the_session_needs_no_uploaded_matrix( + self, client: TestClient, decomp_npz: Path, source_pulse: np.ndarray + ) -> None: + meta = _load(client, decomp_npz) + out = _saved_pulse(_save(client, meta, meta["distime_all"])) + assert np.array_equal(out, source_pulse) + + def test_update_filter_creates_a_revision_and_only_that_row_changes( + self, client: TestClient, decomp_npz: Path, source_pulse: np.ndarray + ) -> None: + meta = _load(client, decomp_npz) + resp = _update_filter(client, meta) + assert resp.status_code == 200, resp.text + data = resp.json()["data"] + assert data["row_rev"] == 1 + recomputed = np.asarray(data["pulse_train"]) + assert recomputed.shape == (N_SAMPLES,) + assert not np.array_equal(recomputed, source_pulse[0]) # the window was recomputed + out = _saved_pulse(_save(client, meta, meta["distime_all"], row_revs={"g0_mu0": 1})) + assert np.array_equal(out[0], recomputed) # exactly what the editor was shown + assert np.array_equal(out[1], source_pulse[1]) # untouched MU: bit-identical + # outside the recomputed window the row is still the file's, bit for bit + assert np.array_equal(out[0][:500], source_pulse[0][:500]) + assert np.array_equal(out[0][4500:], source_pulse[0][4500:]) + + def test_undo_is_a_revision_number_not_a_round_trip( + self, client: TestClient, decomp_npz: Path, source_pulse: np.ndarray + ) -> None: + meta = _load(client, decomp_npz) + assert _update_filter(client, meta).json()["data"]["row_rev"] == 1 + # the editor undid it: it now reports revision 0 again + out = _saved_pulse(_save(client, meta, meta["distime_all"], row_revs={"g0_mu0": 0})) + assert np.array_equal(out, source_pulse) + # ... and a second recompute starts from the version the editor shows + again = _update_filter(client, meta, row_rev=0).json()["data"] + assert again["row_rev"] == 2 + + def test_a_duplicated_motor_unit_is_saved_from_the_session( + self, client: TestClient, decomp_npz: Path, source_pulse: np.ndarray + ) -> None: + meta = _load(client, decomp_npz) + recomputed = np.asarray(_update_filter(client, meta).json()["data"]["pulse_train"]) + dup = client.post( + f"{API}/edit/session/duplicate", + json={ + "session_token": meta["session_token"], + "source_uid": "g0_mu0", + "source_rev": 1, + "new_uid": "g0_mu2", + }, + ) + assert dup.status_code == 200, dup.text + distimes = [*meta["distime_all"], meta["distime_all"][0]] + out = _saved_pulse( + _save( + client, + meta, + distimes, + mu_uids=["g0_mu0", "g0_mu1", "g0_mu2"], + row_revs={"g0_mu0": 1}, + ) + ) + assert out.shape[0] == 3 + assert np.array_equal(out[2], recomputed) # the copy, made from the version shown + assert np.array_equal(out[1], source_pulse[1]) + + def test_removed_motor_units_are_left_out_of_the_saved_matrix( + self, client: TestClient, decomp_npz: Path, source_pulse: np.ndarray + ) -> None: + meta = _load(client, decomp_npz) + resp = _save(client, meta, meta["distime_all"], flagged=[True, False], remove_flagged=True) + assert np.array_equal(_saved_pulse(resp), source_pulse[[1]]) + + def test_unknown_session_is_a_404_that_says_what_to_do( + self, client: TestClient, decomp_npz: Path + ) -> None: + meta = {**_load(client, decomp_npz), "session_token": "expired"} + resp = _save(client, meta, meta["distime_all"]) + assert resp.status_code == 404 + assert "reopen the file" in resp.json()["error"]["detail"]["reason"] + assert _update_filter(client, meta).status_code == 404 + + def test_a_new_file_replaces_the_previous_session( + self, client: TestClient, decomp_npz: Path + ) -> None: + first = _load(client, decomp_npz) + _load(client, decomp_npz) + assert _save(client, first, first["distime_all"]).status_code == 404 + + def test_unknown_motor_unit_id_is_a_400(self, client: TestClient, decomp_npz: Path) -> None: + meta = _load(client, decomp_npz) + resp = _save(client, meta, meta["distime_all"], mu_uids=["g0_mu0", "nope"]) + assert resp.status_code == 400 + assert "nope" in resp.json()["error"]["detail"]["reason"] + + def test_discarded_revision_is_a_409(self, client: TestClient, decomp_npz: Path) -> None: + meta = _load(client, decomp_npz) + resp = _save(client, meta, meta["distime_all"], row_revs={"g0_mu0": 5}) + assert resp.status_code == 409 + + def test_sample_count_mismatch_is_a_400(self, client: TestClient, decomp_npz: Path) -> None: + meta = _load(client, decomp_npz) + resp = _save(client, meta, meta["distime_all"], total_samples=N_SAMPLES + 1) + assert resp.status_code == 400 + + def test_the_uploaded_matrix_still_works_without_a_session( + self, client: TestClient, decomp_npz: Path, source_pulse: np.ndarray + ) -> None: + meta = _load(client, decomp_npz) + meta.pop("session_token") + resp = client.post( + f"{API}/edit/save", + json={ + "distimes": meta["distime_all"], + "pulse_trains": source_pulse.tolist(), + "total_samples": N_SAMPLES, + "fsamp": FSAMP, + "grid_names": [GRID], + "mu_grid_index": [0, 0], + "file_label": FILE_LABEL, + "entity_label": ENTITY, + "project": "session", + "remove_flagged": False, + "remove_duplicates": False, + }, + ) + assert np.array_equal(_saved_pulse(resp), source_pulse) # json floats round-trip exactly