Feature/streaming view

frontend/eslint.config.js

@@ -15,7 +15,9 @@ const filenamesPlugin = {
           Program(node) {
             const filename = context.filename;
             if (!filename) return;
-            const stem = filename.split("/").pop().replace(/\.js$/, "");
+            // Split on both separators: on Windows, ESLint hands us
+            // backslash-joined absolute paths, not just "/"-joined ones.
+            const stem = filename.split(/[/\\]/).pop().replace(/\.js$/, "");
             if (!/^[a-z0-9]+(-[a-z0-9]+)*$/.test(stem)) {
               context.report({
                 node,

frontend/src/api/binary-payloads.js

@@ -1,13 +1,18 @@
 /**
- * Decoders for the compact binary transport the API uses for large float32
+ * Decoders for the compact binary transport the API uses for large float
  * matrices (pulse trains). Each payload is either plain JSON or a binary frame:
  *
  *   magic(4 bytes) | uint32 version | uint32 metaLen | (uint32 rows, uint32 cols)
- *   per array | JSON metadata (metaLen bytes) | float32 array data
+ *   per array | JSON metadata (metaLen bytes) | array data
  *
- * All integers/floats are little-endian. When the magic prefix is absent the
- * server fell back to JSON, so the bytes are parsed as text instead. Mirrors the
- * Python `pack_json_f32_payload` packer in `api/binary.py`.
+ * All integers/floats are little-endian. The array element width depends on
+ * the payload: the edit-load frame (MELD) carries float64 — a display copy of
+ * one file's worth of pulse trains costs nothing extra to widen, and it means
+ * a value shown to the user never differs from what is on disk. QC/preview
+ * frames stay float32, sized for many more samples at once. When the magic
+ * prefix is absent the server fell back to JSON, so the bytes are parsed as
+ * text instead. Mirrors the Python `pack_json_f32_payload` packer in
+ * `api/binary.py`.
  */
 /** @typedef {import("../app/context.js").JsonObject} JsonObject */
 
@@ -42,7 +47,22 @@ function readFloat32Values(view, offset, count) {
 }
 
 /**
- * @param {Float32Array} raw
+ * @param {DataView} view
+ * @param {number} offset
+ * @param {number} count
+ */
+function readFloat64Values(view, offset, count) {
+  const out = new Float64Array(count);
+  let cursor = offset;
+  for (let i = 0; i < count; i++) {
+    out[i] = view.getFloat64(cursor, true);
+    cursor += 8;
+  }
+  return out;
+}
+
+/**
+ * @param {Float32Array | Float64Array} raw
  * @param {number} rows
  * @param {number} cols
  * @returns {number[][]}
@@ -131,8 +151,8 @@ export function decodeQcRawF32(buffer) {
  * @param {ArrayBuffer} buffer
  * @param {string | null} [formatHeader]
  */
-function isEditLoadF32Payload(buffer, formatHeader = "") {
-  return formatHeader === "edit-load-f32-v1" || hasMagic(buffer, "MELD");
+function isEditLoadF64Payload(buffer, formatHeader = "") {
+  return formatHeader === "edit-load-f64-v1" || hasMagic(buffer, "MELD");
 }
 
 /**
@@ -141,7 +161,7 @@ function isEditLoadF32Payload(buffer, formatHeader = "") {
  * @returns {JsonObject}
  */
 export function decodeEditLoadPayload(buffer, formatHeader = "") {
-  if (!isEditLoadF32Payload(buffer, formatHeader)) {
+  if (!isEditLoadF64Payload(buffer, formatHeader)) {
     const text = textDecoder.decode(new Uint8Array(buffer));
     return JSON.parse(text);
   }
@@ -163,7 +183,7 @@ export function decodeEditLoadPayload(buffer, formatHeader = "") {
     textDecoder.decode(new Uint8Array(buffer, offset, metaLen)),
   );
   offset += metaLen;
-  const pulse = to2d(readFloat32Values(view, offset, rows * cols), rows, cols);
+  const pulse = to2d(readFloat64Values(view, offset, rows * cols), rows, cols);
   return { ...meta, pulse_trains_full: pulse };
 }
 

frontend/src/api/client.js

@@ -9,7 +9,6 @@ import {
   decodeQcJsonPayload,
   decodeQcRawF32,
   decodeDecomposePreviewPayload,
-  decodeEditLoadPayload,
 } from "./binary-payloads.js";
 import { normalizePreviewPayload } from "./payloads.js";
 
@@ -150,19 +149,21 @@ export function createApiClient({ apiFetch, apiJson, API_BASE }) {
      * @param {string} filepath
      */
     async editLoadByPath(filepath) {
-      const res = await apiFetch(
+      // Light load: metadata + session token, rows are fetched one at a time
+      // (editSessionRow). When no session could be opened the server sends the
+      // full matrix instead (as JSON), which the editor then holds as before.
+      return apiJson(
         `${API_BASE}${routes.editLoadByPath}`,
         {
           method: "POST",
-          headers: { "Content-Type": "application/json" },
+          headers: {
+            "Content-Type": "application/json",
+            "X-Muedit-Lazy-Rows": "1",
+          },
           body: JSON.stringify({ path: filepath }),
         },
         120000,
       );
-      return decodeEditLoadPayload(
-        await res.arrayBuffer(),
-        res.headers.get("x-muedit-format"),
-      );
     },
 
     /**
@@ -172,6 +173,37 @@ export function createApiClient({ apiFetch, apiJson, API_BASE }) {
       return postJson(`${API_BASE}${routes.editSave}`, payload, 120000);
     },
 
+    /**
+     * @param {JsonObject} payload
+     */
+    editSessionDuplicate(payload) {
+      return postJson(
+        `${API_BASE}${routes.editSessionDuplicate}`,
+        payload,
+        30000,
+      );
+    },
+
+    /**
+     * One motor unit's row (float64 on the wire) from the server-side session.
+     * @param {JsonObject} payload
+     * @returns {Promise<number[]>}
+     */
+    async editSessionRow(payload) {
+      const res = await apiFetch(
+        `${API_BASE}${routes.editSessionRow}`,
+        {
+          method: "POST",
+          headers: { "Content-Type": "application/json" },
+          body: JSON.stringify(payload),
+        },
+        60000,
+      );
+      // Sent at the session's own precision (float64), not narrowed to
+      // float32: see the row rationale in the backend route.
+      return Array.from(new Float64Array(await res.arrayBuffer()));
+    },
+
     healthUrl() {
       return `${API_BASE}${routes.health}`;
     },

frontend/src/api/routes.js

@@ -11,6 +11,8 @@ export const routes = {
   decomposePreview: (/** @type {string} */ token) =>
     `/decompose_preview/${encodeURIComponent(token)}`,
   editSave: "/edit/save",
+  editSessionDuplicate: "/edit/session/duplicate",
+  editSessionRow: "/edit/session/row",
   editAction: (/** @type {string} */ action) => `/edit/${action}`,
   editMode: (/** @type {string} */ mode) => `/edit/${mode}`,
   editRemoveOutliers: "/edit/remove-outliers",

frontend/src/app/context.js

@@ -156,7 +156,7 @@
  * @property {() => Promise<void>} removeOutliers
  * @property {() => Promise<void>} flagMuForDeletion
  * @property {() => void} resetCurrentMuEdits
- * @property {() => void} duplicateMu
+ * @property {() => void | Promise<void>} duplicateMu
  * @property {() => Promise<void>} removeDuplicateMus
  * @property {() => Promise<void>} saveEditedFile
  * @property {(file: FileRef, path: string) => Promise<void>} loadDecompositionForEdit

frontend/src/app/services/editing-service.js

@@ -8,7 +8,7 @@ import { handleError } from "./error-service.js";
 import { normalizeEditLoadPayload } from "../../api/payloads.js";
 import { inferGridCount, normalizeGridNames } from "../../io/grid.js";
 import { getSuggestedNpzName } from "../../io/bids.js";
-import { muUidFor } from "../../state/selectors.js";
+import { muUidFor, pulseSource } from "../../state/selectors.js";
 import { spikesDiff } from "../../editing/operations.js";
 import {
   clearAllEditSelections,
@@ -29,6 +29,7 @@ import {
   setEditFlagForMu,
   setEditFlaggedArray,
   setEditFsamp,
+  setEditLazyRows,
   setEditGridNames,
   setEditHistory,
   keepEditMus,
@@ -39,6 +40,8 @@ import {
   setEditParameters,
   setEditPulseTrainForMu,
   setEditPulseTrains,
+  setEditRowRev,
+  setEditSessionToken,
   setEditSignalToken,
   setEditTotalSamples,
   setEditView,
@@ -106,7 +109,7 @@ export async function requestRoiEdit(app, action, payload) {
     const data = await api.editAction(action, {
       distimes: state.edit.distimes,
       mu_index: payload.muIdx,
-      pulse_train: payload.pulse,
+      ...pulseSource(state, payload.muIdx, payload.pulse),
       fsamp: payload.fs,
       x_start: payload.xStart,
       x_end: payload.xEnd,
@@ -248,7 +251,10 @@ export async function requestFilterUpdate(app, mode) {
       mu_index: muIdx,
       distimes: state.edit.distimes,
       mu_grid_index: state.edit.muGridIndex || [],
-      pulse_train: pulse,
+      ...pulseSource(state, muIdx, pulse),
+      // The row lives on the server: the editor refetches it instead of taking
+      // a 700k-value JSON array back.
+      omit_pulse_train: !!state.edit.sessionToken,
       view_start: start,
       view_end: end,
       use_peeloff: els.editPeelOffToggle?.dataset.state === "on",
@@ -260,6 +266,9 @@ export async function requestFilterUpdate(app, mode) {
     if (data.pulse_train && Array.isArray(data.pulse_train)) {
       setEditPulseTrainForMu(state, muIdx, data.pulse_train);
     }
+    if (state.edit.sessionToken && typeof data.row_rev === "number") {
+      setEditRowRev(state, muUidFor(state, muIdx), data.row_rev);
+    }
     ensureEditFlagged();
     setEditFlagForMu(state, muIdx, false);
     const muUid = muUidFor(state, muIdx);
@@ -321,7 +330,7 @@ export async function removeOutliers(app) {
     const data = await api.editRemoveOutliers({
       distimes: state.edit.distimes,
       mu_index: muIdx,
-      pulse_train: pulse,
+      ...pulseSource(state, muIdx, pulse),
       fsamp: state.edit.fsamp || 0,
     });
     setEditDistimesForMu(state, muIdx, data.distimes || []);
@@ -484,12 +493,15 @@ export async function saveEditedFile(app) {
     return;
   }
   const muscleNames = getBidsMuscleNames();
-  const maxSpike = Math.max(
-    0,
-    ...distimes
-      .flatMap((d) => d || [])
-      .map((v) => (Number.isFinite(v) ? v : 0)),
-  );
+  // A plain loop, not Math.max(0, ...spikes): spreading every spike index of a
+  // large file as call arguments exceeds the engine's argument limit and throws
+  // before the "Saving..." status is shown (this sits above the try block).
+  let maxSpike = 0;
+  for (const d of distimes) {
+    for (const v of d || []) {
+      if (Number.isFinite(v) && v > maxSpike) maxSpike = v;
+    }
+  }
   const totalSamples =
     state.edit.totalSamples ||
     (state.edit.pulseTrains?.[0]?.length ?? 0) ||
@@ -502,7 +514,14 @@ export async function saveEditedFile(app) {
   const payload = {
     distimes,
     flagged: state.edit.flagged || [],
-    pulse_trains: state.edit.pulseTrains || [],
+    // With a server session the file is written from the reference (float64)
+    // rows held there; the browser only names which revision of each row to use.
+    ...(state.edit.sessionToken
+      ? {
+          session_token: state.edit.sessionToken,
+          row_revs: state.edit.rowRevs || {},
+        }
+      : { pulse_trains: state.edit.pulseTrains || [] }),
     total_samples: totalSamples,
     fsamp: state.edit.fsamp,
     grid_names: state.edit.gridNames,
@@ -597,16 +616,22 @@ export async function loadDecompositionForEdit(app, file, filepath) {
     if (els.bidsProject) els.bidsProject.value = loadedProject;
     setEditProject(state, loadedProject);
     setEditSignalToken(state, data.edit_signal_token || "");
+    setEditSessionToken(state, data.session_token || "");
     setEditFile(state, file);
     setEditFilename(state, file.name || data.file_label || "decomposition");
-    setEditPulseTrains(
-      state,
-      data.pulse_trains_full || data.pulse_trains || [],
-    );
-    setEditOriginalPulseTrains(
-      state,
-      (state.edit.pulseTrains || []).map((row) => (row ? [...row] : row)),
-    );
+    if (data.lazy_rows) {
+      // Pulse trains stay on the server; the editor fetches the row it shows.
+      setEditLazyRows(state, (data.distime_all || data.distime || []).length);
+    } else {
+      setEditPulseTrains(
+        state,
+        data.pulse_trains_full || data.pulse_trains || [],
+      );
+      setEditOriginalPulseTrains(
+        state,
+        (state.edit.pulseTrains || []).map((row) => (row ? [...row] : row)),
+      );
+    }
     const dist = data.distime_all || data.distime || [];
     setEditDistimes(
       state,

frontend/src/app/state.js

@@ -20,6 +20,7 @@ import { GRID_COLORS } from "../config.js";
  * @property {number[] | null} pulseTrain
  * @property {number[]} artifactTimes
  * @property {number} historyLength Edit-log length when the backup was taken.
+ * @property {number} rowRev Server-side revision of the row when the backup was taken.
  */
 
 /**
@@ -72,6 +73,11 @@ import { GRID_COLORS } from "../config.js";
  * @property {string} bidsRoot
  * @property {string} project
  * @property {string} editSignalToken
+ * @property {string} sessionToken Server-side edit session holding the reference pulse trains.
+ * @property {Record<string, number>} rowRevs Row revision in force per MU id (missing = as read from the file).
+ * @property {boolean} lazyRows Rows are fetched one at a time from the session (pulseTrains holds only a few).
+ * @property {Record<string, number>} rowCacheRevs Revision of each row currently held in pulseTrains, by MU id.
+ * @property {string[]} rowCacheOrder MU ids of the rows held, oldest first (bounds browser memory).
  * @property {string[]} muUids
  * @property {EditHistoryEntry[]} editHistory
  * @property {Bookmark | null} bookmarkPosition
@@ -149,6 +155,11 @@ export function createEditSlice() {
     bidsRoot: "",
     project: "",
     editSignalToken: "",
+    sessionToken: "",
+    rowRevs: {},
+    lazyRows: false,
+    rowCacheRevs: {},
+    rowCacheOrder: [],
     muUids: [],
     editHistory: [],
     bookmarkPosition: null,

frontend/src/decomp/run.js

@@ -46,13 +46,19 @@ export async function autoSaveRunDecomposition(app) {
   if (state.lastRunDownloadKey === key) return;
 
   const muscleNames = getBidsMuscleNames();
+  // A plain loop, not Math.max(0, ...spikes): see saveEditedFile in
+  // app/services/editing-service.js.
+  let maxSpike = 0;
+  for (const d of state.muDistimes) {
+    for (const v of d || []) {
+      const n = Number(v) || 0;
+      if (n > maxSpike) maxSpike = n;
+    }
+  }
   const totalSamples =
     state.seriesLength ||
     (state.muPulseTrains?.[0]?.length ?? 0) ||
-    Math.max(
-      0,
-      ...state.muDistimes.flatMap((d) => (d || []).map((v) => Number(v) || 0)),
-    ) + 1;
+    maxSpike + 1;
   const fs = state.fsamp;
   const payload = {
     distimes: state.muDistimes || [],