From ae19a6b29ed776db3e9791a86696bc8e9080b904 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Mon, 14 Sep 2026 08:35:55 -0700 Subject: [PATCH 01/57] Add store and unit tests --- dashboard-docs/spike/combined-store.js | 236 ++++++++++++++++++++ nodes/config/ui_base.js | 3 + nodes/store/reactive.js | 118 ++++++++++ test/store/reactive.spec.js | 297 +++++++++++++++++++++++++ 4 files changed, 654 insertions(+) create mode 100644 dashboard-docs/spike/combined-store.js create mode 100644 nodes/store/reactive.js create mode 100644 test/store/reactive.spec.js diff --git a/dashboard-docs/spike/combined-store.js b/dashboard-docs/spike/combined-store.js new file mode 100644 index 000000000..97669e067 --- /dev/null +++ b/dashboard-docs/spike/combined-store.js @@ -0,0 +1,236 @@ +/* + * Reactive data store POC: an Entry (value + metadata) at each top-level key, + * with a deep-reactive .value, injected into a real Node-RED memory context. + * A write fires onChange(key, entry, path). + * Run from the dashboard repo or a Node-RED install: node combined-store.js + * + * Tests: value/$meta reads, history, nested + array mutation (incl. truncation), + * top-level delete, '$'-key reservation, prototype guards, maxHistory eviction, + * callback API, namespace clobber, and serialise + restart with reactivity. + */ + +const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') + +let tick = 0 +const clock = () => tick++ // deterministic for the spike; Date.now() in production +const snapshot = (v) => (v && typeof v === 'object') ? JSON.parse(JSON.stringify(v)) : v + +class Entry { + constructor () { + this.value = undefined + this.quality = 'GOOD' + this.timestamp = 0 + this.history = [] + } +} + +function deepReactive (value, notify, path) { + if (value === null || typeof value !== 'object') return value + for (const k of Object.keys(value)) value[k] = deepReactive(value[k], notify, `${path}.${k}`) + return new Proxy(value, { + get (t, p, r) { return Reflect.get(t, p, r) }, + set (t, p, v) { + if (typeof p === 'symbol') return Reflect.set(t, p, v) + const shrank = p === 'length' && Array.isArray(t) && v < t.length + t[p] = deepReactive(v, notify, `${path}.${String(p)}`) + if (p !== 'length') notify(`${path}.${String(p)}`) // index/prop write + else if (shrank) notify(path) // array truncation (e.g. arr.length = 0); growth from push is ignored + return true + }, + deleteProperty (t, p) { const ok = delete t[p]; notify(`${path}.${String(p)}`); return ok } + }) +} + +function createDataStore ({ maxHistory = 20, onChange } = {}) { + const records = Object.create(null) + const handler = { + get (target, prop, receiver) { + if (typeof prop === 'symbol') return Reflect.get(target, prop, receiver) + if (prop === 'toJSON') { + return () => Object.fromEntries(Object.entries(target).map(([k, r]) => [k, r.value])) + } + if (prop.startsWith('$')) return target[prop.slice(1)] // read-only meta view: $key -> Entry + const rec = target[prop] + return rec ? rec.value : undefined // value by default (deep-reactive if object) + }, + set (target, prop, value) { + if (typeof prop === 'symbol') return Reflect.set(target, prop, value) + if (prop === '__proto__' || prop === 'constructor') return false + if (prop.startsWith('$')) return false // '$' reserved for the meta view; keys can't start with $ + + let rec = target[prop] + if (!rec) { + rec = new Entry() + target[prop] = rec + } else { + rec.history.push({ value: snapshot(rec.value), timestamp: rec.timestamp }) + if (rec.history.length > maxHistory) rec.history.shift() + } + const notify = (path) => { rec.timestamp = clock(); onChange?.(prop, rec, path) } + rec.value = deepReactive(value, notify, prop) + rec.timestamp = clock() + onChange?.(prop, rec, prop) + return true + }, + ownKeys (target) { return Reflect.ownKeys(target) }, + getOwnPropertyDescriptor (target, prop) { + return Object.getOwnPropertyDescriptor(target, prop) ?? { enumerable: true, configurable: true } + }, + deleteProperty (target, prop) { + const existed = prop in target + const ok = delete target[prop] + if (existed) onChange?.(prop, undefined, prop) // notify on top-level delete + return ok + } + } + return new Proxy(records, handler) +} + +// ---- wire into a real Node-RED memory context under global.dashboard ---- +let log = [] +function makeContext (opts = {}) { + log = [] + const ctx = Memory({}) + ctx.set('global', 'dashboard', createDataStore({ ...opts, onChange: (key, entry, path) => log.push(path) })) + return ctx +} +const gset = (ctx, k, v) => ctx.set('global', k, v) +const gget = (ctx, k) => ctx.get('global', k) + +let pass = 0; let fail = 0 +// shape check (JSON equality) — proves stored/serialised shape, NOT liveness. Liveness is proven via `log`. +const check = (n, a, e) => { const ok = JSON.stringify(a) === JSON.stringify(e); console.log(`${ok ? 'PASS' : 'FAIL'} ${n}${ok ? '' : ` (exp ${JSON.stringify(e)}, got ${JSON.stringify(a)})`}`); ok ? pass++ : fail++ } + +console.log('\n--- flat scalar ---') +{ + const ctx = makeContext() + gset(ctx, 'dashboard.k1', 123) + check('scalar set fires onChange at key level', log, ['k1']) + check('value by default', gget(ctx, 'dashboard.k1'), 123) + check('$meta returns the Entry', (() => { const r = gget(ctx, 'dashboard.$k1'); return { value: r.value, quality: r.quality } })(), { value: 123, quality: 'GOOD' }) +} + +console.log('\n--- reassign: history snapshots the prior value (not a live ref) ---') +{ + const ctx = makeContext() + gset(ctx, 'dashboard.obj', { n: 1 }) + gset(ctx, 'dashboard.obj', { n: 2 }) + check('current value', gget(ctx, 'dashboard.obj'), { n: 2 }) + const hist = gget(ctx, 'dashboard.$obj').history + check('history captured prior value', hist.map(h => h.value), [{ n: 1 }]) + // mutate the CURRENT value; history must not change (snapshot, not alias) + gget(ctx, 'dashboard.obj').n = 999 + check('history not corrupted by later mutation', gget(ctx, 'dashboard.$obj').history.map(h => h.value), [{ n: 1 }]) +} + +console.log('\n--- nested object: liveness proven, not just shape ---') +{ + const ctx = makeContext() + gset(ctx, 'dashboard.robot', { temp: 20 }) + log = [] + gget(ctx, 'dashboard.robot').temp = 25 // direct nested mutation on the returned value + check('LIVENESS: nested mutation on returned value fires', log, ['robot.temp']) + check('nested value updated', gget(ctx, 'dashboard.robot').temp, 25) + gset(ctx, 'dashboard.robot.speed', 5) // nested set via context path (change-node style) + check('value-by-default still returns the whole object', gget(ctx, 'dashboard.robot'), { temp: 25, speed: 5 }) +} + +console.log('\n--- array: push, deep mutate, and CLEAR (length=0) ---') +{ + const ctx = makeContext() + gset(ctx, 'dashboard.series', []) + log = [] + gget(ctx, 'dashboard.series').push({ x: 1, y: 2 }) + check('push fires once (length growth ignored)', log, ['series.0']) + log = [] + gget(ctx, 'dashboard.series')[0].y = 99 + check('deep mutation inside array fires', log, ['series.0.y']) + log = [] + gget(ctx, 'dashboard.series').length = 0 // truncation — previously invisible + check('array clear via length=0 fires', log, ['series']) + check('array emptied', gget(ctx, 'dashboard.series'), []) +} + +console.log('\n--- top-level delete notifies ---') +{ + const ctx = makeContext() + gset(ctx, 'dashboard.gone', 1) + log = [] + gset(ctx, 'dashboard.gone', undefined) // change-node style: setObjectProperty deletes on undefined + check('delete fires onChange', log, ['gone']) + check('value removed', gget(ctx, 'dashboard.gone'), undefined) +} + +console.log('\n--- guards: no prototype pollution AND no own-key created ---') +{ + const ctx = makeContext() + const store = gget(ctx, 'dashboard') + store.__proto__ = { polluted: true } // eslint-disable-line no-proto + store.constructor = { polluted: true } + check('Object.prototype not polluted', ({}).polluted, undefined) + check('no __proto__/constructor own-keys created', Reflect.ownKeys(store).filter(k => k === '__proto__' || k === 'constructor'), []) +} + +console.log('\n--- $ prefix reserved (no silent data loss) ---') +{ + const ctx = makeContext() + gset(ctx, 'dashboard.$weird', 1) // reserved: set is a no-op + check('set of $-key is a no-op, not stored-then-lost', gget(ctx, 'dashboard.$weird'), undefined) + gset(ctx, 'dashboard.normal', 7) + check('normal keys unaffected', gget(ctx, 'dashboard.normal'), 7) + check('$normal still reads the Entry', gget(ctx, 'dashboard.$normal').value, 7) +} + +console.log('\n--- maxHistory eviction ---') +{ + const ctx = makeContext({ maxHistory: 2 }) + for (let i = 0; i < 5; i++) gset(ctx, 'dashboard.h', i) + check('history capped at maxHistory', gget(ctx, 'dashboard.$h').history.map(x => x.value), [2, 3]) + check('current value is latest', gget(ctx, 'dashboard.h'), 4) +} + +console.log('\n--- Node-RED async callback API works + reactivity intact ---') +{ + const ctx = makeContext() + let cbVal + ctx.set('global', 'dashboard.cbk', 42, () => {}) // set with callback + ctx.get('global', 'dashboard.cbk', (err, v) => { cbVal = err ? 'ERR' : v }) // get with callback + check('callback get returns value', cbVal, 42) + log = [] + gget(ctx, 'dashboard.cbk') // no-op read + ctx.set('global', 'dashboard.cbk', 43, () => {}) + check('write via callback still fires onChange', log, ['cbk']) +} + +console.log('\n--- namespace clobber (documents Story 2 risk) ---') +{ + const ctx = makeContext() + gset(ctx, 'dashboard.k', 1) + gset(ctx, 'dashboard', { k: 999 }) // a flow overwrites the whole namespace + log = [] + gset(ctx, 'dashboard.k', 2) + check('after clobber, writes are NOT observed (proxy replaced)', log, []) + check('namespace is now a plain object', gget(ctx, 'dashboard.k'), 2) +} + +console.log('\n--- serialise + restart, with nesting + deep reactivity rewired ---') +{ + const ctx = makeContext() + gset(ctx, 'dashboard.a', 10) + gset(ctx, 'dashboard.robot', { temp: 20, axes: [1, 2] }) + const snap = JSON.parse(JSON.stringify(gget(ctx, 'dashboard'))) + check('toJSON flattens Entries -> values (nested preserved)', snap, { a: 10, robot: { temp: 20, axes: [1, 2] } }) + + const ctx2 = Memory({}) + const store2 = createDataStore({ onChange: (k, e, p) => log.push(p) }) + for (const [k, v] of Object.entries(snap)) store2[k] = v // rehydrate through set + ctx2.set('global', 'dashboard', store2) + check('scalar survived restart', ctx2.get('global', 'dashboard.a'), 10) + check('nested survived restart', ctx2.get('global', 'dashboard.robot'), { temp: 20, axes: [1, 2] }) + log = [] + ctx2.get('global', 'dashboard.robot').axes.push(3) // deep reactivity, post-restart + check('LIVENESS: deep reactivity works after restart', log, ['robot.axes.2']) +} + +console.log(`\n===== ${pass} passed, ${fail} failed =====`) +process.exit(fail ? 1 : 0) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index 833f54095..db6f63859 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -5,6 +5,7 @@ const axios = require('axios') const v = require('../../package.json').version const datastore = require('../store/data.js') +const { attachToContext } = require('../store/reactive.js') const statestore = require('../store/state.js') const { appendTopic, addConnectionCredentials, getThirdPartyWidgets } = require('../utils/index.js') @@ -400,6 +401,8 @@ module.exports = function (RED) { node._created = Date.now() + attachToContext(node.context().global, { clone: RED.util.cloneMessage }) + n.root = RED.settings.httpNodeRoot || '/' /** @type {Object.} */ diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js new file mode 100644 index 000000000..f62725c9d --- /dev/null +++ b/nodes/store/reactive.js @@ -0,0 +1,118 @@ +const STORE = Symbol('dashboardDataStore') +const NAMESPACE = 'dashboard' + +class Entry { + constructor () { + this.value = undefined + this.quality = 'GOOD' + this.timestamp = 0 + this.history = [] + } +} + +// only plain objects and arrays are deep-watched; Buffers, Dates, class instances etc. are opaque leaves +function isReactable (v) { + if (Array.isArray(v)) return true + if (v === null || typeof v !== 'object') return false + const proto = Object.getPrototypeOf(v) + return proto === Object.prototype || proto === null +} + +// reads through proxies (structuredClone rejects them); production injects RED.util.cloneMessage +function deepClone (v) { + if (v === null || typeof v !== 'object') return v + if (Buffer.isBuffer(v)) return Buffer.from(v) + if (v instanceof Date) return new Date(v) + if (Array.isArray(v)) return v.map(deepClone) + const out = {} + for (const k of Object.keys(v)) out[k] = deepClone(v[k]) + return out +} + +function deepReactive (value, notify, path) { + if (!isReactable(value)) return value + for (const k of Object.keys(value)) value[k] = deepReactive(value[k], notify, `${path}.${k}`) + return new Proxy(value, { + get (t, p, r) { return Reflect.get(t, p, r) }, + set (t, p, v) { + if (typeof p === 'symbol') return Reflect.set(t, p, v) + // a push writes the new index then grows length; only a shrink is a change worth reporting + const shrank = p === 'length' && Array.isArray(t) && v < t.length + t[p] = deepReactive(v, notify, `${path}.${String(p)}`) + if (p !== 'length') notify(`${path}.${String(p)}`) + else if (shrank) notify(path) + return true + }, + deleteProperty (t, p) { + const existed = p in t + const ok = delete t[p] + if (existed) notify(`${path}.${String(p)}`) + return ok + } + }) +} + +function createDataStore ({ maxHistory = 20, onChange, clone = deepClone, now = Date.now } = {}) { + const cloneValue = (v) => (v && typeof v === 'object') ? clone(v) : v + const records = Object.create(null) + + records[STORE] = true + + const handler = { + get (target, prop, receiver) { + if (typeof prop === 'symbol') return Reflect.get(target, prop, receiver) + if (prop === 'toJSON') { + return () => Object.fromEntries(Object.entries(target).map(([k, r]) => [k, r.value])) + } + if (prop.startsWith('$')) return target[prop.slice(1)] + const rec = target[prop] + return rec ? rec.value : undefined + }, + set (target, prop, value) { + if (typeof prop === 'symbol') return Reflect.set(target, prop, value) + // no-op guards return true so strict-mode callers do not throw + if (prop === '__proto__' || prop === 'constructor') return true + if (prop.startsWith('$')) return true // '$' is reserved for the read-only meta view + + let rec = target[prop] + if (!rec) { + rec = new Entry() + target[prop] = rec + } else { + rec.history.push({ value: cloneValue(rec.value), timestamp: rec.timestamp }) + if (rec.history.length > maxHistory) rec.history.shift() + } + const notify = (path) => { rec.timestamp = now(); onChange?.(prop, rec, path) } + // clone on write so a flow reusing its own object can't mutate stored state + rec.value = deepReactive(cloneValue(value), notify, prop) + rec.timestamp = now() + onChange?.(prop, rec, prop) + return true + }, + ownKeys (target) { return Reflect.ownKeys(target) }, + deleteProperty (target, prop) { + const existed = prop in target + const ok = delete target[prop] + if (existed) onChange?.(prop, undefined, prop) + return ok + } + } + return new Proxy(records, handler) +} + +// Inject the store into global context once. On redeploy the existing store is reused; +// after a restart with a persistent context store, existing plain data is rehydrated. +function attachToContext (globalContext, opts = {}) { + const namespace = opts.namespace || NAMESPACE + const existing = globalContext.get(namespace) + if (existing && existing[STORE]) return existing + + const store = createDataStore(opts) + if (existing && typeof existing === 'object') { + for (const [k, v] of Object.entries(existing)) store[k] = v + } + globalContext.set(namespace, store) + return store +} + +module.exports = { createDataStore, attachToContext, NAMESPACE, STORE } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js new file mode 100644 index 000000000..0b1f4bac1 --- /dev/null +++ b/test/store/reactive.spec.js @@ -0,0 +1,297 @@ +const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') +const { util } = require('@node-red/util') +const should = require('should') // eslint-disable-line no-unused-vars + +const { createDataStore, attachToContext } = require('../../nodes/store/reactive.js') + +function makeStore (opts = {}) { + const log = [] + let tick = 0 + const store = createDataStore({ + now: () => tick++, + onChange: (key, entry, path) => log.push(path), + ...opts + }) + return { store, log } +} + +describe('store: reactive data store', function () { + describe('values and metadata', function () { + it('stores a scalar, fires onChange, and reads it back as a value', function () { + const { store, log } = makeStore() + store.k1 = 123 + log.should.eql(['k1']) + store.k1.should.equal(123) + }) + + it('exposes the full entry under the $ prefix', function () { + const { store } = makeStore() + store.k1 = 123 + store.$k1.value.should.equal(123) + store.$k1.quality.should.equal('GOOD') + store.$k1.timestamp.should.be.a.Number() + }) + + it('returns the whole object by default', function () { + const { store } = makeStore() + store.robot = { temp: 20 } + store.robot.should.eql({ temp: 20 }) + }) + }) + + describe('history', function () { + it('keeps the prior value on reassignment', function () { + const { store } = makeStore() + store.obj = { n: 1 } + store.obj = { n: 2 } + store.obj.should.eql({ n: 2 }) + store.$obj.history.map((h) => h.value).should.eql([{ n: 1 }]) + }) + + it('snapshots history so later mutation cannot corrupt it', function () { + const { store } = makeStore() + store.obj = { n: 1 } + store.obj = { n: 2 } + store.obj.n = 999 + store.$obj.history.map((h) => h.value).should.eql([{ n: 1 }]) + }) + + it('evicts beyond maxHistory', function () { + const { store } = makeStore({ maxHistory: 2 }) + for (let i = 0; i < 5; i++) store.h = i + store.$h.history.map((h) => h.value).should.eql([2, 3]) + store.h.should.equal(4) + }) + }) + + describe('deep reactivity', function () { + it('fires on a nested object mutation', function () { + const { store, log } = makeStore() + store.robot = { temp: 20 } + log.length = 0 + store.robot.temp = 25 + log.should.eql(['robot.temp']) + store.robot.temp.should.equal(25) + }) + + it('fires on array push, deep item mutation, and clear', function () { + const { store, log } = makeStore() + store.series = [] + log.length = 0 + store.series.push({ x: 1, y: 2 }) + log.should.eql(['series.0']) + log.length = 0 + store.series[0].y = 99 + log.should.eql(['series.0.y']) + log.length = 0 + store.series.length = 0 + log.should.eql(['series']) + store.series.should.eql([]) + }) + }) + + describe('delete', function () { + it('fires onChange and removes the value', function () { + const { store, log } = makeStore() + store.gone = 1 + log.length = 0 + delete store.gone + log.should.eql(['gone']) + should(store.gone).be.undefined() + }) + }) + + describe('guards', function () { + it('blocks prototype pollution and creates no bad keys', function () { + const { store } = makeStore() + store.__proto__ = { polluted: true } // eslint-disable-line no-proto + store.constructor = { polluted: true } + should({}.polluted).be.undefined() + Reflect.ownKeys(store).filter((k) => k === '__proto__' || k === 'constructor').should.eql([]) + }) + + it('reserves the $ prefix so keys are never silently lost', function () { + const { store } = makeStore() + store.$weird = 1 + should(store.$weird).be.undefined() + store.normal = 7 + store.normal.should.equal(7) + store.$normal.value.should.equal(7) + }) + }) + + describe('clone on write', function () { + it('does not alias the caller object into the store', function () { + const { store } = makeStore() + const original = { inner: { x: 1 } } + store.obj = original + original.inner.x = 999 + store.obj.inner.x.should.equal(1) + }) + }) + + describe('timestamps', function () { + it('uses Date.now by default', function () { + const store = createDataStore({}) + store.k = 1 + store.$k.timestamp.should.be.a.Number().and.above(0) + }) + + it('advances on each write', function () { + const { store } = makeStore() + store.k = 1 + const first = store.$k.timestamp + store.k = 2 + store.$k.timestamp.should.be.above(first) + }) + }) + + describe('serialise and restart', function () { + it('flattens entries to values, including nested', function () { + const { store } = makeStore() + store.a = 10 + store.robot = { temp: 20, axes: [1, 2] } + JSON.parse(JSON.stringify(store)).should.eql({ a: 10, robot: { temp: 20, axes: [1, 2] } }) + }) + + it('rebuilds reactivity around a restored snapshot', function () { + const first = makeStore() + first.store.a = 10 + first.store.robot = { temp: 20, axes: [1, 2] } + const snapshot = JSON.parse(JSON.stringify(first.store)) + + const { store, log } = makeStore() + for (const [k, v] of Object.entries(snapshot)) store[k] = v + store.a.should.equal(10) + store.robot.should.eql({ temp: 20, axes: [1, 2] }) + log.length = 0 + store.robot.axes.push(3) + log.should.eql(['robot.axes.2']) + }) + }) + + describe('via Node-RED global context', function () { + function makeContext () { + const log = [] + const ctx = Memory({}) + ctx.set('global', 'dashboard', createDataStore({ onChange: (k, e, p) => log.push(p) })) + return { ctx, log } + } + + it('observes a change-node style path write', function () { + const { ctx, log } = makeContext() + ctx.set('global', 'dashboard.tag1', 123) + log.should.eql(['tag1']) + ctx.get('global', 'dashboard.tag1').should.equal(123) + }) + + it('observes a function-node get-then-mutate', function () { + const { ctx, log } = makeContext() + ctx.set('global', 'dashboard.robot', { temp: 20 }) + log.length = 0 + ctx.get('global', 'dashboard.robot').temp = 25 + log.should.eql(['robot.temp']) + }) + + it('creates a brand-new nested path from a change-node write', function () { + const { ctx, log } = makeContext() + ctx.set('global', 'dashboard.newObj.sub', 5) + ctx.get('global', 'dashboard.newObj').should.eql({ sub: 5 }) + log.should.containEql('newObj.sub') + }) + + it('works through the async callback API', function () { + const { ctx, log } = makeContext() + let cbVal + ctx.set('global', 'dashboard.cbk', 42, () => {}) + ctx.get('global', 'dashboard.cbk', (err, v) => { cbVal = err ? 'ERR' : v }) + cbVal.should.equal(42) + log.length = 0 + ctx.set('global', 'dashboard.cbk', 43, () => {}) + log.should.eql(['cbk']) + }) + + it('loses reactivity if a flow clobbers the namespace (known Story 2 risk)', function () { + const { ctx, log } = makeContext() + ctx.set('global', 'dashboard.k', 1) + ctx.set('global', 'dashboard', { k: 999 }) + log.length = 0 + ctx.set('global', 'dashboard.k', 2) + log.should.eql([]) + ctx.get('global', 'dashboard.k').should.equal(2) + }) + }) + + describe('with RED.util.cloneMessage as the cloner', function () { + function makeCloneStore () { + const log = [] + const store = createDataStore({ clone: util.cloneMessage, onChange: (k, e, p) => log.push(p) }) + return { store, log } + } + + it('reassigning an object value fires only the key, no phantom req/res', function () { + const { store, log } = makeCloneStore() + store.robot = { temp: 20 } + log.length = 0 + store.robot = { temp: 21 } + log.should.eql(['robot']) + }) + + it('keeps a Buffer value intact and unproxied', function () { + const { store } = makeCloneStore() + store.buf = Buffer.from([1, 2, 3]) + Buffer.isBuffer(store.buf).should.equal(true) + store.buf.should.eql(Buffer.from([1, 2, 3])) + }) + + it('keeps a Date value intact', function () { + const { store } = makeCloneStore() + const d = new Date('2020-01-01T00:00:00Z') + store.d = d + store.d.should.be.instanceof(Date) + store.d.getTime().should.equal(d.getTime()) + }) + }) + + describe('strict-mode guards', function () { + it('do not throw when a strict caller writes a reserved key', function () { + const { store } = makeStore() + const write = function () { 'use strict'; store.$x = 1; store.__proto__ = {} } // eslint-disable-line no-proto + write.should.not.throw() + should(store.$x).be.undefined() + }) + }) + + describe('attachToContext', function () { + function fakeGlobal () { + const m = {} + return { get: (k) => m[k], set: (k, v) => { m[k] = v } } + } + + it('injects the store under the dashboard namespace', function () { + const g = fakeGlobal() + const store = attachToContext(g, { onChange: () => {} }) + should(g.get('dashboard')).equal(store) + store.a = 1 + g.get('dashboard').a.should.equal(1) + }) + + it('is idempotent, reusing the store across calls', function () { + const g = fakeGlobal() + const first = attachToContext(g) + const second = attachToContext(g) + should(second).equal(first) + }) + + it('rehydrates around existing plain data and rewires reactivity', function () { + const g = fakeGlobal() + g.set('dashboard', { a: 5, robot: { temp: 1 } }) + const log = [] + const store = attachToContext(g, { onChange: (k, e, p) => log.push(p) }) + store.a.should.equal(5) + store.robot.should.eql({ temp: 1 }) + store.a = 6 + log.should.containEql('a') + }) + }) +}) From 647ef4043645233720629d3d2f1286b44a6b8daa Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 15 Sep 2026 11:20:54 -0700 Subject: [PATCH 02/57] Fix clone-on-write for nested assignments and add ui-base store injection tests --- nodes/store/reactive.js | 8 +-- test/nodes/config-nodes/ui_base.spec.js | 78 +++++++++++++++++++++++++ test/store/reactive.spec.js | 18 ++++++ 3 files changed, 100 insertions(+), 4 deletions(-) create mode 100644 test/nodes/config-nodes/ui_base.spec.js diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index f62725c9d..6a3056f42 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -29,16 +29,16 @@ function deepClone (v) { return out } -function deepReactive (value, notify, path) { +function deepReactive (value, notify, path, clone) { if (!isReactable(value)) return value - for (const k of Object.keys(value)) value[k] = deepReactive(value[k], notify, `${path}.${k}`) + for (const k of Object.keys(value)) value[k] = deepReactive(value[k], notify, `${path}.${k}`, clone) return new Proxy(value, { get (t, p, r) { return Reflect.get(t, p, r) }, set (t, p, v) { if (typeof p === 'symbol') return Reflect.set(t, p, v) // a push writes the new index then grows length; only a shrink is a change worth reporting const shrank = p === 'length' && Array.isArray(t) && v < t.length - t[p] = deepReactive(v, notify, `${path}.${String(p)}`) + t[p] = deepReactive(clone(v), notify, `${path}.${String(p)}`, clone) if (p !== 'length') notify(`${path}.${String(p)}`) else if (shrank) notify(path) return true @@ -84,7 +84,7 @@ function createDataStore ({ maxHistory = 20, onChange, clone = deepClone, now = } const notify = (path) => { rec.timestamp = now(); onChange?.(prop, rec, path) } // clone on write so a flow reusing its own object can't mutate stored state - rec.value = deepReactive(cloneValue(value), notify, prop) + rec.value = deepReactive(cloneValue(value), notify, prop, cloneValue) rec.timestamp = now() onChange?.(prop, rec, prop) return true diff --git a/test/nodes/config-nodes/ui_base.spec.js b/test/nodes/config-nodes/ui_base.spec.js new file mode 100644 index 000000000..735d1d4ed --- /dev/null +++ b/test/nodes/config-nodes/ui_base.spec.js @@ -0,0 +1,78 @@ +const helper = require('node-red-node-test-helper') +const should = require('should') // eslint-disable-line no-unused-vars + +const { attachToContext, STORE, NAMESPACE } = require('../../../nodes/store/reactive.js') +const { testData1 } = require('../fixtures/index.js') +const { verifyFlowLoaded } = require('../utils.js') + +const testFlow1 = testData1.flows +const nodeImports = testData1.getImports(null, ['ui_button']) + +helper.init(require.resolve('node-red')) + +describe('ui-base config node: data store injection', function () { + beforeEach(function (done) { + helper.startServer(done) + }) + + afterEach(function (done) { + helper.unload() + helper.stopServer(done) + }) + + const secondBase = { + id: 'config-ui-base-2', + type: 'ui-base', + name: 'UI Name 2', + path: '/ui2', + port: '', + includeClientData: true, + acceptsClientConfig: ['ui-notification', 'ui-control'] + } + + const flow = [ + { id: 'node-ui-button', type: 'ui-button', z: 'tab-id', group: 'config-ui-group' }, + ...testFlow1 + ] + + it('injects the reactive store into global context under the dashboard namespace', async function () { + await helper.load(nodeImports, flow) + verifyFlowLoaded(helper, flow) + const base = helper.getNode('config-ui-base') + const store = base.context().global.get(NAMESPACE) + should(store).be.an.Object() + store[STORE].should.equal(true) + }) + + it('keeps the store reactive through the real context manager, not just the raw memory store', async function () { + await helper.load(nodeImports, flow) + const global = helper.getNode('config-ui-base').context().global + + global.set(`${NAMESPACE}.tag1`, 123) + global.get(`${NAMESPACE}.tag1`).should.equal(123) + global.get(`${NAMESPACE}.$tag1`).value.should.equal(123) + + const store = global.get(NAMESPACE) + store.robot = { temp: 20 } + global.get(`${NAMESPACE}.robot`).temp.should.equal(20) + global.get(`${NAMESPACE}.$robot`).value.temp.should.equal(20) + }) + + it('gives multiple base nodes one shared store, not one each', async function () { + await helper.load(nodeImports, [...flow, secondBase]) + const first = helper.getNode('config-ui-base').context().global.get(NAMESPACE) + const second = helper.getNode('config-ui-base-2').context().global.get(NAMESPACE) + should(first).equal(second) + }) + + it('re-injection reuses the existing store and preserves its values', async function () { + await helper.load(nodeImports, flow) + const global = helper.getNode('config-ui-base').context().global + + const original = global.get(NAMESPACE) + original.reused = 42 + const again = attachToContext(global) + should(again).equal(original) + global.get(`${NAMESPACE}.reused`).should.equal(42) + }) +}) diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 0b1f4bac1..86e5c3434 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -128,6 +128,24 @@ describe('store: reactive data store', function () { original.inner.x = 999 store.obj.inner.x.should.equal(1) }) + + it('does not alias a nested assignment into the store', function () { + const { store } = makeStore() + store.robot = { sensors: {} } + const original = { x: 1 } + store.robot.sensors = original + original.x = 999 + store.robot.sensors.x.should.equal(1) + }) + + it('does not wrap the caller object in place on a nested assignment', function () { + const { store } = makeStore() + store.robot = { sensors: {} } + const original = { inner: { x: 1 } } + const innerRef = original.inner + store.robot.sensors = original + should(original.inner).equal(innerRef) + }) }) describe('timestamps', function () { From 0cf6d39c0a806de70e68792073ae79c1721eb4b1 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 15 Sep 2026 12:19:31 -0700 Subject: [PATCH 03/57] Drop spike artifact from store PR --- dashboard-docs/spike/combined-store.js | 236 ------------------------- 1 file changed, 236 deletions(-) delete mode 100644 dashboard-docs/spike/combined-store.js diff --git a/dashboard-docs/spike/combined-store.js b/dashboard-docs/spike/combined-store.js deleted file mode 100644 index 97669e067..000000000 --- a/dashboard-docs/spike/combined-store.js +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Reactive data store POC: an Entry (value + metadata) at each top-level key, - * with a deep-reactive .value, injected into a real Node-RED memory context. - * A write fires onChange(key, entry, path). - * Run from the dashboard repo or a Node-RED install: node combined-store.js - * - * Tests: value/$meta reads, history, nested + array mutation (incl. truncation), - * top-level delete, '$'-key reservation, prototype guards, maxHistory eviction, - * callback API, namespace clobber, and serialise + restart with reactivity. - */ - -const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') - -let tick = 0 -const clock = () => tick++ // deterministic for the spike; Date.now() in production -const snapshot = (v) => (v && typeof v === 'object') ? JSON.parse(JSON.stringify(v)) : v - -class Entry { - constructor () { - this.value = undefined - this.quality = 'GOOD' - this.timestamp = 0 - this.history = [] - } -} - -function deepReactive (value, notify, path) { - if (value === null || typeof value !== 'object') return value - for (const k of Object.keys(value)) value[k] = deepReactive(value[k], notify, `${path}.${k}`) - return new Proxy(value, { - get (t, p, r) { return Reflect.get(t, p, r) }, - set (t, p, v) { - if (typeof p === 'symbol') return Reflect.set(t, p, v) - const shrank = p === 'length' && Array.isArray(t) && v < t.length - t[p] = deepReactive(v, notify, `${path}.${String(p)}`) - if (p !== 'length') notify(`${path}.${String(p)}`) // index/prop write - else if (shrank) notify(path) // array truncation (e.g. arr.length = 0); growth from push is ignored - return true - }, - deleteProperty (t, p) { const ok = delete t[p]; notify(`${path}.${String(p)}`); return ok } - }) -} - -function createDataStore ({ maxHistory = 20, onChange } = {}) { - const records = Object.create(null) - const handler = { - get (target, prop, receiver) { - if (typeof prop === 'symbol') return Reflect.get(target, prop, receiver) - if (prop === 'toJSON') { - return () => Object.fromEntries(Object.entries(target).map(([k, r]) => [k, r.value])) - } - if (prop.startsWith('$')) return target[prop.slice(1)] // read-only meta view: $key -> Entry - const rec = target[prop] - return rec ? rec.value : undefined // value by default (deep-reactive if object) - }, - set (target, prop, value) { - if (typeof prop === 'symbol') return Reflect.set(target, prop, value) - if (prop === '__proto__' || prop === 'constructor') return false - if (prop.startsWith('$')) return false // '$' reserved for the meta view; keys can't start with $ - - let rec = target[prop] - if (!rec) { - rec = new Entry() - target[prop] = rec - } else { - rec.history.push({ value: snapshot(rec.value), timestamp: rec.timestamp }) - if (rec.history.length > maxHistory) rec.history.shift() - } - const notify = (path) => { rec.timestamp = clock(); onChange?.(prop, rec, path) } - rec.value = deepReactive(value, notify, prop) - rec.timestamp = clock() - onChange?.(prop, rec, prop) - return true - }, - ownKeys (target) { return Reflect.ownKeys(target) }, - getOwnPropertyDescriptor (target, prop) { - return Object.getOwnPropertyDescriptor(target, prop) ?? { enumerable: true, configurable: true } - }, - deleteProperty (target, prop) { - const existed = prop in target - const ok = delete target[prop] - if (existed) onChange?.(prop, undefined, prop) // notify on top-level delete - return ok - } - } - return new Proxy(records, handler) -} - -// ---- wire into a real Node-RED memory context under global.dashboard ---- -let log = [] -function makeContext (opts = {}) { - log = [] - const ctx = Memory({}) - ctx.set('global', 'dashboard', createDataStore({ ...opts, onChange: (key, entry, path) => log.push(path) })) - return ctx -} -const gset = (ctx, k, v) => ctx.set('global', k, v) -const gget = (ctx, k) => ctx.get('global', k) - -let pass = 0; let fail = 0 -// shape check (JSON equality) — proves stored/serialised shape, NOT liveness. Liveness is proven via `log`. -const check = (n, a, e) => { const ok = JSON.stringify(a) === JSON.stringify(e); console.log(`${ok ? 'PASS' : 'FAIL'} ${n}${ok ? '' : ` (exp ${JSON.stringify(e)}, got ${JSON.stringify(a)})`}`); ok ? pass++ : fail++ } - -console.log('\n--- flat scalar ---') -{ - const ctx = makeContext() - gset(ctx, 'dashboard.k1', 123) - check('scalar set fires onChange at key level', log, ['k1']) - check('value by default', gget(ctx, 'dashboard.k1'), 123) - check('$meta returns the Entry', (() => { const r = gget(ctx, 'dashboard.$k1'); return { value: r.value, quality: r.quality } })(), { value: 123, quality: 'GOOD' }) -} - -console.log('\n--- reassign: history snapshots the prior value (not a live ref) ---') -{ - const ctx = makeContext() - gset(ctx, 'dashboard.obj', { n: 1 }) - gset(ctx, 'dashboard.obj', { n: 2 }) - check('current value', gget(ctx, 'dashboard.obj'), { n: 2 }) - const hist = gget(ctx, 'dashboard.$obj').history - check('history captured prior value', hist.map(h => h.value), [{ n: 1 }]) - // mutate the CURRENT value; history must not change (snapshot, not alias) - gget(ctx, 'dashboard.obj').n = 999 - check('history not corrupted by later mutation', gget(ctx, 'dashboard.$obj').history.map(h => h.value), [{ n: 1 }]) -} - -console.log('\n--- nested object: liveness proven, not just shape ---') -{ - const ctx = makeContext() - gset(ctx, 'dashboard.robot', { temp: 20 }) - log = [] - gget(ctx, 'dashboard.robot').temp = 25 // direct nested mutation on the returned value - check('LIVENESS: nested mutation on returned value fires', log, ['robot.temp']) - check('nested value updated', gget(ctx, 'dashboard.robot').temp, 25) - gset(ctx, 'dashboard.robot.speed', 5) // nested set via context path (change-node style) - check('value-by-default still returns the whole object', gget(ctx, 'dashboard.robot'), { temp: 25, speed: 5 }) -} - -console.log('\n--- array: push, deep mutate, and CLEAR (length=0) ---') -{ - const ctx = makeContext() - gset(ctx, 'dashboard.series', []) - log = [] - gget(ctx, 'dashboard.series').push({ x: 1, y: 2 }) - check('push fires once (length growth ignored)', log, ['series.0']) - log = [] - gget(ctx, 'dashboard.series')[0].y = 99 - check('deep mutation inside array fires', log, ['series.0.y']) - log = [] - gget(ctx, 'dashboard.series').length = 0 // truncation — previously invisible - check('array clear via length=0 fires', log, ['series']) - check('array emptied', gget(ctx, 'dashboard.series'), []) -} - -console.log('\n--- top-level delete notifies ---') -{ - const ctx = makeContext() - gset(ctx, 'dashboard.gone', 1) - log = [] - gset(ctx, 'dashboard.gone', undefined) // change-node style: setObjectProperty deletes on undefined - check('delete fires onChange', log, ['gone']) - check('value removed', gget(ctx, 'dashboard.gone'), undefined) -} - -console.log('\n--- guards: no prototype pollution AND no own-key created ---') -{ - const ctx = makeContext() - const store = gget(ctx, 'dashboard') - store.__proto__ = { polluted: true } // eslint-disable-line no-proto - store.constructor = { polluted: true } - check('Object.prototype not polluted', ({}).polluted, undefined) - check('no __proto__/constructor own-keys created', Reflect.ownKeys(store).filter(k => k === '__proto__' || k === 'constructor'), []) -} - -console.log('\n--- $ prefix reserved (no silent data loss) ---') -{ - const ctx = makeContext() - gset(ctx, 'dashboard.$weird', 1) // reserved: set is a no-op - check('set of $-key is a no-op, not stored-then-lost', gget(ctx, 'dashboard.$weird'), undefined) - gset(ctx, 'dashboard.normal', 7) - check('normal keys unaffected', gget(ctx, 'dashboard.normal'), 7) - check('$normal still reads the Entry', gget(ctx, 'dashboard.$normal').value, 7) -} - -console.log('\n--- maxHistory eviction ---') -{ - const ctx = makeContext({ maxHistory: 2 }) - for (let i = 0; i < 5; i++) gset(ctx, 'dashboard.h', i) - check('history capped at maxHistory', gget(ctx, 'dashboard.$h').history.map(x => x.value), [2, 3]) - check('current value is latest', gget(ctx, 'dashboard.h'), 4) -} - -console.log('\n--- Node-RED async callback API works + reactivity intact ---') -{ - const ctx = makeContext() - let cbVal - ctx.set('global', 'dashboard.cbk', 42, () => {}) // set with callback - ctx.get('global', 'dashboard.cbk', (err, v) => { cbVal = err ? 'ERR' : v }) // get with callback - check('callback get returns value', cbVal, 42) - log = [] - gget(ctx, 'dashboard.cbk') // no-op read - ctx.set('global', 'dashboard.cbk', 43, () => {}) - check('write via callback still fires onChange', log, ['cbk']) -} - -console.log('\n--- namespace clobber (documents Story 2 risk) ---') -{ - const ctx = makeContext() - gset(ctx, 'dashboard.k', 1) - gset(ctx, 'dashboard', { k: 999 }) // a flow overwrites the whole namespace - log = [] - gset(ctx, 'dashboard.k', 2) - check('after clobber, writes are NOT observed (proxy replaced)', log, []) - check('namespace is now a plain object', gget(ctx, 'dashboard.k'), 2) -} - -console.log('\n--- serialise + restart, with nesting + deep reactivity rewired ---') -{ - const ctx = makeContext() - gset(ctx, 'dashboard.a', 10) - gset(ctx, 'dashboard.robot', { temp: 20, axes: [1, 2] }) - const snap = JSON.parse(JSON.stringify(gget(ctx, 'dashboard'))) - check('toJSON flattens Entries -> values (nested preserved)', snap, { a: 10, robot: { temp: 20, axes: [1, 2] } }) - - const ctx2 = Memory({}) - const store2 = createDataStore({ onChange: (k, e, p) => log.push(p) }) - for (const [k, v] of Object.entries(snap)) store2[k] = v // rehydrate through set - ctx2.set('global', 'dashboard', store2) - check('scalar survived restart', ctx2.get('global', 'dashboard.a'), 10) - check('nested survived restart', ctx2.get('global', 'dashboard.robot'), { temp: 20, axes: [1, 2] }) - log = [] - ctx2.get('global', 'dashboard.robot').axes.push(3) // deep reactivity, post-restart - check('LIVENESS: deep reactivity works after restart', log, ['robot.axes.2']) -} - -console.log(`\n===== ${pass} passed, ${fail} failed =====`) -process.exit(fail ? 1 : 0) From 1a86cce4206ab4452c140af95cc7b228ac094666 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 15 Sep 2026 12:40:06 -0700 Subject: [PATCH 04/57] Add warning for uncached global context store --- nodes/config/ui_base.js | 8 ++++++-- nodes/store/reactive.js | 12 +++++++++++- test/store/reactive.spec.js | 18 +++++++++++++++++- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index cd0dfc39c..c9e9befc5 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -6,7 +6,7 @@ const axios = require('axios') const v = require('../../package.json').version const { createClientStore } = require('../store/clients.js') const datastore = require('../store/data.js') -const { attachToContext } = require('../store/reactive.js') +const { attachToContext, isInMemoryBacked } = require('../store/reactive.js') const statestore = require('../store/state.js') const { appendTopic, addConnectionCredentials, normalizeClientId, getThirdPartyWidgets } = require('../utils/index.js') @@ -403,7 +403,11 @@ module.exports = function (RED) { node._created = Date.now() - attachToContext(node.context().global, { clone: RED.util.cloneMessage }) + if (isInMemoryBacked(node.context().global)) { + attachToContext(node.context().global, { clone: RED.util.cloneMessage }) + } else { + node.warn('Dashboard data store disabled: the global context store has caching turned off (cache: false), which only supports asynchronous access. Use an in-memory-backed context store for reactive dashboard state.') + } n.root = RED.settings.httpNodeRoot || '/' diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 6a3056f42..fdb16341c 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -100,6 +100,16 @@ function createDataStore ({ maxHistory = 20, onChange, clone = deepClone, now = return new Proxy(records, handler) } +// A cache-off persistent store serves only async access, so a sync get throws; the reactive proxy needs an in-memory store. +function isInMemoryBacked (globalContext, namespace = NAMESPACE) { + try { + globalContext.get(namespace) + return true + } catch (err) { + return false + } +} + // Inject the store into global context once. On redeploy the existing store is reused; // after a restart with a persistent context store, existing plain data is rehydrated. function attachToContext (globalContext, opts = {}) { @@ -115,4 +125,4 @@ function attachToContext (globalContext, opts = {}) { return store } -module.exports = { createDataStore, attachToContext, NAMESPACE, STORE } +module.exports = { createDataStore, attachToContext, isInMemoryBacked, NAMESPACE, STORE } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 86e5c3434..8ea022a35 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -2,7 +2,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, isInMemoryBacked } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -312,4 +312,20 @@ describe('store: reactive data store', function () { log.should.containEql('a') }) }) + + describe('isInMemoryBacked', function () { + it('is true for an in-memory context whose get returns synchronously', function () { + const m = {} + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + isInMemoryBacked(g).should.equal(true) + }) + + it('is false when a cache-off store throws on synchronous get', function () { + const g = { + get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, + set: () => {} + } + isInMemoryBacked(g).should.equal(false) + }) + }) }) From 075fa317ab1a896c8718ff7a504e6b6ede2e67d0 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 15 Sep 2026 12:54:59 -0700 Subject: [PATCH 05/57] Add additional specs + update warning --- nodes/config/ui_base.js | 2 +- test/store/reactive.spec.js | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index c9e9befc5..c0061f439 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -406,7 +406,7 @@ module.exports = function (RED) { if (isInMemoryBacked(node.context().global)) { attachToContext(node.context().global, { clone: RED.util.cloneMessage }) } else { - node.warn('Dashboard data store disabled: the global context store has caching turned off (cache: false), which only supports asynchronous access. Use an in-memory-backed context store for reactive dashboard state.') + node.warn('Dashboard data store disabled: the global context store does not support synchronous in-memory access (commonly a persistent store with cache: false). Reactive dashboard state needs an in-memory-backed context store.') } n.root = RED.settings.httpNodeRoot || '/' diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 8ea022a35..02ba7743a 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -1,3 +1,8 @@ +const fs = require('fs') +const os = require('os') +const path = require('path') + +const LocalFileSystem = require('@node-red/runtime/lib/nodes/context/localfilesystem.js') const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars @@ -327,5 +332,25 @@ describe('store: reactive data store', function () { } isInMemoryBacked(g).should.equal(false) }) + + // a real localfilesystem store, accessed the way the context manager's sync path does (store.get(scope, key)) + function realGlobal (store) { + return { get: (key) => store.get('global', key) } + } + function tmpDir () { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-ctx-')) + after(() => fs.rmSync(dir, { recursive: true, force: true })) + return dir + } + + it('is false for a real cache-off localfilesystem store', function () { + const store = LocalFileSystem({ dir: tmpDir(), cache: false }) + isInMemoryBacked(realGlobal(store)).should.equal(false) + }) + + it('is true for a real cache-backed localfilesystem store', function () { + const store = LocalFileSystem({ dir: tmpDir(), cache: true }) + isInMemoryBacked(realGlobal(store)).should.equal(true) + }) }) }) From c63b81eb43cdd4fef0439065916405bec7e3df1f Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 15 Sep 2026 13:07:19 -0700 Subject: [PATCH 06/57] Update warn text --- nodes/config/ui_base.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index c0061f439..d637466aa 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -406,7 +406,7 @@ module.exports = function (RED) { if (isInMemoryBacked(node.context().global)) { attachToContext(node.context().global, { clone: RED.util.cloneMessage }) } else { - node.warn('Dashboard data store disabled: the global context store does not support synchronous in-memory access (commonly a persistent store with cache: false). Reactive dashboard state needs an in-memory-backed context store.') + node.warn('Dashboard data store disabled: the global context store isn\'t in-memory-backed (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.') } n.root = RED.settings.httpNodeRoot || '/' From c2ee207fe48e8668d0aa51a590c9d653d6f984a9 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 16 Sep 2026 06:34:35 -0700 Subject: [PATCH 07/57] Add onOverwrite protection for users that replace the store --- nodes/config/ui_base.js | 5 ++++- nodes/store/reactive.js | 2 ++ test/store/reactive.spec.js | 28 ++++++++++++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index d637466aa..faf0aa736 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -404,7 +404,10 @@ module.exports = function (RED) { node._created = Date.now() if (isInMemoryBacked(node.context().global)) { - attachToContext(node.context().global, { clone: RED.util.cloneMessage }) + attachToContext(node.context().global, { + clone: RED.util.cloneMessage, + onOverwrite: () => node.warn('Dashboard data store was overwritten in global context by a flow; re-injecting it. Write keys as global.dashboard. rather than replacing global.dashboard itself.') + }) } else { node.warn('Dashboard data store disabled: the global context store isn\'t in-memory-backed (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.') } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index fdb16341c..4da298f2a 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -117,6 +117,8 @@ function attachToContext (globalContext, opts = {}) { const existing = globalContext.get(namespace) if (existing && existing[STORE]) return existing + if (existing && typeof existing === 'object') opts.onOverwrite?.() + const store = createDataStore(opts) if (existing && typeof existing === 'object') { for (const [k, v] of Object.entries(existing)) store[k] = v diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 02ba7743a..a32a22727 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -7,7 +7,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext, isInMemoryBacked } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, isInMemoryBacked, STORE } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -234,7 +234,7 @@ describe('store: reactive data store', function () { log.should.eql(['cbk']) }) - it('loses reactivity if a flow clobbers the namespace (known Story 2 risk)', function () { + it('a raw overwrite drops reactivity until Dashboard re-injects (see attachToContext)', function () { const { ctx, log } = makeContext() ctx.set('global', 'dashboard.k', 1) ctx.set('global', 'dashboard', { k: 999 }) @@ -316,6 +316,30 @@ describe('store: reactive data store', function () { store.a = 6 log.should.containEql('a') }) + + it('reinjects and warns when a flow has overwritten the namespace, preserving data', function () { + const g = fakeGlobal() + const store = attachToContext(g) + store.k = 1 + g.set('dashboard', { k: 1, extra: 2 }) // a flow replaces the proxy with a plain object + + let warned = 0 + const restored = attachToContext(g, { onOverwrite: () => { warned++ } }) + warned.should.equal(1) + restored[STORE].should.equal(true) + should(g.get('dashboard')).equal(restored) + restored.extra.should.equal(2) + should(attachToContext(g)).equal(restored) // idempotent again, no re-overwrite + restored.k = 5 + restored.k.should.equal(5) + }) + + it('does not warn on a clean first injection', function () { + const g = fakeGlobal() + let warned = 0 + attachToContext(g, { onOverwrite: () => { warned++ } }) + warned.should.equal(0) + }) }) describe('isInMemoryBacked', function () { From fbec5e5794f11ec3aa9e3aa132d0b0e7e340fbfa Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 16 Sep 2026 07:59:17 -0700 Subject: [PATCH 08/57] Add fix for clearing keys from store & add write for simple properties --- nodes/config/ui_base.js | 1 + nodes/store/data.js | 26 +++++++- test/nodes/config-nodes/ui_base.spec.js | 29 ++++++++- test/store/data.spec.js | 84 +++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 test/store/data.spec.js diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index faf0aa736..146e844da 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -1174,6 +1174,7 @@ module.exports = function (RED) { // widget has been removed from the Editor // clear any data from datastore datastore.clear(widgetNode.id) + datastore.clearFromStore(widgetNode) } node.deregister(null, null, widgetNode) done() diff --git a/nodes/store/data.js b/nodes/store/data.js index 8404b1502..91defa7d8 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -1,9 +1,31 @@ +const { attachToContext } = require('./reactive.js') + const data = {} const config = { RED: null } +function projectValue (node, msg) { + return msg.payload +} + +function getStore (node) { + return attachToContext(node.context().global, { clone: config.RED.util.cloneMessage }) +} + +function writeToStore (node, msg) { + try { + getStore(node)[node.id] = projectValue(node, msg) + } catch (err) {} +} + +function clearFromStore (node) { + try { + delete getStore(node)[node.id] + } catch (err) {} +} + /** * Checks if a Client/Socket ID has been assigned to this message, * and whether the node type is being scoped to a specific client. @@ -90,6 +112,7 @@ const setters = { ...data[node.id], ...newMsg } + writeToStore(node, msg) } } }, @@ -128,5 +151,6 @@ module.exports = { save: setters.save, append: setters.append, filter: setters.filter, - clear: setters.clear + clear: setters.clear, + clearFromStore } diff --git a/test/nodes/config-nodes/ui_base.spec.js b/test/nodes/config-nodes/ui_base.spec.js index 735d1d4ed..d5f777719 100644 --- a/test/nodes/config-nodes/ui_base.spec.js +++ b/test/nodes/config-nodes/ui_base.spec.js @@ -6,7 +6,21 @@ const { testData1 } = require('../fixtures/index.js') const { verifyFlowLoaded } = require('../utils.js') const testFlow1 = testData1.flows -const nodeImports = testData1.getImports(null, ['ui_button']) +const nodeImports = testData1.getImports(null, ['ui_button', 'ui_text']) + +function waitFor (fn, timeout = 1000) { + return new Promise((resolve, reject) => { + const start = Date.now() + const tick = () => { + let v + try { v = fn() } catch (e) {} + if (v !== undefined) { return resolve(v) } + if (Date.now() - start > timeout) { return reject(new Error('timeout waiting for store value')) } + setTimeout(tick, 5) + } + tick() + }) +} helper.init(require.resolve('node-red')) @@ -32,6 +46,7 @@ describe('ui-base config node: data store injection', function () { const flow = [ { id: 'node-ui-button', type: 'ui-button', z: 'tab-id', group: 'config-ui-group' }, + { id: 'node-ui-text', type: 'ui-text', z: 'tab-id', group: 'config-ui-group', label: 'txt' }, ...testFlow1 ] @@ -75,4 +90,16 @@ describe('ui-base config node: data store injection', function () { should(again).equal(original) global.get(`${NAMESPACE}.reused`).should.equal(42) }) + + it('populates the store with a widget\'s clean payload on input', async function () { + await helper.load(nodeImports, flow) + const text = helper.getNode('node-ui-text') + const global = helper.getNode('config-ui-base').context().global + + text.receive({ payload: 'hello', topic: 't', _msgid: 'm' }) + + const value = await waitFor(() => global.get(NAMESPACE)['node-ui-text']) + value.should.equal('hello') // the clean payload, not the whole message + global.get(`${NAMESPACE}.$node-ui-text`).value.should.equal('hello') + }) }) diff --git a/test/store/data.spec.js b/test/store/data.spec.js new file mode 100644 index 000000000..b841e26c9 --- /dev/null +++ b/test/store/data.spec.js @@ -0,0 +1,84 @@ +const { util } = require('@node-red/util') +const should = require('should') // eslint-disable-line no-unused-vars + +const datastore = require('../../nodes/store/data.js') + +const RED = { util, plugins: { getByType: () => [] } } +datastore.setConfig(RED) + +function fakeGlobal () { + const m = {} + return { get: (k) => m[k], set: (k, v) => { m[k] = v } } +} + +function fakeNode (id, global) { + return { id, type: 'ui-text', context: () => ({ global }) } +} + +const base = { acceptsClientConfig: [] } + +describe('store: data.js reactive-store mirror', function () { + it('mirrors the clean payload into global.dashboard[id], not the whole message', function () { + const global = fakeGlobal() + const node = fakeNode('w1', global) + datastore.save(base, node, { payload: 42, topic: 't', _msgid: 'm', extra: 'x' }) + + const store = global.get('dashboard') + store.w1.should.equal(42) + // the reactive store holds only the value; the whole message stays in the legacy datastore + datastore.get('w1').payload.should.equal(42) + datastore.get('w1').topic.should.equal('t') + }) + + it('keeps the mirrored value in sync on the latest write', function () { + const global = fakeGlobal() + const node = fakeNode('w2', global) + datastore.save(base, node, { payload: 1 }) + datastore.save(base, node, { payload: 2 }) + global.get('dashboard').w2.should.equal(2) + }) + + it('does not mirror a client-scoped message (not stored centrally)', function () { + const global = fakeGlobal() + const node = fakeNode('w3', global) + const scopedBase = { acceptsClientConfig: ['ui-text'] } + datastore.save(scopedBase, node, { payload: 9, _client: { socketId: 's1' } }) + should(global.get('dashboard')).be.undefined() + }) + + it('does not mirror array (chart-style) saves', function () { + const global = fakeGlobal() + const node = fakeNode('w4', global) + datastore.save(base, node, [{ payload: 1 }, { payload: 2 }]) + should(global.get('dashboard')).be.undefined() + }) + + it('does not throw or mirror when the context store cannot hold the proxy', function () { + const throwingGlobal = { + get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, + set: () => {} + } + const node = fakeNode('w5', throwingGlobal) + should(() => datastore.save(base, node, { payload: 7 })).not.throw() + datastore.get('w5').payload.should.equal(7) // legacy datastore still works + }) + + it('clearFromStore removes the widget key so a removed widget leaves no orphan', function () { + const global = fakeGlobal() + const node = fakeNode('w6', global) + datastore.save(base, node, { payload: 5 }) + global.get('dashboard').w6.should.equal(5) + + datastore.clearFromStore(node) + should(global.get('dashboard').w6).be.undefined() + }) + + it('clearFromStore does not throw when the store is unavailable', function () { + const throwingGlobal = { + get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, + set: () => {} + } + const node = fakeNode('w7', throwingGlobal) + should(() => datastore.clearFromStore(node)).not.throw() + }) +}) From d0ed0e98f6ab0afeefc5b30a2938d14832fae2db Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 16 Sep 2026 09:50:11 -0700 Subject: [PATCH 09/57] Add array support, reduce max history, wrap try catch --- nodes/config/ui_base.js | 12 +++-- nodes/store/data.js | 36 +++++++++++++ nodes/store/reactive.js | 5 +- test/store/data.spec.js | 103 +++++++++++++++++++++++++++++++++++- test/store/reactive.spec.js | 21 ++++++++ 5 files changed, 169 insertions(+), 8 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index 146e844da..b8bff3757 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -404,10 +404,14 @@ module.exports = function (RED) { node._created = Date.now() if (isInMemoryBacked(node.context().global)) { - attachToContext(node.context().global, { - clone: RED.util.cloneMessage, - onOverwrite: () => node.warn('Dashboard data store was overwritten in global context by a flow; re-injecting it. Write keys as global.dashboard. rather than replacing global.dashboard itself.') - }) + try { + attachToContext(node.context().global, { + clone: RED.util.cloneMessage, + onOverwrite: () => node.warn('Dashboard data store was overwritten in global context by a flow; re-injecting it. Write keys as global.dashboard. rather than replacing global.dashboard itself.') + }) + } catch (err) { + node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') + } } else { node.warn('Dashboard data store disabled: the global context store isn\'t in-memory-backed (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.') } diff --git a/nodes/store/data.js b/nodes/store/data.js index 91defa7d8..f8b8ddd43 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -20,6 +20,39 @@ function writeToStore (node, msg) { } catch (err) {} } +function pointsFromMessages (messages) { + const points = [] + if (!Array.isArray(messages)) return points + for (const message of messages) { + const datapoint = message && message._datapoint + if (datapoint == null) { + if (message && message.payload !== undefined) points.push(message.payload) + } else if (Array.isArray(datapoint)) { + points.push(...datapoint) + } else { + points.push(datapoint) + } + } + return points +} + +function writeArrayToStore (node) { + try { + getStore(node)[node.id] = pointsFromMessages(data[node.id]) + } catch (err) {} +} + +function appendToStore (node, message) { + try { + const store = getStore(node) + if (Array.isArray(store[node.id])) { + for (const point of pointsFromMessages([message])) store[node.id].push(point) + } else { + store[node.id] = pointsFromMessages(data[node.id]) + } + } catch (err) {} +} + function clearFromStore (node) { try { delete getStore(node)[node.id] @@ -105,6 +138,7 @@ const setters = { } } data[node.id] = filtered + writeArrayToStore(node) } else { if (canSaveInStore(base, node, msg)) { const newMsg = stripMsg(msg) @@ -124,6 +158,7 @@ const setters = { data[node.id] = [] } data[node.id].push(config.RED.util.cloneMessage(msg)) + appendToStore(node, msg) } }, /** @@ -139,6 +174,7 @@ const setters = { if (filteredMessages.length !== currentData.length) { // no need for save operation to process messages - just apply them data[node.id] = filteredMessages + writeArrayToStore(node) } } } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 4da298f2a..88ed25e13 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -52,7 +52,7 @@ function deepReactive (value, notify, path, clone) { }) } -function createDataStore ({ maxHistory = 20, onChange, clone = deepClone, now = Date.now } = {}) { +function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = Date.now } = {}) { const cloneValue = (v) => (v && typeof v === 'object') ? clone(v) : v const records = Object.create(null) @@ -78,7 +78,8 @@ function createDataStore ({ maxHistory = 20, onChange, clone = deepClone, now = if (!rec) { rec = new Entry() target[prop] = rec - } else { + } else if (!Array.isArray(rec.value)) { + // array-valued keys (e.g. a chart series) are their own history; snapshotting them would blow up memory rec.history.push({ value: cloneValue(rec.value), timestamp: rec.timestamp }) if (rec.history.length > maxHistory) rec.history.shift() } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index b841e26c9..5fa1a4757 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -46,11 +46,11 @@ describe('store: data.js reactive-store mirror', function () { should(global.get('dashboard')).be.undefined() }) - it('does not mirror array (chart-style) saves', function () { + it('mirrors array (chart-style) saves as a series', function () { const global = fakeGlobal() const node = fakeNode('w4', global) datastore.save(base, node, [{ payload: 1 }, { payload: 2 }]) - should(global.get('dashboard')).be.undefined() + global.get('dashboard').w4.should.eql([1, 2]) }) it('does not throw or mirror when the context store cannot hold the proxy', function () { @@ -82,3 +82,102 @@ describe('store: data.js reactive-store mirror', function () { should(() => datastore.clearFromStore(node)).not.throw() }) }) + +describe('store: data.js chart array mirror', function () { + const pt = (x, y, category = 'a') => ({ category, x, y }) + + it('mirrors chart points (_datapoint) into the store on append', function () { + const global = fakeGlobal() + const node = fakeNode('c1', global) + datastore.append(base, node, { payload: 2, _datapoint: pt(1, 2) }) + datastore.append(base, node, { payload: 3, _datapoint: pt(2, 3) }) + global.get('dashboard').c1.should.eql([pt(1, 2), pt(2, 3)]) + }) + + it('pushes each point when _datapoint is an array (multi-series)', function () { + const global = fakeGlobal() + const node = fakeNode('c2', global) + datastore.append(base, node, { _datapoint: [pt(1, 2, 'a'), pt(1, 5, 'b')] }) + global.get('dashboard').c2.should.eql([pt(1, 2, 'a'), pt(1, 5, 'b')]) + }) + + it('replaces the store series on an array save (categorical/replace)', function () { + const global = fakeGlobal() + const node = fakeNode('c3', global) + datastore.append(base, node, { _datapoint: pt(1, 2) }) + datastore.save(base, node, [{ _datapoint: pt(9, 9) }]) + global.get('dashboard').c3.should.eql([pt(9, 9)]) + }) + + it('clears the store series on save([])', function () { + const global = fakeGlobal() + const node = fakeNode('c4', global) + datastore.append(base, node, { _datapoint: pt(1, 2) }) + datastore.save(base, node, []) + global.get('dashboard').c4.should.eql([]) + }) + + it('mirrors a trim (filter) into the store', function () { + const global = fakeGlobal() + const node = fakeNode('c5', global) + datastore.append(base, node, { _datapoint: pt(1, 1) }) + datastore.append(base, node, { _datapoint: pt(2, 2) }) + datastore.append(base, node, { _datapoint: pt(3, 3) }) + datastore.filter(base, node, (m, i) => i > 0) // drop the oldest + global.get('dashboard').c5.should.eql([pt(2, 2), pt(3, 3)]) + }) + + it('keeps no history as the series grows', function () { + const global = fakeGlobal() + const node = fakeNode('c6', global) + datastore.append(base, node, { _datapoint: pt(1, 1) }) + datastore.save(base, node, []) // reassigns the array + datastore.append(base, node, { _datapoint: pt(2, 2) }) + global.get('dashboard').$c6.history.should.eql([]) + }) + + it('falls back to payload when _datapoint is null (multi-series without _datapoint)', function () { + const global = fakeGlobal() + const node = fakeNode('c7', global) + datastore.append(base, node, { payload: 42, _datapoint: null }) + global.get('dashboard').c7.should.eql([42]) + }) + + it('adds no point for an empty _datapoint array', function () { + const global = fakeGlobal() + const node = fakeNode('c8', global) + datastore.append(base, node, { _datapoint: [] }) + global.get('dashboard').c8.should.eql([]) + }) + + it('preserves falsy payloads (0/false) and skips a msg with neither field', function () { + const global = fakeGlobal() + const node = fakeNode('c9', global) + datastore.append(base, node, { payload: 0 }) + datastore.append(base, node, { payload: false }) + datastore.append(base, node, { topic: 'no value here' }) + global.get('dashboard').c9.should.eql([0, false]) + }) + + it('does not mirror a client-scoped chart append', function () { + const global = fakeGlobal() + const node = fakeNode('c10', global) + const scopedBase = { acceptsClientConfig: ['ui-text'] } + datastore.append(scopedBase, node, { _datapoint: pt(1, 2), _client: { socketId: 's1' } }) + should(global.get('dashboard')).be.undefined() + }) + + it('stays in sync with a capped streaming chart (append + trim interleaved)', function () { + const global = fakeGlobal() + const node = fakeNode('c-stream', global) + const CAP = 3 + const keepLast = (message, index, arr) => index >= arr.length - CAP + for (let i = 1; i <= 6; i++) { + datastore.append(base, node, { _datapoint: pt(i, i) }) + datastore.filter(base, node, keepLast) + } + // in-place appends interleaved with rebuild-trims must leave the store == the trimmed series + global.get('dashboard')['c-stream'].should.eql([pt(4, 4), pt(5, 5), pt(6, 6)]) + datastore.get('c-stream').map((m) => m._datapoint).should.eql([pt(4, 4), pt(5, 5), pt(6, 6)]) + }) +}) diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index a32a22727..700c89b7f 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -67,6 +67,27 @@ describe('store: reactive data store', function () { store.$h.history.map((h) => h.value).should.eql([2, 3]) store.h.should.equal(4) }) + + it('keeps no history for array-valued keys (a chart series is its own history)', function () { + const { store } = makeStore() + store.series = [1, 2] + store.series = [3, 4, 5] + store.$series.history.should.eql([]) + }) + + it('does not snapshot the old array when it is replaced (even by a scalar)', function () { + const { store } = makeStore() + store.k = [1, 2, 3] + store.k = 9 + store.$k.history.should.eql([]) + }) + + it('does snapshot a scalar when it becomes an array', function () { + const { store } = makeStore() + store.k = 5 + store.k = [1, 2] + store.$k.history.map((h) => h.value).should.eql([5]) + }) }) describe('deep reactivity', function () { From d39c4a8b0e5be9f1f76a02491f8a2e84624c3b88 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 16 Sep 2026 10:12:17 -0700 Subject: [PATCH 10/57] Reduce maxHistory, update name, wrap attachToContext in a try catch --- nodes/config/ui_base.js | 6 ++++- nodes/store/reactive.js | 4 +-- test/nodes/config-nodes/ui_base.spec.js | 2 +- test/store/reactive.spec.js | 34 ++++++++++++------------- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index cd0dfc39c..0d265616b 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -403,7 +403,11 @@ module.exports = function (RED) { node._created = Date.now() - attachToContext(node.context().global, { clone: RED.util.cloneMessage }) + try { + attachToContext(node.context().global, { clone: RED.util.cloneMessage }) + } catch (err) { + node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') + } n.root = RED.settings.httpNodeRoot || '/' diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 6a3056f42..2291aeca0 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -1,5 +1,5 @@ const STORE = Symbol('dashboardDataStore') -const NAMESPACE = 'dashboard' +const NAMESPACE = 'dashboardStore' class Entry { constructor () { @@ -52,7 +52,7 @@ function deepReactive (value, notify, path, clone) { }) } -function createDataStore ({ maxHistory = 20, onChange, clone = deepClone, now = Date.now } = {}) { +function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = Date.now } = {}) { const cloneValue = (v) => (v && typeof v === 'object') ? clone(v) : v const records = Object.create(null) diff --git a/test/nodes/config-nodes/ui_base.spec.js b/test/nodes/config-nodes/ui_base.spec.js index 735d1d4ed..f8b32135f 100644 --- a/test/nodes/config-nodes/ui_base.spec.js +++ b/test/nodes/config-nodes/ui_base.spec.js @@ -35,7 +35,7 @@ describe('ui-base config node: data store injection', function () { ...testFlow1 ] - it('injects the reactive store into global context under the dashboard namespace', async function () { + it('injects the reactive store into global context under the dashboardStore namespace', async function () { await helper.load(nodeImports, flow) verifyFlowLoaded(helper, flow) const base = helper.getNode('config-ui-base') diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 86e5c3434..550b972c4 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -192,51 +192,51 @@ describe('store: reactive data store', function () { function makeContext () { const log = [] const ctx = Memory({}) - ctx.set('global', 'dashboard', createDataStore({ onChange: (k, e, p) => log.push(p) })) + ctx.set('global', 'dashboardStore', createDataStore({ onChange: (k, e, p) => log.push(p) })) return { ctx, log } } it('observes a change-node style path write', function () { const { ctx, log } = makeContext() - ctx.set('global', 'dashboard.tag1', 123) + ctx.set('global', 'dashboardStore.tag1', 123) log.should.eql(['tag1']) - ctx.get('global', 'dashboard.tag1').should.equal(123) + ctx.get('global', 'dashboardStore.tag1').should.equal(123) }) it('observes a function-node get-then-mutate', function () { const { ctx, log } = makeContext() - ctx.set('global', 'dashboard.robot', { temp: 20 }) + ctx.set('global', 'dashboardStore.robot', { temp: 20 }) log.length = 0 - ctx.get('global', 'dashboard.robot').temp = 25 + ctx.get('global', 'dashboardStore.robot').temp = 25 log.should.eql(['robot.temp']) }) it('creates a brand-new nested path from a change-node write', function () { const { ctx, log } = makeContext() - ctx.set('global', 'dashboard.newObj.sub', 5) - ctx.get('global', 'dashboard.newObj').should.eql({ sub: 5 }) + ctx.set('global', 'dashboardStore.newObj.sub', 5) + ctx.get('global', 'dashboardStore.newObj').should.eql({ sub: 5 }) log.should.containEql('newObj.sub') }) it('works through the async callback API', function () { const { ctx, log } = makeContext() let cbVal - ctx.set('global', 'dashboard.cbk', 42, () => {}) - ctx.get('global', 'dashboard.cbk', (err, v) => { cbVal = err ? 'ERR' : v }) + ctx.set('global', 'dashboardStore.cbk', 42, () => {}) + ctx.get('global', 'dashboardStore.cbk', (err, v) => { cbVal = err ? 'ERR' : v }) cbVal.should.equal(42) log.length = 0 - ctx.set('global', 'dashboard.cbk', 43, () => {}) + ctx.set('global', 'dashboardStore.cbk', 43, () => {}) log.should.eql(['cbk']) }) it('loses reactivity if a flow clobbers the namespace (known Story 2 risk)', function () { const { ctx, log } = makeContext() - ctx.set('global', 'dashboard.k', 1) - ctx.set('global', 'dashboard', { k: 999 }) + ctx.set('global', 'dashboardStore.k', 1) + ctx.set('global', 'dashboardStore', { k: 999 }) log.length = 0 - ctx.set('global', 'dashboard.k', 2) + ctx.set('global', 'dashboardStore.k', 2) log.should.eql([]) - ctx.get('global', 'dashboard.k').should.equal(2) + ctx.get('global', 'dashboardStore.k').should.equal(2) }) }) @@ -289,9 +289,9 @@ describe('store: reactive data store', function () { it('injects the store under the dashboard namespace', function () { const g = fakeGlobal() const store = attachToContext(g, { onChange: () => {} }) - should(g.get('dashboard')).equal(store) + should(g.get('dashboardStore')).equal(store) store.a = 1 - g.get('dashboard').a.should.equal(1) + g.get('dashboardStore').a.should.equal(1) }) it('is idempotent, reusing the store across calls', function () { @@ -303,7 +303,7 @@ describe('store: reactive data store', function () { it('rehydrates around existing plain data and rewires reactivity', function () { const g = fakeGlobal() - g.set('dashboard', { a: 5, robot: { temp: 1 } }) + g.set('dashboardStore', { a: 5, robot: { temp: 1 } }) const log = [] const store = attachToContext(g, { onChange: (k, e, p) => log.push(p) }) store.a.should.equal(5) From 7baf786c2a3e3c17eb5c90f037b31fe83e91c939 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 16 Sep 2026 11:19:08 -0700 Subject: [PATCH 11/57] Fix issues with disabled stores & lost options --- nodes/config/ui_base.js | 7 +-- nodes/store/data.js | 46 ++++++++++++------ test/store/data.spec.js | 100 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+), 16 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index a6b152511..b496eec7b 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -6,7 +6,7 @@ const axios = require('axios') const v = require('../../package.json').version const { createClientStore } = require('../store/clients.js') const datastore = require('../store/data.js') -const { attachToContext, isInMemoryBacked } = require('../store/reactive.js') +const { isInMemoryBacked } = require('../store/reactive.js') const statestore = require('../store/state.js') const { appendTopic, addConnectionCredentials, normalizeClientId, getThirdPartyWidgets } = require('../utils/index.js') @@ -405,14 +405,15 @@ module.exports = function (RED) { if (isInMemoryBacked(node.context().global)) { try { - attachToContext(node.context().global, { - clone: RED.util.cloneMessage, + datastore.initStore(node.context().global, { onOverwrite: () => node.warn('Dashboard data store was overwritten in global context by a flow; re-injecting it. Write keys as global.dashboardStore. rather than replacing global.dashboardStore itself.') }) } catch (err) { + datastore.disableStore() node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') } } else { + datastore.disableStore() node.warn('Dashboard data store disabled: the global context store isn\'t in-memory-backed (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.') } diff --git a/nodes/store/data.js b/nodes/store/data.js index f8b8ddd43..a2b1c1b39 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -10,16 +10,39 @@ function projectValue (node, msg) { return msg.payload } +let storeOptions = {} +let storeEnabled = true + +function storeOpts () { + return { clone: config.RED.util.cloneMessage, ...storeOptions } +} + +function initStore (globalContext, opts) { + storeOptions = { ...opts } + storeEnabled = true + return attachToContext(globalContext, storeOpts()) +} + +function disableStore () { + storeEnabled = false +} + function getStore (node) { - return attachToContext(node.context().global, { clone: config.RED.util.cloneMessage }) + return attachToContext(node.context().global, storeOpts()) } -function writeToStore (node, msg) { +function withStore (node, fn) { + if (!storeEnabled) return try { - getStore(node)[node.id] = projectValue(node, msg) + fn(getStore(node)) } catch (err) {} } +function writeToStore (node, msg) { + if (!('payload' in msg)) return + withStore(node, (store) => { store[node.id] = projectValue(node, msg) }) +} + function pointsFromMessages (messages) { const points = [] if (!Array.isArray(messages)) return points @@ -37,26 +60,21 @@ function pointsFromMessages (messages) { } function writeArrayToStore (node) { - try { - getStore(node)[node.id] = pointsFromMessages(data[node.id]) - } catch (err) {} + withStore(node, (store) => { store[node.id] = pointsFromMessages(data[node.id]) }) } function appendToStore (node, message) { - try { - const store = getStore(node) + withStore(node, (store) => { if (Array.isArray(store[node.id])) { for (const point of pointsFromMessages([message])) store[node.id].push(point) } else { store[node.id] = pointsFromMessages(data[node.id]) } - } catch (err) {} + }) } function clearFromStore (node) { - try { - delete getStore(node)[node.id] - } catch (err) {} + withStore(node, (store) => { delete store[node.id] }) } /** @@ -188,5 +206,7 @@ module.exports = { append: setters.append, filter: setters.filter, clear: setters.clear, - clearFromStore + clearFromStore, + initStore, + disableStore } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 589a11496..541933fb5 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -73,6 +73,27 @@ describe('store: data.js reactive-store mirror', function () { should(global.get('dashboardStore').w6).be.undefined() }) + it('keeps the mirrored value when a later message carries no payload', function () { + const global = fakeGlobal() + const node = fakeNode('w8', global) + datastore.save(base, node, { payload: 'keep me' }) + datastore.save(base, node, { topic: 'no payload' }) + + global.get('dashboardStore').w8.should.equal('keep me') + datastore.get('w8').payload.should.equal('keep me') + global.get('dashboardStore').$w8.history.should.eql([]) + }) + + it('blanks the mirrored value when a message explicitly carries payload: undefined', function () { + const global = fakeGlobal() + const node = fakeNode('w11', global) + datastore.save(base, node, { payload: 'keep me' }) + datastore.save(base, node, { payload: undefined, topic: 'explicit' }) + + should(global.get('dashboardStore').w11).be.undefined() + should(datastore.get('w11').payload).be.undefined() + }) + it('clearFromStore does not throw when the store is unavailable', function () { const throwingGlobal = { get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, @@ -83,6 +104,85 @@ describe('store: data.js reactive-store mirror', function () { }) }) +describe('store: data.js store options', function () { + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('re-injects with the registered options when a flow overwrites the namespace', function () { + const global = fakeGlobal() + const node = fakeNode('w9', global) + let overwrites = 0 + datastore.initStore(global, { onOverwrite: () => { overwrites++ } }) + + datastore.save(base, node, { payload: 1 }) + global.set('dashboardStore', { rogue: true }) + datastore.save(base, node, { payload: 2 }) + + overwrites.should.equal(1) + global.get('dashboardStore').w9.should.equal(2) + }) + + it('keeps onChange wired through a widget-triggered re-injection', function () { + const global = fakeGlobal() + const node = fakeNode('w10', global) + const changes = [] + datastore.initStore(global, { onChange: (key) => changes.push(key) }) + + global.set('dashboardStore', { rogue: true }) + datastore.save(base, node, { payload: 'after' }) + + changes.should.containEql('w10') + }) +}) + +describe('store: data.js disabled store', function () { + const cacheOffGlobal = () => ({ + gets: 0, + get () { this.gets++; throw new Error('File Store cache disabled - only asynchronous access supported') }, + set () {} + }) + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('stops touching global context once the store is disabled', function () { + const global = cacheOffGlobal() + const scalar = fakeNode('w12', global) + const chart = fakeNode('w12-chart', global) + datastore.disableStore() + + datastore.save(base, scalar, { payload: 1 }) + datastore.append(base, chart, { _datapoint: { category: 'a', x: 1, y: 1 } }) + datastore.filter(base, chart, () => false) + datastore.clearFromStore(scalar) + + global.gets.should.equal(0) + }) + + it('leaves the legacy datastore working while disabled', function () { + const global = cacheOffGlobal() + const node = fakeNode('w13', global) + datastore.disableStore() + + datastore.save(base, node, { payload: 'legacy still works' }) + + datastore.get('w13').payload.should.equal('legacy still works') + }) + + it('re-enables on a later initStore', function () { + datastore.disableStore() + + const global = fakeGlobal() + const node = fakeNode('w14', global) + datastore.initStore(global, {}) + datastore.save(base, node, { payload: 'back on' }) + + global.get('dashboardStore').w14.should.equal('back on') + }) +}) + describe('store: data.js chart array mirror', function () { const pt = (x, y, category = 'a') => ({ category, x, y }) From 9bf8b8e1bf847571ef8b843644f6656fed16b3f1 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 17 Sep 2026 09:41:43 -0700 Subject: [PATCH 12/57] Update guard to not fire on every restart and only when a users replaces the store --- nodes/config/ui_base.js | 2 +- nodes/store/reactive.js | 11 +++++++++-- test/store/reactive.spec.js | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 5 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index 6297649eb..b26786453 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -407,7 +407,7 @@ module.exports = function (RED) { try { attachToContext(node.context().global, { clone: RED.util.cloneMessage, - onOverwrite: () => node.warn('Dashboard data store was overwritten in global context by a flow; re-injecting it. Write keys as global.dashboardStore. rather than replacing global.dashboardStore itself.') + onReplaced: () => node.warn('A flow replaced global.dashboardStore, discarding the stored value of every widget; the store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') }) } catch (err) { node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index c6dc40a58..1f9d64a5f 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -110,20 +110,27 @@ function isInMemoryBacked (globalContext, namespace = NAMESPACE) { } } +const injected = new WeakMap() + // Inject the store into global context once. On redeploy the existing store is reused; // after a restart with a persistent context store, existing plain data is rehydrated. function attachToContext (globalContext, opts = {}) { const namespace = opts.namespace || NAMESPACE const existing = globalContext.get(namespace) - if (existing && existing[STORE]) return existing + if (existing && existing[STORE]) { + injected.set(globalContext, existing) + return existing + } - if (existing && typeof existing === 'object') opts.onOverwrite?.() + const previous = injected.get(globalContext) + if (previous && existing !== previous) opts.onReplaced?.() const store = createDataStore(opts) if (existing && typeof existing === 'object') { for (const [k, v] of Object.entries(existing)) store[k] = v } globalContext.set(namespace, store) + injected.set(globalContext, store) return store } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index c4c6cd0e8..84990764d 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -324,7 +324,7 @@ describe('store: reactive data store', function () { g.set('dashboardStore', { k: 1, extra: 2 }) // a flow replaces the proxy with a plain object let warned = 0 - const restored = attachToContext(g, { onOverwrite: () => { warned++ } }) + const restored = attachToContext(g, { onReplaced: () => { warned++ } }) warned.should.equal(1) restored[STORE].should.equal(true) should(g.get('dashboardStore')).equal(restored) @@ -337,9 +337,38 @@ describe('store: reactive data store', function () { it('does not warn on a clean first injection', function () { const g = fakeGlobal() let warned = 0 - attachToContext(g, { onOverwrite: () => { warned++ } }) + attachToContext(g, { onReplaced: () => { warned++ } }) warned.should.equal(0) }) + + it('does not warn when a restart rehydrates plain data it never injected', function () { + const g = fakeGlobal() + g.set('dashboardStore', { widget: 'from disk' }) // what a persistent context store hands back + let warned = 0 + const store = attachToContext(g, { onReplaced: () => { warned++ } }) + warned.should.equal(0) + store.widget.should.equal('from disk') + }) + + const replacements = [ + ['an object', { mine: 1 }], + ['an array', [1, 2]], + ['a string', 'hello'], + ['a number', 42], + ['null', null], + ['undefined', undefined] + ] + replacements.forEach(([label, value]) => { + it(`warns when a flow replaces the injected store with ${label}`, function () { + const g = fakeGlobal() + attachToContext(g) + let warned = 0 + g.set('dashboardStore', value) + const restored = attachToContext(g, { onReplaced: () => { warned++ } }) + warned.should.equal(1) + restored[STORE].should.equal(true) + }) + }) }) describe('isInMemoryBacked', function () { From 325aa620fbf29c7b54a224797e624b3c74a048f5 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 17 Sep 2026 11:25:02 -0700 Subject: [PATCH 13/57] Remove charts and update specs --- nodes/store/data.js | 33 --------- nodes/store/reactive.js | 2 +- test/store/data.spec.js | 135 ++++++++++++++++-------------------- test/store/reactive.spec.js | 24 ++++++- 4 files changed, 83 insertions(+), 111 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index a2b1c1b39..561962fc1 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -43,36 +43,6 @@ function writeToStore (node, msg) { withStore(node, (store) => { store[node.id] = projectValue(node, msg) }) } -function pointsFromMessages (messages) { - const points = [] - if (!Array.isArray(messages)) return points - for (const message of messages) { - const datapoint = message && message._datapoint - if (datapoint == null) { - if (message && message.payload !== undefined) points.push(message.payload) - } else if (Array.isArray(datapoint)) { - points.push(...datapoint) - } else { - points.push(datapoint) - } - } - return points -} - -function writeArrayToStore (node) { - withStore(node, (store) => { store[node.id] = pointsFromMessages(data[node.id]) }) -} - -function appendToStore (node, message) { - withStore(node, (store) => { - if (Array.isArray(store[node.id])) { - for (const point of pointsFromMessages([message])) store[node.id].push(point) - } else { - store[node.id] = pointsFromMessages(data[node.id]) - } - }) -} - function clearFromStore (node) { withStore(node, (store) => { delete store[node.id] }) } @@ -156,7 +126,6 @@ const setters = { } } data[node.id] = filtered - writeArrayToStore(node) } else { if (canSaveInStore(base, node, msg)) { const newMsg = stripMsg(msg) @@ -176,7 +145,6 @@ const setters = { data[node.id] = [] } data[node.id].push(config.RED.util.cloneMessage(msg)) - appendToStore(node, msg) } }, /** @@ -192,7 +160,6 @@ const setters = { if (filteredMessages.length !== currentData.length) { // no need for save operation to process messages - just apply them data[node.id] = filteredMessages - writeArrayToStore(node) } } } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 46e785d94..5f7a38753 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -79,7 +79,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D rec = new Entry() target[prop] = rec } else if (!Array.isArray(rec.value)) { - // array-valued keys (e.g. a chart series) are their own history; snapshotting them would blow up memory + // snapshotting a whole array on every write (e.g. a table's rows) would blow up memory rec.history.push({ value: cloneValue(rec.value), timestamp: rec.timestamp }) if (rec.history.length > maxHistory) rec.history.shift() } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index c0e4a6801..d73509acb 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -46,11 +46,28 @@ describe('store: data.js reactive-store mirror', function () { should(global.get('dashboardStore')).be.undefined() }) - it('mirrors array (chart-style) saves as a series', function () { + it('deep-copies a table rows array, so a flow reusing it cannot mutate stored state', function () { const global = fakeGlobal() - const node = fakeNode('w4', global) - datastore.save(base, node, [{ payload: 1 }, { payload: 2 }]) - global.get('dashboardStore').w4.should.eql([1, 2]) + const node = fakeNode('tbl-1', global) + const rows = [{ id: 1, name: 'a', meta: { site: { rack: 'r1' } } }] + + datastore.save(base, node, { payload: rows }) + rows[0].name = 'mutated' + rows[0].meta.site.rack = 'mutated-deep' + + global.get('dashboardStore')['tbl-1'].should.eql([ + { id: 1, name: 'a', meta: { site: { rack: 'r1' } } } + ]) + }) + + it('keeps the mirrored rows independent of the legacy datastore', function () { + const global = fakeGlobal() + const node = fakeNode('tbl-2', global) + + datastore.save(base, node, { payload: [{ id: 1, name: 'original' }] }) + global.get('dashboardStore')['tbl-2'][0].name = 'changed via store' + + datastore.get('tbl-2').payload.should.eql([{ id: 1, name: 'original' }]) }) it('does not throw or mirror when the context store cannot hold the proxy', function () { @@ -183,101 +200,67 @@ describe('store: data.js disabled store', function () { }) }) -describe('store: data.js chart array mirror', function () { +describe('store: data.js chart writes (deferred to Story 3)', function () { const pt = (x, y, category = 'a') => ({ category, x, y }) - it('mirrors chart points (_datapoint) into the store on append', function () { - const global = fakeGlobal() - const node = fakeNode('c1', global) - datastore.append(base, node, { payload: 2, _datapoint: pt(1, 2) }) - datastore.append(base, node, { payload: 3, _datapoint: pt(2, 3) }) - global.get('dashboardStore').c1.should.eql([pt(1, 2), pt(2, 3)]) - }) + // an absent chart key only proves anything if the store is actually working + function liveStore (global, id) { + datastore.save(base, fakeNode(id, global), { payload: 'live' }) + return global.get('dashboardStore')?.[id] + } - it('pushes each point when _datapoint is an array (multi-series)', function () { + it('does not mirror a chart append into the store', function () { const global = fakeGlobal() - const node = fakeNode('c2', global) - datastore.append(base, node, { _datapoint: [pt(1, 2, 'a'), pt(1, 5, 'b')] }) - global.get('dashboardStore').c2.should.eql([pt(1, 2, 'a'), pt(1, 5, 'b')]) - }) + const node = fakeNode('chart-1', global) - it('replaces the store series on an array save (categorical/replace)', function () { - const global = fakeGlobal() - const node = fakeNode('c3', global) - datastore.append(base, node, { _datapoint: pt(1, 2) }) - datastore.save(base, node, [{ _datapoint: pt(9, 9) }]) - global.get('dashboardStore').c3.should.eql([pt(9, 9)]) + datastore.append(base, node, { _datapoint: pt(1, 1) }) + datastore.append(base, node, { _datapoint: pt(2, 2) }) + + liveStore(global, 'canary-1').should.equal('live') + should(global.get('dashboardStore')['chart-1']).be.undefined() }) - it('clears the store series on save([])', function () { + it('does not mirror an array save into the store', function () { const global = fakeGlobal() - const node = fakeNode('c4', global) - datastore.append(base, node, { _datapoint: pt(1, 2) }) - datastore.save(base, node, []) - global.get('dashboardStore').c4.should.eql([]) + const node = fakeNode('chart-2', global) + + datastore.save(base, node, [{ _datapoint: pt(1, 1) }, { _datapoint: pt(2, 2) }]) + + liveStore(global, 'canary-2').should.equal('live') + should(global.get('dashboardStore')['chart-2']).be.undefined() }) - it('mirrors a trim (filter) into the store', function () { + it('does not mirror a trim into the store', function () { const global = fakeGlobal() - const node = fakeNode('c5', global) + const node = fakeNode('chart-3', global) + datastore.append(base, node, { _datapoint: pt(1, 1) }) datastore.append(base, node, { _datapoint: pt(2, 2) }) - datastore.append(base, node, { _datapoint: pt(3, 3) }) - datastore.filter(base, node, (m, i) => i > 0) // drop the oldest - global.get('dashboardStore').c5.should.eql([pt(2, 2), pt(3, 3)]) + datastore.filter(base, node, (m, i) => i > 0) + + liveStore(global, 'canary-3').should.equal('live') + should(global.get('dashboardStore')['chart-3']).be.undefined() }) - it('keeps no history as the series grows', function () { + it('still keeps the legacy chart series intact', function () { const global = fakeGlobal() - const node = fakeNode('c6', global) + const node = fakeNode('chart-4', global) + datastore.append(base, node, { _datapoint: pt(1, 1) }) - datastore.save(base, node, []) // reassigns the array datastore.append(base, node, { _datapoint: pt(2, 2) }) - global.get('dashboardStore').$c6.history.should.eql([]) - }) + datastore.filter(base, node, (m, i) => i > 0) - it('falls back to payload when _datapoint is null (multi-series without _datapoint)', function () { - const global = fakeGlobal() - const node = fakeNode('c7', global) - datastore.append(base, node, { payload: 42, _datapoint: null }) - global.get('dashboardStore').c7.should.eql([42]) - }) - - it('adds no point for an empty _datapoint array', function () { - const global = fakeGlobal() - const node = fakeNode('c8', global) - datastore.append(base, node, { _datapoint: [] }) - global.get('dashboardStore').c8.should.eql([]) + datastore.get('chart-4').map((m) => m._datapoint).should.eql([pt(2, 2)]) }) - it('preserves falsy payloads (0/false) and skips a msg with neither field', function () { + it('clears a stale key if a node id was previously used for a value', function () { const global = fakeGlobal() - const node = fakeNode('c9', global) - datastore.append(base, node, { payload: 0 }) - datastore.append(base, node, { payload: false }) - datastore.append(base, node, { topic: 'no value here' }) - global.get('dashboardStore').c9.should.eql([0, false]) - }) + const node = fakeNode('chart-5', global) - it('does not mirror a client-scoped chart append', function () { - const global = fakeGlobal() - const node = fakeNode('c10', global) - const scopedBase = { acceptsClientConfig: ['ui-text'] } - datastore.append(scopedBase, node, { _datapoint: pt(1, 2), _client: { socketId: 's1' } }) - should(global.get('dashboardStore')).be.undefined() - }) + datastore.save(base, node, { payload: 'scalar first' }) + global.get('dashboardStore')['chart-5'].should.equal('scalar first') - it('stays in sync with a capped streaming chart (append + trim interleaved)', function () { - const global = fakeGlobal() - const node = fakeNode('c-stream', global) - const CAP = 3 - const keepLast = (message, index, arr) => index >= arr.length - CAP - for (let i = 1; i <= 6; i++) { - datastore.append(base, node, { _datapoint: pt(i, i) }) - datastore.filter(base, node, keepLast) - } - // in-place appends interleaved with rebuild-trims must leave the store == the trimmed series - global.get('dashboardStore')['c-stream'].should.eql([pt(4, 4), pt(5, 5), pt(6, 6)]) - datastore.get('c-stream').map((m) => m._datapoint).should.eql([pt(4, 4), pt(5, 5), pt(6, 6)]) + datastore.clearFromStore(node) + should(global.get('dashboardStore')['chart-5']).be.undefined() }) }) diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 65cf83254..d82c50502 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -68,7 +68,7 @@ describe('store: reactive data store', function () { store.h.should.equal(4) }) - it('keeps no history for array-valued keys (a chart series is its own history)', function () { + it('keeps no history for array-valued keys', function () { const { store } = makeStore() store.series = [1, 2] store.series = [3, 4, 5] @@ -114,6 +114,28 @@ describe('store: reactive data store', function () { log.should.eql(['series']) store.series.should.eql([]) }) + + it('fires for an array at a nested path the same as one at a top-level key', function () { + const { store, log } = makeStore() + store.top = [1, 2] + store.nested = { rows: [1, 2] } + + log.length = 0 + store.top.push(3) + store.nested.rows.push(3) + + log.should.eql(['top.2', 'nested.rows.2']) + }) + + it('does not alias an array assigned into the store', function () { + const { store } = makeStore() + const original = [{ x: 1 }] + store.series = original + + original[0].x = 999 + + store.series.should.eql([{ x: 1 }]) + }) }) describe('delete', function () { From c8e5cf75b9fa25f111dde38af86b85e121825ef0 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 17 Sep 2026 11:46:52 -0700 Subject: [PATCH 14/57] Simplify fns in data.js for the store --- nodes/store/data.js | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index 561962fc1..2f90c94e3 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -6,45 +6,36 @@ const config = { RED: null } -function projectValue (node, msg) { - return msg.payload -} - let storeOptions = {} let storeEnabled = true -function storeOpts () { - return { clone: config.RED.util.cloneMessage, ...storeOptions } +function getOrCreateStore (globalContext) { + return attachToContext(globalContext, { clone: config.RED.util.cloneMessage, ...storeOptions }) } function initStore (globalContext, opts) { storeOptions = { ...opts } storeEnabled = true - return attachToContext(globalContext, storeOpts()) + return getOrCreateStore(globalContext) } function disableStore () { storeEnabled = false } -function getStore (node) { - return attachToContext(node.context().global, storeOpts()) -} - -function withStore (node, fn) { +function writeToStore (node, msg) { if (!storeEnabled) return + if (!('payload' in msg)) return try { - fn(getStore(node)) + getOrCreateStore(node.context().global)[node.id] = msg.payload } catch (err) {} } -function writeToStore (node, msg) { - if (!('payload' in msg)) return - withStore(node, (store) => { store[node.id] = projectValue(node, msg) }) -} - function clearFromStore (node) { - withStore(node, (store) => { delete store[node.id] }) + if (!storeEnabled) return + try { + delete getOrCreateStore(node.context().global)[node.id] + } catch (err) {} } /** From e7d3618643bf748ce418d99396e7224123ec1de1 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 17 Sep 2026 12:11:49 -0700 Subject: [PATCH 15/57] Add additional specs + protect for non obj msg --- nodes/store/data.js | 2 +- test/nodes/config-nodes/ui_base.spec.js | 74 ++++++++++++++++++++++++- test/store/data.spec.js | 8 +++ 3 files changed, 81 insertions(+), 3 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index 2f90c94e3..a8d9cdf50 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -25,7 +25,7 @@ function disableStore () { function writeToStore (node, msg) { if (!storeEnabled) return - if (!('payload' in msg)) return + if (!msg || typeof msg !== 'object' || !('payload' in msg)) return try { getOrCreateStore(node.context().global)[node.id] = msg.payload } catch (err) {} diff --git a/test/nodes/config-nodes/ui_base.spec.js b/test/nodes/config-nodes/ui_base.spec.js index 679e3835b..30a38c1e6 100644 --- a/test/nodes/config-nodes/ui_base.spec.js +++ b/test/nodes/config-nodes/ui_base.spec.js @@ -1,7 +1,9 @@ +const context = require('@node-red/runtime/lib/nodes/context') const helper = require('node-red-node-test-helper') const should = require('should') // eslint-disable-line no-unused-vars -const { attachToContext, STORE, NAMESPACE } = require('../../../nodes/store/reactive.js') +const datastore = require('../../../nodes/store/data.js') +const { STORE, NAMESPACE } = require('../../../nodes/store/reactive.js') const { testData1 } = require('../fixtures/index.js') const { verifyFlowLoaded } = require('../utils.js') @@ -86,7 +88,10 @@ describe('ui-base config node: data store injection', function () { const original = global.get(NAMESPACE) original.reused = 42 - const again = attachToContext(global) + + // the path ui-base actually takes on a redeploy + const again = datastore.initStore(global, {}) + should(again).equal(original) global.get(`${NAMESPACE}.reused`).should.equal(42) }) @@ -103,3 +108,68 @@ describe('ui-base config node: data store injection', function () { global.get(`${NAMESPACE}.$node-ui-text`).value.should.equal('hello') }) }) + +describe('ui-base config node: cache-off context store', function () { + const CACHE_OFF = 'File Store cache disabled - only asynchronous access supported' + let realGet + let touched + + beforeEach(function (done) { + touched = { get: 0, set: 0 } + realGet = context.get + // a persistent context store with cache:false serves only async access, so a sync get throws + context.get = function (id, z) { + const real = realGet.call(context, id, z) + return Object.create(real, { + global: { + value: { + get () { touched.get++; throw new Error(CACHE_OFF) }, + set (...args) { touched.set++; return real.global.set(...args) } + } + } + }) + } + helper.startServer(done) + }) + + afterEach(function (done) { + context.get = realGet + datastore.initStore(fakeGlobal(), {}) + helper.unload() + helper.stopServer(done) + }) + + function fakeGlobal () { + const m = {} + return { get: (k) => m[k], set: (k, v) => { m[k] = v } } + } + + const flow = [ + { id: 'node-ui-text', type: 'ui-text', z: 'tab-id', group: 'config-ui-group', label: 'txt' }, + ...testFlow1 + ] + + it('warns and does not inject the store', async function () { + await helper.load(nodeImports, flow) + + const warned = helper.log().args.filter((args) => + args[0].level === helper.log().WARN && /in-memory-backed/.test(args[0].msg) + ) + warned.should.have.length(1) + touched.set.should.equal(0) + }) + + it('stops the write path instead of throwing on every message', async function () { + await helper.load(nodeImports, flow) + const text = helper.getNode('node-ui-text') + + touched.get = 0 + touched.set = 0 + should(() => text.receive({ payload: 'hello' })).not.throw() + await new Promise((resolve) => setTimeout(resolve, 50)) + + touched.get.should.equal(0) + touched.set.should.equal(0) + datastore.get('node-ui-text').payload.should.equal('hello') + }) +}) diff --git a/test/store/data.spec.js b/test/store/data.spec.js index d73509acb..77ff9df2d 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -111,6 +111,14 @@ describe('store: data.js reactive-store mirror', function () { should(datastore.get('w11').payload).be.undefined() }) + it('does not throw when a msg is not an object (unchanged from main)', function () { + const global = fakeGlobal() + for (const msg of ['hello', 42, true]) { + const node = fakeNode('prim-' + String(msg), global) + should(() => datastore.save(base, node, msg)).not.throw() + } + }) + it('clearFromStore does not throw when the store is unavailable', function () { const throwingGlobal = { get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, From f185270a5be62a5b52d8e5f124ae3aa9f5adb654 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Mon, 21 Sep 2026 13:01:27 -0700 Subject: [PATCH 16/57] Store entire msg in the dataStore with specs --- nodes/store/data.js | 9 ++-- nodes/store/reactive.js | 42 ++++++++++++------ test/store/data.spec.js | 86 +++++++++++++++++++++++++++++++++++++ test/store/reactive.spec.js | 39 ++++++++++++++++- 4 files changed, 157 insertions(+), 19 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index a8d9cdf50..babac2d12 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -1,4 +1,4 @@ -const { attachToContext } = require('./reactive.js') +const { attachToContext, SET_ENTRY } = require('./reactive.js') const data = {} @@ -23,11 +23,12 @@ function disableStore () { storeEnabled = false } -function writeToStore (node, msg) { +function writeToStore (node, msg, stored) { if (!storeEnabled) return if (!msg || typeof msg !== 'object' || !('payload' in msg)) return try { - getOrCreateStore(node.context().global)[node.id] = msg.payload + const clone = config.RED.util.cloneMessage + getOrCreateStore(node.context().global)[SET_ENTRY](node.id, msg.payload, clone(stored)) } catch (err) {} } @@ -124,7 +125,7 @@ const setters = { ...data[node.id], ...newMsg } - writeToStore(node, msg) + writeToStore(node, msg, data[node.id]) } } }, diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 5f7a38753..2b07a5ad3 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -1,9 +1,11 @@ const STORE = Symbol('dashboardDataStore') +const SET_ENTRY = Symbol('dashboardSetEntry') const NAMESPACE = 'dashboardStore' class Entry { constructor () { this.value = undefined + this.msg = undefined this.quality = 'GOOD' this.timestamp = 0 this.history = [] @@ -58,6 +60,30 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D records[STORE] = true + // returns the record without notifying, so a caller can finish writing it before onChange fires + const writeValue = (prop, value) => { + let rec = records[prop] + if (!rec) { + rec = new Entry() + records[prop] = rec + } else if (!Array.isArray(rec.value)) { + // snapshotting a whole array on every write (e.g. a table's rows) would blow up memory + rec.history.push({ value: cloneValue(rec.value), timestamp: rec.timestamp }) + if (rec.history.length > maxHistory) rec.history.shift() + } + const notify = (path) => { rec.timestamp = now(); onChange?.(prop, rec, path) } + // clone on write so a flow reusing its own object can't mutate stored state + rec.value = deepReactive(cloneValue(value), notify, prop, cloneValue) + rec.timestamp = now() + return rec + } + + records[SET_ENTRY] = (prop, value, msg) => { + const rec = writeValue(prop, value) + rec.msg = msg + onChange?.(prop, rec, prop) + } + const handler = { get (target, prop, receiver) { if (typeof prop === 'symbol') return Reflect.get(target, prop, receiver) @@ -74,19 +100,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (prop === '__proto__' || prop === 'constructor') return true if (prop.startsWith('$')) return true // '$' is reserved for the read-only meta view - let rec = target[prop] - if (!rec) { - rec = new Entry() - target[prop] = rec - } else if (!Array.isArray(rec.value)) { - // snapshotting a whole array on every write (e.g. a table's rows) would blow up memory - rec.history.push({ value: cloneValue(rec.value), timestamp: rec.timestamp }) - if (rec.history.length > maxHistory) rec.history.shift() - } - const notify = (path) => { rec.timestamp = now(); onChange?.(prop, rec, path) } - // clone on write so a flow reusing its own object can't mutate stored state - rec.value = deepReactive(cloneValue(value), notify, prop, cloneValue) - rec.timestamp = now() + const rec = writeValue(prop, value) onChange?.(prop, rec, prop) return true }, @@ -135,4 +149,4 @@ function attachToContext (globalContext, opts = {}) { return store } -module.exports = { createDataStore, attachToContext, isInMemoryBacked, NAMESPACE, STORE } +module.exports = { createDataStore, attachToContext, isInMemoryBacked, NAMESPACE, STORE, SET_ENTRY } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 77ff9df2d..0528a3d1f 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -38,6 +38,48 @@ describe('store: data.js reactive-store mirror', function () { global.get('dashboardStore').w2.should.equal(2) }) + it('stores the whole message alongside the value', function () { + const global = fakeGlobal() + const node = fakeNode('m1', global) + datastore.save(base, node, { payload: 42, topic: 't', extra: 'x' }) + + const store = global.get('dashboardStore') + store.m1.should.equal(42) + store.$m1.msg.payload.should.equal(42) + store.$m1.msg.topic.should.equal('t') + store.$m1.msg.extra.should.equal('x') + }) + + it('keeps the stored message in step with the legacy merge', function () { + const global = fakeGlobal() + const node = fakeNode('m3', global) + datastore.save(base, node, { payload: 1, topic: 'a', unit: 'bar' }) + datastore.save(base, node, { payload: 2 }) + + global.get('dashboardStore').$m3.msg.should.eql(datastore.get('m3')) + }) + + it('does not alias the legacy store into the mirror', function () { + const global = fakeGlobal() + const node = fakeNode('m4', global) + datastore.save(base, node, { payload: 1, topic: 'a' }) + + global.get('dashboardStore').$m4.msg.topic = 'mutated' + + datastore.get('m4').topic.should.equal('a') + }) + + it('does not alias the flow\'s message into the store', function () { + const global = fakeGlobal() + const node = fakeNode('m2', global) + const sent = { payload: 1, nested: { a: 1 } } + datastore.save(base, node, sent) + + sent.nested.a = 999 + + global.get('dashboardStore').$m2.msg.nested.a.should.equal(1) + }) + it('does not mirror a client-scoped message (not stored centrally)', function () { const global = fakeGlobal() const node = fakeNode('w3', global) @@ -272,3 +314,47 @@ describe('store: data.js chart writes (deferred to Story 3)', function () { should(global.get('dashboardStore')['chart-5']).be.undefined() }) }) + +describe('store: data.js audit invariants', function () { + it('does not store a message carrying only dynamic properties', function () { + const global = fakeGlobal() + const node = fakeNode('a1', global) + datastore.save(base, node, { payload: 'real' }) + datastore.save(base, node, { ui_update: { class: 'red' } }) + + global.get('dashboardStore').a1.should.equal('real') + }) + + it('keeps value and msg in step across writes', function () { + const global = fakeGlobal() + const node = fakeNode('a2', global) + datastore.save(base, node, { payload: 1, topic: 'first' }) + datastore.save(base, node, { payload: 2, topic: 'second' }) + + const store = global.get('dashboardStore') + store.a2.should.equal(2) + store.$a2.msg.topic.should.equal('second') + }) + + it('leaves no key at all when a widget never stores', function () { + const global = fakeGlobal() + datastore.save(base, fakeNode('a3', global), { payload: 1 }) + + const store = global.get('dashboardStore') + store.a3.should.equal(1) + should(store.never).be.undefined() + }) + + it('clears value and msg together when a widget is removed', function () { + const global = fakeGlobal() + const node = fakeNode('a4', global) + datastore.save(base, node, { payload: 5, topic: 't' }) + global.get('dashboardStore').$a4.msg.should.be.an.Object() + + datastore.clearFromStore(node) + + const store = global.get('dashboardStore') + should(store.a4).be.undefined() + should(store.$a4).be.undefined() + }) +}) diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index d82c50502..57017166d 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -7,7 +7,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext, isInMemoryBacked, STORE } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, isInMemoryBacked, STORE, SET_ENTRY } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -37,6 +37,13 @@ describe('store: reactive data store', function () { store.$k1.timestamp.should.be.a.Number() }) + it('exposes an empty msg field on a new entry', function () { + const { store } = makeStore() + store.k1 = 123 + store.$k1.should.have.property('msg') + should(store.$k1.msg).be.undefined() + }) + it('returns the whole object by default', function () { const { store } = makeStore() store.robot = { temp: 20 } @@ -44,6 +51,36 @@ describe('store: reactive data store', function () { }) }) + describe('entry setter', function () { + it('sets value and msg together and fires one change', function () { + const { store, log } = makeStore() + const msg = { payload: 42, topic: 'sensor-A' } + + store[SET_ENTRY]('gauge', msg.payload, msg) + + store.gauge.should.equal(42) + store.$gauge.msg.should.eql({ payload: 42, topic: 'sensor-A' }) + log.should.eql(['gauge']) + }) + + it('keeps history on the value, as a normal write does', function () { + const { store } = makeStore() + store[SET_ENTRY]('gauge', 1, { payload: 1 }) + store[SET_ENTRY]('gauge', 2, { payload: 2 }) + + store.gauge.should.equal(2) + store.$gauge.history.map((h) => h.value).should.eql([1]) + }) + + it('does not clone the message it is given', function () { + const { store } = makeStore() + const msg = { payload: 1, nested: { a: 1 } } + store[SET_ENTRY]('k', msg.payload, msg) + + should(store.$k.msg).equal(msg) + }) + }) + describe('history', function () { it('keeps the prior value on reassignment', function () { const { store } = makeStore() From 153ad4f3e56ff7032e2d47409c7aea56eaf21203 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 08:19:36 -0700 Subject: [PATCH 17/57] Add charts surface to data store --- nodes/store/data.js | 45 +++++++++- nodes/store/reactive.js | 13 ++- test/store/data.spec.js | 168 +++++++++++++++++++++++++++++++++--- test/store/reactive.spec.js | 55 +++++++++++- 4 files changed, 265 insertions(+), 16 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index babac2d12..1adba3f44 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -1,4 +1,4 @@ -const { attachToContext, SET_ENTRY } = require('./reactive.js') +const { attachToContext, SET_ENTRY, APPEND_ENTRY } = require('./reactive.js') const data = {} @@ -8,6 +8,8 @@ const config = { let storeOptions = {} let storeEnabled = true +const trimCounts = {} +const TRIM_BATCH = 60 function getOrCreateStore (globalContext) { return attachToContext(globalContext, { clone: config.RED.util.cloneMessage, ...storeOptions }) @@ -32,10 +34,48 @@ function writeToStore (node, msg, stored) { } catch (err) {} } +function chartPoints (msgs) { + if (!storeEnabled) return + const points = [] + for (const m of msgs) { + const d = m?._datapoint + if (d === undefined || d === null) continue + if (Array.isArray(d)) points.push(...d) + else points.push(d) + } + return points +} + +function appendToStore (node, msg) { + if (!storeEnabled) return + try { + const stored = config.RED.util.cloneMessage(msg) + getOrCreateStore(node.context().global)[APPEND_ENTRY](node.id, chartPoints([stored]), stored) + } catch (err) {} +} + +function replaceInStore (node, msgs) { + if (!storeEnabled) return + try { + const clone = config.RED.util.cloneMessage + const stored = msgs.map((m) => clone(m)) + getOrCreateStore(node.context().global)[SET_ENTRY](node.id, chartPoints(stored), stored) + delete trimCounts[node.id] + } catch (err) {} +} + +function trimStore (node, msgs) { + if (!storeEnabled) return + trimCounts[node.id] = (trimCounts[node.id] || 0) + 1 + if (trimCounts[node.id] < TRIM_BATCH) return + replaceInStore(node, msgs) +} + function clearFromStore (node) { if (!storeEnabled) return try { delete getOrCreateStore(node.context().global)[node.id] + delete trimCounts[node.id] } catch (err) {} } @@ -118,6 +158,7 @@ const setters = { } } data[node.id] = filtered + replaceInStore(node, filtered) } else { if (canSaveInStore(base, node, msg)) { const newMsg = stripMsg(msg) @@ -137,6 +178,7 @@ const setters = { data[node.id] = [] } data[node.id].push(config.RED.util.cloneMessage(msg)) + appendToStore(node, msg) } }, /** @@ -152,6 +194,7 @@ const setters = { if (filteredMessages.length !== currentData.length) { // no need for save operation to process messages - just apply them data[node.id] = filteredMessages + trimStore(node, filteredMessages) } } } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 2b07a5ad3..592611365 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -1,5 +1,6 @@ const STORE = Symbol('dashboardDataStore') const SET_ENTRY = Symbol('dashboardSetEntry') +const APPEND_ENTRY = Symbol('dashboardAppendEntry') const NAMESPACE = 'dashboardStore' class Entry { @@ -84,6 +85,16 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D onChange?.(prop, rec, prop) } + records[APPEND_ENTRY] = (prop, points, msg) => { + let rec = records[prop] + if (!rec) { + rec = writeValue(prop, []) + rec.msg = [] + } + rec.msg.push(msg) + for (const point of points) rec.value.push(point) + } + const handler = { get (target, prop, receiver) { if (typeof prop === 'symbol') return Reflect.get(target, prop, receiver) @@ -149,4 +160,4 @@ function attachToContext (globalContext, opts = {}) { return store } -module.exports = { createDataStore, attachToContext, isInMemoryBacked, NAMESPACE, STORE, SET_ENTRY } +module.exports = { createDataStore, attachToContext, isInMemoryBacked, NAMESPACE, STORE, SET_ENTRY, APPEND_ENTRY } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 0528a3d1f..d06816e0a 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -250,7 +250,7 @@ describe('store: data.js disabled store', function () { }) }) -describe('store: data.js chart writes (deferred to Story 3)', function () { +describe('store: data.js chart writes', function () { const pt = (x, y, category = 'a') => ({ category, x, y }) // an absent chart key only proves anything if the store is actually working @@ -259,37 +259,143 @@ describe('store: data.js chart writes (deferred to Story 3)', function () { return global.get('dashboardStore')?.[id] } - it('does not mirror a chart append into the store', function () { + it('mirrors a chart append into the store', function () { const global = fakeGlobal() const node = fakeNode('chart-1', global) - datastore.append(base, node, { _datapoint: pt(1, 1) }) - datastore.append(base, node, { _datapoint: pt(2, 2) }) + datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) + datastore.append(base, node, { payload: 2, _datapoint: pt(2, 2) }) liveStore(global, 'canary-1').should.equal('live') - should(global.get('dashboardStore')['chart-1']).be.undefined() + const store = global.get('dashboardStore') + store['chart-1'].should.eql([pt(1, 1), pt(2, 2)]) + store['$chart-1'].msg.should.have.length(2) + store['$chart-1'].msg[1].payload.should.equal(2) + }) + + it('flattens a multi-series append into several points', function () { + const global = fakeGlobal() + const node = fakeNode('chart-multi', global) + + datastore.append(base, node, { payload: [1, 5], _datapoint: [pt(1, 1, 'a'), pt(1, 5, 'b')] }) + + liveStore(global, 'canary-multi').should.equal('live') + const store = global.get('dashboardStore') + store['chart-multi'].should.eql([pt(1, 1, 'a'), pt(1, 5, 'b')]) + store['$chart-multi'].msg.should.have.length(1) + }) + + it('stores a message with no datapoint but adds no point', function () { + const global = fakeGlobal() + const node = fakeNode('chart-nodp', global) + + datastore.append(base, node, { payload: 'x' }) + + const store = global.get('dashboardStore') + store['chart-nodp'].should.eql([]) + store['$chart-nodp'].msg.should.have.length(1) + }) + + it('does not alias the flow\'s message into an appended series', function () { + const global = fakeGlobal() + const node = fakeNode('chart-alias', global) + const sent = { payload: 1, _datapoint: pt(1, 1), nested: { a: 1 } } + + datastore.append(base, node, sent) + sent.nested.a = 999 + + global.get('dashboardStore')['$chart-alias'].msg[0].nested.a.should.equal(1) }) - it('does not mirror an array save into the store', function () { + it('mirrors an array save into the store as a whole series', function () { const global = fakeGlobal() const node = fakeNode('chart-2', global) - datastore.save(base, node, [{ _datapoint: pt(1, 1) }, { _datapoint: pt(2, 2) }]) + datastore.save(base, node, [ + { payload: 1, _datapoint: pt(1, 1) }, + { payload: 2, _datapoint: pt(2, 2) } + ]) liveStore(global, 'canary-2').should.equal('live') - should(global.get('dashboardStore')['chart-2']).be.undefined() + const store = global.get('dashboardStore') + store['chart-2'].should.eql([pt(1, 1), pt(2, 2)]) + store['$chart-2'].msg.should.have.length(2) + }) + + it('replaces rather than appends on a second array save', function () { + const global = fakeGlobal() + const node = fakeNode('chart-replace', global) + + datastore.save(base, node, [{ payload: 1, _datapoint: pt(1, 1) }]) + datastore.save(base, node, [{ payload: 9, _datapoint: pt(9, 9) }]) + + const store = global.get('dashboardStore') + store['chart-replace'].should.eql([pt(9, 9)]) + store['$chart-replace'].msg.should.have.length(1) }) - it('does not mirror a trim into the store', function () { + it('empties the series when saved an empty array', function () { + const global = fakeGlobal() + const node = fakeNode('chart-clear', global) + + datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) + datastore.save(base, node, []) + + const store = global.get('dashboardStore') + store['chart-clear'].should.eql([]) + store['$chart-clear'].msg.should.eql([]) + }) + + it('does not rebuild the stored series until the batch is reached', function () { const global = fakeGlobal() const node = fakeNode('chart-3', global) + for (let i = 0; i < 100; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } - datastore.append(base, node, { _datapoint: pt(1, 1) }) - datastore.append(base, node, { _datapoint: pt(2, 2) }) - datastore.filter(base, node, (m, i) => i > 0) + for (let i = 0; i < 59; i++) { + datastore.filter(base, node, (m, idx) => idx > 0) + } liveStore(global, 'canary-3').should.equal('live') - should(global.get('dashboardStore')['chart-3']).be.undefined() + datastore.get('chart-3').should.have.length(41) + global.get('dashboardStore')['chart-3'].should.have.length(100) + }) + + it('rebuilds the stored series on the batch boundary', function () { + const global = fakeGlobal() + const node = fakeNode('chart-batch', global) + for (let i = 0; i < 100; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + + for (let i = 0; i < 60; i++) { + datastore.filter(base, node, (m, idx) => idx > 0) + } + + const expected = [] + for (let i = 60; i < 100; i++) expected.push(pt(i, i)) + + const store = global.get('dashboardStore') + store['chart-batch'].should.eql(expected) + store['$chart-batch'].msg.should.have.length(40) + }) + + it('does not advance the batch when a trim removes nothing', function () { + const global = fakeGlobal() + const node = fakeNode('chart-noop', global) + for (let i = 0; i < 100; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + + for (let i = 0; i < 200; i++) { + datastore.filter(base, node, () => true) + } + for (let i = 0; i < 59; i++) { + datastore.filter(base, node, (m, idx) => idx > 0) + } + + global.get('dashboardStore')['chart-noop'].should.have.length(100) }) it('still keeps the legacy chart series intact', function () { @@ -313,6 +419,42 @@ describe('store: data.js chart writes (deferred to Story 3)', function () { datastore.clearFromStore(node) should(global.get('dashboardStore')['chart-5']).be.undefined() }) + + it('keeps history empty across many appends', function () { + const global = fakeGlobal() + const node = fakeNode('chart-hist', global) + for (let i = 0; i < 20; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + + global.get('dashboardStore')['$chart-hist'].history.should.eql([]) + }) + + it('serialises a chart key to its point array and nothing else', function () { + const global = fakeGlobal() + const node = fakeNode('chart-json', global) + datastore.append(base, node, { payload: 1, topic: 'a', _datapoint: pt(1, 1) }) + + JSON.parse(JSON.stringify(global.get('dashboardStore'))).should.eql({ + 'chart-json': [pt(1, 1)] + }) + }) + + it('leaves the legacy series untouched when the store is disabled', function () { + const global = fakeGlobal() + const node = fakeNode('chart-off', global) + datastore.disableStore() + try { + datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) + datastore.save(base, node, [{ payload: 2, _datapoint: pt(2, 2) }]) + datastore.filter(base, node, () => false) + + datastore.get('chart-off').should.eql([]) + should(global.get('dashboardStore')).be.undefined() + } finally { + datastore.initStore(fakeGlobal(), {}) + } + }) }) describe('store: data.js audit invariants', function () { diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 57017166d..d61fcb641 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -7,7 +7,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext, isInMemoryBacked, STORE, SET_ENTRY } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, isInMemoryBacked, STORE, SET_ENTRY, APPEND_ENTRY } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -81,6 +81,59 @@ describe('store: reactive data store', function () { }) }) + describe('append entry', function () { + const pt = (x, y, category = 'a') => ({ category, x, y }) + + it('creates a series on first append', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', [pt(1, 1)], { payload: 1, _datapoint: pt(1, 1) }) + + store.c1.should.eql([pt(1, 1)]) + store.$c1.msg.should.have.length(1) + store.$c1.msg[0].payload.should.equal(1) + log.should.eql(['c1.0']) + }) + + it('pushes onto an existing series', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', [pt(1, 1)], { payload: 1 }) + log.length = 0 + store[APPEND_ENTRY]('c1', [pt(2, 2)], { payload: 2 }) + + store.c1.should.have.length(2) + store.c1[1].x.should.equal(2) + store.$c1.msg.should.have.length(2) + log.should.eql(['c1.1']) + }) + + it('pushes every point of a multi-series message, but the message once', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', [pt(1, 1, 'a'), pt(1, 5, 'b')], { payload: [1, 5] }) + + store.c1.should.eql([pt(1, 1, 'a'), pt(1, 5, 'b')]) + store.$c1.msg.should.have.length(1) + log.should.eql(['c1.0', 'c1.1']) + }) + + it('keeps the message but fires nothing when there are no points', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', [], { payload: 'no datapoint' }) + + store.c1.should.eql([]) + store.$c1.msg.should.have.length(1) + log.should.eql([]) + }) + + it('keeps history empty, because the value is an array', function () { + const { store } = makeStore() + store[APPEND_ENTRY]('c1', [pt(1, 1)], { payload: 1 }) + store[APPEND_ENTRY]('c1', [pt(2, 2)], { payload: 2 }) + store[APPEND_ENTRY]('c1', [pt(3, 3)], { payload: 3 }) + + store.$c1.history.should.eql([]) + }) + }) + describe('history', function () { it('keeps the prior value on reassignment', function () { const { store } = makeStore() From 35001297736a81e875e687c71d203fbd40750f8f Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 10:10:41 -0700 Subject: [PATCH 18/57] Return a frozen projection from the $ view so writes can't bypass onChange --- nodes/store/reactive.js | 7 ++++-- test/store/reactive.spec.js | 45 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 2291aeca0..24b5f7fca 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -64,7 +64,10 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (prop === 'toJSON') { return () => Object.fromEntries(Object.entries(target).map(([k, r]) => [k, r.value])) } - if (prop.startsWith('$')) return target[prop.slice(1)] + if (prop.startsWith('$')) { + const rec = target[prop.slice(1)] + return rec && Object.freeze({ value: rec.value, quality: rec.quality, timestamp: rec.timestamp, history: Object.freeze(rec.history.slice()) }) + } const rec = target[prop] return rec ? rec.value : undefined }, @@ -79,7 +82,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D rec = new Entry() target[prop] = rec } else { - rec.history.push({ value: cloneValue(rec.value), timestamp: rec.timestamp }) + rec.history.push(Object.freeze({ value: cloneValue(rec.value), timestamp: rec.timestamp })) if (rec.history.length > maxHistory) rec.history.shift() } const notify = (path) => { rec.timestamp = now(); onChange?.(prop, rec, path) } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 550b972c4..6347348fb 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -110,6 +110,51 @@ describe('store: reactive data store', function () { Reflect.ownKeys(store).filter((k) => k === '__proto__' || k === 'constructor').should.eql([]) }) + it('does not let a write through $ replace the stored value', function () { + const { store, log } = makeStore() + store.k = 1 + log.length = 0 + + store.$k.value = 'TAMPERED' + + store.k.should.equal(1) + log.should.eql([]) + }) + + it('does not let history grow past the cap through $', function () { + const { store } = makeStore({ maxHistory: 2 }) + store.k = 1 + store.k = 2 + store.k = 3 + + const push = function () { store.$k.history.push({ value: 'x', timestamp: 0 }) } + + push.should.throw() + store.$k.history.should.have.length(2) + }) + + it('does not let a stored history entry be rewritten through $', function () { + const { store } = makeStore() + store.k = 1 + store.k = 2 + + const write = function () { 'use strict'; store.$k.history[0].value = 'x' } + + write.should.throw() + store.$k.history[0].value.should.equal(1) + }) + + it('still reports a nested write made through the $ value', function () { + const { store, log } = makeStore() + store.k = { n: 1 } + log.length = 0 + + store.$k.value.n = 2 + + log.should.eql(['k.n']) + store.k.n.should.equal(2) + }) + it('reserves the $ prefix so keys are never silently lost', function () { const { store } = makeStore() store.$weird = 1 From ef8c5aa4e20a0859a3bb7fd070c04ee8a818777f Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 10:17:48 -0700 Subject: [PATCH 19/57] Wrap onChange in try catch --- nodes/store/reactive.js | 7 ++++--- test/store/reactive.spec.js | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 24b5f7fca..e5256d859 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -54,6 +54,7 @@ function deepReactive (value, notify, path, clone) { function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = Date.now } = {}) { const cloneValue = (v) => (v && typeof v === 'object') ? clone(v) : v + const emit = (prop, rec, path) => { try { onChange?.(prop, rec, path) } catch (err) {} } const records = Object.create(null) records[STORE] = true @@ -85,18 +86,18 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D rec.history.push(Object.freeze({ value: cloneValue(rec.value), timestamp: rec.timestamp })) if (rec.history.length > maxHistory) rec.history.shift() } - const notify = (path) => { rec.timestamp = now(); onChange?.(prop, rec, path) } + const notify = (path) => { rec.timestamp = now(); emit(prop, rec, path) } // clone on write so a flow reusing its own object can't mutate stored state rec.value = deepReactive(cloneValue(value), notify, prop, cloneValue) rec.timestamp = now() - onChange?.(prop, rec, prop) + emit(prop, rec, prop) return true }, ownKeys (target) { return Reflect.ownKeys(target) }, deleteProperty (target, prop) { const existed = prop in target const ok = delete target[prop] - if (existed) onChange?.(prop, undefined, prop) + if (existed) emit(prop, undefined, prop) return ok } } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 6347348fb..35311f709 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -110,6 +110,39 @@ describe('store: reactive data store', function () { Reflect.ownKeys(store).filter((k) => k === '__proto__' || k === 'constructor').should.eql([]) }) + it('does not let a subscriber error escape a write', function () { + const { store } = makeStore({ onChange: () => { throw new Error('boom') } }) + + const write = function () { store.k = 1 } + + write.should.not.throw() + store.k.should.equal(1) + }) + + it('does not let a subscriber error escape a nested write', function () { + let live = false + const { store } = makeStore({ onChange: () => { if (live) throw new Error('boom') } }) + store.k = { n: 1 } + live = true + + const write = function () { store.k.n = 2 } + + write.should.not.throw() + store.k.n.should.equal(2) + }) + + it('does not let a subscriber error escape a delete', function () { + let live = false + const { store } = makeStore({ onChange: () => { if (live) throw new Error('boom') } }) + store.k = 1 + live = true + + const remove = function () { delete store.k } + + remove.should.not.throw() + should(store.k).be.undefined() + }) + it('does not let a write through $ replace the stored value', function () { const { store, log } = makeStore() store.k = 1 From 89d6e794a9277638689a5ea5d318676f003461db Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 10:29:39 -0700 Subject: [PATCH 20/57] Guard deepReactive and deepClone against cycles so a self-referencing value can't blow the stack --- nodes/store/reactive.js | 19 ++++++++++++------- test/store/reactive.spec.js | 38 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index e5256d859..ee1269447 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -19,20 +19,21 @@ function isReactable (v) { } // reads through proxies (structuredClone rejects them); production injects RED.util.cloneMessage -function deepClone (v) { +function deepClone (v, seen = new WeakMap()) { if (v === null || typeof v !== 'object') return v if (Buffer.isBuffer(v)) return Buffer.from(v) if (v instanceof Date) return new Date(v) - if (Array.isArray(v)) return v.map(deepClone) - const out = {} - for (const k of Object.keys(v)) out[k] = deepClone(v[k]) + if (seen.has(v)) return seen.get(v) + const out = Array.isArray(v) ? [] : {} + seen.set(v, out) + for (const k of Object.keys(v)) out[k] = deepClone(v[k], seen) return out } -function deepReactive (value, notify, path, clone) { +function deepReactive (value, notify, path, clone, seen = new WeakMap()) { if (!isReactable(value)) return value - for (const k of Object.keys(value)) value[k] = deepReactive(value[k], notify, `${path}.${k}`, clone) - return new Proxy(value, { + if (seen.has(value)) return seen.get(value) + const proxy = new Proxy(value, { get (t, p, r) { return Reflect.get(t, p, r) }, set (t, p, v) { if (typeof p === 'symbol') return Reflect.set(t, p, v) @@ -50,6 +51,10 @@ function deepReactive (value, notify, path, clone) { return ok } }) + // registered before walking children so a cycle resolves to this proxy instead of recursing + seen.set(value, proxy) + for (const k of Object.keys(value)) value[k] = deepReactive(value[k], notify, `${path}.${k}`, clone, seen) + return proxy } function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = Date.now } = {}) { diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 35311f709..c7f533947 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -110,6 +110,44 @@ describe('store: reactive data store', function () { Reflect.ownKeys(store).filter((k) => k === '__proto__' || k === 'constructor').should.eql([]) }) + it('stores a self-referencing object without blowing the stack', function () { + const { store } = makeStore() + const a = { n: 1 } + a.self = a + + const write = function () { store.k = a } + + write.should.not.throw() + store.k.n.should.equal(1) + store.k.self.should.equal(store.k) + }) + + it('stores a mutually-referencing pair', function () { + const { store } = makeStore() + const a = { name: 'a' } + const b = { name: 'b', a } + a.b = b + + const write = function () { store.k = a } + + write.should.not.throw() + store.k.b.name.should.equal('b') + store.k.b.a.should.equal(store.k) + }) + + it('keeps a cyclic value reactive', function () { + const { store, log } = makeStore() + const a = { n: 1 } + a.self = a + store.k = a + log.length = 0 + + store.k.self.n = 2 + + log.should.eql(['k.n']) + store.k.n.should.equal(2) + }) + it('does not let a subscriber error escape a write', function () { const { store } = makeStore({ onChange: () => { throw new Error('boom') } }) From 4a0a14c6af2d03a534f8c8f37899fdda12bb1e7b Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 10:35:00 -0700 Subject: [PATCH 21/57] Register the store marker symbol globally so two module copies share one store --- nodes/store/reactive.js | 2 +- test/store/reactive.spec.js | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index ee1269447..0c6038648 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -1,4 +1,4 @@ -const STORE = Symbol('dashboardDataStore') +const STORE = Symbol.for('@flowfuse/node-red-dashboard/store') const NAMESPACE = 'dashboardStore' class Entry { diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index c7f533947..b7d8d31d1 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -397,6 +397,19 @@ describe('store: reactive data store', function () { }) describe('attachToContext', function () { + it('recognises a store injected by another copy of the module', function () { + const m = {} + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + // what a second copy of this module would have put in context + const foreign = { robot: 'from the other copy' } + foreign[Symbol.for('@flowfuse/node-red-dashboard/store')] = true + m.dashboardStore = foreign + + const store = attachToContext(g, {}) + + store.should.equal(foreign) + }) + function fakeGlobal () { const m = {} return { get: (k) => m[k], set: (k, v) => { m[k] = v } } From d39f273430c13ad8cb1ebc5e5784a4daf0c2200b Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 10:40:36 -0700 Subject: [PATCH 22/57] Only rehydrate a plain object so an array at the namespace isn't spread into numeric keys --- nodes/store/reactive.js | 2 +- test/store/reactive.spec.js | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 0c6038648..3e23f6423 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -117,7 +117,7 @@ function attachToContext (globalContext, opts = {}) { if (existing && existing[STORE]) return existing const store = createDataStore(opts) - if (existing && typeof existing === 'object') { + if (isReactable(existing) && !Array.isArray(existing)) { for (const [k, v] of Object.entries(existing)) store[k] = v } globalContext.set(namespace, store) diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index b7d8d31d1..78707cf7b 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -397,6 +397,36 @@ describe('store: reactive data store', function () { }) describe('attachToContext', function () { + it('does not rehydrate an array into numeric keys', function () { + const m = { dashboardStore: ['a', 'b'] } + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + + const store = attachToContext(g, {}) + + Object.keys(store).should.eql([]) + JSON.stringify(store).should.equal('{}') + }) + + it('does not spread a class instance into keys', function () { + const m = { dashboardStore: new Date() } + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + + const store = attachToContext(g, {}) + + Object.keys(store).should.eql([]) + }) + + it('still rehydrates a plain object', function () { + const m = { dashboardStore: { robot: { temp: 20 }, count: 3 } } + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + + const store = attachToContext(g, {}) + + store.count.should.equal(3) + store.robot.temp.should.equal(20) + store.$robot.timestamp.should.be.a.Number() + }) + it('recognises a store injected by another copy of the module', function () { const m = {} const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } From 6cd455e398fd01f941701961283aecb5171b4063 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 10:44:50 -0700 Subject: [PATCH 23/57] Reserve toJSON as a key so a stored value can never shadow the serialiser --- nodes/store/reactive.js | 2 +- test/store/reactive.spec.js | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 3e23f6423..731f74177 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -80,7 +80,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D set (target, prop, value) { if (typeof prop === 'symbol') return Reflect.set(target, prop, value) // no-op guards return true so strict-mode callers do not throw - if (prop === '__proto__' || prop === 'constructor') return true + if (prop === '__proto__' || prop === 'constructor' || prop === 'toJSON') return true if (prop.startsWith('$')) return true // '$' is reserved for the read-only meta view let rec = target[prop] diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 78707cf7b..ccedea39a 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -226,6 +226,19 @@ describe('store: reactive data store', function () { store.k.n.should.equal(2) }) + it('reserves toJSON so a key can never shadow the serialiser', function () { + const { store, log } = makeStore() + store.real = 1 + log.length = 0 + + store.toJSON = 5 + + log.should.eql([]) + should(store.$toJSON).be.undefined() + Object.keys(store).should.eql(['real']) + JSON.stringify(store).should.equal('{"real":1}') + }) + it('reserves the $ prefix so keys are never silently lost', function () { const { store } = makeStore() store.$weird = 1 From df4f7a96eebbf599cf74d01ee2380cceb42a0b7f Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 11:21:04 -0700 Subject: [PATCH 24/57] Ignore change events from a value that has already been replaced --- nodes/store/reactive.js | 10 +++++++-- test/store/reactive.spec.js | 42 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 731f74177..91965384f 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -91,9 +91,15 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D rec.history.push(Object.freeze({ value: cloneValue(rec.value), timestamp: rec.timestamp })) if (rec.history.length > maxHistory) rec.history.shift() } - const notify = (path) => { rec.timestamp = now(); emit(prop, rec, path) } + const notify = (path) => { + // a replaced value keeps its proxies alive; they must not report into the record any more + if (rec.value !== wrapped) return + rec.timestamp = now() + emit(prop, rec, path) + } // clone on write so a flow reusing its own object can't mutate stored state - rec.value = deepReactive(cloneValue(value), notify, prop, cloneValue) + const wrapped = deepReactive(cloneValue(value), notify, prop, cloneValue) + rec.value = wrapped rec.timestamp = now() emit(prop, rec, prop) return true diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index ccedea39a..f380c2431 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -110,6 +110,48 @@ describe('store: reactive data store', function () { Reflect.ownKeys(store).filter((k) => k === '__proto__' || k === 'constructor').should.eql([]) }) + it('does not report a write to a value that has been replaced', function () { + const { store, log } = makeStore() + store.k = { n: 1 } + const detached = store.k + store.k = { n: 2 } + log.length = 0 + const stamp = store.$k.timestamp + + detached.n = 999 + + log.should.eql([]) + detached.n.should.equal(999) + store.k.n.should.equal(2) + store.$k.timestamp.should.equal(stamp) + }) + + it('does not report a write to a child of a replaced value', function () { + const { store, log } = makeStore() + store.k = { child: { n: 1 } } + const detachedChild = store.k.child + store.k = { child: { n: 2 } } + log.length = 0 + + detachedChild.n = 999 + + log.should.eql([]) + detachedChild.n.should.equal(999) + store.k.child.n.should.equal(2) + }) + + it('still reports writes to the current value after a replacement', function () { + const { store, log } = makeStore() + store.k = { n: 1 } + store.k = { n: 2 } + log.length = 0 + + store.k.n = 3 + + log.should.eql(['k.n']) + store.k.n.should.equal(3) + }) + it('stores a self-referencing object without blowing the stack', function () { const { store } = makeStore() const a = { n: 1 } From 2409269124150c8ae40230b759dcd36349f6c7b1 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 22 Sep 2026 12:57:03 -0700 Subject: [PATCH 25/57] Cut comments --- nodes/store/reactive.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index b4dc6cf03..33a5e02e7 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -1,4 +1,3 @@ -// registered globally so a second copy of this module recognises a store the first one injected const STORE = Symbol.for('@flowfuse/node-red-dashboard/store') const SET_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/setEntry') const NAMESPACE = 'dashboardStore' @@ -105,7 +104,6 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D } if (prop.startsWith('$')) { const rec = target[prop.slice(1)] - // a projection, not the record: writing through $ would skip clone-on-write and onChange return rec && Object.freeze({ value: rec.value, msg: rec.msg, quality: rec.quality, timestamp: rec.timestamp, history: Object.freeze(rec.history.slice()) }) } const rec = target[prop] From d2e0331da663e0ef47981d1b70eb76740b4d0efb Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 09:50:12 -0700 Subject: [PATCH 26/57] Update check for isReactable" --- nodes/store/reactive.js | 2 +- test/store/reactive.spec.js | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index cf48b068f..59cbdafa5 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -15,7 +15,7 @@ function isReactable (v) { if (Array.isArray(v)) return true if (v === null || typeof v !== 'object') return false const proto = Object.getPrototypeOf(v) - return proto === Object.prototype || proto === null + return proto === null || Object.getPrototypeOf(proto) === null } // reads through proxies (structuredClone rejects them); production injects RED.util.cloneMessage diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 47e2a6126..13a187eb5 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -2,8 +2,11 @@ const fs = require('fs') const os = require('os') const path = require('path') +const vm = require('vm') + const LocalFileSystem = require('@node-red/runtime/lib/nodes/context/localfilesystem.js') const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') + const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars @@ -20,6 +23,10 @@ function makeStore (opts = {}) { return { store, log } } +// Node-RED runs function nodes in a vm context, so their objects carry that realm's Object.prototype +const fnRealm = vm.createContext({}) +const fnNodeObject = (src) => vm.runInContext(src, fnRealm) + describe('store: reactive data store', function () { describe('values and metadata', function () { it('stores a scalar, fires onChange, and reads it back as a value', function () { @@ -423,6 +430,17 @@ describe('store: reactive data store', function () { return { store, log } } + it('deep-watches an object built in a function node realm', function () { + const { store, log } = makeCloneStore() + store.w = fnNodeObject('({ nested: { temp: 20 } })') + log.length = 0 + + store.w.nested.temp = 25 + + log.should.eql(['w.nested.temp']) + store.w.nested.temp.should.equal(25) + }) + it('reassigning an object value fires only the key, no phantom req/res', function () { const { store, log } = makeCloneStore() store.robot = { temp: 20 } @@ -476,6 +494,16 @@ describe('store: reactive data store', function () { Object.keys(store).should.eql([]) }) + it('rehydrates an object a function node left at the namespace', function () { + const m = { dashboardStore: fnNodeObject('({ robot: "PLAIN", count: 3 })') } + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + + const store = attachToContext(g, {}) + + store.robot.should.equal('PLAIN') + store.count.should.equal(3) + }) + it('still rehydrates a plain object', function () { const m = { dashboardStore: { robot: { temp: 20 }, count: 3 } } const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } From cf1dfd678c9d26a03e27a8e09b1b55f48be0c99c Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 09:54:55 -0700 Subject: [PATCH 27/57] Update wording of warn text for onReplaced --- nodes/config/ui_base.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index 141f910f3..6ee0eccd0 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -406,7 +406,7 @@ module.exports = function (RED) { try { attachToContext(node.context().global, { clone: RED.util.cloneMessage, - onReplaced: () => node.warn('A flow replaced global.dashboardStore, discarding the stored value of every widget; the store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') + onReplaced: () => node.warn('global.dashboardStore was replaced by a flow since the last deploy, discarding the stored value of every widget. The store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') }) } catch (err) { node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') From fb3af541fdab4857776e9d93676eceed0307a773 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 10:39:49 -0700 Subject: [PATCH 28/57] Remove isMemoryBacked, add checks directly to attachToContext with specs --- nodes/config/ui_base.js | 21 ++++++++--------- nodes/store/reactive.js | 22 ++++++++---------- test/store/reactive.spec.js | 46 +++++++++++++++++++++++++++---------- 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index 6ee0eccd0..4e2afe9d1 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -6,7 +6,7 @@ const axios = require('axios') const v = require('../../package.json').version const { createClientStore } = require('../store/clients.js') const datastore = require('../store/data.js') -const { attachToContext, isInMemoryBacked } = require('../store/reactive.js') +const { attachToContext } = require('../store/reactive.js') const statestore = require('../store/state.js') const { appendTopic, addConnectionCredentials, matchesClient, normalizeClientId, getThirdPartyWidgets } = require('../utils/index.js') @@ -402,17 +402,16 @@ module.exports = function (RED) { node._created = Date.now() - if (isInMemoryBacked(node.context().global)) { - try { - attachToContext(node.context().global, { - clone: RED.util.cloneMessage, - onReplaced: () => node.warn('global.dashboardStore was replaced by a flow since the last deploy, discarding the stored value of every widget. The store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') - }) - } catch (err) { - node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') + try { + const store = attachToContext(node.context().global, { + clone: RED.util.cloneMessage, + onReplaced: () => node.warn('global.dashboardStore was replaced by a flow since the last deploy, discarding the stored value of every widget. The store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') + }) + if (!store) { + node.warn('Dashboard data store disabled: the global context store doesn\'t hold objects by reference (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.') } - } else { - node.warn('Dashboard data store disabled: the global context store isn\'t in-memory-backed (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.') + } catch (err) { + node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') } n.root = RED.settings.httpNodeRoot || '/' diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 59cbdafa5..4d7c6d9d7 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -115,23 +115,20 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D return new Proxy(records, handler) } -// A cache-off persistent store serves only async access, so a sync get throws; the reactive proxy needs an in-memory store. -function isInMemoryBacked (globalContext, namespace = NAMESPACE) { - try { - globalContext.get(namespace) - return true - } catch (err) { - return false - } -} - const injected = new WeakMap() // Inject the store into global context once. On redeploy the existing store is reused; // after a restart with a persistent context store, existing plain data is rehydrated. +// Returns null when the context store can't hold a live proxy: a cache-off store serves +// only async access so a sync get throws, and a serialising store hands back a copy. function attachToContext (globalContext, opts = {}) { const namespace = opts.namespace || NAMESPACE - const existing = globalContext.get(namespace) + let existing + try { + existing = globalContext.get(namespace) + } catch (err) { + return null + } if (existing && existing[STORE]) { injected.set(globalContext, existing) return existing @@ -145,8 +142,9 @@ function attachToContext (globalContext, opts = {}) { for (const [k, v] of Object.entries(existing)) store[k] = v } globalContext.set(namespace, store) + if (globalContext.get(namespace) !== store) return null injected.set(globalContext, store) return store } -module.exports = { createDataStore, attachToContext, isInMemoryBacked, NAMESPACE, STORE } +module.exports = { createDataStore, attachToContext, NAMESPACE, STORE } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 13a187eb5..ed92f6066 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -10,7 +10,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext, isInMemoryBacked, STORE } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, STORE } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -613,24 +613,46 @@ describe('store: reactive data store', function () { }) }) - describe('isInMemoryBacked', function () { - it('is true for an in-memory context whose get returns synchronously', function () { + describe('context stores that cannot hold a live proxy', function () { + it('attaches to an in-memory context', function () { const m = {} const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } - isInMemoryBacked(g).should.equal(true) + should(attachToContext(g, {})).not.be.null() }) - it('is false when a cache-off store throws on synchronous get', function () { + it('returns null when a cache-off store throws on synchronous get', function () { const g = { get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, set: () => {} } - isInMemoryBacked(g).should.equal(false) + should(attachToContext(g, {})).be.null() }) - // a real localfilesystem store, accessed the way the context manager's sync path does (store.get(scope, key)) + it('returns null for a store that hands back a serialised copy', function () { + const disk = {} + const g = { + get: (k) => disk[k] === undefined ? undefined : JSON.parse(disk[k]), + set: (k, v) => { disk[k] = JSON.stringify(v) } + } + should(attachToContext(g, {})).be.null() + }) + + it('never reports a replacement for a store it refused to attach to', function () { + const disk = {} + const g = { + get: (k) => disk[k] === undefined ? undefined : JSON.parse(disk[k]), + set: (k, v) => { disk[k] = JSON.stringify(v) } + } + let warned = 0 + + for (let deploy = 0; deploy < 3; deploy++) attachToContext(g, { onReplaced: () => { warned++ } }) + + warned.should.equal(0) + }) + + // a real localfilesystem store, accessed the way the context manager's sync path does function realGlobal (store) { - return { get: (key) => store.get('global', key) } + return { get: (key) => store.get('global', key), set: (key, value) => store.set('global', key, value) } } function tmpDir () { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-ctx-')) @@ -638,14 +660,14 @@ describe('store: reactive data store', function () { return dir } - it('is false for a real cache-off localfilesystem store', function () { + it('returns null for a real cache-off localfilesystem store', function () { const store = LocalFileSystem({ dir: tmpDir(), cache: false }) - isInMemoryBacked(realGlobal(store)).should.equal(false) + should(attachToContext(realGlobal(store), {})).be.null() }) - it('is true for a real cache-backed localfilesystem store', function () { + it('attaches to a real cache-backed localfilesystem store', function () { const store = LocalFileSystem({ dir: tmpDir(), cache: true }) - isInMemoryBacked(realGlobal(store)).should.equal(true) + should(attachToContext(realGlobal(store), {})).not.be.null() }) }) }) From 75c21bd36f127630f9076120032cc506fbaf0f1e Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 10:47:24 -0700 Subject: [PATCH 29/57] Clear value in deleteProperty --- nodes/store/reactive.js | 2 ++ test/store/reactive.spec.js | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 4d7c6d9d7..e6851eb80 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -106,8 +106,10 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D }, ownKeys (target) { return Reflect.ownKeys(target) }, deleteProperty (target, prop) { + const rec = target[prop] const existed = prop in target const ok = delete target[prop] + if (rec) rec.value = undefined if (existed) emit(prop, undefined, prop) return ok } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index ed92f6066..f19b0aa8c 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -138,6 +138,20 @@ describe('store: reactive data store', function () { store.$k.timestamp.should.equal(stamp) }) + it('does not report a write to a value whose key was deleted', function () { + const { store, log } = makeStore() + store.k = { n: 1 } + const held = store.k + delete store.k + log.length = 0 + + held.n = 99 + + log.should.eql([]) + held.n.should.equal(99) + should(store.k).be.undefined() + }) + it('does not report a write to a child of a replaced value', function () { const { store, log } = makeStore() store.k = { child: { n: 1 } } From e8fb8970363e3ec18db29cbbcbe7d854977e9d13 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 11:08:21 -0700 Subject: [PATCH 30/57] Update comment for reserved names --- nodes/store/reactive.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index e6851eb80..f1ff2c6ec 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -79,7 +79,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D }, set (target, prop, value) { if (typeof prop === 'symbol') return Reflect.set(target, prop, value) - // no-op guards return true so strict-mode callers do not throw + // returning true rather than false so a strict caller picking a reserved name doesn't throw if (prop === '__proto__' || prop === 'constructor' || prop === 'toJSON') return true if (prop.startsWith('$')) return true // '$' is reserved for the read-only meta view From c891745d5ce75673bf603bb5e944ba1873e0c11a Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 12:25:49 -0700 Subject: [PATCH 31/57] Swap default to false for storeEnabled --- nodes/store/data.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index 25a5d97b9..0e82a0883 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -9,7 +9,7 @@ const config = { } let storeOptions = {} -let storeEnabled = true +let storeEnabled = false function getOrCreateStore (globalContext) { return attachToContext(globalContext, { clone: config.RED.util.cloneMessage, ...storeOptions }) From 424b9dc35747bc498de5162b9680f7cf00ed79ff Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 12:52:01 -0700 Subject: [PATCH 32/57] Add deepFreeze to msg and history --- nodes/store/reactive.js | 11 ++++++++-- test/store/reactive.spec.js | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 91c7a3495..5d4798059 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -2,6 +2,13 @@ const STORE = Symbol.for('@flowfuse/node-red-dashboard/store') const SET_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/setEntry') const NAMESPACE = 'dashboardStore' +function deepFreeze (v) { + if (!isReactable(v) || Object.isFrozen(v)) return v + Object.freeze(v) + for (const k of Object.keys(v)) deepFreeze(v[k]) + return v +} + class Entry { constructor () { this.value = undefined @@ -74,7 +81,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D records[prop] = rec } else if (!Array.isArray(rec.value)) { // snapshotting a whole array on every write (e.g. a table's rows) would blow up memory - rec.history.push(Object.freeze({ value: cloneValue(rec.value), timestamp: rec.timestamp })) + rec.history.push(Object.freeze({ value: deepFreeze(cloneValue(rec.value)), timestamp: rec.timestamp })) if (rec.history.length > maxHistory) rec.history.shift() } const notify = (path) => { @@ -92,7 +99,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D records[SET_ENTRY] = (prop, value, msg) => { const rec = writeValue(prop, value) - rec.msg = msg + rec.msg = deepFreeze(msg) emit(prop, rec, prop) } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 1bca01605..b031d1553 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -1,4 +1,5 @@ const fs = require('fs') +const http = require('http') const os = require('os') const path = require('path') @@ -363,6 +364,49 @@ describe('store: reactive data store', function () { store.$k.history[0].value.should.equal(1) }) + it('does not let the stored msg be rewritten through $', function () { + const { store, log } = makeStore() + store[SET_ENTRY]('k', 42, { payload: 42, topic: 'boiler' }) + log.length = 0 + + const write = function () { 'use strict'; store.$k.msg.payload = 'TAMPERED' } + + write.should.throw() + store.$k.msg.payload.should.equal(42) + log.should.be.empty() + }) + + it('does not let a nested property of the stored msg be rewritten through $', function () { + const { store } = makeStore() + store[SET_ENTRY]('k', { deep: { n: 1 } }, { payload: { deep: { n: 1 } } }) + + const write = function () { 'use strict'; store.$k.msg.payload.deep.n = 'TAMPERED' } + + write.should.throw() + store.$k.msg.payload.deep.n.should.equal(1) + }) + + it('does not let a nested property of a history entry be rewritten through $', function () { + const { store } = makeStore() + store.k = { n: 1 } + store.k = { n: 2 } + + const write = function () { 'use strict'; store.$k.history[0].value.n = 'TAMPERED' } + + write.should.throw() + store.$k.history[0].value.n.should.equal(1) + }) + + it('leaves msg.req and msg.res usable, since cloneMessage keeps them as live handles', function () { + const { store } = makeStore() + const req = new http.IncomingMessage({ fake: 'socket' }) + const res = new http.ServerResponse(req) + store[SET_ENTRY]('k', 1, { payload: 1, req, res }) + + Object.isFrozen(req).should.be.false() + res.setHeader.bind(res, 'x-test', '1').should.not.throw() + }) + it('still reports a nested write made through the $ value', function () { const { store, log } = makeStore() store.k = { n: 1 } From 3c05e874ede09f44471840e5487c6bfef4a1da29 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 13:03:26 -0700 Subject: [PATCH 33/57] Synthesise msg from the value in writeValue so no write path leaves a stale msg --- nodes/store/reactive.js | 1 + test/store/reactive.spec.js | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 5d4798059..9879ce8c0 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -93,6 +93,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D // clone on write so a flow reusing its own object can't mutate stored state const wrapped = deepReactive(cloneValue(value), notify, prop, cloneValue) rec.value = wrapped + rec.msg = Object.freeze({ payload: wrapped }) rec.timestamp = now() return rec } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index b031d1553..fbeef3312 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -45,11 +45,22 @@ describe('store: reactive data store', function () { store.$k1.timestamp.should.be.a.Number() }) - it('exposes an empty msg field on a new entry', function () { + it('synthesises a msg from the value when a flow writes the key directly', function () { const { store } = makeStore() store.k1 = 123 - store.$k1.should.have.property('msg') - should(store.$k1.msg).be.undefined() + store.$k1.msg.should.eql({ payload: 123 }) + store.$k1.msg.payload.should.equal(store.$k1.value) + }) + + it('does not leave a msg describing the previous value after a flow write', function () { + const { store } = makeStore() + store[SET_ENTRY]('k1', 42, { payload: 42, topic: 'boiler' }) + store.$k1.msg.topic.should.equal('boiler') + + store.k1 = 99 + + store.$k1.value.should.equal(99) + store.$k1.msg.should.eql({ payload: 99 }) }) it('returns the whole object by default', function () { From a09492c400abdb34054863c196c6a1e083b40322 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 23 Sep 2026 13:25:36 -0700 Subject: [PATCH 34/57] Derive the stored value from the cloned msg in SET_ENTRY so one clone serves both --- nodes/store/data.js | 3 +-- nodes/store/reactive.js | 12 +++++----- test/store/reactive.spec.js | 47 +++++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index 224278ecd..d90a67b52 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -29,8 +29,7 @@ function writeToStore (node, msg, stored) { if (!storeEnabled) return if (!msg || typeof msg !== 'object' || !('payload' in msg)) return try { - const clone = config.RED.util.cloneMessage - getOrCreateStore(node.context().global)[SET_ENTRY](node.id, msg.payload, clone(stored)) + getOrCreateStore(node.context().global)[SET_ENTRY](node.id, stored) } catch (err) {} } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 9879ce8c0..6ae5827db 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -90,17 +90,17 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D rec.timestamp = now() emit(prop, rec, path) } - // clone on write so a flow reusing its own object can't mutate stored state - const wrapped = deepReactive(cloneValue(value), notify, prop, cloneValue) + const wrapped = deepReactive(value, notify, prop, cloneValue) rec.value = wrapped rec.msg = Object.freeze({ payload: wrapped }) rec.timestamp = now() return rec } - records[SET_ENTRY] = (prop, value, msg) => { - const rec = writeValue(prop, value) - rec.msg = deepFreeze(msg) + records[SET_ENTRY] = (prop, msg) => { + const { payload, ...rest } = cloneValue(msg) + const rec = writeValue(prop, payload) + rec.msg = Object.freeze({ ...deepFreeze(rest), payload: rec.value }) emit(prop, rec, prop) } @@ -123,7 +123,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (prop === '__proto__' || prop === 'constructor' || prop === 'toJSON') return true if (prop.startsWith('$')) return true // '$' is reserved for the read-only meta view - const rec = writeValue(prop, value) + const rec = writeValue(prop, cloneValue(value)) emit(prop, rec, prop) return true }, diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index fbeef3312..846415483 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -54,7 +54,7 @@ describe('store: reactive data store', function () { it('does not leave a msg describing the previous value after a flow write', function () { const { store } = makeStore() - store[SET_ENTRY]('k1', 42, { payload: 42, topic: 'boiler' }) + store[SET_ENTRY]('k1', { payload: 42, topic: 'boiler' }) store.$k1.msg.topic.should.equal('boiler') store.k1 = 99 @@ -75,7 +75,7 @@ describe('store: reactive data store', function () { const { store, log } = makeStore() const msg = { payload: 42, topic: 'sensor-A' } - store[SET_ENTRY]('gauge', msg.payload, msg) + store[SET_ENTRY]('gauge', msg) store.gauge.should.equal(42) store.$gauge.msg.should.eql({ payload: 42, topic: 'sensor-A' }) @@ -84,19 +84,29 @@ describe('store: reactive data store', function () { it('keeps history on the value, as a normal write does', function () { const { store } = makeStore() - store[SET_ENTRY]('gauge', 1, { payload: 1 }) - store[SET_ENTRY]('gauge', 2, { payload: 2 }) + store[SET_ENTRY]('gauge', { payload: 1 }) + store[SET_ENTRY]('gauge', { payload: 2 }) store.gauge.should.equal(2) store.$gauge.history.map((h) => h.value).should.eql([1]) }) - it('does not clone the message it is given', function () { + it('clones the message it is given, so a caller reusing it cannot mutate stored state', function () { const { store } = makeStore() const msg = { payload: 1, nested: { a: 1 } } - store[SET_ENTRY]('k', msg.payload, msg) + store[SET_ENTRY]('k', msg) - should(store.$k.msg).equal(msg) + should(store.$k.msg).not.equal(msg) + msg.nested.a = 'MUTATED' + store.$k.msg.nested.a.should.equal(1) + }) + + it('serves the stored value as the payload of the stored msg', function () { + const { store } = makeStore() + store[SET_ENTRY]('k', { payload: { n: 1 }, topic: 't' }) + + store.$k.msg.payload.should.equal(store.$k.value) + store.k.n.should.equal(1) }) }) @@ -377,7 +387,7 @@ describe('store: reactive data store', function () { it('does not let the stored msg be rewritten through $', function () { const { store, log } = makeStore() - store[SET_ENTRY]('k', 42, { payload: 42, topic: 'boiler' }) + store[SET_ENTRY]('k', { payload: 42, topic: 'boiler' }) log.length = 0 const write = function () { 'use strict'; store.$k.msg.payload = 'TAMPERED' } @@ -387,14 +397,25 @@ describe('store: reactive data store', function () { log.should.be.empty() }) - it('does not let a nested property of the stored msg be rewritten through $', function () { + it('does not let a non-payload property of the stored msg be rewritten through $', function () { const { store } = makeStore() - store[SET_ENTRY]('k', { deep: { n: 1 } }, { payload: { deep: { n: 1 } } }) + store[SET_ENTRY]('k', { payload: 1, meta: { deep: { n: 1 } } }) - const write = function () { 'use strict'; store.$k.msg.payload.deep.n = 'TAMPERED' } + const write = function () { 'use strict'; store.$k.msg.meta.deep.n = 'TAMPERED' } write.should.throw() - store.$k.msg.payload.deep.n.should.equal(1) + store.$k.msg.meta.deep.n.should.equal(1) + }) + + it('reports a nested write made through the $ msg payload, since it is the value', function () { + const { store, log } = makeStore() + store[SET_ENTRY]('k', { payload: { deep: { n: 1 } } }) + log.length = 0 + + store.$k.msg.payload.deep.n = 2 + + log.should.eql(['k.deep.n']) + store.k.deep.n.should.equal(2) }) it('does not let a nested property of a history entry be rewritten through $', function () { @@ -412,7 +433,7 @@ describe('store: reactive data store', function () { const { store } = makeStore() const req = new http.IncomingMessage({ fake: 'socket' }) const res = new http.ServerResponse(req) - store[SET_ENTRY]('k', 1, { payload: 1, req, res }) + store[SET_ENTRY]('k', { payload: 1, req, res }) Object.isFrozen(req).should.be.false() res.setHeader.bind(res, 'x-test', '1').should.not.throw() From 57c5fe08c4cb35bdecc39d5436d87edf0edf8f2b Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 25 Sep 2026 10:13:25 -0700 Subject: [PATCH 35/57] Update fn names, protect against malformed append, extract helpers --- nodes/store/data.js | 8 +++----- nodes/store/reactive.js | 34 +++++++++++++++------------------- test/store/reactive.spec.js | 11 +++++++++++ 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index c3e8597d2..ad14621fe 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -20,9 +20,7 @@ function getOrCreateStore (globalContext) { function initStore (globalContext, opts) { storeOptions = { ...opts } storeEnabled = true - const store = getOrCreateStore(globalContext) - storeEnabled = !!store - return store + return getOrCreateStore(globalContext) } function disableStore () { @@ -52,7 +50,7 @@ function replaceInStore (node, msgs) { } catch (err) {} } -function trimStore (node, msgs) { +function trimInStore (node, msgs) { if (!storeEnabled) return trimCounts[node.id] = (trimCounts[node.id] || 0) + 1 if (trimCounts[node.id] < TRIM_BATCH) return @@ -182,7 +180,7 @@ const setters = { if (filteredMessages.length !== currentData.length) { // no need for save operation to process messages - just apply them data[node.id] = filteredMessages - trimStore(node, filteredMessages) + trimInStore(node, filteredMessages) } } } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 09210717d..3fb24b73f 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -108,11 +108,14 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D const pointsOf = (d) => (d === undefined || d === null) ? [] : (Array.isArray(d) ? d : [d]) - // the stored msg keeps _datapoint pointing at the live series, the same way an entry's msg.payload is its value - const storeMessage = (owned, live) => { + const toStoredDatapoint = (series, datapoint, start, end) => { + if (end === start) return undefined + return Array.isArray(datapoint) ? Object.freeze(series.slice(start, end)) : series[start] + } + + const toStoredMessage = (owned, datapoint) => { for (const k of Object.keys(owned)) if (k !== '_datapoint') deepFreeze(owned[k]) - // _datapoint is left unfrozen only while it IS the live series; otherwise it is ordinary stored data - if (live !== undefined) owned._datapoint = live + if (datapoint !== undefined) owned._datapoint = datapoint else deepFreeze(owned._datapoint) return Object.freeze(owned) } @@ -126,33 +129,26 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D return [start, points.length] }) const rec = writeValue(prop, points) - rec.msg = owned.map((m, i) => { - const [start, end] = spans[i] - if (end === start) return storeMessage(m) - return storeMessage(m, Array.isArray(m._datapoint) ? Object.freeze(rec.value.slice(start, end)) : rec.value[start]) - }) + rec.msg = owned.map((m, i) => toStoredMessage(m, toStoredDatapoint(rec.value, m._datapoint, ...spans[i]))) emit(prop, rec, prop) } records[APPEND_ENTRY] = (prop, msg) => { let rec = records[prop] - // a flow can leave any shape at a chart's key; re-seed rather than throwing on every append from then on - if (!rec || !Array.isArray(rec.value) || !Array.isArray(rec.msg)) { + const replaced = !!rec && !(Array.isArray(rec.value) && Array.isArray(rec.msg)) + if (!rec || replaced) { rec = writeValue(prop, []) rec.msg = [] } - // the points are pushed through the proxy, which clones them, so only the rest of the message needs cloning here + // the proxy clones on push, so points must not be cloned again here const { _datapoint, ...rest } = msg const owned = cloneValue(rest) const start = rec.value.length for (const p of pointsOf(_datapoint)) rec.value.push(p) - const end = rec.value.length - if (end > start) { - rec.msg.push(storeMessage(owned, Array.isArray(_datapoint) ? Object.freeze(rec.value.slice(start, end)) : rec.value[start])) - } else { - if ('_datapoint' in msg) owned._datapoint = cloneValue(_datapoint) - rec.msg.push(storeMessage(owned)) - } + const stored = toStoredDatapoint(rec.value, _datapoint, start, rec.value.length) + if (stored === undefined && '_datapoint' in msg) owned._datapoint = cloneValue(_datapoint) + rec.msg.push(toStoredMessage(owned, stored)) + if (replaced) emit(prop, rec, prop) } const handler = { diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index e0454367a..2af120e2d 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -268,6 +268,17 @@ describe('store: reactive data store', function () { store.$c1.msg.should.have.length(2) }) + it('reports discarding whatever a flow left at the key', function () { + const { store, log } = makeStore() + store.c1 = { not: 'a series' } + log.length = 0 + + store[APPEND_ENTRY]('c1', { payload: 1 }) + + store.c1.should.eql([]) + log.should.eql(['c1']) + }) + it('does not let a multi-point datapoint be resized through $', function () { const { store } = makeStore() store[SET_SERIES]('c1', [{ payload: 1, _datapoint: [pt(1, 1), pt(2, 2)] }]) From 8fddd989c7ca6b8f98618c04d305337553759b23 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 25 Sep 2026 12:09:57 -0700 Subject: [PATCH 36/57] Clone the points apart from the rest in SET_SERIES so freezing a message can't freeze a live point it shares --- nodes/store/reactive.js | 16 ++++++++++----- test/store/reactive.spec.js | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 3fb24b73f..07f0826f3 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -121,15 +121,21 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D } records[SET_SERIES] = (prop, msgs) => { - const owned = msgs.map((m) => cloneValue(m)) const points = [] - const spans = owned.map((m) => { + const parts = msgs.map((msg) => { + const { _datapoint, ...rest } = msg + const owned = cloneValue(rest) + const datapoint = cloneValue(_datapoint) const start = points.length - for (const p of pointsOf(m._datapoint)) points.push(p) - return [start, points.length] + for (const p of pointsOf(datapoint)) points.push(p) + return { msg, owned, datapoint, start, end: points.length } }) const rec = writeValue(prop, points) - rec.msg = owned.map((m, i) => toStoredMessage(m, toStoredDatapoint(rec.value, m._datapoint, ...spans[i]))) + rec.msg = parts.map(({ msg, owned, datapoint, start, end }) => { + const stored = toStoredDatapoint(rec.value, datapoint, start, end) + if (stored === undefined && '_datapoint' in msg) owned._datapoint = datapoint + return toStoredMessage(owned, stored) + }) emit(prop, rec, prop) } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 2af120e2d..34b44b749 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -279,6 +279,45 @@ describe('store: reactive data store', function () { log.should.eql(['c1']) }) + it('does not freeze a point a message reaches through another key', function () { + for (const write of [ + (store, msg) => store[SET_SERIES]('c1', [msg]), + (store, msg) => store[APPEND_ENTRY]('c1', msg) + ]) { + const { store } = makeStore({ clone: util.cloneMessage }) + const payload = { reading: { v: 1 } } + write(store, { topic: 't', payload, _datapoint: { category: 'a', x: 1, y: payload.reading } }) + + Object.isFrozen(store.c1[0].y).should.be.false() + store.c1[0].y.v = 99 + store.c1[0].y.v.should.equal(99) + } + }) + + it('freezes the stored series message itself', function () { + const { store } = makeStore() + store[APPEND_ENTRY]('c1', { topic: 't', payload: 1, _datapoint: pt(1, 1) }) + + Object.isFrozen(store.$c1.msg[0]).should.be.true() + const write = function () { 'use strict'; store.$c1.msg[0].topic = 'TAMPERED' } + + write.should.throw() + store.$c1.msg[0].topic.should.equal('t') + }) + + it('keeps an empty or null datapoint on the stored message', function () { + for (const datapoint of [null, []]) { + const { store } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 'x', _datapoint: datapoint }) + store[SET_SERIES]('c2', [{ payload: 'x', _datapoint: datapoint }]) + + store.$c1.msg[0].should.have.property('_datapoint') + store.$c2.msg[0].should.have.property('_datapoint') + should(store.$c1.msg[0]._datapoint).eql(datapoint) + should(store.$c2.msg[0]._datapoint).eql(datapoint) + } + }) + it('does not let a multi-point datapoint be resized through $', function () { const { store } = makeStore() store[SET_SERIES]('c1', [{ payload: 1, _datapoint: [pt(1, 1), pt(2, 2)] }]) From 33ed22eac31a55acf2668d72e4b86e9bedc865b8 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 25 Sep 2026 12:28:21 -0700 Subject: [PATCH 37/57] Move trimcounts out of try and add guards --- nodes/store/data.js | 4 +++- test/store/data.spec.js | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index ad14621fe..fdb65b973 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -37,6 +37,7 @@ function writeToStore (node, msg, stored) { function appendToStore (node, msg) { if (!storeEnabled) return + if (!msg || typeof msg !== 'object') return try { getOrCreateStore(node.context().global)[APPEND_ENTRY](node.id, msg) } catch (err) {} @@ -44,6 +45,7 @@ function appendToStore (node, msg) { function replaceInStore (node, msgs) { if (!storeEnabled) return + if (!Array.isArray(msgs) || msgs.some((m) => !m || typeof m !== 'object')) return try { delete trimCounts[node.id] getOrCreateStore(node.context().global)[SET_SERIES](node.id, msgs) @@ -59,9 +61,9 @@ function trimInStore (node, msgs) { function clearFromStore (node) { if (!storeEnabled) return + delete trimCounts[node.id] try { delete getOrCreateStore(node.context().global)[node.id] - delete trimCounts[node.id] } catch (err) {} } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 1dcaae940..738d676b9 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -385,6 +385,43 @@ describe('store: data.js chart writes', function () { store['$chart-batch'].msg.should.have.length(40) }) + it('does not create a key when the appended message is not an object', function () { + const global = fakeGlobal() + const node = fakeNode('chart-junk', global) + datastore.append(base, fakeNode('chart-ok', global), { payload: 1, _datapoint: pt(1, 1) }) + + datastore.append(base, node, 5) + datastore.save(base, node, [5, { payload: 1, _datapoint: pt(1, 1) }]) + + should(global.get('dashboardStore')['chart-junk']).be.undefined() + global.get('dashboardStore')['chart-ok'].should.have.length(1) + }) + + it('clears the trim batch even when the store cannot be reached', function () { + const global = fakeGlobal() + const node = fakeNode('chart-cleared', global) + for (let i = 0; i < 100; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + for (let i = 0; i < 59; i++) { + datastore.filter(base, node, (m, idx) => idx > 0) + } + + const realGet = global.get + global.get = () => { throw new Error('context store unavailable') } + datastore.clear(node.id) + datastore.clearFromStore(node) + global.get = realGet + + for (let i = 0; i < 100; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + const before = global.get('dashboardStore')['chart-cleared'].length + datastore.filter(base, node, (m, idx) => idx > 0) + + global.get('dashboardStore')['chart-cleared'].should.have.length(before) + }) + it('resets the trim batch even when the rebuild fails, so it does not retry on every trim', function () { const global = fakeGlobal() const node = fakeNode('chart-stuck', global) From 870c1bc70b145189ed9f69fc742cd2f124068452 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 25 Sep 2026 12:46:48 -0700 Subject: [PATCH 38/57] Ensure we are not freezing req + res --- nodes/store/reactive.js | 24 ++++++++++++++++-------- test/store/reactive.spec.js | 16 +++++++++++----- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 6ae5827db..c74928a3a 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -2,13 +2,6 @@ const STORE = Symbol.for('@flowfuse/node-red-dashboard/store') const SET_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/setEntry') const NAMESPACE = 'dashboardStore' -function deepFreeze (v) { - if (!isReactable(v) || Object.isFrozen(v)) return v - Object.freeze(v) - for (const k of Object.keys(v)) deepFreeze(v[k]) - return v -} - class Entry { constructor () { this.value = undefined @@ -66,6 +59,21 @@ function deepReactive (value, notify, path, clone, seen = new WeakMap()) { return proxy } +function deepFreeze (v) { + if (!isReactable(v) || Object.isFrozen(v)) return v + Object.freeze(v) + for (const k of Object.keys(v)) deepFreeze(v[k]) + return v +} + +function freezeOwned (owned) { + for (const k of Object.keys(owned)) { + if (k === 'req' || k === 'res') continue + deepFreeze(owned[k]) + } + return owned +} + function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = Date.now } = {}) { const cloneValue = (v) => (v && typeof v === 'object') ? clone(v) : v const emit = (prop, rec, path) => { try { onChange?.(prop, rec, path) } catch (err) {} } @@ -100,7 +108,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D records[SET_ENTRY] = (prop, msg) => { const { payload, ...rest } = cloneValue(msg) const rec = writeValue(prop, payload) - rec.msg = Object.freeze({ ...deepFreeze(rest), payload: rec.value }) + rec.msg = Object.freeze({ ...freezeOwned(rest), payload: rec.value }) emit(prop, rec, prop) } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 846415483..15e79d0f9 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -429,14 +429,20 @@ describe('store: reactive data store', function () { store.$k.history[0].value.n.should.equal(1) }) - it('leaves msg.req and msg.res usable, since cloneMessage keeps them as live handles', function () { - const { store } = makeStore() + it('leaves msg.req and msg.res usable, since cloneMessage keeps them by reference', function () { + // msg.res is an object literal in Node-RED (createResponseWrapper), not a class instance, + // so isReactable accepts it and nothing but an explicit exemption keeps it writable + const { store } = makeStore({ clone: util.cloneMessage }) const req = new http.IncomingMessage({ fake: 'socket' }) - const res = new http.ServerResponse(req) - store[SET_ENTRY]('k', { payload: 1, req, res }) + const res = { _res: new http.ServerResponse(req), set () {}, status () {} } + const msg = { payload: 1, req, res } + store[SET_ENTRY]('k', msg) + store.$k.msg.req.should.equal(req) Object.isFrozen(req).should.be.false() - res.setHeader.bind(res, 'x-test', '1').should.not.throw() + Object.isFrozen(res).should.be.false() + res.statusCode = 404 + res.statusCode.should.equal(404) }) it('still reports a nested write made through the $ value', function () { From 35578b3b7884073d855f00554247310f2c61efce Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Mon, 28 Sep 2026 07:30:08 -0700 Subject: [PATCH 39/57] Add error catch, reflect back symbol --- nodes/config/ui_base.js | 4 +++- nodes/store/reactive.js | 2 ++ test/store/reactive.spec.js | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index 4e2afe9d1..e99e696d0 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -403,12 +403,14 @@ module.exports = function (RED) { node._created = Date.now() try { + let unavailable = null const store = attachToContext(node.context().global, { clone: RED.util.cloneMessage, + onUnavailable: (err) => { unavailable = err }, onReplaced: () => node.warn('global.dashboardStore was replaced by a flow since the last deploy, discarding the stored value of every widget. The store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') }) if (!store) { - node.warn('Dashboard data store disabled: the global context store doesn\'t hold objects by reference (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.') + node.warn('Dashboard data store disabled: the global context store doesn\'t hold objects by reference (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.' + (unavailable ? ' Reading it failed with: ' + unavailable.message : '')) } } catch (err) { node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index f1ff2c6ec..14bb28b10 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -106,6 +106,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D }, ownKeys (target) { return Reflect.ownKeys(target) }, deleteProperty (target, prop) { + if (typeof prop === 'symbol') return Reflect.deleteProperty(target, prop) const rec = target[prop] const existed = prop in target const ok = delete target[prop] @@ -129,6 +130,7 @@ function attachToContext (globalContext, opts = {}) { try { existing = globalContext.get(namespace) } catch (err) { + opts.onUnavailable?.(err) return null } if (existing && existing[STORE]) { diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index f19b0aa8c..b909b1c8d 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -294,6 +294,16 @@ describe('store: reactive data store', function () { store.k.n.should.equal(2) }) + it('passes a symbol delete straight through, like get and set do', function () { + const { store, log } = makeStore() + store.k = 1 + log.length = 0 + + delete store[STORE] + + log.should.be.empty() + }) + it('reserves toJSON so a key can never shadow the serialiser', function () { const { store, log } = makeStore() store.real = 1 @@ -634,6 +644,29 @@ describe('store: reactive data store', function () { should(attachToContext(g, {})).not.be.null() }) + it('reports why the context store could not be read', function () { + let reason = null + const globalContext = { + get () { throw new Error('ECONNREFUSED: context store unreachable') }, + set () {} + } + + const store = attachToContext(globalContext, { onUnavailable: (err) => { reason = err } }) + + should(store).be.null() + reason.message.should.equal('ECONNREFUSED: context store unreachable') + }) + + it('does not report a reason when the context store reads fine', function () { + let reason = null + const m = {} + const globalContext = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + + attachToContext(globalContext, { onUnavailable: (err) => { reason = err } }) + + should(reason).be.null() + }) + it('returns null when a cache-off store throws on synchronous get', function () { const g = { get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, From 708d32798fb05d07922ac7f847f84b78023899a9 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 29 Sep 2026 08:53:21 -0700 Subject: [PATCH 40/57] Add seperate clones to prevent payload freezing --- nodes/store/reactive.js | 6 +++--- test/store/reactive.spec.js | 12 ++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 9f0e60b9e..5803968e6 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -106,9 +106,9 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D } records[SET_ENTRY] = (prop, msg) => { - const { payload, ...rest } = cloneValue(msg) - const rec = writeValue(prop, payload) - rec.msg = Object.freeze({ ...freezeOwned(rest), payload: rec.value }) + const { payload, ...rest } = msg + const rec = writeValue(prop, cloneValue(payload)) + rec.msg = Object.freeze({ ...freezeOwned(cloneValue(rest)), payload: rec.value }) emit(prop, rec, prop) } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index bd188e6d2..68e6e4037 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -429,6 +429,18 @@ describe('store: reactive data store', function () { store.$k.history[0].value.n.should.equal(1) }) + it('does not freeze the payload when the message references it under another key', function () { + // a change node copies by reference unless deep copy is ticked, so msg.copy === msg.payload + const { store } = makeStore({ clone: util.cloneMessage }) + const payload = { temp: 25 } + store[SET_ENTRY]('k', { payload, copy: payload, topic: 't' }) + + Object.isFrozen(store.k).should.be.false() + store.k.temp = 26 + store.k.temp.should.equal(26) + Object.isFrozen(store.$k.msg.copy).should.be.true() + }) + it('leaves msg.req and msg.res usable, since cloneMessage keeps them by reference', function () { // msg.res is an object literal in Node-RED (createResponseWrapper), not a class instance, // so isReactable accepts it and nothing but an explicit exemption keeps it writable From 59af60e23ef74fe0ac6098f780b5c34db2b6c1db Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 1 Oct 2026 06:38:00 -0700 Subject: [PATCH 41/57] Add error handling for methods to node.warn --- nodes/store/data.js | 27 +++++++--- nodes/store/reactive.js | 2 +- test/store/data.spec.js | 105 +++++++++++++++++++++++++++++++++++- test/store/reactive.spec.js | 15 ++++++ 4 files changed, 140 insertions(+), 9 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index fdb65b973..55885a184 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -12,15 +12,30 @@ let storeOptions = {} let storeEnabled = false const trimCounts = {} const TRIM_BATCH = 60 +let warned = false + +function warnOnce (node, problem) { + if (warned) return + warned = true + node.warn(`Dashboard data store: ${problem}. Further store failures are not reported until the next full deploy or restart.`) +} function getOrCreateStore (globalContext) { - return attachToContext(globalContext, { clone: config.RED.util.cloneMessage, ...storeOptions }) + let unavailable = null + const store = attachToContext(globalContext, { + clone: config.RED.util.cloneMessage, + ...storeOptions, + onUnavailable: (err) => { unavailable = err } + }) + if (!store) throw unavailable || new Error('global context saved a copy of the store rather than the store itself, which happens with context stores that serialise values') + return store } function initStore (globalContext, opts) { storeOptions = { ...opts } storeEnabled = true - return getOrCreateStore(globalContext) + warned = false + return attachToContext(globalContext, { clone: config.RED.util.cloneMessage, ...storeOptions }) } function disableStore () { @@ -32,7 +47,7 @@ function writeToStore (node, msg, stored) { if (!msg || typeof msg !== 'object' || !('payload' in msg)) return try { getOrCreateStore(node.context().global)[SET_ENTRY](node.id, stored) - } catch (err) {} + } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function appendToStore (node, msg) { @@ -40,7 +55,7 @@ function appendToStore (node, msg) { if (!msg || typeof msg !== 'object') return try { getOrCreateStore(node.context().global)[APPEND_ENTRY](node.id, msg) - } catch (err) {} + } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function replaceInStore (node, msgs) { @@ -49,7 +64,7 @@ function replaceInStore (node, msgs) { try { delete trimCounts[node.id] getOrCreateStore(node.context().global)[SET_SERIES](node.id, msgs) - } catch (err) {} + } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function trimInStore (node, msgs) { @@ -64,7 +79,7 @@ function clearFromStore (node) { delete trimCounts[node.id] try { delete getOrCreateStore(node.context().global)[node.id] - } catch (err) {} + } catch (err) { warnOnce(node, `clearing a store entry failed (${err.message}), so global.dashboardStore may still hold this widget's old value`) } } /** diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 0fbd6e665..852f27ccc 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -225,7 +225,6 @@ function attachToContext (globalContext, opts = {}) { } const previous = injected.get(globalContext) - if (previous && existing !== previous) opts.onReplaced?.() const store = createDataStore(opts) if (isReactable(existing) && !Array.isArray(existing)) { @@ -233,6 +232,7 @@ function attachToContext (globalContext, opts = {}) { } globalContext.set(namespace, store) if (globalContext.get(namespace) !== store) return null + if (previous && existing !== previous) opts.onReplaced?.() injected.set(globalContext, store) return store } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 738d676b9..a792caba5 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -3,6 +3,7 @@ const should = require('should') // eslint-disable-line no-unused-vars const datastore = require('../../nodes/store/data.js') +const warnings = [] const RED = { util, plugins: { getByType: () => [] } } datastore.setConfig(RED) @@ -12,7 +13,7 @@ function fakeGlobal () { } function fakeNode (id, global) { - return { id, type: 'ui-text', context: () => ({ global }) } + return { id, type: 'ui-text', context: () => ({ global }), warn: (m) => warnings.push(`${id}: ${m}`) } } const base = { acceptsClientConfig: [] } @@ -566,7 +567,7 @@ describe('store: data.js audit invariants', function () { describe('store: data.js client-scoped writes', function () { describe('save', function () { const base = { acceptsClientConfig: ['ui-text'] } - const node = { id: 'w1', type: 'ui-text' } + const node = fakeNode('w1', fakeGlobal()) beforeEach(function () { datastore.clear(node.id) @@ -608,3 +609,103 @@ describe('store: data.js client-scoped writes', function () { }) }) }) + +describe('store: data.js failure reporting', function () { + const unreachable = () => { throw new Error('context store unavailable') } + + beforeEach(function () { + warnings.length = 0 + }) + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('warns once on the widget whose write failed, not once per write', function () { + const global = fakeGlobal() + const node = fakeNode('fail-1', global) + datastore.initStore(global, {}) + global.get = unreachable + + datastore.save(base, node, { payload: 1 }) + datastore.save(base, node, { payload: 2 }) + datastore.append(base, fakeNode('fail-1-chart', global), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + datastore.clearFromStore(node) + + warnings.should.have.length(1) + warnings[0].should.equal('fail-1: Dashboard data store: a store write failed (context store unavailable), so global.dashboardStore may be out of date. Further store failures are not reported until the next full deploy or restart.') + }) + + it('warns when an append fails', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + global.get = unreachable + + datastore.append(base, fakeNode('fail-a', global), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + + warnings.should.have.length(1) + warnings[0].should.startWith('fail-a: Dashboard data store: a store write failed (context store unavailable)') + }) + + it('warns when a clear fails', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + global.get = unreachable + + datastore.clearFromStore(fakeNode('fail-c', global)) + + warnings.should.have.length(1) + warnings[0].should.startWith('fail-c: Dashboard data store: clearing a store entry failed (context store unavailable), so global.dashboardStore may still hold this widget\'s old value.') + }) + + it('explains a context store that stops holding the store by reference', function () { + const m = {} + let copying = false + const global = { get: (k) => copying ? JSON.parse(JSON.stringify(m[k])) : m[k], set: (k, v) => { m[k] = v } } + datastore.initStore(global, {}) + copying = true + + datastore.save(base, fakeNode('fail-r', global), { payload: 1 }) + + warnings.should.have.length(1) + warnings[0].should.startWith('fail-r: Dashboard data store: a store write failed (global context saved a copy of the store rather than the store itself, which happens with context stores that serialise values)') + }) + + it('still writes the legacy datastore when the store write fails', function () { + const global = fakeGlobal() + const node = fakeNode('fail-2', global) + datastore.initStore(global, {}) + global.get = unreachable + + datastore.save(base, node, { payload: 'kept' }) + + datastore.get('fail-2').payload.should.equal('kept') + }) + + it('warns again after the next initStore', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + global.get = unreachable + datastore.save(base, fakeNode('fail-3', global), { payload: 1 }) + + const fresh = fakeGlobal() + datastore.initStore(fresh, {}) + fresh.get = unreachable + datastore.save(base, fakeNode('fail-3', fresh), { payload: 2 }) + + warnings.should.have.length(2) + }) + + it('warns on a failing trim rebuild', function () { + const global = fakeGlobal() + const node = fakeNode('fail-5', global) + datastore.initStore(global, {}) + for (let i = 0; i < 100; i++) datastore.append(base, node, { payload: i, _datapoint: { category: 'a', x: i, y: i } }) + global.get = unreachable + + for (let i = 0; i < 60; i++) datastore.filter(base, node, (m, idx) => idx > 0) + + warnings.should.have.length(1) + warnings[0].should.startWith('fail-5: ') + }) +}) diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 6626d6a8f..e912850d5 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -1047,6 +1047,21 @@ describe('store: reactive data store', function () { log.should.containEql('a') }) + it('does not report a replacement when the context only ever hands back copies', function () { + const m = {} + let copying = false + const g = { get: (k) => copying ? JSON.parse(JSON.stringify(m[k])) : m[k], set: (k, v) => { m[k] = v } } + attachToContext(g) + copying = true + + let warned = 0 + const results = [] + for (let i = 0; i < 3; i++) results.push(attachToContext(g, { onReplaced: () => { warned++ } })) + + warned.should.equal(0) + results.should.eql([null, null, null]) + }) + it('reinjects and warns when a flow has overwritten the namespace, preserving data', function () { const g = fakeGlobal() const store = attachToContext(g) From db7d96312d8365c685e2f2d204e341c2d51db170 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 1 Oct 2026 07:24:50 -0700 Subject: [PATCH 42/57] Move options into smol helper to prevent drift --- nodes/store/data.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index 55885a184..241e29db7 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -20,13 +20,11 @@ function warnOnce (node, problem) { node.warn(`Dashboard data store: ${problem}. Further store failures are not reported until the next full deploy or restart.`) } +const attachOptions = () => ({ clone: config.RED.util.cloneMessage, ...storeOptions }) + function getOrCreateStore (globalContext) { let unavailable = null - const store = attachToContext(globalContext, { - clone: config.RED.util.cloneMessage, - ...storeOptions, - onUnavailable: (err) => { unavailable = err } - }) + const store = attachToContext(globalContext, { ...attachOptions(), onUnavailable: (err) => { unavailable = err } }) if (!store) throw unavailable || new Error('global context saved a copy of the store rather than the store itself, which happens with context stores that serialise values') return store } @@ -35,7 +33,7 @@ function initStore (globalContext, opts) { storeOptions = { ...opts } storeEnabled = true warned = false - return attachToContext(globalContext, { clone: config.RED.util.cloneMessage, ...storeOptions }) + return attachToContext(globalContext, attachOptions()) } function disableStore () { From 5e018a2082698a4a6b928c0e79ccfbd7a069b135 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 1 Oct 2026 08:23:16 -0700 Subject: [PATCH 43/57] Remove cloning for existance check and point trim check --- .../contributing/guides/state-management.md | 20 ++++ nodes/store/data.js | 26 +++++ nodes/widgets/ui_chart.js | 30 +---- test/store/data.spec.js | 103 ++++++++++++++++++ 4 files changed, 151 insertions(+), 28 deletions(-) diff --git a/docs/en/contributing/guides/state-management.md b/docs/en/contributing/guides/state-management.md index 62629cb70..864bbc81d 100644 --- a/docs/en/contributing/guides/state-management.md +++ b/docs/en/contributing/guides/state-management.md @@ -108,6 +108,16 @@ datastore.get(node.id) This ensures, on refresh of the client, or when new clients connect after data has been generated, that the state is presented consistently. +`get` returns a copy, so changing it does not change what is stored. Copying a long history is expensive, so avoid calling it on every message. + +#### `datastore.has` + +Checks whether any data is stored for a widget, without copying it: + +```js +datastore.has(node.id) +``` + #### `datastore.append` With `.append`, we can store multiple messages against the same widget, representing a history of state, rather than a single point reference to the _last_ value only. @@ -122,6 +132,16 @@ datastore.append(base, node, msg) This is used in `ui-chart` to store the history of data points, where each data point could have been an individual message received by the widget. +#### `datastore.keepLatestPerTopic` + +Trims an appended history to the most recent `maxPoints` messages for each `msg.topic`, without copying it: + +```js +datastore.keepLatestPerTopic(base, node, maxPoints) +``` + +This is used in `ui-chart` to apply its "remove older points" limit to each line. + #### `datastore.clear` When a widget is removed from the Editor, we can clear the datastore of any messages stored against that widget using: diff --git a/nodes/store/data.js b/nodes/store/data.js index 241e29db7..43b4208d3 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -131,6 +131,9 @@ const getters = { // given a widget id, return the latest msg received msg (id) { return config.RED.util.cloneMessage(data[id]) + }, + has (id) { + return !!data[id] } } @@ -198,16 +201,39 @@ const setters = { trimInStore(node, filteredMessages) } } + }, + keepLatestPerTopic (base, node, maxPoints) { + const currentData = data[node.id] + if (!Array.isArray(currentData)) { + return + } + const counts = {} + const keep = [] + let trimmed = false + for (let i = currentData.length - 1; i >= 0; i--) { + const topic = currentData[i].topic + counts[topic] = (counts[topic] || 0) + 1 + if (counts[topic] <= maxPoints) { + keep[i] = true + } else { + trimmed = true + } + } + if (trimmed) { + setters.filter(base, node, (m, i) => keep[i]) + } } } module.exports = { get: getters.msg, + has: getters.has, RED: getters.RED, setConfig: setters.setConfig, save: setters.save, append: setters.append, filter: setters.filter, + keepLatestPerTopic: setters.keepLatestPerTopic, clear: setters.clear, clearFromStore, initStore, diff --git a/nodes/widgets/ui_chart.js b/nodes/widgets/ui_chart.js index e5d916898..815126422 100644 --- a/nodes/widgets/ui_chart.js +++ b/nodes/widgets/ui_chart.js @@ -230,7 +230,7 @@ module.exports = function (RED) { }, onInput: function (msg, send, done) { // use our own custom onInput in order to store history of msg payloads - if (!datastore.get(node.id)) { + if (!datastore.has(node.id)) { datastore.save(base, node, []) } // To prevent ui_update messages from deleting old data, skip this section if no msg.payload present @@ -268,33 +268,7 @@ module.exports = function (RED) { const maxPoints = parseInt(config.removeOlderPoints) if (maxPoints && config.removeOlderPoints) { - // account for multiple lines? - // client-side does this for _each_ line - // remove older points using datastore.filter instead of saving the whole array - const lineCounts = {} - const _msg = datastore.get(node.id) || [] - - // determine which message objects to keep (latest maxPoints per label) - const keepIndexes = [] - let doFiltering = false - for (let i = _msg.length - 1; i >= 0; i--) { - const m = _msg[i] - const label = m.topic - lineCounts[label] = lineCounts[label] || 0 - if (lineCounts[label] < maxPoints) { - keepIndexes[i] = true - lineCounts[label]++ - } else { - doFiltering = true - } - } - - // filter the datastore to only keep the selected messages - if (doFiltering) { - datastore.filter(base, node, (m, i) => { - return keepIndexes[i] - }) - } + datastore.keepLatestPerTopic(base, node, maxPoints) } if (config.xAxisType === 'time' && config.removeOlder && config.removeOlderUnit) { diff --git a/test/store/data.spec.js b/test/store/data.spec.js index a792caba5..b3c35e6bf 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -709,3 +709,106 @@ describe('store: data.js failure reporting', function () { warnings[0].should.startWith('fail-5: ') }) }) + +describe('store: data reads that avoid cloning', function () { + let clones = 0 + const countingRED = { + util: { ...util, cloneMessage: (m) => { clones++; return util.cloneMessage(m) } }, + plugins: { getByType: () => [] } + } + const pt = (topic, n) => ({ topic, payload: n }) + const topicsAndPayloads = (id) => datastore.get(id).map((m) => `${m.topic}:${m.payload}`) + + beforeEach(function () { + datastore.setConfig(countingRED) + }) + + afterEach(function () { + datastore.setConfig(RED) + }) + + describe('has', function () { + it('is false for a widget with no data', function () { + datastore.clear('has-1') + datastore.has('has-1').should.equal(false) + }) + + it('is true once a series exists, even an empty one', function () { + const node = fakeNode('has-2', fakeGlobal()) + datastore.save(base, node, []) + datastore.has('has-2').should.equal(true) + }) + + it('does not clone', function () { + const node = fakeNode('has-3', fakeGlobal()) + for (let i = 0; i < 10; i++) datastore.append(base, node, pt('a', i)) + clones = 0 + + datastore.has('has-3') + + clones.should.equal(0) + }) + }) + + describe('keepLatestPerTopic', function () { + it('keeps the newest points of each topic, in their original order', function () { + const node = fakeNode('keep-1', fakeGlobal()) + datastore.save(base, node, []) + for (const [t, n] of [['a', 1], ['b', 1], ['a', 2], ['b', 2], ['a', 3], ['b', 3], ['a', 4]]) datastore.append(base, node, pt(t, n)) + + datastore.keepLatestPerTopic(base, node, 2) + + topicsAndPayloads('keep-1').should.eql(['b:2', 'a:3', 'b:3', 'a:4']) + }) + + it('treats messages without a topic as one series', function () { + const node = fakeNode('keep-2', fakeGlobal()) + datastore.save(base, node, []) + for (let i = 1; i <= 4; i++) datastore.append(base, node, { payload: i }) + + datastore.keepLatestPerTopic(base, node, 3) + + datastore.get('keep-2').map((m) => m.payload).should.eql([2, 3, 4]) + }) + + it('leaves the series alone when no topic is over the limit', function () { + const node = fakeNode('keep-3', fakeGlobal()) + datastore.save(base, node, []) + for (let i = 1; i <= 3; i++) datastore.append(base, node, pt('a', i)) + + datastore.keepLatestPerTopic(base, node, 3) + + topicsAndPayloads('keep-3').should.eql(['a:1', 'a:2', 'a:3']) + }) + + it('does nothing when the widget holds a single message rather than a series', function () { + const node = fakeNode('keep-4', fakeGlobal()) + datastore.save(base, node, { payload: 'x' }) + + datastore.keepLatestPerTopic(base, node, 1) + + datastore.get('keep-4').payload.should.equal('x') + }) + + it('does not clone the series', function () { + const node = fakeNode('keep-5', fakeGlobal()) + datastore.save(base, node, []) + for (let i = 0; i < 50; i++) datastore.append(base, node, pt(i % 2 ? 'a' : 'b', i)) + clones = 0 + + datastore.keepLatestPerTopic(base, node, 10) + + clones.should.equal(0) + }) + }) + + it('still returns a copy from get', function () { + const node = fakeNode('copy-1', fakeGlobal()) + datastore.save(base, node, []) + datastore.append(base, node, pt('a', 1)) + + datastore.get('copy-1')[0].payload = 'mutated' + + datastore.get('copy-1')[0].payload.should.equal(1) + }) +}) From 2890ed482801641f5c8307413fe0d3edb839d119 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 1 Oct 2026 12:11:48 -0700 Subject: [PATCH 44/57] Add testing harness for legacy datastore --- test/store/differential.spec.js | 73 ++++++++ test/store/helpers/legacy-datastore.js | 91 ++++++++++ test/store/helpers/sequences.js | 240 +++++++++++++++++++++++++ 3 files changed, 404 insertions(+) create mode 100644 test/store/differential.spec.js create mode 100644 test/store/helpers/legacy-datastore.js create mode 100644 test/store/helpers/sequences.js diff --git a/test/store/differential.spec.js b/test/store/differential.spec.js new file mode 100644 index 000000000..ca023d219 --- /dev/null +++ b/test/store/differential.spec.js @@ -0,0 +1,73 @@ +const { util } = require('@node-red/util') +const should = require('should') // eslint-disable-line no-unused-vars + +const datastore = require('../../nodes/store/data.js') + +const createLegacy = require('./helpers/legacy-datastore.js') +const { scenarios, generate, generateMessy, runStep } = require('./helpers/sequences.js') + +const RED = { util, plugins: { getByType: () => [] } } +const base = { acceptsClientConfig: ['ui-scoped'] } + +function fakeGlobal () { + const m = {} + return { get: (k) => m[k], set: (k, v) => { m[k] = v } } +} + +const runs = [ + ...Object.entries(scenarios).map(([name, steps]) => ({ name, steps })), + ...Array.from({ length: 200 }, (_, i) => ({ name: `seed ${i + 1}`, steps: generate(i + 1, 120) })), + ...Array.from({ length: 100 }, (_, i) => ({ name: `messy seed ${i + 1}`, steps: generateMessy(i + 1, 150) })) +] + +let runIndex = 0 + +function prepare (steps) { + const prefix = `diff-${runIndex++}:` + const prefixed = steps.map((s) => ({ ...s, id: prefix + s.id })) + return { steps: prefixed, ids: [...new Set(prefixed.map((s) => s.id))] } +} + +function compareAfterEachStep (steps, ids) { + const global = fakeGlobal() + const warnings = [] + datastore.initStore(global, {}) + const nodeFor = (id, type) => ({ id, type, context: () => ({ global }), warn: (m) => warnings.push(m) }) + const legacy = createLegacy(RED) + let comparedWithData = 0 + for (const step of steps) { + runStep(datastore, step, { base, nodeFor }) + runStep(legacy, step, { base, nodeFor }) + should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after ${step.op}`) + datastore.has(step.id).should.equal(legacy.has(step.id), `${step.id} after ${step.op}`) + if (legacy.has(step.id)) { + comparedWithData++ + } + } + for (const id of ids) { + should(datastore.get(id)).eql(legacy.get(id), id) + } + return { comparedWithData, warnings } +} + +describe('store: differential against the legacy datastore', function () { + beforeEach(function () { + datastore.setConfig(RED) + }) + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + describe('data.js matches the legacy reference', function () { + for (const run of runs) { + it(run.name, function () { + const { steps, ids } = prepare(run.steps) + const { comparedWithData, warnings } = compareAfterEachStep(steps, ids) + + comparedWithData.should.be.above(0) + warnings.should.eql([]) + }) + } + }) +}) diff --git a/test/store/helpers/legacy-datastore.js b/test/store/helpers/legacy-datastore.js new file mode 100644 index 000000000..ba51c9b27 --- /dev/null +++ b/test/store/helpers/legacy-datastore.js @@ -0,0 +1,91 @@ +const { isClientScoped } = require('../../../nodes/utils/index.js') + +module.exports = function createLegacy (RED) { + const data = {} + + function canSaveInStore (base, node, msg) { + const checks = [] + if (base.acceptsClientConfig.includes(node.type)) { + if (isClientScoped(msg)) { + checks.push(false) + } + for (const plugin of RED.plugins.getByType('node-red-dashboard-2')) { + if (plugin.hooks?.onCanSaveInStore) { + checks.push(plugin.hooks.onCanSaveInStore(msg)) + } + } + } + return checks.length === 0 || !checks.includes(false) + } + + function stripMsg (msg) { + const newMsg = RED.util.cloneMessage(msg) + delete newMsg.ui_update + return newMsg + } + + function filter (base, node, filterFunction) { + const currentData = data[node.id] + if (filterFunction && Array.isArray(currentData) && currentData.length) { + const filteredMessages = currentData.filter(filterFunction) + if (filteredMessages.length !== currentData.length) { + data[node.id] = filteredMessages + } + } + } + + return { + get (id) { + return RED.util.cloneMessage(data[id]) + }, + has (id) { + return !!data[id] + }, + clear (id) { + delete data[id] + }, + save (base, node, msg) { + if (Array.isArray(msg)) { + const filtered = [] + for (const m of msg) { + if (canSaveInStore(base, node, m)) { + filtered.push(RED.util.cloneMessage(m)) + } + } + data[node.id] = filtered + } else if (canSaveInStore(base, node, msg)) { + data[node.id] = { ...data[node.id], ...stripMsg(msg) } + } + }, + append (base, node, msg) { + if (canSaveInStore(base, node, msg)) { + if (!data[node.id]) { + data[node.id] = [] + } + data[node.id].push(RED.util.cloneMessage(msg)) + } + }, + filter, + keepLatestPerTopic (base, node, maxPoints) { + const currentData = data[node.id] + if (!Array.isArray(currentData)) { + return + } + const counts = {} + const keep = [] + let trimmed = false + for (let i = currentData.length - 1; i >= 0; i--) { + const topic = currentData[i].topic + counts[topic] = (counts[topic] || 0) + 1 + if (counts[topic] <= maxPoints) { + keep[i] = true + } else { + trimmed = true + } + } + if (trimmed) { + filter(base, node, (m, i) => keep[i]) + } + } + } +} diff --git a/test/store/helpers/sequences.js b/test/store/helpers/sequences.js new file mode 100644 index 000000000..c7d33e820 --- /dev/null +++ b/test/store/helpers/sequences.js @@ -0,0 +1,240 @@ +const pt = (x, y, category = 'a') => ({ category, x, y }) + +function mulberry32 (seed) { + return function () { + seed |= 0 + seed = seed + 0x6D2B79F5 | 0 + let t = Math.imul(seed ^ seed >>> 15, 1 | seed) + t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t + return ((t ^ t >>> 14) >>> 0) / 4294967296 + } +} + +const repeat = (n, make) => Array.from({ length: n }, (_, i) => make(i)) + +function hasExtraProps (message) { + const allowed = ['_msgid', 'ui_update', 'class', 'visible', 'enabled'] + const keys = Object.keys(message).filter((key) => message[key] !== undefined) + return keys.length > 0 && keys.some((key) => !allowed.includes(key)) +} + +const chartPoint = (x, topic) => { + const msg = { payload: x, _datapoint: pt(x, x, topic || 'a') } + if (topic !== undefined) { + msg.topic = topic + } + return msg +} + +const scenarios = { + text: [ + { op: 'input', id: 't', msg: { payload: 1, topic: 'first' } }, + { op: 'input', id: 't', msg: { topic: 'second' } }, + { op: 'input', id: 't', msg: { payload: 2, ui_update: { label: 'x' } } }, + { op: 'input', id: 't', msg: { class: 'big' } }, + { op: 'input', id: 't', msg: { payload: { nested: [1, 2] }, extra: true } } + ], + switch: [ + { op: 'switchSave', id: 'sw', msg: { payload: true, topic: 'on' } }, + { op: 'switchSave', id: 'sw', msg: { class: 'highlight' } }, + { op: 'switchSave', id: 'sw', msg: { payload: false } } + ], + 'value-widget-change': [ + { op: 'input', id: 'v', msg: { payload: 10, topic: 'slider' } }, + { op: 'change', id: 'v', value: 20, client: 's1' }, + { op: 'change', id: 'v', value: { payload: 30 } }, + { op: 'input', id: 'v', msg: { payload: 40 } }, + { op: 'change', id: 'v', value: 50, topic: 'configured' } + ], + 'table-append': [ + ...repeat(4, (i) => ({ op: 'table', id: 'tb', action: 'append', msg: { payload: [{ id: i }, { id: i + 100 }], topic: 'rows' } })), + { op: 'table', id: 'tb', action: 'append', msg: { payload: { id: 'single' } } }, + { op: 'table', id: 'tb', action: 'append', msg: { payload: [] } } + ], + 'table-replace': repeat(4, (i) => ({ op: 'table', id: 'tr', action: 'replace', msg: { payload: [{ id: i }] } })), + 'chart-time': repeat(200, (i) => ({ op: 'chart', id: 'ct', msg: chartPoint(i, 'a'), window: 40 })), + 'chart-points': repeat(300, (i) => ({ op: 'chart', id: 'cp', msg: chartPoint(i, ['a', 'b', 'c'][i % 3]), maxPoints: 50 })), + 'chart-both-limits': repeat(200, (i) => ({ op: 'chart', id: 'cb', msg: chartPoint(i, ['a', 'b'][i % 2]), maxPoints: 20, window: 30 })), + 'chart-array-payload': repeat(20, (i) => ({ op: 'chart', id: 'cm', msg: { payload: [i, i * 2], _datapoint: [pt(i, i, 'a'), pt(i, i * 2, 'b')] } })), + 'chart-object-payload': repeat(20, (i) => ({ op: 'chart', id: 'co', msg: { payload: { a: i, b: i * 2 }, _datapoint: [pt(i, i, 'a'), pt(i, i * 2, 'b')] } })), + 'chart-replace-clear': [ + ...repeat(10, (i) => ({ op: 'chart', id: 'cr', msg: chartPoint(i) })), + { op: 'chart', id: 'cr', msg: { ...chartPoint(50), action: 'replace' } }, + ...repeat(5, (i) => ({ op: 'chart', id: 'cr', msg: chartPoint(60 + i) })), + { op: 'chart', id: 'cr', msg: { payload: [] } }, + ...repeat(3, (i) => ({ op: 'chart', id: 'cr', msg: chartPoint(70 + i) })) + ], + 'chart-categorical': repeat(30, (i) => ({ op: 'chart', id: 'cc', categorical: true, msg: { payload: i, _datapoint: pt(['mon', 'tue', 'wed'][i % 3], i, 'sales') } })), + 'client-scoped': [ + { op: 'input', id: 'cs', type: 'ui-scoped', msg: { payload: 'everyone' } }, + { op: 'input', id: 'cs', type: 'ui-scoped', msg: { payload: 'one client', _client: { socketId: 's1' } } }, + { op: 'chart', id: 'csa', type: 'ui-scoped', msg: { ...chartPoint(1), _client: { clientId: 'c1' } } }, + { op: 'chart', id: 'csa', type: 'ui-scoped', msg: chartPoint(2) } + ], + 'audio-clear': [ + { op: 'input', id: 'au', msg: { payload: 'hello' } }, + { op: 'clear', id: 'au' }, + { op: 'input', id: 'au', msg: { topic: 'after clear' } } + ], + 'widget-remove': [ + ...repeat(5, (i) => ({ op: 'chart', id: 'wr', msg: chartPoint(i), maxPoints: 3 })), + { op: 'remove', id: 'wr' }, + ...repeat(5, (i) => ({ op: 'chart', id: 'wr', msg: chartPoint(10 + i), maxPoints: 3 })) + ] +} + +function generate (seed, length) { + const rand = mulberry32(seed) + const pick = (list) => list[Math.floor(rand() * list.length)] + const chartConfig = () => { + const mode = pick(['none', 'time', 'points', 'both']) + return { + window: mode === 'time' || mode === 'both' ? 5 + Math.floor(rand() * 30) : undefined, + maxPoints: mode === 'points' || mode === 'both' ? 1 + Math.floor(rand() * 5) : undefined + } + } + const charts = { g1: chartConfig(), g3: chartConfig() } + const steps = [] + for (let i = 0; i < length; i++) { + const id = pick(['g0', 'g1', 'g2', 'g3']) + const topic = pick(['a', 'b', undefined]) + const roll = rand() + if (roll < 0.03) { + steps.push({ op: pick(['clear', 'remove']), id }) + } else if (id === 'g0') { + const msg = {} + if (rand() < 0.8) { msg.payload = Math.floor(rand() * 100) } + if (topic !== undefined) { msg.topic = topic } + if (rand() < 0.2) { msg.ui_update = { label: 'l' + i } } + if (rand() < 0.2) { msg.extra = { n: i } } + steps.push(roll < 0.5 ? { op: 'input', id, msg } : roll < 0.75 ? { op: 'switchSave', id, msg } : { op: 'change', id, value: Math.floor(rand() * 100), client: rand() < 0.5 ? 's1' : undefined }) + } else if (id === 'g2') { + steps.push({ op: 'table', id, action: roll < 0.7 ? 'append' : 'replace', msg: { payload: rand() < 0.1 ? [] : [{ id: i }] } }) + } else { + const msg = roll < 0.92 ? chartPoint(i, topic) : roll < 0.96 ? { payload: [] } : { ...chartPoint(i, topic), action: 'replace' } + steps.push({ op: 'chart', id, msg, ...charts[id], cutoffAt: i }) + } + } + return steps +} + +function generateMessy (seed, length) { + const rand = mulberry32(seed) + const pick = (list) => list[Math.floor(rand() * list.length)] + const steps = [] + for (let i = 0; i < length; i++) { + const id = pick(['m0', 'm1']) + const roll = rand() + const x = Math.floor(rand() * 200) + if (roll < 0.45) { + steps.push({ op: 'chart', id, msg: chartPoint(x, pick(['a', 'b', undefined])) }) + } else if (roll < 0.6) { + steps.push({ op: 'filterTime', id, cutoff: Math.floor(rand() * 150) }) + } else if (roll < 0.72) { + steps.push({ op: 'keep', id, maxPoints: 1 + Math.floor(rand() * 6) }) + } else if (roll < 0.82) { + steps.push({ op: 'filterIndex', id, skip: Math.floor(rand() * 3) }) + } else if (roll < 0.9) { + steps.push({ op: 'chart', id, categorical: true, msg: chartPoint(x) }) + } else if (roll < 0.95) { + steps.push({ op: 'saveArray', id, msgs: rand() < 0.5 ? [] : [chartPoint(x)] }) + } else { + steps.push({ op: pick(['clear', 'remove']), id }) + } + } + return steps +} + +function chartInput (target, base, node, step) { + const { msg } = step + if (!target.has(step.id)) { + target.save(base, node, []) + } + if (typeof msg.payload === 'undefined') { + return + } + if (Array.isArray(msg.payload) && !msg.payload.length) { + target.save(base, node, []) + return + } + if (msg.action === 'replace') { + target.save(base, node, []) + } + if (!Array.isArray(msg.payload)) { + target.append(base, node, { ...msg }) + } else { + msg.payload.forEach((p, i) => { + target.append(base, node, { ...msg, payload: JSON.parse(JSON.stringify(p)), _datapoint: msg._datapoint ? msg._datapoint[i] : null }) + }) + } + if (step.maxPoints) { + target.keepLatestPerTopic(base, node, step.maxPoints) + } + if (step.window) { + const cutoff = (step.cutoffAt ?? [].concat(msg._datapoint)[0].x) - step.window + target.filter(base, node, (m) => [].concat(m._datapoint)[0]?.x > cutoff) + } else if (step.categorical) { + const latest = {} + for (const item of target.get(step.id) || []) { + latest[JSON.stringify([item._datapoint.category, item._datapoint.x])] = item + } + target.save(base, node, Object.values(latest)) + } +} + +function changeInput (target, base, node, step) { + const msg = target.get(step.id) || {} + if (step.client) { + msg._client = { ...msg._client, socketId: step.client } + } + msg.payload = (typeof step.value === 'object' && step.value !== null && 'payload' in step.value) ? step.value.payload : step.value + if (step.topic !== undefined) { + msg.topic = step.topic + } + if (!('topic' in msg)) { + msg.topic = '' + } + target.save(base, node, msg) +} + +function tableInput (target, base, node, step) { + const existing = target.get(step.id) || [] + const value = step.msg.payload + let payload = (value !== null && typeof value === 'object' && !Array.isArray(value)) ? [value] : value + if (step.action === 'append') { + payload = payload && payload.length > 0 ? [...existing.payload || [], ...payload || []] : payload + } + target.save(base, node, { ...step.msg, payload }) +} + +function runStep (target, step, { base, nodeFor }) { + const node = nodeFor(step.id, step.type || 'ui-test') + if (step.op === 'input') { + if (hasExtraProps(step.msg)) { + target.save(base, node, step.msg) + } + } else if (step.op === 'switchSave') { + target.save(base, node, step.msg) + } else if (step.op === 'change') { + changeInput(target, base, node, step) + } else if (step.op === 'table') { + tableInput(target, base, node, step) + } else if (step.op === 'chart') { + chartInput(target, base, node, step) + } else if (step.op === 'saveArray') { + target.save(base, node, step.msgs) + } else if (step.op === 'filterTime') { + target.filter(base, node, (m) => m._datapoint && [].concat(m._datapoint)[0].x > step.cutoff) + } else if (step.op === 'filterIndex') { + target.filter(base, node, (m, i) => i >= step.skip) + } else if (step.op === 'keep') { + target.keepLatestPerTopic(base, node, step.maxPoints) + } else if (step.op === 'clear') { + target.clear(step.id) + } else if (step.op === 'remove') { + target.clear(step.id) + target.clearFromStore?.(node) + } +} + +module.exports = { scenarios, generate, generateMessy, runStep } From 333ebdd6f455e66f193cbf64fc0e449e7c60478e Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 2 Oct 2026 08:22:38 -0700 Subject: [PATCH 45/57] Merge saved messages inside the store and make clear(id) the single clear path --- nodes/config/ui_base.js | 2 +- nodes/store/data.js | 36 +++++---- nodes/store/reactive.js | 17 ++-- test/store/data.spec.js | 92 +++++++++++++++++++--- test/store/helpers/sequences.js | 1 - test/store/reactive.spec.js | 135 +++++++++++++++++++++++++++++--- 6 files changed, 234 insertions(+), 49 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index e629b71b2..b1f39927b 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -404,6 +404,7 @@ module.exports = function (RED) { try { let unavailable = null const store = datastore.initStore(node.context().global, { + node, onUnavailable: (err) => { unavailable = err }, onReplaced: () => node.warn('global.dashboardStore was replaced by a flow since the last deploy, discarding the stored value of every widget. The store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') }) @@ -1170,7 +1171,6 @@ module.exports = function (RED) { // widget has been removed from the Editor // clear any data from datastore datastore.clear(widgetNode.id) - datastore.clearFromStore(widgetNode) } node.deregister(null, null, widgetNode) done() diff --git a/nodes/store/data.js b/nodes/store/data.js index 43b4208d3..51c63bd50 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -1,6 +1,6 @@ const { isClientScoped } = require('../utils/index.js') -const { attachToContext, SET_ENTRY, APPEND_ENTRY, SET_SERIES } = require('./reactive.js') +const { attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES } = require('./reactive.js') const data = {} @@ -10,12 +10,14 @@ const config = { let storeOptions = {} let storeEnabled = false +let storeContext = null +let storeBase = null const trimCounts = {} const TRIM_BATCH = 60 let warned = false function warnOnce (node, problem) { - if (warned) return + if (warned || !node) return warned = true node.warn(`Dashboard data store: ${problem}. Further store failures are not reported until the next full deploy or restart.`) } @@ -29,9 +31,11 @@ function getOrCreateStore (globalContext) { return store } -function initStore (globalContext, opts) { - storeOptions = { ...opts } +function initStore (globalContext, { node, ...opts } = {}) { + storeOptions = opts storeEnabled = true + storeContext = globalContext + storeBase = node warned = false return attachToContext(globalContext, attachOptions()) } @@ -40,11 +44,11 @@ function disableStore () { storeEnabled = false } -function writeToStore (node, msg, stored) { +function writeToStore (node, msg) { if (!storeEnabled) return - if (!msg || typeof msg !== 'object' || !('payload' in msg)) return + if (!msg || typeof msg !== 'object') return try { - getOrCreateStore(node.context().global)[SET_ENTRY](node.id, stored) + getOrCreateStore(node.context().global)[MERGE_ENTRY](node.id, msg) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } @@ -72,14 +76,6 @@ function trimInStore (node, msgs) { replaceInStore(node, msgs) } -function clearFromStore (node) { - if (!storeEnabled) return - delete trimCounts[node.id] - try { - delete getOrCreateStore(node.context().global)[node.id] - } catch (err) { warnOnce(node, `clearing a store entry failed (${err.message}), so global.dashboardStore may still hold this widget's old value`) } -} - /** * Checks if a Client/Socket ID has been assigned to this message, * and whether the node type is being scoped to a specific client. @@ -145,6 +141,13 @@ const setters = { // remove data associated to a given widget clear (id) { delete data[id] + if (!storeEnabled) { + return + } + delete trimCounts[id] + try { + delete getOrCreateStore(storeContext)[id] + } catch (err) { warnOnce(storeBase, `clearing the store entry for ${id} failed (${err.message}), so global.dashboardStore may still hold its old value`) } }, /** * @@ -170,7 +173,7 @@ const setters = { ...data[node.id], ...newMsg } - writeToStore(node, msg, data[node.id]) + writeToStore(node, newMsg) } } }, @@ -235,7 +238,6 @@ module.exports = { filter: setters.filter, keepLatestPerTopic: setters.keepLatestPerTopic, clear: setters.clear, - clearFromStore, initStore, disableStore } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 852f27ccc..28f9975ce 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -1,7 +1,7 @@ const STORE = Symbol.for('@flowfuse/node-red-dashboard/store') -const SET_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/setEntry') const APPEND_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/appendEntry') const SET_SERIES = Symbol.for('@flowfuse/node-red-dashboard/setSeries') +const MERGE_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/mergeEntry') const NAMESPACE = 'dashboardStore' class Entry { @@ -107,10 +107,15 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D return rec } - records[SET_ENTRY] = (prop, msg) => { - const { payload, ...rest } = msg - const rec = writeValue(prop, cloneValue(payload)) - rec.msg = Object.freeze({ ...freezeOwned(cloneValue(rest)), payload: rec.value }) + records[MERGE_ENTRY] = (prop, msg) => { + const current = records[prop] + const { payload, ...incoming } = msg + const { payload: storedPayload, ...stored } = current?.msg || {} + const keepValue = current && !Array.isArray(current.msg) && !('payload' in msg) + const rec = keepValue ? current : writeValue(prop, cloneValue(payload)) + const fields = { ...stored, ...freezeOwned(cloneValue(incoming)) } + rec.msg = Object.freeze('payload' in msg || (keepValue && 'payload' in current.msg) ? { ...fields, payload: rec.value } : fields) + rec.timestamp = now() emit(prop, rec, prop) } @@ -237,4 +242,4 @@ function attachToContext (globalContext, opts = {}) { return store } -module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, SET_ENTRY, APPEND_ENTRY, SET_SERIES } +module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index b3c35e6bf..2168f1732 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -127,13 +127,14 @@ describe('store: data.js reactive-store mirror', function () { datastore.get('w5').payload.should.equal(7) // legacy datastore still works }) - it('clearFromStore removes the widget key so a removed widget leaves no orphan', function () { + it('clear removes the widget key so a removed widget leaves no orphan', function () { const global = fakeGlobal() + datastore.initStore(global, {}) const node = fakeNode('w6', global) datastore.save(base, node, { payload: 5 }) global.get('dashboardStore').w6.should.equal(5) - datastore.clearFromStore(node) + datastore.clear(node.id) should(global.get('dashboardStore').w6).be.undefined() }) @@ -166,13 +167,15 @@ describe('store: data.js reactive-store mirror', function () { } }) - it('clearFromStore does not throw when the store is unavailable', function () { + it('clear does not throw when the store is unavailable', function () { const throwingGlobal = { get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, set: () => {} } const node = fakeNode('w7', throwingGlobal) - should(() => datastore.clearFromStore(node)).not.throw() + datastore.initStore(throwingGlobal, { node: { warn: () => {} } }) + + should(() => datastore.clear(node.id)).not.throw() }) }) @@ -223,12 +226,14 @@ describe('store: data.js disabled store', function () { const global = cacheOffGlobal() const scalar = fakeNode('w12', global) const chart = fakeNode('w12-chart', global) + datastore.initStore(global, { node: { warn: () => {} } }) + global.gets = 0 datastore.disableStore() datastore.save(base, scalar, { payload: 1 }) datastore.append(base, chart, { _datapoint: { category: 'a', x: 1, y: 1 } }) datastore.filter(base, chart, () => false) - datastore.clearFromStore(scalar) + datastore.clear(scalar.id) global.gets.should.equal(0) }) @@ -400,6 +405,7 @@ describe('store: data.js chart writes', function () { it('clears the trim batch even when the store cannot be reached', function () { const global = fakeGlobal() + datastore.initStore(global, { node: { warn: () => {} } }) const node = fakeNode('chart-cleared', global) for (let i = 0; i < 100; i++) { datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) @@ -411,7 +417,6 @@ describe('store: data.js chart writes', function () { const realGet = global.get global.get = () => { throw new Error('context store unavailable') } datastore.clear(node.id) - datastore.clearFromStore(node) global.get = realGet for (let i = 0; i < 100; i++) { @@ -474,12 +479,13 @@ describe('store: data.js chart writes', function () { it('clears a stale key if a node id was previously used for a value', function () { const global = fakeGlobal() + datastore.initStore(global, {}) const node = fakeNode('chart-5', global) datastore.save(base, node, { payload: 'scalar first' }) global.get('dashboardStore')['chart-5'].should.equal('scalar first') - datastore.clearFromStore(node) + datastore.clear(node.id) should(global.get('dashboardStore')['chart-5']).be.undefined() }) @@ -552,11 +558,12 @@ describe('store: data.js audit invariants', function () { it('clears value and msg together when a widget is removed', function () { const global = fakeGlobal() + datastore.initStore(global, {}) const node = fakeNode('a4', global) datastore.save(base, node, { payload: 5, topic: 't' }) global.get('dashboardStore').$a4.msg.should.be.an.Object() - datastore.clearFromStore(node) + datastore.clear(node.id) const store = global.get('dashboardStore') should(store.a4).be.undefined() @@ -630,7 +637,7 @@ describe('store: data.js failure reporting', function () { datastore.save(base, node, { payload: 1 }) datastore.save(base, node, { payload: 2 }) datastore.append(base, fakeNode('fail-1-chart', global), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) - datastore.clearFromStore(node) + datastore.clear(node.id) warnings.should.have.length(1) warnings[0].should.equal('fail-1: Dashboard data store: a store write failed (context store unavailable), so global.dashboardStore may be out of date. Further store failures are not reported until the next full deploy or restart.') @@ -647,15 +654,23 @@ describe('store: data.js failure reporting', function () { warnings[0].should.startWith('fail-a: Dashboard data store: a store write failed (context store unavailable)') }) - it('warns when a clear fails', function () { + it('does not throw from clear when there is no base node to warn on', function () { const global = fakeGlobal() datastore.initStore(global, {}) global.get = unreachable - datastore.clearFromStore(fakeNode('fail-c', global)) + should(() => datastore.clear('no-base')).not.throw() + }) + + it('warns on the base node when a clear fails, since a clear has no widget node', function () { + const global = fakeGlobal() + datastore.initStore(global, { node: { warn: (m) => warnings.push(`base: ${m}`) } }) + global.get = unreachable + + datastore.clear('fail-c') warnings.should.have.length(1) - warnings[0].should.startWith('fail-c: Dashboard data store: clearing a store entry failed (context store unavailable), so global.dashboardStore may still hold this widget\'s old value.') + warnings[0].should.startWith('base: Dashboard data store: clearing the store entry for fail-c failed (context store unavailable), so global.dashboardStore may still hold its old value.') }) it('explains a context store that stops holding the store by reference', function () { @@ -812,3 +827,56 @@ describe('store: data reads that avoid cloning', function () { datastore.get('copy-1')[0].payload.should.equal(1) }) }) + +describe('store: data.js merges into the store from the store itself', function () { + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('stores a message without a payload, merged onto the stored one', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('merge-1', global) + + datastore.save(base, node, { payload: 1 }) + datastore.save(base, node, { topic: 't' }) + + global.get('dashboardStore')['$merge-1'].msg.should.eql({ payload: 1, topic: 't' }) + }) + + it('merges onto what the store holds, not what the legacy datastore holds', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('merge-2', global) + datastore.save(base, node, { payload: 1 }) + global.get('dashboardStore')['merge-2'] = 5 + + datastore.save(base, node, { topic: 't' }) + + global.get('dashboardStore')['$merge-2'].msg.should.eql({ payload: 5, topic: 't' }) + datastore.get('merge-2').should.eql({ payload: 1, topic: 't' }) + }) + + it('ignores a message that is not an object, without warning', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const warned = [] + const node = { ...fakeNode('merge-4', global), warn: (m) => warned.push(m) } + datastore.save(base, node, { payload: 1 }) + + datastore.save(base, node, 5) + + global.get('dashboardStore')['$merge-4'].msg.should.eql({ payload: 1 }) + warned.should.eql([]) + }) + + it('removes the store entry on clear(id), which has no node', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + datastore.save(base, fakeNode('merge-3', global), { payload: 'hello' }) + + datastore.clear('merge-3') + + should(global.get('dashboardStore')['merge-3']).be.undefined() + }) +}) diff --git a/test/store/helpers/sequences.js b/test/store/helpers/sequences.js index c7d33e820..68d650312 100644 --- a/test/store/helpers/sequences.js +++ b/test/store/helpers/sequences.js @@ -233,7 +233,6 @@ function runStep (target, step, { base, nodeFor }) { target.clear(step.id) } else if (step.op === 'remove') { target.clear(step.id) - target.clearFromStore?.(node) } } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index e912850d5..a592b5d07 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -11,7 +11,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext, STORE, SET_ENTRY, APPEND_ENTRY, SET_SERIES } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -54,7 +54,7 @@ describe('store: reactive data store', function () { it('does not leave a msg describing the previous value after a flow write', function () { const { store } = makeStore() - store[SET_ENTRY]('k1', { payload: 42, topic: 'boiler' }) + store[MERGE_ENTRY]('k1', { payload: 42, topic: 'boiler' }) store.$k1.msg.topic.should.equal('boiler') store.k1 = 99 @@ -75,7 +75,7 @@ describe('store: reactive data store', function () { const { store, log } = makeStore() const msg = { payload: 42, topic: 'sensor-A' } - store[SET_ENTRY]('gauge', msg) + store[MERGE_ENTRY]('gauge', msg) store.gauge.should.equal(42) store.$gauge.msg.should.eql({ payload: 42, topic: 'sensor-A' }) @@ -84,8 +84,8 @@ describe('store: reactive data store', function () { it('keeps history on the value, as a normal write does', function () { const { store } = makeStore() - store[SET_ENTRY]('gauge', { payload: 1 }) - store[SET_ENTRY]('gauge', { payload: 2 }) + store[MERGE_ENTRY]('gauge', { payload: 1 }) + store[MERGE_ENTRY]('gauge', { payload: 2 }) store.gauge.should.equal(2) store.$gauge.history.map((h) => h.value).should.eql([1]) @@ -94,7 +94,7 @@ describe('store: reactive data store', function () { it('clones the message it is given, so a caller reusing it cannot mutate stored state', function () { const { store } = makeStore() const msg = { payload: 1, nested: { a: 1 } } - store[SET_ENTRY]('k', msg) + store[MERGE_ENTRY]('k', msg) should(store.$k.msg).not.equal(msg) store.$k.msg.nested.should.not.equal(msg.nested) @@ -106,7 +106,7 @@ describe('store: reactive data store', function () { it('serves the stored value as the payload of the stored msg', function () { const { store } = makeStore() - store[SET_ENTRY]('k', { payload: { n: 1 }, topic: 't' }) + store[MERGE_ENTRY]('k', { payload: { n: 1 }, topic: 't' }) store.$k.msg.payload.should.equal(store.$k.value) store.k.n.should.equal(1) @@ -670,7 +670,7 @@ describe('store: reactive data store', function () { it('does not let the stored msg be rewritten through $', function () { const { store, log } = makeStore() - store[SET_ENTRY]('k', { payload: 42, topic: 'boiler' }) + store[MERGE_ENTRY]('k', { payload: 42, topic: 'boiler' }) log.length = 0 const write = function () { 'use strict'; store.$k.msg.payload = 'TAMPERED' } @@ -682,7 +682,7 @@ describe('store: reactive data store', function () { it('does not let a non-payload property of the stored msg be rewritten through $', function () { const { store } = makeStore() - store[SET_ENTRY]('k', { payload: 1, meta: { deep: { n: 1 } } }) + store[MERGE_ENTRY]('k', { payload: 1, meta: { deep: { n: 1 } } }) const write = function () { 'use strict'; store.$k.msg.meta.deep.n = 'TAMPERED' } @@ -692,7 +692,7 @@ describe('store: reactive data store', function () { it('reports a nested write made through the $ msg payload, since it is the value', function () { const { store, log } = makeStore() - store[SET_ENTRY]('k', { payload: { deep: { n: 1 } } }) + store[MERGE_ENTRY]('k', { payload: { deep: { n: 1 } } }) log.length = 0 store.$k.msg.payload.deep.n = 2 @@ -716,7 +716,7 @@ describe('store: reactive data store', function () { // a change node copies by reference unless deep copy is ticked, so msg.copy === msg.payload const { store } = makeStore({ clone: util.cloneMessage }) const payload = { temp: 25 } - store[SET_ENTRY]('k', { payload, copy: payload, topic: 't' }) + store[MERGE_ENTRY]('k', { payload, copy: payload, topic: 't' }) Object.isFrozen(store.k).should.be.false() store.k.temp = 26 @@ -728,7 +728,7 @@ describe('store: reactive data store', function () { // msg.res is an object literal in Node-RED (createResponseWrapper), not a class instance, // so isReactable accepts it and nothing but an explicit exemption keeps it writable const paths = [ - (store, msg) => store[SET_ENTRY]('k', msg), + (store, msg) => store[MERGE_ENTRY]('k', msg), (store, msg) => store[SET_SERIES]('k', [msg]), (store, msg) => store[APPEND_ENTRY]('k', msg) ] @@ -1197,3 +1197,114 @@ describe('store: reactive data store', function () { }) }) }) + +describe('store: MERGE_ENTRY', function () { + const merging = () => makeStore({ clone: util.cloneMessage }) + + it('merges a message onto the stored one, as the legacy datastore does', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: 1, topic: 'a', label: 'x' }) + + store[MERGE_ENTRY]('k', { topic: 'b' }) + + store.$k.msg.should.eql({ payload: 1, topic: 'b', label: 'x' }) + }) + + it('keeps the value and adds no history for a message without a payload', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: { n: 1 } }) + const value = store.k + + store[MERGE_ENTRY]('k', { topic: 'b' }) + + store.k.should.equal(value) + store.$k.history.should.eql([]) + }) + + it('records history for every message that carries a payload, even a repeated one', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: true }) + store[MERGE_ENTRY]('k', { payload: true }) + store[MERGE_ENTRY]('k', { payload: false }) + + store.$k.history.map((h) => h.value).should.eql([true, true]) + }) + + it('does not add a payload key that no message had', function () { + const { store } = merging() + + store[MERGE_ENTRY]('k', { topic: 't' }) + + Object.keys(store.$k.msg).should.eql(['topic']) + }) + + it('merges onto a value a flow wrote directly', function () { + const { store } = merging() + store.k = 5 + + store[MERGE_ENTRY]('k', { topic: 't' }) + + store.$k.msg.should.eql({ payload: 5, topic: 't' }) + }) + + it('does not alias or freeze the message it is given', function () { + const { store } = merging() + const msg = { payload: { n: 1 }, nested: { a: 1 } } + + store[MERGE_ENTRY]('k', msg) + msg.nested.a = 2 + + store.$k.msg.nested.a.should.equal(1) + Object.isFrozen(msg.nested).should.equal(false) + }) + + it('updates the timestamp when it keeps the value', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: 1 }) + const before = store.$k.timestamp + + store[MERGE_ENTRY]('k', { topic: 'b' }) + + store.$k.timestamp.should.be.above(before) + }) + + it('does not keep a series as the value when a message without a payload is merged onto it', function () { + const { store } = merging() + store[APPEND_ENTRY]('k', { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + + store[MERGE_ENTRY]('k', { topic: 't' }) + + should(store.k).be.undefined() + store.$k.msg.topic.should.equal('t') + }) + + it('reuses stored fields rather than cloning them again on every merge', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: 'a', options: [{ label: 'one', value: 1 }] }) + const options = store.$k.msg.options + + store[MERGE_ENTRY]('k', { payload: 'b' }) + + store.$k.msg.options.should.equal(options) + Object.isFrozen(store.$k.msg.options).should.equal(true) + }) + + it('keeps stored fields in their original order, with payload last', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { topic: 'a', payload: 1, label: 'x' }) + + store[MERGE_ENTRY]('k', { topic: 'b', extra: true }) + + Object.keys(store.$k.msg).should.eql(['topic', 'label', 'extra', 'payload']) + }) + + it('reports each merge once', function () { + const { store, log } = merging() + store[MERGE_ENTRY]('k', { payload: 1 }) + log.length = 0 + + store[MERGE_ENTRY]('k', { topic: 'b' }) + + log.should.eql(['k']) + }) +}) From fd45995f5a1b0aba7fe9c3442a91dc5222c77426 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 9 Oct 2026 08:22:18 -0700 Subject: [PATCH 46/57] Harden the differential harness with aliasing checks and scoped widgets, and move hasExtraProps to utils --- nodes/config/ui_base.js | 15 +-------------- nodes/utils/index.js | 10 +++++++++- test/store/differential.spec.js | 28 +++++++++++++++++++++++++--- test/store/helpers/sequences.js | 17 +++++++++-------- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index e629b71b2..fcaaf3017 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -7,7 +7,7 @@ const v = require('../../package.json').version const { createClientStore } = require('../store/clients.js') const datastore = require('../store/data.js') const statestore = require('../store/state.js') -const { appendTopic, addConnectionCredentials, matchesClient, normalizeClientId, getThirdPartyWidgets } = require('../utils/index.js') +const { appendTopic, addConnectionCredentials, matchesClient, normalizeClientId, getThirdPartyWidgets, hasExtraProps } = require('../utils/index.js') // from: https://stackoverflow.com/a/28592528/3016654 function join (...paths) { @@ -27,19 +27,6 @@ function hasProperty (obj, prop) { return Object.prototype.hasOwnProperty.call(obj, prop) } -/** - * Test whether a message has any properties other than ui_update, class, visible, enabled and _msgid - * Properties with the value undefined are ignored - * @param {*} msg - * @returns true if other properties found - */ -function hasExtraProps (message) { - const allowed = ['_msgid', 'ui_update', 'class', 'visible', 'enabled'] - const keys = Object.keys(message).filter(key => message[key] !== undefined) - - return keys.length > 0 && keys.some(key => !allowed.includes(key)) -} - module.exports = function (RED) { const express = require('express') const { Server } = require('socket.io') diff --git a/nodes/utils/index.js b/nodes/utils/index.js index d6a6dd7c6..2a53989d4 100644 --- a/nodes/utils/index.js +++ b/nodes/utils/index.js @@ -60,6 +60,13 @@ function isClientScoped (msg) { return clientTargets(msg).length > 0 } +function hasExtraProps (message) { + const allowed = ['_msgid', 'ui_update', 'class', 'visible', 'enabled'] + const keys = Object.keys(message).filter(key => message[key] !== undefined) + + return keys.length > 0 && keys.some(key => !allowed.includes(key)) +} + function matchesClient (conn, msg) { return clientTargets(msg).every((target) => target.value === target.connValue(conn)) } @@ -155,5 +162,6 @@ module.exports = { normalizeClientId, isClientScoped, matchesClient, - getThirdPartyWidgets + getThirdPartyWidgets, + hasExtraProps } diff --git a/test/store/differential.spec.js b/test/store/differential.spec.js index ca023d219..88fcf121f 100644 --- a/test/store/differential.spec.js +++ b/test/store/differential.spec.js @@ -16,7 +16,7 @@ function fakeGlobal () { const runs = [ ...Object.entries(scenarios).map(([name, steps]) => ({ name, steps })), - ...Array.from({ length: 200 }, (_, i) => ({ name: `seed ${i + 1}`, steps: generate(i + 1, 120) })), + ...Array.from({ length: 200 }, (_, i) => ({ name: `seed ${i + 1}`, steps: generate(i + 1, 180) })), ...Array.from({ length: 100 }, (_, i) => ({ name: `messy seed ${i + 1}`, steps: generateMessy(i + 1, 150) })) ] @@ -28,6 +28,23 @@ function prepare (steps) { return { steps: prefixed, ids: [...new Set(prefixed.map((s) => s.id))] } } +function scribble (value, seen = new Set()) { + if (!value || typeof value !== 'object' || seen.has(value)) { + return + } + seen.add(value) + for (const key of Object.keys(value)) { + if (value[key] && typeof value[key] === 'object') { + scribble(value[key], seen) + } else { + value[key] = 'scribbled' + } + } + if (Array.isArray(value)) { + value.push('scribbled') + } +} + function compareAfterEachStep (steps, ids) { const global = fakeGlobal() const warnings = [] @@ -36,9 +53,14 @@ function compareAfterEachStep (steps, ids) { const legacy = createLegacy(RED) let comparedWithData = 0 for (const step of steps) { - runStep(datastore, step, { base, nodeFor }) - runStep(legacy, step, { base, nodeFor }) + const given = structuredClone(step) + runStep(datastore, given, { base, nodeFor }) + runStep(legacy, structuredClone(step), { base, nodeFor }) should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after ${step.op}`) + scribble(given) + should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after scribbling what ${step.op} was given`) + scribble(datastore.get(step.id)) + should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after scribbling what get returned`) datastore.has(step.id).should.equal(legacy.has(step.id), `${step.id} after ${step.op}`) if (legacy.has(step.id)) { comparedWithData++ diff --git a/test/store/helpers/sequences.js b/test/store/helpers/sequences.js index c7d33e820..a82e3eef2 100644 --- a/test/store/helpers/sequences.js +++ b/test/store/helpers/sequences.js @@ -1,3 +1,5 @@ +const { hasExtraProps } = require('../../../nodes/utils/index.js') + const pt = (x, y, category = 'a') => ({ category, x, y }) function mulberry32 (seed) { @@ -12,12 +14,6 @@ function mulberry32 (seed) { const repeat = (n, make) => Array.from({ length: n }, (_, i) => make(i)) -function hasExtraProps (message) { - const allowed = ['_msgid', 'ui_update', 'class', 'visible', 'enabled'] - const keys = Object.keys(message).filter((key) => message[key] !== undefined) - return keys.length > 0 && keys.some((key) => !allowed.includes(key)) -} - const chartPoint = (x, topic) => { const msg = { payload: x, _datapoint: pt(x, x, topic || 'a') } if (topic !== undefined) { @@ -93,14 +89,19 @@ function generate (seed, length) { maxPoints: mode === 'points' || mode === 'both' ? 1 + Math.floor(rand() * 5) : undefined } } - const charts = { g1: chartConfig(), g3: chartConfig() } + const charts = { g1: chartConfig(), g3: chartConfig(), g5: chartConfig() } const steps = [] for (let i = 0; i < length; i++) { - const id = pick(['g0', 'g1', 'g2', 'g3']) + const id = pick(['g0', 'g1', 'g2', 'g3', 'g4', 'g5']) const topic = pick(['a', 'b', undefined]) const roll = rand() + const client = rand() < 0.4 ? { _client: { socketId: pick(['s1', 's2']) } } : {} if (roll < 0.03) { steps.push({ op: pick(['clear', 'remove']), id }) + } else if (id === 'g4') { + steps.push({ op: roll < 0.5 ? 'input' : 'switchSave', id, type: 'ui-scoped', msg: { payload: Math.floor(rand() * 100), ...client } }) + } else if (id === 'g5') { + steps.push({ op: 'chart', id, type: 'ui-scoped', msg: { ...chartPoint(i, topic), ...client }, ...charts[id], cutoffAt: i }) } else if (id === 'g0') { const msg = {} if (rand() < 0.8) { msg.payload = Math.floor(rand() * 100) } From 8399fa013bddc176d46b6ed4730f86ba2053ffd1 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 9 Oct 2026 10:21:58 -0700 Subject: [PATCH 47/57] Trim chart series inside the store through TRIM_SERIES and FLUSH_SERIES instead of copying legacy's results, and stop a replaced series' points staying writable --- nodes/store/data.js | 49 +++++----- nodes/store/reactive.js | 47 +++++++++- test/store/data.spec.js | 111 ++++++++++++----------- test/store/reactive.spec.js | 174 +++++++++++++++++++++++++++++++++++- 4 files changed, 301 insertions(+), 80 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index 51c63bd50..00799efbb 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -1,6 +1,6 @@ const { isClientScoped } = require('../utils/index.js') -const { attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES } = require('./reactive.js') +const { attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES, TRIM_SERIES } = require('./reactive.js') const data = {} @@ -12,8 +12,6 @@ let storeOptions = {} let storeEnabled = false let storeContext = null let storeBase = null -const trimCounts = {} -const TRIM_BATCH = 60 let warned = false function warnOnce (node, problem) { @@ -64,16 +62,28 @@ function replaceInStore (node, msgs) { if (!storeEnabled) return if (!Array.isArray(msgs) || msgs.some((m) => !m || typeof m !== 'object')) return try { - delete trimCounts[node.id] getOrCreateStore(node.context().global)[SET_SERIES](node.id, msgs) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } -function trimInStore (node, msgs) { +function trimInStore (node, trim) { if (!storeEnabled) return - trimCounts[node.id] = (trimCounts[node.id] || 0) + 1 - if (trimCounts[node.id] < TRIM_BATCH) return - replaceInStore(node, msgs) + try { + getOrCreateStore(node.context().global)[TRIM_SERIES](node.id, trim) + } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } +} + +function latestPerTopic (msgs, maxPoints) { + const counts = {} + const kept = [] + for (let i = msgs.length - 1; i >= 0; i--) { + const topic = msgs[i].topic + counts[topic] = (counts[topic] || 0) + 1 + if (counts[topic] <= maxPoints) { + kept.push(msgs[i]) + } + } + return kept.reverse() } /** @@ -144,7 +154,6 @@ const setters = { if (!storeEnabled) { return } - delete trimCounts[id] try { delete getOrCreateStore(storeContext)[id] } catch (err) { warnOnce(storeBase, `clearing the store entry for ${id} failed (${err.message}), so global.dashboardStore may still hold its old value`) } @@ -201,29 +210,21 @@ const setters = { if (filteredMessages.length !== currentData.length) { // no need for save operation to process messages - just apply them data[node.id] = filteredMessages - trimInStore(node, filteredMessages) } } + if (filterFunction) { + trimInStore(node, (msgs) => msgs.filter(filterFunction)) + } }, keepLatestPerTopic (base, node, maxPoints) { + trimInStore(node, (msgs) => latestPerTopic(msgs, maxPoints)) const currentData = data[node.id] if (!Array.isArray(currentData)) { return } - const counts = {} - const keep = [] - let trimmed = false - for (let i = currentData.length - 1; i >= 0; i--) { - const topic = currentData[i].topic - counts[topic] = (counts[topic] || 0) + 1 - if (counts[topic] <= maxPoints) { - keep[i] = true - } else { - trimmed = true - } - } - if (trimmed) { - setters.filter(base, node, (m, i) => keep[i]) + const kept = latestPerTopic(currentData, maxPoints) + if (kept.length !== currentData.length) { + data[node.id] = kept } } } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 28f9975ce..32479a53f 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -2,6 +2,9 @@ const STORE = Symbol.for('@flowfuse/node-red-dashboard/store') const APPEND_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/appendEntry') const SET_SERIES = Symbol.for('@flowfuse/node-red-dashboard/setSeries') const MERGE_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/mergeEntry') +const TRIM_SERIES = Symbol.for('@flowfuse/node-red-dashboard/trimSeries') +const FLUSH_SERIES = Symbol.for('@flowfuse/node-red-dashboard/flushSeries') +const TRIM_BATCH = 60 const NAMESPACE = 'dashboardStore' class Entry { @@ -11,6 +14,7 @@ class Entry { this.quality = 'GOOD' this.timestamp = 0 this.history = [] + this.trims = null } } @@ -102,6 +106,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D } const wrapped = deepReactive(value, notify, prop, cloneValue) rec.value = wrapped + rec.trims = null rec.msg = Object.freeze({ payload: wrapped }) rec.timestamp = now() return rec @@ -110,7 +115,8 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D records[MERGE_ENTRY] = (prop, msg) => { const current = records[prop] const { payload, ...incoming } = msg - const { payload: storedPayload, ...stored } = current?.msg || {} + const previous = Array.isArray(current?.msg) ? freezeOwned(cloneValue({ ...current.msg })) : current?.msg || {} + const { payload: storedPayload, ...stored } = previous const keepValue = current && !Array.isArray(current.msg) && !('payload' in msg) const rec = keepValue ? current : writeValue(prop, cloneValue(payload)) const fields = { ...stored, ...freezeOwned(cloneValue(incoming)) } @@ -170,6 +176,43 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (replaced) emit(prop, rec, prop) } + const rebuildSeries = (prop, rec) => { + const { kept, seen } = rec.trims + rec.trims = null + const msgs = kept.concat(rec.msg.slice(seen)) + if (msgs.length !== rec.msg.length) { + records[SET_SERIES](prop, msgs) + } + } + + records[TRIM_SERIES] = (prop, trim) => { + const rec = records[prop] + if (!rec || !Array.isArray(rec.msg)) { + return + } + const pending = rec.trims || (rec.trims = { calls: 0, kept: [], seen: 0 }) + for (let i = pending.seen; i < rec.msg.length; i++) { + pending.kept.push(rec.msg[i]) + } + pending.seen = rec.msg.length + const before = pending.kept.length + pending.kept = trim(pending.kept) + if (pending.kept.length === before) { + return + } + pending.calls++ + if (pending.calls >= TRIM_BATCH) { + rebuildSeries(prop, rec) + } + } + + records[FLUSH_SERIES] = (prop) => { + const rec = records[prop] + if (rec?.trims) { + rebuildSeries(prop, rec) + } + } + const handler = { get (target, prop, receiver) { if (typeof prop === 'symbol') return Reflect.get(target, prop, receiver) @@ -242,4 +285,4 @@ function attachToContext (globalContext, opts = {}) { return store } -module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY } +module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, FLUSH_SERIES } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 6f73869c6..a257a0708 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -2,6 +2,7 @@ const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars const datastore = require('../../nodes/store/data.js') +const { APPEND_ENTRY, FLUSH_SERIES } = require('../../nodes/store/reactive.js') const warnings = [] const RED = { util, plugins: { getByType: () => [] } } @@ -403,67 +404,19 @@ describe('store: data.js chart writes', function () { global.get('dashboardStore')['chart-ok'].should.have.length(1) }) - it('clears the trim batch even when the store cannot be reached', function () { - const global = fakeGlobal() - datastore.initStore(global, { node: { warn: () => {} } }) - const node = fakeNode('chart-cleared', global) - for (let i = 0; i < 100; i++) { - datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) - } - for (let i = 0; i < 59; i++) { - datastore.filter(base, node, (m, idx) => idx > 0) - } - - const realGet = global.get - global.get = () => { throw new Error('context store unavailable') } - datastore.clear(node.id) - global.get = realGet - - for (let i = 0; i < 100; i++) { - datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) - } - const before = global.get('dashboardStore')['chart-cleared'].length - datastore.filter(base, node, (m, idx) => idx > 0) - - global.get('dashboardStore')['chart-cleared'].should.have.length(before) - }) - - it('resets the trim batch even when the rebuild fails, so it does not retry on every trim', function () { - const global = fakeGlobal() - const node = fakeNode('chart-stuck', global) - for (let i = 0; i < 100; i++) { - datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) - } - - const realGet = global.get - global.get = () => { throw new Error('context store unavailable') } - for (let i = 0; i < 60; i++) { - datastore.filter(base, node, (m, idx) => idx > 0) - } - global.get = realGet - - // the counter was cleared despite the failure, so one more trim must not trigger a rebuild - const before = global.get('dashboardStore')['chart-stuck'].length - datastore.filter(base, node, (m, idx) => idx > 0) - - global.get('dashboardStore')['chart-stuck'].should.have.length(before) - }) - - it('does not advance the batch when a trim removes nothing', function () { + it('leaves the stored series untouched when trims remove nothing', function () { const global = fakeGlobal() const node = fakeNode('chart-noop', global) for (let i = 0; i < 100; i++) { datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) } + const before = global.get('dashboardStore')['$chart-noop'].msg for (let i = 0; i < 200; i++) { datastore.filter(base, node, () => true) } - for (let i = 0; i < 59; i++) { - datastore.filter(base, node, (m, idx) => idx > 0) - } - global.get('dashboardStore')['chart-noop'].should.have.length(100) + global.get('dashboardStore')['$chart-noop'].msg.should.eql(before) }) it('still keeps the legacy chart series intact', function () { @@ -711,18 +664,29 @@ describe('store: data.js failure reporting', function () { warnings.should.have.length(2) }) - it('warns on a failing trim rebuild', function () { + it('warns on the widget when a trim cannot reach the store', function () { const global = fakeGlobal() const node = fakeNode('fail-5', global) datastore.initStore(global, {}) for (let i = 0; i < 100; i++) datastore.append(base, node, { payload: i, _datapoint: { category: 'a', x: i, y: i } }) global.get = unreachable - for (let i = 0; i < 60; i++) datastore.filter(base, node, (m, idx) => idx > 0) + datastore.filter(base, node, (m, idx) => idx > 0) warnings.should.have.length(1) warnings[0].should.startWith('fail-5: ') }) + + it('throws a failing filter to the widget without reporting a store failure', function () { + const global = fakeGlobal() + const node = fakeNode('fail-6', global) + datastore.initStore(global, {}) + datastore.append(base, node, { payload: 1, _datapoint: null }) + + should.throws(() => datastore.filter(base, node, (m) => m._datapoint.x > 0)) + + warnings.should.have.length(0) + }) }) describe('store: data reads that avoid cloning', function () { @@ -883,3 +847,44 @@ describe('store: data.js merges into the store from the store itself', function should(global.get('dashboardStore')['merge-3']).be.undefined() }) }) + +describe('store: data.js trims the stored series itself', function () { + const point = (x, topic = 'a') => ({ payload: x, topic, _datapoint: { category: topic, x, y: x } }) + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('trims what the store holds, not a copy of the legacy result', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('own-trim', global) + datastore.save(base, node, []) + for (let i = 0; i < 6; i++) { + datastore.append(base, node, point(i, i % 2 ? 'b' : 'a')) + } + global.get('dashboardStore')[APPEND_ENTRY]('own-trim', point('store-only')) + + datastore.keepLatestPerTopic(base, node, 2) + global.get('dashboardStore')[FLUSH_SERIES]('own-trim') + + global.get('dashboardStore')['$own-trim'].msg.map((m) => m.payload).should.eql([3, 4, 5, 'store-only']) + datastore.get('own-trim').map((m) => m.payload).should.eql([2, 3, 4, 5]) + }) + + it('applies both a point limit and a time window', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('both-limits', global) + datastore.save(base, node, []) + for (let i = 0; i < 100; i++) { + datastore.append(base, node, point(i)) + datastore.keepLatestPerTopic(base, node, 10) + datastore.filter(base, node, (m) => m._datapoint.x > i - 50) + } + + global.get('dashboardStore')[FLUSH_SERIES]('both-limits') + + global.get('dashboardStore')['$both-limits'].msg.map((m) => m.payload).should.eql(datastore.get('both-limits').map((m) => m.payload)) + }) +}) diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index a592b5d07..c2b40cfb3 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -11,7 +11,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, FLUSH_SERIES } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -1278,6 +1278,18 @@ describe('store: MERGE_ENTRY', function () { store.$k.msg.topic.should.equal('t') }) + it('keeps a replaced series\' messages as legacy does, without leaving their points writable', function () { + const { store, log } = merging() + store[APPEND_ENTRY]('k', { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + + store[MERGE_ENTRY]('k', { payload: 'single' }) + log.length = 0 + try { store.$k.msg[0]._datapoint.y = 777 } catch (err) {} + + store.$k.msg.should.eql({ 0: { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }, payload: 'single' }) + log.should.eql([]) + }) + it('reuses stored fields rather than cloning them again on every merge', function () { const { store } = merging() store[MERGE_ENTRY]('k', { payload: 'a', options: [{ label: 'one', value: 1 }] }) @@ -1308,3 +1320,163 @@ describe('store: MERGE_ENTRY', function () { log.should.eql(['k']) }) }) + +describe('store: TRIM_SERIES', function () { + const point = (x, topic = 'a') => ({ payload: x, topic, _datapoint: { category: topic, x, y: x } }) + const keepLatest = (max) => (msgs) => msgs.filter((m, i) => msgs.slice(i + 1).filter((n) => n.topic === m.topic).length < max) + + function series (n, topicOf = () => 'a', opts = {}) { + const made = makeStore({ clone: util.cloneMessage, ...opts }) + for (let i = 0; i < n; i++) { + made.store[APPEND_ENTRY]('c', point(i, topicOf(i))) + } + return made + } + const payloads = (store) => store.$c.msg.map((m) => m.payload) + + it('leaves the stored series alone for 59 trims and rebuilds it on the 60th', function () { + const { store } = series(100) + for (let i = 0; i < 59; i++) { + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > i)) + } + store.$c.msg.should.have.length(100) + + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > 59)) + + payloads(store).should.eql(Array.from({ length: 40 }, (_, i) => i + 60)) + }) + + it('counts only trims that remove something towards the batch', function () { + const { store } = series(100) + for (let i = 0; i < 59; i++) { + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > i)) + store[TRIM_SERIES]('c', (msgs) => msgs) + } + + store.$c.msg.should.have.length(100) + }) + + it('applies pending trims on flush', function () { + const { store } = series(10) + store[TRIM_SERIES]('c', keepLatest(3)) + + store[FLUSH_SERIES]('c') + + payloads(store).should.eql([7, 8, 9]) + }) + + it('matches trimming after every message when points arrive out of order', function () { + const { store } = series(0) + store[SET_SERIES]('c', []) + for (const [x, cutoff] of [[100, 0], [1, 10], [50, 20]]) { + store[APPEND_ENTRY]('c', point(x)) + store[TRIM_SERIES]('c', keepLatest(2)) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m._datapoint.x > cutoff)) + } + + store[FLUSH_SERIES]('c') + + payloads(store).should.eql([100, 50]) + }) + + it('applies every trim, not just the latest', function () { + const { store } = series(5) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 1)) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 3)) + + store[FLUSH_SERIES]('c') + + payloads(store).should.eql([0, 2, 4]) + }) + + it('gives a position-based trim the positions a per-message trim would see', function () { + const { store } = series(5) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m, i) => i > 0)) + store[APPEND_ENTRY]('c', point(5)) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m, i) => i > 0)) + + store[FLUSH_SERIES]('c') + + payloads(store).should.eql([2, 3, 4, 5]) + }) + + it('keeps points appended after the last trim on flush', function () { + const { store } = series(5) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 0)) + store[APPEND_ENTRY]('c', point(5)) + + store[FLUSH_SERIES]('c') + + payloads(store).should.eql([1, 2, 3, 4, 5]) + }) + + it('does not rebuild or report when trims remove nothing', function () { + const { store, log } = series(10) + log.length = 0 + + for (let i = 0; i < 60; i++) { + store[TRIM_SERIES]('c', (msgs) => msgs) + } + store[FLUSH_SERIES]('c') + + log.should.eql([]) + payloads(store).should.eql([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + }) + + it('drops pending trims when the series is replaced', function () { + const { store } = series(10) + store[TRIM_SERIES]('c', keepLatest(1)) + store[SET_SERIES]('c', [point(1), point(2)]) + + store[TRIM_SERIES]('c', keepLatest(1)) + store[FLUSH_SERIES]('c') + + payloads(store).should.eql([2]) + }) + + it('drops pending trims when the entry is deleted', function () { + const { store } = series(10) + store[TRIM_SERIES]('c', keepLatest(1)) + + delete store.c + store[APPEND_ENTRY]('c', point(1)) + store[APPEND_ENTRY]('c', point(2)) + store[FLUSH_SERIES]('c') + + payloads(store).should.eql([1, 2]) + }) + + it('ignores trims and flushes on an entry that is not a series', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + store.k = 5 + + store[TRIM_SERIES]('k', () => []) + store[FLUSH_SERIES]('k') + store[FLUSH_SERIES]('missing') + + store.k.should.equal(5) + }) + + it('does not clone on trims that do not rebuild', function () { + let clones = 0 + const { store } = series(50, (i) => (i % 2 ? 'a' : 'b'), { clone: (v) => { clones++; return util.cloneMessage(v) } }) + clones = 0 + + for (let i = 0; i < 59; i++) { + store[TRIM_SERIES]('c', keepLatest(10)) + } + + clones.should.equal(0) + }) + + it('reports the rebuild once', function () { + const { store, log } = series(100) + log.length = 0 + + for (let i = 0; i < 60; i++) { + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > i)) + } + + log.should.eql(['c']) + }) +}) From b6252e3cde966f8f018cc1a4cd921a135973be8f Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Wed, 14 Oct 2026 09:11:43 -0700 Subject: [PATCH 48/57] Gate the harness on the store alone matching the reference, and apply pending trims before merging a message onto a chart series --- nodes/store/reactive.js | 7 ++--- test/store/differential.spec.js | 46 +++++++++++++++++++++++++++++++++ test/store/helpers/sequences.js | 2 ++ test/store/reactive.spec.js | 15 +++++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 32479a53f..e9c3ba0ff 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -112,10 +112,12 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D return rec } + const trimmedSeries = (rec) => rec.trims ? rec.trims.kept.concat(rec.msg.slice(rec.trims.seen)) : rec.msg + records[MERGE_ENTRY] = (prop, msg) => { const current = records[prop] const { payload, ...incoming } = msg - const previous = Array.isArray(current?.msg) ? freezeOwned(cloneValue({ ...current.msg })) : current?.msg || {} + const previous = Array.isArray(current?.msg) ? freezeOwned(cloneValue({ ...trimmedSeries(current) })) : current?.msg || {} const { payload: storedPayload, ...stored } = previous const keepValue = current && !Array.isArray(current.msg) && !('payload' in msg) const rec = keepValue ? current : writeValue(prop, cloneValue(payload)) @@ -177,9 +179,8 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D } const rebuildSeries = (prop, rec) => { - const { kept, seen } = rec.trims + const msgs = trimmedSeries(rec) rec.trims = null - const msgs = kept.concat(rec.msg.slice(seen)) if (msgs.length !== rec.msg.length) { records[SET_SERIES](prop, msgs) } diff --git a/test/store/differential.spec.js b/test/store/differential.spec.js index 88fcf121f..6f26a0fb1 100644 --- a/test/store/differential.spec.js +++ b/test/store/differential.spec.js @@ -2,6 +2,7 @@ const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars const datastore = require('../../nodes/store/data.js') +const { FLUSH_SERIES } = require('../../nodes/store/reactive.js') const createLegacy = require('./helpers/legacy-datastore.js') const { scenarios, generate, generateMessy, runStep } = require('./helpers/sequences.js') @@ -45,6 +46,12 @@ function scribble (value, seen = new Set()) { } } +function storeHolds (global, id) { + const store = global.get('dashboardStore') + store[FLUSH_SERIES](id) + return store['$' + id]?.msg +} + function compareAfterEachStep (steps, ids) { const global = fakeGlobal() const warnings = [] @@ -62,16 +69,35 @@ function compareAfterEachStep (steps, ids) { scribble(datastore.get(step.id)) should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after scribbling what get returned`) datastore.has(step.id).should.equal(legacy.has(step.id), `${step.id} after ${step.op}`) + should(storeHolds(global, step.id)).eql(legacy.get(step.id), `store for ${step.id} after ${step.op}`) if (legacy.has(step.id)) { comparedWithData++ } } for (const id of ids) { should(datastore.get(id)).eql(legacy.get(id), id) + should(storeHolds(global, id)).eql(legacy.get(id), `store for ${id}`) } return { comparedWithData, warnings } } +function compareAtEnd (steps, ids) { + const global = fakeGlobal() + const warnings = [] + datastore.initStore(global, {}) + const nodeFor = (id, type) => ({ id, type, context: () => ({ global }), warn: (m) => warnings.push(m) }) + const legacy = createLegacy(RED) + for (const step of steps) { + runStep(datastore, structuredClone(step), { base, nodeFor }) + runStep(legacy, structuredClone(step), { base, nodeFor }) + } + const withData = ids.filter((id) => legacy.has(id)) + for (const id of ids) { + should(storeHolds(global, id)).eql(legacy.get(id), `store for ${id}`) + } + return { withData, warnings } +} + describe('store: differential against the legacy datastore', function () { beforeEach(function () { datastore.setConfig(RED) @@ -92,4 +118,24 @@ describe('store: differential against the legacy datastore', function () { }) } }) + + describe('the store matches the legacy reference when only read at the end', function () { + let runsWithData = 0 + + for (const run of runs) { + it(run.name, function () { + const { steps, ids } = prepare(run.steps) + const { withData, warnings } = compareAtEnd(steps, ids) + + if (withData.length) { + runsWithData++ + } + warnings.should.eql([]) + }) + } + + it('compared stored data at the end of most runs', function () { + runsWithData.should.be.above(runs.length * 0.9) + }) + }) }) diff --git a/test/store/helpers/sequences.js b/test/store/helpers/sequences.js index ef96d25b3..9dfcbe6af 100644 --- a/test/store/helpers/sequences.js +++ b/test/store/helpers/sequences.js @@ -60,6 +60,7 @@ const scenarios = { { op: 'chart', id: 'cr', msg: { payload: [] } }, ...repeat(3, (i) => ({ op: 'chart', id: 'cr', msg: chartPoint(70 + i) })) ], + 'chart-position-trims': repeat(150, (i) => [{ op: 'chart', id: 'pt', msg: chartPoint(i) }, { op: 'filterIndex', id: 'pt', skip: 1 }]).flat(), 'chart-categorical': repeat(30, (i) => ({ op: 'chart', id: 'cc', categorical: true, msg: { payload: i, _datapoint: pt(['mon', 'tue', 'wed'][i % 3], i, 'sales') } })), 'client-scoped': [ { op: 'input', id: 'cs', type: 'ui-scoped', msg: { payload: 'everyone' } }, @@ -143,6 +144,7 @@ function generateMessy (seed, length) { steps.push({ op: pick(['clear', 'remove']), id }) } } + steps.push({ op: 'switchSave', id: 'm0', msg: { topic: 'merged' } }, { op: 'switchSave', id: 'm1', msg: { topic: 'merged' } }) return steps } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index c2b40cfb3..5dc7381d0 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -1290,6 +1290,21 @@ describe('store: MERGE_ENTRY', function () { log.should.eql([]) }) + it('applies pending trims to a series before merging onto it, reporting once', function () { + const { store, log } = merging() + for (let x = 0; x < 5; x++) { + store[APPEND_ENTRY]('k', { payload: x, _datapoint: { category: 'a', x, y: x } }) + } + store[TRIM_SERIES]('k', (msgs) => msgs.filter((m) => m._datapoint.x > 2)) + log.length = 0 + + store[MERGE_ENTRY]('k', { topic: 'later' }) + + Object.keys(store.$k.msg).should.eql(['0', '1', 'topic']) + store.$k.msg[0].payload.should.equal(3) + log.should.eql(['k']) + }) + it('reuses stored fields rather than cloning them again on every merge', function () { const { store } = merging() store[MERGE_ENTRY]('k', { payload: 'a', options: [{ label: 'one', value: 1 }] }) From 88ffee10cf98fe2162cc8481856e0b683fdc1730 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 15 Oct 2026 10:03:09 -0700 Subject: [PATCH 49/57] Fix blank history entries and fix confusing ternary --- nodes/store/reactive.js | 5 +++-- test/store/reactive.spec.js | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index e9c3ba0ff..cf288159c 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -93,7 +93,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (!rec) { rec = new Entry() records[prop] = rec - } else if (!Array.isArray(rec.value)) { + } else if (!Array.isArray(rec.value) && 'payload' in rec.msg) { // snapshotting a whole array on every write (e.g. a table's rows) would blow up memory rec.history.push(Object.freeze({ value: deepFreeze(cloneValue(rec.value)), timestamp: rec.timestamp })) if (rec.history.length > maxHistory) rec.history.shift() @@ -122,7 +122,8 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D const keepValue = current && !Array.isArray(current.msg) && !('payload' in msg) const rec = keepValue ? current : writeValue(prop, cloneValue(payload)) const fields = { ...stored, ...freezeOwned(cloneValue(incoming)) } - rec.msg = Object.freeze('payload' in msg || (keepValue && 'payload' in current.msg) ? { ...fields, payload: rec.value } : fields) + const hasPayload = 'payload' in msg || (keepValue && 'payload' in current.msg) + rec.msg = Object.freeze(hasPayload ? { ...fields, payload: rec.value } : fields) rec.timestamp = now() emit(prop, rec, prop) } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 5dc7381d0..2a63d972e 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -1221,6 +1221,18 @@ describe('store: MERGE_ENTRY', function () { store.$k.history.should.eql([]) }) + it('adds no history for a value that no message ever set', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { topic: 'a' }) + store[MERGE_ENTRY]('u', { payload: undefined }) + + store[MERGE_ENTRY]('k', { payload: 1 }) + store[MERGE_ENTRY]('u', { payload: 1 }) + + store.$k.history.should.eql([]) + store.$u.history.map((h) => h.value).should.eql([undefined]) + }) + it('records history for every message that carries a payload, even a repeated one', function () { const { store } = merging() store[MERGE_ENTRY]('k', { payload: true }) From f5967f8029e14f640a5625101fb13e925ef7dc3c Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Thu, 15 Oct 2026 11:03:46 -0700 Subject: [PATCH 50/57] Fall back to an in-memory store that survives redeploys when global context can't hold the store, instead of disabling it --- nodes/config/ui_base.js | 6 ++-- nodes/store/data.js | 45 +++++++++++++------------ test/nodes/config-nodes/ui_base.spec.js | 2 +- test/store/data.spec.js | 39 ++++++++++++--------- test/store/differential.spec.js | 40 +++++++++++++++++----- 5 files changed, 81 insertions(+), 51 deletions(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index ebedfdd55..852a35e80 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -396,12 +396,10 @@ module.exports = function (RED) { onReplaced: () => node.warn('global.dashboardStore was replaced by a flow since the last deploy, discarding the stored value of every widget. The store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') }) if (!store) { - datastore.disableStore() - node.warn('Dashboard data store disabled: the global context store doesn\'t hold objects by reference (e.g. cache: false), so live state can\'t work. Use the memory store or localfilesystem with cache: true.' + (unavailable ? ' Reading it failed with: ' + unavailable.message : '')) + node.warn('Dashboard data store: flows can\'t use global.dashboardStore, because the default context store doesn\'t keep values in memory (e.g. localfilesystem with cache: false). Dashboards still work normally. To use global.dashboardStore from flows, set contextStorage.default to memory, or localfilesystem with cache: true.' + (unavailable ? ' (Reason: ' + unavailable.message + ')' : '')) } } catch (err) { - datastore.disableStore() - node.warn('Dashboard data store disabled: could not initialise it in global context (' + err.message + ').') + node.warn('Dashboard data store: flows can\'t use global.dashboardStore, because saving it to global context failed (' + err.message + '). Dashboards still work normally.') } n.root = RED.settings.httpNodeRoot || '/' diff --git a/nodes/store/data.js b/nodes/store/data.js index 00799efbb..575105aa5 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -1,6 +1,6 @@ const { isClientScoped } = require('../utils/index.js') -const { attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES, TRIM_SERIES } = require('./reactive.js') +const { createDataStore, attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES, TRIM_SERIES } = require('./reactive.js') const data = {} @@ -9,7 +9,7 @@ const config = { } let storeOptions = {} -let storeEnabled = false +let fallbackStore = null let storeContext = null let storeBase = null let warned = false @@ -29,47 +29,52 @@ function getOrCreateStore (globalContext) { return store } +function fallbackOrContextStore (node) { + return fallbackStore || getOrCreateStore(node ? node.context().global : storeContext) +} + function initStore (globalContext, { node, ...opts } = {}) { storeOptions = opts - storeEnabled = true storeContext = globalContext storeBase = node warned = false - return attachToContext(globalContext, attachOptions()) -} - -function disableStore () { - storeEnabled = false + let store = null + try { + store = attachToContext(globalContext, attachOptions()) + } finally { + if (store) { + fallbackStore = null + } else if (!fallbackStore) { + fallbackStore = createDataStore(attachOptions()) + } + } + return store } function writeToStore (node, msg) { - if (!storeEnabled) return if (!msg || typeof msg !== 'object') return try { - getOrCreateStore(node.context().global)[MERGE_ENTRY](node.id, msg) + fallbackOrContextStore(node)[MERGE_ENTRY](node.id, msg) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function appendToStore (node, msg) { - if (!storeEnabled) return if (!msg || typeof msg !== 'object') return try { - getOrCreateStore(node.context().global)[APPEND_ENTRY](node.id, msg) + fallbackOrContextStore(node)[APPEND_ENTRY](node.id, msg) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function replaceInStore (node, msgs) { - if (!storeEnabled) return if (!Array.isArray(msgs) || msgs.some((m) => !m || typeof m !== 'object')) return try { - getOrCreateStore(node.context().global)[SET_SERIES](node.id, msgs) + fallbackOrContextStore(node)[SET_SERIES](node.id, msgs) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function trimInStore (node, trim) { - if (!storeEnabled) return try { - getOrCreateStore(node.context().global)[TRIM_SERIES](node.id, trim) + fallbackOrContextStore(node)[TRIM_SERIES](node.id, trim) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } @@ -151,11 +156,8 @@ const setters = { // remove data associated to a given widget clear (id) { delete data[id] - if (!storeEnabled) { - return - } try { - delete getOrCreateStore(storeContext)[id] + delete fallbackOrContextStore()[id] } catch (err) { warnOnce(storeBase, `clearing the store entry for ${id} failed (${err.message}), so global.dashboardStore may still hold its old value`) } }, /** @@ -239,6 +241,5 @@ module.exports = { filter: setters.filter, keepLatestPerTopic: setters.keepLatestPerTopic, clear: setters.clear, - initStore, - disableStore + initStore } diff --git a/test/nodes/config-nodes/ui_base.spec.js b/test/nodes/config-nodes/ui_base.spec.js index 84d58f262..ff85897fb 100644 --- a/test/nodes/config-nodes/ui_base.spec.js +++ b/test/nodes/config-nodes/ui_base.spec.js @@ -153,7 +153,7 @@ describe('ui-base config node: cache-off context store', function () { await helper.load(nodeImports, flow) const warned = helper.log().args.filter((args) => - args[0].level === helper.log().WARN && /hold objects by reference/.test(args[0].msg) + args[0].level === helper.log().WARN && /doesn't keep values in memory/.test(args[0].msg) ) warned.should.have.length(1) touched.set.should.equal(0) diff --git a/test/store/data.spec.js b/test/store/data.spec.js index a257a0708..0e80d0a9f 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -212,7 +212,7 @@ describe('store: data.js store options', function () { }) }) -describe('store: data.js disabled store', function () { +describe('store: data.js when global context cannot hold the store', function () { const cacheOffGlobal = () => ({ gets: 0, get () { this.gets++; throw new Error('File Store cache disabled - only asynchronous access supported') }, @@ -223,34 +223,41 @@ describe('store: data.js disabled store', function () { datastore.initStore(fakeGlobal(), {}) }) - it('stops touching global context once the store is disabled', function () { + it('keeps writes off global context and reports no failures', function () { const global = cacheOffGlobal() - const scalar = fakeNode('w12', global) - const chart = fakeNode('w12-chart', global) - datastore.initStore(global, { node: { warn: () => {} } }) + let contextCalls = 0 + const offContext = (id) => ({ ...fakeNode(id, global), context: () => { contextCalls++; return { global } } }) + const scalar = offContext('w12') + const chart = offContext('w12-chart') + should(datastore.initStore(global, { node: { warn: (m) => warnings.push(m) } })).be.null() + warnings.length = 0 global.gets = 0 - datastore.disableStore() datastore.save(base, scalar, { payload: 1 }) datastore.append(base, chart, { _datapoint: { category: 'a', x: 1, y: 1 } }) datastore.filter(base, chart, () => false) + datastore.keepLatestPerTopic(base, chart, 1) datastore.clear(scalar.id) global.gets.should.equal(0) + contextCalls.should.equal(0) + warnings.should.eql([]) }) - it('leaves the legacy datastore working while disabled', function () { - const global = cacheOffGlobal() + it('falls back when writing to global context throws', function () { + const global = { get: () => undefined, set () { throw new Error('read-only context') } } const node = fakeNode('w13', global) - datastore.disableStore() + should(() => datastore.initStore(global, {})).throw('read-only context') + warnings.length = 0 - datastore.save(base, node, { payload: 'legacy still works' }) + datastore.save(base, node, { payload: 'still works' }) - datastore.get('w13').payload.should.equal('legacy still works') + warnings.should.eql([]) + datastore.get('w13').payload.should.equal('still works') }) - it('re-enables on a later initStore', function () { - datastore.disableStore() + it('goes back to global context on a later initStore', function () { + datastore.initStore(cacheOffGlobal(), {}) const global = fakeGlobal() const node = fakeNode('w14', global) @@ -462,10 +469,10 @@ describe('store: data.js chart writes', function () { }) }) - it('leaves the legacy series untouched when the store is disabled', function () { + it('leaves global context untouched when it cannot hold the store', function () { const global = fakeGlobal() const node = fakeNode('chart-off', global) - datastore.disableStore() + datastore.initStore({ get () { throw new Error('cache off') }, set () {} }, {}) try { datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) datastore.save(base, node, [{ payload: 2, _datapoint: pt(2, 2) }]) @@ -701,7 +708,7 @@ describe('store: data reads that avoid cloning', function () { const topicsAndPayloads = (id) => datastore.get(id).map((m) => `${m.topic}:${m.payload}`) beforeEach(function () { - datastore.disableStore() + datastore.initStore({ get () { throw new Error('cache off') }, set () {} }, {}) datastore.setConfig(countingRED) }) diff --git a/test/store/differential.spec.js b/test/store/differential.spec.js index 6f26a0fb1..574f68cc7 100644 --- a/test/store/differential.spec.js +++ b/test/store/differential.spec.js @@ -15,6 +15,10 @@ function fakeGlobal () { return { get: (k) => m[k], set: (k, v) => { m[k] = v } } } +function cacheOffGlobal () { + return { get () { throw new Error('File Store cache disabled - only asynchronous access supported') }, set () {} } +} + const runs = [ ...Object.entries(scenarios).map(([name, steps]) => ({ name, steps })), ...Array.from({ length: 200 }, (_, i) => ({ name: `seed ${i + 1}`, steps: generate(i + 1, 180) })), @@ -52,14 +56,16 @@ function storeHolds (global, id) { return store['$' + id]?.msg } -function compareAfterEachStep (steps, ids) { - const global = fakeGlobal() +function compareAfterEachStep (steps, ids, global = fakeGlobal()) { const warnings = [] - datastore.initStore(global, {}) + const shared = datastore.initStore(global, {}) const nodeFor = (id, type) => ({ id, type, context: () => ({ global }), warn: (m) => warnings.push(m) }) const legacy = createLegacy(RED) let comparedWithData = 0 - for (const step of steps) { + for (const [i, step] of steps.entries()) { + if (i === Math.floor(steps.length / 2)) { + datastore.initStore(global, {}) + } const given = structuredClone(step) runStep(datastore, given, { base, nodeFor }) runStep(legacy, structuredClone(step), { base, nodeFor }) @@ -69,16 +75,20 @@ function compareAfterEachStep (steps, ids) { scribble(datastore.get(step.id)) should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after scribbling what get returned`) datastore.has(step.id).should.equal(legacy.has(step.id), `${step.id} after ${step.op}`) - should(storeHolds(global, step.id)).eql(legacy.get(step.id), `store for ${step.id} after ${step.op}`) + if (shared) { + should(storeHolds(global, step.id)).eql(legacy.get(step.id), `store for ${step.id} after ${step.op}`) + } if (legacy.has(step.id)) { comparedWithData++ } } for (const id of ids) { should(datastore.get(id)).eql(legacy.get(id), id) - should(storeHolds(global, id)).eql(legacy.get(id), `store for ${id}`) + if (shared) { + should(storeHolds(global, id)).eql(legacy.get(id), `store for ${id}`) + } } - return { comparedWithData, warnings } + return { comparedWithData, warnings, shared } } function compareAtEnd (steps, ids) { @@ -111,8 +121,22 @@ describe('store: differential against the legacy datastore', function () { for (const run of runs) { it(run.name, function () { const { steps, ids } = prepare(run.steps) - const { comparedWithData, warnings } = compareAfterEachStep(steps, ids) + const { comparedWithData, warnings, shared } = compareAfterEachStep(steps, ids) + + should(shared).not.be.null() + comparedWithData.should.be.above(0) + warnings.should.eql([]) + }) + } + }) + + describe('data.js matches the legacy reference when global context cannot hold the store', function () { + for (const run of runs) { + it(run.name, function () { + const { steps, ids } = prepare(run.steps) + const { comparedWithData, warnings, shared } = compareAfterEachStep(steps, ids, cacheOffGlobal()) + should(shared).be.null() comparedWithData.should.be.above(0) warnings.should.eql([]) }) From 9dfe94ece0681699962c69b8647fc697bb5cbf95 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 16 Oct 2026 07:31:57 -0700 Subject: [PATCH 51/57] Read get and has from the store and drop the proxies' pass-through get trap --- nodes/store/data.js | 25 ++- nodes/store/reactive.js | 32 ++-- test/store/data.spec.js | 286 ++++++++++++++++++-------------- test/store/differential.spec.js | 2 - test/store/reactive.spec.js | 73 +++++--- 5 files changed, 238 insertions(+), 180 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index 575105aa5..b3b5bd2ac 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -1,6 +1,6 @@ const { isClientScoped } = require('../utils/index.js') -const { createDataStore, attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES, TRIM_SERIES } = require('./reactive.js') +const { createDataStore, attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES, TRIM_SERIES, ENTRY_MSG } = require('./reactive.js') const data = {} @@ -29,8 +29,14 @@ function getOrCreateStore (globalContext) { return store } -function fallbackOrContextStore (node) { - return fallbackStore || getOrCreateStore(node ? node.context().global : storeContext) +function fallbackOrContextStore () { + return fallbackStore || getOrCreateStore(storeContext) +} + +function storeOrWarn (id) { + try { + return fallbackOrContextStore() + } catch (err) { warnOnce(storeBase, `reading ${id} from the store failed (${err.message}), so it reads as empty`) } } function initStore (globalContext, { node, ...opts } = {}) { @@ -54,27 +60,27 @@ function initStore (globalContext, { node, ...opts } = {}) { function writeToStore (node, msg) { if (!msg || typeof msg !== 'object') return try { - fallbackOrContextStore(node)[MERGE_ENTRY](node.id, msg) + fallbackOrContextStore()[MERGE_ENTRY](node.id, msg) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function appendToStore (node, msg) { if (!msg || typeof msg !== 'object') return try { - fallbackOrContextStore(node)[APPEND_ENTRY](node.id, msg) + fallbackOrContextStore()[APPEND_ENTRY](node.id, msg) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function replaceInStore (node, msgs) { if (!Array.isArray(msgs) || msgs.some((m) => !m || typeof m !== 'object')) return try { - fallbackOrContextStore(node)[SET_SERIES](node.id, msgs) + fallbackOrContextStore()[SET_SERIES](node.id, msgs) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } function trimInStore (node, trim) { try { - fallbackOrContextStore(node)[TRIM_SERIES](node.id, trim) + fallbackOrContextStore()[TRIM_SERIES](node.id, trim) } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } } @@ -141,10 +147,11 @@ const getters = { }, // given a widget id, return the latest msg received msg (id) { - return config.RED.util.cloneMessage(data[id]) + return config.RED.util.cloneMessage(storeOrWarn(id)?.[ENTRY_MSG](id)) }, has (id) { - return !!data[id] + const store = storeOrWarn(id) + return !!store && id in store } } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index cf288159c..dfd27aedd 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -3,7 +3,7 @@ const APPEND_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/appendEntry') const SET_SERIES = Symbol.for('@flowfuse/node-red-dashboard/setSeries') const MERGE_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/mergeEntry') const TRIM_SERIES = Symbol.for('@flowfuse/node-red-dashboard/trimSeries') -const FLUSH_SERIES = Symbol.for('@flowfuse/node-red-dashboard/flushSeries') +const ENTRY_MSG = Symbol.for('@flowfuse/node-red-dashboard/entryMsg') const TRIM_BATCH = 60 const NAMESPACE = 'dashboardStore' @@ -42,7 +42,6 @@ function deepReactive (value, notify, path, clone, seen = new WeakMap()) { if (!isReactable(value)) return value if (seen.has(value)) return seen.get(value) const proxy = new Proxy(value, { - get (t, p, r) { return Reflect.get(t, p, r) }, set (t, p, v) { if (typeof p === 'symbol') return Reflect.set(t, p, v) // a push writes the new index then grows length; only a shrink is a change worth reporting @@ -112,7 +111,13 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D return rec } - const trimmedSeries = (rec) => rec.trims ? rec.trims.kept.concat(rec.msg.slice(rec.trims.seen)) : rec.msg + const trimmedSeries = (rec) => rec.trims ? rec.trims.kept.concat(rec.msg.slice(rec.trims.seen)) : rec.msg.slice() + const msgOf = (rec) => Array.isArray(rec.msg) ? Object.freeze(trimmedSeries(rec)) : rec.msg + + records[ENTRY_MSG] = (prop) => { + const rec = records[prop] + return rec ? msgOf(rec) : undefined + } records[MERGE_ENTRY] = (prop, msg) => { const current = records[prop] @@ -179,14 +184,6 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (replaced) emit(prop, rec, prop) } - const rebuildSeries = (prop, rec) => { - const msgs = trimmedSeries(rec) - rec.trims = null - if (msgs.length !== rec.msg.length) { - records[SET_SERIES](prop, msgs) - } - } - records[TRIM_SERIES] = (prop, trim) => { const rec = records[prop] if (!rec || !Array.isArray(rec.msg)) { @@ -204,14 +201,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D } pending.calls++ if (pending.calls >= TRIM_BATCH) { - rebuildSeries(prop, rec) - } - } - - records[FLUSH_SERIES] = (prop) => { - const rec = records[prop] - if (rec?.trims) { - rebuildSeries(prop, rec) + records[SET_SERIES](prop, trimmedSeries(rec)) } } @@ -224,7 +214,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (prop.startsWith('$')) { const rec = target[prop.slice(1)] if (!rec) return undefined - const msg = Array.isArray(rec.msg) ? Object.freeze(rec.msg.slice()) : rec.msg + const msg = msgOf(rec) return Object.freeze({ value: rec.value, msg, quality: rec.quality, timestamp: rec.timestamp, history: Object.freeze(rec.history.slice()) }) } const rec = target[prop] @@ -287,4 +277,4 @@ function attachToContext (globalContext, opts = {}) { return store } -module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, FLUSH_SERIES } +module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, ENTRY_MSG } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 0e80d0a9f..a5b842166 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -2,7 +2,7 @@ const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars const datastore = require('../../nodes/store/data.js') -const { APPEND_ENTRY, FLUSH_SERIES } = require('../../nodes/store/reactive.js') +const { APPEND_ENTRY } = require('../../nodes/store/reactive.js') const warnings = [] const RED = { util, plugins: { getByType: () => [] } } @@ -13,8 +13,14 @@ function fakeGlobal () { return { get: (k) => m[k], set: (k, v) => { m[k] = v } } } -function fakeNode (id, global) { - return { id, type: 'ui-text', context: () => ({ global }), warn: (m) => warnings.push(`${id}: ${m}`) } +function storeGlobal () { + const global = fakeGlobal() + datastore.initStore(global, {}) + return global +} + +function fakeNode (id) { + return { id, type: 'ui-text', warn: (m) => warnings.push(`${id}: ${m}`) } } const base = { acceptsClientConfig: [] } @@ -25,28 +31,27 @@ describe('store: data.js reactive-store mirror', function () { }) it('mirrors the clean payload into global.dashboardStore[id], not the whole message', function () { - const global = fakeGlobal() - const node = fakeNode('w1', global) + const global = storeGlobal() + const node = fakeNode('w1') datastore.save(base, node, { payload: 42, topic: 't', _msgid: 'm', extra: 'x' }) const store = global.get('dashboardStore') store.w1.should.equal(42) - // the reactive store holds only the value; the whole message stays in the legacy datastore datastore.get('w1').payload.should.equal(42) datastore.get('w1').topic.should.equal('t') }) it('keeps the mirrored value in sync on the latest write', function () { - const global = fakeGlobal() - const node = fakeNode('w2', global) + const global = storeGlobal() + const node = fakeNode('w2') datastore.save(base, node, { payload: 1 }) datastore.save(base, node, { payload: 2 }) global.get('dashboardStore').w2.should.equal(2) }) it('stores the whole message alongside the value', function () { - const global = fakeGlobal() - const node = fakeNode('m1', global) + const global = storeGlobal() + const node = fakeNode('m1') datastore.save(base, node, { payload: 42, topic: 't', extra: 'x' }) const store = global.get('dashboardStore') @@ -56,28 +61,18 @@ describe('store: data.js reactive-store mirror', function () { store.$m1.msg.extra.should.equal('x') }) - it('keeps the stored message in step with the legacy merge', function () { - const global = fakeGlobal() - const node = fakeNode('m3', global) + it('merges a later message onto the stored one', function () { + storeGlobal() + const node = fakeNode('m3') datastore.save(base, node, { payload: 1, topic: 'a', unit: 'bar' }) datastore.save(base, node, { payload: 2 }) - global.get('dashboardStore').$m3.msg.should.eql(datastore.get('m3')) - }) - - it('does not alias the legacy store into the mirror', function () { - const global = fakeGlobal() - const node = fakeNode('m4', global) - datastore.save(base, node, { payload: 1, topic: 'a' }) - - global.get('dashboardStore').$m4.msg.topic = 'mutated' - - datastore.get('m4').topic.should.equal('a') + datastore.get('m3').should.eql({ payload: 2, topic: 'a', unit: 'bar' }) }) it('does not alias the flow\'s message into the store', function () { - const global = fakeGlobal() - const node = fakeNode('m2', global) + const global = storeGlobal() + const node = fakeNode('m2') const sent = { payload: 1, nested: { a: 1 } } datastore.save(base, node, sent) @@ -87,16 +82,17 @@ describe('store: data.js reactive-store mirror', function () { }) it('does not mirror a client-scoped message (not stored centrally)', function () { - const global = fakeGlobal() - const node = fakeNode('w3', global) + const global = storeGlobal() + const node = fakeNode('w3') const scopedBase = { acceptsClientConfig: ['ui-text'] } datastore.save(scopedBase, node, { payload: 9, _client: { socketId: 's1' } }) - should(global.get('dashboardStore')).be.undefined() + should(global.get('dashboardStore').w3).be.undefined() + datastore.has('w3').should.equal(false) }) it('deep-copies a table rows array, so a flow reusing it cannot mutate stored state', function () { - const global = fakeGlobal() - const node = fakeNode('tbl-1', global) + const global = storeGlobal() + const node = fakeNode('tbl-1') const rows = [{ id: 1, name: 'a', meta: { site: { rack: 'r1' } } }] datastore.save(base, node, { payload: rows }) @@ -108,30 +104,30 @@ describe('store: data.js reactive-store mirror', function () { ]) }) - it('keeps the mirrored rows independent of the legacy datastore', function () { - const global = fakeGlobal() - const node = fakeNode('tbl-2', global) + it('reads a value only a flow wrote', function () { + const global = storeGlobal() - datastore.save(base, node, { payload: [{ id: 1, name: 'original' }] }) - global.get('dashboardStore')['tbl-2'][0].name = 'changed via store' + global.get('dashboardStore')['flow-only'] = 5 - datastore.get('tbl-2').payload.should.eql([{ id: 1, name: 'original' }]) + datastore.has('flow-only').should.equal(true) + datastore.get('flow-only').should.eql({ payload: 5 }) }) - it('does not throw or mirror when the context store cannot hold the proxy', function () { - const throwingGlobal = { - get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, - set: () => {} - } - const node = fakeNode('w5', throwingGlobal) - should(() => datastore.save(base, node, { payload: 7 })).not.throw() - datastore.get('w5').payload.should.equal(7) // legacy datastore still works + it('reads what a flow wrote into the stored rows, as a copy the caller owns', function () { + const global = storeGlobal() + const node = fakeNode('tbl-2') + + datastore.save(base, node, { payload: [{ id: 1, name: 'original' }] }) + global.get('dashboardStore')['tbl-2'][0].name = 'changed via store' + datastore.get('tbl-2').payload[0].name = 'changed by the caller' + + datastore.get('tbl-2').payload.should.eql([{ id: 1, name: 'changed via store' }]) }) it('clear removes the widget key so a removed widget leaves no orphan', function () { const global = fakeGlobal() datastore.initStore(global, {}) - const node = fakeNode('w6', global) + const node = fakeNode('w6') datastore.save(base, node, { payload: 5 }) global.get('dashboardStore').w6.should.equal(5) @@ -140,8 +136,8 @@ describe('store: data.js reactive-store mirror', function () { }) it('keeps the mirrored value when a later message carries no payload', function () { - const global = fakeGlobal() - const node = fakeNode('w8', global) + const global = storeGlobal() + const node = fakeNode('w8') datastore.save(base, node, { payload: 'keep me' }) datastore.save(base, node, { topic: 'no payload' }) @@ -151,8 +147,8 @@ describe('store: data.js reactive-store mirror', function () { }) it('blanks the mirrored value when a message explicitly carries payload: undefined', function () { - const global = fakeGlobal() - const node = fakeNode('w11', global) + const global = storeGlobal() + const node = fakeNode('w11') datastore.save(base, node, { payload: 'keep me' }) datastore.save(base, node, { payload: undefined, topic: 'explicit' }) @@ -161,7 +157,7 @@ describe('store: data.js reactive-store mirror', function () { }) it('does not throw when a msg is not an object (unchanged from main)', function () { - const global = fakeGlobal() + const global = storeGlobal() for (const msg of ['hello', 42, true]) { const node = fakeNode('prim-' + String(msg), global) should(() => datastore.save(base, node, msg)).not.throw() @@ -173,7 +169,7 @@ describe('store: data.js reactive-store mirror', function () { get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, set: () => {} } - const node = fakeNode('w7', throwingGlobal) + const node = fakeNode('w7') datastore.initStore(throwingGlobal, { node: { warn: () => {} } }) should(() => datastore.clear(node.id)).not.throw() @@ -187,7 +183,7 @@ describe('store: data.js store options', function () { it('re-injects with the registered options when a flow replaces the namespace', function () { const global = fakeGlobal() - const node = fakeNode('w9', global) + const node = fakeNode('w9') let replaced = 0 datastore.initStore(global, { onReplaced: () => { replaced++ } }) @@ -201,7 +197,7 @@ describe('store: data.js store options', function () { it('keeps onChange wired through a widget-triggered re-injection', function () { const global = fakeGlobal() - const node = fakeNode('w10', global) + const node = fakeNode('w10') const changes = [] datastore.initStore(global, { onChange: (key) => changes.push(key) }) @@ -225,10 +221,8 @@ describe('store: data.js when global context cannot hold the store', function () it('keeps writes off global context and reports no failures', function () { const global = cacheOffGlobal() - let contextCalls = 0 - const offContext = (id) => ({ ...fakeNode(id, global), context: () => { contextCalls++; return { global } } }) - const scalar = offContext('w12') - const chart = offContext('w12-chart') + const scalar = fakeNode('w12') + const chart = fakeNode('w12-chart') should(datastore.initStore(global, { node: { warn: (m) => warnings.push(m) } })).be.null() warnings.length = 0 global.gets = 0 @@ -240,13 +234,12 @@ describe('store: data.js when global context cannot hold the store', function () datastore.clear(scalar.id) global.gets.should.equal(0) - contextCalls.should.equal(0) warnings.should.eql([]) }) it('falls back when writing to global context throws', function () { const global = { get: () => undefined, set () { throw new Error('read-only context') } } - const node = fakeNode('w13', global) + const node = fakeNode('w13') should(() => datastore.initStore(global, {})).throw('read-only context') warnings.length = 0 @@ -260,7 +253,7 @@ describe('store: data.js when global context cannot hold the store', function () datastore.initStore(cacheOffGlobal(), {}) const global = fakeGlobal() - const node = fakeNode('w14', global) + const node = fakeNode('w14') datastore.initStore(global, {}) datastore.save(base, node, { payload: 'back on' }) @@ -269,17 +262,21 @@ describe('store: data.js when global context cannot hold the store', function () }) describe('store: data.js chart writes', function () { + beforeEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + const pt = (x, y, category = 'a') => ({ category, x, y }) // an absent chart key only proves anything if the store is actually working function liveStore (global, id) { - datastore.save(base, fakeNode(id, global), { payload: 'live' }) + datastore.save(base, fakeNode(id), { payload: 'live' }) return global.get('dashboardStore')?.[id] } it('mirrors a chart append into the store', function () { - const global = fakeGlobal() - const node = fakeNode('chart-1', global) + const global = storeGlobal() + const node = fakeNode('chart-1') datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) datastore.append(base, node, { payload: 2, _datapoint: pt(2, 2) }) @@ -292,8 +289,8 @@ describe('store: data.js chart writes', function () { }) it('flattens a multi-series append into several points', function () { - const global = fakeGlobal() - const node = fakeNode('chart-multi', global) + const global = storeGlobal() + const node = fakeNode('chart-multi') datastore.append(base, node, { payload: [1, 5], _datapoint: [pt(1, 1, 'a'), pt(1, 5, 'b')] }) @@ -304,8 +301,8 @@ describe('store: data.js chart writes', function () { }) it('stores a message with no datapoint but adds no point', function () { - const global = fakeGlobal() - const node = fakeNode('chart-nodp', global) + const global = storeGlobal() + const node = fakeNode('chart-nodp') datastore.append(base, node, { payload: 'x' }) @@ -315,8 +312,8 @@ describe('store: data.js chart writes', function () { }) it('does not alias the flow\'s message into an appended series', function () { - const global = fakeGlobal() - const node = fakeNode('chart-alias', global) + const global = storeGlobal() + const node = fakeNode('chart-alias') const sent = { payload: 1, _datapoint: pt(1, 1), nested: { a: 1 } } datastore.append(base, node, sent) @@ -326,8 +323,8 @@ describe('store: data.js chart writes', function () { }) it('mirrors an array save into the store as a whole series', function () { - const global = fakeGlobal() - const node = fakeNode('chart-2', global) + const global = storeGlobal() + const node = fakeNode('chart-2') datastore.save(base, node, [ { payload: 1, _datapoint: pt(1, 1) }, @@ -341,8 +338,8 @@ describe('store: data.js chart writes', function () { }) it('replaces rather than appends on a second array save', function () { - const global = fakeGlobal() - const node = fakeNode('chart-replace', global) + const global = storeGlobal() + const node = fakeNode('chart-replace') datastore.save(base, node, [{ payload: 1, _datapoint: pt(1, 1) }]) datastore.save(base, node, [{ payload: 9, _datapoint: pt(9, 9) }]) @@ -353,8 +350,8 @@ describe('store: data.js chart writes', function () { }) it('empties the series when saved an empty array', function () { - const global = fakeGlobal() - const node = fakeNode('chart-clear', global) + const global = storeGlobal() + const node = fakeNode('chart-clear') datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) datastore.save(base, node, []) @@ -365,8 +362,8 @@ describe('store: data.js chart writes', function () { }) it('does not rebuild the stored series until the batch is reached', function () { - const global = fakeGlobal() - const node = fakeNode('chart-3', global) + const global = storeGlobal() + const node = fakeNode('chart-3') for (let i = 0; i < 100; i++) { datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) } @@ -376,13 +373,13 @@ describe('store: data.js chart writes', function () { } liveStore(global, 'canary-3').should.equal('live') - datastore.get('chart-3').should.have.length(41) global.get('dashboardStore')['chart-3'].should.have.length(100) + datastore.get('chart-3').should.have.length(41) }) it('rebuilds the stored series on the batch boundary', function () { - const global = fakeGlobal() - const node = fakeNode('chart-batch', global) + const global = storeGlobal() + const node = fakeNode('chart-batch') for (let i = 0; i < 100; i++) { datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) } @@ -400,9 +397,9 @@ describe('store: data.js chart writes', function () { }) it('does not create a key when the appended message is not an object', function () { - const global = fakeGlobal() - const node = fakeNode('chart-junk', global) - datastore.append(base, fakeNode('chart-ok', global), { payload: 1, _datapoint: pt(1, 1) }) + const global = storeGlobal() + const node = fakeNode('chart-junk') + datastore.append(base, fakeNode('chart-ok'), { payload: 1, _datapoint: pt(1, 1) }) datastore.append(base, node, 5) datastore.save(base, node, [5, { payload: 1, _datapoint: pt(1, 1) }]) @@ -412,8 +409,8 @@ describe('store: data.js chart writes', function () { }) it('leaves the stored series untouched when trims remove nothing', function () { - const global = fakeGlobal() - const node = fakeNode('chart-noop', global) + const global = storeGlobal() + const node = fakeNode('chart-noop') for (let i = 0; i < 100; i++) { datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) } @@ -426,9 +423,9 @@ describe('store: data.js chart writes', function () { global.get('dashboardStore')['$chart-noop'].msg.should.eql(before) }) - it('still keeps the legacy chart series intact', function () { - const global = fakeGlobal() - const node = fakeNode('chart-4', global) + it('keeps the trimmed chart series', function () { + storeGlobal() + const node = fakeNode('chart-4') datastore.append(base, node, { _datapoint: pt(1, 1) }) datastore.append(base, node, { _datapoint: pt(2, 2) }) @@ -440,7 +437,7 @@ describe('store: data.js chart writes', function () { it('clears a stale key if a node id was previously used for a value', function () { const global = fakeGlobal() datastore.initStore(global, {}) - const node = fakeNode('chart-5', global) + const node = fakeNode('chart-5') datastore.save(base, node, { payload: 'scalar first' }) global.get('dashboardStore')['chart-5'].should.equal('scalar first') @@ -450,8 +447,8 @@ describe('store: data.js chart writes', function () { }) it('keeps history empty across many appends', function () { - const global = fakeGlobal() - const node = fakeNode('chart-hist', global) + const global = storeGlobal() + const node = fakeNode('chart-hist') for (let i = 0; i < 20; i++) { datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) } @@ -460,8 +457,8 @@ describe('store: data.js chart writes', function () { }) it('serialises a chart key to its point array and nothing else', function () { - const global = fakeGlobal() - const node = fakeNode('chart-json', global) + const global = storeGlobal() + const node = fakeNode('chart-json') datastore.append(base, node, { payload: 1, topic: 'a', _datapoint: pt(1, 1) }) JSON.parse(JSON.stringify(global.get('dashboardStore'))).should.eql({ @@ -471,7 +468,7 @@ describe('store: data.js chart writes', function () { it('leaves global context untouched when it cannot hold the store', function () { const global = fakeGlobal() - const node = fakeNode('chart-off', global) + const node = fakeNode('chart-off') datastore.initStore({ get () { throw new Error('cache off') }, set () {} }, {}) try { datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) @@ -487,9 +484,13 @@ describe('store: data.js chart writes', function () { }) describe('store: data.js audit invariants', function () { + beforeEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + it('does not store a message carrying only dynamic properties', function () { - const global = fakeGlobal() - const node = fakeNode('a1', global) + const global = storeGlobal() + const node = fakeNode('a1') datastore.save(base, node, { payload: 'real' }) datastore.save(base, node, { ui_update: { class: 'red' } }) @@ -497,8 +498,8 @@ describe('store: data.js audit invariants', function () { }) it('keeps value and msg in step across writes', function () { - const global = fakeGlobal() - const node = fakeNode('a2', global) + const global = storeGlobal() + const node = fakeNode('a2') datastore.save(base, node, { payload: 1, topic: 'first' }) datastore.save(base, node, { payload: 2, topic: 'second' }) @@ -508,8 +509,8 @@ describe('store: data.js audit invariants', function () { }) it('leaves no key at all when a widget never stores', function () { - const global = fakeGlobal() - datastore.save(base, fakeNode('a3', global), { payload: 1 }) + const global = storeGlobal() + datastore.save(base, fakeNode('a3'), { payload: 1 }) const store = global.get('dashboardStore') store.a3.should.equal(1) @@ -519,7 +520,7 @@ describe('store: data.js audit invariants', function () { it('clears value and msg together when a widget is removed', function () { const global = fakeGlobal() datastore.initStore(global, {}) - const node = fakeNode('a4', global) + const node = fakeNode('a4') datastore.save(base, node, { payload: 5, topic: 't' }) global.get('dashboardStore').$a4.msg.should.be.an.Object() @@ -532,9 +533,13 @@ describe('store: data.js audit invariants', function () { }) describe('store: data.js client-scoped writes', function () { + beforeEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + describe('save', function () { const base = { acceptsClientConfig: ['ui-text'] } - const node = fakeNode('w1', fakeGlobal()) + const node = fakeNode('w1') beforeEach(function () { datastore.clear(node.id) @@ -590,13 +595,13 @@ describe('store: data.js failure reporting', function () { it('warns once on the widget whose write failed, not once per write', function () { const global = fakeGlobal() - const node = fakeNode('fail-1', global) + const node = fakeNode('fail-1') datastore.initStore(global, {}) global.get = unreachable datastore.save(base, node, { payload: 1 }) datastore.save(base, node, { payload: 2 }) - datastore.append(base, fakeNode('fail-1-chart', global), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + datastore.append(base, fakeNode('fail-1-chart'), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) datastore.clear(node.id) warnings.should.have.length(1) @@ -608,7 +613,7 @@ describe('store: data.js failure reporting', function () { datastore.initStore(global, {}) global.get = unreachable - datastore.append(base, fakeNode('fail-a', global), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + datastore.append(base, fakeNode('fail-a'), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) warnings.should.have.length(1) warnings[0].should.startWith('fail-a: Dashboard data store: a store write failed (context store unavailable)') @@ -640,40 +645,40 @@ describe('store: data.js failure reporting', function () { datastore.initStore(global, {}) copying = true - datastore.save(base, fakeNode('fail-r', global), { payload: 1 }) + datastore.save(base, fakeNode('fail-r'), { payload: 1 }) warnings.should.have.length(1) warnings[0].should.startWith('fail-r: Dashboard data store: a store write failed (global context saved a copy of the store rather than the store itself, which happens with context stores that serialise values)') }) - it('still writes the legacy datastore when the store write fails', function () { + it('reads as empty and warns once on the base node when the store cannot be reached', function () { const global = fakeGlobal() - const node = fakeNode('fail-2', global) - datastore.initStore(global, {}) + datastore.initStore(global, { node: { warn: (m) => warnings.push(`base: ${m}`) } }) global.get = unreachable - datastore.save(base, node, { payload: 'kept' }) + should(datastore.get('fail-2')).be.undefined() + datastore.has('fail-2').should.equal(false) - datastore.get('fail-2').payload.should.equal('kept') + warnings.should.eql(['base: Dashboard data store: reading fail-2 from the store failed (context store unavailable), so it reads as empty. Further store failures are not reported until the next full deploy or restart.']) }) it('warns again after the next initStore', function () { const global = fakeGlobal() datastore.initStore(global, {}) global.get = unreachable - datastore.save(base, fakeNode('fail-3', global), { payload: 1 }) + datastore.save(base, fakeNode('fail-3'), { payload: 1 }) const fresh = fakeGlobal() datastore.initStore(fresh, {}) fresh.get = unreachable - datastore.save(base, fakeNode('fail-3', fresh), { payload: 2 }) + datastore.save(base, fakeNode('fail-3'), { payload: 2 }) warnings.should.have.length(2) }) it('warns on the widget when a trim cannot reach the store', function () { const global = fakeGlobal() - const node = fakeNode('fail-5', global) + const node = fakeNode('fail-5') datastore.initStore(global, {}) for (let i = 0; i < 100; i++) datastore.append(base, node, { payload: i, _datapoint: { category: 'a', x: i, y: i } }) global.get = unreachable @@ -686,7 +691,7 @@ describe('store: data.js failure reporting', function () { it('throws a failing filter to the widget without reporting a store failure', function () { const global = fakeGlobal() - const node = fakeNode('fail-6', global) + const node = fakeNode('fail-6') datastore.initStore(global, {}) datastore.append(base, node, { payload: 1, _datapoint: null }) @@ -810,7 +815,7 @@ describe('store: data.js merges into the store from the store itself', function it('stores a message without a payload, merged onto the stored one', function () { const global = fakeGlobal() datastore.initStore(global, {}) - const node = fakeNode('merge-1', global) + const node = fakeNode('merge-1') datastore.save(base, node, { payload: 1 }) datastore.save(base, node, { topic: 't' }) @@ -821,21 +826,21 @@ describe('store: data.js merges into the store from the store itself', function it('merges onto what the store holds, not what the legacy datastore holds', function () { const global = fakeGlobal() datastore.initStore(global, {}) - const node = fakeNode('merge-2', global) + const node = fakeNode('merge-2') datastore.save(base, node, { payload: 1 }) global.get('dashboardStore')['merge-2'] = 5 datastore.save(base, node, { topic: 't' }) global.get('dashboardStore')['$merge-2'].msg.should.eql({ payload: 5, topic: 't' }) - datastore.get('merge-2').should.eql({ payload: 1, topic: 't' }) + datastore.get('merge-2').should.eql({ payload: 5, topic: 't' }) }) it('ignores a message that is not an object, without warning', function () { const global = fakeGlobal() datastore.initStore(global, {}) const warned = [] - const node = { ...fakeNode('merge-4', global), warn: (m) => warned.push(m) } + const node = { ...fakeNode('merge-4'), warn: (m) => warned.push(m) } datastore.save(base, node, { payload: 1 }) datastore.save(base, node, 5) @@ -847,7 +852,7 @@ describe('store: data.js merges into the store from the store itself', function it('removes the store entry on clear(id), which has no node', function () { const global = fakeGlobal() datastore.initStore(global, {}) - datastore.save(base, fakeNode('merge-3', global), { payload: 'hello' }) + datastore.save(base, fakeNode('merge-3'), { payload: 'hello' }) datastore.clear('merge-3') @@ -865,7 +870,7 @@ describe('store: data.js trims the stored series itself', function () { it('trims what the store holds, not a copy of the legacy result', function () { const global = fakeGlobal() datastore.initStore(global, {}) - const node = fakeNode('own-trim', global) + const node = fakeNode('own-trim') datastore.save(base, node, []) for (let i = 0; i < 6; i++) { datastore.append(base, node, point(i, i % 2 ? 'b' : 'a')) @@ -873,16 +878,15 @@ describe('store: data.js trims the stored series itself', function () { global.get('dashboardStore')[APPEND_ENTRY]('own-trim', point('store-only')) datastore.keepLatestPerTopic(base, node, 2) - global.get('dashboardStore')[FLUSH_SERIES]('own-trim') global.get('dashboardStore')['$own-trim'].msg.map((m) => m.payload).should.eql([3, 4, 5, 'store-only']) - datastore.get('own-trim').map((m) => m.payload).should.eql([2, 3, 4, 5]) + datastore.get('own-trim').map((m) => m.payload).should.eql([3, 4, 5, 'store-only']) }) it('applies both a point limit and a time window', function () { const global = fakeGlobal() datastore.initStore(global, {}) - const node = fakeNode('both-limits', global) + const node = fakeNode('both-limits') datastore.save(base, node, []) for (let i = 0; i < 100; i++) { datastore.append(base, node, point(i)) @@ -890,8 +894,36 @@ describe('store: data.js trims the stored series itself', function () { datastore.filter(base, node, (m) => m._datapoint.x > i - 50) } - global.get('dashboardStore')[FLUSH_SERIES]('both-limits') - global.get('dashboardStore')['$both-limits'].msg.map((m) => m.payload).should.eql(datastore.get('both-limits').map((m) => m.payload)) }) }) + +describe('store: data.js after a restart with a persistent context store', function () { + const restored = (values) => { + const global = fakeGlobal() + global.set('dashboardStore', JSON.parse(JSON.stringify(values))) + datastore.initStore(global, {}) + } + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('reads restored values as messages holding only the payload', function () { + restored({ text: 'hello', rows: [{ id: 1 }] }) + + datastore.has('text').should.equal(true) + datastore.get('text').should.eql({ payload: 'hello' }) + datastore.get('rows').should.eql({ payload: [{ id: 1 }] }) + }) + + it('reads a restored chart as its points until the next point replaces it with a series', function () { + restored({ chart: [{ category: 'a', x: 1, y: 1 }] }) + + datastore.get('chart').should.eql({ payload: [{ category: 'a', x: 1, y: 1 }] }) + + datastore.append(base, fakeNode('chart'), { payload: 2, _datapoint: { category: 'a', x: 2, y: 2 } }) + + datastore.get('chart').should.eql([{ payload: 2, _datapoint: { category: 'a', x: 2, y: 2 } }]) + }) +}) diff --git a/test/store/differential.spec.js b/test/store/differential.spec.js index 574f68cc7..3652ff846 100644 --- a/test/store/differential.spec.js +++ b/test/store/differential.spec.js @@ -2,7 +2,6 @@ const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars const datastore = require('../../nodes/store/data.js') -const { FLUSH_SERIES } = require('../../nodes/store/reactive.js') const createLegacy = require('./helpers/legacy-datastore.js') const { scenarios, generate, generateMessy, runStep } = require('./helpers/sequences.js') @@ -52,7 +51,6 @@ function scribble (value, seen = new Set()) { function storeHolds (global, id) { const store = global.get('dashboardStore') - store[FLUSH_SERIES](id) return store['$' + id]?.msg } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 2a63d972e..6eeaba8fe 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -11,7 +11,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, FLUSH_SERIES } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, ENTRY_MSG } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -1348,6 +1348,49 @@ describe('store: MERGE_ENTRY', function () { }) }) +describe('store: ENTRY_MSG', function () { + const point = (x) => ({ payload: x, topic: 'a', _datapoint: { category: 'a', x, y: x } }) + + it('reads a stored message without building a new object', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + store[MERGE_ENTRY]('k', { payload: 1, topic: 't' }) + store[MERGE_ENTRY]('k', { payload: 2 }) + + store[ENTRY_MSG]('k').should.equal(store[ENTRY_MSG]('k')) + store[ENTRY_MSG]('k').should.eql({ payload: 2, topic: 't' }) + }) + + it('reads a series with pending trims applied, as a frozen copy', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + for (let x = 0; x < 5; x++) { + store[APPEND_ENTRY]('c', point(x)) + } + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > 2)) + + const msgs = store[ENTRY_MSG]('c') + + msgs.map((m) => m.payload).should.eql([3, 4]) + Object.isFrozen(msgs).should.equal(true) + store[ENTRY_MSG]('c').should.not.equal(msgs) + }) + + it('matches the $ view', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + store[APPEND_ENTRY]('c', point(1)) + store[TRIM_SERIES]('c', (msgs) => msgs) + store[MERGE_ENTRY]('k', { payload: 1 }) + + store[ENTRY_MSG]('c').should.eql(store.$c.msg) + store[ENTRY_MSG]('k').should.equal(store.$k.msg) + }) + + it('reads nothing for a missing entry', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + + should(store[ENTRY_MSG]('missing')).be.undefined() + }) +}) + describe('store: TRIM_SERIES', function () { const point = (x, topic = 'a') => ({ payload: x, topic, _datapoint: { category: topic, x, y: x } }) const keepLatest = (max) => (msgs) => msgs.filter((m, i) => msgs.slice(i + 1).filter((n) => n.topic === m.topic).length < max) @@ -1366,10 +1409,11 @@ describe('store: TRIM_SERIES', function () { for (let i = 0; i < 59; i++) { store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > i)) } - store.$c.msg.should.have.length(100) + store.c.should.have.length(100) store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > 59)) + store.c.should.have.length(40) payloads(store).should.eql(Array.from({ length: 40 }, (_, i) => i + 60)) }) @@ -1380,15 +1424,14 @@ describe('store: TRIM_SERIES', function () { store[TRIM_SERIES]('c', (msgs) => msgs) } - store.$c.msg.should.have.length(100) + store.c.should.have.length(100) }) - it('applies pending trims on flush', function () { + it('shows pending trims in the $ view before the series is rebuilt', function () { const { store } = series(10) store[TRIM_SERIES]('c', keepLatest(3)) - store[FLUSH_SERIES]('c') - + store.c.should.have.length(10) payloads(store).should.eql([7, 8, 9]) }) @@ -1401,8 +1444,6 @@ describe('store: TRIM_SERIES', function () { store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m._datapoint.x > cutoff)) } - store[FLUSH_SERIES]('c') - payloads(store).should.eql([100, 50]) }) @@ -1411,8 +1452,6 @@ describe('store: TRIM_SERIES', function () { store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 1)) store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 3)) - store[FLUSH_SERIES]('c') - payloads(store).should.eql([0, 2, 4]) }) @@ -1422,18 +1461,14 @@ describe('store: TRIM_SERIES', function () { store[APPEND_ENTRY]('c', point(5)) store[TRIM_SERIES]('c', (msgs) => msgs.filter((m, i) => i > 0)) - store[FLUSH_SERIES]('c') - payloads(store).should.eql([2, 3, 4, 5]) }) - it('keeps points appended after the last trim on flush', function () { + it('shows points appended after the last trim', function () { const { store } = series(5) store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 0)) store[APPEND_ENTRY]('c', point(5)) - store[FLUSH_SERIES]('c') - payloads(store).should.eql([1, 2, 3, 4, 5]) }) @@ -1444,7 +1479,6 @@ describe('store: TRIM_SERIES', function () { for (let i = 0; i < 60; i++) { store[TRIM_SERIES]('c', (msgs) => msgs) } - store[FLUSH_SERIES]('c') log.should.eql([]) payloads(store).should.eql([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -1456,7 +1490,6 @@ describe('store: TRIM_SERIES', function () { store[SET_SERIES]('c', [point(1), point(2)]) store[TRIM_SERIES]('c', keepLatest(1)) - store[FLUSH_SERIES]('c') payloads(store).should.eql([2]) }) @@ -1468,18 +1501,16 @@ describe('store: TRIM_SERIES', function () { delete store.c store[APPEND_ENTRY]('c', point(1)) store[APPEND_ENTRY]('c', point(2)) - store[FLUSH_SERIES]('c') payloads(store).should.eql([1, 2]) }) - it('ignores trims and flushes on an entry that is not a series', function () { + it('ignores trims on an entry that is not a series', function () { const { store } = makeStore({ clone: util.cloneMessage }) store.k = 5 store[TRIM_SERIES]('k', () => []) - store[FLUSH_SERIES]('k') - store[FLUSH_SERIES]('missing') + store[TRIM_SERIES]('missing', () => []) store.k.should.equal(5) }) From 23c12e640187f085d068ad4e8fab44c367cf3cd2 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 16 Oct 2026 07:53:38 -0700 Subject: [PATCH 52/57] Remove the legacy datastore, and send operation errors to the widget and unreachable-store warnings to ui-base --- nodes/store/data.js | 112 ++++++++-------------------------------- test/store/data.spec.js | 95 +++++++++++++++------------------- 2 files changed, 64 insertions(+), 143 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index b3b5bd2ac..704474965 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -2,8 +2,6 @@ const { isClientScoped } = require('../utils/index.js') const { createDataStore, attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES, TRIM_SERIES, ENTRY_MSG } = require('./reactive.js') -const data = {} - const config = { RED: null } @@ -14,10 +12,10 @@ let storeContext = null let storeBase = null let warned = false -function warnOnce (node, problem) { - if (warned || !node) return +function warnOnce (problem) { + if (warned || !storeBase) return warned = true - node.warn(`Dashboard data store: ${problem}. Further store failures are not reported until the next full deploy or restart.`) + storeBase.warn(`Dashboard data store: ${problem}. Further store failures are not reported until the next full deploy or restart.`) } const attachOptions = () => ({ clone: config.RED.util.cloneMessage, ...storeOptions }) @@ -33,10 +31,10 @@ function fallbackOrContextStore () { return fallbackStore || getOrCreateStore(storeContext) } -function storeOrWarn (id) { +function storeOrWarn (action, id) { try { return fallbackOrContextStore() - } catch (err) { warnOnce(storeBase, `reading ${id} from the store failed (${err.message}), so it reads as empty`) } + } catch (err) { warnOnce(`${action} ${id} failed (${err.message})`) } } function initStore (globalContext, { node, ...opts } = {}) { @@ -57,33 +55,6 @@ function initStore (globalContext, { node, ...opts } = {}) { return store } -function writeToStore (node, msg) { - if (!msg || typeof msg !== 'object') return - try { - fallbackOrContextStore()[MERGE_ENTRY](node.id, msg) - } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } -} - -function appendToStore (node, msg) { - if (!msg || typeof msg !== 'object') return - try { - fallbackOrContextStore()[APPEND_ENTRY](node.id, msg) - } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } -} - -function replaceInStore (node, msgs) { - if (!Array.isArray(msgs) || msgs.some((m) => !m || typeof m !== 'object')) return - try { - fallbackOrContextStore()[SET_SERIES](node.id, msgs) - } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } -} - -function trimInStore (node, trim) { - try { - fallbackOrContextStore()[TRIM_SERIES](node.id, trim) - } catch (err) { warnOnce(node, `a store write failed (${err.message}), so global.dashboardStore may be out of date`) } -} - function latestPerTopic (msgs, maxPoints) { const counts = {} const kept = [] @@ -131,26 +102,16 @@ function canSaveInStore (base, node, msg) { return checks.length === 0 || !checks.includes(false) } -// Strip msg of properties that are not needed for storage -function stripMsg (msg) { - const newMsg = config.RED.util.cloneMessage(msg) - - // don't need to store ui_updates in the datastore, as this is handled in statestore - delete newMsg.ui_update - - return newMsg -} - const getters = { RED () { return config.RED }, // given a widget id, return the latest msg received msg (id) { - return config.RED.util.cloneMessage(storeOrWarn(id)?.[ENTRY_MSG](id)) + return config.RED.util.cloneMessage(storeOrWarn('reading', id)?.[ENTRY_MSG](id)) }, has (id) { - const store = storeOrWarn(id) + const store = storeOrWarn('reading', id) return !!store && id in store } } @@ -162,10 +123,10 @@ const setters = { }, // remove data associated to a given widget clear (id) { - delete data[id] - try { - delete fallbackOrContextStore()[id] - } catch (err) { warnOnce(storeBase, `clearing the store entry for ${id} failed (${err.message}), so global.dashboardStore may still hold its old value`) } + const store = storeOrWarn('clearing', id) + if (store) { + delete store[id] + } }, /** * @@ -175,35 +136,20 @@ const setters = { */ save (base, node, msg) { if (Array.isArray(msg)) { - /// need to check msg by msg - const filtered = [] - for (const m of msg) { - if (canSaveInStore(base, node, m)) { - filtered.push(config.RED.util.cloneMessage(m)) - } - } - data[node.id] = filtered - replaceInStore(node, filtered) - } else { - if (canSaveInStore(base, node, msg)) { - const newMsg = stripMsg(msg) - data[node.id] = { - ...data[node.id], - ...newMsg - } - writeToStore(node, newMsg) + const msgs = msg.filter((m) => canSaveInStore(base, node, m)) + if (msgs.every((m) => m && typeof m === 'object')) { + storeOrWarn('saving', node.id)?.[SET_SERIES](node.id, msgs) } + } else if (msg && typeof msg === 'object' && canSaveInStore(base, node, msg)) { + const { ui_update: uiUpdate, ...rest } = msg + storeOrWarn('saving', node.id)?.[MERGE_ENTRY](node.id, rest) } }, // given a widget id, and msg, store in an array of history of values // useful for charting widgets append (base, node, msg) { - if (canSaveInStore(base, node, msg)) { - if (!data[node.id]) { - data[node.id] = [] - } - data[node.id].push(config.RED.util.cloneMessage(msg)) - appendToStore(node, msg) + if (msg && typeof msg === 'object' && canSaveInStore(base, node, msg)) { + storeOrWarn('saving', node.id)?.[APPEND_ENTRY](node.id, msg) } }, /** @@ -213,28 +159,12 @@ const setters = { * @param {(msg) => Boolean} filterFunction */ filter (base, node, filterFunction) { - const currentData = data[node.id] - if (filterFunction && Array.isArray(currentData) && currentData.length) { - const filteredMessages = currentData.filter(filterFunction) - if (filteredMessages.length !== currentData.length) { - // no need for save operation to process messages - just apply them - data[node.id] = filteredMessages - } - } if (filterFunction) { - trimInStore(node, (msgs) => msgs.filter(filterFunction)) + storeOrWarn('trimming', node.id)?.[TRIM_SERIES](node.id, (msgs) => msgs.filter(filterFunction)) } }, keepLatestPerTopic (base, node, maxPoints) { - trimInStore(node, (msgs) => latestPerTopic(msgs, maxPoints)) - const currentData = data[node.id] - if (!Array.isArray(currentData)) { - return - } - const kept = latestPerTopic(currentData, maxPoints) - if (kept.length !== currentData.length) { - data[node.id] = kept - } + storeOrWarn('trimming', node.id)?.[TRIM_SERIES](node.id, (msgs) => latestPerTopic(msgs, maxPoints)) } } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index a5b842166..a3d03749a 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -584,6 +584,8 @@ describe('store: data.js client-scoped writes', function () { describe('store: data.js failure reporting', function () { const unreachable = () => { throw new Error('context store unavailable') } + const baseNode = { warn: (m) => warnings.push(`base: ${m}`) } + const SUFFIX = '. Further store failures are not reported until the next full deploy or restart.' beforeEach(function () { warnings.length = 0 @@ -593,111 +595,100 @@ describe('store: data.js failure reporting', function () { datastore.initStore(fakeGlobal(), {}) }) - it('warns once on the widget whose write failed, not once per write', function () { + it('warns once on the base node when the store cannot be reached, not once per call', function () { const global = fakeGlobal() - const node = fakeNode('fail-1') - datastore.initStore(global, {}) + datastore.initStore(global, { node: baseNode }) global.get = unreachable - datastore.save(base, node, { payload: 1 }) - datastore.save(base, node, { payload: 2 }) + datastore.save(base, fakeNode('fail-1'), { payload: 1 }) + datastore.save(base, fakeNode('fail-1'), { payload: 2 }) datastore.append(base, fakeNode('fail-1-chart'), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) - datastore.clear(node.id) + datastore.filter(base, fakeNode('fail-1-chart'), () => true) + datastore.clear('fail-1') + datastore.get('fail-1') - warnings.should.have.length(1) - warnings[0].should.equal('fail-1: Dashboard data store: a store write failed (context store unavailable), so global.dashboardStore may be out of date. Further store failures are not reported until the next full deploy or restart.') + warnings.should.eql([`base: Dashboard data store: saving fail-1 failed (context store unavailable)${SUFFIX}`]) }) - it('warns when an append fails', function () { - const global = fakeGlobal() - datastore.initStore(global, {}) - global.get = unreachable + it('names the operation that failed', function () { + for (const [action, call] of [ + ['saving', () => datastore.append(base, fakeNode('fail-a'), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } })], + ['trimming', () => datastore.keepLatestPerTopic(base, fakeNode('fail-a'), 1)], + ['clearing', () => datastore.clear('fail-a')], + ['reading', () => datastore.has('fail-a')] + ]) { + const global = fakeGlobal() + datastore.initStore(global, { node: baseNode }) + global.get = unreachable + warnings.length = 0 - datastore.append(base, fakeNode('fail-a'), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + call() - warnings.should.have.length(1) - warnings[0].should.startWith('fail-a: Dashboard data store: a store write failed (context store unavailable)') + warnings.should.eql([`base: Dashboard data store: ${action} fail-a failed (context store unavailable)${SUFFIX}`]) + } }) - it('does not throw from clear when there is no base node to warn on', function () { + it('does not throw when there is no base node to warn on', function () { const global = fakeGlobal() datastore.initStore(global, {}) global.get = unreachable should(() => datastore.clear('no-base')).not.throw() - }) - - it('warns on the base node when a clear fails, since a clear has no widget node', function () { - const global = fakeGlobal() - datastore.initStore(global, { node: { warn: (m) => warnings.push(`base: ${m}`) } }) - global.get = unreachable - - datastore.clear('fail-c') - - warnings.should.have.length(1) - warnings[0].should.startWith('base: Dashboard data store: clearing the store entry for fail-c failed (context store unavailable), so global.dashboardStore may still hold its old value.') + should(() => datastore.save(base, fakeNode('no-base'), { payload: 1 })).not.throw() }) it('explains a context store that stops holding the store by reference', function () { const m = {} let copying = false const global = { get: (k) => copying ? JSON.parse(JSON.stringify(m[k])) : m[k], set: (k, v) => { m[k] = v } } - datastore.initStore(global, {}) + datastore.initStore(global, { node: baseNode }) copying = true datastore.save(base, fakeNode('fail-r'), { payload: 1 }) - warnings.should.have.length(1) - warnings[0].should.startWith('fail-r: Dashboard data store: a store write failed (global context saved a copy of the store rather than the store itself, which happens with context stores that serialise values)') + warnings.should.eql([`base: Dashboard data store: saving fail-r failed (global context saved a copy of the store rather than the store itself, which happens with context stores that serialise values)${SUFFIX}`]) }) - it('reads as empty and warns once on the base node when the store cannot be reached', function () { + it('reads as empty when the store cannot be reached', function () { const global = fakeGlobal() - datastore.initStore(global, { node: { warn: (m) => warnings.push(`base: ${m}`) } }) + datastore.initStore(global, { node: baseNode }) global.get = unreachable should(datastore.get('fail-2')).be.undefined() datastore.has('fail-2').should.equal(false) - - warnings.should.eql(['base: Dashboard data store: reading fail-2 from the store failed (context store unavailable), so it reads as empty. Further store failures are not reported until the next full deploy or restart.']) }) it('warns again after the next initStore', function () { const global = fakeGlobal() - datastore.initStore(global, {}) + datastore.initStore(global, { node: baseNode }) global.get = unreachable datastore.save(base, fakeNode('fail-3'), { payload: 1 }) const fresh = fakeGlobal() - datastore.initStore(fresh, {}) + datastore.initStore(fresh, { node: baseNode }) fresh.get = unreachable datastore.save(base, fakeNode('fail-3'), { payload: 2 }) warnings.should.have.length(2) }) - it('warns on the widget when a trim cannot reach the store', function () { - const global = fakeGlobal() - const node = fakeNode('fail-5') - datastore.initStore(global, {}) - for (let i = 0; i < 100; i++) datastore.append(base, node, { payload: i, _datapoint: { category: 'a', x: i, y: i } }) - global.get = unreachable - - datastore.filter(base, node, (m, idx) => idx > 0) - - warnings.should.have.length(1) - warnings[0].should.startWith('fail-5: ') - }) - it('throws a failing filter to the widget without reporting a store failure', function () { - const global = fakeGlobal() + datastore.initStore(fakeGlobal(), { node: baseNode }) const node = fakeNode('fail-6') - datastore.initStore(global, {}) datastore.append(base, node, { payload: 1, _datapoint: null }) should.throws(() => datastore.filter(base, node, (m) => m._datapoint.x > 0)) - warnings.should.have.length(0) + warnings.should.eql([]) + }) + + it('throws a message the store cannot copy to the widget without reporting a store failure', function () { + datastore.initStore(fakeGlobal(), { node: baseNode }) + const unreadable = { get payload () { throw new Error('unreadable') } } + + should.throws(() => datastore.save(base, fakeNode('fail-7'), unreadable), /unreadable/) + + warnings.should.eql([]) }) }) From 684af5ec9a89daae5ff68847de0d194071e97db6 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Fri, 16 Oct 2026 11:25:56 -0700 Subject: [PATCH 53/57] Copy incoming fields before writing an entry so a failed save leaves it unchanged, and make the harness's warning checks real --- nodes/store/reactive.js | 12 +++++++----- test/store/data.spec.js | 32 +++++++++++++++++++++++++++++++- test/store/differential.spec.js | 7 ++++--- 3 files changed, 42 insertions(+), 9 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index dfd27aedd..702848e18 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -125,8 +125,10 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D const previous = Array.isArray(current?.msg) ? freezeOwned(cloneValue({ ...trimmedSeries(current) })) : current?.msg || {} const { payload: storedPayload, ...stored } = previous const keepValue = current && !Array.isArray(current.msg) && !('payload' in msg) - const rec = keepValue ? current : writeValue(prop, cloneValue(payload)) - const fields = { ...stored, ...freezeOwned(cloneValue(incoming)) } + const owned = freezeOwned(cloneValue(incoming)) + const value = keepValue ? undefined : cloneValue(payload) + const rec = keepValue ? current : writeValue(prop, value) + const fields = { ...stored, ...owned } const hasPayload = 'payload' in msg || (keepValue && 'payload' in current.msg) rec.msg = Object.freeze(hasPayload ? { ...fields, payload: rec.value } : fields) rec.timestamp = now() @@ -169,13 +171,13 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D records[APPEND_ENTRY] = (prop, msg) => { let rec = records[prop] const replaced = !!rec && !(Array.isArray(rec.value) && Array.isArray(rec.msg)) + // the proxy clones on push, so points must not be cloned again here + const { _datapoint, ...rest } = msg + const owned = cloneValue(rest) if (!rec || replaced) { rec = writeValue(prop, []) rec.msg = [] } - // the proxy clones on push, so points must not be cloned again here - const { _datapoint, ...rest } = msg - const owned = cloneValue(rest) const start = rec.value.length for (const p of pointsOf(_datapoint)) rec.value.push(p) const stored = toStoredDatapoint(rec.value, _datapoint, start, rec.value.length) diff --git a/test/store/data.spec.js b/test/store/data.spec.js index a3d03749a..021602b80 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -240,7 +240,7 @@ describe('store: data.js when global context cannot hold the store', function () it('falls back when writing to global context throws', function () { const global = { get: () => undefined, set () { throw new Error('read-only context') } } const node = fakeNode('w13') - should(() => datastore.initStore(global, {})).throw('read-only context') + should(() => datastore.initStore(global, { node: { warn: (m) => warnings.push(m) } })).throw('read-only context') warnings.length = 0 datastore.save(base, node, { payload: 'still works' }) @@ -423,6 +423,16 @@ describe('store: data.js chart writes', function () { global.get('dashboardStore')['$chart-noop'].msg.should.eql(before) }) + it('ignores a filter call with no callback', function () { + storeGlobal() + const node = fakeNode('chart-nofilter') + datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) + + should(() => datastore.filter(base, node, undefined)).not.throw() + + datastore.get('chart-nofilter').should.have.length(1) + }) + it('keeps the trimmed chart series', function () { storeGlobal() const node = fakeNode('chart-4') @@ -682,6 +692,26 @@ describe('store: data.js failure reporting', function () { warnings.should.eql([]) }) + it('leaves the stored message untouched when a save cannot be copied', function () { + datastore.initStore(fakeGlobal(), { node: baseNode }) + const node = fakeNode('fail-8') + datastore.save(base, node, { payload: 1, topic: 't', label: 'L' }) + + should.throws(() => datastore.save(base, node, { payload: 2, meta: { get bad () { throw new Error('unreadable') } } })) + + datastore.get('fail-8').should.eql({ payload: 1, topic: 't', label: 'L' }) + }) + + it('leaves the stored message untouched when an append cannot be copied', function () { + datastore.initStore(fakeGlobal(), { node: baseNode }) + const node = fakeNode('fail-9') + datastore.save(base, node, { payload: 1, topic: 't' }) + + should.throws(() => datastore.append(base, node, { payload: 2, meta: { get bad () { throw new Error('unreadable') } }, _datapoint: { x: 1, y: 2 } })) + + datastore.get('fail-9').should.eql({ payload: 1, topic: 't' }) + }) + it('throws a message the store cannot copy to the widget without reporting a store failure', function () { datastore.initStore(fakeGlobal(), { node: baseNode }) const unreadable = { get payload () { throw new Error('unreadable') } } diff --git a/test/store/differential.spec.js b/test/store/differential.spec.js index 3652ff846..cd540769f 100644 --- a/test/store/differential.spec.js +++ b/test/store/differential.spec.js @@ -56,13 +56,14 @@ function storeHolds (global, id) { function compareAfterEachStep (steps, ids, global = fakeGlobal()) { const warnings = [] - const shared = datastore.initStore(global, {}) + const baseNode = { warn: (m) => warnings.push(m) } + const shared = datastore.initStore(global, { node: baseNode }) const nodeFor = (id, type) => ({ id, type, context: () => ({ global }), warn: (m) => warnings.push(m) }) const legacy = createLegacy(RED) let comparedWithData = 0 for (const [i, step] of steps.entries()) { if (i === Math.floor(steps.length / 2)) { - datastore.initStore(global, {}) + datastore.initStore(global, { node: baseNode }) } const given = structuredClone(step) runStep(datastore, given, { base, nodeFor }) @@ -92,7 +93,7 @@ function compareAfterEachStep (steps, ids, global = fakeGlobal()) { function compareAtEnd (steps, ids) { const global = fakeGlobal() const warnings = [] - datastore.initStore(global, {}) + datastore.initStore(global, { node: { warn: (m) => warnings.push(m) } }) const nodeFor = (id, type) => ({ id, type, context: () => ({ global }), warn: (m) => warnings.push(m) }) const legacy = createLegacy(RED) for (const step of steps) { From 6d585e6dcb78b94b51a6ef3c54d1ce6c0445b3bc Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Mon, 19 Oct 2026 08:25:15 -0700 Subject: [PATCH 54/57] Show the trimmed series in the $ view and drop FLUSH_SERIES --- nodes/store/reactive.js | 24 ++++-------------------- test/store/data.spec.js | 5 +---- test/store/differential.spec.js | 2 -- test/store/reactive.spec.js | 30 +++++++++--------------------- 4 files changed, 14 insertions(+), 47 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index cf288159c..858a4fb74 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -3,7 +3,6 @@ const APPEND_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/appendEntry') const SET_SERIES = Symbol.for('@flowfuse/node-red-dashboard/setSeries') const MERGE_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/mergeEntry') const TRIM_SERIES = Symbol.for('@flowfuse/node-red-dashboard/trimSeries') -const FLUSH_SERIES = Symbol.for('@flowfuse/node-red-dashboard/flushSeries') const TRIM_BATCH = 60 const NAMESPACE = 'dashboardStore' @@ -112,7 +111,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D return rec } - const trimmedSeries = (rec) => rec.trims ? rec.trims.kept.concat(rec.msg.slice(rec.trims.seen)) : rec.msg + const trimmedSeries = (rec) => rec.trims ? rec.trims.kept.concat(rec.msg.slice(rec.trims.seen)) : rec.msg.slice() records[MERGE_ENTRY] = (prop, msg) => { const current = records[prop] @@ -179,14 +178,6 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (replaced) emit(prop, rec, prop) } - const rebuildSeries = (prop, rec) => { - const msgs = trimmedSeries(rec) - rec.trims = null - if (msgs.length !== rec.msg.length) { - records[SET_SERIES](prop, msgs) - } - } - records[TRIM_SERIES] = (prop, trim) => { const rec = records[prop] if (!rec || !Array.isArray(rec.msg)) { @@ -204,14 +195,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D } pending.calls++ if (pending.calls >= TRIM_BATCH) { - rebuildSeries(prop, rec) - } - } - - records[FLUSH_SERIES] = (prop) => { - const rec = records[prop] - if (rec?.trims) { - rebuildSeries(prop, rec) + records[SET_SERIES](prop, trimmedSeries(rec)) } } @@ -224,7 +208,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D if (prop.startsWith('$')) { const rec = target[prop.slice(1)] if (!rec) return undefined - const msg = Array.isArray(rec.msg) ? Object.freeze(rec.msg.slice()) : rec.msg + const msg = Array.isArray(rec.msg) ? Object.freeze(trimmedSeries(rec)) : rec.msg return Object.freeze({ value: rec.value, msg, quality: rec.quality, timestamp: rec.timestamp, history: Object.freeze(rec.history.slice()) }) } const rec = target[prop] @@ -287,4 +271,4 @@ function attachToContext (globalContext, opts = {}) { return store } -module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, FLUSH_SERIES } +module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES } diff --git a/test/store/data.spec.js b/test/store/data.spec.js index a257a0708..2bc0acbc6 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -2,7 +2,7 @@ const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars const datastore = require('../../nodes/store/data.js') -const { APPEND_ENTRY, FLUSH_SERIES } = require('../../nodes/store/reactive.js') +const { APPEND_ENTRY } = require('../../nodes/store/reactive.js') const warnings = [] const RED = { util, plugins: { getByType: () => [] } } @@ -866,7 +866,6 @@ describe('store: data.js trims the stored series itself', function () { global.get('dashboardStore')[APPEND_ENTRY]('own-trim', point('store-only')) datastore.keepLatestPerTopic(base, node, 2) - global.get('dashboardStore')[FLUSH_SERIES]('own-trim') global.get('dashboardStore')['$own-trim'].msg.map((m) => m.payload).should.eql([3, 4, 5, 'store-only']) datastore.get('own-trim').map((m) => m.payload).should.eql([2, 3, 4, 5]) @@ -883,8 +882,6 @@ describe('store: data.js trims the stored series itself', function () { datastore.filter(base, node, (m) => m._datapoint.x > i - 50) } - global.get('dashboardStore')[FLUSH_SERIES]('both-limits') - global.get('dashboardStore')['$both-limits'].msg.map((m) => m.payload).should.eql(datastore.get('both-limits').map((m) => m.payload)) }) }) diff --git a/test/store/differential.spec.js b/test/store/differential.spec.js index 6f26a0fb1..6ec849eab 100644 --- a/test/store/differential.spec.js +++ b/test/store/differential.spec.js @@ -2,7 +2,6 @@ const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars const datastore = require('../../nodes/store/data.js') -const { FLUSH_SERIES } = require('../../nodes/store/reactive.js') const createLegacy = require('./helpers/legacy-datastore.js') const { scenarios, generate, generateMessy, runStep } = require('./helpers/sequences.js') @@ -48,7 +47,6 @@ function scribble (value, seen = new Set()) { function storeHolds (global, id) { const store = global.get('dashboardStore') - store[FLUSH_SERIES](id) return store['$' + id]?.msg } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 2a63d972e..703c57ba5 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -11,7 +11,7 @@ const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars -const { createDataStore, attachToContext, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, FLUSH_SERIES } = require('../../nodes/store/reactive.js') +const { createDataStore, attachToContext, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES } = require('../../nodes/store/reactive.js') function makeStore (opts = {}) { const log = [] @@ -1366,10 +1366,11 @@ describe('store: TRIM_SERIES', function () { for (let i = 0; i < 59; i++) { store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > i)) } - store.$c.msg.should.have.length(100) + store.c.should.have.length(100) store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > 59)) + store.c.should.have.length(40) payloads(store).should.eql(Array.from({ length: 40 }, (_, i) => i + 60)) }) @@ -1380,15 +1381,14 @@ describe('store: TRIM_SERIES', function () { store[TRIM_SERIES]('c', (msgs) => msgs) } - store.$c.msg.should.have.length(100) + store.c.should.have.length(100) }) - it('applies pending trims on flush', function () { + it('shows pending trims in the $ view before the series is rebuilt', function () { const { store } = series(10) store[TRIM_SERIES]('c', keepLatest(3)) - store[FLUSH_SERIES]('c') - + store.c.should.have.length(10) payloads(store).should.eql([7, 8, 9]) }) @@ -1401,8 +1401,6 @@ describe('store: TRIM_SERIES', function () { store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m._datapoint.x > cutoff)) } - store[FLUSH_SERIES]('c') - payloads(store).should.eql([100, 50]) }) @@ -1411,8 +1409,6 @@ describe('store: TRIM_SERIES', function () { store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 1)) store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 3)) - store[FLUSH_SERIES]('c') - payloads(store).should.eql([0, 2, 4]) }) @@ -1422,18 +1418,14 @@ describe('store: TRIM_SERIES', function () { store[APPEND_ENTRY]('c', point(5)) store[TRIM_SERIES]('c', (msgs) => msgs.filter((m, i) => i > 0)) - store[FLUSH_SERIES]('c') - payloads(store).should.eql([2, 3, 4, 5]) }) - it('keeps points appended after the last trim on flush', function () { + it('shows points appended after the last trim', function () { const { store } = series(5) store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 0)) store[APPEND_ENTRY]('c', point(5)) - store[FLUSH_SERIES]('c') - payloads(store).should.eql([1, 2, 3, 4, 5]) }) @@ -1444,7 +1436,6 @@ describe('store: TRIM_SERIES', function () { for (let i = 0; i < 60; i++) { store[TRIM_SERIES]('c', (msgs) => msgs) } - store[FLUSH_SERIES]('c') log.should.eql([]) payloads(store).should.eql([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) @@ -1456,7 +1447,6 @@ describe('store: TRIM_SERIES', function () { store[SET_SERIES]('c', [point(1), point(2)]) store[TRIM_SERIES]('c', keepLatest(1)) - store[FLUSH_SERIES]('c') payloads(store).should.eql([2]) }) @@ -1468,18 +1458,16 @@ describe('store: TRIM_SERIES', function () { delete store.c store[APPEND_ENTRY]('c', point(1)) store[APPEND_ENTRY]('c', point(2)) - store[FLUSH_SERIES]('c') payloads(store).should.eql([1, 2]) }) - it('ignores trims and flushes on an entry that is not a series', function () { + it('ignores trims on an entry that is not a series', function () { const { store } = makeStore({ clone: util.cloneMessage }) store.k = 5 store[TRIM_SERIES]('k', () => []) - store[FLUSH_SERIES]('k') - store[FLUSH_SERIES]('missing') + store[TRIM_SERIES]('missing', () => []) store.k.should.equal(5) }) From 8e0fba94da59719102c7fa0742af0621bdb47be5 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Mon, 19 Oct 2026 08:57:52 -0700 Subject: [PATCH 55/57] Update to getStoreOrWarn and remove fn layer --- nodes/store/data.js | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/nodes/store/data.js b/nodes/store/data.js index 704474965..928103a11 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -27,13 +27,9 @@ function getOrCreateStore (globalContext) { return store } -function fallbackOrContextStore () { - return fallbackStore || getOrCreateStore(storeContext) -} - -function storeOrWarn (action, id) { +function getStoreOrWarn (action, id) { try { - return fallbackOrContextStore() + return fallbackStore || getOrCreateStore(storeContext) } catch (err) { warnOnce(`${action} ${id} failed (${err.message})`) } } @@ -108,10 +104,10 @@ const getters = { }, // given a widget id, return the latest msg received msg (id) { - return config.RED.util.cloneMessage(storeOrWarn('reading', id)?.[ENTRY_MSG](id)) + return config.RED.util.cloneMessage(getStoreOrWarn('reading', id)?.[ENTRY_MSG](id)) }, has (id) { - const store = storeOrWarn('reading', id) + const store = getStoreOrWarn('reading', id) return !!store && id in store } } @@ -123,7 +119,7 @@ const setters = { }, // remove data associated to a given widget clear (id) { - const store = storeOrWarn('clearing', id) + const store = getStoreOrWarn('clearing', id) if (store) { delete store[id] } @@ -138,18 +134,18 @@ const setters = { if (Array.isArray(msg)) { const msgs = msg.filter((m) => canSaveInStore(base, node, m)) if (msgs.every((m) => m && typeof m === 'object')) { - storeOrWarn('saving', node.id)?.[SET_SERIES](node.id, msgs) + getStoreOrWarn('saving', node.id)?.[SET_SERIES](node.id, msgs) } } else if (msg && typeof msg === 'object' && canSaveInStore(base, node, msg)) { const { ui_update: uiUpdate, ...rest } = msg - storeOrWarn('saving', node.id)?.[MERGE_ENTRY](node.id, rest) + getStoreOrWarn('saving', node.id)?.[MERGE_ENTRY](node.id, rest) } }, // given a widget id, and msg, store in an array of history of values // useful for charting widgets append (base, node, msg) { if (msg && typeof msg === 'object' && canSaveInStore(base, node, msg)) { - storeOrWarn('saving', node.id)?.[APPEND_ENTRY](node.id, msg) + getStoreOrWarn('saving', node.id)?.[APPEND_ENTRY](node.id, msg) } }, /** @@ -160,11 +156,11 @@ const setters = { */ filter (base, node, filterFunction) { if (filterFunction) { - storeOrWarn('trimming', node.id)?.[TRIM_SERIES](node.id, (msgs) => msgs.filter(filterFunction)) + getStoreOrWarn('trimming', node.id)?.[TRIM_SERIES](node.id, (msgs) => msgs.filter(filterFunction)) } }, keepLatestPerTopic (base, node, maxPoints) { - storeOrWarn('trimming', node.id)?.[TRIM_SERIES](node.id, (msgs) => latestPerTopic(msgs, maxPoints)) + getStoreOrWarn('trimming', node.id)?.[TRIM_SERIES](node.id, (msgs) => latestPerTopic(msgs, maxPoints)) } } From d3605d4ea2f0846e881a6b60de889c7a1c1ecfd7 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 20 Oct 2026 10:50:18 -0700 Subject: [PATCH 56/57] Have reload start fresh --- nodes/store/reactive.js | 4 ++-- test/store/data.spec.js | 15 +++++++-------- test/store/reactive.spec.js | 32 +++++++++++++++++++++----------- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js index 702848e18..a4ca049d6 100644 --- a/nodes/store/reactive.js +++ b/nodes/store/reactive.js @@ -249,7 +249,7 @@ function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = D const injected = new WeakMap() // Inject the store into global context once. On redeploy the existing store is reused; -// after a restart with a persistent context store, existing plain data is rehydrated. +// after a restart it starts fresh and plain data a flow put in its place is copied in. // Returns null when the context store can't hold a live proxy: a cache-off store serves // only async access so a sync get throws, and a serialising store hands back a copy. function attachToContext (globalContext, opts = {}) { @@ -269,7 +269,7 @@ function attachToContext (globalContext, opts = {}) { const previous = injected.get(globalContext) const store = createDataStore(opts) - if (isReactable(existing) && !Array.isArray(existing)) { + if (previous && isReactable(existing) && !Array.isArray(existing)) { for (const [k, v] of Object.entries(existing)) store[k] = v } globalContext.set(namespace, store) diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 021602b80..60bf94019 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -930,19 +930,18 @@ describe('store: data.js after a restart with a persistent context store', funct datastore.initStore(fakeGlobal(), {}) }) - it('reads restored values as messages holding only the payload', function () { - restored({ text: 'hello', rows: [{ id: 1 }] }) + it('starts empty, as main does', function () { + restored({ text: 'hello', rows: [{ id: 1 }], chart: [{ category: 'a', x: 1, y: 1 }] }) - datastore.has('text').should.equal(true) - datastore.get('text').should.eql({ payload: 'hello' }) - datastore.get('rows').should.eql({ payload: [{ id: 1 }] }) + datastore.has('text').should.equal(false) + should(datastore.get('text')).be.undefined() + datastore.has('rows').should.equal(false) + datastore.has('chart').should.equal(false) }) - it('reads a restored chart as its points until the next point replaces it with a series', function () { + it('stores a chart\'s first point after a restart as a fresh series', function () { restored({ chart: [{ category: 'a', x: 1, y: 1 }] }) - datastore.get('chart').should.eql({ payload: [{ category: 'a', x: 1, y: 1 }] }) - datastore.append(base, fakeNode('chart'), { payload: 2, _datapoint: { category: 'a', x: 2, y: 2 } }) datastore.get('chart').should.eql([{ payload: 2, _datapoint: { category: 'a', x: 2, y: 2 } }]) diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js index 6eeaba8fe..64c5d67b0 100644 --- a/test/store/reactive.spec.js +++ b/test/store/reactive.spec.js @@ -963,9 +963,11 @@ describe('store: reactive data store', function () { }) describe('attachToContext', function () { - it('does not rehydrate an array into numeric keys', function () { - const m = { dashboardStore: ['a', 'b'] } + it('does not copy an array a flow put in its place into numeric keys', function () { + const m = {} const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + attachToContext(g, {}) + m.dashboardStore = ['a', 'b'] const store = attachToContext(g, {}) @@ -973,18 +975,22 @@ describe('store: reactive data store', function () { JSON.stringify(store).should.equal('{}') }) - it('does not spread a class instance into keys', function () { - const m = { dashboardStore: new Date() } + it('does not spread a class instance a flow put in its place into keys', function () { + const m = {} const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + attachToContext(g, {}) + m.dashboardStore = new Date() const store = attachToContext(g, {}) Object.keys(store).should.eql([]) }) - it('rehydrates an object a function node left at the namespace', function () { - const m = { dashboardStore: fnNodeObject('({ robot: "PLAIN", count: 3 })') } + it('copies in an object a function node put in its place', function () { + const m = {} const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + attachToContext(g, {}) + m.dashboardStore = fnNodeObject('({ robot: "PLAIN", count: 3 })') const store = attachToContext(g, {}) @@ -992,9 +998,11 @@ describe('store: reactive data store', function () { store.count.should.equal(3) }) - it('still rehydrates a plain object', function () { - const m = { dashboardStore: { robot: { temp: 20 }, count: 3 } } + it('copies in a plain object a flow put in its place', function () { + const m = {} const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + attachToContext(g, {}) + m.dashboardStore = { robot: { temp: 20 }, count: 3 } const store = attachToContext(g, {}) @@ -1036,8 +1044,9 @@ describe('store: reactive data store', function () { should(second).equal(first) }) - it('rehydrates around existing plain data and rewires reactivity', function () { + it('rewires reactivity around plain data a flow put in its place', function () { const g = fakeGlobal() + attachToContext(g) g.set('dashboardStore', { a: 5, robot: { temp: 1 } }) const log = [] const store = attachToContext(g, { onChange: (k, e, p) => log.push(p) }) @@ -1086,13 +1095,14 @@ describe('store: reactive data store', function () { warned.should.equal(0) }) - it('does not warn when a restart rehydrates plain data it never injected', function () { + it('starts fresh after a restart, without warning, rather than restoring plain data it never injected', function () { const g = fakeGlobal() g.set('dashboardStore', { widget: 'from disk' }) // what a persistent context store hands back let warned = 0 const store = attachToContext(g, { onReplaced: () => { warned++ } }) warned.should.equal(0) - store.widget.should.equal('from disk') + should(store.widget).be.undefined() + should(g.get('dashboardStore')).equal(store) }) const replacements = [ From 9a0ce0fac529129cb4e74ecc67ea444653a17541 Mon Sep 17 00:00:00 2001 From: Noley Holland Date: Tue, 20 Oct 2026 11:07:18 -0700 Subject: [PATCH 57/57] Update to only warnedStoreUnavailable on start --- nodes/config/ui_base.js | 5 ++++- test/nodes/config-nodes/ui_base.spec.js | 11 +++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index 852a35e80..7fab5e2b0 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -34,6 +34,8 @@ module.exports = function (RED) { datastore.setConfig(RED) statestore.setConfig(RED) + let warnedStoreUnavailable = false + /** * @typedef {import('socket.io').Socket} Socket * @typedef {import('socket.io').Server} Server @@ -395,7 +397,8 @@ module.exports = function (RED) { onUnavailable: (err) => { unavailable = err }, onReplaced: () => node.warn('global.dashboardStore was replaced by a flow since the last deploy, discarding the stored value of every widget. The store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') }) - if (!store) { + if (!store && !warnedStoreUnavailable) { + warnedStoreUnavailable = true node.warn('Dashboard data store: flows can\'t use global.dashboardStore, because the default context store doesn\'t keep values in memory (e.g. localfilesystem with cache: false). Dashboards still work normally. To use global.dashboardStore from flows, set contextStorage.default to memory, or localfilesystem with cache: true.' + (unavailable ? ' (Reason: ' + unavailable.message + ')' : '')) } } catch (err) { diff --git a/test/nodes/config-nodes/ui_base.spec.js b/test/nodes/config-nodes/ui_base.spec.js index ff85897fb..f6ef5ce72 100644 --- a/test/nodes/config-nodes/ui_base.spec.js +++ b/test/nodes/config-nodes/ui_base.spec.js @@ -159,6 +159,17 @@ describe('ui-base config node: cache-off context store', function () { touched.set.should.equal(0) }) + it('warns once per start, not on every full deploy', async function () { + await helper.load(nodeImports, flow) + + await helper.setFlows(flow, 'full') + + const warned = helper.log().args.filter((args) => + args[0].level === helper.log().WARN && /doesn't keep values in memory/.test(args[0].msg) + ) + warned.should.have.length(1) + }) + it('stops the write path instead of throwing on every message', async function () { await helper.load(nodeImports, flow) const text = helper.getNode('node-ui-text')