diff --git a/nodes/config/ui_base.js b/nodes/config/ui_base.js index 8b6a3f2ad..7fab5e2b0 100644 --- a/nodes/config/ui_base.js +++ b/nodes/config/ui_base.js @@ -7,7 +7,7 @@ const v = require('../../package.json').version const { createClientStore } = require('../store/clients.js') const datastore = require('../store/data.js') const statestore = require('../store/state.js') -const { appendTopic, addConnectionCredentials, matchesClient, normalizeClientId, getThirdPartyWidgets } = require('../utils/index.js') +const { appendTopic, addConnectionCredentials, matchesClient, normalizeClientId, getThirdPartyWidgets, hasExtraProps } = require('../utils/index.js') // from: https://stackoverflow.com/a/28592528/3016654 function join (...paths) { @@ -27,19 +27,6 @@ function hasProperty (obj, prop) { return Object.prototype.hasOwnProperty.call(obj, prop) } -/** - * Test whether a message has any properties other than ui_update, class, visible, enabled and _msgid - * Properties with the value undefined are ignored - * @param {*} msg - * @returns true if other properties found - */ -function hasExtraProps (message) { - const allowed = ['_msgid', 'ui_update', 'class', 'visible', 'enabled'] - const keys = Object.keys(message).filter(key => message[key] !== undefined) - - return keys.length > 0 && keys.some(key => !allowed.includes(key)) -} - module.exports = function (RED) { const express = require('express') const { Server } = require('socket.io') @@ -47,6 +34,8 @@ module.exports = function (RED) { datastore.setConfig(RED) statestore.setConfig(RED) + let warnedStoreUnavailable = false + /** * @typedef {import('socket.io').Socket} Socket * @typedef {import('socket.io').Server} Server @@ -401,6 +390,21 @@ module.exports = function (RED) { node._created = Date.now() + try { + let unavailable = null + const store = datastore.initStore(node.context().global, { + node, + onUnavailable: (err) => { unavailable = err }, + onReplaced: () => node.warn('global.dashboardStore was replaced by a flow since the last deploy, discarding the stored value of every widget. The store has been re-created. To write your own data, set global.dashboardStore. rather than replacing global.dashboardStore itself.') + }) + if (!store && !warnedStoreUnavailable) { + warnedStoreUnavailable = true + node.warn('Dashboard data store: flows can\'t use global.dashboardStore, because the default context store doesn\'t keep values in memory (e.g. localfilesystem with cache: false). Dashboards still work normally. To use global.dashboardStore from flows, set contextStorage.default to memory, or localfilesystem with cache: true.' + (unavailable ? ' (Reason: ' + unavailable.message + ')' : '')) + } + } catch (err) { + node.warn('Dashboard data store: flows can\'t use global.dashboardStore, because saving it to global context failed (' + err.message + '). Dashboards still work normally.') + } + n.root = RED.settings.httpNodeRoot || '/' /** @type {Object.} */ diff --git a/nodes/store/data.js b/nodes/store/data.js index ffcae6632..928103a11 100644 --- a/nodes/store/data.js +++ b/nodes/store/data.js @@ -1,11 +1,69 @@ const { isClientScoped } = require('../utils/index.js') -const data = {} +const { createDataStore, attachToContext, MERGE_ENTRY, APPEND_ENTRY, SET_SERIES, TRIM_SERIES, ENTRY_MSG } = require('./reactive.js') const config = { RED: null } +let storeOptions = {} +let fallbackStore = null +let storeContext = null +let storeBase = null +let warned = false + +function warnOnce (problem) { + if (warned || !storeBase) return + warned = true + storeBase.warn(`Dashboard data store: ${problem}. Further store failures are not reported until the next full deploy or restart.`) +} + +const attachOptions = () => ({ clone: config.RED.util.cloneMessage, ...storeOptions }) + +function getOrCreateStore (globalContext) { + let unavailable = null + const store = attachToContext(globalContext, { ...attachOptions(), onUnavailable: (err) => { unavailable = err } }) + if (!store) throw unavailable || new Error('global context saved a copy of the store rather than the store itself, which happens with context stores that serialise values') + return store +} + +function getStoreOrWarn (action, id) { + try { + return fallbackStore || getOrCreateStore(storeContext) + } catch (err) { warnOnce(`${action} ${id} failed (${err.message})`) } +} + +function initStore (globalContext, { node, ...opts } = {}) { + storeOptions = opts + storeContext = globalContext + storeBase = node + warned = false + let store = null + try { + store = attachToContext(globalContext, attachOptions()) + } finally { + if (store) { + fallbackStore = null + } else if (!fallbackStore) { + fallbackStore = createDataStore(attachOptions()) + } + } + return store +} + +function latestPerTopic (msgs, maxPoints) { + const counts = {} + const kept = [] + for (let i = msgs.length - 1; i >= 0; i--) { + const topic = msgs[i].topic + counts[topic] = (counts[topic] || 0) + 1 + if (counts[topic] <= maxPoints) { + kept.push(msgs[i]) + } + } + return kept.reverse() +} + /** * Checks if a Client/Socket ID has been assigned to this message, * and whether the node type is being scoped to a specific client. @@ -40,26 +98,17 @@ function canSaveInStore (base, node, msg) { return checks.length === 0 || !checks.includes(false) } -// Strip msg of properties that are not needed for storage -function stripMsg (msg) { - const newMsg = config.RED.util.cloneMessage(msg) - - // don't need to store ui_updates in the datastore, as this is handled in statestore - delete newMsg.ui_update - - return newMsg -} - const getters = { RED () { return config.RED }, // given a widget id, return the latest msg received msg (id) { - return config.RED.util.cloneMessage(data[id]) + return config.RED.util.cloneMessage(getStoreOrWarn('reading', id)?.[ENTRY_MSG](id)) }, has (id) { - return !!data[id] + const store = getStoreOrWarn('reading', id) + return !!store && id in store } } @@ -70,7 +119,10 @@ const setters = { }, // remove data associated to a given widget clear (id) { - delete data[id] + const store = getStoreOrWarn('clearing', id) + if (store) { + delete store[id] + } }, /** * @@ -80,32 +132,20 @@ const setters = { */ save (base, node, msg) { if (Array.isArray(msg)) { - /// need to check msg by msg - const filtered = [] - for (const m of msg) { - if (canSaveInStore(base, node, m)) { - filtered.push(config.RED.util.cloneMessage(m)) - } - } - data[node.id] = filtered - } else { - if (canSaveInStore(base, node, msg)) { - const newMsg = stripMsg(msg) - data[node.id] = { - ...data[node.id], - ...newMsg - } + const msgs = msg.filter((m) => canSaveInStore(base, node, m)) + if (msgs.every((m) => m && typeof m === 'object')) { + getStoreOrWarn('saving', node.id)?.[SET_SERIES](node.id, msgs) } + } else if (msg && typeof msg === 'object' && canSaveInStore(base, node, msg)) { + const { ui_update: uiUpdate, ...rest } = msg + getStoreOrWarn('saving', node.id)?.[MERGE_ENTRY](node.id, rest) } }, // given a widget id, and msg, store in an array of history of values // useful for charting widgets append (base, node, msg) { - if (canSaveInStore(base, node, msg)) { - if (!data[node.id]) { - data[node.id] = [] - } - data[node.id].push(config.RED.util.cloneMessage(msg)) + if (msg && typeof msg === 'object' && canSaveInStore(base, node, msg)) { + getStoreOrWarn('saving', node.id)?.[APPEND_ENTRY](node.id, msg) } }, /** @@ -115,35 +155,12 @@ const setters = { * @param {(msg) => Boolean} filterFunction */ filter (base, node, filterFunction) { - const currentData = data[node.id] - if (filterFunction && Array.isArray(currentData) && currentData.length) { - const filteredMessages = currentData.filter(filterFunction) - if (filteredMessages.length !== currentData.length) { - // no need for save operation to process messages - just apply them - data[node.id] = filteredMessages - } + if (filterFunction) { + getStoreOrWarn('trimming', node.id)?.[TRIM_SERIES](node.id, (msgs) => msgs.filter(filterFunction)) } }, keepLatestPerTopic (base, node, maxPoints) { - const currentData = data[node.id] - if (!Array.isArray(currentData)) { - return - } - const counts = {} - const keep = [] - let trimmed = false - for (let i = currentData.length - 1; i >= 0; i--) { - const topic = currentData[i].topic - counts[topic] = (counts[topic] || 0) + 1 - if (counts[topic] <= maxPoints) { - keep[i] = true - } else { - trimmed = true - } - } - if (trimmed) { - setters.filter(base, node, (m, i) => keep[i]) - } + getStoreOrWarn('trimming', node.id)?.[TRIM_SERIES](node.id, (msgs) => latestPerTopic(msgs, maxPoints)) } } @@ -156,5 +173,6 @@ module.exports = { append: setters.append, filter: setters.filter, keepLatestPerTopic: setters.keepLatestPerTopic, - clear: setters.clear + clear: setters.clear, + initStore } diff --git a/nodes/store/reactive.js b/nodes/store/reactive.js new file mode 100644 index 000000000..a4ca049d6 --- /dev/null +++ b/nodes/store/reactive.js @@ -0,0 +1,282 @@ +const STORE = Symbol.for('@flowfuse/node-red-dashboard/store') +const APPEND_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/appendEntry') +const SET_SERIES = Symbol.for('@flowfuse/node-red-dashboard/setSeries') +const MERGE_ENTRY = Symbol.for('@flowfuse/node-red-dashboard/mergeEntry') +const TRIM_SERIES = Symbol.for('@flowfuse/node-red-dashboard/trimSeries') +const ENTRY_MSG = Symbol.for('@flowfuse/node-red-dashboard/entryMsg') +const TRIM_BATCH = 60 +const NAMESPACE = 'dashboardStore' + +class Entry { + constructor () { + this.value = undefined + this.msg = undefined + this.quality = 'GOOD' + this.timestamp = 0 + this.history = [] + this.trims = null + } +} + +// only plain objects and arrays are deep-watched; Buffers, Dates, class instances etc. are opaque leaves +function isReactable (v) { + if (Array.isArray(v)) return true + if (v === null || typeof v !== 'object') return false + const proto = Object.getPrototypeOf(v) + return proto === null || Object.getPrototypeOf(proto) === null +} + +// reads through proxies (structuredClone rejects them); production injects RED.util.cloneMessage +function deepClone (v, seen = new WeakMap()) { + if (v === null || typeof v !== 'object') return v + if (Buffer.isBuffer(v)) return Buffer.from(v) + if (v instanceof Date) return new Date(v) + if (seen.has(v)) return seen.get(v) + const out = Array.isArray(v) ? [] : {} + seen.set(v, out) + for (const k of Object.keys(v)) out[k] = deepClone(v[k], seen) + return out +} + +function deepReactive (value, notify, path, clone, seen = new WeakMap()) { + if (!isReactable(value)) return value + if (seen.has(value)) return seen.get(value) + const proxy = new Proxy(value, { + set (t, p, v) { + if (typeof p === 'symbol') return Reflect.set(t, p, v) + // a push writes the new index then grows length; only a shrink is a change worth reporting + const shrank = p === 'length' && Array.isArray(t) && v < t.length + t[p] = deepReactive(clone(v), notify, `${path}.${String(p)}`, clone) + if (p !== 'length') notify(`${path}.${String(p)}`) + else if (shrank) notify(path) + return true + }, + deleteProperty (t, p) { + const existed = p in t + const ok = delete t[p] + if (existed) notify(`${path}.${String(p)}`) + return ok + } + }) + // registered before walking children so a cycle resolves to this proxy instead of recursing + seen.set(value, proxy) + for (const k of Object.keys(value)) value[k] = deepReactive(value[k], notify, `${path}.${k}`, clone, seen) + return proxy +} + +function deepFreeze (v) { + if (!isReactable(v) || Object.isFrozen(v)) return v + Object.freeze(v) + for (const k of Object.keys(v)) deepFreeze(v[k]) + return v +} + +function freezeOwned (owned, except) { + for (const k of Object.keys(owned)) { + if (k === except || k === 'req' || k === 'res') continue + deepFreeze(owned[k]) + } + return owned +} + +function createDataStore ({ maxHistory = 5, onChange, clone = deepClone, now = Date.now } = {}) { + const cloneValue = (v) => (v && typeof v === 'object') ? clone(v) : v + const emit = (prop, rec, path) => { try { onChange?.(prop, rec, path) } catch (err) {} } + const records = Object.create(null) + + records[STORE] = true + + // returns the record without notifying, so a caller can finish writing it before onChange fires + const writeValue = (prop, value) => { + let rec = records[prop] + if (!rec) { + rec = new Entry() + records[prop] = rec + } else if (!Array.isArray(rec.value) && 'payload' in rec.msg) { + // snapshotting a whole array on every write (e.g. a table's rows) would blow up memory + rec.history.push(Object.freeze({ value: deepFreeze(cloneValue(rec.value)), timestamp: rec.timestamp })) + if (rec.history.length > maxHistory) rec.history.shift() + } + const notify = (path) => { + // a replaced value keeps its proxies alive; they must not report into the record any more + if (rec.value !== wrapped) return + rec.timestamp = now() + emit(prop, rec, path) + } + const wrapped = deepReactive(value, notify, prop, cloneValue) + rec.value = wrapped + rec.trims = null + rec.msg = Object.freeze({ payload: wrapped }) + rec.timestamp = now() + return rec + } + + const trimmedSeries = (rec) => rec.trims ? rec.trims.kept.concat(rec.msg.slice(rec.trims.seen)) : rec.msg.slice() + const msgOf = (rec) => Array.isArray(rec.msg) ? Object.freeze(trimmedSeries(rec)) : rec.msg + + records[ENTRY_MSG] = (prop) => { + const rec = records[prop] + return rec ? msgOf(rec) : undefined + } + + records[MERGE_ENTRY] = (prop, msg) => { + const current = records[prop] + const { payload, ...incoming } = msg + const previous = Array.isArray(current?.msg) ? freezeOwned(cloneValue({ ...trimmedSeries(current) })) : current?.msg || {} + const { payload: storedPayload, ...stored } = previous + const keepValue = current && !Array.isArray(current.msg) && !('payload' in msg) + const owned = freezeOwned(cloneValue(incoming)) + const value = keepValue ? undefined : cloneValue(payload) + const rec = keepValue ? current : writeValue(prop, value) + const fields = { ...stored, ...owned } + const hasPayload = 'payload' in msg || (keepValue && 'payload' in current.msg) + rec.msg = Object.freeze(hasPayload ? { ...fields, payload: rec.value } : fields) + rec.timestamp = now() + emit(prop, rec, prop) + } + + const pointsOf = (d) => (d === undefined || d === null) ? [] : (Array.isArray(d) ? d : [d]) + + const toStoredDatapoint = (series, datapoint, start, end) => { + if (end === start) return undefined + return Array.isArray(datapoint) ? Object.freeze(series.slice(start, end)) : series[start] + } + + const toStoredMessage = (owned, datapoint) => { + freezeOwned(owned, '_datapoint') + if (datapoint !== undefined) owned._datapoint = datapoint + else deepFreeze(owned._datapoint) + return Object.freeze(owned) + } + + records[SET_SERIES] = (prop, msgs) => { + const points = [] + const parts = msgs.map((msg) => { + const { _datapoint, ...rest } = msg + const owned = cloneValue(rest) + const datapoint = cloneValue(_datapoint) + const start = points.length + for (const p of pointsOf(datapoint)) points.push(p) + return { msg, owned, datapoint, start, end: points.length } + }) + const rec = writeValue(prop, points) + rec.msg = parts.map(({ msg, owned, datapoint, start, end }) => { + const stored = toStoredDatapoint(rec.value, datapoint, start, end) + if (stored === undefined && '_datapoint' in msg) owned._datapoint = datapoint + return toStoredMessage(owned, stored) + }) + emit(prop, rec, prop) + } + + records[APPEND_ENTRY] = (prop, msg) => { + let rec = records[prop] + const replaced = !!rec && !(Array.isArray(rec.value) && Array.isArray(rec.msg)) + // the proxy clones on push, so points must not be cloned again here + const { _datapoint, ...rest } = msg + const owned = cloneValue(rest) + if (!rec || replaced) { + rec = writeValue(prop, []) + rec.msg = [] + } + const start = rec.value.length + for (const p of pointsOf(_datapoint)) rec.value.push(p) + const stored = toStoredDatapoint(rec.value, _datapoint, start, rec.value.length) + if (stored === undefined && '_datapoint' in msg) owned._datapoint = cloneValue(_datapoint) + rec.msg.push(toStoredMessage(owned, stored)) + if (replaced) emit(prop, rec, prop) + } + + records[TRIM_SERIES] = (prop, trim) => { + const rec = records[prop] + if (!rec || !Array.isArray(rec.msg)) { + return + } + const pending = rec.trims || (rec.trims = { calls: 0, kept: [], seen: 0 }) + for (let i = pending.seen; i < rec.msg.length; i++) { + pending.kept.push(rec.msg[i]) + } + pending.seen = rec.msg.length + const before = pending.kept.length + pending.kept = trim(pending.kept) + if (pending.kept.length === before) { + return + } + pending.calls++ + if (pending.calls >= TRIM_BATCH) { + records[SET_SERIES](prop, trimmedSeries(rec)) + } + } + + const handler = { + get (target, prop, receiver) { + if (typeof prop === 'symbol') return Reflect.get(target, prop, receiver) + if (prop === 'toJSON') { + return () => Object.fromEntries(Object.entries(target).map(([k, r]) => [k, r.value])) + } + if (prop.startsWith('$')) { + const rec = target[prop.slice(1)] + if (!rec) return undefined + const msg = msgOf(rec) + return Object.freeze({ value: rec.value, msg, quality: rec.quality, timestamp: rec.timestamp, history: Object.freeze(rec.history.slice()) }) + } + const rec = target[prop] + return rec ? rec.value : undefined + }, + set (target, prop, value) { + if (typeof prop === 'symbol') return Reflect.set(target, prop, value) + // returning true rather than false so a strict caller picking a reserved name doesn't throw + if (prop === '__proto__' || prop === 'constructor' || prop === 'toJSON') return true + if (prop.startsWith('$')) return true // '$' is reserved for the read-only meta view + + const rec = writeValue(prop, cloneValue(value)) + emit(prop, rec, prop) + return true + }, + ownKeys (target) { return Reflect.ownKeys(target) }, + deleteProperty (target, prop) { + if (typeof prop === 'symbol') return Reflect.deleteProperty(target, prop) + const rec = target[prop] + const existed = prop in target + const ok = delete target[prop] + if (rec) rec.value = undefined + if (existed) emit(prop, undefined, prop) + return ok + } + } + return new Proxy(records, handler) +} + +const injected = new WeakMap() + +// Inject the store into global context once. On redeploy the existing store is reused; +// after a restart it starts fresh and plain data a flow put in its place is copied in. +// Returns null when the context store can't hold a live proxy: a cache-off store serves +// only async access so a sync get throws, and a serialising store hands back a copy. +function attachToContext (globalContext, opts = {}) { + const namespace = opts.namespace || NAMESPACE + let existing + try { + existing = globalContext.get(namespace) + } catch (err) { + opts.onUnavailable?.(err) + return null + } + if (existing && existing[STORE]) { + injected.set(globalContext, existing) + return existing + } + + const previous = injected.get(globalContext) + + const store = createDataStore(opts) + if (previous && isReactable(existing) && !Array.isArray(existing)) { + for (const [k, v] of Object.entries(existing)) store[k] = v + } + globalContext.set(namespace, store) + if (globalContext.get(namespace) !== store) return null + if (previous && existing !== previous) opts.onReplaced?.() + injected.set(globalContext, store) + return store +} + +module.exports = { createDataStore, attachToContext, NAMESPACE, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, ENTRY_MSG } diff --git a/nodes/utils/index.js b/nodes/utils/index.js index d6a6dd7c6..2a53989d4 100644 --- a/nodes/utils/index.js +++ b/nodes/utils/index.js @@ -60,6 +60,13 @@ function isClientScoped (msg) { return clientTargets(msg).length > 0 } +function hasExtraProps (message) { + const allowed = ['_msgid', 'ui_update', 'class', 'visible', 'enabled'] + const keys = Object.keys(message).filter(key => message[key] !== undefined) + + return keys.length > 0 && keys.some(key => !allowed.includes(key)) +} + function matchesClient (conn, msg) { return clientTargets(msg).every((target) => target.value === target.connValue(conn)) } @@ -155,5 +162,6 @@ module.exports = { normalizeClientId, isClientScoped, matchesClient, - getThirdPartyWidgets + getThirdPartyWidgets, + hasExtraProps } diff --git a/test/nodes/config-nodes/ui_base.spec.js b/test/nodes/config-nodes/ui_base.spec.js new file mode 100644 index 000000000..f6ef5ce72 --- /dev/null +++ b/test/nodes/config-nodes/ui_base.spec.js @@ -0,0 +1,186 @@ +const context = require('@node-red/runtime/lib/nodes/context') +const helper = require('node-red-node-test-helper') +const should = require('should') // eslint-disable-line no-unused-vars + +const datastore = require('../../../nodes/store/data.js') +const { STORE, NAMESPACE } = require('../../../nodes/store/reactive.js') +const { testData1 } = require('../fixtures/index.js') +const { verifyFlowLoaded } = require('../utils.js') + +const testFlow1 = testData1.flows +const nodeImports = testData1.getImports(null, ['ui_button', 'ui_text']) + +function waitFor (fn, timeout = 1000) { + return new Promise((resolve, reject) => { + const start = Date.now() + const tick = () => { + let v + try { v = fn() } catch (e) {} + if (v !== undefined) { return resolve(v) } + if (Date.now() - start > timeout) { return reject(new Error('timeout waiting for store value')) } + setTimeout(tick, 5) + } + tick() + }) +} + +helper.init(require.resolve('node-red')) + +describe('ui-base config node: data store injection', function () { + beforeEach(function (done) { + helper.startServer(done) + }) + + afterEach(function (done) { + helper.unload() + helper.stopServer(done) + }) + + const secondBase = { + id: 'config-ui-base-2', + type: 'ui-base', + name: 'UI Name 2', + path: '/ui2', + port: '', + includeClientData: true, + acceptsClientConfig: ['ui-notification', 'ui-control'] + } + + const flow = [ + { id: 'node-ui-button', type: 'ui-button', z: 'tab-id', group: 'config-ui-group' }, + { id: 'node-ui-text', type: 'ui-text', z: 'tab-id', group: 'config-ui-group', label: 'txt' }, + ...testFlow1 + ] + + it('injects the reactive store into global context under the dashboardStore namespace', async function () { + await helper.load(nodeImports, flow) + verifyFlowLoaded(helper, flow) + const base = helper.getNode('config-ui-base') + const store = base.context().global.get(NAMESPACE) + should(store).be.an.Object() + store[STORE].should.equal(true) + }) + + it('keeps the store reactive through the real context manager, not just the raw memory store', async function () { + await helper.load(nodeImports, flow) + const global = helper.getNode('config-ui-base').context().global + + global.set(`${NAMESPACE}.tag1`, 123) + global.get(`${NAMESPACE}.tag1`).should.equal(123) + global.get(`${NAMESPACE}.$tag1`).value.should.equal(123) + + const store = global.get(NAMESPACE) + store.robot = { temp: 20 } + global.get(`${NAMESPACE}.robot`).temp.should.equal(20) + global.get(`${NAMESPACE}.$robot`).value.temp.should.equal(20) + }) + + it('gives multiple base nodes one shared store, not one each', async function () { + await helper.load(nodeImports, [...flow, secondBase]) + const first = helper.getNode('config-ui-base').context().global.get(NAMESPACE) + const second = helper.getNode('config-ui-base-2').context().global.get(NAMESPACE) + should(first).equal(second) + }) + + it('re-injection reuses the existing store and preserves its values', async function () { + await helper.load(nodeImports, flow) + const global = helper.getNode('config-ui-base').context().global + + const original = global.get(NAMESPACE) + original.reused = 42 + + // the path ui-base actually takes on a redeploy + const again = datastore.initStore(global, {}) + + should(again).equal(original) + global.get(`${NAMESPACE}.reused`).should.equal(42) + }) + + it('populates the store with a widget\'s clean payload on input', async function () { + await helper.load(nodeImports, flow) + const text = helper.getNode('node-ui-text') + const global = helper.getNode('config-ui-base').context().global + + text.receive({ payload: 'hello', topic: 't', _msgid: 'm' }) + + const value = await waitFor(() => global.get(NAMESPACE)['node-ui-text']) + value.should.equal('hello') // the clean payload, not the whole message + global.get(`${NAMESPACE}.$node-ui-text`).value.should.equal('hello') + }) +}) + +describe('ui-base config node: cache-off context store', function () { + const CACHE_OFF = 'File Store cache disabled - only asynchronous access supported' + let realGet + let touched + + beforeEach(function (done) { + touched = { get: 0, set: 0 } + realGet = context.get + // a persistent context store with cache:false serves only async access, so a sync get throws + context.get = function (id, z) { + const real = realGet.call(context, id, z) + return Object.create(real, { + global: { + value: { + get () { touched.get++; throw new Error(CACHE_OFF) }, + set (...args) { touched.set++; return real.global.set(...args) } + } + } + }) + } + helper.startServer(done) + }) + + afterEach(function (done) { + context.get = realGet + datastore.initStore(fakeGlobal(), {}) + helper.unload() + helper.stopServer(done) + }) + + function fakeGlobal () { + const m = {} + return { get: (k) => m[k], set: (k, v) => { m[k] = v } } + } + + const flow = [ + { id: 'node-ui-text', type: 'ui-text', z: 'tab-id', group: 'config-ui-group', label: 'txt' }, + ...testFlow1 + ] + + it('warns and does not inject the store', async function () { + await helper.load(nodeImports, flow) + + const warned = helper.log().args.filter((args) => + args[0].level === helper.log().WARN && /doesn't keep values in memory/.test(args[0].msg) + ) + warned.should.have.length(1) + touched.set.should.equal(0) + }) + + it('warns once per start, not on every full deploy', async function () { + await helper.load(nodeImports, flow) + + await helper.setFlows(flow, 'full') + + const warned = helper.log().args.filter((args) => + args[0].level === helper.log().WARN && /doesn't keep values in memory/.test(args[0].msg) + ) + warned.should.have.length(1) + }) + + it('stops the write path instead of throwing on every message', async function () { + await helper.load(nodeImports, flow) + const text = helper.getNode('node-ui-text') + + touched.get = 0 + touched.set = 0 + should(() => text.receive({ payload: 'hello' })).not.throw() + await new Promise((resolve) => setTimeout(resolve, 50)) + + touched.get.should.equal(0) + touched.set.should.equal(0) + datastore.get('node-ui-text').payload.should.equal('hello') + }) +}) diff --git a/test/store/data.spec.js b/test/store/data.spec.js index 161eac34e..60bf94019 100644 --- a/test/store/data.spec.js +++ b/test/store/data.spec.js @@ -1,24 +1,555 @@ +const { util } = require('@node-red/util') const should = require('should') // eslint-disable-line no-unused-vars const datastore = require('../../nodes/store/data.js') +const { APPEND_ENTRY } = require('../../nodes/store/reactive.js') -const RED = { - util: { - cloneMessage: (msg) => (msg === undefined ? undefined : JSON.parse(JSON.stringify(msg))) - }, - plugins: { - getByType: () => [] - } +const warnings = [] +const RED = { util, plugins: { getByType: () => [] } } +datastore.setConfig(RED) + +function fakeGlobal () { + const m = {} + return { get: (k) => m[k], set: (k, v) => { m[k] = v } } } -describe('store: data', function () { +function storeGlobal () { + const global = fakeGlobal() + datastore.initStore(global, {}) + return global +} + +function fakeNode (id) { + return { id, type: 'ui-text', warn: (m) => warnings.push(`${id}: ${m}`) } +} + +const base = { acceptsClientConfig: [] } + +describe('store: data.js reactive-store mirror', function () { before(function () { - datastore.setConfig(RED) + datastore.initStore(fakeGlobal(), {}) + }) + + it('mirrors the clean payload into global.dashboardStore[id], not the whole message', function () { + const global = storeGlobal() + const node = fakeNode('w1') + datastore.save(base, node, { payload: 42, topic: 't', _msgid: 'm', extra: 'x' }) + + const store = global.get('dashboardStore') + store.w1.should.equal(42) + datastore.get('w1').payload.should.equal(42) + datastore.get('w1').topic.should.equal('t') + }) + + it('keeps the mirrored value in sync on the latest write', function () { + const global = storeGlobal() + const node = fakeNode('w2') + datastore.save(base, node, { payload: 1 }) + datastore.save(base, node, { payload: 2 }) + global.get('dashboardStore').w2.should.equal(2) + }) + + it('stores the whole message alongside the value', function () { + const global = storeGlobal() + const node = fakeNode('m1') + datastore.save(base, node, { payload: 42, topic: 't', extra: 'x' }) + + const store = global.get('dashboardStore') + store.m1.should.equal(42) + store.$m1.msg.payload.should.equal(42) + store.$m1.msg.topic.should.equal('t') + store.$m1.msg.extra.should.equal('x') + }) + + it('merges a later message onto the stored one', function () { + storeGlobal() + const node = fakeNode('m3') + datastore.save(base, node, { payload: 1, topic: 'a', unit: 'bar' }) + datastore.save(base, node, { payload: 2 }) + + datastore.get('m3').should.eql({ payload: 2, topic: 'a', unit: 'bar' }) + }) + + it('does not alias the flow\'s message into the store', function () { + const global = storeGlobal() + const node = fakeNode('m2') + const sent = { payload: 1, nested: { a: 1 } } + datastore.save(base, node, sent) + + sent.nested.a = 999 + + global.get('dashboardStore').$m2.msg.nested.a.should.equal(1) + }) + + it('does not mirror a client-scoped message (not stored centrally)', function () { + const global = storeGlobal() + const node = fakeNode('w3') + const scopedBase = { acceptsClientConfig: ['ui-text'] } + datastore.save(scopedBase, node, { payload: 9, _client: { socketId: 's1' } }) + should(global.get('dashboardStore').w3).be.undefined() + datastore.has('w3').should.equal(false) + }) + + it('deep-copies a table rows array, so a flow reusing it cannot mutate stored state', function () { + const global = storeGlobal() + const node = fakeNode('tbl-1') + const rows = [{ id: 1, name: 'a', meta: { site: { rack: 'r1' } } }] + + datastore.save(base, node, { payload: rows }) + rows[0].name = 'mutated' + rows[0].meta.site.rack = 'mutated-deep' + + global.get('dashboardStore')['tbl-1'].should.eql([ + { id: 1, name: 'a', meta: { site: { rack: 'r1' } } } + ]) + }) + + it('reads a value only a flow wrote', function () { + const global = storeGlobal() + + global.get('dashboardStore')['flow-only'] = 5 + + datastore.has('flow-only').should.equal(true) + datastore.get('flow-only').should.eql({ payload: 5 }) + }) + + it('reads what a flow wrote into the stored rows, as a copy the caller owns', function () { + const global = storeGlobal() + const node = fakeNode('tbl-2') + + datastore.save(base, node, { payload: [{ id: 1, name: 'original' }] }) + global.get('dashboardStore')['tbl-2'][0].name = 'changed via store' + datastore.get('tbl-2').payload[0].name = 'changed by the caller' + + datastore.get('tbl-2').payload.should.eql([{ id: 1, name: 'changed via store' }]) + }) + + it('clear removes the widget key so a removed widget leaves no orphan', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('w6') + datastore.save(base, node, { payload: 5 }) + global.get('dashboardStore').w6.should.equal(5) + + datastore.clear(node.id) + should(global.get('dashboardStore').w6).be.undefined() + }) + + it('keeps the mirrored value when a later message carries no payload', function () { + const global = storeGlobal() + const node = fakeNode('w8') + datastore.save(base, node, { payload: 'keep me' }) + datastore.save(base, node, { topic: 'no payload' }) + + global.get('dashboardStore').w8.should.equal('keep me') + datastore.get('w8').payload.should.equal('keep me') + global.get('dashboardStore').$w8.history.should.eql([]) + }) + + it('blanks the mirrored value when a message explicitly carries payload: undefined', function () { + const global = storeGlobal() + const node = fakeNode('w11') + datastore.save(base, node, { payload: 'keep me' }) + datastore.save(base, node, { payload: undefined, topic: 'explicit' }) + + should(global.get('dashboardStore').w11).be.undefined() + should(datastore.get('w11').payload).be.undefined() + }) + + it('does not throw when a msg is not an object (unchanged from main)', function () { + const global = storeGlobal() + for (const msg of ['hello', 42, true]) { + const node = fakeNode('prim-' + String(msg), global) + should(() => datastore.save(base, node, msg)).not.throw() + } + }) + + it('clear does not throw when the store is unavailable', function () { + const throwingGlobal = { + get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, + set: () => {} + } + const node = fakeNode('w7') + datastore.initStore(throwingGlobal, { node: { warn: () => {} } }) + + should(() => datastore.clear(node.id)).not.throw() + }) +}) + +describe('store: data.js store options', function () { + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('re-injects with the registered options when a flow replaces the namespace', function () { + const global = fakeGlobal() + const node = fakeNode('w9') + let replaced = 0 + datastore.initStore(global, { onReplaced: () => { replaced++ } }) + + datastore.save(base, node, { payload: 1 }) + global.set('dashboardStore', { rogue: true }) + datastore.save(base, node, { payload: 2 }) + + replaced.should.equal(1) + global.get('dashboardStore').w9.should.equal(2) + }) + + it('keeps onChange wired through a widget-triggered re-injection', function () { + const global = fakeGlobal() + const node = fakeNode('w10') + const changes = [] + datastore.initStore(global, { onChange: (key) => changes.push(key) }) + + global.set('dashboardStore', { rogue: true }) + datastore.save(base, node, { payload: 'after' }) + + changes.should.containEql('w10') + }) +}) + +describe('store: data.js when global context cannot hold the store', function () { + const cacheOffGlobal = () => ({ + gets: 0, + get () { this.gets++; throw new Error('File Store cache disabled - only asynchronous access supported') }, + set () {} + }) + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('keeps writes off global context and reports no failures', function () { + const global = cacheOffGlobal() + const scalar = fakeNode('w12') + const chart = fakeNode('w12-chart') + should(datastore.initStore(global, { node: { warn: (m) => warnings.push(m) } })).be.null() + warnings.length = 0 + global.gets = 0 + + datastore.save(base, scalar, { payload: 1 }) + datastore.append(base, chart, { _datapoint: { category: 'a', x: 1, y: 1 } }) + datastore.filter(base, chart, () => false) + datastore.keepLatestPerTopic(base, chart, 1) + datastore.clear(scalar.id) + + global.gets.should.equal(0) + warnings.should.eql([]) + }) + + it('falls back when writing to global context throws', function () { + const global = { get: () => undefined, set () { throw new Error('read-only context') } } + const node = fakeNode('w13') + should(() => datastore.initStore(global, { node: { warn: (m) => warnings.push(m) } })).throw('read-only context') + warnings.length = 0 + + datastore.save(base, node, { payload: 'still works' }) + + warnings.should.eql([]) + datastore.get('w13').payload.should.equal('still works') + }) + + it('goes back to global context on a later initStore', function () { + datastore.initStore(cacheOffGlobal(), {}) + + const global = fakeGlobal() + const node = fakeNode('w14') + datastore.initStore(global, {}) + datastore.save(base, node, { payload: 'back on' }) + + global.get('dashboardStore').w14.should.equal('back on') + }) +}) + +describe('store: data.js chart writes', function () { + beforeEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + const pt = (x, y, category = 'a') => ({ category, x, y }) + + // an absent chart key only proves anything if the store is actually working + function liveStore (global, id) { + datastore.save(base, fakeNode(id), { payload: 'live' }) + return global.get('dashboardStore')?.[id] + } + + it('mirrors a chart append into the store', function () { + const global = storeGlobal() + const node = fakeNode('chart-1') + + datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) + datastore.append(base, node, { payload: 2, _datapoint: pt(2, 2) }) + + liveStore(global, 'canary-1').should.equal('live') + const store = global.get('dashboardStore') + store['chart-1'].should.eql([pt(1, 1), pt(2, 2)]) + store['$chart-1'].msg.should.have.length(2) + store['$chart-1'].msg[1].payload.should.equal(2) + }) + + it('flattens a multi-series append into several points', function () { + const global = storeGlobal() + const node = fakeNode('chart-multi') + + datastore.append(base, node, { payload: [1, 5], _datapoint: [pt(1, 1, 'a'), pt(1, 5, 'b')] }) + + liveStore(global, 'canary-multi').should.equal('live') + const store = global.get('dashboardStore') + store['chart-multi'].should.eql([pt(1, 1, 'a'), pt(1, 5, 'b')]) + store['$chart-multi'].msg.should.have.length(1) + }) + + it('stores a message with no datapoint but adds no point', function () { + const global = storeGlobal() + const node = fakeNode('chart-nodp') + + datastore.append(base, node, { payload: 'x' }) + + const store = global.get('dashboardStore') + store['chart-nodp'].should.eql([]) + store['$chart-nodp'].msg.should.have.length(1) + }) + + it('does not alias the flow\'s message into an appended series', function () { + const global = storeGlobal() + const node = fakeNode('chart-alias') + const sent = { payload: 1, _datapoint: pt(1, 1), nested: { a: 1 } } + + datastore.append(base, node, sent) + sent.nested.a = 999 + + global.get('dashboardStore')['$chart-alias'].msg[0].nested.a.should.equal(1) + }) + + it('mirrors an array save into the store as a whole series', function () { + const global = storeGlobal() + const node = fakeNode('chart-2') + + datastore.save(base, node, [ + { payload: 1, _datapoint: pt(1, 1) }, + { payload: 2, _datapoint: pt(2, 2) } + ]) + + liveStore(global, 'canary-2').should.equal('live') + const store = global.get('dashboardStore') + store['chart-2'].should.eql([pt(1, 1), pt(2, 2)]) + store['$chart-2'].msg.should.have.length(2) + }) + + it('replaces rather than appends on a second array save', function () { + const global = storeGlobal() + const node = fakeNode('chart-replace') + + datastore.save(base, node, [{ payload: 1, _datapoint: pt(1, 1) }]) + datastore.save(base, node, [{ payload: 9, _datapoint: pt(9, 9) }]) + + const store = global.get('dashboardStore') + store['chart-replace'].should.eql([pt(9, 9)]) + store['$chart-replace'].msg.should.have.length(1) + }) + + it('empties the series when saved an empty array', function () { + const global = storeGlobal() + const node = fakeNode('chart-clear') + + datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) + datastore.save(base, node, []) + + const store = global.get('dashboardStore') + store['chart-clear'].should.eql([]) + store['$chart-clear'].msg.should.eql([]) + }) + + it('does not rebuild the stored series until the batch is reached', function () { + const global = storeGlobal() + const node = fakeNode('chart-3') + for (let i = 0; i < 100; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + + for (let i = 0; i < 59; i++) { + datastore.filter(base, node, (m, idx) => idx > 0) + } + + liveStore(global, 'canary-3').should.equal('live') + global.get('dashboardStore')['chart-3'].should.have.length(100) + datastore.get('chart-3').should.have.length(41) + }) + + it('rebuilds the stored series on the batch boundary', function () { + const global = storeGlobal() + const node = fakeNode('chart-batch') + for (let i = 0; i < 100; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + + for (let i = 0; i < 60; i++) { + datastore.filter(base, node, (m, idx) => idx > 0) + } + + const expected = [] + for (let i = 60; i < 100; i++) expected.push(pt(i, i)) + + const store = global.get('dashboardStore') + store['chart-batch'].should.eql(expected) + store['$chart-batch'].msg.should.have.length(40) + }) + + it('does not create a key when the appended message is not an object', function () { + const global = storeGlobal() + const node = fakeNode('chart-junk') + datastore.append(base, fakeNode('chart-ok'), { payload: 1, _datapoint: pt(1, 1) }) + + datastore.append(base, node, 5) + datastore.save(base, node, [5, { payload: 1, _datapoint: pt(1, 1) }]) + + should(global.get('dashboardStore')['chart-junk']).be.undefined() + global.get('dashboardStore')['chart-ok'].should.have.length(1) + }) + + it('leaves the stored series untouched when trims remove nothing', function () { + const global = storeGlobal() + const node = fakeNode('chart-noop') + for (let i = 0; i < 100; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + const before = global.get('dashboardStore')['$chart-noop'].msg + + for (let i = 0; i < 200; i++) { + datastore.filter(base, node, () => true) + } + + global.get('dashboardStore')['$chart-noop'].msg.should.eql(before) + }) + + it('ignores a filter call with no callback', function () { + storeGlobal() + const node = fakeNode('chart-nofilter') + datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) + + should(() => datastore.filter(base, node, undefined)).not.throw() + + datastore.get('chart-nofilter').should.have.length(1) + }) + + it('keeps the trimmed chart series', function () { + storeGlobal() + const node = fakeNode('chart-4') + + datastore.append(base, node, { _datapoint: pt(1, 1) }) + datastore.append(base, node, { _datapoint: pt(2, 2) }) + datastore.filter(base, node, (m, i) => i > 0) + + datastore.get('chart-4').map((m) => m._datapoint).should.eql([pt(2, 2)]) + }) + + it('clears a stale key if a node id was previously used for a value', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('chart-5') + + datastore.save(base, node, { payload: 'scalar first' }) + global.get('dashboardStore')['chart-5'].should.equal('scalar first') + + datastore.clear(node.id) + should(global.get('dashboardStore')['chart-5']).be.undefined() + }) + + it('keeps history empty across many appends', function () { + const global = storeGlobal() + const node = fakeNode('chart-hist') + for (let i = 0; i < 20; i++) { + datastore.append(base, node, { payload: i, _datapoint: pt(i, i) }) + } + + global.get('dashboardStore')['$chart-hist'].history.should.eql([]) + }) + + it('serialises a chart key to its point array and nothing else', function () { + const global = storeGlobal() + const node = fakeNode('chart-json') + datastore.append(base, node, { payload: 1, topic: 'a', _datapoint: pt(1, 1) }) + + JSON.parse(JSON.stringify(global.get('dashboardStore'))).should.eql({ + 'chart-json': [pt(1, 1)] + }) + }) + + it('leaves global context untouched when it cannot hold the store', function () { + const global = fakeGlobal() + const node = fakeNode('chart-off') + datastore.initStore({ get () { throw new Error('cache off') }, set () {} }, {}) + try { + datastore.append(base, node, { payload: 1, _datapoint: pt(1, 1) }) + datastore.save(base, node, [{ payload: 2, _datapoint: pt(2, 2) }]) + datastore.filter(base, node, () => false) + + datastore.get('chart-off').should.eql([]) + should(global.get('dashboardStore')).be.undefined() + } finally { + datastore.initStore(fakeGlobal(), {}) + } + }) +}) + +describe('store: data.js audit invariants', function () { + beforeEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('does not store a message carrying only dynamic properties', function () { + const global = storeGlobal() + const node = fakeNode('a1') + datastore.save(base, node, { payload: 'real' }) + datastore.save(base, node, { ui_update: { class: 'red' } }) + + global.get('dashboardStore').a1.should.equal('real') + }) + + it('keeps value and msg in step across writes', function () { + const global = storeGlobal() + const node = fakeNode('a2') + datastore.save(base, node, { payload: 1, topic: 'first' }) + datastore.save(base, node, { payload: 2, topic: 'second' }) + + const store = global.get('dashboardStore') + store.a2.should.equal(2) + store.$a2.msg.topic.should.equal('second') + }) + + it('leaves no key at all when a widget never stores', function () { + const global = storeGlobal() + datastore.save(base, fakeNode('a3'), { payload: 1 }) + + const store = global.get('dashboardStore') + store.a3.should.equal(1) + should(store.never).be.undefined() + }) + + it('clears value and msg together when a widget is removed', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('a4') + datastore.save(base, node, { payload: 5, topic: 't' }) + global.get('dashboardStore').$a4.msg.should.be.an.Object() + + datastore.clear(node.id) + + const store = global.get('dashboardStore') + should(store.a4).be.undefined() + should(store.$a4).be.undefined() + }) +}) + +describe('store: data.js client-scoped writes', function () { + beforeEach(function () { + datastore.initStore(fakeGlobal(), {}) }) describe('save', function () { const base = { acceptsClientConfig: ['ui-text'] } - const node = { id: 'w1', type: 'ui-text' } + const node = fakeNode('w1') beforeEach(function () { datastore.clear(node.id) @@ -61,6 +592,136 @@ describe('store: data', function () { }) }) +describe('store: data.js failure reporting', function () { + const unreachable = () => { throw new Error('context store unavailable') } + const baseNode = { warn: (m) => warnings.push(`base: ${m}`) } + const SUFFIX = '. Further store failures are not reported until the next full deploy or restart.' + + beforeEach(function () { + warnings.length = 0 + }) + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('warns once on the base node when the store cannot be reached, not once per call', function () { + const global = fakeGlobal() + datastore.initStore(global, { node: baseNode }) + global.get = unreachable + + datastore.save(base, fakeNode('fail-1'), { payload: 1 }) + datastore.save(base, fakeNode('fail-1'), { payload: 2 }) + datastore.append(base, fakeNode('fail-1-chart'), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + datastore.filter(base, fakeNode('fail-1-chart'), () => true) + datastore.clear('fail-1') + datastore.get('fail-1') + + warnings.should.eql([`base: Dashboard data store: saving fail-1 failed (context store unavailable)${SUFFIX}`]) + }) + + it('names the operation that failed', function () { + for (const [action, call] of [ + ['saving', () => datastore.append(base, fakeNode('fail-a'), { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } })], + ['trimming', () => datastore.keepLatestPerTopic(base, fakeNode('fail-a'), 1)], + ['clearing', () => datastore.clear('fail-a')], + ['reading', () => datastore.has('fail-a')] + ]) { + const global = fakeGlobal() + datastore.initStore(global, { node: baseNode }) + global.get = unreachable + warnings.length = 0 + + call() + + warnings.should.eql([`base: Dashboard data store: ${action} fail-a failed (context store unavailable)${SUFFIX}`]) + } + }) + + it('does not throw when there is no base node to warn on', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + global.get = unreachable + + should(() => datastore.clear('no-base')).not.throw() + should(() => datastore.save(base, fakeNode('no-base'), { payload: 1 })).not.throw() + }) + + it('explains a context store that stops holding the store by reference', function () { + const m = {} + let copying = false + const global = { get: (k) => copying ? JSON.parse(JSON.stringify(m[k])) : m[k], set: (k, v) => { m[k] = v } } + datastore.initStore(global, { node: baseNode }) + copying = true + + datastore.save(base, fakeNode('fail-r'), { payload: 1 }) + + warnings.should.eql([`base: Dashboard data store: saving fail-r failed (global context saved a copy of the store rather than the store itself, which happens with context stores that serialise values)${SUFFIX}`]) + }) + + it('reads as empty when the store cannot be reached', function () { + const global = fakeGlobal() + datastore.initStore(global, { node: baseNode }) + global.get = unreachable + + should(datastore.get('fail-2')).be.undefined() + datastore.has('fail-2').should.equal(false) + }) + + it('warns again after the next initStore', function () { + const global = fakeGlobal() + datastore.initStore(global, { node: baseNode }) + global.get = unreachable + datastore.save(base, fakeNode('fail-3'), { payload: 1 }) + + const fresh = fakeGlobal() + datastore.initStore(fresh, { node: baseNode }) + fresh.get = unreachable + datastore.save(base, fakeNode('fail-3'), { payload: 2 }) + + warnings.should.have.length(2) + }) + + it('throws a failing filter to the widget without reporting a store failure', function () { + datastore.initStore(fakeGlobal(), { node: baseNode }) + const node = fakeNode('fail-6') + datastore.append(base, node, { payload: 1, _datapoint: null }) + + should.throws(() => datastore.filter(base, node, (m) => m._datapoint.x > 0)) + + warnings.should.eql([]) + }) + + it('leaves the stored message untouched when a save cannot be copied', function () { + datastore.initStore(fakeGlobal(), { node: baseNode }) + const node = fakeNode('fail-8') + datastore.save(base, node, { payload: 1, topic: 't', label: 'L' }) + + should.throws(() => datastore.save(base, node, { payload: 2, meta: { get bad () { throw new Error('unreadable') } } })) + + datastore.get('fail-8').should.eql({ payload: 1, topic: 't', label: 'L' }) + }) + + it('leaves the stored message untouched when an append cannot be copied', function () { + datastore.initStore(fakeGlobal(), { node: baseNode }) + const node = fakeNode('fail-9') + datastore.save(base, node, { payload: 1, topic: 't' }) + + should.throws(() => datastore.append(base, node, { payload: 2, meta: { get bad () { throw new Error('unreadable') } }, _datapoint: { x: 1, y: 2 } })) + + datastore.get('fail-9').should.eql({ payload: 1, topic: 't' }) + }) + + it('throws a message the store cannot copy to the widget without reporting a store failure', function () { + datastore.initStore(fakeGlobal(), { node: baseNode }) + const unreadable = { get payload () { throw new Error('unreadable') } } + + should.throws(() => datastore.save(base, fakeNode('fail-7'), unreadable), /unreadable/) + + warnings.should.eql([]) + }) +}) + describe('store: data reads that avoid cloning', function () { const { util } = require('@node-red/util') const base = { acceptsClientConfig: [] } @@ -73,6 +734,7 @@ describe('store: data reads that avoid cloning', function () { const topicsAndPayloads = (id) => datastore.get(id).map((m) => `${m.topic}:${m.payload}`) beforeEach(function () { + datastore.initStore({ get () { throw new Error('cache off') }, set () {} }, {}) datastore.setConfig(countingRED) }) @@ -165,3 +827,123 @@ describe('store: data reads that avoid cloning', function () { datastore.get('copy-1')[0].payload.should.equal(1) }) }) + +describe('store: data.js merges into the store from the store itself', function () { + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('stores a message without a payload, merged onto the stored one', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('merge-1') + + datastore.save(base, node, { payload: 1 }) + datastore.save(base, node, { topic: 't' }) + + global.get('dashboardStore')['$merge-1'].msg.should.eql({ payload: 1, topic: 't' }) + }) + + it('merges onto what the store holds, not what the legacy datastore holds', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('merge-2') + datastore.save(base, node, { payload: 1 }) + global.get('dashboardStore')['merge-2'] = 5 + + datastore.save(base, node, { topic: 't' }) + + global.get('dashboardStore')['$merge-2'].msg.should.eql({ payload: 5, topic: 't' }) + datastore.get('merge-2').should.eql({ payload: 5, topic: 't' }) + }) + + it('ignores a message that is not an object, without warning', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const warned = [] + const node = { ...fakeNode('merge-4'), warn: (m) => warned.push(m) } + datastore.save(base, node, { payload: 1 }) + + datastore.save(base, node, 5) + + global.get('dashboardStore')['$merge-4'].msg.should.eql({ payload: 1 }) + warned.should.eql([]) + }) + + it('removes the store entry on clear(id), which has no node', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + datastore.save(base, fakeNode('merge-3'), { payload: 'hello' }) + + datastore.clear('merge-3') + + should(global.get('dashboardStore')['merge-3']).be.undefined() + }) +}) + +describe('store: data.js trims the stored series itself', function () { + const point = (x, topic = 'a') => ({ payload: x, topic, _datapoint: { category: topic, x, y: x } }) + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('trims what the store holds, not a copy of the legacy result', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('own-trim') + datastore.save(base, node, []) + for (let i = 0; i < 6; i++) { + datastore.append(base, node, point(i, i % 2 ? 'b' : 'a')) + } + global.get('dashboardStore')[APPEND_ENTRY]('own-trim', point('store-only')) + + datastore.keepLatestPerTopic(base, node, 2) + + global.get('dashboardStore')['$own-trim'].msg.map((m) => m.payload).should.eql([3, 4, 5, 'store-only']) + datastore.get('own-trim').map((m) => m.payload).should.eql([3, 4, 5, 'store-only']) + }) + + it('applies both a point limit and a time window', function () { + const global = fakeGlobal() + datastore.initStore(global, {}) + const node = fakeNode('both-limits') + datastore.save(base, node, []) + for (let i = 0; i < 100; i++) { + datastore.append(base, node, point(i)) + datastore.keepLatestPerTopic(base, node, 10) + datastore.filter(base, node, (m) => m._datapoint.x > i - 50) + } + + global.get('dashboardStore')['$both-limits'].msg.map((m) => m.payload).should.eql(datastore.get('both-limits').map((m) => m.payload)) + }) +}) + +describe('store: data.js after a restart with a persistent context store', function () { + const restored = (values) => { + const global = fakeGlobal() + global.set('dashboardStore', JSON.parse(JSON.stringify(values))) + datastore.initStore(global, {}) + } + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + it('starts empty, as main does', function () { + restored({ text: 'hello', rows: [{ id: 1 }], chart: [{ category: 'a', x: 1, y: 1 }] }) + + datastore.has('text').should.equal(false) + should(datastore.get('text')).be.undefined() + datastore.has('rows').should.equal(false) + datastore.has('chart').should.equal(false) + }) + + it('stores a chart\'s first point after a restart as a fresh series', function () { + restored({ chart: [{ category: 'a', x: 1, y: 1 }] }) + + datastore.append(base, fakeNode('chart'), { payload: 2, _datapoint: { category: 'a', x: 2, y: 2 } }) + + datastore.get('chart').should.eql([{ payload: 2, _datapoint: { category: 'a', x: 2, y: 2 } }]) + }) +}) diff --git a/test/store/differential.spec.js b/test/store/differential.spec.js new file mode 100644 index 000000000..cd540769f --- /dev/null +++ b/test/store/differential.spec.js @@ -0,0 +1,164 @@ +const { util } = require('@node-red/util') +const should = require('should') // eslint-disable-line no-unused-vars + +const datastore = require('../../nodes/store/data.js') + +const createLegacy = require('./helpers/legacy-datastore.js') +const { scenarios, generate, generateMessy, runStep } = require('./helpers/sequences.js') + +const RED = { util, plugins: { getByType: () => [] } } +const base = { acceptsClientConfig: ['ui-scoped'] } + +function fakeGlobal () { + const m = {} + return { get: (k) => m[k], set: (k, v) => { m[k] = v } } +} + +function cacheOffGlobal () { + return { get () { throw new Error('File Store cache disabled - only asynchronous access supported') }, set () {} } +} + +const runs = [ + ...Object.entries(scenarios).map(([name, steps]) => ({ name, steps })), + ...Array.from({ length: 200 }, (_, i) => ({ name: `seed ${i + 1}`, steps: generate(i + 1, 180) })), + ...Array.from({ length: 100 }, (_, i) => ({ name: `messy seed ${i + 1}`, steps: generateMessy(i + 1, 150) })) +] + +let runIndex = 0 + +function prepare (steps) { + const prefix = `diff-${runIndex++}:` + const prefixed = steps.map((s) => ({ ...s, id: prefix + s.id })) + return { steps: prefixed, ids: [...new Set(prefixed.map((s) => s.id))] } +} + +function scribble (value, seen = new Set()) { + if (!value || typeof value !== 'object' || seen.has(value)) { + return + } + seen.add(value) + for (const key of Object.keys(value)) { + if (value[key] && typeof value[key] === 'object') { + scribble(value[key], seen) + } else { + value[key] = 'scribbled' + } + } + if (Array.isArray(value)) { + value.push('scribbled') + } +} + +function storeHolds (global, id) { + const store = global.get('dashboardStore') + return store['$' + id]?.msg +} + +function compareAfterEachStep (steps, ids, global = fakeGlobal()) { + const warnings = [] + const baseNode = { warn: (m) => warnings.push(m) } + const shared = datastore.initStore(global, { node: baseNode }) + const nodeFor = (id, type) => ({ id, type, context: () => ({ global }), warn: (m) => warnings.push(m) }) + const legacy = createLegacy(RED) + let comparedWithData = 0 + for (const [i, step] of steps.entries()) { + if (i === Math.floor(steps.length / 2)) { + datastore.initStore(global, { node: baseNode }) + } + const given = structuredClone(step) + runStep(datastore, given, { base, nodeFor }) + runStep(legacy, structuredClone(step), { base, nodeFor }) + should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after ${step.op}`) + scribble(given) + should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after scribbling what ${step.op} was given`) + scribble(datastore.get(step.id)) + should(datastore.get(step.id)).eql(legacy.get(step.id), `${step.id} after scribbling what get returned`) + datastore.has(step.id).should.equal(legacy.has(step.id), `${step.id} after ${step.op}`) + if (shared) { + should(storeHolds(global, step.id)).eql(legacy.get(step.id), `store for ${step.id} after ${step.op}`) + } + if (legacy.has(step.id)) { + comparedWithData++ + } + } + for (const id of ids) { + should(datastore.get(id)).eql(legacy.get(id), id) + if (shared) { + should(storeHolds(global, id)).eql(legacy.get(id), `store for ${id}`) + } + } + return { comparedWithData, warnings, shared } +} + +function compareAtEnd (steps, ids) { + const global = fakeGlobal() + const warnings = [] + datastore.initStore(global, { node: { warn: (m) => warnings.push(m) } }) + const nodeFor = (id, type) => ({ id, type, context: () => ({ global }), warn: (m) => warnings.push(m) }) + const legacy = createLegacy(RED) + for (const step of steps) { + runStep(datastore, structuredClone(step), { base, nodeFor }) + runStep(legacy, structuredClone(step), { base, nodeFor }) + } + const withData = ids.filter((id) => legacy.has(id)) + for (const id of ids) { + should(storeHolds(global, id)).eql(legacy.get(id), `store for ${id}`) + } + return { withData, warnings } +} + +describe('store: differential against the legacy datastore', function () { + beforeEach(function () { + datastore.setConfig(RED) + }) + + afterEach(function () { + datastore.initStore(fakeGlobal(), {}) + }) + + describe('data.js matches the legacy reference', function () { + for (const run of runs) { + it(run.name, function () { + const { steps, ids } = prepare(run.steps) + const { comparedWithData, warnings, shared } = compareAfterEachStep(steps, ids) + + should(shared).not.be.null() + comparedWithData.should.be.above(0) + warnings.should.eql([]) + }) + } + }) + + describe('data.js matches the legacy reference when global context cannot hold the store', function () { + for (const run of runs) { + it(run.name, function () { + const { steps, ids } = prepare(run.steps) + const { comparedWithData, warnings, shared } = compareAfterEachStep(steps, ids, cacheOffGlobal()) + + should(shared).be.null() + comparedWithData.should.be.above(0) + warnings.should.eql([]) + }) + } + }) + + describe('the store matches the legacy reference when only read at the end', function () { + let runsWithData = 0 + + for (const run of runs) { + it(run.name, function () { + const { steps, ids } = prepare(run.steps) + const { withData, warnings } = compareAtEnd(steps, ids) + + if (withData.length) { + runsWithData++ + } + warnings.should.eql([]) + }) + } + + it('compared stored data at the end of most runs', function () { + runsWithData.should.be.above(runs.length * 0.9) + }) + }) +}) diff --git a/test/store/helpers/legacy-datastore.js b/test/store/helpers/legacy-datastore.js new file mode 100644 index 000000000..ba51c9b27 --- /dev/null +++ b/test/store/helpers/legacy-datastore.js @@ -0,0 +1,91 @@ +const { isClientScoped } = require('../../../nodes/utils/index.js') + +module.exports = function createLegacy (RED) { + const data = {} + + function canSaveInStore (base, node, msg) { + const checks = [] + if (base.acceptsClientConfig.includes(node.type)) { + if (isClientScoped(msg)) { + checks.push(false) + } + for (const plugin of RED.plugins.getByType('node-red-dashboard-2')) { + if (plugin.hooks?.onCanSaveInStore) { + checks.push(plugin.hooks.onCanSaveInStore(msg)) + } + } + } + return checks.length === 0 || !checks.includes(false) + } + + function stripMsg (msg) { + const newMsg = RED.util.cloneMessage(msg) + delete newMsg.ui_update + return newMsg + } + + function filter (base, node, filterFunction) { + const currentData = data[node.id] + if (filterFunction && Array.isArray(currentData) && currentData.length) { + const filteredMessages = currentData.filter(filterFunction) + if (filteredMessages.length !== currentData.length) { + data[node.id] = filteredMessages + } + } + } + + return { + get (id) { + return RED.util.cloneMessage(data[id]) + }, + has (id) { + return !!data[id] + }, + clear (id) { + delete data[id] + }, + save (base, node, msg) { + if (Array.isArray(msg)) { + const filtered = [] + for (const m of msg) { + if (canSaveInStore(base, node, m)) { + filtered.push(RED.util.cloneMessage(m)) + } + } + data[node.id] = filtered + } else if (canSaveInStore(base, node, msg)) { + data[node.id] = { ...data[node.id], ...stripMsg(msg) } + } + }, + append (base, node, msg) { + if (canSaveInStore(base, node, msg)) { + if (!data[node.id]) { + data[node.id] = [] + } + data[node.id].push(RED.util.cloneMessage(msg)) + } + }, + filter, + keepLatestPerTopic (base, node, maxPoints) { + const currentData = data[node.id] + if (!Array.isArray(currentData)) { + return + } + const counts = {} + const keep = [] + let trimmed = false + for (let i = currentData.length - 1; i >= 0; i--) { + const topic = currentData[i].topic + counts[topic] = (counts[topic] || 0) + 1 + if (counts[topic] <= maxPoints) { + keep[i] = true + } else { + trimmed = true + } + } + if (trimmed) { + filter(base, node, (m, i) => keep[i]) + } + } + } +} diff --git a/test/store/helpers/sequences.js b/test/store/helpers/sequences.js new file mode 100644 index 000000000..9dfcbe6af --- /dev/null +++ b/test/store/helpers/sequences.js @@ -0,0 +1,242 @@ +const { hasExtraProps } = require('../../../nodes/utils/index.js') + +const pt = (x, y, category = 'a') => ({ category, x, y }) + +function mulberry32 (seed) { + return function () { + seed |= 0 + seed = seed + 0x6D2B79F5 | 0 + let t = Math.imul(seed ^ seed >>> 15, 1 | seed) + t = t + Math.imul(t ^ t >>> 7, 61 | t) ^ t + return ((t ^ t >>> 14) >>> 0) / 4294967296 + } +} + +const repeat = (n, make) => Array.from({ length: n }, (_, i) => make(i)) + +const chartPoint = (x, topic) => { + const msg = { payload: x, _datapoint: pt(x, x, topic || 'a') } + if (topic !== undefined) { + msg.topic = topic + } + return msg +} + +const scenarios = { + text: [ + { op: 'input', id: 't', msg: { payload: 1, topic: 'first' } }, + { op: 'input', id: 't', msg: { topic: 'second' } }, + { op: 'input', id: 't', msg: { payload: 2, ui_update: { label: 'x' } } }, + { op: 'input', id: 't', msg: { class: 'big' } }, + { op: 'input', id: 't', msg: { payload: { nested: [1, 2] }, extra: true } } + ], + switch: [ + { op: 'switchSave', id: 'sw', msg: { payload: true, topic: 'on' } }, + { op: 'switchSave', id: 'sw', msg: { class: 'highlight' } }, + { op: 'switchSave', id: 'sw', msg: { payload: false } } + ], + 'value-widget-change': [ + { op: 'input', id: 'v', msg: { payload: 10, topic: 'slider' } }, + { op: 'change', id: 'v', value: 20, client: 's1' }, + { op: 'change', id: 'v', value: { payload: 30 } }, + { op: 'input', id: 'v', msg: { payload: 40 } }, + { op: 'change', id: 'v', value: 50, topic: 'configured' } + ], + 'table-append': [ + ...repeat(4, (i) => ({ op: 'table', id: 'tb', action: 'append', msg: { payload: [{ id: i }, { id: i + 100 }], topic: 'rows' } })), + { op: 'table', id: 'tb', action: 'append', msg: { payload: { id: 'single' } } }, + { op: 'table', id: 'tb', action: 'append', msg: { payload: [] } } + ], + 'table-replace': repeat(4, (i) => ({ op: 'table', id: 'tr', action: 'replace', msg: { payload: [{ id: i }] } })), + 'chart-time': repeat(200, (i) => ({ op: 'chart', id: 'ct', msg: chartPoint(i, 'a'), window: 40 })), + 'chart-points': repeat(300, (i) => ({ op: 'chart', id: 'cp', msg: chartPoint(i, ['a', 'b', 'c'][i % 3]), maxPoints: 50 })), + 'chart-both-limits': repeat(200, (i) => ({ op: 'chart', id: 'cb', msg: chartPoint(i, ['a', 'b'][i % 2]), maxPoints: 20, window: 30 })), + 'chart-array-payload': repeat(20, (i) => ({ op: 'chart', id: 'cm', msg: { payload: [i, i * 2], _datapoint: [pt(i, i, 'a'), pt(i, i * 2, 'b')] } })), + 'chart-object-payload': repeat(20, (i) => ({ op: 'chart', id: 'co', msg: { payload: { a: i, b: i * 2 }, _datapoint: [pt(i, i, 'a'), pt(i, i * 2, 'b')] } })), + 'chart-replace-clear': [ + ...repeat(10, (i) => ({ op: 'chart', id: 'cr', msg: chartPoint(i) })), + { op: 'chart', id: 'cr', msg: { ...chartPoint(50), action: 'replace' } }, + ...repeat(5, (i) => ({ op: 'chart', id: 'cr', msg: chartPoint(60 + i) })), + { op: 'chart', id: 'cr', msg: { payload: [] } }, + ...repeat(3, (i) => ({ op: 'chart', id: 'cr', msg: chartPoint(70 + i) })) + ], + 'chart-position-trims': repeat(150, (i) => [{ op: 'chart', id: 'pt', msg: chartPoint(i) }, { op: 'filterIndex', id: 'pt', skip: 1 }]).flat(), + 'chart-categorical': repeat(30, (i) => ({ op: 'chart', id: 'cc', categorical: true, msg: { payload: i, _datapoint: pt(['mon', 'tue', 'wed'][i % 3], i, 'sales') } })), + 'client-scoped': [ + { op: 'input', id: 'cs', type: 'ui-scoped', msg: { payload: 'everyone' } }, + { op: 'input', id: 'cs', type: 'ui-scoped', msg: { payload: 'one client', _client: { socketId: 's1' } } }, + { op: 'chart', id: 'csa', type: 'ui-scoped', msg: { ...chartPoint(1), _client: { clientId: 'c1' } } }, + { op: 'chart', id: 'csa', type: 'ui-scoped', msg: chartPoint(2) } + ], + 'audio-clear': [ + { op: 'input', id: 'au', msg: { payload: 'hello' } }, + { op: 'clear', id: 'au' }, + { op: 'input', id: 'au', msg: { topic: 'after clear' } } + ], + 'widget-remove': [ + ...repeat(5, (i) => ({ op: 'chart', id: 'wr', msg: chartPoint(i), maxPoints: 3 })), + { op: 'remove', id: 'wr' }, + ...repeat(5, (i) => ({ op: 'chart', id: 'wr', msg: chartPoint(10 + i), maxPoints: 3 })) + ] +} + +function generate (seed, length) { + const rand = mulberry32(seed) + const pick = (list) => list[Math.floor(rand() * list.length)] + const chartConfig = () => { + const mode = pick(['none', 'time', 'points', 'both']) + return { + window: mode === 'time' || mode === 'both' ? 5 + Math.floor(rand() * 30) : undefined, + maxPoints: mode === 'points' || mode === 'both' ? 1 + Math.floor(rand() * 5) : undefined + } + } + const charts = { g1: chartConfig(), g3: chartConfig(), g5: chartConfig() } + const steps = [] + for (let i = 0; i < length; i++) { + const id = pick(['g0', 'g1', 'g2', 'g3', 'g4', 'g5']) + const topic = pick(['a', 'b', undefined]) + const roll = rand() + const client = rand() < 0.4 ? { _client: { socketId: pick(['s1', 's2']) } } : {} + if (roll < 0.03) { + steps.push({ op: pick(['clear', 'remove']), id }) + } else if (id === 'g4') { + steps.push({ op: roll < 0.5 ? 'input' : 'switchSave', id, type: 'ui-scoped', msg: { payload: Math.floor(rand() * 100), ...client } }) + } else if (id === 'g5') { + steps.push({ op: 'chart', id, type: 'ui-scoped', msg: { ...chartPoint(i, topic), ...client }, ...charts[id], cutoffAt: i }) + } else if (id === 'g0') { + const msg = {} + if (rand() < 0.8) { msg.payload = Math.floor(rand() * 100) } + if (topic !== undefined) { msg.topic = topic } + if (rand() < 0.2) { msg.ui_update = { label: 'l' + i } } + if (rand() < 0.2) { msg.extra = { n: i } } + steps.push(roll < 0.5 ? { op: 'input', id, msg } : roll < 0.75 ? { op: 'switchSave', id, msg } : { op: 'change', id, value: Math.floor(rand() * 100), client: rand() < 0.5 ? 's1' : undefined }) + } else if (id === 'g2') { + steps.push({ op: 'table', id, action: roll < 0.7 ? 'append' : 'replace', msg: { payload: rand() < 0.1 ? [] : [{ id: i }] } }) + } else { + const msg = roll < 0.92 ? chartPoint(i, topic) : roll < 0.96 ? { payload: [] } : { ...chartPoint(i, topic), action: 'replace' } + steps.push({ op: 'chart', id, msg, ...charts[id], cutoffAt: i }) + } + } + return steps +} + +function generateMessy (seed, length) { + const rand = mulberry32(seed) + const pick = (list) => list[Math.floor(rand() * list.length)] + const steps = [] + for (let i = 0; i < length; i++) { + const id = pick(['m0', 'm1']) + const roll = rand() + const x = Math.floor(rand() * 200) + if (roll < 0.45) { + steps.push({ op: 'chart', id, msg: chartPoint(x, pick(['a', 'b', undefined])) }) + } else if (roll < 0.6) { + steps.push({ op: 'filterTime', id, cutoff: Math.floor(rand() * 150) }) + } else if (roll < 0.72) { + steps.push({ op: 'keep', id, maxPoints: 1 + Math.floor(rand() * 6) }) + } else if (roll < 0.82) { + steps.push({ op: 'filterIndex', id, skip: Math.floor(rand() * 3) }) + } else if (roll < 0.9) { + steps.push({ op: 'chart', id, categorical: true, msg: chartPoint(x) }) + } else if (roll < 0.95) { + steps.push({ op: 'saveArray', id, msgs: rand() < 0.5 ? [] : [chartPoint(x)] }) + } else { + steps.push({ op: pick(['clear', 'remove']), id }) + } + } + steps.push({ op: 'switchSave', id: 'm0', msg: { topic: 'merged' } }, { op: 'switchSave', id: 'm1', msg: { topic: 'merged' } }) + return steps +} + +function chartInput (target, base, node, step) { + const { msg } = step + if (!target.has(step.id)) { + target.save(base, node, []) + } + if (typeof msg.payload === 'undefined') { + return + } + if (Array.isArray(msg.payload) && !msg.payload.length) { + target.save(base, node, []) + return + } + if (msg.action === 'replace') { + target.save(base, node, []) + } + if (!Array.isArray(msg.payload)) { + target.append(base, node, { ...msg }) + } else { + msg.payload.forEach((p, i) => { + target.append(base, node, { ...msg, payload: JSON.parse(JSON.stringify(p)), _datapoint: msg._datapoint ? msg._datapoint[i] : null }) + }) + } + if (step.maxPoints) { + target.keepLatestPerTopic(base, node, step.maxPoints) + } + if (step.window) { + const cutoff = (step.cutoffAt ?? [].concat(msg._datapoint)[0].x) - step.window + target.filter(base, node, (m) => [].concat(m._datapoint)[0]?.x > cutoff) + } else if (step.categorical) { + const latest = {} + for (const item of target.get(step.id) || []) { + latest[JSON.stringify([item._datapoint.category, item._datapoint.x])] = item + } + target.save(base, node, Object.values(latest)) + } +} + +function changeInput (target, base, node, step) { + const msg = target.get(step.id) || {} + if (step.client) { + msg._client = { ...msg._client, socketId: step.client } + } + msg.payload = (typeof step.value === 'object' && step.value !== null && 'payload' in step.value) ? step.value.payload : step.value + if (step.topic !== undefined) { + msg.topic = step.topic + } + if (!('topic' in msg)) { + msg.topic = '' + } + target.save(base, node, msg) +} + +function tableInput (target, base, node, step) { + const existing = target.get(step.id) || [] + const value = step.msg.payload + let payload = (value !== null && typeof value === 'object' && !Array.isArray(value)) ? [value] : value + if (step.action === 'append') { + payload = payload && payload.length > 0 ? [...existing.payload || [], ...payload || []] : payload + } + target.save(base, node, { ...step.msg, payload }) +} + +function runStep (target, step, { base, nodeFor }) { + const node = nodeFor(step.id, step.type || 'ui-test') + if (step.op === 'input') { + if (hasExtraProps(step.msg)) { + target.save(base, node, step.msg) + } + } else if (step.op === 'switchSave') { + target.save(base, node, step.msg) + } else if (step.op === 'change') { + changeInput(target, base, node, step) + } else if (step.op === 'table') { + tableInput(target, base, node, step) + } else if (step.op === 'chart') { + chartInput(target, base, node, step) + } else if (step.op === 'saveArray') { + target.save(base, node, step.msgs) + } else if (step.op === 'filterTime') { + target.filter(base, node, (m) => m._datapoint && [].concat(m._datapoint)[0].x > step.cutoff) + } else if (step.op === 'filterIndex') { + target.filter(base, node, (m, i) => i >= step.skip) + } else if (step.op === 'keep') { + target.keepLatestPerTopic(base, node, step.maxPoints) + } else if (step.op === 'clear') { + target.clear(step.id) + } else if (step.op === 'remove') { + target.clear(step.id) + } +} + +module.exports = { scenarios, generate, generateMessy, runStep } diff --git a/test/store/reactive.spec.js b/test/store/reactive.spec.js new file mode 100644 index 000000000..64c5d67b0 --- /dev/null +++ b/test/store/reactive.spec.js @@ -0,0 +1,1550 @@ +const fs = require('fs') +const http = require('http') +const os = require('os') +const path = require('path') + +const vm = require('vm') + +const LocalFileSystem = require('@node-red/runtime/lib/nodes/context/localfilesystem.js') +const Memory = require('@node-red/runtime/lib/nodes/context/memory.js') + +const { util } = require('@node-red/util') +const should = require('should') // eslint-disable-line no-unused-vars + +const { createDataStore, attachToContext, STORE, APPEND_ENTRY, SET_SERIES, MERGE_ENTRY, TRIM_SERIES, ENTRY_MSG } = require('../../nodes/store/reactive.js') + +function makeStore (opts = {}) { + const log = [] + let tick = 0 + const store = createDataStore({ + now: () => tick++, + onChange: (key, entry, path) => log.push(path), + ...opts + }) + return { store, log } +} + +// Node-RED runs function nodes in a vm context, so their objects carry that realm's Object.prototype +const fnRealm = vm.createContext({}) +const fnNodeObject = (src) => vm.runInContext(src, fnRealm) + +describe('store: reactive data store', function () { + describe('values and metadata', function () { + it('stores a scalar, fires onChange, and reads it back as a value', function () { + const { store, log } = makeStore() + store.k1 = 123 + log.should.eql(['k1']) + store.k1.should.equal(123) + }) + + it('exposes the full entry under the $ prefix', function () { + const { store } = makeStore() + store.k1 = 123 + store.$k1.value.should.equal(123) + store.$k1.quality.should.equal('GOOD') + store.$k1.timestamp.should.be.a.Number() + }) + + it('synthesises a msg from the value when a flow writes the key directly', function () { + const { store } = makeStore() + store.k1 = 123 + store.$k1.msg.should.eql({ payload: 123 }) + store.$k1.msg.payload.should.equal(store.$k1.value) + }) + + it('does not leave a msg describing the previous value after a flow write', function () { + const { store } = makeStore() + store[MERGE_ENTRY]('k1', { payload: 42, topic: 'boiler' }) + store.$k1.msg.topic.should.equal('boiler') + + store.k1 = 99 + + store.$k1.value.should.equal(99) + store.$k1.msg.should.eql({ payload: 99 }) + }) + + it('returns the whole object by default', function () { + const { store } = makeStore() + store.robot = { temp: 20 } + store.robot.should.eql({ temp: 20 }) + }) + }) + + describe('entry setter', function () { + it('sets value and msg together and fires one change', function () { + const { store, log } = makeStore() + const msg = { payload: 42, topic: 'sensor-A' } + + store[MERGE_ENTRY]('gauge', msg) + + store.gauge.should.equal(42) + store.$gauge.msg.should.eql({ payload: 42, topic: 'sensor-A' }) + log.should.eql(['gauge']) + }) + + it('keeps history on the value, as a normal write does', function () { + const { store } = makeStore() + store[MERGE_ENTRY]('gauge', { payload: 1 }) + store[MERGE_ENTRY]('gauge', { payload: 2 }) + + store.gauge.should.equal(2) + store.$gauge.history.map((h) => h.value).should.eql([1]) + }) + + it('clones the message it is given, so a caller reusing it cannot mutate stored state', function () { + const { store } = makeStore() + const msg = { payload: 1, nested: { a: 1 } } + store[MERGE_ENTRY]('k', msg) + + should(store.$k.msg).not.equal(msg) + store.$k.msg.nested.should.not.equal(msg.nested) + Object.isFrozen(msg).should.be.false() + Object.isFrozen(msg.nested).should.be.false() + msg.nested.a = 'MUTATED' + store.$k.msg.nested.a.should.equal(1) + }) + + it('serves the stored value as the payload of the stored msg', function () { + const { store } = makeStore() + store[MERGE_ENTRY]('k', { payload: { n: 1 }, topic: 't' }) + + store.$k.msg.payload.should.equal(store.$k.value) + store.k.n.should.equal(1) + }) + }) + + describe('series entries', function () { + const pt = (x, y, category = 'a') => ({ category, x, y }) + + it('clones every message it is given, so a caller reusing them cannot mutate stored state', function () { + const { store } = makeStore() + const msgs = [{ payload: 1, _datapoint: pt(1, 1), meta: { n: 1 } }] + store[SET_SERIES]('c1', msgs) + + store.$c1.msg[0].should.not.equal(msgs[0]) + store.$c1.msg[0].meta.should.not.equal(msgs[0].meta) + Object.isFrozen(msgs[0]).should.be.false() + Object.isFrozen(msgs[0].meta).should.be.false() + msgs[0].meta.n = 'MUTATED' + store.$c1.msg[0].meta.n.should.equal(1) + }) + + it('clones the appended message too', function () { + const { store } = makeStore() + const msg = { payload: 1, _datapoint: pt(1, 1), meta: { n: 1 } } + store[APPEND_ENTRY]('c1', msg) + + store.$c1.msg[0].should.not.equal(msg) + store.$c1.msg[0].meta.should.not.equal(msg.meta) + Object.isFrozen(msg).should.be.false() + Object.isFrozen(msg.meta).should.be.false() + msg.meta.n = 'MUTATED' + store.$c1.msg[0].meta.n.should.equal(1) + }) + + it('does not let the stored series messages be replaced through $', function () { + const { store } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(1, 1) }) + + const push = function () { 'use strict'; store.$c1.msg.push({ payload: 'INJECTED' }) } + const truncate = function () { 'use strict'; store.$c1.msg.length = 0 } + + push.should.throw() + truncate.should.throw() + store.$c1.msg.should.have.length(1) + }) + + it('emits once for the whole series, however many points', function () { + const { store, log } = makeStore() + store[SET_SERIES]('c1', [ + { payload: 1, _datapoint: pt(1, 1) }, + { payload: 2, _datapoint: [pt(2, 2), pt(3, 3)] } + ]) + + store.c1.should.eql([pt(1, 1), pt(2, 2), pt(3, 3)]) + store.$c1.msg.should.have.length(2) + log.should.eql(['c1']) + }) + + it('replaces the previous series rather than appending to it', function () { + const { store } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(9, 9) }) + store[SET_SERIES]('c1', [{ payload: 2, _datapoint: pt(2, 2) }]) + + store.c1.should.eql([pt(2, 2)]) + store.$c1.msg.should.have.length(1) + }) + + it('empties the series when given no messages', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(1, 1) }) + log.length = 0 + store[SET_SERIES]('c1', []) + + store.c1.should.eql([]) + store.$c1.msg.should.eql([]) + log.should.eql(['c1']) + }) + + it('keeps a message that carries no datapoint, contributing no points', function () { + const { store } = makeStore() + store[SET_SERIES]('c1', [ + { payload: 'none' }, + { payload: 1, _datapoint: pt(1, 1) }, + { payload: 'null', _datapoint: null }, + { payload: 'empty', _datapoint: [] } + ]) + + store.c1.should.eql([pt(1, 1)]) + store.$c1.msg.should.have.length(4) + }) + + it('points each message at its own slice of the live series', function () { + const { store, log } = makeStore() + store[SET_SERIES]('c1', [ + { payload: 1, _datapoint: pt(1, 1) }, + { payload: 2, _datapoint: [pt(2, 2), pt(3, 3)] } + ]) + log.length = 0 + + store.c1[0].y = 11 + store.c1[2].y = 33 + + store.$c1.msg[0]._datapoint.y.should.equal(11) + store.$c1.msg[1]._datapoint[1].y.should.equal(33) + log.should.eql(['c1.0.y', 'c1.2.y']) + }) + + it('reports a write made through a series message datapoint, since it is the live point', function () { + const { store, log } = makeStore() + store[SET_SERIES]('c1', [ + { payload: 1, _datapoint: pt(1, 1) }, + { payload: 2, _datapoint: [pt(2, 2), pt(3, 3)] } + ]) + log.length = 0 + + store.$c1.msg[0]._datapoint.y = 77 + store.$c1.msg[1]._datapoint[0].y = 88 + + log.should.eql(['c1.0.y', 'c1.1.y']) + store.c1[0].y.should.equal(77) + store.c1[1].y.should.equal(88) + }) + + it('reports a write made through an appended message datapoint', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(1, 1) }) + store[APPEND_ENTRY]('c1', { payload: 2, _datapoint: [pt(2, 2), pt(3, 3)] }) + log.length = 0 + + store.$c1.msg[0]._datapoint.y = 77 + store.$c1.msg[1]._datapoint[1].y = 88 + + log.should.eql(['c1.0.y', 'c1.2.y']) + store.c1[0].y.should.equal(77) + store.c1[2].y.should.equal(88) + }) + + it('re-seeds the series when a flow has left another shape at the key', function () { + for (const seed of [42, 'text', { not: 'a series' }, [1, 2]]) { + const { store } = makeStore() + store.c1 = seed + + const append = function () { store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(1, 1) }) } + + append.should.not.throw() + store.c1.should.eql([pt(1, 1)]) + store.$c1.msg.should.have.length(1) + } + }) + + it('keeps appending after a re-seed, rather than failing once and forever', function () { + const { store } = makeStore() + store.c1 = 42 + store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(1, 1) }) + store[APPEND_ENTRY]('c1', { payload: 2, _datapoint: pt(2, 2) }) + + store.c1.should.eql([pt(1, 1), pt(2, 2)]) + store.$c1.msg.should.have.length(2) + }) + + it('reports discarding whatever a flow left at the key', function () { + const { store, log } = makeStore() + store.c1 = { not: 'a series' } + log.length = 0 + + store[APPEND_ENTRY]('c1', { payload: 1 }) + + store.c1.should.eql([]) + log.should.eql(['c1']) + }) + + it('does not freeze a point a message reaches through another key', function () { + for (const write of [ + (store, msg) => store[SET_SERIES]('c1', [msg]), + (store, msg) => store[APPEND_ENTRY]('c1', msg) + ]) { + const { store } = makeStore({ clone: util.cloneMessage }) + const payload = { reading: { v: 1 } } + write(store, { topic: 't', payload, _datapoint: { category: 'a', x: 1, y: payload.reading } }) + + Object.isFrozen(store.c1[0].y).should.be.false() + store.c1[0].y.v = 99 + store.c1[0].y.v.should.equal(99) + } + }) + + it('freezes the stored series message itself', function () { + const { store } = makeStore() + store[APPEND_ENTRY]('c1', { topic: 't', payload: 1, _datapoint: pt(1, 1) }) + + Object.isFrozen(store.$c1.msg[0]).should.be.true() + const write = function () { 'use strict'; store.$c1.msg[0].topic = 'TAMPERED' } + + write.should.throw() + store.$c1.msg[0].topic.should.equal('t') + }) + + it('keeps an empty or null datapoint on the stored message', function () { + for (const datapoint of [null, []]) { + const { store } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 'x', _datapoint: datapoint }) + store[SET_SERIES]('c2', [{ payload: 'x', _datapoint: datapoint }]) + + store.$c1.msg[0].should.have.property('_datapoint') + store.$c2.msg[0].should.have.property('_datapoint') + should(store.$c1.msg[0]._datapoint).eql(datapoint) + should(store.$c2.msg[0]._datapoint).eql(datapoint) + } + }) + + it('does not let a multi-point datapoint be resized through $', function () { + const { store } = makeStore() + store[SET_SERIES]('c1', [{ payload: 1, _datapoint: [pt(1, 1), pt(2, 2)] }]) + + const push = function () { 'use strict'; store.$c1.msg[0]._datapoint.push(pt(9, 9)) } + + push.should.throw() + store.$c1.msg[0]._datapoint.should.have.length(2) + store.c1.should.have.length(2) + }) + + it('freezes everything on a series message except its datapoint', function () { + const { store } = makeStore() + store[SET_SERIES]('c1', [{ payload: 1, _datapoint: pt(1, 1), meta: { n: 1 } }]) + + const write = function () { 'use strict'; store.$c1.msg[0].meta.n = 'TAMPERED' } + + write.should.throw() + store.$c1.msg[0].meta.n.should.equal(1) + }) + }) + + describe('append entry', function () { + const pt = (x, y, category = 'a') => ({ category, x, y }) + + it('creates a series on first append', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(1, 1) }) + + store.c1.should.eql([pt(1, 1)]) + store.$c1.msg.should.have.length(1) + store.$c1.msg[0].payload.should.equal(1) + log.should.eql(['c1.0']) + }) + + it('pushes onto an existing series', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(1, 1) }) + log.length = 0 + store[APPEND_ENTRY]('c1', { payload: 2, _datapoint: pt(2, 2) }) + + store.c1.should.have.length(2) + store.c1[1].x.should.equal(2) + store.$c1.msg.should.have.length(2) + log.should.eql(['c1.1']) + }) + + it('pushes every point of a multi-series message, but the message once', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', { payload: [1, 5], _datapoint: [pt(1, 1, 'a'), pt(1, 5, 'b')] }) + + store.c1.should.eql([pt(1, 1, 'a'), pt(1, 5, 'b')]) + store.$c1.msg.should.have.length(1) + log.should.eql(['c1.0', 'c1.1']) + }) + + it('keeps the message but fires nothing when there are no points', function () { + const { store, log } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 'no datapoint' }) + + store.c1.should.eql([]) + store.$c1.msg.should.have.length(1) + log.should.eql([]) + }) + + it('keeps history empty, because the value is an array', function () { + const { store } = makeStore() + store[APPEND_ENTRY]('c1', { payload: 1, _datapoint: pt(1, 1) }) + store[APPEND_ENTRY]('c1', { payload: 2, _datapoint: pt(2, 2) }) + store[APPEND_ENTRY]('c1', { payload: 3, _datapoint: pt(3, 3) }) + + store.$c1.history.should.eql([]) + }) + }) + + describe('history', function () { + it('keeps the prior value on reassignment', function () { + const { store } = makeStore() + store.obj = { n: 1 } + store.obj = { n: 2 } + store.obj.should.eql({ n: 2 }) + store.$obj.history.map((h) => h.value).should.eql([{ n: 1 }]) + }) + + it('snapshots history so later mutation cannot corrupt it', function () { + const { store } = makeStore() + store.obj = { n: 1 } + store.obj = { n: 2 } + store.obj.n = 999 + store.$obj.history.map((h) => h.value).should.eql([{ n: 1 }]) + }) + + it('evicts beyond maxHistory', function () { + const { store } = makeStore({ maxHistory: 2 }) + for (let i = 0; i < 5; i++) store.h = i + store.$h.history.map((h) => h.value).should.eql([2, 3]) + store.h.should.equal(4) + }) + + it('keeps no history for array-valued keys', function () { + const { store } = makeStore() + store.series = [1, 2] + store.series = [3, 4, 5] + store.$series.history.should.eql([]) + }) + + it('does not snapshot the old array when it is replaced (even by a scalar)', function () { + const { store } = makeStore() + store.k = [1, 2, 3] + store.k = 9 + store.$k.history.should.eql([]) + }) + + it('does snapshot a scalar when it becomes an array', function () { + const { store } = makeStore() + store.k = 5 + store.k = [1, 2] + store.$k.history.map((h) => h.value).should.eql([5]) + }) + }) + + describe('deep reactivity', function () { + it('fires on a nested object mutation', function () { + const { store, log } = makeStore() + store.robot = { temp: 20 } + log.length = 0 + store.robot.temp = 25 + log.should.eql(['robot.temp']) + store.robot.temp.should.equal(25) + }) + + it('fires on array push, deep item mutation, and clear', function () { + const { store, log } = makeStore() + store.series = [] + log.length = 0 + store.series.push({ x: 1, y: 2 }) + log.should.eql(['series.0']) + log.length = 0 + store.series[0].y = 99 + log.should.eql(['series.0.y']) + log.length = 0 + store.series.length = 0 + log.should.eql(['series']) + store.series.should.eql([]) + }) + + it('fires for an array at a nested path the same as one at a top-level key', function () { + const { store, log } = makeStore() + store.top = [1, 2] + store.nested = { rows: [1, 2] } + + log.length = 0 + store.top.push(3) + store.nested.rows.push(3) + + log.should.eql(['top.2', 'nested.rows.2']) + }) + + it('does not alias an array assigned into the store', function () { + const { store } = makeStore() + const original = [{ x: 1 }] + store.series = original + + original[0].x = 999 + + store.series.should.eql([{ x: 1 }]) + }) + }) + + describe('delete', function () { + it('fires onChange and removes the value', function () { + const { store, log } = makeStore() + store.gone = 1 + log.length = 0 + delete store.gone + log.should.eql(['gone']) + should(store.gone).be.undefined() + }) + }) + + describe('guards', function () { + it('blocks prototype pollution and creates no bad keys', function () { + const { store } = makeStore() + store.__proto__ = { polluted: true } // eslint-disable-line no-proto + store.constructor = { polluted: true } + should({}.polluted).be.undefined() + Reflect.ownKeys(store).filter((k) => k === '__proto__' || k === 'constructor').should.eql([]) + }) + + it('does not report a write to a value that has been replaced', function () { + const { store, log } = makeStore() + store.k = { n: 1 } + const detached = store.k + store.k = { n: 2 } + log.length = 0 + const stamp = store.$k.timestamp + + detached.n = 999 + + log.should.eql([]) + detached.n.should.equal(999) + store.k.n.should.equal(2) + store.$k.timestamp.should.equal(stamp) + }) + + it('does not report a write to a value whose key was deleted', function () { + const { store, log } = makeStore() + store.k = { n: 1 } + const held = store.k + delete store.k + log.length = 0 + + held.n = 99 + + log.should.eql([]) + held.n.should.equal(99) + should(store.k).be.undefined() + }) + + it('does not report a write to a child of a replaced value', function () { + const { store, log } = makeStore() + store.k = { child: { n: 1 } } + const detachedChild = store.k.child + store.k = { child: { n: 2 } } + log.length = 0 + + detachedChild.n = 999 + + log.should.eql([]) + detachedChild.n.should.equal(999) + store.k.child.n.should.equal(2) + }) + + it('still reports writes to the current value after a replacement', function () { + const { store, log } = makeStore() + store.k = { n: 1 } + store.k = { n: 2 } + log.length = 0 + + store.k.n = 3 + + log.should.eql(['k.n']) + store.k.n.should.equal(3) + }) + + it('stores a self-referencing object without blowing the stack', function () { + const { store } = makeStore() + const a = { n: 1 } + a.self = a + + const write = function () { store.k = a } + + write.should.not.throw() + store.k.n.should.equal(1) + store.k.self.should.equal(store.k) + }) + + it('stores a mutually-referencing pair', function () { + const { store } = makeStore() + const a = { name: 'a' } + const b = { name: 'b', a } + a.b = b + + const write = function () { store.k = a } + + write.should.not.throw() + store.k.b.name.should.equal('b') + store.k.b.a.should.equal(store.k) + }) + + it('keeps a cyclic value reactive', function () { + const { store, log } = makeStore() + const a = { n: 1 } + a.self = a + store.k = a + log.length = 0 + + store.k.self.n = 2 + + log.should.eql(['k.n']) + store.k.n.should.equal(2) + }) + + it('does not let a subscriber error escape a write', function () { + const { store } = makeStore({ onChange: () => { throw new Error('boom') } }) + + const write = function () { store.k = 1 } + + write.should.not.throw() + store.k.should.equal(1) + }) + + it('does not let a subscriber error escape a nested write', function () { + let live = false + const { store } = makeStore({ onChange: () => { if (live) throw new Error('boom') } }) + store.k = { n: 1 } + live = true + + const write = function () { store.k.n = 2 } + + write.should.not.throw() + store.k.n.should.equal(2) + }) + + it('does not let a subscriber error escape a delete', function () { + let live = false + const { store } = makeStore({ onChange: () => { if (live) throw new Error('boom') } }) + store.k = 1 + live = true + + const remove = function () { delete store.k } + + remove.should.not.throw() + should(store.k).be.undefined() + }) + + it('does not let a write through $ replace the stored value', function () { + const { store, log } = makeStore() + store.k = 1 + log.length = 0 + + store.$k.value = 'TAMPERED' + + store.k.should.equal(1) + log.should.eql([]) + }) + + it('does not let history grow past the cap through $', function () { + const { store } = makeStore({ maxHistory: 2 }) + store.k = 1 + store.k = 2 + store.k = 3 + + const push = function () { store.$k.history.push({ value: 'x', timestamp: 0 }) } + + push.should.throw() + store.$k.history.should.have.length(2) + }) + + it('does not let a stored history entry be rewritten through $', function () { + const { store } = makeStore() + store.k = 1 + store.k = 2 + + const write = function () { 'use strict'; store.$k.history[0].value = 'x' } + + write.should.throw() + store.$k.history[0].value.should.equal(1) + }) + + it('does not let the stored msg be rewritten through $', function () { + const { store, log } = makeStore() + store[MERGE_ENTRY]('k', { payload: 42, topic: 'boiler' }) + log.length = 0 + + const write = function () { 'use strict'; store.$k.msg.payload = 'TAMPERED' } + + write.should.throw() + store.$k.msg.payload.should.equal(42) + log.should.be.empty() + }) + + it('does not let a non-payload property of the stored msg be rewritten through $', function () { + const { store } = makeStore() + store[MERGE_ENTRY]('k', { payload: 1, meta: { deep: { n: 1 } } }) + + const write = function () { 'use strict'; store.$k.msg.meta.deep.n = 'TAMPERED' } + + write.should.throw() + store.$k.msg.meta.deep.n.should.equal(1) + }) + + it('reports a nested write made through the $ msg payload, since it is the value', function () { + const { store, log } = makeStore() + store[MERGE_ENTRY]('k', { payload: { deep: { n: 1 } } }) + log.length = 0 + + store.$k.msg.payload.deep.n = 2 + + log.should.eql(['k.deep.n']) + store.k.deep.n.should.equal(2) + }) + + it('does not let a nested property of a history entry be rewritten through $', function () { + const { store } = makeStore() + store.k = { n: 1 } + store.k = { n: 2 } + + const write = function () { 'use strict'; store.$k.history[0].value.n = 'TAMPERED' } + + write.should.throw() + store.$k.history[0].value.n.should.equal(1) + }) + + it('does not freeze the payload when the message references it under another key', function () { + // a change node copies by reference unless deep copy is ticked, so msg.copy === msg.payload + const { store } = makeStore({ clone: util.cloneMessage }) + const payload = { temp: 25 } + store[MERGE_ENTRY]('k', { payload, copy: payload, topic: 't' }) + + Object.isFrozen(store.k).should.be.false() + store.k.temp = 26 + store.k.temp.should.equal(26) + Object.isFrozen(store.$k.msg.copy).should.be.true() + }) + + it('leaves msg.req and msg.res usable, since cloneMessage keeps them by reference', function () { + // msg.res is an object literal in Node-RED (createResponseWrapper), not a class instance, + // so isReactable accepts it and nothing but an explicit exemption keeps it writable + const paths = [ + (store, msg) => store[MERGE_ENTRY]('k', msg), + (store, msg) => store[SET_SERIES]('k', [msg]), + (store, msg) => store[APPEND_ENTRY]('k', msg) + ] + for (const write of paths) { + const { store } = makeStore({ clone: util.cloneMessage }) + const req = new http.IncomingMessage({ fake: 'socket' }) + const res = { _res: new http.ServerResponse(req), set () {}, status () {} } + const msg = { payload: 1, req, res, _datapoint: { category: 'a', x: 1, y: 1 } } + write(store, msg) + + store.$k.msg.should.be.ok() + Object.isFrozen(req).should.be.false() + Object.isFrozen(res).should.be.false() + res.statusCode = 404 + res.statusCode.should.equal(404) + } + }) + + it('still reports a nested write made through the $ value', function () { + const { store, log } = makeStore() + store.k = { n: 1 } + log.length = 0 + + store.$k.value.n = 2 + + log.should.eql(['k.n']) + store.k.n.should.equal(2) + }) + + it('passes a symbol delete straight through, like get and set do', function () { + const { store, log } = makeStore() + store.k = 1 + log.length = 0 + + delete store[STORE] + + log.should.be.empty() + }) + + it('reserves toJSON so a key can never shadow the serialiser', function () { + const { store, log } = makeStore() + store.real = 1 + log.length = 0 + + store.toJSON = 5 + + log.should.eql([]) + should(store.$toJSON).be.undefined() + Object.keys(store).should.eql(['real']) + JSON.stringify(store).should.equal('{"real":1}') + }) + + it('reserves the $ prefix so keys are never silently lost', function () { + const { store } = makeStore() + store.$weird = 1 + should(store.$weird).be.undefined() + store.normal = 7 + store.normal.should.equal(7) + store.$normal.value.should.equal(7) + }) + }) + + describe('clone on write', function () { + it('does not alias the caller object into the store', function () { + const { store } = makeStore() + const original = { inner: { x: 1 } } + store.obj = original + original.inner.x = 999 + store.obj.inner.x.should.equal(1) + }) + + it('does not alias a nested assignment into the store', function () { + const { store } = makeStore() + store.robot = { sensors: {} } + const original = { x: 1 } + store.robot.sensors = original + original.x = 999 + store.robot.sensors.x.should.equal(1) + }) + + it('does not wrap the caller object in place on a nested assignment', function () { + const { store } = makeStore() + store.robot = { sensors: {} } + const original = { inner: { x: 1 } } + const innerRef = original.inner + store.robot.sensors = original + should(original.inner).equal(innerRef) + }) + }) + + describe('timestamps', function () { + it('uses Date.now by default', function () { + const store = createDataStore({}) + store.k = 1 + store.$k.timestamp.should.be.a.Number().and.above(0) + }) + + it('advances on each write', function () { + const { store } = makeStore() + store.k = 1 + const first = store.$k.timestamp + store.k = 2 + store.$k.timestamp.should.be.above(first) + }) + }) + + describe('serialise and restart', function () { + it('flattens entries to values, including nested', function () { + const { store } = makeStore() + store.a = 10 + store.robot = { temp: 20, axes: [1, 2] } + JSON.parse(JSON.stringify(store)).should.eql({ a: 10, robot: { temp: 20, axes: [1, 2] } }) + }) + + it('rebuilds reactivity around a restored snapshot', function () { + const first = makeStore() + first.store.a = 10 + first.store.robot = { temp: 20, axes: [1, 2] } + const snapshot = JSON.parse(JSON.stringify(first.store)) + + const { store, log } = makeStore() + for (const [k, v] of Object.entries(snapshot)) store[k] = v + store.a.should.equal(10) + store.robot.should.eql({ temp: 20, axes: [1, 2] }) + log.length = 0 + store.robot.axes.push(3) + log.should.eql(['robot.axes.2']) + }) + }) + + describe('via Node-RED global context', function () { + function makeContext () { + const log = [] + const ctx = Memory({}) + ctx.set('global', 'dashboardStore', createDataStore({ onChange: (k, e, p) => log.push(p) })) + return { ctx, log } + } + + it('observes a change-node style path write', function () { + const { ctx, log } = makeContext() + ctx.set('global', 'dashboardStore.tag1', 123) + log.should.eql(['tag1']) + ctx.get('global', 'dashboardStore.tag1').should.equal(123) + }) + + it('observes a function-node get-then-mutate', function () { + const { ctx, log } = makeContext() + ctx.set('global', 'dashboardStore.robot', { temp: 20 }) + log.length = 0 + ctx.get('global', 'dashboardStore.robot').temp = 25 + log.should.eql(['robot.temp']) + }) + + it('creates a brand-new nested path from a change-node write', function () { + const { ctx, log } = makeContext() + ctx.set('global', 'dashboardStore.newObj.sub', 5) + ctx.get('global', 'dashboardStore.newObj').should.eql({ sub: 5 }) + log.should.containEql('newObj.sub') + }) + + it('works through the async callback API', function () { + const { ctx, log } = makeContext() + let cbVal + ctx.set('global', 'dashboardStore.cbk', 42, () => {}) + ctx.get('global', 'dashboardStore.cbk', (err, v) => { cbVal = err ? 'ERR' : v }) + cbVal.should.equal(42) + log.length = 0 + ctx.set('global', 'dashboardStore.cbk', 43, () => {}) + log.should.eql(['cbk']) + }) + + it('a raw overwrite drops reactivity until Dashboard re-injects (see attachToContext)', function () { + const { ctx, log } = makeContext() + ctx.set('global', 'dashboardStore.k', 1) + ctx.set('global', 'dashboardStore', { k: 999 }) + log.length = 0 + ctx.set('global', 'dashboardStore.k', 2) + log.should.eql([]) + ctx.get('global', 'dashboardStore.k').should.equal(2) + }) + }) + + describe('with RED.util.cloneMessage as the cloner', function () { + function makeCloneStore () { + const log = [] + const store = createDataStore({ clone: util.cloneMessage, onChange: (k, e, p) => log.push(p) }) + return { store, log } + } + + it('deep-watches an object built in a function node realm', function () { + const { store, log } = makeCloneStore() + store.w = fnNodeObject('({ nested: { temp: 20 } })') + log.length = 0 + + store.w.nested.temp = 25 + + log.should.eql(['w.nested.temp']) + store.w.nested.temp.should.equal(25) + }) + + it('reassigning an object value fires only the key, no phantom req/res', function () { + const { store, log } = makeCloneStore() + store.robot = { temp: 20 } + log.length = 0 + store.robot = { temp: 21 } + log.should.eql(['robot']) + }) + + it('keeps a Buffer value intact and unproxied', function () { + const { store } = makeCloneStore() + store.buf = Buffer.from([1, 2, 3]) + Buffer.isBuffer(store.buf).should.equal(true) + store.buf.should.eql(Buffer.from([1, 2, 3])) + }) + + it('keeps a Date value intact', function () { + const { store } = makeCloneStore() + const d = new Date('2020-01-01T00:00:00Z') + store.d = d + store.d.should.be.instanceof(Date) + store.d.getTime().should.equal(d.getTime()) + }) + }) + + describe('strict-mode guards', function () { + it('do not throw when a strict caller writes a reserved key', function () { + const { store } = makeStore() + const write = function () { 'use strict'; store.$x = 1; store.__proto__ = {} } // eslint-disable-line no-proto + write.should.not.throw() + should(store.$x).be.undefined() + }) + }) + + describe('attachToContext', function () { + it('does not copy an array a flow put in its place into numeric keys', function () { + const m = {} + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + attachToContext(g, {}) + m.dashboardStore = ['a', 'b'] + + const store = attachToContext(g, {}) + + Object.keys(store).should.eql([]) + JSON.stringify(store).should.equal('{}') + }) + + it('does not spread a class instance a flow put in its place into keys', function () { + const m = {} + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + attachToContext(g, {}) + m.dashboardStore = new Date() + + const store = attachToContext(g, {}) + + Object.keys(store).should.eql([]) + }) + + it('copies in an object a function node put in its place', function () { + const m = {} + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + attachToContext(g, {}) + m.dashboardStore = fnNodeObject('({ robot: "PLAIN", count: 3 })') + + const store = attachToContext(g, {}) + + store.robot.should.equal('PLAIN') + store.count.should.equal(3) + }) + + it('copies in a plain object a flow put in its place', function () { + const m = {} + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + attachToContext(g, {}) + m.dashboardStore = { robot: { temp: 20 }, count: 3 } + + const store = attachToContext(g, {}) + + store.count.should.equal(3) + store.robot.temp.should.equal(20) + store.$robot.timestamp.should.be.a.Number() + }) + + it('recognises a store injected by another copy of the module', function () { + const m = {} + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + // what a second copy of this module would have put in context + const foreign = { robot: 'from the other copy' } + foreign[Symbol.for('@flowfuse/node-red-dashboard/store')] = true + m.dashboardStore = foreign + + const store = attachToContext(g, {}) + + store.should.equal(foreign) + }) + + function fakeGlobal () { + const m = {} + return { get: (k) => m[k], set: (k, v) => { m[k] = v } } + } + + it('injects the store under the dashboard namespace', function () { + const g = fakeGlobal() + const store = attachToContext(g, { onChange: () => {} }) + should(g.get('dashboardStore')).equal(store) + store.a = 1 + g.get('dashboardStore').a.should.equal(1) + }) + + it('is idempotent, reusing the store across calls', function () { + const g = fakeGlobal() + const first = attachToContext(g) + const second = attachToContext(g) + should(second).equal(first) + }) + + it('rewires reactivity around plain data a flow put in its place', function () { + const g = fakeGlobal() + attachToContext(g) + g.set('dashboardStore', { a: 5, robot: { temp: 1 } }) + const log = [] + const store = attachToContext(g, { onChange: (k, e, p) => log.push(p) }) + store.a.should.equal(5) + store.robot.should.eql({ temp: 1 }) + store.a = 6 + log.should.containEql('a') + }) + + it('does not report a replacement when the context only ever hands back copies', function () { + const m = {} + let copying = false + const g = { get: (k) => copying ? JSON.parse(JSON.stringify(m[k])) : m[k], set: (k, v) => { m[k] = v } } + attachToContext(g) + copying = true + + let warned = 0 + const results = [] + for (let i = 0; i < 3; i++) results.push(attachToContext(g, { onReplaced: () => { warned++ } })) + + warned.should.equal(0) + results.should.eql([null, null, null]) + }) + + it('reinjects and warns when a flow has overwritten the namespace, preserving data', function () { + const g = fakeGlobal() + const store = attachToContext(g) + store.k = 1 + g.set('dashboardStore', { k: 1, extra: 2 }) // a flow replaces the proxy with a plain object + + let warned = 0 + const restored = attachToContext(g, { onReplaced: () => { warned++ } }) + warned.should.equal(1) + restored[STORE].should.equal(true) + should(g.get('dashboardStore')).equal(restored) + restored.extra.should.equal(2) + should(attachToContext(g)).equal(restored) // idempotent again, no re-overwrite + restored.k = 5 + restored.k.should.equal(5) + }) + + it('does not warn on a clean first injection', function () { + const g = fakeGlobal() + let warned = 0 + attachToContext(g, { onReplaced: () => { warned++ } }) + warned.should.equal(0) + }) + + it('starts fresh after a restart, without warning, rather than restoring plain data it never injected', function () { + const g = fakeGlobal() + g.set('dashboardStore', { widget: 'from disk' }) // what a persistent context store hands back + let warned = 0 + const store = attachToContext(g, { onReplaced: () => { warned++ } }) + warned.should.equal(0) + should(store.widget).be.undefined() + should(g.get('dashboardStore')).equal(store) + }) + + const replacements = [ + ['an object', { mine: 1 }], + ['an array', [1, 2]], + ['a string', 'hello'], + ['a number', 42], + ['null', null], + ['undefined', undefined] + ] + replacements.forEach(([label, value]) => { + it(`warns when a flow replaces the injected store with ${label}`, function () { + const g = fakeGlobal() + attachToContext(g) + let warned = 0 + g.set('dashboardStore', value) + const restored = attachToContext(g, { onReplaced: () => { warned++ } }) + warned.should.equal(1) + restored[STORE].should.equal(true) + }) + }) + }) + + describe('context stores that cannot hold a live proxy', function () { + it('attaches to an in-memory context', function () { + const m = {} + const g = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + should(attachToContext(g, {})).not.be.null() + }) + + it('reports why the context store could not be read', function () { + let reason = null + const globalContext = { + get () { throw new Error('ECONNREFUSED: context store unreachable') }, + set () {} + } + + const store = attachToContext(globalContext, { onUnavailable: (err) => { reason = err } }) + + should(store).be.null() + reason.message.should.equal('ECONNREFUSED: context store unreachable') + }) + + it('does not report a reason when the context store reads fine', function () { + let reason = null + const m = {} + const globalContext = { get: (k) => m[k], set: (k, v) => { m[k] = v } } + + attachToContext(globalContext, { onUnavailable: (err) => { reason = err } }) + + should(reason).be.null() + }) + + it('returns null when a cache-off store throws on synchronous get', function () { + const g = { + get: () => { throw new Error('File Store cache disabled - only asynchronous access supported') }, + set: () => {} + } + should(attachToContext(g, {})).be.null() + }) + + it('returns null for a store that hands back a serialised copy', function () { + const disk = {} + const g = { + get: (k) => disk[k] === undefined ? undefined : JSON.parse(disk[k]), + set: (k, v) => { disk[k] = JSON.stringify(v) } + } + should(attachToContext(g, {})).be.null() + }) + + it('never reports a replacement for a store it refused to attach to', function () { + const disk = {} + const g = { + get: (k) => disk[k] === undefined ? undefined : JSON.parse(disk[k]), + set: (k, v) => { disk[k] = JSON.stringify(v) } + } + let warned = 0 + + for (let deploy = 0; deploy < 3; deploy++) attachToContext(g, { onReplaced: () => { warned++ } }) + + warned.should.equal(0) + }) + + // a real localfilesystem store, accessed the way the context manager's sync path does + function realGlobal (store) { + return { get: (key) => store.get('global', key), set: (key, value) => store.set('global', key, value) } + } + function tmpDir () { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ds-ctx-')) + after(() => fs.rmSync(dir, { recursive: true, force: true })) + return dir + } + + it('returns null for a real cache-off localfilesystem store', function () { + const store = LocalFileSystem({ dir: tmpDir(), cache: false }) + should(attachToContext(realGlobal(store), {})).be.null() + }) + + it('attaches to a real cache-backed localfilesystem store', function () { + const store = LocalFileSystem({ dir: tmpDir(), cache: true }) + should(attachToContext(realGlobal(store), {})).not.be.null() + }) + }) +}) + +describe('store: MERGE_ENTRY', function () { + const merging = () => makeStore({ clone: util.cloneMessage }) + + it('merges a message onto the stored one, as the legacy datastore does', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: 1, topic: 'a', label: 'x' }) + + store[MERGE_ENTRY]('k', { topic: 'b' }) + + store.$k.msg.should.eql({ payload: 1, topic: 'b', label: 'x' }) + }) + + it('keeps the value and adds no history for a message without a payload', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: { n: 1 } }) + const value = store.k + + store[MERGE_ENTRY]('k', { topic: 'b' }) + + store.k.should.equal(value) + store.$k.history.should.eql([]) + }) + + it('adds no history for a value that no message ever set', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { topic: 'a' }) + store[MERGE_ENTRY]('u', { payload: undefined }) + + store[MERGE_ENTRY]('k', { payload: 1 }) + store[MERGE_ENTRY]('u', { payload: 1 }) + + store.$k.history.should.eql([]) + store.$u.history.map((h) => h.value).should.eql([undefined]) + }) + + it('records history for every message that carries a payload, even a repeated one', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: true }) + store[MERGE_ENTRY]('k', { payload: true }) + store[MERGE_ENTRY]('k', { payload: false }) + + store.$k.history.map((h) => h.value).should.eql([true, true]) + }) + + it('does not add a payload key that no message had', function () { + const { store } = merging() + + store[MERGE_ENTRY]('k', { topic: 't' }) + + Object.keys(store.$k.msg).should.eql(['topic']) + }) + + it('merges onto a value a flow wrote directly', function () { + const { store } = merging() + store.k = 5 + + store[MERGE_ENTRY]('k', { topic: 't' }) + + store.$k.msg.should.eql({ payload: 5, topic: 't' }) + }) + + it('does not alias or freeze the message it is given', function () { + const { store } = merging() + const msg = { payload: { n: 1 }, nested: { a: 1 } } + + store[MERGE_ENTRY]('k', msg) + msg.nested.a = 2 + + store.$k.msg.nested.a.should.equal(1) + Object.isFrozen(msg.nested).should.equal(false) + }) + + it('updates the timestamp when it keeps the value', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: 1 }) + const before = store.$k.timestamp + + store[MERGE_ENTRY]('k', { topic: 'b' }) + + store.$k.timestamp.should.be.above(before) + }) + + it('does not keep a series as the value when a message without a payload is merged onto it', function () { + const { store } = merging() + store[APPEND_ENTRY]('k', { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + + store[MERGE_ENTRY]('k', { topic: 't' }) + + should(store.k).be.undefined() + store.$k.msg.topic.should.equal('t') + }) + + it('keeps a replaced series\' messages as legacy does, without leaving their points writable', function () { + const { store, log } = merging() + store[APPEND_ENTRY]('k', { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }) + + store[MERGE_ENTRY]('k', { payload: 'single' }) + log.length = 0 + try { store.$k.msg[0]._datapoint.y = 777 } catch (err) {} + + store.$k.msg.should.eql({ 0: { payload: 1, _datapoint: { category: 'a', x: 1, y: 1 } }, payload: 'single' }) + log.should.eql([]) + }) + + it('applies pending trims to a series before merging onto it, reporting once', function () { + const { store, log } = merging() + for (let x = 0; x < 5; x++) { + store[APPEND_ENTRY]('k', { payload: x, _datapoint: { category: 'a', x, y: x } }) + } + store[TRIM_SERIES]('k', (msgs) => msgs.filter((m) => m._datapoint.x > 2)) + log.length = 0 + + store[MERGE_ENTRY]('k', { topic: 'later' }) + + Object.keys(store.$k.msg).should.eql(['0', '1', 'topic']) + store.$k.msg[0].payload.should.equal(3) + log.should.eql(['k']) + }) + + it('reuses stored fields rather than cloning them again on every merge', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { payload: 'a', options: [{ label: 'one', value: 1 }] }) + const options = store.$k.msg.options + + store[MERGE_ENTRY]('k', { payload: 'b' }) + + store.$k.msg.options.should.equal(options) + Object.isFrozen(store.$k.msg.options).should.equal(true) + }) + + it('keeps stored fields in their original order, with payload last', function () { + const { store } = merging() + store[MERGE_ENTRY]('k', { topic: 'a', payload: 1, label: 'x' }) + + store[MERGE_ENTRY]('k', { topic: 'b', extra: true }) + + Object.keys(store.$k.msg).should.eql(['topic', 'label', 'extra', 'payload']) + }) + + it('reports each merge once', function () { + const { store, log } = merging() + store[MERGE_ENTRY]('k', { payload: 1 }) + log.length = 0 + + store[MERGE_ENTRY]('k', { topic: 'b' }) + + log.should.eql(['k']) + }) +}) + +describe('store: ENTRY_MSG', function () { + const point = (x) => ({ payload: x, topic: 'a', _datapoint: { category: 'a', x, y: x } }) + + it('reads a stored message without building a new object', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + store[MERGE_ENTRY]('k', { payload: 1, topic: 't' }) + store[MERGE_ENTRY]('k', { payload: 2 }) + + store[ENTRY_MSG]('k').should.equal(store[ENTRY_MSG]('k')) + store[ENTRY_MSG]('k').should.eql({ payload: 2, topic: 't' }) + }) + + it('reads a series with pending trims applied, as a frozen copy', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + for (let x = 0; x < 5; x++) { + store[APPEND_ENTRY]('c', point(x)) + } + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > 2)) + + const msgs = store[ENTRY_MSG]('c') + + msgs.map((m) => m.payload).should.eql([3, 4]) + Object.isFrozen(msgs).should.equal(true) + store[ENTRY_MSG]('c').should.not.equal(msgs) + }) + + it('matches the $ view', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + store[APPEND_ENTRY]('c', point(1)) + store[TRIM_SERIES]('c', (msgs) => msgs) + store[MERGE_ENTRY]('k', { payload: 1 }) + + store[ENTRY_MSG]('c').should.eql(store.$c.msg) + store[ENTRY_MSG]('k').should.equal(store.$k.msg) + }) + + it('reads nothing for a missing entry', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + + should(store[ENTRY_MSG]('missing')).be.undefined() + }) +}) + +describe('store: TRIM_SERIES', function () { + const point = (x, topic = 'a') => ({ payload: x, topic, _datapoint: { category: topic, x, y: x } }) + const keepLatest = (max) => (msgs) => msgs.filter((m, i) => msgs.slice(i + 1).filter((n) => n.topic === m.topic).length < max) + + function series (n, topicOf = () => 'a', opts = {}) { + const made = makeStore({ clone: util.cloneMessage, ...opts }) + for (let i = 0; i < n; i++) { + made.store[APPEND_ENTRY]('c', point(i, topicOf(i))) + } + return made + } + const payloads = (store) => store.$c.msg.map((m) => m.payload) + + it('leaves the stored series alone for 59 trims and rebuilds it on the 60th', function () { + const { store } = series(100) + for (let i = 0; i < 59; i++) { + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > i)) + } + store.c.should.have.length(100) + + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > 59)) + + store.c.should.have.length(40) + payloads(store).should.eql(Array.from({ length: 40 }, (_, i) => i + 60)) + }) + + it('counts only trims that remove something towards the batch', function () { + const { store } = series(100) + for (let i = 0; i < 59; i++) { + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > i)) + store[TRIM_SERIES]('c', (msgs) => msgs) + } + + store.c.should.have.length(100) + }) + + it('shows pending trims in the $ view before the series is rebuilt', function () { + const { store } = series(10) + store[TRIM_SERIES]('c', keepLatest(3)) + + store.c.should.have.length(10) + payloads(store).should.eql([7, 8, 9]) + }) + + it('matches trimming after every message when points arrive out of order', function () { + const { store } = series(0) + store[SET_SERIES]('c', []) + for (const [x, cutoff] of [[100, 0], [1, 10], [50, 20]]) { + store[APPEND_ENTRY]('c', point(x)) + store[TRIM_SERIES]('c', keepLatest(2)) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m._datapoint.x > cutoff)) + } + + payloads(store).should.eql([100, 50]) + }) + + it('applies every trim, not just the latest', function () { + const { store } = series(5) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 1)) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 3)) + + payloads(store).should.eql([0, 2, 4]) + }) + + it('gives a position-based trim the positions a per-message trim would see', function () { + const { store } = series(5) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m, i) => i > 0)) + store[APPEND_ENTRY]('c', point(5)) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m, i) => i > 0)) + + payloads(store).should.eql([2, 3, 4, 5]) + }) + + it('shows points appended after the last trim', function () { + const { store } = series(5) + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload !== 0)) + store[APPEND_ENTRY]('c', point(5)) + + payloads(store).should.eql([1, 2, 3, 4, 5]) + }) + + it('does not rebuild or report when trims remove nothing', function () { + const { store, log } = series(10) + log.length = 0 + + for (let i = 0; i < 60; i++) { + store[TRIM_SERIES]('c', (msgs) => msgs) + } + + log.should.eql([]) + payloads(store).should.eql([0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) + }) + + it('drops pending trims when the series is replaced', function () { + const { store } = series(10) + store[TRIM_SERIES]('c', keepLatest(1)) + store[SET_SERIES]('c', [point(1), point(2)]) + + store[TRIM_SERIES]('c', keepLatest(1)) + + payloads(store).should.eql([2]) + }) + + it('drops pending trims when the entry is deleted', function () { + const { store } = series(10) + store[TRIM_SERIES]('c', keepLatest(1)) + + delete store.c + store[APPEND_ENTRY]('c', point(1)) + store[APPEND_ENTRY]('c', point(2)) + + payloads(store).should.eql([1, 2]) + }) + + it('ignores trims on an entry that is not a series', function () { + const { store } = makeStore({ clone: util.cloneMessage }) + store.k = 5 + + store[TRIM_SERIES]('k', () => []) + store[TRIM_SERIES]('missing', () => []) + + store.k.should.equal(5) + }) + + it('does not clone on trims that do not rebuild', function () { + let clones = 0 + const { store } = series(50, (i) => (i % 2 ? 'a' : 'b'), { clone: (v) => { clones++; return util.cloneMessage(v) } }) + clones = 0 + + for (let i = 0; i < 59; i++) { + store[TRIM_SERIES]('c', keepLatest(10)) + } + + clones.should.equal(0) + }) + + it('reports the rebuild once', function () { + const { store, log } = series(100) + log.length = 0 + + for (let i = 0; i < 60; i++) { + store[TRIM_SERIES]('c', (msgs) => msgs.filter((m) => m.payload > i)) + } + + log.should.eql(['c']) + }) +})