diff --git a/src/stores/activity.ts b/src/stores/activity.ts index eae1adecb..681845320 100644 --- a/src/stores/activity.ts +++ b/src/stores/activity.ts @@ -22,6 +22,12 @@ import { useBucketsStore } from '~/stores/buckets'; import { useCategoryStore } from '~/stores/categories'; import { getClient } from '~/util/awclient'; +import { PeriodCache } from '~/util/periodCache'; + +const categoryPeriodCaches = new WeakMap>(); +const categoryRequests = new WeakMap(); +// Key of the request that produced the current category.by_period. +const categoryHistoryKeys = new WeakMap(); import { FullDesktopQueryResult, mergeFullDesktopResults, @@ -95,6 +101,7 @@ export interface QueryOptions { filter_afk?: boolean; include_audible?: boolean; include_stopwatch?: boolean; + include_category_history?: boolean; filter_categories?: string[][]; dont_query_inactive?: boolean; force?: boolean; @@ -337,8 +344,19 @@ export const useActivityStore = defineStore('activity', { } // Perform this last, as it takes the longest - if (this.window.available || this.android.available) { + const canQueryCategories = this.window.available || this.android.available; + if (canQueryCategories && query_options.include_category_history !== false) { await this.query_category_time_by_period(query_options); + } else if ( + canQueryCategories && + (query_options.force || + categoryHistoryKeys.get(this) !== this.category_history_request(query_options).key) + ) { + // This load skipped category history, but the retained periods were + // computed for different inputs (range, filters, category rules or + // buckets), so they no longer describe the current query and must not + // be served to views like Report that read them without reloading. + this.query_category_time_by_period_completed({ by_period: null }); } } else { console.warn( @@ -405,6 +423,8 @@ export const useActivityStore = defineStore('activity', { }, async reset() { + categoryRequests.delete(this); + categoryPeriodCaches.delete(this); getClient().abort(); this.query_window_completed({}); this.query_browser_completed({}); @@ -512,16 +532,18 @@ export const useActivityStore = defineStore('activity', { this.query_active_history_completed({ active_history }); }, - async query_category_time_by_period({ + // The periods and query that category history is built from. The key + // identifies everything the result depends on, including the current + // category rules and resolved bucket IDs baked into the query. + category_history_request({ timeperiod, filter_categories, filter_afk, include_stopwatch, - dontQueryInactive, always_active_pattern, - }: QueryOptions & { dontQueryInactive: boolean }) { + }: QueryOptions): { periods: string[]; query: string[]; key: string } { // TODO: Needs to be adapted for Android - let periods: string[]; + let periods: string[] = []; const count = timeperiod.length[0]; const res = timeperiod.length[1]; if (res.startsWith('day') && count == 1) { @@ -544,76 +566,74 @@ export const useActivityStore = defineStore('activity', { // Filter out periods that start in the future periods = periods.filter(period => new Date(period.split('/')[0]) < new Date()); - const signal = getClient().controller.signal; - let cancelled = false; - signal.onabort = () => { - cancelled = true; - console.debug('Request aborted'); - }; + // Prefer ScreenTime bucket over Android watcher for consistency with query_android + const iosBucketForCategory = this.buckets.android.find((id: string) => + id.startsWith('aw-import-screentime') + ); + const iosOrAndroidBucket = iosBucketForCategory || this.buckets.android[0]; + const isAndroid = iosOrAndroidBucket !== undefined; + // ScreenTime (iOS) buckets carry a "title" key; aw-watcher-android buckets do not. + // Pass isIos so canonicalEvents uses the correct merge keys and titles are preserved. + const isIosForCategory = !!iosBucketForCategory; + const categories = useCategoryStore().classes_for_query; + // TODO: Clean up call, pass QueryParams in fullDesktopQuery as well + // TODO: Unify QueryOptions and QueryParams + const query = queries.categoryQuery({ + bid_browsers: this.buckets.browser, + bid_stopwatch: + include_stopwatch && this.buckets.stopwatch.length > 0 + ? this.buckets.stopwatch[0] + : undefined, + categories, + filter_categories, + filter_afk, + always_active_pattern, + ...(isAndroid + ? { + bid_android: iosOrAndroidBucket, + isIos: isIosForCategory, + } + : { + bid_afk: this.buckets.afk[0], + bid_window: this.buckets.window[0], + }), + }); + return { periods, query, key: JSON.stringify([query, periods]) }; + }, - // Query one period at a time, to avoid timeout on slow queries - let data = []; - for (const period of periods) { - // Not stable - //signal.throwIfAborted(); - if (cancelled) { - throw signal['reason'] || 'unknown reason'; - } + async query_category_time_by_period(query_options: QueryOptions) { + const { periods, query, key } = this.category_history_request(query_options); - // Only query periods with known data from AFK bucket - if (dontQueryInactive && this.active.events.length > 0) { - const start = new Date(period.split('/')[0]); - const end = new Date(period.split('/')[1]); + const request = {}; + categoryRequests.set(this, request); + const signal = getClient().controller.signal; + let cache = categoryPeriodCaches.get(this); + if (!cache) { + cache = new PeriodCache(); + categoryPeriodCaches.set(this, cache); + } + if (query_options.force) cache.clear(); - // Retrieve active time in period - const period_activity = this.active.events.find((e: IEvent) => { - return start < new Date(e.timestamp) && new Date(e.timestamp) < end; + const queryKey = JSON.stringify(query); + const data = []; + // Retain sequential requests to avoid long server queries/timeouts. + for (const period of periods) { + if (signal.aborted || categoryRequests.get(this) !== request) return; + const periodKey = JSON.stringify([queryKey, period]); + const periodClosed = new Date(period.split('/')[1]).getTime() <= Date.now(); + let result = periodClosed ? cache.get(periodKey) : undefined; + if (result === undefined) { + const revision = cache.version; + const response = await getClient().query([period], query, { + cache: false, // This bounded cache owns freshness and invalidation. + name: 'categoryQuery', }); - - // Check if there was active time - if (!(period_activity && period_activity.duration > 0)) { - data = data.concat([{ cat_events: [] }]); - continue; - } + if (signal.aborted || categoryRequests.get(this) !== request) return; + result = response[0]; + if (periodClosed && result !== undefined && revision === cache.version) + cache.set(periodKey, result); } - - // Prefer ScreenTime bucket over Android watcher for consistency with query_android - const iosBucketForCategory = this.buckets.android.find((id: string) => - id.startsWith('aw-import-screentime') - ); - const iosOrAndroidBucket = iosBucketForCategory || this.buckets.android[0]; - const isAndroid = iosOrAndroidBucket !== undefined; - // ScreenTime (iOS) buckets carry a "title" key; aw-watcher-android buckets do not. - // Pass isIos so canonicalEvents uses the correct merge keys and titles are preserved. - const isIosForCategory = !!iosBucketForCategory; - const categories = useCategoryStore().classes_for_query; - // TODO: Clean up call, pass QueryParams in fullDesktopQuery as well - // TODO: Unify QueryOptions and QueryParams - const query = queries.categoryQuery({ - bid_browsers: this.buckets.browser, - bid_stopwatch: - include_stopwatch && this.buckets.stopwatch.length > 0 - ? this.buckets.stopwatch[0] - : undefined, - categories, - filter_categories, - filter_afk, - always_active_pattern, - ...(isAndroid - ? { - bid_android: iosOrAndroidBucket, - isIos: isIosForCategory, - } - : { - bid_afk: this.buckets.afk[0], - bid_window: this.buckets.window[0], - }), - }); - const result = await getClient().query([period], query, { - verbose: true, - name: 'categoryQuery', - }); - data = data.concat(result); + data.push(result); } // Zip periods @@ -621,7 +641,7 @@ export const useActivityStore = defineStore('activity', { // Filter out values that are undefined (no longer needed, only used when visualization was progressive (looks buggy)) by_period = _.fromPairs(_.toPairs(by_period).filter(o => o[1])); - this.query_category_time_by_period_completed({ by_period }); + this.query_category_time_by_period_completed({ by_period, key }); }, async query_active_history_android({ timeperiod }: QueryOptions) { @@ -763,6 +783,7 @@ export const useActivityStore = defineStore('activity', { // mutations start_loading(this: State, query_options: QueryOptions) { + categoryRequests.delete(this); this.loaded = true; this.query_options = query_options; @@ -781,7 +802,13 @@ export const useActivityStore = defineStore('activity', { this.editor.top_projects = null; this.category.top = null; - this.category.by_period = null; + // When this load refreshes category history, clear it up front like the + // rest of the state. When it skips it, ensure_loaded decides after buckets + // resolve whether the retained periods still match the current query. + if (query_options.include_category_history !== false) { + this.category.by_period = null; + categoryHistoryKeys.delete(this); + } this.active.duration = null; @@ -841,8 +868,13 @@ export const useActivityStore = defineStore('activity', { }; }, - query_category_time_by_period_completed(this: State, { by_period } = { by_period: [] }) { + query_category_time_by_period_completed( + this: State, + { by_period, key }: { by_period: any; key?: string } = { by_period: [] } + ) { this.category.by_period = by_period; + if (key) categoryHistoryKeys.set(this, key); + else categoryHistoryKeys.delete(this); }, }, }); diff --git a/src/util/awclient.ts b/src/util/awclient.ts index 666534dfa..7eb6c313b 100644 --- a/src/util/awclient.ts +++ b/src/util/awclient.ts @@ -1,4 +1,5 @@ import { AWClient } from 'aw-client'; +import { invalidatePeriodCaches } from './periodCache'; import type { AxiosInstance } from 'axios'; import { useSettingsStore } from '~/stores/settings'; @@ -107,6 +108,15 @@ export function createClient(force?: boolean): AWClient { testing: !production, baseURL, }); + invalidatePeriodCaches(); + _client.req.interceptors.response.use(response => { + const method = (response.config.method || 'get').toLowerCase(); + const path = (response.config.url || '').split('?')[0]; + if (!['get', 'head', 'options'].includes(method) && !/\/query\/?$/.test(path)) { + invalidatePeriodCaches(); + } + return response; + }); applyApiToken(_client, loadApiTokenFromBrowser()); } else { throw 'Tried to instantiate global AWClient twice!'; diff --git a/src/util/classes.ts b/src/util/classes.ts index 94c72aca0..a4a29539e 100644 --- a/src/util/classes.ts +++ b/src/util/classes.ts @@ -462,43 +462,65 @@ function pickHighestRanked(categories: Category[]) { return _.maxBy(categories, categoryRank); } +interface CompiledRule { + source: string; + ignoreCase: boolean; + rawKeys: string[] | undefined; + keys: string[] | undefined; + regex: RegExp; +} + +// Rule objects can be edited in place by the category editor. Compare the +// compilation inputs rather than relying only on array/object identity. +const compiledRules = new WeakMap(); +function compileRule(rule: Rule): CompiledRule { + const cached = compiledRules.get(rule); + const rawKeys = rule.select_keys; + if ( + cached && + cached.source === rule.regex && + cached.ignoreCase === !!rule.ignore_case && + cached.rawKeys?.length === rawKeys?.length && + (rawKeys || []).every((key, i) => key === cached.rawKeys[i]) + ) { + return cached; + } + const compiled = { + source: rule.regex, + ignoreCase: !!rule.ignore_case, + rawKeys: rawKeys?.slice(), + keys: normalizeSelectKeys(rawKeys), + regex: new RegExp(rule.regex, (rule.ignore_case ? 'i' : '') + 'm'), + }; + compiledRules.set(rule, compiled); + return compiled; +} + export function matchString( str: string, categories: Category[] | null, event?: IEvent ): Category | null { - if (!categories) { - console.log( - 'Categories not passed, loading... (if you see this outside of a test, you should probably pass them)' - ); - categories = loadClasses(); - } - - // Compile regexes - const regexes: [Category, RegExp][] = categories - .filter(c => c.rule.type == 'regex') - .map(c => { - // using 'm' flag to make `$` and `^` in rules work - const re = RegExp(c.rule.regex, (c.rule.ignore_case ? 'i' : '') + 'm'); - return [c, re]; - }); - - // Find the matching category. - // If several categories match, explicit priority wins; otherwise depth wins. - const matchingCats: [Category, RegExp][] = regexes.filter(([category, re]) => { - const selectKeys = normalizeSelectKeys(category.rule.select_keys); - if (event && selectKeys) { - return selectKeys.some(key => { - const value = event.data[key]; - return typeof value === 'string' && re.test(value); - }); + categories = categories || loadClasses(); + let best: Category | null = null; + let bestRank = -Infinity; + for (const category of categories) { + if (category.rule.type !== 'regex') continue; + const { regex, keys } = compileRule(category.rule); + const matches = + event && keys + ? keys.some(key => typeof event.data[key] === 'string' && regex.test(event.data[key])) + : regex.test(str); + if (matches) { + const rank = categoryRank(category); + // Strictly greater preserves the original first-match tie breaking. + if (rank > bestRank) { + best = category; + bestRank = rank; + } } - return re.test(str); - }); - if (matchingCats.length > 0) { - return pickHighestRanked(matchingCats.map(c => c[0])); } - return null; + return best; } // this is used only in tests diff --git a/src/util/color.ts b/src/util/color.ts index 431f0f4ef..7465f120d 100644 --- a/src/util/color.ts +++ b/src/util/color.ts @@ -2,7 +2,7 @@ import _ from 'lodash'; import { Category, matchString, loadClasses } from './classes'; import Color from 'color'; import * as d3 from 'd3'; -import { IEvent, IBucket } from './interfaces'; +import { IEvent } from './interfaces'; // See here for examples: // https://bl.ocks.org/pstuffa/3393ff2711a53975040077b7453781a9 @@ -140,7 +140,10 @@ export function getTitleAttr(bucket: { type?: string }, e: IEvent) { } } -export function getCategorizationStringFromEvent(bucket: IBucket, e: IEvent): string | null { +export function getCategorizationStringFromEvent( + bucket: { type?: string }, + e: IEvent +): string | null { if (bucket.type == 'currentwindow') { // using linebreak and "m" regex flag to make `$` and `^` work return e.data.app + '\n' + e.data.title; @@ -156,7 +159,7 @@ export function getCategorizationStringFromEvent(bucket: IBucket, e: IEvent): st return null; } -export function getCategoryColorFromEvent(bucket: IBucket, e: IEvent) { +export function getCategoryColorFromEvent(bucket: { type?: string }, e: IEvent) { const categorizationString = getCategorizationStringFromEvent(bucket, e); if (categorizationString !== null) { const allCats = loadClasses(); diff --git a/src/util/datasets.ts b/src/util/datasets.ts index d47b1ab71..cedf6835c 100644 --- a/src/util/datasets.ts +++ b/src/util/datasets.ts @@ -1,10 +1,7 @@ -import _ from 'lodash'; - import { split_by_hour_into_data } from '~/util/transforms'; import { getColorFromCategory } from '~/util/color'; import { Category } from '~/util/classes'; import { IEvent } from './interfaces'; -import { useCategoryStore } from '~/stores/categories'; interface HourlyData { cat_events: IEvent[]; @@ -16,45 +13,35 @@ interface Dataset { data: number[]; } -export function buildBarchartDataset(data_by_hour: HourlyData[], classes: Category[]): Dataset[] { - const SEP = '>>>'; - const data = data_by_hour; - if (data) { - const category_names: Set = new Set( - Object.values(data) - .map(result => { - return result.cat_events.map(e => e.data['$category'].join(SEP)); - }) - .flat() - ); - const ds: Dataset[] = [...category_names] - .map(c_ => { - const categoryStore = useCategoryStore(); - const c = categoryStore.get_category(c_.split(SEP)); - - if (c) { - const values = Object.values(data).map(results => { - const cat = results.cat_events.find(e => _.isEqual(e.data['$category'], c.name)); - if (cat) return Math.round((cat.duration / (60 * 60)) * 1000) / 1000; - else return null; - }); - return { - label: c.name.join(' > '), - backgroundColor: getColorFromCategory(c, classes), - data: values, - } as Dataset; - } else { - // FIXME: This shouldn't happen - // This may for example happen if one doesn't have an 'Uncategorized' category, - // as can happen when one upgrades from an old version where there wasn't one in the default classes. - console.error('missing category:', c_); - } - }) - .filter(x => x); - return ds; - } else { - return []; +export function buildBarchartDataset( + data_by_hour: HourlyData[] | Record, + classes: Category[] +): Dataset[] { + if (!data_by_hour) return []; + const periods = Object.values(data_by_hour); + const categoryPaths = new Map(); + const totals = periods.map(period => { + const sums = new Map(); + for (const event of period.cat_events) { + const path = event.data.$category || []; + const key = JSON.stringify(path); + if (!categoryPaths.has(key)) categoryPaths.set(key, path); + sums.set(key, (sums.get(key) || 0) + event.duration); + } + return sums; + }); + const categories = new Map(); + for (const category of classes) { + const key = JSON.stringify(category.name); + if (!categories.has(key)) categories.set(key, category); } + return Array.from(categoryPaths, ([key, path]) => ({ + label: path.length ? path.join(' > ') : 'Uncategorized', + backgroundColor: getColorFromCategory(categories.get(key), classes), + data: totals.map(sums => + sums.has(key) ? Math.round((sums.get(key) / 3600) * 1000) / 1000 : null + ), + })); } export function buildBarchartDatasetActive(events_active: IEvent[]) { diff --git a/src/util/graphData.ts b/src/util/graphData.ts new file mode 100644 index 000000000..274cc1c3b --- /dev/null +++ b/src/util/graphData.ts @@ -0,0 +1,38 @@ +import { IEvent } from './interfaces'; + +export function buildGraphData( + events: IEvent[], + maxDepth: number, + colorForCategory: (path: string[]) => string +) { + const categories = new Map(); + const transitions = new Map(); + let previous: string | undefined; + for (const event of events) { + const path = event.data.$category.slice(0, maxDepth); + const id = JSON.stringify(path); + const category = categories.get(id); + if (category) category.duration += event.duration; + else categories.set(id, { path, duration: event.duration }); + + if (previous !== undefined && previous !== id) { + const key = JSON.stringify([previous, id]); + const link = transitions.get(key); + if (link) link.value++; + else transitions.set(key, { source: previous, target: id, value: 1 }); + } + previous = id; + } + const groups = new Map([['Uncategorized', 0]]); + const nodes = Array.from(categories, ([id, category]) => { + const root = category.path[0] || ''; + if (!groups.has(root)) groups.set(root, groups.size); + return { + id, + group: groups.get(root), + color: colorForCategory(category.path), + value: category.duration, + }; + }); + return { nodes, links: Array.from(transitions.values()) }; +} diff --git a/src/util/interfaces.ts b/src/util/interfaces.ts index bde00fd3b..65febc742 100644 --- a/src/util/interfaces.ts +++ b/src/util/interfaces.ts @@ -6,6 +6,7 @@ export interface IEvent { export interface IBucket { id: string; + client?: string; hostname: string; device_id: string; type: string; diff --git a/src/util/periodCache.ts b/src/util/periodCache.ts new file mode 100644 index 000000000..275998a90 --- /dev/null +++ b/src/util/periodCache.ts @@ -0,0 +1,38 @@ +// Shared invalidation for writes made through this UI. A short expiry also +// bounds staleness from external importers/watchers that cannot notify us. +let revision = 0; +export function invalidatePeriodCaches(): void { + revision++; +} + +export class PeriodCache { + private entries = new Map(); + private revision = revision; + constructor(private limit = 256, private ttl = 60_000) {} + + get version(): number { + return revision; + } + + clear(): void { + this.entries.clear(); + this.revision = revision; + } + + get(key: string, now = Date.now()): T | undefined { + if (this.revision !== revision) this.clear(); + const entry = this.entries.get(key); + if (!entry || entry.expires <= now) { + this.entries.delete(key); + return undefined; + } + return entry.value; + } + + set(key: string, value: T, now = Date.now()): void { + if (this.revision !== revision) this.clear(); + this.entries.delete(key); + this.entries.set(key, { value, expires: now + this.ttl }); + while (this.entries.size > this.limit) this.entries.delete(this.entries.keys().next().value); + } +} diff --git a/src/util/timelineIndex.ts b/src/util/timelineIndex.ts new file mode 100644 index 000000000..5797f3415 --- /dev/null +++ b/src/util/timelineIndex.ts @@ -0,0 +1,64 @@ +import { IEvent } from './interfaces'; + +export interface IndexedEvent { + id: string; + bucket: { id: string; type?: string }; + event: IEvent & { id?: number }; + start: number; + end: number; +} + +export function indexTimelineEvents(buckets, filterShort = true) { + const entries: IndexedEvent[] = []; + for (const bucket of buckets) { + (bucket.events || []).forEach((event, index) => { + if (filterShort && event.duration <= 1) return; + const start = new Date(event.timestamp).getTime(); + const end = start + event.duration * 1000; + if (!Number.isFinite(start) || !Number.isFinite(end)) return; + entries.push({ + id: JSON.stringify([bucket.id, event.id ?? [event.timestamp, index]]), + bucket, + event, + start, + end, + }); + }); + } + entries.sort((a, b) => a.start - b.start); + let maxEnd = -Infinity; + const ends = entries.map(e => (maxEnd = Math.max(maxEnd, e.end))); + return { entries, ends, groups: new Set(entries.map(item => item.bucket.id)) }; +} + +// Prefix maximum ends keep events that begin before the viewport but overlap it. +export function visibleTimelineEvents( + index: ReturnType, + start: number, + end: number +): IndexedEvent[] { + let low = 0; + let high = index.ends.length; + while (low < high) { + const mid = (low + high) >>> 1; + if (index.ends[mid] < start) low = mid + 1; + else high = mid; + } + const result: IndexedEvent[] = []; + for (let i = low; i < index.entries.length && index.entries[i].start <= end; i++) { + if (index.entries[i].end >= start) result.push(index.entries[i]); + } + return result; +} + +// DataSet updates notify vis-timeline. Avoid notifications for unchanged items. +export function syncTimelineData(dataset, next: Record[]): void { + const ids = new Set(next.map(item => item.id)); + const removed = dataset.getIds().filter(id => !ids.has(id)); + if (removed.length) dataset.remove(removed); + const changed = next.filter(item => { + const previous = dataset.get(item.id); + return !previous || Object.keys(item).some(key => item[key] !== previous[key]); + }); + if (changed.length) dataset.update(changed); +} diff --git a/src/views/Graph.vue b/src/views/Graph.vue index b57723ce7..9d5ccd4a0 100644 --- a/src/views/Graph.vue +++ b/src/views/Graph.vue @@ -61,6 +61,7 @@ div