From 872cfae78754601e5268faa9372bd77640a76c93 Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Mon, 14 Sep 2026 15:00:48 +0200 Subject: [PATCH 01/13] feat(identities): sync and clean the events collection on identity webhooks - bump @data-fair/lib-express to 1.26.0 (secret in the x-secret-key header) - move the webhook effects to identities/service.ts - rename: sender and originator names (departmentName included) are rewritten on events with their search texts, notifications follow the sender name like subscriptions do - delete: the events of the identity's own feed are removed, the events a deleted user triggered on other feeds only keep the user id (pseudonymized, no name nor email left), pending webhooks and the notifications pointer of the identity are removed too - sparse indexes on originator.user.id and originator.organization.id --- api/package.json | 2 +- api/src/identities/router.ts | 57 +------------- api/src/identities/service.ts | 113 +++++++++++++++++++++++++++ api/src/mongo.ts | 5 +- package-lock.json | 11 +-- tests/identities-webhook.api.spec.ts | 61 +++++++++++++++ 6 files changed, 185 insertions(+), 64 deletions(-) create mode 100644 api/src/identities/service.ts diff --git a/api/package.json b/api/package.json index 3c15fa8..bed32b2 100644 --- a/api/package.json +++ b/api/package.json @@ -15,7 +15,7 @@ }, "dependencies": { "@data-fair/lib-common-types": "^1.20.2", - "@data-fair/lib-express": "^1.22.5", + "@data-fair/lib-express": "^1.26.0", "@data-fair/lib-node": "^2.12.1", "@data-fair/lib-utils": "^1.14.0", "@data-fair/lib-validation": "^1.0.2", diff --git a/api/src/identities/router.ts b/api/src/identities/router.ts index d2d0f03..03dbb8b 100644 --- a/api/src/identities/router.ts +++ b/api/src/identities/router.ts @@ -3,59 +3,6 @@ import config from '#config' import { createIdentitiesRouter } from '@data-fair/lib-express/identities/index.js' -import mongo from '#mongo' +import { updateIdentity, deleteIdentity } from './service.ts' -export default createIdentitiesRouter( - config.secretKeys.identities, - // onUpdate - async (identity) => { - if (identity.type === 'user') { - await mongo.notifications.updateMany({ 'recipient.id': identity.id }, { $set: { 'recipient.name': identity.name } }) - await mongo.subscriptions.updateMany({ 'recipient.id': identity.id }, { $set: { 'recipient.name': identity.name } }) - } - await mongo.subscriptions.updateMany({ 'sender.type': identity.type, 'sender.id': identity.id }, { $set: { 'sender.name': identity.name } }) - await mongo.pushSubscriptions.updateMany({ 'owner.type': identity.type, 'owner.id': identity.id }, { $set: { 'owner.name': identity.name } }) - await mongo.webhookSubscriptions.updateMany({ 'sender.type': identity.type, 'sender.id': identity.id }, { $set: { 'sender.name': identity.name } }) - await mongo.webhookSubscriptions.updateMany({ 'owner.type': identity.type, 'owner.id': identity.id }, { $set: { 'owner.name': identity.name } }) - if (identity.departments) { - for (const department of identity.departments.filter(d => !!d.name)) { - await mongo.subscriptions.updateMany({ 'sender.type': identity.type, 'sender.id': identity.id, 'sender.department': department.id }, { $set: { 'sender.name': identity.name, 'sender.departmentName': department.name } }) - await mongo.pushSubscriptions.updateMany({ 'owner.type': identity.type, 'owner.id': identity.id, 'owner.department': department.id }, { $set: { 'owner.name': identity.name, 'owner.departmentName': department.name } }) - await mongo.webhookSubscriptions.updateMany({ 'sender.type': identity.type, 'sender.id': identity.id, 'sender.department': department.id }, { $set: { 'sender.name': identity.name, 'sender.departmentName': department.name } }) - await mongo.webhookSubscriptions.updateMany({ 'owner.type': identity.type, 'owner.id': identity.id, 'owner.department': department.id }, { $set: { 'owner.name': identity.name, 'owner.departmentName': department.name } }) - } - } - - if (identity.type === 'user' && identity.organizations) { - const privateSubscriptionFilter = { - 'recipient.id': identity.id, - visibility: { $ne: 'public' as const }, - 'sender.type': 'organization' - } - for await (const privateSubscription of mongo.subscriptions.find(privateSubscriptionFilter)) { - let userOrg = identity.organizations.find(o => o.id === privateSubscription.sender?.id && !o.department) - if (privateSubscription.sender?.department) { - userOrg = userOrg || identity.organizations.find(o => o.id === privateSubscription.sender?.id && o.department === privateSubscription.sender.department) - } - if (userOrg && privateSubscription.sender?.role && userOrg.role !== privateSubscription.sender.role && userOrg.role !== 'admin') { - userOrg = undefined - } - if (!userOrg) { - // console.log('remove private subscription that does not match user orgs anymore', identity, privateSubscription) - await mongo.subscriptions.deleteOne({ _id: privateSubscription._id }) - } - } - } - }, - // onDelete - async (identity) => { - if (identity.type === 'user') { - await mongo.notifications.deleteMany({ 'recipient.id': identity.id }) - await mongo.subscriptions.deleteMany({ 'recipient.id': identity.id }) - } - await mongo.subscriptions.deleteMany({ 'sender.type': identity.type, 'sender.id': identity.id }) - await mongo.pushSubscriptions.deleteMany({ 'owner.type': identity.type, 'owner.id': identity.id }) - await mongo.webhookSubscriptions.deleteMany({ 'owner.type': identity.type, 'owner.id': identity.id }) - await mongo.webhookSubscriptions.deleteMany({ 'sender.type': identity.type, 'sender.id': identity.id }) - } -) +export default createIdentitiesRouter(config.secretKeys.identities, updateIdentity, deleteIdentity) diff --git a/api/src/identities/service.ts b/api/src/identities/service.ts new file mode 100644 index 0000000..a8fe17c --- /dev/null +++ b/api/src/identities/service.ts @@ -0,0 +1,113 @@ +// Synchronize the copies of identity data (names on senders, recipients, owners, originators) +// with the users/organizations directory, and remove them when an identity is deleted. + +import type { Filter } from 'mongodb' +import type { IdentityUpdate, IdentityDelete } from '@data-fair/lib-express/identities/index.js' +import type { SearchableEvent } from '#types' +import config from '#config' +import mongo from '#mongo' +import { buildSearchTexts } from '../events/operations.ts' + +// events are rewritten one by one: the names are also part of the _search texts +const rewriteEvents = async (filter: Filter, rewrite: (event: SearchableEvent) => void) => { + for await (const event of mongo.events.find(filter)) { + rewrite(event) + const $set: Partial = { _search: buildSearchTexts(event, config.i18n.locales, config.i18n.defaultLocale) } + if (event.sender) $set.sender = event.sender + if (event.originator) $set.originator = event.originator + await mongo.events.updateOne({ _id: event._id }, { $set }) + } +} + +export const updateIdentity = async (identity: IdentityUpdate) => { + const { type, id, name, departments } = identity + + if (type === 'user') { + await mongo.notifications.updateMany({ 'recipient.id': id }, { $set: { 'recipient.name': name } }) + await mongo.subscriptions.updateMany({ 'recipient.id': id }, { $set: { 'recipient.name': name } }) + } + await mongo.notifications.updateMany({ 'sender.type': type, 'sender.id': id }, { $set: { 'sender.name': name } }) + await mongo.subscriptions.updateMany({ 'sender.type': type, 'sender.id': id }, { $set: { 'sender.name': name } }) + await mongo.pushSubscriptions.updateMany({ 'owner.type': type, 'owner.id': id }, { $set: { 'owner.name': name } }) + await mongo.webhookSubscriptions.updateMany({ 'sender.type': type, 'sender.id': id }, { $set: { 'sender.name': name } }) + await mongo.webhookSubscriptions.updateMany({ 'owner.type': type, 'owner.id': id }, { $set: { 'owner.name': name } }) + if (departments) { + for (const department of departments.filter(d => !!d.name)) { + await mongo.subscriptions.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': department.id }, { $set: { 'sender.name': name, 'sender.departmentName': department.name } }) + await mongo.pushSubscriptions.updateMany({ 'owner.type': type, 'owner.id': id, 'owner.department': department.id }, { $set: { 'owner.name': name, 'owner.departmentName': department.name } }) + await mongo.webhookSubscriptions.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': department.id }, { $set: { 'sender.name': name, 'sender.departmentName': department.name } }) + await mongo.webhookSubscriptions.updateMany({ 'owner.type': type, 'owner.id': id, 'owner.department': department.id }, { $set: { 'owner.name': name, 'owner.departmentName': department.name } }) + } + } + + // events: as sender, and as the user or organization that triggered them + const eventsFilter: Filter[] = [{ 'sender.type': type, 'sender.id': id }] + if (type === 'user') eventsFilter.push({ 'originator.user.id': id }) + else eventsFilter.push({ 'originator.organization.id': id }) + await rewriteEvents({ $or: eventsFilter }, (event) => { + if (event.sender?.type === type && event.sender.id === id) { + event.sender.name = name + const department = event.sender.department && departments?.find(d => d.id === event.sender?.department) + if (department) event.sender.departmentName = department.name + } + if (type === 'user' && event.originator?.user?.id === id) { + event.originator.user.name = name + } + if (type === 'organization' && event.originator?.organization?.id === id) { + event.originator.organization.name = name + const department = event.originator.organization.department && departments?.find(d => d.id === event.originator?.organization?.department) + if (department) event.originator.organization.departmentName = department.name + } + }) + + if (type === 'user' && identity.organizations) { + const privateSubscriptionFilter = { + 'recipient.id': id, + visibility: { $ne: 'public' as const }, + 'sender.type': 'organization' + } + for await (const privateSubscription of mongo.subscriptions.find(privateSubscriptionFilter)) { + let userOrg = identity.organizations.find(o => o.id === privateSubscription.sender?.id && !o.department) + if (privateSubscription.sender?.department) { + userOrg = userOrg || identity.organizations.find(o => o.id === privateSubscription.sender?.id && o.department === privateSubscription.sender.department) + } + if (userOrg && privateSubscription.sender?.role && userOrg.role !== privateSubscription.sender.role && userOrg.role !== 'admin') { + userOrg = undefined + } + if (!userOrg) { + // remove private subscription that does not match user orgs anymore + await mongo.subscriptions.deleteOne({ _id: privateSubscription._id }) + } + } + } +} + +export const deleteIdentity = async (identity: IdentityDelete) => { + const { type, id } = identity + + if (type === 'user') { + await mongo.notifications.deleteMany({ 'recipient.id': id }) + await mongo.subscriptions.deleteMany({ 'recipient.id': id }) + await mongo.pointers.deleteMany({ 'recipient.id': id }) + } + await mongo.subscriptions.deleteMany({ 'sender.type': type, 'sender.id': id }) + await mongo.pushSubscriptions.deleteMany({ 'owner.type': type, 'owner.id': id }) + await mongo.webhookSubscriptions.deleteMany({ 'owner.type': type, 'owner.id': id }) + await mongo.webhookSubscriptions.deleteMany({ 'sender.type': type, 'sender.id': id }) + // pending or failed webhooks of the deleted subscriptions + await mongo.webhooks.deleteMany({ 'owner.type': type, 'owner.id': id }) + await mongo.webhooks.deleteMany({ 'sender.type': type, 'sender.id': id }) + + // the events of the identity are its own feed, nobody else can read them + await mongo.events.deleteMany({ 'sender.type': type, 'sender.id': id }) + // the events a user triggered on other feeds keep the trace of the action without the person: + // only the id remains (pseudonymized), an organization is not personal data and is left as is + if (type === 'user') { + await rewriteEvents({ 'originator.user.id': id }, (event) => { + if (event.originator?.user) { + delete event.originator.user.name + delete event.originator.user.email + } + }) + } +} diff --git a/api/src/mongo.ts b/api/src/mongo.ts index d2522cf..b8c3925 100644 --- a/api/src/mongo.ts +++ b/api/src/mongo.ts @@ -52,7 +52,10 @@ export class EventsMongo { 'main-keys': [ { 'sender.type': 1, 'sender.id': 1, '_search.text': 'text', date: -1 }, { default_language: config.i18n.defaultLocale } - ] + ], + // identity webhooks rewrite the events triggered by a user or an organization + 'originator-user': [{ 'originator.user.id': 1 }, { sparse: true }], + 'originator-organization': [{ 'originator.organization.id': 1 }, { sparse: true }] }, subscriptions: { 'main-keys': [ diff --git a/package-lock.json b/package-lock.json index acab7b3..098cc15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,7 +43,7 @@ "api": { "dependencies": { "@data-fair/lib-common-types": "^1.20.2", - "@data-fair/lib-express": "^1.22.5", + "@data-fair/lib-express": "^1.26.0", "@data-fair/lib-node": "^2.12.1", "@data-fair/lib-utils": "^1.14.0", "@data-fair/lib-validation": "^1.0.2", @@ -491,9 +491,9 @@ } }, "node_modules/@data-fair/lib-express": { - "version": "1.25.0", - "resolved": "https://registry.npmjs.org/@data-fair/lib-express/-/lib-express-1.25.0.tgz", - "integrity": "sha512-Haqd+OUDgFcduVX0K+xs0dP4+FoS1aE2LDTzdoed2wHZO/WwJ60c/e2QtdijzWH04sRarzC9ujKJQxnrBScA6A==", + "version": "1.26.0", + "resolved": "https://registry.npmjs.org/@data-fair/lib-express/-/lib-express-1.26.0.tgz", + "integrity": "sha512-sSPl09vhgaMfAIqZXVcbcCWT6T8G07R/5BBWqunVG1M9PeEWA0R2gShikQmNJM4kKE6Jqe2e0qmXaqzqcNPflg==", "license": "MIT", "dependencies": { "@data-fair/lib-common-types": "^1.7.1", @@ -1491,7 +1491,6 @@ "resolved": "https://registry.npmjs.org/@intlify/core-base/-/core-base-11.4.10.tgz", "integrity": "sha512-+yJ74JRWVJokdgG9zYNMyTSzeNV3O9T4vVxk8PvLFHmI+R/BYA//cITh7vhRK37hWLZ4/kTcKcUz1dlWOpypIg==", "license": "MIT", - "peer": true, "dependencies": { "@intlify/devtools-types": "11.4.10", "@intlify/message-compiler": "11.4.10", @@ -1509,7 +1508,6 @@ "resolved": "https://registry.npmjs.org/@intlify/devtools-types/-/devtools-types-11.4.10.tgz", "integrity": "sha512-xZxzZsAuu6/0zoLRVQWdpXWe5Kjl0LnWpjlQA3r9u9FbLYMhapqt7IwkgQyn0Tm2GUNAqhj9eZiUmYOrB024BQ==", "license": "MIT", - "peer": true, "dependencies": { "@intlify/core-base": "11.4.10", "@intlify/shared": "11.4.10" @@ -11862,7 +11860,6 @@ "resolved": "https://registry.npmjs.org/vue-i18n/-/vue-i18n-11.4.10.tgz", "integrity": "sha512-Lp+BjOxqzOY87DS6Z8KrQrpiTr9IN/Lt4kZEilwyXG2Wrx+AcU6IVsAW92HNXtVcn1HFFPV6ty41p9e/qDpyvg==", "license": "MIT", - "peer": true, "dependencies": { "@intlify/core-base": "11.4.10", "@intlify/devtools-types": "11.4.10", diff --git a/tests/identities-webhook.api.spec.ts b/tests/identities-webhook.api.spec.ts index b155bce..4e2e07d 100644 --- a/tests/identities-webhook.api.spec.ts +++ b/tests/identities-webhook.api.spec.ts @@ -7,6 +7,7 @@ const axIdentities = axios({ params: { key: 'SECRET_IDENTITIES' }, baseURL: devB const axPush = axios({ params: { key: 'SECRET_EVENTS' }, baseURL: devBaseURL }) const user1 = await axiosAuth('test-user1') const admin1 = await axiosAuth('test1-admin1') +admin1.setOrg('test1') test.describe('identities webhooks', () => { test.beforeEach(clean) @@ -74,9 +75,69 @@ test.describe('identities webhooks', () => { }) }) +const postEvents = async () => { + await axPush.post('/api/events', [{ + date: new Date().toISOString(), + topic: { key: 'topic1' }, + title: 'own feed event', + sender: { type: 'user', id: 'test-user1', name: 'Zéphyrine Dubois' }, + originator: { user: { id: 'test-user1', name: 'Zéphyrine Dubois' } } + }, { + date: new Date().toISOString(), + topic: { key: 'topic1' }, + title: 'organization feed event', + sender: { type: 'organization', id: 'test1', name: 'Test Organization 1' }, + originator: { user: { id: 'test-user1', name: 'Zéphyrine Dubois' }, organization: { id: 'test1', name: 'Test Organization 1' } } + }]) +} + +test.describe('identities update webhook on events', () => { + test.beforeEach(clean) + + test('should rename the sender and originator of events, search texts included', async () => { + await postEvents() + await axIdentities.post('/api/identities/user/test-user1', { name: 'Aurélien Lefort' }) + + const ownEvents = (await user1.get('/api/events')).data.results + expect(ownEvents).toHaveLength(1) + expect(ownEvents[0].sender.name).toBe('Aurélien Lefort') + expect(ownEvents[0].originator.user.name).toBe('Aurélien Lefort') + expect((await user1.get('/api/events?q=Lefort')).data.results).toHaveLength(1) + expect((await user1.get('/api/events?q=Dubois')).data.results).toHaveLength(0) + + let orgEvents = (await admin1.get('/api/events')).data.results + expect(orgEvents).toHaveLength(1) + expect(orgEvents[0].originator.user.name).toBe('Aurélien Lefort') + + await axIdentities.post('/api/identities/organization/test1', { name: 'Renamed Organization 1' }) + orgEvents = (await admin1.get('/api/events')).data.results + expect(orgEvents[0].sender.name).toBe('Renamed Organization 1') + expect(orgEvents[0].originator.organization.name).toBe('Renamed Organization 1') + }) +}) + test.describe('identities delete webhook', () => { test.beforeEach(clean) + test('should delete the events of a deleted user and pseudonymize the ones it triggered elsewhere', async () => { + await postEvents() + await axIdentities.delete('/api/identities/user/test-user1') + + expect((await user1.get('/api/events')).data.results).toHaveLength(0) + const orgEvents = (await admin1.get('/api/events')).data.results + expect(orgEvents).toHaveLength(1) + expect(orgEvents[0].originator.user).toEqual({ id: 'test-user1' }) + expect(orgEvents[0].originator.organization.name).toBe('Test Organization 1') + expect((await admin1.get('/api/events?q=Dubois')).data.results).toHaveLength(0) + }) + + test('should delete the events of a deleted organization', async () => { + await postEvents() + await axIdentities.delete('/api/identities/organization/test1') + expect((await admin1.get('/api/events')).data.results).toHaveLength(0) + expect((await user1.get('/api/events')).data.results).toHaveLength(1) + }) + test('should delete subscriptions and notifications of a deleted user before responding', async () => { await user1.post('/api/subscriptions', { topic: { key: 'topic1' }, From 321c6dd23e6de8b05d6659ff0ef398ff347b2438 Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Mon, 14 Sep 2026 15:08:58 +0200 Subject: [PATCH 02/13] refactor(identities): do not rewrite the sender name of notifications Notifications are snapshots taken at delivery and the collection has no index on sender: rewriting it would scan the whole collection on every user webhook. --- api/src/identities/service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/identities/service.ts b/api/src/identities/service.ts index a8fe17c..89e6b08 100644 --- a/api/src/identities/service.ts +++ b/api/src/identities/service.ts @@ -26,7 +26,7 @@ export const updateIdentity = async (identity: IdentityUpdate) => { await mongo.notifications.updateMany({ 'recipient.id': id }, { $set: { 'recipient.name': name } }) await mongo.subscriptions.updateMany({ 'recipient.id': id }, { $set: { 'recipient.name': name } }) } - await mongo.notifications.updateMany({ 'sender.type': type, 'sender.id': id }, { $set: { 'sender.name': name } }) + // notifications are snapshots taken at delivery, their sender name is not rewritten (no index on sender) await mongo.subscriptions.updateMany({ 'sender.type': type, 'sender.id': id }, { $set: { 'sender.name': name } }) await mongo.pushSubscriptions.updateMany({ 'owner.type': type, 'owner.id': id }, { $set: { 'owner.name': name } }) await mongo.webhookSubscriptions.updateMany({ 'sender.type': type, 'sender.id': id }, { $set: { 'sender.name': name } }) From c6b4f15d3a7c5702f2583b0cec0ad1debd2f0cac Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Mon, 14 Sep 2026 16:57:20 +0200 Subject: [PATCH 03/13] chore(dev): simple-directory notifies only this service, directly The shared identities router refuses the calls that come through the proxy, and the other services of the dev environment do not need the webhooks. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 316edb2..c29f10c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,8 @@ services: DEBUG: session ADMINS: '["superadmin@test.com"]' PUBLIC_URL: http://${DEV_HOST}:${NGINX_PORT}/simple-directory + # in dev simple-directory only notifies this service, directly (the shared identities router refuses calls through the proxy) + IDENTITIES_WEBHOOKS: '[{"base":"http://localhost:${DEV_API_PORT}/api/identities","key":"SECRET_IDENTITIES"}]' MAILDEV_ACTIVE: "true" STORAGE_TYPE: file ROLES_DEFAULTS: '["admin", "contrib", "user"]' From 4e74e6ed2c6cd5792c9ec0cbe0177d4118467a0f Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Tue, 15 Sep 2026 09:22:43 +0200 Subject: [PATCH 04/13] feat(identities): forget the name of a deleted department and label it in the embed A department missing from the complete list sent by simple-directory was deleted: subscriptions, push and webhook subscriptions and events keep the department id but lose departmentName. The events embed shows "Former department - " on originators. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017hqsZKwijEBoB3m3v9srVD --- api/src/identities/service.ts | 21 ++++++++++++++---- tests/identities-webhook.api.spec.ts | 29 +++++++++++++++++++++++++ ui/dts/auto-imports.d.ts | 2 ++ ui/src/composables/use-display-owner.ts | 27 +++++++++++++++++++++++ ui/src/pages/embed/events.vue | 3 ++- 5 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 ui/src/composables/use-display-owner.ts diff --git a/api/src/identities/service.ts b/api/src/identities/service.ts index 89e6b08..31f2518 100644 --- a/api/src/identities/service.ts +++ b/api/src/identities/service.ts @@ -38,6 +38,13 @@ export const updateIdentity = async (identity: IdentityUpdate) => { await mongo.webhookSubscriptions.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': department.id }, { $set: { 'sender.name': name, 'sender.departmentName': department.name } }) await mongo.webhookSubscriptions.updateMany({ 'owner.type': type, 'owner.id': id, 'owner.department': department.id }, { $set: { 'owner.name': name, 'owner.departmentName': department.name } }) } + // the directory sends the complete list of departments: a department missing from it was + // deleted, what it owns keeps the id (still reachable by the organization admins) but not the name + const deletedDepartment = { $exists: true, $nin: departments.map(d => d.id) } + await mongo.subscriptions.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': deletedDepartment }, { $unset: { 'sender.departmentName': 1 } }) + await mongo.pushSubscriptions.updateMany({ 'owner.type': type, 'owner.id': id, 'owner.department': deletedDepartment }, { $unset: { 'owner.departmentName': 1 } }) + await mongo.webhookSubscriptions.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': deletedDepartment }, { $unset: { 'sender.departmentName': 1 } }) + await mongo.webhookSubscriptions.updateMany({ 'owner.type': type, 'owner.id': id, 'owner.department': deletedDepartment }, { $unset: { 'owner.departmentName': 1 } }) } // events: as sender, and as the user or organization that triggered them @@ -47,16 +54,22 @@ export const updateIdentity = async (identity: IdentityUpdate) => { await rewriteEvents({ $or: eventsFilter }, (event) => { if (event.sender?.type === type && event.sender.id === id) { event.sender.name = name - const department = event.sender.department && departments?.find(d => d.id === event.sender?.department) - if (department) event.sender.departmentName = department.name + if (event.sender.department && departments) { + const department = departments.find(d => d.id === event.sender?.department) + if (department) event.sender.departmentName = department.name + else delete event.sender.departmentName + } } if (type === 'user' && event.originator?.user?.id === id) { event.originator.user.name = name } if (type === 'organization' && event.originator?.organization?.id === id) { event.originator.organization.name = name - const department = event.originator.organization.department && departments?.find(d => d.id === event.originator?.organization?.department) - if (department) event.originator.organization.departmentName = department.name + if (event.originator.organization.department && departments) { + const department = departments.find(d => d.id === event.originator?.organization?.department) + if (department) event.originator.organization.departmentName = department.name + else delete event.originator.organization.departmentName + } } }) diff --git a/tests/identities-webhook.api.spec.ts b/tests/identities-webhook.api.spec.ts index 4e2e07d..f9d9b4c 100644 --- a/tests/identities-webhook.api.spec.ts +++ b/tests/identities-webhook.api.spec.ts @@ -94,6 +94,35 @@ const postEvents = async () => { test.describe('identities update webhook on events', () => { test.beforeEach(clean) + test('should rename a department and forget the name of a deleted one', async () => { + const subscription = (await admin1.post('/api/subscriptions', { + topic: { key: 'topic1' }, + sender: { type: 'organization', id: 'test1', name: 'Test Organization 1', department: 'dep1', departmentName: 'Department 1' }, + visibility: 'private' + })).data + await axPush.post('/api/events', [{ + date: new Date().toISOString(), + topic: { key: 'topic1' }, + title: 'department feed event', + sender: { type: 'organization', id: 'test1', name: 'Test Organization 1', department: 'dep1', departmentName: 'Department 1' } + }]) + + await axIdentities.post('/api/identities/organization/test1', { name: 'Test Organization 1', departments: [{ id: 'dep1', name: 'Renamed Department' }] }) + let fresh = (await admin1.get('/api/subscriptions')).data.results.find((s: Subscription) => s._id === subscription._id) + expect(fresh.sender.departmentName).toBe('Renamed Department') + let events = (await admin1.get('/api/events')).data.results + expect(events[0].sender.departmentName).toBe('Renamed Department') + + // dep1 is missing from the complete list of departments: it was deleted, only its id remains + await axIdentities.post('/api/identities/organization/test1', { name: 'Test Organization 1', departments: [] }) + fresh = (await admin1.get('/api/subscriptions')).data.results.find((s: Subscription) => s._id === subscription._id) + expect(fresh.sender.department).toBe('dep1') + expect(fresh.sender.departmentName).toBeUndefined() + events = (await admin1.get('/api/events')).data.results + expect(events[0].sender.department).toBe('dep1') + expect(events[0].sender.departmentName).toBeUndefined() + }) + test('should rename the sender and originator of events, search texts included', async () => { await postEvents() await axIdentities.post('/api/identities/user/test-user1', { name: 'Aurélien Lefort' }) diff --git a/ui/dts/auto-imports.d.ts b/ui/dts/auto-imports.d.ts index 731873c..0e32d05 100644 --- a/ui/dts/auto-imports.d.ts +++ b/ui/dts/auto-imports.d.ts @@ -104,6 +104,7 @@ declare global { const useConceptFilters: typeof import('@data-fair/lib-vue/concept-filters.js').useConceptFilters const useCssModule: typeof import('vue').useCssModule const useCssVars: typeof import('vue').useCssVars + const useDisplayOwner: typeof import('../src/composables/use-display-owner').useDisplayOwner const useEditFetch: typeof import('@data-fair/lib-vue/edit-fetch.js').useEditFetch const useFetch: typeof import('@data-fair/lib-vue/fetch.js').useFetch const useI18n: typeof import('vue-i18n').useI18n @@ -223,6 +224,7 @@ declare module 'vue' { readonly useConceptFilters: UnwrapRef readonly useCssModule: UnwrapRef readonly useCssVars: UnwrapRef + readonly useDisplayOwner: UnwrapRef readonly useEditFetch: UnwrapRef readonly useFetch: UnwrapRef readonly useI18n: UnwrapRef diff --git a/ui/src/composables/use-display-owner.ts b/ui/src/composables/use-display-owner.ts new file mode 100644 index 0000000..76207cb --- /dev/null +++ b/ui/src/composables/use-display-owner.ts @@ -0,0 +1,27 @@ +// A department deleted in simple-directory keeps its id on the resources it owned, but not its name: +// the identity webhook removed it. Build the owner as it must be displayed (owner-avatar, labels). +import { useI18n } from 'vue-i18n' + +type Owner = { department?: string, departmentName?: string, [key: string]: unknown } + +export const useDisplayOwner = () => { + const { t } = useI18n({ + useScope: 'local', + messages: { + fr: { formerDepartment: 'Ancien département - {id}' }, + en: { formerDepartment: 'Former department - {id}' } + } + }) + + const departmentLabel = (department?: string, departmentName?: string) => { + if (!department) return undefined + return departmentName || t('formerDepartment', { id: department }) + } + + const displayOwner = (owner: T): T => { + if (!owner.department || owner.departmentName) return owner + return { ...owner, departmentName: departmentLabel(owner.department) } + } + + return { displayOwner, departmentLabel } +} diff --git a/ui/src/pages/embed/events.vue b/ui/src/pages/embed/events.vue index 7366e2c..f96cee7 100644 --- a/ui/src/pages/embed/events.vue +++ b/ui/src/pages/embed/events.vue @@ -76,7 +76,7 @@ /> Date: Tue, 15 Sep 2026 09:44:00 +0200 Subject: [PATCH 05/13] fix(identities): build the former department label without inline i18n messages The production build uses the runtime-only vue-i18n: inline messages given to useI18n are not compiled and the key was displayed instead of the label. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_017hqsZKwijEBoB3m3v9srVD --- ui/src/composables/use-display-owner.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ui/src/composables/use-display-owner.ts b/ui/src/composables/use-display-owner.ts index 76207cb..f6f420b 100644 --- a/ui/src/composables/use-display-owner.ts +++ b/ui/src/composables/use-display-owner.ts @@ -4,18 +4,18 @@ import { useI18n } from 'vue-i18n' type Owner = { department?: string, departmentName?: string, [key: string]: unknown } +// plain strings: the runtime-only vue-i18n build used in production cannot compile inline messages +const formerDepartment: Record = { + fr: 'Ancien département', + en: 'Former department' +} + export const useDisplayOwner = () => { - const { t } = useI18n({ - useScope: 'local', - messages: { - fr: { formerDepartment: 'Ancien département - {id}' }, - en: { formerDepartment: 'Former department - {id}' } - } - }) + const { locale } = useI18n() const departmentLabel = (department?: string, departmentName?: string) => { if (!department) return undefined - return departmentName || t('formerDepartment', { id: department }) + return departmentName || `${formerDepartment[locale.value] ?? formerDepartment.en} - ${department}` } const displayOwner = (owner: T): T => { From 56876183b20eff05a5f931b122566122238259d0 Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Tue, 15 Sep 2026 16:17:08 +0200 Subject: [PATCH 06/13] fix(identities): bulk update event names instead of rewriting every event An organization can own tens of thousands of events and simple-directory waits for the webhook response: rewriting them one by one with their search texts (and the text index behind) took more than ten minutes on staging, delaying every service notified after events. - names are no longer part of the search texts (ids, topic, title and body only), so a rename never touches _search nor the text index - rename and delete use updateMany on sender, originator.user and originator.organization (department names included, which were not propagated on the originator) --- api/src/events/operations.ts | 8 ++-- api/src/identities/service.ts | 61 ++++++++-------------------- tests/events.unit.spec.ts | 12 ++++-- tests/identities-webhook.api.spec.ts | 6 ++- 4 files changed, 34 insertions(+), 53 deletions(-) diff --git a/api/src/events/operations.ts b/api/src/events/operations.ts index 8674279..bf70961 100644 --- a/api/src/events/operations.ts +++ b/api/src/events/operations.ts @@ -77,13 +77,15 @@ export const buildSearchTexts = (event: SearchableEvent, locales: string[], defa const search: SearchableEvent['_search'] = [] for (const locale of locales) { const localizedEvent = localizeEvent(event, locale, defaultLocale) - const searchParts: (string | undefined)[] = [...event.topic.key.split(':'), event.topic.title, localizedEvent.title, localizedEvent.body, event.sender?.id, event.sender?.name] + // ids only, no name: names change and the search texts of every past event would have to be rebuilt + // (the identity webhooks only bulk update the name fields themselves) + const searchParts: (string | undefined)[] = [...event.topic.key.split(':'), event.topic.title, localizedEvent.title, localizedEvent.body, event.sender?.id] if (event.originator) { if (event.originator.organization) { - searchParts.push(event.originator.organization.name, event.originator.organization.id) + searchParts.push(event.originator.organization.id) } if (event.originator.user && (!event.originator.organization || (event.sender?.type === 'organization' && event.sender.id === event.originator.organization.id))) { - searchParts.push(event.originator.user.name, event.originator.user.id) + searchParts.push(event.originator.user.id) } } search.push({ language: locale, text: searchParts.filter(Boolean).join(' ') }) diff --git a/api/src/identities/service.ts b/api/src/identities/service.ts index 31f2518..d2f071a 100644 --- a/api/src/identities/service.ts +++ b/api/src/identities/service.ts @@ -1,23 +1,10 @@ // Synchronize the copies of identity data (names on senders, recipients, owners, originators) // with the users/organizations directory, and remove them when an identity is deleted. +// Everything is done with bulk updates: an organization can own tens of thousands of events and +// simple-directory waits for the response (names are kept out of the search texts for this reason). -import type { Filter } from 'mongodb' import type { IdentityUpdate, IdentityDelete } from '@data-fair/lib-express/identities/index.js' -import type { SearchableEvent } from '#types' -import config from '#config' import mongo from '#mongo' -import { buildSearchTexts } from '../events/operations.ts' - -// events are rewritten one by one: the names are also part of the _search texts -const rewriteEvents = async (filter: Filter, rewrite: (event: SearchableEvent) => void) => { - for await (const event of mongo.events.find(filter)) { - rewrite(event) - const $set: Partial = { _search: buildSearchTexts(event, config.i18n.locales, config.i18n.defaultLocale) } - if (event.sender) $set.sender = event.sender - if (event.originator) $set.originator = event.originator - await mongo.events.updateOne({ _id: event._id }, { $set }) - } -} export const updateIdentity = async (identity: IdentityUpdate) => { const { type, id, name, departments } = identity @@ -48,30 +35,21 @@ export const updateIdentity = async (identity: IdentityUpdate) => { } // events: as sender, and as the user or organization that triggered them - const eventsFilter: Filter[] = [{ 'sender.type': type, 'sender.id': id }] - if (type === 'user') eventsFilter.push({ 'originator.user.id': id }) - else eventsFilter.push({ 'originator.organization.id': id }) - await rewriteEvents({ $or: eventsFilter }, (event) => { - if (event.sender?.type === type && event.sender.id === id) { - event.sender.name = name - if (event.sender.department && departments) { - const department = departments.find(d => d.id === event.sender?.department) - if (department) event.sender.departmentName = department.name - else delete event.sender.departmentName - } - } - if (type === 'user' && event.originator?.user?.id === id) { - event.originator.user.name = name - } - if (type === 'organization' && event.originator?.organization?.id === id) { - event.originator.organization.name = name - if (event.originator.organization.department && departments) { - const department = departments.find(d => d.id === event.originator?.organization?.department) - if (department) event.originator.organization.departmentName = department.name - else delete event.originator.organization.departmentName - } + await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id }, { $set: { 'sender.name': name } }) + if (departments) { + for (const department of departments.filter(d => !!d.name)) { + await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': department.id }, { $set: { 'sender.departmentName': department.name } }) + await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.department': department.id }, { $set: { 'originator.organization.departmentName': department.name } }) } - }) + const deletedDepartment = { $exists: true, $nin: departments.map(d => d.id) } + await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': deletedDepartment }, { $unset: { 'sender.departmentName': 1 } }) + await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.department': deletedDepartment }, { $unset: { 'originator.organization.departmentName': 1 } }) + } + if (type === 'user') { + await mongo.events.updateMany({ 'originator.user.id': id }, { $set: { 'originator.user.name': name } }) + } else { + await mongo.events.updateMany({ 'originator.organization.id': id }, { $set: { 'originator.organization.name': name } }) + } if (type === 'user' && identity.organizations) { const privateSubscriptionFilter = { @@ -116,11 +94,6 @@ export const deleteIdentity = async (identity: IdentityDelete) => { // the events a user triggered on other feeds keep the trace of the action without the person: // only the id remains (pseudonymized), an organization is not personal data and is left as is if (type === 'user') { - await rewriteEvents({ 'originator.user.id': id }, (event) => { - if (event.originator?.user) { - delete event.originator.user.name - delete event.originator.user.email - } - }) + await mongo.events.updateMany({ 'originator.user.id': id }, { $unset: { 'originator.user.name': 1, 'originator.user.email': 1 } }) } } diff --git a/tests/events.unit.spec.ts b/tests/events.unit.spec.ts index 1a5aa12..9b11872 100644 --- a/tests/events.unit.spec.ts +++ b/tests/events.unit.spec.ts @@ -204,6 +204,8 @@ test.describe('buildSearchTexts', () => { expect(result[0].text).toContain('Titre') expect(result[0].text).toContain('Topic') expect(result[0].text).toContain('test-user1') + // names are not searchable: they change, and the search texts would have to be rebuilt on every rename + expect(result[0].text).not.toContain('User 1') expect(result[1].language).toBe('en') expect(result[1].text).toContain('Title') }) @@ -223,8 +225,10 @@ test.describe('buildSearchTexts', () => { } } const result = buildSearchTexts(event, ['fr'], 'fr') - expect(result[0].text).toContain('Org 1') - expect(result[0].text).toContain('User 1') + expect(result[0].text).toContain('org1') + expect(result[0].text).toContain('u1') + expect(result[0].text).not.toContain('Org 1') + expect(result[0].text).not.toContain('User 1') }) test('excludes user when originator is from different org', () => { @@ -242,7 +246,7 @@ test.describe('buildSearchTexts', () => { } } const result = buildSearchTexts(event, ['fr'], 'fr') - expect(result[0].text).toContain('Other Org') - expect(result[0].text).not.toContain('User 1') + expect(result[0].text).toContain('org-other') + expect(result[0].text).not.toContain('u1') }) }) diff --git a/tests/identities-webhook.api.spec.ts b/tests/identities-webhook.api.spec.ts index f9d9b4c..6485b2f 100644 --- a/tests/identities-webhook.api.spec.ts +++ b/tests/identities-webhook.api.spec.ts @@ -123,7 +123,7 @@ test.describe('identities update webhook on events', () => { expect(events[0].sender.departmentName).toBeUndefined() }) - test('should rename the sender and originator of events, search texts included', async () => { + test('should rename the sender and originator of events without touching the search texts', async () => { await postEvents() await axIdentities.post('/api/identities/user/test-user1', { name: 'Aurélien Lefort' }) @@ -131,8 +131,10 @@ test.describe('identities update webhook on events', () => { expect(ownEvents).toHaveLength(1) expect(ownEvents[0].sender.name).toBe('Aurélien Lefort') expect(ownEvents[0].originator.user.name).toBe('Aurélien Lefort') - expect((await user1.get('/api/events?q=Lefort')).data.results).toHaveLength(1) + // names are never part of the search texts: a rename is a cheap bulk update, not a rewrite of every event + expect((await user1.get('/api/events?q=Lefort')).data.results).toHaveLength(0) expect((await user1.get('/api/events?q=Dubois')).data.results).toHaveLength(0) + expect((await user1.get('/api/events?q=test-user1')).data.results).toHaveLength(1) let orgEvents = (await admin1.get('/api/events')).data.results expect(orgEvents).toHaveLength(1) From 767cfb78f9b8be873e2a3052f6c04e4d043ba87a Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Wed, 16 Sep 2026 09:32:30 +0200 Subject: [PATCH 07/13] refactor(ui): let owner-avatar label a deleted department --- package.json | 3 ++- ui/src/composables/use-display-owner.ts | 27 ------------------------- ui/src/pages/embed/events.vue | 3 +-- 3 files changed, 3 insertions(+), 30 deletions(-) delete mode 100644 ui/src/composables/use-display-owner.ts diff --git a/package.json b/package.json index 6b90fdf..21f9326 100644 --- a/package.json +++ b/package.json @@ -62,7 +62,8 @@ }, "relativeDependencies": { "@data-fair/lib-express": "../lib/packages/express", - "@data-fair/lib-vue": "../lib/packages/vue" + "@data-fair/lib-vue": "../lib/packages/vue", + "@data-fair/lib-vuetify": "../lib/packages/vuetify" }, "optionalDependencies": { "sass": "1.98.0" diff --git a/ui/src/composables/use-display-owner.ts b/ui/src/composables/use-display-owner.ts deleted file mode 100644 index f6f420b..0000000 --- a/ui/src/composables/use-display-owner.ts +++ /dev/null @@ -1,27 +0,0 @@ -// A department deleted in simple-directory keeps its id on the resources it owned, but not its name: -// the identity webhook removed it. Build the owner as it must be displayed (owner-avatar, labels). -import { useI18n } from 'vue-i18n' - -type Owner = { department?: string, departmentName?: string, [key: string]: unknown } - -// plain strings: the runtime-only vue-i18n build used in production cannot compile inline messages -const formerDepartment: Record = { - fr: 'Ancien département', - en: 'Former department' -} - -export const useDisplayOwner = () => { - const { locale } = useI18n() - - const departmentLabel = (department?: string, departmentName?: string) => { - if (!department) return undefined - return departmentName || `${formerDepartment[locale.value] ?? formerDepartment.en} - ${department}` - } - - const displayOwner = (owner: T): T => { - if (!owner.department || owner.departmentName) return owner - return { ...owner, departmentName: departmentLabel(owner.department) } - } - - return { displayOwner, departmentLabel } -} diff --git a/ui/src/pages/embed/events.vue b/ui/src/pages/embed/events.vue index f96cee7..ccd1fa8 100644 --- a/ui/src/pages/embed/events.vue +++ b/ui/src/pages/embed/events.vue @@ -76,7 +76,7 @@ /> Date: Wed, 16 Sep 2026 09:56:07 +0200 Subject: [PATCH 08/13] chore(ui): regenerate auto-imports after moving useDisplayOwner to lib-vue --- ui/dts/auto-imports.d.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/dts/auto-imports.d.ts b/ui/dts/auto-imports.d.ts index 0e32d05..82dc3b6 100644 --- a/ui/dts/auto-imports.d.ts +++ b/ui/dts/auto-imports.d.ts @@ -104,7 +104,7 @@ declare global { const useConceptFilters: typeof import('@data-fair/lib-vue/concept-filters.js').useConceptFilters const useCssModule: typeof import('vue').useCssModule const useCssVars: typeof import('vue').useCssVars - const useDisplayOwner: typeof import('../src/composables/use-display-owner').useDisplayOwner + const useDisplayOwner: typeof import('@data-fair/lib-vue/owner.js').useDisplayOwner const useEditFetch: typeof import('@data-fair/lib-vue/edit-fetch.js').useEditFetch const useFetch: typeof import('@data-fair/lib-vue/fetch.js').useFetch const useI18n: typeof import('vue-i18n').useI18n @@ -224,7 +224,7 @@ declare module 'vue' { readonly useConceptFilters: UnwrapRef readonly useCssModule: UnwrapRef readonly useCssVars: UnwrapRef - readonly useDisplayOwner: UnwrapRef + readonly useDisplayOwner: UnwrapRef readonly useEditFetch: UnwrapRef readonly useFetch: UnwrapRef readonly useI18n: UnwrapRef From c9579109f7c2398ed4adc759cadb9d09a3722879 Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Wed, 16 Sep 2026 11:11:03 +0200 Subject: [PATCH 09/13] chore(deps): update lib-vue and lib-vuetify for the deleted department label --- package-lock.json | 18 +++++++++--------- ui/package.json | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index 098cc15..917998b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -663,9 +663,9 @@ } }, "node_modules/@data-fair/lib-vue": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/@data-fair/lib-vue/-/lib-vue-1.30.1.tgz", - "integrity": "sha512-ZGb9LRih5idfHDnIcA6J5pAuIBxVUvx+4bJiqidZIXbuAKt/pvtyM0lkgIW/1wfG5sY05AfeJvYr8890PZpjDQ==", + "version": "1.31.1", + "resolved": "https://registry.npmjs.org/@data-fair/lib-vue/-/lib-vue-1.31.1.tgz", + "integrity": "sha512-SD7jnWerCKZZM658nIvqPK6/efdBfVIA0wln7tRmQyFYdF2PZYTTREbauBT7j92fKDs4Nm7oXJJqeA4yMsoTsA==", "license": "MIT", "dependencies": { "@data-fair/lib-common-types": "^1.7.1", @@ -698,9 +698,9 @@ } }, "node_modules/@data-fair/lib-vuetify": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/@data-fair/lib-vuetify/-/lib-vuetify-2.4.3.tgz", - "integrity": "sha512-GqaiGZ4BxMi1hDo/uzGujYXl23qpp4R5OxMNmI//4380rOD83bvxQUb+12lQENscWMKYwx+3RfYNhDecDJk7Ag==", + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@data-fair/lib-vuetify/-/lib-vuetify-2.5.1.tgz", + "integrity": "sha512-efLqHNIC3B2DsgsRB4TzCuoIaVIC9tcucDEOLDiiWuvx1fHuHi08f0MxM7PjAv1eQmD5a8Dw1g7HJ9u1+1iIlg==", "license": "MIT", "dependencies": { "@data-fair/lib-common-types": "^1.10.4", @@ -708,7 +708,7 @@ "@vueuse/core": "^14.0.0" }, "peerDependencies": { - "@data-fair/lib-vue": "^1.15.0", + "@data-fair/lib-vue": "^1.31.1", "ofetch": "1", "vue-i18n": "10 || 11", "vuetify": "4" @@ -12410,8 +12410,8 @@ "dependencies": { "@data-fair/frame": "^0.17.7", "@data-fair/lib-utils": "^1.14.0", - "@data-fair/lib-vue": "^1.26.0", - "@data-fair/lib-vuetify": "^2.0.0", + "@data-fair/lib-vue": "^1.31.1", + "@data-fair/lib-vuetify": "^2.5.1", "@data-fair/lib-vuetify-events": "*", "@intlify/unplugin-vue-i18n": "^11.0.7", "@mdi/js": "^7.4.47", diff --git a/ui/package.json b/ui/package.json index 582c7ec..ad07308 100644 --- a/ui/package.json +++ b/ui/package.json @@ -14,8 +14,8 @@ "dependencies": { "@data-fair/frame": "^0.17.7", "@data-fair/lib-utils": "^1.14.0", - "@data-fair/lib-vue": "^1.26.0", - "@data-fair/lib-vuetify": "^2.0.0", + "@data-fair/lib-vue": "^1.31.1", + "@data-fair/lib-vuetify": "^2.5.1", "@data-fair/lib-vuetify-events": "*", "@intlify/unplugin-vue-i18n": "^11.0.7", "@mdi/js": "^7.4.47", From b49c302ab86348cc09b51b7059acb6212bcec97e Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Wed, 16 Sep 2026 12:03:45 +0200 Subject: [PATCH 10/13] perf(identities): index the sender of webhooks for the delete webhook --- api/src/mongo.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/src/mongo.ts b/api/src/mongo.ts index b8c3925..02376bf 100644 --- a/api/src/mongo.ts +++ b/api/src/mongo.ts @@ -78,6 +78,8 @@ export class EventsMongo { }, webhooks: { 'main-keys': { 'owner.type': 1, 'owner.id': 1, 'subscription._id': 1, 'notification.date': 1 }, + // identity webhooks drop the webhooks of a deleted sender + 'sender-keys': { 'sender.type': 1, 'sender.id': 1 }, 'loop-keys': { status: 1, nextAttempt: 1 } }, pushSubscriptions: { From 398de63e0b23fc6dd887ef475fccc7ec52a15103 Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Fri, 25 Sep 2026 16:12:09 +0200 Subject: [PATCH 11/13] feat(identities): keep names searchable, rebuilt by a search worker Names are back in the events search texts: they are what makes a user findable in an organization feed. The identity webhook no longer rewrites them: it only bulk updates the names and flags the events (_needsSearch), and a search worker running in the api server rebuilds their texts by batches of 1000. - the webhook only touches the events whose name actually changes (simple-directory also posts on every membership change) - a deleted user's name leaves the search texts too - the worker skips an event flagged again by a newer rename in the meantime, the next batch picks it up - partial index on _needsSearch, hidden from the API responses --- api/src/events/operations.ts | 8 ++--- api/src/events/router.ts | 2 +- api/src/events/search-worker.ts | 54 ++++++++++++++++++++++++++++ api/src/identities/service.ts | 18 ++++++---- api/src/mongo.ts | 4 ++- api/src/server.ts | 3 ++ api/types/index.ts | 3 +- tests/identities-webhook.api.spec.ts | 13 ++++--- 8 files changed, 85 insertions(+), 20 deletions(-) create mode 100644 api/src/events/search-worker.ts diff --git a/api/src/events/operations.ts b/api/src/events/operations.ts index bf70961..8674279 100644 --- a/api/src/events/operations.ts +++ b/api/src/events/operations.ts @@ -77,15 +77,13 @@ export const buildSearchTexts = (event: SearchableEvent, locales: string[], defa const search: SearchableEvent['_search'] = [] for (const locale of locales) { const localizedEvent = localizeEvent(event, locale, defaultLocale) - // ids only, no name: names change and the search texts of every past event would have to be rebuilt - // (the identity webhooks only bulk update the name fields themselves) - const searchParts: (string | undefined)[] = [...event.topic.key.split(':'), event.topic.title, localizedEvent.title, localizedEvent.body, event.sender?.id] + const searchParts: (string | undefined)[] = [...event.topic.key.split(':'), event.topic.title, localizedEvent.title, localizedEvent.body, event.sender?.id, event.sender?.name] if (event.originator) { if (event.originator.organization) { - searchParts.push(event.originator.organization.id) + searchParts.push(event.originator.organization.name, event.originator.organization.id) } if (event.originator.user && (!event.originator.organization || (event.sender?.type === 'organization' && event.sender.id === event.originator.organization.id))) { - searchParts.push(event.originator.user.id) + searchParts.push(event.originator.user.name, event.originator.user.id) } } search.push({ language: locale, text: searchParts.filter(Boolean).join(' ') }) diff --git a/api/src/events/router.ts b/api/src/events/router.ts index d41fa44..407f369 100644 --- a/api/src/events/router.ts +++ b/api/src/events/router.ts @@ -24,7 +24,7 @@ router.get('', async (req, res, next) => { query['resource.id'] = id } - const project = mongoProjection(req.query.select, ['_search', 'htmlBody']) + const project = mongoProjection(req.query.select, ['_search', '_needsSearch', 'htmlBody']) // implement a special pagination based on the fact that we always sort by date const sort: Sort = { date: -1, _id: -1 } diff --git a/api/src/events/search-worker.ts b/api/src/events/search-worker.ts new file mode 100644 index 0000000..276a97a --- /dev/null +++ b/api/src/events/search-worker.ts @@ -0,0 +1,54 @@ +// Rebuilds the search texts of the events flagged by the identity webhooks (_needsSearch), +// so that simple-directory gets its response without waiting for the text index. +// Runs in the api server, like the webhooks worker. + +import config from '#config' +import Debug from 'debug' +import mongo from '#mongo' +import { internalError } from '@data-fair/lib-node/observer.js' +import locks from '@data-fair/lib-node/locks.js' +import { buildSearchTexts } from './operations.ts' + +const debug = Debug('search-worker') +const batchSize = 1000 + +let loopPromise: Promise | null = null +let stopped = false +let acquiredLock = false + +const wait = () => new Promise(resolve => setTimeout(resolve, config.worker.loopInterval)) + +const loop = async () => { + // eslint-disable-next-line no-unmodified-loop-condition + while (!stopped) { + try { + if (!acquiredLock) { + acquiredLock = await locks.acquire('search-loop') + if (!acquiredLock) { await wait(); continue } + } + const events = await mongo.events.find({ _needsSearch: { $exists: true } }).limit(batchSize).toArray() + if (!events.length) { await wait(); continue } + debug('rebuild the search texts of', events.length, 'events') + await mongo.events.bulkWrite(events.map(event => ({ + updateOne: { + // a newer rename flagged the event again in the meantime: skipped, picked up by the next batch + filter: { _id: event._id, _needsSearch: event._needsSearch }, + update: { $set: { _search: buildSearchTexts(event, config.i18n.locales, config.i18n.defaultLocale) }, $unset: { _needsSearch: 1 } } + } + })), { ordered: false }) + } catch (err) { + internalError('search-loop', err) + await wait() + } + } + if (acquiredLock) await locks.release('search-loop') +} + +export const start = () => { + loopPromise = loop() +} + +export const stop = async () => { + stopped = true + await loopPromise +} diff --git a/api/src/identities/service.ts b/api/src/identities/service.ts index d2f071a..e22661b 100644 --- a/api/src/identities/service.ts +++ b/api/src/identities/service.ts @@ -1,8 +1,10 @@ // Synchronize the copies of identity data (names on senders, recipients, owners, originators) // with the users/organizations directory, and remove them when an identity is deleted. // Everything is done with bulk updates: an organization can own tens of thousands of events and -// simple-directory waits for the response (names are kept out of the search texts for this reason). +// simple-directory waits for the response. Names are also part of the events search texts: the +// events are only flagged here (_needsSearch) and the search worker rebuilds the texts. +import { randomUUID } from 'node:crypto' import type { IdentityUpdate, IdentityDelete } from '@data-fair/lib-express/identities/index.js' import mongo from '#mongo' @@ -35,20 +37,22 @@ export const updateIdentity = async (identity: IdentityUpdate) => { } // events: as sender, and as the user or organization that triggered them - await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id }, { $set: { 'sender.name': name } }) + // only the events whose name actually changes: simple-directory also posts on every membership change + const _needsSearch = randomUUID() + await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.name': { $ne: name } }, { $set: { 'sender.name': name, _needsSearch } }) if (departments) { for (const department of departments.filter(d => !!d.name)) { - await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': department.id }, { $set: { 'sender.departmentName': department.name } }) - await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.department': department.id }, { $set: { 'originator.organization.departmentName': department.name } }) + await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': department.id, 'sender.departmentName': { $ne: department.name } }, { $set: { 'sender.departmentName': department.name } }) + await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.department': department.id, 'originator.organization.departmentName': { $ne: department.name } }, { $set: { 'originator.organization.departmentName': department.name } }) } const deletedDepartment = { $exists: true, $nin: departments.map(d => d.id) } await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': deletedDepartment }, { $unset: { 'sender.departmentName': 1 } }) await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.department': deletedDepartment }, { $unset: { 'originator.organization.departmentName': 1 } }) } if (type === 'user') { - await mongo.events.updateMany({ 'originator.user.id': id }, { $set: { 'originator.user.name': name } }) + await mongo.events.updateMany({ 'originator.user.id': id, 'originator.user.name': { $ne: name } }, { $set: { 'originator.user.name': name, _needsSearch } }) } else { - await mongo.events.updateMany({ 'originator.organization.id': id }, { $set: { 'originator.organization.name': name } }) + await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.name': { $ne: name } }, { $set: { 'originator.organization.name': name, _needsSearch } }) } if (type === 'user' && identity.organizations) { @@ -94,6 +98,6 @@ export const deleteIdentity = async (identity: IdentityDelete) => { // the events a user triggered on other feeds keep the trace of the action without the person: // only the id remains (pseudonymized), an organization is not personal data and is left as is if (type === 'user') { - await mongo.events.updateMany({ 'originator.user.id': id }, { $unset: { 'originator.user.name': 1, 'originator.user.email': 1 } }) + await mongo.events.updateMany({ 'originator.user.id': id }, { $set: { _needsSearch: randomUUID() }, $unset: { 'originator.user.name': 1, 'originator.user.email': 1 } }) } } diff --git a/api/src/mongo.ts b/api/src/mongo.ts index 5cd002d..58afb01 100644 --- a/api/src/mongo.ts +++ b/api/src/mongo.ts @@ -55,7 +55,9 @@ export class EventsMongo { ], // identity webhooks rewrite the events triggered by a user or an organization 'originator-user': [{ 'originator.user.id': 1 }, { sparse: true }], - 'originator-organization': [{ 'originator.organization.id': 1 }, { sparse: true }] + 'originator-organization': [{ 'originator.organization.id': 1 }, { sparse: true }], + // the events waiting for the search worker, empty most of the time + 'needs-search': [{ _needsSearch: 1 }, { partialFilterExpression: { _needsSearch: { $exists: true } } }] }, subscriptions: { 'main-keys': [ diff --git a/api/src/server.ts b/api/src/server.ts index ca8144d..f914826 100644 --- a/api/src/server.ts +++ b/api/src/server.ts @@ -12,6 +12,7 @@ import { createHttpTerminator } from 'http-terminator' import app from './app.ts' import config from '#config' import * as webhooksWorker from './webhooks/worker.ts' +import * as searchWorker from './events/search-worker.ts' import * as pushService from './push/service.ts' const server = createServer(app) @@ -41,6 +42,7 @@ export const start = async () => { await wsEmitter.init(mongo.db) await pushService.init() await webhooksWorker.start() + searchWorker.start() server.listen(config.port) await new Promise(resolve => server.once('listening', resolve)) @@ -51,6 +53,7 @@ export const start = async () => { export const stop = async () => { await httpTerminator.terminate() await webhooksWorker.stop() + await searchWorker.stop() await wsServer.stop() if (config.observer.active) await stopObserver() await locks.stop() diff --git a/api/types/index.ts b/api/types/index.ts index 36996c2..fef588e 100644 --- a/api/types/index.ts +++ b/api/types/index.ts @@ -10,4 +10,5 @@ export type { DeviceRegistration } from './device-registration/index.js' export type FullEvent = Event & Required> export type LocalizedEvent = Omit & { title: string, body?: string, htmlBody?: string } -export type SearchableEvent = FullEvent & { _search: { language: string, text: string }[] } +// _needsSearch: set by the identity webhooks when a name changed, cleared by the search worker +export type SearchableEvent = FullEvent & { _search: { language: string, text: string }[], _needsSearch?: string } diff --git a/tests/identities-webhook.api.spec.ts b/tests/identities-webhook.api.spec.ts index 6485b2f..36de1a3 100644 --- a/tests/identities-webhook.api.spec.ts +++ b/tests/identities-webhook.api.spec.ts @@ -123,7 +123,7 @@ test.describe('identities update webhook on events', () => { expect(events[0].sender.departmentName).toBeUndefined() }) - test('should rename the sender and originator of events without touching the search texts', async () => { + test('should rename the sender and originator of events and let the search worker rebuild the search texts', async () => { await postEvents() await axIdentities.post('/api/identities/user/test-user1', { name: 'Aurélien Lefort' }) @@ -131,10 +131,9 @@ test.describe('identities update webhook on events', () => { expect(ownEvents).toHaveLength(1) expect(ownEvents[0].sender.name).toBe('Aurélien Lefort') expect(ownEvents[0].originator.user.name).toBe('Aurélien Lefort') - // names are never part of the search texts: a rename is a cheap bulk update, not a rewrite of every event - expect((await user1.get('/api/events?q=Lefort')).data.results).toHaveLength(0) + // the search texts are rebuilt by the search worker, after the webhook responded + await expect.poll(async () => (await user1.get('/api/events?q=Lefort')).data.results.length, { timeout: 10000 }).toBe(1) expect((await user1.get('/api/events?q=Dubois')).data.results).toHaveLength(0) - expect((await user1.get('/api/events?q=test-user1')).data.results).toHaveLength(1) let orgEvents = (await admin1.get('/api/events')).data.results expect(orgEvents).toHaveLength(1) @@ -144,6 +143,7 @@ test.describe('identities update webhook on events', () => { orgEvents = (await admin1.get('/api/events')).data.results expect(orgEvents[0].sender.name).toBe('Renamed Organization 1') expect(orgEvents[0].originator.organization.name).toBe('Renamed Organization 1') + await expect.poll(async () => (await admin1.get('/api/events?q=Renamed')).data.results.length, { timeout: 10000 }).toBe(1) }) }) @@ -152,6 +152,7 @@ test.describe('identities delete webhook', () => { test('should delete the events of a deleted user and pseudonymize the ones it triggered elsewhere', async () => { await postEvents() + expect((await admin1.get('/api/events?q=Dubois')).data.results).toHaveLength(1) await axIdentities.delete('/api/identities/user/test-user1') expect((await user1.get('/api/events')).data.results).toHaveLength(0) @@ -159,7 +160,9 @@ test.describe('identities delete webhook', () => { expect(orgEvents).toHaveLength(1) expect(orgEvents[0].originator.user).toEqual({ id: 'test-user1' }) expect(orgEvents[0].originator.organization.name).toBe('Test Organization 1') - expect((await admin1.get('/api/events?q=Dubois')).data.results).toHaveLength(0) + // the name also leaves the search texts, once the search worker went through + await expect.poll(async () => (await admin1.get('/api/events?q=Dubois')).data.results.length, { timeout: 10000 }).toBe(0) + expect((await admin1.get('/api/events?q=test-user1')).data.results).toHaveLength(1) }) test('should delete the events of a deleted organization', async () => { From 7eb61d3e0109071736762676001ec7d52698f403 Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Fri, 25 Sep 2026 16:21:07 +0200 Subject: [PATCH 12/13] ci: give the api config dir as a relative path config 5 resolves config 5 no longer resolves NODE_CONFIG_DIR=api/config (Cannot find module 'api/config/default.cjs'): the quality check fails since the dependencies refresh (#15). --- .github/workflows/reuse-quality.yml | 2 +- tests/events.unit.spec.ts | 12 ++++-------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/reuse-quality.yml b/.github/workflows/reuse-quality.yml index ea61ff3..0d7c798 100644 --- a/.github/workflows/reuse-quality.yml +++ b/.github/workflows/reuse-quality.yml @@ -35,7 +35,7 @@ jobs: run: docker compose up -d --wait - name: Start dev API - run: NODE_ENV=development NODE_CONFIG_DIR=api/config node api/index.ts & + run: NODE_ENV=development NODE_CONFIG_DIR=./api/config node api/index.ts & - name: Wait for API to be ready run: | diff --git a/tests/events.unit.spec.ts b/tests/events.unit.spec.ts index 9b11872..1a5aa12 100644 --- a/tests/events.unit.spec.ts +++ b/tests/events.unit.spec.ts @@ -204,8 +204,6 @@ test.describe('buildSearchTexts', () => { expect(result[0].text).toContain('Titre') expect(result[0].text).toContain('Topic') expect(result[0].text).toContain('test-user1') - // names are not searchable: they change, and the search texts would have to be rebuilt on every rename - expect(result[0].text).not.toContain('User 1') expect(result[1].language).toBe('en') expect(result[1].text).toContain('Title') }) @@ -225,10 +223,8 @@ test.describe('buildSearchTexts', () => { } } const result = buildSearchTexts(event, ['fr'], 'fr') - expect(result[0].text).toContain('org1') - expect(result[0].text).toContain('u1') - expect(result[0].text).not.toContain('Org 1') - expect(result[0].text).not.toContain('User 1') + expect(result[0].text).toContain('Org 1') + expect(result[0].text).toContain('User 1') }) test('excludes user when originator is from different org', () => { @@ -246,7 +242,7 @@ test.describe('buildSearchTexts', () => { } } const result = buildSearchTexts(event, ['fr'], 'fr') - expect(result[0].text).toContain('org-other') - expect(result[0].text).not.toContain('u1') + expect(result[0].text).toContain('Other Org') + expect(result[0].text).not.toContain('User 1') }) }) From 66080a106c6ad04ad59921e301e5459bf8f709e8 Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Fri, 25 Sep 2026 16:21:12 +0200 Subject: [PATCH 13/13] fix(identities): harden the search worker - a fresh _needsSearch value per update: an event flagged by two updates of the same webhook could otherwise get the texts of the first one and lose its flag - an event whose texts cannot be built has its flag cleared (like drainSearchIndex in data-fair) instead of coming back first in every batch and blocking the others - read only the fields buildSearchTexts needs - buildSearchTexts unit tests back to names being searchable --- api/src/events/search-worker.ts | 21 ++++++++++++++------- api/src/identities/service.ts | 10 +++++----- tests/identities-webhook.api.spec.ts | 8 ++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/api/src/events/search-worker.ts b/api/src/events/search-worker.ts index 276a97a..6b2eafb 100644 --- a/api/src/events/search-worker.ts +++ b/api/src/events/search-worker.ts @@ -11,6 +11,7 @@ import { buildSearchTexts } from './operations.ts' const debug = Debug('search-worker') const batchSize = 1000 +const searchProjection = { topic: 1, title: 1, body: 1, sender: 1, originator: 1, _needsSearch: 1 } let loopPromise: Promise | null = null let stopped = false @@ -26,16 +27,22 @@ const loop = async () => { acquiredLock = await locks.acquire('search-loop') if (!acquiredLock) { await wait(); continue } } - const events = await mongo.events.find({ _needsSearch: { $exists: true } }).limit(batchSize).toArray() + // only what buildSearchTexts reads: an htmlBody can be heavy + const events = await mongo.events.find({ _needsSearch: { $exists: true } }, { projection: searchProjection }).limit(batchSize).toArray() if (!events.length) { await wait(); continue } debug('rebuild the search texts of', events.length, 'events') - await mongo.events.bulkWrite(events.map(event => ({ - updateOne: { - // a newer rename flagged the event again in the meantime: skipped, picked up by the next batch - filter: { _id: event._id, _needsSearch: event._needsSearch }, - update: { $set: { _search: buildSearchTexts(event, config.i18n.locales, config.i18n.defaultLocale) }, $unset: { _needsSearch: 1 } } + await mongo.events.bulkWrite(events.map(event => { + // a newer rename flagged the event again in the meantime: skipped, picked up by the next batch + const filter = { _id: event._id, _needsSearch: event._needsSearch } + try { + return { updateOne: { filter, update: { $set: { _search: buildSearchTexts(event, config.i18n.locales, config.i18n.defaultLocale) }, $unset: { _needsSearch: 1 } } } } + } catch (err) { + // like drainSearchIndex in data-fair: the event keeps its previous texts, but the flag is cleared + // or this event would come back first in every batch and block the others forever + internalError('search-loop-event', `failed to rebuild the search texts of event ${event._id} - ${(err as Error)?.stack || err}`) + return { updateOne: { filter, update: { $unset: { _needsSearch: 1 } } } } } - })), { ordered: false }) + }), { ordered: false }) } catch (err) { internalError('search-loop', err) await wait() diff --git a/api/src/identities/service.ts b/api/src/identities/service.ts index e22661b..04337ab 100644 --- a/api/src/identities/service.ts +++ b/api/src/identities/service.ts @@ -37,9 +37,9 @@ export const updateIdentity = async (identity: IdentityUpdate) => { } // events: as sender, and as the user or organization that triggered them - // only the events whose name actually changes: simple-directory also posts on every membership change - const _needsSearch = randomUUID() - await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.name': { $ne: name } }, { $set: { 'sender.name': name, _needsSearch } }) + // only the events whose name actually changes: simple-directory also posts on every membership change. + // A fresh _needsSearch per update: the search worker only clears the value it read. + await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.name': { $ne: name } }, { $set: { 'sender.name': name, _needsSearch: randomUUID() } }) if (departments) { for (const department of departments.filter(d => !!d.name)) { await mongo.events.updateMany({ 'sender.type': type, 'sender.id': id, 'sender.department': department.id, 'sender.departmentName': { $ne: department.name } }, { $set: { 'sender.departmentName': department.name } }) @@ -50,9 +50,9 @@ export const updateIdentity = async (identity: IdentityUpdate) => { await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.department': deletedDepartment }, { $unset: { 'originator.organization.departmentName': 1 } }) } if (type === 'user') { - await mongo.events.updateMany({ 'originator.user.id': id, 'originator.user.name': { $ne: name } }, { $set: { 'originator.user.name': name, _needsSearch } }) + await mongo.events.updateMany({ 'originator.user.id': id, 'originator.user.name': { $ne: name } }, { $set: { 'originator.user.name': name, _needsSearch: randomUUID() } }) } else { - await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.name': { $ne: name } }, { $set: { 'originator.organization.name': name, _needsSearch } }) + await mongo.events.updateMany({ 'originator.organization.id': id, 'originator.organization.name': { $ne: name } }, { $set: { 'originator.organization.name': name, _needsSearch: randomUUID() } }) } if (type === 'user' && identity.organizations) { diff --git a/tests/identities-webhook.api.spec.ts b/tests/identities-webhook.api.spec.ts index 36de1a3..5d1cd58 100644 --- a/tests/identities-webhook.api.spec.ts +++ b/tests/identities-webhook.api.spec.ts @@ -145,6 +145,14 @@ test.describe('identities update webhook on events', () => { expect(orgEvents[0].originator.organization.name).toBe('Renamed Organization 1') await expect.poll(async () => (await admin1.get('/api/events?q=Renamed')).data.results.length, { timeout: 10000 }).toBe(1) }) + + test('should end up with the last name when renames follow each other', async () => { + await postEvents() + await axIdentities.post('/api/identities/organization/test1', { name: 'Intermediate Name' }) + await axIdentities.post('/api/identities/organization/test1', { name: 'Final Name' }) + await expect.poll(async () => (await admin1.get('/api/events?q=Final')).data.results.length, { timeout: 10000 }).toBe(1) + expect((await admin1.get('/api/events?q=Intermediate')).data.results).toHaveLength(0) + }) }) test.describe('identities delete webhook', () => {