From 00440c8797c53498344e706436b6334efb593a3b Mon Sep 17 00:00:00 2001 From: BatLeDev Date: Tue, 15 Sep 2026 14:24:52 +0200 Subject: [PATCH 1/3] feat(api): sync artefacts on selection, mirror thumbnails, sortable listing - POST selected-artefacts queues a background sync of that artefact only (pendingSync on the registry doc, drained under the sync lock so quick successive selections are never lost) - sync mirrors upstream thumbnails under the same id (add, replace, remove) - GET remote-artefacts forwards category/format to the upstream and annotates each row with its local mirror state (synced, upToDate, conflict) - GET remote-thumbnails/:id/data proxies upstream thumbnails for the admin table, restricted to image/webp and image/svg+xml with nosniff + CSP sandbox - GET artefacts accepts a whitelisted sort key with a leading dash for descending; unknown keys are 400, admin-only keys fall back for others - store the package.json description as packageDescription (read-only, mirrored by the sync) --- AGENTS.md | 4 +- api/src/artefacts/operations.ts | 6 +- api/src/artefacts/router.ts | 39 +++- api/src/artefacts/service.ts | 4 +- api/src/remote-registries/operations.ts | 30 +++ api/src/remote-registries/router.ts | 56 +++++- api/src/remote-registries/sync.ts | 91 ++++++++- api/types/artefact/schema.js | 35 ++-- api/types/remote-registry/schema.js | 6 + dev/fixtures.ts | 12 +- tests/artefact-admin.e2e.spec.ts | 35 +++- tests/artefacts-operations.unit.spec.ts | 16 ++ tests/artefacts.api.spec.ts | 61 +++++- .../remote-registries-operations.unit.spec.ts | 40 +++- tests/remote-registries-sync.api.spec.ts | 176 +++++++++++++++--- tests/remote-registries.api.spec.ts | 15 +- tests/support/axios.ts | 13 ++ tests/support/test-tarball.ts | 4 +- 18 files changed, 563 insertions(+), 80 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c0537cd..521cea0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,7 +57,9 @@ Test users are defined in @dev/resources/users.json and organizations in @dev/re Sync mirrors artefacts *from* an upstream registry, so it needs two registries. `api-upstream` is a second registry process (same code, `PORT`/`MONGO_URL`/`DATA_DIR` overridden). Pointing a registry at itself cannot work: selecting an artefact that already exists locally without an `origin` returns 409. -`npm run dev:fixtures` seeds the upstream, mints a read key owned by org `test1`, registers the mirror and selects two artefacts. It stops short of syncing — click **Sync now** in the admin UI. +`npm run dev:fixtures` seeds the upstream, mints a read key owned by org `test1`, registers the mirror and selects two artefacts. Selecting an artefact queues a background sync of that artefact alone (`pendingSync` on the registry doc, drained under the sync lock), so the mirrors land without clicking **Sync now** — that button runs a full sync. Sync progress is published on the registry's ws channel; the admin table gets each row's local state from `GET …/remote-artefacts` and loads upstream thumbnails through `GET …/remote-thumbnails/:id/data` (the browser cannot reach the upstream directly). + +Unselecting keeps the local copy but unlocks it (drops `origin`); it then blocks re-mirroring of that id (409) until it is deleted. Deleting a registry unlocks all its mirrors the same way. `tests/remote-registries-sync.api.spec.ts` covers the mirror path end to end against the same upstream. diff --git a/api/src/artefacts/operations.ts b/api/src/artefacts/operations.ts index c9add0d..3352888 100644 --- a/api/src/artefacts/operations.ts +++ b/api/src/artefacts/operations.ts @@ -9,6 +9,9 @@ export interface Manifest { name: string version: string licence?: string + // package.json description, shown as the technical description next to the + // editable, localized one. + description?: string } export interface ExtractManifestResult { @@ -115,7 +118,8 @@ export const extractManifest = async ( manifest = { name: pkg.name, version: pkg.version, - licence: pkg.licence || pkg.license + licence: pkg.licence || pkg.license, + ...(typeof pkg.description === 'string' && pkg.description.trim() ? { description: pkg.description.trim() } : {}) } next() } catch (err) { diff --git a/api/src/artefacts/router.ts b/api/src/artefacts/router.ts index 2dbffab..875b2fa 100644 --- a/api/src/artefacts/router.ts +++ b/api/src/artefacts/router.ts @@ -96,6 +96,36 @@ const tryInternalSecret = (req: import('express').Request): boolean => { return timingSafeEqual(received, expected) } +// `?sort=` with an optional leading `-` for descending. Keys map to the +// Mongo sort they stand for; a name tie-breaker keeps pages stable when many +// docs share a value. `vulnerabilities` and `public` are admin-only: scan data +// is stripped for non-admins (ordering by it would leak) and visibility is not +// shown to them — they silently get the default sort. +const sortKeys: Record, admin?: boolean }> = { + dataUpdatedAt: { sort: { dataUpdatedAt: -1 } }, + name: { sort: { name: 1 } }, + category: { sort: { category: 1 } }, + 'group.en': { sort: { 'group.en': 1 } }, + 'group.fr': { sort: { 'group.fr': 1 } }, + version: { sort: { version: 1 } }, + size: { sort: { size: 1 } }, + public: { sort: { public: 1 }, admin: true }, + vulnerabilities: { sort: { 'scan.summary.critical': -1, 'scan.summary.high': -1, 'scan.summary.medium': -1 }, admin: true } +} +const parseSort = (raw: unknown, caller: Caller): Record => { + const fallback = { dataUpdatedAt: -1 as const, name: 1 as const } + if (typeof raw !== 'string' || !raw) return fallback + const desc = raw.startsWith('-') + const key = desc ? raw.slice(1) : raw + const def = sortKeys[key] + if (!def) throw httpError(400, `invalid sort, must be one of: ${Object.keys(sortKeys).join(', ')} (prefix with - for descending)`) + if (def.admin && !caller.admin) return fallback + const sort: Record = {} + for (const [field, dir] of Object.entries(def.sort)) sort[field] = desc ? (dir === 1 ? -1 : 1) : dir + if (!('name' in sort)) sort.name = 1 + return sort +} + // List artefacts (filtered by access) router.get('/', async (req, res, next) => { try { @@ -103,14 +133,7 @@ router.get('/', async (req, res, next) => { const filter = artefactAccessFilter(caller) const skip = Math.max(0, Math.min(parseInt(req.query.skip as string) || 0, 100000)) const size = Math.min(parseInt(req.query.size as string) || 10, 100) - // `vulnerabilities` is admin-only: scan data is stripped for non-admins, so - // ordering by it would leak. Non-admins silently get the default sort. - let sort: Record = { dataUpdatedAt: -1 } - if (req.query.sort === 'name') { - sort = { name: 1 } - } else if (req.query.sort === 'vulnerabilities' && caller.admin) { - sort = { 'scan.summary.critical': -1, 'scan.summary.high': -1, 'scan.summary.medium': -1, dataUpdatedAt: -1 } - } + const sort = parseSort(req.query.sort, caller) // Text search on name if (req.query.q) { diff --git a/api/src/artefacts/service.ts b/api/src/artefacts/service.ts index 3014a35..0d84b5b 100644 --- a/api/src/artefacts/service.ts +++ b/api/src/artefacts/service.ts @@ -176,6 +176,7 @@ export const commitNpmUpload = async (params: { packageName: manifest.name, version: manifest.version, ...(manifest.licence ? { licence: manifest.licence } : {}), + ...(manifest.description ? { packageDescription: manifest.description } : {}), category, path, size, @@ -191,7 +192,8 @@ export const commitNpmUpload = async (params: { public: false, privateAccess: [], createdAt: now - } + }, + ...(manifest.description ? {} : { $unset: { packageDescription: '' } }) }, { upsert: true } ) diff --git a/api/src/remote-registries/operations.ts b/api/src/remote-registries/operations.ts index c962418..3026efa 100644 --- a/api/src/remote-registries/operations.ts +++ b/api/src/remote-registries/operations.ts @@ -38,3 +38,33 @@ export const syncState = ( if (startedAt && (!registry.lastSyncAt || startedAt > registry.lastSyncAt)) return 'interrupted' return 'idle' } + +// Local mirror state of each remote artefact, for the admin's selection table. +// `upToDate` compares the upstream dataUpdatedAt with the mirrored copy's — the +// same fast-path key the npm sync uses, so a stale row is exactly one the next +// sync would re-download. +// `conflict` flags a local artefact with the same id that this registry does +// not own (uploaded locally, or unlocked when a registry was deleted): the +// select endpoint would answer 409, so the UI can say so up front. +export type LocalState = { synced: boolean, upToDate: boolean, dataUpdatedAt?: string, conflict?: boolean } + +export const annotateLocalState = ( + remoteArtefacts: T[], + localArtefacts: { _id: string, dataUpdatedAt?: string, origin?: string }[], + registryId: string +): (T & { local: LocalState })[] => { + const locals = new Map(localArtefacts.map(a => [a._id, a])) + return remoteArtefacts.map(remote => { + const local = locals.get(remote._id) + if (!local) return { ...remote, local: { synced: false, upToDate: false } } + if (local.origin !== registryId) return { ...remote, local: { synced: false, upToDate: false, conflict: true } } + return { + ...remote, + local: { + synced: true, + upToDate: !!local.dataUpdatedAt && local.dataUpdatedAt === remote.dataUpdatedAt, + dataUpdatedAt: local.dataUpdatedAt + } + } + }) +} diff --git a/api/src/remote-registries/router.ts b/api/src/remote-registries/router.ts index d7db332..341b7b0 100644 --- a/api/src/remote-registries/router.ts +++ b/api/src/remote-registries/router.ts @@ -1,11 +1,12 @@ import { Router } from 'express' +import { pipeline } from 'node:stream/promises' import { session } from '@data-fair/lib-express/index.js' import { httpError } from '@data-fair/lib-utils/http-errors.js' import { axiosBuilder } from '@data-fair/lib-node/axios.js' import mongo from '#mongo' import { cipher, decipher } from '../cipher.ts' -import { startSync } from './sync.ts' -import { filterSuggestedArtefacts, syncLockId, syncState } from './operations.ts' +import { startSync, enqueueArtefactSync } from './sync.ts' +import { filterSuggestedArtefacts, annotateLocalState, syncLockId, syncState } from './operations.ts' import * as postReqBody from '#doc/remote-registries/post-req/index.ts' import * as patchReqBody from '#doc/remote-registries/patch-req/index.ts' @@ -135,10 +136,52 @@ router.get('/:id/remote-artefacts', async (req, res, next) => { // not already selected — a deprecated artefact is not suggested for new // mirroring but stays visible if it is already mirrored. const params: Record = { size: String(size), skip: String(skip), includeDeprecated: 'true' } - if (req.query.q) params.q = req.query.q as string + for (const key of ['q', 'category', 'format'] as const) { + if (typeof req.query[key] === 'string' && req.query[key]) params[key] = req.query[key] as string + } const remote = await ax.get('/api/v1/artefacts', { params }) - res.json(filterSuggestedArtefacts(remote.data, doc.selectedArtefacts)) + const suggested = filterSuggestedArtefacts(remote.data, doc.selectedArtefacts) + // One local read for the page: the admin table shows, per row, whether the + // mirror exists and whether it is behind the upstream. + const locals = await mongo.artefacts + .find({ _id: { $in: suggested.results.map((a: { _id: string }) => a._id) } }, { projection: { dataUpdatedAt: 1, origin: 1 } }) + .toArray() + res.json({ ...suggested, results: annotateLocalState(suggested.results, locals, doc._id) }) + } catch (err) { next(err) } +}) + +const remoteThumbnailTypes = new Set(['image/webp', 'image/svg+xml']) + +// Proxy an upstream thumbnail for the admin's selection table. The upstream +// url is the one the *server* reaches (possibly an internal one), and the +// upstream sets Cross-Origin-Resource-Policy: same-origin on its assets, so the +// browser cannot load them directly. +router.get('/:id/remote-thumbnails/:thumbnailId/data', async (req, res, next) => { + try { + await session.reqAdminMode(req) + const doc = await mongo.remoteRegistries.findOne({ _id: req.params.id }) + if (!doc) throw httpError(404, 'remote registry not found') + if (!/^[\w-]+$/.test(req.params.thumbnailId)) throw httpError(400, 'invalid thumbnail id') + + const ax = axiosBuilder({ baseURL: doc._id, headers: { 'x-api-key': decipher(doc.apiKey) } }) + const remote = await ax.get(`/api/v1/thumbnails/${req.params.thumbnailId}/data`, { responseType: 'stream', validateStatus: () => true }) + if (remote.status !== 200) throw httpError(404, 'thumbnail not found') + // The upstream is another deployment: never relay its Content-Type blindly + // onto our origin. Only the types a registry stores are accepted, and SVG + // is served sandboxed so it cannot run scripts if opened as a document. + const contentType = String(remote.headers['content-type'] || '').split(';')[0].trim().toLowerCase() + if (!remoteThumbnailTypes.has(contentType)) { + remote.data.destroy() + throw httpError(415, 'unsupported thumbnail type') + } + res.set('Content-Type', contentType) + res.set('X-Content-Type-Options', 'nosniff') + res.set('Content-Security-Policy', "default-src 'none'; sandbox") + if (remote.headers['content-length']) res.set('Content-Length', remote.headers['content-length']) + // Upstream ids change on every replace, so the bytes behind one id never do. + res.set('Cache-Control', 'private, max-age=31536000, immutable') + await pipeline(remote.data, res) } catch (err) { next(err) } }) @@ -171,6 +214,9 @@ router.post('/:id/selected-artefacts', async (req, res, next) => { $set: { updatedAt: new Date().toISOString() } } ) + // Mirror it right away rather than waiting for the daily job or a manual + // full sync; progress is published on the registry's ws channel. + await enqueueArtefactSync(req.params.id, artefactId) res.status(201).json({ artefactId }) } catch (err) { next(err) } }) @@ -185,7 +231,7 @@ router.delete('/:id/selected-artefacts/:artefactId', async (req, res, next) => { await mongo.remoteRegistries.updateOne( { _id: req.params.id }, { - $pull: { selectedArtefacts: req.params.artefactId }, + $pull: { selectedArtefacts: req.params.artefactId, pendingSync: req.params.artefactId }, $set: { updatedAt: new Date().toISOString() } } ) diff --git a/api/src/remote-registries/sync.ts b/api/src/remote-registries/sync.ts index d12f68d..b813a5a 100644 --- a/api/src/remote-registries/sync.ts +++ b/api/src/remote-registries/sync.ts @@ -1,4 +1,5 @@ import { randomUUID } from 'node:crypto' +import { Binary } from 'mongodb' import locks from '@data-fair/lib-node/locks.js' import { axiosBuilder } from '@data-fair/lib-node/axios.js' import { internalError } from '@data-fair/lib-node/observer.js' @@ -31,6 +32,40 @@ const emitSync = async (remoteRegistryId: string, event: SyncEvent) => { } } +type RemoteThumbnail = NonNullable + +// Mirror the upstream thumbnail, keeping its id: a thumbnail's id changes on +// every replace upstream, so comparing ids is enough to know whether the local +// copy is current. Runs after the artefact doc exists locally, and outside the +// tarball fast path — a thumbnail change never bumps dataUpdatedAt. +const syncThumbnail = async ( + ax: AxiosInstance, + artefactId: string, + remoteThumbnail: RemoteThumbnail | undefined, + localThumbnail: RemoteThumbnail | undefined +) => { + if (remoteThumbnail?.id === localThumbnail?.id) return + if (!remoteThumbnail) { + await mongo.thumbnails.deleteMany({ artefactId }) + await mongo.artefacts.updateOne({ _id: artefactId }, { $unset: { thumbnail: '' } }) + return + } + const res = await ax.get(`/api/v1/thumbnails/${remoteThumbnail.id}/data`, { responseType: 'arraybuffer' }) + const data = Buffer.from(res.data) + await mongo.thumbnails.deleteMany({ artefactId }) + await mongo.thumbnails.insertOne({ + _id: remoteThumbnail.id, + artefactId, + data: new Binary(data), + mimeType: res.headers['content-type'] === 'image/svg+xml' ? 'image/svg+xml' : 'image/webp', + width: remoteThumbnail.width, + height: remoteThumbnail.height, + byteSize: data.byteLength, + createdAt: new Date().toISOString() + }) + await mongo.artefacts.updateOne({ _id: artefactId }, { $set: { thumbnail: remoteThumbnail } }) +} + const syncNpmArtefact = async (ax: AxiosInstance, remoteUrl: string, artefactId: string) => { const encodedId = encodeURIComponent(artefactId) const remoteRes = await ax.get(`/api/v1/artefacts/${encodedId}`) @@ -40,6 +75,7 @@ const syncNpmArtefact = async (ax: AxiosInstance, remoteUrl: string, artefactId: // Fast path: same upstream dataUpdatedAt means no new upload to mirror. if (local?.path && local.dataUpdatedAt === remoteArtefact.dataUpdatedAt) { + await syncThumbnail(ax, artefactId, remoteArtefact.thumbnail, local.thumbnail) return } @@ -60,6 +96,7 @@ const syncNpmArtefact = async (ax: AxiosInstance, remoteUrl: string, artefactId: packageName: remoteArtefact.packageName, version: remoteArtefact.version, licence: remoteArtefact.licence, + ...(remoteArtefact.packageDescription ? { packageDescription: remoteArtefact.packageDescription } : {}), category: remoteArtefact.category, deprecated: !!remoteArtefact.deprecated, hasNativeModules: !!remoteArtefact.hasNativeModules, @@ -87,6 +124,7 @@ const syncNpmArtefact = async (ax: AxiosInstance, remoteUrl: string, artefactId: if (oldPath && oldPath !== localPath) { await filesStorage.delete(oldPath).catch(() => {}) } + await syncThumbnail(ax, artefactId, remoteArtefact.thumbnail, local?.thumbnail) } const syncFileArtefact = async (ax: AxiosInstance, remoteUrl: string, artefactId: string) => { @@ -145,15 +183,32 @@ const syncFileArtefact = async (ax: AxiosInstance, remoteUrl: string, artefactId { $set: { origin: remoteUrl } } ) } + await syncThumbnail(ax, artefactId, remoteArtefact.thumbnail, local?.thumbnail) +} + +// Atomically take the queued selections, so two drains can't sync the same id twice. +const drainPendingSync = async (remoteRegistryId: string): Promise => { + const doc = await mongo.remoteRegistries.findOneAndUpdate( + { _id: remoteRegistryId }, + { $unset: { pendingSync: '' } }, + { returnDocument: 'before', projection: { pendingSync: 1 } } + ) + return doc?.pendingSync ?? [] } -// The actual work. Callers own the lock. -const runSync = async (remoteRegistryId: string) => { +export type SyncScope = 'all' | 'pending' + +// The actual work. Callers own the lock. `pending` syncs only the artefacts +// queued by selections (see pendingSync); `all` walks every selected artefact. +// Either way, selections queued while this run was in flight are drained before +// returning, so the lock is only released once nothing is left to sync. +const runSync = async (remoteRegistryId: string, scope: SyncScope = 'all') => { const remote = await mongo.remoteRegistries.findOne({ _id: remoteRegistryId }) if (!remote) return + const artefactIds = scope === 'pending' ? await drainPendingSync(remoteRegistryId) : remote.selectedArtefacts const startedAt = new Date().toISOString() - const total = remote.selectedArtefacts.length + const total = artefactIds.length let done = 0 await mongo.remoteRegistries.updateOne( @@ -171,7 +226,7 @@ const runSync = async (remoteRegistryId: string) => { let hasErrors = false let lastError = '' - for (const artefactId of remote.selectedArtefacts) { + for (const artefactId of artefactIds) { await mongo.remoteRegistries.updateOne( { _id: remoteRegistryId }, { $set: { 'syncProgress.currentArtefact': artefactId } } @@ -230,19 +285,41 @@ const runSync = async (remoteRegistryId: string) => { lastSyncStatus, ...(hasErrors ? { lastSyncError: lastError } : {}) }) + + // A full run already covered anything queued meanwhile only if it was + // selected before the run read selectedArtefacts; draining is cheap (the + // dataUpdatedAt fast path) and keeps the queue semantics simple. + const pending = await mongo.remoteRegistries.findOne({ _id: remoteRegistryId }, { projection: { pendingSync: 1 } }) + if (pending?.pendingSync?.length) await runSync(remoteRegistryId, 'pending') } // Returns as soon as the lock is taken; the work continues in the background. // A held lock is a conflict the caller (a human clicking a button) should see. -export const startSync = async (remoteRegistryId: string): Promise => { +export const startSync = async (remoteRegistryId: string, scope: SyncScope = 'all'): Promise => { const lockId = syncLockId(remoteRegistryId) if (!await locks.acquire(lockId)) return false - runSync(remoteRegistryId) + runSync(remoteRegistryId, scope) .catch(err => internalError('sync-remote-registry', err)) - .finally(() => locks.release(lockId).catch(err => internalError('sync-remote-registry-release', err))) + .finally(async () => { + await locks.release(lockId).catch(err => internalError('sync-remote-registry-release', err)) + // A selection can land between the final drain and the release above; + // its own startSync lost the lock race, so pick it up here. + const doc = await mongo.remoteRegistries.findOne({ _id: remoteRegistryId }, { projection: { pendingSync: 1 } }) + if (doc?.pendingSync?.length) await startSync(remoteRegistryId, 'pending') + }) return true } +// Queue one freshly selected artefact and sync it in the background. If a sync +// already holds the lock, that run drains the queue before releasing it. +export const enqueueArtefactSync = async (remoteRegistryId: string, artefactId: string) => { + await mongo.remoteRegistries.updateOne( + { _id: remoteRegistryId }, + { $addToSet: { pendingSync: artefactId } } + ) + await startSync(remoteRegistryId, 'pending') +} + // Awaits completion. Used by the daily job, which syncs registries one at a time. export const syncRemoteRegistry = async (remoteRegistryId: string): Promise => { const lockId = syncLockId(remoteRegistryId) diff --git a/api/types/artefact/schema.js b/api/types/artefact/schema.js index 51b7a41..5f59c8b 100644 --- a/api/types/artefact/schema.js +++ b/api/types/artefact/schema.js @@ -16,6 +16,9 @@ export default { packageName: { type: 'string', readOnly: true }, version: { type: 'string', readOnly: true }, licence: { type: 'string', readOnly: true }, + // The npm manifest's own description (technical, not localized); the + // editable `description` below is the curated one. + packageDescription: { type: 'string', readOnly: true }, category: { type: 'string', enum: ['processing', 'catalog', 'application', 'tileset', 'maplibre-style', 'other'] @@ -33,8 +36,8 @@ export default { additionalProperties: false, layout: { if: '!context.accessOnly' }, properties: { - en: { type: 'string', title: 'Title - English', 'x-i18n-title': { fr: 'Titre - Anglais' }, layout: { cols: { md: 6 } } }, - fr: { type: 'string', title: 'Title - French', 'x-i18n-title': { fr: 'Titre - Français' }, layout: { cols: { md: 6 } } } + en: { type: 'string', title: 'Title - English', 'x-i18n-title': { fr: 'Titre - Anglais' }, layout: { cols: { sm: 6 } } }, + fr: { type: 'string', title: 'Title - French', 'x-i18n-title': { fr: 'Titre - Français' }, layout: { cols: { sm: 6 } } } } }, description: { @@ -42,8 +45,8 @@ export default { additionalProperties: false, layout: { if: '!context.accessOnly' }, properties: { - en: { type: 'string', title: 'Description - English', 'x-i18n-title': { fr: 'Description - Anglais' }, layout: { comp: 'textarea', props: { autoGrow: true, rows: 3 }, cols: { md: 6 } } }, - fr: { type: 'string', title: 'Description - French', 'x-i18n-title': { fr: 'Description - Français' }, layout: { comp: 'textarea', props: { autoGrow: true, rows: 3 }, cols: { md: 6 } } } + en: { type: 'string', title: 'Description - English', 'x-i18n-title': { fr: 'Description - Anglais' }, layout: { comp: 'textarea', props: { autoGrow: true, rows: 3 }, cols: { sm: 6 } } }, + fr: { type: 'string', title: 'Description - French', 'x-i18n-title': { fr: 'Description - Français' }, layout: { comp: 'textarea', props: { autoGrow: true, rows: 3 }, cols: { sm: 6 } } } } }, group: { @@ -57,7 +60,7 @@ export default { 'x-i18n-title': { fr: 'Groupe - Anglais' }, layout: { comp: 'combobox', - cols: { md: 6 }, + cols: { sm: 6 }, getItems: { url: '${context.apiPath}/v1/artefacts/groups?category=${context.category}&locale=en', itemsResults: 'data.results' @@ -70,7 +73,7 @@ export default { 'x-i18n-title': { fr: 'Groupe - Français' }, layout: { comp: 'combobox', - cols: { md: 6 }, + cols: { sm: 6 }, getItems: { url: '${context.apiPath}/v1/artefacts/groups?category=${context.category}&locale=fr', itemsResults: 'data.results' @@ -90,18 +93,27 @@ export default { height: { type: 'integer' } } }, + documentation: { + type: 'string', + format: 'uri', + title: 'Documentation URL', + 'x-i18n-title': { fr: 'URL de documentation' }, + layout: { if: '!context.accessOnly' } + }, + // Property order is form order: documentation above, then the two + // switches side by side. deprecated: { type: 'boolean', title: 'Deprecated', 'x-i18n-title': { fr: 'Déprécié' }, - layout: { comp: 'switch', if: '!context.accessOnly' }, + layout: { comp: 'switch', if: '!context.accessOnly', cols: { sm: 6 } }, default: false }, public: { type: 'boolean', title: 'Public', 'x-i18n-title': { fr: 'Public' }, - layout: { comp: 'switch', if: 'context.accessOnly || !context.mirrored' }, + layout: { comp: 'switch', if: 'context.accessOnly || !context.mirrored', cols: { sm: 6 } }, default: false }, privateAccess: { @@ -132,13 +144,6 @@ export default { } } }, - documentation: { - type: 'string', - format: 'uri', - title: 'Documentation URL', - 'x-i18n-title': { fr: 'URL de documentation' }, - layout: { if: '!context.accessOnly' } - }, origin: { type: 'string', readOnly: true }, // `fileName` is only used by format=file. fileName: { type: 'string', readOnly: true }, diff --git a/api/types/remote-registry/schema.js b/api/types/remote-registry/schema.js index a1e1214..334e38b 100644 --- a/api/types/remote-registry/schema.js +++ b/api/types/remote-registry/schema.js @@ -23,6 +23,12 @@ export default { type: 'array', items: { type: 'string' } }, + // Artefacts selected while a sync was already running. The running sync + // drains this queue before releasing its lock, so a selection is never lost. + pendingSync: { + type: 'array', + items: { type: 'string' } + }, lastSyncAt: { type: 'string', format: 'date-time' }, lastSyncStatus: { type: 'string', enum: ['success', 'error'] }, lastSyncError: { type: 'string' }, diff --git a/dev/fixtures.ts b/dev/fixtures.ts index 6939adf..0877667 100644 --- a/dev/fixtures.ts +++ b/dev/fixtures.ts @@ -102,10 +102,10 @@ async function main () { // In the unified model each artefact id is one major line; uploading to // /npm/:id stores a single tarball (the `noarch` slot). The category is // taken from the multipart form field, never from the package manifest. - const npmSpecs: { name: string, category: string, version: string }[] = [ - { name: '@koumoul/processing-hello', category: 'processing', version: '1.1.0' }, - { name: '@koumoul/application-demo', category: 'application', version: '1.0.0' }, - { name: '@test/catalog-sample', category: 'catalog', version: '2.0.0' } + const npmSpecs: { name: string, category: string, version: string, description: string }[] = [ + { name: '@koumoul/processing-hello', category: 'processing', version: '1.1.0', description: 'Hello world processing plugin for data-fair' }, + { name: '@koumoul/application-demo', category: 'application', version: '1.0.0', description: 'Demo application showcasing the registry' }, + { name: '@test/catalog-sample', category: 'catalog', version: '2.0.0', description: 'Sample catalog connector' } ] for (const spec of npmSpecs) { @@ -115,7 +115,7 @@ async function main () { console.log(` ✓ npm ${id} (skipped)`) continue } - const tarball = await createTestTarball({ name: spec.name, version: spec.version, licence: 'MIT' }) + const tarball = await createTestTarball({ name: spec.name, version: spec.version, licence: 'MIT', description: spec.description }) const form = new FormData() form.append('file', tarball, { filename: 'package.tgz', contentType: 'application/gzip' }) form.append('category', spec.category) @@ -265,7 +265,7 @@ async function main () { if (await upstreamArtefactExists(UPSTREAM_NPM_ID)) { console.log(` ✓ upstream npm ${UPSTREAM_NPM_ID} (skipped)`) } else { - const tarball = await createTestTarball({ name: '@upstream/processing-remote', version: '1.0.0', licence: 'MIT' }) + const tarball = await createTestTarball({ name: '@upstream/processing-remote', version: '1.0.0', licence: 'MIT', description: 'Processing plugin published on the upstream registry' }) const form = new FormData() form.append('file', tarball, { filename: 'package.tgz', contentType: 'application/gzip' }) form.append('category', 'processing') diff --git a/tests/artefact-admin.e2e.spec.ts b/tests/artefact-admin.e2e.spec.ts index a957347..ef77e2d 100644 --- a/tests/artefact-admin.e2e.spec.ts +++ b/tests/artefact-admin.e2e.spec.ts @@ -1,5 +1,6 @@ import { test, expect, type Page } from '@playwright/test' import FormData from 'form-data' +import sharp from 'sharp' import { superAdmin, axiosWithApiKey, clean, setArtefactOrigin } from './support/axios.ts' import { createTestTarball } from './support/test-tarball.ts' @@ -44,7 +45,7 @@ test.describe('Artefact admin metadata editing', () => { // The e2e environment renders the UI in French. test('existing group and documentation values are loaded into the edit form', async ({ page }) => { await page.goto('/registry/artefacts/' + encodeURIComponent(pkgId)) - await expect(page.locator('#artefact-admin')).toBeVisible() + await expect(page.locator('#artefact-edit')).toBeVisible() await expect(page.getByLabel('Groupe - Anglais')).toHaveValue('Geo tools') await expect(page.getByLabel('Groupe - Français')).toHaveValue('Outils géo') await expect(page.getByLabel('URL de documentation')).toHaveValue('https://example.com/docs') @@ -52,12 +53,12 @@ test.describe('Artefact admin metadata editing', () => { test('editing the group and saving persists the new value', async ({ page }) => { await page.goto('/registry/artefacts/' + encodeURIComponent(pkgId)) - await expect(page.locator('#artefact-admin')).toBeVisible() + await expect(page.locator('#artefact-edit')).toBeVisible() const groupEn = page.getByLabel('Groupe - Anglais') await groupEn.fill('Mapping tools') await groupEn.blur() - await page.locator('#artefact-admin').getByRole('button', { name: 'Enregistrer' }).click() + await page.locator('#artefact-edit').getByRole('button', { name: 'Enregistrer' }).click() await expect(page.getByText('Modifications enregistrées')).toBeVisible() await page.reload() @@ -66,7 +67,7 @@ test.describe('Artefact admin metadata editing', () => { test('warns before navigating away with unsaved changes', async ({ page }) => { await page.goto('/registry/artefacts/' + encodeURIComponent(pkgId)) - await expect(page.locator('#artefact-admin')).toBeVisible() + await expect(page.locator('#artefact-edit')).toBeVisible() const groupEn = page.getByLabel('Groupe - Anglais') await groupEn.fill('Unsaved change') @@ -75,13 +76,31 @@ test.describe('Artefact admin metadata editing', () => { // Dismissing the confirmation keeps us on the page. page.once('dialog', dialog => dialog.dismiss()) await page.getByRole('link', { name: 'Administration' }).click() - await expect(page.locator('#artefact-admin')).toBeVisible() + await expect(page.locator('#artefact-edit')).toBeVisible() // Accepting it lets the navigation through. page.once('dialog', dialog => dialog.accept()) await page.getByRole('link', { name: 'Administration' }).click() await expect(page).toHaveURL(/\/admin/) }) + + test('a picked thumbnail is staged and uploaded with the metadata save', async ({ page }) => { + await page.goto('/registry/artefacts/' + encodeURIComponent(pkgId)) + const form = page.locator('#artefact-edit') + await expect(form).toBeVisible() + await expect(form.locator('img.thumbnail-preview')).toHaveCount(0) + + const png = await sharp({ create: { width: 120, height: 80, channels: 3, background: { r: 200, g: 30, b: 30 } } }).png().toBuffer() + await form.locator('input[type=file]').setInputFiles({ name: 'thumb.png', mimeType: 'image/png', buffer: png }) + // picking a file is an unsaved change like any other field + await expect(form.locator('img.thumbnail-preview')).toBeVisible() + await form.getByRole('button', { name: 'Enregistrer' }).click() + await expect(page.getByText('Modifications enregistrées')).toBeVisible() + + await page.reload() + await expect(form.locator('img.thumbnail-preview')).toBeVisible() + await expect(form.locator('img.thumbnail-preview')).toHaveAttribute('src', /\/v1\/thumbnails\/.+\/data$/) + }) }) test.describe('Artefact admin metadata editing for a mirrored artefact', () => { @@ -91,7 +110,7 @@ test.describe('Artefact admin metadata editing for a mirrored artefact', () => { test('remote-owned metadata is read-only while local access stays editable', async ({ page }) => { await page.goto('/registry/artefacts/' + encodeURIComponent(mirrorId)) - await expect(page.locator('#artefact-admin')).toBeVisible() + await expect(page.locator('#artefact-edit')).toBeVisible() // Mirror notice is shown and the remote-owned metadata appears read-only. await expect(page.getByText('mirroré depuis un registre distant')).toBeVisible() @@ -105,12 +124,12 @@ test.describe('Artefact admin metadata editing for a mirrored artefact', () => { test('toggling local access and saving succeeds (only public/privateAccess are sent)', async ({ page }) => { await page.goto('/registry/artefacts/' + encodeURIComponent(mirrorId)) - await expect(page.locator('#artefact-admin')).toBeVisible() + await expect(page.locator('#artefact-edit')).toBeVisible() // Before the fix the form sent the remote-owned fields too and the API // answered 403; toggling Public and saving must now succeed. await page.getByLabel('Public').click() - await page.locator('#artefact-admin').getByRole('button', { name: 'Enregistrer' }).click() + await page.locator('#artefact-edit').getByRole('button', { name: 'Enregistrer' }).click() await expect(page.getByText('Modifications enregistrées')).toBeVisible() await page.reload() diff --git a/tests/artefacts-operations.unit.spec.ts b/tests/artefacts-operations.unit.spec.ts index 451144b..6e7b6f9 100644 --- a/tests/artefacts-operations.unit.spec.ts +++ b/tests/artefacts-operations.unit.spec.ts @@ -55,6 +55,22 @@ test.describe('extractManifest', () => { expect(result.manifest.licence).toBe('MIT') }) + test('extracts the package description, trimmed', async () => { + const tarball = await packTarball([ + { name: 'package/package.json', content: manifest({ description: ' A demo plugin ' }) } + ]) + const result = await extractManifest(Readable.from(tarball)) + expect(result.manifest.description).toBe('A demo plugin') + }) + + test('ignores a non-string package description', async () => { + const tarball = await packTarball([ + { name: 'package/package.json', content: manifest({ description: { fr: 'nope' } }) } + ]) + const result = await extractManifest(Readable.from(tarball)) + expect(result.manifest.description).toBeUndefined() + }) + test('does not extract a category from package.json (category comes from the upload form field)', async () => { const tarball = await packTarball([ { name: 'package/package.json', content: manifest({ registry: { category: 'processing' } }) } diff --git a/tests/artefacts.api.spec.ts b/tests/artefacts.api.spec.ts index 4dd7cdf..9cf2e85 100644 --- a/tests/artefacts.api.spec.ts +++ b/tests/artefacts.api.spec.ts @@ -26,7 +26,8 @@ test.describe('Artefacts', () => { const tarball = await createTestTarball({ name: '@data-fair/processing-gpkg', version: '1.2.3', - licence: 'MIT' + licence: 'MIT', + description: 'GeoPackage import' }) const form = new FormData() form.append('file', tarball, { filename: 'package.tgz', contentType: 'application/gzip' }) @@ -44,6 +45,7 @@ test.describe('Artefacts', () => { expect(res.data.artefact.packageName).toBe('@data-fair/processing-gpkg') expect(res.data.artefact.version).toBe('1.2.3') expect(res.data.artefact.category).toBe('processing') + expect(res.data.artefact.packageDescription).toBe('GeoPackage import') expect(typeof res.data.artefact.path).toBe('string') expect(typeof res.data.artefact.size).toBe('number') expect(res.data.artefact.size).toBeGreaterThan(0) @@ -408,4 +410,61 @@ test.describe('Artefacts', () => { expect(list.data.results[0]._id).toBe('@test/pkg@1') }) }) + + test.describe('Sorting', () => { + const upload = async (name: string, version: string, group?: { en: string, fr: string }) => { + const id = name + '@1' + const ax = axiosWithApiKey(uploadApiKey) + const form = new FormData() + form.append('file', await createTestTarball({ name, version }), { filename: 'p.tgz', contentType: 'application/gzip' }) + await ax.post('/api/v1/artefacts/npm/' + encodeURIComponent(id), form, { headers: form.getHeaders() }) + if (group) { + const admin = await superAdmin + await admin.patch('/api/v1/artefacts/' + encodeURIComponent(id), { group }) + } + return id + } + + test.beforeEach(async () => { + await upload('@test/b', '2.0.0', { en: 'Zebra', fr: 'Zèbre' }) + await upload('@test/a', '1.0.0', { en: 'Apple', fr: 'Pomme' }) + await upload('@test/c', '3.0.0') + }) + + test('sort=group.en orders by the English group (ungrouped first, as Mongo sorts missing fields)', async () => { + const admin = await superAdmin + const res = await admin.get('/api/v1/artefacts?sort=group.en') + expect(res.data.results.map((a: any) => a._id)).toEqual(['@test/c@1', '@test/a@1', '@test/b@1']) + }) + + test('a leading dash reverses the order', async () => { + const admin = await superAdmin + const res = await admin.get('/api/v1/artefacts?sort=-name') + expect(res.data.results.map((a: any) => a._id)).toEqual(['@test/c@1', '@test/b@1', '@test/a@1']) + }) + + test('sort=version orders by version', async () => { + const admin = await superAdmin + const res = await admin.get('/api/v1/artefacts?sort=-version') + expect(res.data.results.map((a: any) => a.version)).toEqual(['3.0.0', '2.0.0', '1.0.0']) + }) + + test('an unknown sort key returns 400', async () => { + const admin = await superAdmin + try { + await admin.get('/api/v1/artefacts?sort=nope') + expect(true).toBe(false) + } catch (err: any) { + expect(err.status).toBe(400) + } + }) + + test('sort=public is admin-only and ignored for others', async () => { + const admin = await superAdmin + await admin.patch('/api/v1/artefacts/' + encodeURIComponent('@test/c@1'), { public: true }) + const res = await anonymousAx.get('/api/v1/artefacts?sort=-public') + // anonymous only sees the public one anyway; the point is no 400 + expect(res.status).toBe(200) + }) + }) }) diff --git a/tests/remote-registries-operations.unit.spec.ts b/tests/remote-registries-operations.unit.spec.ts index 1f30369..004235f 100644 --- a/tests/remote-registries-operations.unit.spec.ts +++ b/tests/remote-registries-operations.unit.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from '@playwright/test' -import { filterSuggestedArtefacts, syncLockId, syncChannel, syncState } from '../api/src/remote-registries/operations.ts' +import { filterSuggestedArtefacts, syncLockId, syncChannel, syncState, annotateLocalState } from '../api/src/remote-registries/operations.ts' test.describe('filterSuggestedArtefacts', () => { test('keeps non-deprecated artefacts and recomputes count', () => { @@ -70,3 +70,41 @@ test.describe('syncState', () => { expect(syncState(false, { syncProgress: { startedAt: '2026-07-09T12:00:00.000Z' } })).toBe('interrupted') }) }) + +test.describe('annotateLocalState', () => { + const remote = [ + { _id: 'a', dataUpdatedAt: '2026-01-02T00:00:00.000Z' }, + { _id: 'b', dataUpdatedAt: '2026-01-02T00:00:00.000Z' }, + { _id: 'c', dataUpdatedAt: '2026-01-02T00:00:00.000Z' } + ] + const origin = 'https://up.example.com' + const local = [ + { _id: 'a', dataUpdatedAt: '2026-01-02T00:00:00.000Z', origin }, + { _id: 'b', dataUpdatedAt: '2026-01-01T00:00:00.000Z', origin } + ] + + test('a mirrored artefact with the same dataUpdatedAt is up to date', () => { + const out = annotateLocalState(remote, local, origin) + expect(out[0].local).toEqual({ synced: true, upToDate: true, dataUpdatedAt: '2026-01-02T00:00:00.000Z' }) + }) + + test('a mirrored artefact with an older dataUpdatedAt is stale', () => { + const out = annotateLocalState(remote, local, origin) + expect(out[1].local).toEqual({ synced: true, upToDate: false, dataUpdatedAt: '2026-01-01T00:00:00.000Z' }) + }) + + test('an artefact absent locally is not synced', () => { + const out = annotateLocalState(remote, local, origin) + expect(out[2].local).toEqual({ synced: false, upToDate: false }) + }) + + test('a local artefact with the same id but another (or no) origin is a conflict', () => { + const out = annotateLocalState(remote, [ + { _id: 'a', dataUpdatedAt: '2026-01-02T00:00:00.000Z' }, + { _id: 'b', dataUpdatedAt: '2026-01-02T00:00:00.000Z', origin: 'https://other.example.com' } + ], origin) + expect(out[0].local).toEqual({ synced: false, upToDate: false, conflict: true }) + expect(out[1].local).toEqual({ synced: false, upToDate: false, conflict: true }) + expect(out[2].local).toEqual({ synced: false, upToDate: false }) + }) +}) diff --git a/tests/remote-registries-sync.api.spec.ts b/tests/remote-registries-sync.api.spec.ts index e7ac176..c2af3f0 100644 --- a/tests/remote-registries-sync.api.spec.ts +++ b/tests/remote-registries-sync.api.spec.ts @@ -7,8 +7,9 @@ import { test, expect } from '@playwright/test' import FormData from 'form-data' +import sharp from 'sharp' import { - superAdmin, clean, + superAdmin, clean, waitSyncIdle, upstreamBaseURL, upstreamSuperAdmin, upstreamAxiosAuth, upstreamAxiosWithApiKey, cleanUpstream } from './support/axios.ts' import { createTestTarball } from './support/test-tarball.ts' @@ -47,23 +48,55 @@ const seedUpstream = async () => { return { readKey: readRes.data.key as string, uploadKey: keyRes.data.key as string } } +const selectArtefact = async (artefactId: string) => { + const admin = await superAdmin + await admin.post( + `/api/v1/remote-registries/${encodeURIComponent(upstreamBaseURL())}/selected-artefacts`, + { artefactId } + ) +} + +// Selecting kicks off a background sync of that artefact; wait for it so the +// tests below start from a settled registry. const registerMirror = async (readKey: string, artefactIds: string[]) => { const admin = await superAdmin await admin.post('/api/v1/remote-registries', { url: upstreamBaseURL(), name: 'Upstream', apiKey: readKey }) - for (const artefactId of artefactIds) { - await admin.post( - `/api/v1/remote-registries/${encodeURIComponent(upstreamBaseURL())}/selected-artefacts`, - { artefactId } - ) - } + for (const artefactId of artefactIds) await selectArtefact(artefactId) + await waitSyncIdle(upstreamBaseURL()) +} + +const republishUpstream = async (uploadKey: string, version: string) => { + const upload = upstreamAxiosWithApiKey(uploadKey) + const tarball = await createTestTarball({ name: '@up/pkg', version, licence: 'MIT' }) + const form = new FormData() + form.append('file', tarball, { filename: 'package.tgz', contentType: 'application/gzip' }) + form.append('category', 'processing') + await upload.post('/api/v1/artefacts/npm/' + encodeURIComponent(NPM_ID), form, { headers: form.getHeaders() }) +} + +const uploadUpstreamThumbnail = async (artefactId: string) => { + const admin = await upstreamSuperAdmin() + const png = await sharp({ create: { width: 200, height: 100, channels: 3, background: { r: 10, g: 120, b: 200 } } }).png().toBuffer() + const form = new FormData() + form.append('file', png, { filename: 'thumb.png', contentType: 'image/png' }) + const res = await admin.post(`/api/v1/artefacts/${encodeURIComponent(artefactId)}/thumbnail`, form, { headers: form.getHeaders() }) + return res.data.thumbnail as { id: string, width: number, height: number } +} + +const listRemote = async (params: Record = {}) => { + const admin = await superAdmin + const res = await admin.get(`/api/v1/remote-registries/${encodeURIComponent(upstreamBaseURL())}/remote-artefacts`, { params }) + return res.data as { results: any[], count: number } } -// Triggers a sync and waits for it to settle. `previousLastSyncAt` distinguishes a -// fresh completion from the previous one — a bare `lastSyncStatus` check would -// return instantly on the second sync within a test. -const runSync = async (previousLastSyncAt?: string) => { +// Triggers a full sync and waits for it to settle. The completion is told apart +// from the previous one (a selection's auto-sync, or an earlier runSync) by a +// fresh lastSyncAt — a bare `lastSyncStatus` check would return instantly. +const runSync = async () => { const admin = await superAdmin const id = encodeURIComponent(upstreamBaseURL()) + const before = await admin.get(`/api/v1/remote-registries/${id}`) + const previousLastSyncAt = before.data.lastSyncAt await admin.post(`/api/v1/remote-registries/${id}/sync`) for (let i = 0; i < 100; i++) { const res = await admin.get(`/api/v1/remote-registries/${id}`) @@ -135,10 +168,10 @@ test.describe('Federation sync against a real upstream registry', () => { test('a re-sync with no upstream change does not re-download', async () => { await registerMirror(readKey, [NPM_ID]) - const first = await runSync() + await runSync() const before = await getLocal(NPM_ID) - await runSync(first.lastSyncAt) + await runSync() const after = await getLocal(NPM_ID) // the dataUpdatedAt fast path in syncNpmArtefact short-circuits @@ -148,17 +181,12 @@ test.describe('Federation sync against a real upstream registry', () => { test('an upstream republish is picked up on the next sync', async () => { await registerMirror(readKey, [NPM_ID]) - const first = await runSync() + await runSync() const before = await getLocal(NPM_ID) - const upload = upstreamAxiosWithApiKey(uploadKey) - const tarball = await createTestTarball({ name: '@up/pkg', version: '2.0.0', licence: 'MIT' }) - const form = new FormData() - form.append('file', tarball, { filename: 'package.tgz', contentType: 'application/gzip' }) - form.append('category', 'processing') - await upload.post('/api/v1/artefacts/npm/' + encodeURIComponent(NPM_ID), form, { headers: form.getHeaders() }) + await republishUpstream(uploadKey, '2.0.0') - await runSync(first.lastSyncAt) + await runSync() const after = await getLocal(NPM_ID) expect(after.version).toBe('2.0.0') @@ -216,4 +244,110 @@ test.describe('Federation sync against a real upstream registry', () => { const local = await getLocal(NPM_ID) expect(local.origin).toBeUndefined() }) + + test('selecting an artefact mirrors it without a manual sync', async () => { + const admin = await superAdmin + await admin.post('/api/v1/remote-registries', { url: upstreamBaseURL(), name: 'Upstream', apiKey: readKey }) + await selectArtefact(NPM_ID) + const registry = await waitSyncIdle(upstreamBaseURL()) + + expect(registry.lastSyncStatus).toBe('success') + const local = await getLocal(NPM_ID) + expect(local.origin).toBe(upstreamBaseURL()) + expect(local.version).toBe('1.0.0') + }) + + test('selecting several artefacts in a row mirrors them all', async () => { + const admin = await superAdmin + await admin.post('/api/v1/remote-registries', { url: upstreamBaseURL(), name: 'Upstream', apiKey: readKey }) + // no await between the two: the second lands while the first sync holds the lock + await Promise.all([selectArtefact(NPM_ID), selectArtefact(FILE_ID)]) + const registry = await waitSyncIdle(upstreamBaseURL()) + + expect(registry.lastSyncStatus).toBe('success') + expect(registry.pendingSync ?? []).toEqual([]) + expect((await getLocal(NPM_ID)).origin).toBe(upstreamBaseURL()) + expect((await getLocal(FILE_ID)).origin).toBe(upstreamBaseURL()) + }) + + test('the upstream thumbnail is mirrored under the same id', async () => { + const upstreamThumb = await uploadUpstreamThumbnail(NPM_ID) + await registerMirror(readKey, [NPM_ID]) + + const local = await getLocal(NPM_ID) + expect(local.thumbnail).toEqual(upstreamThumb) + + const admin = await superAdmin + const res = await admin.get(`/api/v1/thumbnails/${upstreamThumb.id}/data`, { responseType: 'arraybuffer' }) + expect(res.status).toBe(200) + expect(res.headers['content-type']).toBe('image/webp') + expect(Buffer.from(res.data).length).toBeGreaterThan(0) + }) + + test('an upstream thumbnail removal is mirrored on the next sync', async () => { + const upstreamThumb = await uploadUpstreamThumbnail(NPM_ID) + await registerMirror(readKey, [NPM_ID]) + await runSync() + + const upstreamAdmin = await upstreamSuperAdmin() + await upstreamAdmin.delete(`/api/v1/artefacts/${encodeURIComponent(NPM_ID)}/thumbnail`) + await runSync() + + const local = await getLocal(NPM_ID) + expect(local.thumbnail).toBeUndefined() + const admin = await superAdmin + try { + await admin.get(`/api/v1/thumbnails/${upstreamThumb.id}/data`) + expect(true).toBe(false) + } catch (err: any) { + expect(err.status).toBe(404) + } + }) + + test('remote-artefacts reports the local mirror state of each artefact', async () => { + const admin = await superAdmin + await admin.post('/api/v1/remote-registries', { url: upstreamBaseURL(), name: 'Upstream', apiKey: readKey }) + + const before = (await listRemote()).results.find(a => a._id === NPM_ID) + expect(before.local).toEqual({ synced: false, upToDate: false }) + + await selectArtefact(NPM_ID) + await waitSyncIdle(upstreamBaseURL()) + const synced = (await listRemote()).results.find(a => a._id === NPM_ID) + expect(synced.local.synced).toBe(true) + expect(synced.local.upToDate).toBe(true) + expect(synced.local.dataUpdatedAt).toBe(synced.dataUpdatedAt) + + await republishUpstream(uploadKey, '2.0.0') + const stale = (await listRemote()).results.find(a => a._id === NPM_ID) + expect(stale.local.synced).toBe(true) + expect(stale.local.upToDate).toBe(false) + }) + + test('an upstream thumbnail is served through the local proxy for the admin table', async () => { + const upstreamThumb = await uploadUpstreamThumbnail(NPM_ID) + const admin = await superAdmin + await admin.post('/api/v1/remote-registries', { url: upstreamBaseURL(), name: 'Upstream', apiKey: readKey }) + + const res = await admin.get( + `/api/v1/remote-registries/${encodeURIComponent(upstreamBaseURL())}/remote-thumbnails/${upstreamThumb.id}/data`, + { responseType: 'arraybuffer' } + ) + expect(res.status).toBe(200) + expect(res.headers['content-type']).toBe('image/webp') + expect(res.headers['x-content-type-options']).toBe('nosniff') + expect(res.headers['content-security-policy']).toContain('sandbox') + expect(Buffer.from(res.data).length).toBeGreaterThan(0) + }) + + test('remote-artefacts forwards category and format filters to the upstream', async () => { + const admin = await superAdmin + await admin.post('/api/v1/remote-registries', { url: upstreamBaseURL(), name: 'Upstream', apiKey: readKey }) + + const tilesets = await listRemote({ category: 'tileset' }) + expect(tilesets.results.map(a => a._id)).toEqual([FILE_ID]) + + const npm = await listRemote({ format: 'npm' }) + expect(npm.results.map(a => a._id)).toEqual([NPM_ID]) + }) }) diff --git a/tests/remote-registries.api.spec.ts b/tests/remote-registries.api.spec.ts index d137b2e..079a9b4 100644 --- a/tests/remote-registries.api.spec.ts +++ b/tests/remote-registries.api.spec.ts @@ -1,6 +1,6 @@ import { test, expect } from '@playwright/test' import FormData from 'form-data' -import { superAdmin, axiosAuth, axiosWithApiKey, clean, setArtefactOrigin, holdSyncLock, releaseSyncLock, syncLockExists } from './support/axios.ts' +import { superAdmin, axiosAuth, axiosWithApiKey, clean, setArtefactOrigin, holdSyncLock, releaseSyncLock, syncLockExists, waitSyncIdle } from './support/axios.ts' import { createTestTarball } from './support/test-tarball.ts' let uploadApiKey: string @@ -178,6 +178,9 @@ test.describe('Remote registries', () => { const registry = await ax.get(`/api/v1/remote-registries/${encodedRemoteUrl}`) expect(registry.data.selectedArtefacts).toContain('@test/pkg') + // selecting starts a sync of that artefact right away + expect(registry.data.syncState).toBe('running') + await waitSyncIdle(remoteUrl) }) test('select duplicate returns 409', async () => { @@ -193,6 +196,7 @@ test.describe('Remote registries', () => { } catch (err: any) { expect(err.status).toBe(409) } + await waitSyncIdle(remoteUrl) }) test('select conflicts with existing local artefact', async () => { @@ -220,6 +224,7 @@ test.describe('Remote registries', () => { await ax.post(`/api/v1/remote-registries/${encodedRemoteUrl}/selected-artefacts`, { artefactId: '@test/pkg' }) + await waitSyncIdle(remoteUrl) const res = await ax.delete( `/api/v1/remote-registries/${encodedRemoteUrl}/selected-artefacts/${encodeURIComponent('@test/pkg')}` @@ -244,6 +249,7 @@ test.describe('Remote registries', () => { await ax.post(`/api/v1/remote-registries/${encodedRemoteUrl}/selected-artefacts`, { artefactId: '@test/pkg@1' }) + await waitSyncIdle(remoteUrl) await ax.delete( `/api/v1/remote-registries/${encodedRemoteUrl}/selected-artefacts/${encodeURIComponent('@test/pkg@1')}` ) @@ -468,11 +474,12 @@ test.describe('Remote registries', () => { const ax = await superAdmin const artefactId = '@test/pkg@1' - // Selecting doesn't contact the remote — only checks for a conflicting - // local artefact — so this is safe against a non-resolving remote URL. + // Selecting starts a background sync of the artefact, which fails against + // the non-resolving host. Let it settle so the manual sync below owns the lock. await ax.post(`/api/v1/remote-registries/${encodeURIComponent(url)}/selected-artefacts`, { artefactId }) + const settled = await waitSyncIdle(url) await ax.post('/api/v1/remote-registries/' + encodeURIComponent(url) + '/sync') @@ -481,7 +488,7 @@ test.describe('Remote registries', () => { for (let i = 0; i < 50; i++) { const res = await ax.get('/api/v1/remote-registries/' + encodeURIComponent(url)) doc = res.data - if (doc.lastSyncStatus) break + if (doc.lastSyncAt && doc.lastSyncAt !== settled.lastSyncAt) break await new Promise(resolve => setTimeout(resolve, 100)) } diff --git a/tests/support/axios.ts b/tests/support/axios.ts index 438909d..4846a4a 100644 --- a/tests/support/axios.ts +++ b/tests/support/axios.ts @@ -52,6 +52,19 @@ export const syncLockExists = async (registryId: string): Promise => { return res.data.exists } +// Selecting an artefact starts a background sync of it. Tests that then depend on +// the lock (a manual sync, a held-lock assertion) or on the mirrored copy must +// wait for that sync — and any queued follow-up — to settle first. +export const waitSyncIdle = async (registryId: string) => { + const admin = await superAdmin + for (let i = 0; i < 100; i++) { + const res = await admin.get('/api/v1/remote-registries/' + encodeURIComponent(registryId)) + if (res.data.syncState !== 'running' && !res.data.pendingSync?.length) return res.data + await new Promise(resolve => setTimeout(resolve, 100)) + } + throw new Error('sync did not settle within 10s') +} + // --- federation upstream -------------------------------------------------- // A second registry process, used as a mirror source. See // docs/superpowers/specs/2026-07-10-federation-dev-testing-design.md diff --git a/tests/support/test-tarball.ts b/tests/support/test-tarball.ts index c5891ff..f1a9c69 100644 --- a/tests/support/test-tarball.ts +++ b/tests/support/test-tarball.ts @@ -13,6 +13,7 @@ export interface TarballOptions { name: string version: string licence?: string + description?: string /** Additional entries appended after package/package.json. Useful for native-module signal tests. */ extraEntries?: TarballEntry[] } @@ -22,7 +23,8 @@ export const createTestTarball = async (options: TarballOptions): Promise Date: Tue, 15 Sep 2026 14:25:07 +0200 Subject: [PATCH 2/3] feat(ui): federation and admin UI pass - home: sortable column headers bound to the sort search param (three-state), table in a card, edit pencil for admins, count in the card title - artefact page: editable metadata form ahead of the technical sections, with the thumbnail staged and saved by the same button (fixes lost edits when uploading an image first); tarball and download cards merged into the metadata card; technical description from package.json - remote registry page: artefacts table aligned on the home list with category/format filters, pagination, per-row sync state fed by the ws channel, proxied upstream thumbnails, conflict rows, error notifications, confirmations before unselecting; url/name and key/new key rows - admin sections: full-height admin button, Manage button and delete confirmation for remote registries, account avatars and type in the grants picker and table, localized capitalized status chips, plural rules - form fields default to comfortable density and hide-details auto; remaining schema cols use the sm breakpoint so they apply inside the form width - api key fields are masked text instead of type=password so Chrome stops autofilling them --- ui/components.d.ts | 1 + .../admin/access-grants-section.vue | 54 ++- ui/src/components/admin/api-keys-section.vue | 12 +- .../admin/remote-registries-section.vue | 119 +++-- .../admin/vulnerability-section.vue | 5 +- ui/src/components/artefact-admin.vue | 231 +-------- ui/src/components/artefact-edit.vue | 271 +++++++++++ ui/src/components/artefact-metadata.vue | 145 +++--- ui/src/components/default-layout.vue | 21 +- ui/src/components/vjsf/vjsf-patch-req-en.vue | 444 +++++++++--------- ui/src/components/vjsf/vjsf-patch-req-fr.vue | 444 +++++++++--------- ui/src/composables/registry-sync.ts | 9 +- ui/src/main.ts | 18 +- ui/src/pages/admin/remote-registries/[id].vue | 429 +++++++++++++---- ui/src/pages/artefacts/[id].vue | 104 +--- ui/src/pages/index.vue | 157 ++++--- 16 files changed, 1397 insertions(+), 1067 deletions(-) create mode 100644 ui/src/components/artefact-edit.vue diff --git a/ui/components.d.ts b/ui/components.d.ts index 8ac2b59..d90fb94 100644 --- a/ui/components.d.ts +++ b/ui/components.d.ts @@ -10,6 +10,7 @@ declare module 'vue' { AccessGrantsSection: typeof import('./src/components/admin/access-grants-section.vue')['default'] ApiKeysSection: typeof import('./src/components/admin/api-keys-section.vue')['default'] ArtefactAdmin: typeof import('./src/components/artefact-admin.vue')['default'] + ArtefactEdit: typeof import('./src/components/artefact-edit.vue')['default'] ArtefactMetadata: typeof import('./src/components/artefact-metadata.vue')['default'] DefaultLayout: typeof import('./src/components/default-layout.vue')['default'] RemoteRegistriesSection: typeof import('./src/components/admin/remote-registries-section.vue')['default'] diff --git a/ui/src/components/admin/access-grants-section.vue b/ui/src/components/admin/access-grants-section.vue index 622e578..f605496 100644 --- a/ui/src/components/admin/access-grants-section.vue +++ b/ui/src/components/admin/access-grants-section.vue @@ -22,8 +22,6 @@ return-object no-filter clearable - density="compact" - hide-details variant="outlined" /> @@ -52,9 +50,9 @@ /> - + {{ t('existingGrants') }} - ({{ grantsFetch.data.value.count }}) + ({{ grantsFetch.data.value.count }}) @@ -71,17 +69,28 @@ :key="grant._id" > - - {{ grant.account.type }} - - {{ grant.account.name || grant.account.id }} - {{ grant.account.id }} +
+ + + + + + {{ t(grant.account.type === 'organization' ? 'organization' : 'user') }} + + {{ grant.account.name || grant.account.id }} + {{ grant.account.id }} +
{{ grant.grantedBy.name || grant.grantedBy.id }} {{ dayjs(grant.grantedAt).format('L LT') }} @@ -112,6 +121,8 @@ fr: account: Compte grantedBy: Accordé par grantedAt: Accordé le + organization: Organisation + user: Utilisateur en: grantAccess: Grant Access searchAccount: Search an account @@ -121,12 +132,14 @@ en: account: Account grantedBy: Granted by grantedAt: Granted + organization: Organization + user: User + + diff --git a/ui/src/components/admin/vulnerability-section.vue b/ui/src/components/admin/vulnerability-section.vue index 936d078..2b6743e 100644 --- a/ui/src/components/admin/vulnerability-section.vue +++ b/ui/src/components/admin/vulnerability-section.vue @@ -1,5 +1,8 @@