diff --git a/.github/workflows/libraries.yml b/.github/workflows/libraries.yml index 93050ab..1def6ae 100644 --- a/.github/workflows/libraries.yml +++ b/.github/workflows/libraries.yml @@ -1,7 +1,8 @@ name: Library pack # Rebuilds packages/libraries from the H5P hub every week and opens a pull request when a library -# changed. The pack is built deterministically, so an unchanged hub leaves the tree clean. +# changed. The pack is built deterministically, in any time zone, so an unchanged hub leaves the +# tree clean. on: schedule: @@ -39,7 +40,17 @@ jobs: fi branch=libraries/refresh - changes=$(git diff -U0 -- packages/libraries/libraries.txt | grep '^[-+]H5P' | sort -k1.2 || true) + # Every library's line, ` `, FontAwesome and jQuery.ui among them. + changes=$(git diff -U0 -- packages/libraries/libraries.txt | grep -E '^[-+][A-Za-z][^ ]*-[0-9]+\.[0-9]+ ' | sort -k1.2 || true) + if [ -z "$changes" ]; then + # The files changed and no library's version, licence or authors did: the build is not + # deterministic, as it was once in the builder's time zone (#51), or the hub changed a + # library's files without a new version. Either way a pull request would say the hub + # changed when no library did. + echo "::error::The pack's files changed, but no library in libraries.txt did. Rebuild with pnpm libraries and compare the archives' entries before releasing anything." + git status --short -- packages/libraries | head -20 + exit 1 + fi version=$(node -e " const fs = require('fs'), path = 'packages/libraries/package.json' diff --git a/AGENTS.md b/AGENTS.md index 9d948bf..2cb13ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1018,7 +1018,13 @@ element, and the element acts. That relay is why `frame-document.ts` has a `mess requires, its entries copied from the pack as they are (zip.js `passThrough`), so a part's bytes change only when its library does and the weekly refresh commits only those; `parts/index.json` says each library's full version, dependencies and addon patterns and its archive's size and - SHA-256. Measured on the demo's packages: 2 to 16 parts, 0.04 to 1.5 MB, 4.0 MB for formulas + SHA-256. Every entry of the pack and its parts carries `FIXED_DATE` (`scripts/parts.mjs`), built + from local fields, since a zip records a wall-clock time and zip.js reads the Date's local ones: + built from UTC fields, it wrote 02:00 where the pack was built at UTC+2 and 00:00 on CI, whose + first refresh after the split rewrote all 106 archives with no library changed (#48, closed + unmerged; #51). `tests/unit/pack-bytes.test.ts` in the player builds in four zones, and the + workflow stops with an error when the files change and no library's line in `libraries.txt` + does. Measured on the demo's packages: 2 to 16 parts, 0.04 to 1.5 MB, 4.0 MB for formulas (MathDisplay is 2.8 MB), against 14 MB whole. **Planning is the worker's** (`planFromPack` in `mount.ts`, the rules in `sw/pack-plan.ts`): an `index` that offers the pack (`pack` on the message: `{ pinned: true }` for `LIBRARY_PACK_BASE`, whose index the worker carries compiled in diff --git a/apps/demo/scripts/build-libraries.mjs b/apps/demo/scripts/build-libraries.mjs index 024422e..d066479 100644 --- a/apps/demo/scripts/build-libraries.mjs +++ b/apps/demo/scripts/build-libraries.mjs @@ -3,7 +3,7 @@ import { writeFile } from 'node:fs/promises' import { join, resolve } from 'node:path' import { BlobReader, BlobWriter, TextReader, Uint8ArrayReader, Uint8ArrayWriter, ZipReader, ZipWriter, configure } from '@zip.js/zip.js' import { formatBytes } from '@missing-elements/h5p-normalize' -import { splitPack, writePackModule } from '../../../packages/libraries/scripts/parts.mjs' +import { FIXED_DATE, splitPack, writePackModule } from '../../../packages/libraries/scripts/parts.mjs' import { HUB, download as cachedDownload, hubBundle as cachedBundle } from './lib/hub.mjs' /** @@ -159,9 +159,10 @@ if (unknown.length > 0) { // A bundle registered to supply libraries is never held to its own manifest, but the player // refuses any archive without an h5p.json. -// A fixed date, so the pack's bytes change only when a library does: the weekly refresh -// (.github/workflows/libraries.yml) opens a pull request only on a real difference. -const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: new Date(Date.UTC(2000, 0, 1)), extendedTimestamp: false }) +// A fixed date, the same in any time zone (see `FIXED_DATE`), so the pack's bytes change only when +// a library does: the weekly refresh (.github/workflows/libraries.yml) opens a pull request only +// on a real difference. +const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false }) await writer.add( 'h5p.json', new TextReader( diff --git a/packages/libraries/scripts/parts.mjs b/packages/libraries/scripts/parts.mjs index 66090b6..027d178 100644 --- a/packages/libraries/scripts/parts.mjs +++ b/packages/libraries/scripts/parts.mjs @@ -30,8 +30,15 @@ const packageDir = resolve(import.meta.dirname, '..') */ const PART_MANIFEST = JSON.stringify({ title: 'H5P Offline Player library pack', language: 'und', mainLibrary: '', preloadedDependencies: [], embedTypes: ['div'] }) -/** A fixed date, as the pack has: the bytes change only when a library does. */ -const FIXED_DATE = new Date(Date.UTC(2000, 0, 1)) +/** + * The date every entry of the pack and its parts is written with, so the bytes change only when a + * library does. A zip records a wall-clock time with no zone, and zip.js takes it from the Date's + * local fields, so this is built from local fields too: the same bytes in any time zone. It was + * `Date.UTC(2000, 0, 1)` until #51, which wrote 02:00 at UTC+2, where the pack up to 1.0.0 was + * built, and 00:00 on CI, whose first weekly run rewrote all 106 archives with no library changed + * (#48). 02:00 is kept so the bytes stay those pack 1.0.0 shipped. + */ +export const FIXED_DATE = new Date(2000, 0, 1, 2) /** * The smallest a part is written. A bundler inlines a small asset named by `new URL` as a `data:` diff --git a/packages/player/tests/unit/pack-bytes.test.ts b/packages/player/tests/unit/pack-bytes.test.ts new file mode 100644 index 0000000..a3d8953 --- /dev/null +++ b/packages/player/tests/unit/pack-bytes.test.ts @@ -0,0 +1,71 @@ +import { execFileSync } from 'node:child_process' +import { resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { describe, expect, it } from 'vitest' + +/** + * The library pack's bytes, which the player pins by hash, depend on nothing but its libraries. A + * zip records a wall-clock time with no zone, and the pack was once written with a date built + * from UTC fields: the pack built at UTC+2 said 02:00 and CI's said 00:00, so the first weekly + * refresh after the split rewrote every archive with no library changed (#48, #51). + * + * Each time zone runs in a process of its own: whether setting `TZ` inside a running test worker + * moves its clock is the engine's business, and a child process starts in the zone it is given. + */ + +const librariesDir = resolve(import.meta.dirname, '../../../libraries') +const parts = pathToFileURL(resolve(librariesDir, 'scripts/parts.mjs')).href + +// A pack of one library, written the way apps/demo/scripts/build-libraries.mjs writes the real +// one, split by the real `splitPack`; prints the bytes' hashes and the first entry's DOS time and date. +const build = ` +import { createHash } from 'node:crypto' +import { mkdtemp, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { BlobWriter, TextReader, ZipWriter, configure } from '@zip.js/zip.js' +import { FIXED_DATE, splitPack } from ${JSON.stringify(parts)} + +configure({ useWebWorkers: false }) +const writer = new ZipWriter(new BlobWriter('application/zip'), { lastModDate: FIXED_DATE, extendedTimestamp: false }) +await writer.add('h5p.json', new TextReader('{}')) +await writer.add('H5P.Tiny-1.0/library.json', new TextReader(JSON.stringify({ machineName: 'H5P.Tiny', majorVersion: 1, minorVersion: 0, patchVersion: 3 }))) +await writer.add('H5P.Tiny-1.0/tiny.js', new TextReader('H5P.Tiny = function () {}')) +const pack = new Uint8Array(await (await writer.close()).arrayBuffer()) + +const out = await mkdtemp(join(tmpdir(), 'pack-bytes-')) +const [{ sha256 }] = await splitPack(pack, out) +const part = await readFile(join(out, 'H5P.Tiny-1.0.h5p')) +const header = new DataView(pack.buffer) +console.log(JSON.stringify({ + pack: createHash('sha256').update(pack).digest('hex'), + part: createHash('sha256').update(part).digest('hex'), + indexed: sha256, + time: header.getUint16(10, true), + date: header.getUint16(12, true) +})) +` + +function buildIn(timeZone: string) { + const output = execFileSync(process.execPath, ['--input-type=module', '-e', build], { + cwd: librariesDir, + env: { ...process.env, TZ: timeZone }, + encoding: 'utf8' + }) + return JSON.parse(output.trim().split('\n').at(-1)!) as { pack: string; part: string; indexed: string; time: number; date: number } +} + +describe('the library pack', () => { + it('is the same bytes in every time zone', () => { + const [utc, ...others] = ['UTC', 'Europe/Kyiv', 'America/Los_Angeles', 'Pacific/Kiritimati'].map(buildIn) + for (const other of others) expect(other).toEqual(utc) + expect(utc.part).toBe(utc.indexed) + }, 30_000) + + it('records 2000-01-01 02:00, the time pack 1.0.0 shipped with', () => { + // A different time rewrites every part and every hash the player pins, for no library's sake. + const { time, date } = buildIn('UTC') + expect(time).toBe(2 << 11) + expect(date).toBe(((2000 - 1980) << 9) | (1 << 5) | 1) + }, 30_000) +})