diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 802e0a0a..ed729686 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -72,7 +72,10 @@ jobs: - name: Test storage restart and editor recovery run: >- pnpm -C app exec vitest run - src/storage/storageRecovery.test.ts src/lib/storageRecoveryDrafts.test.ts + src/storage/storageRecovery.test.ts src/storage/editorSaveSession.test.ts + src/storage/tabRecoveryCheckpoint.test.ts src/storage/storageOwner.test.ts + src/releases/prepareTabReload.test.ts + src/storage/storageWorkerControl.test.ts src/lib/storageRecoveryDrafts.test.ts src/components/StorageRecoveryBanner.test.tsx --maxWorkers=2 --minWorkers=1 - name: Test explorer folder refresh @@ -118,12 +121,26 @@ jobs: - name: Test release controls and manifest discovery run: >- pnpm -C app exec vitest run - src/releases/control.test.ts src/releases/runtime.test.ts + src/releases/control.test.ts src/releases/runtime.test.ts src/releases/releaseAttention.test.ts + src/components/SidePanel/SidePanel.test.tsx src/releases/bootstrap.test.ts src/releases/launcherSelection.test.ts src/releases/loaderController.test.ts src/releases/releaseDatabase.test.ts src/releases/startupRecovery.test.ts src/releases/recoverControl.test.ts src/releases/controlChunks.test.ts src/releases/recoveryOwnerLock.test.ts src/storage/releaseOwnerFailure.test.ts src/releases/startupDiagnostics.test.ts src/pwa/loader-worker.test.ts src/components/ReleaseControls.test.tsx + src/components/ReleaseRecoveryControls.test.tsx src/components/StorageWorkerControls.test.tsx src/releases/releaseCatalog.test.ts + - name: Use the primary Ubuntu package mirror + shell: bash + run: | + set -euo pipefail + # Match app-tests: the Azure mirror can stall browser dependencies for + # several minutes before apt falls back to the primary archive. + if [[ -f /etc/apt/apt-mirrors.txt ]]; then + sudo sed -i 's|http://azure.archive.ubuntu.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' /etc/apt/apt-mirrors.txt + fi - run: pnpm -C app exec playwright-core install --with-deps chromium + - name: Test independent storage reconnect + if: matrix.baseline == 'current' + run: pnpm -C app run cuj:storage - run: pnpm -C app run cuj:releases env: RUNME_RELEASE_BASELINE_COMMIT: ${{ matrix.baseline != 'current' && matrix.baseline || '' }} @@ -131,4 +148,7 @@ jobs: if: always() with: name: controlled-release-evidence-${{ matrix.baseline }} - path: app/test/browser/test-output/controlled-releases/ + path: | + app/test/browser/test-output/controlled-releases/ + app/test/browser/test-output/storage-worker-startup/ + app/test/browser/test-output/storage-worker-reconnect/ diff --git a/app/package.json b/app/package.json index cec924f0..3f2c9353 100644 --- a/app/package.json +++ b/app/package.json @@ -16,7 +16,8 @@ "format": "prettier --ignore-path .prettierignore --write .", "preview": "vite preview", "test": "pnpm run check:documentation && vitest", - "cuj:releases": "tsc --target es2020 --module nodenext --moduleResolution nodenext --esModuleInterop --skipLibCheck --outDir test/browser/.generated test/browser/run-controlled-releases.ts test/browser/controlled-releases.ts && node test/browser/.generated/run-controlled-releases.js" + "cuj:storage": "tsc --target es2022 --module nodenext --moduleResolution nodenext --esModuleInterop --skipLibCheck --outDir test/browser/.generated test/browser/run-storage-worker-reconnect.ts test/browser/storage-worker-reconnect.ts && node test/browser/.generated/run-storage-worker-reconnect.js", + "cuj:releases": "tsc --target es2020 --module nodenext --moduleResolution nodenext --esModuleInterop --skipLibCheck --outDir test/browser/.generated test/browser/run-controlled-releases.ts test/browser/controlled-releases.ts test/browser/storage-worker-startup.ts && node test/browser/.generated/run-controlled-releases.js" }, "dependencies": { "@buf/googleapis_googleapis.bufbuild_es": "2.6.0-20250411203938-61b203b9a916.1", diff --git a/app/src/components/Actions/Actions.test.tsx b/app/src/components/Actions/Actions.test.tsx index c39ab09c..594d8c92 100644 --- a/app/src/components/Actions/Actions.test.tsx +++ b/app/src/components/Actions/Actions.test.tsx @@ -2767,6 +2767,15 @@ describe('Actions tabs', () => { }) }) + it('renders the Pin Release flow as its own workspace tab', () => { + contextMocks.currentDoc = 'app://pin-release' + contextMocks.workspaceDocuments = [{ uri: 'app://pin-release', title: 'Pin Release' }] + render() + expect(screen.getByRole('tab', { name: 'Pin Release' })).toBeTruthy() + expect(screen.getByRole('button', { name: /Choose another release/ })).toBeTruthy() + expect(screen.queryByText('Recovery and rollback')).toBeNull() + }) + it('renders the version information workspace document as a tab', () => { contextMocks.currentDoc = VERSION_INFO_DOCUMENT_URI contextMocks.workspaceDocuments = [ diff --git a/app/src/components/Actions/Actions.tsx b/app/src/components/Actions/Actions.tsx index dbcd85bc..f38d316f 100644 --- a/app/src/components/Actions/Actions.tsx +++ b/app/src/components/Actions/Actions.tsx @@ -159,6 +159,8 @@ import RunnerStatusTab from '../RunnerStatusTab' import KernelStatusTab from '../KernelStatusTab' import { NotebookDiffContent } from '../NotebookDiff/NotebookDiffView' import VersionInfoTab from '../VersionInfoTab' +import { ReleaseRecoveryControls } from '../ReleaseRecoveryControls' +import { PIN_RELEASE_DOCUMENT_URI } from '../../lib/workspaceDocuments/workspaceDocumentTypes' import { NotebookCommentsPanel } from '../NotebookCommentsPanel' import AppConsole from '../AppConsole/AppConsole' import LogsPane from '../Logs/LogsPane' @@ -4533,6 +4535,8 @@ function renderWorkspaceDocument({ if (isOidcStatusUri(document.uri)) return + if (document.uri === PIN_RELEASE_DOCUMENT_URI) return + if (isVersionInfoUri(document.uri)) { return } diff --git a/app/src/components/ReleaseControls.test.tsx b/app/src/components/ReleaseControls.test.tsx index bf8a2c6e..0c3e6e3e 100644 --- a/app/src/components/ReleaseControls.test.tsx +++ b/app/src/components/ReleaseControls.test.tsx @@ -97,12 +97,16 @@ describe('Information release controls', () => { expect( ( screen.getByRole('button', { - name: 'Upgrade all tabs', + name: 'Activate release and update this tab', }) as HTMLButtonElement ).disabled ).toBe(false) ) - fireEvent.click(screen.getByRole('button', { name: 'Upgrade all tabs' })) + fireEvent.click( + screen.getByRole('button', { + name: 'Activate release and update this tab', + }) + ) await waitFor(() => expect(api.upgrade).toHaveBeenCalledTimes(1)) }) it('keeps update errors visible in Information and permits retry', async () => { diff --git a/app/src/components/ReleaseControls.tsx b/app/src/components/ReleaseControls.tsx index 2ad81557..1f8cbfb8 100644 --- a/app/src/components/ReleaseControls.tsx +++ b/app/src/components/ReleaseControls.tsx @@ -1,3 +1,7 @@ +import { useReleaseAttention } from '../releases/useReleaseAttention' +import { StorageWorkerControls } from './StorageWorkerControls' +import { showWorkspaceDocument } from '../lib/workspaceDocuments/workspaceDocumentController' +import { PIN_RELEASE_DOCUMENT_URI } from '../lib/workspaceDocuments/workspaceDocumentTypes' import { Button } from '@radix-ui/themes' import { useCallback, useEffect, useState } from 'react' @@ -6,7 +10,6 @@ import { type Control, type Manifest, type ReleaseProgress, - releaseIdValid, } from '../releases/control' import '../releases/runtime' import type { OfflineDownloadStatus } from '../releases/runtime' @@ -17,16 +20,51 @@ import type { OfflineDownloadStatus } from '../releases/runtime' * the common commands and can use the independent recovery page for rollback. */ export function ReleaseControls() { + const attention = useReleaseAttention() + return ( + <> +
+
+

Application versions

+ +
+ {attention && ( +

+ {attention.message}. Manage this tab’s version below. +

+ )} + +
+ + + ) +} + +/** Installed-state controls may load independently of recovery controls. */ +function InstalledReleaseControls() { const api = window.runmeVersions const [state, setState] = useState() const [message, setMessage] = useState('') const [progress, setProgress] = useState() const [offline, setOffline] = useState() const [error, setError] = useState('') + const [readError, setReadError] = useState('') const [busy, setBusy] = useState(false) - const [exact, setExact] = useState('') const [cached, setCached] = useState([]) - const [published, setPublished] = useState([]) // Polling also observes other tabs and older loaders without change events. // Ignore late responses after unmount so a closed workspace tab stays closed. @@ -41,6 +79,7 @@ export function ReleaseControls() { const next = await api.get() if (!disposed) { setState(next) + setReadError('') setProgress(api.progress?.()) setOffline( next.active && api.offline?.(next.active.manifest.releaseId) @@ -53,7 +92,7 @@ export function ReleaseControls() { ) } } catch (e) { - if (!disposed) setError(String(e)) + if (!disposed) setReadError(String(e)) } finally { reading = false } @@ -107,20 +146,19 @@ export function ReleaseControls() { if (!api) return

Release channels are available in published builds.

if (!state) - return

{error || 'Loading release information…'}

+ return ( +

+ {readError || error || 'Loading release information…'} +

+ ) const active = state.active?.manifest const candidate = state.candidate const hasUpdate = candidate && candidate.releaseId !== active?.releaseId const selection = state.selection.kind === 'channel' ? state.selection.name : 'Pinned release' - const recoveryUrl = '/recovery.html' + location.search + location.hash return ( -
-

Application versions

+
Current channel
{selection}
@@ -145,9 +183,28 @@ export function ReleaseControls() { )}
+ {window.runmeRelease && + active && + window.runmeRelease.active.manifest.releaseId !== active.releaseId && ( +
+

+ This tab is running{' '} + {window.runmeRelease.active.manifest.webCommit.slice(0, 8)}. The + selected worker uses a different version; other tabs can update + independently. +

+ +
+ )}

Choose a channel to download its release in the background. Your - installed version stays active until you choose Upgrade all tabs. + selected worker version stays active until you activate the release. + Other tabs keep their current UI until you reload them individually.

- {active && ( - - )}
{hasUpdate && ( )} {state.transition && (

- Upgrade {state.transition.phase}. Each tab must preserve its edits - before the upgrade can finish. + {state.transition.phase === 'prepare' + ? 'An older upgrade is pending. Cancel it to start a new upgrade.' + : 'Starting the selected storage worker. Other tabs stay open.'}

{state.transition.phase === 'prepare' && (
)} - {error && ( + {(readError || error) && (

- {error} + {readError || error}

)}

@@ -271,68 +315,6 @@ export function ReleaseControls() { id="release-other-actions" className="mt-3 flex flex-col items-start gap-3" > - - - - {published.map((id) => ( - - ))} {api.activate && cached .filter((m) => m.releaseId !== active?.releaseId) @@ -351,16 +333,8 @@ export function ReleaseControls() { > Remove unused downloads - - Open recovery in another tab -

-
+ ) } diff --git a/app/src/components/ReleaseRecoveryControls.test.tsx b/app/src/components/ReleaseRecoveryControls.test.tsx new file mode 100644 index 00000000..c4c98038 --- /dev/null +++ b/app/src/components/ReleaseRecoveryControls.test.tsx @@ -0,0 +1,202 @@ +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from '@testing-library/react' +import { afterEach, expect, it, vi } from 'vitest' +import { ReleaseControls } from './ReleaseControls' +import { ReleaseRecoveryControls } from './ReleaseRecoveryControls' +import { showWorkspaceDocument } from '../lib/workspaceDocuments/workspaceDocumentController' +import type { Control, Manifest } from '../releases/control' + +vi.mock('../lib/workspaceDocuments/workspaceDocumentController', () => ({ + showWorkspaceDocument: vi.fn(), +})) +const installed = `${'a'.repeat(40)}-${'b'.repeat(64)}` +const id = `${'c'.repeat(40)}-${'d'.repeat(64)}` +function setup( + get = async () => + ({ active: { manifest: { releaseId: installed } } }) as Control +) { + const api = { + get, + check: vi.fn(async () => {}), + pin: vi.fn(async () => {}), + upgrade: vi.fn(), + retry: vi.fn(), + cancel: vi.fn(), + prune: vi.fn(), + prepareRecovery: vi.fn(async () => ({ releaseId: id }) as Manifest), + recover: vi.fn(async () => {}), + } + window.runmeVersions = api + return api +} +afterEach(() => { + cleanup() + delete window.runmeVersions + vi.unstubAllGlobals() + vi.clearAllMocks() +}) +async function selectCommit(value = id) { + fireEvent.click( + screen.getByRole('button', { name: /Choose another release/ }) + ) + fireEvent.click(screen.getByRole('button', { name: 'Enter a commit' })) + fireEvent.change( + screen.getByRole('textbox', { name: 'Commit SHA or release ID' }), + { target: { value } } + ) + fireEvent.click(screen.getByRole('button', { name: 'Review release' })) + await screen.findByRole('heading', { name: 'Pin this release?' }) +} + +it.each(['failed', 'pending', 'missing'])( + 'keeps Pin Release reachable when settings are %s', + async (mode) => { + if (mode !== 'missing') + setup( + mode === 'failed' + ? async () => { + throw new Error('read timeout (get)') + } + : () => new Promise(() => {}) + ) + render() + fireEvent.click(screen.getByRole('button', { name: 'Pin Release' })) + expect(showWorkspaceDocument).toHaveBeenCalledWith('app://pin-release') + expect(screen.queryByText('Recovery and rollback')).toBeNull() + if (mode === 'failed') await screen.findByText(/read timeout/) + } +) +it('pins the installed release only after review without downloading or activating', async () => { + const api = setup() + render() + const current = screen.getByRole('button', { + name: /Keep the installed release/, + }) + await waitFor(() => expect(current.hasAttribute('disabled')).toBe(false)) + fireEvent.click(current) + expect(api.pin).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'Pin installed release' })) + await screen.findByRole('heading', { name: 'Release pinned' }) + expect(api.pin).toHaveBeenCalledOnce() + expect(api.check).not.toHaveBeenCalled() + expect(api.recover).not.toHaveBeenCalled() +}) +it('reviews an exact build, pins it, and activates only on a separate action', async () => { + const api = setup() + render() + await selectCommit() + expect(api.check).not.toHaveBeenCalled() + expect(api.prepareRecovery).not.toHaveBeenCalled() + fireEvent.click(screen.getByRole('button', { name: 'Pin and download' })) + await screen.findByRole('heading', { name: 'Release pinned' }) + expect(api.check).toHaveBeenCalledWith({ kind: 'pinned', releaseId: id }) + expect(api.recover).not.toHaveBeenCalled() + fireEvent.click( + screen.getByRole('button', { name: 'Activate release and update this tab' }) + ) + await waitFor(() => + expect(api.recover).toHaveBeenCalledWith({ kind: 'pinned', releaseId: id }) + ) +}) +it('browses published builds and reviews the exact selection without pinning yet', async () => { + const api = setup() + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response( + JSON.stringify({ format: 1, releases: [{ releaseId: id }] }) + ) + ) + ) + render() + fireEvent.click( + screen.getByRole('button', { name: /Choose another release/ }) + ) + fireEvent.click( + screen.getByRole('button', { name: 'Load published releases' }) + ) + fireEvent.click( + await screen.findByRole('button', { name: /Commit cccccccc/ }) + ) + expect( + screen.getByRole('heading', { name: 'Pin this release?' }) + ).toBeTruthy() + expect(screen.getByText('d'.repeat(64))).toBeTruthy() + expect(api.check).not.toHaveBeenCalled() +}) +it('reports ambiguous commits without guessing a build', async () => { + const api = setup() + vi.stubGlobal( + 'fetch', + vi.fn( + async () => + new Response( + JSON.stringify({ + format: 1, + releases: [ + { releaseId: id }, + { releaseId: `${'c'.repeat(40)}-${'e'.repeat(64)}` }, + ], + }) + ) + ) + ) + render() + fireEvent.click( + screen.getByRole('button', { name: /Choose another release/ }) + ) + fireEvent.click(screen.getByRole('button', { name: 'Enter a commit' })) + fireEvent.change(screen.getByRole('textbox'), { + target: { value: 'ccccccc' }, + }) + fireEvent.click(screen.getByRole('button', { name: 'Review release' })) + expect((await screen.findByRole('alert')).textContent).toContain( + 'Multiple builds' + ) + expect(api.check).not.toHaveBeenCalled() +}) +it('retains a verified recovery download when settings cannot record the pin', async () => { + const api = setup(async () => { + throw new Error('read timeout (get)') + }) + api.check.mockRejectedValue(new Error('read timeout (get)')) + render() + await selectCommit() + fireEvent.click(screen.getByRole('button', { name: 'Pin and download' })) + await screen.findByRole('alert') + expect(api.prepareRecovery).toHaveBeenCalledWith({ + kind: 'pinned', + releaseId: id, + }) + expect(api.recover).not.toHaveBeenCalled() + expect(screen.queryByRole('heading', { name: 'Release pinned' })).toBeNull() + fireEvent.click( + screen.getByRole('button', { name: 'Recover with this release' }) + ) + await waitFor(() => + expect(api.recover).toHaveBeenCalledWith({ kind: 'pinned', releaseId: id }) + ) +}) +it('keeps recovery navigation and diagnostics available during a hung download', async () => { + const api = setup(() => new Promise(() => {})) + api.prepareRecovery.mockImplementation(() => new Promise(() => {})) + render() + await selectCommit() + fireEvent.click(screen.getByRole('button', { name: 'Pin and download' })) + fireEvent.click(screen.getByText('Troubleshooting')) + expect( + screen + .getByRole('link', { name: /Open recovery in another tab/ }) + .getAttribute('href') + ).toContain(`release=${id}`) + fireEvent.click( + screen.getByRole('button', { name: 'Show release settings diagnostics' }) + ) + expect(screen.getByLabelText('Release settings diagnostics')).toBeTruthy() +}) diff --git a/app/src/components/ReleaseRecoveryControls.tsx b/app/src/components/ReleaseRecoveryControls.tsx new file mode 100644 index 00000000..c3b86d9c --- /dev/null +++ b/app/src/components/ReleaseRecoveryControls.tsx @@ -0,0 +1,464 @@ +import { Button, ScrollArea } from '@radix-ui/themes' +import { useEffect, useState } from 'react' +import type { Manifest } from '../releases/control' +import { + publishedReleaseIds, + resolvePublishedRelease, +} from '../releases/releaseCatalog' +import { releaseDatabaseDiagnostics } from '../releases/releaseDiagnostics' +import { showWorkspaceDocument } from '../lib/workspaceDocuments/workspaceDocumentController' +import { VERSION_INFO_DOCUMENT_URI } from '../lib/workspaceDocuments/workspaceDocumentTypes' +import '../releases/runtime' + +type Step = 'choose' | 'select' | 'review' | 'done' + +/** A tab-local wizard separates choosing, downloading and activating a release. + * Discovery and recovery navigation remain usable even if settings never reply. + */ +export function ReleaseRecoveryControls() { + const api = window.runmeVersions + const [step, setStep] = useState('choose') + const [source, setSource] = useState<'browse' | 'commit'>('browse') + const [installed, setInstalled] = useState() + const [settingsError, setSettingsError] = useState('') + const [input, setInput] = useState('') + const [releases, setReleases] = useState([]) + const [listed, setListed] = useState(false) + const [prepared, setPrepared] = useState() + const [selected, setSelected] = useState('') + const [busy, setBusy] = useState(false) + const [message, setMessage] = useState('') + const [error, setError] = useState('') + const [diagnostics, setDiagnostics] = useState('') + + useEffect(() => { + let disposed = false + if (api) + void api + .get() + .then((state) => { + if (!disposed) setInstalled(state.active?.manifest.releaseId) + }) + .catch(() => { + if (!disposed) + setSettingsError( + 'Installed release settings are unavailable. You can still choose another release.' + ) + }) + return () => { + disposed = true + } + }, [api]) + + const recoveryUrl = new URL('/recovery.html', window.location.href) + recoveryUrl.search = window.location.search + recoveryUrl.hash = window.location.hash + if (selected) { + recoveryUrl.searchParams.delete('channel') + recoveryUrl.searchParams.set('release', selected) + } + /** Keep errors in this tab; navigation stays available during stalled work. */ + const run = async (action: () => Promise) => { + setBusy(true) + setError('') + setMessage('') + try { + await action() + } catch (failure) { + setError(failure instanceof Error ? failure.message : String(failure)) + } finally { + setBusy(false) + } + } + const back = () => { + setStep(step === 'review' && selected !== installed ? 'select' : 'choose') + setError('') + setMessage('') + setPrepared(undefined) + } + const review = (id: string) => { + setSelected(id) + setPrepared(undefined) + setStep('review') + setError('') + } + const choiceClass = + 'rounded-xl border border-nb-border bg-white p-5 text-left transition hover:border-violet-500 hover:bg-violet-50 focus-visible:outline focus-visible:outline-2 focus-visible:outline-violet-600 disabled:cursor-not-allowed disabled:opacity-50' + + return ( + +
+
+

Pin Release

+

+ Choose the version Runme should stay on. You can follow a channel + again at any time. +

+
+
    + {['Choose', 'Select release', 'Review', 'Pinned'].map( + (label, index) => ( +
  1. + {index + 1}. {label} +
    2. + ) + )} +
+ + {step === 'choose' && ( +
+ + +
+ )} + {step === 'choose' && settingsError && ( +

{settingsError}

+ )} + + {step === 'select' && ( +
+
+ + +
+ {source === 'browse' ? ( + <> +

+ Choose a published build. Each commit can have more than one + build. +

+ + {listed && !releases.length && ( +

+ No published releases found. You can enter a full release ID + instead. +

+ )} +
+ {releases.map((id) => ( + + ))} +
+ + ) : ( + <> + +

+ Use at least 7 characters of a published commit, or a full + release ID. +

+ + + )} +
+ )} + + {(step === 'review' || step === 'done') && ( +
+

+ {step === 'done' + ? 'Release pinned' + : selected === installed + ? 'Keep this version?' + : 'Pin this release?'} +

+
+
Commit
+
+ {selected.split('-')[0]} +
+
Build
+
+ {selected.split('-')[1]} +
+
+

+ {selected === installed + ? 'Runme will stop following channel updates. Your running app stays unchanged.' + : 'Pinning downloads this build. Your running app stays unchanged until you activate it.'} +

+ {step === 'review' && ( + + )} + {prepared && error && ( +

+ The release was downloaded, but pinning did not finish. Retry or + use recovery below. +

+ )} + {((step === 'done' && selected !== installed) || + (prepared && error)) && + api?.recover && ( + <> +

+ Activating saves and reloads this tab. Other tabs stay open. +

+ + + )} +
+ )} + {error && ( +

+ {error} +

+ )} + {(busy || message) && ( +

+ {busy ? 'Working…' : message} +

+ )} +
+ {step !== 'choose' && step !== 'done' && ( + + )} + {step === 'done' && ( + + )} + +
+
+ + Troubleshooting + +
+

+ If this tab cannot finish, use the independent recovery page. +

+ +
+ +
+ {diagnostics && ( + 
+                {diagnostics}
+
+ )} +
+
+
+ + ) +} diff --git a/app/src/components/SidePanel/SidePanel.test.tsx b/app/src/components/SidePanel/SidePanel.test.tsx index e42be4d7..a0e08ee9 100644 --- a/app/src/components/SidePanel/SidePanel.test.tsx +++ b/app/src/components/SidePanel/SidePanel.test.tsx @@ -1,12 +1,14 @@ // @vitest-environment jsdom import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { ServiceAccountCredentialStatus } from '../../auth/googleServiceAccountImpersonation' const { showToastMock } = vi.hoisted(() => ({ showToastMock: vi.fn(), })) +afterEach(() => { delete window.runmeVersions }) + type PanelKey = | 'explorer' | 'documentation' @@ -529,6 +531,27 @@ describe('SidePanelToolbar drive status button', () => { expect(setCurrentDocMock).toHaveBeenCalledWith('app://version') }) + it('marks version attention on the information icon and clears it without reading storage', async () => { + let attention: import('../../releases/releaseAttention').ReleaseAttention | undefined = { + kind: 'update', message: 'An update is ready to activate', + } + const get = vi.fn() + window.runmeVersions = { get, attention: () => attention } as unknown as NonNullable + render() + const button = screen.getByRole('button', { name: 'Open Version Information' }) + await waitFor(() => expect(button.getAttribute('aria-description')).toBe('An update is ready to activate')) + expect(screen.getByTestId('version-attention-badge').getAttribute('data-kind')).toBe('update') + fireEvent.click(button) + expect(showDocumentMock).toHaveBeenCalledWith('app://version', { title: 'Version Information' }) + attention = {kind: 'reload', message: 'This tab is out of date'} + fireEvent(window, new Event('runme-release-change')) + await waitFor(() => expect(button.getAttribute('aria-description')).toBe('This tab is out of date')) + attention = undefined + fireEvent(window, new Event('runme-release-change')) + await waitFor(() => expect(screen.queryByTestId('version-attention-badge')).toBeNull()) + expect(get).not.toHaveBeenCalled() + }) + it('opens the runner status document from the toolbar', () => { runnersState = [ { diff --git a/app/src/components/SidePanel/SidePanel.tsx b/app/src/components/SidePanel/SidePanel.tsx index 6a9e3d6a..c2aba8c4 100644 --- a/app/src/components/SidePanel/SidePanel.tsx +++ b/app/src/components/SidePanel/SidePanel.tsx @@ -67,6 +67,8 @@ import AuthenticationSettingsPanel from '../AuthenticationSettings/Authenticatio import { SuggestionGraderSettings } from '../SuggestionGraderSettings/SuggestionGraderSettings' import { showToast } from '../../lib/toast' +import { useReleaseAttention } from '../../releases/useReleaseAttention' + const sideButtonBase = 'group side-btn' const sideButtonInactive = 'side-btn-inactive' @@ -238,6 +240,7 @@ function OpenDocumentsPanel() { } export function SidePanelToolbar() { + const versionAttention = useReleaseAttention() // Keep the indicator current even while the vault panel is closed. const vaultSnapshot = useSyncExternalStore(keyVault.subscribe, keyVault.getSnapshot) const vaultUnlocked = vaultSnapshot.status === 'unlocked' @@ -718,10 +721,25 @@ export function SidePanelToolbar() { }`} aria-pressed={versionInfoSelected} aria-label="Open Version Information" + aria-description={versionAttention?.message} onClick={handleVersionInfoClick} > - - Version Information + + + {versionAttention && ( + + )} + + + {versionAttention ? `Version Information — ${versionAttention.message}` : 'Version Information'} + diff --git a/app/src/components/StorageRecoveryBanner.test.tsx b/app/src/components/StorageRecoveryBanner.test.tsx index add7f903..224b60ac 100644 --- a/app/src/components/StorageRecoveryBanner.test.tsx +++ b/app/src/components/StorageRecoveryBanner.test.tsx @@ -31,7 +31,7 @@ it('shows a startup failure detected before mounting in both UI locations', () = ) - const buttons = screen.getAllByRole('button', { name: 'Restart Runme' }) + const buttons = screen.getAllByRole('button', { name: 'Restart storage' }) expect(buttons).toHaveLength(2) fireEvent.click(buttons[1]) expect(restart).toHaveBeenCalledOnce() @@ -40,14 +40,19 @@ it('does not show a recovery control for healthy storage', () => { render() expect(screen.queryByRole('alert')).toBeNull() }) -it('disables duplicate requests while every tab prepares for reload', () => { +it('keeps force restart available while graceful restart is waiting', () => { storageRecovery.set({ phase: 'preparing', message: 'Preserving edits' }) render() expect( screen - .getByRole('button', { name: 'Preparing restart…' }) + .getByRole('button', { name: 'Restarting storage…' }) .hasAttribute('disabled') ).toBe(true) + expect( + screen + .getByRole('button', { name: 'Force restart storage' }) + .hasAttribute('disabled') + ).toBe(false) }) it('offers retry with the reason when preservation or coordination fails', () => { storageRecovery.set({ @@ -58,7 +63,7 @@ it('offers retry with the reason when preservation or coordination fails', () => expect(screen.getByText(/did not acknowledge/)).toBeTruthy() expect( screen - .getByRole('button', { name: 'Restart Runme' }) + .getByRole('button', { name: 'Restart storage' }) .hasAttribute('disabled') ).toBe(false) }) @@ -80,7 +85,18 @@ it('offers current editor downloads while storage is unavailable and restart is ) expect(createURL).toHaveBeenCalledWith(expect.any(Blob)) expect(click).toHaveBeenCalledOnce() - expect(screen.getByText(/Session copies do not survive/)).toBeTruthy() + expect(screen.getByText(/Storage restart keeps this page/)).toBeTruthy() vi.unstubAllGlobals() vi.restoreAllMocks() }) + +it('allows diagnostics while restart is waiting and reports an unresponsive worker', async () => { + storageRecovery.set({ phase: 'preparing', message: 'Waiting for tabs' }) + const dispose = storageRecovery.registerDiagnostics(async () => { + throw new Error('No diagnostic reply') + }) + render() + fireEvent.click(screen.getByRole('button', { name: 'Check storage worker' })) + expect(await screen.findByText('Error: No diagnostic reply')).toBeTruthy() + dispose() +}) diff --git a/app/src/components/StorageRecoveryBanner.tsx b/app/src/components/StorageRecoveryBanner.tsx index 9183e787..96a30ee1 100644 --- a/app/src/components/StorageRecoveryBanner.tsx +++ b/app/src/components/StorageRecoveryBanner.tsx @@ -1,3 +1,4 @@ +import { appLogger } from '../lib/logging/runtime' import { useEffect, useState, useSyncExternalStore } from 'react' import { storageRecovery } from '../storage/storageRecovery' import { @@ -33,6 +34,8 @@ export default function StorageRecoveryBanner({ const [drafts, setDrafts] = useState([]) const [currentDrafts, setCurrentDrafts] = useState([]) const [captureMessage, setCaptureMessage] = useState('') + const [diagnostics, setDiagnostics] = useState('') + const [diagnosing, setDiagnosing] = useState(false) useEffect(() => { try { setDrafts(readRecoveryDrafts()) @@ -53,10 +56,10 @@ export default function StorageRecoveryBanner({

Runme storage needs to reconnect

{state.message}

- Restart first backs up this tab’s editors in session storage. - Connected tabs reload only after all tabs preserve edits and pending - writes finish. Session copies do not survive closing a tab and are - not persistent autosave. + Storage restart keeps this page and its editors open. Pending editor + saves remain in this tab and retry after reconnecting. Keep this tab + open until saves finish, or download recovery copies before closing + it.

+ + +

+ Force restart skips draining pending work. Editor saves can retry; + other operations may have an uncertain outcome. +

)} + {state.phase !== 'healthy' && ( +
+ + {diagnostics && ( + 
+              {diagnostics}
+
+ )} +

+ Diagnostics do not save or retry notebook operations. A reply + confirms the worker can process diagnostic messages, not that + storage writes are working. +

+ {state.phase === 'blocked' && ( +

+ If restart cannot finish, download current editor copies in every + Runme tab first. Then close all Runme tabs for this site and + reopen Runme. If the worker remains stuck, restart the browser + after saving the downloads. Do not clear site data. +

+ )} +
+ )} {captureMessage &&

{captureMessage}

} {currentDrafts.map((draft, index) => ( + + +

+ Restart keeps editors open. Force restart skips waiting for pending + work; retained editor saves retry after reconnecting. Other operations + may have an uncertain outcome. +

+ {[...workers.values()].map((worker) => ( +
+

+ {worker.releaseId ?? 'Development worker'} · epoch {worker.epoch} ·{' '} + {worker.phase} ({Math.round(worker.phaseElapsedMs / 1000)}s at last + reply) +

+

Instance: {worker.instanceId}

+ + +
+ Worker diagnostics + 
+              {JSON.stringify(worker, null, 2)}
+
+
+
+ ))} +

+ A force request still needs a responsive worker message loop. For a + completely unresponsive worker, use the browser’s worker inspection + tools (Chrome: chrome://inspect/#workers). Keep tabs open to retain + unsaved edits; download recovery copies before closing them. Do not + clear site data. +

+ + ) +} diff --git a/app/src/contexts/NotebookContext.tsx b/app/src/contexts/NotebookContext.tsx index c2647da2..c2852bf5 100644 --- a/app/src/contexts/NotebookContext.tsx +++ b/app/src/contexts/NotebookContext.tsx @@ -20,6 +20,7 @@ import { appLogger } from "../lib/logging/runtime"; import { appState } from "../lib/runtime/AppState"; import { isNotebookDocumentUri } from "../lib/workspaceDocuments/workspaceDocumentTypes"; import { prepareRelease } from '../releases/prepare'; +import { prepareTabReload } from '../releases/prepareTabReload'; import { adoptLegacyVersionUi } from '../releases/runtime'; import { useCurrentDoc } from "./CurrentDocContext"; import { useNotebookStore } from "./NotebookStoreContext"; @@ -74,6 +75,7 @@ export function NotebookProvider({ children }: { children: ReactNode }) { const entries = controller.getOpenNotebooks(); if (entries.some(entry => entry.state === 'loading' || entry.state === 'resolving')) return; runtime.prepare = prepareRelease; + runtime.prepareReload = prepareTabReload; if (entries.every(entry => entry.state === 'loaded' || entry.state === 'blocked')) { runtime.ready?.(); releaseLegacyUi ??= adoptLegacyVersionUi(); @@ -87,7 +89,7 @@ export function NotebookProvider({ children }: { children: ReactNode }) { }).catch(error => appLogger.warn('Release startup did not become healthy', { attrs: { scope: 'release.startup', error: String(error) }, })); - return () => { disposed = true; unsubscribe?.(); releaseLegacyUi?.(); delete runtime.prepare; }; + return () => { disposed = true; unsubscribe?.(); releaseLegacyUi?.(); delete runtime.prepare; delete runtime.prepareReload; }; }, [controller, store]); const getNotebookData = useCallback( diff --git a/app/src/lib/appConsole/AppConsoleData.ts b/app/src/lib/appConsole/AppConsoleData.ts index ccb834f5..5bcf1839 100644 --- a/app/src/lib/appConsole/AppConsoleData.ts +++ b/app/src/lib/appConsole/AppConsoleData.ts @@ -425,8 +425,10 @@ export class AppConsoleData { } /** Flush the latest draft after earlier writes; failure is an upgrade blocker. */ - async flushForRelease(): Promise { + async flushForRelease(signal?: AbortSignal): Promise { if (this.hydratePromise) await this.hydratePromise; + // A timed-out reload must not freeze the console when hydration resolves late. + signal?.throwIfAborted(); if (!this.snapshotCache.hydrated) return; if (!this.persistenceEnabled) throw new Error('App Console drafts cannot be saved'); if (this.snapshotCache.cells.some(cell => cell.status === 'running')) throw new Error('App Console is running'); diff --git a/app/src/lib/notebookDataController.test.ts b/app/src/lib/notebookDataController.test.ts index 1c2e7ebd..768d56c9 100644 --- a/app/src/lib/notebookDataController.test.ts +++ b/app/src/lib/notebookDataController.test.ts @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' import { parser_pb } from '../contexts/CellContext' import type { LocalNotebooks } from '../storage/local' import { NotebookStoreItemType } from '../storage/notebook' +import * as checkpoints from '../storage/tabRecoveryCheckpoint' import { googleAnalytics } from './googleAnalytics' import { __resetNotebookDataControllerForTests, @@ -655,6 +656,37 @@ describe('NotebookDataController', () => { ]) }) + it('finishes restoring open editors before scanning orphan recovery checkpoints', async () => { + const uri = 'local://file/recovered' + const localStore = createFakeLocalNotebooks() + localStore.records.set(uri, { + id: uri, + name: 'recovered.runme', + remoteId: uri, + notebook: createNotebook('recovered edits'), + }) + sessionStorage.setItem('runme/openNotebooks', JSON.stringify([ + { uri, requestedUri: uri, name: 'recovered.runme' }, + ])) + const scan = vi.spyOn(checkpoints, 'tabCheckpointUris').mockReturnValue([uri]) + let releaseLoad!: () => void + const loading = new Promise((resolve) => { releaseLoad = resolve }) + localStore.load.mockImplementation(async () => { + await loading + return localStore.records.get(uri)!.notebook + }) + const controller = getNotebookDataController() + controller.configureStores({ localNotebooks: localStore as unknown as LocalNotebooks }) + await waitFor(() => expect(localStore.load).toHaveBeenCalled()) + // A second open creates a second save actor for this same recovery entry. + // Keep the normal restore pending to deterministically expose that race. + expect(scan).not.toHaveBeenCalled() + releaseLoad() + await waitFor(() => expect(scan).toHaveBeenCalledOnce()) + expect(localStore.createOperationLogSaveStore).toHaveBeenCalledTimes(1) + expect(controller.getNotebookData(uri)?.getNotebook().cells[0].value).toBe('recovered edits') + }) + it('restores session open-notebook storage as load intents only', () => { window.sessionStorage.setItem( 'runme/openNotebooks', diff --git a/app/src/lib/notebookDataController.ts b/app/src/lib/notebookDataController.ts index 014506b9..9daab035 100644 --- a/app/src/lib/notebookDataController.ts +++ b/app/src/lib/notebookDataController.ts @@ -2,6 +2,7 @@ import { create } from '@bufbuild/protobuf' import { parser_pb } from '../contexts/CellContext' import type { LocalNotebooks } from '../storage/local' +import { tabCheckpointUris } from '../storage/tabRecoveryCheckpoint' import { type NotebookAnalyticsSource, classifyNotebookAnalyticsSource, @@ -153,6 +154,21 @@ export class NotebookDataController { } if (this.localNotebooks) { void this.loadOpenNotebooks() + .then(async () => { + // A save from an editor closed before reload can still be pending. Its + // checkpoint must be reopened/replayed, not orphaned by the tab list. + for (const uri of tabCheckpointUris()) { + if (!this.notebooks.get(uri)?.loaded) await this.openNotebook(uri) + } + }) + .catch((error) => + appLogger.error( + 'Tab recovery could not restore all editors; checkpoints retained', + { + attrs: { scope: 'notebook-session', error: String(error) }, + } + ) + ) } } @@ -971,15 +987,18 @@ export class NotebookDataController { return } const restored = [...this.openNotebooks] - for (const item of restored) { - if (this.notebooks.get(item.uri)?.loaded) { - continue - } - void this.openNotebook(item.requestedUri || item.uri, { - name: item.name, - loadReadOnly: false, + // Load notebooks independently, but do not report restoration complete until + // every load has settled. The checkpoint scan runs next; returning early can + // reopen a still-loading editor and create two save sessions for one URI. + await Promise.all( + restored.map(async (item) => { + if (this.notebooks.get(item.uri)?.loaded) return + await this.openNotebook(item.requestedUri || item.uri, { + name: item.name, + loadReadOnly: false, + }) }) - } + ) } private ensureRestored(): void { diff --git a/app/src/lib/storageRecoveryDrafts.test.ts b/app/src/lib/storageRecoveryDrafts.test.ts index 6e03a76c..9931b78b 100644 --- a/app/src/lib/storageRecoveryDrafts.test.ts +++ b/app/src/lib/storageRecoveryDrafts.test.ts @@ -96,29 +96,20 @@ it('also blocks restart for running cells in read-only notebooks', async () => { expect(m.setReleasePending).not.toHaveBeenCalled() }) -it('cancels a hung flush promptly and does not unlock a later recovery round', async () => { +it('backs up without flushing and abort does not unlock a later recovery round', async () => { const m = model() - let finish!: () => void - m.flushPendingPersist.mockImplementationOnce( - () => - new Promise((resolve) => { - finish = resolve - }) - ) + m.flushPendingPersist.mockImplementation(() => new Promise(() => {})) const controller = new AbortController() - const preparation = prepareNotebookStorageRestart(controller.signal) - const rejected = expect(preparation).rejects.toThrow('Could not preserve') + const releaseOld = await prepareNotebookStorageRestart(controller.signal) expect(readRecoveryDrafts()).toHaveLength(1) + expect(m.flushPendingPersist).not.toHaveBeenCalled() controller.abort() expect(m.setReleasePending).toHaveBeenLastCalledWith(false) - await rejected const releaseNew = await prepareNotebookStorageRestart() expect(m.setReleasePending).toHaveBeenLastCalledWith(true) - const callsBeforeOldFlush = m.setReleasePending.mock.calls.length - finish() - await Promise.resolve() - await Promise.resolve() - expect(m.setReleasePending).toHaveBeenCalledTimes(callsBeforeOldFlush) + const calls = m.setReleasePending.mock.calls.length + releaseOld() + expect(m.setReleasePending).toHaveBeenCalledTimes(calls) releaseNew() expect(m.setReleasePending).toHaveBeenLastCalledWith(false) }) @@ -132,7 +123,7 @@ it.each(['[null]', '[{"uri":"local://file/a"}]', '{}'])( } ) -it('backs up every notebook before waiting for a hung first save', async () => { +it('backs up every notebook independently of a hung first save', async () => { const first = model() const second = model() second.getSnapshot = () => ({ @@ -149,14 +140,14 @@ it('backs up every notebook before waiting for a hung first save', async () => { ) first.flushPendingPersist.mockImplementation(() => new Promise(() => {})) const controller = new AbortController() - const preparing = prepareNotebookStorageRestart(controller.signal) - const rejected = expect(preparing).rejects.toThrow('Could not preserve') + const release = await prepareNotebookStorageRestart(controller.signal) + expect(first.flushPendingPersist).not.toHaveBeenCalled() expect(readRecoveryDrafts().map((draft) => draft.name)).toEqual([ 'draft.runme', 'second.runme', ]) controller.abort() - await rejected + release() expect(second.setReleasePending).toHaveBeenLastCalledWith(false) }) diff --git a/app/src/lib/storageRecoveryDrafts.ts b/app/src/lib/storageRecoveryDrafts.ts index 01663788..479f3646 100644 --- a/app/src/lib/storageRecoveryDrafts.ts +++ b/app/src/lib/storageRecoveryDrafts.ts @@ -99,16 +99,9 @@ export async function prepareNotebookStorageRestart( try { // Copy all notebooks before a stalled first flush can block the rest. backupNotebookRecoveryDrafts() - for (const model of models) { - signal.throwIfAborted() - if (model!.isReadOnly()) continue - try { - await waitForPersist(model!.flushPendingPersist(), signal) - } catch { - /* The recovery copy remains available after reload. */ - } - signal.throwIfAborted() - } + // Recovery must not wait on the persistence service being restarted. The + // worker drains admitted writes in graceful mode; snapshots cover unsaved edits. + signal.throwIfAborted() return release } catch (error) { release() @@ -117,21 +110,3 @@ export async function prepareNotebookStorageRestart( ) } } - -/** Cancel waiting, not the underlying commit. The owner still drains active writes. */ -async function waitForPersist( - persist: Promise, - signal: AbortSignal -): Promise { - let onAbort!: () => void - const aborted = new Promise((_resolve, reject) => { - onAbort = () => reject(signal.reason) - signal.addEventListener('abort', onAbort, { once: true }) - if (signal.aborted) onAbort() - }) - try { - await Promise.race([persist, aborted]) - } finally { - signal.removeEventListener('abort', onAbort) - } -} diff --git a/app/src/lib/workspaceDocuments/workspaceDocumentTypes.ts b/app/src/lib/workspaceDocuments/workspaceDocumentTypes.ts index 2ee27b80..306c4a6b 100644 --- a/app/src/lib/workspaceDocuments/workspaceDocumentTypes.ts +++ b/app/src/lib/workspaceDocuments/workspaceDocumentTypes.ts @@ -8,6 +8,7 @@ export const DRIVE_LINK_STATUS_DOCUMENT_URI = 'status://drive-link' export const DRIVE_SYNC_STATUS_DOCUMENT_URI = 'status://drive-sync' export const OIDC_STATUS_DOCUMENT_URI = 'status://oidc' export const VERSION_INFO_DOCUMENT_URI = 'app://version' +export const PIN_RELEASE_DOCUMENT_URI = 'app://pin-release' export const RUNNER_STATUS_DOCUMENT_URI = 'status://runners' const RUNNER_KERNELS_DOCUMENT_PREFIX = `${RUNNER_STATUS_DOCUMENT_URI}/` const RUNNER_KERNELS_DOCUMENT_SUFFIX = '/kernels' @@ -187,6 +188,7 @@ export function deriveWorkspaceDocumentTitle(uri: string): string { return 'Google Drive Sync Status' } if (isOidcStatusUri(documentUri)) return 'Runme authentication' + if (documentUri === PIN_RELEASE_DOCUMENT_URI) return 'Pin Release' if (isVersionInfoUri(documentUri)) { return 'Version Information' } diff --git a/app/src/releases/control.ts b/app/src/releases/control.ts index a78680db..d3b4d053 100644 --- a/app/src/releases/control.ts +++ b/app/src/releases/control.ts @@ -25,6 +25,7 @@ export type Manifest = { buildInputsDigest: string publishedAt: string recoveredControlFormat?: 1 + tabRecoveryFormat?: 1 requiredCapabilities: string[] testedRollbackTargets: string[] launcherProtocol: { min: number; max: number } diff --git a/app/src/releases/controlChunks.ts b/app/src/releases/controlChunks.ts index a0585615..6c51c979 100644 --- a/app/src/releases/controlChunks.ts +++ b/app/src/releases/controlChunks.ts @@ -10,7 +10,8 @@ export function readControlValue( table: IDBObjectStore, value: unknown, done: (value: unknown) => void, - failed: (error: unknown) => void + failed: (error: unknown) => void, + progress?: (read: number, total: number) => void ): void { const header = value as Partial
| undefined if (header?.encoding !== 'runme-control-json-chunks-v1') { @@ -24,6 +25,7 @@ export function readControlValue( } const chunks: string[] = new Array(count) let remaining = count + progress?.(0, count) for (let index = 0; index < count; index++) { const request = table.get(`control-chunk-${index}`) request.onerror = () => failed(request.error) @@ -36,7 +38,9 @@ export function readControlValue( return } chunks[index] = request.result - if (--remaining === 0) { + remaining-- + progress?.(count - remaining, count) + if (remaining === 0) { try { done(JSON.parse(chunks.join(''))) } catch (error) { diff --git a/app/src/releases/launcher.ts b/app/src/releases/launcher.ts index e6a025df..d5ae5215 100644 --- a/app/src/releases/launcher.ts +++ b/app/src/releases/launcher.ts @@ -18,17 +18,48 @@ import { import { loaderController } from './loaderController' import { recoverUnreadableControl } from './recoverControl' import { withRecoveryOwner } from './recoveryOwnerLock' +import { type ReleaseAttention, releaseAttention } from './releaseAttention' +import { resolvePublishedRelease } from './releaseCatalog' import { ReleaseControlError } from './releaseDatabase' +import { releaseDatabaseDiagnostics } from './releaseDiagnostics' import type { OfflineDownloadStatus, ReleaseRuntime } from './runtime' import { startupDiagnostics } from './startupDiagnostics' -import { prepareStartupRecovery } from './startupRecovery' +import { + downloadRecoveryRelease, + prepareStartupRecovery, +} from './startupRecovery' const tabId = crypto.randomUUID() +const recovery = location.pathname === '/recovery.html' const panel = document.createElement('section') panel.id = 'runme-release-panel' panel.setAttribute('aria-label', 'Runme versions') panel.style.cssText = 'position:fixed;right:12px;bottom:12px;z-index:2147483647;background:white;color:#222;border:1px solid #aaa;border-radius:8px;padding:12px;max-width:480px;font:14px system-ui;max-height:80vh;overflow:auto' +// Recovery deliberately never imports the app. Keep its explanation and exit +// outside the dynamic controls so errors and collapsed details cannot hide them. +const recoveryIntro = document.createElement('header') +if (recovery) { + recoveryIntro.id = 'runme-recovery-intro' + const heading = document.createElement('h1') + heading.textContent = 'Runme recovery' + heading.style.cssText = 'font-size:20px;margin:0 0 8px' + const explanation = document.createElement('p') + explanation.textContent = + 'This page manages app versions. Your notebooks open in Runme when you return.' + const returnLink = document.createElement('a') + const appUrl = new URL(location.href) + appUrl.pathname = '/' + // Recovery requests have already been applied; returning must not replay them. + appUrl.searchParams.delete('release') + appUrl.searchParams.delete('channel') + returnLink.href = appUrl.pathname + appUrl.search + appUrl.hash + returnLink.textContent = 'Return to Runme' + returnLink.style.cssText = + 'display:inline-block;background:#1463d6;color:white;padding:10px 16px;border-radius:6px;font-weight:600;text-decoration:none' + recoveryIntro.append(heading, explanation, returnLink) + panel.append(recoveryIntro) +} const status = document.createElement('p') status.setAttribute('role', 'status') panel.append(status) @@ -43,20 +74,22 @@ panel.append(diagnostics.element) let progress: ReleaseProgress | undefined document.body.append(panel) let runtime: ReleaseRuntime | undefined -let preparation: string | undefined -let prepared = false +let reloadAfterActivation: string | undefined +let reloadingTab = false +let legacyPrepared = false let uiKey = '' let renderGeneration = 0 -let preparing = false let ticking = false let owner: SharedWorker | undefined let ownerKey = '' let dismissedCandidate: string | undefined let appReady = false let versionUiUsers = 0 +let attentionUiUsers = 0 +let attention: ReleaseAttention | undefined let transitioning = false -let details = location.pathname === '/recovery.html' -const recovery = details +let tabUpdateAvailable = false +let details = recovery let startupPrepared = false let recoveringStartup = false let deferredFonts = false @@ -101,6 +134,11 @@ function completeOffline(manifest: Manifest): Promise { /** Display diagnostics as text, never executable release metadata or user content. */ function report(value: unknown, nextProgress?: ReleaseProgress): void { + if (value instanceof Error) + attention = { + kind: 'error', + message: 'Version information needs attention', + } status.textContent = String(value instanceof Error ? value.message : value) progress = nextProgress progressBar.hidden = !progress @@ -139,7 +177,11 @@ function freeze(): void { * host an older app. Hide fallback controls only after an explicit UI handoff. */ function updatePanelVisibility(): void { - panel.hidden = appReady && versionUiUsers > 0 && !recovery && !transitioning + panel.hidden = + appReady && + versionUiUsers > 0 && + !recovery && + (attentionUiUsers > 0 || (!transitioning && !tabUpdateAvailable)) } /** A held tab lock is proof of liveness, including suspended tabs. No heartbeat @@ -281,9 +323,18 @@ async function downloadSelection(selection?: Selection): Promise { } async function begin(target: Manifest): Promise { - await navigator.locks.request('runme-release-cache', { mode: 'shared' }, () => - beginLocked(target) - ) + try { + await navigator.locks.request( + 'runme-release-cache', + { mode: 'shared' }, + () => beginLocked(target) + ) + } catch (error) { + // Historical apps can only flush to their original worker. A completed + // local flush permits restoring that tab even if activation then failed. + if (legacyPrepared) location.reload() + throw error + } } async function beginLocked(target: Manifest): Promise { report('Verifying the downloaded release…') @@ -297,6 +348,19 @@ async function beginLocked(target: Manifest): Promise { throw new Error( 'This release predates release-settings recovery. Choose a newer release.' ) + if ( + runtime && + (!runtime.prepareReload || target.tabRecoveryFormat !== 1) && + !legacyPrepared + ) { + if (!runtime.prepare) + throw new Error( + 'Wait for this tab to finish loading before activating a release' + ) + await runtime.prepare() + legacyPrepared = true + freeze() + } const pending = current.transition if (pending?.phase === 'handoff') { // Recovery can replace a failed candidate only while no storage owner exists. @@ -325,6 +389,7 @@ async function beginLocked(target: Manifest): Promise { delete s.ownerError }) }) + if (runtime) reloadAfterActivation = target.releaseId ownerKey = '' await render() return @@ -347,58 +412,61 @@ async function beginLocked(target: Manifest): Promise { from: s.active, target, epoch: Math.max(s.active?.epoch ?? 0, s.epochBase ?? 0) + 1, - phase: 'prepare', - participants: Object.fromEntries( - Object.entries(s.tabs).map(([id, tab]) => [id, { epoch: tab.epoch }]) - ), + phase: 'handoff', + participants: {}, } }) + if (runtime) reloadAfterActivation = target.releaseId await render() } -/** Each loop advances one durable phase under a short coordinator lock. Losing - * this tab loses no decision: another launcher resumes the same transition. +/** Activation changes the selected worker without consulting tab readiness. + * Any launcher can start the selected candidate if the initiating tab disappears. + * Tab records only keep live app assets from being pruned. */ async function coordinate(): Promise { - await navigator.locks.request( - 'runme-release-transition', - { ifAvailable: true }, - async (lock) => { - if (!lock) return - const state = await readControl() - const gone: string[] = [] - for (const id of Object.keys(state.tabs)) - if (await closed(id)) gone.push(id) - const transition = state.transition - const allReady = - transition?.phase === 'prepare' && - Object.entries(transition.participants).every( - ([id, p]) => gone.includes(id) || (p.ready !== undefined && !p.error) + const state = await readControl() + const gone: string[] = [] + for (const id of Object.keys(state.tabs)) if (await closed(id)) gone.push(id) + if (gone.length) + await control((s) => { + for (const id of gone) delete s.tabs[id] + }) + if (state.transition?.phase === 'handoff') + startOwner({ + manifest: state.transition.target, + epoch: state.transition.epoch, + }) +} + +/** Only a user action or this tab's own activation can authorize its reload. + * Checkpoint failure leaves the mounted editor alive; no peer participates. + */ +async function reloadThisTab(): Promise { + if (reloadingTab) return + reloadingTab = true + freeze() + try { + if (runtime && !legacyPrepared) { + const selected = (await readControl()).active + if (runtime.prepareReload && selected?.manifest.tabRecoveryFormat !== 1) + throw new Error( + 'The selected release cannot restore this tab’s pending-save checkpoints. Keep this tab open and export edits, or select a release with tab recovery support.' ) - if (allReady) - await verifyCache(transition.target, undefined, startupScope()) - await control((s) => { - for (const id of gone) { - delete s.tabs[id] - if (s.transition) delete s.transition.participants[id] - } - if ( - transition && - allReady && - s.transition?.id === transition.id && - s.transition.phase === 'prepare' && - Object.values(s.transition.participants).every( - (p) => p.ready !== undefined && !p.error - ) + if (runtime.prepareReload) await runtime.prepareReload() + else if (runtime.prepare) await runtime.prepare() + else + throw new Error( + 'This tab is not ready to preserve its editors; wait before reloading' ) - s.transition.phase = 'handoff' - }) } - ) - const state = await readControl() - const pending = state.transition - if (pending?.phase === 'handoff') - startOwner({ manifest: pending.target, epoch: pending.epoch }) + location.reload() + } catch (error) { + reloadingTab = false + const root = document.getElementById('root') + if (root) root.inert = false + throw error + } } /** Starting a worker only contends for the lifetime lock. Its verified startup @@ -472,59 +540,15 @@ async function tick(): Promise { uiKey = nextUiKey await render() } - if (prepared && !transition) { - location.reload() - return - } - if (runtime && state.active?.epoch !== runtime.active.epoch) { - freeze() - location.reload() - return - } + // A different tab's activation never freezes or reloads this page. if ( + !transition && runtime && - transition?.phase === 'prepare' && - transition?.participants[tabId] && - preparation !== transition.id && - !preparing + state.active?.manifest.releaseId === reloadAfterActivation ) { - freeze() - if (!runtime.prepare) { - report('Waiting for application startup before preserving edits…') - return - } - preparing = true - try { - const refs = await runtime.prepare() - prepared = true - await control((s) => { - if ( - s.transition?.id !== transition.id || - s.transition.phase !== 'prepare' - ) - throw new Error('Upgrade preparation changed') - s.transition.participants[tabId].ready = refs - delete s.transition.participants[tabId].error - }) - preparation = transition.id - } catch (error) { - await control((s) => { - if (s.transition?.id === transition.id) - s.transition.participants[tabId].error = String(error) - }) - preparation = - error instanceof Error && error.name === 'UpgradeBusyError' - ? undefined - : transition.id - if (prepared) { - location.reload() - return - } - const root = document.getElementById('root') - if (root) root.inert = false - } finally { - preparing = false - } + reloadAfterActivation = undefined + await reloadThisTab() + return } await coordinate() const latest = await readControl() @@ -536,11 +560,10 @@ async function tick(): Promise { ) { report(latest.ownerError.message) } else if (latest.transition) { - const blocked = Object.entries(latest.transition.participants).filter( - ([, p]) => !p.ready || p.error - ) report( - `Upgrade ${latest.transition.phase}: ${blocked.length ? blocked.map(([id, p]) => `${latest.tabs[id]?.label ?? id.slice(0, 8)}: ${p.error ?? 'waiting for tab'}`).join('; ') : 'waiting for exclusive storage ownership'}` + latest.transition.phase === 'prepare' + ? 'An older upgrade is pending. Cancel it before starting an independent worker upgrade.' + : 'Starting the selected storage worker; other tabs keep their editors open.' ) } } catch (error) { @@ -626,7 +649,13 @@ async function render(): Promise { const generation = ++renderGeneration const s = await readControl() if (generation !== renderGeneration) return + attention = releaseAttention(s, runtime?.active.manifest.releaseId) transitioning = Boolean(s.transition) + tabUpdateAvailable = Boolean( + runtime && + s.active && + runtime.active.manifest.releaseId !== s.active.manifest.releaseId + ) // Normal management belongs to Information. Startup/recovery and frozen-app // transitions retain independent controls, including cancel/resume. updatePanelVisibility() @@ -638,13 +667,24 @@ async function render(): Promise { s.selection, ]) for (const node of [...panel.children]) - if (node !== status && node !== progressBar && node !== diagnostics.element) + if ( + node !== status && + node !== progressBar && + node !== diagnostics.element && + node !== recoveryIntro + ) node.remove() report( s.active ? `Runme ${s.active.manifest.webCommit.slice(0, 8)} · ${s.selection.kind === 'channel' ? s.selection.name : 'pinned'}` : 'Installing Runme…' ) + if (tabUpdateAvailable) { + report( + 'The selected application version has changed. Reload this tab when ready; other tabs stay open.' + ) + button('Preserve edits and reload this tab', reloadThisTab) + } button(details ? 'Hide versions' : 'Versions', () => { details = !details return render() @@ -654,7 +694,7 @@ async function render(): Promise { s.candidate.releaseId !== s.active?.manifest.releaseId && (details || dismissedCandidate !== s.candidate.releaseId) ) { - button('Upgrade all tabs', () => begin(s.candidate!)) + button('Activate release and update this tab', () => begin(s.candidate!)) button('Later', async () => { dismissedCandidate = s.candidate!.releaseId details = false @@ -708,16 +748,18 @@ async function render(): Promise { if (s.transition?.phase === 'prepare') button('Cancel upgrade', () => cancel(s.transition!.id)) const label = document.createElement('label') - label.textContent = 'Exact release ID ' + label.textContent = 'Commit SHA or release ID ' const input = document.createElement('input') input.placeholder = '-' label.append(input) panel.append(label) - button('Download exact release', () => - check({ kind: 'pinned', releaseId: input.value.trim() }) + button('Download exact release', async () => + check({ + kind: 'pinned', + releaseId: await resolvePublishedRelease(input.value), + }) ) - button('Retry preparation / resume', async () => { - preparation = undefined + button('Retry worker activation', async () => { ownerKey = '' await tick() }) @@ -775,8 +817,10 @@ async function main(): Promise { report('Starting the installed release…') diagnostics.record('launcher', 'info', { path: new URL(import.meta.url).pathname, - source: document.documentElement.dataset.runmeLauncherSource ?? 'legacy-shell', - bootstrapLauncher: document.documentElement.dataset.runmeLauncher ?? 'unknown', + source: + document.documentElement.dataset.runmeLauncherSource ?? 'legacy-shell', + bootstrapLauncher: + document.documentElement.dataset.runmeLauncher ?? 'unknown', }) await installLoader() report('Registering this tab…') @@ -823,13 +867,7 @@ async function startRelease(applyRequest = true): Promise { 60 * 60 * 1000 ) window.addEventListener('pageshow', () => { - freeze() - void tick().then(() => { - if (!preparation) { - const root = document.getElementById('root') - if (root) root.inert = false - } - }) + void tick() }) if (s.active) void check().catch((error) => @@ -845,10 +883,16 @@ function showStartupRecovery(error: unknown): void { if (error instanceof ReleaseControlError) diagnostics.record(`release-settings-${error.stage}`, 'failed', { ...error.diagnosticDetails, + trace: JSON.stringify(error.trace ?? null), stage: `release-settings-${error.stage}`, }) for (const node of [...panel.children]) - if (node !== status && node !== progressBar && node !== diagnostics.element) + if ( + node !== status && + node !== progressBar && + node !== diagnostics.element && + node !== recoveryIntro + ) node.remove() report( `${String(error instanceof Error ? error.message : error)} Retry loading stable, or choose another release below. Browser storage must respond before Runme can safely open notebooks.` @@ -858,22 +902,24 @@ function showStartupRecovery(error: unknown): void { recoverStartup({ kind: 'channel', name: 'latest' }) ) const label = document.createElement('label') - label.textContent = 'Exact release ID ' + label.textContent = 'Commit SHA or release ID ' const input = document.createElement('input') input.placeholder = '-' label.append(input) panel.append(label) - button('Try exact release', () => - recoverStartup( - validateSelection({ kind: 'pinned', releaseId: input.value.trim() }) - ) + input.value = new URLSearchParams(location.search).get('release') ?? '' + button('Try exact release', async () => + recoverStartup({ + kind: 'pinned', + releaseId: await resolvePublishedRelease(input.value), + }) ) // An expanded trace must never push the recovery actions below the fold. panel.append(diagnostics.element) } /** Prefer latest stable after a settings failure. Downloads do not need the - * settings DB; activation still uses the usual prepare/handoff and owner lock. + * settings DB; activation still uses worker handoff and the exclusive owner lock. * Existing transitions are resumed, never discarded to force a new version. */ async function recoverStartup( @@ -922,15 +968,35 @@ async function recoverStartup( } window.runmeVersions = { + reloadTab: reloadThisTab, + diagnostics: releaseDatabaseDiagnostics, + prepareRecovery: (selection) => + downloadRecoveryRelease( + validateSelection(selection), + (value) => reportProgress('Preparing recovery release', value), + startupScope() + ), + recover: async (selection) => { + const candidate = await prepareStartupRecovery( + validateSelection(selection), + (value) => reportProgress('Preparing recovery release', value), + startupScope() + ) + await begin(candidate) + await tick() + }, uiVersion: 1, - registerVersionUi: () => { + attention: () => attention, + registerVersionUi: (options) => { versionUiUsers++ + if (options?.attention) attentionUiUsers++ updatePanelVisibility() let released = false return () => { if (released) return released = true versionUiUsers-- + if (options?.attention) attentionUiUsers-- updatePanelVisibility() } }, @@ -969,7 +1035,6 @@ window.runmeVersions = { cancel, prune: pruneUnusedCaches, retry: async () => { - preparation = undefined ownerKey = '' await tick() }, @@ -1007,6 +1072,7 @@ void diagnostics.step('bootstrap', main).catch((error) => { if (startupPrepared && error instanceof ReleaseControlError) { diagnostics.record(`release-settings-${error.stage}`, 'failed', { ...error.diagnosticDetails, + trace: JSON.stringify(error.trace ?? null), stage: `release-settings-${error.stage}`, }) void recoverStartup() diff --git a/app/src/releases/prepare.ts b/app/src/releases/prepare.ts index 9bf4fc08..0c853fcd 100644 --- a/app/src/releases/prepare.ts +++ b/app/src/releases/prepare.ts @@ -5,8 +5,8 @@ import { closeReleaseDatabases } from './databases' import { assertNoUpgradeBlockers, setReleasePreparation } from './runtime' let frozen = false -/** Routine activation requires durable commits. Emergency sessionStorage copies - * are intentionally not involved. Failed/uncertain saves keep this tab open. +/** Compatibility hook for historical launchers that require durable commits. + * Current launchers use prepareTabReload and never ask peers to prepare. */ export async function prepareRelease(): Promise { if (frozen) diff --git a/app/src/releases/prepareTabReload.test.ts b/app/src/releases/prepareTabReload.test.ts new file mode 100644 index 00000000..cc686f63 --- /dev/null +++ b/app/src/releases/prepareTabReload.test.ts @@ -0,0 +1,97 @@ +import { afterEach, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + capture: vi.fn(), + resume: vi.fn(), + consoleFlush: vi.fn(), + consoleResume: vi.fn(), + model: { + getSnapshot: () => ({ loaded: true, notebook: { cells: [] } }), + getUri: () => 'local://file/a', + hasActiveExecutions: () => false, + isReviewPending: () => false, + isReleasePending: () => false, + isReadOnly: () => false, + setReleasePending: vi.fn(), + flushPendingPersist: vi.fn(), + }, +})) +vi.mock('../lib/notebookDataController', () => ({ + getNotebookDataController: () => ({ + getOpenNotebooks: () => [{ uri: 'local://file/a', state: 'loaded' }], + getNotebookData: () => mocks.model, + }), +})) +vi.mock('../lib/appConsole/appConsoleController', () => ({ + getAppConsoleData: () => ({ + flushForRelease: mocks.consoleFlush, + resumeAfterFailedPreparation: mocks.consoleResume, + }), +})) +vi.mock('../storage/tabRecoveryCheckpoint', () => ({ + checkpointEditorSessions: mocks.capture, + checkpointEditorUris: () => new Set(['local://file/a']), + resumeCheckpointTransport: mocks.resume, +})) +afterEach(() => { + vi.resetModules() + vi.clearAllMocks() +}) +it('checkpoints replayable pending saves without flushing through the worker', async () => { + mocks.capture.mockResolvedValue(new Set(['local://file/a'])) + const runtime = await import('./runtime') + const release = runtime.registerUpgradeBlocker( + () => 'Pending editor save', + true, + true + ) + const { prepareTabReload } = await import('./prepareTabReload') + await prepareTabReload() + expect(mocks.capture).toHaveBeenCalledWith( + new Map([['local://file/a', { cells: [] }]]), + expect.any(AbortSignal) + ) + expect(mocks.model.flushPendingPersist).not.toHaveBeenCalled() + expect(mocks.model.setReleasePending).toHaveBeenLastCalledWith(true) + release() +}) +it('keeps editors usable if backup fails', async () => { + mocks.capture.mockRejectedValue(new Error('quota')) + const { prepareTabReload } = await import('./prepareTabReload') + await expect(prepareTabReload()).rejects.toThrow('quota') + expect(mocks.resume).toHaveBeenCalledOnce() + expect(mocks.consoleResume).toHaveBeenCalledOnce() + expect(mocks.model.setReleasePending).toHaveBeenLastCalledWith(false) + const { assertReleaseWritable } = await import('./runtime') + expect(assertReleaseWritable).not.toThrow() +}) +it('unknown external effects block only this tab reload', async () => { + const runtime = await import('./runtime') + const release = runtime.registerUpgradeBlocker( + () => 'Creation outcome unknown' + ) + const { prepareTabReload } = await import('./prepareTabReload') + await expect(prepareTabReload()).rejects.toThrow('Creation outcome unknown') + expect(mocks.capture).not.toHaveBeenCalled() + release() +}) +it('refuses to discard an editor missing from the checkpoint', async () => { + mocks.capture.mockResolvedValue(new Set()) + const { prepareTabReload } = await import('./prepareTabReload') + await expect(prepareTabReload()).rejects.toThrow( + 'without replayable save recovery' + ) + expect(mocks.model.setReleasePending).toHaveBeenLastCalledWith(false) +}) + +it('bounds a stalled console backup and leaves the tab editable', async () => { + vi.useFakeTimers() + mocks.consoleFlush.mockImplementationOnce(() => new Promise(() => {})) + const { prepareTabReload } = await import('./prepareTabReload') + const result = expect(prepareTabReload()).rejects.toThrow('backup timed out') + await vi.advanceTimersByTimeAsync(15_000) + await result + expect(mocks.model.setReleasePending).toHaveBeenLastCalledWith(false) + expect(mocks.capture).not.toHaveBeenCalled() + vi.useRealTimers() +}) diff --git a/app/src/releases/prepareTabReload.ts b/app/src/releases/prepareTabReload.ts new file mode 100644 index 00000000..a0ccb851 --- /dev/null +++ b/app/src/releases/prepareTabReload.ts @@ -0,0 +1,103 @@ +import { getAppConsoleData } from '../lib/appConsole/appConsoleController' +import { getNotebookDataController } from '../lib/notebookDataController' +import { + checkpointEditorSessions, + checkpointEditorUris, + resumeCheckpointTransport, +} from '../storage/tabRecoveryCheckpoint' +import { assertNoUpgradeBlockers, setReleasePreparation } from './runtime' + +/** Bound a local preservation step without interpreting silence as data loss. */ +async function preserveWithinDeadline( + work: Promise, + abort: AbortController +): Promise { + let timer: ReturnType | undefined + try { + return await Promise.race([ + work, + new Promise((_, reject) => { + timer = setTimeout(() => { + const error = new Error( + 'This tab’s recovery backup timed out; no reload performed. Keep it open and retry or export edits.' + ) + abort.abort(error) + reject(error) + }, 15_000) + }), + ]) + } finally { + clearTimeout(timer) + } +} + +/** Preserve only this tab. A broken worker cannot block replayable editor saves + * from being checkpointed; unknown external effects still block a local reload. + */ +export async function prepareTabReload(): Promise { + assertNoUpgradeBlockers(true) + const controller = getNotebookDataController() + const entries = controller.getOpenNotebooks() + if ( + entries.some( + (entry) => entry.state === 'loading' || entry.state === 'resolving' + ) + ) + throw new Error( + 'Wait for this tab’s notebooks to finish loading before reloading' + ) + const models = entries + .map((entry) => controller.getNotebookData(entry.uri)) + .filter((model) => model?.getSnapshot().loaded) + if ( + models.some( + (model) => + model!.hasActiveExecutions() || + model!.isReviewPending() || + model!.isReleasePending() + ) + ) + throw new Error( + 'Stop running cells and finish reviews in this tab before reloading' + ) + const consoleData = getAppConsoleData() + setReleasePreparation(true) + models.forEach((model) => model!.setReleasePending(true)) + const abort = new AbortController() + try { + const replayable = checkpointEditorUris() + // Legacy JSON/Markdown editors have no operation IDs to replay safely. + // Flush those through a healthy worker; failure blocks only this tab. + for (const model of models) + if (!model!.isReadOnly() && !replayable.has(model!.getUri())) + await preserveWithinDeadline(model!.flushPendingPersist(), abort) + // Console persistence is independent of the notebook worker. Failure keeps + // this tab open, and cannot hold up another tab's worker activation. + await preserveWithinDeadline( + consoleData.flushForRelease(abort.signal), + abort + ) + const editors = new Map( + models + .filter( + (model) => !model!.isReadOnly() && replayable.has(model!.getUri()) + ) + .map((model) => [model!.getUri(), model!.getSnapshot().notebook]) + ) + const captured = await preserveWithinDeadline( + checkpointEditorSessions(editors, abort.signal), + abort + ) + if ([...editors.keys()].some((uri) => !captured.has(uri))) + throw new Error( + 'This tab has an editor without replayable save recovery; save or export it before reloading' + ) + assertNoUpgradeBlockers(true) + } catch (error) { + resumeCheckpointTransport() + consoleData.resumeAfterFailedPreparation() + models.forEach((model) => model!.setReleasePending(false)) + setReleasePreparation(false) + throw error + } +} diff --git a/app/src/releases/releaseAttention.test.ts b/app/src/releases/releaseAttention.test.ts new file mode 100644 index 00000000..184568dc --- /dev/null +++ b/app/src/releases/releaseAttention.test.ts @@ -0,0 +1,19 @@ +import { expect, it } from 'vitest' + +import type { Control } from './control' +import { releaseAttention } from './releaseAttention' + +it('distinguishes a downloaded update, an outdated tab, activation and a healthy pin', () => { + const state = { + active: { manifest: { releaseId: 'installed' } }, + selection: { kind: 'pinned', releaseId: 'installed' }, + } as Control + expect(releaseAttention(state, 'installed')).toBeUndefined() + state.candidate = { releaseId: 'new' } as Control['candidate'] + expect(releaseAttention(state, 'installed')?.kind).toBe('update') + expect(releaseAttention(state, 'old')?.kind).toBe('reload') + state.transition = { phase: 'handoff' } as Control['transition'] + expect(releaseAttention(state, 'old')?.kind).toBe('busy') + state.ownerError = { message: 'Failed' } as Control['ownerError'] + expect(releaseAttention(state, 'old')?.kind).toBe('error') +}) diff --git a/app/src/releases/releaseAttention.ts b/app/src/releases/releaseAttention.ts new file mode 100644 index 00000000..0c386b75 --- /dev/null +++ b/app/src/releases/releaseAttention.ts @@ -0,0 +1,35 @@ +import type { Control } from './control' + +export type ReleaseAttention = { + kind: 'update' | 'reload' | 'busy' | 'error' + message: string +} + +/** Describe actionable version state without fetching or activating a release. */ +export function releaseAttention( + state: Control, + runningReleaseId?: string +): ReleaseAttention | undefined { + if (state.ownerError) + return { kind: 'error', message: 'Version activation needs attention' } + if (state.transition) + return { + kind: 'busy', + message: + state.transition.phase === 'prepare' + ? 'An earlier upgrade needs attention' + : 'Version activation in progress', + } + if ( + runningReleaseId && + state.active && + runningReleaseId !== state.active.manifest.releaseId + ) + return { kind: 'reload', message: 'This tab is out of date' } + if ( + state.candidate && + state.candidate.releaseId !== state.active?.manifest.releaseId + ) + return { kind: 'update', message: 'An update is ready to activate' } + return undefined +} diff --git a/app/src/releases/releaseCatalog.test.ts b/app/src/releases/releaseCatalog.test.ts new file mode 100644 index 00000000..b521274a --- /dev/null +++ b/app/src/releases/releaseCatalog.test.ts @@ -0,0 +1,34 @@ +import { afterEach, expect, it, vi } from 'vitest' + +import { resolvePublishedRelease } from './releaseCatalog' + +const id = `${'a'.repeat(40)}-${'b'.repeat(64)}` +afterEach(() => vi.unstubAllGlobals()) +it('accepts a full release ID without requiring catalog or settings access', async () => { + const fetch = vi.fn() + vi.stubGlobal('fetch', fetch) + expect(await resolvePublishedRelease(id)).toBe(id) + expect(fetch).not.toHaveBeenCalled() +}) +it.each([0, 1, 2])( + 'requires exactly one published build for a commit (%s matches)', + async (count) => { + vi.stubGlobal( + 'fetch', + vi.fn(async () => ({ + ok: true, + json: async () => ({ + format: 1, + releases: [id, `${'a'.repeat(40)}-${'c'.repeat(64)}`] + .slice(0, count) + .map((releaseId) => ({ releaseId })), + }), + })) + ) + if (count === 1) expect(await resolvePublishedRelease('aaaaaaa')).toBe(id) + else + await expect(resolvePublishedRelease('aaaaaaa')).rejects.toThrow( + count ? 'Multiple builds' : 'No published release' + ) + } +) diff --git a/app/src/releases/releaseCatalog.ts b/app/src/releases/releaseCatalog.ts new file mode 100644 index 00000000..d190aaec --- /dev/null +++ b/app/src/releases/releaseCatalog.ts @@ -0,0 +1,45 @@ +import { releaseIdValid } from './control' + +/** Discover published builds without opening release settings or notebook storage. */ +export async function publishedReleaseIds(): Promise { + const response = await fetch('/releases/catalog.json', { + cache: 'no-store', + credentials: 'omit', + redirect: 'error', + signal: AbortSignal.timeout(15_000), + }) + if (!response.ok) + throw new Error( + 'Published releases are unavailable. Try an exact release ID.' + ) + const catalog = await response.json() + if (catalog.format !== 1 || !Array.isArray(catalog.releases)) + throw new Error('Unsupported release catalog') + return [ + ...new Set( + catalog.releases + .map((release: { releaseId?: unknown }) => release?.releaseId) + .filter(releaseIdValid) + ), + ].reverse() +} + +/** A commit may have multiple builds; never guess between distinct matches. */ +export async function resolvePublishedRelease(input: string): Promise { + const value = input.trim().toLowerCase() + if (releaseIdValid(value)) return value + if (!/^[a-f0-9]{7,40}$/.test(value)) + throw new Error( + 'Enter a commit SHA (at least 7 characters) or a full release ID.' + ) + const matches = (await publishedReleaseIds()).filter((id) => + id.slice(0, 40).startsWith(value) + ) + if (matches.length !== 1) + throw new Error( + matches.length + ? 'Multiple builds match this commit. Browse published releases and select an exact release ID.' + : 'No published release matches this commit.' + ) + return matches[0] +} diff --git a/app/src/releases/releaseDatabase.test.ts b/app/src/releases/releaseDatabase.test.ts index 0cbb57e4..f046dada 100644 --- a/app/src/releases/releaseDatabase.test.ts +++ b/app/src/releases/releaseDatabase.test.ts @@ -290,3 +290,65 @@ it('keeps exception text and arbitrary names out of visible diagnostics', () => expect(arbitrary.message).not.toContain('secret') expect(arbitrary.diagnosticDetails).not.toHaveProperty('browserError') }) + +it.each(['get', 'chunks', 'commit'] as const)( + 'identifies the last callback reached when %s stalls', + async (phase) => { + vi.useFakeTimers() + const mock = database( + phase === 'chunks' + ? { encoding: 'runme-control-json-chunks-v1', count: 2 } + : undefined + ) + const chunks: any[] = [] + if (phase === 'chunks') + mock.table.get.mockImplementation((key?: string) => { + if (key === 'control') return mock.get + const request = { result: '{}' } + chunks.push(request) + return request + }) + const pending = readControl() + const failed = expect(pending).rejects.toMatchObject({ + diagnosticDetails: { + trace: { + mode: 'readonly', + phase: + phase === 'get' + ? 'get-requested' + : phase === 'chunks' + ? 'reading-chunks' + : 'waiting-for-commit', + ...(phase === 'chunks' ? { chunksRead: 1, chunksTotal: 2 } : {}), + }, + }, + }) + await opened(mock) + if (phase !== 'get') mock.get.onsuccess() + if (phase === 'chunks') chunks[0].onsuccess() + await vi.advanceTimersByTimeAsync(5000) + await failed + expect(mock.tx.abort).toHaveBeenCalledOnce() + } +) + +it('offers memory-only pending diagnostics and keeps stored values out of traces', async () => { + const { releaseDatabaseDiagnostics } = await import('./releaseDiagnostics') + const mock = database({ format: 1, private: 'secret notebook contents' }) + const pending = readControl() + await opened(mock) + const before = mock.factory.open.mock.calls.length + expect(releaseDatabaseDiagnostics().active).toEqual( + expect.arrayContaining([ + expect.objectContaining({ phase: 'get-requested' }), + ]) + ) + expect(mock.factory.open).toHaveBeenCalledTimes(before) + mock.get.onsuccess() + mock.tx.oncomplete() + await pending + expect(releaseDatabaseDiagnostics().activeCount).toBe(0) + expect(JSON.stringify(releaseDatabaseDiagnostics())).not.toContain( + 'secret notebook contents' + ) +}) diff --git a/app/src/releases/releaseDatabase.ts b/app/src/releases/releaseDatabase.ts index e902b2cd..79613fe2 100644 --- a/app/src/releases/releaseDatabase.ts +++ b/app/src/releases/releaseDatabase.ts @@ -1,5 +1,9 @@ import { appLogger } from '../lib/logging/runtime' import { readControlValue, writeControlValue } from './controlChunks' +import { + type ReleaseDatabaseTrace, + createReleaseTrace, +} from './releaseDiagnostics' const DEADLINE_MS = 5000 @@ -38,7 +42,8 @@ export class ReleaseControlError extends Error { readonly stage: 'open' | 'read' | 'commit', readonly reason: 'blocked' | 'timeout' | 'failed', readonly operation: Operation = stage === 'read' ? 'get' : stage, - readonly cause?: unknown + readonly cause?: unknown, + readonly trace?: ReleaseDatabaseTrace ) { super( `Release settings ${stage} ${reason} (${operation}${browserErrorName(cause) ? ': ' + browserErrorName(cause) : ''}). Stored notebook data has not been removed.` @@ -55,6 +60,7 @@ export class ReleaseControlError extends Error { stage: this.stage, reason: this.reason, operation: this.operation, + ...(this.trace ? { trace: this.trace } : {}), ...(browserErrorName(this.cause) ? { browserError: browserErrorName(this.cause) } : {}), @@ -65,7 +71,10 @@ export class ReleaseControlError extends Error { /** An open request cannot be cancelled. Close late connections, and abort late * upgrades so timed-out attempts never create a store or retain a connection. */ -function openDatabase(name: string): Promise { +function openDatabase( + name: string, + trace?: ReturnType +): Promise { return new Promise((resolve, reject) => { let settled = false let request: IDBOpenDBRequest @@ -82,7 +91,15 @@ function openDatabase(name: string): Promise { } catch { /* A completed upgrade has no live transaction. */ } - reject(new ReleaseControlError('open', reason, operation, cause)) + reject( + new ReleaseControlError( + 'open', + reason, + operation, + cause, + trace?.snapshot() + ) + ) } const timer = setTimeout(() => fail('timeout'), DEADLINE_MS) try { @@ -90,6 +107,7 @@ function openDatabase(name: string): Promise { request.onblocked = () => fail('blocked') request.onerror = () => fail('failed', request.error) request.onupgradeneeded = () => { + trace?.mark('schema-upgrade') if (settled) { request.transaction?.abort() return @@ -109,6 +127,8 @@ function openDatabase(name: string): Promise { settled = true clearTimeout(timer) db.onversionchange = () => db.close() + trace?.details({ databaseVersion: db.version }) + trace?.mark('open-succeeded') resolve(db) } } catch (error) { @@ -126,14 +146,30 @@ export async function releaseTransaction( write: boolean, mutate: (value: unknown) => { state: unknown; result: T } ): Promise { - const db = await openDatabase(name) + const trace = createReleaseTrace(write) + let db: IDBDatabase | undefined + let complete = false try { + db = await openDatabase(name, trace) return await new Promise((resolve, reject) => { let settled = false let tx: IDBTransaction | undefined let stage: 'read' | 'commit' = 'read' let operation: Operation = 'transaction' let result: T + const storageError = ( + stage: 'read' | 'commit', + reason: 'timeout' | 'failed', + operation: Operation, + cause?: unknown + ) => + new ReleaseControlError( + stage, + reason, + operation, + cause, + trace.snapshot() + ) const fail = (error: unknown) => { if (settled) return settled = true @@ -146,62 +182,75 @@ export async function releaseTransaction( reject(error) } const timer = setTimeout( - () => fail(new ReleaseControlError(stage, 'timeout', operation)), + () => fail(storageError(stage, 'timeout', operation)), DEADLINE_MS ) try { - tx = db.transaction('state', write ? 'readwrite' : 'readonly') + trace.mark('transaction-requested') + tx = db!.transaction('state', write ? 'readwrite' : 'readonly') tx.onabort = tx.onerror = () => { if (!settled) - fail(new ReleaseControlError(stage, 'failed', operation, tx?.error)) + fail(storageError(stage, 'failed', operation, tx?.error)) } tx.oncomplete = () => { if (settled) return settled = true clearTimeout(timer) + complete = true + trace.mark('transaction-complete') resolve(result) } const table = tx.objectStore('state') operation = 'get' + trace.mark('get-requested') const request = table.get('control') request.onerror = () => - fail(new ReleaseControlError('read', 'failed', 'get', request.error)) + fail(storageError('read', 'failed', 'get', request.error)) const accept = (value: unknown) => { if (settled) return try { + trace.mark('value-decoded') const next = mutate(value) result = next.result stage = 'commit' if (write) { operation = 'put' try { - writeControlValue(table, next.state, db.version > 1, (error) => - fail( - new ReleaseControlError('commit', 'failed', 'put', error) - ) + writeControlValue(table, next.state, db!.version > 1, (error) => + fail(storageError('commit', 'failed', 'put', error)) ) } catch (error) { - fail(new ReleaseControlError('commit', 'failed', 'put', error)) + fail(storageError('commit', 'failed', 'put', error)) return } } operation = 'commit' + trace.mark('waiting-for-commit') } catch (error) { fail(error) } } request.onsuccess = () => { - if (!settled) - readControlValue(table, request.result, accept, (error) => - fail(new ReleaseControlError('read', 'failed', 'get', error)) - ) + if (settled) return + trace.mark('get-succeeded') + readControlValue( + table, + request.result, + accept, + (error) => fail(storageError('read', 'failed', 'get', error)), + (chunksRead, chunksTotal) => { + if (chunksRead === 0) trace.mark('reading-chunks') + trace.details({ chunksRead, chunksTotal }) + } + ) } } catch (error) { - fail(new ReleaseControlError(stage, 'failed', operation, error)) + fail(storageError(stage, 'failed', operation, error)) } }) } finally { - db.close() + trace.finish(complete ? 'complete' : 'failed') + db?.close() } } diff --git a/app/src/releases/releaseDiagnostics.ts b/app/src/releases/releaseDiagnostics.ts new file mode 100644 index 00000000..d1f5996f --- /dev/null +++ b/app/src/releases/releaseDiagnostics.ts @@ -0,0 +1,97 @@ +/** Value-free, realm-local traces. Reading these never touches browser storage. */ +export type ReleaseDatabaseTrace = { + id: number + mode: 'readonly' | 'readwrite' + context: 'window' | 'worker' + visibility: string + elapsedMs: number + startedAt: string + phase: string + phaseElapsedMs: number + databaseVersion?: number + chunksRead?: number + chunksTotal?: number + outcome?: 'complete' | 'failed' + events: { phase: string; elapsedMs: number }[] +} +let sequence = 0 +const active = new Map>() +const recent: ReleaseDatabaseTrace[] = [] +const failures: ReleaseDatabaseTrace[] = [] + +/** Capture the last callback reached, not an inferred browser transaction state. */ +export function createReleaseTrace(write: boolean) { + const started = performance.now() + let changed = started + const value: ReleaseDatabaseTrace = { + id: ++sequence, + mode: write ? 'readwrite' : 'readonly', + context: typeof document === 'undefined' ? 'worker' : 'window', + visibility: + typeof document === 'undefined' + ? 'unavailable' + : document.visibilityState, + elapsedMs: 0, + startedAt: new Date().toISOString(), + phase: 'open-requested', + phaseElapsedMs: 0, + events: [{ phase: 'open-requested', elapsedMs: 0 }], + } + const trace = { + mark(phase: string) { + changed = performance.now() + value.phase = phase + value.events.push({ phase, elapsedMs: Math.round(changed - started) }) + if (value.events.length > 12) value.events.shift() + }, + details( + details: Pick< + ReleaseDatabaseTrace, + 'databaseVersion' | 'chunksRead' | 'chunksTotal' + > + ) { + Object.assign(value, details) + }, + snapshot(): ReleaseDatabaseTrace { + return { + ...value, + elapsedMs: Math.round(performance.now() - started), + phaseElapsedMs: Math.round(performance.now() - changed), + events: value.events.map((event) => ({ ...event })), + } + }, + finish(outcome: 'complete' | 'failed') { + value.outcome = outcome + active.delete(value.id) + recent.push(trace.snapshot()) + if (recent.length > 10) recent.shift() + if (outcome === 'failed') { + failures.push(trace.snapshot()) + if (failures.length > 10) failures.shift() + } + }, + } + active.set(value.id, trace) + return trace +} + +/** Bounded snapshots include reads waiting before any IndexedDB callback fires. */ +export function releaseDatabaseDiagnostics() { + const pending: ReleaseDatabaseTrace[] = [] + for (const trace of active.values()) { + pending.push(trace.snapshot()) + if (pending.length === 10) break + } + return { + activeCount: active.size, + active: pending, + failures: failures.map((trace) => ({ + ...trace, + events: trace.events.map((event) => ({ ...event })), + })), + recent: recent.map((trace) => ({ + ...trace, + events: trace.events.map((event) => ({ ...event })), + })), + } +} diff --git a/app/src/releases/runtime.test.ts b/app/src/releases/runtime.test.ts index cbc4abeb..c3e3078c 100644 --- a/app/src/releases/runtime.test.ts +++ b/app/src/releases/runtime.test.ts @@ -15,7 +15,7 @@ it('explicitly hands version controls to the app and releases them on cleanup', registerVersionUi, } as unknown as NonNullable const cleanup = adoptLegacyVersionUi() - expect(registerVersionUi).toHaveBeenCalledOnce() + expect(registerVersionUi).toHaveBeenCalledWith({ attention: true }) cleanup() expect(release).toHaveBeenCalledOnce() }) diff --git a/app/src/releases/runtime.ts b/app/src/releases/runtime.ts index 3cac762c..a442a68e 100644 --- a/app/src/releases/runtime.ts +++ b/app/src/releases/runtime.ts @@ -6,14 +6,17 @@ import type { ReleaseProgress, Selection, } from './control' +import type { ReleaseAttention } from './releaseAttention' +import type { releaseDatabaseDiagnostics } from './releaseDiagnostics' -/** The independent launcher owns coordination; React registers its durable flush - * hook only after model initialization. No application import is needed by recovery. +/** The independent launcher owns worker activation; React registers this tab’s + * recovery hook after model initialization. No application import is needed by recovery. */ export type ReleaseRuntime = { active: Active tabId: string prepare?: () => Promise + prepareReload?: () => Promise ready?: () => void } export type OfflineDownloadStatus = { @@ -33,8 +36,12 @@ declare global { cancel: () => Promise prune: () => Promise // Optional because an installed older loader can host a newer app bundle. + diagnostics?: typeof releaseDatabaseDiagnostics + prepareRecovery?: (selection: Selection) => Promise + recover?: (selection: Selection) => Promise uiVersion?: number - registerVersionUi?: () => () => void + registerVersionUi?: (options?: { attention?: boolean }) => () => void + attention?: () => ReleaseAttention | undefined status?: () => string progress?: () => ReleaseProgress | undefined offline?: (releaseId: string) => OfflineDownloadStatus | undefined @@ -42,6 +49,7 @@ declare global { cached?: () => Promise activate?: (releaseId: string) => Promise pin?: () => Promise + reloadTab?: () => Promise } } } @@ -51,7 +59,7 @@ declare global { */ export function adoptLegacyVersionUi(): () => void { if (window.runmeVersions?.registerVersionUi) - return window.runmeVersions.registerVersionUi() + return window.runmeVersions.registerVersionUi({ attention: true }) if (window.runmeVersions?.uiVersion) return () => {} const panel = document.getElementById('runme-release-panel') const root = document.getElementById('root') @@ -68,23 +76,29 @@ export function adoptLegacyVersionUi(): () => void { } } -/** All draft surfaces register blockers before permitting a coordinated reload. */ +/** Draft surfaces register blockers before permitting this tab to reload. */ const blockers = new Map< symbol, - { check: () => string | undefined; retryable: boolean } + { + check: () => string | undefined + retryable: boolean + checkpointable: boolean + } >() export function registerUpgradeBlocker( check: () => string | undefined, - retryable = false + retryable = false, + checkpointable = false ): () => void { const id = Symbol() - blockers.set(id, { check, retryable }) + blockers.set(id, { check, retryable, checkpointable }) return () => { blockers.delete(id) } } -export function assertNoUpgradeBlockers(): void { +export function assertNoUpgradeBlockers(checkpointedReload = false): void { for (const blocker of blockers.values()) { + if (checkpointedReload && blocker.checkpointable) continue const reason = blocker.check() if (reason) { const error = new Error(reason) @@ -95,7 +109,7 @@ export function assertNoUpgradeBlockers(): void { } let preparingRelease = false -/** Gate new asynchronous work while an upgrade captures durable local state. */ +/** Gate new asynchronous work while this tab captures its reload checkpoint. */ export function setReleasePreparation(value: boolean): void { preparingRelease = value } diff --git a/app/src/releases/startupRecovery.test.ts b/app/src/releases/startupRecovery.test.ts index 6cef2996..d926489f 100644 --- a/app/src/releases/startupRecovery.test.ts +++ b/app/src/releases/startupRecovery.test.ts @@ -81,3 +81,16 @@ it('accepts an explicit alternative to stable and never retries coordination err expect(discover).toHaveBeenCalledWith(selection) expect(control).toHaveBeenCalledOnce() }) + +it('can download a recovery build while settings are completely unavailable', async () => { + const candidate = setup() + vi.mocked(control).mockRejectedValue( + new ReleaseControlError('read', 'timeout') + ) + const { downloadRecoveryRelease } = await import('./startupRecovery') + expect( + await downloadRecoveryRelease({ kind: 'channel', name: 'latest' }) + ).toBe(candidate) + expect(stage).toHaveBeenCalledOnce() + expect(control).not.toHaveBeenCalled() +}) diff --git a/app/src/releases/startupRecovery.ts b/app/src/releases/startupRecovery.ts index a2d5c9a1..81cbbe45 100644 --- a/app/src/releases/startupRecovery.ts +++ b/app/src/releases/startupRecovery.ts @@ -18,10 +18,7 @@ export async function prepareStartupRecovery( scope: 'all' | 'startup' = 'all', repair?: (candidate: Manifest) => Promise ): Promise { - const candidate = await discover(selection) - await navigator.locks.request('runme-release-cache', { mode: 'shared' }, () => - stage(candidate, progress, scope) - ) + const candidate = await downloadRecoveryRelease(selection, progress, scope) // A transient error can clear while downloading. Retry a second bounded // transaction, always rereading state so a prior uncertain commit is safe. for (let attempt = 0; ; attempt++) { @@ -40,3 +37,16 @@ export async function prepareStartupRecovery( } } } + +/** Download and verify a recovery build without accessing release settings. */ +export async function downloadRecoveryRelease( + selection: Selection, + progress?: (value: ReleaseProgress) => void, + scope: 'all' | 'startup' = 'all' +): Promise { + const candidate = await discover(selection) + await navigator.locks.request('runme-release-cache', { mode: 'shared' }, () => + stage(candidate, progress, scope) + ) + return candidate +} diff --git a/app/src/releases/useReleaseAttention.ts b/app/src/releases/useReleaseAttention.ts new file mode 100644 index 00000000..d2305545 --- /dev/null +++ b/app/src/releases/useReleaseAttention.ts @@ -0,0 +1,52 @@ +import { useEffect, useState } from 'react' + +import { type ReleaseAttention, releaseAttention } from './releaseAttention' +import './runtime' + +/** Current launchers publish a memory-only snapshot, including read failures. + * Older launchers fall back to bounded, non-overlapping settings reads. + */ +export function useReleaseAttention(): ReleaseAttention | undefined { + const api = window.runmeVersions + const [attention, setAttention] = useState() + useEffect(() => { + if (!api) return + let disposed = false + let reading = false + const refresh = async () => { + if (disposed || reading) return + if (api.attention) { + setAttention(api.attention()) + return + } + reading = true + try { + const state = await api.get() + if (!disposed) + setAttention( + releaseAttention( + state, + window.runmeRelease?.active.manifest.releaseId + ) + ) + } catch { + if (!disposed) + setAttention({ + kind: 'error', + message: 'Version information needs attention', + }) + } finally { + reading = false + } + } + void refresh() + const timer = window.setInterval(() => void refresh(), 5000) + window.addEventListener('runme-release-change', refresh) + return () => { + disposed = true + window.clearInterval(timer) + window.removeEventListener('runme-release-change', refresh) + } + }, [api]) + return attention +} diff --git a/app/src/storage/editorSaveSession.test.ts b/app/src/storage/editorSaveSession.test.ts new file mode 100644 index 00000000..d85cf88f --- /dev/null +++ b/app/src/storage/editorSaveSession.test.ts @@ -0,0 +1,152 @@ +// @vitest-environment node +import { create } from '@bufbuild/protobuf' +import { expect, it, vi } from 'vitest' + +import { createInitialNotebookFile } from '../lib/notebookFormat' +import { + type RunmeOperation, + materializeOperationLog, + materializedLogToNotebook, + mergeOperationSets, + parseOperationLog, + serializeOperationLog, +} from '../lib/operationLog' +import { parser_pb } from '../runme/client' +import { EditorSaveSession } from './editorSaveSession' + +it('captures the submitted snapshot while typing continues and retries identical operations after failure', async () => { + let acknowledge!: () => void + const commit = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve) => { + acknowledge = resolve + }) + ) + .mockRejectedValueOnce(new Error('Worker exited')) + .mockResolvedValue(undefined) + const session = new EditorSaveSession( + 'local://file/test', + createInitialNotebookFile('test.runme'), + commit + ) + const editor = session.initialNotebook + editor.cells.push( + create(parser_pb.CellSchema, { + refId: 'one', + kind: parser_pb.CellKind.MARKUP, + value: 'submitted', + }) + ) + const first = session.save('local://file/test', editor) + await vi.waitFor(() => expect(commit).toHaveBeenCalledOnce()) + editor.cells[0].value = 'typed while waiting' + acknowledge() + await first + await expect(session.save('local://file/test', editor)).rejects.toThrow( + 'Worker exited' + ) + expect(session.hasPending).toBe(true) + const failed = structuredClone(commit.mock.calls[1]) + expect( + failed[2].some( + (op: any) => + op.kind === 'cell.update' && + op.payload.cell.value === 'typed while waiting' + ) + ).toBe(true) + await session.resume() + expect(commit.mock.calls[2]).toEqual(failed) + expect(session.hasPending).toBe(false) +}) + +it('restores a lost-ack batch and later typing without overwriting another tab’s edits', async () => { + const uri = 'local://file/recovery' + const initial = createInitialNotebookFile('recovery.runme') + let durable = initial + const commit = vi.fn( + async (_uri: string, _id: string, operations: RunmeOperation[]) => { + const parsed = parseOperationLog(durable) + durable = serializeOperationLog( + parsed.header, + mergeOperationSets(parsed.operations, operations) + ) + } + ) + const first = new EditorSaveSession(uri, initial, async (...args) => { + await commit(...args) + throw new Error('Lost reply after durable commit') + }) + const editor = first.initialNotebook + editor.cells.push( + create(parser_pb.CellSchema, { + refId: 'mine', + kind: parser_pb.CellKind.MARKUP, + value: 'submitted', + }) + ) + await expect(first.save(uri, editor)).rejects.toThrow('Lost reply') + const originalBatch = structuredClone(commit.mock.calls[0][2]) + const other = new EditorSaveSession(uri, initial, commit) + other.initialNotebook.cells.push( + create(parser_pb.CellSchema, { + refId: 'theirs', + kind: parser_pb.CellKind.MARKUP, + value: 'other tab', + }) + ) + await other.save(uri, other.initialNotebook) + editor.cells[0].value = 'typed after submission' + const checkpoint = JSON.parse(JSON.stringify(await first.checkpoint(editor))) + const recovered = vi.fn() + const restored = new EditorSaveSession( + uri, + checkpoint.document, + vi + .fn() + .mockRejectedValueOnce(new Error('Worker unavailable during replay')) + .mockImplementation(commit), + checkpoint.actorId, + checkpoint, + recovered + ) + expect(restored.initialNotebook.cells[0].value).toBe('typed after submission') + await expect(restored.save(uri, restored.initialNotebook)).rejects.toThrow( + 'Worker unavailable during replay' + ) + expect(recovered).not.toHaveBeenCalled() + // No new keystroke/save callback is required to persist the restored snapshot. + await restored.resume() + expect(recovered).toHaveBeenCalledOnce() + expect(restored.hasPending).toBe(false) + expect(commit.mock.calls[2][2]).toEqual(originalBatch) + const parsed = parseOperationLog(durable) + expect(new Set(parsed.operations.map((op) => op.op_id)).size).toBe( + parsed.operations.length + ) + expect( + materializedLogToNotebook(materializeOperationLog(parsed.operations)) + .cells.map((cell) => cell.value) + .sort() + ).toEqual(['other tab', 'typed after submission']) +}) + +it('resumes the original editor after checkpoint persistence fails', async () => { + const commit = vi.fn(async () => {}) + const session = new EditorSaveSession( + 'uri', + createInitialNotebookFile('test.runme'), + commit + ) + await session.checkpoint() + await expect(session.save('uri', session.initialNotebook)).rejects.toThrow( + 'paused' + ) + session.resumeAfterFailedCheckpoint() + session.initialNotebook.cells.push( + create(parser_pb.CellSchema, { refId: 'new', value: 'retained' }) + ) + await session.save('uri', session.initialNotebook) + expect(commit).toHaveBeenCalledOnce() +}) diff --git a/app/src/storage/editorSaveSession.ts b/app/src/storage/editorSaveSession.ts new file mode 100644 index 00000000..70739009 --- /dev/null +++ b/app/src/storage/editorSaveSession.ts @@ -0,0 +1,202 @@ +import { fromJsonString, toJsonString } from '@bufbuild/protobuf' + +import { + type RunmeOperation, + buildOperationLogDiff, + cloneNotebook, + highestActorSequence, + materializeOperationLog, + materializedLogToNotebook, + parseOperationLog, + serializeOperationLog, + upgradeOperationLogToV2, +} from '../lib/operationLog' +import { + captureCommittedRevision, + snapshotHeads, +} from '../lib/operationLog/versions' +import { parser_pb } from '../runme/client' + +/** Versioned, tab-local recovery contract. Operations retain their original IDs. */ +export type EditorSaveCheckpoint = { + format: 1 + uri: string + actorId: string + document: string + baseline: string + latest: string + pending?: { operations: RunmeOperation[]; snapshot: string } +} + +/** Page-owned save state survives replacing the storage connection. The worker + * only commits immutable batches; it never owns this editor's save baseline. + */ +export class EditorSaveSession { + readonly initialNotebook: parser_pb.Notebook + private readonly notebookId: string + private observed: RunmeOperation[] + private baseline: parser_pb.Notebook + private pending?: { + operations: RunmeOperation[] + snapshot: parser_pb.Notebook + } + private queue = Promise.resolve() + private saving = 0 + private suspended = false + private latest: parser_pb.Notebook + private readonly header: ReturnType['header'] + + /** Pending work blocks an app reload, but never prevents stopping the worker. */ + get hasPending(): boolean { + return Boolean(this.pending) || this.saving > 0 + } + + constructor( + private readonly uri: string, + document: string, + private readonly commit: ( + uri: string, + notebookId: string, + operations: RunmeOperation[] + ) => Promise, + private readonly actorId = `editor:${crypto.randomUUID()}`, + checkpoint?: EditorSaveCheckpoint, + private onRecovered?: () => void + ) { + const parsed = parseOperationLog(upgradeOperationLogToV2(document)) + this.header = parsed.header + this.notebookId = parsed.header.notebook_id + this.observed = parsed.operations + this.baseline = materializedLogToNotebook( + materializeOperationLog(this.observed) + ) + if (checkpoint) { + if ( + checkpoint.format !== 1 || + checkpoint.uri !== uri || + checkpoint.actorId !== actorId + ) + throw new Error('Unsupported or mismatched editor recovery checkpoint') + this.baseline = fromJsonString( + parser_pb.NotebookSchema, + checkpoint.baseline + ) + this.pending = checkpoint.pending && { + operations: structuredClone(checkpoint.pending.operations), + snapshot: fromJsonString( + parser_pb.NotebookSchema, + checkpoint.pending.snapshot + ), + } + // Parse the combined history before replaying anything. Corrupt checkpoints + // remain in session storage; never turn them into an empty editor. + parseOperationLog( + serializeOperationLog(this.header, [ + ...this.observed, + ...(this.pending?.operations ?? []), + ]) + ) + } + this.latest = checkpoint + ? fromJsonString(parser_pb.NotebookSchema, checkpoint.latest) + : cloneNotebook(this.baseline) + this.initialNotebook = cloneNotebook(this.latest) + } + + /** Only acknowledged history participates in release handoff verification. */ + getObservedOperationHeads(): string[] { + return snapshotHeads(this.observed, captureCommittedRevision(this.observed)) + } + + /** Retry only the retained immutable batch, never regenerate an uncertain save. */ + private async commitPending(): Promise { + if (this.suspended) + throw new Error('Editor saves paused for this tab’s reload') + const pending = this.pending + if (!pending) return + await this.commit(this.uri, this.notebookId, pending.operations) + this.observed = [...this.observed, ...pending.operations] + this.baseline = pending.snapshot + this.pending = undefined + } + + /** The caller disconnects transport first so a missing reply cannot block a + * checkpoint. Await local diff generation, never worker persistence. + */ + async checkpoint(latest?: parser_pb.Notebook): Promise { + this.suspended = true + await this.queue + return { + format: 1, + uri: this.uri, + actorId: this.actorId, + document: serializeOperationLog(this.header, this.observed), + baseline: toJsonString(parser_pb.NotebookSchema, this.baseline), + latest: toJsonString(parser_pb.NotebookSchema, latest ?? this.latest), + pending: this.pending && { + operations: structuredClone(this.pending.operations), + snapshot: toJsonString(parser_pb.NotebookSchema, this.pending.snapshot), + }, + } + } + + /** Failed backup leaves this editor usable with its original pending batch. */ + resumeAfterFailedCheckpoint(): void { + this.suspended = false + } + + getUri(): string { + return this.uri + } + + /** Replay the original batch first, then save later typing retained in the + * snapshot. A failed replay may have prevented that snapshot's diff entirely. + */ + resume(): Promise { + return this.save(this.uri, this.latest) + } + + /** Capture at call time: keystrokes during an acknowledgement wait belong to + * the next save. Serializing also prevents overlapping actor sequence ranges. + */ + save(saveUri: string, notebook: parser_pb.Notebook): Promise { + if (saveUri !== this.uri) + return Promise.reject( + new Error( + `Operation-log save URI changed from ${this.uri} to ${saveUri}` + ) + ) + const next = cloneNotebook(notebook) + this.latest = next + this.saving += 1 + const operation = this.queue + .then(async () => { + await this.commitPending() + const operations = await buildOperationLogDiff({ + previous: this.baseline, + next, + observedOperations: this.observed, + actorId: this.actorId, + firstActorSequence: + highestActorSequence(this.observed, this.actorId) + 1, + }) + if (!operations.length) { + this.baseline = next + return + } + this.pending = { operations, snapshot: next } + await this.commitPending() + }) + .then(() => { + // A checkpoint is consumed only after its full editor state is durable, + // including when the initial replay failed and reconnect completed it. + this.onRecovered?.() + this.onRecovered = undefined + }) + .finally(() => { + this.saving -= 1 + }) + this.queue = operation.catch(() => undefined) + return operation + } +} diff --git a/app/src/storage/local.test.ts b/app/src/storage/local.test.ts index 84ee5274..f6b17f9a 100644 --- a/app/src/storage/local.test.ts +++ b/app/src/storage/local.test.ts @@ -42,6 +42,7 @@ import { ancestorClosure } from '../lib/operationLog/versions' import { createNotebookCommentsRuntimeApi } from '../lib/runtime/notebookCommentsRuntime' import { MimeType, RunmeMetadataKey, parser_pb } from '../runme/client' import { MemoryConflictDocStorage } from './conflictDocs' +import { EditorSaveSession } from './editorSaveSession' import { DriveNotebookStore } from './drive' import type { DriveSyncCoordinator } from './driveSyncCoordinator' import { @@ -1107,6 +1108,53 @@ describe('LocalNotebooks operation-log storage', () => { ).toHaveLength(2) }) + it('retries the exact pending editor save after owner loss without duplicating or deleting unseen edits', async () => { + const storage = new MemoryOperationLogStorage() + const store = createTestStore({}, { operationLogStorage: storage }) + await store.folders.put({ id: LOCAL_FOLDER_URI, name: 'Local Notebooks', remoteId: '', children: [], lastSynced: '' }) + const created = await store.create(LOCAL_FOLDER_URI, 'resume.runme') + const seed = await store.createOperationLogSaveStore(created.uri, {actorId: 'seed'}) + await seed.save(created.uri, create(parser_pb.NotebookSchema, {cells: [create(parser_pb.CellSchema, { + refId: 'original', kind: parser_pb.CellKind.MARKUP, value: 'Original', + })]})) + let loseReply = true + const commit = vi.fn(async (...args: Parameters) => { + await store.commitEditorOperations(...args) + if (loseReply) throw new Error('Worker stopped before reply') + }) + const first = new EditorSaveSession(created.uri, await store.loadContent(created.uri), commit) + const acknowledged = first.getObservedOperationHeads() + const second = await store.createOperationLogSaveStore(created.uri, {actorId: 'other-editor'}) + const editor = first.initialNotebook + editor.cells[0].value = 'Saved, acknowledgement lost' + await expect(first.save(created.uri, editor)).rejects.toThrow('Worker stopped') + expect(first.getObservedOperationHeads()).toEqual(acknowledged) + const other = second.initialNotebook + other.cells.push(create(parser_pb.CellSchema, {refId: 'unseen', kind: parser_pb.CellKind.MARKUP, value: 'Other tab added this'})) + await second.save(created.uri, other) + const before = await store.loadContent(created.uri) + loseReply = false + await first.resume() + expect(commit.mock.calls[1]).toEqual(commit.mock.calls[0]) + expect(await store.loadContent(created.uri)).toBe(before) + expect(first.getObservedOperationHeads()).not.toEqual(acknowledged) + editor.cells[0].value = 'Edited after reconnect' + await first.save(created.uri, editor) + const result = await store.loadOperationLogSnapshot(created.uri) + expect(result.cells.map(cell => cell.value)).toEqual(['Edited after reconnect', 'Other tab added this']) + const saved = await store.loadContent(created.uri) + const [uri, notebookId, operations] = commit.mock.calls[0] + await expect(store.commitEditorOperations(uri, 'different-notebook', operations)).rejects.toThrow('identity changed') + const conflicting = structuredClone(operations) + conflicting[0].created_at = '2000-01-01T00:00:00Z' + await expect(store.commitEditorOperations(uri, notebookId, conflicting)).rejects.toThrow('Conflicting operations') + const missingHistory = structuredClone(operations) + missingHistory[0].op_id += '-new' + missingHistory[0].deps = ['missing-history'] + await expect(store.commitEditorOperations(uri, notebookId, missingHistory)).rejects.toThrow() + expect(await store.loadContent(created.uri)).toBe(saved) + }) + it('verifies acknowledged history across later commits and rejects missing commits before owner activation', async () => { const actor = vi.spyOn(actorIdentity, 'getNotebookActorId').mockResolvedValue('release_writer') const store = createTestStore({}, { operationLogStorage: new MemoryOperationLogStorage() }) diff --git a/app/src/storage/local.ts b/app/src/storage/local.ts index a000e405..68244df7 100644 --- a/app/src/storage/local.ts +++ b/app/src/storage/local.ts @@ -1,4 +1,3 @@ -import { registerReleaseDatabase } from '../releases/databases' import { create, fromJsonString, toJsonString } from '@bufbuild/protobuf' import Dexie, { Table } from 'dexie' import md5 from 'md5' @@ -89,6 +88,7 @@ import { } from '../lib/outputReference' import { appState } from '../lib/runtime/AppState' import type { ExampleJob } from '../lib/trainingExamples/protocol' +import { registerReleaseDatabase } from '../releases/databases' import { RunmeMetadataKey, parser_pb } from '../runme/client' import { type ConflictDocStorage, @@ -1924,7 +1924,8 @@ export class LocalNotebooks extends Dexie { ? await this.updateFolder(item.uri, item.name) : await this.driveSyncCoordinator.runExclusive( this.driveFolderMembershipLockKey(item.uri), - async () => (await this.ensureFolderMirror(item.uri, item.name)).folderId + async () => + (await this.ensureFolderMirror(item.uri, item.name)).folderId ) childUris.push(localFolderUri) } @@ -1981,7 +1982,10 @@ export class LocalNotebooks extends Dexie { return folderId } - async sync(localUri: string, options: { recursive?: boolean } = {}): Promise { + async sync( + localUri: string, + options: { recursive?: boolean } = {} + ): Promise { if (localUri.startsWith('drive-create:')) { const id = localUri.slice('drive-create:'.length) await this.queueDriveWork( @@ -2296,7 +2300,10 @@ export class LocalNotebooks extends Dexie { /** Create a tab-local snapshot adapter that appends .runme operations. */ async createOperationLogSaveStore( uri: string, - options: { actorId?: string; initialDocument?: string } = {} + options: { + actorId?: string + initialDocument?: string + } = {} ): Promise<{ save(saveUri: string, notebook: parser_pb.Notebook): Promise getObservedOperationHeads(): string[] @@ -2395,6 +2402,63 @@ export class LocalNotebooks extends Dexie { } } + /** Commit a tab-prepared batch by identity, including after a lost reply. + * The tab retains the exact bytes and causal baseline. Matching operations + * are acknowledged without duplication; conflicting IDs or missing history + * fail before writing. No editor identity is used to infer what it saw. + */ + async commitEditorOperations( + uri: string, + notebookId: string, + operations: RunmeOperation[] + ): Promise { + const record = await this.files.get(uri) + if ( + !record?.operationLogRef || + detectNotebookFileFormat(record.name) !== 'runme-operation-log' + ) + throw new Error(`Operation-log reference missing for ${uri}`) + const stored = await this.operationLogStorage.appendTransaction( + record.operationLogRef, + (document) => { + const current = parseOperationLog(document) + if (current.header.notebook_id !== notebookId) + throw new Error(`Operation-log notebook identity changed for ${uri}`) + // mergeOperationSets rejects different contents under the same ID. + const merged = mergeOperationSets(current.operations, operations) + const observed = ancestorClosure( + merged, + operations.map((op) => op.op_id) + ) + snapshotHeads( + observed, + observed.map((op) => op.op_id) + ) + const existing = new Set(current.operations.map((op) => op.op_id)) + const missing = operations.filter((op) => !existing.has(op.op_id)) + return missing.length + ? `${missing.map(encodeOperation).join('\n')}\n` + : '' + }, + { + prepareDocument: upgradeOperationLogToV2, + validate: (document) => void parseOperationLog(document), + } + ) + // Repeat metadata repair even when the OPFS append already succeeded. A + // previous worker may have stopped between these two durable writes. + await this.updateFile(uri, { + doc: '', + md5Checksum: this.localWriteChecksum(stored), + operationLogRef: stored.ref, + }) + this.notifySync(uri) + if (!record.conflict) { + this.enqueueSync(uri) + this.enqueueMarkdownSync(uri) + } + } + operationLogSupportsConcurrentWriters(): boolean { return this.operationLogStorage.supportsConcurrentWriters() } @@ -3300,21 +3364,28 @@ export class LocalNotebooks extends Dexie { * authoritative local history only: a remote refetch must not hide lost commits. * Newer operations may supersede these heads without invalidating their history. */ - async verifyReleaseCommits(refs: { uri: string; heads: string[] }[]): Promise { + async verifyReleaseCommits( + refs: { uri: string; heads: string[] }[] + ): Promise { const byUri = new Map>() for (const ref of refs) { if (!ref.heads.length) continue const heads = byUri.get(ref.uri) ?? new Set() - ref.heads.forEach(head => heads.add(head)) + ref.heads.forEach((head) => heads.add(head)) byUri.set(ref.uri, heads) } for (const [uri, heads] of byUri) { const record = await this.files.get(uri) - if (!record?.operationLogRef) throw new Error(`Confirmed notebook history is unavailable for ${uri}`) + if (!record?.operationLogRef) + throw new Error(`Confirmed notebook history is unavailable for ${uri}`) const stored = await this.operationLogStorage.read(record.operationLogRef) - const operations = new Set(parseOperationLog(stored.document).operations.map(op => op.op_id)) - if ([...heads].some(head => !operations.has(head))) - throw new Error(`Confirmed notebook commits are missing for ${uri}; preserve the existing data and use recovery`) + const operations = new Set( + parseOperationLog(stored.document).operations.map((op) => op.op_id) + ) + if ([...heads].some((head) => !operations.has(head))) + throw new Error( + `Confirmed notebook commits are missing for ${uri}; preserve the existing data and use recovery` + ) } } diff --git a/app/src/storage/storageOwner.test.ts b/app/src/storage/storageOwner.test.ts index 13df191b..a68c7a41 100644 --- a/app/src/storage/storageOwner.test.ts +++ b/app/src/storage/storageOwner.test.ts @@ -52,6 +52,41 @@ function setup() { } describe('SharedWorker message boundary', () => { + it('negotiates across RPC versions without authorization or storage effects', async () => { + const { store } = setup() + const authorize = vi.fn(async () => { + throw new Error('must not authorize negotiation') + }) + const host = new StorageOwnerHost( + store as unknown as LocalNotebooks, + undefined, + authorize + ) + const channel = new MessageChannel() + ports.push( + channel.port1 as unknown as MessagePort, + channel.port2 as unknown as MessagePort + ) + host.attach(channel.port1 as unknown as MessagePort) + const reply = new Promise((resolve) => + channel.port2.once('message', resolve) + ) + channel.port2.postMessage({ + type: 'request', + id: 'negotiate', + method: 'handshake', + version: -1, + args: [], + }) + expect((await reply).value).toMatchObject({ + protocol: 1, + recoveryFormat: 1, + editorCommitVersion: 1, + }) + expect(authorize).not.toHaveBeenCalled() + expect(store.setDriveSyncAvailable).not.toHaveBeenCalled() + expect(store.save).not.toHaveBeenCalled() + }) it('restores the profile boundary before unauthenticated tabs can queue writes', async () => { const { host, store } = setup() const human = connect(host) @@ -382,7 +417,12 @@ it('answers diagnostics outside initialization and reports waiting requests with const diagnostics = await client.diagnose() expect(diagnostics.initialization).toBe('pending') expect(diagnostics.oldestRequests).toEqual([ - { method: 'save', elapsedMs: expect.any(Number), stage: 'initializing' }, + { + method: 'save', + elapsedMs: expect.any(Number), + stage: 'initializing', + stageElapsedMs: expect.any(Number), + }, ]) expect(JSON.stringify(diagnostics)).not.toContain('private-uri') expect(JSON.stringify(diagnostics)).not.toContain('notebook body') @@ -526,6 +566,7 @@ it('blocks upgrade and mutation replay after a timeout until the original acknow ).rejects.toThrow('uncertain') await vi.advanceTimersByTimeAsync(300_000) await outcome + expect(client.needsReconnect).toBe(false) const { assertNoUpgradeBlockers } = await import('../releases/runtime') expect(assertNoUpgradeBlockers).toThrow('unconfirmed') await expect(client.request('save', ['retry', {}])).rejects.toThrow( @@ -544,6 +585,26 @@ it('blocks upgrade and mutation replay after a timeout until the original acknow expect(sent.filter((m) => m.method === 'save')).toHaveLength(1) }) +it('allows reconnect after an editor-save timeout without requiring a stop notification', async () => { + vi.useFakeTimers() + const channel = new MessageChannel() + ports.push(channel.port1 as unknown as MessagePort) + const client = new StorageOwnerClient( + channel.port2 as unknown as MessagePort, + async () => 'token' + ) + clients.push(client) + const outcome = expect( + client.request('commitEditorOperations', ['uri', 'notebook', []]) + ).rejects.toThrow('Pending operations remain') + await vi.advanceTimersByTimeAsync(300_000) + await outcome + expect(client.needsReconnect).toBe(true) + client.close() + const { assertNoUpgradeBlockers } = await import('../releases/runtime') + expect(assertNoUpgradeBlockers).not.toThrow() +}) + it('does not treat disconnect as acknowledgement of an outstanding write', async () => { const channel = new MessageChannel() ports.push(channel.port1 as unknown as MessagePort) @@ -560,3 +621,39 @@ it('does not treat disconnect as acknowledgement of an outstanding write', async const { assertNoUpgradeBlockers } = await import('../releases/runtime') expect(assertNoUpgradeBlockers).toThrow('unconfirmed') }) + +it.each([1, 2])( + 'distinguishes authorization pass %s from store initialization', + async (pass) => { + const { store } = setup() + let resume!: () => void + let calls = 0 + const authorize = async () => { + if (++calls === pass) + await new Promise((resolve) => { + resume = resolve + }) + } + const client = connect( + new StorageOwnerHost( + store as unknown as LocalNotebooks, + undefined, + authorize + ) + ) + const saving = client.request('save', [ + 'private-uri', + { secret: 'notebook contents' }, + ]) + await vi.waitFor(() => expect(calls).toBe(pass)) + const snapshot = await client.diagnose() + expect(snapshot.oldestRequests[0].stage).toBe( + pass === 1 ? 'authorizing' : 'reauthorizing' + ) + expect(store.save).not.toHaveBeenCalled() + expect(JSON.stringify(snapshot)).not.toContain('notebook contents') + resume() + await saving + expect(store.save).toHaveBeenCalledOnce() + } +) diff --git a/app/src/storage/storageOwner.worker.ts b/app/src/storage/storageOwner.worker.ts index 8b6e719f..b9ad8a11 100644 --- a/app/src/storage/storageOwner.worker.ts +++ b/app/src/storage/storageOwner.worker.ts @@ -1,6 +1,8 @@ import { appLogger } from '../lib/logging/runtime' import { OWNER_LOCK, control, readControl } from '../releases/control' +import { releaseDatabaseDiagnostics } from '../releases/releaseDiagnostics' import { retireUnreadableOwner } from './releaseOwnerFailure' +import { installStorageWorkerControl } from './storageWorkerControl' const scope = globalThis as unknown as { onconnect: (event: MessageEvent) => void @@ -9,9 +11,56 @@ const scope = globalThis as unknown as { const releaseId = import.meta.env.VITE_RUNME_RELEASE_ID as string | undefined // CacheStorage responses retain their original URL, which can drop a query. // The browser-assigned worker name is the authoritative instance identity. -const epoch = Number( - (globalThis as unknown as { name: string }).name.split('-').pop() -) +const epoch = releaseId + ? Number((globalThis as unknown as { name: string }).name.split('-').pop()) + : 0 +const instanceId = crypto.randomUUID() +const startedAt = Date.now() +let phase = 'waiting-for-lock' +let phaseStartedAt = startedAt +let runtime: + | Awaited< + ReturnType<(typeof import('./storageOwnerRuntime'))['startStorageOwner']> + > + | undefined +let stopping = false +/** Track the last reached boundary even while an awaited operation is stuck. */ +function setPhase(next: string) { + phase = next + phaseStartedAt = Date.now() + appLogger.info('Storage worker lifecycle', { + attrs: { scope: 'storage.owner', instanceId, phase, releaseId, epoch }, + }) +} +installStorageWorkerControl({ + status: () => ({ + instanceId, + releaseId, + epoch, + phase, + phaseElapsedMs: Date.now() - phaseStartedAt, + uptimeMs: Date.now() - startedAt, + diagnostics: runtime?.diagnostics(), + releaseDatabase: releaseDatabaseDiagnostics(), + }), + phase: setPhase, + drain: async () => { + stopping = true + // Before a runtime is available there are no admitted notebook requests. + // Startup may nevertheless be opening stores: wait for it to settle before + // graceful close. Force-close remains available on this independent channel. + if (!runtime) await runtimeReady + await runtime?.drain() + }, + close: () => { + stopping = true + scope.close() + }, +}) +let finishRuntime!: () => void +const runtimeReady = new Promise((resolve) => { + finishRuntime = resolve +}) const ports: MessagePort[] = [] let attach: ((port: MessagePort) => void) | undefined scope.onconnect = (event) => { @@ -29,6 +78,7 @@ async function start(): Promise { if (!navigator.locks) throw new Error('Web Locks are required for storage ownership') await navigator.locks.request(OWNER_LOCK, async () => { + setPhase('reading-release-settings') if (releaseId) { const state = await readControl() const pending = state.transition @@ -46,14 +96,21 @@ async function start(): Promise { ) throw new Error('Storage candidate is no longer selected') } + setPhase('recording-startup') if (releaseId) await control((s) => { s.ownerStartup = { releaseId, epoch, phase: 'initializing' } }) + setPhase('importing-runtime') const { startStorageOwner } = await import('./storageOwnerRuntime') - const runtime = await startStorageOwner( + setPhase('initializing-storage') + runtime = await startStorageOwner( releaseId ? { releaseId, epoch } : undefined ) + finishRuntime() + if (stopping) await new Promise(() => {}) + const activeRuntime = runtime + setPhase('publishing-ready') try { if (releaseId) await control((state) => { @@ -79,11 +136,12 @@ async function start(): Promise { }) } catch (error) { // If startup lost admission, finish maintenance before releasing ownership. - await runtime.drain() - runtime.close() + await activeRuntime.drain() + activeRuntime.close() throw error } - attach = runtime.attach + setPhase('ready') + attach = activeRuntime.attach for (const port of ports.splice(0)) attach(port) await new Promise((resolve) => { let checking = false @@ -96,9 +154,9 @@ async function start(): Promise { state.transition?.phase === 'handoff' || state.active?.epoch !== epoch ) { - await runtime.drain() + await activeRuntime.drain() clearInterval(timer) - runtime.close() + activeRuntime.close() resolve() } } catch (error) { @@ -106,7 +164,7 @@ async function start(): Promise { // lock and retry reconciliation; never release it on a timeout/error. try { if ( - await retireUnreadableOwner(error, runtime, () => { + await retireUnreadableOwner(error, activeRuntime, () => { clearInterval(timer) resolve() }) @@ -135,6 +193,8 @@ async function start(): Promise { scope.close() } void start().catch(async (error) => { + finishRuntime() + setPhase('failed') await control((state) => { if ( releaseId && diff --git a/app/src/storage/storageOwnerClient.ts b/app/src/storage/storageOwnerClient.ts index 24ca59d7..f8bca004 100644 --- a/app/src/storage/storageOwnerClient.ts +++ b/app/src/storage/storageOwnerClient.ts @@ -1,9 +1,16 @@ import { getWorkloadIdentityConfig } from '../auth/workloadIdentity' import { getGoogleDriveBaseUrl } from '../lib/googleDriveRuntime' import { appLogger } from '../lib/logging/runtime' -import { RELEASE_CAPABILITIES, releasePrefix } from '../releases/control' +import { + type Active, + OWNER_LOCK, + RELEASE_CAPABILITIES, + readControl, + releasePrefix, +} from '../releases/control' import { registerUpgradeBlocker } from '../releases/runtime' import { DriveCreateNotCommittedError, type DriveNotebookStore } from './drive' +import { EditorSaveSession } from './editorSaveSession' import { FilesystemEntryAlreadyExistsError } from './fs' import { readLegacyCreationJournal } from './legacyCreationJournal' import LocalNotebooks, { @@ -14,9 +21,19 @@ import type { StorageOwnerDiagnostics } from './storageOwnerDiagnostics' import { STORAGE_METHODS, STORAGE_OWNER_VERSION } from './storageOwnerProtocol' import { requiresStorageReload, storageRecovery } from './storageRecovery' import { StorageRecoveryClient } from './storageRecoveryClient' -import type { RecoveryMessage } from './storageRecoveryProtocol' +import { + type RecoveryMessage, + recoveryMessage, +} from './storageRecoveryProtocol' +import { StorageWorkerControl, type WorkerStatus } from './storageWorkerControl' +import { + acknowledgeTabCheckpoint, + readTabCheckpoints, + registerCheckpointTransport, +} from './tabRecoveryCheckpoint' type Pending = { + method: string resolve: (value: unknown) => void reject: (error: Error) => void timer: ReturnType @@ -28,6 +45,8 @@ export class StorageOwnerClient { private uncertain = new Set() private unresolvedCommit = false private closed = false + private retryConnection = false + private unregisterDiagnostics?: () => void private unregisterReleaseBlocker = registerUpgradeBlocker(() => this.uncertain.size || this.unresolvedCommit ? 'A storage operation has an unconfirmed outcome; resolve it before upgrading' @@ -46,9 +65,18 @@ export class StorageOwnerClient { constructor( private readonly port: MessagePort, private readonly token: DriveNotebookStore['getAccessToken'], - private readonly getCredentialBoundary: () => string = () => 'default' + private readonly getCredentialBoundary: () => string = () => 'default', + private readonly ownerEpoch?: number ) { port.onmessage = (event) => { + if (event.data?.type === 'release-owner-error') { + this.close( + new Error( + `Notebook storage worker failed: ${String(event.data.error)}` + ) + ) + return + } if (event.data?.type === 'storage-owner-diagnostics') { const diagnostic = this.diagnostic if (diagnostic && diagnostic.id === event.data.id) { @@ -67,10 +95,13 @@ export class StorageOwnerClient { port.onmessageerror = () => this.close( new Error( - 'Storage worker message failed. Local saves may have committed; reload to recover.' + 'Storage worker message failed. Local saves may have committed; reconnect storage to recover.' ) ) port.start() + this.unregisterDiagnostics = storageRecovery.registerDiagnostics(() => + this.diagnose() + ) } /** Coalesce probes outside the data RPC/hello path; never retry the timed-out mutation. */ diagnose(): Promise { @@ -103,10 +134,24 @@ export class StorageOwnerClient { return promise } + /** Missing stop notifications must not strand a tab on a dead MessagePort. + * Safe request timeouts allow the next call to replace this connection. Keep + * generic uncertain mutations on their original correlation until recovery. + */ + get needsReconnect(): boolean { + return this.closed || this.retryConnection + } + request(method: string, args: unknown[] = []): Promise { if ( (this.uncertain.size || this.unresolvedCommit) && - !['disconnect', 'heartbeat', 'hello'].includes(method) + ![ + 'disconnect', + 'heartbeat', + 'handshake', + 'hello', + 'commitEditorOperations', + ].includes(method) ) return Promise.reject( new Error( @@ -116,7 +161,7 @@ export class StorageOwnerClient { if (this.closed) return Promise.reject( new Error( - 'Storage worker disconnected. Reload to recover saved content.' + 'Storage worker disconnected. Reconnect to resume pending editor saves.' ) ) const id = crypto.randomUUID() @@ -124,14 +169,25 @@ export class StorageOwnerClient { return new Promise((resolve, reject) => { const timer = setTimeout( () => { + if ( + [ + 'handshake', + 'hello', + 'heartbeat', + 'commitEditorOperations', + ].includes(method) + ) + this.retryConnection = true // Keep the original correlation until its late acknowledgement. An // upgrade retry must not turn a timed-out mutation into a new mutation. if ( ![ 'hello', + 'handshake', 'heartbeat', 'disconnect', 'getDriveQueueMetrics', + 'commitEditorOperations', ].includes(method) ) this.uncertain.add(id) @@ -139,7 +195,9 @@ export class StorageOwnerClient { const error = new Error( method === 'getDriveQueueMetrics' ? 'Storage worker did not respond to queue diagnostics.' - : 'Storage worker did not acknowledge the request. Its outcome is uncertain; do not repeat creation blindly.' + : method === 'commitEditorOperations' + ? 'Storage worker did not acknowledge an editor save. Pending operations remain in this tab; reconnect storage to retry them.' + : 'Storage worker did not acknowledge the request. Its outcome is uncertain; do not repeat creation blindly.' ) appLogger.warn('Storage owner request timed out', { attrs: { @@ -169,11 +227,13 @@ export class StorageOwnerClient { this.onFailure?.(error) reject(error) }, - method === 'hello' || method === 'getDriveQueueMetrics' + method === 'hello' || + method === 'handshake' || + method === 'getDriveQueueMetrics' ? 10_000 : 300_000 ) - this.pending.set(id, { resolve, reject, timer }) + this.pending.set(id, { method, resolve, reject, timer }) try { this.port.postMessage({ type: 'request', @@ -183,9 +243,10 @@ export class StorageOwnerClient { args, credentialBoundary: this.getCredentialBoundary(), epoch: - typeof window !== 'undefined' + this.ownerEpoch ?? + (typeof window !== 'undefined' ? window.runmeRelease?.active.epoch - : undefined, + : undefined), tabId: typeof window !== 'undefined' ? window.runmeRelease?.tabId @@ -276,13 +337,27 @@ export class StorageOwnerClient { } else pending.resolve(message.value) } close(error = new Error('Storage worker disconnected')): void { + this.unregisterDiagnostics?.() this.recovery?.disconnected() this.onFailure?.(error) this.closed = true // Losing the transport does not settle an outstanding effect. Keep the // upgrade blocker until document reload/recovery; never turn disconnect // into an acknowledgement of an uncertain save or create. - if (this.pending.size || this.uncertain.size || this.unresolvedCommit) + if ( + [...this.pending.values()].some( + (p) => + ![ + 'hello', + 'handshake', + 'heartbeat', + 'disconnect', + 'commitEditorOperations', + ].includes(p.method) + ) || + this.uncertain.size || + this.unresolvedCommit + ) this.unresolvedCommit = true else this.unregisterReleaseBlocker() if (this.diagnostic) { @@ -308,99 +383,374 @@ export function createSharedNotebookStore( drive: DriveNotebookStore ): LocalNotebooks { const release = window.runmeRelease?.active - const worker = release - ? new SharedWorker( - `${releasePrefix(release.manifest.releaseId)}${release.manifest.owner.entry}`, - { + const listeners = new Map void>>() + let recoveryAttemptEpoch: number | undefined + /** Preserve subscriptions when a new transport replaces a stopped worker. */ + function notifyChange(uri: string) { + for (const callback of listeners.get(uri) ?? []) callback() + window.dispatchEvent( + new CustomEvent('local-notebook-sync-updated', { detail: { uri } }) + ) + window.dispatchEvent( + new CustomEvent('local-notebook-updated', { detail: { uri } }) + ) + } + /** A fresh handshake is mandatory for every replacement owner. */ + function connect(selected: Active | undefined = release) { + const release = selected + let ownerInstance: string | undefined + const worker = release + ? new SharedWorker( + `${releasePrefix(release.manifest.releaseId)}${release.manifest.owner.entry}`, + { + type: 'module', + name: `runme-storage-owner-${release.manifest.releaseId}-${release.epoch}`, + } + ) + : new SharedWorker(new URL('./storageOwner.worker.ts', import.meta.url), { type: 'module', - name: `runme-storage-owner-${release.manifest.releaseId}-${release.epoch}`, - } + name: 'runme-storage-owner', + }) + const client = new StorageOwnerClient( + worker.port, + (options) => drive.getAccessToken(options), + credentialBoundary, + release?.epoch + ) + client.onFailure = (error) => { + if (requiresStorageReload(error)) storageRecovery.report(error) + } + worker.port.postMessage(recoveryMessage('register', 'editor-client')) + // Migration is part of the handshake: do not let a recovery scan generate a + // second remote identity while an old reserved ID only exists in localStorage. + const ready = Promise.resolve().then(async () => { + // This small contract is stable across data-RPC versions and has no + // storage effects. Incompatibility is distinct from timeout/unavailability. + if (release && release.manifest.tabRecoveryFormat !== 1) { + const error = new Error( + 'Selected release predates replayable tab recovery. Keep this tab open and export edits, or select a release with tab recovery support.' + ) + error.name = 'StorageWorkerIncompatibleError' + storageRecovery.set({ phase: 'blocked', message: error.message }) + throw error + } + const negotiated = (await client.request('handshake')) as { + protocol: number + rpcVersion: number + recoveryFormat: number + editorCommitVersion: number + } + if ( + negotiated.protocol !== 1 || + negotiated.rpcVersion !== STORAGE_OWNER_VERSION || + negotiated.recoveryFormat !== 1 || + negotiated.editorCommitVersion !== 1 + ) { + const error = new Error( + 'Storage worker is incompatible with this tab. Preserve edits and reload this tab to use the selected application version.' + ) + error.name = 'StorageWorkerIncompatibleError' + storageRecovery.set({ phase: 'blocked', message: error.message }) + throw error + } + let attempts: ReturnType = [] + let creationMigrationError: string | undefined + try { + attempts = readLegacyCreationJournal(localStorage) + } catch (error) { + creationMigrationError = String(error) + } + const hello = await client.request('hello', [ + { + driveBaseUrl: getGoogleDriveBaseUrl(), + credentialBoundary: credentialBoundary(), + profileCredentialBoundary: localStorage.getItem( + 'runme/storageCredentialBoundary' + ), + attempts, + creationMigrationError, + }, + ]) + const handshake = hello as { + editorCommitVersion?: number + version: number + instanceId?: string + release?: { releaseId: string; epoch: number; capabilities: string[] } + } + if ( + release && + (handshake.version !== STORAGE_OWNER_VERSION || + !handshake.instanceId || + handshake.release?.releaseId !== release.manifest.releaseId || + handshake.release.epoch !== release.epoch || + RELEASE_CAPABILITIES.some( + (c) => !handshake.release?.capabilities?.includes(c) + )) ) - : new SharedWorker(new URL('./storageOwner.worker.ts', import.meta.url), { - type: 'module', - name: 'runme-storage-owner', + throw new Error( + 'Storage owner identity or lifecycle capabilities do not match this tab' + ) + for (const record of attempts) { + // Cleanup is optional after durable import. Failure cannot disconnect an + // otherwise healthy editor or discard its notebook persistence channel. + try { + if ( + localStorage.getItem(record.id) === JSON.stringify(record.attempt) + ) + localStorage.removeItem(record.id) + } catch { + /* A later handshake can safely repeat the import. */ + } + } + ownerInstance = handshake.instanceId + return handshake + }) + void ready.catch(async (error) => { + appLogger.error('Notebook storage worker unavailable', { + attrs: { scope: 'storage.owner', error: String(error) }, }) - const client = new StorageOwnerClient( - worker.port, - (options) => drive.getAccessToken(options), - credentialBoundary - ) - client.onFailure = (error) => { - if (requiresStorageReload(error)) storageRecovery.report(error) - } - client.recovery = new StorageRecoveryClient( - (message) => worker.port.postMessage(message), - () => { - // This is an acknowledged shutdown, not worker failure. Keep editors frozen - // until navigation completes so no last keystroke escapes the recovery copy. - client.onFailure = undefined - client.recovery = undefined - client.close() - window.location.reload() - } - ) - storageRecovery.registerRestart(() => client.recovery!.restart()) - // Migration is part of the handshake: do not let a recovery scan generate a - // second remote identity while an old reserved ID only exists in localStorage. - const ready = Promise.resolve().then(async () => { - let attempts: ReturnType = [] - let creationMigrationError: string | undefined - try { - attempts = readLegacyCreationJournal(localStorage) - } catch (error) { - creationMigrationError = String(error) + // Only an explicit capability mismatch with a different selected release + // warrants upgrading this tab. Silence/timeouts never authorize a reload. + if ( + error?.name !== 'StorageWorkerIncompatibleError' || + !selected || + selected.manifest.releaseId === + window.runmeRelease?.active.manifest.releaseId || + recoveryAttemptEpoch === selected.epoch || + !window.runmeRelease?.prepareReload || + !window.runmeVersions?.reloadTab + ) + return + recoveryAttemptEpoch = selected.epoch + try { + await window.runmeVersions.reloadTab() + } catch (failure) { + storageRecovery.set({ + phase: 'blocked', + message: `This tab could not preserve its state for the selected release: ${String(failure)}. Keep it open and retry recovery.`, + }) + } + }) + client.onChange = notifyChange + worker.onerror = (event) => { + if (connection.client === client) needsConnection = true + client.close( + new Error( + `Notebook storage worker failed: ${event.message || 'startup error'} (${event.filename}:${event.lineno}). Reconnect storage to resume saving.` + ) + ) } - const hello = await client.request('hello', [ - { - driveBaseUrl: getGoogleDriveBaseUrl(), - credentialBoundary: credentialBoundary(), - profileCredentialBoundary: localStorage.getItem( - 'runme/storageCredentialBoundary' - ), - attempts, - creationMigrationError, + return { + worker, + client, + ready, + selected, + get instanceId() { + return ownerInstance }, - ]) - const handshake = hello as { - version: number - instanceId?: string - release?: { releaseId: string; epoch: number; capabilities: string[] } } + } + let connection = connect() + let checkpointing = false + let needsConnection = false + let connecting: Promise | undefined + let retrying: Promise | undefined + let restartRequested = false + let driveAvailable = false + let currentWorker: WorkerStatus | undefined + let lastManagementReply = 0 + const sessions = new Map< + EditorSaveSession, + { disposed: boolean; unregister: () => void } + >() + registerCheckpointTransport({ + uris: () => + new Set([...sessions.keys()].map((session) => session.getUri())), + capture: async (editors) => { + checkpointing = true + connection.client.onFailure = undefined + connection.client.close( + new Error('This tab is preserving editor saves before reload') + ) + needsConnection = true + return Promise.all( + [...sessions.keys()].map((session) => + session.checkpoint(editors.get(session.getUri())) + ) + ) + }, + resume: () => { + checkpointing = false + for (const session of sessions.keys()) + session.resumeAfterFailedCheckpoint() + }, + }) + const management = new StorageWorkerControl((event) => { if ( - release && - (handshake.version !== STORAGE_OWNER_VERSION || - !handshake.instanceId || - handshake.release?.releaseId !== release.manifest.releaseId || - handshake.release.epoch !== release.epoch || - RELEASE_CAPABILITIES.some( - (c) => !handshake.release?.capabilities?.includes(c) - )) + event.status.releaseId !== connection.selected?.manifest.releaseId || + event.status.epoch !== (connection.selected?.epoch ?? 0) ) + return + currentWorker = event.status + lastManagementReply = Date.now() + // Status replies are observations only. No tab tells another tab to connect. + if (event.type === 'stopping') + storageRecovery.set({ + phase: 'restarting', + message: + 'Storage is stopping. Editors and pending saves remain in this tab.', + }) + if (event.type === 'stopped') { + connection.client.onFailure = undefined + connection.client.close( + new Error('Storage worker stopped; pending editor saves retained') + ) + needsConnection = true + storageRecovery.set({ + phase: 'required', + message: + 'Storage stopped. Retry a save or restart storage to reconnect; editors remain open.', + }) + if ( + event.restart && + (restartRequested || + [...sessions.keys()].some((session) => session.hasPending)) + ) + void retryPending() + restartRequested = false + } + }) + /** A tab reconnects lazily on its own next request. The browser joins the same + * named SharedWorker; the worker's lifetime lock remains the only writer gate. + */ + async function ensureConnected(): Promise { + if (connection.client.needsReconnect) needsConnection = true + if (connecting) return connecting + const selected = release ? await readControl() : undefined + if (connecting) return connecting + if (selected?.transition?.phase === 'handoff') throw new Error( - 'Storage owner identity or lifecycle capabilities do not match this tab' + 'Storage worker is changing versions. Pending saves remain in this tab; retry after activation.' ) - for (const record of attempts) { - // Cleanup is optional after durable import. Failure cannot disconnect an - // otherwise healthy editor or discard its notebook persistence channel. + if (release && !selected?.active) + throw new Error('No active storage worker is selected') + if ( + selected?.active && + selected.active.epoch !== connection.selected?.epoch + ) + needsConnection = true + if (!needsConnection) { + await connection.ready + return + } + const operation = (async () => { + connection.client.onFailure = undefined + connection.client.close() + connection = connect(selected?.active) + const handshake = await connection.ready + if (handshake.editorCommitVersion !== 1) + throw new Error( + 'This worker cannot resume pending editor saves. Keep this tab open and upgrade Runme.' + ) + needsConnection = false + if (driveAvailable) + await connection.client.request('availability', [ + true, + credentialBoundary(), + ]) + })() + connecting = operation + try { + await operation + } catch (error) { + needsConnection = true + throw error + } finally { + connecting = undefined + } + } + /** Retry only this tab's retained idempotent batches, independently of peers. */ + async function retryPending(): Promise { + if (retrying) return retrying + const operation = (async () => { try { - if (localStorage.getItem(record.id) === JSON.stringify(record.attempt)) - localStorage.removeItem(record.id) - } catch { - /* A later handshake can safely repeat the import. */ + await ensureConnected() + const results = await Promise.allSettled( + [...sessions.keys()].map((session) => session.resume()) + ) + for (const [session, state] of sessions) { + if (state.disposed && !session.hasPending) { + state.unregister() + sessions.delete(session) + } + } + const failed = results.filter((result) => result.status === 'rejected') + storageRecovery.set( + failed.length + ? { + phase: 'blocked', + message: `${failed.length} notebook save(s) still need recovery. Editors and pending batches remain in this tab.`, + } + : { phase: 'healthy', message: '' } + ) + management.discover() + } catch (error) { + storageRecovery.set({ + phase: 'blocked', + message: `Storage reconnect failed: ${String(error)}. Editors and pending saves remain in this tab.`, + }) } + })() + retrying = operation + try { + await operation + } finally { + retrying = undefined } - }) - void ready.catch((error) => - appLogger.error('Notebook storage worker unavailable', { - attrs: { scope: 'storage.owner', error: String(error) }, - }) - ) - worker.onerror = (event) => - client.close( - new Error( - `Notebook storage worker failed: ${event.message || 'startup error'} (${event.filename}:${event.lineno}). Reload to recover durable edits.` + } + storageRecovery.registerRestart((force = false) => { + const requestedAt = Date.now() + const sendStop = async () => { + if ( + currentWorker && + lastManagementReply >= requestedAt && + currentWorker.phase !== 'closing' + ) { + restartRequested = true + management.stop(currentWorker.instanceId, { force, restart: true }) + return + } + // Silence alone is not death. A free ownership lock permits reconnecting + // after a crash without a stopped event; never steal an occupied lock. + const absent = await navigator.locks.request( + OWNER_LOCK, + { mode: 'shared', ifAvailable: true }, + (lock) => Boolean(lock) ) + if (absent) { + connection.client.onFailure = undefined + connection.client.close() + needsConnection = true + void retryPending() + } else + storageRecovery.set({ + phase: 'blocked', + message: + 'No management reply and storage ownership is still held. Keep editors open and use browser worker inspection.', + }) + } + management.discover() + setTimeout( + () => + void sendStop().catch((error) => + storageRecovery.report( + error instanceof Error ? error : new Error(String(error)) + ) + ), + 250 ) + }) + management.discover() const local = new LocalNotebooks( drive, undefined, @@ -411,43 +761,39 @@ export function createSharedNotebookStore( undefined, { client: true } ) - const listeners = new Map void>>() - client.onChange = (uri) => { - for (const callback of listeners.get(uri) ?? []) callback() - window.dispatchEvent( - new CustomEvent('local-notebook-sync-updated', { detail: { uri } }) - ) - window.dispatchEvent( - new CustomEvent('local-notebook-updated', { detail: { uri } }) - ) - } - const call = async (method: string, args: unknown[]) => { - await ready - return client.request(method, args) + const call = async (method: string, args: unknown[]): Promise => { + if (checkpointing) throw new Error('This tab is preserving pending saves') + const previous = connection + const reconnect = needsConnection + await ensureConnected() + if (checkpointing) throw new Error('This tab is preserving pending saves') + if (reconnect || connection !== previous) void retryPending() + return connection.client.request(method, args) } - const heartbeat = setInterval( - () => void call('heartbeat', []).catch(() => {}), - 15_000 - ) + const heartbeat = setInterval(() => { + if (!needsConnection) void call('heartbeat', []).catch(() => {}) + }, 15_000) window.addEventListener( 'pagehide', (event) => { if (event.persisted) return clearInterval(heartbeat) - void client.request('disconnect').catch(() => {}) - client.close() + void connection.client.request('disconnect').catch(() => {}) + connection.client.close() + management.close() } // A BFCache round trip must not remove the eventual disconnect handler. ) const methods = new Set(STORAGE_METHODS) return new Proxy(local, { get(target, key) { - if (key === 'ready') return ready + if (key === 'ready') return ensureConnected().then(() => connection.ready) if (key === 'setFilesystemStore' || key === 'stopSyncQueue') return () => {} if (key === 'operationLogSupportsConcurrentWriters') return () => true if (key === 'setDriveSyncAvailable') return (available: boolean) => { + driveAvailable = available const boundary = credentialBoundary() const saved = localStorage.getItem('runme/storageCredentialBoundary') // Existing queued edits belong to this profile's identity. A different @@ -479,7 +825,71 @@ export function createSharedNotebookStore( } } if (key === 'createOperationLogSaveStore') - return async (uri: string, options: unknown) => { + return async ( + uri: string, + options: Parameters< + LocalNotebooks['createOperationLogSaveStore'] + >[1] = {} + ) => { + await ensureConnected() + const handshake = await connection.ready + if (handshake.editorCommitVersion === 1) { + const checkpoint = readTabCheckpoints(uri).find( + (entry) => entry.uri === uri + ) + const document = + checkpoint?.document ?? + options.initialDocument ?? + ((await call('loadContent', [uri])) as string) + const session = new EditorSaveSession( + uri, + document, + async (...args) => { + await call('commitEditorOperations', args) + }, + checkpoint?.actorId ?? options.actorId, + checkpoint, + checkpoint + ? () => acknowledgeTabCheckpoint(uri, checkpoint.actorId) + : undefined + ) + const state = { + disposed: false, + unregister: registerUpgradeBlocker( + () => + session.hasPending + ? 'A notebook has pending editor saves; reconnect storage before upgrading' + : undefined, + true, + true + ), + } + sessions.set(session, state) + if (checkpoint) { + void session + .save(uri, session.initialNotebook) + .catch((error) => + storageRecovery.report( + error instanceof Error ? error : new Error(String(error)) + ) + ) + } + return { + initialNotebook: session.initialNotebook, + getObservedOperationHeads: () => + session.getObservedOperationHeads(), + save: session.save.bind(session), + dispose: () => { + state.disposed = true + if (!session.hasPending) { + state.unregister() + sessions.delete(session) + } + }, + } + } + // Older releases still support their original view protocol. Never + // assume they understand immutable editor commits merely from version 8. const view = (await call('createView', [uri, options])) as { id: string heads: string[] diff --git a/app/src/storage/storageOwnerDiagnostics.ts b/app/src/storage/storageOwnerDiagnostics.ts index b7f46701..693ce1b6 100644 --- a/app/src/storage/storageOwnerDiagnostics.ts +++ b/app/src/storage/storageOwnerDiagnostics.ts @@ -1,3 +1,5 @@ +import type { releaseDatabaseDiagnostics } from '../releases/releaseDiagnostics' + /** Diagnostics bypass store initialization and never include notebook contents or credentials. */ export type StorageOwnerDiagnostics = { release?: { releaseId: string; epoch: number; capabilities: string[] } @@ -5,10 +7,24 @@ export type StorageOwnerDiagnostics = { instanceId: string uptimeMs: number connectedTabs: number + connectedPorts?: number initialization: 'pending' | 'ready' | 'failed' pendingCredentials: number scanning: boolean recovery: 'idle' | 'preparing' | 'draining' activeRequestCount: number - oldestRequests: { method: string; elapsedMs: number; stage: string }[] + releaseDatabase?: ReturnType + restart?: { + stage: string + elapsedMs: number + readyTabs: number + connectedTabs: number + inFlight: number + } + oldestRequests: { + method: string + elapsedMs: number + stage: string + stageElapsedMs?: number + }[] } diff --git a/app/src/storage/storageOwnerHost.ts b/app/src/storage/storageOwnerHost.ts index 99ccaca1..5421ad34 100644 --- a/app/src/storage/storageOwnerHost.ts +++ b/app/src/storage/storageOwnerHost.ts @@ -1,5 +1,6 @@ -import { control } from '../releases/control' import { setGoogleDriveBaseUrl } from '../lib/googleDriveRuntime' +import { appLogger } from '../lib/logging/runtime' +import { releaseDatabaseDiagnostics } from '../releases/releaseDiagnostics' import type { parser_pb } from '../runme/client' import { type CreationAttemptRecord, @@ -23,7 +24,12 @@ import { SyncDeferred } from './syncWorkQueue' export class StorageOwnerHost { private ports = new Map< MessagePort, - { available: boolean; lastSeen: number; credentialBoundary: string } + { + available: boolean + lastSeen: number + credentialBoundary: string + editor: boolean + } >() private tokenRequests = new Map< string, @@ -55,6 +61,7 @@ export class StorageOwnerHost { method: string startedAt: number stage: string + stageStartedAt: number }>() private inFlight = new Set>() private recovery?: { @@ -64,6 +71,8 @@ export class StorageOwnerHost { timer?: ReturnType } private draining?: Promise + private recoveryStage = 'idle' + private recoveryStageStartedAt = Date.now() constructor( private readonly store: LocalNotebooks, @@ -94,7 +103,7 @@ export class StorageOwnerHost { private releaseDraining = false /** A bounded memory-only snapshot still answers while IndexedDB or an RPC is stalled. */ - private diagnostics(): StorageOwnerDiagnostics { + diagnostics(): StorageOwnerDiagnostics { // Stop iterating at the response limit: a stalled owner may have many // outstanding requests, and diagnosis must not copy the whole backlog. const oldestRequests: StorageOwnerDiagnostics['oldestRequests'] = [] @@ -103,6 +112,7 @@ export class StorageOwnerHost { method: request.method, elapsedMs: Date.now() - request.startedAt, stage: request.stage, + stageElapsedMs: Date.now() - request.stageStartedAt, }) if (oldestRequests.length === 10) break } @@ -111,7 +121,8 @@ export class StorageOwnerHost { release: this.release, instanceId: this.instanceId, uptimeMs: Date.now() - this.startedAt, - connectedTabs: this.ports.size, + connectedTabs: this.editorPorts().length, + connectedPorts: this.ports.size, initialization: this.initialization, pendingCredentials: this.tokenRequests.size, scanning: Boolean(this.scanning), @@ -122,6 +133,14 @@ export class StorageOwnerHost { : 'idle', activeRequestCount: this.activeRequests.size, oldestRequests, + releaseDatabase: releaseDatabaseDiagnostics(), + restart: { + stage: this.recoveryStage, + elapsedMs: Date.now() - this.recoveryStageStartedAt, + readyTabs: this.recovery?.ready.size ?? 0, + connectedTabs: this.editorPorts().length, + inFlight: this.inFlight.size, + }, } } @@ -130,6 +149,7 @@ export class StorageOwnerHost { available: false, lastSeen: Date.now(), credentialBoundary: 'default', + editor: false, }) this.views.set(port, new Map()) port.onmessage = (event) => { @@ -145,7 +165,15 @@ export class StorageOwnerHost { return } if (event.data?.type === 'storage-recovery') { - void this.recover(port, event.data).catch(error => port.postMessage(recoveryMessage('blocked', event.data.id, String(error)))) + void this.recover(port, event.data).catch((error) => { + this.markRecovery('failed') + appLogger.warn('Storage restart failed', { + attrs: { scope: 'storage.owner', diagnostics: this.diagnostics() }, + }) + port.postMessage( + recoveryMessage('blocked', event.data.id, String(error)) + ) + }) return } const operation = this.receive(port, event.data) @@ -157,6 +185,19 @@ export class StorageOwnerHost { } port.onmessageerror = () => this.detach(port) port.start() + } + + /** Only application connections own editors; passive launcher ports do not vote. */ + private editorPorts(): MessagePort[] { + return [...this.ports] + .filter(([, state]) => state.editor) + .map(([port]) => port) + } + + private registerEditor(port: MessagePort): void { + const state = this.ports.get(port) + if (!state || state.editor) return + state.editor = true if (this.recovery) { port.postMessage(recoveryMessage('prepare', this.recovery.id)) this.armRecoveryTimeout() @@ -164,14 +205,18 @@ export class StorageOwnerHost { } /** Unversioned control plane: every attached tab must preserve its editor before shutdown. */ - private async recover(port: MessagePort, message: RecoveryMessage): Promise { + private async recover( + port: MessagePort, + message: RecoveryMessage + ): Promise { if (this.releaseDraining) return if (!this.ports.has(port) || message.protocol !== 1) return + if (message.action === 'register') { + this.registerEditor(port) + return + } if (message.action === 'restart') { - if (this.release) await control(s => { - if (s.transition || s.active?.epoch !== this.release!.epoch) throw new Error('A release transition is pending; use Versions recovery') - s.emergencyRestart = {epoch: this.release!.epoch, id: this.recovery?.id ?? message.id} - }) + this.registerEditor(port) if (this.recovery && !this.recovery.draining) { port.postMessage(recoveryMessage('prepare', this.recovery.id)) return @@ -193,8 +238,9 @@ export class StorageOwnerHost { timer: undefined as ReturnType | undefined, } this.recovery = round + this.markRecovery('waiting-for-tab-backups') this.armRecoveryTimeout() - for (const client of this.ports.keys()) + for (const client of this.editorPorts()) client.postMessage(recoveryMessage('prepare', round.id)) } else if (message.action === 'ready' && message.id === this.recovery?.id) { if (message.error) { @@ -206,6 +252,12 @@ export class StorageOwnerHost { } } + /** Restart phases remain observable even when the release database cannot reply. */ + private markRecovery(stage: string): void { + this.recoveryStage = stage + this.recoveryStageStartedAt = Date.now() + } + private armRecoveryTimeout(): void { const round = this.recovery if (!round) return @@ -219,6 +271,7 @@ export class StorageOwnerHost { } private abortRecovery(error: string): void { + this.markRecovery('blocked') const round = this.recovery if (!round) return clearTimeout(round.timer) @@ -226,29 +279,30 @@ export class StorageOwnerHost { // gather new editor acknowledgements after a late tab fixes its backup error. if (!round.draining) { this.recovery = undefined - if (this.release) void control(s => { if (s.emergencyRestart?.epoch === this.release!.epoch && s.emergencyRestart.id === round.id) delete s.emergencyRestart }).catch(() => {}) - } - else round.ready.clear() - for (const port of this.ports.keys()) + } else round.ready.clear() + for (const port of this.editorPorts()) port.postMessage(recoveryMessage('blocked', round.id, error)) } /** No timer may force termination while a storage or network commit is still active. */ private async finishRecovery(): Promise { const round = this.recovery - if (!round || [...this.ports.keys()].some((port) => !round.ready.has(port))) + if (!round || this.editorPorts().some((port) => !round.ready.has(port))) return clearTimeout(round.timer) - for (const port of this.ports.keys()) + for (const port of this.editorPorts()) port.postMessage(recoveryMessage('draining', round.id)) if (!round.draining) { round.draining = true this.draining = (async () => { + this.markRecovery('waiting-for-initialization') await this.store.ready + this.markRecovery('waiting-for-active-requests') await Promise.allSettled([ ...this.inFlight, ...(this.scanning ? [this.scanning] : []), ]) + this.markRecovery('draining-storage') await this.store.drainForStorageRestart() })() } @@ -257,16 +311,17 @@ export class StorageOwnerHost { // A tab attached during draining must preserve its editor too. if ( this.recovery !== round || - [...this.ports.keys()].some((port) => !round.ready.has(port)) + this.editorPorts().some((port) => !round.ready.has(port)) ) return this.recovery = undefined - for (const port of this.ports.keys()) + for (const port of this.editorPorts()) port.postMessage(recoveryMessage('reload', round.id)) + this.markRecovery('shutdown') this.shutdown?.() } catch { // Remain quiescent rather than start a second owner after an uncertain drain. - for (const port of this.ports.keys()) + for (const port of this.editorPorts()) port.postMessage( recoveryMessage( 'blocked', @@ -363,12 +418,12 @@ export class StorageOwnerHost { } event(type: string, detail: unknown): void { - for (const port of this.ports.keys()) + for (const port of this.editorPorts()) port.postMessage({ type: 'event', eventType: type, detail }) } notify(uri: string): void { - for (const port of this.ports.keys()) + for (const port of this.editorPorts()) port.postMessage({ type: 'changed', uri }) } @@ -394,6 +449,21 @@ export class StorageOwnerHost { return } if (request.type !== 'request' || typeof request.id !== 'string') return + // Stable negotiation precedes data-RPC version checks and all storage work. + if (request.method === 'handshake') { + port.postMessage({ + type: 'result', + id: request.id, + value: { + protocol: 1, + rpcVersion: STORAGE_OWNER_VERSION, + recoveryFormat: 1, + editorCommitVersion: 1, + }, + }) + return + } + if (request.method !== 'disconnect') this.registerEditor(port) // Closing a stale tab must release its port even when its data protocol differs. if (request.method === 'disconnect') { this.detach(port) @@ -412,10 +482,15 @@ export class StorageOwnerHost { ? request.method : 'unknown', startedAt: Date.now(), - stage: 'initializing', + stage: 'authorizing', + stageStartedAt: Date.now(), } this.activeRequests.add(active) try { + if (request.version !== STORAGE_OWNER_VERSION) + throw new Error( + 'Storage worker version mismatch. Preserve edits and reload this tab to reconnect.' + ) if (this.releaseDraining) throw new Error('Release handoff is draining storage') await this.authorize?.(request) @@ -433,10 +508,6 @@ export class StorageOwnerHost { state.credentialBoundary = request.credentialBoundary this.updateAvailability() } - if (request.version !== STORAGE_OWNER_VERSION) - throw new Error( - 'Storage worker version mismatch. Reload all Runme tabs to reconnect.' - ) if (!Array.isArray(request.args)) throw new Error('Invalid storage request arguments') if ( @@ -449,11 +520,16 @@ export class StorageOwnerHost { throw new Error( 'Drive identity differs from the storage owner; use an isolated browser profile' ) + active.stage = 'initializing' + active.stageStartedAt = Date.now() await this.store.ready + active.stage = 'reauthorizing' + active.stageStartedAt = Date.now() if (this.releaseDraining) throw new Error('Release handoff is draining storage') await this.authorize?.(request) active.stage = 'executing' + active.stageStartedAt = Date.now() let value: unknown switch (request.method) { case 'hello': @@ -510,7 +586,12 @@ export class StorageOwnerHost { )?.creationMigrationError this.configuredBaseUrl = baseUrl setGoogleDriveBaseUrl(baseUrl) - value = { version: STORAGE_OWNER_VERSION, instanceId: this.instanceId, release: this.release } + value = { + version: STORAGE_OWNER_VERSION, + editorCommitVersion: 1, + instanceId: this.instanceId, + release: this.release, + } } break case 'heartbeat': diff --git a/app/src/storage/storageOwnerProtocol.ts b/app/src/storage/storageOwnerProtocol.ts index c702daea..f6b5da47 100644 --- a/app/src/storage/storageOwnerProtocol.ts +++ b/app/src/storage/storageOwnerProtocol.ts @@ -20,6 +20,7 @@ export const STORAGE_METHODS = [ 'listFileSyncStatusPage', 'getMetadata', 'save', + 'commitEditorOperations', 'isOperationLogNotebook', 'listOperationLogComments', 'addOperationLogComment', diff --git a/app/src/storage/storageOwnerRuntime.ts b/app/src/storage/storageOwnerRuntime.ts index 65623b82..dcccf5a9 100644 --- a/app/src/storage/storageOwnerRuntime.ts +++ b/app/src/storage/storageOwnerRuntime.ts @@ -51,7 +51,6 @@ export async function startStorageOwner(identity?: { if ( request.epoch !== identity.epoch || !tab || - tab.epoch !== identity.epoch || state.active?.epoch !== identity.epoch || state.active.manifest.releaseId !== identity.releaseId || state.transition?.phase === 'handoff' || @@ -97,6 +96,7 @@ export async function startStorageOwner(identity?: { } return { attach: (port: MessagePort) => host.attach(port), + diagnostics: () => host.diagnostics(), drain: () => { clearInterval(timer) return host.drainForRelease() diff --git a/app/src/storage/storageRecovery.test.ts b/app/src/storage/storageRecovery.test.ts index fdb1d70f..37409810 100644 --- a/app/src/storage/storageRecovery.test.ts +++ b/app/src/storage/storageRecovery.test.ts @@ -53,6 +53,7 @@ function connect(host: StorageOwnerHost, prepare = vi.fn(async () => vi.fn())) { reload, prepare ) + pagePort.postMessage(recoveryMessage('register', 'editor-client')) clients.push(client) return { client, reload, prepare, pagePort } } @@ -145,6 +146,16 @@ describe('coordinated storage restart', () => { channel.port2 as unknown as MessagePort ) host.attach(channel.port1 as unknown as MessagePort) + channel.port2.postMessage({ + type: 'request', + id: 'legacy-hello', + method: 'hello', + version: 0, + args: [], + }) + await vi.waitFor(async () => + expect((await first.client.diagnose()).connectedTabs).toBe(2) + ) vi.useFakeTimers() first.client.recovery!.restart() await vi.waitFor(() => expect(first.prepare).toHaveBeenCalled()) @@ -419,3 +430,66 @@ it('releases preserved editors if sending the ready acknowledgement throws', asy expect(release).toHaveBeenCalledOnce() expect(storageRecovery.getSnapshot().phase).toBe('blocked') }) + +it('offers recovery for a release-settings failure before the RPC deadline', async () => { + const { requiresStorageReload } = await import('./storageRecovery') + expect( + requiresStorageReload( + new Error( + 'Release settings read timeout (get). Stored notebook data has not been removed.' + ) + ) + ).toBe(true) + expect( + requiresStorageReload( + new Error( + 'Stale or frozen release: storage request rejected before effects' + ) + ) + ).toBe(false) +}) + +it('does not count the launcher control connection as an editor tab', async () => { + const { host, shutdown } = setup() + const editor = connect(host) + const launcher = new MessageChannel() + ports.push( + launcher.port1 as unknown as MessagePort, + launcher.port2 as unknown as MessagePort + ) + host.attach(launcher.port1 as unknown as MessagePort) + const messages: unknown[] = [] + launcher.port2.on('message', (message) => messages.push(message)) + await vi.waitFor(async () => + expect((await editor.client.diagnose()).connectedTabs).toBe(1) + ) + expect((await editor.client.diagnose()).connectedPorts).toBe(2) + editor.client.recovery!.restart() + await vi.waitFor(() => expect(editor.reload).toHaveBeenCalledOnce()) + expect(shutdown).toHaveBeenCalledOnce() + expect(messages).toEqual([]) +}) + +it('backs up and restarts an admitted owner without consulting release settings', async () => { + const { store, shutdown } = setup() + const module = await import('../releases/control') + const control = vi + .spyOn(module, 'control') + .mockImplementation(() => new Promise(() => {})) + try { + const host = new StorageOwnerHost( + store as unknown as LocalNotebooks, + shutdown, + undefined, + { releaseId: 'test', epoch: 1, capabilities: [] } + ) + const editor = connect(host) + editor.client.recovery!.restart() + await vi.waitFor(() => expect(editor.reload).toHaveBeenCalledOnce()) + expect(editor.prepare).toHaveBeenCalledOnce() + expect(control).not.toHaveBeenCalled() + expect(shutdown).toHaveBeenCalledOnce() + } finally { + control.mockRestore() + } +}) diff --git a/app/src/storage/storageRecovery.ts b/app/src/storage/storageRecovery.ts index 759cf325..e60917ae 100644 --- a/app/src/storage/storageRecovery.ts +++ b/app/src/storage/storageRecovery.ts @@ -1,3 +1,5 @@ +import type { StorageOwnerDiagnostics } from './storageOwnerDiagnostics' + /** Page-local recovery status shared by the notebook banner and storage status page. */ export type StorageRecoveryState = { phase: 'healthy' | 'required' | 'preparing' | 'restarting' | 'blocked' @@ -5,7 +7,8 @@ export type StorageRecoveryState = { } let state: StorageRecoveryState = { phase: 'healthy', message: '' } const listeners = new Set<() => void>() -let restart: (() => void) | undefined +let restart: ((force?: boolean) => void) | undefined +let diagnose: (() => Promise) | undefined let backup: (() => number) | undefined let prepare: ((signal: AbortSignal) => Promise<() => void>) | undefined @@ -31,7 +34,21 @@ export const storageRecovery = { return this.set({ phase: 'required', message: error.message }) }, - registerRestart(callback: () => void) { + /** Independent of normal RPC admission and release-settings reads. */ + registerDiagnostics(callback: () => Promise) { + diagnose = callback + return () => { + if (diagnose === callback) diagnose = undefined + } + }, + diagnose() { + return diagnose + ? diagnose() + : Promise.reject( + new Error('Storage worker diagnostics are not connected.') + ) + }, + registerRestart(callback: (force?: boolean) => void) { restart = callback }, registerPreparation( @@ -61,14 +78,14 @@ export const storageRecovery = { ) return prepare(signal) }, - restart() { + restart(force = false) { if (!restart) return - restart() + restart(force) }, } export function requiresStorageReload(error: Error): boolean { - return /Storage worker version mismatch|Storage worker disconnected|Storage worker message failed|Notebook storage worker failed|Storage worker did not acknowledge|Storage worker did not respond|Drive endpoint differs from the storage owner/.test( + return /Release settings (?:open|read|commit) (?:blocked|timeout|failed)|Storage worker version mismatch|Storage worker disconnected|Storage worker message failed|Notebook storage worker failed|Storage worker did not acknowledge|Storage worker did not respond|Drive endpoint differs from the storage owner/.test( error.message ) } diff --git a/app/src/storage/storageRecoveryProtocol.ts b/app/src/storage/storageRecoveryProtocol.ts index f41a349b..c91ce092 100644 --- a/app/src/storage/storageRecoveryProtocol.ts +++ b/app/src/storage/storageRecoveryProtocol.ts @@ -3,7 +3,14 @@ export const STORAGE_RECOVERY_PROTOCOL = 1 export type RecoveryMessage = { type: 'storage-recovery' protocol: 1 - action: 'restart' | 'prepare' | 'ready' | 'blocked' | 'draining' | 'reload' + action: + | 'register' + | 'restart' + | 'prepare' + | 'ready' + | 'blocked' + | 'draining' + | 'reload' id: string error?: string } diff --git a/app/src/storage/storageWorkerControl.test.ts b/app/src/storage/storageWorkerControl.test.ts new file mode 100644 index 00000000..ce85711d --- /dev/null +++ b/app/src/storage/storageWorkerControl.test.ts @@ -0,0 +1,58 @@ +// @vitest-environment node +import { expect, it, vi } from 'vitest' + +import { + StorageWorkerControl, + type WorkerControlEvent, + installStorageWorkerControl, +} from './storageWorkerControl' + +it('answers diagnostics during a stalled drain and allows force close without tab acknowledgements', async () => { + let phase = 'initializing-storage' + const close = vi.fn() + const drain = vi.fn(() => new Promise(() => {})) + const dispose = installStorageWorkerControl({ + status: () => ({ + instanceId: 'stalled', + epoch: 0, + phase, + phaseElapsedMs: 100, + uptimeMs: 200, + }), + phase: (value) => { + phase = value + }, + drain, + close, + }) + const events: WorkerControlEvent[] = [] + const client = new StorageWorkerControl((event) => { + events.push(event) + }) + try { + client.discover() + await vi.waitFor(() => + expect(events.at(-1)?.status.phase).toBe('initializing-storage') + ) + client.stop('unrelated-worker', { force: true, restart: true }) + client.stop('stalled', { force: false, restart: true }) + await vi.waitFor(() => expect(drain).toHaveBeenCalledOnce()) + expect(close).not.toHaveBeenCalled() + client.discover() + await vi.waitFor(() => + expect(events.at(-1)).toMatchObject({ + type: 'status', + status: { phase: 'draining' }, + }) + ) + client.stop('stalled', { force: true, restart: true }) + await vi.waitFor(() => + expect(events.at(-1)).toMatchObject({ type: 'stopped', restart: true }) + ) + expect(close).toHaveBeenCalledOnce() + expect(drain).toHaveBeenCalledOnce() + } finally { + client.close() + dispose() + } +}) diff --git a/app/src/storage/storageWorkerControl.ts b/app/src/storage/storageWorkerControl.ts new file mode 100644 index 00000000..f9a875d1 --- /dev/null +++ b/app/src/storage/storageWorkerControl.ts @@ -0,0 +1,106 @@ +import type { StorageOwnerDiagnostics } from './storageOwnerDiagnostics' + +export const STORAGE_WORKER_CONTROL = 'runme-storage-worker-control-v1' +export type WorkerStatus = { + instanceId: string + releaseId?: string + epoch: number + phase: string + phaseElapsedMs: number + uptimeMs: number + diagnostics?: StorageOwnerDiagnostics + releaseDatabase?: StorageOwnerDiagnostics['releaseDatabase'] + error?: string +} +export type WorkerControlEvent = + | { type: 'status'; status: WorkerStatus } + | { type: 'stopping' | 'stopped'; status: WorkerStatus; restart: boolean } +type WorkerCommand = + | { type: 'discover' } + | { type: 'stop'; instanceId: string; force: boolean; restart: boolean } + +/** A memory-only management connection, independent of storage RPC and IndexedDB. + * Discovery includes only responding Runme workers implementing this protocol. + */ +export class StorageWorkerControl { + private readonly channel = new BroadcastChannel(STORAGE_WORKER_CONTROL) + constructor(onEvent: (event: WorkerControlEvent) => void) { + this.channel.onmessage = ({ data }) => { + if (['status', 'stopping', 'stopped'].includes(data?.type) && data.status) + onEvent(data as WorkerControlEvent) + } + } + discover(): void { + this.channel.postMessage({ type: 'discover' } satisfies WorkerCommand) + } + stop( + instanceId: string, + options: { force: boolean; restart: boolean } + ): void { + this.channel.postMessage({ + type: 'stop', + instanceId, + ...options, + } satisfies WorkerCommand) + } + close(): void { + this.channel.close() + } +} + +/** Install before lock acquisition/imports so a stalled startup remains visible. + * Force-close is cooperative: the worker event loop must still process messages. + * No tab acknowledgements are needed because this never reloads a page. + */ +export function installStorageWorkerControl(options: { + status: () => WorkerStatus + drain: () => Promise + close: () => void + phase: (phase: string) => void +}): () => void { + const channel = new BroadcastChannel(STORAGE_WORKER_CONTROL) + let stopping = false + let closed = false + const publish = (type: 'stopping' | 'stopped', restart: boolean) => + channel.postMessage({ + type, + status: options.status(), + restart, + } satisfies WorkerControlEvent) + channel.onmessage = async ({ data }: MessageEvent) => { + if (data?.type === 'discover') { + channel.postMessage({ + type: 'status', + status: options.status(), + } satisfies WorkerControlEvent) + return + } + if ( + data?.type !== 'stop' || + data.instanceId !== options.status().instanceId || + closed + ) + return + // A force request can interrupt a graceful drain stuck in storage/network IO. + if (stopping && !data.force) return + stopping = true + options.phase(data.force ? 'force-closing' : 'draining') + publish('stopping', data.restart) + try { + if (!data.force) await options.drain() + if (closed) return + closed = true + options.phase('closing') + publish('stopped', data.restart) + options.close() + } catch (error) { + stopping = false + options.phase('drain-failed') + channel.postMessage({ + type: 'status', + status: { ...options.status(), error: String(error) }, + } satisfies WorkerControlEvent) + } + } + return () => channel.close() +} diff --git a/app/src/storage/tabRecoveryCheckpoint.test.ts b/app/src/storage/tabRecoveryCheckpoint.test.ts new file mode 100644 index 00000000..5544b812 --- /dev/null +++ b/app/src/storage/tabRecoveryCheckpoint.test.ts @@ -0,0 +1,56 @@ +import { afterEach, expect, it, vi } from 'vitest' + +import { createInitialNotebookFile } from '../lib/notebookFormat' +import { EditorSaveSession } from './editorSaveSession' +import { + acknowledgeTabCheckpoint, + readTabCheckpoints, + writeTabCheckpoints, +} from './tabRecoveryCheckpoint' + +afterEach(() => { + vi.restoreAllMocks() + sessionStorage.clear() +}) + +it('retains every checkpoint until its own restored save is acknowledged', async () => { + const a = await new EditorSaveSession( + 'a', + createInitialNotebookFile('a.runme'), + async () => {} + ).checkpoint() + const b = await new EditorSaveSession( + 'b', + createInitialNotebookFile('b.runme'), + async () => {} + ).checkpoint() + writeTabCheckpoints([a]) + writeTabCheckpoints([b]) + expect(readTabCheckpoints()).toEqual([a, b]) + acknowledgeTabCheckpoint('a', 'wrong-actor') + expect(readTabCheckpoints()).toHaveLength(2) + acknowledgeTabCheckpoint('a', a.actorId) + expect(readTabCheckpoints()).toEqual([b]) +}) + +it('rejects quota failure and failed write verification instead of authorizing reload', async () => { + const entry = await new EditorSaveSession( + 'a', + createInitialNotebookFile('a.runme'), + async () => {} + ).checkpoint() + vi.spyOn(Storage.prototype, 'setItem').mockImplementation(() => { + throw new DOMException('Full', 'QuotaExceededError') + }) + expect(() => writeTabCheckpoints([entry])).toThrow('Full') + vi.restoreAllMocks() + vi.spyOn(Storage.prototype, 'setItem').mockImplementation(() => {}) + expect(() => writeTabCheckpoints([entry])).toThrow('could not be verified') +}) + +it('retains unsupported recovery bytes', () => { + const raw = '{"format":99,"editors":[]}' + sessionStorage.setItem('runme/tab-editor-recovery/v1', raw) + expect(readTabCheckpoints).toThrow('Unsupported') + expect(sessionStorage.getItem('runme/tab-editor-recovery/v1')).toBe(raw) +}) diff --git a/app/src/storage/tabRecoveryCheckpoint.ts b/app/src/storage/tabRecoveryCheckpoint.ts new file mode 100644 index 00000000..e06345ca --- /dev/null +++ b/app/src/storage/tabRecoveryCheckpoint.ts @@ -0,0 +1,135 @@ +import type { parser_pb } from '../runme/client' +import { + type EditorSaveCheckpoint, + EditorSaveSession, +} from './editorSaveSession' + +const KEY = 'runme/tab-editor-recovery/v1' +type Envelope = { format: 1; editors: EditorSaveCheckpoint[] } + +/** Fail closed on unknown formats or corrupt data; leave the original bytes intact. */ +export function readTabCheckpoints(uri?: string): EditorSaveCheckpoint[] { + const raw = sessionStorage.getItem(KEY) + if (!raw) return [] + const value = JSON.parse(raw) as Envelope + if (value?.format !== 1 || !Array.isArray(value.editors)) + throw new Error( + 'Unsupported tab recovery format; keep this tab’s recovery data' + ) + const entries = + uri === undefined + ? value.editors + : value.editors.filter((entry) => entry?.uri === uri) + const uris = new Set() + for (const entry of entries) { + if (typeof entry?.uri !== 'string' || uris.has(entry.uri)) + throw new Error('Invalid tab recovery notebook identity') + uris.add(entry.uri) + // Validate all editor state before returning any entry for restoration. + new EditorSaveSession( + entry.uri, + entry.document, + async () => {}, + entry.actorId, + entry + ) + } + return entries +} + +/** Enumerate identities without letting a damaged editor block unrelated loads. + * Each notebook validates its own checkpoint when it opens; bytes stay intact. + */ +export function tabCheckpointUris(): string[] { + const raw = sessionStorage.getItem(KEY) + if (!raw) return [] + const value = JSON.parse(raw) as Envelope + if (value?.format !== 1 || !Array.isArray(value.editors)) + throw new Error( + 'Unsupported tab recovery format; keep this tab’s recovery data' + ) + return [ + ...new Set( + value.editors + .map((entry) => entry?.uri) + .filter((uri) => typeof uri === 'string') + ), + ] +} + +/** Synchronous write/read-back is the reload boundary, independent of the worker. + * Keep unconsumed entries, including saves from editors closed before recovery. + */ +export function writeTabCheckpoints(entries: EditorSaveCheckpoint[]): void { + const retained = new Map( + readTabCheckpoints().map((entry) => [entry.uri, entry]) + ) + const captured = new Set() + for (const entry of entries) { + if (captured.has(entry.uri)) + throw new Error( + 'Multiple pending editors for one notebook; reconnect saves before reloading this tab' + ) + captured.add(entry.uri) + retained.set(entry.uri, entry) + } + const raw = JSON.stringify({ + format: 1, + editors: [...retained.values()], + } satisfies Envelope) + sessionStorage.setItem(KEY, raw) + if (sessionStorage.getItem(KEY) !== raw) + throw new Error( + 'Tab recovery checkpoint could not be verified; no reload performed' + ) + readTabCheckpoints() +} + +/** Clear only after replay and the restored editor snapshot have been acknowledged. */ +export function acknowledgeTabCheckpoint(uri: string, actorId: string): void { + const entries = readTabCheckpoints() + const remaining = entries.filter( + (entry) => entry.uri !== uri || entry.actorId !== actorId + ) + if (remaining.length === entries.length) return + if (!remaining.length) sessionStorage.removeItem(KEY) + else + sessionStorage.setItem( + KEY, + JSON.stringify({ format: 1, editors: remaining } satisfies Envelope) + ) +} + +type CheckpointTransport = { + uris: () => ReadonlySet + capture: ( + editors: Map + ) => Promise + resume: () => void +} +let transport: CheckpointTransport | undefined + +/** The storage adapter owns save sessions; preparation owns the visible editors. */ +export function registerCheckpointTransport(value: CheckpointTransport): void { + transport = value +} + +/** Legacy snapshot stores must acknowledge a durable flush before local reload. */ +export function checkpointEditorUris(): ReadonlySet { + return transport?.uris() ?? new Set() +} + +export async function checkpointEditorSessions( + editors: Map, + signal?: AbortSignal +): Promise> { + if (!transport) throw new Error('Editor recovery transport is not ready') + const checkpoints = await transport.capture(editors) + signal?.throwIfAborted() + writeTabCheckpoints(checkpoints) + return new Set(checkpoints.map((entry) => entry.uri)) +} + +export function resumeCheckpointTransport(): void { + transport?.resume() +} diff --git a/app/test/browser/controlled-releases.ts b/app/test/browser/controlled-releases.ts index 4f5533f1..65162cc7 100644 --- a/app/test/browser/controlled-releases.ts +++ b/app/test/browser/controlled-releases.ts @@ -628,6 +628,36 @@ async function main() { console.log('PASS: deploy B; refresh and new tabs retain A') await openVersions(one) await one.screenshot({ path: output + '/00-version-information.png' }) + const pinFlow = one.getByRole('button', { + name: 'Pin Release', + exact: true, + }) + const hasPinFlow = (await pinFlow.count()) > 0 + if (hasPinFlow) { + await pinFlow.click() + await one.getByRole('tab', { name: 'Pin Release', exact: true }).waitFor() + await one.screenshot({ path: output + '/00-pin-release-choices.png' }) + await one.getByRole('button', { name: /Choose another release/ }).click() + await one + .getByRole('button', { name: 'Load published releases', exact: true }) + .click() + await one + .getByRole('button', { + name: new RegExp(`Commit ${expected.b.slice(0, 8)}`), + }) + .click() + await one.screenshot({ path: output + '/00-pin-release-review.png' }) + assert.equal( + (await one.evaluate(() => (window as any).runmeVersions.get())) + .selection.kind, + 'channel' + ) + await one.getByRole('button', { name: 'Back', exact: true }).click() + await one.getByRole('button', { name: 'Back', exact: true }).click() + await one + .getByRole('button', { name: /Keep the installed release/ }) + .click() + } await one .getByRole('button', { name: 'Pin installed release', exact: true }) .click() @@ -638,6 +668,13 @@ async function main() { s.selection.releaseId === a && !s.candidate ) + if (hasPinFlow) + await one + .getByRole('button', { + name: 'Back to version information', + exact: true, + }) + .click() await one .getByRole('button', { name: 'Check for updates', exact: true }) .click() @@ -660,6 +697,17 @@ async function main() { s.selection.name === 'stable' && s.candidate?.releaseId === expected.b ) assert.equal(await active(one), a) + if (!process.env.RUNME_RELEASE_BASELINE) { + await one.getByTestId('version-attention-badge').waitFor() + assert.equal( + await one + .getByTestId('version-attention-badge') + .getAttribute('data-kind'), + 'update' + ) + await one.locator('#runme-release-panel').waitFor({ state: 'hidden' }) + await one.screenshot({ path: output + '/00-version-attention.png' }) + } console.log( 'PASS: Information view owns controls; exact pin and channel selection never activate implicitly' ) @@ -669,119 +717,176 @@ async function main() { `const d=await notebooks.createLocal('upgrade-one.runme'); await notebooks.appendCell({target:{uri:d.handle.uri},kind:'markup',value:'Before upgrade: durable edit'}); await notebooks.show(d.handle.uri); console.log(d.handle.uri);` ) const uri = created.match(/local:\/\/file\/[^\s]+/)![0] - await execute( + const secondCreated = await execute( two, - `const d=await notebooks.createLocal('upgrade-two.runme');await notebooks.appendCell({target:{uri:d.handle.uri},kind:'markup',value:'Second tab survives'});await notebooks.show(d.handle.uri);` + `const d=await notebooks.createLocal('upgrade-two.runme');await notebooks.appendCell({target:{uri:d.handle.uri},kind:'markup',value:'Second tab survives'});await notebooks.show(d.handle.uri);console.log(d.handle.uri);` ) + const secondUri = secondCreated.match(/local:\/\/file\/[^\s]+/)![0] await one.screenshot({ path: output + '/01-two-tabs-a.png' }) - // A failed preparation must block all tabs, even though the other tabs save. - await two.evaluate(() => { - const runtime = (window as any).runmeRelease - ;(window as any).__realPrepare = runtime.prepare - runtime.prepare = async () => { - throw new Error('Injected durable-save failure') - } - }) + const historicalApp = Boolean(process.env.RUNME_RELEASE_BASELINE) + let incompatible: Page | undefined + if (!historicalApp) { + incompatible = await context.newPage() + pages.push(incompatible) + await incompatible.goto(base + '/?session=incompatible-peer') + await ready(incompatible) + await execute( + incompatible, + `const d=await notebooks.createLocal('incompatible.runme');await notebooks.appendCell({target:{uri:d.handle.uri},kind:'markup',value:'Preserved during compatibility recovery'});await notebooks.show(d.handle.uri);` + ) + await incompatible.evaluate(() => { + // Inject only the stable capability reply, leaving the actual worker, + // lock, checkpoint, navigation and persistence paths unchanged. + const NativeWorker = SharedWorker + ;(window as any).SharedWorker = class extends NativeWorker { + constructor(...args: ConstructorParameters) { + super(...args) + const send = this.port.postMessage.bind(this.port) + this.port.postMessage = (message: any) => { + if ( + message.type === 'request' && + !['handshake', 'disconnect'].includes(message.method) + ) + sessionStorage.setItem('incompatible-data-rpc', message.method) + send(message) + } + const descriptor = Object.getOwnPropertyDescriptor( + MessagePort.prototype, + 'onmessage' + )! + Object.defineProperty(this.port, 'onmessage', { + set(handler) { + descriptor.set!.call(this, (event: MessageEvent) => { + if (event.data?.value?.protocol === 1) + event.data.value.rpcVersion = -1 + handler?.(event) + }) + }, + }) + } + } + }) + } + const peerId = await two.evaluate(() => (window as any).runmeRelease.tabId) + let suspended: Awaited> | undefined + if (historicalApp) { + // Historical launchers implement their own all-tab reload policy. Preserve + // and close those already-durable peers before testing the migration floor. + await Promise.all([two, three].map((page) => page.close())) + } else { + await two.evaluate(() => { + const runtime = (window as any).runmeRelease + runtime.prepare = runtime.prepareReload = async () => { + throw new Error('Peer must never be asked to prepare') + } + }) + suspended = await context.newCDPSession(two) + await suspended.send('Page.setWebLifecycleState', { state: 'frozen' }) + } await openVersions(one) await one - .getByRole('button', { name: 'Upgrade all tabs', exact: true }) - .click() - await one - .locator('#runme-release-panel') - .getByRole('status') - .filter({ hasText: 'Injected durable-save failure' }) - .waitFor({ timeout: 30000 }) - assert.equal(await active(one), a) - assert.equal(await active(two), a) - await one.screenshot({ path: output + '/02-blocked-save.png' }) - console.log('PASS: unconfirmed durable save blocks activation') - await two.evaluate(() => { - ;(window as any).runmeRelease.prepare = (window as any).__realPrepare - }) - const oldTabId = await one.evaluate( - () => (window as any).runmeRelease.tabId - ) - await one - .locator('#runme-release-panel') - .getByRole('button', { name: 'Cancel upgrade', exact: true }) + .getByRole('button', { + name: /^(Activate release and update this tab|Upgrade all tabs)$/, + exact: true, + }) .click() await one.waitForFunction( - (id) => - (window as any).runmeRelease?.tabId && - (window as any).runmeRelease.tabId !== id, - oldTabId + (id) => (window as any).runmeRelease?.active.manifest.releaseId === id, + expected.b, + { timeout: 60000 } ) await ready(one) - assert.match( + const b = await active(one) + assert.notEqual(a, b) + if (!historicalApp) { + await suspended!.send('Page.setWebLifecycleState', { state: 'active' }) + await suspended!.detach() + assert.equal(await active(two), a) + assert.equal(await active(three), a) + assert.equal( + await two.evaluate(() => (window as any).runmeRelease.tabId), + peerId + ) + assert.equal( + await two.evaluate(() => document.getElementById('root')?.inert), + false + ) await execute( - one, - `console.log((await notebooks.get({uri:${JSON.stringify(uri)}})).notebook.cells.map(c=>c.value).join('\\n'))` - ), - /Before upgrade/ - ) - console.log('PASS: cancellation reloads prepared tabs from durable state') - // Hold readiness as well as requesting browser freeze: Chromium may wake a - // frozen page when another tab opens. Neither missing readiness nor loss of - // the initiating tab permits the coordinator to discard this participant. - await ready(two) - await ready(three) - await waitControl(three, (s) => !s.transition) - await two.evaluate(() => { - const real = (window as any).runmeRelease.prepare - ;(window as any).runmeRelease.prepare = () => - new Promise((resolve, reject) => { - ;(window as any).__resumeReleasePreparation = () => - real().then(resolve, reject) + two, + `await notebooks.appendCell({target:{uri:${JSON.stringify(secondUri)}},kind:'markup',value:'Old UI saves through new worker'});` + ) + await waitControl( + two, + (s) => !s.transition && s.active?.manifest.releaseId === b + ) + assert.equal(await active(two), a, 'Saving must not reload the older UI') + await two.waitForFunction( + () => + document + .querySelector('[data-testid="version-attention-badge"]') + ?.getAttribute('data-kind') === 'reload' + ) + await two.locator('#runme-release-panel').waitFor({ state: 'hidden' }) + await two + .getByRole('button', { name: 'Open Version Information', exact: true }) + .click() + await two + .getByRole('button', { + name: 'Preserve edits and reload this tab', + exact: true, }) - }) - const suspended = await context.newCDPSession(two) - await suspended.send('Page.setWebLifecycleState', { state: 'frozen' }) - await openVersions(one) - await one - .getByRole('button', { name: 'Upgrade all tabs', exact: true }) - .click() - await waitControl( - three, - (s) => - s.transition?.phase === 'prepare' && - Object.values(s.transition.participants).some((p: any) => p.ready) - ) - assert.equal( - await three.evaluate( - async () => - (await (window as any).runmeVersions.get()).active.manifest.releaseId - ), - a - ) - await one.close() - one = await context.newPage() - pages[0] = one - await one.goto(base + '/?session=release-one') - await one - .getByRole('status') - .filter({ hasText: 'waiting for tab' }) - .waitFor({ timeout: 30000 }) - assert.equal(await active(one), undefined) - await one.screenshot({ path: output + '/02-frozen-tab-new-tab-waits.png' }) - await suspended.send('Page.setWebLifecycleState', { state: 'active' }) - await two.waitForFunction( - () => typeof (window as any).__resumeReleasePreparation === 'function' - ) - await two.evaluate(() => (window as any).__resumeReleasePreparation()) - await suspended.detach() - console.log( - 'PASS: frozen tab blocks; new tab waits; another coordinator resumes after initiator closes' - ) - for (const page of pages) { - await page.waitForFunction( + .waitFor() + await two.screenshot({ + path: output + '/02-outdated-tab-information.png', + }) + // A local backup failure must not reload this page or roll back the worker. + const thirdId = await three.evaluate( + () => (window as any).runmeRelease.tabId + ) + const failedBackup = await three.evaluate(async () => { + const set = Storage.prototype.setItem + Storage.prototype.setItem = function (key, value) { + if (key === 'runme/tab-editor-recovery/v1') + throw new DOMException( + 'Injected recovery quota', + 'QuotaExceededError' + ) + return set.call(this, key, value) + } + try { + await (window as any).runmeVersions.reloadTab() + return 'unexpected reload' + } catch (error) { + return String(error) + } finally { + Storage.prototype.setItem = set + } + }) + assert.match(failedBackup, /Injected recovery quota/) + assert.equal( + await three.evaluate(() => (window as any).runmeRelease.tabId), + thirdId + ) + assert.equal(await active(three), a) + assert.equal( + await three.evaluate(() => document.getElementById('root')?.inert), + false + ) + await three.evaluate(() => (window as any).runmeVersions.reloadTab()) + await three.waitForFunction( (id) => (window as any).runmeRelease?.active.manifest.releaseId === id, - expected.b, - { timeout: 60000 } + b + ) + await ready(three) + assert.equal( + await active(two), + a, + 'Independent reload must leave its peer alone' + ) + console.log( + 'PASS: frozen peer cannot block activation; old UI saves to new worker; quota failure preserves its tab; independent reload leaves peer alive' ) - await ready(page) } - const b = await active(one) - assert.notEqual(a, b) assert.match( await execute( one, @@ -789,8 +894,34 @@ async function main() { ), /Before upgrade/ ) - await one.screenshot({ path: output + '/03-upgraded-b.png' }) - console.log('PASS: all tabs upgraded and notebook edits retained') + if (incompatible) { + await incompatible.waitForFunction( + (id) => (window as any).runmeRelease?.active.manifest.releaseId === id, + b, + { timeout: 60000 } + ) + await ready(incompatible) + assert.equal( + await incompatible.evaluate(() => + sessionStorage.getItem('incompatible-data-rpc') + ), + null, + 'Incompatible connection cannot send data RPCs before recovery' + ) + await incompatible + .getByText('Preserved during compatibility recovery', { exact: true }) + .first() + .waitFor() + await incompatible.screenshot({ + path: output + '/03-incompatible-tab-recovered.png', + }) + await incompatible.close() + console.log( + 'PASS: confirmed capability mismatch recovers only that tab before any data RPC' + ) + } + await one.screenshot({ path: output + '/03-independent-upgrade-b.png' }) + console.log('PASS: initiating tab upgrades and restores notebook edits') await one.waitForFunction( () => { const id = (window as any).runmeRelease?.active.manifest.releaseId @@ -801,8 +932,39 @@ async function main() { undefined, { timeout: 60000 } ) + let rollbackPeer: Page | undefined + let rollbackPeerId: string | undefined + if (historicalApp) { + rollbackPeer = await context.newPage() + pages.push(rollbackPeer) + // The frozen service worker still serves its historical root loader. + // Explicitly enter the current launcher to test the new peer contract. + await rollbackPeer.goto( + `${base}/version/${expected.b}/launcher.html?session=historical-rollback-peer` + ) + await ready(rollbackPeer) + assert.equal( + await rollbackPeer.evaluate( + () => typeof (window as any).runmeVersions.reloadTab + ), + 'function' + ) + rollbackPeerId = await rollbackPeer.evaluate( + () => (window as any).runmeRelease.tabId + ) + await execute( + rollbackPeer, + `const d=await notebooks.createLocal('rollback-peer.runme');await notebooks.appendCell({target:{uri:d.handle.uri},kind:'markup',value:'Keep this editor alive during historical rollback'});await notebooks.show(d.handle.uri);` + ) + } await context.setOffline(true) - await one.reload() + if (historicalApp) { + // Its historical service worker caches the old root launcher. Restart + // through the cached migration entry so this tab uses the current contract. + await one.goto( + `${base}/version/${expected.b}/launcher.html?session=release-one` + ) + } else await one.reload() await ready(one) assert.equal(await active(one), b) await one.screenshot({ path: output + '/04-offline-restart.png' }) @@ -819,13 +981,15 @@ async function main() { exact: true, }) .click() - for (const page of pages) { - await page.waitForFunction( - (id) => (window as any).runmeRelease?.active.manifest.releaseId === id, - a, - { timeout: 60000 } - ) - await ready(page) + await one.waitForFunction( + (id) => (window as any).runmeRelease?.active.manifest.releaseId === id, + a, + { timeout: 60000 } + ) + await ready(one) + if (!historicalApp) { + assert.equal(await active(two), a) + assert.equal(await active(three), b, 'Rollback must not reload a peer') } assert.match( await execute( @@ -834,26 +998,95 @@ async function main() { ), /Edited on B/ ) + if (rollbackPeer) { + await waitControl( + rollbackPeer, + (s) => !s.transition && s.active?.manifest.releaseId === a + ) + const reloadError = await rollbackPeer.evaluate(async () => { + try { + await (window as any).runmeVersions.reloadTab() + return 'unexpected reload' + } catch (error) { + return String(error) + } + }) + assert.match(reloadError, /cannot restore this tab/) + assert.equal( + await rollbackPeer.evaluate(() => (window as any).runmeRelease.tabId), + rollbackPeerId + ) + assert.equal(await active(rollbackPeer), b) + await rollbackPeer + .getByText('Keep this editor alive during historical rollback', { + exact: true, + }) + .first() + .waitFor() + await rollbackPeer.screenshot({ + path: output + '/05-historical-rollback-peer-retained.png', + }) + await rollbackPeer.close() + console.log( + 'PASS: historical rollback flushes the initiator and refuses to discard a peer into an unsupported checkpoint reader' + ) + } await one.screenshot({ path: output + '/05-offline-rollback.png' }) console.log('PASS: compatible offline rollback keeps edits made on B') const recovery = await context.newPage() pages.push(recovery) - await recovery.goto(base + '/recovery.html') + await recovery.goto( + base + '/recovery.html?session=recovery-return#notebooks' + ) await recovery .getByRole('button', { name: 'Check for updates', exact: true }) .waitFor({ timeout: 30000 }) assert.equal(await recovery.locator('#root').textContent(), '') await recovery.screenshot({ path: output + '/06-independent-recovery.png' }) - console.log( - 'PASS: recovery selector works without importing the application' - ) + // The frozen pre-bootstrap service worker still serves its old root + // recovery page offline. Only the current launcher contains this new UI. + if (!historicalApp) { + // Return explicitly enters the notebook app, preserving this tab's session. + // Leave the independent selector open afterward for broken-release recovery. + const returnLink = recovery.getByRole('link', { + name: 'Return to Runme', + exact: true, + }) + assert.equal( + await returnLink.getAttribute('href'), + '/?session=recovery-return#notebooks' + ) + await returnLink.click() + await ready(recovery) + assert.equal(new URL(recovery.url()).pathname, '/') + assert.equal( + new URL(recovery.url()).searchParams.get('session'), + 'recovery-return' + ) + assert.equal(new URL(recovery.url()).hash, '#notebooks') + await recovery.screenshot({ path: output + '/06-return-to-runme.png' }) + await recovery.goto(base + '/recovery.html') + await recovery + .getByRole('link', { name: 'Return to Runme', exact: true }) + .waitFor() + console.log( + 'PASS: recovery stays independent and explicitly returns to the app with the same session' + ) + } else { + console.log( + 'PASS: frozen historical recovery remains independent of app imports' + ) + } if (process.env.RUNME_TEST_BROKEN) { await context.setOffline(false) await fetch(base + '/test/promote?release=broken', { method: 'POST' }) await one.evaluate(() => (window as any).runmeVersions.check()) await openVersions(one) await one - .getByRole('button', { name: 'Upgrade all tabs', exact: true }) + .getByRole('button', { + name: /^(Activate release and update this tab|Upgrade all tabs)$/, + exact: true, + }) .click() await one.waitForFunction( () => @@ -883,6 +1116,21 @@ async function main() { exact: true, }) .click() + // Clicking starts asynchronous cache verification and handoff. Opening a + // new app immediately can admit it to the still-active broken release and + // make it a participant before recovery intent has been committed. Observe + // completed activation, as shown by recovery status, before opening it. + const recoveredDeadline = Date.now() + 60000 + let recovered = false + while (Date.now() < recoveredDeadline) { + recovered = await recovery.evaluate(async (id) => { + const state = await (window as any).runmeVersions.get() + return !state.transition && state.active?.manifest.releaseId === id + }, a) + if (recovered) break + await new Promise((resolve) => setTimeout(resolve, 100)) + } + assert.ok(recovered, 'Recovery must activate A before opening a new app') const restored = await context.newPage() pages.splice(0, 3, restored) await restored.goto(base + '/?session=release-one') @@ -904,7 +1152,10 @@ async function main() { await restored.evaluate(() => (window as any).runmeVersions.check()) await openVersions(restored) await restored - .getByRole('button', { name: 'Upgrade all tabs', exact: true }) + .getByRole('button', { + name: /^(Activate release and update this tab|Upgrade all tabs)$/, + exact: true, + }) .click() await waitControl( recovery, diff --git a/app/test/browser/reconnect-fixture.ts b/app/test/browser/reconnect-fixture.ts new file mode 100644 index 00000000..7ec48ce0 --- /dev/null +++ b/app/test/browser/reconnect-fixture.ts @@ -0,0 +1,2 @@ +// Let Vite resolve the dependency; do not rely on prebundled cache filenames. +export { create } from '@bufbuild/protobuf' diff --git a/app/test/browser/run-controlled-releases.ts b/app/test/browser/run-controlled-releases.ts index 17f15b2e..89854c83 100644 --- a/app/test/browser/run-controlled-releases.ts +++ b/app/test/browser/run-controlled-releases.ts @@ -130,6 +130,24 @@ async function main() { ) const code = await new Promise((resolve) => test.once('exit', resolve)) if (code !== 0) throw new Error(`Controlled release CUJ failed: ${code}`) + const startup = spawn( + process.execPath, + [ + join( + dirname(fileURLToPath(import.meta.url)), + 'storage-worker-startup.js' + ), + ], + { + env: { ...process.env, RUNME_TEST_URL: 'http://127.0.0.1:5198' }, + stdio: 'inherit', + } + ) + const startupCode = await new Promise((resolve) => + startup.once('exit', resolve) + ) + if (startupCode !== 0) + throw new Error(`Storage startup CUJ failed: ${startupCode}`) } finally { server.kill('SIGTERM') } diff --git a/app/test/browser/run-storage-worker-reconnect.ts b/app/test/browser/run-storage-worker-reconnect.ts new file mode 100644 index 00000000..1231058d --- /dev/null +++ b/app/test/browser/run-storage-worker-reconnect.ts @@ -0,0 +1,57 @@ +import { spawn } from 'node:child_process' +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +/** Run the real app's Vite server and browser test with isolated storage. */ +async function main() { + const output = 'test/browser/test-output/storage-worker-reconnect' + await mkdir(output, { recursive: true }) + const logs: string[] = [] + const server = spawn( + 'pnpm', + ['exec', 'vite', '--host', '127.0.0.1', '--port', '5194', '--strictPort'], + { + env: { ...process.env, CHOKIDAR_USEPOLLING: 'true' }, + stdio: ['ignore', 'pipe', 'pipe'], + } + ) + server.stdout.on('data', (chunk) => logs.push(String(chunk))) + server.stderr.on('data', (chunk) => logs.push(String(chunk))) + try { + for (let attempt = 0; ; attempt++) { + if (server.exitCode !== null) + throw new Error('Vite exited before the test') + if ( + await fetch('http://127.0.0.1:5194/') + .then((r) => r.ok) + .catch(() => false) + ) + break + if (attempt === 99) throw new Error('Vite did not become ready') + await new Promise((resolve) => setTimeout(resolve, 100)) + } + const test = spawn( + process.execPath, + [ + join( + dirname(fileURLToPath(import.meta.url)), + 'storage-worker-reconnect.js' + ), + ], + { + env: { ...process.env, RUNME_TEST_URL: 'http://127.0.0.1:5194' }, + stdio: 'inherit', + } + ) + const code = await new Promise((resolve) => test.once('exit', resolve)) + if (code !== 0) throw new Error(`Storage reconnect test failed: ${code}`) + } finally { + server.kill('SIGTERM') + await writeFile(`${output}/vite.log`, logs.join('')) + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/app/test/browser/storage-worker-reconnect.ts b/app/test/browser/storage-worker-reconnect.ts new file mode 100644 index 00000000..b0f9ba29 --- /dev/null +++ b/app/test/browser/storage-worker-reconnect.ts @@ -0,0 +1,470 @@ +import assert from 'node:assert/strict' +import { mkdir, writeFile } from 'node:fs/promises' +import { type Page, chromium } from 'playwright-core' + +/** Await the actual asynchronous probe result, rather than a truthy Promise. */ +async function waitForOwner(page: Page, previous: string) { + const deadline = Date.now() + 30000 + while (Date.now() < deadline) { + const ready = await page.evaluate(async (old) => { + const w = window as any + if (w.recovery.getSnapshot().phase === 'required') + await w.store.getMetadata(w.uri) + if (w.recovery.getSnapshot().phase !== 'healthy') return false + try { + return (await w.recovery.diagnose()).instanceId !== old + } catch { + return false + } + }, previous) + if (ready) return + await new Promise((resolve) => setTimeout(resolve, 100)) + } + throw new Error( + `Owner did not reconnect: ${JSON.stringify(await page.evaluate(() => (window as any).recovery.getSnapshot()))}` + ) +} + +/** Exercise the actual tab store, SharedWorker and OPFS in a disposable profile. + * Run against Vite development serving this checkout; no live credentials/data. + */ +async function main() { + const browser = await chromium.launch({ + executablePath: process.env.CHROMIUM_PATH, + headless: true, + }) + const output = 'test/browser/test-output/storage-worker-reconnect' + await mkdir(output, { recursive: true }) + const watchdog = setTimeout(() => void browser.close(), 90000) + const logs: string[] = [] + try { + const context = await browser.newContext() + const base = process.env.RUNME_TEST_URL ?? 'http://127.0.0.1:5194' + await context.route('**/reconnect-test', (route) => + route.fulfill({ + contentType: 'text/html', + body: 'Storage reconnect

Disposable storage lifecycle test

', + }) + ) + const pages = [await context.newPage(), await context.newPage()] + async function initialize(page: Page) { + page.on('console', (message) => logs.push(message.text())) + page.on('pageerror', (error) => logs.push(String(error))) + await page.goto(`${base}/reconnect-test`) + await page.evaluate(async () => { + const w = window as any + w.pageIdentity = crypto.randomUUID() + // Exercise a missed lifecycle notification without waiting five minutes + // for the real RPC timeout. All other timer behavior remains unchanged. + const nativeTimeout = window.setTimeout.bind(window) + w.setTimeout = ( + callback: TimerHandler, + delay: number, + ...args: any[] + ) => + nativeTimeout( + callback, + w.fastTimeout && delay === 300000 ? 1000 : delay, + ...args + ) + const NativeChannel = BroadcastChannel + w.BroadcastChannel = class extends NativeChannel { + constructor(name: string) { + super(name) + const descriptor = Object.getOwnPropertyDescriptor( + NativeChannel.prototype, + 'onmessage' + )! + Object.defineProperty(this, 'onmessage', { + set(handler) { + descriptor.set!.call(this, (event: MessageEvent) => { + if ( + w.dropManagement && + name === 'runme-storage-worker-control-v1' + ) { + w.droppedManagement = (w.droppedManagement ?? 0) + 1 + return + } + handler?.(event) + }) + }, + }) + } + } + // Drop only a successful editor-commit reply, after the actual OPFS write. + // The tab must keep the same batch and deduplicate it on the new owner. + const NativeWorker = SharedWorker + w.SharedWorker = class extends NativeWorker { + constructor(...args: ConstructorParameters) { + super(...args) + w.lastWorkerArgs = args + const ids = new Set() + const send = this.port.postMessage.bind(this.port) + this.port.postMessage = (message: any) => { + if (message.method === 'commitEditorOperations') + ids.add(message.id) + send(message) + } + const descriptor = Object.getOwnPropertyDescriptor( + MessagePort.prototype, + 'onmessage' + )! + Object.defineProperty(this.port, 'onmessage', { + set(handler) { + descriptor.set!.call(this, (event: MessageEvent) => { + if ( + w.dropReply && + event.data.type === 'result' && + !event.data.error && + ids.has(event.data.id) + ) { + w.lostReply = true + return + } + handler?.(event) + }) + }, + }) + } + } + const clientPath = '/src/storage/storageOwnerClient.ts' + const drivePath = '/src/storage/drive.ts' + const protoPath = '/src/runme/client.ts' + const controlPath = '/src/storage/storageRecovery.ts' + const protobufPath = '/test/browser/reconnect-fixture.ts' + w.createProto = (await import(protobufPath)).create + const { createSharedNotebookStore } = await import(clientPath) + const { DriveNotebookStore } = await import(drivePath) + w.proto = (await import(protoPath)).parser_pb + w.recovery = (await import(controlPath)).storageRecovery + w.recovery.subscribe(() => + console.log( + 'RECOVERY', + w.pageIdentity, + JSON.stringify(w.recovery.getSnapshot()) + ) + ) + const watch = new BroadcastChannel('runme-storage-worker-control-v1') + watch.onmessage = ({ data }) => + console.log('MANAGEMENT', w.pageIdentity, JSON.stringify(data)) + w.store = createSharedNotebookStore( + new DriveNotebookStore(async () => { + throw new Error('No test credentials') + }) + ) + await w.store.ready + w.passive = new NativeWorker( + ...(w.lastWorkerArgs as ConstructorParameters) + ) + w.passive.port.start() + }) + } + for (const page of pages) await initialize(page) + const identities = await Promise.all( + pages.map((page) => page.evaluate(() => (window as any).pageIdentity)) + ) + const original = await pages[0].evaluate(async () => { + const w = window as any + const file = await w.store.create('local://folder/local', 'restart.runme') + w.uri = file.uri + w.view = await w.store.createOperationLogSaveStore(file.uri) + w.editor = w.view.initialNotebook + w.editor.cells.push( + w.createProto(w.proto.CellSchema, { + refId: 'first', + kind: w.proto.CellKind.MARKUP, + value: 'Original', + }) + ) + await w.view.save(file.uri, w.editor) + return { + uri: file.uri, + instance: (await w.recovery.diagnose()).instanceId, + } + }) + await pages[1].evaluate(async (uri) => { + const w = window as any + w.uri = uri + w.view = await w.store.createOperationLogSaveStore(uri) + w.editor = w.view.initialNotebook + }, original.uri) + await pages[0].evaluate(() => { + const w = window as any + w.dropReply = true + w.editor.cells[0].value = 'Committed before worker stop' + w.saveResult = w.view.save(w.uri, w.editor).catch((error: Error) => { + w.saveError = String(error) + }) + }) + await pages[0].waitForFunction(() => (window as any).lostReply, undefined, { + timeout: 20000, + }) + await pages[1].evaluate(async () => { + const w = window as any + w.editor.cells.push( + w.createProto(w.proto.CellSchema, { + refId: 'other', + kind: w.proto.CellKind.MARKUP, + value: 'Other tab content', + }) + ) + await w.view.save(w.uri, w.editor) + }) + const before = await pages[1].evaluate(() => + (window as any).store.loadContent((window as any).uri) + ) + await pages[0].evaluate(() => { + const w = window as any + w.dropReply = false + w.recovery.restart(true) + }) + for (const page of pages) await waitForOwner(page, original.instance) + assert.deepEqual( + await Promise.all( + pages.map((page) => page.evaluate(() => (window as any).pageIdentity)) + ), + identities, + 'Neither page may reload' + ) + assert.equal( + await pages[0].evaluate(() => + (window as any).store.loadContent((window as any).uri) + ), + before, + 'Retry must not duplicate the committed batch' + ) + await pages[0].evaluate(async () => { + const w = window as any + w.editor.cells[0].value = 'Edited after reconnect' + await w.view.save(w.uri, w.editor) + }) + const cells = await pages[1].evaluate(async () => + ( + await (window as any).store.loadOperationLogSnapshot( + (window as any).uri + ) + ).cells.map((cell: any) => cell.value) + ) + console.log('Force restart completed') + assert.deepEqual(cells, ['Edited after reconnect', 'Other tab content']) + const forceInstance = await pages[0].evaluate( + async () => (await (window as any).recovery.diagnose()).instanceId + ) + await pages[0].evaluate(() => (window as any).recovery.restart()) + for (const page of pages) await waitForOwner(page, forceInstance) + assert.deepEqual( + await Promise.all( + pages.map((page) => page.evaluate(() => (window as any).pageIdentity)) + ), + identities + ) + console.log('Graceful restart completed') + // Idle tabs do not need immediate reconnection. Each can join on its next + // request without waiting for, or sending readiness messages to, another tab. + const beforeStop = await pages[0].evaluate( + async () => (await (window as any).recovery.diagnose()).instanceId + ) + await pages[0].evaluate(async () => { + const path = '/src/storage/storageWorkerControl.ts' + const { StorageWorkerControl } = await import(path) + await new Promise((resolve) => { + const control = new StorageWorkerControl((event: any) => { + if (event.type !== 'status') return + control.stop(event.status.instanceId, { force: true, restart: false }) + control.close() + resolve() + }) + control.discover() + }) + }) + for (const page of pages) + await page.waitForFunction( + () => (window as any).recovery.getSnapshot().phase === 'required' + ) + await pages[1].evaluate(async () => { + const w = window as any + w.editor.cells[1].value = 'Saved by idle tab after stop' + await w.view.save(w.uri, w.editor) + }) + for (const page of pages) await waitForOwner(page, beforeStop) + assert.deepEqual( + await Promise.all( + pages.map((page) => page.evaluate(() => (window as any).pageIdentity)) + ), + identities + ) + const finalCells = await pages[0].evaluate(async () => + ( + await (window as any).store.loadOperationLogSnapshot( + (window as any).uri + ) + ).cells.map((cell: any) => cell.value) + ) + assert.deepEqual(finalCells, [ + 'Edited after reconnect', + 'Saved by idle tab after stop', + ]) + const ownerIds = await Promise.all( + pages.map((page) => + page.evaluate( + async () => (await (window as any).recovery.diagnose()).instanceId + ) + ) + ) + assert.equal( + ownerIds[0], + ownerIds[1], + 'Independent retries join the same worker' + ) + await pages[1].evaluate(() => { + const w = window as any + w.dropManagement = true + w.fastTimeout = true + }) + await pages[0].evaluate(async () => { + const path = '/src/storage/storageWorkerControl.ts' + const { StorageWorkerControl } = await import(path) + await new Promise((resolve) => { + const control = new StorageWorkerControl((event: any) => { + if (event.type !== 'status') return + control.stop(event.status.instanceId, { force: true, restart: false }) + control.close() + resolve() + }) + control.discover() + }) + }) + await pages[0].waitForFunction( + () => (window as any).recovery.getSnapshot().phase === 'required' + ) + const missedNotification = await pages[1].evaluate(async () => { + const w = window as any + w.editor.cells[1].value = 'Saved after missing the stop notification' + let timedOut = false + try { + await w.view.save(w.uri, w.editor) + } catch (error) { + timedOut = String(error).includes('did not acknowledge an editor save') + } + if (!timedOut) throw new Error('Stale port should time out before retry') + await w.view.save(w.uri, w.editor) + return { + timedOut, + droppedManagement: w.droppedManagement, + instanceId: (await w.recovery.diagnose()).instanceId, + cells: (await w.store.loadOperationLogSnapshot(w.uri)).cells.map( + (cell: any) => cell.value + ), + } + }) + assert.ok(missedNotification.droppedManagement > 0) + assert.notEqual(missedNotification.instanceId, ownerIds[1]) + assert.deepEqual(missedNotification.cells, [ + 'Edited after reconnect', + 'Saved after missing the stop notification', + ]) + assert.deepEqual( + await Promise.all( + pages.map((page) => page.evaluate(() => (window as any).pageIdentity)) + ), + identities + ) + // An uncertain committed save plus subsequent typing survives a real page + // reload. Capture and replay use the production adapter and browser storage. + await pages[0].evaluate(() => { + const w = window as any + w.lostReply = false + w.dropReply = true + w.editor.cells[0].value = 'Committed before local reload' + void w.view.save(w.uri, w.editor).catch(() => {}) + }) + await pages[0].waitForFunction(() => (window as any).lostReply) + const pendingBefore = await pages[1].evaluate(() => + (window as any).store.loadContent((window as any).uri) + ) + await pages[0].evaluate(async () => { + const w = window as any + w.editor.cells[0].value = 'Typed after lost acknowledgement' + const path = '/src/storage/tabRecoveryCheckpoint.ts' + const { checkpointEditorSessions } = await import(path) + await checkpointEditorSessions(new Map([[w.uri, w.editor]])) + }) + await initialize(pages[0]) + const recovered = await pages[0].evaluate(async (uri) => { + const w = window as any + w.uri = uri + w.view = await w.store.createOperationLogSaveStore(uri) + w.editor = w.view.initialNotebook + const restored = w.editor.cells[0].value + await w.view.save(uri, w.editor) + return { + restored, + document: await w.store.loadContent(uri), + cells: (await w.store.loadOperationLogSnapshot(uri)).cells.map( + (c: any) => c.value + ), + } + }, original.uri) + assert.equal(recovered.restored, 'Typed after lost acknowledgement') + assert.deepEqual(recovered.cells, [ + 'Typed after lost acknowledgement', + 'Saved after missing the stop notification', + ]) + const previousIds = pendingBefore + .trim() + .split('\n') + .slice(1) + .map((line: string) => JSON.parse(line).op_id) + const replayedIds = recovered.document + .trim() + .split('\n') + .slice(1) + .map((line: string) => JSON.parse(line).op_id) + for (const id of previousIds) + assert.equal( + replayedIds.filter((value: string) => value === id).length, + 1 + ) + assert.equal( + await pages[1].evaluate(() => (window as any).pageIdentity), + identities[1] + ) + await pages[0].waitForFunction( + () => !sessionStorage.getItem('runme/tab-editor-recovery/v1') + ) + console.log( + 'PASS: local page reload restores later typing, replays exact pending saves once, and preserves concurrent peer edits' + ) + await writeFile( + `${output}/result.json`, + JSON.stringify( + { + passed: true, + pagesPreserved: identities, + cells, + lostReplyDeduplicated: true, + forceRestart: true, + gracefulRestart: true, + independentRetryAfterStop: true, + finalCells, + ownerIds, + missedNotification, + localReloadRecovery: recovered, + }, + null, + 2 + ) + ) + console.log( + 'PASS: force and graceful restarts preserve both pages; lost save reply is deduplicated; concurrent edits survive' + ) + } finally { + clearTimeout(watchdog) + await writeFile(`${output}/browser.log`, logs.join('\n')) + await browser.close() + } +} +main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/app/test/browser/storage-worker-startup.ts b/app/test/browser/storage-worker-startup.ts new file mode 100644 index 00000000..a80fe231 --- /dev/null +++ b/app/test/browser/storage-worker-startup.ts @@ -0,0 +1,269 @@ +import assert from 'node:assert/strict' +import { mkdir, writeFile } from 'node:fs/promises' +import { type Page, chromium } from 'playwright-core' + +/** Poll an asynchronous browser observation with an actual awaited result. */ +async function until( + page: Page, + predicate: () => Promise, + label: string +) { + const deadline = Date.now() + 20000 + while (Date.now() < deadline) { + if (await page.evaluate(predicate)) return + await new Promise((resolve) => setTimeout(resolve, 100)) + } + throw new Error(`Timed out: ${label}`) +} + +/** Hold a real IndexedDB transaction ahead of the emitted worker's settings read. + * This exercises Chromium scheduling rather than mocking a request callback. + */ +async function main() { + const base = process.env.RUNME_TEST_URL ?? 'http://127.0.0.1:5198' + await fetch(`${base}/test/promote?release=b`, { method: 'POST' }) + const browser = await chromium.launch({ + executablePath: process.env.CHROMIUM_PATH, + headless: true, + }) + const output = 'test/browser/test-output/storage-worker-startup' + await mkdir(output, { recursive: true }) + const events: unknown[] = [] + const watchdog = setTimeout(() => void browser.close(), 90000) + let contextForDiagnostics: + | import('playwright-core').BrowserContext + | undefined + try { + const context = await browser.newContext() + contextForDiagnostics = context + const page = await context.newPage() + page.on('console', (message) => events.push(message.text())) + await page.goto(`${base}/recovery.html`) + await until( + page, + async () => Boolean((await (window as any).runmeVersions?.get())?.active), + 'initial owner activation' + ) + const identity = await page.evaluate(async () => { + const w = window as any + w.active = (await w.runmeVersions.get()).active + w.pageIdentity = crypto.randomUUID() + w.events = [] + w.management = new BroadcastChannel('runme-storage-worker-control-v1') + w.management.onmessage = ({ data }: MessageEvent) => { + if (!data.status) return + w.events.push(data) + w.workerStatus = data.status + } + w.startOwner = () => { + const { manifest, epoch } = w.active + w.worker = new SharedWorker( + `/version/${manifest.releaseId}/${manifest.owner.entry}`, + { + type: 'module', + name: `runme-storage-owner-${manifest.releaseId}-${epoch}`, + } + ) + w.worker.port.start() + } + return w.pageIdentity + }) + await until( + page, + async () => { + const w = window as any + w.management.postMessage({ type: 'discover' }) + return w.workerStatus?.phase === 'ready' + }, + 'ready owner diagnostics' + ) + await page.evaluate(() => { + const w = window as any + w.management.postMessage({ + type: 'stop', + instanceId: w.workerStatus.instanceId, + force: true, + restart: false, + }) + }) + await until( + page, + async () => + navigator.locks.request( + 'runme-storage-owner-lifetime-v1', + { ifAvailable: true }, + (lock) => Boolean(lock) + ), + 'old owner exit' + ) + await page.evaluate(async () => { + const w = window as any + const db = await new Promise((resolve, reject) => { + const request = indexedDB.open('runme-release-control') + request.onsuccess = () => resolve(request.result) + request.onerror = () => reject(request.error) + }) + const tx = db.transaction('state', 'readwrite') + const table = tx.objectStore('state') + // Keep this transaction active without changing any settings or notebook. + const keepAlive = () => { + table.get('control').onsuccess = keepAlive + } + keepAlive() + w.releaseDatabase = () => { + tx.abort() + db.close() + } + w.workerStatus = undefined + w.startOwner() + }) + await until( + page, + async () => { + const w = window as any + w.management.postMessage({ type: 'discover' }) + return ( + w.workerStatus?.phase === 'reading-release-settings' && + w.workerStatus.releaseDatabase?.active?.some( + (trace: any) => trace.phase === 'get-requested' + ) + ) + }, + 'queued settings get visible before runtime initialization' + ) + const queued = await page.evaluate(() => (window as any).workerStatus) + await page.evaluate(() => { + const w = window as any + w.management.postMessage({ + type: 'stop', + instanceId: w.workerStatus.instanceId, + force: false, + restart: false, + }) + }) + await until( + page, + async () => { + const w = window as any + w.management.postMessage({ type: 'discover' }) + return w.workerStatus?.phase === 'draining' + }, + 'graceful drain still observable' + ) + await page.evaluate(() => { + const w = window as any + w.management.postMessage({ + type: 'stop', + instanceId: w.workerStatus.instanceId, + force: true, + restart: false, + }) + }) + await until( + page, + async () => + navigator.locks.request( + 'runme-storage-owner-lifetime-v1', + { ifAvailable: true }, + (lock) => Boolean(lock) + ), + 'force close bypasses stalled initialization' + ) + await page.evaluate(() => { + const w = window as any + w.workerStatus = undefined + w.startOwner() + }) + await until( + page, + async () => { + const w = window as any + w.management.postMessage({ type: 'discover' }) + return ( + w.workerStatus?.phase === 'failed' && + w.workerStatus.releaseDatabase?.failures?.length > 0 + ) + }, + 'real queued get timeout reported' + ) + const timedOut = await page.evaluate(() => (window as any).workerStatus) + await page.evaluate(() => { + const w = window as any + w.management.postMessage({ + type: 'stop', + instanceId: w.workerStatus.instanceId, + force: true, + restart: false, + }) + }) + await until( + page, + async () => + navigator.locks.request( + 'runme-storage-owner-lifetime-v1', + { ifAvailable: true }, + (lock) => Boolean(lock) + ), + 'failed owner exit' + ) + await page.evaluate(() => { + const w = window as any + w.releaseDatabase() + w.workerStatus = undefined + w.startOwner() + }) + await until( + page, + async () => { + const w = window as any + w.management.postMessage({ type: 'discover' }) + return w.workerStatus?.phase === 'ready' + }, + 'replacement initializes after the blocking transaction ends' + ) + assert.equal( + await page.evaluate(() => (window as any).pageIdentity), + identity + ) + await writeFile( + `${output}/result.json`, + JSON.stringify( + { + passed: true, + queued, + timedOut, + replacement: await page.evaluate(() => (window as any).workerStatus), + }, + null, + 2 + ) + ) + console.log( + 'PASS: real IndexedDB get stall is observable; force close bypasses a stalled drain; replacement starts without page reload' + ) + } finally { + const diagnostics = await Promise.all( + contextForDiagnostics + ?.pages() + .map(async (page) => + page + .evaluate(() => ({ + status: (window as any).workerStatus, + events: (window as any).events, + })) + .catch((error) => ({ error: String(error) })) + ) ?? [] + ) + await writeFile( + `${output}/diagnostics.json`, + JSON.stringify(diagnostics, null, 2) + ) + clearTimeout(watchdog) + await writeFile(`${output}/browser.log`, events.join('\n')) + await browser.close() + } +} +void main().catch((error) => { + console.error(error) + process.exitCode = 1 +}) diff --git a/app/vite.config.mts b/app/vite.config.mts index b4e4c7e8..372e2cfc 100644 --- a/app/vite.config.mts +++ b/app/vite.config.mts @@ -198,6 +198,7 @@ function pwaServiceWorkerPlugin(): Plugin { releaseId, appEntry: entry.fileName, styles: [...(entry.viteMetadata?.importedCss ?? [])], recoveredControlFormat: 1, + tabRecoveryFormat: 1, requiredCapabilities: ['durable-prepare-v1', 'epoch-fence-v1', 'owner-lock-v1'], testedRollbackTargets: [], owner: { buildId: releaseId.split('-')[1], entry: 'storage-owner.js', rpcMin: 8, rpcMax: 8 }, launcherProtocol: { min: 1, max: 1 }, diff --git a/docs-dev/CUJs/controlled-app-releases.md b/docs-dev/CUJs/controlled-app-releases.md index 851f26ca..bdaffbfa 100644 --- a/docs-dev/CUJs/controlled-app-releases.md +++ b/docs-dev/CUJs/controlled-app-releases.md @@ -21,7 +21,10 @@ Acceptance: - The floating launcher disappears after startup only when the loaded app explicitly registers its version controls. Older clients retain the fallback controls. With a current app, the left Information icon opens version controls - inside the Version Information document. + inside the Version Information document. A badge and descriptive tooltip on the + Information icon signal a downloaded update, an outdated tab, or a version + problem. Apps supporting this affordance keep the overlay hidden during + updates and handoffs; actions and status remain in Version Information. - Startup diagnostics expose timestamped stage, duration, and waiting/failure events before the app starts. The trace is bounded, stays in memory, and does not include notebook data, credentials, or URL query parameters. @@ -53,15 +56,19 @@ Acceptance: through the real service worker. It must answer before importing the launcher, then open the selected app when discovery resumes. Slow discovery must never misclassify a controlled bootstrap as an incompatible legacy window. -- Channel changes and exact pins only stage; activation requires Upgrade all tabs. -- Pending notebook edits in multiple tabs persist before upgrade readiness. -- A failed preparation blocks activation and leaves that tab editable. -- Cancel before handoff: prepared tabs reopen their durable A state. -- Freeze a tab and explicitly withhold its readiness acknowledgement: it retains membership. A new tab waits outside the app. -- Close the initiating tab: another launcher resumes the persisted transition. -- All admitted tabs move to B under a new storage-owner epoch. -- After full offline readiness, offline restart uses B; offline rollback to A preserves edits made on B. -- The recovery selector works independently of React/application imports. +- Channel changes and exact pins only stage; activation requires an explicit action. +- Activating B replaces the worker under its lifetime lock and reloads only the initiating tab. +- A frozen peer with a failing preparation hook cannot block activation and is never asked to prepare. +- The old peer keeps its tab identity and mounted editor, then saves through worker B on retry. +- Reloading another tab is an independent action. A session-storage quota error leaves that tab editable and does not roll back the selected worker. +- A successful local reload checkpoints and restores editor state without involving a peer. +- A confirmed capability mismatch triggers local checkpoint/reload recovery before any data RPC. Timeouts alone never do. +- Historical rollback requires a durable flush of the initiating tab. Current peers refuse checkpoint reload into an app without `tabRecoveryFormat: 1`, retaining their mounted editors. +- Historical launchers retain their original behavior. The frozen-baseline migration closes durable legacy peers before activation; the initiating historical app flushes locally before worker replacement. +- After full offline readiness, offline restart uses B; offline rollback to A preserves edits made on B and leaves other tabs on their loaded UI versions. +- The current recovery selector works independently of React/application imports, explains + that notebooks open in the app, and keeps **Return to Runme** visible. Returning + opens the app in the same tab with its session and hash preserved. - A checksum-valid but throwing app entry can be recovered through the selector after failed app tabs close, retaining current notebook content. @@ -85,3 +92,42 @@ Launcher selection tests cover bounded network/cache waits, incompatible descriptors, offline fallback, corrupted modules, interrupted publication, and reuse of verified modules on refresh. Releaser tests verify that the launcher advances even when the stable application channel is pinned. + +## Recovery while the app is already open + +- Inject a rejected or never-settling `runmeVersions.get()` after opening Version + Information. The **Pin Release** button must remain available and open a + separate workspace tab. Users choose the installed release or select another + build by browsing or entering a commit. Reviewing a selection has no side + effects; pinning and activation are separate actions. Recovery navigation and + memory-only diagnostics remain reachable under **Troubleshooting**, even + while settings or a download are stalled. Also cover an + older launcher without the new recovery API: the selected release is carried + to the independent recovery page without navigating the editor tab away. +- Preparing a build downloads and verifies it without accessing release settings. + It must not activate the build. Explicit activation still uses the existing + tab preparation, compatibility checks and exclusive storage-owner handoff. +- A commit prefix with several published builds must require selecting a full + release ID rather than guessing a build. An unpublished commit is not runnable. +- Diagnostic traces distinguish missing initial get callbacks, incomplete chunk + reads and pending transaction completion. Include elapsed time, realm and mode; + exclude record values, notebook contents, credentials and arbitrary keys. +- These failure-injection checks are automated in `ReleaseRecoveryControls.test.tsx`, + `releaseCatalog.test.ts`, `releaseDatabase.test.ts` and `startupRecovery.test.ts`. + +## Unresponsive storage owner + +- The recovery banner's **Check storage worker** action works while restart or + normal RPC admission is waiting. A diagnostic reply must not replay a mutation. +- Report authorization before initialization, initialization, authorization after + initialization, and execution separately. Restart diagnostics identify release + settings, waiting for tab backups, waiting for active requests and draining. +- Keep diagnostic snapshots bounded and memory-only, including release database + traces from the worker realm. A diagnostic reply proves message-loop liveness, + not persistence health; absence of a reply does not prove the worker is dead. +- Never steal the ownership lock or launch a competing writer after a timeout. + Explain that users must download current editor copies in every tab before + closing tabs or restarting the browser; session copies alone are insufficient. +- Automated coverage: `storageOwner.test.ts`, `storageRecovery.test.ts`, and + `StorageRecoveryBanner.test.tsx`. Existing browser restart and upgrade journeys + remain in `storage-owner-recovery.ts` and `controlled-releases.ts`. diff --git a/docs-dev/CUJs/drive-sync-recovery.md b/docs-dev/CUJs/drive-sync-recovery.md index dd4e8d1c..a7fafefb 100644 --- a/docs-dev/CUJs/drive-sync-recovery.md +++ b/docs-dev/CUJs/drive-sync-recovery.md @@ -351,26 +351,38 @@ all notebooks copied before a hung first flush, cancellation after worker loss, downloads with unavailable session storage, and bounded diagnostic probes outside storage initialization. A timeout never retries the uncertain operation. -On an isolated test origin, reproduce a storage protocol mismatch and verify -**Restart Runme** appears above the notebook and inside Google Drive Sync Status. -With two tabs open, make an edit, click the button, and verify both tabs reload -without closing the browser, notebook storage works again, and recovery JSON -contains the editor content. Block one tab's acknowledgement: neither tab should -reload. Block a storage/network write: restart must wait for the write to settle. -Repeat against an older worker without the recovery protocol: show the actionable -fallback and retain editor content rather than silently reloading in a loop. -Verify the initiating tab has session copies even without a worker response. Fill -session storage, click **Get recovery copies**, and download the current editors; -verify exact cell text in the JSON. The banner must distinguish earlier snapshots -from current captures, and explain that session copies do not survive tab closure. -Perform these destructive failure injections only in an isolated test profile. - -The emitted-worker/browser check is `test/browser/storage-owner-recovery.ts`. -Against a production preview on port 5194, compile it with the same `tsc` flags -above and run `node test/browser/.generated/storage-owner-recovery.js`. It asserts -that two pages reload, the old SharedWorker target terminates, recovery copies -survive, durable notebook bytes are unchanged, and a new owner accepts requests. -A frontend worker fixture then verifies the actual notebook and status-page -buttons and legacy fallback. Screenshots and JSON results are written under -`test/browser/test-output/storage-*recovery*`. Use an isolated Chromium for Testing -binary via `CHROMIUM_PATH`; the test never uses an existing browser profile. +On an isolated test origin, verify **Restart storage** and **Force restart +storage** appear above the notebook when storage fails. Open the Version +Information page and verify worker discovery and stop controls remain available +while release settings fail. Discovery must label nonresponders as unknown. + +With two tabs open, save a notebook and discard its successful worker reply. +Let the other tab save an additional cell. Force restart storage: both pages and +editor objects must remain open, the worker instance must change, the retained +batch must be acknowledged without changing persisted bytes, and a subsequent +edit must preserve the other tab's cell. Repeat with graceful restart. Neither +path may call page reload. Block a drain and verify diagnostics and force restart +still work. Block release settings during startup and inspect the last reached +phase. App upgrade/reload continues to require its separate preservation protocol. + +Run `pnpm -C app run cuj:storage` for `storage-worker-reconnect.ts`. Its driver +starts the real Vite app on localhost:5194 and uses disposable Chromium storage. +Set `CHROMIUM_PATH` when using a Chromium for Testing binary instead of Playwright's +installed browser. Evidence goes to `test/browser/test-output/storage-worker-reconnect`. +The test also stops the worker without requesting a restart, then verifies that +each tab independently reconnects on its next save and joins the same replacement. +It then drops management notifications for one tab, lets a save on its stale +port time out, and verifies that retrying replaces the connection and commits +the retained batch without losing either tab's edits. + +`pnpm -C app run cuj:releases` also runs `storage-worker-startup.ts` against emitted +production assets and the Go release fixture server. It holds a real IndexedDB +transaction ahead of the worker's settings read. It verifies the pending +`get-requested` trace, force-close during a stalled graceful drain, a timeout trace, +and successful replacement after releasing the transaction without reloading the +page. Evidence goes to `test/browser/test-output/storage-worker-startup`. + +`storage-owner-recovery.ts` retains coverage of the older coordinated reload wire +protocol for compatibility. It is not evidence for the new in-place restart path. +Current editor downloads remain available independently of session storage and +worker health. Never inject failures into a live user's browser profile. diff --git a/docs-dev/design/20261027_storage_lifecycle.md b/docs-dev/design/20261027_storage_lifecycle.md new file mode 100644 index 00000000..d6f2535e --- /dev/null +++ b/docs-dev/design/20261027_storage_lifecycle.md @@ -0,0 +1,108 @@ +# Storage lifecycle and recovery + +## Decision + +Tabs upgrade independently. Activating a published release replaces the shared +storage worker without waiting for any tab's readiness or acknowledgement. The +initiating tab then preserves its own state and reloads. Other tabs remain mounted +and editable, with an explicit control to update that tab when the user is ready. +A frozen or disconnected peer cannot block worker activation. + +The selected worker and each tab's loaded application have separate identities. +Every reconnect resolves the selected worker, then negotiates a stable handshake +before sending data RPCs. Compatible old tabs keep working. An incompatible tab +attempts local checkpoint recovery once for that selected worker epoch; if preservation fails it keeps its editors and offers a retry; a timeout alone never proves a +version mismatch or authorizes automatic reload. + +Local recovery freezes only that tab's edits and disconnects its transport. It +records a versioned session-storage checkpoint containing each editor's current +contents, observed operation history, acknowledged baseline, actor ID, and exact +pending operations. It verifies the written checkpoint before reloading. The new +app restores the captured editor, retries identical operation IDs, then persists +later edits relative to the captured baseline. It removes a checkpoint only after +those saves are acknowledged. Quota, malformed data, unsupported formats, active +executions, and uncertain non-editor operations keep the tab open. Checkpoints +survive reload, not closing the tab; recovery downloads remain available. + +The worker's lifetime Web Lock remains the exclusive storage ownership authority. +The old owner stops admission and drains admitted work before releasing the lock. +Force-close is separately available through its management channel, provided the +worker event loop responds. It never asks tabs to reload. Generic mutations are +never replayed just because transport failed. + +The launcher retains its independent update path and settings-failure recovery. +Release discovery and download remain independent of notebook storage. Startup +phase, queued settings reads, elapsed time, and RPC diagnostics are observable +without issuing another database read. No new tab-readiness coordinator is added. + +### Compatibility boundary + +The independent lifecycle applies to tabs running this launcher and recovery +contract. Already-running historical launcher code can still execute its old +reload policy; code in another tab cannot rewrite it. Old data-RPC versions are +rejected before effects, and unknown checkpoint formats are retained for recovery. +Legacy JSON/Markdown editors without replayable operation IDs require a durable local flush; if storage cannot acknowledge it, that tab stays open for export/retry. +Published manifests advertise `tabRecoveryFormat: 1`. Activation of a historical +release without that contract first durably flushes the initiating tab to its +current worker. Other current tabs refuse checkpoint-based reload into that +historical app; their editors remain open for export or selection of a supported +release. Compatibility of the notebook schema alone does not imply compatibility +of the recovery envelope. +Data-schema compatibility remains an activation prerequisite; independent tab +upgrades do not authorize concurrent incompatible storage formats. + +## Failure analysis + +Recent releases added editor snapshots and diagnostics (#417), immutable app +releases and multi-tab handoff (#418), version UI and partial offline readiness +(#419), loading diagnostics/UI handoff (#421), bounded release-settings access +(#423), unreadable-record quarantine (#425), and an independently updated launcher +(#428). #430 fixed the publication runner. These changes each address a real +failure, but recovery became dependent on the release database it was recovering. + +The previous restart handler writes `emergencyRestart` before requesting editor +snapshots. If the release database stalls, no other tab is asked to preserve its +editors. This is a shared failure dependency, not proof of an IndexedDB lock cycle. +The editor preparation routine also awaits persistence through the failing worker. +That defeats the purpose of its independent session-storage snapshot. + +A second bug treats MessagePorts as tabs. The launcher holds one control port and +the notebook app holds another editor port to the same worker. The launcher cannot +acknowledge editor preparation. Counting its control port as a participant can +block every restart. Diagnostics must report connections and editor clients +separately. A control connection is not evidence of unsaved editors. + +## Invariants + +- Only the worker holding `runme-storage-owner-lifetime-v1` opens notebook stores. +- Worker activation has no tab-readiness barrier and no peer reload command. +- A tab reloads for its own action, its own activation, or locally confirmed RPC incompatibility. Timeouts never trigger automatic reload. +- A reload checkpoint is verified before the page discards JavaScript memory. +- Save retries retain operation IDs and content; lost replies cannot duplicate edits. +- A restored baseline is the tab's captured baseline, not latest merged storage. +- RPC compatibility is checked before notebook or external effects. +- App and worker identities are independent; request fencing uses the selected worker epoch. +- Session-storage failure keeps the tab and its unsaved edits alive. +- Unknown mutation outcomes and unsupported recovery formats are never silently discarded. + +## Browser boundary + +Web pages cannot enumerate all browser SharedWorkers or externally terminate them. +Runme can list workers that implement its own discovery protocol, and request a +cooperative stop or force-close from those workers. A nonresponse is unknown status, +not proof of death. Browser DevTools can enumerate and terminate workers with +privileges unavailable to the page. The UI must state this limitation explicitly. + +A dedicated storage worker with a supervisor could support external termination, +but introduces supervisor election, port forwarding and a second lifecycle. Chrome +also differs from the current HTML specification on Worker construction inside a +SharedWorker. Do not add this architecture as an untested recovery patch. + +## Verification required + +Real-browser tests must cover launcher plus editor ports, multiple editor tabs, +pending release-settings reads, pending persistence, new clients during recovery, +failed snapshots, worker initialization failure, and replacement identity under +the lifetime lock. Verify OPFS notebook contents after restart, and recovery copies +when an edit had not committed. Existing upgrade/offline/rollback journeys must +still pass. Unit tests alone do not establish lifecycle correctness. diff --git a/releaser/README.md b/releaser/README.md index ea6d4a3f..2f76d4a4 100644 --- a/releaser/README.md +++ b/releaser/README.md @@ -191,20 +191,20 @@ Future schema changes require a new compatibility contract, resumable migration, and real old/new artifact tests before eligibility changes. `testedRollbackTargets` records individually qualified historical releases; an empty list makes no historical qualification claim. Previous locally healthy releases with the same -storage contract remain usable through the coordinated selector. +storage contract remain usable through the release selector. The root loader has its own service-worker lifecycle and cache. It never calls `skipWaiting`: existing pre-contract tabs must close before initial installation. A legacy-worker retirement handshake blocks versioned owners until the old owner is gone. This one-time transition may require saving recovery copies and closing -all Runme tabs. Application upgrades afterwards use the durable coordinator. +all Runme tabs. Application upgrades afterwards replace the worker under its lifetime lock; each tab preserves and reloads independently. ### Release acceptance gate Run `pnpm -C app run cuj:releases` after building workspace packages and installing Playwright Chromium. The driver creates independent immutable A/B bundles and a third bundle with a deliberately broken application entry, serves them with the -Go fixture, and verifies coordination, offline restart, current-data rollback and +Go fixture, and verifies independent tab upgrades, offline restart, current-data rollback and independent recovery. Screenshots, recordings and machine-readable results go to `app/test/browser/test-output/controlled-releases/`. CI runs this gate on every PR and main update. Do not promote a release when this gate fails. diff --git a/releaser/channels.go b/releaser/channels.go index 04fc6f2b..079602aa 100644 --- a/releaser/channels.go +++ b/releaser/channels.go @@ -18,6 +18,7 @@ import ( // Runtime manifests are separate from archives: only lifecycle-aware builds can activate. type runtimeManifest struct { + TabRecoveryFormat int `json:"tabRecoveryFormat,omitempty"` RecoveredControlFormat int `json:"recoveredControlFormat,omitempty"` Format int `json:"format"` ReleaseID string `json:"releaseId"` diff --git a/releaser/channels_test.go b/releaser/channels_test.go index 00251510..006cab4e 100644 --- a/releaser/channels_test.go +++ b/releaser/channels_test.go @@ -57,7 +57,7 @@ func TestRuntimePublicationAndStablePin(t *testing.T) { dist := desiredBuildFixture(t, inputs) buildID, _ := inputs.id() id := inputs.WebCommit + "-" + buildID - metadata := map[string]any{"recoveredControlFormat": 1, "releaseId": id, "appEntry": "app.js", "styles": []string{}, "requiredCapabilities": []string{"durable-prepare-v1", "epoch-fence-v1", "owner-lock-v1"}, "owner": map[string]any{"buildId": strings.Split(id, "-")[1], "entry": "storage-owner.js", "rpcMin": 8, "rpcMax": 8}, "launcherProtocol": map[string]any{"min": 1, "max": 1}, "storageCompatibilityId": "runme-schema9-oplog2-payload1-console1"} + metadata := map[string]any{"recoveredControlFormat": 1, "tabRecoveryFormat": 1, "releaseId": id, "appEntry": "app.js", "styles": []string{}, "requiredCapabilities": []string{"durable-prepare-v1", "epoch-fence-v1", "owner-lock-v1"}, "owner": map[string]any{"buildId": strings.Split(id, "-")[1], "entry": "storage-owner.js", "rpcMin": 8, "rpcMax": 8}, "launcherProtocol": map[string]any{"min": 1, "max": 1}, "storageCompatibilityId": "runme-schema9-oplog2-payload1-console1"} receipt, _ := json.Marshal(releaseAcceptance{1, id, "controlled-releases-v1"}) writeArchiveFixture(t, dist, "release-acceptance.json", string(receipt)) raw, _ := json.Marshal(metadata) @@ -75,7 +75,7 @@ func TestRuntimePublicationAndStablePin(t *testing.T) { } first := publish(true) manifestRaw, _, err := readObject(ctx, inputs.Bucket, "version/"+first+"/release.json") - if err != nil || !strings.Contains(string(manifestRaw), `"recoveredControlFormat": 1`) { + if err != nil || !strings.Contains(string(manifestRaw), `"recoveredControlFormat": 1`) || !strings.Contains(string(manifestRaw), `"tabRecoveryFormat": 1`) { t.Fatal("recovery capability missing from published manifest", err) } stable, _, err := readObject(ctx, inputs.Bucket, "channels/stable.json") diff --git a/testing/release-server/main.go b/testing/release-server/main.go index 8d62a53c..b0d351b9 100644 --- a/testing/release-server/main.go +++ b/testing/release-server/main.go @@ -99,6 +99,16 @@ func main() { mu.Unlock() w.WriteHeader(204) }) + // Release browsing exercises the same immutable IDs as the real publisher. + mux.HandleFunc("/releases/catalog.json", func(w http.ResponseWriter, r *http.Request) { + entries := make([]map[string]string, 0, len(releases)) + for _, rel := range releases { + parts := strings.SplitN(rel.id, "-", 2) + entries = append(entries, map[string]string{"releaseId": rel.id, "webCommit": parts[0], "buildId": parts[1]}) + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{"format": 1, "releases": entries}) + }) mux.HandleFunc("/test/releases", func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]string{"a": first.id, "b": second.id})