From 3952041a2b6cb86699d9e71296b7ce9ab5ae2d99 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 19:49:57 +0000 Subject: [PATCH 01/47] Launcher test compares the opened download folder as a path, so Windows slashes match --- docs/community-results.md | 4 ---- docs/zh-CN/community-results.md | 4 ---- 2 files changed, 8 deletions(-) diff --git a/docs/community-results.md b/docs/community-results.md index d6194362..02c06546 100644 --- a/docs/community-results.md +++ b/docs/community-results.md @@ -71,10 +71,6 @@ Thanks to the community members who sent diagnostic reports. FreeVideo already r | 3070 Laptop | 8 GB | 27 GB | 1344×768 | 10 s | Light⁵ | 0.4.4 | 625 s | 0.4.4 | Already optimal | | 5070 Laptop | 8 GB | 47 GB | 960×544 | 5.2 s | Light | 0.4.4 | 554 s | 0.4.4 | Already optimal | | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | Medium | 0.4.4 | 753 s | 0.4.4 | Already optimal | -| 5090 | 32 GB | 96 GB | 1344×768 | 10 s | Light | 0.4.4 | 400 s | 0.4.4 | Already optimal | -| 3060 | 12 GB | 32 GB | 1088×1920 | 15 s | Light | 0.4.3 | 4292 s | 0.4.5 | 4245 s (−47 s) | -| 5090 | 32 GB | 96 GB | 1344×768 | 10 s | Max | 0.4.4 | 517 s | 0.4.4 | Already optimal | -| 5070 Ti Laptop | 12 GB | 32 GB | 1344×768 | 4.5 s | Max | 0.4.5 | 819 s | 0.4.5 | Already optimal | - Time: end to end, from submission until the video is saved. - After: the estimated time, derived from the report's stage times and from complete planned runs on our test machines (RTX 3090 and RTX 5060 Ti on Windows, RTX PRO 6000 on Linux) under the reporting machine's VRAM and RAM limits, not from reruns on that machine. diff --git a/docs/zh-CN/community-results.md b/docs/zh-CN/community-results.md index babe92d0..34e591f7 100644 --- a/docs/zh-CN/community-results.md +++ b/docs/zh-CN/community-results.md @@ -71,10 +71,6 @@ | 3070 笔⁠记⁠本 | 8 GB | 27 GB | 1344×768 | 10 s | 轻⁠量⁵ | 0.4.4 | 625 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5070 笔⁠记⁠本 | 8 GB | 47 GB | 960×544 | 5.2 s | 轻⁠量 | 0.4.4 | 554 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | 标⁠准 | 0.4.4 | 753 s | 0.4.4 | 已⁠是⁠最⁠优 | -| 5090 | 32 GB | 96 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.4 | 400 s | 0.4.4 | 已⁠是⁠最⁠优 | -| 3060 | 12 GB | 32 GB | 1088×1920 | 15 s | 轻⁠量 | 0.4.3 | 4292 s | 0.4.5 | 4245 s⁠（−47 s⁠） | -| 5090 | 32 GB | 96 GB | 1344×768 | 10 s | 极⁠致 | 0.4.4 | 517 s | 0.4.4 | 已⁠是⁠最⁠优 | -| 5070 Ti 笔⁠记⁠本 | 12 GB | 32 GB | 1344×768 | 4.5 s | 极⁠致 | 0.4.5 | 819 s | 0.4.5 | 已⁠是⁠最⁠优 | - 用时：从提交到视频保存完成的端到端用时。 - 优化后：预估用时，根据报告中的分段用时，结合测试机（Windows RTX 3090 和 RTX 5060 Ti、Linux RTX PRO 6000）在报告机器的显存和内存上限下运行完整规划的实测推算，不是在报告机器上重跑。 From 5a8cfdf23f6421729fe64ced40bd505bf0648f0c Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 21:13:32 +0000 Subject: [PATCH 02/47] Sync community result row from the public repo --- docs/community-results.md | 1 + docs/zh-CN/community-results.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/community-results.md b/docs/community-results.md index 02c06546..9644e841 100644 --- a/docs/community-results.md +++ b/docs/community-results.md @@ -71,6 +71,7 @@ Thanks to the community members who sent diagnostic reports. FreeVideo already r | 3070 Laptop | 8 GB | 27 GB | 1344×768 | 10 s | Light⁵ | 0.4.4 | 625 s | 0.4.4 | Already optimal | | 5070 Laptop | 8 GB | 47 GB | 960×544 | 5.2 s | Light | 0.4.4 | 554 s | 0.4.4 | Already optimal | | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | Medium | 0.4.4 | 753 s | 0.4.4 | Already optimal | +| 5090 | 32 GB | 96 GB | 1344×768 | 10 s | Light | 0.4.4 | 400 s | 0.4.4 | Already optimal | - Time: end to end, from submission until the video is saved. - After: the estimated time, derived from the report's stage times and from complete planned runs on our test machines (RTX 3090 and RTX 5060 Ti on Windows, RTX PRO 6000 on Linux) under the reporting machine's VRAM and RAM limits, not from reruns on that machine. diff --git a/docs/zh-CN/community-results.md b/docs/zh-CN/community-results.md index 34e591f7..82c482a2 100644 --- a/docs/zh-CN/community-results.md +++ b/docs/zh-CN/community-results.md @@ -71,6 +71,7 @@ | 3070 笔⁠记⁠本 | 8 GB | 27 GB | 1344×768 | 10 s | 轻⁠量⁵ | 0.4.4 | 625 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5070 笔⁠记⁠本 | 8 GB | 47 GB | 960×544 | 5.2 s | 轻⁠量 | 0.4.4 | 554 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | 标⁠准 | 0.4.4 | 753 s | 0.4.4 | 已⁠是⁠最⁠优 | +| 5090 | 32 GB | 96 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.4 | 400 s | 0.4.4 | 已⁠是⁠最⁠优 | - 用时：从提交到视频保存完成的端到端用时。 - 优化后：预估用时，根据报告中的分段用时，结合测试机（Windows RTX 3090 和 RTX 5060 Ti、Linux RTX PRO 6000）在报告机器的显存和内存上限下运行完整规划的实测推算，不是在报告机器上重跑。 From 51cf581194d2ca2e2551b0d6f8545a2a9108cffa Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 21:24:41 +0000 Subject: [PATCH 03/47] Sync community result row from the public repo --- docs/community-results.md | 1 + docs/zh-CN/community-results.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/community-results.md b/docs/community-results.md index 9644e841..d1a5b7a8 100644 --- a/docs/community-results.md +++ b/docs/community-results.md @@ -72,6 +72,7 @@ Thanks to the community members who sent diagnostic reports. FreeVideo already r | 5070 Laptop | 8 GB | 47 GB | 960×544 | 5.2 s | Light | 0.4.4 | 554 s | 0.4.4 | Already optimal | | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | Medium | 0.4.4 | 753 s | 0.4.4 | Already optimal | | 5090 | 32 GB | 96 GB | 1344×768 | 10 s | Light | 0.4.4 | 400 s | 0.4.4 | Already optimal | +| 3060 | 12 GB | 32 GB | 1088×1920 | 15 s | Light | 0.4.3 | 4292 s | 0.4.5 | 4245 s (−47 s) | - Time: end to end, from submission until the video is saved. - After: the estimated time, derived from the report's stage times and from complete planned runs on our test machines (RTX 3090 and RTX 5060 Ti on Windows, RTX PRO 6000 on Linux) under the reporting machine's VRAM and RAM limits, not from reruns on that machine. diff --git a/docs/zh-CN/community-results.md b/docs/zh-CN/community-results.md index 82c482a2..a80d4e40 100644 --- a/docs/zh-CN/community-results.md +++ b/docs/zh-CN/community-results.md @@ -72,6 +72,7 @@ | 5070 笔⁠记⁠本 | 8 GB | 47 GB | 960×544 | 5.2 s | 轻⁠量 | 0.4.4 | 554 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | 标⁠准 | 0.4.4 | 753 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5090 | 32 GB | 96 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.4 | 400 s | 0.4.4 | 已⁠是⁠最⁠优 | +| 3060 | 12 GB | 32 GB | 1088×1920 | 15 s | 轻⁠量 | 0.4.3 | 4292 s | 0.4.5 | 4245 s (−47 s) | - 用时：从提交到视频保存完成的端到端用时。 - 优化后：预估用时，根据报告中的分段用时，结合测试机（Windows RTX 3090 和 RTX 5060 Ti、Linux RTX PRO 6000）在报告机器的显存和内存上限下运行完整规划的实测推算，不是在报告机器上重跑。 From 08e24818899ce22e86ecc1985353605f88b8a817 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 21:33:20 +0000 Subject: [PATCH 04/47] Sync community result row format from the public repo --- docs/zh-CN/community-results.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh-CN/community-results.md b/docs/zh-CN/community-results.md index a80d4e40..70032d0a 100644 --- a/docs/zh-CN/community-results.md +++ b/docs/zh-CN/community-results.md @@ -72,7 +72,7 @@ | 5070 笔⁠记⁠本 | 8 GB | 47 GB | 960×544 | 5.2 s | 轻⁠量 | 0.4.4 | 554 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | 标⁠准 | 0.4.4 | 753 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5090 | 32 GB | 96 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.4 | 400 s | 0.4.4 | 已⁠是⁠最⁠优 | -| 3060 | 12 GB | 32 GB | 1088×1920 | 15 s | 轻⁠量 | 0.4.3 | 4292 s | 0.4.5 | 4245 s (−47 s) | +| 3060 | 12 GB | 32 GB | 1088×1920 | 15 s | 轻⁠量 | 0.4.3 | 4292 s | 0.4.5 | 4245 s⁠（−47 s⁠） | - 用时：从提交到视频保存完成的端到端用时。 - 优化后：预估用时，根据报告中的分段用时，结合测试机（Windows RTX 3090 和 RTX 5060 Ti、Linux RTX PRO 6000）在报告机器的显存和内存上限下运行完整规划的实测推算，不是在报告机器上重跑。 From 529949f399569fd3b4b41a311ccde98b6a86134c Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 21:38:15 +0000 Subject: [PATCH 05/47] Sync community result row from the public repo --- docs/community-results.md | 1 + docs/zh-CN/community-results.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/community-results.md b/docs/community-results.md index d1a5b7a8..479b3c98 100644 --- a/docs/community-results.md +++ b/docs/community-results.md @@ -73,6 +73,7 @@ Thanks to the community members who sent diagnostic reports. FreeVideo already r | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | Medium | 0.4.4 | 753 s | 0.4.4 | Already optimal | | 5090 | 32 GB | 96 GB | 1344×768 | 10 s | Light | 0.4.4 | 400 s | 0.4.4 | Already optimal | | 3060 | 12 GB | 32 GB | 1088×1920 | 15 s | Light | 0.4.3 | 4292 s | 0.4.5 | 4245 s (−47 s) | +| 5090 | 32 GB | 96 GB | 1344×768 | 10 s | Max | 0.4.4 | 517 s | 0.4.4 | Already optimal | - Time: end to end, from submission until the video is saved. - After: the estimated time, derived from the report's stage times and from complete planned runs on our test machines (RTX 3090 and RTX 5060 Ti on Windows, RTX PRO 6000 on Linux) under the reporting machine's VRAM and RAM limits, not from reruns on that machine. diff --git a/docs/zh-CN/community-results.md b/docs/zh-CN/community-results.md index 70032d0a..edcba286 100644 --- a/docs/zh-CN/community-results.md +++ b/docs/zh-CN/community-results.md @@ -73,6 +73,7 @@ | 5060 Ti | 16 GB | 126 GB | 1344×768 | 10 s | 标⁠准 | 0.4.4 | 753 s | 0.4.4 | 已⁠是⁠最⁠优 | | 5090 | 32 GB | 96 GB | 1344×768 | 10 s | 轻⁠量 | 0.4.4 | 400 s | 0.4.4 | 已⁠是⁠最⁠优 | | 3060 | 12 GB | 32 GB | 1088×1920 | 15 s | 轻⁠量 | 0.4.3 | 4292 s | 0.4.5 | 4245 s⁠（−47 s⁠） | +| 5090 | 32 GB | 96 GB | 1344×768 | 10 s | 极⁠致 | 0.4.4 | 517 s | 0.4.4 | 已⁠是⁠最⁠优 | - 用时：从提交到视频保存完成的端到端用时。 - 优化后：预估用时，根据报告中的分段用时，结合测试机（Windows RTX 3090 和 RTX 5060 Ti、Linux RTX PRO 6000）在报告机器的显存和内存上限下运行完整规划的实测推算，不是在报告机器上重跑。 From b5382b0812f3ec696a9aca774797869eda53d25a Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:21 +0000 Subject: [PATCH 06/47] Reopening the creation panel keeps the previous project's media instead of opening the bundled... Reopening the creation panel keeps the previous project's media instead of opening the bundled workflow over it --- web/launcher.js | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/web/launcher.js b/web/launcher.js index 5c32c475..cc7a8081 100644 --- a/web/launcher.js +++ b/web/launcher.js @@ -10,6 +10,23 @@ const studioWorkflow = graph => (graph?._nodes || graph?.nodes || []).some(n => const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); let configured = performance.now(); +// ComfyUI's own session storage holds the last project's tabs and drafts under +// several keys that differ between frontend versions; any of them naming one of +// our nodes (the generate node may have been removed) means a project from the previous visit is waiting to be restored. +function cachedStudioProject() { + for (const area of ['localStorage', 'sessionStorage']) { + try { + const store = globalThis[area]; + for (let index = 0; index < (store?.length || 0); index++) { + const key = store.key(index); + if (key !== 'workflow' && !key?.startsWith('Comfy.')) continue; + if (/FreeVideo(Generate|Media)|freevideo_studio/.test(store.getItem(key) || '')) return true; + } + } catch { /* Storage blocked: open the bundled workflow right away. */ } + } + return false; +} + async function existingStudioWorkflow(target, bundled) { // Restored background tabs are lazy: load() reads the host's saved draft, // whereas the workflow store's openWorkflow() only selects the tab record. @@ -97,18 +114,22 @@ app.registerExtension({ url.searchParams.delete('freevideo'); window.history.replaceState(window.history.state, '', url); let workflow; + const cached = cachedStudioProject(); try { const response = await api.fetchApi('/freevideo/launcher/workflow'); if (!response.ok) throw new Error(`FreeVideo workflow: HTTP ${response.status}`); // A fresh copy of the bundled workflow, with the prompt last written. workflow = await restorePromptDraft(api, await response.json()); - await app.loadGraphData(structuredClone(workflow), true, true, 'FreeVideo'); + // Opening the bundled graph now would overwrite the cached project's + // media and connections before ComfyUI restores them; keepInFront + // brings the restored tab forward instead. + if (!cached) await app.loadGraphData(structuredClone(workflow), true, true, 'FreeVideo'); } catch (error) { console.error('FreeVideo launcher:', error); health({type: 'launch', error}); return; } - const opened = app.extensionManager?.workflow?.activeWorkflow; + const opened = cached ? undefined : app.extensionManager?.workflow?.activeWorkflow; keepInFront(workflow, opened).then(() => health({type: 'launch'}), error => { console.error('FreeVideo launcher:', error); health({type: 'launch', error}); From dedb9ba403935f09cf0fe1ebd1072040a930a328 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:21 +0000 Subject: [PATCH 07/47] Say what the install and model-reuse waits are doing: progress words in Settings and counts in... Say what the install and model-reuse waits are doing: progress words in Settings and counts in launcher steps --- freevideo_engine/desktop_runtime.py | 4 ++-- freevideo_engine/setup_progress.py | 23 ++++++++++++++++------- web/setup.js | 22 ++++++++++++++-------- 3 files changed, 32 insertions(+), 17 deletions(-) diff --git a/freevideo_engine/desktop_runtime.py b/freevideo_engine/desktop_runtime.py index 9edc47b6..551464fa 100644 --- a/freevideo_engine/desktop_runtime.py +++ b/freevideo_engine/desktop_runtime.py @@ -235,18 +235,18 @@ def local_model_ui(message): label = {'scan': 'Find existing models', 'verify': 'Verify local model', 'copy': 'Copy local model', 'link': 'Reuse local model', 'ready': 'Existing models checked'}.get(phase, 'Check local models') detail = message.get('file', '') + scanned = done if phase == 'scan' else 0 if phase == 'scan': detail = '%d files / folders scanned' % done done = 0 elif phase in ('verify', 'copy'): detail = '%.1f / %.1f MiB · %.1f MiB/s · %s' % (done/2**20, total/2**20, (rate or 0)/2**20, detail) return dict(kind='task_end' if phase == 'ready' else 'progress', key='local-models', - # The card names the step; byte counts, speed and scan counts stay in the detail. activity={'verify': 'check', 'copy': 'copy', 'scan': 'note'}.get(phase, ''), name=((message.get('name') or message.get('file', '')) if phase in ('verify', 'copy') else 'Finding existing models' if phase == 'scan' else ''), unit='bytes', label=label, detail=detail, state='complete' if phase == 'ready' else 'running', - done=done, total=total, elapsed_seconds=message.get('elapsed_seconds', 0), + done=done, total=total, rate=rate, scanned=scanned, elapsed_seconds=message.get('elapsed_seconds', 0), remaining_seconds=(total-done)/rate if total and rate and phase in ('verify', 'copy') else None) diff --git a/freevideo_engine/setup_progress.py b/freevideo_engine/setup_progress.py index e1c38d22..fb3baa72 100644 --- a/freevideo_engine/setup_progress.py +++ b/freevideo_engine/setup_progress.py @@ -3,6 +3,7 @@ import time from .launcher_copy import size_text +from .model_status import NAMES def number(value): @@ -97,7 +98,7 @@ def transfer_parts(done, total, rate, decimal_sizes=False): def progress_text(event, zh=False, *, decimal_sizes=False): - """One activity line for the launcher; step counts and ETAs remain in worker events/logs.""" + """One activity line for the launcher, including known transfer and scan counts.""" t = lambda en, cn: cn if zh else en activity, name = event.get('activity'), event.get('name') or '' if activity == 'count': @@ -117,13 +118,21 @@ def progress_text(event, zh=False, *, decimal_sizes=False): if activity == 'resume': return t('The connection dropped; resuming the download and keeping what was already downloaded', '连接中断，正在继续下载，已下载的部分会保留') - if activity == 'check': - return t('Checking ', '正在校验 ') + name - if activity == 'copy': - return t('Copying ', '正在复制 ') + name + if activity in ('check', 'copy'): + # Mid-sentence, a model family name starts in lower case in English; Chinese has no space after the verb. + name = name[:1].lower() + name[1:] if any(name == en for en, _ in NAMES.values()) else name + parts = [t('Checking ', '正在校验') + name if activity == 'check' + else t('Copying ', '正在复制') + name] + done, total = event.get('done'), event.get('total') + if event.get('unit') == 'bytes' and number(done) and number(total) and total > 0 and done <= total: + parts.extend(transfer_parts(done, total, event.get('rate'), decimal_sizes)) + return ' · '.join(parts) if activity == 'note': - # A fixed description of the step (already translated with the event), and its speed if any. - return ' · '.join([name] + transfer_parts(0, 0, event.get('rate'), decimal_sizes)) + parts = [name] + scanned = event.get('scanned') + if number(scanned) and scanned > 0: + parts.append(t('{:,} checked', '已查看 {} 项').format(scanned)) + return ' · '.join(parts + transfer_parts(0, 0, event.get('rate'), decimal_sizes)) if activity == 'download': total = event.get('total') if event.get('unit') == 'bytes' else 0 return ' · '.join([t('Downloading ', '正在下载 ') + name] + diff --git a/web/setup.js b/web/setup.js index 0093e327..3b3c7453 100644 --- a/web/setup.js +++ b/web/setup.js @@ -161,8 +161,10 @@ export async function openSetup() { dialog.addEventListener('cancel', e => { e.preventDefault(); closeDialog(dialog); }); document.body.append(dialog); dialog.showModal(); let savedFolders = "", savedMissing = false; - let token, planId, busy = false, probeBusy = false, pollTimer, lastPlanId, planErrors = false, planStale = false, ready = false; - const fail = error => { installation.open = true; state.textContent = failureText(error.message); state.className = "fv-failure"; state.scrollIntoView({block:'nearest'}); }; + let token, planId, busy = false, probeBusy = false, pollTimer, lastPlanId, planErrors = false, planStale = false, ready = false, waitSince; + // A click names what is happening at once, before the server has answered. + const doing = (en, zh) => { waitSince ??= Date.now(); state.className = ''; state.textContent = t(en, zh); }; + const fail = error => { waitSince = undefined; installation.open = true; state.textContent = failureText(error.message); state.className = "fv-failure"; state.scrollIntoView({block:'nearest'}); }; const request = async (action, value) => { const response = await api.fetchApi("/freevideo/setup" + (action ? "/" + action : root.value ? '?root='+encodeURIComponent(root.value) : ""), action ? { method: "POST", headers: { "Content-Type": "application/json", "X-FreeVideo-Setup": token }, body: JSON.stringify(value || {}), @@ -323,7 +325,8 @@ export async function openSetup() { if (value.errors?.length) { const e = node("p", value.errors.map(failureText).join("\n")); e.className = "fv-failure"; plan.append(e); } } function render(task) { - busy = Boolean(task.busy); ready = Boolean(task.ready) || ready && task.status !== 'failed'; + busy = Boolean(task.busy); if (!busy) waitSince = undefined; else waitSince ??= Date.now(); + ready = Boolean(task.ready) || ready && task.status !== 'failed'; cancel.textContent = task.action === 'calibrate' ? t('Stop', '停止') : t('Stop setup', '停止安装'); inputs.hidden = busy; plan.hidden = busy; if (task.plan_id && task.plan_id !== lastPlanId) { @@ -342,8 +345,8 @@ export async function openSetup() { overallText.textContent = phases.total > 0 ? `${t("Overall", "整体进度")} · ${phases.done} / ${phases.total} ${t("stages complete", "阶段已完成")} · ${Math.floor(100 * phases.done / phases.total)}%` : t("Preparing installation plan…", "正在准备安装计划…"); bar.hidden = !busy; bar.max = valid ? p.total : 1; bar.value = valid ? p.done : 0; const amount = n => p.total >= 2 ** 30 ? gib(n) : `${(n / 2 ** 20).toFixed(1)} MiB`; - const count = valid ? `${Math.floor(100 * p.done / p.total)}% · ${p.unit === "bytes" ? `${amount(p.done)} / ${amount(p.total)}` : `${p.done} / ${p.total}`}` : t("Measuring progress…", "正在获取进度…"); - const eta = valid && Number.isFinite(p.remaining_seconds) && p.remaining_seconds > 0 ? `${p.unit === "bytes" ? t("File ETA", "本文件预计剩余") : t("ETA", "预计剩余")} ~${Math.ceil(p.remaining_seconds)} s` : t("Estimating remaining time…", "正在估算剩余时间…"); + const count = valid ? `${Math.floor(100 * p.done / p.total)}% · ${p.unit === "bytes" ? `${amount(p.done)} / ${amount(p.total)}` : `${p.done} / ${p.total}`}` : t("Preparing…", "正在准备…"); + const eta = valid && Number.isFinite(p.remaining_seconds) && p.remaining_seconds > 0 ? `${p.unit === "bytes" ? t("File ETA", "本文件预计剩余") : t("ETA", "预计剩余")} ~${Math.ceil(p.remaining_seconds)} s` : valid ? t("Estimating remaining time…", "正在估算剩余时间…") : `${t("Elapsed", "已用时")} ${Math.floor((Date.now() - (waitSince || Date.now())) / 1000)} s`; counters.textContent = busy ? `${count} · ${eta}` : ""; const others = (task.active_tasks || []).slice(1).map(p => p.label).filter(Boolean); const progressDetail = (p.detail || '').replace(/^Keeping the original file: /, t('Keeping the original file: ', '正在保留原文件：')); @@ -375,16 +378,18 @@ export async function openSetup() { } saveFolders.onclick = async () => { invalidate(true); busy = true; controls(); + const before = [state.textContent, state.className]; doing('Saving model folders…', '正在保存模型目录…'); try { await saveModelFolders(); } catch (error) { savedFolders = ''; savedMissing = false; folderStatus.textContent = failureText(error.message); folderStatus.className = 'fv-failure'; } - finally { busy = false; controls(); } + finally { busy = false; waitSince = undefined; [state.textContent, state.className] = before; controls(); } }; inspect.onclick = async () => { - invalidate(); busy = true; controls(); + invalidate(); busy = true; controls(); doing('Checking installation…', '正在检查安装…'); try { await saveModelFolders(); render(await request("inspect", {root: root.value, extra_libraries: extra.value.split("\n").map(s => s.trim()).filter(Boolean), copy: copy.checked, sampling_caches: samplingCaches.checked})); poll(); } - catch (error) { busy = false; controls(); fail(error); } + catch (error) { busy = false; waitSince = undefined; controls(); fail(error); } }; install.onclick = async () => { + doing('Starting installation…', '正在开始安装…'); try { render(await request("install", {plan_id: planId, accept_licenses: accept.checked})); invalidate(); poll(); } catch (error) { fail(error); } }; @@ -394,6 +399,7 @@ export async function openSetup() { }; cancel.onclick = async () => { try { render(await request("cancel")); poll(); } catch (error) { fail(error); } }; dialog.onclose = () => { clearTimeout(pollTimer); for (const dispose of motions) dispose(); dialog.remove(); if (current === dialog) current = undefined; }; + doing('Reading installation settings…', '正在读取安装设置…'); try { const info = await request(); token = info.discovery.token; root.value = info.discovery.root; extra.value = (info.discovery.extra_libraries || []).join("\n"); From 957d251fbb39160ee67d47170c61e22f808a2732 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:22 +0000 Subject: [PATCH 08/47] AI autopilot: every click answers, sends are bounded, and failures name the step with a report --- freevideo_engine/agent/cast.py | 9 +- freevideo_engine/agent/errors.py | 78 ++++++++- freevideo_engine/agent/jobs.py | 30 +++- freevideo_engine/agent/productions.py | 35 +++- freevideo_engine/agent/routes.py | 43 ++++- freevideo_engine/agent/schema.py | 5 +- web/agent_activity.js | 30 ++-- web/agent_connect.css | 2 + web/agent_connect.js | 24 ++- web/agent_session.js | 15 ++ web/autopilot.js | 4 +- web/autopilot_chat.css | 9 +- web/autopilot_chat.js | 228 +++++++++++++++++++------- web/generation_progress.js | 97 ++++++----- 14 files changed, 469 insertions(+), 140 deletions(-) diff --git a/freevideo_engine/agent/cast.py b/freevideo_engine/agent/cast.py index 7ed1f999..1b32163d 100644 --- a/freevideo_engine/agent/cast.py +++ b/freevideo_engine/agent/cast.py @@ -98,6 +98,7 @@ def view(card): if card.get('voice'): value['voice'] = dict(deepcopy(card['voice']), media=media_view(card['voice']['media']), clip=assets.view(card['voice']['clip']['file'])) + value['generate_error'] = errors.with_reason(value.get('generate_error')) return value @@ -112,7 +113,8 @@ def row_view(card): return dict(id=card['id'], kind=card['kind'], name=card['name'], description=card['description'], origin=card['origin'], revision=card['revision'], updated_at=card['updated_at'], images=len(card['images']), voice=card['voice'] is not None, - generating=card.get('generating') is not None, thumbnail=thumbnail(card)) + generating=card.get('generating') is not None, thumbnail=thumbnail(card), + generate_error=errors.with_reason(card.get('generate_error'))) def _plain(item): @@ -729,7 +731,7 @@ def _land_image(self, identity, generating, hidden, rows): code = (row.get('error') or {}).get('code') if row else 'not_found' code = code or (row['status'] if row else 'not_found') detail = errors.image_failure(code, (row.get('error') or {}).get('report_id') if row else None) - failure = failure or dict(code=code or row['status'], job=job, message=( + failure = failure or dict(code=code, job=job, error_zh=detail['reason']['zh'], message=( 'The generation was cancelled.' if row and row['status'] == 'cancelled' else detail['reason']['en']), **detail) continue @@ -764,7 +766,8 @@ def _land(self, identity, generating, hidden, rows): face = job == generating.get('face_job') if row is None or row['status'] != 'succeeded': code = (row.get('error') or {}).get('code') if row else 'not_found' - failure = failure or dict(code=code or row['status'], job=job, message=( + failure = failure or dict(code=code or row['status'], job=job, + error_zh=errors.failure_reason((row.get('error') or dict(code=code or row['status'])) if row else dict(code=code)), message=( 'The generation was cancelled.' if row and row['status'] == 'cancelled' else 'The reference clip did not finish.')) continue diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index f79540a1..8ec96ead 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -248,6 +248,82 @@ def safe_details(value): return '[redacted]' +# One public sentence shared by conversation cards, HTTP and MCP. Backend text +# is classified, never echoed: it can contain prompts, filenames or credentials. +FAILURE_REASONS_ZH = { + 'out_of_memory': '生成时显存或内存不足', + 'insufficient_memory': '可用显存或内存不够生成这段内容', + 'no_memory': '可用显存或内存不够生成这张图片', + 'ram_space': '加载模型时系统内存不足', + 'model_missing': '需要的模型还没有安装好', + 'pack_missing': '图片模型还没有安装好', + 'not_ready': '生成所需的环境还没有安装好', + 'cancelled': '生成已被停止', + 'stopped_by_user': '生成已被停止', + 'interrupted': '生成服务中途重启，任务被打断', + 'worker_exit': '图片生成进程中途退出', + 'closed': '图片生成服务已经关闭', + 'image_unavailable': '图片生成服务还没有启动', + 'vlm_unavailable': '读取画面的模型服务还没有启动', + 'vlm_failed': '读取画面的模型没有完成请求', + 'vlm_invalid_json': '模型返回的结果格式无法读取', + 'context_length': '描述和参考图超过了模型可读取的长度', + 'dependency_failed': '前面依赖的任务没有完成', + 'lost': '生成服务没有保留这次任务的结果', + 'unknown_job': '图片生成服务找不到这次任务', + 'not_found': '找不到这次任务或素材', + 'source_missing': '接续所需的上一段视频已经不在', + 'chain_missing': '接续所需的生成记录已经不在', + 'chain_unavailable': '分段生成服务还没有启动', + 'continuation_unavailable': '当前生成方式不支持接续视频', + 'join_failed': '片段已生成，但拼接成片时出错', + 'invalid_output': '生成的结果无法读取或保存', + 'storage_unavailable': '任务记录或素材无法写入磁盘', + 'submit_interrupted': '提交生成时服务中断，尚未确认任务是否开始', + 'timeout': '生成等待超时', + 'invalid_media': '参考素材无法读取', + 'clip_unreadable': '生成的角色参考视频无法读取', + 'invalid_request': '生成参数不符合支持的范围', + 'invalid_drama': '短剧计划中有无法使用的参数', + 'reshoot_limit': '这个镜头已达到可生成的次数上限', + 'production_limit': '这部短剧已达到可生成的任务数量上限', + 'export_failed': '导出成片时出错', + 'invalid_prompt': '图片描述为空或超过了支持的长度', + 'invalid_dimensions': '图片尺寸超出了支持的范围', + 'too_many_images': '参考图片数量超过了支持的范围', + 'chain_too_long': '接续后的视频超过了支持的段数', + 'chain_rejected': '生成服务无法使用这组参数', + 'generation_failed': '生成过程中出错，未能完成结果', + 'internal_error': '处理任务时出错，未能完成结果', +} +FAILURE_MESSAGES_ZH = { + 'The previous video is too short to continue; it needs to be at least 2 s long.': '接续的上一段视频太短，至少需要 2 秒', + 'The previous video has no usable continuation data. Generate it again with this version of FreeVideo.': '上一段视频缺少可用的接续数据', + 'The previous video cannot be read.': '接续的上一段视频无法读取', + 'Use the same width and height as the previous video.': '接续的画面大小与上一段视频不一致', + 'The reference video is too short; it needs to be at least about 1 s long.': '参考视频太短，至少需要约 1 秒', +} + + +def failure_reason(error): + if isinstance(error.get('error_zh'), str) and error['error_zh'].strip(): + return error['error_zh'] + lines = str(error.get('message') or '').strip().splitlines() + line = lines[0] if lines else '' + if line.startswith('FreeVideo generation failed') and len(lines) > 1: + line = lines[1].strip() + line = re.sub(r'^(?:[\w.]*Error|[\w.]*Exception):\s*', '', line) + if line in FAILURE_MESSAGES_ZH: + return FAILURE_MESSAGES_ZH[line] + if line.startswith(('CUDA out of memory', 'MPS backend out of memory')): + return FAILURE_REASONS_ZH['out_of_memory'] + return FAILURE_REASONS_ZH.get(error.get('code'), '处理任务时出错，未能完成结果') + + +def with_reason(error): + return dict(error, error_zh=failure_reason(error)) if error else error + + def build(code, *, details=None, request=None, validation=False, message=None, next_action=None, action_required=True, retry_after_seconds=None, owner=None, kind=None, stage='request', resumable=True, report_id=None): @@ -270,7 +346,7 @@ def build(code, *, details=None, request=None, validation=False, message=None, n result['retry_after_seconds'] = retry_after_seconds or 1 result['details'].append(dict(owner=owner if owner in ('agent', 'ui') else 'ui')) result['message'] += ' An Agent job is ahead.' if owner == 'agent' else ' A job in the user interface is ahead.' - return result + return with_reason(result) def validation(errors, code='invalid_request'): diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 2e12cdb2..0f517fbb 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -14,6 +14,7 @@ import logging import math import os +import re import secrets import shutil import stat @@ -592,15 +593,21 @@ def touch(self, row, **changes): def finish(self, row, status, *, result=None, error=None, save=True): if error is not None: code = error.get('code', 'internal_error') + report_id = error.get('report_id') if code == 'invalid_json' and row['kind'] in ('vlm', 'enhance'): code = 'vlm_invalid_json' + reason_zh = errors.failure_reason(dict(error, code=code)) if code != error.get('code') or not error.get('request_id') or row['kind'] == 'image': error = errors.build(code, details=error.get('details'), request=row.get('request_id'), action_required=code != 'cancelled', kind=row['kind'], stage='terminal', resumable=not row['spec'].get('refine_from'), report_id=error.get('report_id')) + if row['kind'] == 'image': + reason_zh = error['error_zh'] if error is not None: - error = dict(error, request_id=row.get('request_id') or error['request_id']) + error = dict(error, request_id=row.get('request_id') or error['request_id'], error_zh=reason_zh) + if row['kind'] != 'image' and isinstance(report_id, str) and re.fullmatch(r'[a-f0-9]{32}', report_id): + error['report_id'] = report_id if error.get('next'): error['next'] = errors.recovery(error['code'], kind=row['kind'], stage='terminal', resumable=not row['spec'].get('refine_from')).replace('{id}', row['id']) @@ -1068,6 +1075,11 @@ def view(self, row, *, detail=True): result['error'] = errors.build(result['error']['code'], kind='image', stage='terminal', details=result['error'].get('details'), request=row.get('request_id'), action_required=result['error']['code'] != 'cancelled', report_id=row['error'].get('report_id')) + else: + result['error'] = dict(result['error'], error_zh=errors.failure_reason(row['error'])) + report_id = row['error'].get('report_id') + if isinstance(report_id, str) and re.fullmatch(r'[a-f0-9]{32}', report_id): + result['error']['report_id'] = report_id if result['error'].get('next'): result['error'] = dict(result['error'], next=errors.recovery(result['error']['code'], kind=row['kind'], stage='terminal', resumable=not row['spec'].get('refine_from')).replace('{id}', row['id'])) @@ -1696,6 +1708,10 @@ def generation_failure(self, row, chain): reason = self.port.generation_error(history, chain['node']) except Exception as error: self.log_poll_failure(row, error) + try: + report_id = (self.port.progress(chain['node']) or {}).get('report_id') + except Exception: + report_id = None code = reason.get('code', 'generation_failed') if code not in ('insufficient_memory', 'out_of_memory', 'generation_failed'): code = 'generation_failed' @@ -1718,8 +1734,12 @@ def generation_failure(self, row, chain): 'POST /freevideo/agent/v1/jobs/%s/resume to retry unfinished work.' % row['id']) if code == 'insufficient_memory' and reason.get('alternatives'): next_action = 'Use the alternatives in details to lower size or duration, then POST /freevideo/agent/v1/jobs.' - return chain_error(code, errors.ERRORS[code][2], next_action, details=[detail], - retryable=code != 'insufficient_memory').view() + failure = dict(chain_error(code, errors.ERRORS[code][2], next_action, details=[detail], + retryable=code != 'insufficient_memory').view(), + error_zh=errors.failure_reason(dict(code=code, message=message))) + if isinstance(report_id, str) and re.fullmatch(r'[a-f0-9]{32}', report_id): + failure['report_id'] = report_id + return failure # Local image generation ---------------------------------------------------------------------------- def wait_image(self, row, code, *, waiting_for=None): @@ -1781,7 +1801,9 @@ def poll_image(self, row): elif phase == 'cancelled': self.finish(row, 'cancelled', error=dict(code='cancelled')) else: - code = state.get('error') or 'generation_failed' + failure = state.get('error') + code = failure.get('code') if isinstance(failure, dict) else failure + code = code or 'generation_failed' if not self.wait_image(row, code): self.finish(row, 'failed', error=dict(code=code, report_id=state.get('report_id'))) diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py index 54c794f2..fab1b081 100644 --- a/freevideo_engine/agent/productions.py +++ b/freevideo_engine/agent/productions.py @@ -140,7 +140,22 @@ def _safe_error(error): rows = [_safe_row(row) for row in details] if isinstance(details, list) and details else [] if rows and all(rows): result.update(details=rows, message='; '.join(row['message'] for row in rows)) - return result + return dict(result, error_zh=errors.failure_reason(error)) + + +def failure(view): + """The current failure, excluding old takes superseded by a usable take.""" + if view.get('stalled'): + return view['stalled'], None + join = view.get('join') or {} + if join.get('state') in ('failed', 'interrupted') and join.get('error'): + return join['error'], join.get('task') + for shot in view.get('shots', []): + if shot['state'] == 'failed': + take = next((t for t in reversed(shot['takes']) if t.get('error') and not t['stale']), {}) + if take: + return take['error'], take.get('task') + return None, None class Productions: @@ -194,7 +209,7 @@ def load(self): def _stalled(self, error): code = error.code if isinstance(getattr(error, 'code', None), str) and error.code in errors.ERRORS else 'internal_error' - return dict(code=code, message=errors.ERRORS[code][2], at=iso(self.jobs.clock())) + return errors.with_reason(dict(code=code, message=errors.ERRORS[code][2], at=iso(self.jobs.clock()))) def _save(self, record): self.directory.mkdir(parents=True, exist_ok=True) @@ -588,7 +603,7 @@ def _view(self, record): value = production.view(record['state']) tasks = record.get('tasks', {}) def contextual(error, join=False): - error = deepcopy(error) + error = errors.with_reason(deepcopy(error)) if error and 'next' in error: action = self._context_next(record, join) if action: @@ -604,9 +619,11 @@ def contextual(error, join=False): ref = record['state']['current_join'] join = next(j for j in record['state']['joins'] if j['ref'] == ref) value['join'].update(ref=ref, task=tasks.get(ref), error=contextual(join['error'], True)) - value['stalled'] = deepcopy(record.get('stalled')) + value['stalled'] = errors.with_reason(deepcopy(record.get('stalled'))) value['progress'] = self._progress(record) value['over_budget'] = self._elapsed_budget(value['progress']) + error, _ = failure(value) + value['error_zh'] = errors.failure_reason(error) if error else None return value def get(self, identity, *, since=None, wait=0, compact=True): @@ -625,9 +642,15 @@ def _reply(self, identity, since, compact): changed = record['state']['revision'] != since if not changed and compact: progress = self._progress(record) - return dict(id=identity, revision=since, changed=False, status=record['state']['status'], + value = dict(id=identity, revision=since, changed=False, status=record['state']['status'], paused=deepcopy(record['state'].get('paused')), fraction=progress['fraction'], elapsed_seconds=progress['elapsed_seconds']) + state = record['state'] + join = next((j for j in state['joins'] if j['ref'] == state['current_join']), None) + error, _ = failure(dict(stalled=record.get('stalled'), join=join, shots=state['shots'])) + if error: + value['error_zh'] = errors.failure_reason(error) + return value return dict(self._view(record), changed=changed) async def get_async(self, identity, *, since=None, wait=0, compact=True): @@ -676,7 +699,7 @@ def list(self, *, status='all', limit=50): shot = next((s for s in view['shots'] if s['chosen'] is None), view['shots'][-1]) result.append(dict(id=view['id'], title=view['title'], status=view['status'], revision=view['revision'], shot=shot['index'], shots=len(view['shots']), next=view['todo'][0] if view['todo'] else None, - paused=view['paused'], stalled=view['stalled'], session=record.get('session'))) + paused=view['paused'], stalled=view['stalled'], session=record.get('session'), error_zh=view['error_zh'])) return result def _mutate(self, action, identity, *, by, ref=None, client=None, **values): diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index e47d59bf..ef1e9261 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -586,6 +586,12 @@ def response_refs(value): for key in ('drama', 'project', 'image'): if isinstance(value.get(key), str): refs[key] = value[key] + identity = value.get('id') + if isinstance(identity, str): + if identity.startswith('d') and 'shots' in value: + refs['drama'] = identity + elif identity.startswith('c') and value.get('kind') in ('character', 'setting'): + refs['cast'] = identity return refs async def session_after(request, response): @@ -726,6 +732,9 @@ def activity_item(row, *, card=False): value['waiting_for'] = len(view['waiting_for']) if view.get('error'): value['error'] = view['error'] + report_id = (view.get('error') or {}).get('report_id') or (view.get('result') or {}).get('report_id') + if isinstance(report_id, str) and re.fullmatch(r'[a-f0-9]{32}', report_id): + value['report_id'] = report_id video = (view.get('result') or {}).get('video') or {} if video.get('url'): value['video'] = video['url'] @@ -792,7 +801,8 @@ def chat_fold(store, sid): if fold is None: while len(chat['folds']) >= 8: chat['folds'].pop(next(iter(chat['folds']))) - fold = chat['folds'][sid] = dict(seq=0, items=[], jobs={}, asked={}, sent={}, ids={}, names={}, said=None) + # Message IDs persist in session events and rebuild deduplication when a fold is evicted. + fold = chat['folds'][sid] = dict(seq=0, items=[], jobs={}, entities={}, asked={}, sent={}, ids={}, names={}, said=None) while store.get(sid)['last_seq'] > fold['seq']: rows = store.events(sid, after=fold['seq'], limit=500) if not rows: @@ -810,11 +820,18 @@ def chat_fold(store, sid): while len(fold['sent']) > 500: fold['ids'].pop(fold['sent'].pop(next(iter(fold['sent']))), None) elif row['type'] == 'step' and row['state'] == 'done': - job = (row.get('refs') or {}).get('job') + refs = row.get('refs') or {} + job = refs.get('job') + entity = next(((refs[key], kind) for key, kind in (('drama', 'production'), ('cast', 'cast')) if refs.get(key)), None) + new_entity = entity and entity[0] not in fold['entities'] + if new_entity: + fold['entities'][entity[0]] = entity[1] if job and job not in fold['jobs']: fold['jobs'][job] = row['seq'] fold['names'][job] = chat_name(fold['said']) - fold['items'].append(dict(base, type='job', job=job)) + fold['items'].append(dict(base, type='job', job=job, **({'entity': entity[0]} if new_entity else {}))) + elif new_entity: + fold['items'].append(dict(base, type=entity[1], entity=entity[0])) elif row['type'] == 'confirm': if row['state'] == 'asked': item = dict(base, id=row['id'], question=row.get('question'), @@ -854,7 +871,7 @@ def read(): store = work_sessions() sid = chat_session(store, wanted) rows = jobs.rows() - value = dict(agent=agent_state(), session=sid, here=chat_session(store), last_seq=0, read_seq=0, items=[], jobs={}, confirms={}, current=None, + value = dict(agent=agent_state(), session=sid, here=chat_session(store), last_seq=0, read_seq=0, items=[], jobs={}, failures={}, confirms={}, current=None, active=sum(1 for r in rows if r['status'] in ACTIVE)) if sid is None: return value @@ -883,6 +900,24 @@ def read(): shown = {r['id']: r for r in mine if r['status'] in ACTIVE or (r.get('finished_at') or 0) > now - 1800} shown.update((i['job'], known[i['job']]) for i in items if i['type'] == 'job') value['jobs'] = {identity: chat_card(row, fold) for identity, row in shown.items()} + for identity, kind in fold['entities'].items(): + try: + entity = jobs.productions.get(identity, compact=False) if kind == 'production' else jobs.cast.get(identity) + except JobError: + continue # A deleted card is no longer part of this conversation. + error, task = None, None + if kind == 'cast': + error = entity.get('generate_error') + task = (error or {}).get('job') + else: + from .productions import failure + error, task = failure(entity) + if error: + task_view = jobs.view(known[task], detail=False) if task in known else {} + report_id = ((task_view.get('error') or {}).get('report_id') + or (task_view.get('result') or {}).get('report_id')) + value['failures'][identity] = dict(id=identity, kind=kind, status='failed', + label=entity.get('title') or entity.get('name'), error=errors.with_reason(error), report_id=report_id) value['current'] = current['id'] if current else None if current and current['id'] not in value['jobs']: value['jobs'][current['id']] = chat_card(current, fold) diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 11ef962f..1e3d01c8 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -510,7 +510,7 @@ def cast_view(): description=nullable(string(1000)), revision=number(1, integer=True), images=array(picture, 0, 6), candidates=array(picture, 0, 6), generating=nullable(generating), generated=nullable(generated), - generate_error=nullable(obj(dict(code=string(), job=JOB, message=string()), ('code', 'job', 'message')))), + generate_error=nullable(obj(dict(code=string(), job=JOB, message=string(), error_zh=string()), ('code', 'job', 'message')))), ('id', 'kind', 'name', 'revision', 'images', 'candidates', 'generating')), additionalProperties=True) @@ -1332,7 +1332,8 @@ def openapi(): report_id={'type': ['string', 'null'], 'pattern': '^[0-9a-f]{32}$', 'description': 'Image diagnostic report ID, or null when no report exists.'}, request_id=string(), docs=string(), retry_after_seconds=number(0), card={'type': 'object'}, job=JOB, - project={'type': 'object'}, current_revision=number(1, integer=True)), + project={'type': 'object'}, current_revision=number(1, integer=True), + error_zh=string()), ('code', 'message', 'details', 'retryable', 'request_id', 'docs'))})) diff --git a/web/agent_activity.js b/web/agent_activity.js index 5785391a..224ca578 100644 --- a/web/agent_activity.js +++ b/web/agent_activity.js @@ -1,4 +1,4 @@ -import {agentPost} from './agent_session.js'; +import {retryAgentPost} from './agent_session.js'; // What agents connected to this FreeVideo are doing. Their requests share the // studio's queue, so people can see them, follow progress and stop them. const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./agent_activity.css', import.meta.url).href; document.head.append(css); @@ -45,6 +45,7 @@ function phaseText(p, t) { function status(job, t) { const [en, zh] = KINDS[job.kind] || KINDS.generate; if (job.status === 'running') return t(en, zh); + if (job.status === 'cancelling') return t('Stopping', '正在停止'); if (job.status === 'queued') return job.progress?.queue_position ? t(`In queue · #${job.progress.queue_position}`, `排队 · 第 ${job.progress.queue_position} 位`) : t('Queued', '排队中'); if (job.status === 'waiting') return job.waiting_for ? t('Waiting for earlier steps', '等待前序任务') : t('Waiting for the GPU', '等待显卡'); @@ -54,7 +55,7 @@ function title(job, t) { const name = NAMES[job.kind] || NAMES.generate; return job.label || (t(name[0], name[1]) + (job.take ? ' · ' + t(`Take ${job.take}`, `第 ${job.take} 条`) : '')); } -function row(job, t, api, refresh, open) { +function row(job, t, api, refresh, open, cancellationErrors) { const item = el('li', 'fv-agent-job'); item.dataset.status = job.status; item.dataset.job = job.id; item.append(glyph((KINDS[job.kind] || KINDS.generate)[2])); const body = el('div', 'fv-agent-body'); @@ -75,7 +76,7 @@ function row(job, t, api, refresh, open) { if (job.status === 'running' && p.remaining_seconds > 0) meta.push(left(p.remaining_seconds, t)); const reason = job.status === 'failed' && REASONS[job.error?.code]; if (reason) meta.push(t(...reason)); - else if (job.error?.message && job.status === 'failed') meta.push(job.error.message); + else if (job.status === 'failed') meta.push(t(job.error?.message || 'The task failed.', job.error?.error_zh || '处理任务时出错，未能完成结果')); if (job.project) meta.push(job.project); const foot = el('div', 'fv-agent-foot'); const details = el('p', 'fv-agent-meta', meta.join(' · ')); @@ -85,10 +86,16 @@ function row(job, t, api, refresh, open) { const stop = el('button', 'fv-agent-stop', t('Stop', '停止')); stop.type = 'button'; stop.setAttribute('aria-label', t('Stop ', '停止 ') + title(job, t)); stop.onclick = async () => { - stop.disabled = true; + stop.disabled = true; stop.textContent = t('Stopping', '正在停止'); try { - await agentPost(api, '/freevideo/agent/activity/cancel', {id: job.id}); + const response = await retryAgentPost(api, '/freevideo/agent/activity/cancel', {id: job.id}); + if (!response?.ok) { + const message = t('Stopping failed after a retry. If still running, click again.', '停止没成功，已重试；仍在运行请再点一次'); + details.textContent = message; + cancellationErrors.set(job.id, message); + } else cancellationErrors.delete(job.id); } finally { refresh(); } + stop.disabled = false; stop.textContent = t('Stop', '停止'); }; foot.append(stop); } @@ -100,21 +107,22 @@ function row(job, t, api, refresh, open) { foot.append(show); } body.append(foot); + if (cancellationErrors.has(job.id)) details.textContent = cancellationErrors.get(job.id); item.append(body); return item; } // Running first, then the rest in the order they will run; finished ones after, newest first. -function rows(jobs, t, api, refresh, open) { - const rank = {running: 0, queued: 1, waiting: 2}; +function rows(jobs, t, api, refresh, open, cancellationErrors) { + const rank = {running: 0, cancelling: 0, queued: 1, waiting: 2}; const active = jobs.filter(j => j.status in rank) .sort((a, b) => rank[a.status] - rank[b.status] || String(a.created_at).localeCompare(String(b.created_at))); const done = jobs.filter(j => !(j.status in rank)) .sort((a, b) => String(b.finished_at).localeCompare(String(a.finished_at))); - const items = active.map(job => row(job, t, api, refresh, open)); + const items = active.map(job => row(job, t, api, refresh, open, cancellationErrors)); if (done.length) { const divider = el('li', 'fv-agent-divider', t('Last 30 minutes', '最近 30 分钟')); divider.setAttribute('role', 'presentation'); - items.push(divider, ...done.map(job => row(job, t, api, refresh, open))); + items.push(divider, ...done.map(job => row(job, t, api, refresh, open, cancellationErrors))); } return items; } @@ -140,18 +148,20 @@ export function createAgentActivity(t, api, host, {interval = [1500, 8000], open panel.setAttribute('aria-label', t('Agent jobs', 'Agent 任务')); host.append(panel); let data = {active: 0, jobs: []}, timer = null, disposed = false, failures = 0; + const cancellationErrors = new Map(); function renderPanel() { const header = el('header'); header.append(el('h2', null, t('Agent jobs', 'Agent 任务'))); const list = el('ol'); - const key = refill(panel, list, rows(data.jobs, t, api, poll, job => { setOpen(false); open(job); })); + const key = refill(panel, list, rows(data.jobs, t, api, poll, job => { setOpen(false); open(job); }, cancellationErrors)); panel.replaceChildren(); panel.append(header, list); if (key) (panel.querySelector(`[data-job="${key[0]}"] .${key[1].split(' ')[0]}`) || panel).focus?.(); } function render() { const jobs = data.jobs || []; + for (const job of jobs) if (!['waiting', 'queued', 'running', 'cancelling'].includes(job.status)) cancellationErrors.delete(job.id); element.hidden = !jobs.length; if (!jobs.length) { setOpen(false); return; } const running = jobs.find(j => j.status === 'running'), active = data.active || 0; diff --git a/web/agent_connect.css b/web/agent_connect.css index 2d0c8ca2..f6f5d75f 100644 --- a/web/agent_connect.css +++ b/web/agent_connect.css @@ -30,6 +30,8 @@ .fv-studio .fv-autopilot-now-stop{flex:none;width:30px;height:30px;padding:0} .fv-autopilot-now-stop::after{content:"";width:18px;height:18px;border-radius:5px;background:#1b2a3a radial-gradient(circle,var(--fv-danger) 0 0) center/7px 7px no-repeat} .fv-studio .fv-autopilot-now-stop:hover::after{background-color:var(--fv-danger-subtle)} +.fv-studio .fv-autopilot-now-stop[aria-busy=true]{width:auto;padding:0 8px;font-size:12px;color:var(--fv-muted);opacity:1} +.fv-autopilot-now-stop[aria-busy=true]::after{display:none} .fv-autopilot-line{position:absolute;left:0;bottom:-1px;height:2px;border-radius:0 2px 2px 0;background:linear-gradient(90deg,var(--fv-accent),var(--fv-accent-hover));transition:width .6s var(--fv-ease),opacity .4s;pointer-events:none} .fv-autopilot-line[data-on=false]{opacity:0} @media(max-width:1050px){.fv-autopilot-now-go>em{display:none}.fv-autopilot-now-name{max-width:120px}} diff --git a/web/agent_connect.js b/web/agent_connect.js index b8f8e9f1..65bf2cd0 100644 --- a/web/agent_connect.js +++ b/web/agent_connect.js @@ -43,15 +43,22 @@ export function agentConnectCard(t, api, shown = () => true) { const arrow = el('span', 'fv-agent-connect-arrow', '→'); arrow.setAttribute('aria-hidden', 'true'); actions.append(button, arrow, el('span', 'fv-agent-connect-hint', t('Paste to your agent', '粘贴给 Agent'))); const sentence = el('p', 'fv-agent-sentence'); sentence.hidden = true; - body.append(mark, title, subtitle, actions, sentence); + const status = el('p', 'fv-agent-connect-status', t('Getting the connection address…', '正在获取连接地址…')); status.setAttribute('role', 'status'); + body.append(mark, title, subtitle, actions, sentence, status); - let feedback = false, text = ''; + let feedback = false, text = '', failures = 0; let timer, feedbackTimer, controller, generation = 0; const active = () => body.isConnected && shown(); - const render = () => { label.textContent = feedback ? t('Copied', '已复制') : t('Copy', '复制'); }; + const render = () => { + label.textContent = !text && failures >= 3 ? t('Could not get address. Retry', '获取失败，重试') : feedback ? t('Copied', '已复制') : t('Copy', '复制'); + button.disabled = !text && failures < 3; + button.title = !text && failures < 3 ? t('Getting the connection address…', '正在获取连接地址…') : ''; + status.hidden = !!text; status.textContent = !text && failures < 3 ? t('Getting the connection address…', '正在获取连接地址…') : ''; + }; render(); button.onclick = async () => { + if (!text) { failures = 0; render(); await load(); return; } try { await navigator.clipboard.writeText(text); sentence.hidden = true; } catch { sentence.textContent = text; sentence.hidden = false; @@ -77,20 +84,21 @@ export function agentConnectCard(t, api, shown = () => true) { // The sentence carries no parameter: the server gives the agent the language this card showed. const response = await api.fetchApi('/freevideo/agent/connect?lang=' + t('en', 'zh'), {cache: 'no-store', signal: request.signal}); const connect = response.ok ? await response.json() : null; - if (current !== generation || !connect?.address) return; + if (current !== generation) return; + if (!connect?.address) throw new Error('Missing address'); text = connectionSentence(connect.address, t); - button.disabled = false; - } catch { /* Try again while the card is in sight. */ } + failures = 0; render(); + } catch { if (current === generation) { failures++; render(); } } finally { if (current === generation) { controller = null; - if (!text && active()) timer = setTimeout(load, 3000); + if (!text && active() && failures < 3) timer = setTimeout(() => { timer = null; void load(); }, 3000); } } } function sync() { if (!active()) { stop(); return; } - if (!text && !controller && !timer) load(); + if (!text && failures < 3 && !controller && !timer) load(); } // Resolves to whether this FreeVideo has the Agent interface on; a surface shows nothing when it is off. const enabled = agentInterfaceEnabled(api); diff --git a/web/agent_session.js b/web/agent_session.js index c4a69799..b2330045 100644 --- a/web/agent_session.js +++ b/web/agent_session.js @@ -37,3 +37,18 @@ export async function agentPost(api, path, body) { sessions.delete(api); } } + +// UI actions retry once, including HTTP refusals. Callers keep the same +// idempotency id/body and show the result where the user clicked. +export async function retryAgentPost(api, path, body, {retry = async () => body, accept = () => true} = {}) { + for (let attempt = 0; attempt < 2; attempt++) { + let response = null; + try { + response = await agentPost(api, path, body); + if (response?.ok && await accept(response)) return response; + } catch { /* Retry below, including an unreadable response body. */ } + if (attempt) return response?.ok ? null : response; + body = await retry(response); + if (!body) return response; + } +} diff --git a/web/autopilot.js b/web/autopilot.js index fe348b62..9453dd5d 100644 --- a/web/autopilot.js +++ b/web/autopilot.js @@ -68,7 +68,9 @@ export function openAutopilotView(node, {studio = null} = {}) { const job = talking ? last.current : null; now.hidden = !job; line.dataset.on = String(!!job); halt.disabled = !!last.stopping; - halt.title = last.stopping ? t('Stopping', '正在停止') : t('Stop all', '全部停止'); halt.setAttribute('aria-label', halt.title); + halt.setAttribute('aria-busy', String(!!last.stopping)); + halt.textContent = last.stopping ? t('Stopping…', '正在停止…') : ''; + halt.title = last.stopping ? t('Stopping…', '正在停止…') : t('Stop all', '全部停止'); halt.setAttribute('aria-label', halt.title); if (job) { const fraction = last.fraction || 0, running = job.status === 'running'; currentId = job.id; name.textContent = jobName(job, t); more.textContent = last.active > 1 ? `+${last.active - 1}` : ''; diff --git a/web/autopilot_chat.css b/web/autopilot_chat.css index db8f83b2..7f2fa371 100644 --- a/web/autopilot_chat.css +++ b/web/autopilot_chat.css @@ -109,6 +109,13 @@ .fv-studio .fv-chat-down{position:absolute;right:max(40px,calc(50% - 360px));bottom:96px;width:34px;height:34px;min-height:0;padding:0;border-radius:50%;background:var(--fv-raised);box-shadow:0 6px 18px #0008} .fv-chat svg{width:16px;height:16px;fill:none;stroke:currentColor;stroke-linecap:round;stroke-linejoin:round} .fv-chat-input{box-sizing:border-box;justify-self:center;width:100%;max-width:720px;min-height:54px;display:grid;grid-template-columns:minmax(0,1fr) auto;gap:10px;align-items:end;border:1px solid var(--fv-border);background:#0e151e;border-radius:14px;padding:8px 8px 8px 16px;transition:border-color var(--fv-quick),box-shadow var(--fv-quick)} +.fv-chat-notice{justify-self:center;width:100%;max-width:720px;margin:6px 0;font-size:14px;color:var(--fv-ink)} +.fv-chat-preview[data-loading=true] .fv-chat-play::before{border-color:var(--fv-accent);animation:fv-chat-breathe 1s infinite} +.fv-chat-job:is([data-mode=failed],[data-mode=stopped]){display:block;width:100%}.fv-chat-job:is([data-mode=failed],[data-mode=stopped]) .fv-chat-actions{margin-top:10px;flex-wrap:wrap} +.fv-studio .fv-chat-job:is([data-mode=failed],[data-mode=stopped]) .fv-result-links{justify-content:flex-start} +.fv-chat-job .fv-report-tip{max-width:100%} +.fv-chat-main{grid-template-rows:auto minmax(0,1fr) auto auto} +.fv-chat-tools{grid-row:1}.fv-chat-scroll{grid-row:2}.fv-chat-notice{grid-row:3}.fv-chat-input{grid-row:4} .fv-chat-input:focus-within{border-color:var(--fv-accent-dim);box-shadow:0 0 0 3px color-mix(in srgb,var(--fv-accent) 13%,transparent)} .fv-studio.fv-autopilot .fv-chat-input textarea,.fv-chat-input textarea{box-sizing:border-box;width:100%;height:36px;min-height:36px;max-height:none;padding:7px 0;border:0;border-radius:0;background:none;color:var(--fv-ink);font-size:14.5px;line-height:22px;resize:none;outline:none;overflow-y:auto} .fv-chat-input textarea::placeholder{color:var(--fv-disabled)} @@ -120,7 +127,7 @@ .fv-studio .fv-chat-conversations .fv-chat-conversation-row:has(> .fv-chat-more:not([hidden])) .fv-chat-conversation{padding-right:calc(var(--fv-chats-pad) + 36px)} .fv-studio .fv-chat-conversations .fv-chat-more,.fv-studio .fv-chat-conversation-row:is(:hover,:focus-within,[data-menu-open=true]) .fv-chat-more{width:36px;height:36px;min-height:36px;opacity:1;pointer-events:auto} .fv-chat[data-conversations=true]{grid-template-columns:minmax(0,1fr)} - .fv-chat[data-conversations=true] .fv-chat-main{grid-template-rows:auto minmax(0,1fr) auto} + .fv-chat[data-conversations=true] .fv-chat-main{grid-template-rows:auto minmax(0,1fr) auto auto} .fv-chat-tools:not([hidden]){display:flex;padding-top:12px;margin-bottom:-12px} .fv-studio .fv-chat-chats{position:relative;width:var(--fv-chats-toggle);height:var(--fv-chats-toggle);min-height:0;padding:0;border-radius:var(--fv-chats-radius);background:var(--fv-card)} .fv-chat-chats .fv-chat-running{position:absolute;top:5px;right:5px} diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index 39cccda6..6be090bf 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -1,9 +1,10 @@ // The conversation with the connected agent: what the user says, what the agent answers, and the agent's jobs as // cards at the place they happened. Reads and sends use the page session; the agent's token never enters the page. -import {agentGet, agentPost} from './agent_session.js'; +import {agentGet, agentPost, retryAgentPost} from './agent_session.js'; import {isolateDialogKeys, routeDialogKey} from './dialog_keys.js'; import {closeDialog} from './motion.js'; -import { estimatedFraction, remainingAbove, remainingShown } from './generation_progress.js'; +import { estimatedFraction, remainingAbove, remainingShown, createDiagnosticReportControl } from './generation_progress.js'; +import {createErrorReport} from './error_panel.js'; import {canDeleteConversation, conversationMenu, emptyConversations, hasConversations, mergeConversations, removeConversation, runningElsewhere, selectConversation} from './autopilot_conversations.js'; import {renderImageFailure} from './image_generation.js'; @@ -19,9 +20,26 @@ function glyph(path, width = 2.2) { const press = (cls, text, action) => { const b = el('button', cls, text); b.type = 'button'; b.onclick = action; return b; }; let chatNumber = 0; -const WORKING = ['running', 'queued', 'waiting'], VIDEOS = ['generate', 'assemble']; +const WORKING = ['running', 'queued', 'waiting', 'cancelling'], VIDEOS = ['generate', 'assemble']; // What a job with no picture of its own is doing, in the reader's language. const QUIET = {vlm: ['Checking', '检查中'], enhance: ['Polishing', '润色中']}; +const SEND_ATTEMPTS = 3, WAIT_FOR_AGENT = ['not_connected', 'not_listening']; +const GENERIC_FAILURE_ZH = '处理任务时出错，未能完成结果'; +// A listening/connection race is queued; other 4xx refusals never repeat automatically. +const sendRefused = (error, response) => error?.code === 'session_full' || error?.retryable === false + || response?.status >= 400 && response.status < 500 && !WAIT_FOR_AGENT.includes(error?.code); +async function sendError(response) { + if (!response) return null; + try { return (await (response.clone?.() || response).json())?.error || null; } + catch { return null; } +} +// Sends and failure cards use the same public reason, including the engine's Chinese mapping. +function failureReason(value, t) { + const error = value.error, reason = error?.error_zh || value.error_zh; + if (error?.code === 'session_full' && (!reason || reason === GENERIC_FAILURE_ZH)) + return t('This conversation is full. Start a new chat.', '这个对话已达到消息数量上限，请新建对话'); + return t(error?.message || 'The task failed.', reason || GENERIC_FAILURE_ZH); +} export const about = (seconds, t) => seconds >= 60 ? t(`About ${Math.ceil(seconds / 60)} min`, `约 ${Math.ceil(seconds / 60)} 分钟`) : t('Under 1 min', '不到 1 分钟'); export function clock(seconds) { const s = Math.max(0, Math.round(seconds)); return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`; } @@ -78,6 +96,7 @@ const SAVING = ['decode', 'finishing', 'assembling'], stepped = p => p.phase === // The one line that says where a job is. No step count is written unless the engine gave one. export function jobLine(job, t) { const p = job.progress || {}; + if (job.status === 'cancelling') return t('Stopping', '正在停止'); if (job.status !== 'running') return t('Queued', '排队中'); if (stepped(p)) return t(`Step ${p.step.done} / ${p.step.total}`, `生成 ${p.step.done} / ${p.step.total}`); if (SAVING.includes(p.phase)) return t('Saving', '保存中'); @@ -110,14 +129,14 @@ export function jobParts(job) { return Array.from({length: block.total}, (_, i) => i < (block.done || 0) ? 1 : i === block.index - 1 ? inside : 0); } -export const emptyChat = () => ({session: undefined, items: new Map(), jobs: {}, confirms: {}, last: 0, read: 0, agent: {}, current: null, active: 0}); +export const emptyChat = () => ({session: undefined, items: new Map(), jobs: {}, failures: {}, confirms: {}, last: 0, read: 0, agent: {}, current: null, active: 0}); // One answer from the server folded into what the page already has: entries add up by their number, jobs are // replaced by id (a card keeps its last state after the server stops listing its job), and so are the answers to its questions. export function chatMerge(state, data) { const items = new Map(state.items); - for (const item of data.items || []) if (['message', 'job', 'confirm'].includes(item.type) && Number.isFinite(item.seq)) items.set(item.seq, item); + for (const item of data.items || []) if (['message', 'job', 'confirm', 'production', 'cast'].includes(item.type) && Number.isFinite(item.seq)) items.set(item.seq, item); const read = Math.max(state.read, Number(data.read_seq) || 0, ...(data.items || []).filter(item => item.read === true).map(item => item.seq)); - return {session: data.session ?? null, items, jobs: {...state.jobs, ...(data.jobs || {})}, confirms: {...state.confirms, ...(data.confirms || {})}, read, + return {session: data.session ?? null, items, jobs: {...state.jobs, ...(data.jobs || {})}, failures: data.failures || {}, confirms: {...state.confirms, ...(data.confirms || {})}, read, last: Math.max(state.last, Number(data.last_seq) || 0, ...items.keys()), agent: data.agent || {}, current: data.current || null, active: Number(data.active) || 0}; } @@ -222,6 +241,8 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () input.rows = 1; input.maxLength = 8000; input.setAttribute('aria-label', t('Message the agent', '对 Agent 说')); sendButton.type = 'submit'; sendButton.setAttribute('aria-label', t('Send', '发送')); sendButton.append(glyph('M12 19V5M5 12l7-7 7 7')); form.append(input, sendButton); scroller.append(log); main.append(scroller, down, form); element.append(main); + const notice = el('p', 'fv-chat-notice'); notice.hidden = true; notice.setAttribute('role', 'status'); main.insertBefore(notice, form); + function say(text) { notice.textContent = text; notice.hidden = !text; } let dismissImage = null; let state = emptyChat(), pending = [], known = false, stopped = false, timer = null, busy = false, failures = 0; @@ -433,7 +454,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } } function chooseChat(session, focus = false) { - closeMenu(); + closeMenu(); say(''); showDraft(session); rememberSelection(session); selectedSession = session; selection++; pinned = true; away = null; conversations = selectConversation(conversations, session); renderConversations(); @@ -512,7 +533,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () // Whether the reader is at the newest entry. It holds while the conversation is out of sight, which forgets where it was scrolled. let pinned = true; const closed = new AbortController(); - const nodes = new Map(), answers = new Map(), stopping = new Set(), stopErrors = new Map(), retrying = new Set(), confirming = new Set(), confirmErrors = new Map(); + const nodes = new Map(), answers = new Map(), stopping = new Set(), retrying = new Set(), confirming = new Set(), confirmErrors = new Map(); let bulkStopping = false; const cardKey = (session, id) => JSON.stringify([session, id]); // The fraction each job shows: the estimate, never backwards, short of 100% until the video is saved. @@ -539,19 +560,20 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } const active = () => !stopped && element.isConnected && !document.hidden && shown(); const inSelectedChat = () => !creating && (!selectedSession || selectedSession === state.session); + const composeAllowed = () => inSelectedChat() && state.agent.connected === true; const sendingAllowed = () => inSelectedChat() && canSend(state); const draftSending = () => pending.some(entry => entry.sending && entry.draft && entry.draft === drafts.get(draftSession)); - async function cancelJob(job, session) { - if (stopping.has(job.id)) return; - stopping.add(job.id); stopErrors.delete(job.id); render(); report(); - try { - const response = await agentPost(api, '/freevideo/agent/activity/cancel', {id: job.id}); - if (!response.ok) throw await requestError(response); - } catch (error) { - stopping.delete(job.id); - stopErrors.set(job.id, {session, text: t('Could not stop "' + jobName(job, t) + '": ', '没有停止“' + jobName(job, t) + '”：') + reason(error)}); - } + async function cancelJobs(ids) { + if (!ids.length) { say(t('No tasks are running.', '当前没有在运行的任务')); return; } + ids = ids.filter(id => !stopping.has(id)); + if (!ids.length) return; + say(t('Stopping…', '正在停止…')); + ids.forEach(id => stopping.add(id)); render(); report(); + const results = await Promise.all(ids.map(id => retryAgentPost(api, '/freevideo/agent/activity/cancel', {id}))); + results.forEach((response, index) => { if (!response?.ok) stopping.delete(ids[index]); }); + if (results.some(response => !response?.ok)) say(t('Stopping failed after a retry. If still running, click again.', '停止没成功，已重试；仍在运行请再点一次')); + else say(''); if (!stopped) { render(); report(); void poll(); } } async function retryJob(key, text) { @@ -601,10 +623,8 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const middle = el('div', 'fv-chat-where'), parts = el('div', 'fv-chat-parts'), line = el('p', 'fv-chat-line'), part = el('span'), step = el('span'); line.append(part, step); middle.append(parts, line); const foot = el('div', 'fv-chat-actions'), left = el('span', 'fv-chat-left'); - const session = state.session; - const stop = press('fv-chat-stop', null, () => void cancelJob(job, session)); - const error = el('p', 'fv-chat-error'); error.setAttribute('role', 'alert'); - foot.append(left, stop); info.append(head, middle, foot, error); item.replaceChildren(box, info); + const stop = press('fv-chat-stop', null, () => void cancelJobs([job.id])); + foot.append(left, stop); info.append(head, middle, foot); item.replaceChildren(box, info); const said = {}; // The number rides on the water, and goes under the line once there is no room left above it. const level = next => { @@ -629,9 +649,9 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const where = cardLine(said, next, t); part.textContent = cells.length ? t(`Part ${block.index} / ${block.total}`, `第 ${block.index} / ${block.total} 段`) : where; step.textContent = cells.length ? where : ''; - error.textContent = stopErrors.get(next.id)?.text || ''; error.hidden = !error.textContent; - const going = stopping.has(next.id); - stop.disabled = going; stop.textContent = going ? t('Stopping', '正在停止') : t('Stop', '停止'); + const going = stopping.has(next.id) || next.status === 'cancelling'; + stop.disabled = going; stop.textContent = going ? t('Stopping…', '正在停止…') : t('Stop', '停止'); + stop.setAttribute('aria-busy', String(going)); stop.setAttribute('aria-label', t('Stop ', '停止 ') + jobName(next, t)); }; } @@ -661,6 +681,24 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () if (job.play) { // The video plays inside its frame, with its sound; the line under it follows and can be dragged. let video = null; + let playing = false, loadRetried = false; + const playbackFailure = () => { + if (item.dataset.mode !== 'done') return; + playback.textContent = t('Cannot play right now. Open it in Creations.', '暂时播放不了，可在作品库中打开'); playback.hidden = false; + show.textContent = t('Open in Creations', '在作品库打开'); if (!show.parentNode) foot.append(show); + if (!foot.parentNode) info.append(foot); + }; + const playback = el('p', 'fv-chat-playback'); playback.hidden = true; info.append(playback); + const play = async () => { + if (playing) return; + playing = true; box.dataset.loading = 'true'; + try { await video.play(); playback.hidden = true; } + catch { + if (loadRetried) playbackFailure(); + else try { loadRetried = true; video.load(); await video.play(); playback.hidden = true; } catch { playbackFailure(); } + } + finally { playing = false; delete box.dataset.loading; } + }; const toggle = press('fv-chat-play', null, () => { if (!video) { video = el('video'); video.playsInline = true; video.src = api.apiURL(job.play); @@ -669,10 +707,15 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () video.onloadedmetadata = () => shape(video.videoWidth, video.videoHeight); video.ontimeupdate = () => { if (video.duration) seek.value = String(Math.round(video.currentTime / video.duration * 1000)); }; video.onplay = video.onpause = video.onended = () => { box.dataset.playing = String(!video.paused && !video.ended); label(); }; + video.onerror = () => { + if (playing) return; + if (loadRetried) { playbackFailure(); return; } + loadRetried = true; video.load(); void play(); + }; seek.oninput = () => { if (video.duration) video.currentTime = seek.value / 1000 * video.duration; }; box.prepend(video); box.append(seek); box.dataset.video = 'true'; } - if (video.paused || video.ended) void video.play().catch(() => {}); else video.pause(); + if (video.paused || video.ended) { loadRetried = false; void play(); } else video.pause(); }); const label = () => toggle.setAttribute('aria-label', (box.dataset.playing === 'true' ? t('Pause ', '暂停 ') : t('Play ', '播放 ')) + jobName(job, t)); label(); box.append(toggle); @@ -693,27 +736,43 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () return {element: item, update(job, {latest, retry, key}) { // Jobs without a picture (a check, a prompt) are one quiet line while they run, and nothing after. // A picture takes seconds: the same quiet line while it is made, then the picture. - const card = VIDEOS.includes(job.kind || 'generate') || (job.kind === 'image' && !WORKING.includes(job.status)); + const card = VIDEOS.includes(job.kind || 'generate') || (['image', 'production', 'cast'].includes(job.kind) && !WORKING.includes(job.status)); const next = card ? jobMode(job) + (jobMode(job) === 'done' ? `:${job.play}:${job.thumbnail}:${job.work}` - : jobMode(job) === 'failed' ? `:${job.error?.code}:${job.error?.report_id}` : '') + : jobMode(job) === 'failed' && job.kind === 'image' ? `:${job.error?.code}:${job.error?.report_id}` : '') : WORKING.includes(job.status) ? 'quiet' : 'none'; if (next !== mode) { mode = next; item.dataset.job = job.id; item.dataset.mode = next.split(':')[0]; item.hidden = next === 'none'; if (job.kind === 'image') item.dataset.kind = 'image'; else delete item.dataset.kind; - if (next !== 'working') { stopping.delete(job.id); stopErrors.delete(job.id); } + if (!WORKING.includes(job.status)) stopping.delete(job.id); if (next === 'working') apply = working(item, job); else if (next.startsWith('done')) apply = done(item, job); - else if (next === 'quiet') { item.replaceChildren(el('span', 'fv-chat-dots'), el('span', null, t(...(QUIET[job.kind] || ['Working', '进行中'])))); apply = null; } - else if (next === 'stopped') { item.replaceChildren(el('span', null, t('Stopped', '已停止'))); apply = null; } - else if (next.startsWith('failed')) { - if (job.kind === 'image') { - const host = el('div'); - renderImageFailure(host, {error: job.error?.code || 'generation_failed', report_id: job.error?.report_id}, t, - {productVersion, fetchApi: (path, options) => api.fetchApi(path, options)}); - item.replaceChildren(host); - } else item.replaceChildren(el('span', null, t('Could not generate', '没有生成出来'))); + else if (next === 'quiet') { + const label = el('span'); item.replaceChildren(el('span', 'fv-chat-dots'), label); + apply = next => { label.textContent = stopping.has(next.id) || next.status === 'cancelling' + ? t('Stopping', '正在停止') : t(...(QUIET[next.kind] || ['Working', '进行中'])); }; + } + else if (next === 'stopped' && (!job.error || job.kind === 'image')) { item.replaceChildren(el('span', null, t('Stopped', '已停止'))); apply = null; } + else if (next.startsWith('failed') && job.kind === 'image') { + // Picture failures name their step with the picture's own report card. + const host = el('div'); + renderImageFailure(host, {error: job.error?.code || 'generation_failed', report_id: job.error?.report_id}, t, + {productVersion, fetchApi: (path, options) => api.fetchApi(path, options)}); + item.replaceChildren(host); if (retry) item.append(press('fv-chat-retry', t('Retry', '重试'), retry)); apply = null; + } + else if (next === 'failed' || next === 'stopped') { + const reason = el('p'), actions = el('div', 'fv-chat-actions fv-actions'), links = el('div', 'fv-result-links'); + let current = job; + const diagnostic = createDiagnosticReportControl(t, {api, tip: true, version: productVersion, + label: () => t('Download diagnostic report', '下载诊断报告'), + fallback: () => new Blob([JSON.stringify(createErrorReport({error: current.error, exception_type: current.error?.code, + details: current.error?.request_id}, {stage: 'execution'}), null, 2)], {type: 'application/json'})}); + actions.append(diagnostic.report); + if (retry) actions.append(press('fv-chat-retry', t('Retry', '重试'), retry)); + links.append(actions); item.replaceChildren(reason, links); + apply = next => { current = next; reason.textContent = failureReason(next, t); + diagnostic.updateReport({report_id: next.report_id}); }; } else { item.replaceChildren(); apply = null; } } apply?.(job, latest); @@ -771,7 +830,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () item.setAttribute('role', 'status'); item.setAttribute('aria-label', t('Thinking', '正在思考')); item.append(mark); return {element: item, update() {}}; } - const MAKE = {error: () => { const element = el('li', 'fv-chat-error'); element.setAttribute('role', 'alert'); return {element, update(text) { element.textContent = text; }}; }, hello, examples, dots, user: () => message('user'), agent: () => message('agent'), job: jobCard, confirm: confirmCard}; + const MAKE = {hello, examples, dots, user: () => message('user'), agent: () => message('agent'), job: jobCard, confirm: confirmCard}; function render(follow = false) { const entries = chatEntries(state), ids = new Set(entries.map(entry => entry.id).filter(Boolean)); @@ -792,12 +851,16 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const text = lastUser?.text; const key = cardKey(state.session, job.id); wanted.push([`s:${entry.seq}`, 'job', job, {latest: entry === finished, key, retry: text ? () => retryJob(key, text) : null}]); + const failure = state.failures[entry.entity]; + if (failure) wanted.push([`e:${entry.seq}`, 'job', failure]); + } else if (entry.type === 'production' || entry.type === 'cast') { + const failure = state.failures[entry.entity]; + if (failure) wanted.push([`s:${entry.seq}`, 'job', failure]); } else wanted.push([`s:${entry.seq}`, 'confirm', entry]); } for (const entry of localPending) wanted.push([`m:${entry.id}`, 'user', {...entry, taken: false}]); if (!said) wanted.push(['examples', 'examples', null]); // The agent has the sentence, or is waiting for one, and has not answered yet. - for (const [id, error] of stopErrors) if (error.session !== state.session) wanted.push([`stop:${id}`, 'error', error.text]); const tail = localPending.at(-1) || entries.at(-1); if (tail?.type !== 'job' && tail?.type !== 'confirm' && (localPending.some(entry => !entry.error) || !tail?.error && tail?.role === 'user') && state.agent.connected === true && (state.agent.listening !== false || tail.taken === true)) wanted.push(['dots', 'dots', null]); @@ -814,7 +877,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const sendable = sendingAllowed(); input.disabled = state.agent.connected !== true || creating || !!selectedSession && selectedSession !== state.session; input.placeholder = state.agent.connected === true && !sendable ? t('Ask your agent to continue', '请在 Agent 里让它继续') : t('Message the agent…', '对 Agent 说…'); - sendButton.disabled = !sendable || draftSending() || !input.value.trim(); + sendButton.disabled = !composeAllowed() || draftSending() || !input.value.trim(); if (focusSession === state.session && focusSession && !input.disabled && active()) { focusSession = null; input.focus(); } if (stick) { pinned = true; scroller.scrollTop = scroller.scrollHeight; down.hidden = true; } else if (log.children.length > before) down.hidden = false; @@ -847,34 +910,63 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () stopping: bulkStopping, any: state.items.size > 0 || pending.some(entry => entry.session === state.session)}); } function acknowledge(entry) { - entry.acknowledged = true; entry.error = null; + entry.acknowledged = true; entry.error = null; entry.automatic = false; + if (entry.session === state.session) say(''); unsaidSessions.delete(entry.session); if (entry.draft && drafts.get(entry.session) === entry.draft) { drafts.delete(entry.session); if (draftSession === entry.session) { input.value = ''; grow(); } } } - async function send(text, original = null, draft = null) { + const waitingNotice = code => (code ? code === 'not_listening' : state.agent.connected === true) + ? t('Not sent: Agent is not listening yet. It will send automatically once listening resumes.', '没发出去：Agent 还没在听，继续监听后会自动发送') + : t('Not sent: Agent is not connected. It will send automatically once connected.', '没发出去：Agent 还没连接，连接后会自动发送'); + async function send(text, original = null, draft = null, automatic = false) { text = String(text).trim(); - original ||= pending.find(entry => entry.session === (state.session ?? null) && entry.text === text && (entry.error || entry.sending)); - if (!text || !sendingAllowed() || original?.sending || original && original.session !== state.session) return false; + original ||= pending.find(entry => entry.session === (state.session ?? null) && entry.text === text && (entry.error || entry.sending || entry.automatic)); + if (!text || original?.sending || original && original.session !== state.session) return false; + if (!inSelectedChat()) { + say(creating ? t('Creating a new chat…', '正在新建对话…') : t('Loading this chat. Please send again shortly.', '正在读取对话，请稍后再发一次')); + return false; + } const entry = original || {id: newId(), text, session: state.session ?? null, draft}; if (draft) entry.draft = draft; if (!original) pending.push(entry); - entry.sending = true; entry.retried = !!original; entry.error = null; render(true); report(); + if (!automatic) entry.attempts = 0; + if (!sendingAllowed()) { + entry.automatic = true; entry.error = waitingNotice(); + say(entry.error); render(); report(); return false; + } + entry.sending = true; entry.retried = !!original; entry.error = null; entry.automatic = false; render(true); report(); try { const body = {id: entry.id, text: entry.text}; if (conversations.supported === true && entry.session) body.session = entry.session; - const response = await agentPost(api, '/freevideo/agent/chat', body); - if (!response.ok) throw await requestError(response); - const value = await response.json().catch(() => ({})); - acknowledge(entry); - if (!entry.session && value.session) entry.session = value.session; - if (!stopped) { void poll(); void readConversations(true); } - return true; - } catch (error) { + entry.attempts++; + const response = await retryAgentPost(api, '/freevideo/agent/chat', body, {retry: async response => { + if (stopped || entry.acknowledged || sendRefused(await sendError(response), response) || entry.attempts >= SEND_ATTEMPTS) return null; + entry.attempts++; return body; + }}); if (entry.acknowledged) return true; - entry.error = t('Could not send: ', '没有发送成功：') + reason(error); + if (response?.ok) { + const value = await response.json().catch(() => ({})); + acknowledge(entry); + if (!entry.session && value.session) entry.session = value.session; + if (!stopped) { void poll(); void readConversations(true); } + return true; + } + const error = await sendError(response), refused = sendRefused(error, response); + const waiting = !refused && (WAIT_FOR_AGENT.includes(error?.code) || !error && state.agent.connected !== true); + const exhausted = entry.attempts >= SEND_ATTEMPTS; + entry.automatic = waiting && !exhausted; + const explanation = refused || error && !waiting ? t('Not sent: ', '没发出去：') + failureReason({error}, t) + : exhausted ? t('Not sent: tried 3 times. Automatic sending stopped; please send again.', '没发出去：已尝试 3 次，已停止自动发送，请再发一次') + : waiting ? waitingNotice(error?.code) : t('The network is unstable. Retried; please send again.', '网络不稳，已重试，请再发一次'); + entry.error = waiting ? explanation : t('Could not send: ', '没有发送成功：') + (error?.message || explanation); + if (entry.session === state.session) { + if (waiting && error?.code === 'not_connected') state = {...state, agent: {connected: false, name: null}}; + if (waiting && error?.code === 'not_listening') state = {...state, agent: {...state.agent, listening: false}}; + say(explanation); + } return false; } finally { entry.sending = false; @@ -886,9 +978,19 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () // At most five lines; the box scrolls after that. const line = (typeof getComputedStyle === 'function' && parseFloat(getComputedStyle(input).lineHeight)) || 22; input.style.height = `${Math.min(input.scrollHeight, Math.round(line * 5) + 14)}px`; - sendButton.disabled = !sendingAllowed() || draftSending() || !input.value.trim(); + sendButton.disabled = !composeAllowed() || draftSending() || !input.value.trim(); } - input.addEventListener('input', () => { saveDraft(); grow(); if (unsaidSessions.has(draftSession)) renderConversations(); }); + input.addEventListener('input', () => { + saveDraft(); + // Editing a queued draft cancels its automatic delivery. Its original message remains available for manual retry. + for (const entry of pending) if (entry.automatic && entry.session === draftSession && entry.draft !== drafts.get(draftSession)) { + entry.automatic = false; + entry.error = t('Not sent: the queued draft changed. Retry to send the original.', '没发出去：排队草稿已修改，原消息需手动重试'); + say(''); + } + if (!input.value.trim()) say(''); + grow(); if (unsaidSessions.has(draftSession)) renderConversations(); + }); // Enter sends, Shift+Enter starts a new line; Enter that confirms an input-method candidate does neither. input.addEventListener('keydown', event => { if (event.key !== 'Enter' || event.shiftKey || event.isComposing || event.keyCode === 229) return; @@ -898,7 +1000,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () event.preventDefault(); const text = input.value; const draft = saveDraft(); - if (!text.trim() || !sendingAllowed() || draftSending()) return; + if (!text.trim() || !composeAllowed() || draftSending()) return; const failed = pending.find(entry => entry.session === state.session && entry.draft === draft && entry.error); void send(text, failed, draft); }); @@ -968,6 +1070,11 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () const idle = state.agent.connected !== true && !state.active; timer = setTimeout(poll, (idle ? Math.max(interval, 3000) : interval) * Math.min(4, 1 + failures)); render(); report(); + if (sendingAllowed()) for (const entry of [...pending]) { + if (entry.session !== state.session || !entry.automatic || entry.sending || entry.acknowledged) continue; + await send(entry.text, entry, entry.draft, true); + if (!sendingAllowed() || stopped) break; + } } // Between reads the estimate keeps moving, on the cards and in the page's header alike. const moving = setInterval(() => { @@ -990,10 +1097,9 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () }, async stopAll() { if (bulkStopping) return; - const jobs = new Map(Object.values(state.jobs).filter(job => WORKING.includes(job.status)).map(job => [job.id, {job, session: state.session}])); - for (const id of away?.ids || []) if (!jobs.has(id)) jobs.set(id, {job: away.jobs[id], session: away.session}); + const ids = [...new Set([...Object.values(state.jobs).filter(job => WORKING.includes(job.status)).map(job => job.id), ...(away?.ids || [])])]; bulkStopping = true; report(); - try { await Promise.all([...jobs.values()].map(({job, session}) => cancelJob(job, session))); } + try { await cancelJobs(ids); } finally { bulkStopping = false; if (!stopped) { report(); void poll(); } } }, stop() { closeMenu(); stopped = true; dismissImage?.(); closed.abort(); clearInterval(moving); clearTimeout(timer); timer = null; diff --git a/web/generation_progress.js b/web/generation_progress.js index e6314f01..e398d2a2 100644 --- a/web/generation_progress.js +++ b/web/generation_progress.js @@ -1,5 +1,5 @@ // One whole-video estimate and the same five rows in Studio and the node view. -import { saveReport } from './report_issue.js'; +import { saveReport, reportIssue, showReportTip } from './report_issue.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./generation_progress.css', import.meta.url).href; document.head.append(css); @@ -349,6 +349,59 @@ export function estimatedFraction(state, overall, stateAge, overallAge) { return candidate; } +// The video's report download, also used by Agent failure cards. A task that +// failed before creating an engine report supplies the shared redacted summary. +export function createDiagnosticReportControl(t, {api = null, fallback = null, tip = false, version = null, + label = () => t('Download report', '下载报告')} = {}) { + if (tip) { + const href = new URL('./result_actions.css', import.meta.url).href; + if (![...document.head.querySelectorAll('link')].some(link => link.href === href)) { + const style = document.createElement('link'); style.rel = 'stylesheet'; style.href = href; document.head.append(style); + } + } + const report = el('button', 'fv-report-download fv-quiet'); report.type = 'button'; report.hidden = true; + let reportId = null, revision = 0; + report.textContent = label(); + function updateReport(message) { + if (message.new_request === true || message.overall?.reset === true || message.reset === true && !message.phase) { + reportId = null; revision++; report.hidden = true; + } + if (typeof message.report_id === 'string' && /^[a-f0-9]{32}$/.test(message.report_id)) { + if (reportId !== message.report_id) { revision++; report.disabled = false; report.textContent = label(); } + reportId = message.report_id; + } + report.hidden = !reportId && !fallback; + return reportId; + } + async function fetchReport() { + if (reportId) { + const path = `/freevideo/report/${reportId}`; + const response = await (api ? api.fetchApi(path) : fetch(path)); + if (!response.ok) throw new Error('Report unavailable'); + return response.blob(); + } + if (fallback) return fallback(); + throw new Error('Report unavailable'); + } + async function downloadReport() { saveReport(await fetchReport()); } + report.onclick = async () => { + if (report.disabled || report.hidden) return; + const current = revision; + report.disabled = true; report.textContent = t('Preparing report…', '正在整理报告…'); + try { + await downloadReport(); + if (current === revision) { + report.textContent = label(); + if (tip && report.isConnected) showReportTip(report.parentElement, report, t, {mode: 'downloaded', + submit: () => reportIssue(fetchReport, typeof version === 'function' ? version() : version, {save: false})}); + } + } catch { + if (current === revision) report.textContent = t('Report unavailable · retry', '报告暂不可用 · 重试'); + } finally { if (current === revision) report.disabled = false; } + }; + return {report, updateReport, fetchReport, downloadReport, hasReport: () => reportId !== null}; +} + export function createGenerationProgress(t, now = () => Date.now(), {compact = false, api = null, onLayout = null} = {}) { const element = el('section', 'fv-generation-progress'); element.hidden = true; if (compact) element.classList.add('fv-generation-compact'); @@ -422,42 +475,8 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f const fraction = value => Math.min(1, Math.max(0, value)); const newRequest = message => message.new_request === true || message.overall?.reset === true || (message.reset === true && !message.phase); - const report = el('button', 'fv-report-download fv-quiet'); report.type = 'button'; report.hidden = true; - let reportId = null; - const reportLabel = () => t('Download report', '下载报告'); - report.textContent = reportLabel(); - function updateReport(message) { - if (newRequest(message)) { reportId = null; report.hidden = true; } - if (typeof message.report_id === 'string' && /^[a-f0-9]{32}$/.test(message.report_id)) { - if (reportId !== message.report_id) { report.disabled = false; report.textContent = reportLabel(); } - reportId = message.report_id; report.hidden = false; - } - // The saved video's request record is different from the redacted - // diagnostic download. Keep this token available after completion; - // only a new request clears it. - return reportId; - } - // The redacted diagnostic export; throws when it is unavailable. - async function fetchReport() { - const id = reportId; - if (!id) throw new Error('Report unavailable'); - const path = `/freevideo/report/${id}`; - const response = await (api ? api.fetchApi(path) : fetch(path)); - if (!response.ok) throw new Error('Report unavailable'); - return response.blob(); - } - async function downloadReport() { saveReport(await fetchReport()); } - report.onclick = async () => { - if (!reportId || report.disabled) return; - const id = reportId; - report.disabled = true; report.textContent = t('Preparing report…', '正在整理报告…'); - try { - await downloadReport(); - if (id === reportId) report.textContent = reportLabel(); - } catch { - if (id === reportId) report.textContent = t('Report unavailable · retry', '报告暂不可用 · 重试'); - } finally { if (id === reportId) report.disabled = false; } - }; + const diagnostic = createDiagnosticReportControl(t, {api}); + const {report, updateReport, fetchReport, downloadReport} = diagnostic; const hasCount = value => Number.isInteger(value.total) && value.total > 0 && Number.isInteger(value.done) && value.done >= 0 && value.done <= value.total; @@ -513,7 +532,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f const threshold = Math.max(floor, valid(typical) ? 2 * typical : 0); const shown = !ended && age >= threshold; const wasShown = !stall.hidden; - const hasReport = reportId !== null; + const hasReport = diagnostic.hasReport(); // The separate notice never adds another clock or blocks the run. if (shown && (stall.hidden || hasReport !== stallHasReport)) { const first = t('No progress for a while. Generation is still running, so you can keep waiting.', @@ -987,6 +1006,6 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f mini_fraction: visible && !mini.hidden && miniMode === 'determinate' ? miniShown : null, mini_shimmer: visible && !mini.hidden && miniShimmer}; } - return {element, stall, report, updateReport, fetchReport, downloadReport, hasReport: () => reportId !== null, update, hide, dispose: hide, + return {element, stall, report, updateReport, fetchReport, downloadReport, hasReport: diagnostic.hasReport, update, hide, dispose: hide, noticeCount: () => trims.size, stallShown: () => !stall.hidden, percentShown: () => !heading.hidden, snapshot}; } From cdb7ab67fbc6bb7f3b33da2e571b43887a1cb5ec Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:22 +0000 Subject: [PATCH 09/47] Let a waiting video move to the front of the queue --- freevideo_engine/chain_runner.py | 6 +++++ freevideo_engine/comfy_progress.py | 36 ++++++++++++++++++++++++++++++ web/studio.css | 4 +++- web/studio.js | 13 ++++++++++- web/studio_queue.js | 15 +++++++++++-- 5 files changed, 70 insertions(+), 4 deletions(-) diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index f48c478b..cf71f203 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -300,6 +300,12 @@ def _find(self, prompt_id, node): return record, block return None, None + def owns(self, prompt_id): + """True while a chain block submitted this host request; its order is the chain's.""" + with self.lock: + return any(row['prompt_id'] == prompt_id and row['state'] in ('queued', 'running') + for record in self.records.values() for row in record['blocks']) + def _start(self, block): if block['started'] is None: block['started'] = self.wall() diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.py index 32126c37..12b49a03 100644 --- a/freevideo_engine/comfy_progress.py +++ b/freevideo_engine/comfy_progress.py @@ -1,6 +1,7 @@ """Transient progress snapshots for reopened ComfyUI tabs; no request inputs.""" from copy import deepcopy from pathlib import Path +import heapq import threading import time import uuid @@ -242,6 +243,30 @@ def matches(row): return 'finished' +def move_request_first(queue, prompt_id, node_id, protected=lambda prompt_id: False): + """Put exactly this waiting FreeVideo request ahead of every other waiting one. + + Comfy runs the smallest queue number first, so the request takes a number + below the current smallest. Everything else keeps its relative order. + """ + with queue.mutex: + waiting = queue.queue + row = next((row for row in waiting if row[1] == prompt_id + and row[2].get(str(node_id), {}).get('class_type') == 'FreeVideoGenerate'), None) + if row is None: + running = any(item[1] == prompt_id for item in queue.currently_running.values()) + return 'running' if running else 'finished' + if protected(prompt_id): + return 'unavailable' + if row is min(waiting): + return 'first' + index = next(i for i, item in enumerate(waiting) if item is row) + waiting[index] = (min(item[0] for item in waiting) - 1,) + tuple(row[1:]) + heapq.heapify(waiting) + queue.server.queue_updated() + return 'moved' + + def register(): import asyncio from aiohttp import web @@ -279,3 +304,14 @@ async def cancel(request): import nodes return web.json_response({'status': cancel_request(server.prompt_queue, prompt_id, node_id, nodes.interrupt_processing)}) + + @server.routes.post('/freevideo/queue/first') + async def queue_first(request): + value = await request.json() + prompt_id, node_id = value.get('prompt_id'), value.get('node_id') + if not isinstance(prompt_id, str) or not prompt_id or not isinstance(node_id, str): + return web.json_response({'error': 'A request and node ID are required'}, status=400) + from . import comfy_chain + runner = comfy_chain.RUNNER + return web.json_response({'status': move_request_first(server.prompt_queue, prompt_id, node_id, + runner.owns if runner is not None else (lambda prompt_id: False))}) diff --git a/web/studio.css b/web/studio.css index f5992760..1a8b7b60 100644 --- a/web/studio.css +++ b/web/studio.css @@ -112,9 +112,11 @@ .fv-studio .fv-run-menu .fv-field{padding:10px 8px 0}.fv-studio .fv-run-menu p{margin:10px 8px 2px;line-height:1.6} .fv-studio .fv-queue-actions{display:flex;align-items:center;justify-content:center;gap:4px;flex-wrap:wrap;margin-top:8px} .fv-studio .fv-queue-actions button,.fv-studio .fv-queue-details>summary{padding:4px 8px;min-height:28px;font-size:var(--fv-micro);color:var(--fv-muted)} -.fv-studio .fv-queue-details{position:relative}.fv-studio .fv-queue-details>summary{cursor:pointer;list-style:none} +.fv-studio .fv-queue-actions{position:relative}.fv-studio .fv-queue-details{position:static}.fv-studio .fv-queue-details>summary{cursor:pointer;list-style:none} .fv-studio .fv-queue-list{left:0;right:auto;bottom:calc(100% + var(--fv-h-lg) + 24px);width:300px;max-height:260px;overflow:auto;scrollbar-width:thin} .fv-studio .fv-queue-item{display:flex;align-items:center;justify-content:space-between;gap:8px;font-size:var(--fv-micro);padding:3px 0;white-space:nowrap} +.fv-studio .fv-queue-item span{flex:1;min-width:0;overflow:hidden;text-overflow:ellipsis} +.fv-studio .fv-queue-item button{flex:none} .fv-studio .fv-status{min-height:22px;text-align:center;font-size:var(--fv-micro);color:var(--fv-muted);overflow-wrap:anywhere}.fv-studio .fv-status[data-error=true]{color:var(--fv-danger)} .fv-studio .fv-progress{width:100%;height:4px;accent-color:var(--fv-accent);border:0;margin:0 0 10px}.fv-studio .fv-budget,.fv-studio .fv-prewarm{text-align:center;font-size:var(--fv-micro);color:var(--fv-muted)} .fv-studio .fv-panel{height:auto;padding:0;border:none;border-radius:0;background:transparent;color:var(--fv-ink);font-size:var(--fv-micro);overflow:visible} diff --git a/web/studio.js b/web/studio.js index 702026ab..23f501e0 100644 --- a/web/studio.js +++ b/web/studio.js @@ -1019,6 +1019,7 @@ export function openStudio(node, {extendFrom = null} = {}) { const queueError = error => ({ queue_unavailable: t('Could not read the queue. Any loop has been stopped; already submitted videos stay queued.', '暂时无法读取队列。循环已停止追加，已提交的任务仍在队列中。'), cancel_failed: t('Could not cancel. Try again.', '暂时未能取消，请重试。'), + move_failed: t('Could not move it. Try again.', '暂时未能移动，请重试。'), submit_failed: t('The workflow was not queued. Check the node settings.', '工作流未能提交，请检查节点设置。'), loop_active: t('Stop the current loop before starting another.', '请先停止当前循环。'), loop_stopped: t('The last request did not complete. The loop has stopped.', '上一条任务未完成，循环已停止。'), @@ -1113,7 +1114,17 @@ export function openStudio(node, {extendFrom = null} = {}) { catch (error) { showQueueError(error); remove.disabled = false; } }, 'fv-quiet'); remove.setAttribute('aria-label', t(`Remove queued video ${index + 1}`, `移除第 ${index + 1} 条排队任务`)); - item.append(description, remove); queueList.append(item); + item.append(description); + if (index > 0) { + const first = button(t('Move to front', '排到最前'), async () => { + first.disabled = remove.disabled = true; + try { await queueController.moveFirst(row.id); } + catch (error) { showQueueError(error); first.disabled = remove.disabled = false; } + }, 'fv-quiet'); + first.setAttribute('aria-label', t(`Move queued video ${index + 1} to front`, `把第 ${index + 1} 条排队任务排到最前`)); + item.append(first); + } + item.append(remove); queueList.append(item); }); } if (!busy) cancelling = false; diff --git a/web/studio_queue.js b/web/studio_queue.js index 25a7bd74..dd7796a6 100644 --- a/web/studio_queue.js +++ b/web/studio_queue.js @@ -124,6 +124,16 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 if (!response.ok) throw new Error('cancel_failed'); return response.json(); } + // The page changes its list only after the server says where the request now is. + async function moveFirst(id) { + const response = await api.fetchApi('/freevideo/queue/first', {method: 'POST', + headers: {'Content-Type': 'application/json'}, body: JSON.stringify({prompt_id: id, node_id: nodeId})}); + if (!response.ok) throw new Error('move_failed'); + const reply = await response.json(); + if (reply.status === 'unavailable') throw new Error('move_failed'); + await refresh(); + return reply; + } async function submit(session, randomized) { let seed = session.snapshot.output[nodeId].inputs.seed; if (randomized) { @@ -176,7 +186,8 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 if (!response.ok) throw new Error('queue_unavailable'); const queue = await response.json(); running = (queue.queue_running || []).filter(matches).map(entry)[0] || null; - pending = (queue.queue_pending || []).filter(matches).map(entry); + // Comfy returns its heap array; the queue number gives the real running order. + pending = (queue.queue_pending || []).filter(matches).sort((a, b) => a[0] - b[0]).map(entry); for (const row of [running, ...pending]) if (row) remember(row.id); await advanceLoop(); publish(); } @@ -223,7 +234,7 @@ export function createStudioQueue(api, nodeId, {random = randomSeed, pollMs = 20 api.addEventListener('execution_interrupted', onFailure); api.addEventListener('executed', onExecuted); return { - state, refresh, start, stopLoop, + state, refresh, start, stopLoop, moveFirst, subscribe(listener) { listeners.add(listener); listener(state()); schedule(); return () => { listeners.delete(listener); schedule(); }; }, async cancel(id) { if (loop?.waiting === id) { loop.active = false; publish(); } From c14a48bb115994d90640b04f4b8a8e61ce8cd63d Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:22 +0000 Subject: [PATCH 10/47] Editor: once a cancel is accepted the generate bar reads Stopping and offers no second Cancel --- web/editor_genbar.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/web/editor_genbar.js b/web/editor_genbar.js index 2e2c459e..0bfb6904 100644 --- a/web/editor_genbar.js +++ b/web/editor_genbar.js @@ -84,10 +84,13 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { : ''; input.setAttribute('aria-label', t('Description', '描述')); generate.classList.toggle('fv-primary', selected && !busy); - cancel.hidden = !busy || opening; resume.hidden = !stopped; + // A cancel already accepted is not offered again; the bar says it is stopping until the run ends. + const cancelling = status?.state === 'cancelling' && !held; + cancel.hidden = !busy || opening || cancelling; resume.hidden = !stopped; const waiting = status?.state === 'queued' && !(status.blocks || []).some(block => ['queued', 'running'].includes(block.state) && block.prompt_id); if (opening) hint.textContent = t('Starting…', '正在开始…'); else if (held || waiting) hint.textContent = t('Queued', '排队中'); + else if (cancelling) hint.textContent = t('Stopping…', '正在停止…'); else if (running) { const blocks = status?.blocks || [], done = blocks.filter(block => block.state === 'done').length; const number = Math.min(blocks.length || 1, Math.max(done, status?.chainProgress?.index || 0) + 1), count = blocks.length || 1; @@ -178,7 +181,11 @@ export function editorBlockStates(job, messages = [], resolve = null) { if (resolve) return resolve(job, messages); const states = {pending: 'waiting', queued: 'waiting', running: 'generating', done: 'done', failed: 'failed', cancelled: 'stopped', interrupted: 'stopped'}; + // A part that never started in a run that was stopped or failed is stopped too, not still queued. + // Once a cancel is accepted (cancelling), such a part can only end cancelled, so it reads stopped at once. + const ended = ['failed', 'cancelled', 'interrupted', 'cancelling'].includes(job.state); return Object.fromEntries((job.blocks || []).map(block => { + if (ended && ['pending', 'queued'].includes(block.state)) return [block.id, {state: 'stopped', fraction: 0, remainingSeconds: null}]; const live = messages.find(message => message.chain?.id === job.id && message.chain?.block === block.id); const running = block.state === 'running' || block.state === 'queued'; const fraction = block.state === 'done' ? 1 : running && live From 30edc7f35623af11491dc81d9aaf91745256d743 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:22 +0000 Subject: [PATCH 11/47] Editor: a failed save before export names a full disk Editor: a failed save before export names a full disk; picking a clip without sound for Replace audio says so --- freevideo_engine/comfy_editor.py | 14 ++++++++++---- web/editor.js | 15 +++++++++------ web/editor_audio.js | 2 +- web/editor_audio_replace.js | 9 +++++---- web/editor_store.js | 13 ++++++++++--- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index 619b9b2e..57053c62 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -5,6 +5,7 @@ """ import asyncio import copy +import errno import hashlib import json import logging @@ -48,6 +49,11 @@ _LOG = logging.getLogger(__name__) +def _disk_full(error): + # The page names a full disk; other write errors stay a plain retry. + return {'reason': 'disk_full'} if getattr(error, 'errno', None) == errno.ENOSPC else {} + + class Conflict(ValueError): def __init__(self, revision, identity=None): super().__init__('Editor project has changed') @@ -1173,8 +1179,8 @@ async def guarded(request): return web.json_response(dict(error='Editor request is too large'), status=413, headers=_HEADERS) except FileNotFoundError: return web.json_response(dict(error='Editor item unavailable'), status=404, headers=_HEADERS) - except OSError: - return web.json_response(dict(saved=False, retry=True), status=503, headers=_HEADERS) + except OSError as error: + return web.json_response(dict(saved=False, retry=True, **_disk_full(error)), status=503, headers=_HEADERS) except (ValueError, TypeError, KeyError, RecursionError): return web.json_response(dict(error='Invalid editor request'), status=400, headers=_HEADERS) except Exception: @@ -1205,9 +1211,9 @@ async def persist(request): result = await asyncio.to_thread(save, root(), body.get('project'), body.get('create'), body.get('operation')) except MissingProject: raise - except OSError: + except OSError as error: # A missing staging directory is a retryable write error, not 404. - return web.json_response(dict(saved=False, retry=True), status=503, headers=_HEADERS) + return web.json_response(dict(saved=False, retry=True, **_disk_full(error)), status=503, headers=_HEADERS) return web.json_response(dict(project=result), headers=_HEADERS) @endpoint('get', '/freevideo/editor/sources') diff --git a/web/editor.js b/web/editor.js index 385f35fe..28bfca76 100644 --- a/web/editor.js +++ b/web/editor.js @@ -71,6 +71,11 @@ function exportErrorText(value, t) { // A bare status code means nothing to the user; the server's own words do. return message ? t(`Export failed: ${message}`, `导出时出错：${message}`) : t('Export failed. Try again.', '导出时出错，请再试一次。'); } +// The save route's own messages are internal English, so only a full disk is named. +function saveErrorText(value, t) { + return value?.reason === 'disk_full' ? t('Saving changes failed: the disk is full. Free some space, then export again.', '保存改动时出错：磁盘空间不够，空出空间后再导出。') + : t('Saving changes failed. Please export again.', '保存改动时出错，请再导出一次。'); +} function restoreGeneration(record, before, prepared) { const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); const entries = value => new Map(value.tracks.flatMap(track => track.clips.map(clip => [clip.id, {track: track.id, clip}]))); @@ -260,11 +265,12 @@ export function openEditor({api, app, from = null, panel = {}, components: given : t(en || 'Each continued part must be at least 10 seconds. Make it longer.', zh || '接着生成的每一段至少 10 秒，请加长一些。'); notify(text); return {ok: false, reason: code, say: text}; } - if (['busy', 'locked', 'text', 'audio_full', 'overlap', 'range'].includes(code)) { + if (['busy', 'locked', 'text', 'audio_full', 'overlap', 'range', 'no_audio'].includes(code)) { if (['locked', 'audio_full', 'overlap', 'range'].includes(code)) tracks?.flashBlocked?.(value.displayFrame); const said = {busy: t('This part is generating.', '这一段正在生成。'), locked: t('This track is locked.', '这条轨道已锁定。'), text: t('Write a description first.', '先写一句描述。'), overlap: t('That spot is taken. Use an empty spot.', '这里已经有片段了，请放到空白处。'), - audio_full: t('Both audio tracks are full here. Move an audio clip first.', '这里两条声音轨都满了，请先挪开一段声音。')}[code]; + audio_full: t('Both audio tracks are full here. Move an audio clip first.', '这里两条声音轨都满了，请先挪开一段声音。'), + no_audio: t('This clip has no sound. Please choose one with sound.', '这个素材没有声音，请选一段有声音的。')}[code]; if (said) notify(said); return {ok: false, reason: code}; } @@ -443,10 +449,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given await queue.flush(); } if (disposed || closing || epoch !== projectEpoch) return false; - if (queue.dirty) { - notify(t('Changes could not be saved after three tries. Check disk space, then export again.', '改动没有保存成功，已重试三次。请检查磁盘空间后再导出。')); - return false; - } + if (queue.dirty) { notify(saveErrorText(queue.failure, t)); return false; } return !!state.record.id; } function updateTop() { diff --git a/web/editor_audio.js b/web/editor_audio.js index 0a619697..ec0e2326 100644 --- a/web/editor_audio.js +++ b/web/editor_audio.js @@ -66,7 +66,7 @@ export function createAudioIntegration({state, player, t, error, adjust, library const unregisterMenu = () => { unregisterItem?.(); unregisterItem = null; stopMenu(); }; const stopMenu = state.subscribe(event => { if (event.type === 'record') syncMenu(); }); syncMenu(); - const detachPicker = library && store ? attachReplacePicker({state, library, store, actions: integration.actions, t}) : null; + const detachPicker = library && store ? attachReplacePicker({state, library, store, actions: integration.actions, t, error}) : null; const showJunction = () => { if (value.junction) { adjust?.element.querySelector('[data-adjust-tab="audio"]').click(); adjust?.open(); } }; if (adjust) { unregister = adjust.register('audio', { diff --git a/web/editor_audio_replace.js b/web/editor_audio_replace.js index f61e6d34..cf07a1dc 100644 --- a/web/editor_audio_replace.js +++ b/web/editor_audio_replace.js @@ -25,14 +25,15 @@ const payloadOf = (key, metadata) => { return {source: {kind: key.slice(0, separator), id: key.slice(separator + 1)}, frames: metadata.frames, hasAudio: true, hasVideo: false}; }; // The library cards keep their own gestures; while the mode is on, "add" gestures pick instead. -export function attachReplacePicker({state, library, store, actions, t}) { +export function attachReplacePicker({state, library, store, actions, t, error = () => {}}) { const mode = replaceMode(state); async function pick(key) { const id = mode.id; await store.getSources([key]); if (mode.id !== id || id === null) return; - const payload = payloadOf(key, store.sources.get(key)); - if (!payload) return; + const metadata = store.sources.get(key), payload = payloadOf(key, metadata); + // A pick that cannot replace says why; the mode stays on for another pick. + if (!payload) { error(metadata && !metadata._fallback ? {reason: 'no_audio'} : new Error('source')); return; } if (actions.replaceAudio(id, payload).ok !== false) mode.cancel(); } const choose = event => { @@ -40,7 +41,7 @@ export function attachReplacePicker({state, library, store, actions, t}) { const button = event.target.closest?.('.fv-ed-source-card')?.querySelector('.fv-ed-source'); if (!button || button.disabled) return; event.preventDefault(); event.stopImmediatePropagation(); - pick(button.dataset.key).catch(() => {}); + pick(button.dataset.key).catch(value => error(value)); }; const onClick = event => { if (event.target.closest?.('.fv-ed-source-add')) choose(event); }; // The keys a card answers to: Enter on the card, Enter or Space on its add button. diff --git a/web/editor_store.js b/web/editor_store.js index ea90d2cd..16f5e522 100644 --- a/web/editor_store.js +++ b/web/editor_store.js @@ -5,6 +5,11 @@ const clockDefault = {now: () => Date.now(), setTimeout: (fn, ms) => setTimeout( const copy = value => structuredClone(value); const memoryProjects = new Map(); const errorFor = response => Object.assign(new Error('editor_request'), {status: response.status}); +// A failed save keeps the server's reason so the page can say what went wrong. +async function saveFailure(response) { + let body = null; try { body = await response.json(); } catch {} + return Object.assign(errorFor(response), body && typeof body === 'object' ? {reason: body.reason, response: body} : {}); +} const post = value => ({method: 'POST', headers: {'Content-Type': 'application/json'}, body: JSON.stringify(value)}); export const sourceKey = source => source ? `${source.kind}:${source.id}` : null; import {chainRoute, createPoller, interpolate, capabilityPoints} from './chain_support.js'; @@ -57,7 +62,7 @@ export function createSaveQueue({write, read, onSaved = () => {}, onConflict = ( initial = null, getRecord = null, copySuffix = ' (copy)', session = null, create = null, attempt: resumedAttempt = null}) { const owner = session || holdSession(); let pending = null, inFlight = null, timer = null, heartbeat = null, meta = null, retry = 0, disposed = false; - let memory = false, memoryNoticed = false, lastSaved = initial && copy(initial), failed = false, copied = false, retained = false; + let memory = false, memoryNoticed = false, lastSaved = initial && copy(initial), failed = false, failure = null, copied = false, retained = false; let createOperation = create || (initial?.id ? null : operationId()); let attempt = resumedAttempt && copy(resumedAttempt); const localKey = owner.session; @@ -108,7 +113,7 @@ export function createSaveQueue({write, read, onSaved = () => {}, onConflict = ( } attempt = null; pending = {...latest, ...meta}; retain(pending); retry = 0; return; } - if (!response.ok) throw errorFor(response); + if (!response.ok) throw await saveFailure(response); const saved = (await response.json()).project; if (disposed) return; attempt = null; @@ -119,7 +124,7 @@ export function createSaveQueue({write, read, onSaved = () => {}, onConflict = ( if (pending) retain({...pending, ...meta}); else { saveOutbox(localKey, null); retained = false; clock.clearTimeout(heartbeat); heartbeat = null; } } catch (error) { - failed = true; + failed = true; failure = error; if (!disposed) { retain({...(pending || sent), ...(meta || {})}); schedule(Math.min(30000, 1000 * 2 ** retry++)); onError(error); @@ -154,6 +159,8 @@ export function createSaveQueue({write, read, onSaved = () => {}, onConflict = ( get recovery() { return {project: copy({...(pending || attempt?.project), ...(meta || {})}), baseline: copy(lastSaved), create: createOperation, attempt: copy(attempt)}; }, dispose({discard = false} = {}) { disposed = true; clock.clearTimeout(timer); timer = null; clock.clearTimeout(heartbeat); heartbeat = null; if (discard) saveOutbox(localKey, null); if (!session) owner.dispose(); }, get dirty() { return !!pending || !!attempt || !!inFlight; }, + // Why the last write failed, while that failure still stands. + get failure() { return failed ? failure : null; }, get locallyRetained() { return retained; }, }; } From 43f768d409b6234a88683b6eb3ae877e6bb4f279 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:31 +0000 Subject: [PATCH 12/47] Voice reference: keep it when the context falls back to the encoded ending Voice reference: keep it when the context falls back to the encoded ending; skip with preencoded conditioning --- freevideo_engine/cli.py | 2 + freevideo_engine/comfy_bridge.py | 6 +- freevideo_engine/comfy_chain.py | 4 +- freevideo_engine/comfy_nodes.py | 5 +- freevideo_engine/continuation.py | 124 +++++++++++++++++++++++++-- freevideo_engine/generate.py | 49 ++++++++++- freevideo_engine/prompt_vlm/rules.py | 9 ++ freevideo_engine/runtime.py | 17 +++- freevideo_engine/worker.py | 12 +++ 9 files changed, 211 insertions(+), 17 deletions(-) diff --git a/freevideo_engine/cli.py b/freevideo_engine/cli.py index 8da891a8..f60ab43e 100644 --- a/freevideo_engine/cli.py +++ b/freevideo_engine/cli.py @@ -97,6 +97,8 @@ def main(): help='With --continue-from: also give the model the first frame of the whole film as a ' 'reference picture. Reduces drift over many parts; holds framing and motion close to ' 'the opening, so leave it off when later parts should move away from it') + generate.add_argument('--voice-reference', action='store_true', + help='With --continue-from: use the opening six seconds as a voice reference when the prompt has dialogue') generate.add_argument('--continue-anchor', type=Path, metavar='IMAGE', help='Experimental: a reference picture of your own for a continuation, with no prompt note') generate.add_argument('--allocator-limit-gib', type=float, diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index 59eb37c1..d2fb88ca 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -1155,9 +1155,11 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, release_models=None, export_inputs=None, two_pass=True, encoder_prewarm=None, force_regenerate=False, base_steps=8, refine_steps=3, comfy_metadata=None, prompt_rewrite_report=None, preview=False, refine_from=None, report_extra=None, - continue_from=None, frames=None, first_frame_reference=False): + continue_from=None, frames=None, first_frame_reference=False, voice_reference=False): if type(first_frame_reference) is not bool: raise ValueError('First-frame reference must be a boolean') + if type(voice_reference) is not bool: + raise ValueError('Voice reference must be a boolean') if continue_from is not None: from .continuation import validate_route validate_route(two_pass=two_pass, preview=preview, refine_from=refine_from) @@ -1249,6 +1251,8 @@ def check_ending(): command += ['--continue-from', str(continue_from)] if first_frame_reference: command.append('--first-frame-reference') + if voice_reference: + command.append('--voice-reference') if continuation is not None: from .continuation import persist_snapshot continuation = persist_snapshot(continuation, run / 'continuation-source.safetensors') diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py index 18ceae9c..1e07903d 100644 --- a/freevideo_engine/comfy_chain.py +++ b/freevideo_engine/comfy_chain.py @@ -39,7 +39,7 @@ def seconds_for_frames(frames, minimum=1.625, maximum=60., step=.1): class ComfyHost: - continuation_inputs = tuple(CONTINUE_INPUTS.values()) + continuation_inputs = (*CONTINUE_INPUTS.values(), 'voice_reference') media_inputs = ('media', 'first', 'last', 'references', 'conditioning') def __init__(self, server): @@ -59,6 +59,8 @@ def inputs(self, block_inputs): # Each direction is filled once the node offers its input. if name in schema: values[name] = '' if independent else block_inputs[key] + if 'voice_reference' in schema and values.get('continue_from'): + values['voice_reference'] = True return values def work_exists(self, path): diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index eacab22f..983e4bd9 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -64,6 +64,8 @@ def define_schema(cls): 'Includes the 22 overlap frames when continuing; the saved MP4 contains only new frames.'), io.Boolean.Input('first_frame_reference', display_name='First-frame reference', default=False, optional=True, advanced=True, tooltip='Reduces drift.'), + io.Boolean.Input('voice_reference', display_name='Voice reference', default=False, + optional=True, advanced=True, tooltip='Keeps the opening voice consistent in continued dialogue.'), ], outputs=[io.Video.Output('video'), io.String.Output('report', display_name='Report JSON')], hidden=[io.Hidden.unique_id, io.Hidden.prompt, io.Hidden.extra_pnginfo], @@ -109,7 +111,7 @@ def fingerprint_inputs(cls, **kwargs): def execute(cls, text, width, height, seconds, seed, first=None, last=None, references=None, loras=None, conditioning=None, media=None, two_pass=True, force_regenerate=False, base_steps=8, refine_steps=3, preview=False, refine_from='', - continue_from='', frames=0, first_frame_reference=False): + continue_from='', frames=0, first_frame_reference=False, voice_reference=False): from server import PromptServer from .comfy_chain import block_context, block_finished, report_fields node_id = cls.hidden.unique_id @@ -221,6 +223,7 @@ def release(): **options, # Only a part that continues another can use it; other requests keep the existing bridge call. **({'first_frame_reference': True} if first_frame_reference is True and options.get('continue_from') else {}), + **({'voice_reference': True} if voice_reference is True and options.get('continue_from') else {}), **({'report_extra': {'chain': report_fields(chain)}} if chain is not None else {})) relative = output.relative_to(output_root) report = output.with_suffix('.request.json').read_text(encoding='utf-8') diff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.py index b262bb04..1905c886 100644 --- a/freevideo_engine/continuation.py +++ b/freevideo_engine/continuation.py @@ -22,6 +22,8 @@ 'location, framing and colour tone as in that image throughout.') FIRST_FRAME_MAX_BYTES = 8 * 1024 * 1024 FIRST_FRAME_MAX_SIDE = 4096 +FIRST_SOUND_MAX_BYTES = 4 * 1024 * 1024 +FIRST_SOUND_SECONDS = 6 # The picture the model is given is half the side of the film's first frame, but not under this # on its short side: on a real six-part chain it held framing and colour as well as the full # picture for a third of the extra sampling time. Smaller pictures are not tested. @@ -68,13 +70,18 @@ def validate_frames(frames): def validate_route(*, task='t2va', two_pass=True, preview=False, refine_from=None, - media=None, backend=None, anchor=False): + media=None, backend=None, anchor=False, sound=False): media = media or {} references = media.get('references') valid_reference = (anchor and isinstance(references, list) and len(references) == 1 and isinstance(references[0], dict) and references[0].get('kind') == 'image') - if (not supported()[0] or backend not in (None, 'cuda') or task != ('ref2va' if anchor else 't2va') + valid_task = task == ('ref2va' if anchor else 't2va') + if sound: + valid_reference = (not anchor and isinstance(references, list) and len(references) == 1 + and isinstance(references[0], dict) and references[0].get('kind') == 'audio') + valid_task = task in ('t2va', 'ref2va', 'ref2va_audio') and valid_reference + if (not supported()[0] or backend not in (None, 'cuda') or not valid_task or not two_pass or preview or refine_from or any(media.get(key) for key in ('first', 'last', 'conditioning_info')) or (references and not valid_reference)): @@ -198,6 +205,101 @@ def _chain_first_frame(request, video_path): return pack_first_frame(first_frame_png(video_path)) +def pack_first_sound(raw): + """A bounded stereo PCM16 WAV, starting at the film's first audio sample.""" + import io + import wave + try: + if not isinstance(raw, bytes) or len(raw) > FIRST_SOUND_MAX_BYTES: + raise ValueError('Invalid first sound') + with wave.open(io.BytesIO(raw), 'rb') as source: + rate, samples = source.getframerate(), source.getnframes() + if (source.getnchannels() != 2 or source.getsampwidth() != 2 or source.getcomptype() != 'NONE' + or rate <= 0 or not 0 < samples <= round(FIRST_SOUND_SECONDS * rate) + or len(source.readframes(samples + 1)) != samples * 4): + raise ValueError('Invalid first sound') + return dict(format='wav', rate=rate, data=base64.b64encode(raw).decode('ascii')) + except (ValueError, EOFError, wave.Error, OSError): + raise ValueError('Invalid first sound') from None + + +def unpack_first_sound(value): + """Recheck the record's WAV, including its declared rate and decoded size.""" + try: + if (value['format'] != 'wav' or type(value['rate']) is not int + or not isinstance(value['data'], str) + or len(value['data']) > (FIRST_SOUND_MAX_BYTES + 2) // 3 * 4): + raise ValueError('Invalid first sound') + raw = base64.b64decode(value['data'], validate=True) + if pack_first_sound(raw) != value: + raise ValueError('Invalid first sound') + return raw + except (KeyError, TypeError, ValueError, OSError): + raise ValueError('Invalid first sound') from None + + +def first_sound_wav(video_path): + """Decode at most six seconds, at the source rate; duplicate mono without changing its gain.""" + import io + import wave + import av + with av.open(str(video_path)) as container: + stream = container.streams.audio[0] + rate = stream.codec_context.sample_rate + if rate <= 0: + raise ValueError('Invalid first sound') + # Read one sample past the size limit to reject an oversized opening, while still + # accepting a short high-rate source whose actual WAV fits within the bound. + limit = (FIRST_SOUND_MAX_BYTES - 44) // 4 + count = min(round(FIRST_SOUND_SECONDS * rate), limit + 1) + saved = 0 + mono = len(stream.codec_context.layout.channels) == 1 + resampler = av.AudioResampler(format='s16', layout='mono' if mono else 'stereo', rate=rate) + chunks = [] + def keep(frame): + nonlocal count, saved + samples = frame.to_ndarray().reshape(-1, 1 if mono else 2)[:count] + if mono: + samples = samples.repeat(2, axis=1) + chunks.append(samples.astype(' limit: + raise ValueError('Invalid first sound') + for frame in container.decode(stream): + for converted in resampler.resample(frame): + keep(converted) + if count == 0: + break + if count: + for converted in resampler.resample(None): + keep(converted) + target = io.BytesIO() + with wave.open(target, 'wb') as output: + output.setparams((2, 2, rate, 0, 'NONE', 'not compressed')) + output.writeframes(b''.join(chunks)) + return target.getvalue() + + +def chain_first_sound(request, video_path=None): + """Carry the opening sound unchanged; only the first part may supply it from its own file.""" + try: + metadata = _snapshot_value(request)['metadata'] + if metadata.get('first_sound') is not None: + unpack_first_sound(metadata['first_sound']) + return metadata['first_sound'] + if metadata['chain']['segment'] != 1 or video_path is None: + return None + from .continuation_store import video_id + if video_id(video_path) != request.get('source_id'): + raise ValueError(MISSING) + return pack_first_sound(first_sound_wav(video_path)) + except (ValueError, MemoryError): + raise + except Exception: + raise ValueError('Invalid first sound') from None + + def validate_path(video_path): path = Path(video_path) if not path.is_absolute() or path.suffix.lower() != '.mp4' or not path.is_file(): @@ -375,13 +477,16 @@ class Prefix: update. Audio is either kept clean or re-noised at each call, and restored on return. Only re-noising owns an independent RNG. """ - def __init__(self, transformer, data, seed, device, *, video_offset=0): + def __init__(self, transformer, data, seed, device, *, video_offset=0, audio_offset=0): import torch from diffusers.modular_pipelines.minimax_h3.before_denoise import patchify_video_latents self.audio_context = validate_audio_context(data.get('audio_context', AUDIO_CONTEXT_DEFAULT)) if type(video_offset) is not int or video_offset < 0: raise ValueError('Invalid continuation reference row offset') self.video_offset = video_offset + if type(audio_offset) is not int or audio_offset < 0: + raise ValueError('Invalid continuation reference row offset') + self.audio_offset = audio_offset self.video = data['video'].to(device=device, dtype=torch.float32) self.audio = data['audio'].to(device=device, dtype=torch.float32) if (self.video.ndim != 5 or self.video.shape[:3] != (1, transformer.config.in_channels, LATENT_FRAMES) @@ -400,18 +505,20 @@ def before(self, model, args, kwargs): import torch video, audio = kwargs['hidden_states'][0], kwargs['audio_hidden_states'][0] count, ticks = self.video_rows.shape[0], self.audio.shape[2] - if video.shape[0] <= self.video_offset + count or audio.shape[0] // 2 < ticks: + generated = audio.shape[0] - self.audio_offset + if video.shape[0] <= self.video_offset + count or generated // 2 < ticks: raise ValueError(MISSING) self.current_video = video self.restore_video() - audio_rows = torch.cat([torch.arange(ticks, device=audio.device) + channel * (audio.shape[0] // 2) + audio_rows = torch.cat([torch.arange(ticks, device=audio.device) + self.audio_offset + channel * (generated // 2) for channel in range(2)]) row_times = kwargs['timestep'][kwargs['timestep_indices']].clone() if self.audio_context == 'clean': audio[audio_rows] = self.audio_rows row_times[kwargs['audio_indices'][audio_rows]] = CLEAN_T else: - audio_t = row_times[kwargs['audio_indices'][0]] + # The first generated row: a voice reference ahead of it stays clean. + audio_t = row_times[kwargs['audio_indices'][self.audio_offset]] # scale_noise uses H3's clean-at-one convention, not a standard DDPM clock. audio[audio_rows] = self.scheduler.scale_noise(self.audio_rows, audio_t, self.noise) row_times[kwargs['video_indices'][self.video_offset:self.video_offset + count]] = CLEAN_T @@ -429,12 +536,13 @@ def finish(self, video, audio): @contextmanager -def sampling(transformer, data, seed, device, *, video_offset=0): +def sampling(transformer, data, seed, device, *, video_offset=0, audio_offset=0): if data is None: yield None return # Keep the legacy constructor call identical for existing continuations. - prefix = Prefix(transformer, data, seed, device, **({'video_offset': video_offset} if video_offset else {})) + prefix = Prefix(transformer, data, seed, device, **({'video_offset': video_offset} if video_offset else {}), + **({'audio_offset': audio_offset} if audio_offset else {})) handle = transformer.register_forward_pre_hook(prefix.before, with_kwargs=True, prepend=True) try: yield prefix diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index 47f7275e..c581f6d9 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -8,6 +8,7 @@ import argparse import json import os +import re from pathlib import Path import signal import subprocess @@ -364,7 +365,7 @@ def usable_continuation(value, *check): raise # A record of another version, other weights or another first-pass size. value = dict(encoded_continuation(continue_from, canvas), - **{key: value[key] for key in ('audio_context', 'source_video', 'anchor') if key in value}) + **{key: value[key] for key in ('audio_context', 'source_video', 'anchor', 'sound') if key in value}) return freeze_continuation(value) if continuation_directory is not None else value frames = getattr(args, 'frames', None) @@ -374,6 +375,8 @@ def usable_continuation(value, *check): raise ValueError('--continue-anchor requires --continue-from') # Every part of a long video is asked alike; only a part that continues another can use it. first_frame = None + voice_reference = (dict(requested=True, used=False, reason=None) + if getattr(args, 'voice_reference', False) and continue_from is not None else None) if getattr(args, 'first_frame_reference', False) and continue_from is not None: if continue_anchor is not None: raise ValueError('Choose --first-frame-reference or --continue-anchor, not both') @@ -446,6 +449,20 @@ def usable_continuation(value, *check): first_frame.update(used=True, width=picture_width, height=picture_height) if continue_anchor is not None: continuation['anchor'] = media['references'][0] + if voice_reference is not None: + from .prompt_vlm.rules import split_source + if first_frame is not None: + voice_reference['reason'] = 'first_frame' + elif continue_anchor is not None: + voice_reference['reason'] = 'anchor' + elif args.conditioning: + # Preencoded conditioning cannot take another reference. + voice_reference['reason'] = 'conditioning' + elif not args.prompt_file or not split_source(args.prompt_file.read_text(encoding='utf-8'))[2]: + voice_reference['reason'] = 'no_lines' + elif re.search(r'(?i)\b(?:sing|sings|singing|sang|sung|lyrics)\b|唱', args.prompt_file.read_text(encoding='utf-8')): + # Quoted lyrics read as lines, but on music the opening sound did not hold steady. + voice_reference['reason'] = 'singing' active_loras = [row for row in media.get('loras', []) if row['strength'] != 0] if media: inferred_task = task_for(media) @@ -531,6 +548,8 @@ def usable_continuation(value, *check): report['continue_anchor'] = continuation['anchor'] if first_frame is not None: report['first_frame_reference'] = first_frame + if voice_reference is not None: + report['voice_reference'] = voice_reference from .support_report import code_identity, write as write_debug report['runtime_hardware'] = hardware.to_dict() report['runtime_code'] = code_identity() @@ -565,6 +584,31 @@ def interrupted(signum, frame): if args._lora_max_block_bytes or args._lora_root_bytes: profile.setdefault('policy', {})['lora_max_block_bytes'] = args._lora_max_block_bytes profile['policy']['lora_root_bytes'] = args._lora_root_bytes + if voice_reference is not None and voice_reference['reason'] is None: + # Model identity, first-pass size and LoRA admission may have switched this + # request to tail encoding. Decide only after those checks, before encoding media. + if 'tail' in continuation: + voice_reference['reason'] = 'source_predates' + else: + from .continuation import chain_first_sound, unpack_first_sound + try: + kept = chain_first_sound(continuation, continue_from) + sound = unpack_first_sound(kept) if kept is not None else None + except ValueError: + sound = None + if sound is None: + voice_reference['reason'] = 'unreadable' + else: + path = destination.with_suffix('.first-sound.wav') + path.write_bytes(sound) + from .media_request import digest + media = dict(media, version=1, references=[dict(kind='audio', path=str(path), + sha256=digest(path), bytes=path.stat().st_size)]) + validate_route(task=getattr(args, 'task', None) or 't2va', sound=True, two_pass=two_pass, + preview=preview, refine_from=getattr(args, 'refine_from', None), media=media) + continuation['sound'] = media['references'][0] + args.task = profile['engine']['task'] = task_for(media) + voice_reference['used'] = True resource_identity = local_identity(hardware, args.cache) history = ResourceHistory(history_path()) history.recover_pending() @@ -729,7 +773,8 @@ def interrupted(signum, frame): staged=bool(preview or refine_source)) report['sampling_plan'] = sampling_plan if continuation is not None: - validate_route(task=profile['engine'].get('task', 't2va'), anchor=continue_anchor is not None) + validate_route(task=profile['engine'].get('task', 't2va'), anchor=continue_anchor is not None, + **(dict(sound=True, media=media) if voice_reference and voice_reference['used'] else {})) continuation = usable_continuation(continuation, canvas, json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8')), sampling_plan) if 'tail' in continuation: diff --git a/freevideo_engine/prompt_vlm/rules.py b/freevideo_engine/prompt_vlm/rules.py index 7b6c5ced..32d576bb 100644 --- a/freevideo_engine/prompt_vlm/rules.py +++ b/freevideo_engine/prompt_vlm/rules.py @@ -204,6 +204,15 @@ def spoken_lines(scene): return lines +def split_source(text): + """Description, sound sections and spoken lines, using only the prompt's text.""" + sound = re.search(r'(?m)^[ \t]*overall_soundscape[ \t]*:', text) + scene, soundscape = (text[:sound.start()], text[sound.start():]) if sound else (text, '') + tagged = re.findall(r'<(?:d|en|zh|ja|ko|cn)>\s*(.*?)\s*', scene, re.S) + lines = [line for line in tagged if plain(line)] + spoken_lines(scene) + return scene, soundscape, list(dict.fromkeys(lines)) + + def dialogue(text, scene): """Turn the user's spoken lines that came back quoted into H3 dialogue, verbatim. diff --git a/freevideo_engine/runtime.py b/freevideo_engine/runtime.py index 64188b4f..b8b929ca 100644 --- a/freevideo_engine/runtime.py +++ b/freevideo_engine/runtime.py @@ -550,7 +550,9 @@ def _sample(self, conditioning, seed, frames=243, width=1344, height=768, *, fin refining = initial_latents is not None if continuation is not None: from .continuation import validate_route - validate_route(task=self.task, anchor=bool(continuation.get('anchor'))) + # Audio references are admitted below, after inspecting the encoded modalities. + if self.task != 'ref2va_audio': + validate_route(task=self.task, anchor=bool(continuation.get('anchor'))) if tuple(continuation['video'].shape[-2:]) != (height // 16, width // 16): from .continuation import SIZE_MISMATCH raise ValueError(SIZE_MISMATCH) @@ -584,7 +586,7 @@ def _sample(self, conditioning, seed, frames=243, width=1344, height=768, *, fin progress = SamplingProgress(active_steps, len(self.transformer.transformer_blocks), offset=progress_offset, total=progress_total) progress.start() - continuation_offset = 0 + continuation_offset = continuation_audio_offset = 0 early_receipt = None condition_schedule = None sampling_kwargs = {} @@ -616,12 +618,18 @@ def _sample(self, conditioning, seed, frames=243, width=1344, height=768, *, fin sampling_kwargs.update(condition_schedule=condition_schedule, condition_trace=early_receipt['steps']) conditions = value.get('references') or None if continuation is not None: - if conditions is None or len(conditions) != 1 or conditions[0]['kind'] != 'image' or conditions[0].get('audio_latent') is not None: + sound = (not continuation.get('anchor') and self.task == 'ref2va_audio' + and conditions is not None and len(conditions) == 1 + and conditions[0]['kind'] == 'audio' and conditions[0].get('audio_latent') is not None) + if not sound and (conditions is None or len(conditions) != 1 or conditions[0]['kind'] != 'image' or conditions[0].get('audio_latent') is not None): raise ValueError('Continuation anchor requires exactly one image reference without audio') + if sound: + validate_route(task=self.task, sound=True, media=dict(references=conditions)) # Ref2VA packs independent-size references before generated # video rows in both passes. Their latents do not need the # keyframe resize used by first/last-frame conditioning. continuation_offset = info['reference_video_tokens'] + continuation_audio_offset = info['reference_audio_tokens'] prompt, tags = value['prompt_embeds'].to(self.device_backend.device, torch.bfloat16), value['text_token_tags'] else: prompt, tags, conditions = load_prompt(str(conditioning), self.device_backend.device) @@ -821,7 +829,8 @@ def append(self, value): with progress.layers(self.transformer.transformer_blocks), self.device_backend.make_offloader( self.offload_layers, prefetch=self.prefetch, weight_source=self.stream_weight_source) as offloader, sampling( self.transformer, continuation, seed, self.device_backend.device, - **({'video_offset': continuation_offset} if continuation_offset else {})) as prefix, \ + **({'video_offset': continuation_offset} if continuation_offset else {}), + **({'audio_offset': continuation_audio_offset} if continuation_audio_offset else {})) as prefix, \ (watch.layers(self.transformer.transformer_blocks) if watch is not None else nullcontext()): latents, audio = generate_latents(self.transformer, prompt, tags, frames, self.steps, seed, self.device_backend.device, step_seconds=step_seconds, diff --git a/freevideo_engine/worker.py b/freevideo_engine/worker.py index 537c0c09..8cc52ea5 100644 --- a/freevideo_engine/worker.py +++ b/freevideo_engine/worker.py @@ -104,6 +104,8 @@ def released(engine): # single-pass request does not. validate_route(task=request['engine_options'].get('task', 't2va'), anchor=bool(continuation.get('anchor')), + **(dict(sound=True, media=dict(references=[continuation['sound']])) + if continuation.get('sound') else {}), two_pass=bool(request.get('sampling_plan', {}).get('requested')), preview=request.get('preview', False), refine_from=request.get('resume_first_pass')) if isinstance(getattr(torch.version, 'hip', None), str): @@ -652,6 +654,16 @@ def keep_audio_lead_in(value): 'OutOfMemoryError', 'CancelledError', 'InterruptProcessingException'): raise metrics['first_frame_kept'] = dict(saved=False, error=type(error).__name__) + try: + from .continuation import chain_first_sound + sound = chain_first_sound(continuation, continuation.get('source_video')) + if sound is not None: + continuation_data['metadata']['first_sound'] = sound + except Exception as error: + if isinstance(error, MemoryError) or type(error).__name__ in ( + 'OutOfMemoryError', 'CancelledError', 'InterruptProcessingException'): + raise + metrics['first_sound_kept'] = dict(saved=False, error=type(error).__name__) try: from .continuation_store import save as save_continuation metrics['continuation_store'] = dict(save_continuation(request['output'], continuation_data), saved=True) From abc34d479a5652bd42d3729dd893cd633badf4d1 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:47 +0000 Subject: [PATCH 13/47] Hidden tails: hide 34 frames so dialogue parts keep moving through the seam --- freevideo_engine/adaptive.py | 6 ++- freevideo_engine/chain_runner.py | 1 + freevideo_engine/cli.py | 2 + freevideo_engine/comfy_assets.py | 4 ++ freevideo_engine/comfy_bridge.py | 28 +++++++++++-- freevideo_engine/comfy_chain.py | 4 +- freevideo_engine/comfy_nodes.py | 5 ++- freevideo_engine/continuation.py | 35 ++++++++++++---- freevideo_engine/decode.py | 25 +++++++++--- freevideo_engine/decode_resume.py | 2 + freevideo_engine/generate.py | 67 +++++++++++++++++++++++++++---- freevideo_engine/worker.py | 14 +++++-- 12 files changed, 160 insertions(+), 33 deletions(-) diff --git a/freevideo_engine/adaptive.py b/freevideo_engine/adaptive.py index 6fa6011f..37d9a9b8 100644 --- a/freevideo_engine/adaptive.py +++ b/freevideo_engine/adaptive.py @@ -660,11 +660,13 @@ def complete_observation(metrics, canvas, output, artifacts, telemetry): from .validation import _retained_array_header, finite_number expected = canvas['frames'] continuation = metrics.get('continuation') - output_frames = expected + from .continuation import validate_hide_tail + hidden = validate_hide_tail(metrics.get('hide_tail_frames', 0)) + output_frames = expected - hidden if continuation is not None: from .continuation import OVERLAP_FRAMES if (not isinstance(continuation, dict) or continuation.get('overlap_frames') != OVERLAP_FRAMES - or continuation.get('new_frames') != expected - OVERLAP_FRAMES): + or continuation.get('new_frames') != expected - hidden - OVERLAP_FRAMES): raise ValueError('Invalid continuation output geometry') output_frames = continuation['new_frames'] preview = metrics.get('preview') is True diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index cf71f203..4ab52cf3 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -503,6 +503,7 @@ def _step_chain(self, identity): self._save(record) prompt = deepcopy(record['snapshot']['prompt']) block_inputs = dict(block['inputs']) + block_inputs.update(index=block['index'], count=len(record['blocks'])) sources = self._sources(record, block) extra = deepcopy(record['snapshot']['extra_data']) if record['client_id'] is not None: diff --git a/freevideo_engine/cli.py b/freevideo_engine/cli.py index f60ab43e..2059597d 100644 --- a/freevideo_engine/cli.py +++ b/freevideo_engine/cli.py @@ -97,6 +97,8 @@ def main(): help='With --continue-from: also give the model the first frame of the whole film as a ' 'reference picture. Reduces drift over many parts; holds framing and motion close to ' 'the opening, so leave it off when later parts should move away from it') + generate.add_argument('--hide-tail', action='store_true', + help='Generate and discard 34 extra frames for dialogue segments.') generate.add_argument('--voice-reference', action='store_true', help='With --continue-from: use the opening six seconds as a voice reference when the prompt has dialogue') generate.add_argument('--continue-anchor', type=Path, metavar='IMAGE', diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index 88d142ea..844b379c 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -162,6 +162,10 @@ def output_summary(report, relative_video): video=relative_video.as_posix(), report=relative_video.with_suffix( '.debug.json' if report.get('diagnostic_file') == relative_video.with_suffix('.debug.json').name else '.request.json').as_posix()) + if type(report.get('output_frames')) is int and report['output_frames'] > 0: + # Resource reports keep the generated canvas; result cards show only saved frames. + result['geometry'].update(frames=report['output_frames'], + seconds=report['output_frames'] / (result['geometry'].get('fps') or 24)) checkpoint = encoding.get('checkpoint') or {} sizes = checkpoint.get('bytes'), checkpoint.get('expected_bytes') # Reports are read back from disk; a malformed one gets no notice rather than failing the library. diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index d2fb88ca..cfa14427 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -1155,11 +1155,13 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, release_models=None, export_inputs=None, two_pass=True, encoder_prewarm=None, force_regenerate=False, base_steps=8, refine_steps=3, comfy_metadata=None, prompt_rewrite_report=None, preview=False, refine_from=None, report_extra=None, - continue_from=None, frames=None, first_frame_reference=False, voice_reference=False): + continue_from=None, frames=None, first_frame_reference=False, voice_reference=False, hide_tail=False): if type(first_frame_reference) is not bool: raise ValueError('First-frame reference must be a boolean') if type(voice_reference) is not bool: raise ValueError('Voice reference must be a boolean') + if type(hide_tail) is not bool: + raise ValueError('Hide tail must be a boolean') if continue_from is not None: from .continuation import validate_route validate_route(two_pass=two_pass, preview=preview, refine_from=refine_from) @@ -1266,6 +1268,8 @@ def check_ending(): command += ['--' + field.replace('_', '-'), str(reserve)] if preview: command.append('--preview') + if hide_tail: + command.append('--hide-tail') state = {'status': 'starting', 'geometry': canvas, 'seed': seed, 'installation': str(root), 'command': command, 'output': str(output), 'source': str(source), 'resources': resources} if preview: @@ -1344,8 +1348,12 @@ def poll(): 'references': any(media.get(key) for key in ('first', 'last', 'references'))}) from .media_request import task_for from .two_pass import plan, sampling_canvas + from .generate import hide_tail_geometry + generated_canvas, hidden = hide_tail_geometry(canvas, None if extra.get('conditioning') else prompt, + hide_tail, preview=bool(preview or upscale)) + hide_tail_frames = hidden['frames'] if hidden and hidden['used'] else 0 # The engine samples this canvas; results, cache keys and labels keep the requested size. - sampled, output_size = sampling_canvas(canvas, staged=bool(preview or upscale), media=extra.get('media'), + sampled, output_size = sampling_canvas(generated_canvas, staged=bool(preview or upscale), media=extra.get('media'), conditioned=bool(extra.get('conditioning'))) planned = plan(sampled, two_pass, 'ref2va' if anchored else task_for(extra.get('media', {})), base_steps=base_steps, refine_steps=refine_steps, staged=bool(preview or upscale)) @@ -1356,7 +1364,7 @@ def poll(): if continuation is not None: try: continuation = validate_snapshot(continuation, sampled, sampling_plan=planned) - state['continuation'] = summary(continuation) + state['continuation'] = summary(continuation, hide_tail_frames=hide_tail_frames) except ValueError as error: if str(error) != MISSING: raise @@ -1371,9 +1379,11 @@ def poll(): if continuation is None: from .continuation import OVERLAP_FRAMES state['continuation'] = dict(overlap_frames=OVERLAP_FRAMES, - new_frames=sampled['frames'] - OVERLAP_FRAMES, encoded=True) + new_frames=sampled['frames'] - hide_tail_frames - OVERLAP_FRAMES, encoded=True) # A preview is its own result; an upscale is always run from its preview. keyed = dict(planned, preview=True) if preview else planned + if hide_tail_frames: + keyed = dict(keyed, hide_tail_frames=hide_tail_frames) identify = lambda check: request_key(prompt, seed, canvas, keyed, extra, machine, resources, source, environment, inspection=check) state['result_cache'] = dict(enabled=False, hit=False, forced=force_regenerate) @@ -1457,6 +1467,15 @@ def asset_progress(message): raise RuntimeError(generation_failure(run, process.returncode, repair=hint)) report = json.loads(output.with_suffix('.request.json').read_text(encoding='utf-8')) engine = json.loads(output.with_suffix('.engine.json').read_text(encoding='utf-8')) + if hide_tail_frames and report.get('hide_tail') == dict(requested=True, used=False, reason='too_long'): + # Admission before encoding can decline the extra frames after the bridge's + # estimate. Validate the ordinary plan and don't cache it as a hidden-tail result. + # After encoding the conditioning is tied to the longer canvas, so a later refusal + # fails like any other request. + sampled, output_size = sampling_canvas(canvas, staged=bool(preview or upscale), + media=extra.get('media'), conditioned=bool(extra.get('conditioning'))) + hidden, hide_tail_frames = report['hide_tail'], 0 + cache_key = None from .two_pass import steps, plan completed_steps = steps(report.get('sampling_plan')) expected_plan = plan(sampled, two_pass, report.get('profile', {}).get('engine', {}).get('task', 't2va'), @@ -1472,6 +1491,7 @@ def asset_progress(message): or engine.get('sampling_plan') != report.get('sampling_plan') or report.get('sampling_plan') != expected_plan or bool(report.get('preview')) != preview or bool(report.get('upscaled_preview')) != bool(upscale) + or (hide_tail and report.get('hide_tail') != hidden) or any(engine.get('geometry', {}).get(k) != shown[k] for k in ('width', 'height', 'frames')) or (report.get('output_size') or None) != output_size): from .comfy_launcher_api import repair_hint diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py index 1e07903d..1eee6717 100644 --- a/freevideo_engine/comfy_chain.py +++ b/freevideo_engine/comfy_chain.py @@ -39,7 +39,7 @@ def seconds_for_frames(frames, minimum=1.625, maximum=60., step=.1): class ComfyHost: - continuation_inputs = (*CONTINUE_INPUTS.values(), 'voice_reference') + continuation_inputs = (*CONTINUE_INPUTS.values(), 'voice_reference', 'hide_tail') media_inputs = ('media', 'first', 'last', 'references', 'conditioning') def __init__(self, server): @@ -61,6 +61,8 @@ def inputs(self, block_inputs): values[name] = '' if independent else block_inputs[key] if 'voice_reference' in schema and values.get('continue_from'): values['voice_reference'] = True + if 'hide_tail' in schema and not independent and block_inputs.get('index', 0) < block_inputs.get('count', 0) - 1: + values['hide_tail'] = True return values def work_exists(self, path): diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index 983e4bd9..cd978980 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -66,6 +66,8 @@ def define_schema(cls): optional=True, advanced=True, tooltip='Reduces drift.'), io.Boolean.Input('voice_reference', display_name='Voice reference', default=False, optional=True, advanced=True, tooltip='Keeps the opening voice consistent in continued dialogue.'), + io.Boolean.Input('hide_tail', display_name='Hide dialogue tail', default=False, + optional=True, advanced=True, tooltip='Generates and discards 34 extra frames to keep dialogue moving between parts.'), ], outputs=[io.Video.Output('video'), io.String.Output('report', display_name='Report JSON')], hidden=[io.Hidden.unique_id, io.Hidden.prompt, io.Hidden.extra_pnginfo], @@ -111,7 +113,7 @@ def fingerprint_inputs(cls, **kwargs): def execute(cls, text, width, height, seconds, seed, first=None, last=None, references=None, loras=None, conditioning=None, media=None, two_pass=True, force_regenerate=False, base_steps=8, refine_steps=3, preview=False, refine_from='', - continue_from='', frames=0, first_frame_reference=False, voice_reference=False): + continue_from='', frames=0, first_frame_reference=False, voice_reference=False, hide_tail=False): from server import PromptServer from .comfy_chain import block_context, block_finished, report_fields node_id = cls.hidden.unique_id @@ -224,6 +226,7 @@ def release(): # Only a part that continues another can use it; other requests keep the existing bridge call. **({'first_frame_reference': True} if first_frame_reference is True and options.get('continue_from') else {}), **({'voice_reference': True} if voice_reference is True and options.get('continue_from') else {}), + **({'hide_tail': True} if hide_tail is True else {}), **({'report_extra': {'chain': report_fields(chain)}} if chain is not None else {})) relative = output.relative_to(output_root) report = output.with_suffix('.request.json').read_text(encoding='utf-8') diff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.py index 1905c886..b8b60b1e 100644 --- a/freevideo_engine/continuation.py +++ b/freevideo_engine/continuation.py @@ -415,9 +415,12 @@ def validate_snapshot(request, canvas, manifest=None, sampling_plan=None): return _prepare_snapshot(request, _snapshot_value(request), canvas, manifest, sampling_plan) -def summary(request): - return dict({key: request[key] for key in ('overlap_frames', 'new_frames', 'frame_origin', 'audio_origin', 'segment')}, - audio_lead_in_samples=request.get('audio_lead_in_samples', 0)) +def summary(request, *, hide_tail_frames=0): + result = dict({key: request[key] for key in ('overlap_frames', 'new_frames', 'frame_origin', 'audio_origin', 'segment')}, + audio_lead_in_samples=request.get('audio_lead_in_samples', 0)) + if hide_tail_frames: + result['new_frames'] -= validate_hide_tail(hide_tail_frames) + return result def load(request, stage): @@ -431,17 +434,33 @@ def load(request, stage): audio=torch.from_numpy(value['audio'].copy())) -def tail(video): +# Dialogue parts settle into an ending over their last ~1.5 s; 17 frames hid only the pause, 34 also the pose reset. +HIDE_TAIL_FRAMES = 34 + + +def validate_hide_tail(frames): + if type(frames) is not int or frames not in (0, 17, HIDE_TAIL_FRAMES): + raise ValueError('Hidden tail must be 0, 17 or 34 frames') + return frames + + +def tail(video, *, hide_tail_frames=0): # clone avoids keeping the entire clip's storage behind a small view. + hidden = validate_hide_tail(hide_tail_frames) // 17 * 5 + if hidden: + return video[:, :, -hidden - LATENT_FRAMES:-hidden].detach().cpu().clone() return video[:, :, -LATENT_FRAMES:].detach().cpu().clone() -def tail_data(first, video, audio, canvas, sampling_plan, manifest, request=None): +def tail_data(first, video, audio, canvas, sampling_plan, manifest, request=None, *, hide_tail_frames=0): chain = ({key: request[key] for key in ('segment', 'frame_origin', 'audio_origin')} if request else dict(segment=1, frame_origin=0, audio_origin=0)) - start = audio_origin(chain['frame_origin'] + canvas['frames'] - OVERLAP_FRAMES) - chain['audio_origin'] - return dict(video_lo=first, video_hi=tail(video), audio=audio[:, :, start:].detach().cpu().clone(), - metadata=dict(identity=identity(manifest), geometry={key: canvas[key] for key in ('width', 'height', 'frames')}, + kept = canvas['frames'] - validate_hide_tail(hide_tail_frames) + start = audio_origin(chain['frame_origin'] + kept - OVERLAP_FRAMES) - chain['audio_origin'] + end = round(kept * 40 / 24) if hide_tail_frames else audio.shape[2] + return dict(video_lo=first, video_hi=tail(video, hide_tail_frames=hide_tail_frames), + audio=audio[:, :, start:end].detach().cpu().clone(), + metadata=dict(identity=identity(manifest), geometry=dict(width=canvas['width'], height=canvas['height'], frames=kept), first_canvas={key: sampling_plan['first'][key] for key in ('width', 'height')}, chain=chain, **({'source_id': request['source_id']} if request and 'source_id' in request else {}))) diff --git a/freevideo_engine/decode.py b/freevideo_engine/decode.py index cedb2228..2c6d79f4 100644 --- a/freevideo_engine/decode.py +++ b/freevideo_engine/decode.py @@ -25,7 +25,10 @@ def decode_to_file(latents, audio_latents, out_path, *, base, offload=False, prefetch=True, tile_group=False, preload=False, pin_weights=False, artifacts_dir=None, phase_callback=None, stream_output=False, stream_weights=False, resident_blocks=0, model_cache=None, - linear_compute_cache=False, continuation=None, audio_lead_in_callback=None, output_size=None): + linear_compute_cache=False, continuation=None, audio_lead_in_callback=None, output_size=None, + hide_tail_frames=0): + from .continuation import validate_hide_tail + validate_hide_tail(hide_tail_frames) def phase(name): if phase_callback is not None: phase_callback(name) @@ -171,6 +174,9 @@ def decoded(done, total, elapsed): chunk = (video[:, :, start:start + 8].float() * pixel_std + pixel_mean).clamp_(0, 1) frames[start:start + chunk.shape[2]] = (chunk[0].permute(1, 2, 3, 0) * 255).round().to(torch.uint8) del video + if hide_tail_frames: + kept_frames = len(frames) - hide_tail_frames + frames = frames[:kept_frames] if continuation is not None: from .continuation import OVERLAP_FRAMES frames = frames[OVERLAP_FRAMES:] @@ -181,12 +187,17 @@ def decoded(done, total, elapsed): tick = time.perf_counter() artifacts_dir = Path(artifacts_dir) artifacts_dir.mkdir(parents=True, exist_ok=True) - if not stream_output or continuation is not None: - # A streamed mapping still contains the overlap. Replace its + if not stream_output or continuation is not None or hide_tail_frames: + # A streamed mapping still contains overlap/hidden frames. Replace its # retained artifact after the mapping closes, below. rgb_path = artifacts_dir / ('rgb-new.npy' if stream_output else 'rgb.npy') np.save(rgb_path, frames.numpy() if hasattr(frames, 'numpy') else frames, allow_pickle=False) artifact_seconds += time.perf_counter() - tick + elif stream_output and hide_tail_frames: + # render_rgb also retains a mapping when no explicit artifacts directory was requested. + import numpy as np + np.save(directory / 'rgb-new.npy', frames.numpy() if hasattr(frames, 'numpy') else frames, + allow_pickle=False) audio_start = time.perf_counter() phase('audio_load_decode') audio_vae = None @@ -212,6 +223,9 @@ def decoded(done, total, elapsed): if audio_lead_in_callback is not None: from .continuation import capture_audio_lead_in audio_lead_in_callback(capture_audio_lead_in(audio, continuation, rate)) + if hide_tail_frames: + audio = audio[:, :round(kept_frames * 40 / 24) * 800] + if continuation is not None: audio = audio[:, continuation['audio_latents'] * 800:] del audio_vae, audio_mean, audio_std gc.collect() @@ -248,8 +262,9 @@ def decoded(done, total, elapsed): if rgb_mapping is not None: del frames rgb_mapping._mmap.close() - if continuation is not None and artifacts_dir and (artifacts_dir / 'rgb-new.npy').is_file(): - (artifacts_dir / 'rgb-new.npy').replace(artifacts_dir / 'rgb.npy') + retained = Path(artifacts_dir) if artifacts_dir else directory + if (continuation is not None or hide_tail_frames) and (retained / 'rgb-new.npy').is_file(): + (retained / 'rgb-new.npy').replace(retained / 'rgb.npy') return {'vae_load_seconds': load_seconds, 'video_decode_seconds': video_decode_seconds, 'vae_linear_compute_cache': linear_compute_cache, 'vae_compute_cache_preparation': compute_cache_metrics, 'resident_vae_cache_hit': vae_cache_hit, 'resident_audio_vae_cache_hit': audio_cache_hit, diff --git a/freevideo_engine/decode_resume.py b/freevideo_engine/decode_resume.py index 4a27033b..58fc2416 100644 --- a/freevideo_engine/decode_resume.py +++ b/freevideo_engine/decode_resume.py @@ -94,6 +94,8 @@ def sampling_provenance(request): result['upscaler_sha256'] = _digest(request['upscaler_checkpoint']) if result['upscaler_sha256'] != request['sampling_plan']['upscaler_sha256']: raise ValueError('Latent upscaler differs from the planned checkpoint') + if request.get('hide_tail_frames'): + result['hide_tail_frames'] = request['hide_tail_frames'] if request.get('continuation') is not None: # A retry's identity binds the accepted bytes, without embedding the # snapshot again or making a relocation change computation identity. diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index c581f6d9..727c4076 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -340,6 +340,33 @@ def run(args): raise +def dialogue_reason(prompt): + """Shared eligibility for opening-voice references and hidden dialogue tails.""" + from .prompt_vlm.rules import split_source + if not prompt or not split_source(prompt)[2]: + return 'no_lines' + if re.search(r'(?i)\b(?:sing|sings|singing|sang|sung|lyrics)\b|唱', prompt): + return 'singing' + return None + + +def hide_tail_geometry(canvas, prompt, requested=False, *, preview=False, max_frames=None): + """Plan the generated canvas while keeping the ordinary output duration.""" + if not requested: + return canvas, None + from .continuation import HIDE_TAIL_FRAMES + from .geometry import geometry + # The product's longest selectable request is 60 seconds on the H3 grid. + limit = geometry(seconds=60)['frames'] if max_frames is None else max_frames + reason = 'preview' if preview else dialogue_reason(prompt) + if reason is None and canvas['frames'] + HIDE_TAIL_FRAMES > limit: + reason = 'too_long' + if reason is not None: + return canvas, dict(requested=True, used=False, reason=reason) + generated = dict(canvas, **geometry(canvas['width'], canvas['height'], frames=canvas['frames'] + HIDE_TAIL_FRAMES)) + return generated, dict(requested=True, used=True, frames=HIDE_TAIL_FRAMES) + + def _run(args): from .hardware import detect from .geometry import geometry @@ -385,6 +412,12 @@ def usable_continuation(value, *check): validate_frames(frames) canvas = geometry(getattr(args, 'width', 1344), getattr(args, 'height', 768), frames=frames, seconds=getattr(args, 'seconds', None)) + visible_canvas = canvas + canvas, hide_tail = hide_tail_geometry(canvas, + args.prompt_file.read_text(encoding='utf-8') if getattr(args, 'hide_tail', False) and args.prompt_file else None, + getattr(args, 'hide_tail', False), + preview=bool(getattr(args, 'preview', False) or getattr(args, 'refine_from', None))) + hide_tail_frames = hide_tail['frames'] if hide_tail and hide_tail['used'] else 0 continuation = None from .two_pass import validate_steps requested_steps = getattr(args, 'base_steps', None) @@ -450,7 +483,6 @@ def usable_continuation(value, *check): if continue_anchor is not None: continuation['anchor'] = media['references'][0] if voice_reference is not None: - from .prompt_vlm.rules import split_source if first_frame is not None: voice_reference['reason'] = 'first_frame' elif continue_anchor is not None: @@ -458,11 +490,9 @@ def usable_continuation(value, *check): elif args.conditioning: # Preencoded conditioning cannot take another reference. voice_reference['reason'] = 'conditioning' - elif not args.prompt_file or not split_source(args.prompt_file.read_text(encoding='utf-8'))[2]: - voice_reference['reason'] = 'no_lines' - elif re.search(r'(?i)\b(?:sing|sings|singing|sang|sung|lyrics)\b|唱', args.prompt_file.read_text(encoding='utf-8')): - # Quoted lyrics read as lines, but on music the opening sound did not hold steady. - voice_reference['reason'] = 'singing' + else: + voice_reference['reason'] = dialogue_reason( + args.prompt_file.read_text(encoding='utf-8') if args.prompt_file else None) active_loras = [row for row in media.get('loras', []) if row['strength'] != 0] if media: inferred_task = task_for(media) @@ -479,7 +509,21 @@ def usable_continuation(value, *check): hardware, resident_credit = credit(detect()) profile = json.loads(args.profile.read_text(encoding='utf-8')) else: - hardware, resident_credit, profile = automatic_profile(args, canvas, stage='before-encoding', evidence=planning) + from .policy import ResourceBudgetError + try: + hardware, resident_credit, profile = automatic_profile(args, canvas, stage='before-encoding', evidence=planning) + except ResourceBudgetError: + if not hide_tail_frames: + raise + # The optional tail must fit this size/step tier too. Re-admit the + # original duration before encoding anything; never shorten it. + canvas, output_size = sampling_canvas(dict(canvas, **visible_canvas), + staged=bool(preview or refine_source), media=media, conditioned=bool(args.conditioning)) + hardware, resident_credit, profile = automatic_profile(args, canvas, stage='before-encoding', evidence=planning) + hide_tail, hide_tail_frames = dict(requested=True, used=False, reason='too_long'), 0 + if continuation is not None: + from .continuation import OVERLAP_FRAMES + continuation = dict(continuation, new_frames=canvas['frames'] - OVERLAP_FRAMES) print(json.dumps(dict(event='compute_device', backend='CUDA', name=getattr(hardware, 'gpu_name', None), uuid=getattr(hardware, 'gpu_uuid', None), vram_total_bytes=getattr(hardware, 'vram_total', None), vram_free_bytes=getattr(hardware, 'vram_free', None))), flush=True) @@ -540,7 +584,7 @@ def usable_continuation(value, *check): 'reclaimable_resident_models': resident_credit, 'encoder_mode': 'native H3 encoder library child' if args.prompt_file else 'preencoded shared conditioning'} if continuation is not None: - report['continuation'] = continuation_summary(continuation) + report['continuation'] = continuation_summary(continuation, hide_tail_frames=hide_tail_frames) if 'tail' in continuation: report['continuation']['context'] = 'encoded' report['audio_context'] = continuation['audio_context'] @@ -550,6 +594,11 @@ def usable_continuation(value, *check): report['first_frame_reference'] = first_frame if voice_reference is not None: report['voice_reference'] = voice_reference + if hide_tail is not None: + report['hide_tail'] = hide_tail + if hide_tail_frames: + report['output_frames'] = (report['continuation']['new_frames'] if continuation is not None + else canvas['frames'] - hide_tail_frames) from .support_report import code_identity, write as write_debug report['runtime_hardware'] = hardware.to_dict() report['runtime_code'] = code_identity() @@ -874,6 +923,8 @@ def validate(metrics, telemetry, selected): 'metrics': str(destination.with_suffix('.engine.json'))} if continuation is not None: request['continuation'] = continuation + if hide_tail_frames: + request['hide_tail_frames'] = hide_tail_frames if preview: request['preview'] = True if allocator_limit_bytes is not None: diff --git a/freevideo_engine/worker.py b/freevideo_engine/worker.py index 8cc52ea5..fb30a15d 100644 --- a/freevideo_engine/worker.py +++ b/freevideo_engine/worker.py @@ -85,6 +85,10 @@ def released(engine): 'torch_num_threads': torch.get_num_threads()}}} save(request['metrics'], metrics) try: + from .continuation import validate_hide_tail + hide_tail_frames = validate_hide_tail(request.get('hide_tail_frames', 0)) + if hide_tail_frames: + metrics['hide_tail_frames'] = hide_tail_frames if 'sampling_plan' in request: from .two_pass import plan expected = plan(request['geometry'], request['sampling_plan']['requested'], @@ -115,7 +119,7 @@ def released(engine): request['continuation'] = continuation request['engine_options'] = dict(request['engine_options'], adaln_task=ANCHOR_ADALN_TASK if continuation.get('anchor') else ADALN_TASK) - metrics['continuation'] = summary(continuation) + metrics['continuation'] = summary(continuation, hide_tail_frames=hide_tail_frames) metrics['audio_context'] = continuation['audio_context'] metrics['device_memory'] = device_memory_report(request['gpu_budget_bytes'], request.get('allocator_limit_bytes'), reserve_bytes=request.get('gpu_reserve_bytes', 0), capacity_trial=request.get('capacity_trial', False)) @@ -240,7 +244,7 @@ def capture_first(video): return try: from .continuation import tail - first_tail = tail(video) + first_tail = tail(video, **({'hide_tail_frames': hide_tail_frames} if hide_tail_frames else {})) except Exception as error: tail_failure(error) first_tail = None @@ -253,7 +257,7 @@ def capture_first(video): **({'continuation_tail_callback': restore_tail} if continuable else {})) metrics.update(sampled, phase='decode', decode_phase='latent_transfer') if continuation is not None: - metrics['continuation'] = summary(continuation) + metrics['continuation'] = summary(continuation, hide_tail_frames=hide_tail_frames) save(request['metrics'], metrics) print(json.dumps(dict(event='decode_resume', sampling_source_attempt=metrics['sampling_source_attempt'])), flush=True) fit_decoder() @@ -321,7 +325,8 @@ def sampling_complete(latents, audio, sampled): try: from .continuation import tail_data continuation_data = tail_data(first_tail, latents, audio, canvas, - request['sampling_plan'], manifest, continuation) + request['sampling_plan'], manifest, continuation, + **({'hide_tail_frames': hide_tail_frames} if hide_tail_frames else {})) except Exception as error: tail_failure(error) continuation_data = None @@ -636,6 +641,7 @@ def keep_audio_lead_in(value): # Older retained sampling tails predate the predecessor binding. continuation_data['metadata']['source_id'] = continuation['source_id'] metrics.update(decode_to_file(latents, audio, request['output'], base=base, artifacts_dir=artifacts, + **({'hide_tail_frames': hide_tail_frames} if hide_tail_frames else {}), **({'continuation': continuation} if continuation is not None else {}), **({'audio_lead_in_callback': keep_audio_lead_in} if continuation is not None and continuation_data is not None else {}), From 01d7bd67b01b7e39bc5944432eb98c9f5e2e31e6 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:47 +0000 Subject: [PATCH 14/47] A continued part's media panel keeps only its pictures before the Media node checks --- freevideo_engine/chain_runner.py | 6 +++-- freevideo_engine/comfy_assets.py | 22 +++++++++++++++++ freevideo_engine/comfy_bridge.py | 12 ++++++--- freevideo_engine/comfy_chain.py | 18 +++++++++++++- freevideo_engine/comfy_media.py | 17 +++++++++++++ freevideo_engine/comfy_nodes.py | 12 ++++----- freevideo_engine/continuation.py | 36 ++++++++++++++++++++++----- freevideo_engine/generate.py | 42 ++++++++++++++++++++++---------- freevideo_engine/runtime.py | 13 ++++------ 9 files changed, 139 insertions(+), 39 deletions(-) diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index 4ab52cf3..e5e41793 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -531,11 +531,13 @@ def _step_chain(self, identity): block_inputs[key] = (source['output'] or '') if source is not None else '' for name in getattr(self.host, 'continuation_inputs', ()): prompt[node]['inputs'].pop(name, None) - # A part that continues footage takes no keyframes or references of its own: - # only a part that starts fresh keeps the request's media. + # A part that continues footage takes no keyframes of its own: it starts from the footage. + # The host's media inputs name what only a part that starts fresh keeps. if any(source is not None for source in sources.values()): for name in getattr(self.host, 'media_inputs', ()): prompt[node]['inputs'].pop(name, None) + if hasattr(self.host, 'continued'): + self.host.continued(prompt, node) prompt[node]['inputs'].update(self.inputs(block_inputs)) self.host.submit(prompt_id, node_prompt(prompt, node), extra) except Exception as error: diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index 844b379c..29d2cb8c 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -106,6 +106,28 @@ def resolve_assets(serialized, input_directory): return result +def continued_assets(serialized): + """The panel rows a continued part uses: enabled image references in panel order, at most three. + + Its frames and sounds stay in the panel for parts that start fresh, but never reach a continued part's checks.""" + from .continuation import PICTURES_MAX + try: + rows = json.loads(serialized) + except (TypeError, ValueError): + return '[]' + kept = [] + for row in rows if isinstance(rows, list) else (): + if (isinstance(row, dict) and row.get('enabled', True) is True and row.get('role') == 'reference' + and isinstance(row.get('file'), str)): + try: + image = media_kind(row['file']) == 'image' + except ValueError: + image = False + if image and len(kept) < PICTURES_MAX: + kept.append(dict(file=row['file'], role='reference')) + return json.dumps(kept) + + def connect_assets(assets, *, first=None, last=None, reference=None, reference_audio=None, **images): """Keep connected IMAGE/AUDIO values native until request input export.""" result = dict(assets) diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index cfa14427..bcd77402 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -1346,6 +1346,11 @@ def poll(): media = extra.get('media') or {} send_progress({'label': 'Retaining and checking input media', 'timing_phase': 'encoding', 'references': any(media.get(key) for key in ('first', 'last', 'references'))}) + # A continued part keeps the cast's pictures; the engine then leaves out the film's first frame. + pictured = continue_from is not None and any( + row.get('kind') == 'image' for row in (extra.get('media') or {}).get('references', [])) + if pictured: + anchored = False from .media_request import task_for from .two_pass import plan, sampling_canvas from .generate import hide_tail_geometry @@ -1355,11 +1360,12 @@ def poll(): # The engine samples this canvas; results, cache keys and labels keep the requested size. sampled, output_size = sampling_canvas(generated_canvas, staged=bool(preview or upscale), media=extra.get('media'), conditioned=bool(extra.get('conditioning'))) - planned = plan(sampled, two_pass, 'ref2va' if anchored else task_for(extra.get('media', {})), + planned = plan(sampled, two_pass, 'ref2va' if anchored or pictured else task_for(extra.get('media', {})), base_steps=base_steps, refine_steps=refine_steps, staged=bool(preview or upscale)) if continue_from is not None: from .continuation import ADALN_TASK, ANCHOR_ADALN_TASK, MISSING, validate_snapshot, summary - validate_route(task='ref2va' if anchored else task_for(extra.get('media', {})), anchor=anchored, + validate_route(task='ref2va' if anchored or pictured else task_for(extra.get('media', {})), + anchor=anchored or pictured, media=extra.get('media')) if continuation is not None: try: @@ -1414,7 +1420,7 @@ def asset_progress(message): preparation_started = time.monotonic() try: # Reference audio selects its own tables; match the encoder's choice. - asset_task = ((ANCHOR_ADALN_TASK if anchored else ADALN_TASK) if continue_from is not None + asset_task = ((ANCHOR_ADALN_TASK if anchored or pictured else ADALN_TASK) if continue_from is not None else engine_task(extra.get('media', {}), media_base)) preparation = prepare_sampling_assets(root, machine, planned, asset_task, progress=asset_progress, interrupted=interrupted, environ=environment) diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py index 1eee6717..3afecef7 100644 --- a/freevideo_engine/comfy_chain.py +++ b/freevideo_engine/comfy_chain.py @@ -40,11 +40,27 @@ def seconds_for_frames(frames, minimum=1.625, maximum=60., step=.1): class ComfyHost: continuation_inputs = (*CONTINUE_INPUTS.values(), 'voice_reference', 'hide_tail') - media_inputs = ('media', 'first', 'last', 'references', 'conditioning') + # Inputs a continued part drops. The media panel and references stay: the Generate node keeps their pictures. + media_inputs = ('first', 'last', 'conditioning') def __init__(self, server): self.server = server + def continued(self, prompt, node): + """A continued part's media panel keeps only the pictures it uses (prompt is this submission's copy).""" + link = prompt[node]['inputs'].get('media') + if not (isinstance(link, list) and len(link) == 2 and link[0] in prompt): + return + media = prompt[link[0]] + if not isinstance(media, dict) or media.get('class_type') != 'FreeVideoMedia': + return + inputs = media.setdefault('inputs', {}) + for name in ('first', 'last', 'reference_audio'): + inputs.pop(name, None) + if isinstance(inputs.get('assets'), str): + from .comfy_assets import continued_assets + inputs['assets'] = continued_assets(inputs['assets']) + def inputs(self, block_inputs): schema = _generate_inputs() values = {key: block_inputs[key] for key in ('text', 'seed')} diff --git a/freevideo_engine/comfy_media.py b/freevideo_engine/comfy_media.py index d7abd950..ac37d2ba 100644 --- a/freevideo_engine/comfy_media.py +++ b/freevideo_engine/comfy_media.py @@ -1,4 +1,5 @@ """Native Comfy sockets -> retained local files, without model duplication.""" +import logging from pathlib import Path from .comfy_assets import REFERENCE_INPUTS @@ -43,6 +44,22 @@ def _audio(value, path): container.mux(packet) +def continued(exported): + """What a continued part keeps of exported inputs: the pictures (at most three, one per person) and LoRAs. + + It starts from the footage, so keyframes, pre-encoded conditioning, clips and sound are left out.""" + from .continuation import PICTURES_MAX + media = (exported or {}).get('media') or {} + kept = {key: value for key, value in media.items() + if key not in ('first', 'last', 'references', 'conditioning_info')} + pictures = [row for row in media.get('references', []) if row.get('kind') == 'image'] + if len(pictures) > PICTURES_MAX: + logging.warning('[FreeVideo] A continued part keeps the first %d of %d pictures', PICTURES_MAX, len(pictures)) + if pictures: + kept['references'] = pictures[:PICTURES_MAX] + return {'media': kept} if len(kept) > 1 else {} + + def export(run, canvas, *, first=None, last=None, references=None, loras=None, conditioning=None, assets=None): media = {'version': 1} if assets: diff --git a/freevideo_engine/comfy_nodes.py b/freevideo_engine/comfy_nodes.py index cd978980..06f103ad 100644 --- a/freevideo_engine/comfy_nodes.py +++ b/freevideo_engine/comfy_nodes.py @@ -82,13 +82,12 @@ def validate_inputs(cls, text, width, height, seconds, seed, **kwargs): # Use presence checks: tensor inputs have no scalar truth value. assets = kwargs.get('media') connected = assets.get('_connected', {}) if isinstance(assets, dict) else {} + # References are not checked here: a continued part keeps only their pictures. media = dict(first=kwargs.get('first') is not None or connected.get('first') is not None, last=kwargs.get('last') is not None or connected.get('last') is not None, - references=bool(kwargs.get('references')) or any( - connected.get(key) is not None for key in ('reference', 'reference_audio')), conditioning_info=kwargs.get('conditioning') is not None) if isinstance(assets, dict): - for key in ('first', 'last', 'references', 'conditioning_info'): + for key in ('first', 'last', 'conditioning_info'): media[key] = media[key] or bool(assets.get(key)) options.update(two_pass=kwargs.get('two_pass', True), preview=kwargs.get('preview', False), refine_from=kwargs.get('refine_from'), media=media) @@ -125,7 +124,7 @@ def execute(cls, text, width, height, seconds, seed, first=None, last=None, from .background_compile import COMPILE # Returns once the idle compile released the GPU and the engine lease. COMPILE.stop(final=True) - from .comfy_media import export + from .comfy_media import continued, export import folder_paths import comfy.model_management as memory from comfy.utils import ProgressBar @@ -220,8 +219,9 @@ def release(): base_steps=base_steps, refine_steps=refine_steps, comfy_metadata=graph, preview=bool(preview), refine_from=refine_from or None, interrupted=memory.throw_exception_if_processing_interrupted, release_models=release, - export_inputs=lambda run, canvas: export(run, canvas, first=first, last=last, - references=references, loras=loras, conditioning=conditioning, assets=media), + export_inputs=lambda run, canvas: (continued if options.get('continue_from') else dict)( + export(run, canvas, first=first, last=last, references=references, loras=loras, + conditioning=conditioning, assets=media)), **options, # Only a part that continues another can use it; other requests keep the existing bridge call. **({'first_frame_reference': True} if first_frame_reference is True and options.get('continue_from') else {}), diff --git a/freevideo_engine/continuation.py b/freevideo_engine/continuation.py index b8b60b1e..4e1d26a4 100644 --- a/freevideo_engine/continuation.py +++ b/freevideo_engine/continuation.py @@ -33,6 +33,8 @@ MISSING = 'The previous video has no usable continuation data. Generate it again with this version of FreeVideo.' SIZE_MISMATCH = 'Use the same width and height as the previous video.' UNSUPPORTED = 'Video continuation requires full two-pass text generation on an NVIDIA GPU, without first frames, last frames or reference media.' +# A continued part keeps the cast's pictures, one per person and at most three people in a frame. +PICTURES_MAX = 3 INVALID_FRAMES = 'Frame count must be 5 + 17n and at least 39 (39, 56, 73, 90, ...).' INVALID_FILE = 'Choose an existing MP4 video using its absolute path.' @@ -71,16 +73,18 @@ def validate_frames(frames): def validate_route(*, task='t2va', two_pass=True, preview=False, refine_from=None, media=None, backend=None, anchor=False, sound=False): + """anchor: the part also takes pictures (an explicit anchor or the cast's pictures); + sound: the engine adds the film's opening sound as the one audio reference.""" media = media or {} references = media.get('references') - valid_reference = (anchor and isinstance(references, list) and len(references) == 1 - and isinstance(references[0], dict) - and references[0].get('kind') == 'image') + kinds = ([row.get('kind') if isinstance(row, dict) else None for row in references] + if isinstance(references, list) else None) + valid_reference = (kinds is not None and kinds.count('image') + kinds.count('audio') == len(kinds) + and kinds.count('image') <= (PICTURES_MAX if anchor else 0) + and kinds.count('audio') == (1 if sound else 0)) valid_task = task == ('ref2va' if anchor else 't2va') if sound: - valid_reference = (not anchor and isinstance(references, list) and len(references) == 1 - and isinstance(references[0], dict) and references[0].get('kind') == 'audio') - valid_task = task in ('t2va', 'ref2va', 'ref2va_audio') and valid_reference + valid_task = valid_reference and task in (('ref2va', 'ref2va_av') if anchor else ('t2va', 'ref2va', 'ref2va_audio')) if (not supported()[0] or backend not in (None, 'cuda') or not valid_task or not two_pass or preview or refine_from or any(media.get(key) for key in ('first', 'last', 'conditioning_info')) @@ -88,6 +92,26 @@ def validate_route(*, task='t2va', two_pass=True, preview=False, refine_from=Non raise ValueError(UNSUPPORTED) +def encoded_references(conditions, task, anchor): + """Check a continued part's encoded references: its pictures, then the engine's sound. True when the sound is there.""" + rows = conditions or [] + images = [row for row in rows if row['kind'] == 'image' and row.get('audio_latent') is None] + sound = (task == ('ref2va_av' if anchor else 'ref2va_audio') and len(rows) == len(images) + 1 + and rows[-1]['kind'] == 'audio' and rows[-1].get('audio_latent') is not None) + if not rows or anchor != bool(images) or len(images) + sound != len(rows) or len(images) > PICTURES_MAX: + raise ValueError('Continuation pictures must be one to %d images, followed only by its own sound' % PICTURES_MAX) + return sound + + +def continuation_pictures(media): + """The pictures a continued part takes: image references only, at most PICTURES_MAX (one per person).""" + references = media.get('references') or [] + pictures = [row for row in references if row.get('kind') == 'image'] + if len(pictures) != len(references) or len(pictures) > PICTURES_MAX: + raise ValueError(UNSUPPORTED) + return pictures + + def anchor_media(media, path): """Add exactly one explicit global first-frame reference through the media encoder.""" if any(media.get(key) for key in ('first', 'last', 'references', 'conditioning_info')): diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index 727c4076..6585e63d 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -392,7 +392,7 @@ def usable_continuation(value, *check): raise # A record of another version, other weights or another first-pass size. value = dict(encoded_continuation(continue_from, canvas), - **{key: value[key] for key in ('audio_context', 'source_video', 'anchor', 'sound') if key in value}) + **{key: value[key] for key in ('audio_context', 'source_video', 'anchor', 'pictures', 'sound') if key in value}) return freeze_continuation(value) if continuation_directory is not None else value frames = getattr(args, 'frames', None) @@ -436,14 +436,20 @@ def usable_continuation(value, *check): if continue_anchor is not None: from .continuation import anchor_media media = anchor_media(media, continue_anchor) + # The cast's pictures, sent with a continued part like with a part that starts fresh. + pictures = [] + if continue_from is not None and continue_anchor is None and media.get('references'): + from .continuation import continuation_pictures + pictures = continuation_pictures(media) + anchored = continue_anchor is not None or bool(pictures) # Too small to sample cleanly: render larger, scale the picture down. Decided before anything # compares or stores the canvas, so every later check sees the size that is sampled. from .two_pass import sampling_canvas canvas, output_size = sampling_canvas(canvas, staged=bool(preview or refine_source), media=media, conditioned=bool(args.conditioning)) if continue_from is not None: - validate_route(task=getattr(args, 'task', None) or ('ref2va' if continue_anchor else 't2va'), - anchor=continue_anchor is not None, two_pass=two_pass, + validate_route(task=getattr(args, 'task', None) or ('ref2va' if anchored else 't2va'), + anchor=anchored, two_pass=two_pass, preview=preview, refine_from=getattr(args, 'refine_from', None), media=media) # Reject absent/mismatched context before text encoding or model loading. if getattr(args, 'continuation_snapshot', None): @@ -458,7 +464,10 @@ def usable_continuation(value, *check): # Nothing was stored for this file: its own ending is encoded when the run starts. continuation = encoded_continuation(continue_from, canvas) continuation = dict(continuation, audio_context=audio_context(), source_video=str(continue_from)) - if first_frame is not None: + if first_frame is not None and pictures: + # The cast's pictures already hold the people; the film's first frame is not added to them. + first_frame['reason'] = 'pictures' + elif first_frame is not None: from .continuation import anchor_media, chain_first_frame_picture try: # A context encoded from the file carries no first frame; whoever planned this run saw none either. @@ -480,10 +489,13 @@ def usable_continuation(value, *check): validate_route(task=getattr(args, 'task', None) or 'ref2va', anchor=True, two_pass=two_pass, preview=preview, refine_from=getattr(args, 'refine_from', None), media=media) first_frame.update(used=True, width=picture_width, height=picture_height) - if continue_anchor is not None: + anchored = True + if anchored: continuation['anchor'] = media['references'][0] + if pictures: + continuation['pictures'] = len(pictures) if voice_reference is not None: - if first_frame is not None: + if first_frame is not None and not pictures: voice_reference['reason'] = 'first_frame' elif continue_anchor is not None: voice_reference['reason'] = 'anchor' @@ -553,7 +565,7 @@ def usable_continuation(value, *check): manifest = json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8')) if continuation is not None: from .two_pass import plan as continuation_plan - validate_route(task=profile['engine'].get('task', 't2va'), anchor=continue_anchor is not None) + validate_route(task=profile['engine'].get('task', 't2va'), anchor=anchored) planned = continuation_plan(canvas, two_pass, base_steps=base_steps, refine_steps=refine_steps) continuation_first = planned['first'] continuation = usable_continuation(continuation, canvas, None if active_loras else manifest, planned) @@ -590,6 +602,8 @@ def usable_continuation(value, *check): report['audio_context'] = continuation['audio_context'] if continue_anchor is not None: report['continue_anchor'] = continuation['anchor'] + if pictures: + report['continue_pictures'] = len(pictures) if first_frame is not None: report['first_frame_reference'] = first_frame if voice_reference is not None: @@ -651,11 +665,13 @@ def interrupted(signum, frame): path = destination.with_suffix('.first-sound.wav') path.write_bytes(sound) from .media_request import digest - media = dict(media, version=1, references=[dict(kind='audio', path=str(path), - sha256=digest(path), bytes=path.stat().st_size)]) - validate_route(task=getattr(args, 'task', None) or 't2va', sound=True, two_pass=two_pass, + # After the cast's pictures: the sound is the last reference. + media = dict(media, version=1, references=[*media.get('references', []), dict(kind='audio', + path=str(path), sha256=digest(path), bytes=path.stat().st_size)]) + validate_route(task=getattr(args, 'task', None) or ('ref2va' if anchored else 't2va'), sound=True, + anchor=anchored, two_pass=two_pass, preview=preview, refine_from=getattr(args, 'refine_from', None), media=media) - continuation['sound'] = media['references'][0] + continuation['sound'] = media['references'][-1] args.task = profile['engine']['task'] = task_for(media) voice_reference['used'] = True resource_identity = local_identity(hardware, args.cache) @@ -822,7 +838,7 @@ def interrupted(signum, frame): staged=bool(preview or refine_source)) report['sampling_plan'] = sampling_plan if continuation is not None: - validate_route(task=profile['engine'].get('task', 't2va'), anchor=continue_anchor is not None, + validate_route(task=profile['engine'].get('task', 't2va'), anchor=anchored, **(dict(sound=True, media=media) if voice_reference and voice_reference['used'] else {})) continuation = usable_continuation(continuation, canvas, json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8')), sampling_plan) @@ -948,7 +964,7 @@ def launch(selected, attempt): if continuation is not None: from .continuation import ADALN_TASK, ANCHOR_ADALN_TASK selected['engine'] = dict(selected['engine'], - adaln_task=ANCHOR_ADALN_TASK if continue_anchor is not None else ADALN_TASK) + adaln_task=ANCHOR_ADALN_TASK if anchored else ADALN_TASK) request.update(engine_options=selected['engine'], decoder_options=selected['decoder'], ram_budget_bytes=round(selected['inference_ram_budget_gb'] * 1e9), ram_budget_is_estimate=ram_budget_is_estimate(args), diff --git a/freevideo_engine/runtime.py b/freevideo_engine/runtime.py index b8b929ca..c1f43d24 100644 --- a/freevideo_engine/runtime.py +++ b/freevideo_engine/runtime.py @@ -551,7 +551,7 @@ def _sample(self, conditioning, seed, frames=243, width=1344, height=768, *, fin if continuation is not None: from .continuation import validate_route # Audio references are admitted below, after inspecting the encoded modalities. - if self.task != 'ref2va_audio': + if self.task not in ('ref2va_audio', 'ref2va_av'): validate_route(task=self.task, anchor=bool(continuation.get('anchor'))) if tuple(continuation['video'].shape[-2:]) != (height // 16, width // 16): from .continuation import SIZE_MISMATCH @@ -618,13 +618,10 @@ def _sample(self, conditioning, seed, frames=243, width=1344, height=768, *, fin sampling_kwargs.update(condition_schedule=condition_schedule, condition_trace=early_receipt['steps']) conditions = value.get('references') or None if continuation is not None: - sound = (not continuation.get('anchor') and self.task == 'ref2va_audio' - and conditions is not None and len(conditions) == 1 - and conditions[0]['kind'] == 'audio' and conditions[0].get('audio_latent') is not None) - if not sound and (conditions is None or len(conditions) != 1 or conditions[0]['kind'] != 'image' or conditions[0].get('audio_latent') is not None): - raise ValueError('Continuation anchor requires exactly one image reference without audio') - if sound: - validate_route(task=self.task, sound=True, media=dict(references=conditions)) + from .continuation import encoded_references + if encoded_references(conditions, self.task, bool(continuation.get('anchor'))): + validate_route(task=self.task, sound=True, anchor=bool(continuation.get('anchor')), + media=dict(references=[dict(kind=row['kind']) for row in conditions])) # Ref2VA packs independent-size references before generated # video rows in both passes. Their latents do not need the # keyframe resize used by first/last-frame conditioning. From aafae0aa4c71d02e11fb77f1264ea92840496b2a Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:58 +0000 Subject: [PATCH 15/47] Long videos: name the part-description parser apart from the voice reference's line finder --- freevideo_engine/chain_runner.py | 78 +++++++++ freevideo_engine/chain_store.py | 35 +++- freevideo_engine/comfy_chain.py | 62 ++++++++ freevideo_engine/prompt_vlm/rules.py | 211 +++++++++++++++++++++++++ freevideo_engine/prompt_vlm/service.py | 21 ++- freevideo_engine/prompt_vlm/worker.py | 58 +++++++ 6 files changed, 455 insertions(+), 10 deletions(-) diff --git a/freevideo_engine/chain_runner.py b/freevideo_engine/chain_runner.py index e5e41793..64e809df 100644 --- a/freevideo_engine/chain_runner.py +++ b/freevideo_engine/chain_runner.py @@ -212,6 +212,30 @@ def cancel(self, chain_id): self.wake.set() return self._view(record) + def set_text(self, chain_id, block_id, text): + """The description of a part that has not started; the user's own words, used as written.""" + if not isinstance(text, str) or not text.strip() or len(text) > 20000: + raise ValueError('Text must be a nonempty string of at most 20000 characters') + with self.lock: + record = self.records[chain_id] + block = next((row for row in record['blocks'] if row['id'] == block_id), None) + if block is None: + raise KeyError(block_id) + if (record.get('split') or {}).get('state') in ('pending', 'running'): + raise ChainBusy('The descriptions of the parts are not written yet') + if block['state'] != 'pending' or block['prompt_id'] is not None: + raise ChainBusy('This part has already started') + previous = block['inputs']['text'] + block['inputs']['text'] = text + try: + from .lora_triggers import apply_chain_triggers + apply_chain_triggers(record) + except Exception: + block['inputs']['text'] = previous + raise ValueError('Text must be at most 20000 characters with LoRA triggers') from None + self._save(record) + return self._view(record) + def resume(self, chain_id, client_id=None): with self.lock: record = self.records[chain_id] @@ -231,6 +255,12 @@ def resume(self, chain_id, client_id=None): seconds=None, output=None, report_id=None, error=None, fraction=0., remaining_seconds=None, continuation=None, skip_frames=0, skip_audio_samples=0) + split = record.get('split') + if (split is not None and split['state'] == 'failed' and split['reason'] in ('busy', 'memory') + and all(row['state'] != 'done' for row in record['blocks']) + and len({row['inputs']['text'] for row in record['blocks']}) == 1): + # Busy and memory pass; nothing has started and no part was rewritten, so split again. + split.update(state='pending', reason=None) if client_id is not None: record['client_id'] = client_id record['state'] = 'queued' @@ -491,8 +521,18 @@ def _step_chain(self, identity): else: prompt_id, node = block['prompt_id'], record['node'] action = 'status' + split = record.get('split') if action == 'join': pass + elif (prompt_id is None and split is not None and split['state'] == 'pending' + and record['state'] != 'cancelling' and block['index'] == 0): + # Before the first part: one description per part, so later parts carry on instead of replaying. + split['state'] = 'running' + record['state'] = 'running' + self._save(record) + value = dict(text=block['inputs']['text'], continued=block['after'] is not None, + seconds=[row['new_frames'] / chain_plan.FPS for row in record['blocks']]) + action = 'split' elif prompt_id is None: self.progress_messages.pop(identity, None) self.progress_values.get(identity, {}).pop('remaining_seconds', None) @@ -520,6 +560,11 @@ def _step_chain(self, identity): if action == 'join': self._join(identity, snapshot) return + if action == 'split': + self._split(record, value) + if record['split']['state'] != 'pending': + self._step_chain(identity) # The first part starts in the same pass. + return if action == 'submit': try: for key, source in sources.items(): @@ -587,6 +632,39 @@ def _step_chain(self, identity): block['state'] = desired self._save(record) + def _split(self, record, value): + def cancelled(): + return record['state'] == 'cancelling' or self.closed.is_set() + split = getattr(self.host, 'split_texts', None) + parts, reason = None, 'unavailable' + if split is not None: + try: + parts, reason = split(value, cancelled) + except Exception: + logging.exception('[FreeVideo] Splitting the description failed') + parts, reason = None, 'failed' + with self.lock: + if parts is None and cancelled(): + record['split']['state'] = 'pending' # Split again when resumed. + elif parts is not None and len(parts) == len(record['blocks']) and all( + isinstance(part, str) and part.strip() for part in parts): + for row, part in zip(record['blocks'], parts): + row['inputs']['text'] = part + try: + from .lora_triggers import apply_chain_triggers + apply_chain_triggers(record) + record['split'].update(state='done', reason=None) + except Exception: + for row in record['blocks']: + row['inputs']['text'] = value['text'] + record['split'].update(state='failed', reason='failed') + else: + # The whole description goes to every part, as before; the page says why. + if parts is not None: + reason = 'unusable' + record['split'].update(state='failed', reason=reason if reason in store.SPLIT_REASONS else 'failed') + self._save(record) + def _join(self, identity, snapshot): final_state = snapshot.get('join_final_state', 'done') def cancelled(): diff --git a/freevideo_engine/chain_store.py b/freevideo_engine/chain_store.py index 065a8ad8..468e49e5 100644 --- a/freevideo_engine/chain_store.py +++ b/freevideo_engine/chain_store.py @@ -16,6 +16,16 @@ CHAIN_STATES = (*ACTIVE, 'cancelled', 'failed', 'interrupted', 'done') BLOCK_STATES = ('pending', 'queued', 'running', 'done', 'cancelled', 'failed', 'interrupted') OUTPUT_STATES = ('none', 'pending', 'joining', 'done', 'failed') +# pending: split before the first part; given: the request gave one text per part; none: nothing to split. +SPLIT_STATES = ('pending', 'running', 'done', 'failed', 'given', 'none') +SPLIT_REASONS = { + 'model_missing': 'The prompt model is not installed, so every part used the whole description.', + 'busy': 'The prompt model was busy, so every part used the whole description.', + 'memory': 'There was not enough memory to split the description, so every part used the whole description.', + 'unusable': 'The description could not be split into parts, so every part used the whole description.', + 'unavailable': 'Splitting is not available here, so every part used the whole description.', + 'failed': 'Splitting the description failed, so every part used the whole description.', +} # Parts one video may be continued through: the first part and four continuations. MAX_PARTS = 5 @@ -63,7 +73,7 @@ def validate(payload): raise ValueError('Invalid extra data or client ID') if ('blocks' in payload) == ('seconds' in payload): raise ValueError('Specify blocks or seconds, not both') - planned = None + planned = split = None join = payload.get('join', 'seconds' in payload) if type(join) is not bool: raise ValueError('Join must be a bool') @@ -81,9 +91,19 @@ def validate(payload): raise ValueError('Text must be a string') if type(seed) is not int or seed < 0: raise ValueError('A nonnegative integer seed is required') + # Each part plays its own share of the description; one text for all parts replays the story + # from the start in every part. Without texts the runner splits it before the first part. + texts = payload.get('texts') + if texts is not None and (not isinstance(texts, list) or len(texts) != len(planned['blocks']) + or any(not isinstance(row, str) or not row.strip() for row in texts)): + raise ValueError('Texts must give one description for each planned part') rows = [dict(id='b%s' % (index + 1), after='b%s' % index if index else source, - frames=row['frames'], text=text, seed=seed + index) + frames=row['frames'], text=texts[index] if texts else text, seed=seed + index) for index, row in enumerate(planned['blocks'])] + if len(rows) > 1: + split = dict(state='given' if texts else 'pending' if text.strip() else 'none', reason=None) + elif 'texts' in payload: + raise ValueError('Texts are given with seconds; blocks carry their own text') else: rows = payload['blocks'] if not isinstance(rows, list) or not 1 <= len(rows) <= chain_plan.MAX_BLOCKS: @@ -143,7 +163,7 @@ def visit(identity): updated=now, state='queued', client_id=client, snapshot=dict(prompt=deepcopy(prompt), extra_data=deepcopy(extra)), blocks=blocks, plan=planned, join=join, output=None, output_state='pending' if join else 'none', - output_error=None, output_blocks=0) + output_error=None, output_blocks=0, split=split) try: json.dumps(record, allow_nan=False) except (ValueError, TypeError, OverflowError) as error: @@ -166,7 +186,7 @@ def summary(record): return dict({key: deepcopy(record[key]) for key in ('version', 'id', 'node', 'created', 'updated', 'state', 'join', 'output', 'output_state', 'output_error', 'output_blocks')}, - output_seams=deepcopy(record.get('output_seams')), + output_seams=deepcopy(record.get('output_seams')), split=deepcopy(record.get('split')), count=len(blocks), done=sum(row['state'] == 'done' for row in blocks), current=current, elapsed_seconds=sum(row['seconds'] or 0 for row in blocks if row['state'] == 'done'), last_report_id=record.get('last_report_id') or next((row['report_id'] for row in reversed(blocks) @@ -224,6 +244,13 @@ def load_all(directory, limit=200, *, identity=None): if (type(record['output_blocks']) is not int or not 0 <= record['output_blocks'] <= len(record['blocks']) or record['output'] is None and record['output_blocks'] != 0): raise ValueError('Invalid chain output block count') + split = record.setdefault('split', None) + if split is not None and (not isinstance(split, dict) or set(split) != {'state', 'reason'} + or split['state'] not in SPLIT_STATES + or split['reason'] is not None and split['reason'] not in SPLIT_REASONS): + raise ValueError('Invalid chain split') + if split is not None and split['state'] == 'running': + split['state'] = 'pending' # Stopped while splitting: split again when resumed. seams = record.get('output_seams') if seams is not None and (not isinstance(seams, dict) or set(seams) != {'expected', 'kept'} or any(type(value) is not int or value < 0 for value in seams.values())): diff --git a/freevideo_engine/comfy_chain.py b/freevideo_engine/comfy_chain.py index 3afecef7..f1c86cb9 100644 --- a/freevideo_engine/comfy_chain.py +++ b/freevideo_engine/comfy_chain.py @@ -6,6 +6,7 @@ import os from pathlib import Path, PurePosixPath import ssl +import time from urllib import error, request from .chain_runner import ChainBusy, ChainPartTooShort, ChainRunner, ChainTooLong @@ -177,6 +178,54 @@ def join(self, record, cancelled): return join_outputs([block['output'] for block in blocks], record['id'], cancelled, frames=frames, replace=record.get('output'), **kept) + def split_texts(self, value, cancelled): + """One description per part from the local prompt model: (parts, None) or (None, reason).""" + service = getattr(self.server, '_freevideo_prompt_vlm', None) + if service is None: + return None, 'unavailable' + # A rewrite or a single video the user started a moment ago finishes first; busy is said only after + # ten minutes of waiting. Up to ten minutes waiting plus fifteen splitting hold the runner thread; there + # is one active chain, so nothing else waits behind it. + waiting = time.monotonic() + 600 + while True: + try: + job = service.start('split', value)['job'] + except ValueError as failure: + if str(failure) != 'busy' or time.monotonic() > waiting or cancelled(): + return None, {'model_missing': 'model_missing', 'busy': 'busy'}.get(str(failure), 'failed') + time.sleep(2) + continue + parts, reason = self._split_job(service, job, cancelled) + if reason != 'busy' or time.monotonic() > waiting or cancelled(): + return parts, reason + time.sleep(2) + + def _split_job(self, service, job, cancelled): + deadline = time.monotonic() + 900 + while True: + try: + row = service.status(job) + except ValueError: + return None, 'failed' + phase = row.get('phase') + if phase == 'complete': + return row.get('parts'), None + if phase in ('failed', 'cancelled'): + error = row.get('error') + return None, ('memory' if error in ('ram_space', 'disk_space') else 'busy' if error == 'busy' + else 'unusable' if error == 'split_failed' else 'failed') + if cancelled() or time.monotonic() > deadline: + service.cancel(job) + stopping = time.monotonic() + 30 + try: + while (time.monotonic() < stopping + and service.status(job).get('phase') not in ('complete', 'failed', 'cancelled')): + time.sleep(.2) + except ValueError: + pass + return None, 'failed' + time.sleep(.5) + def lead(self, output, previous): """A finished block's seam sound, read while its record is still in the shared store.""" from .comfy_continuation import audio_lead_in @@ -332,6 +381,19 @@ async def cancel(request): except KeyError: return response({'error': 'Unknown chain'}, 404) + @server.routes.post('/freevideo/chain/{id}/blocks/{block}/text') + async def text(request): + try: + value = await body(request) + return response({'chain': runner.set_text(request.match_info['id'], request.match_info['block'], + value.get('text') if isinstance(value, dict) else None)}) + except KeyError: + return response({'error': 'Unknown chain or part'}, 404) + except ChainBusy as failure: + return response({'error': str(failure)}, 409) + except ValueError as failure: + return response({'error': str(failure)}, 400) + @server.routes.post('/freevideo/chain/{id}/resume') async def resume(request): try: diff --git a/freevideo_engine/prompt_vlm/rules.py b/freevideo_engine/prompt_vlm/rules.py index 32d576bb..e1bfa77a 100644 --- a/freevideo_engine/prompt_vlm/rules.py +++ b/freevideo_engine/prompt_vlm/rules.py @@ -411,3 +411,214 @@ def validate_answers(answers, value): if 'error' in answer and answer['error'] not in ('invalid_json', 'context_length'): raise ValueError('invalid_output') return answers + + +# Long videos: one description per part -------------------------------------------------------------------- +# Every part of a long video is generated from its own text. Handing each part the whole description makes +# the model play the story from the start again in every part: the opening pause, the first line and the +# first action come back after each seam. Each part instead gets the shared look and sound plus only what +# happens during its own seconds. +# +# The small local model only sorts the events: spoken lines become placeholders it places, and the shared +# description, the sound sections and the words themselves are put back by code, so they cannot drift. +SPLIT_MAX_PARTS = 8 +CONTINUE = 'Continuing directly from the previous moment, with no pause: everything already in progress keeps going.' +SPEAKER = re.compile(r'(?:\(S\d+(?:,S\d+)*\)\s*)?(\[[A-Za-z]+\])?\s*([^<\n]{1,400}?)\s*') + + +def split_request(value): + if not isinstance(value, dict) or set(value) - {'text', 'seconds', 'mode', 'continued'}: + raise ValueError('invalid_request') + text, seconds = value.get('text'), value.get('seconds') + if type(value.get('continued', False)) is not bool: + raise ValueError('invalid_request') + if not isinstance(text, str) or not text.strip() or len(text) > 12000: + raise ValueError('invalid_prompt') + if (not isinstance(seconds, list) or not 2 <= len(seconds) <= SPLIT_MAX_PARTS or + any(type(s) not in (float, int) or not math.isfinite(s) or not 1 <= s <= 60 for s in seconds)): + raise ValueError('invalid_duration') + return dict(mode='split', text=text, seconds=[float(s) for s in seconds], + continued=value.get('continued', False)) + + +def speech_seconds(words): + """Rough speaking time: about 2.6 English words or 4.5 CJK characters a second.""" + cjk = len(re.findall(r'[぀-ヿ㐀-鿿가-힯]', words)) + return round(cjk / 4.5 + len(re.findall(r'[A-Za-z0-9À-ɏ]+', words)) / 2.6, 1) + + +def part_source(text): + """(body, sound sections, lines): the description with each spoken line as [L1], [L2], ... + + H3 prompts keep their sound sections apart; they describe the whole video and are repeated as written. + """ + body, sound = text.strip(), '' + found = re.search(r'(?m)^overall_soundscape:', body) + if body.startswith('integrated_multimodal_description:') and found: + body, sound = body[:found.start()].strip(), body[found.start():].strip() + body = body[len('integrated_multimodal_description:'):].strip() + body = re.sub(r'^\[Shot 1\]\s*', '', body) + lines = [] + def tagged(match): + lines.append(match.group(0)) + return '[L%d]' % len(lines) + body = SPEAKER.sub(tagged, body) + spoken = set(spoken_lines(body)) + def quoted(match): + words = next(group for group in match.groups() if group is not None).strip() + if words not in spoken: + return match.group(0) + lines.append(match.group(0)) + return '[L%d]' % len(lines) + if spoken: + body = QUOTED.sub(quoted, body) + numbers = [int(n) for n in re.findall(r'\[L(\d+)\]', body)] + if numbers != list(range(1, len(lines) + 1)): + raise ValueError('invalid_prompt') + return body, sound, lines + + +def split_instruction(value): + count = len(value['seconds']) + rows, start = [], 0. + for i, length in enumerate(value['seconds'], 1): + rows.append(f'Part {i}: {start:.0f}-{start + length:.0f} s') + start += length + opening = 'continues an earlier video' if value.get('continued') else 'is the opening' + return f"""A long video is made of {count} parts generated one after another; each part continues where the +previous one ended. Sort the user's description into what stays the same and what happens in each part: +{chr(10).join(rows)} +Shared: who is in the scene and how they look, the place, the lighting, the camera, the style and the +background sound. Copy these words from the description. Leave out every action, event, change and every [L] marker. +Part N: only what happens during that part, in the order of the description, so each action happens +once. Part 1 {opening}; each later part picks up where the previous part ended. When the story is +shorter than the video, the later parts continue the same activity calmly instead of repeating it. +The markers [L1], [L2], ... stand for spoken lines; their speaking time is given. Put each marker in +exactly one part, in order, where it is said, as "he says [L2]" or "她说 [L2]"; spread them so each +part's speech fits its seconds. Do not write any other speech. +Example for a 2-part video. Description: A girl in a yellow raincoat stands on a wooden pier at dawn, +filmed in a wide shot, gulls crying. She waves and shouts [L1], runs to the end of the pier and jumps +into the sea. Speaking time: [L1] 2 s +Shared: A girl in a yellow raincoat on a wooden pier at dawn, filmed in a wide shot, gulls crying. +Part 1: She waves and shouts [L1], then starts running along the pier. +Part 2: She reaches the end of the pier and jumps into the sea. +Write in the language of the description. Answer with exactly these lines and nothing else: +Shared: ... +{chr(10).join(f'Part {i}: ...' for i in range(1, count + 1))}""" + + +def split_user_text(value): + body, _, lines = part_source(value['text']) + text = f'Description of the whole {sum(value["seconds"]):.0f}-second video:\n{body}' + if lines: + text += '\n\nSpeaking time: ' + ', '.join('[L%d] %.0f s' % (i, max(1, speech_seconds(line))) + for i, line in enumerate(lines, 1)) + return text + + +def split_answer(text, count): + if not isinstance(text, str): + raise ValueError('invalid_output') + text = text.strip().strip('`').strip() + found = list(re.finditer(r'(?m)^[ \t*#]*(Shared|Part[ \t]+(\d+))[ \t*]*[:：]', text)) + names = [m.group(2) or 'shared' for m in found] + if names != ['shared'] + [str(i) for i in range(1, count + 1)] or text[:found[0].start()].strip(' \t\n*#'): + raise ValueError('invalid_output') + ends = [m.start() for m in found[1:]] + [len(text)] + values = [' '.join(text[m.end():end].split()).strip('* ') for m, end in zip(found, ends)] + return values[0], values[1:] + + +def place_lines(body, lines, seconds): + """Spoken lines to parts, in order, when the model did not place them. + + A line starts halfway between where an even spread of the speech puts it and where it stands in the + story between the first and the last line; a line that would run into the next part starts early enough to fit. + """ + total, lengths = sum(seconds), [max(1., speech_seconds(line)) for line in lines] + ends = [sum(seconds[:i + 1]) for i in range(len(seconds))] + gap = max(0., total - sum(lengths)) / len(lines) + first = body.index('[L1]') + span = max(1, body.index('[L%d]' % len(lines)) - first) # Scenery after the last line is not story time. + placed, clock = [], 0. + for index, length in enumerate(lengths): + even = sum(lengths[:index]) + index * gap + story = (body.index('[L%d]' % (index + 1)) - first) / span * total + start = max(clock, min((even + story) / 2, total - sum(lengths[index:]))) + part = next((n for n, end in enumerate(ends) if start < end), len(ends) - 1) + opening = ends[part - 1] if part else 0. + if start + length > ends[part] and part < len(ends) - 1: + if ends[part] - length >= max(clock, opening): + start = ends[part] - length + elif start - opening > seconds[part] / 2: # A long line starting early in a part stays there. + part, start = part + 1, max(start, ends[part]) + placed.append(part) + clock = start + length + return placed + + +def spoken(line, words): + """A line put back by code: H3 lines carry their own speaker tag; plain quotes need a speech verb.""" + if '' in line: + return line + return ('说：' if language(words) in ('Chinese', 'Japanese') else 'says: ') + line + + +def validate_split(text, value, *, strict=False): + """Part texts built from the model's sorting; every spoken line kept once, in order. + + A part with nothing new carries on from the previous one; a misplaced line is placed by speaking time. + Only an answer without the expected layout or a usable shared description fails; strict also + fails when the model did not place every line once and in order, so it can be asked again. + """ + body, sound, lines = part_source(value['text']) + shared, events = split_answer(text, len(value['seconds'])) + shared = ' '.join(re.sub(r'\[L\d+\]', '', shared).split()) + if not 20 <= len(shared) <= 4000: + raise ValueError('invalid_output') + seen, cleaned = {plain(shared), plain(body)}, [] + for event in events: + key = plain(re.sub(r'\[L\d+\]', '', event)) + if len(event) > 4000 or not key or key in seen or key in plain(shared): + key = '' + seen.add(key) + cleaned.append(event if key else ' '.join(re.findall(r'\[L\d+\]', event))) + events = cleaned + placed = [int(n) for event in events for n in re.findall(r'\[L(\d+)\]', event)] + if placed != list(range(1, len(lines) + 1)) and strict: + raise ValueError('invalid_output') + if placed != list(range(1, len(lines) + 1)): + # The model lost or reordered a line: keep its events, place the lines by time. + events = [' '.join(re.sub(r'\s*\[L\d+\]\s*[:：,，]?', ' ', event).split()).rstrip(':：,，') for event in events] + for number, part in enumerate(place_lines(body, lines, value['seconds']), 1): + events[part] = (events[part] + ' [P%d]' % number).strip() + def restore(words): + # [L] stands where the model put a line in its own words; [P] where code placed it. + return re.sub(r'\[([LP])(\d+)\]', lambda m: (lines[int(m.group(2)) - 1] if m.group(1) == 'L' + else spoken(lines[int(m.group(2)) - 1], shared)), words) + parts = [] + for index, event in enumerate(events): + if index == 0 and not re.sub(r'\[[LP]\d+\]', '', event).strip(): + # Nothing sorted into the opening: it is the user's own description, without later lines. + mine = set(re.findall(r'\[[LP](\d+)\]', event)) + words = re.sub(r'\[L(\d+)\]', lambda m: m.group(0) if m.group(1) in mine else '', body) + words = restore(' '.join(words.split())) + else: + words = ' '.join(([CONTINUE] if index or value.get('continued') else []) + [shared] + + ([restore(event)] if event else [])) + if sound: + words = 'integrated_multimodal_description: [Shot 1] ' + words + '\n\n' + sound + parts.append(words) + return parts + + +def validate_parts(parts, value): + """Worker output shape for a split job: one non-empty text per part.""" + if (not isinstance(parts, list) or len(parts) != len(value['seconds']) + or any(not isinstance(part, str) or not 40 <= len(part) <= 20000 for part in parts)): + raise ValueError('invalid_output') + return parts + + +SPLIT_RETRY = ('Answer again with exactly one "Shared:" line and the "Part N:" lines, every [L] marker exactly once ' + 'and in order, and no marker in Shared.') diff --git a/freevideo_engine/prompt_vlm/service.py b/freevideo_engine/prompt_vlm/service.py index fc64c895..cad2b225 100644 --- a/freevideo_engine/prompt_vlm/service.py +++ b/freevideo_engine/prompt_vlm/service.py @@ -262,7 +262,7 @@ def progress(self, row): def start(self, action, value): received_epoch = time.time() - if action not in ('install', 'rewrite', 'ask'): + if action not in ('install', 'rewrite', 'ask', 'split'): raise ValueError('invalid_request') if self.busy(): raise ValueError('busy') @@ -274,6 +274,10 @@ def start(self, action, value): if not assets.ready(root): raise ValueError('model_missing') value = self.ask_images(rules.ask_request(value)) + elif action == 'split': + if not assets.ready(root): + raise ValueError('model_missing') + value = rules.split_request(value) else: if not assets.ready(root): raise ValueError('model_missing') @@ -290,7 +294,7 @@ def start(self, action, value): with self.mutex: if self.busy(): raise ValueError('busy') - if action in ('rewrite', 'ask'): + if action != 'install': self.prestart_ticket += 1 # Invalidate pending imports before the worker starts. self.key = secrets.token_urlsafe(24) self.stop.clear() @@ -326,6 +330,7 @@ def status(self, key): if self.result_expires and time.monotonic() > self.result_expires: self.state.pop('text', None) self.state.pop('answers', None) + self.state.pop('parts', None) self.state.update(phase='failed', error='expired') row = dict(self.state) # The browser may immediately queue video work or the next rewrite. @@ -334,6 +339,7 @@ def status(self, key): row['phase'] = 'finishing' row.pop('text', None) row.pop('answers', None) + row.pop('parts', None) return row def cancel(self, key): @@ -368,7 +374,7 @@ def work(self, root, machine, action, value): from ..background_compile import COMPILE COMPILE.stop() self.release_idle_image_worker() - if action in ('rewrite', 'ask'): + if action != 'install': from ..resident_process import OWNER if OWNER.active: raise BlockingIOError('busy') @@ -388,13 +394,14 @@ def work(self, root, machine, action, value): except BlockingIOError: self.progress(dict(phase='failed', error='busy')) except Exception as error: - known = {'disk_space', 'ram_space', 'rewrite_failed', 'worker_exit', 'timeout', 'worker_cleanup_failed'} + known = {'disk_space', 'ram_space', 'rewrite_failed', 'split_failed', 'worker_exit', 'timeout', 'worker_cleanup_failed'} self.progress(dict(phase='failed', error=str(error) if str(error) in known else 'download_failed' if action == 'install' - else 'ask_failed' if action == 'ask' else 'rewrite_failed', + else 'ask_failed' if action == 'ask' else 'split_failed' if action == 'split' + else 'rewrite_failed', exception=type(error).__name__)) finally: - if action in ('rewrite', 'ask'): + if action != 'install': self.stop_prestart(self.prewarm_state.get('reason', 'none')) if self.stop.is_set(): self.progress(dict(phase='cancelled')) @@ -564,6 +571,8 @@ def read_rows(): if result.get('phase') == 'complete': if value.get('mode') == 'ask': rules.validate_answers(result.get('answers'), value) + elif value.get('mode') == 'split': + rules.validate_parts(result.get('parts'), value) else: rules.validate_output(result.get('text'), value) self.progress(dict(result, worker_returncode=child.returncode)) diff --git a/freevideo_engine/prompt_vlm/worker.py b/freevideo_engine/prompt_vlm/worker.py index 0fe96cee..34595365 100644 --- a/freevideo_engine/prompt_vlm/worker.py +++ b/freevideo_engine/prompt_vlm/worker.py @@ -343,6 +343,8 @@ def run(folder, value, *, stats=None): stats = {} if stats is None else stats if value.get('mode') == 'ask': return ask(folder, value, stats) + if value.get('mode') == 'split': + return split(folder, value, stats) return rewrite(folder, value, stats) @@ -409,6 +411,62 @@ def on_token(timing): emit(phase='complete', text=rewritten, diagnostics=dict(stats)) +def split(folder, value, stats): + """One description per part of a long video; text leaves only via stdout.""" + from .rules import SPLIT_RETRY, split_instruction, split_user_text, validate_split + count = len(value['seconds']) + limit = min(2 * OUTPUT_TOKENS - 200, 600 + 300 * count) # Within the cache reserved for a rewrite and its retry. + stats.update(stage='imports', pid=os.getpid(), mode='split', part_count=count, dtype='bfloat16', + attention='sdpa', kv_cache='dynamic', backend='transformers', accelerations=[], + persistent_model=False) + torch, checkpoint = begin(stats) + from transformers import AutoProcessor + checkpoint('preprocess') + preparing = time.monotonic() + processor = AutoProcessor.from_pretrained(folder, local_files_only=True, trust_remote_code=False) + messages = [{'role': 'system', 'content': split_instruction(value)}, + {'role': 'user', 'content': [{'type': 'text', 'text': split_user_text(value)}]}] + inputs = tokenize(processor, messages) + if inputs.input_ids.shape[-1] > 6500: + raise ValueError('context_length') + stats.update(input_tokens=int(inputs.input_ids.shape[-1]), preprocess_seconds=time.monotonic() - preparing) + model, device = load_model(folder, torch, stats, checkpoint) + loaded = time.monotonic() + checkpoint('prefill', 'splitting') + shown = [0.] + def on_token(timing): + stats.update(output_tokens=timing['output_tokens'], decode_seconds=timing['decode_seconds']) + if time.monotonic() - shown[0] > .5: + checkpoint('decode', 'splitting') + shown[0] = time.monotonic() + ids, _ = generate(torch, model, inputs.to(device), limit, loaded, on_token) + answers = [processor.decode(ids, skip_special_tokens=True)] if len(ids) < limit else [] + stats.update(split_seconds=time.monotonic() - loaded, output_tokens=len(ids)) + parts = None + try: + parts = validate_split(answers[0], value, strict=True) + except (ValueError, IndexError): + # One more try with the rules restated; resending the wrong answer would not fit the cache. + retry = [messages[0], {'role': 'user', 'content': [{'type': 'text', 'text': ( + split_user_text(value) + '\n\n' + SPLIT_RETRY)}]}] + retry_inputs = tokenize(processor, retry) + if retry_inputs.input_ids.shape[-1] + limit <= stats.get('kv_positions', 10 ** 9): + ids, _ = generate(torch, model, retry_inputs.to(device), limit, time.monotonic(), lambda timing: None) + stats.update(split_retry=True, retry_output_tokens=len(ids), split_seconds=time.monotonic() - loaded) + if len(ids) < limit: + answers.insert(0, processor.decode(ids, skip_special_tokens=True)) + for strict in (True, False): + for answer in answers: + try: + parts = parts or validate_split(answer, value, strict=strict) + except ValueError: + pass + if parts is None: + raise ValueError('split_failed') + stats['stage'] = 'complete' + emit(phase='complete', parts=parts, diagnostics=dict(stats)) + + def ask(folder, value, stats): """Answer each question with one loaded model; answers leave only via stdout.""" from .rules import ask_instruction, image_tokens, parse_json, ASK_JSON_RETRY From eebae0441555f6b26459ab22287d03779bdf17de Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:58 +0000 Subject: [PATCH 16/47] Agent: a long video can take one description per part --- freevideo_engine/agent/guide.py | 3 +++ freevideo_engine/agent/jobs.py | 23 +++++++++++++++++++++++ freevideo_engine/agent/routes.py | 4 ++-- freevideo_engine/agent/schema.py | 5 ++++- freevideo_engine/agent/spec.py | 2 ++ 5 files changed, 34 insertions(+), 3 deletions(-) diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 92301670..e94e4777 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -77,6 +77,9 @@ 'Multiple blocks or continue_from need two-pass text generation: quality "light" ' 'or sampling.two_pass=true, no preview and no media. ' 'Multi-block output is cut to the planned duration. POST /plan shows the chain plan. ' + 'For a video in several parts, part_prompts gives each part its own description in order (as many as ' + 'chain_plan.blocks from POST /plan), so each part continues the story instead of replaying it; ' + 'without it FreeVideo splits prompt itself. ' 'A continue_from job saves a new video that holds only the added part; to hand the user one video, ' 'insert both into a project and export it.', 'first_frame_reference (default false) reduces drift in a continued video: each continuing part also ' diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 0f517fbb..c02cf7a4 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -157,6 +157,24 @@ def cut_problems(spec, seconds, prefix='', *, editor=None): return [dict(problem, got=got, message=problem['message'].replace('received None', 'received ' + got))] +def part_prompt_problems(spec, prefix=''): + """part_prompts gives one description per planned part; the chain backend refuses any other count.""" + if spec['kind'] != 'generate' or not spec.get('part_prompts'): + return [] + from ..chain_plan import PlanError, plan + try: + count = len(plan(spec['seconds'], max_frames=single_frame_limit(spec), + continued=bool(spec.get('continue_from')))['blocks']) + except PlanError: + return [] # parts_problems names the length. + given = len(spec['part_prompts']) + if given == count: + return [] + allowed = '%d description%s, one for each of the chain_plan.blocks POST /plan gives for %g s' % ( + count, '' if count == 1 else 's', spec['seconds']) + return [schema.diagnostic(prefix + '/part_prompts', allowed, given)] + + def _continued_part_problem(seconds, chain): """True when a video that continues another has a part the backend refuses, or is under 10 s. @@ -733,6 +751,7 @@ def submit_request(self, value, *, client=None, key=None, parent=None, editor=No work=self.continued_work(normalized), planned=self.planned_parts(normalized, rows))) problems.extend(cut_problems(normalized, normalized.get('seconds'), prefix, editor=editor)) + problems.extend(part_prompt_problems(normalized, prefix)) try: self.port.check(normalized) except specs.SpecError as error: @@ -1476,6 +1495,10 @@ def resolve(): # Planned as a continuation, so the new video is as long as the request says. payload.update(after=dict(work=self.port.output_relative(self.source_video(spec['continue_from']))), text=prompt, seed=spec['seed']) + if spec.get('part_prompts') and 'seconds' in payload: + # Each part plays its own description, with the same LoRA trigger words the whole prompt gets. + from ..lora_triggers import apply_triggers + payload['texts'] = [apply_triggers(text, spec.get('loras', [])) for text in spec['part_prompts']] self.start_chain(row, payload=payload) work = asyncio.create_task(asyncio.to_thread(prepare)) try: diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index ef1e9261..0dfc2474 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -231,8 +231,8 @@ def plan(port, value, jobs=None): result = dict(spec={k: v for k, v in spec.items() if k != 'depends_on'}, warnings=[]) if spec['kind'] != 'generate' or spec.get('refine_from'): return result - from .jobs import continuation_problems, cut_problems, parts_problems - problems = continuation_problems(spec) + cut_problems(spec, spec['seconds']) + from .jobs import continuation_problems, cut_problems, part_prompt_problems, parts_problems + problems = continuation_problems(spec) + cut_problems(spec, spec['seconds']) + part_prompt_problems(spec) if jobs is not None: problems += parts_problems(spec, jobs.port.chain_runner(), work=jobs.continued_work(spec), planned=jobs.planned_parts(spec)) diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 1e3d01c8..6e6b423e 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -162,11 +162,14 @@ def generate(*, local=False): first_frame_reference={'type': 'boolean', 'default': False, 'description': 'Reduces drift over a continued video: every continuing part also sees the first frame of the whole ' 'video. It holds framing and motion close to that frame.'}, + part_prompts=array(string(20000), 1, 5, description='One description per part of a video made in several ' + 'parts, in order, so each part plays its own share of the story. The count must equal the ' + 'chain_plan.blocks POST /plan gives for this length. Omitted: FreeVideo splits prompt into parts before the first part.'), takes=number(1, 8, integer=True, default=1), **common()), allOf=[{'if': {'required': ['refine_from'], 'properties': {'refine_from': {'type': 'string'}}}, 'then': {'properties': {k: False for k in ('prompt', 'width', 'height', 'aspect', 'megapixels', 'seconds', 'seed', 'quality', 'sampling', 'media', 'loras', 'preview', 'continue_from', - 'first_frame_reference')}}, + 'first_frame_reference', 'part_prompts')}}, 'else': {'required': ['prompt']}}, {'if': {'anyOf': [{'required': [k]} for k in ('aspect', 'megapixels')]}, 'then': {'not': {'anyOf': [{'required': [k]} for k in ('width', 'height')]}}}, diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index 96ab196a..e1af7de1 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -191,6 +191,8 @@ def normalize(value, *, port=None): result['continue_from'] = dict(value['continue_from']) if value['first_frame_reference']: result['first_frame_reference'] = True + if value.get('part_prompts'): + result['part_prompts'] = list(value['part_prompts']) result['loras'] = [dict(name=row, strength=1.) if isinstance(row, str) else {'strength': 1., **row} for row in value['loras']] if port is not None: From 8aa13d281913ebd7d93b3a589d3d987f718769e6 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:58 +0000 Subject: [PATCH 17/47] Editing view: a continuing part counts only reference images toward its three --- web/editor.js | 19 +++++++++---------- web/editor_genbar.js | 2 +- web/editor_references.js | 40 +++++++++++++++++++++++++--------------- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/web/editor.js b/web/editor.js index 28bfca76..ff98c0c3 100644 --- a/web/editor.js +++ b/web/editor.js @@ -319,9 +319,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given const start = Math.min(...clips.map(clip => clip.start)); return {start, frames: Math.max(...clips.map(clip => clip.start + clip.frames)) - start}; } - // Whether the part about to be made starts fresh instead of continuing footage: only then can pictures guide it. - function startsFresh() { - const range = selectionRange(); if (!range) return false; + // How the part about to be made starts: 'fresh' (pictures can guide it), 'continues' (from the footage before it) or null. + function partStart() { + const range = selectionRange(); if (!range) return null; try { const selected = selectionClips(); const planned = selected.length && selected.every(clip => clip.pending && !clip.source) @@ -329,8 +329,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given : model.planRange(state.record, {...range, target: 'both', text: 'x', maxSeconds: capability.maxSeconds, maxFollowSeconds: capability.maxFollowSeconds}); const ids = planned.tracks[0].clips.filter(clip => clip.start >= range.start && clip.start < range.start + range.frames && clip.pending && !['queued', 'running'].includes(clip.pending)).map(clip => clip.id); - return ids.length > 0 && model.generationBefore(model.prepare(planned, ids), ids).some(before => before == null); - } catch (value) { if (value instanceof model.EditError) return false; throw value; } + if (!ids.length) return null; + return model.generationBefore(model.prepare(planned, ids), ids).some(before => before == null) ? 'fresh' : 'continues'; + } catch (value) { if (value instanceof model.EditError) return null; throw value; } } const jobActive = () => generationBusy || [...jobStatuses.values()].some(job => ['queued', 'running', 'cancelling'].includes(job.state)); // Parts pressed while they cannot be sent yet: the engine is still starting, or they continue a part @@ -473,7 +474,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given if (!value.node || !value.prompt?.[value.node]) throw new Error('missing_node'); const inputs = value.prompt[value.node].inputs; Object.assign(inputs, state.record.settings || {}, capability.generationSize || generationSize(state.record)); - // The media panel's pictures guide a part that starts fresh (generate() drops them when every part continues footage); + // The media panel's pictures guide every part (the engine keeps only the references for a part that continues); // without a panel to show them they stay out of the request. if (!panel.references?.available()) delete inputs.media; delete inputs.continue_from; delete inputs.continue_to; @@ -730,9 +731,9 @@ export function openEditor({api, app, from = null, panel = {}, components: given clip.start + clip.frames === range.start && clip.source && clip.extend && clip.in + clip.frames === clip.length); return before || model.rangeParts(state.record, {...range, ...capability}).length > 1; }, - // Pictures that guide a new part: the creative panel's media rows, shown while the part starts fresh. + // Pictures that guide a new part: the creative panel's media rows; a part that continues shows only its references. references: panel.references ? {source: panel.references, notify, size: () => capability.generationSize || generationSize(state.record), - visible: () => store.generationAvailable && actions.supportedSelection() && startsFresh()} : null, + visible: () => store.generationAvailable && actions.supportedSelection() ? partStart() : null} : null, setFirstFrame: value => { state.apply(t('First-frame reference', '首帧参考'), record => model.setFlag(record, 'first_frame_reference', value)); }, previewRange: selection => genbar?.preview(selection), async estimate(selection) { @@ -768,8 +769,6 @@ export function openEditor({api, app, from = null, panel = {}, components: given submittedSelection = shown && state.selection === selection ? state.select({kind: 'clip', ids}) : state.selection; await completeSources(); const paths = Object.fromEntries([...store.sources].map(([key, value]) => [key, value.path])); const request = model.generationRequest(prepared, ids, {text, paths}); - // Pictures guide parts that start fresh; a request whose parts all continue footage carries none. - if (!request.blocks.some(block => !block.after)) delete frozen.prompt[frozen.node].inputs.media; const follow = continuedSize(model.generationBefore(prepared, ids), store.sources), edit = capability.generationSize || generationSize(state.record); Object.assign(frozen.prompt[frozen.node].inputs, follow || {}); // Parts made at another size than the edit's have that size's limit. diff --git a/web/editor_genbar.js b/web/editor_genbar.js index 0bfb6904..d9aeb56b 100644 --- a/web/editor_genbar.js +++ b/web/editor_genbar.js @@ -108,7 +108,7 @@ export function createEditorGenbar(host, {state, t, actions, text = ''}) { hint.title = shown.map(([, text]) => text).join(''); } firstFrame.update(busy); - references?.update(busy, busy ? undefined : !!actions.references.visible()); + references?.update(busy, busy ? undefined : actions.references.visible() || false); generate.textContent = t('Generate', '生成'); // Never disabled for a reason the press itself can explain or fix. generate.disabled = busy || !!previewSelection; diff --git a/web/editor_references.js b/web/editor_references.js index 5f2f8ec3..4bd6f808 100644 --- a/web/editor_references.js +++ b/web/editor_references.js @@ -1,8 +1,11 @@ // Reference pictures for a new part, kept small inside the generation bar: thumbnails and one "+". +// A part that continues the one before keeps its reference pictures (people stay the same) but not its first or last frame. // `source` is the media rows of the creative panel (editor_media_references.js); `size()` is the video's shape. const el = (tag, cls, text) => { const e = document.createElement(tag); if (cls) e.className = cls; if (text != null) e.textContent = text; return e; }; const IMAGE = /\.(png|jpe?g|webp|bmp|tiff?)$/i; const MAX = 32; +// The engine keeps the first three references of a part that continues. +const CONTINUE_MAX = 3; export function createEditorReferences({t, source, size = () => null, error = () => {}, notify = () => {}}) { const root = el('div', 'fv-ed-refs'), chips = el('div', 'fv-ed-ref-chips'); const add = el('button', 'fv-ed-ref-add'); add.type = 'button'; @@ -12,12 +15,16 @@ export function createEditorReferences({t, source, size = () => null, error = () const menu = el('div', 'fv-ed-ref-menu'); menu.hidden = true; menu.setAttribute('role', 'menu'); const picker = el('input'); picker.type = 'file'; picker.accept = 'image/*'; picker.multiple = true; picker.hidden = true; root.append(add, chips, menu, picker); - let shown = false, busy = false, working = false, opener = null, disposed = false, entries = [], waited = false; + let shown = false, continuing = false, busy = false, working = false, opener = null, disposed = false, entries = [], waited = false; const active = () => source.rows().map((row, index) => ({row, index})).filter(({row}) => row.enabled !== false); const isImage = file => IMAGE.test(file || ''); + // The pictures this part uses: all of them, or the first reference images of a part that continues + // (sound or video references do not count there, as the engine skips them). + const used = () => continuing ? active().filter(({row}) => (row.role || 'reference') === 'reference' && isImage(row.file)).slice(0, CONTINUE_MAX) : active(); + const full = () => continuing ? used().length >= CONTINUE_MAX : active().length >= MAX; // On hover without a full stop; in the notice line with one. const waitText = () => t('Available after generation finishes', '生成完成后可改'); - const fullText = () => t('Up to 32 images', '最多 32\u00a0张'); + const fullText = () => continuing ? t('Up to 3 images when continuing', '接续时最多 3\u00a0张') : t('Up to 32 images', '最多 32\u00a0张'); const sentence = text => text + t('.', '。'); const label = role => role === 'first' ? t('First frame', '首帧') : role === 'last' ? t('Last frame', '尾帧') : t('Reference', '参考'); function close(focus = true) { @@ -58,7 +65,7 @@ export function createEditorReferences({t, source, size = () => null, error = () working = true; update(); try { for (const file of files) { - if (active().length >= MAX) break; + if (full()) break; const result = await source.upload(file); write([...source.rows(), {file: result.file, role: 'reference', enabled: true}]); } @@ -93,6 +100,11 @@ export function createEditorReferences({t, source, size = () => null, error = () } function chipMenu(entry) { entries = []; + const remove = () => item(t('Remove', '移除'), () => { + const rows = source.rows(); + if (rows[entry.index]?.file === entry.row.file) { rows[entry.index] = {...rows[entry.index], enabled: false}; write(rows); } + }); + if (continuing) return [remove()]; const roles = ['reference', 'first', 'last'].map(role => { const b = item(label(role), () => { const rows = source.rows(); @@ -104,25 +116,23 @@ export function createEditorReferences({t, source, size = () => null, error = () b.setAttribute('aria-checked', String((entry.row.role || 'reference') === role)); return b; }); const separator = el('div'); separator.setAttribute('role', 'separator'); - return [...roles, separator, item(t('Remove', '移除'), () => { - const rows = source.rows(); - if (rows[entry.index]?.file === entry.row.file) { rows[entry.index] = {...rows[entry.index], enabled: false}; write(rows); } - })]; + return [...roles, separator, remove()]; } // The line said on a click while a part is made; it goes away with the busy state it describes. const wait = () => { waited = true; notify(sentence(waitText())); }; function update(isBusy = busy, visible = shown) { if (waited && busy && !isBusy) { waited = false; notify(''); } - busy = isBusy; shown = visible; + const was = continuing; + busy = isBusy; shown = visible; continuing = shown === 'continues'; root.hidden = !shown || !source.available(); - if ((root.hidden || busy) && !menu.hidden) close(false); - // The panel holds at most MAX pictures: a full list greys the plus out rather than ignoring a click. - const full = active().length >= MAX; + if ((root.hidden || busy || was !== continuing) && !menu.hidden) close(false); + // The panel holds at most MAX pictures, a continuing part uses three: a full list greys the plus out rather than ignoring a click. + const isFull = full(); // While a part is being made the controls stay clickable and say why they do nothing. add.disabled = working; - add.setAttribute('aria-disabled', String(busy || full)); - add.title = busy ? waitText() : full ? fullText() : t('Reference image', '参考图'); - chips.replaceChildren(...active().map(entry => { + add.setAttribute('aria-disabled', String(busy || isFull)); + add.title = busy ? waitText() : isFull ? fullText() : t('Reference image', '参考图'); + chips.replaceChildren(...used().map(entry => { const chip = el('button', 'fv-ed-ref-chip'); chip.type = 'button'; chip.disabled = working; chip.setAttribute('aria-disabled', String(busy)); chip.dataset.role = entry.row.role || 'reference'; chip.title = busy ? waitText() : label(entry.row.role); chip.setAttribute('aria-haspopup', 'menu'); @@ -134,7 +144,7 @@ export function createEditorReferences({t, source, size = () => null, error = () } add.onclick = () => { if (busy) { wait(); return; } - if (active().length >= MAX) { notify(sentence(fullText())); return; } + if (full()) { notify(sentence(fullText())); return; } if (!menu.hidden && opener === add) close(); else open(add, addMenu()); }; picker.onchange = () => addFiles(Array.from(picker.files)); // Keys stay in the menu: Escape closes it only, and the arrows move through it. From eb932534a7d188e0c5ab59d69ecae0fd1dad44dd Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:43:58 +0000 Subject: [PATCH 18/47] Agent: reference pictures give way to first_frame_reference as the engine does Agent: reference pictures give way to first_frame_reference as the engine does; frame times have an upper bound --- freevideo_engine/agent/drama.py | 37 +++++++++++++++++----------- freevideo_engine/agent/guide.py | 6 +++-- freevideo_engine/agent/jobs.py | 9 +++++-- freevideo_engine/agent/production.py | 9 +++---- freevideo_engine/agent/schema.py | 11 +++++---- freevideo_engine/agent/spec.py | 15 +++++++++++ 6 files changed, 59 insertions(+), 28 deletions(-) diff --git a/freevideo_engine/agent/drama.py b/freevideo_engine/agent/drama.py index 8d1e1154..0ec7e133 100644 --- a/freevideo_engine/agent/drama.py +++ b/freevideo_engine/agent/drama.py @@ -150,16 +150,16 @@ def validate(function, *args, **kwargs): validate(specs._int, shot[field], path + '/' + field, low, high) if 'seconds' in shot: validate(specs._number, shot['seconds'], path + '/seconds', 1.625, 60.) - modes = int(shot.get('first') is not None) + int(bool(shot.get('continue'))) + int('references' in shot) - if modes > 1: - problem(path, 'first, continue and references are mutually exclusive') + # A continuing shot starts from the previous one and may add reference pictures, never a first frame. + if shot.get('first') is not None and (shot.get('continue') or shot.get('references')): + problem(path, 'first is exclusive with continue and references') # Continuing a shot, or one longer than one 15 s block, needs two-pass text generation. seconds = shot.get('seconds', play.get('seconds')) long = isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and seconds > 15 if ((shot.get('continue') or long) and play.get('quality') in specs.QUALITY and not specs.QUALITY[play['quality']][0] and all(p['path'] != '/quality' for p in problems)): problem('/quality', '"light" (two-pass) when a shot continues another or is longer than 15 s') - if long and (shot.get('first') is not None or 'references' in shot): + if long and (shot.get('first') is not None or shot.get('references') and not shot.get('continue')): problem(path + '/seconds', 'at most 15 with a first frame or references') elif long and not shot.get('continue') and _carded(play, shot): # Cast cards travel as references, which one generation of at most 15 s carries. @@ -171,9 +171,13 @@ def validate(function, *args, **kwargs): refs = shot['references'] if not isinstance(refs, list) or len(refs) > 32: problem(path + '/references', 'must be a list of at most 32 items') + elif shot.get('continue') and len(refs) > specs.CONTINUED_PICTURES: + problem(path + '/references', 'at most %d pictures when the shot continues' % specs.CONTINUED_PICTURES) if isinstance(refs, list): for j, ref in enumerate(refs): - validate(specs.media_item, ref, path + '/references/%d' % j) + validate(specs.media_item, ref, path + '/references/%d' % j, image=bool(shot.get('continue'))) + if shot.get('continue') and not specs.still_picture(ref): + problem(path + '/references/%d' % j, 'must be a picture when the shot continues') if 'cast' in shot: if not isinstance(shot['cast'], list): problem(path + '/cast', 'must be a list of character names') @@ -291,21 +295,20 @@ def cast_references(play, shot, cards, *, continues=False): cards maps a card ID to the current card, or None once it is gone. The same function serves plan previews and the moment a take is submitted. - cast_reference: None (no character in the shot has a card), 'inherited' - (continues the previous shot), 'own' (the shot names its own first frame or - references), 'missing' (a card was deleted), 'partial' (not everyone fit) - or 'cards'. 'omitted' lists people left out entirely; 'voice_omitted' lists - people whose pictures are carried but whose voice did not fit.""" + cast_reference: None (no character in the shot has a card), 'own' (the shot + names its own first frame or references), 'missing' (a card was deleted), + 'partial' (not everyone fit) or 'cards'. 'omitted' lists people left out + entirely; 'voice_omitted' lists people whose pictures are carried but whose + voice did not fit. A shot that continues the previous one carries one picture + per person (the sheet, else the front view) for at most CONTINUED_PICTURES + people, and no voice clip: its voices carry on from the previous shot.""" characters = play.get('characters', {}) keys = [k for k in cast(play, shot) if isinstance(characters[k].get('card'), str)] result = dict(cast_reference=None, people=[], omitted=[], voice_omitted=[], missing=[], references=[], prefix=None, subjects=[], cards={}) if not keys: return result - if continues: - result['cast_reference'] = 'inherited' - return result - if shot.get('first') is not None or 'references' in shot: + if shot.get('first') is not None or shot.get('references'): result['cast_reference'] = 'own' return result spoken = {} @@ -320,6 +323,12 @@ def cast_references(play, shot, cards, *, continues=False): result['missing'].append(key) continue images = _pictures(card) + if continues: + if images and len(chosen) < specs.CONTINUED_PICTURES: + chosen.append((key, card, images[:1], False)) + else: + result['omitted'].append(key) + continue voice = card.get('voice') is not None and voices < MAX_VOICES if pictures + len(images) + voices + voice > MAX_REFERENCES: result['omitted'].append(key) diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index e94e4777..d9817494 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -75,7 +75,8 @@ 'Under 30 s defaults to one generation when the device supports this size and quality; unknown limits use 15 s. ' 'Otherwise it uses as few parts as possible, each at least 10 s; how many depends on how long this computer can generate in one run, and the parts continue and join automatically. ' 'Multiple blocks or continue_from need two-pass text generation: quality "light" ' - 'or sampling.two_pass=true, no preview and no media. ' + 'or sampling.two_pass=true and no preview. Their only media is media.references with at most 3 pictures, ' + 'which every continuing part also sees (keeps the same people); first_frame_reference then gives way to them. ' 'Multi-block output is cut to the planned duration. POST /plan shows the chain plan. ' 'For a video in several parts, part_prompts gives each part its own description in order (as many as ' 'chain_plan.blocks from POST /plan), so each part continues the story instead of replaying it; ' @@ -533,7 +534,8 @@ def connection_text(value): 'Changes need the revision you read (body revision or If-Match); stale_revision returns the current card in ' 'error.card. Fields the user changed return changed_by_user: keep the user\'s version.', 'Film with a card by setting characters..card. Cards are read when each take is submitted. Shots that ' - 'continue the previous one carry no references (cast_reference inherited); POST /drama/plan shows each shot\'s cast. ' + 'continue the previous one carry one card picture per person (the sheet, else the front) for at most 3 people, ' + 'and keep the voices of the previous shot; POST /drama/plan shows each shot\'s cast. ' 'A shot with a carded character is at most 15 s; continue it in the next shot to go longer.', 'POST /cast/{id}/generate makes reference pictures from the description as a normal job (face=true adds a close-up). ' 'It defaults to the installed image model, otherwise video frames (labels guessed, label_guess). ' diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index c02cf7a4..15451274 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -136,9 +136,14 @@ def continuation_problems(spec, prefix='', *, editor=None): problems.append(dict(problem, got=got, message=problem['message'].replace('received None', 'received ' + got))) if spec['preview']: problems.append(schema.diagnostic(prefix + '/preview', 'false' + why, True)) - if spec['media']: - problems.append(schema.diagnostic(prefix + '/media', 'none' + why + ' (continue_from carries the previous video)', + others = [key for key, value in spec['media'].items() if key != 'references' and value] + pictures = spec['media'].get('references') or [] + if others: + problems.append(schema.diagnostic(prefix + '/media', 'only references' + why + ' (continue_from carries the previous video)', None)) + elif len(pictures) > specs.CONTINUED_PICTURES or not all(specs.still_picture(item) for item in pictures): + problems.append(schema.diagnostic(prefix + '/media/references', 'at most %d pictures%s' % (specs.CONTINUED_PICTURES, why), + len(pictures))) return problems diff --git a/freevideo_engine/agent/production.py b/freevideo_engine/agent/production.py index 07a5c72e..4c0a5158 100644 --- a/freevideo_engine/agent/production.py +++ b/freevideo_engine/agent/production.py @@ -266,11 +266,10 @@ def _schedule(state, shot, count, by, note=None, *, automatic=False): job.pop('media', None) if job['project'] is None: job.pop('project') - if not shot['continues']: - if source.get('first') is not None: - job['media'] = {'first': deepcopy(source['first'])} - elif 'references' in source: - job['media'] = {'references': deepcopy(source['references'])} + if source.get('first') is not None and not shot['continues']: + job['media'] = {'first': deepcopy(source['first'])} + elif source.get('references'): + job['media'] = {'references': deepcopy(source['references'])} jobs.append(dict(take=None, job=None, generation=None, seed=None, state='queued', automatic=automatic, stale=False, continues_take=None, note=note, by=by, library_id=None, seconds=None, cancel_requested=False, error=None, spec=job)) diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 6e6b423e..79f73768 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -57,7 +57,7 @@ def enum(values, **kw): REF = string(128, pattern=r'^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$', description='Idempotency key; reuse only with the same request content. Batch row refs also name dependencies.') FILE = string(1024, pattern=r'^(?!/)(?!.*(?:\\|:))(?!.*(?:^|/)\.\.(?:/|$)).+$') -FRAME = {'oneOf': [enum(('first', 'last')), number(0)]} +FRAME = {'oneOf': [enum(('first', 'last')), number(0, 86400)]} # Seconds into the video; the bound keeps it a float. WAIT = number(0, default=0, description='x-clamp caps accepted wait values at 25 seconds without rejecting larger values.', **{'x-clamp': 25}) CLIENT_WAIT = number(0, 25, default=25) # Waiting for the user's words is longer: a person types slowly, and each wait is one tool call. @@ -859,8 +859,8 @@ def drama_route_table(): CAST_REFERENCE = {'enum': [None, 'cards', 'partial', 'missing', 'inherited', 'own'], 'description': 'How the shot carries cast cards: cards; partial (cast_omitted did not fit at all, or ' 'cast_voice_omitted came without their voice); missing ' - '(a card was deleted, cast_missing shoot from text); inherited (continues the ' - 'previous shot); own (the shot names its own first frame or references); null.'} + '(a card was deleted, cast_missing shoot from text); inherited (a film made before continuing ' + 'shots carried cards); own (the shot names its own first frame or references); null.'} def production_response(): @@ -1087,8 +1087,9 @@ def bad(field, allowed, got=None): if value.get('scene_image') and not value.get('people'): bad('scene_image', 'omitted unless people is supplied', value['scene_image']) if kind == 'generate': - if value.get('continue_from') and value.get('media'): - bad('media', 'omitted when continue_from is supplied', value['media']) + if value.get('continue_from') and isinstance(value.get('media'), dict) and any( + v is not None for k, v in value['media'].items() if k != 'references'): + bad('media', 'only references when continue_from is supplied', value['media']) if value.get('refine_from') is None: custom = value.get('sampling', {}) two, base, refine = QUALITY[value.get('quality', 'light')] diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index e1af7de1..39973501 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -9,6 +9,8 @@ QUALITY, ASPECTS = contract.QUALITY, contract.ASPECTS KINDS = tuple(contract.KINDS) FRAMES = ('first', 'last') +# A part that continues a video carries at most this many reference pictures, and no other reference media. +CONTINUED_PICTURES = 3 MAX_SEED = contract.generate()['properties']['seed']['maximum'] JOB_ID = re.compile(contract.JOB['pattern'].strip('^$')) LIBRARY_ID = re.compile(contract.SAVED['pattern'].strip('^$')) @@ -142,6 +144,19 @@ def generation_media(value): return result +def still_picture(item): + """A reference that is one picture: a frame of a video, or a picture file.""" + if isinstance(item, str): + item = {'file': item} + if not isinstance(item, dict): + return False + if 'frame' in item: + return True + from ..comfy_assets import EXTENSIONS + source = item.get('file') or item.get('path') + return isinstance(source, str) and source.lower().endswith(tuple(EXTENSIONS['image'])) + + def task(media): if media.get('references'): return 'ref2va' From fb84ad1d181885811da6677d3c0b4f544e25ee27 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:44:08 +0000 Subject: [PATCH 19/47] Show and edit each part's description of a long video --- web/generation_progress.js | 21 +++++--- web/studio.css | 15 ++++++ web/studio.js | 100 ++++++++++++++++++++++++++++++++++++- web/studio_chain.js | 23 ++++++++- 4 files changed, 150 insertions(+), 9 deletions(-) diff --git a/web/generation_progress.js b/web/generation_progress.js index e398d2a2..38d1c67a 100644 --- a/web/generation_progress.js +++ b/web/generation_progress.js @@ -94,6 +94,9 @@ const STALL_SECONDS_BY_PHASE = Object.freeze({ starting: 180, encoding: 180, load: 180, sampling: 180, refinement: 180, latent_upscale: 180, sample_finalize: 180, decode: 180, save: 180, dependencies: 180, recovery: 180, other: 180, + // A long video's part descriptions: the server waits at most 10 min for the model and + // writes for at most 15 min, then falls back to the whole description on its own. + describe: 1500, }); // Normal stage sentences keep the sampling step or a download's size and speed. // After 15 s of engine silence SUBACTIVITY names the work without counts. @@ -101,6 +104,7 @@ const STALL_SECONDS_BY_PHASE = Object.freeze({ // Stage words the product already uses: settings say 一采 / 二采, results say 解码. const ACTIVITY = { prepare: ['Preparing', '正在准备'], + describe: ['Writing part descriptions', '正在写各段描述'], prompt: ['Encoding the prompt', '正在编码提示词'], combined: ['Encoding the prompt and references', '正在编码提示词和参考素材'], load: ['Loading the video model', '正在载入视频模型'], @@ -194,7 +198,7 @@ function activityLabel(message, t, plan = {}, secondPass = false, references = f return steps('pass1', twoPass ? 'first' : 'single', step, count); } const key = labelActivities.get(message.label) || { - starting: 'prepare', prepare: 'prepare', prediction: 'prepare', encoding: 'combined', + starting: 'prepare', prepare: 'prepare', prediction: 'prepare', describe: 'describe', encoding: 'combined', load: 'load', decode: 'decode', save: 'save', }[phase] || labelActivities.get(message.stage); if (!key) return null; @@ -425,7 +429,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f const stallText = el('span', ''); stall.append(stallText); const trims = new Map(); element.append(heading, track, label, mini, times, compatibilityNote, status, live); - let recovery = '', lowMemory = false, compatibility = null, samplingPlan = {}, secondPass = false, references = false; + let recovery = '', lowMemory = false, compatibility = null, samplingPlan = {}, secondPass = false, references = false, splitFailed = null; let hasHeartbeat = false; // Mac: the unified-memory notice of this request, and whether its first pass has ended. let memoryTight = null, firstPassOver = false; @@ -698,7 +702,8 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f const progressText = [paused ? pauseText : '', decodeEta].filter(Boolean).join(t(' · ', '，')); const level = ended ? '' : recovery ? 'retry' : paused ? 'pause' : lowMemory ? 'memory' : memoryTight ? 'unified-memory' : automatic ? 'automatic' - : compatibility && compatibility.level > 0 ? 'compatibility' : trims.size ? 'notice' : ''; + : compatibility && compatibility.level > 0 ? 'compatibility' : trims.size ? 'notice' + : splitFailed && !progressText ? 'split' : ''; const explanation = {retry: recovery, pause: progressText, memory: t('This video uses a memory-saving mode, so it takes longer than usual but will finish.', @@ -714,7 +719,10 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f compatibility: compatibility ? t(`Compatibility setting ${compatibility.name_en} is on, so generation is slower than usual.`, `已开启兼容性设置（${compatibility.name_zh}档），生成会比平时慢。`) : '', automatic: compatibilityText, - notice: trimStatus([...trims.values()], t)}[level] || (!ended ? progressText : ''); + notice: trimStatus([...trims.values()], t), + // A long video whose part descriptions could not be written: the server's own words stay in the report. + split: splitFailed === 'model_missing' ? t('Prompt model not installed. Every part uses the same description.', '未安装提示词模型，各段用同一段描述') + : t('Every part uses the same description.', '各段用同一段描述')}[level] || (!ended ? progressText : ''); if (status.textContent !== explanation) status.textContent = explanation; status.hidden = !explanation; status.dataset.level = level; const announcements = []; @@ -847,7 +855,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f pauseAnnounced = false; lastExplanation = ''; announcedMessages.clear(); live.textContent = ''; status.textContent = ''; resetRemaining(); lastActivity = null; resetMini(); stopCount(); countShown = null; - recovery = ''; lowMemory = false; compatibility = null; references = false; trims.clear(); + recovery = ''; lowMemory = false; compatibility = null; references = false; trims.clear(); splitFailed = null; memoryTight = null; firstPassOver = false; samplingPlan = {}; secondPass = false; hasHeartbeat = false; detail = null; localShare = null; workShares.clear(); decodeMinutes = null; decodeMinutesAt = -Infinity; @@ -892,6 +900,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f completedSteps = Math.max(completedSteps, message.done); if (message.low_memory && typeof message.low_memory === 'object') lowMemory = true; if (message.compatibility && typeof message.compatibility === 'object') compatibility = message.compatibility; + if (message.chain && typeof message.chain === 'object') splitFailed = typeof message.chain.split_failed === 'string' ? message.chain.split_failed : null; // Whether the request has reference media (first or last frame, references). if (typeof message.references === 'boolean') references = message.references; if (message.reference_trimmed || REFERENCE_WORK.has(message.label)) references = true; @@ -979,7 +988,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f tick(); if (timer === null) timer = setInterval(tick, 250); } function hide() { - element.hidden = true; status.hidden = true; recovery = ''; placeholder = false; elapsedLead = 0; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false; + element.hidden = true; status.hidden = true; recovery = ''; placeholder = false; elapsedLead = 0; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false; splitFailed = null; compatibilityNote.hidden = true; compatibilityNote.textContent = ''; memoryTight = null; firstPassOver = false; elapsedShown = 0; diff --git a/web/studio.css b/web/studio.css index 1a8b7b60..5f9c8229 100644 --- a/web/studio.css +++ b/web/studio.css @@ -186,3 +186,18 @@ .fv-studio .fv-first-frame input:checked{background:var(--fv-accent,#6db8fa)} .fv-studio .fv-first-frame input:checked::before{transform:translateX(10px)} .fv-studio .fv-first-frame:has(input:disabled){opacity:.4} +.fv-studio .fv-chain-parts{display:grid;gap:2px;margin-top:-7px} +.fv-studio .fv-chain-parts[hidden]{display:none} +.fv-studio .fv-chain-part{display:grid;grid-template-columns:auto minmax(0,1fr) auto;align-items:start;gap:14px;padding:4px 4px 4px 12px;border-radius:var(--fv-r-sm);font-size:var(--fv-micro);line-height:20px} +.fv-studio .fv-chain-part-name{padding-top:7px;color:var(--fv-muted);white-space:nowrap} +.fv-studio .fv-chain-part .fv-chain-part-text{display:block;min-width:0;min-height:32px;padding:6px 10px;border-color:transparent;background:transparent;color:var(--fv-ink);font-weight:var(--fv-regular);text-align:left;white-space:nowrap;overflow:hidden;text-overflow:ellipsis} +.fv-studio .fv-chain-part:has(.fv-chain-part-text:not(:disabled):not([hidden])):hover{background:var(--fv-hover);cursor:text} +.fv-studio .fv-chain-part:has(.fv-chain-part-text:not(:disabled):not([hidden])):hover .fv-chain-part-text{background:transparent;cursor:text} +.fv-studio .fv-chain-part .fv-chain-part-text:disabled{opacity:1;color:var(--fv-disabled)} +.fv-studio .fv-chain-part:has(.fv-chain-part-text:disabled) :is(.fv-chain-part-name,.fv-chain-part-tag:not([data-kind=error])){color:var(--fv-disabled)} +.fv-studio .fv-chain-part-text[hidden]{display:none} +.fv-studio .fv-chain-part .fv-chain-part-editor{min-height:76px;height:96px;max-height:220px;padding:6px 10px;font-size:var(--fv-micro);line-height:20px;outline:none} +.fv-studio .fv-chain-part-tag{padding:7px 8px 0 0;color:var(--fv-muted);white-space:nowrap} +.fv-studio .fv-chain-part-tag:empty{display:none} +.fv-studio .fv-chain-part-tag[data-kind=error]{color:var(--fv-warning);white-space:normal;max-width:16em;text-align:right} +@media(max-width:680px){.fv-studio .fv-chain-part{gap:8px;padding-left:4px}} diff --git a/web/studio.js b/web/studio.js index 23f501e0..e6533e8f 100644 --- a/web/studio.js +++ b/web/studio.js @@ -31,7 +31,7 @@ import { frameShapes, frameSize } from './frame_sizes.js'; import { createFirstRunGuide } from './first_run_guide.js'; import { createStudioLength, savedSeconds } from './studio_length.js?v=20261011-length'; import { firstFrameRow } from './first_frame_row.js'; -import { createStudioChain, chainPosition, readChainResult, sumPartEstimates, singleRunSnapshot } from './studio_chain.js?v=20261011-length'; +import { createStudioChain, chainPosition, chainParts, chainSplit, readChainResult, sumPartEstimates, singleRunSnapshot } from './studio_chain.js?v=20261014-parts'; import { createExtendState, extendText, extendSourceExists, readExtendRecord, extendSnapshot, mountExtendThumbnail, videoUnavailableText } from './studio_extend.js'; const languageOverride = typeof location !== 'undefined' @@ -43,6 +43,97 @@ const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = n const effortCss = document.createElement('link'); effortCss.rel = 'stylesheet'; effortCss.href = new URL('./sampling_effort.css', import.meta.url).href; document.head.append(effortCss); const enhanceCss = document.createElement('link'); enhanceCss.rel = 'stylesheet'; enhanceCss.href = new URL('./prompt_enhance.css', import.meta.url).href; document.head.append(enhanceCss); const el = (tag, label, cls) => { const e = document.createElement(tag); if (label != null) e.textContent = label; if (cls) e.className = cls; return e; }; +// One line per part of a long video, under the preview. A part that has not started opens in +// place for editing and saves when it loses focus; the server's reply decides what stays. +function createChainPartList(t, save, {onShow = () => {}, onHide = () => {}} = {}) { + const element = el('div', null, 'fv-chain-parts'); element.hidden = true; + const rows = new Map(), timers = new Map(); + let editing = null, showing = null; + function tag(row, text, kind = '') { row.tag.textContent = text; row.tag.dataset.kind = kind; } + function draw(row) { + const {part} = row, writing = editing === row; + row.element.dataset.state = part.editable ? 'pending' : part.state; + row.text.textContent = part.text; + row.text.disabled = !part.editable; + if (!writing && row.editor) { row.editor.remove(); row.editor = null; row.text.hidden = false; } + // A note about a lost edit stays on its line until that part is done. + if (row.tag.dataset.kind === 'error' && part.state === 'done') row.tag.dataset.kind = ''; + if (timers.has(part.id) || row.tag.dataset.kind === 'error') return; + tag(row, part.state === 'done' ? t('Done', '已完成') : part.editable || !['queued', 'running', 'pending'].includes(part.state) ? '' : t('Generating', '生成中')); + } + function flash(row, text, kind) { + clearTimeout(timers.get(row.part.id)); tag(row, text, kind); + if (kind === 'saved') timers.set(row.part.id, setTimeout(() => { timers.delete(row.part.id); tag(row, ''); draw(row); }, 2000)); + } + async function finish(row, keep) { + if (editing !== row) return; + const before = row.part.text, value = row.editor.value; + editing = null; + if (!keep || !value.trim() || value === before) { draw(row); return; } + row.part = {...row.part, text: value}; row.saving = value; draw(row); + try { + await save(row.part.id, value); + row.saving = null; + if (rows.get(row.part.id) === row) flash(row, t('Saved', '已保存'), 'saved'); + } catch (error) { + row.saving = null; + if (rows.get(row.part.id) !== row) return; + row.part = {...row.part, text: before}; draw(row); + flash(row, error.status === 409 ? t('This part has started. Your edit was not saved.', '这一段已开始，修改未保存') + : t('Your edit was not saved.', '修改未保存'), 'error'); + } + } + function open(row) { + if (!row.part.editable || editing === row) return; + if (editing) finish(editing, true); + editing = row; clearTimeout(timers.get(row.part.id)); timers.delete(row.part.id); tag(row, ''); + const editor = row.editor = el('textarea', null, 'fv-chain-part-editor'); + editor.maxLength = 20000; editor.value = row.part.text; + editor.setAttribute('aria-label', t(`Part ${row.part.index + 1} description`, `第 ${row.part.index + 1} 段描述`)); + editor.addEventListener('blur', () => finish(row, true)); + editor.addEventListener('keydown', event => { + // Escape leaves the edit, never the studio. + if (event.key === 'Escape') { event.preventDefault(); event.stopPropagation(); finish(row, false); row.text.focus(); } + }); + row.text.hidden = true; row.text.after(editor); editor.focus(); + editor.setSelectionRange(editor.value.length, editor.value.length); + editor.scrollIntoView({block: 'nearest'}); + } + function clear() { + if (!element.hidden) onHide(); + element.hidden = true; element.replaceChildren(); rows.clear(); editing = null; showing = null; + for (const id of timers.values()) clearTimeout(id); timers.clear(); + } + return {element, clear, update(chain, active) { + // The list belongs to one generation: it shows once the descriptions are written and closes when that ends. + if (!active || !chain?.split || chainSplit(chain).writing) { if (!active || showing !== chain?.id) clear(); return; } + if (showing !== chain.id) clear(); + showing = chain.id; + const first = element.hidden; element.hidden = false; + chainParts(chain).forEach((part, index) => { + part.state = chain.blocks[index].state === 'pending' && chain.blocks[index].prompt_id != null ? 'queued' : chain.blocks[index].state; + let row = rows.get(part.id); + if (!row) { + row = {element: el('div', null, 'fv-chain-part'), tag: el('span', null, 'fv-chain-part-tag')}; + row.text = el('button', null, 'fv-chain-part-text'); row.text.type = 'button'; + row.element.onclick = () => open(row); + row.element.append(el('span', t(`Part ${index + 1}`, `第 ${index + 1} 段`), 'fv-chain-part-name'), row.text, row.tag); + rows.set(part.id, row); element.append(row.element); + } + if (editing === row && !part.editable) { + // The part began while it was open: the words typed so far cannot reach it. + const changed = row.editor.value !== row.part.text; + editing = null; row.part = part; draw(row); + if (changed) flash(row, t('This part has started. Your edit was not saved.', '这一段已开始，修改未保存'), 'error'); + return; + } + if (editing === row) { row.part = {...part, text: row.part.text}; return; } + // A poll answered before the save shows the words being saved, not the older ones. + row.part = row.saving != null ? {...part, text: row.saving} : part; draw(row); + }); + if (first) onShow(); + }}; +} const button = (label, action, cls = '') => { const e = el('button', label, cls); e.type = 'button'; e.onclick = action; return e; }; const widget = (node, name) => node?.widgets?.find(w => w.name === name); const linked = (node, name) => node?.inputs?.some(input => input.name === name && input.link != null); @@ -733,6 +824,12 @@ export function openStudio(node, {extendFrom = null} = {}) { extend: (saved, options) => openExtendEditor(saved, node, {studio: dialog, ...options})}); })); previewHead.append(previewTools); output.append(previewHead); const previewSpace = el('div', null, 'fv-preview-space'); output.append(previewSpace); + // The preview keeps the size it has once the list is in; an opened part grows the column, which scrolls. + const partList = createChainPartList(t, (block, text) => chainController.setText(block, text), { + onShow: () => { previewSpace.style.flex = `0 0 ${previewSpace.getBoundingClientRect().height}px`; }, + onHide: () => { previewSpace.style.flex = ''; }}); + output.append(partList.element); + cleanup.push(() => partList.clear()); const stage = el('div', null, 'fv-stage'); stage.setAttribute('aria-label', t('Video preview', '视频预览')); const stageMedia = el('div', null, 'fv-stage-media'); @@ -940,6 +1037,7 @@ export function openStudio(node, {extendFrom = null} = {}) { lastQueueError = null; status.textContent = ''; status.dataset.error = 'false'; } chainWasActive = active; + partList.update(chain, active); busy = active || Boolean(queueState?.running || queueState?.pending.length); queued = chain?.state === 'queued'; chainLine.hidden = !active; chainLine.textContent = outcome?.joining ? t('Joining the parts.', '正在接合各段。') diff --git a/web/studio_chain.js b/web/studio_chain.js index 7168c418..a6117860 100644 --- a/web/studio_chain.js +++ b/web/studio_chain.js @@ -29,6 +29,17 @@ export function readChainResult(chain) { error: failedJoin ? chain.output_error || 'join_failed' : chain.error || chain.blocks?.find(block => block.state === 'failed')?.error || null}; } +// Before the first part starts the server writes one description per part; until then no part can be changed. +export function chainSplit(chain) { + const split = chain?.split; + return {writing: ['pending', 'running'].includes(split?.state), reason: split?.state === 'failed' ? split.reason || 'failed' : null}; +} +export function chainParts(chain) { + const writing = chainSplit(chain).writing; + return (chain?.blocks || []).map((block, index) => ({id: block.id, index, text: block.inputs?.text ?? '', + editable: !writing && block.state === 'pending' && block.prompt_id == null})); +} + export function sumPartEstimates(plan, estimate) { if (!plan || plan.count < 2 || typeof estimate !== 'function') return null; const values = plan.blocks.map(block => estimate(block.frames)); @@ -80,10 +91,10 @@ export function createStudioChain({api, node, onState = () => {}, onProgress = ( if (Object.keys(overall).length) Object.assign(overall, {status: 'running', reset: false, estimated: false, sampling_time_weighted: true}); previous = overall; lastMessage = {...lastMessage, ...message}; - const phase = readChainResult(chain).joining ? 'save' + const phase = readChainResult(chain).joining ? 'save' : chain.split?.state === 'running' ? 'describe' : ['complete', 'failed', 'cancelled'].includes(lastMessage.phase) ? 'prepare' : lastMessage.phase || 'prepare'; onProgress({...lastMessage, phase, result: null, reset: false, new_request: fresh, - chain: {id: chain.id, ...position}, overall}, chain); + chain: {id: chain.id, ...position, split_failed: chainSplit(chain).reason}, overall}, chain); } function deliver(value, fresh = false) { if (disposed) return false; @@ -182,6 +193,14 @@ export function createStudioChain({api, node, onState = () => {}, onProgress = ( } finally { submitting = false; } }, refresh, + // The reply carries the whole chain, so the part shows the words the server keeps. + async setText(block, text) { + if (!chain) throw new Error('chain_missing'); + const id = chain.id, version = epoch; + const value = (await json(chainRoute(id, `blocks/${encodeURIComponent(block)}/text`), post({text})))?.chain; + if (value?.id === id && version === epoch && !disposed) deliver(value); + return value; + }, async cancel() { if (!chain) return; await changeState('cancel', {}); From 7e4c93ef708747744b8d38e893c2e9750277e2e6 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Wed, 14 Oct 2026 23:44:20 +0000 Subject: [PATCH 20/47] Cast library: a character row in the media area with create, edit and delete dialogs --- scripts/test_freevideo_entry.mjs | 16 +- web/agent_session.js | 9 +- web/cast_library.css | 44 +++++ web/cast_library.js | 296 +++++++++++++++++++++++++++++++ web/freevideo.js | 5 +- web/health.js | 2 +- web/theme.css | 2 +- 7 files changed, 368 insertions(+), 6 deletions(-) create mode 100644 web/cast_library.css create mode 100644 web/cast_library.js diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index bc4f4ea1..0d0003d9 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -27,6 +27,7 @@ class Element { test('main browser entry registers both views and preserves node hooks', async t => { const extensions = [], opened = [], installed = [], events = new EventTarget(); let refreshed = 0, compatibilityChecks = 0, updateChecks = 0, doneNotifyStarts = 0, rememberedView = null; + const casts = []; let progressView, progressOptions, realProgressFactory; const app = { registerExtension(extension) { extensions.push(extension); }, @@ -37,6 +38,10 @@ test('main browser entry registers both views and preserves node hooks', async t createRange: () => ({createContextualFragment: () => new Element('span')})}; const dependencies = { app, api, outputDownloadURL, regenerateResult, upscaleResult, createSecondPassQueue, referenceInputs, cleanOptionalPrompt, cleanOptionalWidgets, + createCastLibrary: options => { + const cast = {element: new Element('cast'), options, destroyed: false, destroy() { this.destroyed = true; }}; + casts.push(cast); return cast; + }, secondPassAction: () => new Element('button'), createErrorPanel: () => ({element: new Element('error'), show() {}, clear() {}}), errorText: value => String(value), @@ -104,6 +109,7 @@ test('main browser entry registers both views and preserves node hooks', async t .replace("import { openImageEditor, icon } from './image_editor.js';", 'const {openImageEditor,icon} = globalThis.__freevideoEntryTest;') .replace("import { installNavigation, refreshNavigation, preferredView } from './view_navigation.js';", 'const {installNavigation,refreshNavigation,preferredView} = globalThis.__freevideoEntryTest;') .replace("import { imageGenerationSupported } from './image_generation.js';", 'const imageGenerationSupported = () => true;') + .replace("import { createCastLibrary } from './cast_library.js';", 'const {createCastLibrary} = globalThis.__freevideoEntryTest;') .replace("import { agentInterfaceEnabled } from './agent_connect.js';", 'const agentInterfaceEnabled = async () => false;'); await import('data:text/javascript;base64,' + Buffer.from(source).toString('base64')); const extension = extensions.find(row => row.name === 'FreeVideo.UnifiedMedia'); @@ -295,7 +301,11 @@ test('main browser entry registers both views and preserves node hooks', async t await extension.beforeRegisterNodeDef(MediaNode, {name: 'FreeVideoMedia'}); const media = new MediaNode(); media.onNodeCreated(); const panel = media.dom.find(row => row.name === 'freevideo_media').element; - const [toolbar, mode, note, audioHelp, connections] = panel.children; + const [castRow, toolbar, mode, note, audioHelp, connections] = panel.children; + assert.equal(castRow, casts[0].element); + assert.equal(casts[0].options.api, api); + assert.equal(typeof casts[0].options.uploadOne, 'function'); + assert.equal(typeof casts[0].options.viewURL, 'function'); assert.match(mode.textContent, /Reference/); assert.equal(audioHelp.hidden, false); assert.match(audioHelp.textContent, /