From d87f6461a1f7f2fe96dc413799ec17191e91629b Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 12:37:16 +0000 Subject: [PATCH 01/27] Print the reused bootstrap Python path as UTF-8 The reuse step runs the prepared interpreter with -I, which ignores PYTHONIOENCODING, so a fresh install into a folder the system code page cannot spell failed on the print. Run it through python_command so it gets -X utf8 like the other Python children. --- freevideo_engine/bootstrap.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index 9243ab0a..cb56322c 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -1541,8 +1541,10 @@ def python_install(self, uv): 'import platform; print(platform.python_version())'], capture_output=True, timeout=30) if check.returncode == 0 and check.stdout.strip() == self.versions['python']: self.env['UV_PYTHON_INSTALL_DIR'] = str(prepared) - self.command('python-bootstrap-reuse', [candidate, '-I', '-B', '-c', - 'import sys; print("Reusing verified bootstrap Python: " + sys.executable)']) + # -I ignores PYTHONIOENCODING: without -X utf8 a path the + # system code page cannot spell fails the print. + self.command('python-bootstrap-reuse', subprocess_output.python_command([candidate, '-I', '-B', '-c', + 'import sys; print("Reusing verified bootstrap Python: " + sys.executable)'])) return custom = self.env.get('UV_PYTHON_INSTALL_MIRROR') if self.network.get('mode') != 'official' else None isolation = ['--no-bin', '--no-registry'] From 8818f5ca8c3abdaf809e7d1a89b110648c1ef5fc Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 12:37:39 +0000 Subject: [PATCH 02/27] Whole-video progress keeps its stage order and is never taken back - A stage that is interrupted and resumes goes on from the time it already has - A stage that outruns its forecast keeps raising the fraction while enough of the run is ahead - Each decoder step corrects the time left - The wait between the two passes is not taken out of the second pass --- freevideo_engine/comfy_bridge.py | 125 ++++++++++++++++++++++++++----- 1 file changed, 108 insertions(+), 17 deletions(-) diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index 231064b6..be111457 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -75,6 +75,11 @@ def progress_history_forecast(root, machine, canvas): return {} # A progress estimate cannot prevent generation. +# The decoder's phase events, as decode.py and macos_decode.py print them. +_DECODE_PARTS = {'Loading video VAE': 'vae_load', 'Decoding the video': 'video_decode', + 'Loading and decoding audio': 'audio_load_decode', 'Saving MP4 and audio': 'encode'} + + class WholeVideoProgress: """Annotate phase events with one report-weighted whole-video progress. @@ -82,6 +87,10 @@ class WholeVideoProgress: rough decoder estimate which real decoded-frame timings then replace. These presentation estimates never change resource admission or placement. """ + ORDER = ('encoding', 'load', 'sampling', 'decode') + # The decoder's own steps, in the order it runs them, by the name of each one's forecast. + DECODE_PARTS = ('vae_load', 'video_decode', 'audio_load_decode', 'encode') + def __init__(self, clock=time.perf_counter): self.clock = clock self.started = clock() @@ -90,15 +99,34 @@ def __init__(self, clock=time.perf_counter): self.completed = set() self.weights = {} self.measured = {} + # Seconds each phase has run over every stretch of it, the forecast's + # own weights, phases the forecast closed and phases a retry does not repeat. + self.spent = {} + self.forecasts = {} + self.closed = set() + self.reused = set() + self.decode_parts = {} + self.decode_rest = None self.total = 0. self.forecast_ready = False self.failed = False self.sampling_plan = {} self.decode_from_sampling = False self.sampling_time_weighted = False + self.sampling_rest = None self.decode_tail = 0. + self.shown = 0. def _sampling_estimate(self, message, now): + if self._step_estimate(message, now) or not self.sampling_rest or self.sampling_rest[0] != message.get('done'): + return + # No new step time from this message. Time that passes outside a step, as between the two passes, is not + # taken out of the steps still to run. + inside = _timing_seconds(message.get('step_elapsed_seconds')) or 0. + self.weights['sampling'] = max(self.weights.get('sampling', 0.), + now - self.phase_started + max(0., self.sampling_rest[1] - inside)) + + def _step_estimate(self, message, now): estimate = _timing_seconds(message.get('estimated_step_seconds')) done, total = message.get('done'), message.get('total') if estimate is None or type(done) is not int or type(total) is not int or not 0 <= done <= total: @@ -122,6 +150,7 @@ def _sampling_estimate(self, message, now): return # Do not treat the two larger refinement NFEs as small NFEs. full_step *= second['width'] * second['height'] / area remaining = max(0, plan['base_steps'] - done) * estimate + plan['refine_steps'] * full_step + self.sampling_rest = done, remaining remaining = max(0., remaining - (_timing_seconds(message.get('step_elapsed_seconds')) or 0.)) self.weights['sampling'] = max(.001, now - self.phase_started + remaining) self.sampling_time_weighted = True @@ -133,6 +162,7 @@ def _sampling_estimate(self, message, now): self.weights['decode'] = 1.9 * full_step self.decode_from_sampling = True self.forecast_ready = True + return True @staticmethod def _forecast_weights(forecast): @@ -156,6 +186,13 @@ def _forecast_weights(forecast): values['decode'] = values.get('decode', 0.) + residual return values + @classmethod + def _forecast_parts(cls, forecast): + stages = forecast.get('stages') if isinstance(forecast, dict) else None + values = {name: _timing_seconds(stages.get(name + '_seconds')) for name in cls.DECODE_PARTS} \ + if isinstance(stages, dict) else {} + return {name: value for name, value in values.items() if value is not None} + def forecast(self, value): value = value.get('forecast') if isinstance(value, dict) and 'forecast' in value else value weights = self._forecast_weights(value) @@ -164,25 +201,56 @@ def forecast(self, value): # Close that phase before the first video-model event so its # measured weight is credited instead of starting a second timer. if self.phase == 'encoding': - self.measured['encoding'] = max(.001, self.clock() - self.phase_started) - self.completed.add('encoding') + self._close(self.clock()) + self.closed.add('encoding') self.phase = None + self.forecasts.update(weights) + self.decode_parts = self._forecast_parts(value) self.weights.update(weights) if 'decode' in weights: self.decode_from_sampling = False - self.weights.update(self.measured) - self.total = sum(self.weights.values()) self.forecast_ready = all(name in self.weights for name in ('sampling', 'decode')) + self._settle() + self.total = sum(self.weights.values()) def _base(self): return sum(self.weights.get(name, 0.) for name in self.completed) + def _close(self, now): + if self.phase is not None: + self.spent[self.phase] = max(0., now - self.phase_started) + + def _enter(self, phase, now): + """Phases do not always come one after the other: a preset check + runs for a moment while the prompt is encoded, and an encoder event + can follow the forecast. A phase that comes back goes on from the + time it already has, and only the phases before the one now running + are finished; a later one that ran for a moment keeps its forecast.""" + self._close(now) + self.phase, self.phase_started = phase, now - self.spent.get(phase, 0.) + self.closed.discard(phase) + self._settle() + + def _settle(self): + before = self.ORDER[:self.ORDER.index(self.phase)] if self.phase else () + self.completed = (set(before) | self.closed | self.reused) - {self.phase} + for name in self.ORDER: + seconds = self.spent.get(name) + if name in self.completed: + if seconds is not None: + self.measured[name] = self.weights[name] = max(.001, seconds) + elif name not in self.reused: + self.weights.pop(name, None) # Passed without running: nothing of it is left to wait for. + elif name in self.measured: + del self.measured[name] + self.weights[name] = max(self.forecasts.get(name, 0.), seconds or 0., .001) + def _message_phase(self, message): if message.get('timing_phase') in ('encoding', 'load', 'sampling', 'decode'): return message['timing_phase'] phase = message.get('phase') return {'sampling': 'sampling', 'sample_finalize': 'decode', - 'complete': 'decode'}.get(phase) + 'complete': 'decode'}.get(phase) if isinstance(phase, str) else None def annotate(self, message): message = dict(message) @@ -192,15 +260,12 @@ def annotate(self, message): self.forecast(message['prediction']) retry = message.get('retry') if isinstance(message.get('retry'), dict) else {} if retry.get('reuse_sampling') is True: + self.reused.add('sampling') self.completed.add('sampling') phase = self._message_phase(message) now = self.clock() if phase and phase != self.phase: - if self.phase is not None: - self.completed.add(self.phase) - self.measured[self.phase] = max(.001, now - self.phase_started) - self.weights[self.phase] = self.measured[self.phase] - self.phase, self.phase_started = phase, now + self._enter(phase, now) if phase == 'sampling': self._sampling_estimate(message, now) if phase == 'decode' and message.get('stage') == 'video': @@ -210,11 +275,19 @@ def annotate(self, message): video_seconds = elapsed * total / done # Reports put audio/MP4/post-decode cleanup at about 6-8% of # video decode time. Retain this tail until audio/save starts. - self.decode_tail = video_seconds * .08 + self.decode_tail = max(video_seconds * .08, self.decode_rest or 0.) self.weights['decode'] = now - self.phase_started + video_seconds - elapsed + self.decode_tail self.forecast_ready = True elif phase == 'decode': self.decode_tail = 0. + part = message.get('decode_part') + rest = self.DECODE_PARTS[self.DECODE_PARTS.index(part):] if part in self.DECODE_PARTS else () + if rest and all(name in self.decode_parts for name in rest): + # The decoder says which of its steps begins: what is left of + # the phase is this step and the ones after it, not the whole + # forecast made before the run. + self.decode_rest = sum(self.decode_parts[name] for name in rest[1:]) + self.weights['decode'] = now - self.phase_started + self.decode_parts[part] + self.decode_rest self.total = sum(self.weights.values()) if message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete': message['overall'] = {'status': 'complete', 'fraction': 1., 'estimated': False, @@ -223,18 +296,33 @@ def annotate(self, message): overall = {'status': 'running', 'estimated': self.forecast_ready, 'elapsed_seconds': now - self.started} if self.forecast_ready and self.total > 0: - start = self._base() / self.total weight = self.weights.get(self.phase, 0.) elapsed = max(0., now - self.phase_started) seconds = self.weights.get(self.phase) - fraction = start - if weight: - fraction += weight / self.total * min(.95, elapsed / max(seconds, 1e-9)) remaining = max(0., self.total - self._base() - min(elapsed, seconds or 0.)) future = sum(value for name, value in self.weights.items() if name not in self.completed and name != self.phase) + total = self.total + if weight and elapsed > .95 * weight and .95 * future > .05 * self._base(): + # A phase that outruns its forecast (a model read from a cold + # disk) is given the time it has taken, so the fraction keeps + # rising instead of standing at the forecast's end; it does so + # only while enough of the run is still ahead for that to + # raise it. The time left is unchanged: what follows is. + seconds = elapsed / .95 + total += seconds - weight + weight = seconds + start = self._base() / total + fraction = start + if weight: + fraction += weight / total * min(.95, elapsed / max(seconds, 1e-9)) + # What has been given out is not taken back. A phase that + # returns (a model loaded again between the passes), a decoder + # step that ran past its forecast and a retry all add time + # left; the fraction waits for the run to catch up with it. + fraction = self.shown = max(self.shown, fraction) overall.update(fraction=fraction, phase_start_fraction=start, - phase_weight=weight / self.total if self.total else 0., + phase_weight=weight / total, phase_elapsed_seconds=elapsed, phase_seconds=seconds, remaining_seconds=remaining, remaining_floor_seconds=max(future, self.decode_tail), @@ -462,7 +550,10 @@ def progress_message(event): 'lora_prepare_start': 'Loading LoRAs', 'resource_retry': 'Adjusting memory placement and retrying'} if name == 'decode_phase': - return {'label': str(event.get('phase', 'Decoding video and audio')), 'timing_phase': 'decode'} + message = {'label': str(event.get('phase', 'Decoding video and audio')), 'timing_phase': 'decode'} + if isinstance(event.get('phase'), str) and event['phase'] in _DECODE_PARTS: + message['decode_part'] = _DECODE_PARTS[event['phase']] + return message if name == 'decode_progress': return dict(label='Decoding the video', timing_phase='decode', phase='decode', stage='video', done=event.get('done'), total=event.get('total'), unit='frames', From fd6ce454fd63cdd8a7d13a76eb5157eb32dfce80 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 12:37:39 +0000 Subject: [PATCH 03/27] Autopilot conversation: card progress through a cold run, and the narrow finished card - The level rises through every stage of a cold run, and the step before the first one reads as loading - A card's line does not go back to the lines from before the first step - A job tried again starts its level from the beginning - No length badge on the small picture of a narrow page - Before the first step a row in the activity list says "Preparing" --- web/agent_activity.js | 3 +- web/autopilot_chat.css | 2 +- web/autopilot_chat.js | 65 ++++++++++++++++++++++++++++-------------- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/web/agent_activity.js b/web/agent_activity.js index 811c7bd5..0f5d08e7 100644 --- a/web/agent_activity.js +++ b/web/agent_activity.js @@ -32,7 +32,8 @@ const left = (s, t) => s >= 60 ? t(`About ${Math.ceil(s / 60)} min`, `约 ${Math // The engine's phases in the reader's language. function phaseText(p, t) { const step = p.step; - // The first step is still getting ready: the line does not count from zero. + // The first step is still getting ready: the line does not count from zero. A row here does not know which + // part of a long video it is on, and a later part loads nothing, so it does not say the model is loading. if (p.phase === 'sampling' && step?.total) return step.done >= 1 ? t(`Step ${step.done} / ${step.total}`, `生成 ${step.done} / ${step.total}`) : t('Preparing', '准备中'); return {encoding: t('Reading prompt and media', '读取提示词和素材'), load: t('Loading the model', '加载模型'), decode: t('Saving video and audio', '保存视频与音频'), starting: t('Starting', '启动中'), diff --git a/web/autopilot_chat.css b/web/autopilot_chat.css index eea7bf42..934f04d1 100644 --- a/web/autopilot_chat.css +++ b/web/autopilot_chat.css @@ -70,5 +70,5 @@ .fv-studio .fv-chat-send:disabled{opacity:1;background:#22344a;color:#9fb3c8} .fv-studio .fv-chat-send:focus-visible{outline-offset:2px} @media(max-width:480px){.fv-chat{padding:0 14px 14px}.fv-chat-scroll{margin:0 -14px;padding:16px 14px 2px}.fv-chat-hello{padding-top:32px}.fv-chat-job{max-width:none;gap:12px}.fv-chat-confirm{max-width:none} - .fv-chat-preview,.fv-chat-preview[data-shape=wide]{width:98px}.fv-chat-percent{font-size:22px}.fv-studio.fv-autopilot .fv-chat-input textarea{font-size:16px}.fv-studio .fv-chat-down{right:14px;bottom:84px}} + .fv-chat-preview,.fv-chat-preview[data-shape=wide]{width:98px}.fv-chat-length{display:none}.fv-chat-percent{font-size:22px}.fv-studio.fv-autopilot .fv-chat-input textarea{font-size:16px}.fv-studio .fv-chat-down{right:14px;bottom:84px}} @media(prefers-reduced-motion:reduce){.fv-chat-water,.fv-chat-percent,.fv-chat-parts>i::after{transition:none}.fv-chat-water::after,.fv-chat-dots>i,.fv-chat-preview>img{animation:none}} diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index f39a5ca9..4f041164 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -22,30 +22,51 @@ export const about = (seconds, t) => seconds >= 60 ? t(`About ${Math.ceil(second export function clock(seconds) { const s = Math.max(0, Math.round(seconds)); return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`; } export const jobName = (job, t) => job.label || t('Video', '视频'); export const jobFraction = job => Math.max(0, Math.min(1, Number(job.progress?.fraction) || 0)); -// What a job's last progress stands for `age` seconds after it was made, by the creation panel's rule. While the -// engine has no timing for what it is doing (a model loading for the first time, a video in several parts), the -// level still rises over the seconds the reported fraction has stood (`still`): at the pace of the time the engine -// said is left, slowing down, and by no more than a quarter of what remains. -const DRIFT = .25; -export function jobEstimate(job, age, still = age) { - const p = job.progress || {}, e = p.estimate, overall = e?.overall, left = p.remaining_seconds; - const sent = overall ? estimatedFraction(e, overall, age, age) : null; - if (overall?.estimated === true && overall.phase_weight > 0) return sent ?? jobFraction(job); - const base = Math.max(jobFraction(job), sent ?? 0); - return left > 0 ? base + (1 - base) * DRIFT * (1 - Math.exp(-Math.max(0, still) / (DRIFT * left))) : base; +// What a job's last progress stands for `age` seconds after it was made, by the creation panel's rule. +export function jobEstimate(job, age) { + const e = job.progress?.estimate, sent = e?.overall ? estimatedFraction(e, e.overall, age, age) : null; + return Math.max(jobFraction(job), sent ?? 0); +} +// The level a job shows, given the engine's number (`base`), the time it says is left and the clock. The engine's +// number can stand for a long while: a model loading for the first time runs far past the time the engine gave it, +// and a video in several parts has no timing at all. While it stands the level still rises, at the pace of the time +// left, slowing down, and by no more than a quarter of what remains. What was added is kept when the number moves +// again: the rest of the way is drawn shorter, so the level neither waits nor goes back, and is full at the end. +const DRIFT = .25, RISE = .0005; +export const newLevel = () => ({value: 0, top: null, since: 0, scale: 1}); +export function jobLevel(row, base, left, time) { + if (row.top === null || base > row.top + RISE) { + if (row.top !== null && row.top < 1) row.scale = (1 - row.value) / (1 - row.top); + row.top = base; row.since = time; + } + const level = 1 - row.scale * (1 - Math.max(base, row.top)); + const more = left > 0 ? (1 - level) * DRIFT * (1 - Math.exp(-Math.max(0, time - row.since) / (DRIFT * left))) : 0; + return row.value = Math.max(row.value, Math.min(.99, level + more)); } // working, done, stopped or failed: which card a job shows. export const jobMode = job => WORKING.includes(job.status) ? 'working' : job.status === 'succeeded' ? 'done' : job.status === 'cancelled' ? 'stopped' : 'failed'; +const SAVING = ['decode', 'finishing', 'assembling'], stepped = p => p.phase === 'sampling' && p.step?.total && p.step.done >= 1; // The one line that says where a job is. No step count is written unless the engine gave one. export function jobLine(job, t) { const p = job.progress || {}; if (job.status !== 'running') return t('Queued', '排队中'); - if (p.phase === 'sampling' && p.step?.total && p.step.done >= 1) return t(`Step ${p.step.done} / ${p.step.total}`, `生成 ${p.step.done} / ${p.step.total}`); - if (['decode', 'finishing', 'assembling'].includes(p.phase)) return t('Saving', '保存中'); + if (stepped(p)) return t(`Step ${p.step.done} / ${p.step.total}`, `生成 ${p.step.done} / ${p.step.total}`); + if (SAVING.includes(p.phase)) return t('Saving', '保存中'); if (p.phase === 'encoding') return t('Reading the prompt', '读取提示词'); - if (['load', 'loading'].includes(p.phase)) return t('Loading the model', '加载模型'); + // Before the first step of the first part the model is still settling in: the line does not turn back to "Preparing". + if (['load', 'loading'].includes(p.phase) || p.phase === 'sampling' && !(p.block?.done > 0)) return t('Loading the model', '加载模型'); return t('Preparing', '准备中'); } +// The line a card shows. After the last step, and again after saving, the engine has a moment with no step and no +// stage to report: a card that has counted steps or said "Saving" for a part keeps that line until there is a newer +// one, and does not go back to the lines from before the first step. `said` is the card's own memory. +export function cardLine(said, job, t) { + const p = job.progress || {}, part = p.block?.index ?? 0; + if (job.status !== 'running' || said.part !== part) said.last = null; + said.part = part; + if (job.status === 'running' && (stepped(p) || SAVING.includes(p.phase))) said.last = job; + return jobLine(said.last || job, t); +} export function jobMeta(job, t) { const parts = []; if (job.width > 0 && job.height > 0) parts.push(`${job.width} × ${job.height}`); @@ -110,14 +131,14 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () function fractionOf(job) { const p = job.progress || {}, {age_seconds: sent, ...sample} = p.estimate || {fraction: p.fraction}; const key = JSON.stringify(sample), time = performance.now() / 1000; + // A job that stopped or failed and is tried again starts from the beginning, not from where the last try got to. + if (job.status !== 'running' && job.status !== 'succeeded') levels.delete(job.id); let row = levels.get(job.id); - if (!row) levels.set(job.id, row = {key: null, at: 0, job: null, value: 0, raw: null, since: time}); + if (!row) levels.set(job.id, row = {key: null, at: 0, job: null, level: newLevel()}); // remaining_seconds counts down between samples; the one that came with the sample is the one to move by. if (row.key !== key) { row.key = key; row.at = time - (sent > 0 ? sent : 0); row.job = job; } - // A loading model sends its own counts on and on while the fraction stands: that standing time is its own clock. - if (row.raw !== p.fraction) { row.raw = p.fraction; row.since = time; } - if (job.status === 'running') row.value = Math.max(row.value, Math.min(.99, jobEstimate(row.job, time - row.at, time - row.since))); - return row.value; + if (job.status === 'running') jobLevel(row.level, jobEstimate(row.job, time - row.at), row.job.progress?.remaining_seconds, time); + return row.level.value; } // The time left each job shows, by the creation panel's rule: never raised, and gone once its last minute has passed. const times = new Map(); @@ -160,6 +181,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () poll(); }); foot.append(left, stop); info.append(head, middle, foot); item.replaceChildren(box, info); + const said = {}; // The number rides on the water, and goes under the line once there is no room left above it. const level = next => { const fraction = fractionOf(next); @@ -180,8 +202,9 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () cells.forEach((value, index) => parts.children[index].style.setProperty('--w', `${value * 100}%`)); parts.hidden = !cells.length; const block = next.progress?.block; - part.textContent = cells.length ? t(`Part ${block.index} / ${block.total}`, `第 ${block.index} / ${block.total} 段`) : jobLine(next, t); - step.textContent = cells.length ? jobLine(next, t) : ''; + const where = cardLine(said, next, t); + part.textContent = cells.length ? t(`Part ${block.index} / ${block.total}`, `第 ${block.index} / ${block.total} 段`) : where; + step.textContent = cells.length ? where : ''; const going = stopping.has(next.id); stop.disabled = going; stop.textContent = going ? t('Stopping', '正在停止') : t('Stop', '停止'); stop.setAttribute('aria-label', t('Stop ', '停止 ') + jobName(next, t)); From 96a135f9f5a89679a83d25d578053a21a77ac3aa Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 12:44:26 +0000 Subject: [PATCH 04/27] Agent conversation: an empty wait says in words to keep waiting - CI: run the Autopilot conversation and editor continue checks --- freevideo_engine/agent/routes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 2638001c..b7cd39b8 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -29,6 +29,9 @@ from .jobs import ACTIVE, DONE, JobError, Jobs, chain_error, library_view PREFIX = '/freevideo/agent/v1' +# What an empty wait tells the agent: the conversation is open for as long as the user keeps the page. +KEEP_WAITING = ('Nothing new yet. The user is still in FreeVideo and may speak at any moment: call this wait ' + 'again now. An empty wait never ends the conversation; do not end your turn or write a summary.') def local_address(request): @@ -1496,8 +1499,9 @@ def read(): return reply(dict(session=sid, inbox=pending[-20:])) left = deadline - time.monotonic() if left <= 0: - # again: nothing new this time; the caller simply waits once more. - return reply(dict(session=sid, inbox=[], again=True)) + # again: nothing new this time; the caller simply waits once more. An agent decides what to + # do next from the last answer it read, so the empty one says it in words as well. + return reply(dict(session=sid, inbox=[], again=True, next=KEEP_WAITING)) if store is None or sid is None: await asyncio.sleep(min(left, 1.)) else: From 75d58b9fed5448f9ea54ccd4406fb6d1a2bf12b2 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 12:44:26 +0000 Subject: [PATCH 05/27] Use device limits for Agent videos under 30 seconds - Agent: the first plan for a size waits for its device limit --- freevideo_engine/agent/errors.py | 2 +- freevideo_engine/agent/forecast.py | 7 ++-- freevideo_engine/agent/guide.py | 15 ++++---- freevideo_engine/agent/jobs.py | 33 +++++++++++------- freevideo_engine/agent/length.py | 49 +++++++++++++++++++++++++++ freevideo_engine/agent/mcp.py | 2 +- freevideo_engine/agent/productions.py | 4 +-- freevideo_engine/agent/routes.py | 8 +++-- freevideo_engine/agent/spec.py | 18 ++++++---- 9 files changed, 100 insertions(+), 38 deletions(-) create mode 100644 freevideo_engine/agent/length.py diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index b5b3fdbc..0ca9a443 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -55,7 +55,7 @@ 'internal_error': (500, True, 'An internal error prevented this operation from completing.', 'Read the job list and report request_id if the error persists.'), 'local_only': (403, False, 'This endpoint does not allow requests from this peer address.', 'Run the client on the FreeVideo computer and connect using its loopback address.'), 'origin_forbidden': (403, False, 'Origin must match this FreeVideo instance; a different origin was received.', 'Open the page served by this FreeVideo instance, or use a local command-line client without Origin.'), - 'continuation_unavailable': (409, False, 'This backend cannot continue video blocks.', 'Use an independent shot with seconds<=15, or enable a backend with continuation inputs and resume.'), + 'continuation_unavailable': (409, False, 'This backend cannot continue video blocks.', 'Use POST /plan for a supported single block (unknown limit: seconds<=15), or enable continuation and resume.'), 'not_resumable': (409, False, 'Only failed, interrupted or cancelled generation jobs can resume.', 'Read the job; wait for active jobs or submit a new job for a different request.'), 'idempotency_conflict': (409, False, 'This idempotency key was already used for different request content.', 'Resend the original content with this key, or use a new ref for the changed request.'), 'source_missing': (409, False, 'A completed source block is no longer available.', 'Restore the completed block before resuming, or submit a new job.'), diff --git a/freevideo_engine/agent/forecast.py b/freevideo_engine/agent/forecast.py index 44ba77a3..cb914c19 100644 --- a/freevideo_engine/agent/forecast.py +++ b/freevideo_engine/agent/forecast.py @@ -1,5 +1,6 @@ """Machine-local timing forecasts shared by planning and durable films.""" import math +from .length import capability_table, single_frame_limit def generation_estimate(port, spec, chain=None): @@ -9,9 +10,9 @@ def generation_estimate(port, spec, chain=None): reasons here so a film never invents a time for unsupported conditioning. """ from ..chain_plan import plan as chain_plan - chain = chain if chain is not None else chain_plan(spec['seconds'], continued=bool(spec.get('continue_from'))) - table = port.capability_snapshot(dict(width=str(spec['width']), height=str(spec['height']), - task=spec['task'], adapters='1' if spec['loras'] else '0')) + table = capability_table(port, spec) + chain = chain if chain is not None else chain_plan(spec['seconds'], max_frames=single_frame_limit(spec, table=table), + continued=bool(spec.get('continue_from'))) seconds = None reason = table['status'] if table['status'] in ('computing', 'unavailable') else 'no_matching_estimate' if table['status'] == 'ready' and not spec['preview'] and not spec['media']: diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 85d2feb6..610aef24 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -60,9 +60,9 @@ ], 'timing': [ 'Single-block frames are 24 fps on a 17n+5 grid (6 s -> 6.583 s, 10 s -> 10.125 s). ' - 'Generation uses the continuous backend. A request of up to 15 s is one generation; a longer one is made ' - 'in parts of up to 15 s (30 s is 15 + 15) that continue one another and join automatically. ' - 'Continuing (longer than 15 s, or continue_from) needs two-pass text generation: quality "light" ' + 'Under 30 s defaults to one generation when the device supports this size and quality; unknown limits use 15 s. ' + 'Otherwise it uses parts of up to 15 s (30 s is 15 + 15), continued and joined automatically. ' + 'Multiple blocks or continue_from need two-pass text generation: quality "light" ' 'or sampling.two_pass=true, no preview and no media. ' 'Multi-block output is cut to the planned duration. POST /plan shows the chain plan. ' 'A continue_from job saves a new video that holds only the added part; to hand the user one video, ' @@ -119,7 +119,7 @@ for name, (method, path, summary) in SESSION_OPERATIONS.items()], 'filming': [ 'Filming is for a film of several shots, each picked from takes. One video of up to 60 seconds is one generate ' - 'job (longer requests continue and join automatically); to polish an idea first, run enhance, then generate. ' + 'job (30 s or more, or beyond the device limit, continues and joins automatically); to polish an idea first, run enhance, then generate. ' 'A film is cut into shots: see the rule "Cut by default" before making one long continued video.', 'Use drama plan → start → get → sheet → choose/reshoot → finish; inspect todo after every change.', 'At start tell the user the estimated total time; on over_budget decide whether to ask them; ' @@ -176,8 +176,7 @@ 'Cut by default. Wherever the film can change shot, make the new shot from the reference pictures it should ' 'have (cast cards, scene pictures) instead of continuing the previous video. Continue a video only for a ' 'long take that cannot be cut, an action that must stay unbroken, or sound that must stay unbroken, and ' - 'continuing again and again is not recommended. Content of 16 to 29 seconds is better made as shots of ' - 'up to 15 seconds each. One video has at most 5 parts (the first and 4 ' + 'continuing again and again is not recommended. One video has at most 5 parts (the first and 4 ' 'continuations), each continued part at least 10 seconds, about 70 seconds in all; filling that is not ' 'recommended. Past it, cut to a new shot.', ] @@ -470,8 +469,8 @@ def connection_text(value): 'Keys survive restart. Batch-level ref names the whole request; row refs still name earlier dependencies.', 'Upload media content with POST /media. HTTP accepts no local absolute paths; MCP and CLI read local files ' 'before uploading. Unknown fields are rejected and every batch is validated before it is recorded.', - 'limits.seconds in the status reply is what this installation can make now. While continuation is false, ' - 'one block (15 s) is the longest video; plan and submit refuse longer requests with continuation_unavailable.', + 'POST /plan gives the length supported for the current size and quality. Without continuation, only a ' + 'single block can run; unknown device limits use 15 s. Multiple blocks return continuation_unavailable.', 'One video has at most 5 parts (60 s in one request). continue_from counts the parts of the video it ' 'continues; plan and submit refuse a longer one and name the longest seconds that still fits.', 'features in the status reply names what this installation offers; false means not usable now (for example ' diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index a7631a9c..143f3b17 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -24,6 +24,7 @@ from . import spec as specs from . import schema, errors, assets +from .length import single_frame_limit from ..system import windows ACTIVE = ('waiting', 'queued', 'running', 'cancelling') @@ -60,9 +61,10 @@ def chain_error(code, message, next_action, *, details=None, retryable=False): return JobError(code, message, details=details or [], next=next_action, retryable=retryable) -def single_block(seconds): +def single_block(spec, seconds=None): from ..chain_plan import plan - return plan(seconds)['single'] + return plan(spec['seconds'] if seconds is None else seconds, max_frames=single_frame_limit(spec), + continued=bool(spec.get('continue_from')))['single'] def continuation_problems(spec, prefix='', *, editor=None): @@ -73,9 +75,9 @@ def continuation_problems(spec, prefix='', *, editor=None): if spec['kind'] != 'generate' or spec.get('refine_from'): return [] if not (any(block['after'] for block in editor['blocks']) if editor is not None - else spec.get('continue_from') or not single_block(spec['seconds'])): + else spec.get('continue_from') or not single_block(spec)): return [] - why = ' when continuing a video or making one longer than 15 s' + why = ' when continuing a saved video or generating multiple blocks' problems = [] if not spec['two_pass']: # The quality name is a contract enum, never caller text. @@ -94,7 +96,7 @@ def cut_problems(spec, seconds, prefix='', *, editor=None): """A shot cut at or after the end of the video never appears; refuse it before anything is generated.""" if spec['kind'] != 'generate' or spec.get('refine_from') or spec.get('continue_from') or seconds is None: return [] - if any(block['after'] for block in editor['blocks']) if editor is not None else not single_block(seconds): + if any(block['after'] for block in editor['blocks']) if editor is not None else not single_block(spec, seconds): return [] # Every block of a continued video reads the same text. late = [cut for cut in specs.cut_times(spec.get('prompt')) if cut >= seconds] if not late: @@ -118,7 +120,7 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N from ..chain_runner import ChainTooLong from ..chain_store import reference limit, continued = runner.max_parts, bool(spec.get('continue_from')) - count = lambda seconds: len(plan(seconds, continued=continued)['blocks']) + count = lambda seconds: len(plan(seconds, max_frames=single_frame_limit(spec), continued=continued)['blocks']) try: with runner.lock: if editor is not None: @@ -148,7 +150,7 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N def continuation_unavailable(): return chain_error('continuation_unavailable', 'This backend cannot continue multiple video blocks.', - 'Submit a new job with seconds<=15, or enable a backend with continuation inputs and resume.') + 'Use POST /plan for a supported single block (unknown limit: seconds<=15), or enable continuation and resume.') def library_id(relative): @@ -403,6 +405,9 @@ def load(self): self.finish(row, 'failed', error=dict(code='interrupted', message='ComfyUI restarted while this job ran; submit it again'), save=False) row['_restored'] = True + if row['kind'] == 'generate' and not row['spec'].get('refine_from'): + # Before device-aware Agent planning, every accepted job used the unknown limit. + row['spec'].setdefault('single_max_frames', None) self.jobs[row['id']] = row for row in self.jobs.values(): if (row['kind'] != 'generate' or row['spec'].get('refine_from') @@ -569,7 +574,7 @@ def submit_request(self, value, *, client=None, key=None, parent=None, editor=No from .routes import substitute item = substitute(item, names) try: - normalized = specs.normalize(item) + normalized = specs.normalize(item, port=self.port) except specs.SpecError as error: problems.extend(schema.prefix_diagnostic(e, prefix) for e in error.errors) continue @@ -619,7 +624,7 @@ def submit_request(self, value, *, client=None, key=None, parent=None, editor=No if any(block['after'] for block in row['editor']['blocks']) and not self.continuation_ready(): raise continuation_unavailable() continue - self.require_continuation(row['spec']['seconds']) + self.require_continuation(row['spec']) if row['spec'].get('continue_from') and not self.continuation_ready(): raise continuation_unavailable() if receipt: @@ -697,8 +702,8 @@ def continuation_ready(self): from ..comfy_chain import continuation_available return continuation_available() and os.environ.get('FREEVIDEO_CHAIN_INDEPENDENT') != '1' - def require_continuation(self, seconds): - if not single_block(seconds) and not self.continuation_ready(): + def require_continuation(self, spec): + if not single_block(spec) and not self.continuation_ready(): raise continuation_unavailable() def require_runner(self): @@ -1197,7 +1202,8 @@ def planned_parts(self, spec, rows=()): spec = source['spec'] if spec['kind'] != 'generate' or spec.get('refine_from') or source.get('editor'): return None - total += len(plan(spec['seconds'], continued=bool(spec.get('continue_from')))['blocks']) + total += len(plan(spec['seconds'], max_frames=single_frame_limit(spec), + continued=bool(spec.get('continue_from')))['blocks']) if not spec.get('continue_from'): return total work = self.continued_work(spec) @@ -1278,6 +1284,7 @@ def resolve(): return graph, extra, node else: payload = dict(node=node, prompt=graph, seconds=spec['seconds'], + max_frames=single_frame_limit(spec), client_id='agent:' + row['id'], extra_data=extra) if row.get('editor'): payload.pop('seconds') @@ -1315,7 +1322,7 @@ def start_chain(self, row, *, payload=None, resume=False): if (any(b['state'] != 'done' and (b['after'] or b['before']) for b in runner.records[row['chain_id']]['blocks']) if resume else any(b['after'] for b in payload['blocks']) if row.get('editor') - else bool(row['spec'].get('continue_from')) or not single_block(payload['seconds'])) and not self.continuation_ready(): + else bool(row['spec'].get('continue_from')) or not single_block(row['spec'])) and not self.continuation_ready(): raise continuation_unavailable() self.touch(row, chain_submission='creating', status='queued', queued_at=self.clock()) from ..chain_runner import ChainBusy, ChainTooLong diff --git a/freevideo_engine/agent/length.py b/freevideo_engine/agent/length.py new file mode 100644 index 00000000..eb4a4923 --- /dev/null +++ b/freevideo_engine/agent/length.py @@ -0,0 +1,49 @@ +"""Studio capability limits for Agent generation plans, without hardware probing.""" +import time + +# A size asked for the first time is worked out in the background in about 0.1 s. The plan that asks first +# waits for it, so it answers what the submit after it will do. +WAIT_SECONDS = 3. + + +def capability_table(port, spec, *, wait=0.): + query = dict(width=str(spec['width']), height=str(spec['height']), + task=spec['task'], adapters='1' if spec['loras'] else '0') + deadline = time.monotonic() + wait + try: + while True: + table = port.capability_snapshot(query) + if table.get('status') != 'computing' or time.monotonic() >= deadline: + return table + time.sleep(.05) + except Exception: + # The studio cache is advisory. Its failure retains the unknown-limit policy. + return dict(status='unavailable') + + +def single_frame_limit(spec, *, port=None, table=None): + """Frame limit for this normalized size/sampling/task; unknown keeps the 15 s default. + + A submitted spec pins even an unknown limit: a later cache refresh must not + change the accepted canvas or number of parts. Old receipts also keep their + unknown limit when no capability source is supplied. + """ + if 'single_max_frames' in spec: + return spec['single_max_frames'] + if table is None: + table = capability_table(port, spec, wait=WAIT_SECONDS) if port is not None else {} + if table.get('status') != 'ready': + return None + tiers = table.get('tiers', {}) + tier = tiers.get(spec['quality']) + if tier is None: + tier = next((row for row in tiers.values() + if row.get('steps') == spec['base_steps'] and row.get('two_pass') == spec['two_pass'] + and (not spec['two_pass'] or spec['refine_steps'] == 3)), {}) + index = next((i for i, row in enumerate(table.get('sizes', [])) + if (row['width'], row['height']) == (spec['width'], spec['height'])), None) + if index is None: + return None + maxima = tier.get('max_frames', []) + limit = maxima[index] if isinstance(maxima, list) and index < len(maxima) else None + return limit if type(limit) is int and limit > 0 else None diff --git a/freevideo_engine/agent/mcp.py b/freevideo_engine/agent/mcp.py index c59d2599..2b683e7a 100644 --- a/freevideo_engine/agent/mcp.py +++ b/freevideo_engine/agent/mcp.py @@ -44,7 +44,7 @@ def tool_list(): dict(name='freevideo_prompt_guide', title='H3 prompt guide', description='How to write prompts: sections, shots and cuts, camera motion, dialogue, references, short-drama tips, and a full example.', annotations=read), dict(name='freevideo_schema', title='Job JSON Schemas', description='JSON Schemas of every job kind and of image edit recipes.', annotations=read), dict(name='freevideo_plan', title='Plan a job', description="Validate a job without running it: actual frames/seconds, sampling passes, warnings and a time estimate from this computer's history. Same fields as freevideo_generate.", annotations=read), - dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds: a longer request continues and joins into one file automatically. Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt; the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write), + dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds. Under 30 s defaults to one generation if the device supports this size and quality; unknown limits use 15 s. Otherwise parts continue and join automatically (30 s is 15 + 15). Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt; the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write), dict(name='freevideo_submit_batch', title='Submit a storyboard', description='Submit several jobs at once. Each row is a job object with "kind" (generate default, assemble, vlm, enhance) and an optional "ref"; later rows use "@ref" wherever a job id is expected (from_job, after, clips, refine_from). FreeVideo runs them in dependency order.', annotations=write), dict(name='freevideo_job', title='Job status', description='Status, progress, result or error of a job.', annotations=read), dict(name='freevideo_wait', title='Wait for a job or the user', description="Wait at most 25 seconds for a job (with id) or, without id, at most 50 seconds for what the user says in FreeVideo (inbox). again=true means nothing new yet: call it again. A job wait returns early when the user says something. If the job is still active, call it again with the same ID; do not resubmit.", annotations=read), diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py index 92af651a..809b2213 100644 --- a/freevideo_engine/agent/productions.py +++ b/freevideo_engine/agent/productions.py @@ -360,7 +360,7 @@ def _take_estimate(self, state, shot, take, item=None): continues=shot['continues']) if resolved['references']: item['media'] = dict(references=deepcopy(resolved['references'])) - estimate, revision = generation_estimate(self.jobs.port, spec.normalize(item)) + estimate, revision = generation_estimate(self.jobs.port, spec.normalize(item, port=self.jobs.port)) return dict(estimate, revision=revision) except Exception: # Timing is optional; a forecast failure must not reject a film or job. return dict(seconds=None, basis=None, reason='unavailable') @@ -375,7 +375,7 @@ def _join_estimate(self, state, previous=None): return dict(seconds=None, basis=None, reason='no_matching_estimate') item = deepcopy(take['spec']) item.pop('continue_from', None) - normalized = spec.normalize(item) + normalized = spec.normalize(item, port=self.jobs.port) shapes.append([normalized[k] for k in ('width', 'height', 'frames')]) current = next((j for j in state['joins'] if j['ref'] == state['current_join']), None) transition = current['transition'] if current else state['screenplay'].get('transition', 'cut') diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index b7cd39b8..cb3ba8d1 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -214,7 +214,7 @@ def capabilities(port, jobs): def plan(port, value, jobs=None): from ..two_pass import plan as sampling_plan from ..geometry import geometry - spec = specs.normalize(value) + spec = specs.normalize(value, port=port) result = dict(spec={k: v for k, v in spec.items() if k != 'depends_on'}, warnings=[]) if spec['kind'] != 'generate' or spec.get('refine_from'): return result @@ -226,7 +226,7 @@ def plan(port, value, jobs=None): if problems: raise specs.SpecError(problems) if jobs is not None: - jobs.require_continuation(spec['seconds']) + jobs.require_continuation(spec) if spec.get('continue_from') and not jobs.continuation_ready(): from .jobs import continuation_unavailable raise continuation_unavailable() @@ -243,7 +243,9 @@ def plan(port, value, jobs=None): if spec['width'] * spec['height'] > 2 * 1024 ** 2: result['warnings'].append('Above 2 MP takes much longer and needs more memory') from ..chain_plan import plan as chain_plan - result['chain_plan'] = chain_plan(spec['seconds'], continued=bool(spec.get('continue_from'))) + from .length import single_frame_limit + result['chain_plan'] = chain_plan(spec['seconds'], max_frames=single_frame_limit(spec), + continued=bool(spec.get('continue_from'))) from .forecast import generation_estimate result['estimate'], result['capabilities_revision'] = generation_estimate(port, spec, result['chain_plan']) return result diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index 8a9608c5..8da22005 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -4,6 +4,7 @@ import re import secrets from . import schema as contract +from .length import single_frame_limit QUALITY, ASPECTS = contract.QUALITY, contract.ASPECTS KINDS = tuple(contract.KINDS) @@ -149,12 +150,13 @@ def task(media): return 'i2va' if 'first' in media else 'l2va' if 'last' in media else 't2va' -def saved_length(canvas, seconds, continued=False): - """Frames and seconds of the saved video. One generation keeps its aligned canvas; a longer video is - made in blocks and joined to exactly the planned whole seconds. A continuation saves only what it adds.""" +def saved_length(canvas, spec): + """Saved length: one generation keeps its aligned canvas; multiple blocks are joined to the planned + whole seconds. A continuation saves only what it adds.""" from ..chain_plan import PlanError, plan + continued = bool(spec.get('continue_from')) try: - planned = plan(seconds, continued=continued) + planned = plan(spec['seconds'], max_frames=single_frame_limit(spec), continued=continued) except PlanError: return canvas['frames'], canvas['seconds'] if planned['single'] and not continued: @@ -162,7 +164,7 @@ def saved_length(canvas, seconds, continued=False): return planned['frames'], planned['frames'] / canvas['fps'] -def normalize(value): +def normalize(value, *, port=None): # Local Python callers retain local media refs. HTTP validates its stricter # (content-only) schema first; the client adapts local refs before transport. contract.validate_job(value, local=True) @@ -180,9 +182,8 @@ def normalize(value): from ..geometry import geometry width, height = canvas_size(original) canvas = geometry(width, height, seconds=value['seconds']) - frames, output_seconds = saved_length(canvas, value['seconds'], bool(value.get('continue_from'))) result.update(prompt=value['prompt'], width=width, height=height, seconds=float(value['seconds']), - frames=frames, output_seconds=output_seconds, seed_random=value['seed'] is None, + seed_random=value['seed'] is None, seed=secrets.randbelow(MAX_SEED + 1) if value['seed'] is None else value['seed'], preview=value['preview'], media=generation_media(value.get('media')), **sampling(value)) result['task'] = task(result['media']) @@ -192,6 +193,9 @@ def normalize(value): result['first_frame_reference'] = True result['loras'] = [dict(name=row, strength=1.) if isinstance(row, str) else {'strength': 1., **row} for row in value['loras']] + if port is not None: + result['single_max_frames'] = single_frame_limit(result, port=port) + result['frames'], result['output_seconds'] = saved_length(canvas, result) elif kind == 'assemble': result['clips'] = [] for row in value['clips']: From b2d1d3c5f4d509cee1640b4e1c57c2220abca775 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 12:44:26 +0000 Subject: [PATCH 06/27] Agent guide: a rough camera draft as an optional way to get a camera path right - Agent guide: the camera draft takes 10 to 30 minutes --- freevideo_engine/agent/guide.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 610aef24..ae08a12b 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -58,6 +58,18 @@ 'Name subjects , in subject_definitions and reuse those labels in every section.', 'Clips are cut to the generated length (at most 15 s); the job result lists any cut.', ], + 'camera_draft': [ + 'Optional, never the default. When the user asks for a specific camera path, placement or order of actions ' + 'and two direct generations did not get it, offer to make a rough camera draft first and tell them it ' + 'takes you 10 to 30 minutes, more the more there is in the shot. Do not offer it for an ordinary shot.', + 'The draft is a video you render yourself from a simple 3D scene: 960x544, 24 fps, 124 frames. Only the ' + 'camera path, the positions of things and the timing of the action need to be right.', + 'Generate at 960x544 for 5 s with the draft as the only media.references video. Write the description as ' + 'one plain paragraph without sections and end it with: "The reference video is a rough 3D stand-in of this ' + 'scene: follow its camera path, object positions and motion timing, but render real materials, colours, ' + 'lighting and surroundings as described above; do not copy its simplified, flat look." The result follows ' + 'the draft and looks plainer than a direct generation, so keep the direct one when the user prefers its look.', + ], 'timing': [ 'Single-block frames are 24 fps on a 17n+5 grid (6 s -> 6.583 s, 10 s -> 10.125 s). ' 'Under 30 s defaults to one generation when the device supports this size and quality; unknown limits use 15 s. ' @@ -156,7 +168,9 @@ 'Solve problems yourself first. Every error has a next field, and often details with alternatives; these are ' 'steps that work on this computer. Not enough graphics memory: use a listed alternative (a lower resolution ' 'or a shorter video) and try again. A take is not good: make another; if it is still not good after two more, ' - 'describe it a different way instead of repeating the request. The queue is busy: wait with wait=25 on the ' + 'describe it a different way instead of repeating the request. When what stays wrong is the camera path, where ' + 'things are or the order of actions, the guide (camera_draft) has a slower way: offer it to the user. ' + 'The queue is busy: wait with wait=25 on the ' 'job, never in a tight loop. The connection dropped or FreeVideo restarted: connect again, read the job list, ' 'and continue; resend an unanswered request with the same ref, and never generate again what is already done.', 'Do not stop to report progress. A long task is not a reason to stop or to ask whether to go on.', @@ -188,7 +202,8 @@ '提交的每条视频都起个名字:label 写拍的是什么,用用户的语言,2 到 8 个字,例如“清晨海边”。', '先自己解决问题。每个错误都有 next 字段,details 里也常有可选方案;这些是在本机可行的处理步骤。' '显存不足时,从列出的方案中选择较低分辨率或较短时长重试。一条拍得不好就再拍;再拍两条仍不满意,' - '换一种描述,不要反复提交同一个要求。队列繁忙时,用 wait=25 等待该任务,不要不停查询。' + '换一种描述,不要反复提交同一个要求。一直不对的是镜头路线、位置或动作先后时,指南的 camera_draft 一节有一种更慢的做法,' + '可以向用户提议。队列繁忙时,用 wait=25 等待该任务,不要不停查询。' '连接中断或 FreeVideo 重启后,重新连接,读取任务列表并继续;未收到响应的请求用相同的 ref 重发,' '已经完成的内容不要重新生成。', '不要为了汇报进度而停下。任务耗时较长,不是停止工作或询问是否继续的理由。', From c3b443a447ee9cc8d78c549e76230b3361c1a943 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 12:44:26 +0000 Subject: [PATCH 07/27] Agent activity: remove the always-open list nothing shows any more --- web/agent_activity.css | 6 ----- web/agent_activity.js | 54 +----------------------------------------- 2 files changed, 1 insertion(+), 59 deletions(-) diff --git a/web/agent_activity.css b/web/agent_activity.css index 094555cb..3b2efcac 100644 --- a/web/agent_activity.css +++ b/web/agent_activity.css @@ -47,9 +47,3 @@ @media(max-width:820px){.fv-studio .fv-header:has(>.fv-view-switch):has(.fv-agent-pill:not([hidden])){height:auto;min-height:98px;grid-template-columns:1fr auto;padding-block:10px}.fv-studio .fv-header:has(.fv-agent-pill:not([hidden]))>.fv-view-switch{grid-column:1/-1;grid-row:2;justify-self:center}.fv-studio .fv-header:has(.fv-agent-pill:not([hidden]))>.fv-header-actions{grid-column:2;grid-row:1}} @media(max-width:480px){.fv-agent-panel{top:104px;left:16px;right:16px;width:auto}} .fv-agent-panel:focus{outline:none} -/* The same rows as an always-open list inside a view. */ -.fv-agent-list{display:grid;gap:10px;min-width:0} -.fv-agent-list[hidden],.fv-agent-list [hidden]{display:none} -.fv-agent-list h2{margin:0;font-size:var(--fv-strong);font-weight:var(--fv-semibold)} -.fv-agent-list ol{list-style:none;margin:0 -12px;padding:0;display:grid;gap:2px} -.fv-agent-empty{margin:0;font-size:var(--fv-body);line-height:1.5;color:var(--fv-muted)} diff --git a/web/agent_activity.js b/web/agent_activity.js index 0f5d08e7..5785391a 100644 --- a/web/agent_activity.js +++ b/web/agent_activity.js @@ -1,4 +1,4 @@ -import {agentGet, agentPost} from './agent_session.js'; +import {agentPost} from './agent_session.js'; // What agents connected to this FreeVideo are doing. Their requests share the // studio's queue, so people can see them, follow progress and stop them. const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./agent_activity.css', import.meta.url).href; document.head.append(css); @@ -131,58 +131,6 @@ async function showWork(t, job) { (await import('./library.js')).openLibrary(t, {selectedWork: job.work}); } -// The same jobs as a list that is always open, for a view of its own: what the agents are doing, how far -// along, where the result is, and a stop on each; open(job) can replace where a finished video opens. It shows -// once an agent is connected or has jobs, and asks nothing while it is out of sight. -export function agentActivityList(t, api, {interval = [1500, 8000], open = job => showWork(t, job)} = {}) { - const element = el('section', 'fv-agent-list'); element.hidden = true; - const heading = el('h2', null, t('Jobs', '任务')), list = el('ol'); - const empty = el('p', 'fv-agent-empty', t('Videos the agent makes will appear here.', 'Agent 生成的视频会显示在这里。')); - element.append(heading, list, empty); - let timer = null, stopped = false, on = null, failures = 0, active = 0; - // Hidden itself, the list is in sight when the view that holds it is. - const inSight = () => element.isConnected && (element.hidden ? element.parentElement : element)?.getClientRects().length > 0; - async function connected() { - try { - const response = await agentGet(api, '/freevideo/agent/v1/connect/status'); - return response.ok && (await response.json()).connected === true; - } catch { return false; } - } - async function poll() { - clearTimeout(timer); timer = null; - if (stopped) return; - const shown = inSight(); - if (shown && on === null) { - // Agent features can be off in this FreeVideo: ask once. - try { - const response = await api.fetchApi('/freevideo/agent', {cache: 'no-store'}); - on = response.ok && (await response.json()).enabled === true; - } catch { on = false; } - } - if (stopped || on === false) return; - if (shown && on) { - try { - const response = await api.fetchApi('/freevideo/agent/activity', {cache: 'no-store'}); - if (response.ok) { - const data = await response.json(), jobs = data.jobs || []; - const show = jobs.length > 0 || await connected(); - if (stopped) return; - failures = 0; active = data.active || 0; - const key = refill(element, list, rows(jobs, t, api, poll, open)); - list.hidden = !jobs.length; empty.hidden = !!jobs.length; element.hidden = !show; - if (key) list.querySelector(`[data-job="${key[0]}"] .${key[1].split(' ')[0]}`)?.focus?.(); - } else failures++; - } catch { failures++; } - } - if (stopped) return; - // Fast while agents work; every 3 s while empty, like the connection card, so connecting shows at once. - const wait = !shown ? 500 : active ? interval[0] : list.hidden ? Math.min(3000, interval[1]) : interval[1] * Math.min(4, 1 + failures); - timer = setTimeout(poll, wait); - } - poll(); - return {element, refresh: poll, stop() { stopped = true; clearTimeout(timer); }}; -} - export function createAgentActivity(t, api, host, {interval = [1500, 8000], open = job => showWork(t, job)} = {}) { const element = el('button', 'fv-agent-pill'); element.type = 'button'; element.hidden = true; element.setAttribute('aria-haspopup', 'dialog'); element.setAttribute('aria-expanded', 'false'); From 3dcec1d892f2a3262f142e72c579a0898873d43d Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 13:16:40 +0000 Subject: [PATCH 08/27] Agent: a name is enough to start a new timeline --- freevideo_engine/agent/guide.py | 1 + freevideo_engine/agent/projects.py | 13 ++++++++++--- freevideo_engine/agent/schema.py | 6 ++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index ae08a12b..ad420ec2 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -103,6 +103,7 @@ ], 'editing_projects': [ 'GET /projects lists timelines shared with the editor page. First GET /projects/{id} to read revision and clip IDs.', + 'For a new timeline POST /projects with {"project": {"name": "..."}} (width and height too if not 1344x768); it comes back empty at revision 1. Then add each clip with an insert operation.', 'POST /projects/{id}/operations sends revision and 1-64 operations, using whole frames at 24 fps. The batch saves once or changes nothing.', 'On 409 project_changed, recompute edits from error.project and its current_revision, then send a new request. On clip_busy wait for generation to finish.', 'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.', diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py index 348d5b8a..afd71807 100644 --- a/freevideo_engine/agent/projects.py +++ b/freevideo_engine/agent/projects.py @@ -35,6 +35,12 @@ } +def new_project(given): + """The page's empty timeline with what the caller sent on top; a name alone is enough to start one.""" + return dict(dict(version=1, id=None, revision=0, name='', fps=comfy_editor.FPS, width=1344, height=768, settings=None, serial=0, + tracks=[dict(id='v1', kind='video', clips=[]), dict(id='a1', kind='audio', clips=[])]), **given) + + class ProjectError(JobError): """A project error; a conflict carries the current project so the caller can retry from it.""" @@ -64,7 +70,7 @@ def _refused(project, error): reason = str(error) if type(error) is ValueError and re.fullmatch(r'[A-Z][A-Za-z ]{2,60}', str(error)) else '' row = schema.diagnostic('/project', 'a valid editor project' + (' (%s)' % (reason[:1].lower() + reason[1:]) if reason else ''), project) - row['next'] = ('Correct that part of /project. To place clips, create the project with empty tracks, ' + row['next'] = ('Correct that part of /project. For a new timeline send only {"project": {"name": "..."}}; ' 'then add each clip with an insert operation.') return spec.SpecError([row]) @@ -196,8 +202,9 @@ def create(self, value, *, ref=None, client=None): if replay is not None: return replay project = value.get('project') if isinstance(value, dict) else None - if not isinstance(project, dict) or project.get('id') is not None or project.get('revision') != 0: - raise _invalid('/project', 'a new project with id null and revision 0') + project = new_project(project) if isinstance(project, dict) else None + if project is None or project.get('id') is not None or project.get('revision') != 0: + raise _invalid('/project', 'a new project: its name is enough, with no id and no revision') # The page's receipt makes a repeated create return the first project. create = _operation_key(ref, client) if ref is not None else None self._lengths(project) diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 38410e1a..1ca01744 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -635,7 +635,7 @@ def image_schema(name): PROJECT_OPERATIONS = { 'list': ('GET', '/projects', 'List editing projects shared with the editor page. Then get one to read its revision and clip IDs.'), 'get': ('GET', '/projects/{id}', 'Read a timeline and authenticated source URLs. Read its revision and clip IDs before editing.'), - 'create': ('POST', '/projects', 'Create an editor timeline. Send a valid editor project with id null and revision 0.'), + 'create': ('POST', '/projects', 'Create an editor timeline. Send {"project": {"name": "..."}}; the rest of an empty timeline is filled in.'), 'write': ('PUT', '/projects/{id}', 'Replace a timeline. First get the project and preserve its id and revision; recompute from error.project on project_changed.'), 'operations': ('POST', '/projects/{id}/operations', 'Apply 1-64 timeline edits atomically and save once. First get revision and clip IDs; on project_changed recompute from error.project, and on clip_busy wait for generation to finish.'), 'generate': ('POST', '/projects/{id}/generate', 'Make new picture and sound for selected clips or a frame range, as the page Generate does. First get revision and clip IDs; the parts are reserved at once and fill in as the returned job finishes.'), @@ -730,7 +730,9 @@ def session_schema(name): def project_schema(name): if name in ('create', 'write'): - return obj(dict(project={'type': 'object', 'description': 'The editor page project record; validated by the shared editor save.'}, ref=REF), ('project',)) + text = ('A new timeline: {"name": "..."} is enough (width and height if not 1344x768); the rest is filled in. Add clips with insert operations afterwards.' + if name == 'create' else 'The whole project record as you read it, with your changes; validated by the shared editor save.') + return obj(dict(project={'type': 'object', 'description': text}, ref=REF), ('project',)) ids = array(CLIP, 1, 500) revision = number(1, integer=True, description='The saved revision you read; a later save returns project_changed.') if name == 'generate': From 9fd875ba4a3c4f644b671bcec3e884e9f86ec056 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 13:48:51 +0000 Subject: [PATCH 09/27] Editing view: a block can carry a short note, shown on the strip under the pictures - Agent: write the short note shown under a timeline clip - Agent: a clip note says what is in the picture, and the answer asks for a missing one - Editing view: the note examples in the tests say what a block is for - Editing view: a note written while its block is being replaced is kept, and a note is one line on both sides --- freevideo_engine/agent/editor_runtime.py | 1 + freevideo_engine/agent/guide.py | 1 + freevideo_engine/agent/projects.py | 19 +++++++++++++++- freevideo_engine/agent/schema.py | 3 ++- freevideo_engine/comfy_editor.py | 8 +++++++ freevideo_engine/editor_text.py | 2 +- web/editor_model.js | 28 +++++++++++++++++++++--- web/editor_notes.js | 10 +++++---- web/editor_text_model.js | 2 +- 9 files changed, 63 insertions(+), 11 deletions(-) diff --git a/freevideo_engine/agent/editor_runtime.py b/freevideo_engine/agent/editor_runtime.py index 6af6b6f5..2f793a32 100644 --- a/freevideo_engine/agent/editor_runtime.py +++ b/freevideo_engine/agent/editor_runtime.py @@ -137,6 +137,7 @@ case 'move': record = actions.moveParts(record, op.clip_ids, {frames: op.frames, track: null}); break; case 'describe': record = model.describe(record, actions.linkedIds(record, op.clip_ids), op.text, by); break; case 'set_ref': record = model.setRef(record, actions.linkedIds(record, op.clip_ids), op.ref); break; + case 'note': record = model.setNote(record, actions.linkedIds(record, op.clip_ids), op.text); break; case 'use_version': record = model.restoreVersion(record, op.clip_id, op.index, by); break; // One undo on the page; rejects as subtitles / range / text / locked / overlap. case 'subtitles': record = texts.addSubtitles(record, op.cues, {replace: op.replace === true}); break; diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index ad420ec2..3d683b80 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -108,6 +108,7 @@ 'On 409 project_changed, recompute edits from error.project and its current_revision, then send a new request. On clip_busy wait for generation to finish.', 'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.', 'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.', + 'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.', 'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.', ], 'project_routes': [dict(operation=name, method=method, path=path, description=summary) diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py index afd71807..26a02528 100644 --- a/freevideo_engine/agent/projects.py +++ b/freevideo_engine/agent/projects.py @@ -33,6 +33,19 @@ 'source': 'a source that has a picture or sound', 'invalid': 'an op named in the operations schema with its fields', } +# The note operation shares the page's 'text' code with describe; what it needed is its own. +NOTE_EXPECTED = 'picture clips and a text of at most 120 characters on one line (an empty text removes the note)' +# An agent acts on the last answer it read, so the answer that brings new picture clips asks for their notes. +NOTE_NEXT = ('Picture clips %s have no note yet. Write each one now with a note operation: in the user\'s language, ' + 'what is in the picture, subject first; not its position and not its progress.') + + +def note_next(result, project, ids): + missing = [clip['id'] for track in project['tracks'] if track['kind'] == 'video' for clip in track['clips'] + if clip['id'] in ids and not clip.get('note')] + if missing: + result['next'] = NOTE_NEXT % ', '.join(missing) + return result def new_project(given): @@ -168,6 +181,8 @@ def _save(self, project, *, ref=None, create=None, client=None): def _rejected(self, error, stored, path, received, *, operation=None): if error['code'] in STATE_CODES: raise ProjectError(STATE_CODES[error['code']], project=stored, operation=operation) + if error['code'] == 'text' and isinstance(received, dict) and received.get('op') == 'note': + raise _invalid(path, NOTE_EXPECTED, received) raise _invalid(path, RULE_EXPECTED.get(error['code'], 'an operation this timeline allows'), received) # Reads --------------------------------------------------------------------------------------------- @@ -270,6 +285,7 @@ def operations(self, identity, value, *, client=None, ref=None): operations[index] if index is not None else None, operation=index) saved = self._save(reply['project'], ref=ref, client=client) result = dict(project=saved, changed_clip_ids=changed_clip_ids(stored, saved), results=reply['results']) + note_next(result, saved, set(result['changed_clip_ids']) - set(_clips(stored))) self._remember(ref, digest, result, client) return result @@ -331,7 +347,8 @@ def generate(self, identity, value, *, jobs, capability=None, ref=None, client=N raise with self.lock: self.prepared[job['id']] = saved - result = dict(job=jobs.view(job), jobs=[jobs.view(job)], project=saved, clip_ids=plan['ids']) + result = note_next(dict(job=jobs.view(job), jobs=[jobs.view(job)], project=saved, clip_ids=plan['ids']), + saved, set(plan['ids'])) self._remember(ref, digest, result, client) return result diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 1ca01744..2fa23133 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -753,7 +753,7 @@ def project_schema(name): if name != 'operations': return EMPTY fields = { - 'insert': (dict(source=EDITOR_SOURCE, at=EDITOR_FRAME), ('source',), 'Insert picture and sound from a source. Omit at to append at the timeline end.'), + 'insert': (dict(source=EDITOR_SOURCE, at=EDITOR_FRAME), ('source',), 'Insert picture and sound from a source. Omit at to append at the timeline end. Then write the new picture clip\'s note.'), 'split': (dict(clip_ids=ids, frame=EDITOR_FRAME), ('clip_ids', 'frame'), 'Cut selected clips and their linked partners. The frame must be inside a selected segment.'), 'trim': (dict(clip_ids=ids, frame=EDITOR_FRAME, edge=enum(('start', 'end'))), ('clip_ids', 'frame', 'edge'), 'Trim a selected edge at the frame. The frame must be inside a selected segment.'), 'remove': (dict(clip_ids=ids), ('clip_ids',), 'Remove selected clips and linked partners. Use clip IDs from this revision.'), @@ -763,6 +763,7 @@ def project_schema(name): 'move': (dict(clip_ids=ids, frames=number(-10**7, 10**7, integer=True)), ('clip_ids', 'frames'), 'Move selected clips by a signed relative frame offset. Use clip IDs from this revision; linked partners move together.'), 'describe': (dict(clip_ids=ids, text=dict(type='string', maxLength=8000)), ('clip_ids', 'text'), 'Set the description of selected clips. Use clip IDs from this revision and one paragraph of text.'), 'set_ref': (dict(clip_ids=ids, ref=dict(type='string', maxLength=128, pattern=r'^[A-Za-z0-9_-]{0,128}$')), ('clip_ids', 'ref'), 'Set an outside shot reference on selected clips. Use clip IDs from this revision; an empty ref removes it.'), + 'note': (dict(clip_ids=ids, text=dict(type='string', maxLength=120)), ('clip_ids', 'text'), 'Write the one line the user sees under selected picture clips; without it they see the start of the generation text. Do it after you add, generate or change a clip: in the user\'s language, what is in the picture, subject first ("Train through morning mist"); a short clip shows only the first few characters. Not the clip\'s position (opening, part 2) and not its progress. Up to 120 characters on one line; an empty text removes it. It never changes the description used for generation.'), 'use_version': (dict(clip_id=CLIP, index=number(0, 49, integer=True)), ('clip_id', 'index'), 'Restore a stored clip version. First get the clip ID and its versions array; index is zero based.'), 'subtitles': (dict(cues=array(obj(dict(start=EDITOR_FRAME, frames=number(1, 10**7, integer=True), text=dict(type='string', minLength=1, maxLength=400)), ('start', 'frames', 'text')), 1, 500), replace=dict(type='boolean')), ('cues',), diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index 852d9e39..df799b49 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -39,6 +39,8 @@ _TOKEN = re.compile(r'[A-Za-z0-9_-]{1,128}') _REASON = re.compile(r'[a-z_]{1,80}') _REF = re.compile(r'[A-Za-z0-9_-]{1,128}') +# A note is one line: the page refuses the same characters (editor_model.js NOTE_BREAK) and the same ends. +_NOTE_BREAK = re.compile('[\x00-\x1f\x7f-\x9f\u2028\u2029]') _STATE = threading.RLock() _TRANSCODING = threading.Lock() _FAILED_IMPORTS = {} # terminal statuses awaiting writable storage @@ -279,6 +281,12 @@ def bounded(value, depth=0): if not isinstance(raw['ref'], str) or not _REF.fullmatch(raw['ref']): raise ValueError('Invalid block reference') clip['ref'] = raw['ref'] + if 'note' in raw: + note = raw['note'] + if (not _text(note, 120) or not note or note != note.strip() or '\ufeff' in (note[0], note[-1]) + or _NOTE_BREAK.search(note)): + raise ValueError('Invalid block note') + clip['note'] = note if source is None and ('take' in raw or 'made_text' in raw): raise ValueError('Invalid clip versions') _made(raw, clip) diff --git a/freevideo_engine/editor_text.py b/freevideo_engine/editor_text.py index 08720601..84fea212 100644 --- a/freevideo_engine/editor_text.py +++ b/freevideo_engine/editor_text.py @@ -79,7 +79,7 @@ def validate_text_project(raw, validate_media): _check(all(c.get(k) is None for k in ('source','pending','link','part','job','reason')) and c.get('extend',False) is False and ('versions' not in c or isinstance(c['versions'],list) and not c['versions'])) _check(type(c.get('in',0)) is int and c.get('in',0) == 0 and type(c.get('length',c['frames'])) is int and c.get('length',c['frames']) == c['frames']) - _check(not any(k in c for k in ('volume','fit','splice','target_frames','text_by','ref','take','made_text'))) + _check(not any(k in c for k in ('volume','fit','splice','target_frames','text_by','ref','note','take','made_text'))) clip = dict(id=identity, start=c['start'], frames=c['frames'], length=c['frames'], source=None, link=None, text=text, part=None, pending=None, reason=None, job=None, extend=False, versions=[], text_kind=kind, **{'in':0}) _check(kind != 'subtitle' or 'text_style' not in c) diff --git a/web/editor_model.js b/web/editor_model.js index 67031600..26e821d2 100644 --- a/web/editor_model.js +++ b/web/editor_model.js @@ -24,6 +24,10 @@ const check = (yes, code = 'invalid', message = code) => { if (!yes) reject(code const ID = /^c[1-9]\d{0,8}$/; // Who last wrote a block's description (its note on the timeline), and an outside reference (a shot id). const TEXT_BY = ['user', 'agent'], REF = /^[A-Za-z0-9_-]{1,128}$/; +// A short line about a block for whoever looks at the timeline; regenerating never reads it. +// One line: no control or line-break character anywhere, nothing at its ends that either side would strip. +const NOTE_BREAK = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/; +const validNote = text => validText(text) && text === text.trim() && !NOTE_BREAK.test(text) && Array.from(text).length >= 1 && Array.from(text).length <= 120; const WORK = /^\d{4}-\d{2}-\d{2}\/[0-9a-f]{32}$/; const PENDING = [null, 'draft', 'queued', 'running', 'failed', 'stopped']; const key = source => `${source.kind}:${source.id}`; @@ -168,6 +172,7 @@ export function parse(value) { // Who last wrote the description, and an outside reference (a shot id) for callers. if (item.text_by !== undefined) { check(TEXT_BY.includes(item.text_by), 'text'); clip.text_by = item.text_by; } if (item.ref !== undefined) { check(typeof item.ref === 'string' && REF.test(item.ref), 'ref'); clip.ref = item.ref; } + if (item.note !== undefined) { check(validNote(item.note), 'text'); clip.note = item.note; } // The part this one was pulled out of, or the part before it in the same range. if (item.follows !== undefined) { check(kind === 'video' && typeof item.follows === 'string' && ID.test(item.follows) && item.follows !== item.id, 'follows'); clip.follows = item.follows; } check(source !== null || (item.take === undefined && item.made_text === undefined), 'versions'); madeFields(item, clip); @@ -335,14 +340,31 @@ export function setRef(record, ids, ref) { check(typeof ref === 'string' && (!ref || REF.test(ref)), 'ref'); return onBlocks(record, ids, clip => { if (ref) clip.ref = ref; else delete clip.ref; }, false); } +// A note kept with the block; an empty value removes it. A block being generated can still be noted. +export function setNote(record, ids, text) { + check(typeof text === 'string' && (!text.trim() || validNote(text.trim())), 'text'); + return onBlocks(record, ids, clip => { if (text.trim()) clip.note = text.trim(); else delete clip.note; }, false, {picture: true, kept: true}); +} // Only an agent's description is marked; no mark means the user wrote it, as in every earlier edit. -function written(clip, text, by) { clip.text = text; if (text && by === 'agent') clip.text_by = by; else delete clip.text_by; } -function onBlocks(record, ids, change, refuseBusy) { +// A note was written about the description it came with, so it goes when the user writes another one. +function written(clip, text, by) { + if (by === 'user' && text !== clip.text) delete clip.note; + clip.text = text; if (text && by === 'agent') clip.text_by = by; else delete clip.text_by; +} +function onBlocks(record, ids, change, refuseBusy, {picture = false, kept = false} = {}) { check(Array.isArray(ids) && ids.length > 0 && ids.every(id => typeof id === 'string'), 'id'); for (const id of ids) check(find(record, id), 'missing'); const selected = linkedIds(record, ids), clips = all(record).filter(clip => selected.has(clip.id)); - check(!clips.some(isText), 'text'); assertUnlocked(record, [...selected]); if (refuseBusy) busy(clips); + check(!clips.some(isText), 'text'); + // A sound block takes the change only together with the picture it is linked to. + check(!picture || clips.every(clip => trackOf(record, clip.id).kind === 'video' || clip.link), 'text'); + assertUnlocked(record, [...selected]); if (refuseBusy) busy(clips); const next = copy(record); for (const clip of all(next)) if (selected.has(clip.id)) change(clip); + // The blocks set aside while a part is being replaced take the change too: whichever of them comes back shows it. + if (kept) for (const tx of next.replacements || []) for (const row of [...tx.originals, ...tx.clips]) { + const shown = clips.filter(clip => trackOf(record, clip.id).id === row.id); + for (const clip of row.clips) if (shown.some(v => v.id === clip.id || (clip.start >= v.start && end(clip) <= end(v)))) change(clip); + } return finish(next); } export function linkedIds(record, ids) { diff --git a/web/editor_notes.js b/web/editor_notes.js index 54bdc670..46e71322 100644 --- a/web/editor_notes.js +++ b/web/editor_notes.js @@ -12,8 +12,10 @@ function glyph(d) { const AGAIN = 'M13.5 2.5v3.5H10M13.2 6A5.5 5.5 0 1 0 13.5 9.5', PENCIL = 'M10.5 2.5l3 3-8 8h-3v-3z'; const generating = clip => ['queued', 'running'].includes(clip?.pending); const firstLine = text => (text || '').trim().split('\n')[0]; -// Only the picture track grows a description strip, and only once one block has a description. -export const describedTrack = track => track.kind === 'video' && track.clips.some(clip => !isText(clip) && clip.text?.trim()); +// A note written about a block (by an agent working on the project) is what the strip shows; without one, the description. +export const stripText = clip => clip.note || firstLine(clip.text); +// Only the picture track grows a description strip, and only once one block has something to show. +export const describedTrack = track => track.kind === 'video' && track.clips.some(clip => !isText(clip) && stripText(clip)); const agentWrote = record => record.tracks.some(track => track.clips.some(clip => clip.text_by === 'agent')); // Status words appear only in the hover tip and the panels (Iris 10-11). export function blockStatus(clip, t, fraction = 0) { @@ -101,7 +103,7 @@ export function createBlockNotes({state, t, actions}) { function showTip(item) { const view = item.element.closest('.fv-ed-track-view'), clip = item.clip; if (!view || !item.element.isConnected) return; const fraction = Number(item.element.style.getPropertyValue('--fv-ed-progress')) || 0; - const text = [clip.text.trim(), blockStatus(clip, t, fraction)].filter(Boolean).join(' · '); + const text = [clip.note || clip.text.trim(), blockStatus(clip, t, fraction)].filter(Boolean).join(' · '); if (!text) return; if (!tip) { tip = el('div', 'fv-ed-note-tip'); tip.setAttribute('role', 'tooltip'); } if (tip.parentElement !== view) view.append(tip); @@ -120,7 +122,7 @@ export function createBlockNotes({state, t, actions}) { items.set(item.clip.id, item); item.element.dataset.noted = String(noted); item.note.hidden = item.number.hidden = !noted; - item.note.textContent = firstLine(item.clip.text); item.number.textContent = String(index + 1); + item.note.textContent = stripText(item.clip); item.number.textContent = String(index + 1); // The pencil marks blocks an agent leaves alone; a project nobody else wrote in has none. item.mark.hidden = !agentWrote(record) || !!item.clip.text_by; } diff --git a/web/editor_text_model.js b/web/editor_text_model.js index 58626532..cf421837 100644 --- a/web/editor_text_model.js +++ b/web/editor_text_model.js @@ -48,7 +48,7 @@ export function readTextProject(raw, parseMedia) { check(typeof c.text === 'string' && Array.from(c.text).length <= TEXT_LIMIT && !/[\u0000-\u0008\u000b-\u001f\u007f\ud800-\udfff]/u.test(c.text)); check(['source','pending','link','part','job','reason'].every(k => c[k] == null) && (c.extend === undefined || c.extend === false) && (c.versions === undefined || (Array.isArray(c.versions) && c.versions.length === 0))); check((c.in === undefined || c.in === 0) && (c.length === undefined || c.length === c.frames)); - check(!['volume', 'fit', 'splice', 'target_frames', 'text_by', 'ref', 'take', 'made_text'].some(k => c[k] !== undefined)); + check(!['volume', 'fit', 'splice', 'target_frames', 'text_by', 'ref', 'note', 'take', 'made_text'].some(k => c[k] !== undefined)); const clip = {id: c.id, start: c.start, frames: c.frames, in: 0, length: c.frames, source: null, link: null, text: c.text, part: null, pending: null, reason: null, job: null, extend: false, versions: [], text_kind: track.kind}; check(track.kind !== 'subtitle' || c.text_style === undefined); From 68e2a4a0760df1d191353c5097b434a0dda69326 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 14:39:03 +0000 Subject: [PATCH 10/27] Editing view: a saved video put on the tracks brings the description it was generated from - Agent projects: a work placed on the timeline keeps the description it was generated from - Editing view: a description that would make the project too large to save, or a description file too long to be one, is left out when a saved video goes on the tracks - Editing view: a long description ends on half a line in its box, so it reads as continuing below --- freevideo_engine/agent/projects.py | 13 ++++++++----- freevideo_engine/agent/schema.py | 2 +- freevideo_engine/comfy_editor.py | 21 +++++++++++++++++++++ web/editor.js | 4 ++-- web/editor_model.js | 21 ++++++++++++++++++--- web/editor_notes.css | 2 +- 6 files changed, 51 insertions(+), 12 deletions(-) diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py index 26a02528..2dd233f4 100644 --- a/freevideo_engine/agent/projects.py +++ b/freevideo_engine/agent/projects.py @@ -36,6 +36,8 @@ # The note operation shares the page's 'text' code with describe; what it needed is its own. NOTE_EXPECTED = 'picture clips and a text of at most 120 characters on one line (an empty text removes the note)' # An agent acts on the last answer it read, so the answer that brings new picture clips asks for their notes. +# It comes first in that answer: the project after it carries every clip's description, and a real agent +# that printed only the start of a long answer never saw a request placed after it. NOTE_NEXT = ('Picture clips %s have no note yet. Write each one now with a note operation: in the user\'s language, ' 'what is in the picture, subject first; not its position and not its progress.') @@ -43,9 +45,7 @@ def note_next(result, project, ids): missing = [clip['id'] for track in project['tracks'] if track['kind'] == 'video' for clip in track['clips'] if clip['id'] in ids and not clip.get('note')] - if missing: - result['next'] = NOTE_NEXT % ', '.join(missing) - return result + return dict(next=NOTE_NEXT % ', '.join(missing), **result) if missing else result def new_project(given): @@ -244,7 +244,8 @@ def write(self, identity, value, *, ref=None, client=None): # Operations ---------------------------------------------------------------------------------------- def _media(self, operations): - """Insert operations name a source key; the server fills in its length and streams.""" + """Insert operations name a source key; the server fills in its length and streams, and for a + saved video the description it was generated from.""" keys = [op['source'] for op in operations if op.get('op') == 'insert'] found = comfy_editor.sources(self.root, keys) if keys else {} resolved = [] @@ -258,6 +259,8 @@ def _media(self, operations): kind, identity = op['source'].split(':', 1) media = dict(source=dict(kind=kind, id=identity), frames=info['frames'], hasVideo=info['has_video'], hasAudio=info['has_audio'], width=info['width'], height=info['height']) + if info.get('text'): + media['text'] = info['text'] resolved.append(dict({k: v for k, v in op.items() if k != 'source'}, media=media)) return resolved @@ -285,7 +288,7 @@ def operations(self, identity, value, *, client=None, ref=None): operations[index] if index is not None else None, operation=index) saved = self._save(reply['project'], ref=ref, client=client) result = dict(project=saved, changed_clip_ids=changed_clip_ids(stored, saved), results=reply['results']) - note_next(result, saved, set(result['changed_clip_ids']) - set(_clips(stored))) + result = note_next(result, saved, set(result['changed_clip_ids']) - set(_clips(stored))) self._remember(ref, digest, result, client) return result diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 2fa23133..f8d8fb4c 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -753,7 +753,7 @@ def project_schema(name): if name != 'operations': return EMPTY fields = { - 'insert': (dict(source=EDITOR_SOURCE, at=EDITOR_FRAME), ('source',), 'Insert picture and sound from a source. Omit at to append at the timeline end. Then write the new picture clip\'s note.'), + 'insert': (dict(source=EDITOR_SOURCE, at=EDITOR_FRAME), ('source',), 'Insert picture and sound from a source. Omit at to append at the timeline end. A work brings the description it was generated from. Then write the new picture clip\'s note.'), 'split': (dict(clip_ids=ids, frame=EDITOR_FRAME), ('clip_ids', 'frame'), 'Cut selected clips and their linked partners. The frame must be inside a selected segment.'), 'trim': (dict(clip_ids=ids, frame=EDITOR_FRAME, edge=enum(('start', 'end'))), ('clip_ids', 'frame', 'edge'), 'Trim a selected edge at the frame. The frame must be inside a selected segment.'), 'remove': (dict(clip_ids=ids), ('clip_ids',), 'Remove selected clips and linked partners. Use clip IDs from this revision.'), diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index df799b49..d915d2c6 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -653,6 +653,10 @@ def sources(root, keys): if _read_json(report, 4 * 1024**2).get('success') is not True: raise ValueError('Creation is incomplete') media = probe(path) + if source['kind'] == 'work': + text = _work_text(root, relative) + if text: + media['text'] = text if source['kind'] == 'import': metadata = _read_import(root, source['id']) media.update(preview_width=media['width'], preview_height=media['height']) @@ -682,6 +686,23 @@ def sources(root, keys): return result +def _work_text(root, relative): + """The text a saved video was generated from; a block made from it starts with this description.""" + try: + path = _safe(root, (PurePosixPath(relative).parent / 'prompt.txt').as_posix()) + with path.open('rb') as stream: + data = stream.read(4 * 8000 + 1) + if len(data) > 4 * 8000: + # Longer than any description a block keeps; its beginning alone is not the description. + return '' + text = data.decode('utf-8').strip() + except (OSError, ValueError): + # Optional: an old, joined or hand-copied video has none. + return '' + # The same limit as a block's own description, so the project it lands in still saves. + return text if _text(text, 8000) else '' + + def _read_import(root, identity): key = (str(Path(root).resolve()), identity) with _STATE: diff --git a/web/editor.js b/web/editor.js index c5d2f9fc..d1560955 100644 --- a/web/editor.js +++ b/web/editor.js @@ -435,7 +435,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given } const metadata = store.sources.get(sourceKey(value.source)); try { state.apply(t('Insert source', '放入素材'), record => model.insertSource(record, { - ...value, width: metadata?.width ?? value.width, height: metadata?.height ?? value.height, + ...value, width: metadata?.width ?? value.width, height: metadata?.height ?? value.height, text: value.text ?? metadata?.text, at: value.at ?? model.duration(record)}), {clearRange: state.selection !== initialSelection}); return {ok: true}; } catch (valueError) { @@ -856,7 +856,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given record = model.emptyProject({width: even(metadata?.width || g.width || panel.width || 1344), height: even(metadata?.height || g.height || panel.height || 768), settings: panel.settings || null, name: t('Untitled edit', '未命名剪辑')}); record = model.insertSource(record, {source: {kind: 'work', id}, frames: metadata?.frames || g.frames || Math.round((g.seconds || 15) * 24), - hasVideo: true, hasAudio: metadata?.has_audio ?? work.has_audio !== false, at: 0}); + hasVideo: true, hasAudio: metadata?.has_audio ?? work.has_audio !== false, text: metadata?.text, at: 0}); if (!metadata) store.cache({[key]: {video: mediaURL(api, work.video), audio: mediaURL(api, work.video), frames: g.frames, has_video: true, has_audio: true, path: work.video, missing: false, thumbnail: '/freevideo/library/thumbnail?' + new URLSearchParams({id})}}); } diff --git a/web/editor_model.js b/web/editor_model.js index 26e821d2..cb83814b 100644 --- a/web/editor_model.js +++ b/web/editor_model.js @@ -433,9 +433,19 @@ function sourceAudioTrack(record, start, frames, allowRoom = false) { } throw new EditError('audio_full'); } -export function insertSource(record, {source, frames, hasVideo, hasAudio, width, height, at: position = 0}) { - source = sourceData(source); check(whole(frames, 1) && whole(position), 'range'); +// A saved project file holds 8 MiB; every character outside plain ASCII is written there as six. +const STORED_ROOM = 8 * 1024 * 1024 - 64 * 1024; +function storedSize(record) { + const written = JSON.stringify(record); let size = written.length; + for (let i = 0; i < written.length; i++) { const code = written.charCodeAt(i); size += code > 126 ? 5 : code === 44 || code === 58 ? 1 : 0; } + return size; +} +// A saved video brings the description it was generated from; one that cannot be kept is left out. +export function insertSource(record, {source, frames, hasVideo, hasAudio, width, height, at: position = 0, text = '', by = 'user'}) { + const given = source; source = sourceData(source); check(whole(frames, 1) && whole(position), 'range'); check(typeof hasVideo === 'boolean' && typeof hasAudio === 'boolean' && (hasVideo || hasAudio), 'source'); + check(TEXT_BY.includes(by), 'text'); + const words = hasVideo && typeof text === 'string' && validText(text.trim()) ? text.trim() : ''; const next = copy(record); if (hasVideo) followFirstSize(next, width, height); let track = trackBy(next, hasVideo ? 'v1' : 'a1'); @@ -445,11 +455,16 @@ export function insertSource(record, {source, frames, hasVideo, hasAudio, width, const sound = available ? audioTrack(next, sourceAudioTrack(available, start, frames, true).id) : null; applyRoom(next, start, frames, {tracks: [track.id, ...(sound ? [sound.id] : [])]}); const primary = newClip(next, track, {source, frames, start}); + if (words) written(primary, words, by); if (hasVideo && hasAudio) { const mate = newClip(next, sound, {source, frames, start}); primary.link = mate.id; mate.link = primary.id; + if (words) written(mate, words, by); } - return finish(next); + const result = finish(next); + // The description is a convenience: a project it would make too large to save takes the video without it. + if (words && storedSize(result) > STORED_ROOM) return insertSource(record, {source: given, frames, hasVideo, hasAudio, width, height, at: position}); + return result; } export function move(record, id, start) { if (isText(find(record, id))) return changeText(record, id, {start}); diff --git a/web/editor_notes.css b/web/editor_notes.css index e61a0063..48bdd29c 100644 --- a/web/editor_notes.css +++ b/web/editor_notes.css @@ -4,7 +4,7 @@ .fv-editor .fv-ed-notes-again{display:inline-flex;align-items:center;gap:4px;height:26px;padding:0 8px;font-size:12px} .fv-ed-notes-again .fv-ed-glyph{width:12px;height:12px} .fv-editor .fv-ed-notes-again:disabled{opacity:.4} -.fv-editor .fv-ed-notes-text{display:block;height:auto;box-sizing:border-box;width:100%;min-height:34px;max-height:94px;padding:6px 10px;resize:none;overflow-y:auto;font:13px/20px var(--fv-font);color:var(--fv-ink);background:var(--fv-bg);border:1px solid var(--fv-border);border-radius:8px} +.fv-editor .fv-ed-notes-text{display:block;height:auto;box-sizing:border-box;width:100%;min-height:34px;max-height:98px;padding:6px 10px;resize:none;overflow-y:auto;font:13px/20px var(--fv-font);color:var(--fv-ink);background:var(--fv-bg);border:1px solid var(--fv-border);border-radius:8px} .fv-editor .fv-ed-notes-text:focus-visible{outline:1px solid var(--fv-accent);outline-offset:-1px;border-color:var(--fv-accent)} .fv-editor .fv-ed-notes-text:disabled{opacity:.4} .fv-ed-notes-versions{display:flex;flex-direction:column;gap:6px;margin-top:6px} From 8357b7d1e046135a8ba924cef8bf6aaac5ddeb49 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 15:16:07 +0000 Subject: [PATCH 11/27] Install the patched Diffusers as a regular copy where the Windows code page cannot spell its folder --- freevideo_engine/install.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/freevideo_engine/install.py b/freevideo_engine/install.py index 73385e8f..4d4b00da 100644 --- a/freevideo_engine/install.py +++ b/freevideo_engine/install.py @@ -32,7 +32,13 @@ def install_packages(env, source): if env.get('FREEVIDEO_UV') else [sys.executable, '-m', 'pip', 'install']) # uv is a native program; pip runs in this interpreter's own Python. run = subprocess_output.run_external if env.get('FREEVIDEO_UV') else subprocess_output.run_python - result = run([*command, '-e', str(diffusers), '--no-deps'], + if pth_spellable(diffusers): + target = ['-e', str(diffusers)] + else: + # A regular copy carries no path; replace any copy from an older tree. + target = [str(diffusers), *(['--reinstall-package', 'diffusers'] if env.get('FREEVIDEO_UV') + else ['--force-reinstall'])] + result = run([*command, *target, '--no-deps'], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(result.stdout, flush=True) if result.returncode: @@ -46,6 +52,26 @@ def install_packages(env, source): 'models_downloaded': False, 'environment_variable': 'FREEVIDEO_VDN_ROOT'})) +def pth_spellable(path, *, windows=None, encoding=None): + """Whether an editable install may name this folder in a .pth file. + + setuptools writes a src-layout folder into the .pth in the locale code page, + and Python 3.12 reads it the same way; UTF-8 mode changes neither. Where the + Windows code page cannot spell the folder, a regular copy is installed + instead: add_vdn() puts the pinned source first on sys.path either way. + """ + if not (sys.platform == 'win32' if windows is None else windows): + return True + if encoding is None: + import locale + encoding = locale.getencoding() if hasattr(locale, 'getencoding') else locale.getpreferredencoding(False) + try: + str(path).encode(encoding) + except (UnicodeEncodeError, LookupError): + return False + return True + + def prepare_diffusers(root, dependencies): """Build the official two-patch tree atomically; never reset a user checkout.""" spec = dependencies['diffusers'] From 2fbd57955bc24ba42e4fc371a1b4e9012ac0d659 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 15:36:08 +0000 Subject: [PATCH 12/27] Windows int8: count resident blocks from the measured small-canvas workspace --- freevideo_engine/policy.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/freevideo_engine/policy.py b/freevideo_engine/policy.py index b2bdceb2..1c1720d7 100644 --- a/freevideo_engine/policy.py +++ b/freevideo_engine/policy.py @@ -427,8 +427,23 @@ def int8_gpu_output_need(tokens, windows=False, full_tail=False): FF_STASH_WIDTH = 14336 # BF16 SwiGLU output channels kept for the per-tensor FP8 scale +# Int8 GPU-output workspace in GiB, second transfer slot included, for +# canvases below the envelope's anchors: (video tokens, GiB). A two-pass +# 960x544 / 4.5 s request on a Windows RTX 5060 Ti 16 GB peaked 2.48 GiB at +# 4320 tokens and 3.46 GiB at 16320 tokens beyond its sixteen resident +# blocks, against 7.14 and 7.82 from the envelope, so the plan held sixteen +# and eighteen blocks where the budget held about twenty-five and streamed +# the rest from disk on every step. Each figure carries one GiB of margin. +# The curve joins the envelope at its lowest anchor (896 square, 243 +# frames) and never exceeds it. It rests on one card. +SMALL_CANVAS_INT8_WORKSPACE = ( + (4320, 3.48), + (16320, 4.46), + (56448, 6.5 + 6. * 56448 / 72576 + BLOCK_BYTES / GiB - 56448 * FF_STASH_WIDTH * 2 / GiB), +) -def windows_gpu_output_workspace(video_tokens, *, prefetch=False, ff_stash=True): + +def windows_gpu_output_workspace(video_tokens, *, prefetch=False, ff_stash=True, small_canvas=False): """Starting workspace for head-8 GPU outputs with the full per-tensor FF stash. Int8 projections scale each activation row on its own, so every FF row tile @@ -442,8 +457,17 @@ def windows_gpu_output_workspace(video_tokens, *, prefetch=False, ff_stash=True) The 6.5 + 6 * token-ratio envelope covers both anchors. It is a cold-start estimate, not a fitted throughput model or proof of capacity elsewhere. The OS growth reserve has already been subtracted from the budget. + + small_canvas=True counts resident blocks below the anchors from the + measured int8 curve (SMALL_CANVAS_INT8_WORKSPACE) instead of the envelope. + Head-group choices keep using the envelope. """ tokens = 72576 if video_tokens is None else video_tokens + if small_canvas and not ff_stash and tokens < SMALL_CANVAS_INT8_WORKSPACE[-1][0]: + for (low, at_low), (high, at_high) in zip(SMALL_CANVAS_INT8_WORKSPACE, SMALL_CANVAS_INT8_WORKSPACE[1:]): + if tokens <= high: + at = at_low + (at_high - at_low) * max(0, tokens - low) / (high - low) + return int(at * GiB) - (0 if prefetch else BLOCK_BYTES) workspace = int((6.5 + 6. * tokens / 72576) * GiB) + (BLOCK_BYTES if prefetch else 0) return workspace if ff_stash else workspace - tokens * FF_STASH_WIDTH * 2 @@ -1220,7 +1244,8 @@ def held(group): # FF stash. In addition, crediting a shorter geometry against its old # 345-frame anchor added five blocks to a request that already OOMed. # Budget the current execution path before host placement is planned. - windows_workspace = windows_gpu_output_workspace(effective_tokens, prefetch=prefetch, ff_stash=not int8) + windows_workspace = windows_gpu_output_workspace(effective_tokens, prefetch=prefetch, ff_stash=not int8, + small_canvas=True) windows_workspace += max(0, activation_bytes(head, effective_tokens) - activation_bytes(8, effective_tokens)) if prefetch and gpu_budget < 14 * GiB: # Saving a transfer slot must not turn fully retained host weights From c8926144f7d3ffd207011ae247bcabff3482b4d4 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 15:40:12 +0000 Subject: [PATCH 13/27] Background clean-up check finds the engine on installs where it starts without it - The offer to free the space of unused video model files could not appear on such installs - Choosing Delete also removes the small records that interrupted downloads left behind --- freevideo_engine/adaln_reclaim.py | 31 +++++++++++++++++++++++++++++-- freevideo_engine/paths.py | 4 ++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/freevideo_engine/adaln_reclaim.py b/freevideo_engine/adaln_reclaim.py index b95112aa..70e8f334 100644 --- a/freevideo_engine/adaln_reclaim.py +++ b/freevideo_engine/adaln_reclaim.py @@ -39,7 +39,7 @@ from . import adaln_assets as assets from .long_paths import io_path -from .variant_cleanup import inside, redirected +from .variant_cleanup import SIDECAR, SIDECAR_LIMIT, inside, redirected RECEIPT = 'adaln-reclaim.json' OFFER = 'adaln-reclaim-offer.json' @@ -286,6 +286,18 @@ def verify_sources(cache, manifest, should_stop=lambda: False): return None +def _remove_sidecars(cache, listed): + """The small records an earlier download left beside a source (.partial.source.json), so the folder can go.""" + for _, path in listed: + sidecar = path.with_name(path.name + SIDECAR) + info = _regular(sidecar) + if info is not None and info.st_size <= SIDECAR_LIMIT and _contained(cache, sidecar): + try: + io_path(sidecar).unlink() + except OSError: + pass + + def _remove_tables(cache, folders): removed = 0 for folder, _ in folders: @@ -373,6 +385,7 @@ def reclaim(cache, *, should_stop=lambda: False, schedules=None, lock=None, ask= return kept('redirected') freed += info.st_size io_path(path).unlink() + _remove_sidecars(cache, listed) try: (io_path(cache) / SOURCES).rmdir() except OSError: @@ -423,11 +436,25 @@ def apply(machine, *, should_stop=lambda: False, schedules=None): return result +def _installed_engine_schedules(machine): + """engine_schedules for a process that has not put the engine's dependencies on its path (the background check).""" + ready = [] + def answer(steps): + if not ready: + from . import paths + vdn = Path(machine['root']) / 'vendor' / 'vdn' if machine.get('root') else None + paths.add_vdn(vdn if vdn is not None and (vdn / 'src/models/hybrid_attention.py').is_file() else None) + ready.append(True) + return engine_schedules(steps) + return answer + + def run(machine, *, should_stop=lambda: False): """Entry point of the background precompile: check and leave the offer, report one event, never raise.""" started = time.time() try: - result = reclaim(machine['cache'], should_stop=should_stop, ask=True, release=installed_release()) + result = reclaim(machine['cache'], should_stop=should_stop, ask=True, release=installed_release(), + schedules=_installed_engine_schedules(machine)) except Exception as error: # Best effort: the installation works either way. result = dict(action='kept', reason='error:' + type(error).__name__, bytes=0, tables_bytes=0) if result['action'] == 'kept' and result['reason'] not in ('declined', 'stopped', 'busy') \ diff --git a/freevideo_engine/paths.py b/freevideo_engine/paths.py index f5bdbe5c..b7ce2b25 100644 --- a/freevideo_engine/paths.py +++ b/freevideo_engine/paths.py @@ -33,8 +33,8 @@ def installed_model_root(): return model_root() -def add_vdn(): - root = vdn_root().resolve() +def add_vdn(root=None): + root = (root or vdn_root()).resolve() if not (root / 'src/models/hybrid_attention.py').is_file(): raise RuntimeError('VDN dependency is missing. Run freevideo setup or set FREEVIDEO_VDN_ROOT.') if str(root) not in sys.path: From f87f7642cc588be1e351090d36119abc9b4d6763 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 15:41:57 +0000 Subject: [PATCH 14/27] First-run guide: keep the first step's Chinese line together and wait for anchors still loading - A guide that starts by itself waits up to five seconds for the part of the page it points at - Tests for the wait and for the line break --- web/first_run_guide.js | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/web/first_run_guide.js b/web/first_run_guide.js index 4efc7c76..41d452e9 100644 --- a/web/first_run_guide.js +++ b/web/first_run_guide.js @@ -12,7 +12,7 @@ export const REPLAY_TEXT = { export const CHAPTERS = [ {id: 'describe', anchor: '[data-fv-guide="describe"]', added: false, en: 'Type your prompt here. You can have an agent follow the Skill at the top right, or turn on FreeToken enhancement.', - zh: '在这里输入提示词。可以让 Agent 参考右⁠上⁠角的 Skill,或开启 FreeToken 提示词增强。'}, + zh: '在这里输入提示词。可以让 Agent 参⁠考 Skill,或开启 FreeToken 提示词增强。'}, {id: 'range', anchor: '[data-fv-guide="range"]', added: true, en: 'See the available sizes and lengths here, and how long each takes.', zh: '在这里查看可生成的尺\u2060寸、时长和预计用时。'}, {id: 'generate', anchor: '[data-fv-guide="generate"]', added: false, @@ -320,10 +320,18 @@ export function createFirstRunGuide({dialog, api, cn, isBusy = () => false}) { if (current === handle) current = null; }}; current = handle; - const startTimer = setTimeout(() => { - if (!automatic || attempted || manual || layer) return; - attempted = true; - show().then(shown => { if (!shown) stopped = true; }); - }, 1000); + // Steps are fixed when the guide starts, so give an anchor that is still waiting for its data up to 5 s. + // The Agent tab is not waited for: its discovery has already finished when the guide is created. + const pending = () => CHAPTERS.some(c => c.id !== 'agent' && target(c) && !visible(target(c))); + function begin(waited = 0) { + startTimer = setTimeout(() => { + if (!automatic || attempted || manual || layer || disposed) return; + if (pending() && waited < 5000) { begin(waited + 250); return; } + attempted = true; + show().then(shown => { if (!shown) stopped = true; }); + }, waited ? 250 : 1000); + } + let startTimer = null; + begin(); return handle; } From 97879123859629abebef9e10c1d7038c044f28e6 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 15:44:25 +0000 Subject: [PATCH 15/27] Whole-video progress and the time left are estimated from this computer's earlier generations - A percentage and a remaining time are shown from the moment a video starts - The time left comes back after it ran out or after a retry starts - A measured time well above the shown one raises it gently --- freevideo_engine/agent/jobs.py | 2 +- freevideo_engine/comfy_bridge.py | 960 +++++++++++++++++++------- freevideo_engine/comfy_progress.py | 29 +- freevideo_engine/effort_forecast.py | 87 ++- freevideo_engine/progress_timeline.py | 9 +- freevideo_engine/stage_estimate.py | 5 + web/autopilot_chat.js | 40 +- web/generation_progress.js | 108 ++- web/progress_connection.js | 4 +- 9 files changed, 953 insertions(+), 291 deletions(-) diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index 143f3b17..0d7cfce8 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -992,7 +992,7 @@ def estimate(self, row): 'step_elapsed_seconds', 'estimated_step_seconds') if message.get(key) is not None} overall = message.get('overall') if isinstance(message.get('overall'), dict) else {} overall = {key: overall[key] for key in ('fraction', 'estimated', 'status', 'phase_start_fraction', 'phase_weight', - 'phase_seconds', 'phase_elapsed_seconds', 'sampling_time_weighted') + 'phase_seconds', 'phase_elapsed_seconds', 'sampling_time_weighted', 'remaining_source') if overall.get(key) is not None} if overall: value['overall'] = overall diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index be111457..954a74c1 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -11,6 +11,7 @@ import sys import time import uuid +from collections import deque from .long_paths import io_path, resolved, plain, plain_data, plain_errors from . import processes @@ -75,260 +76,729 @@ def progress_history_forecast(root, machine, canvas): return {} # A progress estimate cannot prevent generation. -# The decoder's phase events, as decode.py and macos_decode.py print them. +class _SafeProgress: + """Any failure of the estimate falls back to plain progress messages.""" + def __init__(self, inner): + self.inner = inner + + def _call(self, name, *args, **kwargs): + try: + return getattr(self.inner, name)(*args, **kwargs) + except Exception: + self.inner = _PlainProgress() # A progress estimate cannot prevent generation. + return getattr(self.inner, name)(*args, **kwargs) + + def annotate(self, message, *, event=None): + return self._call('annotate', message, event=event) + + def observe(self, event): + return self._call('observe', event) + + def heartbeat(self): + return self._call('heartbeat') + + def configure(self, **facts): + return self._call('configure', **facts) + + +class _PlainProgress: + """Pass engine messages through unchanged when no estimate can be built.""" + def annotate(self, message, *, event=None): + return dict(message) + + def observe(self, event): + pass + + def heartbeat(self): + return None + + def configure(self, **facts): + pass + + +def _engine_version(): + from . import __version__ + return __version__ + + +# Names emitted by both decoders; forecast keys use the same suffixes. _DECODE_PARTS = {'Loading video VAE': 'vae_load', 'Decoding the video': 'video_decode', 'Loading and decoding audio': 'audio_load_decode', 'Saving MP4 and audio': 'encode'} class WholeVideoProgress: - """Annotate phase events with one report-weighted whole-video progress. + """One elapsed/remaining clock, bounded by evidence of completed work. - Local reports take precedence. Without history, live sampling supplies a - rough decoder estimate which real decoded-frame timings then replace. - These presentation estimates never change resource admission or placement. + This display estimator never changes the inference plan. Stage predictions + set the initial ruler; only real events release its count ceiling. Replay + options control smoothing, silent countdown and the ceiling's time shares. """ - ORDER = ('encoding', 'load', 'sampling', 'decode') - # The decoder's own steps, in the order it runs them, by the name of each one's forecast. - DECODE_PARTS = ('vae_load', 'video_decode', 'audio_load_decode', 'encode') - - def __init__(self, clock=time.perf_counter): - self.clock = clock - self.started = clock() - self.phase_started = self.started - self.phase = None - self.completed = set() - self.weights = {} + DECODE_PARTS = tuple(_DECODE_PARTS.values()) + + def __init__(self, clock=time.perf_counter, *, hardware=None, request=None, + cold_flags=None, records=(), rate=.005, smoothing='track', + silence='stage', ceiling='stretch', tau=10., max_rise=.019, final=.98, slow_start=True, started=None): + from . import stage_estimate + from .effort_forecast import stage_forecast + if not math.isfinite(rate) or rate < 0: + raise ValueError('Progress smoothing rate must be finite and nonnegative') + for name, value, choices in (('smoothing', smoothing, ('countdown', 'hold', 'track')), + ('silence', silence, ('freeze', 'stage')), + ('ceiling', ceiling, ('prior', 'current', 'stretch'))): + if value not in choices: + raise ValueError(f'Invalid progress {name}: {value!r}') + self.clock, self.rate = clock, rate + self.smoothing, self.silence, self.ceiling = smoothing, silence, ceiling + # tau: time constant of a downward correction ('track'); max_rise: the + # most the shown fraction may rise in any one second (None: no cap). + self.tau, self.max_rise = tau, max_rise + # final: once every frame is decoded, the bar approaches this fraction + # over the predicted audio and save time (None: no approach). + self.final = final + # slow_start: on a 16 GB machine, a load much slower than predicted + # scales the later stages until history or measured speed takes over. + self.slow_start = slow_start + self.started = self.phase_started = self.last_tick = self.last_event = clock() if started is None else started + self.last_publish = self.started + self.rise_window = deque() + self.hardware = dict(hardware or {}) + self.request = dict(width=1344, height=768, frames=243, base_steps=8, + refine_steps=3, two_pass=True, flow='full', task='t2va') + self.request.update({k: v for k, v in (request or {}).items() if v is not None}) + self.cold_flags = dict(cold_flags or {}) + self.records = records + self.params = stage_estimate._params(None) + self.prior = stage_estimate.predict_prior(self.hardware, self.request, self.cold_flags, self.params) + self.weights = {k: self.prior['stages'][k]['seconds'] for k in stage_estimate.STAGES} + local = stage_forecast(records, self.request, self.cold_flags, self.weights, version=_engine_version()) + self.remaining_source = 'local' if local.get('stages') else 'similar' + self.typical = local['typical'] + self.local = local + self.weights.update(local.get('stages', {})) + self.completed = {k for k, v in self.weights.items() if v == 0} self.measured = {} - # Seconds each phase has run over every stretch of it, the forecast's - # own weights, phases the forecast closed and phases a retry does not repeat. self.spent = {} - self.forecasts = {} - self.closed = set() + self.forecasts = dict(self.weights) + self.phase_counts = {} self.reused = set() + self.phase = 'encode' + self.completed.discard(self.phase) + self.sampling_plan = {} + self.steps = {'pass1': {}, 'pass2': {}} + self.live_calibration = True + self.count = None + self.fraction = 0. + self.ceiling_base = 0. + self.last_ceiling = 0. + self.ceiling_weights = dict(self.weights) + self.remaining_target = self.remaining_shown = sum(self.weights.values()) + self.total = self.remaining_target + self.last_message = {} + self.finished = False + self.decode_remaining = None self.decode_parts = {} + self.decode_part = None self.decode_rest = None - self.total = 0. - self.forecast_ready = False - self.failed = False - self.sampling_plan = {} - self.decode_from_sampling = False - self.sampling_time_weighted = False - self.sampling_rest = None - self.decode_tail = 0. - self.shown = 0. - - def _sampling_estimate(self, message, now): - if self._step_estimate(message, now) or not self.sampling_rest or self.sampling_rest[0] != message.get('done'): - return - # No new step time from this message. Time that passes outside a step, as between the two passes, is not - # taken out of the steps still to run. - inside = _timing_seconds(message.get('step_elapsed_seconds')) or 0. - self.weights['sampling'] = max(self.weights.get('sampling', 0.), - now - self.phase_started + max(0., self.sampling_rest[1] - inside)) - - def _step_estimate(self, message, now): - estimate = _timing_seconds(message.get('estimated_step_seconds')) - done, total = message.get('done'), message.get('total') - if estimate is None or type(done) is not int or type(total) is not int or not 0 <= done <= total: + self.decode_tail = max(0., self.weights['decode_save'] - self.prior['stages']['video_decode']['seconds']) + self.retry_decode_only = False + self.retry_reuse_first = False + self.history = None # (output directory, machine, opened): read late by _late_history. + self.shown_source = self.remaining_source + + @classmethod + def for_request(cls, machine, canvas, output_directory, *, base_steps=8, + refine_steps=3, two_pass=True, preview=False, upscale=False, + encoder_prewarm=None, options=None): + import platform + from .effort_forecast import start_context + started, opened = time.perf_counter(), time.monotonic() + observed, records = start_context(output_directory, machine) + hardware = cls._hardware(observed, machine) + # machine.vram_gib/ram_gib can be user policy caps, not physical totals. + # A worker process being alive is not proof that a model is resident. + warm = encoder_prewarm or {} + flags = {'encode_cached': True if upscale else None} + if warm.get('gpu_preload') is True and warm.get('state') in ('ready', 'complete') and not warm.get('worker_released'): + flags['encoder_cold'] = False + tracker = cls(hardware=hardware, records=records, cold_flags=flags, started=started, + request=dict(canvas, base_steps=base_steps, refine_steps=refine_steps, + two_pass=two_pass, flow='upscale' if upscale else 'preview' if preview else 'full'), + **(options or {})) + tracker.history = (output_directory, machine, opened) + return tracker + + @staticmethod + def _hardware(observed, machine): + import platform + observed = dict(observed) + if isinstance(machine.get('hardware'), dict): + observed.update(machine['hardware']) + device = machine.get('device_identity') + device = device if isinstance(device, dict) else {} + hardware = dict(gpu_name=observed.get('gpu_name') or machine.get('gpu_name') or device.get('name'), + compute_capability=observed.get('capability'), + os=observed.get('system') or machine.get('system') or platform.system()) + for target, source in (('ram_total_gib', 'ram_total'), ('vram_total_gib', 'vram_total')): + value = observed.get(source) or device.get('unified_ram_bytes') + if _timing_seconds(value): + hardware[target] = value / 2**30 + return hardware + + def _late_history(self): + """Use history the background read finished after this request began.""" + from .effort_forecast import start_context, history_read_at + if self.history is None or self.remaining_source == 'measured': return - # Within a pass's first step, a layer event extrapolates the whole step - # from the first layers, which can include compilation and read-ahead - # (one slow layer once became a 142-minute estimate). With an estimate - # already in hand, use layer-based step times only after the pass has - # completed a step; a first run without one still gets its first number. - base = self.sampling_plan.get('base_steps') - if (self.forecast_ready and message.get('stage') == 'layers' - and (done == 0 or type(base) is int and 0 < base == done < total)): + output_directory, machine, opened = self.history + if (history_read_at(output_directory, machine) or 0.) < opened: return - full_step = estimate - remaining = (total - done) * estimate - if message.get('uniform_remaining_steps') is False: - plan = self.sampling_plan - first, second = plan.get('first', {}), plan.get('second') or {} - area = first.get('width', 0) * first.get('height', 0) - if not area or not second.get('width') or not second.get('height'): - return # Do not treat the two larger refinement NFEs as small NFEs. - full_step *= second['width'] * second['height'] / area - remaining = max(0, plan['base_steps'] - done) * estimate + plan['refine_steps'] * full_step - self.sampling_rest = done, remaining - remaining = max(0., remaining - (_timing_seconds(message.get('step_elapsed_seconds')) or 0.)) - self.weights['sampling'] = max(.001, now - self.phase_started + remaining) - self.sampling_time_weighted = True - if 'decode' not in self.weights or self.decode_from_sampling: - # Windows 1344x768x243 measurements: decode/save was 1.71, 1.90 - # and 2.01 final-resolution warm NFEs on 4060 Ti / 5060 Ti / 5090. - # This is an approximate first-run ruler, not a GPU speed claim. - # It scales with this request's work, then yields to frame timing. - self.weights['decode'] = 1.9 * full_step - self.decode_from_sampling = True - self.forecast_ready = True - return True + self.history = None # One late update per request. + # Work on the live state, but restore all of it if anything fails. + saved = {key: dict(value) if isinstance(value, dict) else value for key, value in vars(self).items()} + try: + observed, rows = start_context(output_directory, machine, timeout=0.) + for key, value in self._hardware(observed, machine).items(): + if value is not None and self.hardware.get(key) is None: + self.hardware[key] = value + self.records = rows + self._reforecast() + self.remaining_target = self._target(self.clock()) + except Exception: + # Damaged history keeps the estimate the request already has. + vars(self).clear() + vars(self).update(saved) - @staticmethod - def _forecast_weights(forecast): - stages = forecast.get('stages') if isinstance(forecast, dict) else None - if not isinstance(stages, dict): - return {} - values = { - 'encoding': _timing_seconds(stages.get('text_encode_seconds')), - 'load': _timing_seconds(stages.get('load_seconds')), - 'sampling': _timing_seconds(stages.get('sample_seconds')), - 'decode': _timing_seconds(stages.get('decode_save_seconds')), - } - values = {key: value for key, value in values.items() if value is not None} - # Sparse phase forecasts expose work_seconds plus its independently - # predicted components. Keep the measured residual as the final - # save/cleanup tail instead of silently dropping it or double-counting - # the worker total. - work = _timing_seconds(stages.get('work_seconds')) - residual = work - sum(values.get(key, 0.) for key in ('load', 'sampling', 'decode')) if work is not None else 0. - if residual > 0: - values['decode'] = values.get('decode', 0.) + residual - return values + def configure(self, *, plan=None, media=None, conditioning=False, placement=None, sampled=None): + """Use facts already obtained by the bridge; never inspect media/models.""" + self._late_history() + from .media_request import task_for + if sampled: + # The engine samples this canvas, which can differ from the requested size. + self.request.update({k: sampled[k] for k in ('width', 'height', 'frames') if sampled.get(k) is not None}) + if plan: + self.sampling_plan = plan + self.request.update(two_pass=bool(plan.get('enabled')), base_steps=plan.get('base_steps', 8), + refine_steps=plan.get('refine_steps', 0), + sampling_geometry={k: plan[k] for k in ('first', 'second') if plan.get(k)}) + for key in ('pass1_resident_blocks', 'pass2_resident_blocks'): + value = plan.get(key) + if type(value) in (int, float) and math.isfinite(value) and 0 <= value <= 50: + self.request[key] = value + if placement: + # generate.py's video_start/decode_resume carry the selected full + # canvas engine/decoder plan. Pass 1 can admit extra temporary + # cache, so its residency stays unknown for a two-pass request. + engine, decoder = placement.get('engine'), placement.get('decoder') + engine = engine if isinstance(engine, dict) else {} + decoder = decoder if isinstance(decoder, dict) else {} + blocks = engine.get('resident_blocks') + if type(blocks) in (int, float) and math.isfinite(blocks) and 0 <= blocks <= 50: + key = 'pass2_resident_blocks' if self.request['two_pass'] else 'pass1_resident_blocks' + self.request[key] = blocks + blocks = decoder.get('resident_blocks') + if type(blocks) in (int, float) and math.isfinite(blocks) and 0 <= blocks <= 36: + self.request['decoder_resident_blocks'] = blocks + if type(decoder.get('offload')) is bool: + self.request['decoder_offload'] = decoder['offload'] + if media is not None: + self.request.update(task=task_for(media), lora_count=len(media.get('loras') or []), + reference_count=len(media.get('references') or []) + + sum(bool(media.get(k)) for k in ('first', 'last'))) + if conditioning: + self.cold_flags['encode_cached'] = True + self._reforecast() - @classmethod - def _forecast_parts(cls, forecast): - stages = forecast.get('stages') if isinstance(forecast, dict) else None - values = {name: _timing_seconds(stages.get(name + '_seconds')) for name in cls.DECODE_PARTS} \ - if isinstance(stages, dict) else {} - return {name: value for name, value in values.items() if value is not None} + def _reforecast(self, *, reset_source=False): + from .stage_estimate import predict_prior, STAGES + from .effort_forecast import stage_forecast + self.prior = predict_prior(self.hardware, self.request, self.cold_flags, self.params) + weights = {k: self.prior['stages'][k]['seconds'] for k in STAGES} + local = stage_forecast(self.records, self.request, self.cold_flags, weights, version=_engine_version()) + weights.update(local.get('stages', {})) + self.typical = local['typical'] + self.local = local + if reset_source or self.remaining_source != 'measured': + self.remaining_source = 'local' if local.get('stages') else 'similar' + for key, value in weights.items(): + if key not in self.completed: + self.weights[key] = value + self.forecasts[key] = value + self.decode_tail = max(0., self.weights['decode_save'] - self.prior['stages']['video_decode']['seconds']) def forecast(self, value): - value = value.get('forecast') if isinstance(value, dict) and 'forecast' in value else value - weights = self._forecast_weights(value) - if weights: - # The prediction is emitted after prompt encoding has completed. - # Close that phase before the first video-model event so its - # measured weight is credited instead of starting a second timer. - if self.phase == 'encoding': - self._close(self.clock()) - self.closed.add('encoding') - self.phase = None - self.forecasts.update(weights) - self.decode_parts = self._forecast_parts(value) - self.weights.update(weights) - if 'decode' in weights: - self.decode_from_sampling = False - self.forecast_ready = all(name in self.weights for name in ('sampling', 'decode')) - self._settle() - self.total = sum(self.weights.values()) - - def _base(self): - return sum(self.weights.get(name, 0.) for name in self.completed) + """Consume older worker forecasts without closing or crediting a phase.""" + value = value.get('forecast', value) if isinstance(value, dict) else {} + if not isinstance(value, dict): + return + stages = value.get('stages') or {} + if not isinstance(stages, dict): + return + # Decoder substeps remain useful after sampling speed is measured. + for part in self.DECODE_PARTS: + seconds = _timing_seconds(stages.get(part + '_seconds')) + if seconds is not None: + self.decode_parts[part] = seconds + # Worker predictions lack the bridge's two-record similarity evidence; + # they must never create a local typical or replace live measurements. + if self.remaining_source == 'measured': + return + mapping = {'encode': 'text_encode_seconds', 'load': 'load_seconds', 'decode_save': 'decode_save_seconds'} + for key, source in mapping.items(): + seconds = _timing_seconds(stages.get(source)) + if seconds is not None and key not in self.completed: + self.weights[key] = seconds + sample = _timing_seconds(stages.get('sample_seconds')) + if sample: + keys = [k for k in ('pass1', 'upscale', 'pass2') if k not in self.completed] + total = sum(self.weights[k] for k in keys) + if total: + for key in keys: + self.weights[key] *= sample / total + work = _timing_seconds(stages.get('work_seconds')) + parts = [_timing_seconds(stages.get(k)) for k in ('load_seconds', 'sample_seconds', 'decode_save_seconds')] + if work and all(v is not None for v in parts) and 'decode_save' not in self.completed: + self.weights['decode_save'] += max(0., work - sum(parts)) + self.forecasts.update({k: v for k, v in self.weights.items() if k not in self.completed}) - def _close(self, now): + def _enter(self, phase, now): + from .stage_estimate import STAGES + if phase == self.phase: + return + self._late_history() + if phase == 'load' and self.phase == 'upscale': + # The model handed over between the passes belongs to the gap + # between them, which is a stage of its own; a retry reloads + # through _retry instead. + return if self.phase is not None: self.spent[self.phase] = max(0., now - self.phase_started) - - def _enter(self, phase, now): - """Phases do not always come one after the other: a preset check - runs for a moment while the prompt is encoded, and an encoder event - can follow the forecast. A phase that comes back goes on from the - time it already has, and only the phases before the one now running - are finished; a later one that ran for a moment keeps its forecast.""" - self._close(now) + self.phase_counts[self.phase] = self.count + self.forecasts[self.phase] = self.weights[self.phase] + # A brief model check can interrupt encoding, and loading can return + # between passes. Only predecessors of the active phase are finished. + self.completed = (set(STAGES[:STAGES.index(phase)]) | self.reused) - {phase} + for key in STAGES: + if key in self.completed and key in self.spent: + self.measured[key] = self.weights[key] = max(.001, self.spent[key]) + elif key in self.measured: + self.measured.pop(key) + self.weights[key] = max(self.forecasts.get(key, 0.), self.spent.get(key, 0.)) self.phase, self.phase_started = phase, now - self.spent.get(phase, 0.) - self.closed.discard(phase) - self._settle() - - def _settle(self): - before = self.ORDER[:self.ORDER.index(self.phase)] if self.phase else () - self.completed = (set(before) | self.closed | self.reused) - {self.phase} - for name in self.ORDER: - seconds = self.spent.get(name) - if name in self.completed: - if seconds is not None: - self.measured[name] = self.weights[name] = max(.001, seconds) - elif name not in self.reused: - self.weights.pop(name, None) # Passed without running: nothing of it is left to wait for. - elif name in self.measured: - del self.measured[name] - self.weights[name] = max(self.forecasts.get(name, 0.), seconds or 0., .001) - - def _message_phase(self, message): - if message.get('timing_phase') in ('encoding', 'load', 'sampling', 'decode'): - return message['timing_phase'] - phase = message.get('phase') - return {'sampling': 'sampling', 'sample_finalize': 'decode', - 'complete': 'decode'}.get(phase) if isinstance(phase, str) else None - - def annotate(self, message): - message = dict(message) - if isinstance(message.get('sampling_plan'), dict): - self.sampling_plan = message['sampling_plan'] + self.count = self.phase_counts.get(phase, self._initial_count()) + self.decode_remaining = None + + def _initial_count(self): + from .stage_estimate import canvases, step_counts + if self.phase == 'load': + return (0, 50, 1) + if self.phase in ('pass1', 'pass2'): + count = step_counts(self.request)[0 if self.phase == 'pass1' else 1] + return (0, count * 50, 1) + if self.phase == 'decode_save': + return (0, canvases(self.request)[2]['frames'], 1) + return None + + def _set_count(self, done, total, margin=1): + # Duplicate/delayed logs cannot undo already witnessed counts. Using + # the smallest known unit avoids lowering the ceiling on the first + # counted event after an uncounted stage-start notification. + candidate = (max(0, done), total, margin) + old = self.count + if old is None or (candidate[0] + margin) / max(total, 1) >= (old[0] + old[2]) / max(old[1], 1): + self.count = candidate + + def _phase_for(self, message, event): + name = event.get('event') + if self.retry_decode_only: + return 'decode_save' + if name == 'decode_resume' or message.get('phase') == 'sample_finalize': + return 'decode_save' + if message.get('stage') in ('latent_upscale', 'first_pass_reused'): + return 'upscale' + phase = message.get('timing_phase') or message.get('phase') + if phase == 'encoding' and any(self.steps.values()): + # After synchronized sampling steps, encoder-file lines are late + # notifications from the finished prompt, not a new encoding run. + return None + if phase == 'sampling': + # `loaded` falsely looks like sampling 0/N in the legacy converter. + if name == 'loaded': + return 'load' + done = message.get('done', 0) + base = self.request['base_steps'] + if self.request['flow'] == 'upscale' or self.retry_reuse_first: + return 'pass2' + if self.request['flow'] != 'preview' and self.request['two_pass']: + if done > base or (done == base and (name == 'sampling_start' or message.get('stage') in ('layers', 'inputs'))): + return 'pass2' + return 'pass1' + return {'encoding': 'encode', 'load': 'load', 'decode': 'decode_save'}.get(phase) if isinstance(phase, str) else None + + def _retry(self, retry, now): + # A retry changes the plan; the earlier load no longer says how fast + # this machine is, and measured steps arrive again soon. + self.slow_start = False + self.retry_decode_only = retry.get('reuse_sampling') is True + self.retry_reuse_first = retry.get('reuse_first_pass') is True + start = 'decode_save' if self.retry_decode_only else 'load' + if not self.retry_decode_only: + self.steps['pass2'].clear() + if not retry.get('reuse_first_pass'): + self.steps['pass1'].clear() + from .stage_estimate import STAGES + self.spent = {k: v for k, v in self.spent.items() if STAGES.index(k) < STAGES.index(start)} + self.phase_counts = {} + self.decode_part, self.decode_rest = None, None + self.reused = set(STAGES[:STAGES.index(start)]) + self.completed = set(self.reused) + if self.retry_reuse_first and not self.retry_decode_only: + self.completed.add('pass1') + self.reused.add('pass1') + self.phase, self.phase_started = start, now + self.count, self.decode_remaining = self._initial_count(), None + self.live_calibration = False + # Recovery can change placement. Wait for the new attempt's plan; + # retained step durations are not new measurements of pending work. + for key in ('pass1_resident_blocks', 'pass2_resident_blocks', + 'decoder_resident_blocks', 'decoder_offload'): + self.request.pop(key, None) + self._reforecast(reset_source=True) + self.ceiling_base = self.fraction + self.ceiling_weights = {k: v for k, v in self.weights.items() if k not in self.completed} + + def _counts_and_live(self, message, event, now): + from .stage_estimate import predict_live, step_counts + phase = self.phase + done, total = message.get('done'), message.get('total') + if phase in ('pass1', 'pass2'): + index = 0 if phase == 'pass1' else 1 + count = step_counts(self.request)[index] + offset = self.request['base_steps'] if index else 0 + if type(done) is int: + local_done = max(0, min(count, done - offset)) + blocks, block = message.get('blocks'), message.get('block') + if type(blocks) is int and blocks > 0 and type(block) is int: + # One step of margin: layer events are dispatch estimates, + # steps are the synchronized counts (a first step may compile). + self._set_count(local_done * blocks + max(0, block - 1), count * blocks, + blocks if self.ceiling == 'stretch' else 1) + else: + # Keep the same one-layer margin across step boundaries; + # changing from one step to one layer could lower a ceiling. + self._set_count(local_done * 50, count * 50, 50 if self.ceiling == 'stretch' else 1) + seconds = _timing_seconds(message.get('last_step_seconds')) + if seconds and local_done > 0: + self.steps[phase][local_done] = seconds + if 1 in self.steps[phase] and 2 in self.steps[phase]: + self.live_calibration = True + elif phase == 'load': + if event.get('event') == 'loaded': + done, total = 50, 50 + elif event.get('event') == 'prepared_blocks': + done, total = event.get('blocks'), 50 + elif event.get('event') == 'model_load_phase': + done, total = event.get('done'), event.get('total') + # LoRA groups describe a substage, not completion of the whole load. + if type(done) is int and type(total) is int and total > 0 and message.get('unit') != 'bytes': + self._set_count(done, total) + elif phase == 'decode_save' and message.get('stage') == 'video': + elapsed = _timing_seconds(message.get('elapsed_seconds')) + if type(done) is int and type(total) is int and 0 <= done <= total and total > 0: + self._set_count(done, total) + if elapsed and done > 0: + self.decode_remaining = elapsed * (total - done) / done + self._after_frames(total) + self.weights[phase] = now - self.phase_started + self.decode_remaining + self.remaining_source = 'measured' + elif phase == 'decode_save' and (event.get('event') == 'decode_phase' or message.get('decode_part')): + # These real transitions finish video frames even for a decoder + # that does not stream progress callbacks. + part = message.get('decode_part') or _DECODE_PARTS.get(event.get('phase') if isinstance(event.get('phase'), str) else '') + if part in self.DECODE_PARTS and part != self.decode_part: + rest = self.DECODE_PARTS[self.DECODE_PARTS.index(part):] + self.decode_part = part + if all(k in self.decode_parts for k in rest): + self.decode_rest = sum(self.decode_parts[k] for k in rest[1:]) + self.decode_remaining = self.decode_parts[part] + self.decode_rest + self.weights[phase] = now - self.phase_started + self.decode_remaining + if part in ('audio_load_decode', 'encode'): + total = self.count[1] if self.count else self.request['frames'] + self._set_count(total, total) + if self.decode_remaining is None: + # Without frame callbacks this is the first sign that the + # video is decoded; only the measured tail remains. + self.decode_remaining = self._after_frames(total) + self.weights[phase] = now - self.phase_started + self.decode_remaining + self.remaining_source = 'measured' + # Only contiguous, actually completed steps calibrate the model. + steps = {} + for key, values in self.steps.items(): + steps[key] = [] + for i in range(1, len(values) + 1): + if i not in values: + break + steps[key].append(values[i]) + if self.live_calibration and any(len(v) >= 2 for v in steps.values()): + done_stages = [('decode' if k == 'decode_save' else k) for k in self.completed] + live = predict_live(dict(hardware=self.hardware, request=self.request, cold_flags=self.cold_flags, + pass1_steps=steps['pass1'], pass2_steps=steps['pass2'], completed_stages=done_stages, + current_stage=phase, step_elapsed_seconds=message.get('step_elapsed_seconds', 0.), + pass_wall_overhead_done=[k for k, v in steps.items() if v]), self.params) + # With matching history on this machine, scale its own stage times + # by the measured warm step instead of the general model. + from statistics import median + history, warm, ratio = self.local.get('stages') or {}, self.local.get('warm') or {}, None + for key in ('pass2', 'pass1'): + if len(steps[key]) >= 2 and warm.get(key): + ratio = min(2., max(.5, median(steps[key][1:]) / warm[key])) + break + for key in ('pass1', 'upscale', 'pass2', 'decode_save'): + if key in self.completed or (key == 'decode_save' and self.decode_remaining is not None): + continue + if ratio is not None and key in history: + self.weights[key] = history[key] * (1. if key == 'decode_save' else ratio) + continue + remaining = live['stages'][key]['seconds'] + if key == phase: + if key == 'decode_save': + # decode/save prediction is a full duration until frames arrive. + self.weights[key] = remaining + else: + self.weights[key] = now - self.phase_started + remaining + else: + self.weights[key] = remaining + self.decode_tail = max(0., live['stages']['decode_save']['seconds'] - live['stages']['video_decode']['seconds']) + self.remaining_source = 'measured' + + def _after_frames(self, frames): + # Audio, MP4 and saving after the last decoded frame: about 2 s plus + # 0.03 s per frame (measured, slightly below the median, so the bar + # rarely jumps at saving), never more than the model's tail. + if self.decode_rest is not None and self.decode_part == 'video_decode': + return self.decode_rest + return min(self.decode_tail, 2. + .03 * frames) + + def _slow_factor(self, now): + # A disk-bound 16 GB machine that loads much slower than predicted is + # slower in every later stage too (reports: about the square root of + # the load ratio). Larger memory or a slow encode does not predict it. + if not self.slow_start or self.remaining_source != 'similar': + return 1. + ram = self.hardware.get('ram_total_gib') + if ram is None or ram > 16.5: + return 1. + predicted = self.prior['stages']['load']['seconds'] + actual = self.measured.get('load') or (now - self.phase_started if self.phase == 'load' else None) + if not actual or predicted <= 1 or actual / predicted <= 2: + return 1. + return min(4., math.sqrt(actual / predicted)) + + def _target(self, now): + factor = self._slow_factor(now) + later = ('pass1', 'upscale', 'pass2', 'decode_save') + remaining = sum(v * (factor if k in later else 1.) + for k, v in self.weights.items() if k not in self.completed and k != self.phase) + elapsed = max(0., now - self.phase_started) + # The stage under way is scaled too until its speed is measured. + current = self.weights.get(self.phase, 0.) * (factor if self.phase in later else 1.) - elapsed + if current < 0: + current = max(3., .1 * elapsed) + return max(0., remaining + current) + + def _ceiling(self, now): + weights = (self.ceiling_weights if self.ceiling == 'prior' else + {k: self.weights[k] for k in self.ceiling_weights}) + stretch = self.ceiling == 'stretch' and (self.count is None or self.count[0] <= 0) + if stretch and self.phase in weights: + # An overrunning stage without counted work keeps its end ahead of + # it, by the same remainder the remaining-time target assumes. + elapsed = max(0., now - self.phase_started) + weights[self.phase] = max(weights[self.phase], elapsed + max(3., .1 * elapsed)) + total = sum(weights.values()) + if total <= 0: + if self.ceiling in ('current', 'stretch'): + self.last_ceiling = .99 + return .99 + credit = sum(v for k, v in weights.items() if k in self.completed) + # An uncounted stage earns its allowance over time, not at its entry. + # This also avoids releasing all of upscale's share at a boundary. + duration = weights.get(self.phase, 0.) + fraction = min(1., max(0., now - self.phase_started) / max(duration, .001)) + if self.count is not None and not stretch: + done, count, margin = self.count + fraction = min(1., max(0., (done + margin) / max(count, 1))) + credit += duration * fraction + value = min(.99, self.ceiling_base + (1. - self.ceiling_base) * credit / total) + if self.ceiling in ('current', 'stretch'): + self.last_ceiling = max(self.last_ceiling, value) + value = self.last_ceiling + return value + + def _smooth(self, dt): + if self.smoothing == 'track': + shown, target = self.remaining_shown, self.remaining_target + if target > shown: + # Slower than predicted: stop counting down, then rise gently. + # A time that ran out starts again from the remainder an + # overrunning stage keeps; from zero it could never rise. + shown = max(shown, min(target, 3.)) + self.remaining_shown = min(target, shown + self.rate * shown * dt) + else: + shown = max(target, shown - dt) + self.remaining_shown = max(0., shown - (shown - target) * (1. - math.exp(-dt / max(self.tau, .001)))) + return + # Hold suppresses countdown only for an upward target correction. + # Both policies approach the target without crossing it by correction; + # countdown itself can pass below it. Keep a positive running remainder. + if self.smoothing == 'hold' and self.remaining_target > self.remaining_shown: + shown = self.remaining_shown + else: + floor = 3. if self.smoothing == 'countdown' else min(3., self.remaining_shown) + shown = max(floor, self.remaining_shown - dt) + limit = self.rate * shown * dt + self.remaining_shown = max(0., shown + max(-limit, min(limit, self.remaining_target - shown))) + + def _render(self, message, now, *, heartbeat=False): + if self.remaining_source == 'local' and self.shown_source == 'similar': + # History read after the start replaces the estimate from similar + # settings: this computer's record corrects it once, so do not smooth. + # (A first measurement keeps smoothing; two steps are still noisy.) + self.remaining_shown = self.remaining_target + self.shown_source = self.remaining_source + elapsed = max(0., now - self.started) + candidate = elapsed / max(elapsed + self.remaining_shown, .001) + if (self.final is not None and self.phase == 'decode_save' and self.count + and self.count[0] >= self.count[1] > 0): + # Every frame is decoded; only audio, MP4 and saving remain, often + # for a few seconds only. Move there as fast as the rise limit allows. + candidate = max(candidate, self.final) + ceiling = self._ceiling(now) + limit = .99 + # Retain the value in force one second ago and all newer changes. + while len(self.rise_window) > 1 and self.rise_window[1][0] <= now - 1.: + self.rise_window.popleft() + if self.max_rise is not None: + base = self.rise_window[0][1] if self.rise_window else self.fraction + limit = min(limit, base + self.max_rise) + self.fraction = max(self.fraction, min(limit, candidate, ceiling)) + if not self.rise_window or self.rise_window[-1][1] != self.fraction: + if len(self.rise_window) > 1 and self.rise_window[-1][0] == now: + self.rise_window.pop() + self.rise_window.append((now, self.fraction)) + overall = dict(status='complete' if self.finished else 'running', estimated=True, + fraction=1. if self.finished else self.fraction, + remaining_seconds=0. if self.finished else self.remaining_shown, + elapsed_seconds=elapsed, remaining_source=self.remaining_source, + phase_start_fraction=self.fraction, phase_weight=0., remaining_floor_seconds=0., + at_ceiling=not self.finished and candidate >= ceiling, + sampling_time_weighted=True) + key = self.phase + if key in self.steps: + key += '_first' if not self.steps[key] else '_warm' + if key in self.typical: + overall.update(typical_seconds=self.typical[key], typical_source='local') + if heartbeat: + overall['heartbeat'] = True + self.last_publish = now + return dict(message, overall=overall) + + def annotate(self, message, *, event=None): + message, event = dict(message), event or {} + now = self.clock() + if self.finished: + return self._render(message, now) + dt = max(0., now - self.last_tick) + old_phase, old_part = self.phase, self.decode_part + waiting_for_pass = (self.phase in self.steps and self.count is not None + and self.count[0] >= self.count[1] > 0) + plan = event if event.get('event') == 'sampling_plan' else message.get('sampling_plan') + if isinstance(plan, dict): + self.configure(plan=plan) if isinstance(message.get('prediction'), dict): self.forecast(message['prediction']) - retry = message.get('retry') if isinstance(message.get('retry'), dict) else {} - if retry.get('reuse_sampling') is True: - self.reused.add('sampling') - self.completed.add('sampling') - phase = self._message_phase(message) - now = self.clock() - if phase and phase != self.phase: + retried = isinstance(message.get('retry'), dict) or message.get('stage') == 'preview_mismatch' + if isinstance(message.get('retry'), dict): + self._retry(message['retry'], now) + if message.get('stage') == 'preview_mismatch': + self.request['flow'] = 'full' + self._retry({}, now) + if event.get('event') in ('video_start', 'decode_resume'): + self.configure(placement=event) + phase = self._phase_for(message, event) + if phase: self._enter(phase, now) - if phase == 'sampling': - self._sampling_estimate(message, now) - if phase == 'decode' and message.get('stage') == 'video': - done, total = message.get('done'), message.get('total') - elapsed = _timing_seconds(message.get('elapsed_seconds')) - if elapsed and type(done) is int and type(total) is int and 0 < done <= total: - video_seconds = elapsed * total / done - # Reports put audio/MP4/post-decode cleanup at about 6-8% of - # video decode time. Retain this tail until audio/save starts. - self.decode_tail = max(video_seconds * .08, self.decode_rest or 0.) - self.weights['decode'] = now - self.phase_started + video_seconds - elapsed + self.decode_tail - self.forecast_ready = True - elif phase == 'decode': - self.decode_tail = 0. - part = message.get('decode_part') - rest = self.DECODE_PARTS[self.DECODE_PARTS.index(part):] if part in self.DECODE_PARTS else () - if rest and all(name in self.decode_parts for name in rest): - # The decoder says which of its steps begins: what is left of - # the phase is this step and the ones after it, not the whole - # forecast made before the run. - self.decode_rest = sum(self.decode_parts[name] for name in rest[1:]) - self.weights['decode'] = now - self.phase_started + self.decode_parts[part] + self.decode_rest + if phase in self.steps and phase != old_phase and phase not in self.spent: + # A first layer reports time inside its step, not the model + # handover before it. Backdate only that witnessed work. + inside = _timing_seconds(message.get('step_elapsed_seconds')) or 0. + self.phase_started = now - inside + if waiting_for_pass: + dt = min(dt, inside) + if event.get('event') == 'conditioning_cache_hit': + self.cold_flags['encode_cached'] = True + if event.get('event') == 'conditioning_reused': + self.cold_flags['encode_cached'] = True + if event.get('event') == 'encoder_device_reuse': + self.cold_flags['encoder_cold'] = False + if event.get('event') == 'compute_device': + self.hardware.update(gpu_name=event.get('name') or self.hardware.get('gpu_name')) + if _timing_seconds(event.get('vram_total_bytes')): + self.hardware['vram_total_gib'] = event['vram_total_bytes'] / 2**30 + self._reforecast() + if not retried: + self._counts_and_live(message, event, now) + self.remaining_target = self._target(now) + self._smooth(dt) + if retried: + # A new attempt is estimated afresh: the time left of the attempt + # that failed says nothing about it, and may already have run out. + self.remaining_shown = self.remaining_target + if self.phase == 'decode_save' and self.decode_rest is not None and ( + self.decode_part != old_part or message.get('stage') == 'video'): + # A decoder boundary gives the pending substeps directly; the + # fraction still obeys its monotonic, per-second rise limit. + self.remaining_shown = self.remaining_target + self.last_tick = self.last_event = now self.total = sum(self.weights.values()) - if message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete': - message['overall'] = {'status': 'complete', 'fraction': 1., 'estimated': False, - 'elapsed_seconds': now - self.started, 'remaining_seconds': 0.} - return message - overall = {'status': 'running', 'estimated': self.forecast_ready, - 'elapsed_seconds': now - self.started} - if self.forecast_ready and self.total > 0: - weight = self.weights.get(self.phase, 0.) - elapsed = max(0., now - self.phase_started) - seconds = self.weights.get(self.phase) - remaining = max(0., self.total - self._base() - min(elapsed, seconds or 0.)) - future = sum(value for name, value in self.weights.items() - if name not in self.completed and name != self.phase) - total = self.total - if weight and elapsed > .95 * weight and .95 * future > .05 * self._base(): - # A phase that outruns its forecast (a model read from a cold - # disk) is given the time it has taken, so the fraction keeps - # rising instead of standing at the forecast's end; it does so - # only while enough of the run is still ahead for that to - # raise it. The time left is unchanged: what follows is. - seconds = elapsed / .95 - total += seconds - weight - weight = seconds - start = self._base() / total - fraction = start - if weight: - fraction += weight / total * min(.95, elapsed / max(seconds, 1e-9)) - # What has been given out is not taken back. A phase that - # returns (a model loaded again between the passes), a decoder - # step that ran past its forecast and a retry all add time - # left; the fraction waits for the run to catch up with it. - fraction = self.shown = max(self.shown, fraction) - overall.update(fraction=fraction, phase_start_fraction=start, - phase_weight=weight / total, - phase_elapsed_seconds=elapsed, phase_seconds=seconds, - remaining_seconds=remaining, - remaining_floor_seconds=max(future, self.decode_tail), - sampling_time_weighted=self.sampling_time_weighted) - message['overall'] = overall - return message + self.finished = message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete' + if self.finished: + self.measured[self.phase] = max(.001, now - self.phase_started) + self.weights[self.phase] = self.measured[self.phase] + self.total = sum(self.weights.values()) + self.completed.update(self.weights) + self.remaining_target = self.remaining_shown = 0. + self.fraction = 1. + if message.keys() - {'report_id'}: + self.last_message = {k: v for k, v in message.items() if k not in ('overall', 'reset', 'new_request', 'prediction', 'sampling_plan')} + else: + message = dict(self.last_message, **message) + return self._render(message, now) + + def observe(self, event): + """Use an engine event that has no progress message, without publishing.""" + published, window = self.last_publish, self.rise_window.copy() + self.annotate({}, event=event) + self.last_publish, self.rise_window = published, window + + def heartbeat(self): + now = self.clock() + if self.finished or now - self.last_publish < 1.: + return None + if (self.phase in self.steps and self.count is not None + and self.count[0] >= self.count[1] > 0): + # Between passes no pending step time is being spent. + self.last_tick = now + elif self.silence == 'stage': + # Admit only the part of this interval inside the predicted stage. + # Crossing its end freezes at the boundary, even with sparse polls. + end = self.phase_started + self.weights.get(self.phase, 0.) + dt = max(0., min(now, end) - max(self.last_tick, self.phase_started)) + if dt: + self.remaining_target = max(0., self.remaining_target - dt) + self._smooth(dt) + self.last_tick = max(self.last_tick, min(now, end)) + # Silent time is not counted down until the next engine event shows + # the engine is still working; that event then counts all of it. + return self._render(self.last_message, now, heartbeat=True) def source_root(): @@ -550,10 +1020,7 @@ def progress_message(event): 'lora_prepare_start': 'Loading LoRAs', 'resource_retry': 'Adjusting memory placement and retrying'} if name == 'decode_phase': - message = {'label': str(event.get('phase', 'Decoding video and audio')), 'timing_phase': 'decode'} - if isinstance(event.get('phase'), str) and event['phase'] in _DECODE_PARTS: - message['decode_part'] = _DECODE_PARTS[event['phase']] - return message + return {'label': str(event.get('phase', 'Decoding video and audio')), 'timing_phase': 'decode'} if name == 'decode_progress': return dict(label='Decoding the video', timing_phase='decode', phase='decode', stage='video', done=event.get('done'), total=event.get('total'), unit='frames', @@ -782,12 +1249,22 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, state['encoder_prewarm'] = prewarm_summary(encoder_prewarm) save(run / 'comfy-request.json', state) started = time.monotonic() - whole_progress = WholeVideoProgress() + try: + whole_progress = WholeVideoProgress.for_request(machine, canvas, output_directory, + base_steps=base_steps, refine_steps=refine_steps, two_pass=two_pass, + preview=preview, upscale=bool(upscale), encoder_prewarm=state['encoder_prewarm']) + except Exception: + # A progress estimate cannot prevent generation, even without its parameters. + try: + whole_progress = WholeVideoProgress() + except Exception: + whole_progress = _PlainProgress() + whole_progress = _SafeProgress(whole_progress) from .comfy_progress import REPORTS report_id = REPORTS.register(output) - def send_progress(message): + def send_progress(message, event=None): if progress: - progress(whole_progress.annotate(dict(message, report_id=report_id))) + progress(whole_progress.annotate(dict(message, report_id=report_id), event=event)) process = None from .result_cache import ResultCache, request_key, inspect_bounded cache = ResultCache(output_directory) @@ -802,8 +1279,14 @@ def poll(): tails[2].reset() message = progress_message(event) if message: - send_progress(message) + send_progress(message, event) + else: + whole_progress.observe(event) + heartbeat = whole_progress.heartbeat() + if progress and heartbeat: + progress(heartbeat) try: + send_progress({'label': 'Preparing video'}) if interrupted: interrupted() extra = {} @@ -889,10 +1372,9 @@ def asset_progress(message): state['sampling_cache_install'] = preparation if preparation['seconds']: started += preparation['seconds'] - whole_progress = WholeVideoProgress() send_progress({'reset': True, 'label': 'Preparing video'}) - whole_progress.sampling_plan = planned - whole_progress.forecast(progress_history_forecast(root, machine, dict(sampled, sampling_plan=planned))) + whole_progress.configure(plan=planned, media=extra.get('media', {}), + conditioning=bool(extra.get('conditioning')), sampled=sampled) send_progress({'label': 'Preparing %.3f s video + audio · %d × %d' % (canvas['seconds'], width, height)}) if release_models: diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.py index fb0cd9b5..32126c37 100644 --- a/freevideo_engine/comfy_progress.py +++ b/freevideo_engine/comfy_progress.py @@ -83,7 +83,18 @@ def publish(self, node, message): old = self.rows.get(node) metadata_only = ('compatibility' in value and not value.get('new_request') and not any(key in value for key in ('phase', 'stage', 'timing_phase', 'label', 'reset'))) - if old and metadata_only: + # A heartbeat carries a newer estimate, not engine progress (Harper's + # bridge repeats the last message's fields with a fresh overall). + heartbeat = isinstance(value.get('overall'), dict) and value['overall'].get('heartbeat') is True + overall_measured = now + if old and heartbeat: + # Replace only overall: the row and its age keep describing the + # last engine message, so a reopened page still sees a quiet engine. + value = dict(old['message'], overall=value['overall']) + value.pop('new_request', None) + value.pop('reset', None) + measured = old['measured'] + elif old and metadata_only: # A settings notice is not engine progress. Keep the current # activity and its timestamp, including in reopened snapshots. value = dict(old['message'], compatibility=value['compatibility']) @@ -91,11 +102,11 @@ def publish(self, node, message): value.pop('reset', None) if isinstance(value.get('overall'), dict) and 'reset' in value['overall']: value['overall'] = {key: item for key, item in value['overall'].items() if key != 'reset'} - measured = old['measured'] + measured, overall_measured = old['measured'], old.get('overall_measured', old['measured']) elif old and value.get('warning'): # A memory warning must not replace the current sampling step. value = dict(old['message'], warning=value['warning']) - measured = old['measured'] + measured, overall_measured = old['measured'], old.get('overall_measured', old['measured']) else: measured = now # The page starts a new request on these too and clears its notes. @@ -134,16 +145,22 @@ def publish(self, node, message): self._record_timeline(node, value, message, now, measured) self.sequence += 1 value.update(node=node, stream_id=self.stream_id, sequence=self.sequence) - self.rows[node] = {'message': value, 'measured': measured} + self.rows[node] = {'message': value, 'measured': measured, 'overall_measured': overall_measured} while len(self.rows) > 32: oldest = min(self.rows, key=lambda key: self.rows[key]['message']['sequence']) del self.rows[oldest] - return dict(value, age_seconds=max(0., now - measured)) + return self._aged(value, now, measured, overall_measured) + + @staticmethod + def _aged(message, now, measured, overall_measured): + # age_seconds: since the last engine message; overall_age_seconds: since + # the estimate in `overall` (a heartbeat refreshes only that one). + return dict(message, age_seconds=max(0., now - measured), overall_age_seconds=max(0., now - overall_measured)) def snapshot(self): with self.lock: now = self.clock() - return [dict(deepcopy(row['message']), age_seconds=max(0., now-row['measured'])) + return [self._aged(deepcopy(row['message']), now, row['measured'], row.get('overall_measured', row['measured'])) for row in self.rows.values()] diff --git a/freevideo_engine/effort_forecast.py b/freevideo_engine/effort_forecast.py index a94108ca..71d959b8 100644 --- a/freevideo_engine/effort_forecast.py +++ b/freevideo_engine/effort_forecast.py @@ -10,6 +10,7 @@ import platform from pathlib import Path from statistics import median +import threading import time @@ -48,7 +49,8 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True): if len(attempts) > 1: return None passes = video.get('sampling_passes') if plan.get('enabled') else [video] - if not isinstance(passes, list) or len(passes) != (2 if plan.get('enabled') else 1): + preview = bool(report.get('preview') or plan.get('preview')) + if not isinstance(passes, list) or len(passes) != (2 if plan.get('enabled') and not preview else 1): return None measured = [] for part in passes: @@ -61,7 +63,7 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True): if sum(p['seconds'] for p in measured) > total * 1.01: return None base = plan.get('base_steps', video.get('config', {}).get('steps', 8)) - refine = plan.get('refine_steps', 0) if plan.get('enabled') else 0 + refine = plan.get('refine_steps', 0) if plan.get('enabled') and not preview else 0 if measured[0]['steps'] != base or (refine and measured[1]['steps'] != refine): return None encoding = report.get('encoding') or {} @@ -118,7 +120,7 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True): version=version if isinstance(version, str) and len(version) <= 64 else None, in_use=in_use, reference_rows=reference_rows, - flow='preview' if report.get('preview') or plan.get('preview') else 'full', stages=stages, + flow='preview' if preview else 'full', stages=stages, sampling_geometry={k: plan[k] for k in ('first', 'second') if plan.get(k)}, references=task_has_references(video.get('config', {}).get('task', 't2va')), model_resident=video.get('resident_engine_cache_hit')) @@ -130,7 +132,7 @@ def estimate(records, canvas, *, base_steps=8, refine_steps=3, two_pass=True, ta validate_steps(base_steps, refine_steps, two_pass) candidates = [] for row in records: - if (row['task'] != task or row['adapters'] != adapters + if (row.get('flow', 'full') != 'full' or row['task'] != task or row['adapters'] != adapters or row['shape'] != {k: canvas[k] for k in ('width', 'height', 'frames')}): continue # Both passes retain their own resolution costs. Never multiply the @@ -319,6 +321,76 @@ def _stage_forecast(records, request, cold_flags, prior): return result +_START_CONTEXT = {} +# Guards the entries above: only one background read per key, and readers +# always see a complete hardware/records pair. File IO stays outside it. +_START_LOCK = threading.Lock() + + +def _context_key(output_directory, machine): + device = machine.get('device_identity') if machine.get('device_backend') == 'mps' else machine.get('gpu_uuid') + system = machine.get('system') or platform.system() + return (str(output_directory), json.dumps(device, sort_keys=True), system, machine.get('setup_run')), device, system + + +def start_context(output_directory, machine, *, timeout=.035): + """Bound foreground IO to ~35 ms, even on slow/unavailable output disks. + + One daemon per key reads in the background. Its last result stays in use + while a newer read runs (at most one every 15 s), so a request that starts + minutes after the previous one still sees that run. A request that started + before a read finished can pick it up later (history_read_at). + No GPU calls, resident RPC, model scans, hashing or directory-wide cache + inspection. Installation inventory describes identity, not live free RAM. + """ + from threading import Event, Thread + key, device, system = _context_key(output_directory, machine) + start = False + with _START_LOCK: + entry = _START_CONTEXT.get(key) + if entry is None: + entry = dict(ready=Event(), started=None, read_at=None, hardware={}, records=[], reading=False) + _START_CONTEXT[key] = entry + now = time.monotonic() + if not entry['reading'] and (entry['started'] is None or now - entry['started'] > 15): + entry['reading'], entry['started'], start = True, now, True + if start: + def read(): + hardware, records = dict(entry['hardware']), list(entry['records']) + try: + if machine.get('setup_run'): + with (Path(machine['setup_run']) / 'plan.json').open('rb') as stream: + raw = stream.read(1024 * 1024 + 1) + if len(raw) <= 1024 * 1024: + found = json.loads(raw).get('inventory', {}).get('hardware', {}) + if same_device(found, device): + hardware = {k: found[k] for k in + ('gpu_name', 'capability', 'ram_total', 'vram_total', 'system') if k in found} + except (OSError, ValueError, TypeError, AttributeError): + pass + try: + records = local_records(output_directory, device, system=system, stage_details=True) + except (OSError, ValueError, TypeError, AttributeError): + pass + finally: + with _START_LOCK: + entry['hardware'], entry['records'] = hardware, records + entry['read_at'], entry['reading'] = time.monotonic(), False + entry['ready'].set() + Thread(target=read, name='freevideo-progress-history', daemon=True).start() + if timeout and not entry['ready'].is_set(): + entry['ready'].wait(timeout) + with _START_LOCK: + return dict(entry['hardware']), list(entry['records']) + + +def history_read_at(output_directory, machine): + """Monotonic time the last background read finished, or None.""" + with _START_LOCK: + entry = _START_CONTEXT.get(_context_key(output_directory, machine)[0]) + return entry and entry['read_at'] + + def local_records(output_directory, gpu_uuid, *, system=None, stage_details=False, refresh=False): """Bounded recent JSON reads, cached between slider changes; never open MP4s.""" system = system or platform.system() @@ -338,7 +410,10 @@ def local_records(output_directory, gpu_uuid, *, system=None, stage_details=Fals except OSError: continue records = [] - for _, path in sorted(paths, reverse=True)[:48]: + # Slider estimates keep their original window of the 48 newest files; the + # progress history reads on until it has 48 usable rows. + candidates = sorted(paths, reverse=True) + for _, path in (candidates if stage_details else candidates[:48]): try: with path.open('rb') as stream: data = stream.read(4 * 1024 * 1024 + 1) @@ -347,6 +422,8 @@ def local_records(output_directory, gpu_uuid, *, system=None, stage_details=Fals row = timing_record(json.loads(data), gpu_uuid, system, stage_details=stage_details) if row: records.append(row) + if stage_details and len(records) == 48: + break except (OSError, ValueError, TypeError, AttributeError, KeyError): continue if not stage_details: diff --git a/freevideo_engine/progress_timeline.py b/freevideo_engine/progress_timeline.py index bb2bab87..7fb3870e 100644 --- a/freevideo_engine/progress_timeline.py +++ b/freevideo_engine/progress_timeline.py @@ -290,7 +290,7 @@ SEGMENT_KEYS = ('phase', 'stage', 'timing_phase', 'label_key') ENUMS = dict(phase=PHASES, stage=STAGES, timing_phase=TIMING_PHASES, label_key=LABEL_KEYS, unit=UNITS, warning=WARNINGS) -OVERALL_ENUMS = dict(status=STATUSES) +OVERALL_ENUMS = dict(status=STATUSES, remaining_source=frozenset(('local', 'similar', 'measured', 'other'))) RETRY_ENUMS = dict(kind=RETRY_KINDS, failed_phase=PHASES) COUNTERS = frozenset(('done', 'total', 'block', 'blocks', 'reference_trimmed')) FLAGS = frozenset(('low_memory', 'kernel_cache_note', 'estimated', 'new_request', 'reset', 'result')) @@ -349,7 +349,7 @@ def _fields(value, *, strict=False, nested=None): (nested is None and (key in COUNTERS or key in FLAGS or key in SECONDS or key in FRACTIONS or key in ('t', 'bytes_per_second', 'overall', 'retry'))) or (nested == 'overall' and (key in OVERALL_SECONDS or key in OVERALL_FRACTIONS - or key in ('estimated', 'sampling_time_weighted'))) or + or key in ('estimated', 'sampling_time_weighted', 'at_ceiling'))) or (nested == 'retry' and key in ('attempt', 'max_attempts', 'reuse_sampling'))) if not known: # Heartbeat and the bridge's extra overall fields are intentionally omitted. @@ -370,8 +370,9 @@ def _fields(value, *, strict=False, nested=None): raise ValueError('Invalid progress mapping') result[key], count = _fields(raw, strict=strict, nested=key) errors += count - elif (nested is None and key in FLAGS) or key in ('estimated', 'reuse_sampling', 'sampling_time_weighted'): - if (strict or key == 'sampling_time_weighted') and type(raw) is not bool: + elif (nested is None and key in FLAGS) or key in ('estimated', 'reuse_sampling', 'sampling_time_weighted', + 'at_ceiling'): + if (strict or key in ('sampling_time_weighted', 'at_ceiling')) and type(raw) is not bool: raise ValueError('Invalid progress flag') result[key] = bool(raw) elif (nested is None and key in COUNTERS) or key in ('attempt', 'max_attempts'): diff --git a/freevideo_engine/stage_estimate.py b/freevideo_engine/stage_estimate.py index c06f1cbd..5b5dded9 100644 --- a/freevideo_engine/stage_estimate.py +++ b/freevideo_engine/stage_estimate.py @@ -124,6 +124,11 @@ def canvases(r): first = dict(full, width=32*uw*m, height=32*uh*m) else: first = dict(full) + # two_pass.plan() runs a first pass under SMALL_FIRST_PASS_PIXELS (138,240) as one pass at the + # target size, except when the first pass itself is wanted (a preview or the upscale of one). + if (not sg.get('first') and two and r.get('flow') not in ('preview', 'upscale') + and first != full and first['width']*first['height'] < 138240): + return full, full, full second = dict(full, **(sg.get('second') or {})) decode = first if r.get('flow') == 'preview' else full return first, second, decode diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index 4f041164..09f68fbe 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -1,7 +1,7 @@ // The conversation with the connected agent: what the user says, what the agent answers, and the agent's jobs as // cards at the place they happened. Reads and sends use the page session; the agent's token never enters the page. import {agentGet, agentPost} from './agent_session.js'; -import { estimatedFraction, remainingShown } from './generation_progress.js'; +import { estimatedFraction, remainingAbove, remainingShown } from './generation_progress.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./autopilot_chat.css', import.meta.url).href; document.head.append(css); @@ -34,6 +34,28 @@ export function jobEstimate(job, age) { // again: the rest of the way is drawn shorter, so the level neither waits nor goes back, and is full at the end. const DRIFT = .25, RISE = .0005; export const newLevel = () => ({value: 0, top: null, since: 0, scale: 1}); +// The time left a running job shows, by the creation panel's rule: once measured never raised, and gone once its last +// minute has passed. `row` is what the job showed before (none yet: it starts at the engine's number); `seconds` is null +// from the moment the time has run out. +export function jobLeft(row, sent, source, time) { + if (!row) { + if (!(sent > 0)) return null; + row = {seconds: sent, at: time, minute: null, last: 0, source: null, above: 0}; + } + if (row.seconds === null) return row; + // Before the run is measured the engine's number is an estimate from earlier runs: follow it up as well as down. + const named = typeof source === 'string', initial = source === 'similar'; + // The first number that is not that estimate replaces it once, up or down, and its last minute starts over. + if (named && !initial && row.source === 'similar' && sent > 0) { row.seconds = sent; row.minute = null; } + if (named) row.source = source; + const target = sent > 0 ? sent : null; row.above = remainingAbove(row.above, row.seconds, target, time - row.at); + row.seconds = remainingShown(row.seconds, target, time - row.at, initial, row.above); row.at = time; + if (row.seconds < 60 && row.minute === null) { row.minute = time; row.last = row.seconds; } + // An estimate that rises out of its last minute starts that minute over. + else if (row.seconds >= 60 && initial) row.minute = null; + if (row.seconds <= 0 || row.minute !== null && time - row.minute >= row.last) row.seconds = null; + return row; +} export function jobLevel(row, base, left, time) { if (row.top === null || base > row.top + RISE) { if (row.top !== null && row.top < 1) row.scale = (1 - row.value) / (1 - row.top); @@ -140,21 +162,13 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () if (job.status === 'running') jobLevel(row.level, jobEstimate(row.job, time - row.at), row.job.progress?.remaining_seconds, time); return row.level.value; } - // The time left each job shows, by the creation panel's rule: never raised, and gone once its last minute has passed. const times = new Map(); function leftOf(job) { - const time = performance.now() / 1000, sent = job.progress?.remaining_seconds; if (job.status !== 'running') { times.delete(job.id); return null; } - let row = times.get(job.id); - if (!row) { - if (!(sent > 0)) return null; - times.set(job.id, row = {seconds: sent, at: time, minute: null, last: 0}); - } - if (row.seconds === null) return null; - row.seconds = remainingShown(row.seconds, sent > 0 ? sent : null, time - row.at); row.at = time; - if (row.seconds < 60 && row.minute === null) { row.minute = time; row.last = row.seconds; } - if (row.seconds <= 0 || row.minute !== null && time - row.minute >= row.last) row.seconds = null; - return row.seconds; + const row = jobLeft(times.get(job.id), job.progress?.remaining_seconds, + job.progress?.estimate?.overall?.remaining_source, performance.now() / 1000); + if (row) times.set(job.id, row); + return row ? row.seconds : null; } const active = () => !stopped && element.isConnected && !document.hidden && shown(); diff --git a/web/generation_progress.js b/web/generation_progress.js index 2fee372b..bb281d42 100644 --- a/web/generation_progress.js +++ b/web/generation_progress.js @@ -216,17 +216,24 @@ export function referenceTrimText(row, t) { `${name} 比生成的视频长,已截取前 ${used} 秒${rule}`); } -// The time left on show `dt` seconds on, given the engine's newest estimate (null when it sent none). Never raised: -// an estimate at or above the shown time holds it, so a slower run does not count down to "less than a minute" -// early. A lower one is eased into. -export function remainingShown(shown, target, dt) { +// The time left on show `dt` seconds on, given the engine's newest estimate (null when it sent none). Once measured, an +// estimate at or above the shown time holds it, so a slower run does not count down to "less than a minute" +// early; one that has stayed well above it for a minute (`above` seconds so far) raises it gently, or a run +// measured much slower would leave a low number standing for minutes. A lower one is eased into. An initial +// estimate (made before this run was measured) is followed both ways: a slow machine must not keep a low number. +export function remainingShown(shown, target, dt, initial = false, above = 0) { if (target === null) return Math.max(0, shown - dt); + if (initial) return shown + (target - shown) * (1 - Math.exp(-dt / 15)); if (target < shown - dt) { const counted = Math.max(0, shown - dt); return counted + (target - counted) * (1 - Math.exp(-dt / 15)); } - return target < shown ? target : shown; + if (target < shown) return target; + return above >= 60 ? Math.min(target, shown + Math.max(1, shown * .003) * dt) : shown; } +// The seconds the engine's estimate has stayed well above the shown time (a quarter more), `dt` seconds on. +export const remainingAbove = (above, shown, target, dt) => + target !== null && target > Math.max(shown * 1.25, shown + 30) ? above + dt : 0; // The whole-video fraction the engine's last message stands for now: its estimate moved on by the seconds since the // step sample (stateAge) and the whole-video sample (overallAge) arrived. Null when the message carries none. export function estimatedFraction(state, overall, stateAge, overallAge) { @@ -294,6 +301,13 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f // Mac: the unified-memory notice of this request, and whether its first pass has ended. let memoryTight = null, firstPassOver = false; let state = {}, overall = {}, started = 0, sampledAt = 0, overallAt = 0, timer = null, elapsedShown = 0; + // Where the bridge's remaining time comes from: 'similar' (no record of these + // settings here yet), 'local' (this computer's record) or 'measured' (this run). + // Absent from older bridges, which keep the two-steps rule. + let estimateSource = null, etaChangedAt = -Infinity; + // The page shows its own "preparing" row at the click; the engine's first + // message for that request follows and must not restart the elapsed time. + let placeholder = false, elapsedLead = 0; let lastProgressAt = 0, previousProgress = null, segmentStartedAt = 0, segmentKey = ''; let pauseAnnounced = false, lastExplanation = '', announcedMessages = new Set(); let announcementRecord = null; @@ -301,7 +315,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f let completedSteps = 0, previousRemaining = null, remainingAt = 0, lastActivity = null; // Once the shown time reaches zero before the video is saved, it stays hidden for this request. // The last minute runs on real time even while an estimate holds the number. - let remainingRetired = false, lastMinuteAt = null, lastMinute = 0; + let remainingRetired = false, lastMinuteAt = null, lastMinute = 0, aboveSeconds = 0; let shownFraction = null; // The visible percentage counts toward its value with the bar instead of // jumping. Accessible values are set exactly and immediately elsewhere. @@ -600,7 +614,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f } // Preserve the server's heartbeat/floor policy for the raw target only. - // The displayed estimate has its own clock and never increases. + // The displayed estimate has its own clock; once it is no longer initial it never increases. function rawRemaining() { if (!valid(overall.remaining_seconds)) return null; return hasHeartbeat ? overall.remaining_seconds : Math.max( @@ -608,8 +622,25 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f overall.remaining_seconds - Math.max(0, (now() - overallAt) / 1000)); } function observeRemaining(message) { - if (completedSteps < 2 || remainingSeconds !== null || remainingRetired || !message.overall) return; + if (!message.overall) return; + const source = typeof message.overall.remaining_source === 'string' ? message.overall.remaining_source : null; const raw = valid(message.overall.remaining_seconds) ? rawRemaining() : null; + if (source) { + // The bridge estimates the whole request from the first message: + // show it at once. Leaving the initial (similar-settings) estimate replaces + // the number once, up or down; after that it only falls. + const leaving = estimateSource === 'similar' && source !== 'similar'; + estimateSource = source; + // The correction starts the last minute over. Once the shown time has run + // out it stays hidden for this attempt, as without an initial estimate. + if (leaving && raw !== null && !remainingRetired) { lastMinuteAt = null; lastMinute = 0; } + if (raw === null || remainingRetired) return; + // A retry's fresh estimate still waits for the 5 s spacing of the shown + // minutes; only a new request starts the line over (in update()). + if (remainingSeconds === null || leaving) { remainingSeconds = raw; remainingAt = now(); } + return; + } + if (completedSteps < 2 || remainingSeconds !== null || remainingRetired) return; // Compare distinct received messages, never repeated timer ticks. // Zero is stable only against another zero. if (raw !== null && previousRemaining !== null @@ -620,7 +651,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f } function resetRemaining() { completedSteps = 0; previousRemaining = null; remainingSeconds = null; remainingAt = now(); - remainingRetired = false; lastMinuteAt = null; lastMinute = 0; + remainingRetired = false; lastMinuteAt = null; lastMinute = 0; aboveSeconds = 0; } function tick() { if (element.hidden) { drawStall(0, true); return; } @@ -631,12 +662,16 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f // The page counts from its first message until the engine's own elapsed time // arrives; that switch never moves the shown time back within a request. const seconds = elapsedShown = Math.max(elapsedShown, - valid(overall.elapsed_seconds) ? overall.elapsed_seconds + overallAge : (now()-started)/1000); + valid(overall.elapsed_seconds) ? overall.elapsed_seconds + overallAge + elapsedLead : (now()-started)/1000); elapsed.textContent = `${t('Elapsed', '已用时')} ${duration(Math.max(0, seconds))}`; if (ended) remainingSeconds = null; if (remainingSeconds !== null) { - remainingSeconds = remainingShown(remainingSeconds, rawRemaining(), Math.max(0, (now() - remainingAt) / 1000)); + const sent = rawRemaining(), passed = Math.max(0, (now() - remainingAt) / 1000); + aboveSeconds = remainingAbove(aboveSeconds, remainingSeconds, sent, passed); + remainingSeconds = remainingShown(remainingSeconds, sent, passed, estimateSource === 'similar', aboveSeconds); if (remainingSeconds < 60 && lastMinuteAt === null) { lastMinuteAt = now(); lastMinute = remainingSeconds; } + // An initial estimate may rise again: its last minute starts over. + else if (remainingSeconds >= 60 && estimateSource === 'similar') lastMinuteAt = null; // A run slower than estimated would leave "less than a minute" up for minutes: // once that minute has passed, or the time reaches zero, stop showing it. if (!ended && (remainingSeconds <= 0 @@ -646,9 +681,14 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f } remainingAt = now(); eta.hidden = remainingSeconds === null; - eta.textContent = remainingSeconds === null ? '' : remainingSeconds >= 60 + const etaText = remainingSeconds === null ? '' : remainingSeconds >= 60 ? t(`About ${Math.ceil(remainingSeconds / 60)}\u00a0min left`, `预计还需约 ${Math.ceil(remainingSeconds / 60)}\u00a0分钟`) : t('Less than a minute left', '预计还需不到 1\u00a0分钟'); + const initial = remainingSeconds !== null && estimateSource === 'similar'; + // While it follows the bridge, the shown minutes change at most every 5 s. + if (eta.textContent !== etaText && (!initial || !eta.textContent || now() - etaChangedAt >= 5000)) { + eta.textContent = etaText; etaChangedAt = now(); + } elapsed.hidden = false; times.hidden = elapsed.hidden && eta.hidden; drawStall(age, ended); drawActivity(age, ended); @@ -662,8 +702,21 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f // not a stage change or something the user needs to act on. if (message.warning === 'ram_budget_warning' && !message.phase) return; const reset = newRequest(message); - if (element.hidden || reset) { - state = {}; overall = {}; started = now(); overallAt = now(); shownFraction = null; elapsedShown = 0; + const placeholderReset = element.hidden || reset; + const terminal = ['complete', 'failed', 'cancelled'].includes(message.phase); + if (placeholderReset) { + // The engine's first message after the page's own placeholder is the + // same click: keep counting from it. The placeholder lasts from this + // node's own start until any engine message, its end, or hide(). + const sameClick = placeholder && message.stream_id !== undefined; + // From then on the engine's elapsed time is offset by the wait before it began. + const firstAt = valid(message.received_at) ? message.received_at : now(); + elapsedLead = sameClick ? Math.max(0, (firstAt - started) / 1000 + - (valid(message.overall?.elapsed_seconds) ? message.overall.elapsed_seconds : 0)) : 0; + if (!sameClick) { started = now(); elapsedShown = 0; } + placeholder = message.stream_id === undefined && !terminal; + state = {}; overall = {}; overallAt = now(); shownFraction = null; estimateSource = null; + eta.textContent = ''; etaChangedAt = -Infinity; lastProgressAt = valid(message.received_at) ? message.received_at : now(); previousProgress = null; segmentKey = ''; segmentStartedAt = lastProgressAt; pauseAnnounced = false; lastExplanation = ''; announcedMessages.clear(); @@ -674,21 +727,32 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f samplingPlan = {}; secondPass = false; hasHeartbeat = false; announcementRecord = null; } + if (!placeholderReset && (message.stream_id !== undefined || terminal)) placeholder = false; const record = announcementsFor(message, reset); if (record) { announcementRecord = record; for (const text of record.messages) announcedMessages.add(text); } const receivedAt = valid(message.received_at) ? message.received_at : now(); - const heartbeat = message.overall?.heartbeat === true; + // The estimate in `overall` can be newer than the engine message it rides + // on: a heartbeat refreshes only that (overall_received_at, from publish). + const overallReceivedAt = valid(message.overall_received_at) ? message.overall_received_at : receivedAt; + const beat = message.overall?.heartbeat === true; + // A heartbeat repeats the last engine message. A page that has not seen + // that message yet (reopened, or opened mid-run) takes it as the message. + const heartbeat = beat && previousProgress !== null; if (message.reset === true) { - hasHeartbeat = false; resetRemaining(); lastActivity = null; + hasHeartbeat = false; lastActivity = null; + // A stage restarting its own count (the sampler's start) leaves the + // bridge's whole-request estimate standing; a retry starts afresh. + if (estimateSource && !message.retry && message.phase !== 'recovery') { completedSteps = 0; previousRemaining = null; } + else resetRemaining(); // A reset that would move a shown share back starts its stage afresh // (hidden, 5 s again). One before anything was counted, such as the // sampler's own start, keeps the stage's clock and the 2 s rule. if (miniFraction > 0) resetMini(); } - if (heartbeat) hasHeartbeat = true; + if (beat) hasHeartbeat = true; if (!heartbeat && message.phase === 'sampling' && valid(message.done)) completedSteps = Math.max(completedSteps, message.done); if (message.low_memory && typeof message.low_memory === 'object') lowMemory = true; @@ -714,8 +778,8 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f samplingPlan = {...message.plan_context}; // A heartbeat carries a newer estimate, never a newer engine event. // Keep sampledAt paired with the sampling counters/elapsed sample. - if (heartbeat && previousProgress !== null) { - overall = {...overall, ...message.overall}; overallAt = receivedAt; + if (heartbeat) { + overall = {...overall, ...message.overall}; overallAt = overallReceivedAt; observeRemaining(message); tick(); return; } @@ -724,7 +788,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f || message.prediction || message.sampling_plan || reset; if (!activityEvent && previousProgress !== null) { if (message.overall && typeof message.overall === 'object') { - overall = {...message.overall}; overallAt = receivedAt; observeRemaining(message); + overall = {...message.overall}; overallAt = overallReceivedAt; observeRemaining(message); } tick(); return; } @@ -759,14 +823,14 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f } element.dataset.phase = message.phase || message.timing_phase || ''; if (message.overall && typeof message.overall === 'object') { - overall = {...message.overall}; overallAt = sampledAt; + overall = {...message.overall}; overallAt = overallReceivedAt; observeRemaining(message); } element.hidden = false; tick(); if (timer === null) timer = setInterval(tick, 250); } function hide() { - element.hidden = true; status.hidden = true; recovery = ''; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false; + element.hidden = true; status.hidden = true; recovery = ''; placeholder = false; elapsedLead = 0; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false; compatibilityNote.hidden = true; compatibilityNote.textContent = ''; memoryTight = null; firstPassOver = false; elapsedShown = 0; diff --git a/web/progress_connection.js b/web/progress_connection.js index 078c3ee9..04e6ad10 100644 --- a/web/progress_connection.js +++ b/web/progress_connection.js @@ -10,7 +10,9 @@ export function createProgressConnection(api, consume, now = () => Date.now()) { // The graph may still be loading. Do not acknowledge an undelivered // snapshot; the next poll restores it once the node exists. const age = Number.isFinite(message.age_seconds) ? Math.max(0, message.age_seconds) : 0; - if (consume({...message, received_at: now() - age * 1000}) !== false) + // A heartbeat refreshes only the estimate: it is younger than the engine message. + const overallAge = Number.isFinite(message.overall_age_seconds) ? Math.max(0, message.overall_age_seconds) : age; + if (consume({...message, received_at: now() - age * 1000, overall_received_at: now() - overallAge * 1000}) !== false) seen.set(key, message); } async function refresh() { From 280bc75600292f8e9967859dffa28cb1d8d411f3 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 17:14:10 +0000 Subject: [PATCH 16/27] Autopilot conversation: a job card's level follows the engine's number while that number is moving - What a standing number added is kept only as far as the number has not caught up with it - Tests: a job card's level is never below the engine's number --- web/autopilot_chat.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index 09f68fbe..b70a3e2a 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -30,8 +30,9 @@ export function jobEstimate(job, age) { // The level a job shows, given the engine's number (`base`), the time it says is left and the clock. The engine's // number can stand for a long while: a model loading for the first time runs far past the time the engine gave it, // and a video in several parts has no timing at all. While it stands the level still rises, at the pace of the time -// left, slowing down, and by no more than a quarter of what remains. What was added is kept when the number moves -// again: the rest of the way is drawn shorter, so the level neither waits nor goes back, and is full at the end. +// left, slowing down, and by no more than a quarter of what remains. When the number moves again, what was added is +// kept only as far as the number has not caught up with it: the rest of the way is then drawn shorter, so the level +// neither waits nor goes back and is full at the end, and a run that goes as the engine said shows the engine's number. const DRIFT = .25, RISE = .0005; export const newLevel = () => ({value: 0, top: null, since: 0, scale: 1}); // The time left a running job shows, by the creation panel's rule: once measured never raised, and gone once its last @@ -58,12 +59,13 @@ export function jobLeft(row, sent, source, time) { } export function jobLevel(row, base, left, time) { if (row.top === null || base > row.top + RISE) { - if (row.top !== null && row.top < 1) row.scale = (1 - row.value) / (1 - row.top); + if (row.top !== null && base < 1) row.scale = Math.min(row.scale, (1 - row.value) / (1 - base)); row.top = base; row.since = time; } - const level = 1 - row.scale * (1 - Math.max(base, row.top)); - const more = left > 0 ? (1 - level) * DRIFT * (1 - Math.exp(-Math.max(0, time - row.since) / (DRIFT * left))) : 0; - return row.value = Math.max(row.value, Math.min(.99, level + more)); + // The number itself, or where standing since it last moved has brought the level: whichever is further on. + const from = 1 - row.scale * (1 - row.top), level = 1 - row.scale * (1 - Math.max(base, row.top)); + const more = left > 0 ? (1 - from) * DRIFT * (1 - Math.exp(-Math.max(0, time - row.since) / (DRIFT * left))) : 0; + return row.value = Math.max(row.value, Math.min(.99, Math.max(level, from + more))); } // working, done, stopped or failed: which card a job shows. export const jobMode = job => WORKING.includes(job.status) ? 'working' : job.status === 'succeeded' ? 'done' : job.status === 'cancelled' ? 'stopped' : 'failed'; From 30bc619ed9f078b0b196ee2bacdec13fa6f703be Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 17:17:09 +0000 Subject: [PATCH 17/27] A slow download says on the main card that the network is slow and it is trying again - The bootstrap scripts report the longer round of retries as its own step, and report when it is over --- freevideo.ps1 | 8 +++++++ freevideo_engine/bootstrap.py | 32 ++++++++++++++++++++------ freevideo_engine/desktop_runtime.py | 31 ++++++++++++++++--------- freevideo_engine/launcher_copy.py | 1 + freevideo_engine/network.py | 4 ++-- freevideo_engine/setup_progress.py | 14 +++++++++++ freevideo_engine/windows_bootstrap.ps1 | 4 +++- 7 files changed, 73 insertions(+), 21 deletions(-) diff --git a/freevideo.ps1 b/freevideo.ps1 index f112fbf4..5a3e24a8 100644 --- a/freevideo.ps1 +++ b/freevideo.ps1 @@ -75,6 +75,7 @@ try { Write-Host 'Preparing isolated Python for setup. Model/environment installation requires the setup confirmation.' Write-Host "Bootstrap cache and log: $bootstrapRoot" $bootstrapLock = [IO.File]::Open((Join-Path $bootstrapRoot 'bootstrap.lock'), 'OpenOrCreate', 'ReadWrite', 'None') + $patientDownloadNotice = $false try { $versions = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'freevideo_engine\bootstrap_versions.json') -Raw -Encoding UTF8 | ConvertFrom-Json $uvSpec = $versions.windows.uv @@ -110,6 +111,10 @@ try { if ($profile.Mode -eq 'patient') { if (-not $allNetworkFailures) { break } Write-Host 'The network is slow. Downloading again; this may take a while.' + $patientDownloadNotice = $true + if ($env:FREEVIDEO_UI_EVENTS -eq '1') { + Write-Host '{"event":"freevideo_ui","kind":"progress","key":"download-patient-python","label":"Download","detail":"The network is slow. Downloading again; this may take a while."}' + } } foreach ($name in @('UV_HTTP_TIMEOUT', 'UV_HTTP_RETRIES')) { $value = $freevideoSavedEnvironment[$name] @@ -139,6 +144,9 @@ try { $freevideoPython = (& $uv python find --managed-python $versions.python | Select-Object -Last 1).Trim() if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $freevideoPython)) { throw 'Managed Python could not be located.' } } finally { + if ($patientDownloadNotice -and $env:FREEVIDEO_UI_EVENTS -eq '1') { + Write-Host '{"event":"freevideo_ui","kind":"progress","key":"download-patient-python","label":"Download","detail":"","state":"complete"}' + } $bootstrapLock.Dispose() Write-Progress -Activity 'FreeVideo bootstrap' -Completed } diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index cb56322c..ba7395c6 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -2,7 +2,7 @@ from __future__ import annotations import argparse import csv -from contextlib import ExitStack +from contextlib import ExitStack, contextmanager import hashlib import json import os @@ -1517,10 +1517,28 @@ def clone(self, name, url, commit, sparse=None): return network.clone(self.root / 'vendor' / name, url, commit, sparse=sparse, run=self.command, network=self.network, env=self.env) + @contextmanager + def download_notice(self, key): + """Keep the patient-round notice visible across command heartbeats.""" + active = False + + def notice(message): + nonlocal active + self.ui.write(message + '\n') + active = True + self.ui.event('progress', key=key, label='Download', detail=message) + + try: + yield notice + finally: + if active: + self.ui.event('progress', key=key, label='Download', detail='', state='complete') + def packages(self, label, args, family='pypi'): - return network.package_command(self.network, family, - lambda env, source: self.command(label if source == 'official' else label + '-' + source, args, env=env), - self.env, notice=lambda message: self.ui.write(message + '\n')) + with self.download_notice('download-patient-' + label) as notice: + return network.package_command(self.network, family, + lambda env, source: self.command(label if source == 'official' else label + '-' + source, args, env=env), + self.env, notice=notice) def python_install(self, uv): from .system import bootstrap_root @@ -1555,9 +1573,9 @@ def attempt(source, route, retry_mode): return self.command('python-' + source + '-' + route, [uv, 'python', 'install', *isolation, self.versions['python']], env=env) try: - return network.download_command(self.network, 'github', attempt, self.env, category='python', - extra_sources=['user-python'] if custom else [], - notice=lambda message: self.ui.write(message + '\n')) + with self.download_notice('download-patient-python') as notice: + return network.download_command(self.network, 'github', attempt, self.env, category='python', + extra_sources=['user-python'] if custom else [], notice=notice) except network.PackageSourcesError as error: failure = RuntimeError('Python download failed on every route; see retained logs') failure.package_attempts = error.attempts diff --git a/freevideo_engine/desktop_runtime.py b/freevideo_engine/desktop_runtime.py index 76815e37..48c082df 100644 --- a/freevideo_engine/desktop_runtime.py +++ b/freevideo_engine/desktop_runtime.py @@ -20,6 +20,20 @@ from .system import install_root, windows +def parse_progress_line(line): + """Decode the worker's UI protocol; ordinary terminal output stays a log.""" + if line.startswith('{'): + try: + message = json.loads(line) + if message.get('event') == 'freevideo_ui': + return message + if message.get('event') == 'local_model_progress': + return local_model_ui(message) + except (ValueError, AttributeError): + pass + return None + + def launcher_root(): if os.environ.get('FREEVIDEO_LAUNCHER_HOME'): return Path(os.environ['FREEVIDEO_LAUNCHER_HOME']) @@ -334,17 +348,12 @@ def progress(message): log.write(line) output = (output + line)[-2 * 1024 * 1024:] self.events.put(('log', line)) - if line.startswith('{'): - try: - message = plain_message(json.loads(line)) - if message.get('event') == 'freevideo_ui': - if message.get('kind') == 'failure' and isinstance(message.get('error'), str): - error = message['error'] - self.events.put(('progress', message)) - elif message.get('event') == 'local_model_progress': - self.events.put(('progress', local_model_ui(message))) - except (ValueError, AttributeError): - pass + message = parse_progress_line(line) + if message is not None: + message = plain_message(message) + if message.get('kind') == 'failure' and isinstance(message.get('error'), str): + error = message['error'] + self.events.put(('progress', message)) code = process.wait(timeout=15) process.stdout.close() except BaseException as failure: diff --git a/freevideo_engine/launcher_copy.py b/freevideo_engine/launcher_copy.py index 6e8b991c..fd1009ca 100644 --- a/freevideo_engine/launcher_copy.py +++ b/freevideo_engine/launcher_copy.py @@ -13,6 +13,7 @@ } ZH = dict([ + ('The network is slow. Downloading again; this may take a while.', '网络较慢,正在重试'), ('Prepare Python', '准备 Python'), ('Install Python', '安装 Python'), ('Prepare download tools', '准备下载工具'), ('Tsinghua mirror', '清华镜像'), ('Preparing installation', '准备安装'), ('Checking the environment', '正在检查运行环境'), diff --git a/freevideo_engine/network.py b/freevideo_engine/network.py index f72c702e..38058520 100644 --- a/freevideo_engine/network.py +++ b/freevideo_engine/network.py @@ -586,8 +586,8 @@ def download_environment(environ, retry_mode='fast'): r"\b(?:invalid|bad|wrong|incorrect|mismatched?)\s+checksums?\b" r"|\bchecksums?\s+(?:mismatch|error|failed|failure)\b" r"|\bchecksums?\s+(?:verification|validation)\s+(?:failed|failure|error)\b" - r"|\bchecksums?\s+(?:does not|doesn't|did not|didn't|do not|don't)\s+match\b" - r"|\b(?:does not|doesn't|did not|didn't|do not|don't)\s+match\s+(?:the\s+|its\s+|expected\s+)?(?:checksums?|hash(?:es)?)\b" + r"|\bchecksums?\s+(?:does\s+not|doesn't|did\s+not|didn't|do\s+not|don't)\s+match\b" + r"|\b(?:does\s+not|doesn't|did\s+not|didn't|do\s+not|don't)\s+match\s+(?:the\s+|its\s+|expected\s+)?(?:checksums?|hash(?:es)?)\b" r"|\bhash\s+(?:mismatch|check\s+failed|does\s+not\s+match)\b|\bhashes\s+(?:do\s+not|didn't)\s+match\b" r"|\bsha-?256\s+mismatch\b|\bdigest\s+mismatch\b" r"|\bintegrity\s+(?:check|verification)\s+failed\b") diff --git a/freevideo_engine/setup_progress.py b/freevideo_engine/setup_progress.py index 723fbf76..44dc6d40 100644 --- a/freevideo_engine/setup_progress.py +++ b/freevideo_engine/setup_progress.py @@ -16,6 +16,7 @@ def __init__(self): self.tasks = {} self.last = {} self.model_groups = [] + self.download_notices = {} def update(self, event, now=None): now = time.monotonic() if now is None else now @@ -25,8 +26,15 @@ def update(self, event, now=None): return if event.get('kind') == 'phase': self.phase = event + self.download_notices.clear() return key = str(event.get('key', 'current')) + if event.get('kind') == 'progress' and key.startswith('download-patient-'): + if event.get('state', 'running') == 'running' and event.get('detail'): + self.download_notices[key] = event + else: + self.download_notices.pop(key, None) + return if event.get('kind') == 'task': self.tasks.pop(key, None) previous = self.tasks.get(key, {}) @@ -48,6 +56,12 @@ def snapshot(self, now=None): active.sort(key=lambda row: (number(row.get('total')) and row['total'] > 0, row.get('unit') == 'bytes'), reverse=True) current = active[0] if active else self._view(self.last, now) + if self.download_notices: + notice = next(reversed(self.download_notices.values())) + # A temporary activity line, with the underlying stage and counters + # retained so clearing it restores the current operation immediately. + current = dict(current, label=current.get('label') or notice['label'], + detail=notice['detail'], activity='', name='') return dict(phase_progress=dict(self.phase), active_tasks=active, progress=current, model_groups=self.model_groups) @staticmethod diff --git a/freevideo_engine/windows_bootstrap.ps1 b/freevideo_engine/windows_bootstrap.ps1 index b1bcb68b..b7cc3e9f 100644 --- a/freevideo_engine/windows_bootstrap.ps1 +++ b/freevideo_engine/windows_bootstrap.ps1 @@ -67,9 +67,11 @@ function Invoke-FreeVideoBootstrapCommand { function Test-FreeVideoSlowDownloadFailure { param([string]$Detail) # Keep the patient-round gate aligned with network.slow_download_failure. + # URL paths may contain validation words without a validation failure. + $Detail = $Detail -replace 'https?://\S+', '' return ($Detail -match 'timed out|timeout|connection|could not resolve|failed to download|failed to fetch|network is unreachable|ssl|tls|http status|status code|requested url returned error|rpc failed|early eof|unable to access|remote end hung up|couldn''t connect|proxy' -and $Detail -notmatch '\b(401|403|404|410)\b' -and - $Detail -notmatch 'no solution found|no matching distribution|could not find a version|checksum|hash mismatch|hashes do not match|hashes didn''t match|hash does not match|hash check failed|sha256 mismatch|sha-256 mismatch|digest mismatch|integrity|certificate verify failed|certificate_verify_failed|invalid peer certificate|unknown issuer|not found|resolutionimpossible|failed to build|invalid wheel|permission denied|no space left on device|disk full') + $Detail -notmatch 'no solution found|no matching distribution|could not find a version|\b(?:invalid|bad|wrong|incorrect|mismatched?)\s+checksums?\b|\bchecksums?\s+(?:mismatch|error|failed|failure)\b|\bchecksums?\s+(?:verification|validation)\s+(?:failed|failure|error)\b|\bchecksums?\s+(?:does\s+not|doesn''t|did\s+not|didn''t|do\s+not|don''t)\s+match\b|\b(?:does\s+not|doesn''t|did\s+not|didn''t|do\s+not|don''t)\s+match\s+(?:the\s+|its\s+|expected\s+)?(?:checksums?|hash(?:es)?)\b|\bhash\s+(?:mismatch|check\s+failed|does\s+not\s+match)\b|\bhashes\s+(?:do\s+not|didn''t)\s+match\b|\bsha-?256\s+mismatch\b|\bdigest\s+mismatch\b|\bintegrity\s+(?:check|verification)\s+failed\b|certificate verify failed|certificate_verify_failed|invalid peer certificate|unknown issuer|not found|resolutionimpossible|failed to build|invalid wheel|permission denied|no space left on device|disk full') } function Test-FreeVideoUvArchive { From 930ccc9fbefa106b4db43228e375b4b115e8da23 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 17:17:09 +0000 Subject: [PATCH 18/27] The long waits while an install runs are split into visible steps with real counts - Python, packages, the encoder, models and the GPU check each report what they are doing - Install waits: the reviewed Chinese wording for the new activity lines --- freevideo_engine/bootstrap.py | 31 ++++++++++++-- freevideo_engine/doctor.py | 16 ++++++- freevideo_engine/encode_worker.py | 8 ++++ freevideo_engine/install_activity.py | 29 +++++++++++++ freevideo_engine/launcher_copy.py | 17 ++++++++ freevideo_engine/launcher_session.py | 4 ++ freevideo_engine/model_transfer.py | 31 ++++++++++---- freevideo_engine/network.py | 10 ++++- freevideo_engine/offline_packages.py | 42 ++++++++++++++----- freevideo_engine/provider_resume.py | 21 ++++++++++ freevideo_engine/provision.py | 28 ++++++++++--- freevideo_engine/runtime_libraries.py | 11 +++-- freevideo_engine/setup_progress.py | 14 +++++++ freevideo_engine/storage.py | 8 ++-- freevideo_engine/terminal_ui.py | 60 +++++++++++++++++++++++---- 15 files changed, 286 insertions(+), 44 deletions(-) create mode 100644 freevideo_engine/install_activity.py diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index ba7395c6..e36988fb 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -998,6 +998,8 @@ def setup_task(label): routes = {'official': 'official source', 'nju': 'Nanjing University mirror', 'tuna': 'Tsinghua mirror', 'ghfast': 'GitHub mirror', 'user-python': 'your Python mirror', 'user': 'your configured source'} + # Python route retries append the route after the source. + label = re.sub(r'-(?:direct|proxy)$', '', label) source = next((name for name in routes if label.endswith('-' + name)), None) key = label[:-(len(source) + 1)] if source else label for prefix in ('unified-', 'engine-', 'encoder-'): @@ -1098,7 +1100,8 @@ def initialize(self, value, ui): from .runtime_libraries import damaged if ui is not None: prepare_step(ui, 'Checking the environment') - self.damaged_before = damaged(self.pythons['engine']) + self.damaged_before = damaged(self.pythons['engine'], progress=( + lambda done, total: prepare_step(ui, 'Checking the environment', done, total)) if ui is not None else None) # Moving a working installation to int8 checks the int8 kernels first # (execute), so a GPU that fails them keeps its configuration as it was. # With changed files that check cannot run; the one after installing still does. @@ -1304,6 +1307,7 @@ def command(self, label, args, env=None, cwd=None): raise RuntimeError(self.state.get('resource_guard', 'Installation cancelled.')) from .triton_compat import temporary_environment env = temporary_environment(self.root, env or self.env) + env = dict(env, FREEVIDEO_INSTALL_PROGRESS='1') args, cwd = uv_file_arguments(args, cwd) with self.state_lock: index = len(self.state['steps']) @@ -1316,6 +1320,10 @@ def command(self, label, args, env=None, cwd=None): if label == 'models' and self.plan.get('local_models'): title, detail = 'Reuse models / download missing files', 'Import verified local models; source files stay in place' self.ui.begin(key, title, detail=str(log) if self.ui.verbose else detail) + initial = ('Python downloading' if title == 'Install Python' else + 'Environment creating' if title == 'Create Python environment' else '') + if initial: + self.ui.update(key, activity='note', name=initial, detail=initial) progress = LogProgress(log) parallel_tasks = set() models_step = label == 'models' @@ -1445,11 +1453,13 @@ def check(): constraints=['-c', constraints_file(label, self.system)], check=check): return from .runtime_libraries import damaged, reinstall_arguments - fresh = damaged(executable) + fresh = self.check_runtime_libraries(executable, label) result = self.packages(label + '-torch', [*argv[:5], *reinstall_arguments(fresh), *argv[5:]], group) # Files linked from a changed uv cache arrive with a new environment. - if not fresh and damaged(executable): - result = self.packages(label + '-torch', [*argv[:5], *reinstall_arguments(damaged(executable)), *argv[5:]], group) + if not fresh: + changed = self.check_runtime_libraries(executable, label) + if changed: + result = self.packages(label + '-torch', [*argv[:5], *reinstall_arguments(changed), *argv[5:]], group) return result task(name + '-torch', torch_packages, (name + '-venv',), writer) @@ -1502,6 +1512,18 @@ def encoder_packages(): tasks['models']['after'] = ('package-cache',) return tasks, pythons, comfy + def check_runtime_libraries(self, python, role): + from .runtime_libraries import damaged + key = role + '-runtime-libraries' + self.ui.begin(key, 'Runtime libraries verifying') + def progress(done, total): + self.ui.update(key, activity='count', name='Runtime libraries verifying', + detail='Runtime libraries verifying', done=done, total=total, unit='items') + try: + return damaged(python, progress=progress) + finally: + self.ui.end(key) + def release_package_cache(self, uv): cache = self.root / 'downloads' / 'uv-cache' if cache.is_symlink() or not cache.resolve().is_relative_to(self.root.resolve()): @@ -1851,6 +1873,7 @@ def prepare_tools(self): def execute(self): if self.int8_check_first: + prepare_end(self.ui) # The GPU precheck is already execution work. # The installed environment runs this source's probes (PYTHONPATH); # the receipt is reused by the check after installation. self.ui.phase('Verify installation on your GPU', 0, 8) diff --git a/freevideo_engine/doctor.py b/freevideo_engine/doctor.py index ccd492b1..ef3c0b63 100644 --- a/freevideo_engine/doctor.py +++ b/freevideo_engine/doctor.py @@ -11,7 +11,12 @@ from . import processes, subprocess_output -def doctor(*, probe=False, hardware=None, backends=None): +def doctor(*, probe=False, hardware=None, backends=None, progress=None): + if progress is None and os.environ.get('FREEVIDEO_INSTALL_PROGRESS') == '1': + from .install_activity import step + progress = step + if progress: + progress('GPU information reading') from .triton_compat import own_temporary_environment own_temporary_environment() versions = package_versions() @@ -28,7 +33,10 @@ def doctor(*, probe=False, hardware=None, backends=None): with runtime_lock() as descriptor: env = dict(os.environ, **{LOCK_ENV: str(descriptor)}) linear = ['linear'] + (['linear-int8'] if hardware.system in ('Windows', 'Linux') else []) - for backend in [*sorted(backends), *linear]: + probes = [*sorted(backends), *linear] + for index, backend in enumerate(probes): + if progress: + progress('GPU tests running', index, len(probes)) command = [sys.executable, '-m', 'freevideo_engine.probe', backend] try: child = subprocess_output.run_python(command, capture_output=True, timeout=180, env=env, @@ -52,6 +60,8 @@ def tail(value): except OSError as error: row = {'backend': backend, 'status': 'error', 'error': 'Could not start kernel probe: ' + str(error)} result['kernel_probes'].append(row) + if progress: + progress('GPU tests running', index + 1, len(probes)) result.update(readiness(result['kernel_probes']), identity=identity(hardware, versions), view_root=view_root()) if temp_root(): @@ -59,4 +69,6 @@ def tail(value): result['receipt'] = str(receipt_path()) result['validation'] = 'Small kernels executed on the listed GPU; this does not validate full-video speed, quality or capacity.' save(receipt_path(), result) + if progress: + progress('GPU tests finished', len(probes), len(probes)) return result diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py index 5744ba30..464453c5 100644 --- a/freevideo_engine/encode_worker.py +++ b/freevideo_engine/encode_worker.py @@ -236,6 +236,12 @@ def _encode(args, request, resident, trace): loading = trace.data def phase(name, **metrics): trace.stage(name, **metrics) + if args.check_library: + from .install_activity import step + labels = {'encoder_torch_import': 'Runtime libraries importing', + 'encoder_options': 'Encoder configuring', 'encoder_native_import': 'Encoder importing'} + if name in labels: + step(labels[name]) if name in ('encoder_oom', 'encoder_retry'): print(json.dumps(dict(event=name, attempt=len(loading.get('encoder_attempts', [])))), flush=True) if name == 'encoder_oom' and request.get('metrics') and request.get('diagnostic_output'): @@ -306,6 +312,8 @@ def phase(name, **metrics): raise RuntimeError('Unexpected application modules: ' + repr(forbidden)) print(json.dumps({'native_encoder_library_import': True, 'application_modules_loaded': [], 'torch': str(torch.__version__), 'source': str(args.comfy_root)}), flush=True) + from .install_activity import step + step('Encoder ready') return phase('encoder_media_prepare') keyframes = request.get('keyframes') diff --git a/freevideo_engine/install_activity.py b/freevideo_engine/install_activity.py new file mode 100644 index 00000000..d8e3f81a --- /dev/null +++ b/freevideo_engine/install_activity.py @@ -0,0 +1,29 @@ +"""Small installation worker events: actual work, never elapsed-time counters.""" +import json +import sys +import time + + +def emit(event, **row): + sys.stdout.write(json.dumps(dict(event=event, **row)) + '\n') + sys.stdout.flush() + + +def step(name, done=0, total=0): + emit('installation_step', name=name, done=done, total=total) + + +def byte_progress(event, file, *, name='Model files', report=None): + """Report bounded reads/writes, including both endpoints, at most twice/s.""" + last = None + def progress(done, total): + nonlocal last + now = time.monotonic() + if last is None or done == total or now - last >= .5: + row = dict(file=str(file), name=name, done_bytes=done, total_bytes=total) + if report is None: + emit(event, **row) + else: + report(**row) + last = now + return progress diff --git a/freevideo_engine/launcher_copy.py b/freevideo_engine/launcher_copy.py index fd1009ca..3cccaa0a 100644 --- a/freevideo_engine/launcher_copy.py +++ b/freevideo_engine/launcher_copy.py @@ -120,6 +120,23 @@ ('Hugging Face · Video', 'Hugging Face · 视频'), ('Hugging Face · Audio', 'Hugging Face · 音频'), ]) ZH.update(NAMES.values()) +ZH.update([ + ('Completed ', '已完成 '), ('Verifying ', '正在校验'), ('Assembling ', '正在合并'), + ('Copying ', '正在复制'), ('Model files', '模型文件'), + ('Packages resolving', '正在检查组件'), ('Packages installing', '正在安装组件'), ('Packages ready', '组件已就绪'), + ('Environment creating', '正在创建运行环境'), ('Base packages installing', '正在安装基础组件'), + ('Python ready', 'Python 已就绪'), ('Python downloading', '正在下载 Python'), + ('GPU acceleration building', '正在准备显卡加速'), + ('Runtime libraries verifying', '正在校验运行组件'), ('Runtime libraries importing', '正在载入运行组件'), + ('Encoder configuring', '正在准备文本编码器'), ('Encoder importing', '正在载入文本编码器'), + ('Encoder ready', '文本编码器已就绪'), + ('GPU information reading', '正在读取显卡信息'), ('GPU tests running', '正在检查显卡'), + ('GPU tests finished', '显卡检查已完成'), + ('Packages preparing', '正在准备组件'), ('Packages removing', '正在移除旧组件'), + ('Model files verifying', '正在校验模型文件'), ('Model files preparing', '正在准备模型文件'), + ('Model files assembling', '正在合并模型文件'), + ('Package files', '离线包文件'), +]) # Card lines for steps that only report counts (the terminal keeps the counts). ZH.update([('Downloading components', '正在下载组件'), ('Preparing components', '正在准备组件'), ('Downloading runtime packages', '正在下载运行组件'), ('Finding existing models', '正在查找已有模型')]) diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index 3ca99ac6..16b3bd69 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -1632,6 +1632,10 @@ def snapshot(self): done=min(group['download_bytes'], group.get('downloaded_bytes') or 0), rate=group.get('bytes_per_second') or progress.get('rate')) file_text = progress_text(current, zh, decimal_sizes=sys.platform == 'darwin') + if self.importer.busy and self.importer.state.get('activity'): + imported = self.importer.state + file_text = progress_text(dict(activity=imported['activity'], unit='bytes', + name=display('Package files', zh), done=imported.get('activity_done'), total=imported.get('activity_total')), zh) # The time row's estimate: only while downloading, from what is left at the current speed. from .setup_progress import number left, rate = (current.get('total') or 0) - (current.get('done') or 0), current.get('rate') diff --git a/freevideo_engine/model_transfer.py b/freevideo_engine/model_transfer.py index d1a9ff30..328e519d 100644 --- a/freevideo_engine/model_transfer.py +++ b/freevideo_engine/model_transfer.py @@ -135,10 +135,11 @@ def update(self, amount, *, transfer=False): self.last = now -def verify(path, row): +def verify(path, row, progress=None): if path.stat().st_size != row['bytes']: raise ValueError('Downloaded size differs from the pinned manifest') - return network.hash_file(path, 'sha256' if row.get('sha256') else 'sha1', not row.get('sha256'), discard_cache=True) == (row.get('sha256') or row['git_blob']) + return network.hash_file(path, 'sha256' if row.get('sha256') else 'sha1', not row.get('sha256'), + discard_cache=True, progress=progress) == (row.get('sha256') or row['git_blob']) def checked_range_response(response, requested, total, progress): @@ -187,10 +188,13 @@ def sdk_worker(request): progress = Progress(row['bytes']) backend = [] started = time.monotonic() + from .install_activity import byte_progress + verifying = byte_progress('verify_progress', target, + report=lambda **row: emit(phase='verify', **row)) try: if target.is_file(): - emit(phase='verify') - if verify(target, row): + emit(phase='verify', done_bytes=0, total_bytes=row['bytes']) + if verify(target, row, progress=verifying): emit(phase='ready', backend='verified-stage', seconds=time.monotonic()-started, fingerprint=fingerprint(target)) return 0 network.retain_partial(target, 'hash-rejected') @@ -213,7 +217,14 @@ def __init__(self, filename, total): def update(self, amount): progress.update(0) def end(self): - emit(phase='reconstruct', done_bytes=row['bytes'], total_bytes=row['bytes']) + emit(phase='reconstruct', done_bytes=0, total_bytes=row['bytes']) + def merge_progress(self, done, total): + merging(done, total) + def verify_progress(self, done, total): + verifying(done, total) + + merging = byte_progress('assemble_progress', target, + report=lambda **row: emit(phase='reconstruct', **row)) original_get, original_parallel = _download.requests.get, getattr(_download, '_parallel_download', None) @@ -318,8 +329,8 @@ def report(done, total, rate): # count alone does not prove that the file descriptor is closed. emit(phase='transfer_complete', done_bytes=row['bytes'], total_bytes=row['bytes']) downloaded = time.monotonic() - emit(phase='verify') - if not verify(target, row): + emit(phase='verify', done_bytes=0, total_bytes=row['bytes']) + if not verify(target, row, progress=verifying): raise ValueError('Downloaded hash differs from the pinned manifest') emit(phase='ready', backend=backend[-1] if backend else 'hf-cache', download_seconds=downloaded-started, verification_seconds=time.monotonic()-downloaded, @@ -524,6 +535,12 @@ def read_output(): network_bytes_per_second=current.get('transfer_bytes', 0) / max(.001, current.get('elapsed_seconds', time.monotonic()-started)), method=current.get('backend')) + if current['phase'] in ('verify', 'reconstruct') and current.get('total_bytes'): + from .model_status import NAMES, family + sys.stdout.write(json.dumps(dict(event='verify_progress' if current['phase'] == 'verify' else 'assemble_progress', + file=str(path), name=NAMES[family(row)][0], done_bytes=current.get('done_bytes', 0), + total_bytes=current['total_bytes'])) + '\n') + sys.stdout.flush() prior_phase = current['phase'] last_emit = time.monotonic() # The SDK reports the transfer complete before it starts its final diff --git a/freevideo_engine/network.py b/freevideo_engine/network.py index 38058520..452dbb37 100644 --- a/freevideo_engine/network.py +++ b/freevideo_engine/network.py @@ -772,8 +772,16 @@ def matches(candidate, source='local'): return False event(network, category=category, source=source, file=path.name, action='verifying', algorithm=algorithm) check = network.get('resource_check') if keep_partial else None + from .install_activity import byte_progress + report = byte_progress('verify_progress', path.name, report=lambda **row: + event(network, category=category, source=source, action='verify-progress', **row)) + def verification(done, total): + if check: + check() + if model: + report(done, total) return hash_file(candidate, algorithm, git_blob, discard_cache=category in ('models', 'vdn-models'), - progress=(lambda done, total: check()) if check else None) == expected + progress=verification if model or check else None) == expected if path.is_file(): if matches(path): return diff --git a/freevideo_engine/offline_packages.py b/freevideo_engine/offline_packages.py index e02e93e0..d304b651 100644 --- a/freevideo_engine/offline_packages.py +++ b/freevideo_engine/offline_packages.py @@ -220,7 +220,7 @@ def stamp(path): return [info.st_size, info.st_mtime_ns] -def unchanged(path, row, known, current=None): +def unchanged(path, row, known, current=None, progress=None): """Still the file its import checked: the size matches, and the recorded time too. Only a file whose record says it was rewritten since is hashed. `current` @@ -235,7 +235,24 @@ def unchanged(path, row, known, current=None): if known.get(row['path']) in (None, current): return True from .network import hash_file - return hash_file(path) == row['sha256'] + return hash_file(path, **({'progress': progress} if progress else {})) == row['sha256'] + + +def file_progress(progress, activity, total): + from .install_activity import byte_progress + return byte_progress('', '', report=lambda **row: + progress(activity=activity, activity_done=row['done_bytes'], activity_total=row['total_bytes'])) + + +def copy_progress(source, target, progress): + """The existing cross-disk copy, with a bounded buffer and actual bytes.""" + total, done = source.stat().st_size, 0 + progress(0, total) + with source.open('rb') as input_file, target.open('wb') as output: + for chunk in iter(lambda: input_file.read(4 * 2**20), b''): + output.write(chunk) + done += len(chunk) + progress(done, total) def import_archive(path, destination, progress=lambda **kw: None, cancelled=lambda: False): @@ -265,18 +282,19 @@ def import_archive(path, destination, progress=lambda **kw: None, cancelled=lamb current = present.get(row['path']) if current is None: continue - if current[0] != row['bytes'] or not unchanged(root / row['path'], row, known, current): + if current[0] != row['bytes'] or not unchanged(root / row['path'], row, known, current, + progress=file_progress(progress, 'verify', row['bytes'])): raise ValueError('已有导入文件已改变,未覆盖:' + plain(root / row['path'])) verified[row['path']] = current marker = root / value['marker'] if not missing and marker.is_file() and marker.read_bytes() == value['raw']: # Imported before and complete: nothing to read again. save(root / VERIFIED, verified) - progress(done=total, total=total, detail=path.name) + progress(done=total, total=total, detail=path.name, activity='') return dict(root=plain(root), kind=value['kind'], variant=value['variant'], name=path.name, **origin) done = sum(r['bytes'] for r in value['files'] if r['path'] in verified) root.mkdir(parents=True, exist_ok=True) - progress(done=done, total=total, detail=path.name) + progress(done=done, total=total, detail=path.name, activity='') with zipfile.ZipFile(path) as archive: for row in value['files']: if cancelled(): @@ -374,7 +392,8 @@ def assemble(runtime, models, source, progress=lambda **kw: None): def source_for(row): for root in models: current = walks[root].get(row['path']) - if current and unchanged(io_path(root) / row['path'], row, known[root], current): + if current and unchanged(io_path(root) / row['path'], row, known[root], current, + progress=file_progress(progress, 'verify', row['bytes'])): return io_path(root) / row['path'] return None @@ -383,19 +402,20 @@ def place(src, target): try: os.link(src, target) except OSError: - shutil.copyfile(src, target) + copy_progress(src, target, file_progress(progress, 'copy_bytes', src.stat().st_size)) for row in model['files']: target, src = runtime / row['path'], source_for(row) if src is None: raise ValueError('还需导入「%s」包,缺少文件:%s' % (pack_name(row['path']), row['path'])) if row['path'] in here: - if not os.path.samefile(target, src) and hash_file(target) != row['sha256']: + if not os.path.samefile(target, src) and hash_file(target, + progress=file_progress(progress, 'verify', row['bytes'])) != row['sha256']: raise ValueError('离线安装已有不同模型,未覆盖:' + plain(target)) else: place(src, target) done += row['bytes'] - progress(done=done, total=total, detail='检查配套包') + progress(done=done, total=total, detail='检查配套包', activity='') listed = {row['path'] for row in model['files']} for root in models: package = io_path(root) / 'offline-package.json' @@ -405,10 +425,12 @@ def place(src, target): if row['path'] in listed or not row['path'].startswith(OPTIONAL_PREFIX) or row['path'] in here: continue current = walks[root].get(row['path']) - if not current or not unchanged(io_path(root) / row['path'], row, known[root], current): + if not current or not unchanged(io_path(root) / row['path'], row, known[root], current, + progress=file_progress(progress, 'verify', row['bytes'])): continue # The engine fetches a table it cannot find when a request needs it. place(io_path(root) / row['path'], runtime / row['path']) listed.add(row['path']) + progress(done=done, total=total, detail='检查配套包', activity='') return finish(runtime, config, model, source, here) diff --git a/freevideo_engine/provider_resume.py b/freevideo_engine/provider_resume.py index c8ed27de..4c5a6f25 100644 --- a/freevideo_engine/provider_resume.py +++ b/freevideo_engine/provider_resume.py @@ -46,6 +46,11 @@ def ms_parallel(module, url, target, file_size, headers, cookies, progress_callb offset = merged.stat().st_size if merged.exists() else 0 if offset > file_size: raise ValueError('Retained merge exceeds the pinned model size') + def verifying(done, total): + for callback in callbacks: + report = getattr(callback, 'verify_progress', None) + if report: + report(done, total) if offset % module.DOWNLOAD_PART_SIZE: # A switched HTTP download may end in the middle of an SDK part. # Seed that part so its first Range starts after the retained prefix. @@ -59,11 +64,14 @@ def ms_parallel(module, url, target, file_size, headers, cookies, progress_callb with merged.open('rb') as prefix, part.open('a+b') as output: prefix.seek(start); output.seek(0) remaining=min(existing,offset-start) + total = remaining + verifying(0, total) while remaining: count=min(BLOCK,remaining) if prefix.read(count)!=output.read(count): raise ValueError('Retained part differs from switched prefix') remaining-=count + verifying(total-remaining, total) if existing 0 and done <= total: + return name + t(' · Completed ', ' · 已完成 ') + '%d / %d' % (done, total) + return name + if activity in ('verify', 'assemble', 'copy_bytes'): + verb = {'verify': ('Verifying ', '正在校验'), 'assemble': ('Assembling ', '正在合并'), + 'copy_bytes': ('Copying ', '正在复制')}[activity] + # MiB rather than tenths of GiB: even slow disks change the visible text. + parts = [t(*verb) + name] + done, total = event.get('done'), event.get('total') + if event.get('unit') == 'bytes' and number(done) and number(total) and total > 0 and done <= total: + parts.append('%d / %d %s' % (done // 2**20, total // 2**20, 'MiB')) + return ' · '.join(parts) if activity == 'resume': return t('The connection dropped; resuming the download and keeping what was already downloaded', '连接中断,正在继续下载,已下载的部分会保留') diff --git a/freevideo_engine/storage.py b/freevideo_engine/storage.py index 9d58d050..3b878c8e 100644 --- a/freevideo_engine/storage.py +++ b/freevideo_engine/storage.py @@ -83,7 +83,7 @@ def record_download(root, path, row, *, origin='download'): save(ledger, {'root': plain(resolved(root)), 'files': files}) -def compact_sources(plan, rows, report): +def compact_sources(plan, rows, report, progress=None): """Caller must first verify the complete FP8 cache and pass GPU setup probes. Deletion uses an allowlist of pinned conversion-only filenames and a matching @@ -138,9 +138,11 @@ def compact_sources(plan, rows, report): # Destructive cleanup cannot trust a metadata cache hit. Equal-size # edits may share a filesystem timestamp tick with the receipt. if row.get('sha256'): - matches = hash_file(path, discard_cache=True) == row['sha256'] + matches = hash_file(path, discard_cache=True, progress=( + lambda done, total: progress(path, done, total)) if progress else None) == row['sha256'] elif row.get('git_blob'): - matches = hash_file(path, 'sha1', git_blob=True, discard_cache=True) == row['git_blob'] + matches = hash_file(path, 'sha1', git_blob=True, discard_cache=True, progress=( + lambda done, total: progress(path, done, total)) if progress else None) == row['git_blob'] else: raise OSError('No pinned content hash; source retained') if not matches: diff --git a/freevideo_engine/terminal_ui.py b/freevideo_engine/terminal_ui.py index 593c4141..730e16f8 100644 --- a/freevideo_engine/terminal_ui.py +++ b/freevideo_engine/terminal_ui.py @@ -322,6 +322,7 @@ class LogProgress: def __init__(self, path): self.path, self.offset, self.pending = Path(path), 0, '' self.packages = {} + self.downloaded_packages = set() self.installing_packages = False self.model_source = '' self.model_backend = '' @@ -382,23 +383,48 @@ def update(**values): self.packages[name] = float(size) * {'KiB': 2**10, 'MiB': 2**20, 'GiB': 2**30}[unit] amount = sum(self.packages.values()) / 2**30 size_text = '%.1f GiB' % amount if amount >= 1 else '%.0f MiB' % (amount * 1024) - update(done=0, total=0, scope='package-download', activity='note', name='Downloading runtime packages', + update(done=len(self.downloaded_packages), total=len(self.packages), unit='items', + scope='package-download', activity='count', name='Downloading runtime packages', detail='Downloading packages · about %s across %d files' % (size_text, len(self.packages))) + elif re.fullmatch(r'Downloaded [\w.+-]+', text): + name = text.split(' ', 1)[1] + if name in self.packages: + self.downloaded_packages.add(name) + update(done=len(self.downloaded_packages), total=len(self.packages), unit='items', + scope='package-download', activity='count', name='Downloading runtime packages', + detail='Downloading runtime packages') elif re.match(r'Resolved \d+ packages? in ', text): self.packages.clear() + self.downloaded_packages.clear() self.installing_packages = False - update(done=0, total=0, scope='package-resolve', detail='Dependencies resolved · preparing downloads') + update(done=0, total=0, scope='package-resolve', activity='note', name='Packages resolving', + detail='Dependencies resolved · preparing downloads') elif re.match(r'Prepared \d+ packages? in ', text): self.installing_packages = True - update(done=0, total=0, scope='package-install', detail='Downloads ready · installing packages') + update(done=0, total=0, scope='package-install', activity='note', name='Packages installing', + detail='Downloads ready · installing packages') elif re.match(r'Installed \d+ packages? in ', text): - update(done=0, total=0, scope='package-finish', detail='Packages installed · finishing checks') + update(done=0, total=0, scope='package-finish', activity='note', name='Packages ready', + detail='Packages installed · finishing checks') + elif re.match(r'Uninstalled \d+ packages? in ', text): + update(done=0, total=0, scope='package-remove', activity='note', name='Packages installing', + detail='Packages installing') + elif text.startswith('Creating virtual environment'): + update(done=0, total=0, scope='venv-create', activity='note', name='Environment creating', detail='Environment creating') + elif text.startswith('Installing seed packages'): + update(done=0, total=0, scope='venv-seed', activity='note', name='Base packages installing', detail='Base packages installing') + elif text.startswith('Installed Python '): + update(done=0, total=0, scope='python-ready', activity='note', name='Python ready', detail='Python ready') match = re.search(r'\[(\d+)/(\d+)\]', line) if match: self.installing_packages |= 'Installing wheels' in text + package_phase = ('remove', 'Packages removing') if 'Uninstalling' in text else ( + ('prepare', 'Packages preparing') if 'Preparing packages' in text else ( + ('install', 'Packages installing') if self.installing_packages else ('compile', 'GPU acceleration building'))) update(done=int(match[1]), total=int(match[2]), + activity='count', name=package_phase[1], detail='Installing packages' if self.installing_packages else 'Compiling CUDA kernels', - scope='package-install' if self.installing_packages else 'compile', unit='items') + scope='package-' + package_phase[0], unit='items') try: event = json.loads(line) except (ValueError, TypeError): @@ -406,7 +432,23 @@ def update(**values): if not isinstance(event, dict): continue kind = event.get('event') - if kind == 'model_groups': + if kind == 'installation_step': + update(done=event.get('done', 0), total=event.get('total', 0), unit='items', + activity='count' if event.get('total') else 'note', name=event['name'], + detail=event['name'], scope='install-' + event['name']) + elif kind == 'freevideo_ui' and event.get('kind') in ('task', 'progress'): + # A nested Comfy environment has its own phases. Forward work, + # never its percentage into the outer installation's one bar. + update(done=event.get('done', 0), total=event.get('total', 0), unit=event.get('unit'), + activity=event.get('activity') or 'note', name=event.get('name') or event.get('label', ''), + detail=event.get('detail', ''), scope='comfy-' + str(event.get('key', ''))) + elif kind in ('verify_progress', 'assemble_progress') or (kind == 'network' and event.get('action') == 'verify-progress'): + update(done=event['done_bytes'], total=event['total_bytes'], unit='bytes', + activity='assemble' if kind == 'assemble_progress' else 'verify', + name=event.get('name') or model_download_name(event['file']).split(' · ')[0], + detail='Model files verifying' if kind != 'assemble_progress' else 'Model files assembling', + scope=kind + '-' + event['file']) + elif kind == 'model_groups': self.model_groups = event['groups'] continue if kind in ('loaded', 'sampling_start', 'sampling_progress', 'step', 'sample_finalize'): @@ -441,9 +483,11 @@ def update(**values): elif kind == 'prepare_fp8_phase': work = 'Verify cached weights' if event['phase'] == 'verify_cache' else 'Read and convert source weights' update(done=max(0, event['index'] - 1), total=event['total'], + activity='count', name='Model files verifying' if event['phase'] == 'verify_cache' else 'Model files preparing', detail=work + ' · ' + event['group'], scope='prepare-fp8-' + event['phase'], unit='items') elif kind == 'prepared_fp8_group': update(done=event.get('index', 0), total=event.get('total', 0), + activity='count', name='Model files preparing', detail=('Reuse verified FP8 ' if event.get('reused') else 'Prepare FP8 ') + event['group'], scope='prepare-fp8', unit='items') elif kind == 'storage_compacted': update(done=0, total=0, activity='', name='', @@ -513,7 +557,7 @@ def update(**values): state = event.get('state', 'running') active = state == 'running' self.parallel_tasks['verify-models'] = dict(label='Verify / reuse local models', state=state, - activity='check' if active else '', + activity='verify' if active else '', name=(event.get('name') or model_download_name(event.get('file', '')).split(' · ')[0]) if active else '', done=event.get('done_bytes', 0) if active else event.get('files_done', 0), total=event.get('total_bytes', 0) if active else event.get('files_total', 0), @@ -540,7 +584,7 @@ def update(**values): speed = event.get('network_bytes_per_second') shown_speed = speed if speed is not None else event['bytes_per_second'] update(done=done, total=total, scope=description + self.model_source, rate=event['bytes_per_second'], unit='bytes', - activity='download', name=event.get('name') or ('PyTorch' if description == 'torch' + activity='download', name=event.get('name') or ('Python' if description.startswith('cpython-') else 'PyTorch' if description == 'torch' else model_download_name(description).split(' · ')[0]), detail='%s%s%.1f MiB/s · %.1f / %.1f MiB · %s' % (source + ' · ' if source else '', From 3e7bdc4a3fdb936760881cc46460c061d2355773 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 17:25:46 +0000 Subject: [PATCH 19/27] Page check: wait out a slow startup before reporting that FreeVideo was not started - Another extension's slow setup delays FreeVideo's own; the check keeps waiting and clears the card once FreeVideo starts --- web/health.js | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/web/health.js b/web/health.js index 39870fe3..0207a59b 100644 --- a/web/health.js +++ b/web/health.js @@ -599,12 +599,26 @@ async function check() { // Opening the workspace waits for an animation frame. await sleep(1500); const problems = []; - const ui = window.FreeVideoUI || {}; - if (!ui.ready) { + if (!window.FreeVideoUI?.ready) { problems.push(...await diagnoseModules()); if (problems.length && problems.every(p => p.retry && !extensionProblem(p)) && await reloadOnce(problems)) return; } - if (!ui.ready && !problems.length) problems.push({text: t('FreeVideo’s scripts loaded, but ComfyUI did not start them.', 'FreeVideo 的脚本已下载,但 ComfyUI 没有启动它们。'), advice: 'reload'}); + // Every file loads, so ComfyUI is most likely still working through other + // extensions: it starts them one after another, and a slow one ahead of + // FreeVideo's delays it. Give it the rest of the minute before naming it. + if (!window.FreeVideoUI?.ready && !problems.length) await waitFor(() => window.FreeVideoUI?.ready, 50000); + const notStarted = {text: t('FreeVideo’s scripts loaded, but ComfyUI did not start them.', 'FreeVideo 的脚本已下载,但 ComfyUI 没有启动它们。'), advice: 'reload'}; + if (!window.FreeVideoUI?.ready && !problems.length) { + problems.push(notStarted); + // A start later than that clears the card instead of leaving it up. + const timer = setInterval(() => { + if (!window.FreeVideoUI?.ready) return; + clearInterval(timer); + if (!card || !lastProblems.length || !lastProblems.every(p => p === notStarted || p.advice === 'plugin')) return; + card.remove(); card = null; lastProblems = []; + send('ready', []); + }, 1000); + } if (!problems.length) problems.push(...await diagnoseStyles()); let checkStudio = false; if (launchVisit && !problems.length) { @@ -637,6 +651,8 @@ async function check() { && !reports.some(item => item.type === 'error' && item.where === 'studio')) problems.unshift({text: t('The creative workspace did not open.', '创作面板没有打开。'), advice: 'reload'}); problems.push(...lastProblems); + // Started while this check was waiting for the page to be in front. + if (window.FreeVideoUI?.ready && problems.includes(notStarted)) problems.splice(problems.indexOf(notStarted), 1); if (problems.length) { problems.push(...pluginErrors()); show(distinct(problems)); send('error', distinct(problems)); From 17f12c21c2844a10ada7e19ecdf9705a61f128bc Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 17:38:48 +0000 Subject: [PATCH 20/27] Third-party downloads list only the files this computer needs - The list can be exported as a download list, and a folder of downloaded files can be used directly - A folder this account cannot read says so - No separate sampling-cache option for third-party and offline-package installs; a missing level is fetched when a video needs it, and says what to do if that fails - The download list dialog keeps its buttons and list inside the dialog when labels are wide --- freevideo_engine/bootstrap.py | 54 +++- freevideo_engine/comfy_launcher_runtime.py | 15 +- freevideo_engine/comfy_setup.py | 2 + freevideo_engine/failure_details.py | 74 +++++ freevideo_engine/launcher/Main.qml | 143 ++++++--- freevideo_engine/launcher_session.py | 99 ++++++- freevideo_engine/model_files.json | 51 ++-- freevideo_engine/model_guidance.py | 322 +++++++++++++++++++-- freevideo_engine/modern_launcher.py | 45 ++- freevideo_engine/offline_packages.py | 9 +- freevideo_engine/provision.py | 11 +- freevideo_engine/sampling_assets.py | 97 ++++++- web/error_panel.js | 46 ++- 13 files changed, 855 insertions(+), 113 deletions(-) diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index e36988fb..ead734a7 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -354,6 +354,18 @@ def model_target(row, model_dir, encoder_dir, prepared_dir=None): return (model_dir if row['repo'].startswith('OpenVDN/') else encoder_dir) / row['file'] +def installation_model_files(prepared, reuse_cache, sampling_caches): + """The file selection shared by planning and provisioning.""" + from . import prepared_model + from .sampling_assets import install_files, usable_with + files = list(required_models(json.loads((PACKAGE / 'model_files.json').read_text(encoding='utf-8')), + reuse_cache or prepared)) + files += prepared_model.files(prepared) + if prepared or (reuse_cache and usable_with(reuse_cache)): + files += install_files(bool(sampling_caches)) + return files + + def resolve_prepared_format(requested, saved, hardware): """The prepared model format of this setup run: 'int8_convrot' or 'fp8'. @@ -530,18 +542,13 @@ def plan(args, *, local_progress=None, stage=None): prepared_dir = prepared['directory'] if prepared else None if mac_target and reuse_cache is None and prepared is None: errors.append('No verified native-compatible prepared model is available. Source conversion is not supported on Mac.') - files = required_models(json.loads((PACKAGE / 'model_files.json').read_text(encoding='utf-8')), reuse_cache or prepared) - files += prepared_model.files(prepared) # New installations prepare every quality level; an existing one keeps its # earlier choice (off if it predates the option) unless the flag says otherwise. sampling_caches = getattr(args, 'sampling_caches', None) if sampling_caches is None: sampling_caches = prior['sampling_caches'] if isinstance(prior.get('sampling_caches'), bool) else not saved.get('ready') sampling_caches = bool(sampling_caches) - from .sampling_assets import install_files, usable_with - if prepared or (reuse_cache and usable_with(reuse_cache)): - # Published tables match these weights; other caches compute their own. - files += install_files(sampling_caches) + files = installation_model_files(prepared, reuse_cache, sampling_caches) local_reuse = None local_folder = getattr(args, 'reuse_models', None) local_manifest = getattr(args, 'reuse_models_manifest', None) @@ -559,9 +566,19 @@ def plan(args, *, local_progress=None, stage=None): if not isinstance(libraries, dict) or libraries.get('version') != 1 or not isinstance(libraries.get('roots'), list): raise ValueError('Invalid model library manifest') directories = ([str(local_folder)] if local_folder else []) + libraries['roots'] - local_reuse = scan_many(directories, needed, prior=previous_reuse, callback=local_progress) + scan_library = lambda: scan_many(directories, needed, prior=previous_reuse, callback=local_progress) else: - local_reuse = scan(local_folder, needed, prior=previous_reuse, callback=local_progress) + scan_library = lambda: scan(local_folder, needed, prior=previous_reuse, callback=local_progress) + try: + local_reuse = scan_library() + except OSError as error: + if not getattr(args, 'download_list', False): + raise + from .model_guidance import DownloadListError + original = str(error) + if error.__cause__ is not None: + original += '\n' + str(error.__cause__) + raise DownloadListError('read', original_error=original) from error local_reuse['copy_mode'] = bool(getattr(args, 'copy_existing_models', False)) local_reuse['copy_bytes'] = local_reuse['linked_bytes'] = 0 for row in needed: @@ -577,18 +594,27 @@ def plan(args, *, local_progress=None, stage=None): mismatches = model_repair.read(root) replace_models = [] reserved = set() + check_download_list = bool(getattr(args, 'download_list', False)) + stamps_path = root / 'verified-models.json' + stamps = json.loads(stamps_path.read_text(encoding='utf-8')) if check_download_list and stamps_path.is_file() else {} for row in files: path = model_target(row, model_dir, encoder_dir, prepared_dir) replacement = model_repair.replacement(path, row, mismatches, reserved) if replacement: replace_models.append(replacement) size_matches = not replacement and path.is_file() and path.stat().st_size == row['bytes'] + if size_matches and check_download_list: + # Only the explicit third-party list needs this check during planning. + # Normal setup keeps its cheap estimate and deferred verification. + from .provision import verified + hf_directory = Path(prepared_dir) if row.get('prepared') else model_dir if row['repo'].startswith('OpenVDN/') else encoder_dir + size_matches = verified(path, row, stamps, mode=getattr(args, 'verify', 'auto'), hf_directory=hf_directory) present += row['bytes'] if size_matches else 0 device = existing_parent(path.parent) entry = groups.setdefault(str(device), {'path': str(device), 'needed_bytes': 0, 'free_bytes': disk_space.free_bytes(device)}) local = (local_reuse or {}).get('matches', {}).get(row['repo'] + '/' + row['file']) - model_entries.append((row, 'found' if size_matches else 'verified' if local else 'download')) + model_entries.append((row, ('verified' if check_download_list else 'found') if size_matches else 'verified' if local else 'download')) entry['needed_bytes'] += 0 if size_matches or local and local['method'] == 'hardlink' else row['bytes'] partial = path.with_suffix(path.suffix + '.partial') if not size_matches and not local and partial.is_file() and partial.stat().st_size <= row['bytes']: @@ -717,6 +743,9 @@ def plan(args, *, local_progress=None, stage=None): 'licenses': ['https://huggingface.co/OpenVDN/vdn-minimax-h3/blob/751739ee5b9e3ac802dca5d5111075fdaeb47885/LICENSE', 'https://huggingface.co/t8star/Vdn-Minimax-H3-Comfy', 'https://docs.nvidia.com/cuda/eula/index.html']} + from .model_guidance import download_files + value['download_files'] = download_files(model_entries) + value['download_files_verified'] = all(status != 'found' for _, status in model_entries) if mac_target: from .macos_bootstrap import describe_plan describe_plan(value) @@ -2081,6 +2110,7 @@ def run_main(argv, shown): parser.add_argument('--model-downloader', choices=['auto', 'xet'], default='auto', help='xet: require official HF Xet for large weights; no ModelScope/HTTP fallback. Small configs use HTTP; existing files/partials retained') parser.add_argument('--plan', '--check', action='store_true', help='Read-only preflight with small network samples; no installation files are written') + parser.add_argument('--download-list', action='store_true', help='Verify existing managed files while planning a third-party download list') parser.add_argument('--json', action='store_true', help='Print preflight as JSON') parser.add_argument('--plain', action='store_true', help='Disable live terminal rendering; append readable log lines') parser.add_argument('--verbose', action='store_true', help='Show full setup details, license links and per-step log paths') @@ -2118,7 +2148,11 @@ def curl_progress(message): stage=None if args.plan else lambda *step: prepare_step(ui, *step)) except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as error: prepare_end(ui, False) - print('Preflight failed: ' + str(error), file=sys.stderr) + from .model_guidance import DownloadListError + if isinstance(error, DownloadListError): + print('Download list error: ' + error.key, file=sys.stderr) + print('Preflight failed: ' + (error.original_error if isinstance(error, DownloadListError) else str(error)), + file=sys.stderr) return 1 print(json.dumps(value, indent=2)) if args.json else display(value, ui, verbose=args.verbose) if args.plan: diff --git a/freevideo_engine/comfy_launcher_runtime.py b/freevideo_engine/comfy_launcher_runtime.py index 03c9418b..7cefc0ed 100644 --- a/freevideo_engine/comfy_launcher_runtime.py +++ b/freevideo_engine/comfy_launcher_runtime.py @@ -958,7 +958,11 @@ def _inspect(self, values): self.selection = dict(descriptor, engine=str(engine), source=str(source), url=url, ready=ready, repair_comfy=repair_comfy, lan_access=values.get('lan_access') is True) self.state = dict(self.state, selection=dict(self.selection), host=host) - if not ready: + # Third-party downloads (manual models, no offline packages) are checked + # file by file, so files the user downloaded are found on a ready engine too. + checked = bool(values.get('download_list') or ( + values.get('model_method') == 'manual' and not values.get('offline_models'))) + if not ready or checked: extra = values.get('model_dirs', []) if not isinstance(extra, list) or any(not isinstance(p, str) for p in extra): raise ValueError('Model folders must be a list of directory paths') @@ -968,10 +972,19 @@ def _inspect(self, values): extra = [p for p in dict.fromkeys(extra) if p.strip() and io_path(Path(p).expanduser()).is_dir()] self.setup.inspect(dict(root=str(engine), extra_libraries=extra, copy=False, sampling_caches=bool(values.get('sampling_caches')), prepared_format=values.get('prepared_format'), + **({'download_list': True} if checked else {}), frontend=dict(root=descriptor['root'], separate=descriptor['separate'], download=fresh, repair_python=host['python'] if repair_comfy else None))) row = self._wait_setup() self.state = dict(self.state, plan=row['plan']) + # A ready receipt covers the engine, not every model file: when the + # file-by-file check finds models to place or download, accepting the + # plan prepares them before connecting. + plan = row['plan'] + if ready and checked and ( + plan.get('download_files') or (plan.get('local_models') or {}).get('matches')): + self.selection = dict(self.selection, ready=False) + self.state = dict(self.state, selection=dict(self.selection)) extra_disk, disks, disk_errors = disk_review(self.state.get('plan', {}), engine, descriptor['root'], separate=descriptor['separate'], new_comfy=fresh and not io_path(descriptor['root']).exists()) errors = list(self.state.get('plan', {}).get('errors', [])) diff --git a/freevideo_engine/comfy_setup.py b/freevideo_engine/comfy_setup.py index 9e881cc8..77f9104c 100644 --- a/freevideo_engine/comfy_setup.py +++ b/freevideo_engine/comfy_setup.py @@ -277,6 +277,8 @@ def inspect(self, value): arguments += ['--prepared-format', value['prepared_format']] if value.get('copy'): arguments.append('--copy-existing-models') + if value.get('download_list'): + arguments.append('--download-list') self.selection = dict(root=str(root), arguments=arguments) self.plan = None self.state = dict(status='running', action='plan', libraries=libraries) diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index c9f18d9a..455eabfa 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -9,6 +9,68 @@ STEP_OUTPUT = '\n\nFull step output:\n' +QUALITY_LEVELS = (('Light', '轻量'), ('Medium', '标准'), ('High', '精细'), ('Max', '极致')) + + +def sampling_table_details(value): + """Read the worker's bounded, numeric sampling-table failure metadata.""" + from .sampling_assets import UNAVAILABLE + for match in re.finditer(r'^(?:(?:[\w.]*(?:Error|Exception)|(?:[\w.]+\.)?SamplingTableUnavailable):\s*)?' + + re.escape(UNAVAILABLE) + r'(\{[^\n]*\})$', + str(value)[:65536], re.M): + try: + details = json.loads(match[1]) + if (type(details['base_steps']) is int and 1 <= details['base_steps'] <= 32 + and type(details['two_pass']) is bool + and type(details['refine_steps']) is int and 0 <= details['refine_steps'] <= 31 + and type(details['missing_bytes']) is int and details['missing_bytes'] > 0 + and type(details['decimal_sizes']) is bool and isinstance(details['available'], list) + and all(type(i) is int and 0 <= i <= 3 for i in details['available'])): + return {key: details[key] for key in + ('base_steps', 'two_pass', 'refine_steps', 'missing_bytes', 'available', 'decimal_sizes')} + except (ValueError, KeyError, TypeError): + pass + return None + + +def sampling_table_failure(value, *, zh=False): + details = sampling_table_details(value) + if details is None: + return None + from .launcher_copy import size_text + language = 1 if zh else 0 + index = next((i for i, steps in enumerate((8, 12, 16, 20)) + if details['base_steps'] == steps and details['two_pass'] == (i == 0) + and (i != 0 or details['refine_steps'] == 3)), None) + names = [('“%s”' if zh else '%s') % QUALITY_LEVELS[i][language] for i in sorted(set(details['available']))] + available = '' + if names: + available = names[0] if len(names) == 1 else ('、' if zh else ', ').join(names[:-1]) + ('或' if zh else ' or ') + names[-1] + size = size_text(details['missing_bytes'], details['decimal_sizes']) + # A custom step count is not a quality level (Iris): no "档" / "quality level" for it. + if zh: + if index is None: + title = '“自定义(%d 步)”的采样表还没有下载' % details['base_steps'] + what = '这个步数的采样表' + else: + title = '“%s”档的采样表还没有下载' % QUALITY_LEVELS[index][1] + what = '这一档的采样表' + detail = ('这次生成需要先下载%s(约 %s),但这次没能下载。' + '请确认网络连接后再生成一次,FreeVideo 会自动下载。') % (what, size) + if available: + detail += '也可以改用已经可以使用的%s档。' % available + else: + if index is None: + title = 'The sampling table for Custom (%d steps) is not downloaded yet' % details['base_steps'] + what = 'the sampling table for this step count' + else: + title = "The %s quality level's sampling table is not downloaded yet" % QUALITY_LEVELS[index][0] + what = "that level's sampling table" + detail = ('This video first needs %s (about %s), and it could not be downloaded this time. ' + 'Check the network connection and generate again; FreeVideo downloads it automatically.') % (what, size) + if available: + detail += ' You can also switch to a quality level that is already available: %s.' % available + return dict(kind='sampling-table', title=title, detail=detail, action='') def redacted_launcher_error(value, replacements=()): @@ -320,6 +382,9 @@ def launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_rep return dict(title='GPU 检查未通过' if zh else 'GPU validation failed', detail=detail, action='展开详情或导出报告,查看完整检查结果。' if zh else 'Open the details or export the report for the complete results.', kind='kernels') + if 'Download list error: read' in text.splitlines(): + from .model_guidance import DOWNLOAD_COPY + return dict(title=DOWNLOAD_COPY['error_read'][zh], detail='', action='', kind='download-list-read') rules = ( (r'The previous video has no usable continuation data', 'continuation-missing', ('This video cannot be continued', '这个视频暂时不能接续'), @@ -578,7 +643,16 @@ def generation_failure(run, exit_code=None, *, repair=None): if len(lines) == 1: lines.append('The worker ended without a readable error report.') lines.append('\nRetained outputs: ' + str(run)) + # The table failure can be in a retained child log rather than the report. + # Keep its numeric metadata ahead of technical output for all interactive callers. + sampling = sampling_table_details('\n'.join(lines)) value = redactor.text('\n'.join(lines)) + if sampling: + from .sampling_assets import UNAVAILABLE + marker = UNAVAILABLE + json.dumps(sampling, separators=(',', ':')) + card = sampling_table_failure(marker) + first, _, rest = value.partition('\n') + value = '\n'.join((first, marker, card['title'], card['detail'], rest)) if explained: from .runtime_libraries import REINSTALL value = value.replace(REPAIR_MARK, repair or REINSTALL, 1) diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml index 1d05b042..22495fb6 100644 --- a/freevideo_engine/launcher/Main.qml +++ b/freevideo_engine/launcher/Main.qml @@ -9,6 +9,9 @@ ApplicationWindow { color: unifiedChrome ? theme.canvas : theme.bg width: 1280; height: 860 property var s: initialState + readonly property var downloads: backend.downloadList + // Keep expansion by family across language changes and new checked lists. + property var downloadExpanded: ({}) Connections { target: backend; function onChanged() { win.s = backend.state } } property bool settingsOpen: false property bool terminalOpen: false @@ -29,7 +32,7 @@ ApplicationWindow { property string previousReview: "" property bool compact: width < 1000 property bool shortWindow: height < 700 - property bool otherModelLinksOpen: false + onModelInfoOpenChanged: if (modelInfoOpen) backend.prepareDownloadList() readonly property bool manualEnvironment: s.offline.runtime_supported !== false && s.form.new_comfy && s.form.environment_method === "manual" readonly property bool usingRuntime: manualEnvironment && s.offline.runtime readonly property bool needsRuntime: s.page === "comfy" && manualEnvironment && !s.offline.runtime @@ -597,19 +600,7 @@ ApplicationWindow { delegate: FButton { required property var modelData; required property int index; objectName: "offlineShare-" + index; text: t("Download from Quark ↗", "打开夸克网盘 ↗"); onClicked: backend.link(modelData.url) } } FText { objectName: "offlinePackageGuide"; text: s.offline.guide; color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true } - FButton { objectName: "otherModelLinks"; text: t("Alternative download sources", "其他下载渠道") + (otherModelLinksOpen ? " −" : " +"); flat: true; implicitHeight: theme.heightSm - 2; leftPadding: 0; font.pixelSize: theme.micro + 1; onClicked: otherModelLinksOpen = !otherModelLinksOpen } - FText { visible: otherModelLinksOpen; text: t("The same models are also available from Hugging Face or ModelScope.", "同一套模型,也可从 Hugging Face 或魔搭下载。"); color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true } - Flow { - visible: otherModelLinksOpen; Layout.fillWidth: true; spacing: 8 - Repeater { - model: ["video", "encoder", "decoder"] - delegate: FButton { - required property string modelData - text: modelData === "video" ? t("Video model ↗", "视频模型 ↗") : modelData === "encoder" ? t("Text encoder ↗", "文本编码器 ↗") : t("Video & audio decoders ↗", "音视频解码器 ↗") - implicitHeight: theme.heightSm; font.pixelSize: theme.micro + 1; onClicked: { modelInfo = modelData; modelInfoOpen = true } - } - } - } + FButton { objectName: "otherModelLinks"; text: downloads.copy.title + " ›"; flat: true; enabled: !s.busy; implicitHeight: theme.heightSm; leftPadding: 0; font.pixelSize: theme.micro + 1; onClicked: modelInfoOpen = true } } ColumnLayout { Layout.fillWidth: true; Layout.preferredWidth: 1; Layout.alignment: Qt.AlignTop; spacing: 10 @@ -635,7 +626,7 @@ ApplicationWindow { FText { visible: !!s.offline.detail; text: s.offline.detail + (number(s.offline.total) ? " · " + bytes(s.offline.done) + " / " + bytes(s.offline.total) : ""); color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true } } FCard { - visible: !usingRuntime; Layout.fillWidth: true; padding: 16 + visible: !manualEnvironment && !offlineSelected; Layout.fillWidth: true; padding: 16 FSwitch { objectName: "samplingCaches"; Layout.fillWidth: true text: t("Prepare all quality levels", "提前下载全部质量档位") @@ -740,7 +731,7 @@ ApplicationWindow { } // What this group downloads, as in the plan; nothing for one already here. FText { objectName: "modelSize-" + modelData.id; visible: modelData.download > 0; text: bytes(modelData.download); font.pixelSize: theme.micro; color: theme.muted; font.features: { "tnum": 1 } } - FButton { text: t("Download links ↗", "下载地址 ↗"); implicitHeight: theme.heightSm; flat: true; font.pixelSize: theme.micro + 1; visible: s.form.model_method === "manual"; onClicked: { modelInfo = modelData.id === "text" ? "encoder" : modelData.id; modelInfoOpen = true } } + FButton { objectName: "modelFiles-" + modelData.id; text: downloads.copy.title + " ›"; implicitHeight: theme.heightSm; flat: true; font.pixelSize: theme.micro + 1; visible: s.form.model_method === "manual"; onClicked: modelInfoOpen = true } } } } @@ -1278,25 +1269,111 @@ ApplicationWindow { FPopup { id: modelDialog; objectName: "modelDialog" visible: modelInfoOpen; onClosed: modelInfoOpen = false - width: Math.min(540, win.width-50) - height: Math.min(modelContents.implicitHeight + padding*2, win.height-48) - contentItem: ScrollView { - id: modelScroll; clip: true; contentWidth: availableWidth - ColumnLayout { - id: modelContents; width: modelScroll.availableWidth - spacing: 12 - FText { text: t("Download models", "下载模型"); font.pixelSize: theme.section + 2; font.weight: Font.DemiBold } - FText { objectName: "videoModelGuide"; text: modelInfo === "video" ? s.video_model_guide : modelInfo === "decoder" ? t("Download the vae and audio_vae folders.", "下载 vae 和 audio_vae 文件夹。") : t("Download the text encoder to your model folder.", "下载文本编码器,放入模型目录。"); color: theme.muted; Layout.fillWidth: true; Layout.bottomMargin: 4 } - Repeater { model: modelLinks[modelInfo]; delegate: FButton { required property var modelData; required property int index; objectName: "modelLink-" + index; text: t(modelData.label, modelData.label_zh) + " ↗"; Layout.fillWidth: true; onClicked: backend.link(modelData.url) } } - Repeater { model: cloudLinks; delegate: FButton { required property var modelData; text: t("Quark · ", "夸克 · ") + modelData.label + " ↗"; Layout.fillWidth: true; onClicked: backend.link(modelData.url) } } - FText { text: t("When your download finishes, import a FreeVideo ZIP or choose the folder containing your models.", "下载完成后,导入 FreeVideo ZIP 或选择存放模型的文件夹。"); color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true; Layout.topMargin: 6 } + width: Math.min(940, win.width-50) + readonly property real maximumHeight: Math.min(680, win.height-48) + implicitHeight: Math.min(maximumHeight, modelContent.implicitHeight + topPadding + bottomPadding) + contentItem: ColumnLayout { + id: modelContent; spacing: 12 + ScrollView { + id: modelScroll; objectName: "downloadListScroll" + Layout.fillWidth: true; Layout.fillHeight: true; Layout.minimumWidth: 0; Layout.minimumHeight: 0 + implicitHeight: downloadBody.implicitHeight + clip: true; contentWidth: availableWidth + ColumnLayout { + id: downloadBody; width: modelScroll.availableWidth; spacing: 12 + FText { text: downloads.copy.title; font.pixelSize: theme.section + 2; font.weight: Font.DemiBold; Layout.fillWidth: true } + FText { objectName: "downloadListTotal"; visible: downloads.available; text: downloads.total; font.weight: Font.DemiBold; color: theme.accent; Layout.fillWidth: true } + FText { objectName: "videoModelGuide"; text: downloads.copy.guide; color: theme.muted; Layout.fillWidth: true; font.pixelSize: theme.micro + 1 } + FText { objectName: "downloadListCheck"; visible: !downloads.available && !s.download_list.inspection_error; text: s.download_list.checking ? downloads.copy.checking : downloads.copy.unavailable; Layout.fillWidth: true; color: theme.muted } + FText { objectName: "downloadListReuse"; visible: downloads.available && text.length > 0; text: downloads.reused; color: theme.success; Layout.fillWidth: true; font.pixelSize: theme.micro + 1 } + ColumnLayout { + Layout.fillWidth: true; spacing: 8 + Repeater { + model: downloads.available ? downloads.groups : [] + delegate: ColumnLayout { + id: downloadGroup + required property var modelData + readonly property bool expanded: !!win.downloadExpanded[modelData.id] + Layout.fillWidth: true; spacing: 4 + FButton { + id: groupButton + objectName: "downloadGroup-" + downloadGroup.modelData.id + text: (downloadGroup.expanded ? "− " : "+ ") + downloadGroup.modelData.title + Accessible.name: text + " · " + downloadGroup.modelData.summary + Layout.fillWidth: true; flat: true; leftPadding: 8; rightPadding: 8 + contentItem: RowLayout { + spacing: 8 + FText { objectName: groupButton.objectName + "-title"; text: groupButton.text; font: groupButton.font; Layout.fillWidth: true; horizontalAlignment: Text.AlignLeft; color: groupButton.hovered ? theme.text : theme.muted } + FText { objectName: groupButton.objectName + "-summary"; text: downloadGroup.modelData.summary; font: groupButton.font; horizontalAlignment: Text.AlignRight; color: theme.muted } + } + onClicked: { + var expanded = Object.assign({}, win.downloadExpanded) + expanded[downloadGroup.modelData.id] = !downloadGroup.expanded + win.downloadExpanded = expanded + } + } + Loader { + active: downloadGroup.expanded; visible: active; Layout.fillWidth: true + sourceComponent: ColumnLayout { + spacing: 2 + Repeater { + model: downloadGroup.modelData.files + delegate: RowLayout { + required property var modelData; required property int index + objectName: "downloadFile-" + downloadGroup.modelData.id + "-" + index + Layout.fillWidth: true; spacing: 8 + FText { objectName: parent.objectName + "-name"; text: modelData.name; Layout.fillWidth: true; elide: Text.ElideMiddle; maximumLineCount: 1; font.pixelSize: theme.micro } + FText { text: modelData.size; Layout.preferredWidth: 104; font.pixelSize: theme.micro; color: theme.muted; horizontalAlignment: Text.AlignRight } + FButton { objectName: "downloadHF-" + downloadGroup.modelData.id + "-" + index; text: "Hugging Face ↗"; flat: true; implicitHeight: theme.heightSm; font.pixelSize: theme.micro; onClicked: backend.link(modelData.hf_url) } + Item { + id: mirrorSlot + readonly property var file: modelData + readonly property string suffix: downloadGroup.modelData.id + "-" + index + objectName: "downloadMSSlot-" + suffix + // Reserve the same width whether a mirror exists or not. + implicitWidth: Math.max(80, mirrorMetrics.width + 32, missingMetrics.width + 32) + implicitHeight: theme.heightSm + TextMetrics { id: mirrorMetrics; text: sourceName("modelscope") + " ↗"; font.pixelSize: theme.micro; font.weight: Font.Medium } + TextMetrics { id: missingMetrics; text: downloads.copy.missing_mirror; font.pixelSize: theme.micro } + Loader { + anchors.fill: parent + sourceComponent: mirrorSlot.file.modelscope_url ? mirrorButton : mirrorUnavailable + } + Component { + id: mirrorButton + FButton { objectName: "downloadMS-" + mirrorSlot.suffix; text: sourceName("modelscope") + " ↗"; flat: true; implicitHeight: theme.heightSm; font.pixelSize: theme.micro; onClicked: backend.link(mirrorSlot.file.modelscope_url) } + } + Component { + id: mirrorUnavailable + FText { objectName: "downloadMSMissing-" + mirrorSlot.suffix; text: downloads.copy.missing_mirror; color: theme.muted; font.pixelSize: theme.micro; horizontalAlignment: Text.AlignHCenter; verticalAlignment: Text.AlignVCenter } + } + } + } + } + } + } + } + } + } + RowLayout { + visible: s.download_list.folder.length > 0; Layout.fillWidth: true; spacing: 8 + FText { objectName: "downloadListNotice"; visible: text.length > 0; text: s.download_list.notice; color: theme.success; Layout.fillWidth: true; font.pixelSize: theme.micro + 1 } + FButton { objectName: "openDownloadFolder"; text: downloads.copy.open_folder; flat: true; implicitHeight: theme.heightSm; font.pixelSize: theme.micro + 1; onClicked: backend.openDownloadFolder() } + } + FText { objectName: "downloadListError"; visible: text.length > 0; text: s.download_list.error; color: theme.danger; Layout.fillWidth: true; wrapMode: Text.WordWrap; font.pixelSize: theme.micro + 1 } + FText { objectName: "downloadListRawError"; visible: s.error.length > 0 && !s.download_list.inspection_error; text: s.error; color: theme.danger; Layout.fillWidth: true; maximumLineCount: 2; elide: Text.ElideRight; font.pixelSize: theme.micro } + } + } RowLayout { - Layout.fillWidth: true; spacing: 8 - FButton { text: t("Close", "关闭"); flat: true; onClicked: modelInfoOpen = false } + objectName: "downloadListActions"; Layout.fillWidth: true; Layout.minimumWidth: 0; spacing: 8 + // The actions share what the Close button leaves, so a long label wraps instead of leaving the dialog. + readonly property int actionCount: checkDownloadList.visible ? 3 : 2 + readonly property real actionWidth: Math.max(80, Math.floor((width - closeDownloadList.implicitWidth - (actionCount + 1) * spacing) / actionCount)) + FButton { id: closeDownloadList; text: downloads.copy.close; flat: true; onClicked: modelInfoOpen = false } Item { Layout.fillWidth: true } - FButton { text: t("Choose model folder", "选择模型目录"); onClicked: { modelInfoOpen = false; backend.browse("model_dirs") } } - FButton { text: t("Import ZIPs", "导入 ZIP"); primary: true; onClicked: { modelInfoOpen = false; backend.browsePackages() } } - } + FButton { id: checkDownloadList; objectName: "checkDownloadList"; Layout.maximumWidth: parent.actionWidth; visible: !downloads.available || s.download_list.needs_check; text: downloads.copy.check; enabled: !s.busy; onClicked: backend.prepareDownloadList() } + FButton { objectName: "useDownloadedFolder"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.use_folder; enabled: !s.busy; onClicked: backend.useDownloadedFolder() } + FButton { objectName: "exportDownloadList"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.export; primary: true; enabled: downloads.available && !s.busy; onClicked: backend.exportDownloadList() } } } } diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index 16b3bd69..a23e4e8a 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -1,4 +1,5 @@ """Widget-independent state for the Qt launcher; uses the existing installer.""" +from copy import deepcopy import json import os from pathlib import Path @@ -17,7 +18,8 @@ from .launcher_copy import size_text from .launcher_terminal import Tail from .model_status import FAMILIES, NAMES -from .model_guidance import package_instructions, video_instructions, runtime_packages_supported +from .model_guidance import (package_instructions, video_instructions, runtime_packages_supported, download_view, + export_download_list, DownloadListError, DOWNLOAD_COPY) from .setup_progress import progress_text from .terminal_ui import clean, duration from .model_upgrade import TARGET @@ -109,6 +111,8 @@ def __init__(self, source=None, *, controller=None, store=None, updater=None, sm self.tail = Tail() self.log_source = None self.model_groups = [] + self.download_export = self.download_error = self.download_error_key = '' + self._download_view = None self.browser_attempted = False self.browser_url = None self.port_notice_url = None @@ -258,6 +262,7 @@ def edit(self, key, value): self.cleaner.state = dict(status='idle', bytes=0, error='') self.controller.selection = None self.controller.state = dict(status='idle') + self.download_export = self.download_error = self.download_error_key = '' self.browser_attempted = False self.port_notice_url = None self.browser_error = self.error = '' @@ -284,6 +289,65 @@ def add_folder(self, folder): kept = [p for p in self.form['model_dirs'] if io_path(Path(p).expanduser()).is_dir()] self.edit('model_dirs', library_roots(kept + [folder])) + def prepare_download_list(self): + if self.controller.busy or self.importer.busy or self.cleaner.busy: + return + if (self.controller.state.get('plan', {}).get('download_files_verified') + and self.controller.state.get('status') == 'review' + and self.download_error_key not in ('unchecked', 'changed')): + return + self.error = '' + self.download_export = self.download_error = self.download_error_key = '' + self.persist() + self.controller.run('inspect', self.setup_values(download_list=True)) + + def export_downloads(self, folder): + """The dialog shows a worded error; nothing raises into a traceback.""" + self.download_export = self.download_error = self.download_error_key = '' + if self.controller.busy or self.controller.state.get('status') != 'review': + self.download_error = self.t('Wait for the file check to finish.', '请等待文件检查完成。') + return + preferences = download_preferences(self.engine_root() / 'download-settings.json') + try: + self.download_export = export_download_list(self.controller.state.get('plan', {}), folder, preferences['source']) + except DownloadListError as error: + self.download_error_key = error.key + reason = '' + if error.key == 'write': + reason = self.t(*DOWNLOAD_COPY.get('write_' + error.reason, DOWNLOAD_COPY['write_other'])) + # Session has no log writer; keep the OS detail on stderr, without the user's paths. + from .failure_details import redacted_launcher_error + print('Download list export failed (%s): %s' % (error.reason, redacted_launcher_error(error.original_error)), + file=sys.stderr) + self.download_error = self.t(*DOWNLOAD_COPY['error_' + error.key]).format(reason=reason) + + def use_downloaded_folder(self, folder): + if self.controller.busy or self.importer.busy or self.cleaner.busy: + return + self.add_folder(folder) + # Even a folder already in the list may have received new downloads. + self.controller.state = dict(status='idle') + self.prepare_download_list() + + def downloads_view(self): + row = self.controller.state + plan = row.get('plan', {}) if row.get('status') == 'review' else {} + if not plan.get('download_files_verified', True): + plan = {} + zh = self.language.startswith('zh') + decimal_sizes = sys.platform == 'darwin' + # Only inputs that affect list content belong here. Disk/progress/notice + # updates must not replace the QML model or its file delegates. + key = ('download_files' in plan, plan.get('download_files', []), + (plan.get('local_models') or {}).get('matches', {}), zh, decimal_sizes) + if self._download_view is None or self._download_view[0] != key: + view = download_view(plan, zh, decimal_sizes=decimal_sizes) + if self._download_view is None or self._download_view[1] != view: + self._download_view = (deepcopy(key), view) + else: + self._download_view = (deepcopy(key), self._download_view[1]) + return self._download_view[1] + def remove_folder(self, index): removed = self.form['model_dirs'][index] self.edit('model_dirs', [p for i, p in enumerate(self.form['model_dirs']) if i != index]) @@ -319,10 +383,20 @@ def prepare_offline(self): if self.archives_busy: return if self.form['model_method'] == 'auto' and not self.form['offline_models']: - self.importer.download(self.form['offline_runtime'], self.source, self.form['sampling_caches']) + self.importer.download(self.form['offline_runtime'], self.source, False) else: self.importer.prepare(self.form['offline_runtime'], self.form['offline_models'], self.source) + def setup_values(self, form=None, **extra): + """Only the visible automatic-setup option can request every quality level.""" + form = self.form if form is None else form + values = dict(form, token=self.token, **extra) + if (form['model_method'] == 'manual' or + (runtime_packages_supported() and form['new_comfy'] + and form['environment_method'] == 'manual')): + values['sampling_caches'] = False + return values + def activate_offline(self, root): from .offline_packages import check_runtime_platform check_runtime_platform() @@ -370,7 +444,7 @@ def action(self, name, accepted=False): self.remember(self.selected, persist=False) self.repair_request = dict(self.form, repair=True) self.persist() - self.controller.run('inspect', dict(self.repair_request, token=self.token)) + self.controller.run('inspect', self.setup_values(self.repair_request)) self.page = 'progress' self.model_groups = [] self.started = time.monotonic() @@ -390,7 +464,7 @@ def action(self, name, accepted=False): # Recompute the plan and live disk space; never install a stale # failed review or discard completed/resumable downloads. self.persist() - self.controller.run('inspect', dict(self.repair_request or self.form, token=self.token)) + self.controller.run('inspect', self.setup_values(self.repair_request or self.form)) self.page = 'progress' self.started = time.monotonic() return @@ -471,7 +545,7 @@ def action(self, name, accepted=False): self.controller.run('connect') else: self.persist() - self.controller.run('inspect', dict(self.repair_request or self.form, token=self.token)) + self.controller.run('inspect', self.setup_values(self.repair_request or self.form)) self.page = 'progress' self.model_groups = [] self.started = time.monotonic() @@ -605,7 +679,7 @@ def _tick_model_upgrade(self, row, busy): self.model_switch = 'inspect' self.repair_request = None upgrade.switching() - self.controller.run('inspect', dict(self.form, token=self.token, repair=True, prepared_format='int8')) + self.controller.run('inspect', self.setup_values(repair=True, prepared_format='int8')) self.page = 'progress' self.model_groups = [] self.started = time.monotonic() @@ -944,7 +1018,7 @@ def _update_engine_when_idle(self): self.port_notice_url = None self.browser_wait = None self.persist() - self.controller.run('inspect', dict(self.form, token=self.token)) + self.controller.run('inspect', self.setup_values()) self.page = 'progress' self.model_groups = [] self.started = time.monotonic() @@ -1712,7 +1786,16 @@ def snapshot(self): self.archives_prompt_offer if self.archives_retry_dialog else self.archive_offer()) archives_idle = bool(archive_offer and archive_offer['paths']) and self.archives_idle() archive_dialog = self.archives_dialog_view(archive_offer, archives_idle, error, update) - return plain_data(dict(sampling_cache_bytes=total_bytes(), version=__version__, + downloads = self.downloads_view() + inspection_read_error = (row.get('status') == 'failed' and row.get('action') == 'inspect' + and 'Download list error: read' in row.get('error', '').splitlines()) + download_state = dict(checking=bool(self.controller.busy), error=self.download_error, + needs_check=self.download_error_key in ('unchecked', 'changed'), folder=self.download_export, + inspection_error=inspection_read_error, + notice=downloads['copy']['exported'].format(folder=self.download_export) if self.download_export else '') + if inspection_read_error: + download_state.update(error=self.t(*DOWNLOAD_COPY['error_read']), needs_check=True) + return plain_data(dict(download_list=download_state, sampling_cache_bytes=total_bytes(), version=__version__, disk=disk, decimal_sizes=sys.platform == 'darwin', zh=self.language.startswith('zh'), form=dict(self.form), page=self.page, status=row.get('status', 'idle'), busy=self.controller.busy or self.importer.busy or self.cleaner.busy, cleanup=cleanup, diff --git a/freevideo_engine/model_files.json b/freevideo_engine/model_files.json index 51a4513a..9e215d8e 100644 --- a/freevideo_engine/model_files.json +++ b/freevideo_engine/model_files.json @@ -5,7 +5,8 @@ "file": "LICENSE", "bytes": 17604, "sha256": null, - "git_blob": "c389b45855337ccec8ddeb389f4fe902abcd0b19" + "git_blob": "c389b45855337ccec8ddeb389f4fe902abcd0b19", + "download_sha256": "59b99642b95ea21630e311198ddbfffbfe05aadba0c2f5d884cbdf4efcc90f44" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -13,7 +14,8 @@ "file": "h3-base/LICENSE", "bytes": 17604, "sha256": null, - "git_blob": "c389b45855337ccec8ddeb389f4fe902abcd0b19" + "git_blob": "c389b45855337ccec8ddeb389f4fe902abcd0b19", + "download_sha256": "59b99642b95ea21630e311198ddbfffbfe05aadba0c2f5d884cbdf4efcc90f44" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -21,7 +23,8 @@ "file": "h3-base/README.md", "bytes": 38391, "sha256": null, - "git_blob": "aa9bba6233ed8d0b486fb43e1a35b4fb5ede2a60" + "git_blob": "aa9bba6233ed8d0b486fb43e1a35b4fb5ede2a60", + "download_sha256": "89a98af9e91fcb54c46cbcb88f8b8aa7aaf3a461714dbf21d2d3ef4b39312ee3" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -29,7 +32,8 @@ "file": "h3-base/audio_scheduler/scheduler_config.json", "bytes": 96, "sha256": null, - "git_blob": "28a3d4ed602d925031c7e5d6eeb07df8958ff0f1" + "git_blob": "28a3d4ed602d925031c7e5d6eeb07df8958ff0f1", + "download_sha256": "804780f7133477067bd6bbfbc02dc8b3cf9feeb400f97c08f5b1d5f6cbab3840" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -37,7 +41,8 @@ "file": "h3-base/audio_vae/config.json", "bytes": 2271, "sha256": null, - "git_blob": "15df6fe0b288b260dca0f2aebf1cd81334d77e96" + "git_blob": "15df6fe0b288b260dca0f2aebf1cd81334d77e96", + "download_sha256": "9a3c645ff892b376c6f5f4c8685964cd75474731af594ff058492a0000caabb6" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -53,7 +58,8 @@ "file": "h3-base/model_index.json", "bytes": 2936, "sha256": null, - "git_blob": "454be19164878173a962c39c81633e4377880fca" + "git_blob": "454be19164878173a962c39c81633e4377880fca", + "download_sha256": "5a587fe13b2371427415ac892463142683aefcd8d322e274a3a095eac37ac7d2" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -61,7 +67,8 @@ "file": "h3-base/modular_model_index.json", "bytes": 2935, "sha256": null, - "git_blob": "5b69f1e801bd0a7e9733c48704c732339f2002fe" + "git_blob": "5b69f1e801bd0a7e9733c48704c732339f2002fe", + "download_sha256": "a2b6a210e482ffb78e613b553f570c44e101afce6741bd4ed91429d0559af031" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -69,7 +76,8 @@ "file": "h3-base/scheduler/scheduler_config.json", "bytes": 97, "sha256": null, - "git_blob": "1d44f95b955a648ad4fab455e4fe11053888c30b" + "git_blob": "1d44f95b955a648ad4fab455e4fe11053888c30b", + "download_sha256": "8fa6c3aa70dc9e691e1a6df899fd1b6f75f70481a27cee6e18a303817075c304" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -77,7 +85,8 @@ "file": "h3-base/transformer/config.json", "bytes": 546, "sha256": null, - "git_blob": "c251a4133c81852a150c6668a570e142dc776b6e" + "git_blob": "c251a4133c81852a150c6668a570e142dc776b6e", + "download_sha256": "74c11bff524336576096993cbfcdcdc2ef4fa2fa4409df693bdcbc6c666282ae" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -197,7 +206,8 @@ "file": "h3-base/transformer/diffusion_pytorch_model.safetensors.index.json", "bytes": 64488, "sha256": null, - "git_blob": "da038def25abb5a5e4be509816ee55a0b7105e09" + "git_blob": "da038def25abb5a5e4be509816ee55a0b7105e09", + "download_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -205,7 +215,8 @@ "file": "h3-base/vae/config.json", "bytes": 2011, "sha256": null, - "git_blob": "ed4874a5886972f4029fcd99cbbe72de0177b2c9" + "git_blob": "ed4874a5886972f4029fcd99cbbe72de0177b2c9", + "download_sha256": "78f67deec3d63aae807f2bfe7154bc1e26f6372cb20b63265fcbae1b62bb5745" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -237,7 +248,8 @@ "file": "h3-base/vae/diffusion_pytorch_model.safetensors.index.json", "bytes": 74228, "sha256": null, - "git_blob": "05dfb255c44daa4f1ac487f57f18210fa5c19439" + "git_blob": "05dfb255c44daa4f1ac487f57f18210fa5c19439", + "download_sha256": "15f6d44553c3c616b0dc999920aa784f92ecee7e4201f1f99ac405cfbf3061ca" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -245,7 +257,8 @@ "file": "stage-dmd-step-250/adapters/default/adapter_config.json", "bytes": 415, "sha256": null, - "git_blob": "ef3c876ab5a596a1b6b4b28f3443bbe30c5455d8" + "git_blob": "ef3c876ab5a596a1b6b4b28f3443bbe30c5455d8", + "download_sha256": "89e0ff8920b9629b826eb99ab6150cce6924a53aa445186d1b493225fd091b96" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -261,7 +274,8 @@ "file": "stage-dmd-step-250/adapters/turbo/adapter_config.json", "bytes": 22264, "sha256": null, - "git_blob": "dbd7be544c9f3092b229f5cb6487f8da2f4ccc05" + "git_blob": "dbd7be544c9f3092b229f5cb6487f8da2f4ccc05", + "download_sha256": "627968f670747c29cd7a0d3f8c75166e501d70f9e829b5cd3242a8f14cefbc18" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -277,7 +291,8 @@ "file": "stage-dmd-step-250/linear_branch/config.json", "bytes": 465, "sha256": null, - "git_blob": "314502a7c9d403f28493ac1cb5054032c8c7b545" + "git_blob": "314502a7c9d403f28493ac1cb5054032c8c7b545", + "download_sha256": "decb06ac7e664610f677fb445318502b3c51f9c1b8603a2cdd00b16042be5bc8" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -293,7 +308,8 @@ "file": "stage-dmd-step-250/metadata.json", "bytes": 463, "sha256": null, - "git_blob": "27ec152079222e3f2a83b8ee0bc634a77da0df01" + "git_blob": "27ec152079222e3f2a83b8ee0bc634a77da0df01", + "download_sha256": "54054ceb1c91b3fdf7fa0278e4a8841c127e8cf666b5e240d69613661f9d3e9e" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -301,7 +317,8 @@ "file": "stage-dmd-step-250/model_spec.json", "bytes": 25705, "sha256": null, - "git_blob": "e039c13987797f91e11a3477c38fa8fd1f6a506f" + "git_blob": "e039c13987797f91e11a3477c38fa8fd1f6a506f", + "download_sha256": "4171f4384e952f1f73467981a893440c03298af4956b947e2c8a857ba9f5a62b" }, { "repo": "t8star/Vdn-Minimax-H3-Comfy", diff --git a/freevideo_engine/model_guidance.py b/freevideo_engine/model_guidance.py index 2c50da50..99dc560c 100644 --- a/freevideo_engine/model_guidance.py +++ b/freevideo_engine/model_guidance.py @@ -1,9 +1,302 @@ """User-facing model download destinations, from the installation manifests.""" +from contextlib import ExitStack +import errno +import hashlib import json -from pathlib import Path +import os +from pathlib import Path, PurePosixPath +import re +import shutil import sys -from urllib.parse import quote, urlsplit +import tempfile +import time +import uuid +from urllib.parse import urlsplit + +from . import network from .long_paths import io_path, plain_errors +from .model_status import FAMILIES, NAMES, family + +# Iris can replace the third-party download copy here, including export guidance. +DOWNLOAD_COPY = { + 'title': ('Download list', '下载清单'), + 'guide': ('The list is filtered for this GPU, so the whole model repository is not needed. ' + 'When the download finishes, choose "Use downloaded folder"; FreeVideo checks every file before using it.', + '清单已按显卡筛选,无需下载整个模型仓库。下载完成后点“使用已下载的文件夹”,FreeVideo 会逐个校验后使用。'), + 'export': ('Export download list', '导出下载清单'), + 'use_folder': ('Use downloaded folder', '使用已下载的文件夹'), + 'checking': ('Checking the hardware and the downloaded files…', '正在检查硬件和已下载的文件…'), + 'check': ('Check again', '重新检查'), + 'unavailable': ('The download list is not ready yet. Choose "Check again".', '下载清单尚未生成,请点“重新检查”。'), + 'total': ('To download: {count} files ({size})', '需要下载:{count} 个文件({size})'), + 'total_remaining': ('Still to download: {count} files ({size})', '还需下载:{count} 个文件({size})'), + 'group': ('{count} files · {size}', '{count} 个文件 · {size}'), + 'reused': ('Checked and ready to use: {count} files ({size})', '已校验、可直接使用:{count} 个文件({size})'), + 'exported': ('Download list exported to {folder}', '下载清单已导出到 {folder}'), + 'open_folder': ('Open folder', '打开文件夹'), + 'close': ('Close', '关闭'), + 'missing_mirror': ('Not on ModelScope', '魔搭暂无'), + 'error_folder': ('Choose an existing folder to save the download list in.', '请选择一个已有的文件夹来保存下载清单。'), + 'error_unchecked': ('Choose "Check again" first, then export the download list.', '请先点“重新检查”,再导出下载清单。'), + 'error_changed': ('The required files have changed. Choose "Check again", then export the download list.', + '所需文件有变化。请点“重新检查”,再导出下载清单。'), + 'error_write': ('The download list could not be saved in this folder: {reason}', '下载清单未能保存到这个文件夹:{reason}'), + 'error_read': ('FreeVideo does not have permission to read some files in the selected folder. Choose another folder, or download the files again without administrator rights.', + 'FreeVideo 没有权限读取所选文件夹里的部分文件。请选择其他文件夹,或不使用管理员身份重新下载。'), + 'write_permission': ('this account cannot write to it. Choose another folder.', '当前账户没有写入权限,请换一个文件夹。'), + 'write_disk_full': ('the drive is full.', '磁盘空间不足。'), + 'write_path_long': ('the path is too long. Choose a folder closer to the top of the drive.', '路径太长,请选择层级更浅的文件夹。'), + 'write_other': ('Choose another folder and try again.', '请换一个文件夹再试一次。'), + 'video': NAMES['video'], 'text': NAMES['text'], 'decoder': NAMES['decoder'], 'sampling': NAMES['sampling'], + 'readme': ( + 'FreeVideo download list\n' + '\n' + 'Run this command in this folder:\n' + 'aria2c --input-file=aria2.txt --continue=true --check-integrity=true\n' + '\n' + 'You can also import urls.txt into another download tool. urls.txt uses the download source chosen in Settings (Automatic uses Hugging Face); a few files are only available from Hugging Face. Keep all downloaded files in this folder or its subfolders.\n' + '\n' + 'When the download finishes, return to the launcher, choose "Use downloaded folder", then "Check & continue". FreeVideo recognizes files by size and full checksum, so different file names are fine; missing or damaged files are downloaded as usual.\n' + '\n' + 'The paths in SHA256SUMS are where aria2 saves the files. With urls.txt the file names may differ, which does not matter.\n' + '\n' + 'There is no need to download the whole model repository. This folder is not an offline package; do not import it with "Choose offline packages".\n', + 'FreeVideo 下载清单\n' + '\n' + '在这个文件夹里运行下面的命令:\n' + 'aria2c --input-file=aria2.txt --continue=true --check-integrity=true\n' + '\n' + '也可以把 urls.txt 导入其他下载工具。urls.txt 用的是设置里选中的下载源(选“自动”时用 Hugging Face);个别文件只能从 Hugging Face 下载。请把下载的文件都放在这个文件夹或它的子文件夹里。\n' + '\n' + '下载完成后,回到启动器点“使用已下载的文件夹”,再点“检查并继续”。FreeVideo 按文件大小和完整校验值识别文件,文件名不同也可以;缺少或损坏的文件会照常下载。\n' + '\n' + 'SHA256SUMS 里的路径是 aria2 的存放路径。用 urls.txt 下载时文件名可能不同,不影响使用。\n' + '\n' + '不必下载整个模型仓库。这个文件夹不是离线包,不要用“选择离线包”导入。\n' + '\n'), +} + + +# The export goes into a folder of its own inside the chosen one, so files the +# user keeps there (a README.txt, say) are never overwritten. +EXPORT_FOLDER = 'FreeVideo-download-list' +GENERATOR = 'FreeVideo download list' +EXPORT_FILES = ('manifest.json', 'aria2.txt', 'SHA256SUMS', 'urls.txt', 'README.txt') + + +class DownloadListError(ValueError): + """key names the DOWNLOAD_COPY 'error_' text the launcher shows.""" + def __init__(self, key, reason='', original_error=''): + self.key, self.reason = key, reason + self.original_error = original_error + super().__init__(DOWNLOAD_COPY['error_' + key][0].format(reason=reason)) + + +def download_write_reason(error, folder): + """Stable reason codes; OS messages are retained only for diagnostics.""" + if isinstance(error, PermissionError) or error.errno in (errno.EACCES, errno.EPERM): + return 'permission' + winerror = getattr(error, 'winerror', None) + if error.errno == errno.ENOSPC or winerror in (112, 39): + return 'disk_full' + paths = (error.filename, error.filename2, folder) + if (error.errno == errno.ENAMETOOLONG or winerror == 206 + or winerror == 3 and any(path is not None and len(str(path)) > 260 for path in paths)): + return 'path_long' + return 'other' + + +def download_copy(zh=False): + return {key: value[bool(zh)] for key, value in DOWNLOAD_COPY.items()} + + +def download_files(entries): + """Project the install plan's placements; never select another GPU variant.""" + result = [] + for row, status in entries: + if status != 'download': + continue + mirror = None if row.get('official_only') else network.modelscope_mapping(row) + result.append(dict(group=family(row), repo=row['repo'], hf_revision=row['revision'], + modelscope_repo=mirror['repo'] if mirror else '', modelscope_revision=mirror['revision'] if mirror else '', + file=row['file'], bytes=row['bytes'], sha256=row.get('sha256') or row.get('download_sha256'), + relative_path='/'.join(('payload', row['repo'], row['file'])), + hf_url=network.model_url(row, 'official', env={}), + modelscope_url=network.model_url(row, 'modelscope', env={}) if mirror else '')) + return result + + +def download_view(plan, zh=False, *, decimal_sizes=False): + from .launcher_copy import size_text + def size(value): + if value < (10**6 if decimal_sizes else 2**20): + unit = 'MB' if decimal_sizes else 'MiB' + return ('不到 1 ' if zh else 'under 1 ') + unit + return size_text(value, decimal_sizes=decimal_sizes) + copy = download_copy(zh) + rows = plan.get('download_files', []) + groups = [] + for group in FAMILIES: + files = [dict(row, name=row['file'], size=size(row['bytes'])) for row in rows if row['group'] == group] + amount = sum(row['bytes'] for row in files) + groups.append(dict(id=group, title=copy[group], files=files, count=len(files), bytes=amount, + summary=copy['group'].format(count=len(files), size=size(amount)))) + reused = (plan.get('local_models') or {}).get('matches', {}) + return dict(available='download_files' in plan, groups=groups, copy=copy, + total_bytes=sum(row['bytes'] for row in rows), count=len(rows), + total=copy['total_remaining' if reused else 'total'].format(size=size(sum(row['bytes'] for row in rows)), count=len(rows)), + reused=copy['reused'].format(count=len(reused), size=size(sum(r['bytes'] for r in reused.values()))) if reused else '') + + +def export_download_list(plan, folder, source='auto'): + """Offline export. The imported manifest is never trusted as a hash source.""" + folder = Path(folder).expanduser().resolve() + if not folder.is_dir(): + raise DownloadListError('folder') + if 'download_files' not in plan or not plan.get('download_files_verified', True): + raise DownloadListError('unchecked') + if source not in ('auto', 'official', 'hf-mirror', 'modelscope'): + raise DownloadListError('changed') + rows = plan['download_files'] + outputs = set() + aria, sums, urls = [], [], [] + for row in rows: + relative = row['relative_path'] + path = PurePosixPath(relative) + if (path.is_absolute() or any(part in ('', '.', '..') for part in relative.split('/')) + or any(char in relative for char in '\\:\r\n\t\x00') or relative in outputs + or not re.fullmatch('[0-9a-f]{64}', row.get('sha256') or '') + or not re.fullmatch('[0-9a-f]{40}', row['hf_revision']) + or type(row['bytes']) is not int or row['bytes'] <= 0): + raise DownloadListError('changed') + outputs.add(relative) + # Use only catalog pins and URLs generated by network.model_url. + original = dict(repo=row['repo'], revision=row['hf_revision'], file=row['file']) + hf = network.model_url(original, 'official', env={}) + mirror = network.model_url(original, 'modelscope', env={}) if row['modelscope_url'] else '' + if row['hf_url'] != hf or row['modelscope_url'] != mirror: + raise DownloadListError('changed') + aria.extend(['\t'.join(url for url in (hf, mirror) if url), + ' dir=' + path.parent.as_posix(), ' out=' + path.name, + ' checksum=sha-256=' + row['sha256']]) + sums.append(row['sha256'] + ' ' + relative) + chosen = 'official' if source == 'auto' or (source == 'modelscope' and not mirror) else source + urls.append(network.model_url(original, chosen, env={})) + prepared = plan.get('prepared_model') or {} + manifest = dict(schema_version=1, generator=GENERATOR, engine_version=plan.get('engine_version'), + video_model=dict(format=prepared.get('scale_granularity') or plan.get('model_scale_granularity') or plan.get('prepared_format'), + reason=plan.get('model_source_reason', ''), source=plan.get('model_source')), + sampling_caches=bool(plan.get('sampling_caches')), source=source, + total_bytes=sum(row['bytes'] for row in rows), files=rows) + documents = {'manifest.json': json.dumps(manifest, ensure_ascii=False, indent=2) + '\n', + 'aria2.txt': '\n'.join(aria) + ('\n' if aria else ''), + 'SHA256SUMS': '\n'.join(sums) + ('\n' if sums else ''), + 'urls.txt': '\n'.join(urls) + ('\n' if urls else ''), + 'README.txt': DOWNLOAD_COPY['readme'][1] + '\n' + DOWNLOAD_COPY['readme'][0]} + # Finish every file before anything appears or is replaced. No payload is downloaded. + stage = None + try: + # Not mkdtemp: Python 3.12.4 protects its Windows ACL from inheritance. + # Downloads made by an administrator must remain readable by the account + # that chose this folder, as with desktop_shortcut's launcher staging. + temporary = folder / ('.freevideo-list-' + uuid.uuid4().hex) + temporary.mkdir() + stage = temporary + for name, content in documents.items(): + (stage / name).write_text(content, encoding='utf-8', newline='\n') + # Two launchers exporting into one folder take turns: the five files + # always come from the same export. + with export_lock(folder): + target, earlier = export_target(folder) + if earlier: + replace_export(stage, target) + else: + os.rename(stage, target) # One step: the folder appears complete or not at all. + stage = None + except OSError as error: + raise DownloadListError('write', download_write_reason(error, folder), str(error)) from error + finally: + if stage is not None: + shutil.rmtree(stage, ignore_errors=True) + return str(target) + + +def export_lock(folder, wait=30.): + """Hold the cross-process lock of one export folder; waits for another export. + + The lock file lives in the temporary directory, not in the user's folder. + """ + from .locking import runtime_lock + key = hashlib.sha256(os.path.normcase(str(folder)).encode('utf-8')).hexdigest()[:16] + path = Path(tempfile.gettempdir()) / ('freevideo-download-list-%s.lock' % key) + deadline = time.monotonic() + wait + while True: + stack = ExitStack() + try: + stack.enter_context(runtime_lock(path, inherit=False)) + return stack + except BlockingIOError: + stack.close() + if time.monotonic() > deadline: + raise OSError(errno.EBUSY, 'Another download list export into this folder did not finish') + time.sleep(.1) + + +def earlier_export(path): + """A folder this launcher exported before, identified by its manifest.""" + try: + if path.is_symlink() or not path.is_dir(): + return False + value = json.loads((path / 'manifest.json').read_text(encoding='utf-8')) + except (OSError, ValueError): + return False + return isinstance(value, dict) and value.get('generator') == GENERATOR + + +def export_target(folder): + """(folder, earlier): our earlier export is refreshed; anything else is left alone.""" + for index in range(1, 100): + path = folder / (EXPORT_FOLDER if index == 1 else '%s-%d' % (EXPORT_FOLDER, index)) + if not os.path.lexists(path): + return path, False + if earlier_export(path): + return path, True + raise OSError(errno.EEXIST, 'Too many folders named ' + EXPORT_FOLDER) + + +def replace_export(stage, target): + """Swap in the five list files; downloads already in the folder stay. + + The files always belong to one export: if any move fails, the earlier ones + are put back before the error is raised. + """ + previous, placed = [], [] + try: + for name in EXPORT_FILES: + current = target / name + if os.path.lexists(current): + kept = target / ('.%s.previous-%s' % (name, os.urandom(4).hex())) + os.replace(current, kept) + previous.append((kept, current)) + os.replace(stage / name, current) + placed.append(current) + except OSError: + for current in placed: + try: + os.unlink(current) + except OSError: + pass + for kept, current in previous: + try: + os.replace(kept, current) + except OSError: + pass + raise + for kept, _ in previous: + try: + os.unlink(kept) + except OSError: + pass def runtime_packages_supported(): @@ -48,22 +341,12 @@ def video_instructions(zh=False): '按模型页说明选择显卡版本,保留下载的目录结构。')[bool(zh)] -@plain_errors -def links(): - root = io_path(Path(__file__).parent) - spec = json.loads((root / 'dependencies.json').read_text(encoding='utf-8'))['models'] - edge = json.loads((root / 'prepared_models.json').read_text(encoding='utf-8')) - return { - 'video': [('Hugging Face', 'https://huggingface.co/' + edge['repo']), - ('ModelScope', 'https://modelscope.ai/models/' + spec['edge_modelscope']['repo'])], - 'encoder': [('Hugging Face', 'https://huggingface.co/' + spec['encoder_repo'] + '/blob/' + - spec['encoder_revision'] + '/' + quote(spec['encoder_file'], safe='/'))], - 'decoder': [('Hugging Face · Video', 'https://huggingface.co/' + spec['vdn_repo'] + '/tree/' + - spec['vdn_revision'] + '/h3-base/vae'), - ('Hugging Face · Audio', 'https://huggingface.co/' + spec['vdn_repo'] + '/tree/' + - spec['vdn_revision'] + '/h3-base/audio_vae'), - ('ModelScope', 'https://modelscope.ai/models/' + spec['vdn_modelscope']['repo'])], - } +def links(plan=None): + """Compatibility for the Tk launcher: fixed files from a checked plan only.""" + return {group: [(row['file'] + ' · ' + source, row[key]) + for row in (plan or {}).get('download_files', []) if row['group'] == group + for source, key in (('Hugging Face', 'hf_url'), ('ModelScope', 'modelscope_url')) if row[key]] + for group in FAMILIES} @plain_errors @@ -103,7 +386,8 @@ def show_component(app, component): body = app.ttk.Frame(dialog, padding=24); body.pack(fill='both', expand=True) app.ttk.Label(body, text=app.t(*titles[component]), style='Heading.TLabel').pack(anchor='w') app.ttk.Label(body, text=app.t(*instructions[component]), wraplength=490).pack(fill='x', pady=12) - for title, url in links()[component]: + checked = app.controller.state.get('plan', {}) + for title, url in links(checked)['text' if component == 'encoder' else component]: app.ttk.Button(body, text=display(title, app.zh) + ' ↗', command=lambda u=url: webbrowser.open(u)).pack(fill='x', pady=4) for row in cloud_models(): app.ttk.Button(body, text=app.t('Quark model pack · ', '夸克模型包 · ') + row['label'] + ' ↗', diff --git a/freevideo_engine/modern_launcher.py b/freevideo_engine/modern_launcher.py index 5122321f..a651d0c3 100644 --- a/freevideo_engine/modern_launcher.py +++ b/freevideo_engine/modern_launcher.py @@ -77,10 +77,12 @@ def create_ui(session, *, show=True): class Bridge(QObject): changed = Signal() + downloadsChanged = Signal() def __init__(self): super().__init__(engine) self.alerted = None self.value = session.snapshot() + self.download_value = session.downloads_view() self.timer = QTimer(self) self.timer.setInterval(250) self.timer.timeout.connect(self.refresh) @@ -90,6 +92,10 @@ def __init__(self): def state(self): return self.value + @Property('QVariantMap', notify=downloadsChanged) + def downloadList(self): + return self.download_value + def invoke(self, function, *args, **kwargs): try: function(*args, **kwargs) @@ -105,6 +111,10 @@ def refresh(self): if value != self.value: self.value = value self.changed.emit() + downloads = session.downloads_view() + if downloads is not self.download_value: + self.download_value = downloads + self.downloadsChanged.emit() update = value.get('update') or {} if update.get('remind') and update.get('key') != self.alerted and engine.rootObjects(): # Flash the taskbar button once per new reminder when the @@ -166,6 +176,26 @@ def browse(self, key): if path: self.invoke(session.add_folder, path) if key == 'model_dirs' else self.invoke(session.edit, key, path) + @Slot() + def prepareDownloadList(self): + self.invoke(session.prepare_download_list) + + @Slot() + def exportDownloadList(self): + if session.controller.busy: + return + path = QFileDialog.getExistingDirectory(None, session.downloads_view()['copy']['export'], '', QFileDialog.ShowDirsOnly) + if path: + self.invoke(session.export_downloads, path) + + @Slot() + def useDownloadedFolder(self): + if session.controller.busy or session.importer.busy: + return + path = QFileDialog.getExistingDirectory(None, session.downloads_view()['copy']['use_folder'], '', QFileDialog.ShowDirsOnly) + if path: + self.invoke(session.use_downloaded_folder, path) + @Slot() def browsePackages(self): if session.controller.busy or session.importer.busy: @@ -237,6 +267,13 @@ def link(self, value): else: self.invoke(open_browser, value) + @Slot() + def openDownloadFolder(self): + from PySide6.QtGui import QDesktopServices + folder = session.download_export + if folder and Path(folder).is_dir(): + self.invoke(QDesktopServices.openUrl, QUrl.fromLocalFile(str(Path(folder).resolve()))) + @Slot(str) def selectSource(self, value): self.invoke(session.select_source, value) @@ -292,17 +329,11 @@ def shutdown(self): self.timer.stop() session.close() - from .model_guidance import links, cloud_models - from .launcher_copy import display + from .model_guidance import cloud_models engine = QQmlApplicationEngine() bridge = Bridge() engine.rootContext().setContextProperty('backend', bridge) engine.rootContext().setContextProperty('initialState', bridge.value) - # Nested Python tuples remain opaque objects in QML. Named maps become - # QVariantMaps, so both the displayed label and clicked URL are available. - engine.rootContext().setContextProperty('modelLinks', { - component: [dict(label=label, label_zh=display(label, True), url=url) for label, url in rows] - for component, rows in links().items()}) engine.rootContext().setContextProperty('cloudLinks', cloud_models()) engine.rootContext().setContextProperty('theme', theme()) engine.load(QUrl.fromLocalFile(str(Path(__file__).parent / 'launcher/Main.qml'))) diff --git a/freevideo_engine/offline_packages.py b/freevideo_engine/offline_packages.py index d304b651..4f6c5267 100644 --- a/freevideo_engine/offline_packages.py +++ b/freevideo_engine/offline_packages.py @@ -404,9 +404,14 @@ def place(src, target): except OSError: copy_progress(src, target, file_progress(progress, 'copy_bytes', src.stat().st_size)) + included = [] for row in model['files']: target, src = runtime / row['path'], source_for(row) if src is None: + if pack_name(row['path']) == '采样缓存': + total -= row['bytes'] + progress(done=done, total=total, detail='检查配套包', activity='') + continue raise ValueError('还需导入「%s」包,缺少文件:%s' % (pack_name(row['path']), row['path'])) if row['path'] in here: if not os.path.samefile(target, src) and hash_file(target, @@ -414,6 +419,7 @@ def place(src, target): raise ValueError('离线安装已有不同模型,未覆盖:' + plain(target)) else: place(src, target) + included.append(row) done += row['bytes'] progress(done=done, total=total, detail='检查配套包', activity='') listed = {row['path'] for row in model['files']} @@ -431,7 +437,8 @@ def place(src, target): place(io_path(root) / row['path'], runtime / row['path']) listed.add(row['path']) progress(done=done, total=total, detail='检查配套包', activity='') - return finish(runtime, config, model, source, here) + # Only placed rows are recorded: a skipped Sampling caches table is fetched when needed. + return finish(runtime, config, dict(model, files=included), source, here) def runtime_config(runtime): diff --git a/freevideo_engine/provision.py b/freevideo_engine/provision.py index 583b17b5..86d6d0b1 100644 --- a/freevideo_engine/provision.py +++ b/freevideo_engine/provision.py @@ -7,10 +7,10 @@ import threading from concurrent.futures import ThreadPoolExecutor, CancelledError -from .bootstrap import PACKAGE, model_target +from .bootstrap import PACKAGE, model_target, installation_model_files from .locking import runtime_lock from .monitoring import save -from .install_tuning import required_models, hf_receipt, unchanged_before +from .install_tuning import hf_receipt, unchanged_before from . import network from .storage import record_download, compact_sources, fingerprint from . import model_transfer @@ -68,13 +68,8 @@ def models(plan): ledger = root / 'verified-models.json' stamps = json.loads(ledger.read_text(encoding='utf-8')) if ledger.is_file() else {} model_dir, encoder_dir = (Path(plan[name]).expanduser().resolve() for name in ('model_dir', 'encoder_dir')) - from . import prepared_model prepared = plan.get('prepared_model') - files = list(required_models(json.loads((PACKAGE / 'model_files.json').read_text(encoding='utf-8')), plan.get('reuse_cache') or prepared)) - files += prepared_model.files(prepared) - from .sampling_assets import install_files, usable_with - if prepared or (plan.get('reuse_cache') and usable_with(plan['reuse_cache'])): - files += install_files(bool(plan.get('sampling_caches'))) + files = installation_model_files(prepared, plan.get('reuse_cache'), plan.get('sampling_caches')) mode = plan.get('verification', 'auto') from .local_models import key as local_key, import_file lock = threading.RLock() diff --git a/freevideo_engine/sampling_assets.py b/freevideo_engine/sampling_assets.py index e6114dd0..1020391c 100644 --- a/freevideo_engine/sampling_assets.py +++ b/freevideo_engine/sampling_assets.py @@ -5,14 +5,82 @@ resumable transfers. No model import, GPU work or prompt text is needed here. """ import json +import errno from functools import lru_cache from pathlib import Path +import re +import sys import time from . import disk_space from .long_paths import io_path, plain_errors COMMUNITY_PREFIX = 'community-sigma3-' WORKERS = 6 +UNAVAILABLE = 'SAMPLING_TABLE_UNAVAILABLE: ' + + +class SamplingTableUnavailable(RuntimeError): + """A request's tables could not be downloaded; keep the transfer cause copyable.""" + def __init__(self, details, cause): + self.details = details + super().__init__(UNAVAILABLE + json.dumps(details, separators=(',', ':')) + '\n' + str(cause)) + + +class _DownloadFailure(Exception): + def __init__(self, cause): + self.cause = cause + + +def _cancel_or_disk(error): + return (isinstance(error, (InterruptedError, KeyboardInterrupt)) or + isinstance(error, OSError) and error.errno == errno.ENOSPC or + bool(re.search(r'interrupt|cancel', type(error).__name__, re.I)) or + bool(re.search(r'cancelled|canceled|interrupted|not enough disk space|insufficient disk space|' + r'no space left|\[Errno 28\]|\[WinError 112\]|取消|磁盘空间不足', str(error), re.I))) + + +def available_levels(cache, shared, manifest, kind, catalog): + """Complete tables for this task and these weights, including the bundled 8 steps.""" + from . import adaln_assets as assets + weights = assets.weight_identity(manifest) + # Bundled tables have no task field; their exact clock identifies the task. + # Match its published refinement clock's channels/dtype/modality contract. + task_tables = [t for t in catalog if t.get('task') == kind and t['identity']['weights'] == weights] + identities = [t['identity'] for t in task_tables] + bundled = [t for t in manifest.get('adaln_tables', []) if kind in ('t2va', 'i2va') and + t['identity']['weights'] == weights and any( + all(t['identity'].get(k) == identity.get(k) for k in ('channels', 'dtype', 'modality_rows')) + and len(t['identity']['timesteps'][0]) == len(identity['timesteps'][0]) + for identity in identities)] + + def ready(table, embedded=False): + for row in table['files']: + found = False + for folder in ((cache,) if embedded else (cache, shared)): + path = assets.asset_path(folder, row['file']) + try: + marker = path.with_suffix('.json') + # Imported shared tables need no receipt yet: prepare() + # verifies them and writes one without downloading. + if not embedded and (folder == cache or marker.is_file()): + receipt = json.loads(marker.read_text(encoding='utf-8')) + if any(receipt.get(k) != row[k] for k in ('bytes', 'sha256')): + continue + assets.check_table(path, row, table['identity']) + found = True + break + except (OSError, ValueError, KeyError, TypeError): + continue + if not found: + return False + return bool(table['files']) + + def base_ready(steps): + return any(len(t['identity']['timesteps']) == steps and ready(t, True) for t in bundled) or any( + not refinement(t) and len(t['identity']['timesteps']) == steps and ready(t) for t in task_tables) + + refine_ready = any(refinement(t) and ready(t) for t in task_tables) + return [i for i, steps in enumerate((8, 12, 16, 20)) if base_ready(steps) and (i != 0 or refine_ready)] @plain_errors @@ -171,7 +239,8 @@ def needed(table): count = len(table['identity']['timesteps']) return (count == 3 and sampling.get('refine_schedule') == COMMUNITY if refinement(table) else count == sampling['base_steps']) - selected = [t for t in tables() if t.get('task') == kind + catalog = tables() + selected = [t for t in catalog if t.get('task') == kind and needed(t) and t['identity']['weights'] == weights] root = Path(root) shared = cache_root(machine['model_root']) @@ -241,9 +310,14 @@ def fetch(index, table, row, path): def moved(current, size, speed, **_): done[index], speeds[index] = current, speed or 0. emit() - network.download(network.model_urls(networking, spec, environ), path, row['sha256'], moved, - network=networking, size=row['bytes'], category=network.model_family(spec), headers_for=headers, - keep_partial=True, stall_seconds=30, slow_seconds=15, low_speed_limit=64*1024) + try: + network.download(network.model_urls(networking, spec, environ), path, row['sha256'], moved, + network=networking, size=row['bytes'], category=network.model_family(spec), headers_for=headers, + keep_partial=True, stall_seconds=30, slow_seconds=15, low_speed_limit=64*1024) + except Exception as error: + if _cancel_or_disk(error): + raise + raise _DownloadFailure(error) from error # network.download already verified the SHA-256. assets.check_table(path, row, table['identity'], verify_hash=False) save(path.with_suffix('.json'), dict(bytes=row['bytes'], sha256=row['sha256'])) @@ -254,17 +328,26 @@ def moved(current, size, speed, **_): emit(force=True) emit(force=True) # Each table is fifty small files; several transfers hide per-file latency. + failure = None with ThreadPoolExecutor(max_workers=min(WORKERS, len(missing)), thread_name_prefix='sampling-cache') as pool: futures = [pool.submit(fetch, index, *item) for index, item in enumerate(missing)] # A cancel reaches one worker (ComfyUI clears its flag once raised): stop # the rest at once instead of waiting for earlier transfers to finish. - done, _ = wait(futures, return_when=FIRST_EXCEPTION) - failed = next((future for future in futures if future in done and future.exception()), None) + completed, _ = wait(futures, return_when=FIRST_EXCEPTION) + failed = next((future for future in futures if future in completed and future.exception()), None) if failed is not None: stop.set() for future in futures: future.cancel() - raise failed.exception() + failure = failed.exception() + if isinstance(failure, _DownloadFailure): + details = dict(base_steps=sampling['base_steps'], two_pass=bool(sampling.get('enabled')), + refine_steps=sampling.get('refine_steps', 0), missing_bytes=total, + available=available_levels(cache, shared, manifest, kind, catalog), + decimal_sizes=sys.platform == 'darwin') + raise SamplingTableUnavailable(details, failure.cause) from failure.cause + if failure is not None: + raise failure seconds = time.monotonic() - started retire_superseded(machine) return dict(seconds=seconds, downloaded_bytes=total) diff --git a/web/error_panel.js b/web/error_panel.js index 201756ed..dc158539 100644 --- a/web/error_panel.js +++ b/web/error_panel.js @@ -50,7 +50,18 @@ const PLAIN_ERRORS = [ function knownError(text) { const lines = String(text).slice(0, 65536).split(/\r?\n/).map(line => line.trim()).filter(Boolean); const index = /^FreeVideo generation failed(?: \(exit -?\d+\))?$/.test(lines[0] ?? '') ? 1 : 0; - const line = (lines[index] ?? '').replace(/^[A-Za-z_][\w.]*(?:Error|Exception):\s*/, ''); + const line = (lines[index] ?? '').replace(/^(?:[A-Za-z_][\w.]*(?:Error|Exception)|(?:[\w.]+\.)?SamplingTableUnavailable):\s*/, ''); + if (line.startsWith('SAMPLING_TABLE_UNAVAILABLE: ')) { + try { + const d = JSON.parse(line.slice('SAMPLING_TABLE_UNAVAILABLE: '.length)); + if (Number.isInteger(d.base_steps) && d.base_steps >= 1 && d.base_steps <= 32 + && typeof d.two_pass === 'boolean' && Number.isInteger(d.refine_steps) && d.refine_steps >= 0 && d.refine_steps <= 31 + && Number.isSafeInteger(d.missing_bytes) && d.missing_bytes > 0 && typeof d.decimal_sizes === 'boolean' + && Array.isArray(d.available) && d.available.every(i => Number.isInteger(i) && i >= 0 && i <= 3)) + return {kind: 'sampling-table', base_steps: d.base_steps, two_pass: d.two_pass, refine_steps: d.refine_steps, + missing_bytes: d.missing_bytes, decimal_sizes: d.decimal_sizes, available: [...new Set(d.available)].sort()}; + } catch { /* Malformed metadata stays in technical details. */ } + } return PLAIN_ERRORS.find(error => line === error.message || error.aliases?.includes(line))?.key ?? null; } @@ -230,6 +241,37 @@ export function createErrorReport(detail, context = {}) { export function failureAdvice(value, t, known = knownError(value)) { const text = String(value).slice(0, 65536); const row = (kind, title, summary, action) => ({kind, title: t(...title), summary: t(...summary), action: t(...action)}); + if (known?.kind === 'sampling-table') { + // Names and plan matching follow sampling_effort.js; metadata carries no user text. + const levels = [['Light', '轻量'], ['Medium', '标准'], ['High', '精细'], ['Max', '极致']]; + const index = [8, 12, 16, 20].findIndex((steps, i) => known.base_steps === steps + && known.two_pass === (i === 0) && (i !== 0 || known.refine_steps === 3)); + const custom = index < 0; + // Iris: a level name and its 档 never break apart. The summary is set as text, + // so word joiners (U+2060) keep each group on one line instead of markup. + const keep = value => [...value].join('\u2060'); + const join = language => { + const names = known.available.map(i => language ? `“${levels[i][1]}”` : levels[i][0]); + if (language) names[names.length - 1] += '档'; + const kept = language ? names.map(keep) : names; + return kept.length < 2 ? kept[0] || '' : kept.slice(0, -1).join(language ? '、' : ', ') + (language ? '或' : ' or ') + kept.at(-1); + }; + const bytes = known.missing_bytes; + // As launcher_copy.size_text: one decimal for GiB, whole MiB; decimal units on macOS. + let size = bytes >= 2**30 ? (bytes / 2**30).toFixed(1) + ' GiB' : Math.floor(bytes / 2**20) + ' MiB'; + if (known.decimal_sizes) { + const unit = [[1e12, 'TB', 2], [1e9, 'GB', 2], [1e6, 'MB', 1]].find(([scale]) => bytes >= scale); + size = unit ? Number((bytes / unit[0]).toFixed(unit[2])) + ' ' + unit[1] : Math.round(bytes / 1e3) + ' KB'; + } + // A custom step count is not a quality level: no "档" / "quality level" for it. + const title = custom + ? [`The sampling table for Custom (${known.base_steps} steps) is not downloaded yet`, `“自定义(${known.base_steps} 步)”的采样表还没有下载`] + : [`The ${levels[index][0]} quality level's sampling table is not downloaded yet`, `${keep(`“${levels[index][1]}”档`)}的采样表还没有下载`]; + const what = custom ? ['the sampling table for this step count', '这个步数的采样表'] : ["that level's sampling table", '这一档的采样表']; + return {...row('sampling-table', title, + [`This video first needs ${what[0]} (about ${size}), and it could not be downloaded this time. Check the network connection and generate again; FreeVideo downloads it automatically.${known.available.length ? ` You can also switch to a quality level that is already available: ${join(0)}.` : ''}`, + `这次生成需要先下载${what[1]}(约 ${size}),但这次没能下载。请确认网络连接后再生成一次,FreeVideo 会自动下载。${known.available.length ? `也可以改用已经可以使用的${join(1)}。` : ''}`], ['', '']), plain: true}; + } // ComfyUI and worker tracebacks carry English exceptions. Translate these // actionable failures through the existing signature/advice path. for (const {key, title, summary} of PLAIN_ERRORS) { @@ -471,7 +513,7 @@ export function createErrorPanel(t) { ? t('Use Download report in the preview or node panel for the diagnostic report.', '完整诊断请使用预览区或节点面板中的“下载报告”。') : t('Known input text, paths and common credentials are removed. Export saves a local error summary; review it before sharing.', '已去除已知输入文本、路径和常见凭据。导出会保存本地错误摘要,分享前可查看内容。'); - copy.textContent = t('Copy error', '复制错误'); + copy.textContent = advice.kind === 'sampling-table' ? t('Copy full details', '复制完整详情') : t('Copy error', '复制错误'); if (popup) showFailureDialog(report, t); return report; }, clear() { currentReport = null; details.value = ''; element.hidden = true; }}; From c57739263237787e2468389475964b90425c609c Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 17:38:48 +0000 Subject: [PATCH 21/27] Editing view: a part that continues a finished video is generated at that video's picture size - A part made at the continued video's size is held to that size's length limit --- web/editor.js | 9 +++++++-- web/editor_generation.js | 9 +++++++++ web/editor_model.js | 6 ++++++ web/editor_store.js | 12 ++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) diff --git a/web/editor.js b/web/editor.js index d1560955..2014834f 100644 --- a/web/editor.js +++ b/web/editor.js @@ -6,7 +6,7 @@ import {openSetup} from './setup.js'; import * as model from './editor_model.js'; import {createEditorState} from './editor_state.js'; import {createEditorStore, sourceKey, mediaURL} from './editor_store.js'; -import {generationSize} from './editor_generation.js'; +import {generationSize, continuedSize} from './editor_generation.js'; import {createEditorLibrary} from './editor_library.js'; import {createEditorPlayer} from './editor_player.js'; import {createEditorTransport, registerEditorShortcuts} from './editor_transport.js'; @@ -506,7 +506,12 @@ export function openEditor({api, app, from = null, panel = {}, components: given submittedSelection = state.select({kind: 'clip', ids}); await completeSources(); const paths = Object.fromEntries([...store.sources].map(([key, value]) => [key, value.path])); const request = model.generationRequest(prepared, ids, {text, paths}); - const tooLong = request.blocks.filter(block => capability.maxRequestFrames != null && block.frames > capability.maxRequestFrames); + const follow = continuedSize(model.generationBefore(prepared, ids), store.sources), edit = capability.generationSize || generationSize(state.record); + Object.assign(frozen.prompt[frozen.node].inputs, follow || {}); + // Parts made at another size than the edit's have that size's limit. + const limit = (follow && (follow.width !== edit.width || follow.height !== edit.height) + ? await store.requestFrames(follow, state.record.settings) : null) ?? capability.maxRequestFrames; + const tooLong = request.blocks.filter(block => limit != null && block.frames > limit); if (tooLong.length) { state.settle(record => tooLong.reduce((next, block) => model.fail(next, block.id, 'too_long', null), record)); return; diff --git a/web/editor_generation.js b/web/editor_generation.js index 65512c22..4bf818da 100644 --- a/web/editor_generation.js +++ b/web/editor_generation.js @@ -13,3 +13,12 @@ export function generationSize(project, sizes = []) { || Math.abs(a.width * a.height - area) - Math.abs(b.width * b.height - area)); return {width: candidates[0].width, height: candidates[0].height}; } +// A part that continues finished footage is made at that footage's size, whatever the edit's own: the size all the +// footage a request continues has in common. null when a video in the request starts fresh or the sizes differ; the +// edit's size then stands. +export function continuedSize(before, sources) { + const sizes = before.map(key => key && sources.get(key)); + if (!sizes.length || !sizes.every(valid)) return null; + const {width, height} = sizes[0]; + return sizes.every(size => size.width === width && size.height === height) ? {width, height} : null; +} diff --git a/web/editor_model.js b/web/editor_model.js index cb83814b..0c80ecf3 100644 --- a/web/editor_model.js +++ b/web/editor_model.js @@ -933,6 +933,12 @@ export function prepare(record, ids) { } return finish(next); } +// The finished footage each video of a generation request continues, as source keys; null for one that starts fresh. +export function generationBefore(record, ids) { + const clips = generationClips(record, ids); + return generationPlan(record, clips).filter(row => !row.before || !clips.some(clip => clip.id === row.before.id)) + .map(row => row.before ? key(row.before.source) : null); +} export function generationRequest(record, ids, {text, paths = {}} = {}) { check(object(paths), 'source'); if (text !== undefined) check(validText(text), 'text'); diff --git a/web/editor_store.js b/web/editor_store.js index c27ba232..eae650d1 100644 --- a/web/editor_store.js +++ b/web/editor_store.js @@ -391,6 +391,18 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener // these limits; only the callback publishes a measured capability. read(); return fallback; }, + // The longest single part this computer makes at a size other than the edit's; null when it cannot be told. + async requestFrames(size, settings) { + try { + for (let tries = 0; tries < 5 && !disposed; tries++) { + const reply = await json('/freevideo/capabilities?' + new URLSearchParams(size)); + if (reply?.status !== 'computing') return !reply || (reply.status && reply.status !== 'ready') ? null + : capabilityPoints(reply, {...size, settings}).maxRequestFrames; + await new Promise(done => clock.setTimeout(done, 1000)); + } + } catch { /* not known */ } + return null; + }, async startChain(payload) { if (standIn) return standIn.startChain(payload); if (!generationAvailable) return null; From b7f3713fdf3ec037140a767da5d77949454b3f0f Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 17:54:56 +0000 Subject: [PATCH 22/27] A short note under the result says when other programs used a large part of the GPU - The note that the GPU was held back by heat reads the request's own telemetry - The notes under a result are centred, and the temperature stays on one line - Saved telemetry with damaged numbers is read as missing --- freevideo_engine/comfy_assets.py | 5 ++-- freevideo_engine/monitoring.py | 42 ++++++++++++++++++++++++-------- scripts/test_freevideo_entry.mjs | 13 +++++++--- web/freevideo.js | 6 +++-- web/generation_progress.js | 12 ++++++++- web/studio.css | 2 ++ web/studio.js | 6 +++-- 7 files changed, 65 insertions(+), 21 deletions(-) diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index 51d13de7..88d142ea 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -4,7 +4,7 @@ import os from pathlib import Path, PurePosixPath, PureWindowsPath from . import launcher_bridge -from .monitoring import thermal_slowdown +from .monitoring import thermal_slowdown, shared_gpu_note, worker_gpu from .paths import data_root from .support_report import encoder_sizes @@ -155,7 +155,8 @@ def output_summary(report, relative_video): geometry=dict(report.get('geometry', {}), **(report.get('output_size') or {})), sampling_plan=report.get('sampling_plan'), low_memory=bool(report.get('low_memory_mode')), - thermal=thermal_slowdown(((report.get('memory') or {}).get('video') or {}).get('gpu')), + thermal=thermal_slowdown(worker_gpu(report)), + shared_gpu=shared_gpu_note(worker_gpu(report)), reference_trims=report.get('encoding', {}).get('reference_trims') or [], preview=bool(report.get('preview')), upscaled_preview=bool(report.get('upscaled_preview')), video=relative_video.as_posix(), report=relative_video.with_suffix( diff --git a/freevideo_engine/monitoring.py b/freevideo_engine/monitoring.py index b3d92c0c..2b4ce75a 100644 --- a/freevideo_engine/monitoring.py +++ b/freevideo_engine/monitoring.py @@ -93,18 +93,21 @@ def thermal_slowdown(measured, *, share=.5, clock_ratio=.7, minimum_samples=200) busy = measured.get('busy_activity') if isinstance(measured, dict) else None if not isinstance(busy, dict): return None - samples = busy.get('clock_reason_samples') or 0 - counts = busy.get('clock_reason_counts') or {} - clock = busy.get('sm_clock_mhz') or {} - mean, peak = clock.get('mean'), clock.get('maximum') + samples = _count(busy.get('clock_reason_samples')) + counts = busy.get('clock_reason_counts') + counts = counts if isinstance(counts, dict) else {} + clock = busy.get('sm_clock_mhz') + clock = clock if isinstance(clock, dict) else {} + mean, peak = _count(clock.get('mean')), _count(clock.get('maximum')) if samples < minimum_samples or not mean or not peak: return None - thermal = min(samples, max(counts.get('sw_thermal') or 0, counts.get('hw_thermal') or 0)) + thermal = min(samples, max(_count(counts.get('sw_thermal')), _count(counts.get('hw_thermal')))) if thermal < share * samples or mean > clock_ratio * peak: return None - temperature = (busy.get('temperature_c') or {}).get('mean') + temperature = busy.get('temperature_c') + temperature = temperature.get('mean') if isinstance(temperature, dict) else None return dict(share=round(thermal / samples, 3), sm_clock_mean_mhz=round(mean), sm_clock_max_mhz=round(peak), - temperature_mean_c=round(temperature, 1) if isinstance(temperature, (int, float)) else None) + temperature_mean_c=round(temperature, 1) if type(temperature) in (int, float) and 0 <= temperature <= 1000 else None) class OtherPrograms: @@ -231,14 +234,33 @@ def shared_gpu(measured, *, share=.5, minimum_checks=30): if not isinstance(programs, dict): return None checks, shared, mean = (programs.get(key) for key in ('checks', 'shared_checks', 'other_mean_percent')) - if (any(type(value) not in (int, float) or not value >= 0 for value in (checks, shared, mean)) - or checks < minimum_checks or shared < share * checks): + if (any(type(value) not in (int, float) for value in (checks, shared, mean)) + or not minimum_checks <= checks <= 1e9 or not 0 <= shared <= checks or not 0 <= mean <= 100 + or shared < share * checks): return None return dict(share=round(shared / checks, 3), other_mean_percent=round(mean)) +# Other programs' mean share of the GPU, in percent, from which the result says so: +# the run took at least about a quarter longer than with the GPU to itself. +SHARED_NOTE_PERCENT = 20 + + +def shared_gpu_note(measured): + """{other_mean_percent} for the note under a result, or None. + + Shown when other programs averaged at least SHARED_NOTE_PERCENT of the + GPU, and shared_gpu holds: at least 30 checks, with other programs using + at least 20% in at least half of them. + """ + if shared_gpu(measured) is None: + return None + mean = measured['other_programs']['other_mean_percent'] + return dict(other_mean_percent=mean) if mean >= SHARED_NOTE_PERCENT else None + + def _count(value): - return value if type(value) in (int, float) and 0 <= value < float('inf') else 0 + return value if type(value) in (int, float) and 0 <= value <= 1e12 else 0 def combined(rows): diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index 0b867324..d386fb3a 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -53,6 +53,7 @@ test('main browser entry registers both views and preserves node hooks', async t }, referenceTrimText: row => `trimmed ${row.kind} ${row.number}`, thermalText: thermal => `thermal ${thermal.sm_clock_mean_mhz}`, + sharedGpuText: shared => `shared GPU ${shared.other_mean_percent}%`, encoderFileText: file => `encoder ${file.sizes.join(' / ')} GiB`, encoderFilePath: file => file?.offline_installation ? file.path || '' : '', createProgressConnection: () => ({start() {}, refresh() {}, reset() {}}), @@ -82,7 +83,7 @@ test('main browser entry registers both views and preserves node hooks', async t .replace('import { app } from "../../scripts/app.js";', 'const {app} = globalThis.__freevideoEntryTest;') .replace('import { api } from "../../scripts/api.js";', 'const {api} = globalThis.__freevideoEntryTest;') .replace("import { openStudio, loraPanel, loraWarning, promptGuide } from \"./studio.js\";", 'const {openStudio,loraPanel,loraWarning,promptGuide} = globalThis.__freevideoEntryTest;') - .replace("import { createGenerationProgress, referenceTrimText, thermalText, encoderFileText, encoderFilePath } from './generation_progress.js';", 'const {createGenerationProgress,referenceTrimText,thermalText,encoderFileText,encoderFilePath} = globalThis.__freevideoEntryTest;') + .replace("import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, encoderFileText, encoderFilePath } from './generation_progress.js';", 'const {createGenerationProgress,referenceTrimText,thermalText,sharedGpuText,encoderFileText,encoderFilePath} = globalThis.__freevideoEntryTest;') .replace("import { createProgressConnection } from './progress_connection.js';", 'const {createProgressConnection} = globalThis.__freevideoEntryTest;') .replace("import { notifyCompatibility } from './compatibility.js';", 'const {notifyCompatibility} = globalThis.__freevideoEntryTest;') .replace("import { startUpdateChecks } from './updates.js';", 'const {startUpdateChecks} = globalThis.__freevideoEntryTest;') @@ -191,12 +192,14 @@ test('main browser entry registers both views and preserves node hooks', async t launcher: true, offline_installation: false}; const resultWidth = node.size[0]; node.setSize([resultWidth, 100]); - node.freevideoShowResult({freevideo_summary: [{video, result_cache_hit: true, sample_seconds: 99, encoder_file}]}); + node.freevideoShowResult({freevideo_summary: [{video, result_cache_hit: true, sample_seconds: 99, encoder_file, + thermal: {sm_clock_mean_mhz: 900}, shared_gpu: {other_mean_percent: 42}}]}); const reused = node.dom.find(row => row.name === 'freevideo_result').element; assert.equal(reused.children[0].hidden, true, 'Do not show the old sampling time as current work'); assert.ok(reused.children[1].children.some(e => e.textContent === 'Reused previous result')); assert.ok(reused.children.some(e => e.className === 'fv-note' && e.textContent === 'encoder 25.3 / 14.6 GiB'), 'A cached result still came from the same encoder file'); + assert.ok(!reused.children.some(e => e.textContent === 'shared GPU 42%'), 'A cached result did not share the GPU this time'); assert.deepEqual(node.size, [resultWidth, node.computeSize()[1]], 'Grow the node after appending the encoder notice'); const again = reused.children[1].children.at(-1); assert.equal(again.textContent, 'Regenerate'); @@ -218,11 +221,12 @@ test('main browser entry registers both views and preserves node hooks', async t assert.ok(node.computeSize()[1] < progressHeight); assert.deepEqual(node.size, [resultWidth, progressHeight], 'Shorter results must not shrink the node'); node.freevideoShowResult({freevideo_summary: [{video, sample_seconds: 5, encoder_file, thermal: {sm_clock_mean_mhz: 900}, + shared_gpu: {other_mean_percent: 42}, reference_trims: [{kind: 'video', number: 1, seconds: 20.3, used_seconds: 5.17}]}]}); assert.deepEqual(reused.children.filter(e => e.className === 'fv-note fv-trim-note').map(e => e.textContent), ['trimmed video 1'], 'Say which reference was shortened, never cut silently'); assert.deepEqual(reused.children.filter(e => e.className === 'fv-note' && !e.hidden).map(e => e.textContent), - ['thermal 900', 'encoder 25.3 / 14.6 GiB'], 'Show the encoder notice after the thermal notice'); + ['thermal 900', 'shared GPU 42%', 'encoder 25.3 / 14.6 GiB'], 'Show thermal, shared GPU and encoder notices in order'); assert.ok(node.computeSize()[1] > progressHeight); assert.deepEqual(node.size, [resultWidth, node.computeSize()[1]], 'Measure the result height after all notices are appended'); @@ -265,8 +269,9 @@ test('main browser entry registers both views and preserves node hooks', async t node.freevideoShowFailure({exception_message: 'Failed'}); assert.equal(options.getMinHeight(), 420, 'Failure keeps its existing height'); node.freevideoShowResult({freevideo_summary: [{video, low_memory: true, thermal: {sm_clock_mean_mhz: 900}, + shared_gpu: {other_mean_percent: 42}, reference_trims: [{kind: 'video', number: 1}]}]}); - assert.equal(options.getMinHeight(), 461, 'Measure memory, thermal and reference trim notices'); + assert.equal(options.getMinHeight(), 543, 'Add 82 px for the shared GPU notice: 66 px text height and two 8 px margins'); }); class MediaNode extends GenerateNode { diff --git a/web/freevideo.js b/web/freevideo.js index 8a962606..7678efee 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -3,7 +3,7 @@ import { app } from "../../scripts/app.js"; import { api } from "../../scripts/api.js"; import { openEditorView } from './editor_loader.js'; import { openStudio, loraPanel, loraWarning, promptGuide } from "./studio.js"; -import { createGenerationProgress, referenceTrimText, thermalText, encoderFileText, encoderFilePath } from './generation_progress.js'; +import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, encoderFileText, encoderFilePath } from './generation_progress.js'; import { createProgressConnection } from './progress_connection.js'; import { notifyCompatibility } from './compatibility.js'; import { startUpdateChecks } from './updates.js'; @@ -26,7 +26,7 @@ const style = document.createElement("style"); style.textContent = ` .fv-panel{box-sizing:border-box;width:100%;height:100%;padding:16px;background:var(--fv-card);color:var(--fv-ink);border:1px solid var(--fv-border);border-radius:var(--fv-r-md,12px);font:13px/1.6 var(--fv-font);overflow:auto;color-scheme:dark} .fv-panel *{box-sizing:border-box}.fv-toolbar,.fv-row,.fv-links{display:flex;align-items:center;gap:8px;flex-wrap:wrap} -.fv-panel .fv-links{justify-content:center}.fv-panel[data-freevideo=result]>.fv-note{text-align:center} +.fv-panel .fv-links{justify-content:center}.fv-panel[data-freevideo=result]>.fv-note{text-align:center;text-wrap:balance} .fv-panel .fv-toolbar{padding:4px 0 8px}.fv-panel .fv-stats{text-align:center}.fv-panel .fv-stat{padding:10px 3px;border-radius:var(--fv-r-md,12px);background:var(--fv-raised)} .fv-panel .fv-stats[hidden]{display:none} .fv-panel .fv-connected{display:flex;align-items:center;justify-content:center;gap:6px;flex-wrap:wrap;margin:10px 0}.fv-panel .fv-connection{border:1px solid var(--fv-border);border-radius:var(--fv-r-sm,8px);padding:5px 9px;background:var(--fv-selected);color:var(--fv-accent);font-size:var(--fv-micro,12px)} @@ -504,6 +504,8 @@ function resultPanel(node) { for (const row of value.reference_trims || []) panel.append(el("div", referenceTrimText(row, text), "fv-note fv-trim-note")); if (value.low_memory && !value.result_cache_hit) resultNotes.push(el("div", text('This video used the low-memory mode for this GPU, so it took longer than usual. A lower resolution or a shorter video runs faster.', '这段视频使用了省显存方式,所以比平时慢。降低分辨率或缩短时长会更快。'), "fv-note")); if (value.thermal && !value.result_cache_hit) resultNotes.push(el("div", thermalText(value.thermal, text), "fv-note")); + const shared = value.shared_gpu && !value.result_cache_hit ? sharedGpuText(value.shared_gpu, text) : ''; + if (shared) resultNotes.push(el("div", shared, "fv-note")); if (value.encoder_file) { const note = el("div", encoderFileText(value.encoder_file, text), "fv-note"); const path = encoderFilePath(value.encoder_file); diff --git a/web/generation_progress.js b/web/generation_progress.js index bb281d42..6c32e8a7 100644 --- a/web/generation_progress.js +++ b/web/generation_progress.js @@ -177,11 +177,21 @@ function trimStatus(rows, t) { export function thermalText(thermal, t) { const share = thermal.sm_clock_max_mhz ? Math.round(100 * thermal.sm_clock_mean_mhz / thermal.sm_clock_max_mhz) : null; const heat = Number.isFinite(thermal.temperature_mean_c) ? Math.round(thermal.temperature_mean_c) : null; - const detail = [heat !== null ? `${heat} °C` : '', share !== null ? t(`${share}% of its peak clock`, `频率只有峰值的 ${share}%`) : ''].filter(Boolean).join(t(', ', ',')); + const detail = [heat !== null ? `${heat}\u00a0°C` : '', share !== null ? t(`${share}% of its peak clock`, `频率只有峰值的 ${share}%`) : ''].filter(Boolean).join(t(', ', ',')); return t(`The GPU spent most of this video in thermal slowdown${detail ? ` (${detail})` : ''}, so it took much longer than usual. Improve cooling; on a laptop, plug in and choose a high-performance power mode.`, `生成这段视频时,显卡大部分时间因过热降频${detail ? `(${detail})` : ''},所以比平时慢很多。请改善散热;笔记本请接通电源并选择高性能模式。`); } +// Other programs held a substantial share of the GPU during this run (summary.shared_gpu). +// The share is rounded to 5% and written as at most 95%; without a usable number there is no note. +export function sharedGpuText(shared, t) { + const mean = shared?.other_mean_percent; + if (typeof mean !== 'number' || !Number.isFinite(mean)) return ''; + const percent = Math.min(95, Math.round(mean / 5) * 5); + return t(`Other programs were using about ${percent}% of the GPU.`, + `其他程序占用了约 ${percent}% 的显卡。`); +} + export function encoderFilePath(file) { return file?.offline_installation === true && typeof file.path === 'string' ? file.path : ''; } diff --git a/web/studio.css b/web/studio.css index 86cf8b23..b20eb021 100644 --- a/web/studio.css +++ b/web/studio.css @@ -119,6 +119,8 @@ .fv-studio .fv-progress{width:100%;height:4px;accent-color:var(--fv-accent);border:0;margin:0 0 10px}.fv-studio .fv-budget,.fv-studio .fv-prewarm{text-align:center;font-size:var(--fv-micro);color:var(--fv-muted)} .fv-studio .fv-panel{height:auto;padding:0;border:none;border-radius:0;background:transparent;color:var(--fv-ink);font-size:var(--fv-micro);overflow:visible} .fv-studio .fv-panel .fv-toolbar{justify-content:flex-start}.fv-studio .fv-panel .fv-note{font-size:var(--fv-micro);margin:8px 0;text-align:left} +.fv-studio .fv-preview-column>.fv-note{text-align:center;text-wrap:balance} +.fv-studio .fv-preview-column>.fv-note:not([hidden])+.fv-note{margin-top:calc(0px - 19px)} .fv-studio .fv-panel .fv-empty{display:grid;place-items:center;padding:22px 15px;min-height:92px;margin-top:12px;font-size:var(--fv-micro);white-space:pre-line;border-color:var(--fv-border);background:var(--fv-card)} .fv-studio .fv-panel .fv-preview{max-height:170px;margin:10px auto;object-fit:contain}.fv-studio .fv-panel .fv-card{background:var(--fv-card);border-color:var(--fv-border)} .fv-studio .fv-panel select{min-height:var(--fv-h-sm);padding:5px 8px;max-width:145px;width:auto}.fv-studio .fv-panel button{font-size:var(--fv-micro);padding:5px 10px;min-height:var(--fv-h-sm)} diff --git a/web/studio.js b/web/studio.js index a2731268..b779a979 100644 --- a/web/studio.js +++ b/web/studio.js @@ -3,7 +3,7 @@ import { app } from '../../scripts/app.js'; import { api } from '../../scripts/api.js'; import { openSetup } from './setup.js'; import { wordmark } from './wordmark.js'; -import { createGenerationProgress, referenceTrimText, thermalText, encoderFileText, encoderFilePath } from './generation_progress.js'; +import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, encoderFileText, encoderFilePath } from './generation_progress.js'; import { viewSwitch, viewChanged, agentTabReady, openAutopilot } from './view_navigation.js'; import { agentInterfaceEnabled } from './agent_connect.js'; import { createUpdateNotice, createVersionInfo, productVersion } from './updates.js'; @@ -647,8 +647,9 @@ export function openStudio(node, {extendFrom = null} = {}) { }; const memoryNote = el('div', t('This video used the low-memory mode for this GPU, so it took longer than usual. A lower resolution or a shorter video runs faster.', '这段视频使用了省显存方式,所以比平时慢。降低分辨率或缩短时长会更快。'), 'fv-note'); memoryNote.hidden = true; const thermalNote = el('div', '', 'fv-note'); thermalNote.hidden = true; + const sharedGpuNote = el('div', '', 'fv-note'); sharedGpuNote.hidden = true; const encoderNote = el('div', '', 'fv-note'); encoderNote.hidden = true; - output.append(status, reuseRow, trimNote, memoryNote, thermalNote, encoderNote, links, stats, budget, progress.stall, progress.report, prewarm); + output.append(status, reuseRow, trimNote, memoryNote, thermalNote, sharedGpuNote, encoderNote, links, stats, budget, progress.stall, progress.report, prewarm); const failure = createErrorPanel(t); output.append(failure.element); // With the picture, the result's actions and numbers form one block, centred below the header. output.append(el('div', null, 'fv-preview-end')); @@ -680,6 +681,7 @@ export function openStudio(node, {extendFrom = null} = {}) { trimNote.textContent = trims.map(row => referenceTrimText(row, t)).join('\n'); trimNote.hidden = !trims.length; memoryNote.hidden = !r.low_memory || !!r.result_cache_hit; thermalNote.textContent = r.thermal ? thermalText(r.thermal, t) : ''; thermalNote.hidden = !r.thermal || !!r.result_cache_hit; + sharedGpuNote.textContent = r.shared_gpu ? sharedGpuText(r.shared_gpu, t) : ''; sharedGpuNote.hidden = !sharedGpuNote.textContent || !!r.result_cache_hit; encoderNote.textContent = r.encoder_file ? encoderFileText(r.encoder_file, t) : ''; encoderNote.hidden = !r.encoder_file; const encoderPath = encoderFilePath(r.encoder_file); if (encoderPath) encoderNote.append(el('pre', encoderPath, 'fv-encoder-path')); From 189c7d8557d8b53dccdfb724545203861e2395c7 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 18:00:44 +0000 Subject: [PATCH 23/27] Extend: image-to-video, first/last-frame and reference videos keep their continuation data - Every two-pass route stores the tail a later continuation needs - A video with a stored record whose first-pass size differs is continued from its own ending --- freevideo_engine/comfy_bridge.py | 64 ++++++++++++++++++++++++++------ freevideo_engine/worker.py | 3 +- 2 files changed, 54 insertions(+), 13 deletions(-) diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index 954a74c1..f7404d67 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -1182,16 +1182,35 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, canvas = validate_request(prompt, width, height, seconds, seed, continue_from=continue_from, frames=frames) source = Path(source or source_root()).resolve() root, machine = installation(source, environ) + continuation = None + + def check_ending(): + try: + from .continuation_tail import check + except ImportError: + return + from .two_pass import sampling_canvas + check(continue_from, sampling_canvas(canvas)[0]) + if continue_from is not None: validate_route(backend=machine.get('device_backend', 'cuda')) - from .continuation import prepare, summary + from .continuation import MISSING, prepare, summary from .two_pass import sampling_canvas # A continuation takes no media or preview, so its sampled size is known here; the snapshot # of the previous segment was written for that canvas, not for the requested size. - continuation = prepare(continue_from, sampling_canvas(canvas)[0], directory=root / 'continuations') + try: + continuation = prepare(continue_from, sampling_canvas(canvas)[0], directory=root / 'continuations') + except ValueError as error: + if str(error) != MISSING: + raise + # A video with no stored context (single-pass, an earlier version, a replaced file) is + # continued from its own ending, which the engine reads and encodes. Only a video whose + # ending cannot be used stops here, before anything is loaded. + check_ending() # The engine decides the same way from the same record: whether the film's first frame is there to use. + # An ending encoded from the file has no first frame of the film to give. anchored = False - if continue_from is not None and first_frame_reference: + if continue_from is not None and continuation is not None and first_frame_reference: from .continuation import chain_first_frame_picture try: anchored = chain_first_frame_picture(continuation, continue_from) is not None @@ -1227,10 +1246,11 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, command += ['--continue-from', str(continue_from)] if first_frame_reference: command.append('--first-frame-reference') - from .continuation import persist_snapshot - continuation = persist_snapshot(continuation, run / 'continuation-source.safetensors') - command += ['--continuation-snapshot', continuation['snapshot'], - '--continuation-snapshot-sha256', continuation['snapshot_sha256']] + if continuation is not None: + from .continuation import persist_snapshot + continuation = persist_snapshot(continuation, run / 'continuation-source.safetensors') + command += ['--continuation-snapshot', continuation['snapshot'], + '--continuation-snapshot-sha256', continuation['snapshot_sha256']] if upscale: command = command[:command.index('generate') + 1] + upscale_command(upscale[1], output, upscale[0], resources) else: @@ -1323,11 +1343,28 @@ def poll(): planned = plan(sampled, two_pass, 'ref2va' if anchored else task_for(extra.get('media', {})), base_steps=base_steps, refine_steps=refine_steps, staged=bool(preview or upscale)) if continue_from is not None: - from .continuation import ADALN_TASK, ANCHOR_ADALN_TASK, validate_snapshot, summary + from .continuation import ADALN_TASK, ANCHOR_ADALN_TASK, MISSING, validate_snapshot, summary validate_route(task='ref2va' if anchored else task_for(extra.get('media', {})), anchor=anchored, media=extra.get('media')) - continuation = validate_snapshot(continuation, sampled, sampling_plan=planned) - state['continuation'] = summary(continuation) + if continuation is not None: + try: + continuation = validate_snapshot(continuation, sampled, sampling_plan=planned) + state['continuation'] = summary(continuation) + except ValueError as error: + if str(error) != MISSING: + raise + # Stored for another first-pass size: the engine sets it aside and encodes the ending. + check_ending() + continuation = None + if anchored: + # An encoded ending carries no first frame of the film, so nothing anchors this run. + anchored = False + planned = plan(sampled, two_pass, task_for(extra.get('media', {})), + base_steps=base_steps, refine_steps=refine_steps, staged=bool(preview or upscale)) + if continuation is None: + from .continuation import OVERLAP_FRAMES + state['continuation'] = dict(overlap_frames=OVERLAP_FRAMES, + new_frames=sampled['frames'] - OVERLAP_FRAMES, encoded=True) # A preview is its own result; an upscale is always run from its preview. keyed = dict(planned, preview=True) if preview else planned identify = lambda check: request_key(prompt, seed, canvas, keyed, extra, machine, @@ -1437,8 +1474,11 @@ def asset_progress(message): report.update(report_extra) save(output.with_suffix('.request.json'), report) if continue_from is not None: - state['continuation']['audio_lead_in_samples'] = engine.get('continuation', {}).get( - 'audio_lead_in_samples', 0) + receipt = engine.get('continuation') or {} + if continuation is None and receipt: + # The ending was encoded by the engine, which is where the real counters come from. + state['continuation'] = dict(receipt) + state['continuation']['audio_lead_in_samples'] = receipt.get('audio_lead_in_samples', 0) if comfy_metadata: # Dropping the video on the canvas restores this graph, as with ComfyUI's own video nodes. # Written before the result cache records the file's size and hash. diff --git a/freevideo_engine/worker.py b/freevideo_engine/worker.py index ddb2d773..db0b8e20 100644 --- a/freevideo_engine/worker.py +++ b/freevideo_engine/worker.py @@ -204,9 +204,10 @@ def fit_decoder(): # A two-pass request that runs as one pass (small canvases) continues # too: its first and final canvases are the same one, so one tail # serves both stages. An explicit single-pass request keeps nothing. + # Whatever media conditioned the request, the sampler returns only the + # generated picture and sound rows, so the tail is the same. plan_flags = request.get('sampling_plan') or {} continuable = (bool(plan_flags.get('enabled') or plan_flags.get('requested')) - and (options.get('task', 't2va') == 't2va' or bool(continuation and continuation.get('anchor'))) and not request.get('preview')) first_tail = continuation_data = None def tail_failure(error): From 9ee81f4ee78796760d9f5106cc9d552dc6556af3 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 18:00:44 +0000 Subject: [PATCH 24/27] Extend: a video without stored continuation data is continued from its own ending - The picture and sound of the ending are read and encoded when nothing usable was stored - A previous video that is too short or cannot be read gets its own message - A retry after such a continuation reuses its sampling - The ending is read with exact colour arithmetic --- freevideo_engine/continuation_tail.py | 267 ++++++++++++++++++++++++++ freevideo_engine/decode_resume.py | 9 +- freevideo_engine/failure_details.py | 9 +- freevideo_engine/generate.py | 55 +++++- freevideo_engine/media_encoding.py | 6 +- freevideo_engine/worker.py | 7 + web/error_panel.js | 8 +- 7 files changed, 347 insertions(+), 14 deletions(-) create mode 100644 freevideo_engine/continuation_tail.py diff --git a/freevideo_engine/continuation_tail.py b/freevideo_engine/continuation_tail.py new file mode 100644 index 00000000..6c508db1 --- /dev/null +++ b/freevideo_engine/continuation_tail.py @@ -0,0 +1,267 @@ +"""A continuation context encoded from the ending of a finished video. + +A video made by an earlier version, or one whose file was replaced after it +was made, has no stored context. Its last 22 frames and the sound under them +are read back and encoded instead. The video encoder works on independent +17-frame clips, so those frames (17 + 5) fall on the same clip grid as the +last 7 latent frames of a generation: nothing before them has to be encoded. +The sound encoder looks back, so it is given up to 15 s and only its last +latents are kept. + +The record describes the ending as a first block of 39 frames. Only the sound +clock reads that length: the kept sound ends exactly where the picture ends. +""" +import math +from pathlib import Path + +from .continuation import (LATENT_FRAMES, MISSING, OVERLAP_FRAMES, SIZE_MISMATCH, audio_origin, validate_frames, + validate_path) + + +NOMINAL_FRAMES = 39 +AUDIO_TICKS = round(NOMINAL_FRAMES * 40 / 24) - audio_origin(NOMINAL_FRAMES - OVERLAP_FRAMES) +AUDIO_RATE = 32000 +TICK_SAMPLES = 800 +AUDIO_WINDOW_TICKS = 600 # 15 s, the longest sound H3 generates +FPS = 24 +TOO_SHORT = 'The previous video is too short to continue; it needs to be at least 2 s long.' +UNREADABLE = 'The previous video cannot be read.' + + +def _fit(image, width, height): + from PIL import Image + return image if image.size == (width, height) else image.resize((width, height), Image.Resampling.LANCZOS) + + +def _picture(frame): + """The frame as an RGB image. + + The decoder library's own conversion comes out about 1/255 darker than the picture that was saved, + which would show as a step where the next part begins. An 8-bit 4:2:0 frame of the usual range is + converted with the exact matrix instead. + """ + import numpy as np + from PIL import Image + if frame.format.name != 'yuv420p' or getattr(frame.color_range, 'value', frame.color_range) not in (0, 1): + return frame.to_image().convert('RGB') + y, cb, cr = frame.reformat(format='yuv444p', interpolation='BICUBIC').to_ndarray().astype(np.float32) + kr, kb = (.2126, .0722) if getattr(frame.colorspace, 'value', frame.colorspace) == 1 else (.299, .114) + y, cb, cr = (y - 16) * (255 / 219), (cb - 128) * (255 / 224), (cr - 128) * (255 / 224) + red, blue = y + 2 * (1 - kr) * cr, y + 2 * (1 - kb) * cb + green = (y - kr * red - kb * blue) / (1 - kr - kb) + return Image.fromarray(np.clip(np.stack([red, green, blue], -1) + .5, 0, 255).astype(np.uint8)) + + +def _rate(stream): + rate = float(stream.average_rate or stream.guessed_rate or 0) + if not 10 <= rate <= 121: + raise ValueError(UNREADABLE) + return rate + + +def _admit(count, rate, width, height, canvas): + """The length of the video as 24 fps frames, when its ending can be continued on `canvas`.""" + frames = count if abs(rate - FPS) <= .01 else round(count * FPS / rate) + if frames < NOMINAL_FRAMES: + raise ValueError(TOO_SHORT) + # The same shape within one 32-pixel step of the canvas; another size of the same shape is scaled. + if min(width, height) < 1 or abs(width * canvas['height'] - height * canvas['width']) > 32 * max(width, height): + raise ValueError(SIZE_MISMATCH) + return frames + + +def check(video_path, canvas): + """Whether the ending of this video can be encoded for `canvas`; a ValueError says why not. + + Reads the file's packets and decodes nothing, so a caller can ask before it queues a run. + """ + import av + try: + with av.open(str(validate_path(video_path))) as container: + if not container.streams.video: + raise ValueError(UNREADABLE) + stream = container.streams.video[0] + count = sum(1 for packet in container.demux(stream) if packet.size) + return _admit(count, _rate(stream), stream.width or 0, stream.height or 0, canvas) + except (av.FFmpegError, OSError) as error: + raise ValueError(UNREADABLE) from error + + +def pending(video_path, canvas): + """A continuation request whose context is encoded from the video when the run starts.""" + validate_frames(canvas['frames']) + check(video_path, canvas) + origin = NOMINAL_FRAMES - OVERLAP_FRAMES + return dict(tail=dict(video=str(Path(video_path).resolve())), overlap_frames=OVERLAP_FRAMES, + new_frames=canvas['frames'] - OVERLAP_FRAMES, frame_origin=origin, + audio_origin=audio_origin(origin), segment=2, audio_latents=AUDIO_TICKS) + + +def read(video_path, canvas, first, directory): + """The last 22 frames at both pass sizes and the sound under them, as files of this run. + + `first` is the first pass's width and height. A video at another frame rate gives the + pictures a 24 fps copy of it would end with. Runs without a GPU. + """ + import av + import numpy as np + from collections import deque + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + try: + with av.open(str(video_path)) as container: + if not container.streams.video: + raise ValueError(UNREADABLE) + rate = _rate(container.streams.video[0]) + kept, count = deque(maxlen=math.ceil(OVERLAP_FRAMES * rate / FPS) + 2), 0 + for frame in container.decode(video=0): + kept.append(frame) + count += 1 + if not count: + raise ValueError(UNREADABLE) + frames = _admit(count, rate, kept[-1].width, kept[-1].height, canvas) + oldest = count - len(kept) + if abs(rate - FPS) <= .01: + picked = range(count - OVERLAP_FRAMES, count) + else: + picked = [min(count - 1, max(oldest, round((frames - OVERLAP_FRAMES + index) * rate / FPS))) + for index in range(OVERLAP_FRAMES)] + images = [_picture(kept[index - oldest]) for index in picked] + del kept + sound = _sound(video_path, count / rate) + except (av.FFmpegError, OSError) as error: + raise ValueError(UNREADABLE) from error + width, height = images[0].size + result = {'frames': frames, 'source': {'width': width, 'height': height, 'fps': rate}} + for name, size in (('hi', canvas), ('lo', first)): + target = directory / ('continuation-tail-%s.npy' % name) + np.save(target, np.stack([np.asarray(_fit(image, size['width'], size['height'])) for image in images]), + allow_pickle=False) + result[name] = str(target) + target = directory / 'continuation-tail-audio.npy' + np.save(target, sound, allow_pickle=False) + result['audio'] = str(target) + return result + + +def _sound(video_path, seconds): + """Up to 15 s of 32 kHz stereo ending where the picture ends; silence where the file has none.""" + import av + import numpy as np + end = round(seconds * AUDIO_RATE) + window = min(AUDIO_WINDOW_TICKS, end // TICK_SAMPLES) * TICK_SAMPLES + begin = end - window + data = np.zeros((2, window), dtype=np.float32) + with av.open(str(video_path)) as container: + if not container.streams.audio: + return data + stream = container.streams.audio[0] + mono = stream.layout.name == 'mono' or stream.channels == 1 + # A mono track has its channel repeated, as reference sound is; a stereo resampler would lower it by 3 dB. + resampler = av.AudioResampler(format='fltp', layout='mono' if mono else 'stereo', rate=AUDIO_RATE) + position = 0 + def place(item): + nonlocal position + block = item.to_ndarray() + start, stop = max(position, begin), min(position + block.shape[1], end) + if stop > start: + data[:, start - begin:stop - begin] = block[:, start - position:stop - position] + position += block.shape[1] + for frame in container.decode(audio=0): + for item in resampler.resample(frame): + place(item) + if position >= end: + break + else: + for item in resampler.resample(None): + place(item) + if not np.isfinite(data).all(): + raise ValueError(UNREADABLE) + return data + + +def encode(tail, base, *, device='cuda', before_upload=None, posterior='mode'): + """Encode the files of `read` into the three tensors of a continuation record.""" + import gc + import numpy as np + import torch + from .media_encoding import encode_visual + from .vae_weights import load_audio_vae, load_video_encoder + empty = torch.mps.empty_cache if torch.device(device).type == 'mps' else ( + torch.cuda.empty_cache if torch.device(device).type == 'cuda' else lambda: None) + result = {} + vae, _ = load_video_encoder(base, before_upload=None if before_upload is None else + lambda model: before_upload(model, False), device=device) + for name in ('hi', 'lo'): + pixels = np.load(tail[name], mmap_mode='r', allow_pickle=False) + latent = encode_visual(vae, pixels, device=device, posterior=posterior) + if latent.shape[2] != LATENT_FRAMES: + raise ValueError('Unexpected continuation latent frames') + result['video_' + name] = latent.numpy() + del pixels, latent + del vae + gc.collect() + empty() + vae, _ = load_audio_vae(base, before_upload=None if before_upload is None else + lambda model: before_upload(model, True), device=device) + if vae.config.sampling_rate != AUDIO_RATE: + raise ValueError('Continuation sound requires the pinned 32 kHz H3 VAE') + mean = torch.tensor(vae.config.latents_mean).view(1, 1, -1) + std = torch.tensor(vae.config.latents_std).view(1, 1, -1) + audio = torch.from_numpy(np.load(tail['audio'], allow_pickle=False)).to(device) + latent = vae.encode(audio[:, None], return_dict=False)[0].mode().float().cpu().transpose(1, 2) + latent = ((latent - mean) / std).transpose(1, 2) + if latent.shape[2] < AUDIO_TICKS: + raise ValueError('Unexpected continuation sound latents') + result['audio'] = latent[:, :, -AUDIO_TICKS:].contiguous().numpy() + del vae, audio, latent + gc.collect() + empty() + return result + + +def metadata(identity, canvas, first): + """The record's description: a first block whose last 22 frames are the file's.""" + return dict(identity=identity, + geometry=dict(width=canvas['width'], height=canvas['height'], frames=NOMINAL_FRAMES), + first_canvas=dict(width=first['width'], height=first['height']), + chain=dict(segment=1, frame_origin=0, audio_origin=0), context='encoded') + + +def freeze(request, canvas, first, directory, base): + """Keep the ending `pending` asked for as files of this run; the worker encodes them.""" + from .continuation_store import video_id + directory = Path(directory) + video = request['tail']['video'] + size = lambda value: {key: value[key] for key in ('width', 'height')} + tail = read(video, canvas, first, directory) + return dict(request, tail=dict(tail, video=video, canvas=size(canvas), first=size(first), base=str(base), + source_id=video_id(video), + snapshot=str((directory / 'continuation-encoded.safetensors').resolve()))) + + +def complete(request, canvas, manifest, sampling_plan=None, *, resident=None, device='cuda'): + """The ordinary snapshot request for a frozen ending, and how long encoding it took. + + The first attempt of a run encodes and writes the record beside its other files; a retry reads it. + """ + import hashlib + import time + from .continuation import identity, validate_snapshot + from .continuation_store import write + tail = request['tail'] + path = Path(tail['snapshot']) + if sampling_plan is not None and any(sampling_plan['first'][key] != tail['first'][key] for key in ('width', 'height')): + raise ValueError(MISSING) + seconds = None + if not path.is_file(): + started = time.perf_counter() + data = encode(tail, tail['base'], device=device, before_upload=None if resident is None else + lambda model, audio: resident.input_vae_room(model, tail['canvas'], audio=audio)) + write(path, dict(data, metadata=dict(metadata(identity(manifest), tail['canvas'], tail['first']), + source_id=tail['source_id']))) + path.chmod(0o400) + seconds = time.perf_counter() - started + result = dict({key: value for key, value in request.items() if key != 'tail'}, snapshot=str(path), + snapshot_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), source_id=tail['source_id']) + return validate_snapshot(result, canvas, manifest, sampling_plan), dict(seconds=seconds, reused=seconds is None) diff --git a/freevideo_engine/decode_resume.py b/freevideo_engine/decode_resume.py index eff55816..4a27033b 100644 --- a/freevideo_engine/decode_resume.py +++ b/freevideo_engine/decode_resume.py @@ -97,7 +97,14 @@ def sampling_provenance(request): if request.get('continuation') is not None: # A retry's identity binds the accepted bytes, without embedding the # snapshot again or making a relocation change computation identity. - result['continuation'] = {key: value for key, value in request['continuation'].items() + value = request['continuation'] + tail = value.get('tail') + if isinstance(tail, dict) and Path(tail.get('snapshot') or '.').is_file(): + # The request as it was sent asks for the ending to be encoded; the worker goes on with the + # record it wrote. A retry compares the two, so both are named by that record. + value = dict({key: item for key, item in value.items() if key != 'tail'}, + snapshot_sha256=_digest(tail['snapshot']), source_id=tail['source_id']) + result['continuation'] = {key: item for key, item in value.items() if key not in ('snapshot', 'snapshot_data', 'source_video')} return result diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index 455eabfa..ef0772d3 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -388,8 +388,13 @@ def launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_rep rules = ( (r'The previous video has no usable continuation data', 'continuation-missing', ('This video cannot be continued', '这个视频暂时不能接续'), - ('Generate the previous video again with this version of FreeVideo. Older videos, replaced files and expired continuation data cannot be used.', - '请用当前版本的 FreeVideo 重新生成上一段视频。旧版本的视频、被替换的文件或已清理接续数据的视频不能接续。')), + ('Generate the previous video again, then continue it.', '请重新生成上一段视频,再接续。')), + (r'The previous video is too short to continue', 'continuation-short', + ('The previous video is too short', '上一段视频太短'), + ('Choose a video that is at least 2 seconds long.', '请选择至少 2 秒的视频。')), + (r'The previous video cannot be read', 'continuation-unreadable', + ('The previous video cannot be read', '无法读取上一段视频'), + ('Choose an MP4 video that plays normally.', '请选择能正常播放的 MP4 视频。')), (r'Use the same width and height as the previous video', 'continuation-size', ('The video sizes differ', '两段视频的画面大小不一致'), ('Set the width and height to the previous video’s size.', '请把宽度和高度设为与上一段视频相同。')), diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index 758b8d5f..1603fc77 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -344,7 +344,29 @@ def _run(args): from .geometry import geometry from .continuation import (validate_frames, validate_route, prepare as prepare_continuation, summary as continuation_summary, validate_snapshot, persist_snapshot, restore_snapshot, - audio_context) + audio_context, MISSING as CONTINUATION_MISSING) + from .continuation_tail import pending as encoded_continuation, freeze as freeze_tail + continuation_directory = continuation_first = None + + def freeze_continuation(value): + from .paths import base_path + return freeze_tail(value, canvas, continuation_first, continuation_directory, + profile['engine'].get('base') or base_path()) + + def usable_continuation(value, *check): + """The stored context when it fits this run, otherwise the previous video's own ending.""" + if 'tail' in value: + return value + try: + return validate_snapshot(value, *check) + except ValueError as error: + if str(error) != CONTINUATION_MISSING: + raise + # A record of another version, other weights or another first-pass size. + value = dict(encoded_continuation(continue_from, canvas), + **{key: value[key] for key in ('audio_context', 'source_video', 'anchor') if key in value}) + return freeze_continuation(value) if continuation_directory is not None else value + frames = getattr(args, 'frames', None) continue_from = getattr(args, 'continue_from', None) continue_anchor = getattr(args, 'continue_anchor', None) @@ -392,12 +414,19 @@ def _run(args): continuation = restore_snapshot(args.continuation_snapshot, getattr(args, 'continuation_snapshot_sha256', None), canvas) else: - continuation = prepare_continuation(continue_from, canvas) + try: + continuation = prepare_continuation(continue_from, canvas) + except ValueError as error: + if str(error) != CONTINUATION_MISSING: + raise + # Nothing was stored for this file: its own ending is encoded when the run starts. + continuation = encoded_continuation(continue_from, canvas) continuation = dict(continuation, audio_context=audio_context(), source_video=str(continue_from)) if first_frame is not None: from .continuation import anchor_media, chain_first_frame_picture try: - kept = chain_first_frame_picture(continuation, continue_from) + # A context encoded from the file carries no first frame; whoever planned this run saw none either. + kept = None if 'tail' in continuation else chain_first_frame_picture(continuation, continue_from) except ValueError: kept = None first_frame['reason'] = 'unreadable' @@ -464,8 +493,9 @@ def _run(args): if continuation is not None: from .two_pass import plan as continuation_plan validate_route(task=profile['engine'].get('task', 't2va'), anchor=continue_anchor is not None) - continuation = validate_snapshot(continuation, canvas, None if active_loras else manifest, - continuation_plan(canvas, two_pass, base_steps=base_steps, refine_steps=refine_steps)) + planned = continuation_plan(canvas, two_pass, base_steps=base_steps, refine_steps=refine_steps) + continuation_first = planned['first'] + continuation = usable_continuation(continuation, canvas, None if active_loras else manifest, planned) if manifest.get('precision') not in ('fp8', 'int8', 'bf16'): # BF16: the unquantized reference raise ValueError('Select the prepared official FP8 or int8 cache') destination = args.out.resolve() @@ -474,7 +504,10 @@ def _run(args): destination.parent.mkdir(parents=True, exist_ok=True) artifacts = destination.with_suffix('.artifacts') artifacts.mkdir(exist_ok=False) - if continuation is not None and 'snapshot' not in continuation: + continuation_directory = artifacts + if continuation is not None and 'tail' in continuation: + continuation = freeze_continuation(continuation) + elif continuation is not None and 'snapshot' not in continuation: continuation = persist_snapshot(continuation, artifacts / 'continuation-source.safetensors') if args.prompt_file: shutil.copyfile(args.prompt_file, artifacts / 'prompt.txt') @@ -491,6 +524,8 @@ def _run(args): 'encoder_mode': 'native H3 encoder library child' if args.prompt_file else 'preencoded shared conditioning'} if continuation is not None: report['continuation'] = continuation_summary(continuation) + if 'tail' in continuation: + report['continuation']['context'] = 'encoded' report['audio_context'] = continuation['audio_context'] if continue_anchor is not None: report['continue_anchor'] = continuation['anchor'] @@ -523,8 +558,10 @@ def interrupted(signum, frame): args._lora_max_block_bytes = prepared['report'].get('max_block_bytes', 0) args._lora_root_bytes = prepared['report'].get('root_bytes', 0) if continuation is not None: - continuation = validate_snapshot(continuation, canvas, + continuation = usable_continuation(continuation, canvas, json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8'))) + if 'tail' in continuation: + report['continuation']['context'] = 'encoded' if args._lora_max_block_bytes or args._lora_root_bytes: profile.setdefault('policy', {})['lora_max_block_bytes'] = args._lora_max_block_bytes profile['policy']['lora_root_bytes'] = args._lora_root_bytes @@ -691,8 +728,10 @@ def interrupted(signum, frame): report['sampling_plan'] = sampling_plan if continuation is not None: validate_route(task=profile['engine'].get('task', 't2va'), anchor=continue_anchor is not None) - continuation = validate_snapshot(continuation, canvas, + continuation = usable_continuation(continuation, canvas, json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8')), sampling_plan) + if 'tail' in continuation: + report['continuation']['context'] = 'encoded' if preview: report['preview'] = dict(geometry=dict(sampling_plan['first'], fps=canvas['fps'])) if refine_source: diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index a67d4e34..a81c7798 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -34,7 +34,7 @@ def reference_size(width, height, canvas): return max(32, round(width * scale / 32) * 32), max(32, round(height * scale / 32) * 32) -def encode_visual(vae, pixels, *, device='cuda'): +def encode_visual(vae, pixels, *, device='cuda', posterior='sample'): """The pinned VAE chunk/padding/posterior recipe with one live pixel chunk. Posterior sampling still occurs once on the original device with seed 42; @@ -58,7 +58,9 @@ def encode_visual(vae, pixels, *, device='cuda'): moments = torch.cat(moments, dim=2).to(device) if len(pixels) > 1 and vae.config.token_drop: moments = moments[:, :, :-vae.config.token_drop] - latent = DiagonalGaussianDistribution(moments).sample(generator=torch.Generator().manual_seed(42)) + distribution = DiagonalGaussianDistribution(moments) + # References keep the pipeline's seeded sample; a continuation context wants the picture itself. + latent = distribution.mode() if posterior == 'mode' else distribution.sample(generator=torch.Generator().manual_seed(42)) latent = latent.to(torch.float16).float().cpu() mean = torch.tensor(vae.config.latents_mean).view(1, -1, 1, 1, 1) std = torch.tensor(vae.config.latents_std).view(1, -1, 1, 1, 1) diff --git a/freevideo_engine/worker.py b/freevideo_engine/worker.py index db0b8e20..5b25e9ff 100644 --- a/freevideo_engine/worker.py +++ b/freevideo_engine/worker.py @@ -164,6 +164,13 @@ def release_idle(): resident.gpu_budget = min(resident.gpu_budget, limit) save(request['metrics'], metrics) manifest = json.loads((Path(request['cache']) / 'manifest.json').read_text(encoding='utf-8')) + if continuation is not None and 'tail' in continuation: + # Nothing was stored for the previous video: its ending is encoded here, before any model of + # this run is loaded, and every later attempt reads the same record. + from .continuation_tail import complete + continuation, metrics['continuation_encode'] = complete( + continuation, request['geometry'], manifest, request.get('sampling_plan'), resident=resident) + request['continuation'] = continuation if manifest.get('precision') not in ('fp8', 'int8', 'bf16'): raise ValueError('The Engine requires an official FP8 or int8 cache') options, decoder = request['engine_options'], request['decoder_options'] diff --git a/web/error_panel.js b/web/error_panel.js index dc158539..ef625d68 100644 --- a/web/error_panel.js +++ b/web/error_panel.js @@ -11,7 +11,13 @@ const record = value => value && typeof value === 'object' && !Array.isArray(val const PLAIN_ERRORS = [ {key: 'continuation-missing', message: 'The previous video has no usable continuation data. Generate it again with this version of FreeVideo.', title: ['This video cannot be continued', '这个视频暂时不能接续'], - summary: ['Generate the previous video again with this version of FreeVideo. Older videos, replaced files and expired continuation data cannot be used.', '请用当前版本的 FreeVideo 重新生成上一段视频。旧版本的视频、被替换的文件或已清理接续数据的视频不能接续。']}, + summary: ['Generate the previous video again, then continue it.', '请重新生成上一段视频,再接续。']}, + {key: 'continuation-short', message: 'The previous video is too short to continue; it needs to be at least 2 s long.', + title: ['The previous video is too short', '上一段视频太短'], + summary: ['Choose a video that is at least 2 seconds long.', '请选择至少 2 秒的视频。']}, + {key: 'continuation-unreadable', message: 'The previous video cannot be read.', + title: ['The previous video cannot be read', '无法读取上一段视频'], + summary: ['Choose an MP4 video that plays normally.', '请选择能正常播放的 MP4 视频。']}, {key: 'continuation-size', message: 'Use the same width and height as the previous video.', title: ['The video sizes differ', '两段视频的画面大小不一致'], summary: ['Set the width and height to the previous video’s size.', '请把宽度和高度设为与上一段视频相同。']}, From 9cdcf43649fb3c01af7d4b4013ac03bae239e41c Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 18:32:09 +0000 Subject: [PATCH 25/27] Extend on a result or a saved video opens the editing view with a draft part ready - The video goes on the timeline with a short draft after it, already selected - Entering again selects the waiting draft instead of adding another - A saved video that is no longer on top, or that cannot be found, does not open the editing view --- web/autopilot.js | 6 +++--- web/editor.js | 33 ++++++++++++++++++++++++++++----- web/editor_entry.js | 4 ++-- web/editor_loader.js | 2 ++ web/editor_tracks.js | 27 +++++++++++++++------------ web/library.js | 7 +++++-- web/result_actions.js | 4 ++-- web/studio.js | 15 +++++++++++++-- web/studio_extend.js | 4 ++-- 9 files changed, 72 insertions(+), 30 deletions(-) diff --git a/web/autopilot.js b/web/autopilot.js index 4cfe7d03..c3829b46 100644 --- a/web/autopilot.js +++ b/web/autopilot.js @@ -5,7 +5,7 @@ import { about, agentChat, jobName } from './autopilot_chat.js'; import { openEditorView } from './editor_loader.js'; import { closeDialog } from './motion.js'; import { openSetup } from './setup.js'; -import { extendVideo, openStudio, showNodes } from './studio.js'; +import { openExtendEditor, openStudio, showNodes } from './studio.js'; import { viewChanged, viewSwitch } from './view_navigation.js'; import { wordmark } from './wordmark.js'; @@ -45,9 +45,9 @@ export function openAutopilotView(node, {studio = null} = {}) { // Leaving for another view waits until this page has closed, so the view underneath is never covered twice. const closed = () => dialog.open ? new Promise(done => { dialog.addEventListener('close', done, {once: true}); closeDialog(dialog); }) : null; const chat = agentChat(t, api, {shown: inSight, - // A finished video opens as its work in Creations. Continuing it there belongs to the studio, so this page gives way first. + // Creations continues the work on the editor timeline; this page gives way first. open: async job => (await import('./library.js')).openLibrary(t, {selectedWork: job.work, back: t('Back to chat', '返回对话'), - extend: async record => { await closed(); return extendVideo(record, node); }}), + extend: async (record, options) => { await closed(); return openExtendEditor(record, node, {studio, ...options}); }}), edit: job => leave(() => openEditorView(node, {studio: studio?.open ? studio : undefined, work: {id: job.work, video: `FreeVideo/${job.work}/video.mp4`, geometry: {width: job.width, height: job.height, seconds: job.seconds}}})), changed: view => { last = view; show(); }}); // With no agent the card stands alone in the middle. Once one has connected the conversation stays, also after diff --git a/web/editor.js b/web/editor.js index 2014834f..a119d602 100644 --- a/web/editor.js +++ b/web/editor.js @@ -141,7 +141,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given tracksHost.inert = genHost.inert = true; title.disabled = true; // CSS delays the pending indication; fast opens stay at full opacity. tracksHost.classList.add('fv-ed-pending'); genHost.classList.add('fv-ed-pending'); - let discardOnSwitch = false, initialSelection = null, offerInitial = null; + let discardOnSwitch = false, initialSelection = null, offerInitial = null, extendEntry = null; let viewKey = null, viewReady = false; let library = null, closing = false, closePromise = null, exportSnapshot = null, lastSeed = null, projectEpoch = 0; let disposed = false, suppressSave = false, savedContent = '', initializing = true, tracks = null, genbar = null, adjustPanel = null, exportStatus = null, pointerEditing = false, deferredStamp = null; @@ -199,7 +199,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given const store = createEditorStore(api, {t, notify, standIn: given?.generationStandIn, onGenerationReady: ready => { if (disposed) return; generationAsked = true; genHost.hidden = !ready; element.dataset.generationReady = String(ready); - settleGenbar(); if (ready) offerInitial?.(); + settleGenbar(); if (ready) { offerInitial?.(); extendEntry?.(); } tracks?.refresh?.(); genbar?.update(mostRecentJob ? barJob(jobStatuses.get(mostRecentJob)) : null); updateTransport(); adjustPanel?.update(); }}); @@ -332,7 +332,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given viewKey = record.id || key || 'draft:' + (globalThis.crypto?.randomUUID?.() || `${Date.now()}-${Math.random()}`); let remembered = editorViews.get(viewKey); if (!remembered) try { remembered = JSON.parse(localStorage.getItem(viewPrefix + viewKey)); } catch {} - initialSelection = offerInitial = null; + initialSelection = offerInitial = extendEntry = null; projectEpoch++; player.pause(); for (const stop of jobWatchers.values()) stop(); jobWatchers.clear(); jobStatuses.clear(); progress.clear(); runProgress.clear(); progressConnection?.reset(); mostRecentJob = null; genbar?.update(null); queue.reset(record); stopExport?.(); stopExport = null; exportStatus = exportSnapshot = null; exportButton.textContent = t('Export', '导出'); @@ -610,10 +610,10 @@ export function openEditor({api, app, from = null, panel = {}, components: given // A sound-only export is not a work: it opens as a file the browser plays and can save. if (value.only === 'sound') { if (value.url) window.open(mediaURL(api, value.url), '_blank', 'noopener'); return; } const module = await import('./library.js'); - module.openLibrary(t, {selectedWork: value.work, extend: async record => { + module.openLibrary(t, {selectedWork: value.work, extend: async (record, options) => { if (await handle.close() === false) throw new Error(saveWarning.textContent); await onView('studio'); - return (await import('./studio.js')).extendVideo(record); + return (await import('./studio.js')).openExtendEditor(record, undefined, options); }}); } function updateExport(value) { @@ -914,6 +914,29 @@ export function openEditor({api, app, from = null, panel = {}, components: given } for (const [id, value] of progress) tracks.setProgress(id, value); tracks.setScrollLeft(editorViews.get(viewKey)?.scrollLeft); + if (from?.work && from.extend) { + const id = from.work.id || from.work.video?.replace(/^FreeVideo\//, '').replace(/\/video\.mp4$/, ''); + const epoch = projectEpoch; + // Selects the work and, when nothing follows it yet, adds one minimum draft. Runs once; when the + // generation probe failed this time it runs after a later probe succeeds. + const enter = (select = true) => { + if (closing || disposed || epoch !== projectEpoch) return true; + const clip = state.record.tracks[0].clips.findLast(clip => clip.source?.kind === 'work' && clip.source.id === id); + if (!clip) return true; + // Coming back to a work that already has parts after it selects the waiting draft instead of adding another. + const edge = clip.start + clip.frames; + const after = state.record.tracks[0].clips.filter(other => other.id !== clip.id && other.start >= edge); + const waiting = after.find(other => other.pending === 'draft'); + if (select) state.select({kind: 'clip', ids: model.pair(state.record, (waiting || clip).id).map(member => member.id)}); + if (after.length || !clip.extend || !clip.source || store.sources.get(sourceKey(clip.source))?.missing) return true; + if (!store.generationAvailable) return false; + const spare = Math.min(...model.pair(state.record, clip.id).map(member => member.length - member.in - member.frames)); + tracks.extendClip(clip.id, spare + model.RULES.minSeconds * model.FPS, {silent: true}); + return true; + }; + await store.generationReady(); + if (!enter() && !closing && !disposed) extendEntry = () => { if (enter(false) || closing || disposed) extendEntry = null; }; + } })().catch(error).finally(() => { // Every exit (including close and partial failure) settles readiness. // After a completed start the saved baseline is already set; edits made since must stay unsaved. diff --git a/web/editor_entry.js b/web/editor_entry.js index 82081776..9d2926e9 100644 --- a/web/editor_entry.js +++ b/web/editor_entry.js @@ -15,7 +15,7 @@ function editorPanel(node) { return {node: String(node.id), prompt: value.output, extra_data: {extra_pnginfo: {workflow: value.workflow || {}}}}; }}; } -export function openEditorView(node, {studio, work = null} = {}) { +export function openEditorView(node, {studio, work = null, extend = false} = {}) { studio?.querySelectorAll('video,audio').forEach(media => media.pause()); const panel = editorPanel(node); const progressElement = studio?.querySelector('.fv-generation-progress'); @@ -27,7 +27,7 @@ export function openEditorView(node, {studio, work = null} = {}) { return number ? {percent: Number(number[1])} : {percent: 0}; }; } - const handle = openEditor({api, app, panel, from: work ? {work} : null, onView(view) { + const handle = openEditor({api, app, panel, from: work ? {work, ...(extend ? {extend: true} : {})} : null, onView(view) { if (view === 'nodes') showNodes(node, studio); else if (view === 'autopilot') openAutopilot(node, {studio}); else if (studio?.open) viewChanged('studio', node); diff --git a/web/editor_loader.js b/web/editor_loader.js index 8221aea1..3b2bc61e 100644 --- a/web/editor_loader.js +++ b/web/editor_loader.js @@ -5,8 +5,10 @@ import {app} from '../../scripts/app.js'; export async function openEditorView(node, options) { try { const entry = await import('./editor_entry.js'); + if (options?.isCurrent?.() === false) return null; return entry.openEditorView(node, options); } catch (error) { + if (options?.isCurrent?.() === false) return null; console.warn('FreeVideo edit view:', error); const language = new URLSearchParams(location.search).get('freevideo_lang'); const cn = language === 'zh' || (language !== 'en' && String(navigator.language || '').toLowerCase().startsWith('zh')); diff --git a/web/editor_tracks.js b/web/editor_tracks.js index efde6c30..c0a821c5 100644 --- a/web/editor_tracks.js +++ b/web/editor_tracks.js @@ -212,6 +212,19 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = actions.editError?.(error); return false; } } + function extendClip(id, frames, {silent = false} = {}) { + if (disposed || actions.generationAvailable?.() === false) return false; + let next; + try { next = model.extendClip(state.record, id, frames, actions.generationLimits?.() || {}); } + catch (error) { + if (!(error instanceof model.EditError)) throw error; + if (!silent) actions.editError?.(error); + return false; + } + const ids = next.tracks[0].clips.filter(clip => clip.pending === 'draft' && !model.find(state.record, clip.id)) + .flatMap(clip => model.pair(next, clip.id).map(member => member.id)); + return attempt(t('Extend', '拉长'), () => next, ids.length ? {selectionAfter: {kind: 'clip', ids}} : undefined); + } function position(node, start, frames) { node.style.left = `${pixels(start)}px`; node.style.width = `${Math.max(2, pixels(frames))}px`; } @@ -840,18 +853,8 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = else if (finished.type === 'plannedBoundary') attempt(t('Resize parts', '调整段长'), record => model.resizePlannedBoundary(record, finished.id, finished.neighbor.clip.id, result.frame, actions.generationLimits?.())); else if (result.kind === 'planned') attempt(t('Resize', '改长短'), record => model.resizePlanned(record, finished.id, finished.edge, result.frame, actions.generationLimits?.())); else if (result.kind === 'extend') { - if (actions.generationAvailable?.() === false) return; const frames = result.frame - finished.clip.start - finished.clip.frames + result.frames; - let next = null; - try { next = model.extendClip(state.record, finished.id, frames, actions.generationLimits?.() || {}); } - catch (error) { if (!(error instanceof model.EditError)) throw error; actions.editError?.(error); } - if (next) { - // The new parts stay selected through undo and redo, so Generate stays usable. - const clip = model.find(next, finished.id), start = clip.start + clip.frames; - const ids = next.tracks[0].clips.filter(c => c.pending === 'draft' && c.start >= start && c.start < start + result.frames) - .flatMap(c => model.pair(next, c.id).map(member => member.id)); - attempt(t('Extend', '拉长'), () => next, ids.length ? {selectionAfter: {kind: 'clip', ids}} : undefined); - } + extendClip(finished.id, frames); } else attempt(t('Trim', '裁短'), record => model.trim(record, finished.id, finished.edge, result.frame)); renderSelection(); } finally { actions.gestureEnd?.(); } @@ -1097,7 +1100,7 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = showInsertion(null); insertion.classList.remove('fv-ed-blocked-insertion'); }, 700); } - return {element, keyboard, menu, dismissPopup, flashBlocked, seekPlayhead, + return {element, keyboard, menu, dismissPopup, flashBlocked, seekPlayhead, extendClip, setSnapping(enabled) { if (typeof enabled !== 'boolean') return {ok: false, reason: 'snapping'}; setTransportSnapping(state, enabled); cachedPoints = allPoints(gesture?.ids || []); diff --git a/web/library.js b/web/library.js index adf7d66d..b79f229a 100644 --- a/web/library.js +++ b/web/library.js @@ -2,6 +2,7 @@ import { api } from '../../scripts/api.js'; import { closeDialog } from './motion.js'; import { resultActions, previewBadge } from './result_actions.js'; import { extendSourceExists } from './studio_extend.js'; +import { topDialog } from './dialog_keys.js'; const style = document.createElement('link'); style.rel = 'stylesheet'; style.href = new URL('./library.css', import.meta.url).href; document.head.append(style); @@ -152,9 +153,11 @@ export function openLibrary(t, {upscale = null, selectedWork = null, extend = nu date.textContent = timestamp(row); geometry.textContent = dimensions(row); links.replaceChildren(resultActions(row, t, { extend: async saved => { + const isCurrent = () => !disposed && dialog.open && dialog.dataset.closing !== 'true' && topDialog() === dialog && selected === saved.video; + if (!isCurrent()) return; player?.pause(); - const run = extend || (async record => (await import('./studio.js')).extendVideo(record)); - try { const studio = await run(saved); dialog.close(); studio?.querySelector('.fv-prompt-editor textarea')?.focus({preventScroll: true}); } + const run = extend || (async (record, options) => (await import('./studio.js')).openExtendEditor(record, undefined, options)); + try { const editor = await run(saved, {isCurrent}); if (editor && !disposed) { dialog.close(); editor.element?.focus({preventScroll: true}); } } catch (error) { notice.textContent = error.message; } }, secondPass: row.preview && upscale ? () => secondPass(row) : null, diff --git a/web/result_actions.js b/web/result_actions.js index 12433ab0..2bbe5f2e 100644 --- a/web/result_actions.js +++ b/web/result_actions.js @@ -1,7 +1,7 @@ import { api } from '../../scripts/api.js'; import { outputDownloadURL } from './output_download.js'; import { shareButton } from './share.js'; -import { EXTEND_ENTRY_OPEN, canExtend, extendText, extendSourceExists } from './studio_extend.js'; +import { canExtend, extendText, extendSourceExists } from './studio_extend.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./result_actions.css', import.meta.url).href; document.head.append(css); @@ -115,7 +115,7 @@ export function resultActions(record, t, {diagnostic = null, secondPass = null, download.href = outputDownloadURL(api, record.video); download.download = ''; bar.append(download); } - if (EXTEND_ENTRY_OPEN && extend && canExtend(record)) { + if (extend && canExtend(record)) { const button = action('button', extendText(t, 'action'), 'extend', 'fv-action-secondary fv-action-extend'); button.hidden = true; extendSourceExists(api, record).then(exists => { button.hidden = !exists; }); diff --git a/web/studio.js b/web/studio.js index b779a979..eced36b2 100644 --- a/web/studio.js +++ b/web/studio.js @@ -1,4 +1,5 @@ import { createErrorPanel, errorText, createErrorReport } from './error_panel.js'; +import {topDialog} from './dialog_keys.js'; import { app } from '../../scripts/app.js'; import { api } from '../../scripts/api.js'; import { openSetup } from './setup.js'; @@ -190,6 +191,14 @@ export async function extendVideo(record, node = current?.node || app.graph?._no return openStudio(node, {extendFrom: await readExtendRecord(api, record)}); } +export async function openExtendEditor(record, node = current?.node || app.graph?._nodes?.find(row => row.type === 'FreeVideoGenerate'), + {studio = current?.node === node ? current?.dialog : undefined, isCurrent} = {}) { + if (!node || isCurrent?.() === false) return null; + const handle = await openEditorView(node, {studio, work: record, extend: true, isCurrent}); + await handle?.ready; + return handle; +} + export function openStudio(node, {extendFrom = null} = {}) { if (current?.node === node && current.dialog.open) { if (extendFrom) current.enterExtend(extendFrom); else current.dialog.focus(); @@ -541,7 +550,8 @@ export function openStudio(node, {extendFrom = null} = {}) { const previewCaption = el('span'), previewTools = el('div', null, 'fv-preview-tools'); previewTools.append(previewCaption); previewTools.append(button(t('Creations', '作品'), () => { - stage.querySelector('video')?.pause(); openLibrary(t, {upscale: value => node.freevideoUpscaleResult(value), extend: saved => extendVideo(saved, node)}); + stage.querySelector('video')?.pause(); openLibrary(t, {upscale: value => node.freevideoUpscaleResult(value), + extend: (saved, options) => openExtendEditor(saved, node, {studio: dialog, ...options})}); })); previewHead.append(previewTools); output.append(previewHead); const previewSpace = el('div', null, 'fv-preview-space'); output.append(previewSpace); const stage = el('div', null, 'fv-stage'); @@ -698,7 +708,8 @@ export function openStudio(node, {extendFrom = null} = {}) { const reported = progress.hasReport(); const tipShowing = !!links.querySelector('.fv-report-tip:not(.fv-report-tip-leaving)'); links.replaceChildren(resultActions(r, t, {diagnostic: reported ? downloadReport : null, - extend: async saved => { try { await extendVideo(saved, node); } catch (error) { resultActionFailed(error); } }, + extend: saved => openExtendEditor(saved, node, {studio: dialog, + isCurrent: () => dialog.open && dialog.dataset.closing !== 'true' && topDialog() === dialog && result === r}), secondPass: r.preview ? secondPass(r) : null})); if (runButtonReady) updateExtendButton(); // After a video finished just now, point at the report menu until the diff --git a/web/studio_extend.js b/web/studio_extend.js index c1e555dc..840a5e41 100644 --- a/web/studio_extend.js +++ b/web/studio_extend.js @@ -8,8 +8,8 @@ export const EXTEND_TEXT = Object.freeze({ }); export const extendText = (t, key) => t(...EXTEND_TEXT[key]); -// The saved video of a continuation holds only what it adds, so the result card and the library keep this entry -// closed until the joined whole video is made; the editor's own extension is unaffected. +// Only the legacy in-studio continuation entry waits for a joined whole video. +// Result cards and Creations continue on the editor timeline independently. export const EXTEND_ENTRY_OPEN = false; export function canExtend(record) { return Boolean(record?.video && /\.mp4$/i.test(record.video) && !record.preview From 48ea8420bf948c53f3688f596c55006a76f31c49 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 21:05:40 +0000 Subject: [PATCH 26/27] AI Autopilot: start a new conversation and return to earlier ones; the agent follows - Earlier conversations stay in a list, newest first - The agent receives the last lines of the conversation the user returned to - A job running in another conversation stays in the header, and the chats button is marked on a narrow page --- freevideo_engine/agent/guide.py | 30 +++-- freevideo_engine/agent/routes.py | 126 ++++++++++++++++--- freevideo_engine/agent/schema.py | 10 +- freevideo_engine/agent/sessions.py | 63 +++++++++- web/autopilot_chat.css | 26 +++- web/autopilot_chat.js | 194 +++++++++++++++++++++++++---- web/autopilot_conversations.js | 21 ++++ web/health.js | 2 +- 8 files changed, 413 insertions(+), 59 deletions(-) create mode 100644 web/autopilot_conversations.js diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 3d683b80..3ac4ba2a 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -351,7 +351,7 @@ def introduction(zh, offers): return rows -def connection_guide(root, base, lang=None, offers=None): +def connection_guide(root, base, lang=None, offers=None, resumed=False): """Public bootstrap text contains locations and commands, never credential values.""" import json from pathlib import Path @@ -403,17 +403,27 @@ def connection_guide(root, base, lang=None, offers=None): description=CAPABILITIES_ZH.get((method, suffix), row['summary']) if zh else row['summary']) for (method, suffix), row in contracts().items() if not suffix.startswith('/freevideo/') and not suffix.startswith('/connect')], - first_reply='用中文回复用户;如果用户改用其他语言,就跟随用户。连接后先用通俗的话告诉用户你能做“先介绍这几样”里的事,' - '只说这几样,并举两三个用户可以提出的要求。不要报版本号,不要念接口名。' - '不要自行启动任务,等待用户提出要求。' if zh else - 'Reply to the user in English; if the user switches languages, follow their language. ' - 'After connecting, first reply in plain words: tell the user you can do the things under ' - '"Introduce these", only those, and give two or three requests they can say. Do not quote version ' - 'numbers or endpoint names. Do not start any task yourself; wait for the user to ask.', + # An agent connecting again while the user's conversation is under way has greeted there once already. + first_reply=(('用中文回复用户;如果用户改用其他语言,就跟随用户。用户所在的对话已经开始了:连接后不要自我介绍,' + '也不要说“已连接”,先调一次等待接口,回应里的 before 是这段对话的最后几句;什么都不说,等用户开口再接着做。' + '不要自行启动任务。' if zh else + 'Reply to the user in English; if the user switches languages, follow their language. ' + 'The user is in a conversation that has already begun: after connecting do not introduce yourself ' + 'and do not say that you are connected. Call the wait once; "before" in its answer holds the last ' + 'lines of that conversation. Say nothing until the user speaks, then carry on from there. ' + 'Do not start any task yourself.') if resumed else ( + '用中文回复用户;如果用户改用其他语言,就跟随用户。连接后先用通俗的话告诉用户你能做“先介绍这几样”里的事,' + '只说这几样,并举两三个用户可以提出的要求。不要报版本号,不要念接口名。' + '不要自行启动任务,等待用户提出要求。如果第一次等待的回应说这段对话已经在进行,就不再介绍,接着之前的做。' if zh else + 'Reply to the user in English; if the user switches languages, follow their language. ' + 'After connecting, first reply in plain words: tell the user you can do the things under ' + '"Introduce these", only those, and give two or three requests they can say. Do not quote version ' + 'numbers or endpoint names. Do not start any task yourself; wait for the user to ask. If the first wait says ' + 'the conversation is already under way, do not introduce yourself again; carry on from it.')), introduce=introduction(zh, offers or {}), conversation='用户会在 FreeVideo 的“AI 全自动”页里直接和你说话,你的话也显示在那里。' '连接后先调一次等待接口 GET ' + base + PREFIX + '/inbox?wait=50(MCP 用不带 id 的 freevideo_wait),' - '回应里的 session 就是这段对话;往 POST ' + base + PREFIX + '/sessions/{session}/events 发 ' + '回应里的 session 就是用户现在所在的对话(用户新开或换到另一段对话时它会变,话总是发到最近一次等待给的那个 session);往 POST ' + base + PREFIX + '/sessions/{session}/events 发 ' '{"type":"message","text":"…"}(MCP 用 freevideo_session_post)。在这里先只说一句话,不超过 20 个字,' '不列条目、不举例(页面上已经有例句),例如“连上了。告诉我想做什么视频。”' '你自己窗口里的那段介绍照常说,但它不是这一轮的结尾:说完紧接着去调等待接口。' @@ -430,7 +440,7 @@ def connection_guide(root, base, lang=None, offers=None): '这段对话里不要新建会话。只有用户明确让你停,才停。' if zh else 'The user talks to you inside FreeVideo, in the Autopilot view, and your words are shown there. ' 'After connecting, call the wait once: GET ' + base + PREFIX + '/inbox?wait=50 (MCP: freevideo_wait ' - 'without id). Its "session" is this conversation; POST ' + base + PREFIX + '/sessions/{session}/events ' + 'without id). Its "session" is the conversation the user is in (it changes when the user opens or returns to another one; always post to the session the latest wait named); POST ' + base + PREFIX + '/sessions/{session}/events ' 'with {"type":"message","text":"..."} (MCP: freevideo_session_post). Say only one short sentence ' 'there first, at most 12 words, no list and no examples (the page already shows examples), for example ' '"Connected. Tell me what video you want.", in the language the user writes to you in. Give the introduction in your own window as usual, but it ' diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index cb3ba8d1..96400e7e 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -32,6 +32,16 @@ # What an empty wait tells the agent: the conversation is open for as long as the user keeps the page. KEEP_WAITING = ('Nothing new yet. The user is still in FreeVideo and may speak at any moment: call this wait ' 'again now. An empty wait never ends the conversation; do not end your turn or write a summary.') +# The user may open another conversation while the agent waits; its words go where the user now is. +MOVED = ('The user is now in another conversation, session %s. Post your words to that session from now on; ' + 'the user no longer sees the earlier one.') +MOVED_NEW = ' It starts afresh: do not carry requests over from the earlier conversation unless the user refers to them.' +MOVED_BACK = (' It is one the user was in before, and the user means what was said there, not the conversation you just ' + 'left. Its last lines are in "before"; all of it: GET /sessions/%s/events.') +# An agent that connects again finds the conversation where the user left it, and has greeted there once already. +RESUMED = ('This conversation is already under way; its last lines are in "before". Do not introduce yourself again: ' + 'say nothing until the user speaks, then carry on from there.') +BEFORE_LINES, BEFORE_TEXT = 8, 600 def local_address(request): @@ -391,7 +401,7 @@ def reply(value, status=200): page_sessions = {} connection = dict(connected=False, client=None, via=None, last_seen_at=None) # The conversation in the Autopilot view: the page and the connected agent talk in the agent's latest session. - chat = dict(client=None, waiting=0, folds={}) + chat = dict(client=None, waiting=0, folds={}, told={}) server._freevideo_agent_connection = connection def problem(code, message=None, status=None, **details): @@ -529,7 +539,11 @@ def attach(): if sid: store.get(sid) # not_found for an unknown session else: - sid = store.latest_for(client) or (store.create(client=client)['id'] if write else None) + # Work belongs to the conversation the agent last heard from; until its next wait it has + # not learned that the user opened another one. + told = chat['told'].get(client) + sid = told if told is not None and store.has(client, told) else ( + store.current_for(client) or (store.create(client=client)['id'] if write else None)) if sid is None: return None request['agent_session'] = sid @@ -787,18 +801,30 @@ def chat_current(rows, sid, fold): key=lambda r: (r['status'] in ('running', 'cancelling'), r['created_at'])) return mine, active[-1] if active else None - def chat_session(store): - return store.latest_for(chat['client']) if store is not None and chat['client'] is not None else None + def chat_session(store, sid=None): + # The conversation the user is in, or an earlier one of the connected agent that the page names. + if store is None or chat['client'] is None: + return None + if sid is None: + return store.current_for(chat['client']) + if not store.has(chat['client'], sid): + raise JobError('not_found', 'No such conversation') + return sid + + def chat_moved(store, before): + # The agent waits in the conversation the user was in; let that wait look again. + if store.current_for(chat['client']) != before: + store.wake(chat['client']) @routes.get('/freevideo/agent/chat') @guarded async def chat_read(request): - after = request[query_key].get('after') + after, wanted = request[query_key].get('after'), request[query_key].get('session') def read(): store = work_sessions() - sid = chat_session(store) + sid = chat_session(store, wanted) rows = jobs.rows() - value = dict(agent=agent_state(), session=sid, last_seq=0, read_seq=0, items=[], jobs={}, confirms={}, current=None, + value = dict(agent=agent_state(), session=sid, here=chat_session(store), last_seq=0, read_seq=0, items=[], jobs={}, confirms={}, current=None, active=sum(1 for r in rows if r['status'] in ACTIVE)) if sid is None: return value @@ -854,7 +880,8 @@ def write(): store, client = work_sessions(), chat['client'] if store is None or client is None: return None - sid = store.latest_for(client) or store.create(client=client, by='user')['id'] + before = store.current_for(client) + sid = chat_session(store, value.get('session')) or store.create(client=client, by='user')['id'] fold = chat_fold(store, sid) if value['id'] in fold['sent']: seq = fold['sent'][value['id']] @@ -869,6 +896,7 @@ def write(): fold['ids'][row['seq']] = value['id'] while len(fold['sent']) > 500: fold['ids'].pop(fold['sent'].pop(next(iter(fold['sent']))), None) + chat_moved(store, before) return 201, dict(seq=row['seq'], id=value['id'], at=row['at'], session=sid, read=False, agent=agent_state()) done = await asyncio.to_thread(write) if done is None: @@ -884,14 +912,49 @@ async def chat_confirm(request): value = await body(request) def write(): store = work_sessions() - sid = chat_session(store) + before = chat_session(store) + sid = chat_session(store, value.get('session')) if sid is None: raise JobError('not_found', 'No conversation') row = store.append(sid, dict(type='confirm', id=value['id'], state='approved' if value['approve'] else 'declined'), by='user') + chat_moved(store, before) return dict(id=row['id'], state=row['state']) return reply(await asyncio.to_thread(write)) + @routes.get('/freevideo/agent/chat/list') + @guarded + async def chat_list(request): + def read(): + store = work_sessions() + here = chat_session(store) + rows = store.history(chat['client']) if here else [] + active = [r.get('session') for r in jobs.rows() if r['status'] in ACTIVE] + return dict(agent=agent_state(), here=here, + conversations=[dict(row, running=active.count(row['id'])) for row in rows]) + return reply(await asyncio.to_thread(read)) + + @routes.post('/freevideo/agent/chat/new') + @guarded + async def chat_new(request): + def write(): + store, client = work_sessions(), chat['client'] + if store is None or client is None: + return None + before = store.current_for(client) + # A conversation the user has not spoken in yet is the new one already. + if before is not None and not store.spoken(before): + return 200, dict(session=before, agent=agent_state()) + sid = store.create(client=client, by='user')['id'] + chat_moved(store, before) + return 201, dict(session=sid, agent=agent_state()) + done = await asyncio.to_thread(write) + if done is None: + return reply({'error': dict(code='not_connected', message='No agent is connected.', retryable=True, + details=[], next='Connect an agent in the Autopilot view, then try again.', + request_id=errors.request_id())}, 409) + return reply(done[1], done[0]) + @routes.get('/freevideo/agent/chat/video') @guarded async def chat_video(request): @@ -975,8 +1038,13 @@ async def connect_guide(request): root = installation_root() prepare_token(root) from .editor_runtime import available + store = work_sessions() + def begun(): + here = store.current_for(chat['client']) if store is not None and chat['client'] else None + return bool(here and store.get(here)['last_seq']) value = connection_guide(root, request.scheme + '://' + request.host, request[query_key].get('lang'), - dict(continuation=await asyncio.to_thread(jobs.continuation_ready), editing=available())) + dict(continuation=await asyncio.to_thread(jobs.continuation_ready), editing=available()), + resumed=await asyncio.to_thread(begun)) if request[query_key].get('format') == 'json': return reply(value) return web.Response(text=connection_text(value), content_type='text/plain', headers={'Cache-Control': 'no-store'}) @@ -986,6 +1054,7 @@ async def connect_guide(request): async def hello(request): value = await body(request) connection.update(connected=True, client=value['client'], via=value['via'], last_seen_at=time.time()) + chat['told'].pop(request['agent_client'], None) # a newly connected agent has heard from no conversation yet unfinished = await asyncio.to_thread(jobs.productions.list, status='active', limit=5) return reply(dict(connection, unfinished=unfinished)) @@ -1198,8 +1267,15 @@ async def session_post(request): return reply(await stop_session(sid), 201) if by == 'agent' and value['type'] == 'message' and isinstance(value.get('text'), str): value['text'] = chat_text(value['text']) - return reply(await asyncio.to_thread(sessions.append, sid, value, - by=by, client=session_client(request, by)), 201) + client = session_client(request, by) + def write(): + row = sessions.append(sid, value, by=by, client=client) + here = sessions.current_for(client) if client and value['type'] == 'message' else None + if here not in (None, sid): + # Said where the user no longer looks: the answer itself tells the agent where to say it. + row['next'] = MOVED % here + return row + return reply(await asyncio.to_thread(write), 201) @routes.get(PREFIX + '/confirmations') @guarded @@ -1485,12 +1561,29 @@ async def inbox(request): client = request['agent_client'] if store is not None and sid is None: # An agent that only listens has no session yet: give it one, so the page can speak into it. - sid = await asyncio.to_thread(lambda: store.latest_for(client) or store.create(client=client)['id']) + sid = await asyncio.to_thread(lambda: store.current_for(client) or store.create(client=client)['id']) request['agent_session'] = sid + # Without a session of its own choosing, the agent listens wherever its user is. + follow = store is not None and not request.headers.get('X-FreeVideo-Session', '').strip() + def answer(value): + told, chat['told'][client] = chat['told'].get(client), value['session'] + if follow and told != value['session']: + first = value['inbox'][0]['seq'] if value['inbox'] else store.get(value['session'])['last_seq'] + 1 + if first > 1: + # What the user sees above these words. An agent that has to fetch it answers from the conversation it left. + value['before'] = [dict(e, text=e['text'][:BEFORE_TEXT]) if isinstance(e.get('text'), str) else e + for e in store.events(value['session'], after=max(0, first - 1 - BEFORE_LINES), + limit=BEFORE_LINES) if e['seq'] < first] + said = (RESUMED if first > 1 else '') if told is None else ( + MOVED % value['session'] + (MOVED_BACK % value['session'] if first > 1 else MOVED_NEW)) + value['next'] = (said + ' ' + value.get('next', '')).strip() + return reply(value) deadline = time.monotonic() + min(50., float(request[query_key].get('wait', '0') or 0)) chat['waiting'] += 1 try: while True: + if follow: + sid = request['agent_session'] = await asyncio.to_thread(store.current_for, client) or sid if store is not None and sid is not None: # Mark before reading so words arriving in between wake the wait below. def read(): @@ -1498,16 +1591,17 @@ def read(): return mark, store.unread(sid, client) mark, pending = await asyncio.to_thread(read) if pending: - return reply(dict(session=sid, inbox=pending[-20:])) + return answer(dict(session=sid, inbox=pending[-20:])) left = deadline - time.monotonic() if left <= 0: # again: nothing new this time; the caller simply waits once more. An agent decides what to # do next from the last answer it read, so the empty one says it in words as well. - return reply(dict(session=sid, inbox=[], again=True, next=KEEP_WAITING)) + return answer(dict(session=sid, inbox=[], again=True, next=KEEP_WAITING)) if store is None or sid is None: await asyncio.sleep(min(left, 1.)) else: - await store.wait(sid, after=mark, seconds=min(left, 25.), by='user') + await store.wait(sid, after=mark, seconds=min(left, 25.), by='user', + leave=(lambda here=sid: store.current_for(client) != here) if follow else None) finally: chat['waiting'] -= 1 # The agent was here for the whole wait, not only when it began. diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index f8d8fb4c..2cdd74e8 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -354,12 +354,12 @@ def route(summary, body=None, query=None, **kw): for suffix, method, body in (('', 'GET', None), ('/session', 'GET', None), ('/activity', 'GET', None), ('/activity/cancel', 'POST', obj({'id': JOB}, ('id',))), ('/connect', 'GET', None), ('/connect/reset', 'POST', EMPTY), ('/chat', 'GET', None), ('/chat/now', 'GET', None), - ('/chat/video', 'GET', None), - ('/chat', 'POST', obj({'text': string(8000), 'id': CHAT_ID}, ('text', 'id'))), - ('/chat/confirm', 'POST', obj({'id': string(12, pattern=r'^cf[0-9a-f]{10}$'), 'approve': {'type': 'boolean'}}, - ('id', 'approve')))): + ('/chat/video', 'GET', None), ('/chat/list', 'GET', None), ('/chat/new', 'POST', EMPTY), + ('/chat', 'POST', obj({'text': string(8000), 'id': CHAT_ID, 'session': SESSION}, ('text', 'id'))), + ('/chat/confirm', 'POST', obj({'id': string(12, pattern=r'^cf[0-9a-f]{10}$'), 'approve': {'type': 'boolean'}, + 'session': SESSION}, ('id', 'approve')))): routes[(method, '/freevideo/agent' + suffix)] = route('Studio Agent controls', body=body, auth=False, local=True) - routes['GET', '/freevideo/agent/chat']['query'] = obj({'after': number(0, integer=True)}) + routes['GET', '/freevideo/agent/chat']['query'] = obj({'after': number(0, integer=True), 'session': SESSION}) routes['GET', '/freevideo/agent/chat/video']['query'] = obj({'id': string(43, pattern=r'^[0-9]{4}-[0-9]{2}-[0-9]{2}/[0-9a-f]{32}$')}, ('id',)) for suffix in ('', '/connect'): routes['GET', '/freevideo/agent' + suffix]['query'] = obj({'lang': enum(('zh', 'en'))}) diff --git a/freevideo_engine/agent/sessions.py b/freevideo_engine/agent/sessions.py index 5d48ab05..5fd002eb 100644 --- a/freevideo_engine/agent/sessions.py +++ b/freevideo_engine/agent/sessions.py @@ -58,6 +58,11 @@ def __init__(self, root: Path, *, clock=time.time, max_events=20000): self._initial_seq = {} self._readers = {} self._user_seq = {} # Last event the user wrote; unread() skips the file when nothing is newer. + # When each session began or its user last acted in it (spoke, answered), and the user's first words + # there: the conversation the user is in, and what the page calls each one in its history. + self._acted = {} + self._said = {} + self._chosen = set() self._waiters = {} self._partial = set() self.root.mkdir(parents=True, exist_ok=True) @@ -91,6 +96,9 @@ def _load(self): updated_at=meta['created_at'], client=meta['client'], last_seq=0, stopped=False) self._ids[sid] = {} self._clients[sid] = {meta['client']} if meta['client'] else set() + self._acted[sid] = (meta['created_at'], 1) + if meta.get('by') == 'user': + self._chosen.add(sid) for row in stream: if not row.endswith(b'\n'): self._partial.add(sid) @@ -111,6 +119,12 @@ def _fold(self, sid, event): meta.update(last_seq=event['seq'], updated_at=event['at']) if event['by'] == 'user': self._user_seq[sid] = event['seq'] + # No counter here: the order must come out the same when the files are read again after a + # restart, so two things in one clock tick are ordered by what they are, then by session. + self._acted[sid] = max(self._acted[sid], (event['at'], 0)) + self._chosen.add(sid) + if event['type'] == 'message' and sid not in self._said: + self._said[sid] = event['text'].strip()[:60] if event.get('client'): self._clients[sid].add(event['client']) if event.get('id'): @@ -149,6 +163,9 @@ def create(self, *, title=None, client=None, by='agent') -> dict: updated_at=meta['created_at'], client=client, last_seq=0, stopped=False) self._ids[sid], self._clients[sid] = {}, {client} if client else set() self._initial_seq[sid] = 0 + self._acted[sid] = (meta['created_at'], 1) + if by == 'user': + self._chosen.add(sid) return deepcopy(self._meta[sid]) def list(self, *, limit=50, before=None) -> dict: @@ -244,7 +261,9 @@ def events(self, sid, *, after=0, limit=500) -> list[dict]: self._get(sid) return self._events(sid, after, limit) - async def wait(self, sid, *, after, seconds, types=None, by=None) -> list[dict]: + async def wait(self, sid, *, after, seconds, types=None, by=None, leave=None) -> list[dict]: + """Events after a cursor, waiting up to some seconds. leave, when given, ends an empty wait early + once it answers true: wake() asks for that look when the user moves to another conversation.""" _check(schema.session_schema('events'), dict(after=after, wait=seconds)) if types is not None: _check(schema.array(schema.enum(('message', 'plan', 'step', 'confirm', 'control')), high=5), types) @@ -294,6 +313,8 @@ async def offload(work): found, cursor = await offload(read) if found: return found + if leave is not None and await offload(leave): + return [] remaining = deadline - loop.time() if remaining <= 0: return [] @@ -316,6 +337,46 @@ def latest_for(self, client) -> str | None: rows = [row for sid, row in self._meta.items() if client in self._clients[sid]] return max(rows, key=lambda r: (r['updated_at'], r['id']))['id'] if rows else None + def current_for(self, client) -> str | None: + """The conversation this client's user is in: the newest one, or an older one the user spoke or + answered in after that; what the agent or the server writes never moves the user. Until the user + has opened a conversation or spoken on the page, it is wherever the latest work is, as before.""" + with self.lock: + rows = [sid for sid in self._meta if client in self._clients[sid]] + if not rows: + return None + if not self._chosen.intersection(rows): + return max(rows, key=lambda sid: (self._meta[sid]['updated_at'], sid)) + return max(rows, key=lambda sid: (self._acted[sid], sid)) + + def has(self, client, sid) -> bool: + with self.lock: + return sid in self._meta and client in self._clients[sid] + + def spoken(self, sid) -> bool: + """Whether the user has said anything in this conversation.""" + with self.lock: + return sid in self._said + + def wake(self, client) -> None: + """Let this client's waits look again: its user moved to another conversation.""" + with self.lock: + waiters = [w for sid, group in self._waiters.items() if client in self._clients[sid] for w in group] + for loop, signal in waiters: + try: + loop.call_soon_threadsafe(signal.set) + except RuntimeError: + pass # A closed client loop no longer needs notification. + + def history(self, client, *, limit=50) -> list[dict]: + """This client's conversations the user has spoken in, for the page: latest change first.""" + current = self.current_for(client) + with self.lock: + rows = sorted((row for sid, row in self._meta.items() if client in self._clients[sid] and sid in self._said), + key=lambda r: (r['updated_at'], r['id']), reverse=True) + return [dict(id=r['id'], title=self._said[r['id']], updated_at=r['updated_at'], current=r['id'] == current) + for r in rows[:limit]] + def read_until(self, sid, reader) -> int: """The last event this reader has been given; unread() moves it.""" with self.lock: diff --git a/web/autopilot_chat.css b/web/autopilot_chat.css index 934f04d1..ea0c8026 100644 --- a/web/autopilot_chat.css +++ b/web/autopilot_chat.css @@ -1,6 +1,19 @@ /* The conversation with an agent: one column, the box to type in fixed under it. */ -.fv-chat{box-sizing:border-box;position:relative;min-height:0;min-width:0;display:grid;grid-template-rows:minmax(0,1fr) auto;grid-template-columns:minmax(0,1fr);gap:16px;padding:0 40px 22px} +.fv-chat{--fv-chats-width:232px;--fv-chats-pad:12px;--fv-chats-gap:6px;--fv-chats-row:40px;--fv-chats-radius:9px;--fv-chats-font:14px;--fv-chats-dot:6px;--fv-chats-toggle:34px;box-sizing:border-box;position:relative;min-height:0;min-width:0;display:grid;grid-template-columns:minmax(0,1fr)} +.fv-chat-main{position:relative;min-height:0;min-width:0;display:grid;grid-template-rows:minmax(0,1fr) auto;grid-template-columns:minmax(0,1fr);gap:16px;padding:0 40px 22px} .fv-chat[hidden],.fv-chat [hidden]{display:none} +/* List dimensions are kept here for art review; the breakpoint below is shared with the drawer's keyboard behavior. */ +.fv-chat[data-conversations=true]{grid-template-columns:var(--fv-chats-width) minmax(0,1fr)} +.fv-chat-conversations{box-sizing:border-box;min-height:0;min-width:0;display:flex;flex-direction:column;gap:var(--fv-chats-gap);padding:16px var(--fv-chats-pad);border-right:1px solid var(--fv-border);background:var(--fv-bg)} +.fv-chat-conversation-list{min-height:0;overflow:auto;display:grid;align-content:start;gap:var(--fv-chats-gap)} +.fv-studio .fv-chat-conversations button{box-sizing:border-box;flex:none;justify-content:flex-start;min-width:0;width:100%;min-height:var(--fv-chats-row);padding:8px var(--fv-chats-pad);border-radius:var(--fv-chats-radius);border:1px solid transparent;background:transparent;font-size:var(--fv-chats-font);font-weight:var(--fv-regular);color:var(--fv-ink);text-align:left;gap:10px} +.fv-studio .fv-chat-conversations .fv-chat-new{border-color:var(--fv-border);margin-bottom:6px} +.fv-studio .fv-chat-conversations button:hover:not(:disabled){background:var(--fv-raised)} +.fv-studio .fv-chat-conversations button:focus-visible{outline-offset:-2px} +.fv-studio .fv-chat-conversation[aria-current]{background:var(--fv-raised);border-color:var(--fv-accent-dim)} +.fv-chat-conversation span{min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +.fv-chat-running{flex:none;width:var(--fv-chats-dot);height:var(--fv-chats-dot);border-radius:50%;background:var(--fv-accent)} +.fv-chat-tools{display:none} .fv-chat-scroll{min-height:0;overflow:auto;margin:0 -40px;padding:24px 40px 2px;scrollbar-gutter:stable both-edges} .fv-chat-log{list-style:none;margin:0 auto;padding:0;width:100%;max-width:720px;display:grid;grid-template-columns:minmax(0,1fr);align-content:start;gap:18px} .fv-chat-log>li{min-width:0}.fv-chat-log p{margin:0;line-height:1.6} @@ -69,6 +82,15 @@ .fv-studio .fv-chat-send:hover:not(:disabled){background:var(--fv-accent-hover)} .fv-studio .fv-chat-send:disabled{opacity:1;background:#22344a;color:#9fb3c8} .fv-studio .fv-chat-send:focus-visible{outline-offset:2px} -@media(max-width:480px){.fv-chat{padding:0 14px 14px}.fv-chat-scroll{margin:0 -14px;padding:16px 14px 2px}.fv-chat-hello{padding-top:32px}.fv-chat-job{max-width:none;gap:12px}.fv-chat-confirm{max-width:none} +@media(max-width:720px){ + .fv-chat[data-conversations=true]{grid-template-columns:minmax(0,1fr)} + .fv-chat[data-conversations=true] .fv-chat-main{grid-template-rows:auto minmax(0,1fr) auto} + .fv-chat-tools:not([hidden]){display:flex;padding-top:12px;margin-bottom:-12px} + .fv-studio .fv-chat-chats{position:relative;width:var(--fv-chats-toggle);height:var(--fv-chats-toggle);min-height:0;padding:0;border-radius:var(--fv-chats-radius);background:var(--fv-card)} + .fv-chat-chats .fv-chat-running{position:absolute;top:5px;right:5px} + .fv-chat-conversations{display:none;position:absolute;inset:0 auto 0 0;z-index:3;width:var(--fv-chats-width);box-shadow:16px 0 32px #0005} + .fv-chat[data-chats-open=true] .fv-chat-conversations:not([hidden]){display:flex} +} +@media(max-width:480px){.fv-chat-main{padding:0 14px 14px}.fv-chat-scroll{margin:0 -14px;padding:16px 14px 2px}.fv-chat-hello{padding-top:32px}.fv-chat-job{max-width:none;gap:12px}.fv-chat-confirm{max-width:none} .fv-chat-preview,.fv-chat-preview[data-shape=wide]{width:98px}.fv-chat-length{display:none}.fv-chat-percent{font-size:22px}.fv-studio.fv-autopilot .fv-chat-input textarea{font-size:16px}.fv-studio .fv-chat-down{right:14px;bottom:84px}} @media(prefers-reduced-motion:reduce){.fv-chat-water,.fv-chat-percent,.fv-chat-parts>i::after{transition:none}.fv-chat-water::after,.fv-chat-dots>i,.fv-chat-preview>img{animation:none}} diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index b70a3e2a..533ac560 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -2,6 +2,7 @@ // cards at the place they happened. Reads and sends use the page session; the agent's token never enters the page. import {agentGet, agentPost} from './agent_session.js'; import { estimatedFraction, remainingAbove, remainingShown } from './generation_progress.js'; +import {emptyConversations, hasConversations, mergeConversations, runningElsewhere, selectConversation} from './autopilot_conversations.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./autopilot_chat.css', import.meta.url).href; document.head.append(css); @@ -13,6 +14,7 @@ function glyph(path, width = 2.2) { return svg; } const press = (cls, text, action) => { const b = el('button', cls, text); b.type = 'button'; b.onclick = action; return b; }; +let chatNumber = 0; const WORKING = ['running', 'queued', 'waiting'], VIDEOS = ['generate', 'assemble']; // What a job with no picture of its own is doing, in the reader's language. @@ -136,16 +138,110 @@ function newId() { // user can see the conversation; the page calls sync() whenever that may have changed. changed(view) follows every // answer: {known, agent, current, active, any}. export function agentChat(t, api, {open, edit, shown = () => true, changed = () => {}, interval = 1500} = {}) { - const element = el('section', 'fv-chat'), scroller = el('div', 'fv-chat-scroll'), log = el('ol', 'fv-chat-log'); + const element = el('section', 'fv-chat'), main = el('div', 'fv-chat-main'), scroller = el('div', 'fv-chat-scroll'), log = el('ol', 'fv-chat-log'); log.setAttribute('role', 'log'); log.setAttribute('aria-label', t('Conversation', '对话')); const down = press('fv-chat-down', null, () => { scroller.scrollTo({top: scroller.scrollHeight, behavior: 'smooth'}); }); down.append(glyph('M12 5v14M5 12l7 7 7-7')); down.setAttribute('aria-label', t('Latest', '最新')); down.hidden = true; const form = el('form', 'fv-chat-input'), input = el('textarea'), sendButton = el('button', 'fv-chat-send'); input.rows = 1; input.maxLength = 8000; input.setAttribute('aria-label', t('Message the agent', '对 Agent 说')); sendButton.type = 'submit'; sendButton.setAttribute('aria-label', t('Send', '发送')); sendButton.append(glyph('M12 19V5M5 12l7-7 7 7')); - form.append(input, sendButton); scroller.append(log); element.append(scroller, down, form); + form.append(input, sendButton); scroller.append(log); main.append(scroller, down, form); element.append(main); let state = emptyChat(), pending = [], known = false, stopped = false, timer = null, busy = false, failures = 0; + let conversations = emptyConversations(), selectedSession, selection = 0, reread = false, focusSession = null, creating = false; + // Work under way in another conversation while this one has none: {session, job, ids}. The page's header keeps it in sight. + let away = null; + let listBusy = false, listAgain = false, listAt = -Infinity; + const listInterval = 10000; + const sidebar = el('nav', 'fv-chat-conversations'), list = el('div', 'fv-chat-conversation-list'), tools = el('div', 'fv-chat-tools'); + sidebar.id = `fv-chat-conversations-${++chatNumber}`; sidebar.setAttribute('aria-label', t('Chats', '对话')); sidebar.hidden = true; + const newChat = press('fv-chat-new', t('New chat', '新对话'), () => void startChat()); + newChat.prepend(glyph('M12 5v14M5 12h14')); newChat.title = t('New chat', '新对话'); + const chats = press('fv-chat-chats', null, () => drawer(chats.getAttribute('aria-expanded') !== 'true')); + const chatsDot = el('i', 'fv-chat-running'); chatsDot.setAttribute('aria-hidden', 'true'); chatsDot.hidden = true; + chats.append(glyph('M4 5h16v12H9l-5 4V5M8 9h8M8 13h5'), chatsDot); + chats.title = t('Chats', '对话'); chats.setAttribute('aria-label', chats.title); + chats.setAttribute('aria-controls', sidebar.id); chats.setAttribute('aria-expanded', 'false'); + tools.hidden = true; tools.append(chats); main.prepend(tools); sidebar.append(newChat, list); element.prepend(sidebar); + const listNodes = new Map(), narrow = () => window.matchMedia('(max-width: 720px)').matches; + function drawer(on, restore = false) { + element.dataset.chatsOpen = String(on); chats.setAttribute('aria-expanded', String(on)); + // Only the overlay needs to keep keyboard focus in its list. + main.inert = on && narrow(); + if (on) (list.querySelector('[aria-current]') || newChat).focus(); + else if (restore) chats.focus(); + } + function renderConversations() { + const visible = hasConversations(conversations); + element.dataset.conversations = String(visible); sidebar.hidden = tools.hidden = !visible; + if (!visible) drawer(false); + newChat.disabled = creating; chatsDot.hidden = !conversations.conversations.some(row => row.running > 0); + const keep = new Set(conversations.conversations.map(row => row.id)); + for (const [id, node] of listNodes) if (!keep.has(id)) { node.button.remove(); listNodes.delete(id); } + conversations.conversations.forEach((row, index) => { + let node = listNodes.get(row.id); + if (!node) { + const button = press('fv-chat-conversation', null, () => chooseChat(row.id)), dot = el('i', 'fv-chat-running'), title = el('span'); + button.dataset.session = row.id; dot.setAttribute('aria-hidden', 'true'); button.append(dot, title); + listNodes.set(row.id, node = {button, dot, title}); + } + node.title.textContent = row.title; node.button.title = row.title; node.dot.hidden = row.running <= 0; + if (row.id === conversations.current) node.button.setAttribute('aria-current', 'true'); else node.button.removeAttribute('aria-current'); + if (list.children[index] !== node.button) list.insertBefore(node.button, list.children[index] || null); + }); + } + function chooseChat(session, focus = false) { + selectedSession = session; selection++; pinned = true; away = null; + conversations = selectConversation(conversations, session); renderConversations(); + input.value = ''; grow(); drawer(false); + focusSession = focus ? session : null; + if (focus && session === state.session) input.focus(); else if (narrow()) chats.focus(); + render(); void readConversations(true); void poll(); + } + async function startChat() { + if (creating || !active() || conversations.supported !== true) return; + creating = true; renderConversations(); render(); + try { + const response = await agentPost(api, '/freevideo/agent/chat/new', {}); + if (stopped) return; + if (response.ok) { + const data = await response.json(); + if (typeof data.session === 'string' && data.session) chooseChat(data.session, true); + } else if (response.status === 409) { state = {...state, agent: {connected: false, name: null}}; report(); } + } catch { /* Keep the current conversation available for another try. */ } + finally { creating = false; if (!stopped) { renderConversations(); render(); } } + } + async function readConversations(force = false) { + if (!active() || conversations.supported === false) return; + if (listBusy) { if (force) listAgain = true; return; } + if (!force && performance.now() - listAt < listInterval) return; + listBusy = true; listAt = performance.now(); + try { + const response = await agentGet(api, '/freevideo/agent/chat/list', {signal: closed.signal}); + if (stopped) return; + conversations = mergeConversations(conversations, response.ok ? await response.json() : null, response.status); + renderConversations(); + if (conversations.supported === false && selectedSession !== undefined) { selectedSession = undefined; selection++; void poll(); } + } catch { /* List failures do not interrupt reading or sending. */ } + finally { listBusy = false; } + if (listAgain) { listAgain = false; void readConversations(true); } + } + const outside = event => { + if (chats.getAttribute('aria-expanded') === 'true' && !sidebar.contains(event.target) && !chats.contains(event.target)) drawer(false, true); + }; + const keys = event => { + if (chats.getAttribute('aria-expanded') !== 'true') return; + if (event.key === 'Escape') { event.preventDefault(); event.stopPropagation(); drawer(false, true); } + else if (event.key === 'Tab' && narrow()) { + const buttons = [...sidebar.querySelectorAll('button')].filter(button => !button.disabled); + if (!buttons.length) return; + if (event.shiftKey && document.activeElement === buttons[0]) { event.preventDefault(); buttons.at(-1).focus(); } + else if (!event.shiftKey && document.activeElement === buttons.at(-1)) { event.preventDefault(); buttons[0].focus(); } + } + }; + const resize = () => { if (!narrow()) drawer(false); }; + document.addEventListener('pointerdown', outside, true); document.addEventListener('keydown', keys, true); + window.addEventListener('resize', resize); // Whether the reader is at the newest entry. It holds while the conversation is out of sight, which forgets where it was scrolled. let pinned = true; const closed = new AbortController(); @@ -173,6 +269,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () return row ? row.seconds : null; } const active = () => !stopped && element.isConnected && !document.hidden && shown(); + const sendingAllowed = () => !creating && (!selectedSession || selectedSession === state.session) && canSend(state); // ---- one entry of the conversation each ---- function message(role) { @@ -291,7 +388,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } else { item.replaceChildren(); apply = null; } } apply?.(job, latest); - const again = item.querySelector('.fv-chat-retry'); if (again) again.disabled = !canSend(state); + const again = item.querySelector('.fv-chat-retry'); if (again) again.disabled = !sendingAllowed(); }}; } function confirmCard() { @@ -329,7 +426,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () t('Make a 1-minute vertical short film: a day in a coffee shop', '做一条 1 分钟的竖屏短片:咖啡店的一天')]; list.append(...lines.map(line => press(null, line, () => send(line)))); item.append(el('p', 'fv-chat-meta', t('Try saying:', '可以这样说:')), list); - return {element: item, update() { list.querySelectorAll('button').forEach(button => { button.disabled = !canSend(state); }); }}; + return {element: item, update() { list.querySelectorAll('button').forEach(button => { button.disabled = !sendingAllowed(); }); }}; } function dots() { const item = el('li', 'fv-chat-thinking'), mark = el('span', 'fv-chat-dots'); mark.append(el('i'), el('i'), el('i')); @@ -372,10 +469,11 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () node.update(value, extra || {}); if (log.children[index] !== node.element) log.insertBefore(node.element, log.children[index] || null); }); - const sendable = canSend(state); - input.disabled = state.agent.connected !== true; + const sendable = sendingAllowed(); + input.disabled = state.agent.connected !== true || creating || !!selectedSession && selectedSession !== state.session; input.placeholder = state.agent.connected === true && !sendable ? t('Ask your agent to keep listening', '回到 Agent 那边让它继续') : t('Message the agent…', '对 Agent 说…'); sendButton.disabled = !sendable || !input.value.trim(); + if (focusSession === state.session && focusSession && !input.disabled && active()) { focusSession = null; input.focus(); } if (stick) { pinned = true; scroller.scrollTop = scroller.scrollHeight; down.hidden = true; } else if (log.children.length > before) down.hidden = false; } @@ -385,18 +483,39 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () if (pinned) down.hidden = true; }, {passive: true}); + async function readAway() { + const row = state.current ? null : runningElsewhere(conversations, state.session); + if (!row) { away = null; return; } + try { + // Past the last message: only that conversation's jobs come back. + const query = new URLSearchParams({session: row.id, after: String(Number.MAX_SAFE_INTEGER)}); + const response = await agentGet(api, `/freevideo/agent/chat?${query}`, {signal: closed.signal}); + if (!response.ok) { away = null; return; } + const data = await response.json(), jobs = Object.values(data.jobs || {}).filter(job => WORKING.includes(job?.status)); + const job = data.jobs?.[data.current] || jobs[0]; + away = job ? {session: row.id, job, ids: jobs.map(job => job.id)} : null; + } catch { /* The last reading stays until the next one. */ } + } function report() { - const current = state.current ? state.jobs[state.current] || null : null; + const current = (state.current ? state.jobs[state.current] : away?.job) || null; changed({known, agent: state.agent, current, fraction: current ? fractionOf(current) : 0, left: current ? leftOf(current) : null, active: state.active, any: state.items.size > 0 || pending.length > 0}); } async function send(text) { text = String(text).trim(); - if (!text || !canSend(state)) return false; + if (!text || !sendingAllowed()) return false; + const sentSelection = selection; const entry = {id: newId(), text}; pending.push(entry); render(true); report(); let response = null; - try { response = await agentPost(api, '/freevideo/agent/chat', entry); } catch { /* Handled below as not sent. */ } - if (response?.ok) { const value = await response.json().catch(() => ({})); if (value.agent) state.agent = value.agent; poll(); return true; } + const body = conversations.supported === true && state.session ? {...entry, session: state.session} : entry; + try { response = await agentPost(api, '/freevideo/agent/chat', body); } catch { /* Handled below as not sent. */ } + if (stopped) return false; + if (response?.ok) { + const value = await response.json().catch(() => ({})); + if (sentSelection === selection) { if (value.agent) state.agent = value.agent; void poll(); } + void readConversations(true); return true; + } + if (sentSelection !== selection) return false; // Not sent: the sentence goes back to the box. An agent that was never connected leads back to the card. pending = pending.filter(item => item !== entry); if (!input.value) { input.value = text; grow(); } @@ -409,7 +528,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () // At most five lines; the box scrolls after that. const line = parseFloat(getComputedStyle(input).lineHeight) || 22; input.style.height = `${Math.min(input.scrollHeight, Math.round(line * 5) + 14)}px`; - sendButton.disabled = !canSend(state) || !input.value.trim(); + sendButton.disabled = !sendingAllowed() || !input.value.trim(); } input.addEventListener('input', grow); // Enter sends, Shift+Enter starts a new line; Enter that confirms an input-method candidate does neither. @@ -420,33 +539,53 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () form.addEventListener('submit', event => { event.preventDefault(); const text = input.value; - if (!text.trim() || !canSend(state)) return; + if (!text.trim() || !sendingAllowed()) return; input.value = ''; grow(); void send(text); }); + // Both a server-side session change and a list selection reset the same reader. Job clocks stay keyed by job id. + function resetChat() { + state = emptyChat(); pending = []; answers.clear(); nodes.clear(); + log.querySelectorAll('video').forEach(video => video.pause()); log.replaceChildren(); pinned = true; + } async function poll() { clearTimeout(timer); timer = null; - if (stopped || busy) return; + if (stopped) return; + if (busy) { reread = true; return; } if (!active()) return; - busy = true; + busy = true; reread = false; + void readConversations(); + const readSelection = selection, requested = selectedSession; let again = false; try { - const after = state.session ? state.last : 0; - const response = await agentGet(api, '/freevideo/agent/chat' + (after ? `?after=${after}` : ''), {signal: closed.signal}); + const after = state.session && (!requested || requested === state.session) ? state.last : 0; + const query = new URLSearchParams(); + if (requested) query.set('session', requested); + if (after) query.set('after', String(after)); + const response = await agentGet(api, '/freevideo/agent/chat' + (query.size ? `?${query}` : ''), {signal: closed.signal}); if (stopped) return; - if (response.ok) { + if (readSelection !== selection) again = true; + else if (response.ok) { const data = await response.json(); + if (stopped) return; + if (readSelection !== selection) { again = true; } // Another conversation began: forget this one and read the new one whole. - if (after && (data.session ?? null) !== state.session) { state = emptyChat(); again = true; } + else if (after && (data.session ?? null) !== state.session) { resetChat(); again = true; } else { - if ((data.session ?? null) !== state.session) state = emptyChat(); + if ((data.session ?? null) !== state.session) resetChat(); state = chatMerge(state, data); failures = 0; + conversations = selectConversation(conversations, state.session); renderConversations(); + await readAway(); + if (stopped) return; + if (readSelection !== selection) { away = null; again = true; } } + } else if (response.status === 404 && requested) { + selectedSession = undefined; selection++; resetChat(); again = true; void readConversations(true); } else failures++; } catch { failures++; } finally { busy = false; } if (stopped) return; - if (again) return poll(); + if (again || reread) return poll(); known = true; // The next read is booked before the page hears of this one, so the page's sync() does not start another. const idle = state.agent.connected !== true && !state.active; @@ -455,7 +594,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } // Between reads the estimate keeps moving, on the cards and in the page's header alike. const moving = setInterval(() => { - if (!active() || !Object.values(state.jobs).some(job => job.status === 'running')) return; + if (!active() || ![...Object.values(state.jobs), ...(away ? [away.job] : [])].some(job => job.status === 'running')) return; for (const item of log.children) item.tick?.(); report(); }, 250); @@ -467,12 +606,19 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } render(); return {element, input, sync, refresh: poll, - scrollToJob(id) { log.querySelector(`[data-job="${CSS.escape(id)}"]`)?.scrollIntoView({block: 'center', behavior: 'smooth'}); }, + scrollToJob(id) { + // A job of another conversation: go to that conversation, which opens at its end where the job is. + if (away?.job.id === id && !state.jobs[id]) chooseChat(away.session); + else log.querySelector(`[data-job="${CSS.escape(id)}"]`)?.scrollIntoView({block: 'center', behavior: 'smooth'}); + }, async stopAll() { - const ids = Object.values(state.jobs).filter(job => WORKING.includes(job.status)).map(job => job.id); + const ids = [...new Set([...Object.values(state.jobs).filter(job => WORKING.includes(job.status)).map(job => job.id), ...(away?.ids || [])])]; ids.forEach(id => stopping.add(id)); render(); await Promise.allSettled(ids.map(id => agentPost(api, '/freevideo/agent/activity/cancel', {id}))); poll(); }, - stop() { stopped = true; closed.abort(); clearInterval(moving); clearTimeout(timer); timer = null; document.removeEventListener('visibilitychange', visible); log.querySelectorAll('video').forEach(video => video.pause()); }}; + stop() { stopped = true; closed.abort(); clearInterval(moving); clearTimeout(timer); timer = null; + document.removeEventListener('visibilitychange', visible); document.removeEventListener('pointerdown', outside, true); + document.removeEventListener('keydown', keys, true); window.removeEventListener('resize', resize); + log.querySelectorAll('video').forEach(video => video.pause()); }}; } diff --git a/web/autopilot_conversations.js b/web/autopilot_conversations.js new file mode 100644 index 00000000..ce2baca9 --- /dev/null +++ b/web/autopilot_conversations.js @@ -0,0 +1,21 @@ +// The list is a server snapshot, in the server's order. An unsaid current conversation need not be in it. +export const emptyConversations = () => ({supported: null, current: null, conversations: []}); +export const selectConversation = (state, session) => ({...state, current: session ?? null}); +export const hasConversations = state => state.supported === true && state.conversations.length > 0; +// The first other conversation with work under way; null when there is none. +export const runningElsewhere = (state, session) => state.conversations.find(row => row.running > 0 && row.id !== session) || null; + +export function mergeConversations(state, data, status = 200) { + // A legacy server is probed once per page; ordinary errors retain the last good snapshot. + if (status === 404) return {...state, supported: false, conversations: []}; + if (state.supported === false || status !== 200 || !Array.isArray(data?.conversations)) return state; + const seen = new Set(), conversations = []; + for (const row of data.conversations) { + if (typeof row?.id !== 'string' || !row.id || typeof row.title !== 'string' || !row.title.trim() || seen.has(row.id)) continue; + seen.add(row.id); + conversations.push({id: row.id, title: row.title, updated_at: row.updated_at, + running: Math.max(0, Math.floor(Number(row.running) || 0))}); + if (conversations.length === 50) break; + } + return {...state, supported: true, conversations}; +} diff --git a/web/health.js b/web/health.js index 0207a59b..111fe4b4 100644 --- a/web/health.js +++ b/web/health.js @@ -13,7 +13,7 @@ const t = (en, zh) => cn ? zh : en; // launcher.js removes this flag during setup; read it before that. const launchVisit = query.get('freevideo') === 'launch'; // Every script this folder ships; a test keeps the list in step with web/*.js. -export const MODULES = ['agent_activity.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'browser_extension.js', 'chain_support.js', 'capability_view.js', 'compatibility.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js', +export const MODULES = ['agent_activity.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'autopilot_conversations.js', 'browser_extension.js', 'chain_support.js', 'capability_view.js', 'compatibility.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js', 'editor.js', 'editor_actions.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_genbar.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'launcher.js', 'length_plan.js', 'library.js', 'motion.js', 'output_download.js', 'port_notice.js', 'preview_scene.js', 'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'report_issue.js', 'studio_recent.js', From 7fc6887911a153e720fb7e6da631b8378982d480 Mon Sep 17 00:00:00 2001 From: Bowen Xue Date: Tue, 13 Oct 2026 21:05:51 +0000 Subject: [PATCH 27/27] Release 0.4.1 --- freevideo_engine/release_notes.json | 121 +++++++++++++++------------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/freevideo_engine/release_notes.json b/freevideo_engine/release_notes.json index 40987da1..7922bc6f 100644 --- a/freevideo_engine/release_notes.json +++ b/freevideo_engine/release_notes.json @@ -1,41 +1,78 @@ { "schema": 1, - "product_version": "0.4.0", + "product_version": "0.4.1", "en": { - "summary": "Adds an editing view, long videos generated one part after another, and AI agent connections, and makes generation faster.", + "summary": "Fixes Extend on videos made earlier, and improves progress display and several agent details.", "changes": [ - "New editing view: arrange generated clips on a timeline, add text and subtitles, change speed or reverse a clip, and choose the size and quality when exporting.", - "Long videos can be generated one part after another: select a range on the timeline in the editing view and press Generate, and each part continues from the one before. A 30-second video is generated in two parts.", - "Clips can be extended in the editing view: drag a new part out of the end of a clip and press Generate, and it continues from the clip. A \"First-frame reference\" switch reduces drift when a clip is extended.", - "The editing view can create subtitles automatically: it recognizes Chinese, English, Japanese, Korean and Cantonese speech and gives each line its text and timing.", - "A step-by-step guide appears the first time FreeVideo opens, and can be viewed again from Settings.", - "Connect an AI agent and let it generate, edit and export videos.", - "A new \"LAN access\" switch in Settings lets phones and other computers on the same network open FreeVideo with a link.", - "After an offline package is imported, FreeVideo offers to delete the ZIP files that are no longer needed, freeing up disk space.", - "LoRAs can be added directly in the creation panel.", - "The FreeVideo · Media node accepts several reference images.", - "Faster generation: the wait before decoding is shorter; when generating several videos in a row, the models load faster for the later ones. On computers with less memory, background compiling after an update uses less memory.", - "Fixes: generating with several large reference images no longer runs out of memory; 16:9 videos can be saved as share videos; making the full video from a preview no longer stalls on a single step now and then; when setup cannot find a graphics driver, the error says so; setup shows the current step; when security software removes one of FreeVideo's files, the error says why and how to restore it; on a slow network, setup waits longer and tries once more when no download source responds; on English Windows, FreeVideo starts when the install folder or the user name contains Chinese characters." + "Extend works again on videos made earlier, including ones whose size was changed, and the new part now uses the previous video's picture size automatically. Clicking Extend on a result or in Creations now opens Edit with the video on the timeline and a new part already placed after it, ready to generate.", + "Autopilot can start a new conversation, and earlier ones stay in a list to return to; the agent also keeps listening after it answers, so it still responds when the conversation resumes later.", + "A percentage and the time left appear as soon as a video starts; progress keeps moving while the model loads and runs smoothly to the end, on Autopilot job cards as well.", + "In Edit, an agent can start a new timeline from just a name, and writes a short note on each clip so it is clear at a glance what the clip is.", + "A video added to the timeline from Creations now brings the description it was generated from, so it can be regenerated right away.", + "A video of 16 to 29 seconds requested through an agent is now made in one generation when the computer allows it, the same as in Create.", + "When a specific camera path, placement or order of actions keeps coming out wrong, the agent can offer to make a rough camera draft first and generate from it.", + "The question about freeing the disk space used by old model files now appears after updating.", + "At smaller sizes, more of the model stays in GPU memory, so computers with less memory read less from the drive while generating.", + "When the network is slow or a step takes a while, the install page now shows what it is doing.", + "Third-party download during setup now lists only the files this computer needs, and the list can be exported; after downloading, choose the folder and the files that pass the check are used directly.", + "When other programs use a large part of the GPU during a generation, or the GPU overheats, a short note appears under the result." ] }, "zh": { - "summary": "新增剪辑视图、一段接一段的长视频生成,并可以连接 AI Agent;生成也更快。", + "summary": "修复了之前生成的视频不能“接着生成”的问题,并改进了进度显示和 Agent 的几处体验。", "changes": [ - "新增剪辑视图:把生成的片段排在时间线上剪辑,可以加文字和字幕、变速和倒放;导出时可以选画面大小和画质。", - "长视频可以一段接一段生成:在剪辑视图的时间线上选好范围再点生成,每一段都接着前一段往下生成;30 秒的视频分两段生成。", - "剪辑视图里可以延长片段:从片段尾巴拖出一段再点生成,接在原片段后面往下生成;延长片段时可以打开“首帧参考”,减小画面漂移。", - "剪辑视图可以自动生成字幕:识别中文、英文、日文、韩文和粤语语音,得到每句的文字和时间。", - "第一次打开时有分步指引,设置里可以再看一遍。", - "可以连接 AI Agent,让它生成、剪辑和导出视频。", - "设置里新增“局域网访问”开关,打开后,同一网络里的手机和其他电脑可以用链接打开 FreeVideo。", - "导入离线包以后,可以删掉用不到的压缩包,释放磁盘空间。", - "创作面板里可以直接添加 LoRA。", - "FreeVideo · Media 节点可以接多张参考图。", - "生成更快:解码前的等待更短;连续生成时,后面的视频加载模型更快。内存较小的电脑上,更新后的后台编译占用的内存更少。", - "修复:用多张较大的参考图生成时不再内存不足;16:9 的视频可以保存分享视频;先出预览再生成正式视频时,不再偶尔在某一步停留很久;安装时没有找到显卡驱动,会直接说明;安装时会显示当前步骤;FreeVideo 的文件被安全软件移走时,会说明原因和恢复的办法;网络较慢、下载源都连不上时,安装会多等一会儿再试一轮;英文 Windows 上,安装文件夹或用户名带中文时也能正常启动。" + "之前生成的视频(包括换过尺寸的)都可以“接着生成”,不再提示不能接续;新的一段会自动用上一段的画面大小。在结果或作品上点“接着生成”,会打开剪辑视图:原视频已在时间线上,后面接好了新的一段,点“生成”继续。", + "“AI 全自动”里可以开新对话,旧对话留在列表里,可以回去接着做;Agent 答完以后会一直等下一句,隔一会儿再说也有回应。", + "点“生成”后马上显示百分比和预计剩余时间;加载模型时进度照常前进,平稳走到结束,“AI 全自动”的任务卡也一样。", + "剪辑视图里,Agent 只凭一个名字就能新建时间线,并会在每一段上写一句说明,一眼就能看出这一段是什么。", + "从“作品”放上时间线的视频会带上当时的生成描述,可以直接重新生成。", + "通过 Agent 生成 16–29 秒的视频时,电脑条件允许就一次生成,和创作面板一致。", + "指定的镜头路线、位置或动作顺序总是生成得不对时,Agent 会提议先做一个粗略的镜头草稿,再照草稿生成。", + "更新后询问是否释放旧模型文件所占硬盘空间的提示,现在会正常出现。", + "画面较小时,更多模型留在显存里,内存较小的电脑生成时读取硬盘更少。", + "安装时网络较慢或某一步较久时,安装页会显示当前正在做什么。", + "安装时选第三方下载,现在只列出这台电脑需要的文件,并可以导出下载清单;下载完成后选中那个文件夹,校验通过的文件直接使用。", + "生成期间其他程序占用了较多显卡,或显卡过热时,结果下方会出现一句简短的提示。" ] }, "history": [ + { + "product_version": "0.4.0", + "en": { + "summary": "Adds an editing view, long videos generated one part after another, and AI agent connections, and makes generation faster.", + "changes": [ + "New editing view: arrange generated clips on a timeline, add text and subtitles, change speed or reverse a clip, and choose the size and quality when exporting.", + "Long videos can be generated one part after another: select a range on the timeline in the editing view and press Generate, and each part continues from the one before. A 30-second video is generated in two parts.", + "Clips can be extended in the editing view: drag a new part out of the end of a clip and press Generate, and it continues from the clip. A \"First-frame reference\" switch reduces drift when a clip is extended.", + "The editing view can create subtitles automatically: it recognizes Chinese, English, Japanese, Korean and Cantonese speech and gives each line its text and timing.", + "A step-by-step guide appears the first time FreeVideo opens, and can be viewed again from Settings.", + "Connect an AI agent and let it generate, edit and export videos.", + "A new \"LAN access\" switch in Settings lets phones and other computers on the same network open FreeVideo with a link.", + "After an offline package is imported, FreeVideo offers to delete the ZIP files that are no longer needed, freeing up disk space.", + "LoRAs can be added directly in the creation panel.", + "The FreeVideo · Media node accepts several reference images.", + "Faster generation: the wait before decoding is shorter; when generating several videos in a row, the models load faster for the later ones. On computers with less memory, background compiling after an update uses less memory.", + "Fixes: generating with several large reference images no longer runs out of memory; 16:9 videos can be saved as share videos; making the full video from a preview no longer stalls on a single step now and then; when setup cannot find a graphics driver, the error says so; setup shows the current step; when security software removes one of FreeVideo's files, the error says why and how to restore it; on a slow network, setup waits longer and tries once more when no download source responds; on English Windows, FreeVideo starts when the install folder or the user name contains Chinese characters." + ] + }, + "zh": { + "summary": "新增剪辑视图、一段接一段的长视频生成,并可以连接 AI Agent;生成也更快。", + "changes": [ + "新增剪辑视图:把生成的片段排在时间线上剪辑,可以加文字和字幕、变速和倒放;导出时可以选画面大小和画质。", + "长视频可以一段接一段生成:在剪辑视图的时间线上选好范围再点生成,每一段都接着前一段往下生成;30 秒的视频分两段生成。", + "剪辑视图里可以延长片段:从片段尾巴拖出一段再点生成,接在原片段后面往下生成;延长片段时可以打开“首帧参考”,减小画面漂移。", + "剪辑视图可以自动生成字幕:识别中文、英文、日文、韩文和粤语语音,得到每句的文字和时间。", + "第一次打开时有分步指引,设置里可以再看一遍。", + "可以连接 AI Agent,让它生成、剪辑和导出视频。", + "设置里新增“局域网访问”开关,打开后,同一网络里的手机和其他电脑可以用链接打开 FreeVideo。", + "导入离线包以后,可以删掉用不到的压缩包,释放磁盘空间。", + "创作面板里可以直接添加 LoRA。", + "FreeVideo · Media 节点可以接多张参考图。", + "生成更快:解码前的等待更短;连续生成时,后面的视频加载模型更快。内存较小的电脑上,更新后的后台编译占用的内存更少。", + "修复:用多张较大的参考图生成时不再内存不足;16:9 的视频可以保存分享视频;先出预览再生成正式视频时,不再偶尔在某一步停留很久;安装时没有找到显卡驱动,会直接说明;安装时会显示当前步骤;FreeVideo 的文件被安全软件移走时,会说明原因和恢复的办法;网络较慢、下载源都连不上时,安装会多等一会儿再试一轮;英文 Windows 上,安装文件夹或用户名带中文时也能正常启动。" + ] + } + }, { "product_version": "0.3.25", "en": { @@ -220,36 +257,6 @@ "在 Windows 上,第一遍步与步之间保留在显卡上的模型层不再逐步减少,每步需要重新传输的层更少。" ] } - }, - { - "product_version": "0.3.18", - "en": { - "summary": "On Windows computers with little RAM, loading the video model reads much less from the disk.", - "changes": [ - "On Windows, when RAM cannot hold the whole video model, loading no longer reads the model layers that sampling streams from the disk anyway; each is read when sampling first needs it. With 16 GB of RAM, that is about 13 GB less to read every time the model loads: on an RTX 5060 Ti test machine limited to the RAM of a 16 GB computer, loading took 7 seconds instead of 11, and on a slower drive the saving is a few minutes. Videos are unchanged. Computers that keep the whole model in RAM, Linux and Mac load as before." - ] - }, - "zh": { - "summary": "在内存较小的 Windows 电脑上,加载视频模型时从硬盘读取的数据少了很多。", - "changes": [ - "在 Windows 上,内存放不下整个视频模型时,加载阶段不再预先读取那些采样时本来就要从硬盘读取的模型层,改为采样第一次用到时再读。16 GB 内存的电脑每次加载模型可少读约 13 GB:在按 16 GB 电脑的内存条件运行的 RTX 5060 Ti 测试机上,加载从 11 秒缩短到 7 秒;硬盘较慢时可节省几分钟。生成的视频不变。内存能放下整个模型的电脑、Linux 和 Mac 的加载方式不变。" - ] - } - }, - { - "product_version": "0.3.17", - "en": { - "summary": "On Windows, GPUs with 12 to 20 GB of memory sample a little faster: attention runs 16 heads at a time where memory allows.", - "changes": [ - "On Windows, attention now runs 16 heads at a time instead of 8 whenever the GPU memory budget leaves room for it. This covers the first pass on 12 to 20 GB GPUs and the second pass on most 16 and 20 GB GPUs. On a 16 GB GPU, a 10-second 1344×768 video took about 6% less time per first-pass step and about 2% less per second-pass step, without using more video memory. FP8 models, requests with a LoRA, Linux and GPUs below 12 GB keep their current settings. The same seed can give a slightly different video than before, as it already does between GPUs with different memory." - ] - }, - "zh": { - "summary": "在 Windows 上,12–20 GB 显存的显卡采样稍快一些:显存允许时,注意力每次计算 16 个头。", - "changes": [ - "在 Windows 上,只要显存预算有余量,注意力改为每次计算 16 个头(原来每次 8 个)。12–20 GB 显卡的第一遍,以及大多数 16 GB 和 20 GB 显卡的第二遍都会用上。在 16 GB 显卡上生成 10 秒 1344×768 视频,第一遍每步缩短约 6%,第二遍每步缩短约 2%,显存占用没有增加。FP8 模型、使用 LoRA 的生成、Linux 和 12 GB 以下的显卡保持现有设置。同一个种子生成的视频可能与以前略有不同,就像在显存不同的显卡上本来就会有差异一样。" - ] - } } ] }