diff --git a/freevideo.ps1 b/freevideo.ps1 index f112fbf4..5a3e24a8 100644 --- a/freevideo.ps1 +++ b/freevideo.ps1 @@ -75,6 +75,7 @@ try { Write-Host 'Preparing isolated Python for setup. Model/environment installation requires the setup confirmation.' Write-Host "Bootstrap cache and log: $bootstrapRoot" $bootstrapLock = [IO.File]::Open((Join-Path $bootstrapRoot 'bootstrap.lock'), 'OpenOrCreate', 'ReadWrite', 'None') + $patientDownloadNotice = $false try { $versions = Get-Content -LiteralPath (Join-Path $PSScriptRoot 'freevideo_engine\bootstrap_versions.json') -Raw -Encoding UTF8 | ConvertFrom-Json $uvSpec = $versions.windows.uv @@ -110,6 +111,10 @@ try { if ($profile.Mode -eq 'patient') { if (-not $allNetworkFailures) { break } Write-Host 'The network is slow. Downloading again; this may take a while.' + $patientDownloadNotice = $true + if ($env:FREEVIDEO_UI_EVENTS -eq '1') { + Write-Host '{"event":"freevideo_ui","kind":"progress","key":"download-patient-python","label":"Download","detail":"The network is slow. Downloading again; this may take a while."}' + } } foreach ($name in @('UV_HTTP_TIMEOUT', 'UV_HTTP_RETRIES')) { $value = $freevideoSavedEnvironment[$name] @@ -139,6 +144,9 @@ try { $freevideoPython = (& $uv python find --managed-python $versions.python | Select-Object -Last 1).Trim() if ($LASTEXITCODE -ne 0 -or -not (Test-Path -LiteralPath $freevideoPython)) { throw 'Managed Python could not be located.' } } finally { + if ($patientDownloadNotice -and $env:FREEVIDEO_UI_EVENTS -eq '1') { + Write-Host '{"event":"freevideo_ui","kind":"progress","key":"download-patient-python","label":"Download","detail":"","state":"complete"}' + } $bootstrapLock.Dispose() Write-Progress -Activity 'FreeVideo bootstrap' -Completed } diff --git a/freevideo_engine/adaln_reclaim.py b/freevideo_engine/adaln_reclaim.py index b95112aa..70e8f334 100644 --- a/freevideo_engine/adaln_reclaim.py +++ b/freevideo_engine/adaln_reclaim.py @@ -39,7 +39,7 @@ from . import adaln_assets as assets from .long_paths import io_path -from .variant_cleanup import inside, redirected +from .variant_cleanup import SIDECAR, SIDECAR_LIMIT, inside, redirected RECEIPT = 'adaln-reclaim.json' OFFER = 'adaln-reclaim-offer.json' @@ -286,6 +286,18 @@ def verify_sources(cache, manifest, should_stop=lambda: False): return None +def _remove_sidecars(cache, listed): + """The small records an earlier download left beside a source (.partial.source.json), so the folder can go.""" + for _, path in listed: + sidecar = path.with_name(path.name + SIDECAR) + info = _regular(sidecar) + if info is not None and info.st_size <= SIDECAR_LIMIT and _contained(cache, sidecar): + try: + io_path(sidecar).unlink() + except OSError: + pass + + def _remove_tables(cache, folders): removed = 0 for folder, _ in folders: @@ -373,6 +385,7 @@ def reclaim(cache, *, should_stop=lambda: False, schedules=None, lock=None, ask= return kept('redirected') freed += info.st_size io_path(path).unlink() + _remove_sidecars(cache, listed) try: (io_path(cache) / SOURCES).rmdir() except OSError: @@ -423,11 +436,25 @@ def apply(machine, *, should_stop=lambda: False, schedules=None): return result +def _installed_engine_schedules(machine): + """engine_schedules for a process that has not put the engine's dependencies on its path (the background check).""" + ready = [] + def answer(steps): + if not ready: + from . import paths + vdn = Path(machine['root']) / 'vendor' / 'vdn' if machine.get('root') else None + paths.add_vdn(vdn if vdn is not None and (vdn / 'src/models/hybrid_attention.py').is_file() else None) + ready.append(True) + return engine_schedules(steps) + return answer + + def run(machine, *, should_stop=lambda: False): """Entry point of the background precompile: check and leave the offer, report one event, never raise.""" started = time.time() try: - result = reclaim(machine['cache'], should_stop=should_stop, ask=True, release=installed_release()) + result = reclaim(machine['cache'], should_stop=should_stop, ask=True, release=installed_release(), + schedules=_installed_engine_schedules(machine)) except Exception as error: # Best effort: the installation works either way. result = dict(action='kept', reason='error:' + type(error).__name__, bytes=0, tables_bytes=0) if result['action'] == 'kept' and result['reason'] not in ('declined', 'stopped', 'busy') \ diff --git a/freevideo_engine/agent/editor_runtime.py b/freevideo_engine/agent/editor_runtime.py index 6af6b6f5..2f793a32 100644 --- a/freevideo_engine/agent/editor_runtime.py +++ b/freevideo_engine/agent/editor_runtime.py @@ -137,6 +137,7 @@ case 'move': record = actions.moveParts(record, op.clip_ids, {frames: op.frames, track: null}); break; case 'describe': record = model.describe(record, actions.linkedIds(record, op.clip_ids), op.text, by); break; case 'set_ref': record = model.setRef(record, actions.linkedIds(record, op.clip_ids), op.ref); break; + case 'note': record = model.setNote(record, actions.linkedIds(record, op.clip_ids), op.text); break; case 'use_version': record = model.restoreVersion(record, op.clip_id, op.index, by); break; // One undo on the page; rejects as subtitles / range / text / locked / overlap. case 'subtitles': record = texts.addSubtitles(record, op.cues, {replace: op.replace === true}); break; diff --git a/freevideo_engine/agent/errors.py b/freevideo_engine/agent/errors.py index b5b3fdbc..0ca9a443 100644 --- a/freevideo_engine/agent/errors.py +++ b/freevideo_engine/agent/errors.py @@ -55,7 +55,7 @@ 'internal_error': (500, True, 'An internal error prevented this operation from completing.', 'Read the job list and report request_id if the error persists.'), 'local_only': (403, False, 'This endpoint does not allow requests from this peer address.', 'Run the client on the FreeVideo computer and connect using its loopback address.'), 'origin_forbidden': (403, False, 'Origin must match this FreeVideo instance; a different origin was received.', 'Open the page served by this FreeVideo instance, or use a local command-line client without Origin.'), - 'continuation_unavailable': (409, False, 'This backend cannot continue video blocks.', 'Use an independent shot with seconds<=15, or enable a backend with continuation inputs and resume.'), + 'continuation_unavailable': (409, False, 'This backend cannot continue video blocks.', 'Use POST /plan for a supported single block (unknown limit: seconds<=15), or enable continuation and resume.'), 'not_resumable': (409, False, 'Only failed, interrupted or cancelled generation jobs can resume.', 'Read the job; wait for active jobs or submit a new job for a different request.'), 'idempotency_conflict': (409, False, 'This idempotency key was already used for different request content.', 'Resend the original content with this key, or use a new ref for the changed request.'), 'source_missing': (409, False, 'A completed source block is no longer available.', 'Restore the completed block before resuming, or submit a new job.'), diff --git a/freevideo_engine/agent/forecast.py b/freevideo_engine/agent/forecast.py index 44ba77a3..cb914c19 100644 --- a/freevideo_engine/agent/forecast.py +++ b/freevideo_engine/agent/forecast.py @@ -1,5 +1,6 @@ """Machine-local timing forecasts shared by planning and durable films.""" import math +from .length import capability_table, single_frame_limit def generation_estimate(port, spec, chain=None): @@ -9,9 +10,9 @@ def generation_estimate(port, spec, chain=None): reasons here so a film never invents a time for unsupported conditioning. """ from ..chain_plan import plan as chain_plan - chain = chain if chain is not None else chain_plan(spec['seconds'], continued=bool(spec.get('continue_from'))) - table = port.capability_snapshot(dict(width=str(spec['width']), height=str(spec['height']), - task=spec['task'], adapters='1' if spec['loras'] else '0')) + table = capability_table(port, spec) + chain = chain if chain is not None else chain_plan(spec['seconds'], max_frames=single_frame_limit(spec, table=table), + continued=bool(spec.get('continue_from'))) seconds = None reason = table['status'] if table['status'] in ('computing', 'unavailable') else 'no_matching_estimate' if table['status'] == 'ready' and not spec['preview'] and not spec['media']: diff --git a/freevideo_engine/agent/guide.py b/freevideo_engine/agent/guide.py index 85d2feb6..3ac4ba2a 100644 --- a/freevideo_engine/agent/guide.py +++ b/freevideo_engine/agent/guide.py @@ -58,11 +58,23 @@ 'Name subjects , in subject_definitions and reuse those labels in every section.', 'Clips are cut to the generated length (at most 15 s); the job result lists any cut.', ], + 'camera_draft': [ + 'Optional, never the default. When the user asks for a specific camera path, placement or order of actions ' + 'and two direct generations did not get it, offer to make a rough camera draft first and tell them it ' + 'takes you 10 to 30 minutes, more the more there is in the shot. Do not offer it for an ordinary shot.', + 'The draft is a video you render yourself from a simple 3D scene: 960x544, 24 fps, 124 frames. Only the ' + 'camera path, the positions of things and the timing of the action need to be right.', + 'Generate at 960x544 for 5 s with the draft as the only media.references video. Write the description as ' + 'one plain paragraph without sections and end it with: "The reference video is a rough 3D stand-in of this ' + 'scene: follow its camera path, object positions and motion timing, but render real materials, colours, ' + 'lighting and surroundings as described above; do not copy its simplified, flat look." The result follows ' + 'the draft and looks plainer than a direct generation, so keep the direct one when the user prefers its look.', + ], 'timing': [ 'Single-block frames are 24 fps on a 17n+5 grid (6 s -> 6.583 s, 10 s -> 10.125 s). ' - 'Generation uses the continuous backend. A request of up to 15 s is one generation; a longer one is made ' - 'in parts of up to 15 s (30 s is 15 + 15) that continue one another and join automatically. ' - 'Continuing (longer than 15 s, or continue_from) needs two-pass text generation: quality "light" ' + 'Under 30 s defaults to one generation when the device supports this size and quality; unknown limits use 15 s. ' + 'Otherwise it uses parts of up to 15 s (30 s is 15 + 15), continued and joined automatically. ' + 'Multiple blocks or continue_from need two-pass text generation: quality "light" ' 'or sampling.two_pass=true, no preview and no media. ' 'Multi-block output is cut to the planned duration. POST /plan shows the chain plan. ' 'A continue_from job saves a new video that holds only the added part; to hand the user one video, ' @@ -91,10 +103,12 @@ ], 'editing_projects': [ 'GET /projects lists timelines shared with the editor page. First GET /projects/{id} to read revision and clip IDs.', + 'For a new timeline POST /projects with {"project": {"name": "..."}} (width and height too if not 1344x768); it comes back empty at revision 1. Then add each clip with an insert operation.', 'POST /projects/{id}/operations sends revision and 1-64 operations, using whole frames at 24 fps. The batch saves once or changes nothing.', 'On 409 project_changed, recompute edits from error.project and its current_revision, then send a new request. On clip_busy wait for generation to finish.', 'POST /projects/{id}/generate reserves pending parts for clip_ids or a start/frames range and fills them as the returned job finishes. POST /projects/{id}/export renders the revision you read; wait for its job.', 'Subtitles: POST /projects/{id}/listen with mode transcribe (writes down what is said) or align (times the lines you give) and the revision you read; wait for the job and read result.lines, each with start and end frames. Then add them with a subtitles operation of cues {start, frames, text}; replace true swaps the whole track. With no speech model the job fails with no_speech_model: write the lines yourself.', + 'The user sees one line under each picture clip: its note, or else the first line of its description. A description is generation text, often long and in English, so after you add, generate or change a clip, write a note (operation note) in the user\'s language saying what is in the picture: the subject first, then the setting, for example "Train through morning mist" (in Chinese "小火车穿过晨雾"). A short clip shows only the first few characters, about four in Chinese, and those must already make sense on their own. Do not write where the clip sits (opening, middle, ending, part 2): the order of the clips shows that. Do not write how far along it is: the clip already shows its progress. Keep it true when you change the clip again.', 'Reuse ref or Idempotency-Key only for the identical request. Source video/audio/thumbnail URLs require the same Agent token.', ], 'project_routes': [dict(operation=name, method=method, path=path, description=summary) @@ -119,7 +133,7 @@ for name, (method, path, summary) in SESSION_OPERATIONS.items()], 'filming': [ 'Filming is for a film of several shots, each picked from takes. One video of up to 60 seconds is one generate ' - 'job (longer requests continue and join automatically); to polish an idea first, run enhance, then generate. ' + 'job (30 s or more, or beyond the device limit, continues and joins automatically); to polish an idea first, run enhance, then generate. ' 'A film is cut into shots: see the rule "Cut by default" before making one long continued video.', 'Use drama plan → start → get → sheet → choose/reshoot → finish; inspect todo after every change.', 'At start tell the user the estimated total time; on over_budget decide whether to ask them; ' @@ -156,7 +170,9 @@ 'Solve problems yourself first. Every error has a next field, and often details with alternatives; these are ' 'steps that work on this computer. Not enough graphics memory: use a listed alternative (a lower resolution ' 'or a shorter video) and try again. A take is not good: make another; if it is still not good after two more, ' - 'describe it a different way instead of repeating the request. The queue is busy: wait with wait=25 on the ' + 'describe it a different way instead of repeating the request. When what stays wrong is the camera path, where ' + 'things are or the order of actions, the guide (camera_draft) has a slower way: offer it to the user. ' + 'The queue is busy: wait with wait=25 on the ' 'job, never in a tight loop. The connection dropped or FreeVideo restarted: connect again, read the job list, ' 'and continue; resend an unanswered request with the same ref, and never generate again what is already done.', 'Do not stop to report progress. A long task is not a reason to stop or to ask whether to go on.', @@ -176,8 +192,7 @@ 'Cut by default. Wherever the film can change shot, make the new shot from the reference pictures it should ' 'have (cast cards, scene pictures) instead of continuing the previous video. Continue a video only for a ' 'long take that cannot be cut, an action that must stay unbroken, or sound that must stay unbroken, and ' - 'continuing again and again is not recommended. Content of 16 to 29 seconds is better made as shots of ' - 'up to 15 seconds each. One video has at most 5 parts (the first and 4 ' + 'continuing again and again is not recommended. One video has at most 5 parts (the first and 4 ' 'continuations), each continued part at least 10 seconds, about 70 seconds in all; filling that is not ' 'recommended. Past it, cut to a new shot.', ] @@ -189,7 +204,8 @@ '提交的每条视频都起个名字：label 写拍的是什么，用用户的语言，2 到 8 个字，例如“清晨海边”。', '先自己解决问题。每个错误都有 next 字段，details 里也常有可选方案；这些是在本机可行的处理步骤。' '显存不足时，从列出的方案中选择较低分辨率或较短时长重试。一条拍得不好就再拍；再拍两条仍不满意，' - '换一种描述，不要反复提交同一个要求。队列繁忙时，用 wait=25 等待该任务，不要不停查询。' + '换一种描述，不要反复提交同一个要求。一直不对的是镜头路线、位置或动作先后时，指南的 camera_draft 一节有一种更慢的做法，' + '可以向用户提议。队列繁忙时，用 wait=25 等待该任务，不要不停查询。' '连接中断或 FreeVideo 重启后，重新连接，读取任务列表并继续；未收到响应的请求用相同的 ref 重发，' '已经完成的内容不要重新生成。', '不要为了汇报进度而停下。任务耗时较长，不是停止工作或询问是否继续的理由。', @@ -335,7 +351,7 @@ def introduction(zh, offers): return rows -def connection_guide(root, base, lang=None, offers=None): +def connection_guide(root, base, lang=None, offers=None, resumed=False): """Public bootstrap text contains locations and commands, never credential values.""" import json from pathlib import Path @@ -387,17 +403,27 @@ def connection_guide(root, base, lang=None, offers=None): description=CAPABILITIES_ZH.get((method, suffix), row['summary']) if zh else row['summary']) for (method, suffix), row in contracts().items() if not suffix.startswith('/freevideo/') and not suffix.startswith('/connect')], - first_reply='用中文回复用户；如果用户改用其他语言，就跟随用户。连接后先用通俗的话告诉用户你能做“先介绍这几样”里的事，' - '只说这几样，并举两三个用户可以提出的要求。不要报版本号，不要念接口名。' - '不要自行启动任务，等待用户提出要求。' if zh else - 'Reply to the user in English; if the user switches languages, follow their language. ' - 'After connecting, first reply in plain words: tell the user you can do the things under ' - '"Introduce these", only those, and give two or three requests they can say. Do not quote version ' - 'numbers or endpoint names. Do not start any task yourself; wait for the user to ask.', + # An agent connecting again while the user's conversation is under way has greeted there once already. + first_reply=(('用中文回复用户；如果用户改用其他语言，就跟随用户。用户所在的对话已经开始了：连接后不要自我介绍，' + '也不要说“已连接”，先调一次等待接口，回应里的 before 是这段对话的最后几句；什么都不说，等用户开口再接着做。' + '不要自行启动任务。' if zh else + 'Reply to the user in English; if the user switches languages, follow their language. ' + 'The user is in a conversation that has already begun: after connecting do not introduce yourself ' + 'and do not say that you are connected. Call the wait once; "before" in its answer holds the last ' + 'lines of that conversation. Say nothing until the user speaks, then carry on from there. ' + 'Do not start any task yourself.') if resumed else ( + '用中文回复用户；如果用户改用其他语言，就跟随用户。连接后先用通俗的话告诉用户你能做“先介绍这几样”里的事，' + '只说这几样，并举两三个用户可以提出的要求。不要报版本号，不要念接口名。' + '不要自行启动任务，等待用户提出要求。如果第一次等待的回应说这段对话已经在进行，就不再介绍，接着之前的做。' if zh else + 'Reply to the user in English; if the user switches languages, follow their language. ' + 'After connecting, first reply in plain words: tell the user you can do the things under ' + '"Introduce these", only those, and give two or three requests they can say. Do not quote version ' + 'numbers or endpoint names. Do not start any task yourself; wait for the user to ask. If the first wait says ' + 'the conversation is already under way, do not introduce yourself again; carry on from it.')), introduce=introduction(zh, offers or {}), conversation='用户会在 FreeVideo 的“AI 全自动”页里直接和你说话，你的话也显示在那里。' '连接后先调一次等待接口 GET ' + base + PREFIX + '/inbox?wait=50（MCP 用不带 id 的 freevideo_wait），' - '回应里的 session 就是这段对话；往 POST ' + base + PREFIX + '/sessions/{session}/events 发 ' + '回应里的 session 就是用户现在所在的对话（用户新开或换到另一段对话时它会变，话总是发到最近一次等待给的那个 session）；往 POST ' + base + PREFIX + '/sessions/{session}/events 发 ' '{"type":"message","text":"…"}（MCP 用 freevideo_session_post）。在这里先只说一句话，不超过 20 个字，' '不列条目、不举例（页面上已经有例句），例如“连上了。告诉我想做什么视频。”' '你自己窗口里的那段介绍照常说，但它不是这一轮的结尾：说完紧接着去调等待接口。' @@ -414,7 +440,7 @@ def connection_guide(root, base, lang=None, offers=None): '这段对话里不要新建会话。只有用户明确让你停，才停。' if zh else 'The user talks to you inside FreeVideo, in the Autopilot view, and your words are shown there. ' 'After connecting, call the wait once: GET ' + base + PREFIX + '/inbox?wait=50 (MCP: freevideo_wait ' - 'without id). Its "session" is this conversation; POST ' + base + PREFIX + '/sessions/{session}/events ' + 'without id). Its "session" is the conversation the user is in (it changes when the user opens or returns to another one; always post to the session the latest wait named); POST ' + base + PREFIX + '/sessions/{session}/events ' 'with {"type":"message","text":"..."} (MCP: freevideo_session_post). Say only one short sentence ' 'there first, at most 12 words, no list and no examples (the page already shows examples), for example ' '"Connected. Tell me what video you want.", in the language the user writes to you in. Give the introduction in your own window as usual, but it ' @@ -470,8 +496,8 @@ def connection_text(value): 'Keys survive restart. Batch-level ref names the whole request; row refs still name earlier dependencies.', 'Upload media content with POST /media. HTTP accepts no local absolute paths; MCP and CLI read local files ' 'before uploading. Unknown fields are rejected and every batch is validated before it is recorded.', - 'limits.seconds in the status reply is what this installation can make now. While continuation is false, ' - 'one block (15 s) is the longest video; plan and submit refuse longer requests with continuation_unavailable.', + 'POST /plan gives the length supported for the current size and quality. Without continuation, only a ' + 'single block can run; unknown device limits use 15 s. Multiple blocks return continuation_unavailable.', 'One video has at most 5 parts (60 s in one request). continue_from counts the parts of the video it ' 'continues; plan and submit refuse a longer one and name the longest seconds that still fits.', 'features in the status reply names what this installation offers; false means not usable now (for example ' diff --git a/freevideo_engine/agent/jobs.py b/freevideo_engine/agent/jobs.py index a7631a9c..0d7cfce8 100644 --- a/freevideo_engine/agent/jobs.py +++ b/freevideo_engine/agent/jobs.py @@ -24,6 +24,7 @@ from . import spec as specs from . import schema, errors, assets +from .length import single_frame_limit from ..system import windows ACTIVE = ('waiting', 'queued', 'running', 'cancelling') @@ -60,9 +61,10 @@ def chain_error(code, message, next_action, *, details=None, retryable=False): return JobError(code, message, details=details or [], next=next_action, retryable=retryable) -def single_block(seconds): +def single_block(spec, seconds=None): from ..chain_plan import plan - return plan(seconds)['single'] + return plan(spec['seconds'] if seconds is None else seconds, max_frames=single_frame_limit(spec), + continued=bool(spec.get('continue_from')))['single'] def continuation_problems(spec, prefix='', *, editor=None): @@ -73,9 +75,9 @@ def continuation_problems(spec, prefix='', *, editor=None): if spec['kind'] != 'generate' or spec.get('refine_from'): return [] if not (any(block['after'] for block in editor['blocks']) if editor is not None - else spec.get('continue_from') or not single_block(spec['seconds'])): + else spec.get('continue_from') or not single_block(spec)): return [] - why = ' when continuing a video or making one longer than 15 s' + why = ' when continuing a saved video or generating multiple blocks' problems = [] if not spec['two_pass']: # The quality name is a contract enum, never caller text. @@ -94,7 +96,7 @@ def cut_problems(spec, seconds, prefix='', *, editor=None): """A shot cut at or after the end of the video never appears; refuse it before anything is generated.""" if spec['kind'] != 'generate' or spec.get('refine_from') or spec.get('continue_from') or seconds is None: return [] - if any(block['after'] for block in editor['blocks']) if editor is not None else not single_block(seconds): + if any(block['after'] for block in editor['blocks']) if editor is not None else not single_block(spec, seconds): return [] # Every block of a continued video reads the same text. late = [cut for cut in specs.cut_times(spec.get('prompt')) if cut >= seconds] if not late: @@ -118,7 +120,7 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N from ..chain_runner import ChainTooLong from ..chain_store import reference limit, continued = runner.max_parts, bool(spec.get('continue_from')) - count = lambda seconds: len(plan(seconds, continued=continued)['blocks']) + count = lambda seconds: len(plan(seconds, max_frames=single_frame_limit(spec), continued=continued)['blocks']) try: with runner.lock: if editor is not None: @@ -148,7 +150,7 @@ def parts_problems(spec, runner, prefix='', *, editor=None, work=None, planned=N def continuation_unavailable(): return chain_error('continuation_unavailable', 'This backend cannot continue multiple video blocks.', - 'Submit a new job with seconds<=15, or enable a backend with continuation inputs and resume.') + 'Use POST /plan for a supported single block (unknown limit: seconds<=15), or enable continuation and resume.') def library_id(relative): @@ -403,6 +405,9 @@ def load(self): self.finish(row, 'failed', error=dict(code='interrupted', message='ComfyUI restarted while this job ran; submit it again'), save=False) row['_restored'] = True + if row['kind'] == 'generate' and not row['spec'].get('refine_from'): + # Before device-aware Agent planning, every accepted job used the unknown limit. + row['spec'].setdefault('single_max_frames', None) self.jobs[row['id']] = row for row in self.jobs.values(): if (row['kind'] != 'generate' or row['spec'].get('refine_from') @@ -569,7 +574,7 @@ def submit_request(self, value, *, client=None, key=None, parent=None, editor=No from .routes import substitute item = substitute(item, names) try: - normalized = specs.normalize(item) + normalized = specs.normalize(item, port=self.port) except specs.SpecError as error: problems.extend(schema.prefix_diagnostic(e, prefix) for e in error.errors) continue @@ -619,7 +624,7 @@ def submit_request(self, value, *, client=None, key=None, parent=None, editor=No if any(block['after'] for block in row['editor']['blocks']) and not self.continuation_ready(): raise continuation_unavailable() continue - self.require_continuation(row['spec']['seconds']) + self.require_continuation(row['spec']) if row['spec'].get('continue_from') and not self.continuation_ready(): raise continuation_unavailable() if receipt: @@ -697,8 +702,8 @@ def continuation_ready(self): from ..comfy_chain import continuation_available return continuation_available() and os.environ.get('FREEVIDEO_CHAIN_INDEPENDENT') != '1' - def require_continuation(self, seconds): - if not single_block(seconds) and not self.continuation_ready(): + def require_continuation(self, spec): + if not single_block(spec) and not self.continuation_ready(): raise continuation_unavailable() def require_runner(self): @@ -987,7 +992,7 @@ def estimate(self, row): 'step_elapsed_seconds', 'estimated_step_seconds') if message.get(key) is not None} overall = message.get('overall') if isinstance(message.get('overall'), dict) else {} overall = {key: overall[key] for key in ('fraction', 'estimated', 'status', 'phase_start_fraction', 'phase_weight', - 'phase_seconds', 'phase_elapsed_seconds', 'sampling_time_weighted') + 'phase_seconds', 'phase_elapsed_seconds', 'sampling_time_weighted', 'remaining_source') if overall.get(key) is not None} if overall: value['overall'] = overall @@ -1197,7 +1202,8 @@ def planned_parts(self, spec, rows=()): spec = source['spec'] if spec['kind'] != 'generate' or spec.get('refine_from') or source.get('editor'): return None - total += len(plan(spec['seconds'], continued=bool(spec.get('continue_from')))['blocks']) + total += len(plan(spec['seconds'], max_frames=single_frame_limit(spec), + continued=bool(spec.get('continue_from')))['blocks']) if not spec.get('continue_from'): return total work = self.continued_work(spec) @@ -1278,6 +1284,7 @@ def resolve(): return graph, extra, node else: payload = dict(node=node, prompt=graph, seconds=spec['seconds'], + max_frames=single_frame_limit(spec), client_id='agent:' + row['id'], extra_data=extra) if row.get('editor'): payload.pop('seconds') @@ -1315,7 +1322,7 @@ def start_chain(self, row, *, payload=None, resume=False): if (any(b['state'] != 'done' and (b['after'] or b['before']) for b in runner.records[row['chain_id']]['blocks']) if resume else any(b['after'] for b in payload['blocks']) if row.get('editor') - else bool(row['spec'].get('continue_from')) or not single_block(payload['seconds'])) and not self.continuation_ready(): + else bool(row['spec'].get('continue_from')) or not single_block(row['spec'])) and not self.continuation_ready(): raise continuation_unavailable() self.touch(row, chain_submission='creating', status='queued', queued_at=self.clock()) from ..chain_runner import ChainBusy, ChainTooLong diff --git a/freevideo_engine/agent/length.py b/freevideo_engine/agent/length.py new file mode 100644 index 00000000..eb4a4923 --- /dev/null +++ b/freevideo_engine/agent/length.py @@ -0,0 +1,49 @@ +"""Studio capability limits for Agent generation plans, without hardware probing.""" +import time + +# A size asked for the first time is worked out in the background in about 0.1 s. The plan that asks first +# waits for it, so it answers what the submit after it will do. +WAIT_SECONDS = 3. + + +def capability_table(port, spec, *, wait=0.): + query = dict(width=str(spec['width']), height=str(spec['height']), + task=spec['task'], adapters='1' if spec['loras'] else '0') + deadline = time.monotonic() + wait + try: + while True: + table = port.capability_snapshot(query) + if table.get('status') != 'computing' or time.monotonic() >= deadline: + return table + time.sleep(.05) + except Exception: + # The studio cache is advisory. Its failure retains the unknown-limit policy. + return dict(status='unavailable') + + +def single_frame_limit(spec, *, port=None, table=None): + """Frame limit for this normalized size/sampling/task; unknown keeps the 15 s default. + + A submitted spec pins even an unknown limit: a later cache refresh must not + change the accepted canvas or number of parts. Old receipts also keep their + unknown limit when no capability source is supplied. + """ + if 'single_max_frames' in spec: + return spec['single_max_frames'] + if table is None: + table = capability_table(port, spec, wait=WAIT_SECONDS) if port is not None else {} + if table.get('status') != 'ready': + return None + tiers = table.get('tiers', {}) + tier = tiers.get(spec['quality']) + if tier is None: + tier = next((row for row in tiers.values() + if row.get('steps') == spec['base_steps'] and row.get('two_pass') == spec['two_pass'] + and (not spec['two_pass'] or spec['refine_steps'] == 3)), {}) + index = next((i for i, row in enumerate(table.get('sizes', [])) + if (row['width'], row['height']) == (spec['width'], spec['height'])), None) + if index is None: + return None + maxima = tier.get('max_frames', []) + limit = maxima[index] if isinstance(maxima, list) and index < len(maxima) else None + return limit if type(limit) is int and limit > 0 else None diff --git a/freevideo_engine/agent/mcp.py b/freevideo_engine/agent/mcp.py index c59d2599..2b683e7a 100644 --- a/freevideo_engine/agent/mcp.py +++ b/freevideo_engine/agent/mcp.py @@ -44,7 +44,7 @@ def tool_list(): dict(name='freevideo_prompt_guide', title='H3 prompt guide', description='How to write prompts: sections, shots and cuts, camera motion, dialogue, references, short-drama tips, and a full example.', annotations=read), dict(name='freevideo_schema', title='Job JSON Schemas', description='JSON Schemas of every job kind and of image edit recipes.', annotations=read), dict(name='freevideo_plan', title='Plan a job', description="Validate a job without running it: actual frames/seconds, sampling passes, warnings and a time estimate from this computer's history. Same fields as freevideo_generate.", annotations=read), - dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds: a longer request continues and joins into one file automatically. Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt; the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write), + dict(name='freevideo_generate', title='Generate a video', description='Generate one video from an H3 prompt, up to 60 seconds. Under 30 s defaults to one generation if the device supports this size and quality; unknown limits use 15 s. Otherwise parts continue and join automatically (30 s is 15 + 15). Prefer a new shot with its reference pictures wherever the film can cut; continue only a take, action or sound that must stay unbroken. Use this for any single video, also after freevideo_enhance_prompt; the drama tools are only for a film of several separately chosen shots. Optionally from first/last frames or references (files, local paths, or frames of earlier jobs). Returns the job; with wait, its result (video path and URL).', annotations=write), dict(name='freevideo_submit_batch', title='Submit a storyboard', description='Submit several jobs at once. Each row is a job object with "kind" (generate default, assemble, vlm, enhance) and an optional "ref"; later rows use "@ref" wherever a job id is expected (from_job, after, clips, refine_from). FreeVideo runs them in dependency order.', annotations=write), dict(name='freevideo_job', title='Job status', description='Status, progress, result or error of a job.', annotations=read), dict(name='freevideo_wait', title='Wait for a job or the user', description="Wait at most 25 seconds for a job (with id) or, without id, at most 50 seconds for what the user says in FreeVideo (inbox). again=true means nothing new yet: call it again. A job wait returns early when the user says something. If the job is still active, call it again with the same ID; do not resubmit.", annotations=read), diff --git a/freevideo_engine/agent/productions.py b/freevideo_engine/agent/productions.py index 92af651a..809b2213 100644 --- a/freevideo_engine/agent/productions.py +++ b/freevideo_engine/agent/productions.py @@ -360,7 +360,7 @@ def _take_estimate(self, state, shot, take, item=None): continues=shot['continues']) if resolved['references']: item['media'] = dict(references=deepcopy(resolved['references'])) - estimate, revision = generation_estimate(self.jobs.port, spec.normalize(item)) + estimate, revision = generation_estimate(self.jobs.port, spec.normalize(item, port=self.jobs.port)) return dict(estimate, revision=revision) except Exception: # Timing is optional; a forecast failure must not reject a film or job. return dict(seconds=None, basis=None, reason='unavailable') @@ -375,7 +375,7 @@ def _join_estimate(self, state, previous=None): return dict(seconds=None, basis=None, reason='no_matching_estimate') item = deepcopy(take['spec']) item.pop('continue_from', None) - normalized = spec.normalize(item) + normalized = spec.normalize(item, port=self.jobs.port) shapes.append([normalized[k] for k in ('width', 'height', 'frames')]) current = next((j for j in state['joins'] if j['ref'] == state['current_join']), None) transition = current['transition'] if current else state['screenplay'].get('transition', 'cut') diff --git a/freevideo_engine/agent/projects.py b/freevideo_engine/agent/projects.py index 348d5b8a..2dd233f4 100644 --- a/freevideo_engine/agent/projects.py +++ b/freevideo_engine/agent/projects.py @@ -33,6 +33,25 @@ 'source': 'a source that has a picture or sound', 'invalid': 'an op named in the operations schema with its fields', } +# The note operation shares the page's 'text' code with describe; what it needed is its own. +NOTE_EXPECTED = 'picture clips and a text of at most 120 characters on one line (an empty text removes the note)' +# An agent acts on the last answer it read, so the answer that brings new picture clips asks for their notes. +# It comes first in that answer: the project after it carries every clip's description, and a real agent +# that printed only the start of a long answer never saw a request placed after it. +NOTE_NEXT = ('Picture clips %s have no note yet. Write each one now with a note operation: in the user\'s language, ' + 'what is in the picture, subject first; not its position and not its progress.') + + +def note_next(result, project, ids): + missing = [clip['id'] for track in project['tracks'] if track['kind'] == 'video' for clip in track['clips'] + if clip['id'] in ids and not clip.get('note')] + return dict(next=NOTE_NEXT % ', '.join(missing), **result) if missing else result + + +def new_project(given): + """The page's empty timeline with what the caller sent on top; a name alone is enough to start one.""" + return dict(dict(version=1, id=None, revision=0, name='', fps=comfy_editor.FPS, width=1344, height=768, settings=None, serial=0, + tracks=[dict(id='v1', kind='video', clips=[]), dict(id='a1', kind='audio', clips=[])]), **given) class ProjectError(JobError): @@ -64,7 +83,7 @@ def _refused(project, error): reason = str(error) if type(error) is ValueError and re.fullmatch(r'[A-Z][A-Za-z ]{2,60}', str(error)) else '' row = schema.diagnostic('/project', 'a valid editor project' + (' (%s)' % (reason[:1].lower() + reason[1:]) if reason else ''), project) - row['next'] = ('Correct that part of /project. To place clips, create the project with empty tracks, ' + row['next'] = ('Correct that part of /project. For a new timeline send only {"project": {"name": "..."}}; ' 'then add each clip with an insert operation.') return spec.SpecError([row]) @@ -162,6 +181,8 @@ def _save(self, project, *, ref=None, create=None, client=None): def _rejected(self, error, stored, path, received, *, operation=None): if error['code'] in STATE_CODES: raise ProjectError(STATE_CODES[error['code']], project=stored, operation=operation) + if error['code'] == 'text' and isinstance(received, dict) and received.get('op') == 'note': + raise _invalid(path, NOTE_EXPECTED, received) raise _invalid(path, RULE_EXPECTED.get(error['code'], 'an operation this timeline allows'), received) # Reads --------------------------------------------------------------------------------------------- @@ -196,8 +217,9 @@ def create(self, value, *, ref=None, client=None): if replay is not None: return replay project = value.get('project') if isinstance(value, dict) else None - if not isinstance(project, dict) or project.get('id') is not None or project.get('revision') != 0: - raise _invalid('/project', 'a new project with id null and revision 0') + project = new_project(project) if isinstance(project, dict) else None + if project is None or project.get('id') is not None or project.get('revision') != 0: + raise _invalid('/project', 'a new project: its name is enough, with no id and no revision') # The page's receipt makes a repeated create return the first project. create = _operation_key(ref, client) if ref is not None else None self._lengths(project) @@ -222,7 +244,8 @@ def write(self, identity, value, *, ref=None, client=None): # Operations ---------------------------------------------------------------------------------------- def _media(self, operations): - """Insert operations name a source key; the server fills in its length and streams.""" + """Insert operations name a source key; the server fills in its length and streams, and for a + saved video the description it was generated from.""" keys = [op['source'] for op in operations if op.get('op') == 'insert'] found = comfy_editor.sources(self.root, keys) if keys else {} resolved = [] @@ -236,6 +259,8 @@ def _media(self, operations): kind, identity = op['source'].split(':', 1) media = dict(source=dict(kind=kind, id=identity), frames=info['frames'], hasVideo=info['has_video'], hasAudio=info['has_audio'], width=info['width'], height=info['height']) + if info.get('text'): + media['text'] = info['text'] resolved.append(dict({k: v for k, v in op.items() if k != 'source'}, media=media)) return resolved @@ -263,6 +288,7 @@ def operations(self, identity, value, *, client=None, ref=None): operations[index] if index is not None else None, operation=index) saved = self._save(reply['project'], ref=ref, client=client) result = dict(project=saved, changed_clip_ids=changed_clip_ids(stored, saved), results=reply['results']) + result = note_next(result, saved, set(result['changed_clip_ids']) - set(_clips(stored))) self._remember(ref, digest, result, client) return result @@ -324,7 +350,8 @@ def generate(self, identity, value, *, jobs, capability=None, ref=None, client=N raise with self.lock: self.prepared[job['id']] = saved - result = dict(job=jobs.view(job), jobs=[jobs.view(job)], project=saved, clip_ids=plan['ids']) + result = note_next(dict(job=jobs.view(job), jobs=[jobs.view(job)], project=saved, clip_ids=plan['ids']), + saved, set(plan['ids'])) self._remember(ref, digest, result, client) return result diff --git a/freevideo_engine/agent/routes.py b/freevideo_engine/agent/routes.py index 2638001c..96400e7e 100644 --- a/freevideo_engine/agent/routes.py +++ b/freevideo_engine/agent/routes.py @@ -29,6 +29,19 @@ from .jobs import ACTIVE, DONE, JobError, Jobs, chain_error, library_view PREFIX = '/freevideo/agent/v1' +# What an empty wait tells the agent: the conversation is open for as long as the user keeps the page. +KEEP_WAITING = ('Nothing new yet. The user is still in FreeVideo and may speak at any moment: call this wait ' + 'again now. An empty wait never ends the conversation; do not end your turn or write a summary.') +# The user may open another conversation while the agent waits; its words go where the user now is. +MOVED = ('The user is now in another conversation, session %s. Post your words to that session from now on; ' + 'the user no longer sees the earlier one.') +MOVED_NEW = ' It starts afresh: do not carry requests over from the earlier conversation unless the user refers to them.' +MOVED_BACK = (' It is one the user was in before, and the user means what was said there, not the conversation you just ' + 'left. Its last lines are in "before"; all of it: GET /sessions/%s/events.') +# An agent that connects again finds the conversation where the user left it, and has greeted there once already. +RESUMED = ('This conversation is already under way; its last lines are in "before". Do not introduce yourself again: ' + 'say nothing until the user speaks, then carry on from there.') +BEFORE_LINES, BEFORE_TEXT = 8, 600 def local_address(request): @@ -211,7 +224,7 @@ def capabilities(port, jobs): def plan(port, value, jobs=None): from ..two_pass import plan as sampling_plan from ..geometry import geometry - spec = specs.normalize(value) + spec = specs.normalize(value, port=port) result = dict(spec={k: v for k, v in spec.items() if k != 'depends_on'}, warnings=[]) if spec['kind'] != 'generate' or spec.get('refine_from'): return result @@ -223,7 +236,7 @@ def plan(port, value, jobs=None): if problems: raise specs.SpecError(problems) if jobs is not None: - jobs.require_continuation(spec['seconds']) + jobs.require_continuation(spec) if spec.get('continue_from') and not jobs.continuation_ready(): from .jobs import continuation_unavailable raise continuation_unavailable() @@ -240,7 +253,9 @@ def plan(port, value, jobs=None): if spec['width'] * spec['height'] > 2 * 1024 ** 2: result['warnings'].append('Above 2 MP takes much longer and needs more memory') from ..chain_plan import plan as chain_plan - result['chain_plan'] = chain_plan(spec['seconds'], continued=bool(spec.get('continue_from'))) + from .length import single_frame_limit + result['chain_plan'] = chain_plan(spec['seconds'], max_frames=single_frame_limit(spec), + continued=bool(spec.get('continue_from'))) from .forecast import generation_estimate result['estimate'], result['capabilities_revision'] = generation_estimate(port, spec, result['chain_plan']) return result @@ -386,7 +401,7 @@ def reply(value, status=200): page_sessions = {} connection = dict(connected=False, client=None, via=None, last_seen_at=None) # The conversation in the Autopilot view: the page and the connected agent talk in the agent's latest session. - chat = dict(client=None, waiting=0, folds={}) + chat = dict(client=None, waiting=0, folds={}, told={}) server._freevideo_agent_connection = connection def problem(code, message=None, status=None, **details): @@ -524,7 +539,11 @@ def attach(): if sid: store.get(sid) # not_found for an unknown session else: - sid = store.latest_for(client) or (store.create(client=client)['id'] if write else None) + # Work belongs to the conversation the agent last heard from; until its next wait it has + # not learned that the user opened another one. + told = chat['told'].get(client) + sid = told if told is not None and store.has(client, told) else ( + store.current_for(client) or (store.create(client=client)['id'] if write else None)) if sid is None: return None request['agent_session'] = sid @@ -782,18 +801,30 @@ def chat_current(rows, sid, fold): key=lambda r: (r['status'] in ('running', 'cancelling'), r['created_at'])) return mine, active[-1] if active else None - def chat_session(store): - return store.latest_for(chat['client']) if store is not None and chat['client'] is not None else None + def chat_session(store, sid=None): + # The conversation the user is in, or an earlier one of the connected agent that the page names. + if store is None or chat['client'] is None: + return None + if sid is None: + return store.current_for(chat['client']) + if not store.has(chat['client'], sid): + raise JobError('not_found', 'No such conversation') + return sid + + def chat_moved(store, before): + # The agent waits in the conversation the user was in; let that wait look again. + if store.current_for(chat['client']) != before: + store.wake(chat['client']) @routes.get('/freevideo/agent/chat') @guarded async def chat_read(request): - after = request[query_key].get('after') + after, wanted = request[query_key].get('after'), request[query_key].get('session') def read(): store = work_sessions() - sid = chat_session(store) + sid = chat_session(store, wanted) rows = jobs.rows() - value = dict(agent=agent_state(), session=sid, last_seq=0, read_seq=0, items=[], jobs={}, confirms={}, current=None, + value = dict(agent=agent_state(), session=sid, here=chat_session(store), last_seq=0, read_seq=0, items=[], jobs={}, confirms={}, current=None, active=sum(1 for r in rows if r['status'] in ACTIVE)) if sid is None: return value @@ -849,7 +880,8 @@ def write(): store, client = work_sessions(), chat['client'] if store is None or client is None: return None - sid = store.latest_for(client) or store.create(client=client, by='user')['id'] + before = store.current_for(client) + sid = chat_session(store, value.get('session')) or store.create(client=client, by='user')['id'] fold = chat_fold(store, sid) if value['id'] in fold['sent']: seq = fold['sent'][value['id']] @@ -864,6 +896,7 @@ def write(): fold['ids'][row['seq']] = value['id'] while len(fold['sent']) > 500: fold['ids'].pop(fold['sent'].pop(next(iter(fold['sent']))), None) + chat_moved(store, before) return 201, dict(seq=row['seq'], id=value['id'], at=row['at'], session=sid, read=False, agent=agent_state()) done = await asyncio.to_thread(write) if done is None: @@ -879,14 +912,49 @@ async def chat_confirm(request): value = await body(request) def write(): store = work_sessions() - sid = chat_session(store) + before = chat_session(store) + sid = chat_session(store, value.get('session')) if sid is None: raise JobError('not_found', 'No conversation') row = store.append(sid, dict(type='confirm', id=value['id'], state='approved' if value['approve'] else 'declined'), by='user') + chat_moved(store, before) return dict(id=row['id'], state=row['state']) return reply(await asyncio.to_thread(write)) + @routes.get('/freevideo/agent/chat/list') + @guarded + async def chat_list(request): + def read(): + store = work_sessions() + here = chat_session(store) + rows = store.history(chat['client']) if here else [] + active = [r.get('session') for r in jobs.rows() if r['status'] in ACTIVE] + return dict(agent=agent_state(), here=here, + conversations=[dict(row, running=active.count(row['id'])) for row in rows]) + return reply(await asyncio.to_thread(read)) + + @routes.post('/freevideo/agent/chat/new') + @guarded + async def chat_new(request): + def write(): + store, client = work_sessions(), chat['client'] + if store is None or client is None: + return None + before = store.current_for(client) + # A conversation the user has not spoken in yet is the new one already. + if before is not None and not store.spoken(before): + return 200, dict(session=before, agent=agent_state()) + sid = store.create(client=client, by='user')['id'] + chat_moved(store, before) + return 201, dict(session=sid, agent=agent_state()) + done = await asyncio.to_thread(write) + if done is None: + return reply({'error': dict(code='not_connected', message='No agent is connected.', retryable=True, + details=[], next='Connect an agent in the Autopilot view, then try again.', + request_id=errors.request_id())}, 409) + return reply(done[1], done[0]) + @routes.get('/freevideo/agent/chat/video') @guarded async def chat_video(request): @@ -970,8 +1038,13 @@ async def connect_guide(request): root = installation_root() prepare_token(root) from .editor_runtime import available + store = work_sessions() + def begun(): + here = store.current_for(chat['client']) if store is not None and chat['client'] else None + return bool(here and store.get(here)['last_seq']) value = connection_guide(root, request.scheme + '://' + request.host, request[query_key].get('lang'), - dict(continuation=await asyncio.to_thread(jobs.continuation_ready), editing=available())) + dict(continuation=await asyncio.to_thread(jobs.continuation_ready), editing=available()), + resumed=await asyncio.to_thread(begun)) if request[query_key].get('format') == 'json': return reply(value) return web.Response(text=connection_text(value), content_type='text/plain', headers={'Cache-Control': 'no-store'}) @@ -981,6 +1054,7 @@ async def connect_guide(request): async def hello(request): value = await body(request) connection.update(connected=True, client=value['client'], via=value['via'], last_seen_at=time.time()) + chat['told'].pop(request['agent_client'], None) # a newly connected agent has heard from no conversation yet unfinished = await asyncio.to_thread(jobs.productions.list, status='active', limit=5) return reply(dict(connection, unfinished=unfinished)) @@ -1193,8 +1267,15 @@ async def session_post(request): return reply(await stop_session(sid), 201) if by == 'agent' and value['type'] == 'message' and isinstance(value.get('text'), str): value['text'] = chat_text(value['text']) - return reply(await asyncio.to_thread(sessions.append, sid, value, - by=by, client=session_client(request, by)), 201) + client = session_client(request, by) + def write(): + row = sessions.append(sid, value, by=by, client=client) + here = sessions.current_for(client) if client and value['type'] == 'message' else None + if here not in (None, sid): + # Said where the user no longer looks: the answer itself tells the agent where to say it. + row['next'] = MOVED % here + return row + return reply(await asyncio.to_thread(write), 201) @routes.get(PREFIX + '/confirmations') @guarded @@ -1480,12 +1561,29 @@ async def inbox(request): client = request['agent_client'] if store is not None and sid is None: # An agent that only listens has no session yet: give it one, so the page can speak into it. - sid = await asyncio.to_thread(lambda: store.latest_for(client) or store.create(client=client)['id']) + sid = await asyncio.to_thread(lambda: store.current_for(client) or store.create(client=client)['id']) request['agent_session'] = sid + # Without a session of its own choosing, the agent listens wherever its user is. + follow = store is not None and not request.headers.get('X-FreeVideo-Session', '').strip() + def answer(value): + told, chat['told'][client] = chat['told'].get(client), value['session'] + if follow and told != value['session']: + first = value['inbox'][0]['seq'] if value['inbox'] else store.get(value['session'])['last_seq'] + 1 + if first > 1: + # What the user sees above these words. An agent that has to fetch it answers from the conversation it left. + value['before'] = [dict(e, text=e['text'][:BEFORE_TEXT]) if isinstance(e.get('text'), str) else e + for e in store.events(value['session'], after=max(0, first - 1 - BEFORE_LINES), + limit=BEFORE_LINES) if e['seq'] < first] + said = (RESUMED if first > 1 else '') if told is None else ( + MOVED % value['session'] + (MOVED_BACK % value['session'] if first > 1 else MOVED_NEW)) + value['next'] = (said + ' ' + value.get('next', '')).strip() + return reply(value) deadline = time.monotonic() + min(50., float(request[query_key].get('wait', '0') or 0)) chat['waiting'] += 1 try: while True: + if follow: + sid = request['agent_session'] = await asyncio.to_thread(store.current_for, client) or sid if store is not None and sid is not None: # Mark before reading so words arriving in between wake the wait below. def read(): @@ -1493,15 +1591,17 @@ def read(): return mark, store.unread(sid, client) mark, pending = await asyncio.to_thread(read) if pending: - return reply(dict(session=sid, inbox=pending[-20:])) + return answer(dict(session=sid, inbox=pending[-20:])) left = deadline - time.monotonic() if left <= 0: - # again: nothing new this time; the caller simply waits once more. - return reply(dict(session=sid, inbox=[], again=True)) + # again: nothing new this time; the caller simply waits once more. An agent decides what to + # do next from the last answer it read, so the empty one says it in words as well. + return answer(dict(session=sid, inbox=[], again=True, next=KEEP_WAITING)) if store is None or sid is None: await asyncio.sleep(min(left, 1.)) else: - await store.wait(sid, after=mark, seconds=min(left, 25.), by='user') + await store.wait(sid, after=mark, seconds=min(left, 25.), by='user', + leave=(lambda here=sid: store.current_for(client) != here) if follow else None) finally: chat['waiting'] -= 1 # The agent was here for the whole wait, not only when it began. diff --git a/freevideo_engine/agent/schema.py b/freevideo_engine/agent/schema.py index 38410e1a..2cdd74e8 100644 --- a/freevideo_engine/agent/schema.py +++ b/freevideo_engine/agent/schema.py @@ -354,12 +354,12 @@ def route(summary, body=None, query=None, **kw): for suffix, method, body in (('', 'GET', None), ('/session', 'GET', None), ('/activity', 'GET', None), ('/activity/cancel', 'POST', obj({'id': JOB}, ('id',))), ('/connect', 'GET', None), ('/connect/reset', 'POST', EMPTY), ('/chat', 'GET', None), ('/chat/now', 'GET', None), - ('/chat/video', 'GET', None), - ('/chat', 'POST', obj({'text': string(8000), 'id': CHAT_ID}, ('text', 'id'))), - ('/chat/confirm', 'POST', obj({'id': string(12, pattern=r'^cf[0-9a-f]{10}$'), 'approve': {'type': 'boolean'}}, - ('id', 'approve')))): + ('/chat/video', 'GET', None), ('/chat/list', 'GET', None), ('/chat/new', 'POST', EMPTY), + ('/chat', 'POST', obj({'text': string(8000), 'id': CHAT_ID, 'session': SESSION}, ('text', 'id'))), + ('/chat/confirm', 'POST', obj({'id': string(12, pattern=r'^cf[0-9a-f]{10}$'), 'approve': {'type': 'boolean'}, + 'session': SESSION}, ('id', 'approve')))): routes[(method, '/freevideo/agent' + suffix)] = route('Studio Agent controls', body=body, auth=False, local=True) - routes['GET', '/freevideo/agent/chat']['query'] = obj({'after': number(0, integer=True)}) + routes['GET', '/freevideo/agent/chat']['query'] = obj({'after': number(0, integer=True), 'session': SESSION}) routes['GET', '/freevideo/agent/chat/video']['query'] = obj({'id': string(43, pattern=r'^[0-9]{4}-[0-9]{2}-[0-9]{2}/[0-9a-f]{32}$')}, ('id',)) for suffix in ('', '/connect'): routes['GET', '/freevideo/agent' + suffix]['query'] = obj({'lang': enum(('zh', 'en'))}) @@ -635,7 +635,7 @@ def image_schema(name): PROJECT_OPERATIONS = { 'list': ('GET', '/projects', 'List editing projects shared with the editor page. Then get one to read its revision and clip IDs.'), 'get': ('GET', '/projects/{id}', 'Read a timeline and authenticated source URLs. Read its revision and clip IDs before editing.'), - 'create': ('POST', '/projects', 'Create an editor timeline. Send a valid editor project with id null and revision 0.'), + 'create': ('POST', '/projects', 'Create an editor timeline. Send {"project": {"name": "..."}}; the rest of an empty timeline is filled in.'), 'write': ('PUT', '/projects/{id}', 'Replace a timeline. First get the project and preserve its id and revision; recompute from error.project on project_changed.'), 'operations': ('POST', '/projects/{id}/operations', 'Apply 1-64 timeline edits atomically and save once. First get revision and clip IDs; on project_changed recompute from error.project, and on clip_busy wait for generation to finish.'), 'generate': ('POST', '/projects/{id}/generate', 'Make new picture and sound for selected clips or a frame range, as the page Generate does. First get revision and clip IDs; the parts are reserved at once and fill in as the returned job finishes.'), @@ -730,7 +730,9 @@ def session_schema(name): def project_schema(name): if name in ('create', 'write'): - return obj(dict(project={'type': 'object', 'description': 'The editor page project record; validated by the shared editor save.'}, ref=REF), ('project',)) + text = ('A new timeline: {"name": "..."} is enough (width and height if not 1344x768); the rest is filled in. Add clips with insert operations afterwards.' + if name == 'create' else 'The whole project record as you read it, with your changes; validated by the shared editor save.') + return obj(dict(project={'type': 'object', 'description': text}, ref=REF), ('project',)) ids = array(CLIP, 1, 500) revision = number(1, integer=True, description='The saved revision you read; a later save returns project_changed.') if name == 'generate': @@ -751,7 +753,7 @@ def project_schema(name): if name != 'operations': return EMPTY fields = { - 'insert': (dict(source=EDITOR_SOURCE, at=EDITOR_FRAME), ('source',), 'Insert picture and sound from a source. Omit at to append at the timeline end.'), + 'insert': (dict(source=EDITOR_SOURCE, at=EDITOR_FRAME), ('source',), 'Insert picture and sound from a source. Omit at to append at the timeline end. A work brings the description it was generated from. Then write the new picture clip\'s note.'), 'split': (dict(clip_ids=ids, frame=EDITOR_FRAME), ('clip_ids', 'frame'), 'Cut selected clips and their linked partners. The frame must be inside a selected segment.'), 'trim': (dict(clip_ids=ids, frame=EDITOR_FRAME, edge=enum(('start', 'end'))), ('clip_ids', 'frame', 'edge'), 'Trim a selected edge at the frame. The frame must be inside a selected segment.'), 'remove': (dict(clip_ids=ids), ('clip_ids',), 'Remove selected clips and linked partners. Use clip IDs from this revision.'), @@ -761,6 +763,7 @@ def project_schema(name): 'move': (dict(clip_ids=ids, frames=number(-10**7, 10**7, integer=True)), ('clip_ids', 'frames'), 'Move selected clips by a signed relative frame offset. Use clip IDs from this revision; linked partners move together.'), 'describe': (dict(clip_ids=ids, text=dict(type='string', maxLength=8000)), ('clip_ids', 'text'), 'Set the description of selected clips. Use clip IDs from this revision and one paragraph of text.'), 'set_ref': (dict(clip_ids=ids, ref=dict(type='string', maxLength=128, pattern=r'^[A-Za-z0-9_-]{0,128}$')), ('clip_ids', 'ref'), 'Set an outside shot reference on selected clips. Use clip IDs from this revision; an empty ref removes it.'), + 'note': (dict(clip_ids=ids, text=dict(type='string', maxLength=120)), ('clip_ids', 'text'), 'Write the one line the user sees under selected picture clips; without it they see the start of the generation text. Do it after you add, generate or change a clip: in the user\'s language, what is in the picture, subject first ("Train through morning mist"); a short clip shows only the first few characters. Not the clip\'s position (opening, part 2) and not its progress. Up to 120 characters on one line; an empty text removes it. It never changes the description used for generation.'), 'use_version': (dict(clip_id=CLIP, index=number(0, 49, integer=True)), ('clip_id', 'index'), 'Restore a stored clip version. First get the clip ID and its versions array; index is zero based.'), 'subtitles': (dict(cues=array(obj(dict(start=EDITOR_FRAME, frames=number(1, 10**7, integer=True), text=dict(type='string', minLength=1, maxLength=400)), ('start', 'frames', 'text')), 1, 500), replace=dict(type='boolean')), ('cues',), diff --git a/freevideo_engine/agent/sessions.py b/freevideo_engine/agent/sessions.py index 5d48ab05..5fd002eb 100644 --- a/freevideo_engine/agent/sessions.py +++ b/freevideo_engine/agent/sessions.py @@ -58,6 +58,11 @@ def __init__(self, root: Path, *, clock=time.time, max_events=20000): self._initial_seq = {} self._readers = {} self._user_seq = {} # Last event the user wrote; unread() skips the file when nothing is newer. + # When each session began or its user last acted in it (spoke, answered), and the user's first words + # there: the conversation the user is in, and what the page calls each one in its history. + self._acted = {} + self._said = {} + self._chosen = set() self._waiters = {} self._partial = set() self.root.mkdir(parents=True, exist_ok=True) @@ -91,6 +96,9 @@ def _load(self): updated_at=meta['created_at'], client=meta['client'], last_seq=0, stopped=False) self._ids[sid] = {} self._clients[sid] = {meta['client']} if meta['client'] else set() + self._acted[sid] = (meta['created_at'], 1) + if meta.get('by') == 'user': + self._chosen.add(sid) for row in stream: if not row.endswith(b'\n'): self._partial.add(sid) @@ -111,6 +119,12 @@ def _fold(self, sid, event): meta.update(last_seq=event['seq'], updated_at=event['at']) if event['by'] == 'user': self._user_seq[sid] = event['seq'] + # No counter here: the order must come out the same when the files are read again after a + # restart, so two things in one clock tick are ordered by what they are, then by session. + self._acted[sid] = max(self._acted[sid], (event['at'], 0)) + self._chosen.add(sid) + if event['type'] == 'message' and sid not in self._said: + self._said[sid] = event['text'].strip()[:60] if event.get('client'): self._clients[sid].add(event['client']) if event.get('id'): @@ -149,6 +163,9 @@ def create(self, *, title=None, client=None, by='agent') -> dict: updated_at=meta['created_at'], client=client, last_seq=0, stopped=False) self._ids[sid], self._clients[sid] = {}, {client} if client else set() self._initial_seq[sid] = 0 + self._acted[sid] = (meta['created_at'], 1) + if by == 'user': + self._chosen.add(sid) return deepcopy(self._meta[sid]) def list(self, *, limit=50, before=None) -> dict: @@ -244,7 +261,9 @@ def events(self, sid, *, after=0, limit=500) -> list[dict]: self._get(sid) return self._events(sid, after, limit) - async def wait(self, sid, *, after, seconds, types=None, by=None) -> list[dict]: + async def wait(self, sid, *, after, seconds, types=None, by=None, leave=None) -> list[dict]: + """Events after a cursor, waiting up to some seconds. leave, when given, ends an empty wait early + once it answers true: wake() asks for that look when the user moves to another conversation.""" _check(schema.session_schema('events'), dict(after=after, wait=seconds)) if types is not None: _check(schema.array(schema.enum(('message', 'plan', 'step', 'confirm', 'control')), high=5), types) @@ -294,6 +313,8 @@ async def offload(work): found, cursor = await offload(read) if found: return found + if leave is not None and await offload(leave): + return [] remaining = deadline - loop.time() if remaining <= 0: return [] @@ -316,6 +337,46 @@ def latest_for(self, client) -> str | None: rows = [row for sid, row in self._meta.items() if client in self._clients[sid]] return max(rows, key=lambda r: (r['updated_at'], r['id']))['id'] if rows else None + def current_for(self, client) -> str | None: + """The conversation this client's user is in: the newest one, or an older one the user spoke or + answered in after that; what the agent or the server writes never moves the user. Until the user + has opened a conversation or spoken on the page, it is wherever the latest work is, as before.""" + with self.lock: + rows = [sid for sid in self._meta if client in self._clients[sid]] + if not rows: + return None + if not self._chosen.intersection(rows): + return max(rows, key=lambda sid: (self._meta[sid]['updated_at'], sid)) + return max(rows, key=lambda sid: (self._acted[sid], sid)) + + def has(self, client, sid) -> bool: + with self.lock: + return sid in self._meta and client in self._clients[sid] + + def spoken(self, sid) -> bool: + """Whether the user has said anything in this conversation.""" + with self.lock: + return sid in self._said + + def wake(self, client) -> None: + """Let this client's waits look again: its user moved to another conversation.""" + with self.lock: + waiters = [w for sid, group in self._waiters.items() if client in self._clients[sid] for w in group] + for loop, signal in waiters: + try: + loop.call_soon_threadsafe(signal.set) + except RuntimeError: + pass # A closed client loop no longer needs notification. + + def history(self, client, *, limit=50) -> list[dict]: + """This client's conversations the user has spoken in, for the page: latest change first.""" + current = self.current_for(client) + with self.lock: + rows = sorted((row for sid, row in self._meta.items() if client in self._clients[sid] and sid in self._said), + key=lambda r: (r['updated_at'], r['id']), reverse=True) + return [dict(id=r['id'], title=self._said[r['id']], updated_at=r['updated_at'], current=r['id'] == current) + for r in rows[:limit]] + def read_until(self, sid, reader) -> int: """The last event this reader has been given; unread() moves it.""" with self.lock: diff --git a/freevideo_engine/agent/spec.py b/freevideo_engine/agent/spec.py index 8a9608c5..8da22005 100644 --- a/freevideo_engine/agent/spec.py +++ b/freevideo_engine/agent/spec.py @@ -4,6 +4,7 @@ import re import secrets from . import schema as contract +from .length import single_frame_limit QUALITY, ASPECTS = contract.QUALITY, contract.ASPECTS KINDS = tuple(contract.KINDS) @@ -149,12 +150,13 @@ def task(media): return 'i2va' if 'first' in media else 'l2va' if 'last' in media else 't2va' -def saved_length(canvas, seconds, continued=False): - """Frames and seconds of the saved video. One generation keeps its aligned canvas; a longer video is - made in blocks and joined to exactly the planned whole seconds. A continuation saves only what it adds.""" +def saved_length(canvas, spec): + """Saved length: one generation keeps its aligned canvas; multiple blocks are joined to the planned + whole seconds. A continuation saves only what it adds.""" from ..chain_plan import PlanError, plan + continued = bool(spec.get('continue_from')) try: - planned = plan(seconds, continued=continued) + planned = plan(spec['seconds'], max_frames=single_frame_limit(spec), continued=continued) except PlanError: return canvas['frames'], canvas['seconds'] if planned['single'] and not continued: @@ -162,7 +164,7 @@ def saved_length(canvas, seconds, continued=False): return planned['frames'], planned['frames'] / canvas['fps'] -def normalize(value): +def normalize(value, *, port=None): # Local Python callers retain local media refs. HTTP validates its stricter # (content-only) schema first; the client adapts local refs before transport. contract.validate_job(value, local=True) @@ -180,9 +182,8 @@ def normalize(value): from ..geometry import geometry width, height = canvas_size(original) canvas = geometry(width, height, seconds=value['seconds']) - frames, output_seconds = saved_length(canvas, value['seconds'], bool(value.get('continue_from'))) result.update(prompt=value['prompt'], width=width, height=height, seconds=float(value['seconds']), - frames=frames, output_seconds=output_seconds, seed_random=value['seed'] is None, + seed_random=value['seed'] is None, seed=secrets.randbelow(MAX_SEED + 1) if value['seed'] is None else value['seed'], preview=value['preview'], media=generation_media(value.get('media')), **sampling(value)) result['task'] = task(result['media']) @@ -192,6 +193,9 @@ def normalize(value): result['first_frame_reference'] = True result['loras'] = [dict(name=row, strength=1.) if isinstance(row, str) else {'strength': 1., **row} for row in value['loras']] + if port is not None: + result['single_max_frames'] = single_frame_limit(result, port=port) + result['frames'], result['output_seconds'] = saved_length(canvas, result) elif kind == 'assemble': result['clips'] = [] for row in value['clips']: diff --git a/freevideo_engine/bootstrap.py b/freevideo_engine/bootstrap.py index 9243ab0a..ead734a7 100644 --- a/freevideo_engine/bootstrap.py +++ b/freevideo_engine/bootstrap.py @@ -2,7 +2,7 @@ from __future__ import annotations import argparse import csv -from contextlib import ExitStack +from contextlib import ExitStack, contextmanager import hashlib import json import os @@ -354,6 +354,18 @@ def model_target(row, model_dir, encoder_dir, prepared_dir=None): return (model_dir if row['repo'].startswith('OpenVDN/') else encoder_dir) / row['file'] +def installation_model_files(prepared, reuse_cache, sampling_caches): + """The file selection shared by planning and provisioning.""" + from . import prepared_model + from .sampling_assets import install_files, usable_with + files = list(required_models(json.loads((PACKAGE / 'model_files.json').read_text(encoding='utf-8')), + reuse_cache or prepared)) + files += prepared_model.files(prepared) + if prepared or (reuse_cache and usable_with(reuse_cache)): + files += install_files(bool(sampling_caches)) + return files + + def resolve_prepared_format(requested, saved, hardware): """The prepared model format of this setup run: 'int8_convrot' or 'fp8'. @@ -530,18 +542,13 @@ def plan(args, *, local_progress=None, stage=None): prepared_dir = prepared['directory'] if prepared else None if mac_target and reuse_cache is None and prepared is None: errors.append('No verified native-compatible prepared model is available. Source conversion is not supported on Mac.') - files = required_models(json.loads((PACKAGE / 'model_files.json').read_text(encoding='utf-8')), reuse_cache or prepared) - files += prepared_model.files(prepared) # New installations prepare every quality level; an existing one keeps its # earlier choice (off if it predates the option) unless the flag says otherwise. sampling_caches = getattr(args, 'sampling_caches', None) if sampling_caches is None: sampling_caches = prior['sampling_caches'] if isinstance(prior.get('sampling_caches'), bool) else not saved.get('ready') sampling_caches = bool(sampling_caches) - from .sampling_assets import install_files, usable_with - if prepared or (reuse_cache and usable_with(reuse_cache)): - # Published tables match these weights; other caches compute their own. - files += install_files(sampling_caches) + files = installation_model_files(prepared, reuse_cache, sampling_caches) local_reuse = None local_folder = getattr(args, 'reuse_models', None) local_manifest = getattr(args, 'reuse_models_manifest', None) @@ -559,9 +566,19 @@ def plan(args, *, local_progress=None, stage=None): if not isinstance(libraries, dict) or libraries.get('version') != 1 or not isinstance(libraries.get('roots'), list): raise ValueError('Invalid model library manifest') directories = ([str(local_folder)] if local_folder else []) + libraries['roots'] - local_reuse = scan_many(directories, needed, prior=previous_reuse, callback=local_progress) + scan_library = lambda: scan_many(directories, needed, prior=previous_reuse, callback=local_progress) else: - local_reuse = scan(local_folder, needed, prior=previous_reuse, callback=local_progress) + scan_library = lambda: scan(local_folder, needed, prior=previous_reuse, callback=local_progress) + try: + local_reuse = scan_library() + except OSError as error: + if not getattr(args, 'download_list', False): + raise + from .model_guidance import DownloadListError + original = str(error) + if error.__cause__ is not None: + original += '\n' + str(error.__cause__) + raise DownloadListError('read', original_error=original) from error local_reuse['copy_mode'] = bool(getattr(args, 'copy_existing_models', False)) local_reuse['copy_bytes'] = local_reuse['linked_bytes'] = 0 for row in needed: @@ -577,18 +594,27 @@ def plan(args, *, local_progress=None, stage=None): mismatches = model_repair.read(root) replace_models = [] reserved = set() + check_download_list = bool(getattr(args, 'download_list', False)) + stamps_path = root / 'verified-models.json' + stamps = json.loads(stamps_path.read_text(encoding='utf-8')) if check_download_list and stamps_path.is_file() else {} for row in files: path = model_target(row, model_dir, encoder_dir, prepared_dir) replacement = model_repair.replacement(path, row, mismatches, reserved) if replacement: replace_models.append(replacement) size_matches = not replacement and path.is_file() and path.stat().st_size == row['bytes'] + if size_matches and check_download_list: + # Only the explicit third-party list needs this check during planning. + # Normal setup keeps its cheap estimate and deferred verification. + from .provision import verified + hf_directory = Path(prepared_dir) if row.get('prepared') else model_dir if row['repo'].startswith('OpenVDN/') else encoder_dir + size_matches = verified(path, row, stamps, mode=getattr(args, 'verify', 'auto'), hf_directory=hf_directory) present += row['bytes'] if size_matches else 0 device = existing_parent(path.parent) entry = groups.setdefault(str(device), {'path': str(device), 'needed_bytes': 0, 'free_bytes': disk_space.free_bytes(device)}) local = (local_reuse or {}).get('matches', {}).get(row['repo'] + '/' + row['file']) - model_entries.append((row, 'found' if size_matches else 'verified' if local else 'download')) + model_entries.append((row, ('verified' if check_download_list else 'found') if size_matches else 'verified' if local else 'download')) entry['needed_bytes'] += 0 if size_matches or local and local['method'] == 'hardlink' else row['bytes'] partial = path.with_suffix(path.suffix + '.partial') if not size_matches and not local and partial.is_file() and partial.stat().st_size <= row['bytes']: @@ -717,6 +743,9 @@ def plan(args, *, local_progress=None, stage=None): 'licenses': ['https://huggingface.co/OpenVDN/vdn-minimax-h3/blob/751739ee5b9e3ac802dca5d5111075fdaeb47885/LICENSE', 'https://huggingface.co/t8star/Vdn-Minimax-H3-Comfy', 'https://docs.nvidia.com/cuda/eula/index.html']} + from .model_guidance import download_files + value['download_files'] = download_files(model_entries) + value['download_files_verified'] = all(status != 'found' for _, status in model_entries) if mac_target: from .macos_bootstrap import describe_plan describe_plan(value) @@ -998,6 +1027,8 @@ def setup_task(label): routes = {'official': 'official source', 'nju': 'Nanjing University mirror', 'tuna': 'Tsinghua mirror', 'ghfast': 'GitHub mirror', 'user-python': 'your Python mirror', 'user': 'your configured source'} + # Python route retries append the route after the source. + label = re.sub(r'-(?:direct|proxy)$', '', label) source = next((name for name in routes if label.endswith('-' + name)), None) key = label[:-(len(source) + 1)] if source else label for prefix in ('unified-', 'engine-', 'encoder-'): @@ -1098,7 +1129,8 @@ def initialize(self, value, ui): from .runtime_libraries import damaged if ui is not None: prepare_step(ui, 'Checking the environment') - self.damaged_before = damaged(self.pythons['engine']) + self.damaged_before = damaged(self.pythons['engine'], progress=( + lambda done, total: prepare_step(ui, 'Checking the environment', done, total)) if ui is not None else None) # Moving a working installation to int8 checks the int8 kernels first # (execute), so a GPU that fails them keeps its configuration as it was. # With changed files that check cannot run; the one after installing still does. @@ -1304,6 +1336,7 @@ def command(self, label, args, env=None, cwd=None): raise RuntimeError(self.state.get('resource_guard', 'Installation cancelled.')) from .triton_compat import temporary_environment env = temporary_environment(self.root, env or self.env) + env = dict(env, FREEVIDEO_INSTALL_PROGRESS='1') args, cwd = uv_file_arguments(args, cwd) with self.state_lock: index = len(self.state['steps']) @@ -1316,6 +1349,10 @@ def command(self, label, args, env=None, cwd=None): if label == 'models' and self.plan.get('local_models'): title, detail = 'Reuse models / download missing files', 'Import verified local models; source files stay in place' self.ui.begin(key, title, detail=str(log) if self.ui.verbose else detail) + initial = ('Python downloading' if title == 'Install Python' else + 'Environment creating' if title == 'Create Python environment' else '') + if initial: + self.ui.update(key, activity='note', name=initial, detail=initial) progress = LogProgress(log) parallel_tasks = set() models_step = label == 'models' @@ -1445,11 +1482,13 @@ def check(): constraints=['-c', constraints_file(label, self.system)], check=check): return from .runtime_libraries import damaged, reinstall_arguments - fresh = damaged(executable) + fresh = self.check_runtime_libraries(executable, label) result = self.packages(label + '-torch', [*argv[:5], *reinstall_arguments(fresh), *argv[5:]], group) # Files linked from a changed uv cache arrive with a new environment. - if not fresh and damaged(executable): - result = self.packages(label + '-torch', [*argv[:5], *reinstall_arguments(damaged(executable)), *argv[5:]], group) + if not fresh: + changed = self.check_runtime_libraries(executable, label) + if changed: + result = self.packages(label + '-torch', [*argv[:5], *reinstall_arguments(changed), *argv[5:]], group) return result task(name + '-torch', torch_packages, (name + '-venv',), writer) @@ -1502,6 +1541,18 @@ def encoder_packages(): tasks['models']['after'] = ('package-cache',) return tasks, pythons, comfy + def check_runtime_libraries(self, python, role): + from .runtime_libraries import damaged + key = role + '-runtime-libraries' + self.ui.begin(key, 'Runtime libraries verifying') + def progress(done, total): + self.ui.update(key, activity='count', name='Runtime libraries verifying', + detail='Runtime libraries verifying', done=done, total=total, unit='items') + try: + return damaged(python, progress=progress) + finally: + self.ui.end(key) + def release_package_cache(self, uv): cache = self.root / 'downloads' / 'uv-cache' if cache.is_symlink() or not cache.resolve().is_relative_to(self.root.resolve()): @@ -1517,10 +1568,28 @@ def clone(self, name, url, commit, sparse=None): return network.clone(self.root / 'vendor' / name, url, commit, sparse=sparse, run=self.command, network=self.network, env=self.env) + @contextmanager + def download_notice(self, key): + """Keep the patient-round notice visible across command heartbeats.""" + active = False + + def notice(message): + nonlocal active + self.ui.write(message + '\n') + active = True + self.ui.event('progress', key=key, label='Download', detail=message) + + try: + yield notice + finally: + if active: + self.ui.event('progress', key=key, label='Download', detail='', state='complete') + def packages(self, label, args, family='pypi'): - return network.package_command(self.network, family, - lambda env, source: self.command(label if source == 'official' else label + '-' + source, args, env=env), - self.env, notice=lambda message: self.ui.write(message + '\n')) + with self.download_notice('download-patient-' + label) as notice: + return network.package_command(self.network, family, + lambda env, source: self.command(label if source == 'official' else label + '-' + source, args, env=env), + self.env, notice=notice) def python_install(self, uv): from .system import bootstrap_root @@ -1541,8 +1610,10 @@ def python_install(self, uv): 'import platform; print(platform.python_version())'], capture_output=True, timeout=30) if check.returncode == 0 and check.stdout.strip() == self.versions['python']: self.env['UV_PYTHON_INSTALL_DIR'] = str(prepared) - self.command('python-bootstrap-reuse', [candidate, '-I', '-B', '-c', - 'import sys; print("Reusing verified bootstrap Python: " + sys.executable)']) + # -I ignores PYTHONIOENCODING: without -X utf8 a path the + # system code page cannot spell fails the print. + self.command('python-bootstrap-reuse', subprocess_output.python_command([candidate, '-I', '-B', '-c', + 'import sys; print("Reusing verified bootstrap Python: " + sys.executable)'])) return custom = self.env.get('UV_PYTHON_INSTALL_MIRROR') if self.network.get('mode') != 'official' else None isolation = ['--no-bin', '--no-registry'] @@ -1553,9 +1624,9 @@ def attempt(source, route, retry_mode): return self.command('python-' + source + '-' + route, [uv, 'python', 'install', *isolation, self.versions['python']], env=env) try: - return network.download_command(self.network, 'github', attempt, self.env, category='python', - extra_sources=['user-python'] if custom else [], - notice=lambda message: self.ui.write(message + '\n')) + with self.download_notice('download-patient-python') as notice: + return network.download_command(self.network, 'github', attempt, self.env, category='python', + extra_sources=['user-python'] if custom else [], notice=notice) except network.PackageSourcesError as error: failure = RuntimeError('Python download failed on every route; see retained logs') failure.package_attempts = error.attempts @@ -1831,6 +1902,7 @@ def prepare_tools(self): def execute(self): if self.int8_check_first: + prepare_end(self.ui) # The GPU precheck is already execution work. # The installed environment runs this source's probes (PYTHONPATH); # the receipt is reused by the check after installation. self.ui.phase('Verify installation on your GPU', 0, 8) @@ -2038,6 +2110,7 @@ def run_main(argv, shown): parser.add_argument('--model-downloader', choices=['auto', 'xet'], default='auto', help='xet: require official HF Xet for large weights; no ModelScope/HTTP fallback. Small configs use HTTP; existing files/partials retained') parser.add_argument('--plan', '--check', action='store_true', help='Read-only preflight with small network samples; no installation files are written') + parser.add_argument('--download-list', action='store_true', help='Verify existing managed files while planning a third-party download list') parser.add_argument('--json', action='store_true', help='Print preflight as JSON') parser.add_argument('--plain', action='store_true', help='Disable live terminal rendering; append readable log lines') parser.add_argument('--verbose', action='store_true', help='Show full setup details, license links and per-step log paths') @@ -2075,7 +2148,11 @@ def curl_progress(message): stage=None if args.plan else lambda *step: prepare_step(ui, *step)) except (OSError, RuntimeError, ValueError, subprocess.CalledProcessError) as error: prepare_end(ui, False) - print('Preflight failed: ' + str(error), file=sys.stderr) + from .model_guidance import DownloadListError + if isinstance(error, DownloadListError): + print('Download list error: ' + error.key, file=sys.stderr) + print('Preflight failed: ' + (error.original_error if isinstance(error, DownloadListError) else str(error)), + file=sys.stderr) return 1 print(json.dumps(value, indent=2)) if args.json else display(value, ui, verbose=args.verbose) if args.plan: diff --git a/freevideo_engine/comfy_assets.py b/freevideo_engine/comfy_assets.py index 51d13de7..88d142ea 100644 --- a/freevideo_engine/comfy_assets.py +++ b/freevideo_engine/comfy_assets.py @@ -4,7 +4,7 @@ import os from pathlib import Path, PurePosixPath, PureWindowsPath from . import launcher_bridge -from .monitoring import thermal_slowdown +from .monitoring import thermal_slowdown, shared_gpu_note, worker_gpu from .paths import data_root from .support_report import encoder_sizes @@ -155,7 +155,8 @@ def output_summary(report, relative_video): geometry=dict(report.get('geometry', {}), **(report.get('output_size') or {})), sampling_plan=report.get('sampling_plan'), low_memory=bool(report.get('low_memory_mode')), - thermal=thermal_slowdown(((report.get('memory') or {}).get('video') or {}).get('gpu')), + thermal=thermal_slowdown(worker_gpu(report)), + shared_gpu=shared_gpu_note(worker_gpu(report)), reference_trims=report.get('encoding', {}).get('reference_trims') or [], preview=bool(report.get('preview')), upscaled_preview=bool(report.get('upscaled_preview')), video=relative_video.as_posix(), report=relative_video.with_suffix( diff --git a/freevideo_engine/comfy_bridge.py b/freevideo_engine/comfy_bridge.py index 231064b6..f7404d67 100644 --- a/freevideo_engine/comfy_bridge.py +++ b/freevideo_engine/comfy_bridge.py @@ -11,6 +11,7 @@ import sys import time import uuid +from collections import deque from .long_paths import io_path, resolved, plain, plain_data, plain_errors from . import processes @@ -75,172 +76,729 @@ def progress_history_forecast(root, machine, canvas): return {} # A progress estimate cannot prevent generation. +class _SafeProgress: + """Any failure of the estimate falls back to plain progress messages.""" + def __init__(self, inner): + self.inner = inner + + def _call(self, name, *args, **kwargs): + try: + return getattr(self.inner, name)(*args, **kwargs) + except Exception: + self.inner = _PlainProgress() # A progress estimate cannot prevent generation. + return getattr(self.inner, name)(*args, **kwargs) + + def annotate(self, message, *, event=None): + return self._call('annotate', message, event=event) + + def observe(self, event): + return self._call('observe', event) + + def heartbeat(self): + return self._call('heartbeat') + + def configure(self, **facts): + return self._call('configure', **facts) + + +class _PlainProgress: + """Pass engine messages through unchanged when no estimate can be built.""" + def annotate(self, message, *, event=None): + return dict(message) + + def observe(self, event): + pass + + def heartbeat(self): + return None + + def configure(self, **facts): + pass + + +def _engine_version(): + from . import __version__ + return __version__ + + +# Names emitted by both decoders; forecast keys use the same suffixes. +_DECODE_PARTS = {'Loading video VAE': 'vae_load', 'Decoding the video': 'video_decode', + 'Loading and decoding audio': 'audio_load_decode', 'Saving MP4 and audio': 'encode'} + + class WholeVideoProgress: - """Annotate phase events with one report-weighted whole-video progress. + """One elapsed/remaining clock, bounded by evidence of completed work. - Local reports take precedence. Without history, live sampling supplies a - rough decoder estimate which real decoded-frame timings then replace. - These presentation estimates never change resource admission or placement. + This display estimator never changes the inference plan. Stage predictions + set the initial ruler; only real events release its count ceiling. Replay + options control smoothing, silent countdown and the ceiling's time shares. """ - def __init__(self, clock=time.perf_counter): - self.clock = clock - self.started = clock() - self.phase_started = self.started - self.phase = None - self.completed = set() - self.weights = {} + DECODE_PARTS = tuple(_DECODE_PARTS.values()) + + def __init__(self, clock=time.perf_counter, *, hardware=None, request=None, + cold_flags=None, records=(), rate=.005, smoothing='track', + silence='stage', ceiling='stretch', tau=10., max_rise=.019, final=.98, slow_start=True, started=None): + from . import stage_estimate + from .effort_forecast import stage_forecast + if not math.isfinite(rate) or rate < 0: + raise ValueError('Progress smoothing rate must be finite and nonnegative') + for name, value, choices in (('smoothing', smoothing, ('countdown', 'hold', 'track')), + ('silence', silence, ('freeze', 'stage')), + ('ceiling', ceiling, ('prior', 'current', 'stretch'))): + if value not in choices: + raise ValueError(f'Invalid progress {name}: {value!r}') + self.clock, self.rate = clock, rate + self.smoothing, self.silence, self.ceiling = smoothing, silence, ceiling + # tau: time constant of a downward correction ('track'); max_rise: the + # most the shown fraction may rise in any one second (None: no cap). + self.tau, self.max_rise = tau, max_rise + # final: once every frame is decoded, the bar approaches this fraction + # over the predicted audio and save time (None: no approach). + self.final = final + # slow_start: on a 16 GB machine, a load much slower than predicted + # scales the later stages until history or measured speed takes over. + self.slow_start = slow_start + self.started = self.phase_started = self.last_tick = self.last_event = clock() if started is None else started + self.last_publish = self.started + self.rise_window = deque() + self.hardware = dict(hardware or {}) + self.request = dict(width=1344, height=768, frames=243, base_steps=8, + refine_steps=3, two_pass=True, flow='full', task='t2va') + self.request.update({k: v for k, v in (request or {}).items() if v is not None}) + self.cold_flags = dict(cold_flags or {}) + self.records = records + self.params = stage_estimate._params(None) + self.prior = stage_estimate.predict_prior(self.hardware, self.request, self.cold_flags, self.params) + self.weights = {k: self.prior['stages'][k]['seconds'] for k in stage_estimate.STAGES} + local = stage_forecast(records, self.request, self.cold_flags, self.weights, version=_engine_version()) + self.remaining_source = 'local' if local.get('stages') else 'similar' + self.typical = local['typical'] + self.local = local + self.weights.update(local.get('stages', {})) + self.completed = {k for k, v in self.weights.items() if v == 0} self.measured = {} - self.total = 0. - self.forecast_ready = False - self.failed = False + self.spent = {} + self.forecasts = dict(self.weights) + self.phase_counts = {} + self.reused = set() + self.phase = 'encode' + self.completed.discard(self.phase) self.sampling_plan = {} - self.decode_from_sampling = False - self.sampling_time_weighted = False - self.decode_tail = 0. + self.steps = {'pass1': {}, 'pass2': {}} + self.live_calibration = True + self.count = None + self.fraction = 0. + self.ceiling_base = 0. + self.last_ceiling = 0. + self.ceiling_weights = dict(self.weights) + self.remaining_target = self.remaining_shown = sum(self.weights.values()) + self.total = self.remaining_target + self.last_message = {} + self.finished = False + self.decode_remaining = None + self.decode_parts = {} + self.decode_part = None + self.decode_rest = None + self.decode_tail = max(0., self.weights['decode_save'] - self.prior['stages']['video_decode']['seconds']) + self.retry_decode_only = False + self.retry_reuse_first = False + self.history = None # (output directory, machine, opened): read late by _late_history. + self.shown_source = self.remaining_source - def _sampling_estimate(self, message, now): - estimate = _timing_seconds(message.get('estimated_step_seconds')) - done, total = message.get('done'), message.get('total') - if estimate is None or type(done) is not int or type(total) is not int or not 0 <= done <= total: + @classmethod + def for_request(cls, machine, canvas, output_directory, *, base_steps=8, + refine_steps=3, two_pass=True, preview=False, upscale=False, + encoder_prewarm=None, options=None): + import platform + from .effort_forecast import start_context + started, opened = time.perf_counter(), time.monotonic() + observed, records = start_context(output_directory, machine) + hardware = cls._hardware(observed, machine) + # machine.vram_gib/ram_gib can be user policy caps, not physical totals. + # A worker process being alive is not proof that a model is resident. + warm = encoder_prewarm or {} + flags = {'encode_cached': True if upscale else None} + if warm.get('gpu_preload') is True and warm.get('state') in ('ready', 'complete') and not warm.get('worker_released'): + flags['encoder_cold'] = False + tracker = cls(hardware=hardware, records=records, cold_flags=flags, started=started, + request=dict(canvas, base_steps=base_steps, refine_steps=refine_steps, + two_pass=two_pass, flow='upscale' if upscale else 'preview' if preview else 'full'), + **(options or {})) + tracker.history = (output_directory, machine, opened) + return tracker + + @staticmethod + def _hardware(observed, machine): + import platform + observed = dict(observed) + if isinstance(machine.get('hardware'), dict): + observed.update(machine['hardware']) + device = machine.get('device_identity') + device = device if isinstance(device, dict) else {} + hardware = dict(gpu_name=observed.get('gpu_name') or machine.get('gpu_name') or device.get('name'), + compute_capability=observed.get('capability'), + os=observed.get('system') or machine.get('system') or platform.system()) + for target, source in (('ram_total_gib', 'ram_total'), ('vram_total_gib', 'vram_total')): + value = observed.get(source) or device.get('unified_ram_bytes') + if _timing_seconds(value): + hardware[target] = value / 2**30 + return hardware + + def _late_history(self): + """Use history the background read finished after this request began.""" + from .effort_forecast import start_context, history_read_at + if self.history is None or self.remaining_source == 'measured': return - # Within a pass's first step, a layer event extrapolates the whole step - # from the first layers, which can include compilation and read-ahead - # (one slow layer once became a 142-minute estimate). With an estimate - # already in hand, use layer-based step times only after the pass has - # completed a step; a first run without one still gets its first number. - base = self.sampling_plan.get('base_steps') - if (self.forecast_ready and message.get('stage') == 'layers' - and (done == 0 or type(base) is int and 0 < base == done < total)): + output_directory, machine, opened = self.history + if (history_read_at(output_directory, machine) or 0.) < opened: return - full_step = estimate - remaining = (total - done) * estimate - if message.get('uniform_remaining_steps') is False: - plan = self.sampling_plan - first, second = plan.get('first', {}), plan.get('second') or {} - area = first.get('width', 0) * first.get('height', 0) - if not area or not second.get('width') or not second.get('height'): - return # Do not treat the two larger refinement NFEs as small NFEs. - full_step *= second['width'] * second['height'] / area - remaining = max(0, plan['base_steps'] - done) * estimate + plan['refine_steps'] * full_step - remaining = max(0., remaining - (_timing_seconds(message.get('step_elapsed_seconds')) or 0.)) - self.weights['sampling'] = max(.001, now - self.phase_started + remaining) - self.sampling_time_weighted = True - if 'decode' not in self.weights or self.decode_from_sampling: - # Windows 1344x768x243 measurements: decode/save was 1.71, 1.90 - # and 2.01 final-resolution warm NFEs on 4060 Ti / 5060 Ti / 5090. - # This is an approximate first-run ruler, not a GPU speed claim. - # It scales with this request's work, then yields to frame timing. - self.weights['decode'] = 1.9 * full_step - self.decode_from_sampling = True - self.forecast_ready = True + self.history = None # One late update per request. + # Work on the live state, but restore all of it if anything fails. + saved = {key: dict(value) if isinstance(value, dict) else value for key, value in vars(self).items()} + try: + observed, rows = start_context(output_directory, machine, timeout=0.) + for key, value in self._hardware(observed, machine).items(): + if value is not None and self.hardware.get(key) is None: + self.hardware[key] = value + self.records = rows + self._reforecast() + self.remaining_target = self._target(self.clock()) + except Exception: + # Damaged history keeps the estimate the request already has. + vars(self).clear() + vars(self).update(saved) - @staticmethod - def _forecast_weights(forecast): - stages = forecast.get('stages') if isinstance(forecast, dict) else None + def configure(self, *, plan=None, media=None, conditioning=False, placement=None, sampled=None): + """Use facts already obtained by the bridge; never inspect media/models.""" + self._late_history() + from .media_request import task_for + if sampled: + # The engine samples this canvas, which can differ from the requested size. + self.request.update({k: sampled[k] for k in ('width', 'height', 'frames') if sampled.get(k) is not None}) + if plan: + self.sampling_plan = plan + self.request.update(two_pass=bool(plan.get('enabled')), base_steps=plan.get('base_steps', 8), + refine_steps=plan.get('refine_steps', 0), + sampling_geometry={k: plan[k] for k in ('first', 'second') if plan.get(k)}) + for key in ('pass1_resident_blocks', 'pass2_resident_blocks'): + value = plan.get(key) + if type(value) in (int, float) and math.isfinite(value) and 0 <= value <= 50: + self.request[key] = value + if placement: + # generate.py's video_start/decode_resume carry the selected full + # canvas engine/decoder plan. Pass 1 can admit extra temporary + # cache, so its residency stays unknown for a two-pass request. + engine, decoder = placement.get('engine'), placement.get('decoder') + engine = engine if isinstance(engine, dict) else {} + decoder = decoder if isinstance(decoder, dict) else {} + blocks = engine.get('resident_blocks') + if type(blocks) in (int, float) and math.isfinite(blocks) and 0 <= blocks <= 50: + key = 'pass2_resident_blocks' if self.request['two_pass'] else 'pass1_resident_blocks' + self.request[key] = blocks + blocks = decoder.get('resident_blocks') + if type(blocks) in (int, float) and math.isfinite(blocks) and 0 <= blocks <= 36: + self.request['decoder_resident_blocks'] = blocks + if type(decoder.get('offload')) is bool: + self.request['decoder_offload'] = decoder['offload'] + if media is not None: + self.request.update(task=task_for(media), lora_count=len(media.get('loras') or []), + reference_count=len(media.get('references') or []) + + sum(bool(media.get(k)) for k in ('first', 'last'))) + if conditioning: + self.cold_flags['encode_cached'] = True + self._reforecast() + + def _reforecast(self, *, reset_source=False): + from .stage_estimate import predict_prior, STAGES + from .effort_forecast import stage_forecast + self.prior = predict_prior(self.hardware, self.request, self.cold_flags, self.params) + weights = {k: self.prior['stages'][k]['seconds'] for k in STAGES} + local = stage_forecast(self.records, self.request, self.cold_flags, weights, version=_engine_version()) + weights.update(local.get('stages', {})) + self.typical = local['typical'] + self.local = local + if reset_source or self.remaining_source != 'measured': + self.remaining_source = 'local' if local.get('stages') else 'similar' + for key, value in weights.items(): + if key not in self.completed: + self.weights[key] = value + self.forecasts[key] = value + self.decode_tail = max(0., self.weights['decode_save'] - self.prior['stages']['video_decode']['seconds']) + + def forecast(self, value): + """Consume older worker forecasts without closing or crediting a phase.""" + value = value.get('forecast', value) if isinstance(value, dict) else {} + if not isinstance(value, dict): + return + stages = value.get('stages') or {} if not isinstance(stages, dict): - return {} - values = { - 'encoding': _timing_seconds(stages.get('text_encode_seconds')), - 'load': _timing_seconds(stages.get('load_seconds')), - 'sampling': _timing_seconds(stages.get('sample_seconds')), - 'decode': _timing_seconds(stages.get('decode_save_seconds')), - } - values = {key: value for key, value in values.items() if value is not None} - # Sparse phase forecasts expose work_seconds plus its independently - # predicted components. Keep the measured residual as the final - # save/cleanup tail instead of silently dropping it or double-counting - # the worker total. + return + # Decoder substeps remain useful after sampling speed is measured. + for part in self.DECODE_PARTS: + seconds = _timing_seconds(stages.get(part + '_seconds')) + if seconds is not None: + self.decode_parts[part] = seconds + # Worker predictions lack the bridge's two-record similarity evidence; + # they must never create a local typical or replace live measurements. + if self.remaining_source == 'measured': + return + mapping = {'encode': 'text_encode_seconds', 'load': 'load_seconds', 'decode_save': 'decode_save_seconds'} + for key, source in mapping.items(): + seconds = _timing_seconds(stages.get(source)) + if seconds is not None and key not in self.completed: + self.weights[key] = seconds + sample = _timing_seconds(stages.get('sample_seconds')) + if sample: + keys = [k for k in ('pass1', 'upscale', 'pass2') if k not in self.completed] + total = sum(self.weights[k] for k in keys) + if total: + for key in keys: + self.weights[key] *= sample / total work = _timing_seconds(stages.get('work_seconds')) - residual = work - sum(values.get(key, 0.) for key in ('load', 'sampling', 'decode')) if work is not None else 0. - if residual > 0: - values['decode'] = values.get('decode', 0.) + residual - return values + parts = [_timing_seconds(stages.get(k)) for k in ('load_seconds', 'sample_seconds', 'decode_save_seconds')] + if work and all(v is not None for v in parts) and 'decode_save' not in self.completed: + self.weights['decode_save'] += max(0., work - sum(parts)) + self.forecasts.update({k: v for k, v in self.weights.items() if k not in self.completed}) - def forecast(self, value): - value = value.get('forecast') if isinstance(value, dict) and 'forecast' in value else value - weights = self._forecast_weights(value) - if weights: - # The prediction is emitted after prompt encoding has completed. - # Close that phase before the first video-model event so its - # measured weight is credited instead of starting a second timer. - if self.phase == 'encoding': - self.measured['encoding'] = max(.001, self.clock() - self.phase_started) - self.completed.add('encoding') - self.phase = None - self.weights.update(weights) - if 'decode' in weights: - self.decode_from_sampling = False - self.weights.update(self.measured) - self.total = sum(self.weights.values()) - self.forecast_ready = all(name in self.weights for name in ('sampling', 'decode')) - - def _base(self): - return sum(self.weights.get(name, 0.) for name in self.completed) - - def _message_phase(self, message): - if message.get('timing_phase') in ('encoding', 'load', 'sampling', 'decode'): - return message['timing_phase'] - phase = message.get('phase') - return {'sampling': 'sampling', 'sample_finalize': 'decode', - 'complete': 'decode'}.get(phase) - - def annotate(self, message): - message = dict(message) - if isinstance(message.get('sampling_plan'), dict): - self.sampling_plan = message['sampling_plan'] - if isinstance(message.get('prediction'), dict): - self.forecast(message['prediction']) - retry = message.get('retry') if isinstance(message.get('retry'), dict) else {} - if retry.get('reuse_sampling') is True: - self.completed.add('sampling') - phase = self._message_phase(message) - now = self.clock() - if phase and phase != self.phase: - if self.phase is not None: - self.completed.add(self.phase) - self.measured[self.phase] = max(.001, now - self.phase_started) - self.weights[self.phase] = self.measured[self.phase] - self.phase, self.phase_started = phase, now + def _enter(self, phase, now): + from .stage_estimate import STAGES + if phase == self.phase: + return + self._late_history() + if phase == 'load' and self.phase == 'upscale': + # The model handed over between the passes belongs to the gap + # between them, which is a stage of its own; a retry reloads + # through _retry instead. + return + if self.phase is not None: + self.spent[self.phase] = max(0., now - self.phase_started) + self.phase_counts[self.phase] = self.count + self.forecasts[self.phase] = self.weights[self.phase] + # A brief model check can interrupt encoding, and loading can return + # between passes. Only predecessors of the active phase are finished. + self.completed = (set(STAGES[:STAGES.index(phase)]) | self.reused) - {phase} + for key in STAGES: + if key in self.completed and key in self.spent: + self.measured[key] = self.weights[key] = max(.001, self.spent[key]) + elif key in self.measured: + self.measured.pop(key) + self.weights[key] = max(self.forecasts.get(key, 0.), self.spent.get(key, 0.)) + self.phase, self.phase_started = phase, now - self.spent.get(phase, 0.) + self.count = self.phase_counts.get(phase, self._initial_count()) + self.decode_remaining = None + + def _initial_count(self): + from .stage_estimate import canvases, step_counts + if self.phase == 'load': + return (0, 50, 1) + if self.phase in ('pass1', 'pass2'): + count = step_counts(self.request)[0 if self.phase == 'pass1' else 1] + return (0, count * 50, 1) + if self.phase == 'decode_save': + return (0, canvases(self.request)[2]['frames'], 1) + return None + + def _set_count(self, done, total, margin=1): + # Duplicate/delayed logs cannot undo already witnessed counts. Using + # the smallest known unit avoids lowering the ceiling on the first + # counted event after an uncounted stage-start notification. + candidate = (max(0, done), total, margin) + old = self.count + if old is None or (candidate[0] + margin) / max(total, 1) >= (old[0] + old[2]) / max(old[1], 1): + self.count = candidate + + def _phase_for(self, message, event): + name = event.get('event') + if self.retry_decode_only: + return 'decode_save' + if name == 'decode_resume' or message.get('phase') == 'sample_finalize': + return 'decode_save' + if message.get('stage') in ('latent_upscale', 'first_pass_reused'): + return 'upscale' + phase = message.get('timing_phase') or message.get('phase') + if phase == 'encoding' and any(self.steps.values()): + # After synchronized sampling steps, encoder-file lines are late + # notifications from the finished prompt, not a new encoding run. + return None if phase == 'sampling': - self._sampling_estimate(message, now) - if phase == 'decode' and message.get('stage') == 'video': - done, total = message.get('done'), message.get('total') + # `loaded` falsely looks like sampling 0/N in the legacy converter. + if name == 'loaded': + return 'load' + done = message.get('done', 0) + base = self.request['base_steps'] + if self.request['flow'] == 'upscale' or self.retry_reuse_first: + return 'pass2' + if self.request['flow'] != 'preview' and self.request['two_pass']: + if done > base or (done == base and (name == 'sampling_start' or message.get('stage') in ('layers', 'inputs'))): + return 'pass2' + return 'pass1' + return {'encoding': 'encode', 'load': 'load', 'decode': 'decode_save'}.get(phase) if isinstance(phase, str) else None + + def _retry(self, retry, now): + # A retry changes the plan; the earlier load no longer says how fast + # this machine is, and measured steps arrive again soon. + self.slow_start = False + self.retry_decode_only = retry.get('reuse_sampling') is True + self.retry_reuse_first = retry.get('reuse_first_pass') is True + start = 'decode_save' if self.retry_decode_only else 'load' + if not self.retry_decode_only: + self.steps['pass2'].clear() + if not retry.get('reuse_first_pass'): + self.steps['pass1'].clear() + from .stage_estimate import STAGES + self.spent = {k: v for k, v in self.spent.items() if STAGES.index(k) < STAGES.index(start)} + self.phase_counts = {} + self.decode_part, self.decode_rest = None, None + self.reused = set(STAGES[:STAGES.index(start)]) + self.completed = set(self.reused) + if self.retry_reuse_first and not self.retry_decode_only: + self.completed.add('pass1') + self.reused.add('pass1') + self.phase, self.phase_started = start, now + self.count, self.decode_remaining = self._initial_count(), None + self.live_calibration = False + # Recovery can change placement. Wait for the new attempt's plan; + # retained step durations are not new measurements of pending work. + for key in ('pass1_resident_blocks', 'pass2_resident_blocks', + 'decoder_resident_blocks', 'decoder_offload'): + self.request.pop(key, None) + self._reforecast(reset_source=True) + self.ceiling_base = self.fraction + self.ceiling_weights = {k: v for k, v in self.weights.items() if k not in self.completed} + + def _counts_and_live(self, message, event, now): + from .stage_estimate import predict_live, step_counts + phase = self.phase + done, total = message.get('done'), message.get('total') + if phase in ('pass1', 'pass2'): + index = 0 if phase == 'pass1' else 1 + count = step_counts(self.request)[index] + offset = self.request['base_steps'] if index else 0 + if type(done) is int: + local_done = max(0, min(count, done - offset)) + blocks, block = message.get('blocks'), message.get('block') + if type(blocks) is int and blocks > 0 and type(block) is int: + # One step of margin: layer events are dispatch estimates, + # steps are the synchronized counts (a first step may compile). + self._set_count(local_done * blocks + max(0, block - 1), count * blocks, + blocks if self.ceiling == 'stretch' else 1) + else: + # Keep the same one-layer margin across step boundaries; + # changing from one step to one layer could lower a ceiling. + self._set_count(local_done * 50, count * 50, 50 if self.ceiling == 'stretch' else 1) + seconds = _timing_seconds(message.get('last_step_seconds')) + if seconds and local_done > 0: + self.steps[phase][local_done] = seconds + if 1 in self.steps[phase] and 2 in self.steps[phase]: + self.live_calibration = True + elif phase == 'load': + if event.get('event') == 'loaded': + done, total = 50, 50 + elif event.get('event') == 'prepared_blocks': + done, total = event.get('blocks'), 50 + elif event.get('event') == 'model_load_phase': + done, total = event.get('done'), event.get('total') + # LoRA groups describe a substage, not completion of the whole load. + if type(done) is int and type(total) is int and total > 0 and message.get('unit') != 'bytes': + self._set_count(done, total) + elif phase == 'decode_save' and message.get('stage') == 'video': elapsed = _timing_seconds(message.get('elapsed_seconds')) - if elapsed and type(done) is int and type(total) is int and 0 < done <= total: - video_seconds = elapsed * total / done - # Reports put audio/MP4/post-decode cleanup at about 6-8% of - # video decode time. Retain this tail until audio/save starts. - self.decode_tail = video_seconds * .08 - self.weights['decode'] = now - self.phase_started + video_seconds - elapsed + self.decode_tail - self.forecast_ready = True - elif phase == 'decode': - self.decode_tail = 0. - self.total = sum(self.weights.values()) - if message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete': - message['overall'] = {'status': 'complete', 'fraction': 1., 'estimated': False, - 'elapsed_seconds': now - self.started, 'remaining_seconds': 0.} - return message - overall = {'status': 'running', 'estimated': self.forecast_ready, - 'elapsed_seconds': now - self.started} - if self.forecast_ready and self.total > 0: - start = self._base() / self.total - weight = self.weights.get(self.phase, 0.) + if type(done) is int and type(total) is int and 0 <= done <= total and total > 0: + self._set_count(done, total) + if elapsed and done > 0: + self.decode_remaining = elapsed * (total - done) / done + self._after_frames(total) + self.weights[phase] = now - self.phase_started + self.decode_remaining + self.remaining_source = 'measured' + elif phase == 'decode_save' and (event.get('event') == 'decode_phase' or message.get('decode_part')): + # These real transitions finish video frames even for a decoder + # that does not stream progress callbacks. + part = message.get('decode_part') or _DECODE_PARTS.get(event.get('phase') if isinstance(event.get('phase'), str) else '') + if part in self.DECODE_PARTS and part != self.decode_part: + rest = self.DECODE_PARTS[self.DECODE_PARTS.index(part):] + self.decode_part = part + if all(k in self.decode_parts for k in rest): + self.decode_rest = sum(self.decode_parts[k] for k in rest[1:]) + self.decode_remaining = self.decode_parts[part] + self.decode_rest + self.weights[phase] = now - self.phase_started + self.decode_remaining + if part in ('audio_load_decode', 'encode'): + total = self.count[1] if self.count else self.request['frames'] + self._set_count(total, total) + if self.decode_remaining is None: + # Without frame callbacks this is the first sign that the + # video is decoded; only the measured tail remains. + self.decode_remaining = self._after_frames(total) + self.weights[phase] = now - self.phase_started + self.decode_remaining + self.remaining_source = 'measured' + # Only contiguous, actually completed steps calibrate the model. + steps = {} + for key, values in self.steps.items(): + steps[key] = [] + for i in range(1, len(values) + 1): + if i not in values: + break + steps[key].append(values[i]) + if self.live_calibration and any(len(v) >= 2 for v in steps.values()): + done_stages = [('decode' if k == 'decode_save' else k) for k in self.completed] + live = predict_live(dict(hardware=self.hardware, request=self.request, cold_flags=self.cold_flags, + pass1_steps=steps['pass1'], pass2_steps=steps['pass2'], completed_stages=done_stages, + current_stage=phase, step_elapsed_seconds=message.get('step_elapsed_seconds', 0.), + pass_wall_overhead_done=[k for k, v in steps.items() if v]), self.params) + # With matching history on this machine, scale its own stage times + # by the measured warm step instead of the general model. + from statistics import median + history, warm, ratio = self.local.get('stages') or {}, self.local.get('warm') or {}, None + for key in ('pass2', 'pass1'): + if len(steps[key]) >= 2 and warm.get(key): + ratio = min(2., max(.5, median(steps[key][1:]) / warm[key])) + break + for key in ('pass1', 'upscale', 'pass2', 'decode_save'): + if key in self.completed or (key == 'decode_save' and self.decode_remaining is not None): + continue + if ratio is not None and key in history: + self.weights[key] = history[key] * (1. if key == 'decode_save' else ratio) + continue + remaining = live['stages'][key]['seconds'] + if key == phase: + if key == 'decode_save': + # decode/save prediction is a full duration until frames arrive. + self.weights[key] = remaining + else: + self.weights[key] = now - self.phase_started + remaining + else: + self.weights[key] = remaining + self.decode_tail = max(0., live['stages']['decode_save']['seconds'] - live['stages']['video_decode']['seconds']) + self.remaining_source = 'measured' + + def _after_frames(self, frames): + # Audio, MP4 and saving after the last decoded frame: about 2 s plus + # 0.03 s per frame (measured, slightly below the median, so the bar + # rarely jumps at saving), never more than the model's tail. + if self.decode_rest is not None and self.decode_part == 'video_decode': + return self.decode_rest + return min(self.decode_tail, 2. + .03 * frames) + + def _slow_factor(self, now): + # A disk-bound 16 GB machine that loads much slower than predicted is + # slower in every later stage too (reports: about the square root of + # the load ratio). Larger memory or a slow encode does not predict it. + if not self.slow_start or self.remaining_source != 'similar': + return 1. + ram = self.hardware.get('ram_total_gib') + if ram is None or ram > 16.5: + return 1. + predicted = self.prior['stages']['load']['seconds'] + actual = self.measured.get('load') or (now - self.phase_started if self.phase == 'load' else None) + if not actual or predicted <= 1 or actual / predicted <= 2: + return 1. + return min(4., math.sqrt(actual / predicted)) + + def _target(self, now): + factor = self._slow_factor(now) + later = ('pass1', 'upscale', 'pass2', 'decode_save') + remaining = sum(v * (factor if k in later else 1.) + for k, v in self.weights.items() if k not in self.completed and k != self.phase) + elapsed = max(0., now - self.phase_started) + # The stage under way is scaled too until its speed is measured. + current = self.weights.get(self.phase, 0.) * (factor if self.phase in later else 1.) - elapsed + if current < 0: + current = max(3., .1 * elapsed) + return max(0., remaining + current) + + def _ceiling(self, now): + weights = (self.ceiling_weights if self.ceiling == 'prior' else + {k: self.weights[k] for k in self.ceiling_weights}) + stretch = self.ceiling == 'stretch' and (self.count is None or self.count[0] <= 0) + if stretch and self.phase in weights: + # An overrunning stage without counted work keeps its end ahead of + # it, by the same remainder the remaining-time target assumes. elapsed = max(0., now - self.phase_started) - seconds = self.weights.get(self.phase) - fraction = start - if weight: - fraction += weight / self.total * min(.95, elapsed / max(seconds, 1e-9)) - remaining = max(0., self.total - self._base() - min(elapsed, seconds or 0.)) - future = sum(value for name, value in self.weights.items() - if name not in self.completed and name != self.phase) - overall.update(fraction=fraction, phase_start_fraction=start, - phase_weight=weight / self.total if self.total else 0., - phase_elapsed_seconds=elapsed, phase_seconds=seconds, - remaining_seconds=remaining, - remaining_floor_seconds=max(future, self.decode_tail), - sampling_time_weighted=self.sampling_time_weighted) - message['overall'] = overall - return message + weights[self.phase] = max(weights[self.phase], elapsed + max(3., .1 * elapsed)) + total = sum(weights.values()) + if total <= 0: + if self.ceiling in ('current', 'stretch'): + self.last_ceiling = .99 + return .99 + credit = sum(v for k, v in weights.items() if k in self.completed) + # An uncounted stage earns its allowance over time, not at its entry. + # This also avoids releasing all of upscale's share at a boundary. + duration = weights.get(self.phase, 0.) + fraction = min(1., max(0., now - self.phase_started) / max(duration, .001)) + if self.count is not None and not stretch: + done, count, margin = self.count + fraction = min(1., max(0., (done + margin) / max(count, 1))) + credit += duration * fraction + value = min(.99, self.ceiling_base + (1. - self.ceiling_base) * credit / total) + if self.ceiling in ('current', 'stretch'): + self.last_ceiling = max(self.last_ceiling, value) + value = self.last_ceiling + return value + + def _smooth(self, dt): + if self.smoothing == 'track': + shown, target = self.remaining_shown, self.remaining_target + if target > shown: + # Slower than predicted: stop counting down, then rise gently. + # A time that ran out starts again from the remainder an + # overrunning stage keeps; from zero it could never rise. + shown = max(shown, min(target, 3.)) + self.remaining_shown = min(target, shown + self.rate * shown * dt) + else: + shown = max(target, shown - dt) + self.remaining_shown = max(0., shown - (shown - target) * (1. - math.exp(-dt / max(self.tau, .001)))) + return + # Hold suppresses countdown only for an upward target correction. + # Both policies approach the target without crossing it by correction; + # countdown itself can pass below it. Keep a positive running remainder. + if self.smoothing == 'hold' and self.remaining_target > self.remaining_shown: + shown = self.remaining_shown + else: + floor = 3. if self.smoothing == 'countdown' else min(3., self.remaining_shown) + shown = max(floor, self.remaining_shown - dt) + limit = self.rate * shown * dt + self.remaining_shown = max(0., shown + max(-limit, min(limit, self.remaining_target - shown))) + + def _render(self, message, now, *, heartbeat=False): + if self.remaining_source == 'local' and self.shown_source == 'similar': + # History read after the start replaces the estimate from similar + # settings: this computer's record corrects it once, so do not smooth. + # (A first measurement keeps smoothing; two steps are still noisy.) + self.remaining_shown = self.remaining_target + self.shown_source = self.remaining_source + elapsed = max(0., now - self.started) + candidate = elapsed / max(elapsed + self.remaining_shown, .001) + if (self.final is not None and self.phase == 'decode_save' and self.count + and self.count[0] >= self.count[1] > 0): + # Every frame is decoded; only audio, MP4 and saving remain, often + # for a few seconds only. Move there as fast as the rise limit allows. + candidate = max(candidate, self.final) + ceiling = self._ceiling(now) + limit = .99 + # Retain the value in force one second ago and all newer changes. + while len(self.rise_window) > 1 and self.rise_window[1][0] <= now - 1.: + self.rise_window.popleft() + if self.max_rise is not None: + base = self.rise_window[0][1] if self.rise_window else self.fraction + limit = min(limit, base + self.max_rise) + self.fraction = max(self.fraction, min(limit, candidate, ceiling)) + if not self.rise_window or self.rise_window[-1][1] != self.fraction: + if len(self.rise_window) > 1 and self.rise_window[-1][0] == now: + self.rise_window.pop() + self.rise_window.append((now, self.fraction)) + overall = dict(status='complete' if self.finished else 'running', estimated=True, + fraction=1. if self.finished else self.fraction, + remaining_seconds=0. if self.finished else self.remaining_shown, + elapsed_seconds=elapsed, remaining_source=self.remaining_source, + phase_start_fraction=self.fraction, phase_weight=0., remaining_floor_seconds=0., + at_ceiling=not self.finished and candidate >= ceiling, + sampling_time_weighted=True) + key = self.phase + if key in self.steps: + key += '_first' if not self.steps[key] else '_warm' + if key in self.typical: + overall.update(typical_seconds=self.typical[key], typical_source='local') + if heartbeat: + overall['heartbeat'] = True + self.last_publish = now + return dict(message, overall=overall) + + def annotate(self, message, *, event=None): + message, event = dict(message), event or {} + now = self.clock() + if self.finished: + return self._render(message, now) + dt = max(0., now - self.last_tick) + old_phase, old_part = self.phase, self.decode_part + waiting_for_pass = (self.phase in self.steps and self.count is not None + and self.count[0] >= self.count[1] > 0) + plan = event if event.get('event') == 'sampling_plan' else message.get('sampling_plan') + if isinstance(plan, dict): + self.configure(plan=plan) + if isinstance(message.get('prediction'), dict): + self.forecast(message['prediction']) + retried = isinstance(message.get('retry'), dict) or message.get('stage') == 'preview_mismatch' + if isinstance(message.get('retry'), dict): + self._retry(message['retry'], now) + if message.get('stage') == 'preview_mismatch': + self.request['flow'] = 'full' + self._retry({}, now) + if event.get('event') in ('video_start', 'decode_resume'): + self.configure(placement=event) + phase = self._phase_for(message, event) + if phase: + self._enter(phase, now) + if phase in self.steps and phase != old_phase and phase not in self.spent: + # A first layer reports time inside its step, not the model + # handover before it. Backdate only that witnessed work. + inside = _timing_seconds(message.get('step_elapsed_seconds')) or 0. + self.phase_started = now - inside + if waiting_for_pass: + dt = min(dt, inside) + if event.get('event') == 'conditioning_cache_hit': + self.cold_flags['encode_cached'] = True + if event.get('event') == 'conditioning_reused': + self.cold_flags['encode_cached'] = True + if event.get('event') == 'encoder_device_reuse': + self.cold_flags['encoder_cold'] = False + if event.get('event') == 'compute_device': + self.hardware.update(gpu_name=event.get('name') or self.hardware.get('gpu_name')) + if _timing_seconds(event.get('vram_total_bytes')): + self.hardware['vram_total_gib'] = event['vram_total_bytes'] / 2**30 + self._reforecast() + if not retried: + self._counts_and_live(message, event, now) + self.remaining_target = self._target(now) + self._smooth(dt) + if retried: + # A new attempt is estimated afresh: the time left of the attempt + # that failed says nothing about it, and may already have run out. + self.remaining_shown = self.remaining_target + if self.phase == 'decode_save' and self.decode_rest is not None and ( + self.decode_part != old_part or message.get('stage') == 'video'): + # A decoder boundary gives the pending substeps directly; the + # fraction still obeys its monotonic, per-second rise limit. + self.remaining_shown = self.remaining_target + self.last_tick = self.last_event = now + self.total = sum(self.weights.values()) + self.finished = message.get('phase') == 'complete' or message.get('overall', {}).get('status') == 'complete' + if self.finished: + self.measured[self.phase] = max(.001, now - self.phase_started) + self.weights[self.phase] = self.measured[self.phase] + self.total = sum(self.weights.values()) + self.completed.update(self.weights) + self.remaining_target = self.remaining_shown = 0. + self.fraction = 1. + if message.keys() - {'report_id'}: + self.last_message = {k: v for k, v in message.items() if k not in ('overall', 'reset', 'new_request', 'prediction', 'sampling_plan')} + else: + message = dict(self.last_message, **message) + return self._render(message, now) + + def observe(self, event): + """Use an engine event that has no progress message, without publishing.""" + published, window = self.last_publish, self.rise_window.copy() + self.annotate({}, event=event) + self.last_publish, self.rise_window = published, window + + def heartbeat(self): + now = self.clock() + if self.finished or now - self.last_publish < 1.: + return None + if (self.phase in self.steps and self.count is not None + and self.count[0] >= self.count[1] > 0): + # Between passes no pending step time is being spent. + self.last_tick = now + elif self.silence == 'stage': + # Admit only the part of this interval inside the predicted stage. + # Crossing its end freezes at the boundary, even with sparse polls. + end = self.phase_started + self.weights.get(self.phase, 0.) + dt = max(0., min(now, end) - max(self.last_tick, self.phase_started)) + if dt: + self.remaining_target = max(0., self.remaining_target - dt) + self._smooth(dt) + self.last_tick = max(self.last_tick, min(now, end)) + # Silent time is not counted down until the next engine event shows + # the engine is still working; that event then counts all of it. + return self._render(self.last_message, now, heartbeat=True) def source_root(): @@ -624,16 +1182,35 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, canvas = validate_request(prompt, width, height, seconds, seed, continue_from=continue_from, frames=frames) source = Path(source or source_root()).resolve() root, machine = installation(source, environ) + continuation = None + + def check_ending(): + try: + from .continuation_tail import check + except ImportError: + return + from .two_pass import sampling_canvas + check(continue_from, sampling_canvas(canvas)[0]) + if continue_from is not None: validate_route(backend=machine.get('device_backend', 'cuda')) - from .continuation import prepare, summary + from .continuation import MISSING, prepare, summary from .two_pass import sampling_canvas # A continuation takes no media or preview, so its sampled size is known here; the snapshot # of the previous segment was written for that canvas, not for the requested size. - continuation = prepare(continue_from, sampling_canvas(canvas)[0], directory=root / 'continuations') + try: + continuation = prepare(continue_from, sampling_canvas(canvas)[0], directory=root / 'continuations') + except ValueError as error: + if str(error) != MISSING: + raise + # A video with no stored context (single-pass, an earlier version, a replaced file) is + # continued from its own ending, which the engine reads and encodes. Only a video whose + # ending cannot be used stops here, before anything is loaded. + check_ending() # The engine decides the same way from the same record: whether the film's first frame is there to use. + # An ending encoded from the file has no first frame of the film to give. anchored = False - if continue_from is not None and first_frame_reference: + if continue_from is not None and continuation is not None and first_frame_reference: from .continuation import chain_first_frame_picture try: anchored = chain_first_frame_picture(continuation, continue_from) is not None @@ -669,10 +1246,11 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, command += ['--continue-from', str(continue_from)] if first_frame_reference: command.append('--first-frame-reference') - from .continuation import persist_snapshot - continuation = persist_snapshot(continuation, run / 'continuation-source.safetensors') - command += ['--continuation-snapshot', continuation['snapshot'], - '--continuation-snapshot-sha256', continuation['snapshot_sha256']] + if continuation is not None: + from .continuation import persist_snapshot + continuation = persist_snapshot(continuation, run / 'continuation-source.safetensors') + command += ['--continuation-snapshot', continuation['snapshot'], + '--continuation-snapshot-sha256', continuation['snapshot_sha256']] if upscale: command = command[:command.index('generate') + 1] + upscale_command(upscale[1], output, upscale[0], resources) else: @@ -691,12 +1269,22 @@ def generate(prompt, width, height, seconds, seed, output_directory, *, state['encoder_prewarm'] = prewarm_summary(encoder_prewarm) save(run / 'comfy-request.json', state) started = time.monotonic() - whole_progress = WholeVideoProgress() + try: + whole_progress = WholeVideoProgress.for_request(machine, canvas, output_directory, + base_steps=base_steps, refine_steps=refine_steps, two_pass=two_pass, + preview=preview, upscale=bool(upscale), encoder_prewarm=state['encoder_prewarm']) + except Exception: + # A progress estimate cannot prevent generation, even without its parameters. + try: + whole_progress = WholeVideoProgress() + except Exception: + whole_progress = _PlainProgress() + whole_progress = _SafeProgress(whole_progress) from .comfy_progress import REPORTS report_id = REPORTS.register(output) - def send_progress(message): + def send_progress(message, event=None): if progress: - progress(whole_progress.annotate(dict(message, report_id=report_id))) + progress(whole_progress.annotate(dict(message, report_id=report_id), event=event)) process = None from .result_cache import ResultCache, request_key, inspect_bounded cache = ResultCache(output_directory) @@ -711,8 +1299,14 @@ def poll(): tails[2].reset() message = progress_message(event) if message: - send_progress(message) + send_progress(message, event) + else: + whole_progress.observe(event) + heartbeat = whole_progress.heartbeat() + if progress and heartbeat: + progress(heartbeat) try: + send_progress({'label': 'Preparing video'}) if interrupted: interrupted() extra = {} @@ -749,11 +1343,28 @@ def poll(): planned = plan(sampled, two_pass, 'ref2va' if anchored else task_for(extra.get('media', {})), base_steps=base_steps, refine_steps=refine_steps, staged=bool(preview or upscale)) if continue_from is not None: - from .continuation import ADALN_TASK, ANCHOR_ADALN_TASK, validate_snapshot, summary + from .continuation import ADALN_TASK, ANCHOR_ADALN_TASK, MISSING, validate_snapshot, summary validate_route(task='ref2va' if anchored else task_for(extra.get('media', {})), anchor=anchored, media=extra.get('media')) - continuation = validate_snapshot(continuation, sampled, sampling_plan=planned) - state['continuation'] = summary(continuation) + if continuation is not None: + try: + continuation = validate_snapshot(continuation, sampled, sampling_plan=planned) + state['continuation'] = summary(continuation) + except ValueError as error: + if str(error) != MISSING: + raise + # Stored for another first-pass size: the engine sets it aside and encodes the ending. + check_ending() + continuation = None + if anchored: + # An encoded ending carries no first frame of the film, so nothing anchors this run. + anchored = False + planned = plan(sampled, two_pass, task_for(extra.get('media', {})), + base_steps=base_steps, refine_steps=refine_steps, staged=bool(preview or upscale)) + if continuation is None: + from .continuation import OVERLAP_FRAMES + state['continuation'] = dict(overlap_frames=OVERLAP_FRAMES, + new_frames=sampled['frames'] - OVERLAP_FRAMES, encoded=True) # A preview is its own result; an upscale is always run from its preview. keyed = dict(planned, preview=True) if preview else planned identify = lambda check: request_key(prompt, seed, canvas, keyed, extra, machine, @@ -798,10 +1409,9 @@ def asset_progress(message): state['sampling_cache_install'] = preparation if preparation['seconds']: started += preparation['seconds'] - whole_progress = WholeVideoProgress() send_progress({'reset': True, 'label': 'Preparing video'}) - whole_progress.sampling_plan = planned - whole_progress.forecast(progress_history_forecast(root, machine, dict(sampled, sampling_plan=planned))) + whole_progress.configure(plan=planned, media=extra.get('media', {}), + conditioning=bool(extra.get('conditioning')), sampled=sampled) send_progress({'label': 'Preparing %.3f s video + audio · %d × %d' % (canvas['seconds'], width, height)}) if release_models: @@ -864,8 +1474,11 @@ def asset_progress(message): report.update(report_extra) save(output.with_suffix('.request.json'), report) if continue_from is not None: - state['continuation']['audio_lead_in_samples'] = engine.get('continuation', {}).get( - 'audio_lead_in_samples', 0) + receipt = engine.get('continuation') or {} + if continuation is None and receipt: + # The ending was encoded by the engine, which is where the real counters come from. + state['continuation'] = dict(receipt) + state['continuation']['audio_lead_in_samples'] = receipt.get('audio_lead_in_samples', 0) if comfy_metadata: # Dropping the video on the canvas restores this graph, as with ComfyUI's own video nodes. # Written before the result cache records the file's size and hash. diff --git a/freevideo_engine/comfy_editor.py b/freevideo_engine/comfy_editor.py index 852d9e39..d915d2c6 100644 --- a/freevideo_engine/comfy_editor.py +++ b/freevideo_engine/comfy_editor.py @@ -39,6 +39,8 @@ _TOKEN = re.compile(r'[A-Za-z0-9_-]{1,128}') _REASON = re.compile(r'[a-z_]{1,80}') _REF = re.compile(r'[A-Za-z0-9_-]{1,128}') +# A note is one line: the page refuses the same characters (editor_model.js NOTE_BREAK) and the same ends. +_NOTE_BREAK = re.compile('[\x00-\x1f\x7f-\x9f\u2028\u2029]') _STATE = threading.RLock() _TRANSCODING = threading.Lock() _FAILED_IMPORTS = {} # terminal statuses awaiting writable storage @@ -279,6 +281,12 @@ def bounded(value, depth=0): if not isinstance(raw['ref'], str) or not _REF.fullmatch(raw['ref']): raise ValueError('Invalid block reference') clip['ref'] = raw['ref'] + if 'note' in raw: + note = raw['note'] + if (not _text(note, 120) or not note or note != note.strip() or '\ufeff' in (note[0], note[-1]) + or _NOTE_BREAK.search(note)): + raise ValueError('Invalid block note') + clip['note'] = note if source is None and ('take' in raw or 'made_text' in raw): raise ValueError('Invalid clip versions') _made(raw, clip) @@ -645,6 +653,10 @@ def sources(root, keys): if _read_json(report, 4 * 1024**2).get('success') is not True: raise ValueError('Creation is incomplete') media = probe(path) + if source['kind'] == 'work': + text = _work_text(root, relative) + if text: + media['text'] = text if source['kind'] == 'import': metadata = _read_import(root, source['id']) media.update(preview_width=media['width'], preview_height=media['height']) @@ -674,6 +686,23 @@ def sources(root, keys): return result +def _work_text(root, relative): + """The text a saved video was generated from; a block made from it starts with this description.""" + try: + path = _safe(root, (PurePosixPath(relative).parent / 'prompt.txt').as_posix()) + with path.open('rb') as stream: + data = stream.read(4 * 8000 + 1) + if len(data) > 4 * 8000: + # Longer than any description a block keeps; its beginning alone is not the description. + return '' + text = data.decode('utf-8').strip() + except (OSError, ValueError): + # Optional: an old, joined or hand-copied video has none. + return '' + # The same limit as a block's own description, so the project it lands in still saves. + return text if _text(text, 8000) else '' + + def _read_import(root, identity): key = (str(Path(root).resolve()), identity) with _STATE: diff --git a/freevideo_engine/comfy_launcher_runtime.py b/freevideo_engine/comfy_launcher_runtime.py index 03c9418b..7cefc0ed 100644 --- a/freevideo_engine/comfy_launcher_runtime.py +++ b/freevideo_engine/comfy_launcher_runtime.py @@ -958,7 +958,11 @@ def _inspect(self, values): self.selection = dict(descriptor, engine=str(engine), source=str(source), url=url, ready=ready, repair_comfy=repair_comfy, lan_access=values.get('lan_access') is True) self.state = dict(self.state, selection=dict(self.selection), host=host) - if not ready: + # Third-party downloads (manual models, no offline packages) are checked + # file by file, so files the user downloaded are found on a ready engine too. + checked = bool(values.get('download_list') or ( + values.get('model_method') == 'manual' and not values.get('offline_models'))) + if not ready or checked: extra = values.get('model_dirs', []) if not isinstance(extra, list) or any(not isinstance(p, str) for p in extra): raise ValueError('Model folders must be a list of directory paths') @@ -968,10 +972,19 @@ def _inspect(self, values): extra = [p for p in dict.fromkeys(extra) if p.strip() and io_path(Path(p).expanduser()).is_dir()] self.setup.inspect(dict(root=str(engine), extra_libraries=extra, copy=False, sampling_caches=bool(values.get('sampling_caches')), prepared_format=values.get('prepared_format'), + **({'download_list': True} if checked else {}), frontend=dict(root=descriptor['root'], separate=descriptor['separate'], download=fresh, repair_python=host['python'] if repair_comfy else None))) row = self._wait_setup() self.state = dict(self.state, plan=row['plan']) + # A ready receipt covers the engine, not every model file: when the + # file-by-file check finds models to place or download, accepting the + # plan prepares them before connecting. + plan = row['plan'] + if ready and checked and ( + plan.get('download_files') or (plan.get('local_models') or {}).get('matches')): + self.selection = dict(self.selection, ready=False) + self.state = dict(self.state, selection=dict(self.selection)) extra_disk, disks, disk_errors = disk_review(self.state.get('plan', {}), engine, descriptor['root'], separate=descriptor['separate'], new_comfy=fresh and not io_path(descriptor['root']).exists()) errors = list(self.state.get('plan', {}).get('errors', [])) diff --git a/freevideo_engine/comfy_progress.py b/freevideo_engine/comfy_progress.py index fb0cd9b5..32126c37 100644 --- a/freevideo_engine/comfy_progress.py +++ b/freevideo_engine/comfy_progress.py @@ -83,7 +83,18 @@ def publish(self, node, message): old = self.rows.get(node) metadata_only = ('compatibility' in value and not value.get('new_request') and not any(key in value for key in ('phase', 'stage', 'timing_phase', 'label', 'reset'))) - if old and metadata_only: + # A heartbeat carries a newer estimate, not engine progress (Harper's + # bridge repeats the last message's fields with a fresh overall). + heartbeat = isinstance(value.get('overall'), dict) and value['overall'].get('heartbeat') is True + overall_measured = now + if old and heartbeat: + # Replace only overall: the row and its age keep describing the + # last engine message, so a reopened page still sees a quiet engine. + value = dict(old['message'], overall=value['overall']) + value.pop('new_request', None) + value.pop('reset', None) + measured = old['measured'] + elif old and metadata_only: # A settings notice is not engine progress. Keep the current # activity and its timestamp, including in reopened snapshots. value = dict(old['message'], compatibility=value['compatibility']) @@ -91,11 +102,11 @@ def publish(self, node, message): value.pop('reset', None) if isinstance(value.get('overall'), dict) and 'reset' in value['overall']: value['overall'] = {key: item for key, item in value['overall'].items() if key != 'reset'} - measured = old['measured'] + measured, overall_measured = old['measured'], old.get('overall_measured', old['measured']) elif old and value.get('warning'): # A memory warning must not replace the current sampling step. value = dict(old['message'], warning=value['warning']) - measured = old['measured'] + measured, overall_measured = old['measured'], old.get('overall_measured', old['measured']) else: measured = now # The page starts a new request on these too and clears its notes. @@ -134,16 +145,22 @@ def publish(self, node, message): self._record_timeline(node, value, message, now, measured) self.sequence += 1 value.update(node=node, stream_id=self.stream_id, sequence=self.sequence) - self.rows[node] = {'message': value, 'measured': measured} + self.rows[node] = {'message': value, 'measured': measured, 'overall_measured': overall_measured} while len(self.rows) > 32: oldest = min(self.rows, key=lambda key: self.rows[key]['message']['sequence']) del self.rows[oldest] - return dict(value, age_seconds=max(0., now - measured)) + return self._aged(value, now, measured, overall_measured) + + @staticmethod + def _aged(message, now, measured, overall_measured): + # age_seconds: since the last engine message; overall_age_seconds: since + # the estimate in `overall` (a heartbeat refreshes only that one). + return dict(message, age_seconds=max(0., now - measured), overall_age_seconds=max(0., now - overall_measured)) def snapshot(self): with self.lock: now = self.clock() - return [dict(deepcopy(row['message']), age_seconds=max(0., now-row['measured'])) + return [self._aged(deepcopy(row['message']), now, row['measured'], row.get('overall_measured', row['measured'])) for row in self.rows.values()] diff --git a/freevideo_engine/comfy_setup.py b/freevideo_engine/comfy_setup.py index 9e881cc8..77f9104c 100644 --- a/freevideo_engine/comfy_setup.py +++ b/freevideo_engine/comfy_setup.py @@ -277,6 +277,8 @@ def inspect(self, value): arguments += ['--prepared-format', value['prepared_format']] if value.get('copy'): arguments.append('--copy-existing-models') + if value.get('download_list'): + arguments.append('--download-list') self.selection = dict(root=str(root), arguments=arguments) self.plan = None self.state = dict(status='running', action='plan', libraries=libraries) diff --git a/freevideo_engine/continuation_tail.py b/freevideo_engine/continuation_tail.py new file mode 100644 index 00000000..6c508db1 --- /dev/null +++ b/freevideo_engine/continuation_tail.py @@ -0,0 +1,267 @@ +"""A continuation context encoded from the ending of a finished video. + +A video made by an earlier version, or one whose file was replaced after it +was made, has no stored context. Its last 22 frames and the sound under them +are read back and encoded instead. The video encoder works on independent +17-frame clips, so those frames (17 + 5) fall on the same clip grid as the +last 7 latent frames of a generation: nothing before them has to be encoded. +The sound encoder looks back, so it is given up to 15 s and only its last +latents are kept. + +The record describes the ending as a first block of 39 frames. Only the sound +clock reads that length: the kept sound ends exactly where the picture ends. +""" +import math +from pathlib import Path + +from .continuation import (LATENT_FRAMES, MISSING, OVERLAP_FRAMES, SIZE_MISMATCH, audio_origin, validate_frames, + validate_path) + + +NOMINAL_FRAMES = 39 +AUDIO_TICKS = round(NOMINAL_FRAMES * 40 / 24) - audio_origin(NOMINAL_FRAMES - OVERLAP_FRAMES) +AUDIO_RATE = 32000 +TICK_SAMPLES = 800 +AUDIO_WINDOW_TICKS = 600 # 15 s, the longest sound H3 generates +FPS = 24 +TOO_SHORT = 'The previous video is too short to continue; it needs to be at least 2 s long.' +UNREADABLE = 'The previous video cannot be read.' + + +def _fit(image, width, height): + from PIL import Image + return image if image.size == (width, height) else image.resize((width, height), Image.Resampling.LANCZOS) + + +def _picture(frame): + """The frame as an RGB image. + + The decoder library's own conversion comes out about 1/255 darker than the picture that was saved, + which would show as a step where the next part begins. An 8-bit 4:2:0 frame of the usual range is + converted with the exact matrix instead. + """ + import numpy as np + from PIL import Image + if frame.format.name != 'yuv420p' or getattr(frame.color_range, 'value', frame.color_range) not in (0, 1): + return frame.to_image().convert('RGB') + y, cb, cr = frame.reformat(format='yuv444p', interpolation='BICUBIC').to_ndarray().astype(np.float32) + kr, kb = (.2126, .0722) if getattr(frame.colorspace, 'value', frame.colorspace) == 1 else (.299, .114) + y, cb, cr = (y - 16) * (255 / 219), (cb - 128) * (255 / 224), (cr - 128) * (255 / 224) + red, blue = y + 2 * (1 - kr) * cr, y + 2 * (1 - kb) * cb + green = (y - kr * red - kb * blue) / (1 - kr - kb) + return Image.fromarray(np.clip(np.stack([red, green, blue], -1) + .5, 0, 255).astype(np.uint8)) + + +def _rate(stream): + rate = float(stream.average_rate or stream.guessed_rate or 0) + if not 10 <= rate <= 121: + raise ValueError(UNREADABLE) + return rate + + +def _admit(count, rate, width, height, canvas): + """The length of the video as 24 fps frames, when its ending can be continued on `canvas`.""" + frames = count if abs(rate - FPS) <= .01 else round(count * FPS / rate) + if frames < NOMINAL_FRAMES: + raise ValueError(TOO_SHORT) + # The same shape within one 32-pixel step of the canvas; another size of the same shape is scaled. + if min(width, height) < 1 or abs(width * canvas['height'] - height * canvas['width']) > 32 * max(width, height): + raise ValueError(SIZE_MISMATCH) + return frames + + +def check(video_path, canvas): + """Whether the ending of this video can be encoded for `canvas`; a ValueError says why not. + + Reads the file's packets and decodes nothing, so a caller can ask before it queues a run. + """ + import av + try: + with av.open(str(validate_path(video_path))) as container: + if not container.streams.video: + raise ValueError(UNREADABLE) + stream = container.streams.video[0] + count = sum(1 for packet in container.demux(stream) if packet.size) + return _admit(count, _rate(stream), stream.width or 0, stream.height or 0, canvas) + except (av.FFmpegError, OSError) as error: + raise ValueError(UNREADABLE) from error + + +def pending(video_path, canvas): + """A continuation request whose context is encoded from the video when the run starts.""" + validate_frames(canvas['frames']) + check(video_path, canvas) + origin = NOMINAL_FRAMES - OVERLAP_FRAMES + return dict(tail=dict(video=str(Path(video_path).resolve())), overlap_frames=OVERLAP_FRAMES, + new_frames=canvas['frames'] - OVERLAP_FRAMES, frame_origin=origin, + audio_origin=audio_origin(origin), segment=2, audio_latents=AUDIO_TICKS) + + +def read(video_path, canvas, first, directory): + """The last 22 frames at both pass sizes and the sound under them, as files of this run. + + `first` is the first pass's width and height. A video at another frame rate gives the + pictures a 24 fps copy of it would end with. Runs without a GPU. + """ + import av + import numpy as np + from collections import deque + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + try: + with av.open(str(video_path)) as container: + if not container.streams.video: + raise ValueError(UNREADABLE) + rate = _rate(container.streams.video[0]) + kept, count = deque(maxlen=math.ceil(OVERLAP_FRAMES * rate / FPS) + 2), 0 + for frame in container.decode(video=0): + kept.append(frame) + count += 1 + if not count: + raise ValueError(UNREADABLE) + frames = _admit(count, rate, kept[-1].width, kept[-1].height, canvas) + oldest = count - len(kept) + if abs(rate - FPS) <= .01: + picked = range(count - OVERLAP_FRAMES, count) + else: + picked = [min(count - 1, max(oldest, round((frames - OVERLAP_FRAMES + index) * rate / FPS))) + for index in range(OVERLAP_FRAMES)] + images = [_picture(kept[index - oldest]) for index in picked] + del kept + sound = _sound(video_path, count / rate) + except (av.FFmpegError, OSError) as error: + raise ValueError(UNREADABLE) from error + width, height = images[0].size + result = {'frames': frames, 'source': {'width': width, 'height': height, 'fps': rate}} + for name, size in (('hi', canvas), ('lo', first)): + target = directory / ('continuation-tail-%s.npy' % name) + np.save(target, np.stack([np.asarray(_fit(image, size['width'], size['height'])) for image in images]), + allow_pickle=False) + result[name] = str(target) + target = directory / 'continuation-tail-audio.npy' + np.save(target, sound, allow_pickle=False) + result['audio'] = str(target) + return result + + +def _sound(video_path, seconds): + """Up to 15 s of 32 kHz stereo ending where the picture ends; silence where the file has none.""" + import av + import numpy as np + end = round(seconds * AUDIO_RATE) + window = min(AUDIO_WINDOW_TICKS, end // TICK_SAMPLES) * TICK_SAMPLES + begin = end - window + data = np.zeros((2, window), dtype=np.float32) + with av.open(str(video_path)) as container: + if not container.streams.audio: + return data + stream = container.streams.audio[0] + mono = stream.layout.name == 'mono' or stream.channels == 1 + # A mono track has its channel repeated, as reference sound is; a stereo resampler would lower it by 3 dB. + resampler = av.AudioResampler(format='fltp', layout='mono' if mono else 'stereo', rate=AUDIO_RATE) + position = 0 + def place(item): + nonlocal position + block = item.to_ndarray() + start, stop = max(position, begin), min(position + block.shape[1], end) + if stop > start: + data[:, start - begin:stop - begin] = block[:, start - position:stop - position] + position += block.shape[1] + for frame in container.decode(audio=0): + for item in resampler.resample(frame): + place(item) + if position >= end: + break + else: + for item in resampler.resample(None): + place(item) + if not np.isfinite(data).all(): + raise ValueError(UNREADABLE) + return data + + +def encode(tail, base, *, device='cuda', before_upload=None, posterior='mode'): + """Encode the files of `read` into the three tensors of a continuation record.""" + import gc + import numpy as np + import torch + from .media_encoding import encode_visual + from .vae_weights import load_audio_vae, load_video_encoder + empty = torch.mps.empty_cache if torch.device(device).type == 'mps' else ( + torch.cuda.empty_cache if torch.device(device).type == 'cuda' else lambda: None) + result = {} + vae, _ = load_video_encoder(base, before_upload=None if before_upload is None else + lambda model: before_upload(model, False), device=device) + for name in ('hi', 'lo'): + pixels = np.load(tail[name], mmap_mode='r', allow_pickle=False) + latent = encode_visual(vae, pixels, device=device, posterior=posterior) + if latent.shape[2] != LATENT_FRAMES: + raise ValueError('Unexpected continuation latent frames') + result['video_' + name] = latent.numpy() + del pixels, latent + del vae + gc.collect() + empty() + vae, _ = load_audio_vae(base, before_upload=None if before_upload is None else + lambda model: before_upload(model, True), device=device) + if vae.config.sampling_rate != AUDIO_RATE: + raise ValueError('Continuation sound requires the pinned 32 kHz H3 VAE') + mean = torch.tensor(vae.config.latents_mean).view(1, 1, -1) + std = torch.tensor(vae.config.latents_std).view(1, 1, -1) + audio = torch.from_numpy(np.load(tail['audio'], allow_pickle=False)).to(device) + latent = vae.encode(audio[:, None], return_dict=False)[0].mode().float().cpu().transpose(1, 2) + latent = ((latent - mean) / std).transpose(1, 2) + if latent.shape[2] < AUDIO_TICKS: + raise ValueError('Unexpected continuation sound latents') + result['audio'] = latent[:, :, -AUDIO_TICKS:].contiguous().numpy() + del vae, audio, latent + gc.collect() + empty() + return result + + +def metadata(identity, canvas, first): + """The record's description: a first block whose last 22 frames are the file's.""" + return dict(identity=identity, + geometry=dict(width=canvas['width'], height=canvas['height'], frames=NOMINAL_FRAMES), + first_canvas=dict(width=first['width'], height=first['height']), + chain=dict(segment=1, frame_origin=0, audio_origin=0), context='encoded') + + +def freeze(request, canvas, first, directory, base): + """Keep the ending `pending` asked for as files of this run; the worker encodes them.""" + from .continuation_store import video_id + directory = Path(directory) + video = request['tail']['video'] + size = lambda value: {key: value[key] for key in ('width', 'height')} + tail = read(video, canvas, first, directory) + return dict(request, tail=dict(tail, video=video, canvas=size(canvas), first=size(first), base=str(base), + source_id=video_id(video), + snapshot=str((directory / 'continuation-encoded.safetensors').resolve()))) + + +def complete(request, canvas, manifest, sampling_plan=None, *, resident=None, device='cuda'): + """The ordinary snapshot request for a frozen ending, and how long encoding it took. + + The first attempt of a run encodes and writes the record beside its other files; a retry reads it. + """ + import hashlib + import time + from .continuation import identity, validate_snapshot + from .continuation_store import write + tail = request['tail'] + path = Path(tail['snapshot']) + if sampling_plan is not None and any(sampling_plan['first'][key] != tail['first'][key] for key in ('width', 'height')): + raise ValueError(MISSING) + seconds = None + if not path.is_file(): + started = time.perf_counter() + data = encode(tail, tail['base'], device=device, before_upload=None if resident is None else + lambda model, audio: resident.input_vae_room(model, tail['canvas'], audio=audio)) + write(path, dict(data, metadata=dict(metadata(identity(manifest), tail['canvas'], tail['first']), + source_id=tail['source_id']))) + path.chmod(0o400) + seconds = time.perf_counter() - started + result = dict({key: value for key, value in request.items() if key != 'tail'}, snapshot=str(path), + snapshot_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), source_id=tail['source_id']) + return validate_snapshot(result, canvas, manifest, sampling_plan), dict(seconds=seconds, reused=seconds is None) diff --git a/freevideo_engine/decode_resume.py b/freevideo_engine/decode_resume.py index eff55816..4a27033b 100644 --- a/freevideo_engine/decode_resume.py +++ b/freevideo_engine/decode_resume.py @@ -97,7 +97,14 @@ def sampling_provenance(request): if request.get('continuation') is not None: # A retry's identity binds the accepted bytes, without embedding the # snapshot again or making a relocation change computation identity. - result['continuation'] = {key: value for key, value in request['continuation'].items() + value = request['continuation'] + tail = value.get('tail') + if isinstance(tail, dict) and Path(tail.get('snapshot') or '.').is_file(): + # The request as it was sent asks for the ending to be encoded; the worker goes on with the + # record it wrote. A retry compares the two, so both are named by that record. + value = dict({key: item for key, item in value.items() if key != 'tail'}, + snapshot_sha256=_digest(tail['snapshot']), source_id=tail['source_id']) + result['continuation'] = {key: item for key, item in value.items() if key not in ('snapshot', 'snapshot_data', 'source_video')} return result diff --git a/freevideo_engine/desktop_runtime.py b/freevideo_engine/desktop_runtime.py index 76815e37..48c082df 100644 --- a/freevideo_engine/desktop_runtime.py +++ b/freevideo_engine/desktop_runtime.py @@ -20,6 +20,20 @@ from .system import install_root, windows +def parse_progress_line(line): + """Decode the worker's UI protocol; ordinary terminal output stays a log.""" + if line.startswith('{'): + try: + message = json.loads(line) + if message.get('event') == 'freevideo_ui': + return message + if message.get('event') == 'local_model_progress': + return local_model_ui(message) + except (ValueError, AttributeError): + pass + return None + + def launcher_root(): if os.environ.get('FREEVIDEO_LAUNCHER_HOME'): return Path(os.environ['FREEVIDEO_LAUNCHER_HOME']) @@ -334,17 +348,12 @@ def progress(message): log.write(line) output = (output + line)[-2 * 1024 * 1024:] self.events.put(('log', line)) - if line.startswith('{'): - try: - message = plain_message(json.loads(line)) - if message.get('event') == 'freevideo_ui': - if message.get('kind') == 'failure' and isinstance(message.get('error'), str): - error = message['error'] - self.events.put(('progress', message)) - elif message.get('event') == 'local_model_progress': - self.events.put(('progress', local_model_ui(message))) - except (ValueError, AttributeError): - pass + message = parse_progress_line(line) + if message is not None: + message = plain_message(message) + if message.get('kind') == 'failure' and isinstance(message.get('error'), str): + error = message['error'] + self.events.put(('progress', message)) code = process.wait(timeout=15) process.stdout.close() except BaseException as failure: diff --git a/freevideo_engine/doctor.py b/freevideo_engine/doctor.py index ccd492b1..ef3c0b63 100644 --- a/freevideo_engine/doctor.py +++ b/freevideo_engine/doctor.py @@ -11,7 +11,12 @@ from . import processes, subprocess_output -def doctor(*, probe=False, hardware=None, backends=None): +def doctor(*, probe=False, hardware=None, backends=None, progress=None): + if progress is None and os.environ.get('FREEVIDEO_INSTALL_PROGRESS') == '1': + from .install_activity import step + progress = step + if progress: + progress('GPU information reading') from .triton_compat import own_temporary_environment own_temporary_environment() versions = package_versions() @@ -28,7 +33,10 @@ def doctor(*, probe=False, hardware=None, backends=None): with runtime_lock() as descriptor: env = dict(os.environ, **{LOCK_ENV: str(descriptor)}) linear = ['linear'] + (['linear-int8'] if hardware.system in ('Windows', 'Linux') else []) - for backend in [*sorted(backends), *linear]: + probes = [*sorted(backends), *linear] + for index, backend in enumerate(probes): + if progress: + progress('GPU tests running', index, len(probes)) command = [sys.executable, '-m', 'freevideo_engine.probe', backend] try: child = subprocess_output.run_python(command, capture_output=True, timeout=180, env=env, @@ -52,6 +60,8 @@ def tail(value): except OSError as error: row = {'backend': backend, 'status': 'error', 'error': 'Could not start kernel probe: ' + str(error)} result['kernel_probes'].append(row) + if progress: + progress('GPU tests running', index + 1, len(probes)) result.update(readiness(result['kernel_probes']), identity=identity(hardware, versions), view_root=view_root()) if temp_root(): @@ -59,4 +69,6 @@ def tail(value): result['receipt'] = str(receipt_path()) result['validation'] = 'Small kernels executed on the listed GPU; this does not validate full-video speed, quality or capacity.' save(receipt_path(), result) + if progress: + progress('GPU tests finished', len(probes), len(probes)) return result diff --git a/freevideo_engine/editor_text.py b/freevideo_engine/editor_text.py index 08720601..84fea212 100644 --- a/freevideo_engine/editor_text.py +++ b/freevideo_engine/editor_text.py @@ -79,7 +79,7 @@ def validate_text_project(raw, validate_media): _check(all(c.get(k) is None for k in ('source','pending','link','part','job','reason')) and c.get('extend',False) is False and ('versions' not in c or isinstance(c['versions'],list) and not c['versions'])) _check(type(c.get('in',0)) is int and c.get('in',0) == 0 and type(c.get('length',c['frames'])) is int and c.get('length',c['frames']) == c['frames']) - _check(not any(k in c for k in ('volume','fit','splice','target_frames','text_by','ref','take','made_text'))) + _check(not any(k in c for k in ('volume','fit','splice','target_frames','text_by','ref','note','take','made_text'))) clip = dict(id=identity, start=c['start'], frames=c['frames'], length=c['frames'], source=None, link=None, text=text, part=None, pending=None, reason=None, job=None, extend=False, versions=[], text_kind=kind, **{'in':0}) _check(kind != 'subtitle' or 'text_style' not in c) diff --git a/freevideo_engine/effort_forecast.py b/freevideo_engine/effort_forecast.py index a94108ca..71d959b8 100644 --- a/freevideo_engine/effort_forecast.py +++ b/freevideo_engine/effort_forecast.py @@ -10,6 +10,7 @@ import platform from pathlib import Path from statistics import median +import threading import time @@ -48,7 +49,8 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True): if len(attempts) > 1: return None passes = video.get('sampling_passes') if plan.get('enabled') else [video] - if not isinstance(passes, list) or len(passes) != (2 if plan.get('enabled') else 1): + preview = bool(report.get('preview') or plan.get('preview')) + if not isinstance(passes, list) or len(passes) != (2 if plan.get('enabled') and not preview else 1): return None measured = [] for part in passes: @@ -61,7 +63,7 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True): if sum(p['seconds'] for p in measured) > total * 1.01: return None base = plan.get('base_steps', video.get('config', {}).get('steps', 8)) - refine = plan.get('refine_steps', 0) if plan.get('enabled') else 0 + refine = plan.get('refine_steps', 0) if plan.get('enabled') and not preview else 0 if measured[0]['steps'] != base or (refine and measured[1]['steps'] != refine): return None encoding = report.get('encoding') or {} @@ -118,7 +120,7 @@ def timing_record(report, gpu_uuid, system, *, stage_details=True): version=version if isinstance(version, str) and len(version) <= 64 else None, in_use=in_use, reference_rows=reference_rows, - flow='preview' if report.get('preview') or plan.get('preview') else 'full', stages=stages, + flow='preview' if preview else 'full', stages=stages, sampling_geometry={k: plan[k] for k in ('first', 'second') if plan.get(k)}, references=task_has_references(video.get('config', {}).get('task', 't2va')), model_resident=video.get('resident_engine_cache_hit')) @@ -130,7 +132,7 @@ def estimate(records, canvas, *, base_steps=8, refine_steps=3, two_pass=True, ta validate_steps(base_steps, refine_steps, two_pass) candidates = [] for row in records: - if (row['task'] != task or row['adapters'] != adapters + if (row.get('flow', 'full') != 'full' or row['task'] != task or row['adapters'] != adapters or row['shape'] != {k: canvas[k] for k in ('width', 'height', 'frames')}): continue # Both passes retain their own resolution costs. Never multiply the @@ -319,6 +321,76 @@ def _stage_forecast(records, request, cold_flags, prior): return result +_START_CONTEXT = {} +# Guards the entries above: only one background read per key, and readers +# always see a complete hardware/records pair. File IO stays outside it. +_START_LOCK = threading.Lock() + + +def _context_key(output_directory, machine): + device = machine.get('device_identity') if machine.get('device_backend') == 'mps' else machine.get('gpu_uuid') + system = machine.get('system') or platform.system() + return (str(output_directory), json.dumps(device, sort_keys=True), system, machine.get('setup_run')), device, system + + +def start_context(output_directory, machine, *, timeout=.035): + """Bound foreground IO to ~35 ms, even on slow/unavailable output disks. + + One daemon per key reads in the background. Its last result stays in use + while a newer read runs (at most one every 15 s), so a request that starts + minutes after the previous one still sees that run. A request that started + before a read finished can pick it up later (history_read_at). + No GPU calls, resident RPC, model scans, hashing or directory-wide cache + inspection. Installation inventory describes identity, not live free RAM. + """ + from threading import Event, Thread + key, device, system = _context_key(output_directory, machine) + start = False + with _START_LOCK: + entry = _START_CONTEXT.get(key) + if entry is None: + entry = dict(ready=Event(), started=None, read_at=None, hardware={}, records=[], reading=False) + _START_CONTEXT[key] = entry + now = time.monotonic() + if not entry['reading'] and (entry['started'] is None or now - entry['started'] > 15): + entry['reading'], entry['started'], start = True, now, True + if start: + def read(): + hardware, records = dict(entry['hardware']), list(entry['records']) + try: + if machine.get('setup_run'): + with (Path(machine['setup_run']) / 'plan.json').open('rb') as stream: + raw = stream.read(1024 * 1024 + 1) + if len(raw) <= 1024 * 1024: + found = json.loads(raw).get('inventory', {}).get('hardware', {}) + if same_device(found, device): + hardware = {k: found[k] for k in + ('gpu_name', 'capability', 'ram_total', 'vram_total', 'system') if k in found} + except (OSError, ValueError, TypeError, AttributeError): + pass + try: + records = local_records(output_directory, device, system=system, stage_details=True) + except (OSError, ValueError, TypeError, AttributeError): + pass + finally: + with _START_LOCK: + entry['hardware'], entry['records'] = hardware, records + entry['read_at'], entry['reading'] = time.monotonic(), False + entry['ready'].set() + Thread(target=read, name='freevideo-progress-history', daemon=True).start() + if timeout and not entry['ready'].is_set(): + entry['ready'].wait(timeout) + with _START_LOCK: + return dict(entry['hardware']), list(entry['records']) + + +def history_read_at(output_directory, machine): + """Monotonic time the last background read finished, or None.""" + with _START_LOCK: + entry = _START_CONTEXT.get(_context_key(output_directory, machine)[0]) + return entry and entry['read_at'] + + def local_records(output_directory, gpu_uuid, *, system=None, stage_details=False, refresh=False): """Bounded recent JSON reads, cached between slider changes; never open MP4s.""" system = system or platform.system() @@ -338,7 +410,10 @@ def local_records(output_directory, gpu_uuid, *, system=None, stage_details=Fals except OSError: continue records = [] - for _, path in sorted(paths, reverse=True)[:48]: + # Slider estimates keep their original window of the 48 newest files; the + # progress history reads on until it has 48 usable rows. + candidates = sorted(paths, reverse=True) + for _, path in (candidates if stage_details else candidates[:48]): try: with path.open('rb') as stream: data = stream.read(4 * 1024 * 1024 + 1) @@ -347,6 +422,8 @@ def local_records(output_directory, gpu_uuid, *, system=None, stage_details=Fals row = timing_record(json.loads(data), gpu_uuid, system, stage_details=stage_details) if row: records.append(row) + if stage_details and len(records) == 48: + break except (OSError, ValueError, TypeError, AttributeError, KeyError): continue if not stage_details: diff --git a/freevideo_engine/encode_worker.py b/freevideo_engine/encode_worker.py index 5744ba30..464453c5 100644 --- a/freevideo_engine/encode_worker.py +++ b/freevideo_engine/encode_worker.py @@ -236,6 +236,12 @@ def _encode(args, request, resident, trace): loading = trace.data def phase(name, **metrics): trace.stage(name, **metrics) + if args.check_library: + from .install_activity import step + labels = {'encoder_torch_import': 'Runtime libraries importing', + 'encoder_options': 'Encoder configuring', 'encoder_native_import': 'Encoder importing'} + if name in labels: + step(labels[name]) if name in ('encoder_oom', 'encoder_retry'): print(json.dumps(dict(event=name, attempt=len(loading.get('encoder_attempts', [])))), flush=True) if name == 'encoder_oom' and request.get('metrics') and request.get('diagnostic_output'): @@ -306,6 +312,8 @@ def phase(name, **metrics): raise RuntimeError('Unexpected application modules: ' + repr(forbidden)) print(json.dumps({'native_encoder_library_import': True, 'application_modules_loaded': [], 'torch': str(torch.__version__), 'source': str(args.comfy_root)}), flush=True) + from .install_activity import step + step('Encoder ready') return phase('encoder_media_prepare') keyframes = request.get('keyframes') diff --git a/freevideo_engine/failure_details.py b/freevideo_engine/failure_details.py index c9f18d9a..ef0772d3 100644 --- a/freevideo_engine/failure_details.py +++ b/freevideo_engine/failure_details.py @@ -9,6 +9,68 @@ STEP_OUTPUT = '\n\nFull step output:\n' +QUALITY_LEVELS = (('Light', '轻量'), ('Medium', '标准'), ('High', '精细'), ('Max', '极致')) + + +def sampling_table_details(value): + """Read the worker's bounded, numeric sampling-table failure metadata.""" + from .sampling_assets import UNAVAILABLE + for match in re.finditer(r'^(?:(?:[\w.]*(?:Error|Exception)|(?:[\w.]+\.)?SamplingTableUnavailable):\s*)?' + + re.escape(UNAVAILABLE) + r'(\{[^\n]*\})$', + str(value)[:65536], re.M): + try: + details = json.loads(match[1]) + if (type(details['base_steps']) is int and 1 <= details['base_steps'] <= 32 + and type(details['two_pass']) is bool + and type(details['refine_steps']) is int and 0 <= details['refine_steps'] <= 31 + and type(details['missing_bytes']) is int and details['missing_bytes'] > 0 + and type(details['decimal_sizes']) is bool and isinstance(details['available'], list) + and all(type(i) is int and 0 <= i <= 3 for i in details['available'])): + return {key: details[key] for key in + ('base_steps', 'two_pass', 'refine_steps', 'missing_bytes', 'available', 'decimal_sizes')} + except (ValueError, KeyError, TypeError): + pass + return None + + +def sampling_table_failure(value, *, zh=False): + details = sampling_table_details(value) + if details is None: + return None + from .launcher_copy import size_text + language = 1 if zh else 0 + index = next((i for i, steps in enumerate((8, 12, 16, 20)) + if details['base_steps'] == steps and details['two_pass'] == (i == 0) + and (i != 0 or details['refine_steps'] == 3)), None) + names = [('“%s”' if zh else '%s') % QUALITY_LEVELS[i][language] for i in sorted(set(details['available']))] + available = '' + if names: + available = names[0] if len(names) == 1 else ('、' if zh else ', ').join(names[:-1]) + ('或' if zh else ' or ') + names[-1] + size = size_text(details['missing_bytes'], details['decimal_sizes']) + # A custom step count is not a quality level (Iris): no "档" / "quality level" for it. + if zh: + if index is None: + title = '“自定义（%d 步）”的采样表还没有下载' % details['base_steps'] + what = '这个步数的采样表' + else: + title = '“%s”档的采样表还没有下载' % QUALITY_LEVELS[index][1] + what = '这一档的采样表' + detail = ('这次生成需要先下载%s（约 %s），但这次没能下载。' + '请确认网络连接后再生成一次，FreeVideo 会自动下载。') % (what, size) + if available: + detail += '也可以改用已经可以使用的%s档。' % available + else: + if index is None: + title = 'The sampling table for Custom (%d steps) is not downloaded yet' % details['base_steps'] + what = 'the sampling table for this step count' + else: + title = "The %s quality level's sampling table is not downloaded yet" % QUALITY_LEVELS[index][0] + what = "that level's sampling table" + detail = ('This video first needs %s (about %s), and it could not be downloaded this time. ' + 'Check the network connection and generate again; FreeVideo downloads it automatically.') % (what, size) + if available: + detail += ' You can also switch to a quality level that is already available: %s.' % available + return dict(kind='sampling-table', title=title, detail=detail, action='') def redacted_launcher_error(value, replacements=()): @@ -320,11 +382,19 @@ def launcher_failure(value, *, zh=False, other_disk=False, disk_name='', can_rep return dict(title='GPU 检查未通过' if zh else 'GPU validation failed', detail=detail, action='展开详情或导出报告，查看完整检查结果。' if zh else 'Open the details or export the report for the complete results.', kind='kernels') + if 'Download list error: read' in text.splitlines(): + from .model_guidance import DOWNLOAD_COPY + return dict(title=DOWNLOAD_COPY['error_read'][zh], detail='', action='', kind='download-list-read') rules = ( (r'The previous video has no usable continuation data', 'continuation-missing', ('This video cannot be continued', '这个视频暂时不能接续'), - ('Generate the previous video again with this version of FreeVideo. Older videos, replaced files and expired continuation data cannot be used.', - '请用当前版本的 FreeVideo 重新生成上一段视频。旧版本的视频、被替换的文件或已清理接续数据的视频不能接续。')), + ('Generate the previous video again, then continue it.', '请重新生成上一段视频，再接续。')), + (r'The previous video is too short to continue', 'continuation-short', + ('The previous video is too short', '上一段视频太短'), + ('Choose a video that is at least 2 seconds long.', '请选择至少 2 秒的视频。')), + (r'The previous video cannot be read', 'continuation-unreadable', + ('The previous video cannot be read', '无法读取上一段视频'), + ('Choose an MP4 video that plays normally.', '请选择能正常播放的 MP4 视频。')), (r'Use the same width and height as the previous video', 'continuation-size', ('The video sizes differ', '两段视频的画面大小不一致'), ('Set the width and height to the previous video’s size.', '请把宽度和高度设为与上一段视频相同。')), @@ -578,7 +648,16 @@ def generation_failure(run, exit_code=None, *, repair=None): if len(lines) == 1: lines.append('The worker ended without a readable error report.') lines.append('\nRetained outputs: ' + str(run)) + # The table failure can be in a retained child log rather than the report. + # Keep its numeric metadata ahead of technical output for all interactive callers. + sampling = sampling_table_details('\n'.join(lines)) value = redactor.text('\n'.join(lines)) + if sampling: + from .sampling_assets import UNAVAILABLE + marker = UNAVAILABLE + json.dumps(sampling, separators=(',', ':')) + card = sampling_table_failure(marker) + first, _, rest = value.partition('\n') + value = '\n'.join((first, marker, card['title'], card['detail'], rest)) if explained: from .runtime_libraries import REINSTALL value = value.replace(REPAIR_MARK, repair or REINSTALL, 1) diff --git a/freevideo_engine/generate.py b/freevideo_engine/generate.py index 758b8d5f..1603fc77 100644 --- a/freevideo_engine/generate.py +++ b/freevideo_engine/generate.py @@ -344,7 +344,29 @@ def _run(args): from .geometry import geometry from .continuation import (validate_frames, validate_route, prepare as prepare_continuation, summary as continuation_summary, validate_snapshot, persist_snapshot, restore_snapshot, - audio_context) + audio_context, MISSING as CONTINUATION_MISSING) + from .continuation_tail import pending as encoded_continuation, freeze as freeze_tail + continuation_directory = continuation_first = None + + def freeze_continuation(value): + from .paths import base_path + return freeze_tail(value, canvas, continuation_first, continuation_directory, + profile['engine'].get('base') or base_path()) + + def usable_continuation(value, *check): + """The stored context when it fits this run, otherwise the previous video's own ending.""" + if 'tail' in value: + return value + try: + return validate_snapshot(value, *check) + except ValueError as error: + if str(error) != CONTINUATION_MISSING: + raise + # A record of another version, other weights or another first-pass size. + value = dict(encoded_continuation(continue_from, canvas), + **{key: value[key] for key in ('audio_context', 'source_video', 'anchor') if key in value}) + return freeze_continuation(value) if continuation_directory is not None else value + frames = getattr(args, 'frames', None) continue_from = getattr(args, 'continue_from', None) continue_anchor = getattr(args, 'continue_anchor', None) @@ -392,12 +414,19 @@ def _run(args): continuation = restore_snapshot(args.continuation_snapshot, getattr(args, 'continuation_snapshot_sha256', None), canvas) else: - continuation = prepare_continuation(continue_from, canvas) + try: + continuation = prepare_continuation(continue_from, canvas) + except ValueError as error: + if str(error) != CONTINUATION_MISSING: + raise + # Nothing was stored for this file: its own ending is encoded when the run starts. + continuation = encoded_continuation(continue_from, canvas) continuation = dict(continuation, audio_context=audio_context(), source_video=str(continue_from)) if first_frame is not None: from .continuation import anchor_media, chain_first_frame_picture try: - kept = chain_first_frame_picture(continuation, continue_from) + # A context encoded from the file carries no first frame; whoever planned this run saw none either. + kept = None if 'tail' in continuation else chain_first_frame_picture(continuation, continue_from) except ValueError: kept = None first_frame['reason'] = 'unreadable' @@ -464,8 +493,9 @@ def _run(args): if continuation is not None: from .two_pass import plan as continuation_plan validate_route(task=profile['engine'].get('task', 't2va'), anchor=continue_anchor is not None) - continuation = validate_snapshot(continuation, canvas, None if active_loras else manifest, - continuation_plan(canvas, two_pass, base_steps=base_steps, refine_steps=refine_steps)) + planned = continuation_plan(canvas, two_pass, base_steps=base_steps, refine_steps=refine_steps) + continuation_first = planned['first'] + continuation = usable_continuation(continuation, canvas, None if active_loras else manifest, planned) if manifest.get('precision') not in ('fp8', 'int8', 'bf16'): # BF16: the unquantized reference raise ValueError('Select the prepared official FP8 or int8 cache') destination = args.out.resolve() @@ -474,7 +504,10 @@ def _run(args): destination.parent.mkdir(parents=True, exist_ok=True) artifacts = destination.with_suffix('.artifacts') artifacts.mkdir(exist_ok=False) - if continuation is not None and 'snapshot' not in continuation: + continuation_directory = artifacts + if continuation is not None and 'tail' in continuation: + continuation = freeze_continuation(continuation) + elif continuation is not None and 'snapshot' not in continuation: continuation = persist_snapshot(continuation, artifacts / 'continuation-source.safetensors') if args.prompt_file: shutil.copyfile(args.prompt_file, artifacts / 'prompt.txt') @@ -491,6 +524,8 @@ def _run(args): 'encoder_mode': 'native H3 encoder library child' if args.prompt_file else 'preencoded shared conditioning'} if continuation is not None: report['continuation'] = continuation_summary(continuation) + if 'tail' in continuation: + report['continuation']['context'] = 'encoded' report['audio_context'] = continuation['audio_context'] if continue_anchor is not None: report['continue_anchor'] = continuation['anchor'] @@ -523,8 +558,10 @@ def interrupted(signum, frame): args._lora_max_block_bytes = prepared['report'].get('max_block_bytes', 0) args._lora_root_bytes = prepared['report'].get('root_bytes', 0) if continuation is not None: - continuation = validate_snapshot(continuation, canvas, + continuation = usable_continuation(continuation, canvas, json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8'))) + if 'tail' in continuation: + report['continuation']['context'] = 'encoded' if args._lora_max_block_bytes or args._lora_root_bytes: profile.setdefault('policy', {})['lora_max_block_bytes'] = args._lora_max_block_bytes profile['policy']['lora_root_bytes'] = args._lora_root_bytes @@ -691,8 +728,10 @@ def interrupted(signum, frame): report['sampling_plan'] = sampling_plan if continuation is not None: validate_route(task=profile['engine'].get('task', 't2va'), anchor=continue_anchor is not None) - continuation = validate_snapshot(continuation, canvas, + continuation = usable_continuation(continuation, canvas, json.loads((args.cache / 'manifest.json').read_text(encoding='utf-8')), sampling_plan) + if 'tail' in continuation: + report['continuation']['context'] = 'encoded' if preview: report['preview'] = dict(geometry=dict(sampling_plan['first'], fps=canvas['fps'])) if refine_source: diff --git a/freevideo_engine/install.py b/freevideo_engine/install.py index 73385e8f..4d4b00da 100644 --- a/freevideo_engine/install.py +++ b/freevideo_engine/install.py @@ -32,7 +32,13 @@ def install_packages(env, source): if env.get('FREEVIDEO_UV') else [sys.executable, '-m', 'pip', 'install']) # uv is a native program; pip runs in this interpreter's own Python. run = subprocess_output.run_external if env.get('FREEVIDEO_UV') else subprocess_output.run_python - result = run([*command, '-e', str(diffusers), '--no-deps'], + if pth_spellable(diffusers): + target = ['-e', str(diffusers)] + else: + # A regular copy carries no path; replace any copy from an older tree. + target = [str(diffusers), *(['--reinstall-package', 'diffusers'] if env.get('FREEVIDEO_UV') + else ['--force-reinstall'])] + result = run([*command, *target, '--no-deps'], env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) print(result.stdout, flush=True) if result.returncode: @@ -46,6 +52,26 @@ def install_packages(env, source): 'models_downloaded': False, 'environment_variable': 'FREEVIDEO_VDN_ROOT'})) +def pth_spellable(path, *, windows=None, encoding=None): + """Whether an editable install may name this folder in a .pth file. + + setuptools writes a src-layout folder into the .pth in the locale code page, + and Python 3.12 reads it the same way; UTF-8 mode changes neither. Where the + Windows code page cannot spell the folder, a regular copy is installed + instead: add_vdn() puts the pinned source first on sys.path either way. + """ + if not (sys.platform == 'win32' if windows is None else windows): + return True + if encoding is None: + import locale + encoding = locale.getencoding() if hasattr(locale, 'getencoding') else locale.getpreferredencoding(False) + try: + str(path).encode(encoding) + except (UnicodeEncodeError, LookupError): + return False + return True + + def prepare_diffusers(root, dependencies): """Build the official two-patch tree atomically; never reset a user checkout.""" spec = dependencies['diffusers'] diff --git a/freevideo_engine/install_activity.py b/freevideo_engine/install_activity.py new file mode 100644 index 00000000..d8e3f81a --- /dev/null +++ b/freevideo_engine/install_activity.py @@ -0,0 +1,29 @@ +"""Small installation worker events: actual work, never elapsed-time counters.""" +import json +import sys +import time + + +def emit(event, **row): + sys.stdout.write(json.dumps(dict(event=event, **row)) + '\n') + sys.stdout.flush() + + +def step(name, done=0, total=0): + emit('installation_step', name=name, done=done, total=total) + + +def byte_progress(event, file, *, name='Model files', report=None): + """Report bounded reads/writes, including both endpoints, at most twice/s.""" + last = None + def progress(done, total): + nonlocal last + now = time.monotonic() + if last is None or done == total or now - last >= .5: + row = dict(file=str(file), name=name, done_bytes=done, total_bytes=total) + if report is None: + emit(event, **row) + else: + report(**row) + last = now + return progress diff --git a/freevideo_engine/launcher/Main.qml b/freevideo_engine/launcher/Main.qml index 1d05b042..22495fb6 100644 --- a/freevideo_engine/launcher/Main.qml +++ b/freevideo_engine/launcher/Main.qml @@ -9,6 +9,9 @@ ApplicationWindow { color: unifiedChrome ? theme.canvas : theme.bg width: 1280; height: 860 property var s: initialState + readonly property var downloads: backend.downloadList + // Keep expansion by family across language changes and new checked lists. + property var downloadExpanded: ({}) Connections { target: backend; function onChanged() { win.s = backend.state } } property bool settingsOpen: false property bool terminalOpen: false @@ -29,7 +32,7 @@ ApplicationWindow { property string previousReview: "" property bool compact: width < 1000 property bool shortWindow: height < 700 - property bool otherModelLinksOpen: false + onModelInfoOpenChanged: if (modelInfoOpen) backend.prepareDownloadList() readonly property bool manualEnvironment: s.offline.runtime_supported !== false && s.form.new_comfy && s.form.environment_method === "manual" readonly property bool usingRuntime: manualEnvironment && s.offline.runtime readonly property bool needsRuntime: s.page === "comfy" && manualEnvironment && !s.offline.runtime @@ -597,19 +600,7 @@ ApplicationWindow { delegate: FButton { required property var modelData; required property int index; objectName: "offlineShare-" + index; text: t("Download from Quark ↗", "打开夸克网盘 ↗"); onClicked: backend.link(modelData.url) } } FText { objectName: "offlinePackageGuide"; text: s.offline.guide; color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true } - FButton { objectName: "otherModelLinks"; text: t("Alternative download sources", "其他下载渠道") + (otherModelLinksOpen ? " −" : " +"); flat: true; implicitHeight: theme.heightSm - 2; leftPadding: 0; font.pixelSize: theme.micro + 1; onClicked: otherModelLinksOpen = !otherModelLinksOpen } - FText { visible: otherModelLinksOpen; text: t("The same models are also available from Hugging Face or ModelScope.", "同一套模型，也可从 Hugging Face 或魔搭下载。"); color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true } - Flow { - visible: otherModelLinksOpen; Layout.fillWidth: true; spacing: 8 - Repeater { - model: ["video", "encoder", "decoder"] - delegate: FButton { - required property string modelData - text: modelData === "video" ? t("Video model ↗", "视频模型 ↗") : modelData === "encoder" ? t("Text encoder ↗", "文本编码器 ↗") : t("Video & audio decoders ↗", "音视频解码器 ↗") - implicitHeight: theme.heightSm; font.pixelSize: theme.micro + 1; onClicked: { modelInfo = modelData; modelInfoOpen = true } - } - } - } + FButton { objectName: "otherModelLinks"; text: downloads.copy.title + " ›"; flat: true; enabled: !s.busy; implicitHeight: theme.heightSm; leftPadding: 0; font.pixelSize: theme.micro + 1; onClicked: modelInfoOpen = true } } ColumnLayout { Layout.fillWidth: true; Layout.preferredWidth: 1; Layout.alignment: Qt.AlignTop; spacing: 10 @@ -635,7 +626,7 @@ ApplicationWindow { FText { visible: !!s.offline.detail; text: s.offline.detail + (number(s.offline.total) ? " · " + bytes(s.offline.done) + " / " + bytes(s.offline.total) : ""); color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true } } FCard { - visible: !usingRuntime; Layout.fillWidth: true; padding: 16 + visible: !manualEnvironment && !offlineSelected; Layout.fillWidth: true; padding: 16 FSwitch { objectName: "samplingCaches"; Layout.fillWidth: true text: t("Prepare all quality levels", "提前下载全部质量档位") @@ -740,7 +731,7 @@ ApplicationWindow { } // What this group downloads, as in the plan; nothing for one already here. FText { objectName: "modelSize-" + modelData.id; visible: modelData.download > 0; text: bytes(modelData.download); font.pixelSize: theme.micro; color: theme.muted; font.features: { "tnum": 1 } } - FButton { text: t("Download links ↗", "下载地址 ↗"); implicitHeight: theme.heightSm; flat: true; font.pixelSize: theme.micro + 1; visible: s.form.model_method === "manual"; onClicked: { modelInfo = modelData.id === "text" ? "encoder" : modelData.id; modelInfoOpen = true } } + FButton { objectName: "modelFiles-" + modelData.id; text: downloads.copy.title + " ›"; implicitHeight: theme.heightSm; flat: true; font.pixelSize: theme.micro + 1; visible: s.form.model_method === "manual"; onClicked: modelInfoOpen = true } } } } @@ -1278,25 +1269,111 @@ ApplicationWindow { FPopup { id: modelDialog; objectName: "modelDialog" visible: modelInfoOpen; onClosed: modelInfoOpen = false - width: Math.min(540, win.width-50) - height: Math.min(modelContents.implicitHeight + padding*2, win.height-48) - contentItem: ScrollView { - id: modelScroll; clip: true; contentWidth: availableWidth - ColumnLayout { - id: modelContents; width: modelScroll.availableWidth - spacing: 12 - FText { text: t("Download models", "下载模型"); font.pixelSize: theme.section + 2; font.weight: Font.DemiBold } - FText { objectName: "videoModelGuide"; text: modelInfo === "video" ? s.video_model_guide : modelInfo === "decoder" ? t("Download the vae and audio_vae folders.", "下载 vae 和 audio_vae 文件夹。") : t("Download the text encoder to your model folder.", "下载文本编码器，放入模型目录。"); color: theme.muted; Layout.fillWidth: true; Layout.bottomMargin: 4 } - Repeater { model: modelLinks[modelInfo]; delegate: FButton { required property var modelData; required property int index; objectName: "modelLink-" + index; text: t(modelData.label, modelData.label_zh) + " ↗"; Layout.fillWidth: true; onClicked: backend.link(modelData.url) } } - Repeater { model: cloudLinks; delegate: FButton { required property var modelData; text: t("Quark · ", "夸克 · ") + modelData.label + " ↗"; Layout.fillWidth: true; onClicked: backend.link(modelData.url) } } - FText { text: t("When your download finishes, import a FreeVideo ZIP or choose the folder containing your models.", "下载完成后，导入 FreeVideo ZIP 或选择存放模型的文件夹。"); color: theme.muted; font.pixelSize: theme.micro; Layout.fillWidth: true; Layout.topMargin: 6 } + width: Math.min(940, win.width-50) + readonly property real maximumHeight: Math.min(680, win.height-48) + implicitHeight: Math.min(maximumHeight, modelContent.implicitHeight + topPadding + bottomPadding) + contentItem: ColumnLayout { + id: modelContent; spacing: 12 + ScrollView { + id: modelScroll; objectName: "downloadListScroll" + Layout.fillWidth: true; Layout.fillHeight: true; Layout.minimumWidth: 0; Layout.minimumHeight: 0 + implicitHeight: downloadBody.implicitHeight + clip: true; contentWidth: availableWidth + ColumnLayout { + id: downloadBody; width: modelScroll.availableWidth; spacing: 12 + FText { text: downloads.copy.title; font.pixelSize: theme.section + 2; font.weight: Font.DemiBold; Layout.fillWidth: true } + FText { objectName: "downloadListTotal"; visible: downloads.available; text: downloads.total; font.weight: Font.DemiBold; color: theme.accent; Layout.fillWidth: true } + FText { objectName: "videoModelGuide"; text: downloads.copy.guide; color: theme.muted; Layout.fillWidth: true; font.pixelSize: theme.micro + 1 } + FText { objectName: "downloadListCheck"; visible: !downloads.available && !s.download_list.inspection_error; text: s.download_list.checking ? downloads.copy.checking : downloads.copy.unavailable; Layout.fillWidth: true; color: theme.muted } + FText { objectName: "downloadListReuse"; visible: downloads.available && text.length > 0; text: downloads.reused; color: theme.success; Layout.fillWidth: true; font.pixelSize: theme.micro + 1 } + ColumnLayout { + Layout.fillWidth: true; spacing: 8 + Repeater { + model: downloads.available ? downloads.groups : [] + delegate: ColumnLayout { + id: downloadGroup + required property var modelData + readonly property bool expanded: !!win.downloadExpanded[modelData.id] + Layout.fillWidth: true; spacing: 4 + FButton { + id: groupButton + objectName: "downloadGroup-" + downloadGroup.modelData.id + text: (downloadGroup.expanded ? "− " : "+ ") + downloadGroup.modelData.title + Accessible.name: text + " · " + downloadGroup.modelData.summary + Layout.fillWidth: true; flat: true; leftPadding: 8; rightPadding: 8 + contentItem: RowLayout { + spacing: 8 + FText { objectName: groupButton.objectName + "-title"; text: groupButton.text; font: groupButton.font; Layout.fillWidth: true; horizontalAlignment: Text.AlignLeft; color: groupButton.hovered ? theme.text : theme.muted } + FText { objectName: groupButton.objectName + "-summary"; text: downloadGroup.modelData.summary; font: groupButton.font; horizontalAlignment: Text.AlignRight; color: theme.muted } + } + onClicked: { + var expanded = Object.assign({}, win.downloadExpanded) + expanded[downloadGroup.modelData.id] = !downloadGroup.expanded + win.downloadExpanded = expanded + } + } + Loader { + active: downloadGroup.expanded; visible: active; Layout.fillWidth: true + sourceComponent: ColumnLayout { + spacing: 2 + Repeater { + model: downloadGroup.modelData.files + delegate: RowLayout { + required property var modelData; required property int index + objectName: "downloadFile-" + downloadGroup.modelData.id + "-" + index + Layout.fillWidth: true; spacing: 8 + FText { objectName: parent.objectName + "-name"; text: modelData.name; Layout.fillWidth: true; elide: Text.ElideMiddle; maximumLineCount: 1; font.pixelSize: theme.micro } + FText { text: modelData.size; Layout.preferredWidth: 104; font.pixelSize: theme.micro; color: theme.muted; horizontalAlignment: Text.AlignRight } + FButton { objectName: "downloadHF-" + downloadGroup.modelData.id + "-" + index; text: "Hugging Face ↗"; flat: true; implicitHeight: theme.heightSm; font.pixelSize: theme.micro; onClicked: backend.link(modelData.hf_url) } + Item { + id: mirrorSlot + readonly property var file: modelData + readonly property string suffix: downloadGroup.modelData.id + "-" + index + objectName: "downloadMSSlot-" + suffix + // Reserve the same width whether a mirror exists or not. + implicitWidth: Math.max(80, mirrorMetrics.width + 32, missingMetrics.width + 32) + implicitHeight: theme.heightSm + TextMetrics { id: mirrorMetrics; text: sourceName("modelscope") + " ↗"; font.pixelSize: theme.micro; font.weight: Font.Medium } + TextMetrics { id: missingMetrics; text: downloads.copy.missing_mirror; font.pixelSize: theme.micro } + Loader { + anchors.fill: parent + sourceComponent: mirrorSlot.file.modelscope_url ? mirrorButton : mirrorUnavailable + } + Component { + id: mirrorButton + FButton { objectName: "downloadMS-" + mirrorSlot.suffix; text: sourceName("modelscope") + " ↗"; flat: true; implicitHeight: theme.heightSm; font.pixelSize: theme.micro; onClicked: backend.link(mirrorSlot.file.modelscope_url) } + } + Component { + id: mirrorUnavailable + FText { objectName: "downloadMSMissing-" + mirrorSlot.suffix; text: downloads.copy.missing_mirror; color: theme.muted; font.pixelSize: theme.micro; horizontalAlignment: Text.AlignHCenter; verticalAlignment: Text.AlignVCenter } + } + } + } + } + } + } + } + } + } + RowLayout { + visible: s.download_list.folder.length > 0; Layout.fillWidth: true; spacing: 8 + FText { objectName: "downloadListNotice"; visible: text.length > 0; text: s.download_list.notice; color: theme.success; Layout.fillWidth: true; font.pixelSize: theme.micro + 1 } + FButton { objectName: "openDownloadFolder"; text: downloads.copy.open_folder; flat: true; implicitHeight: theme.heightSm; font.pixelSize: theme.micro + 1; onClicked: backend.openDownloadFolder() } + } + FText { objectName: "downloadListError"; visible: text.length > 0; text: s.download_list.error; color: theme.danger; Layout.fillWidth: true; wrapMode: Text.WordWrap; font.pixelSize: theme.micro + 1 } + FText { objectName: "downloadListRawError"; visible: s.error.length > 0 && !s.download_list.inspection_error; text: s.error; color: theme.danger; Layout.fillWidth: true; maximumLineCount: 2; elide: Text.ElideRight; font.pixelSize: theme.micro } + } + } RowLayout { - Layout.fillWidth: true; spacing: 8 - FButton { text: t("Close", "关闭"); flat: true; onClicked: modelInfoOpen = false } + objectName: "downloadListActions"; Layout.fillWidth: true; Layout.minimumWidth: 0; spacing: 8 + // The actions share what the Close button leaves, so a long label wraps instead of leaving the dialog. + readonly property int actionCount: checkDownloadList.visible ? 3 : 2 + readonly property real actionWidth: Math.max(80, Math.floor((width - closeDownloadList.implicitWidth - (actionCount + 1) * spacing) / actionCount)) + FButton { id: closeDownloadList; text: downloads.copy.close; flat: true; onClicked: modelInfoOpen = false } Item { Layout.fillWidth: true } - FButton { text: t("Choose model folder", "选择模型目录"); onClicked: { modelInfoOpen = false; backend.browse("model_dirs") } } - FButton { text: t("Import ZIPs", "导入 ZIP"); primary: true; onClicked: { modelInfoOpen = false; backend.browsePackages() } } - } + FButton { id: checkDownloadList; objectName: "checkDownloadList"; Layout.maximumWidth: parent.actionWidth; visible: !downloads.available || s.download_list.needs_check; text: downloads.copy.check; enabled: !s.busy; onClicked: backend.prepareDownloadList() } + FButton { objectName: "useDownloadedFolder"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.use_folder; enabled: !s.busy; onClicked: backend.useDownloadedFolder() } + FButton { objectName: "exportDownloadList"; Layout.maximumWidth: parent.actionWidth; text: downloads.copy.export; primary: true; enabled: downloads.available && !s.busy; onClicked: backend.exportDownloadList() } } } } diff --git a/freevideo_engine/launcher_copy.py b/freevideo_engine/launcher_copy.py index 6e8b991c..3cccaa0a 100644 --- a/freevideo_engine/launcher_copy.py +++ b/freevideo_engine/launcher_copy.py @@ -13,6 +13,7 @@ } ZH = dict([ + ('The network is slow. Downloading again; this may take a while.', '网络较慢，正在重试'), ('Prepare Python', '准备 Python'), ('Install Python', '安装 Python'), ('Prepare download tools', '准备下载工具'), ('Tsinghua mirror', '清华镜像'), ('Preparing installation', '准备安装'), ('Checking the environment', '正在检查运行环境'), @@ -119,6 +120,23 @@ ('Hugging Face · Video', 'Hugging Face · 视频'), ('Hugging Face · Audio', 'Hugging Face · 音频'), ]) ZH.update(NAMES.values()) +ZH.update([ + ('Completed ', '已完成 '), ('Verifying ', '正在校验'), ('Assembling ', '正在合并'), + ('Copying ', '正在复制'), ('Model files', '模型文件'), + ('Packages resolving', '正在检查组件'), ('Packages installing', '正在安装组件'), ('Packages ready', '组件已就绪'), + ('Environment creating', '正在创建运行环境'), ('Base packages installing', '正在安装基础组件'), + ('Python ready', 'Python 已就绪'), ('Python downloading', '正在下载 Python'), + ('GPU acceleration building', '正在准备显卡加速'), + ('Runtime libraries verifying', '正在校验运行组件'), ('Runtime libraries importing', '正在载入运行组件'), + ('Encoder configuring', '正在准备文本编码器'), ('Encoder importing', '正在载入文本编码器'), + ('Encoder ready', '文本编码器已就绪'), + ('GPU information reading', '正在读取显卡信息'), ('GPU tests running', '正在检查显卡'), + ('GPU tests finished', '显卡检查已完成'), + ('Packages preparing', '正在准备组件'), ('Packages removing', '正在移除旧组件'), + ('Model files verifying', '正在校验模型文件'), ('Model files preparing', '正在准备模型文件'), + ('Model files assembling', '正在合并模型文件'), + ('Package files', '离线包文件'), +]) # Card lines for steps that only report counts (the terminal keeps the counts). ZH.update([('Downloading components', '正在下载组件'), ('Preparing components', '正在准备组件'), ('Downloading runtime packages', '正在下载运行组件'), ('Finding existing models', '正在查找已有模型')]) diff --git a/freevideo_engine/launcher_session.py b/freevideo_engine/launcher_session.py index 3ca99ac6..a23e4e8a 100644 --- a/freevideo_engine/launcher_session.py +++ b/freevideo_engine/launcher_session.py @@ -1,4 +1,5 @@ """Widget-independent state for the Qt launcher; uses the existing installer.""" +from copy import deepcopy import json import os from pathlib import Path @@ -17,7 +18,8 @@ from .launcher_copy import size_text from .launcher_terminal import Tail from .model_status import FAMILIES, NAMES -from .model_guidance import package_instructions, video_instructions, runtime_packages_supported +from .model_guidance import (package_instructions, video_instructions, runtime_packages_supported, download_view, + export_download_list, DownloadListError, DOWNLOAD_COPY) from .setup_progress import progress_text from .terminal_ui import clean, duration from .model_upgrade import TARGET @@ -109,6 +111,8 @@ def __init__(self, source=None, *, controller=None, store=None, updater=None, sm self.tail = Tail() self.log_source = None self.model_groups = [] + self.download_export = self.download_error = self.download_error_key = '' + self._download_view = None self.browser_attempted = False self.browser_url = None self.port_notice_url = None @@ -258,6 +262,7 @@ def edit(self, key, value): self.cleaner.state = dict(status='idle', bytes=0, error='') self.controller.selection = None self.controller.state = dict(status='idle') + self.download_export = self.download_error = self.download_error_key = '' self.browser_attempted = False self.port_notice_url = None self.browser_error = self.error = '' @@ -284,6 +289,65 @@ def add_folder(self, folder): kept = [p for p in self.form['model_dirs'] if io_path(Path(p).expanduser()).is_dir()] self.edit('model_dirs', library_roots(kept + [folder])) + def prepare_download_list(self): + if self.controller.busy or self.importer.busy or self.cleaner.busy: + return + if (self.controller.state.get('plan', {}).get('download_files_verified') + and self.controller.state.get('status') == 'review' + and self.download_error_key not in ('unchecked', 'changed')): + return + self.error = '' + self.download_export = self.download_error = self.download_error_key = '' + self.persist() + self.controller.run('inspect', self.setup_values(download_list=True)) + + def export_downloads(self, folder): + """The dialog shows a worded error; nothing raises into a traceback.""" + self.download_export = self.download_error = self.download_error_key = '' + if self.controller.busy or self.controller.state.get('status') != 'review': + self.download_error = self.t('Wait for the file check to finish.', '请等待文件检查完成。') + return + preferences = download_preferences(self.engine_root() / 'download-settings.json') + try: + self.download_export = export_download_list(self.controller.state.get('plan', {}), folder, preferences['source']) + except DownloadListError as error: + self.download_error_key = error.key + reason = '' + if error.key == 'write': + reason = self.t(*DOWNLOAD_COPY.get('write_' + error.reason, DOWNLOAD_COPY['write_other'])) + # Session has no log writer; keep the OS detail on stderr, without the user's paths. + from .failure_details import redacted_launcher_error + print('Download list export failed (%s): %s' % (error.reason, redacted_launcher_error(error.original_error)), + file=sys.stderr) + self.download_error = self.t(*DOWNLOAD_COPY['error_' + error.key]).format(reason=reason) + + def use_downloaded_folder(self, folder): + if self.controller.busy or self.importer.busy or self.cleaner.busy: + return + self.add_folder(folder) + # Even a folder already in the list may have received new downloads. + self.controller.state = dict(status='idle') + self.prepare_download_list() + + def downloads_view(self): + row = self.controller.state + plan = row.get('plan', {}) if row.get('status') == 'review' else {} + if not plan.get('download_files_verified', True): + plan = {} + zh = self.language.startswith('zh') + decimal_sizes = sys.platform == 'darwin' + # Only inputs that affect list content belong here. Disk/progress/notice + # updates must not replace the QML model or its file delegates. + key = ('download_files' in plan, plan.get('download_files', []), + (plan.get('local_models') or {}).get('matches', {}), zh, decimal_sizes) + if self._download_view is None or self._download_view[0] != key: + view = download_view(plan, zh, decimal_sizes=decimal_sizes) + if self._download_view is None or self._download_view[1] != view: + self._download_view = (deepcopy(key), view) + else: + self._download_view = (deepcopy(key), self._download_view[1]) + return self._download_view[1] + def remove_folder(self, index): removed = self.form['model_dirs'][index] self.edit('model_dirs', [p for i, p in enumerate(self.form['model_dirs']) if i != index]) @@ -319,10 +383,20 @@ def prepare_offline(self): if self.archives_busy: return if self.form['model_method'] == 'auto' and not self.form['offline_models']: - self.importer.download(self.form['offline_runtime'], self.source, self.form['sampling_caches']) + self.importer.download(self.form['offline_runtime'], self.source, False) else: self.importer.prepare(self.form['offline_runtime'], self.form['offline_models'], self.source) + def setup_values(self, form=None, **extra): + """Only the visible automatic-setup option can request every quality level.""" + form = self.form if form is None else form + values = dict(form, token=self.token, **extra) + if (form['model_method'] == 'manual' or + (runtime_packages_supported() and form['new_comfy'] + and form['environment_method'] == 'manual')): + values['sampling_caches'] = False + return values + def activate_offline(self, root): from .offline_packages import check_runtime_platform check_runtime_platform() @@ -370,7 +444,7 @@ def action(self, name, accepted=False): self.remember(self.selected, persist=False) self.repair_request = dict(self.form, repair=True) self.persist() - self.controller.run('inspect', dict(self.repair_request, token=self.token)) + self.controller.run('inspect', self.setup_values(self.repair_request)) self.page = 'progress' self.model_groups = [] self.started = time.monotonic() @@ -390,7 +464,7 @@ def action(self, name, accepted=False): # Recompute the plan and live disk space; never install a stale # failed review or discard completed/resumable downloads. self.persist() - self.controller.run('inspect', dict(self.repair_request or self.form, token=self.token)) + self.controller.run('inspect', self.setup_values(self.repair_request or self.form)) self.page = 'progress' self.started = time.monotonic() return @@ -471,7 +545,7 @@ def action(self, name, accepted=False): self.controller.run('connect') else: self.persist() - self.controller.run('inspect', dict(self.repair_request or self.form, token=self.token)) + self.controller.run('inspect', self.setup_values(self.repair_request or self.form)) self.page = 'progress' self.model_groups = [] self.started = time.monotonic() @@ -605,7 +679,7 @@ def _tick_model_upgrade(self, row, busy): self.model_switch = 'inspect' self.repair_request = None upgrade.switching() - self.controller.run('inspect', dict(self.form, token=self.token, repair=True, prepared_format='int8')) + self.controller.run('inspect', self.setup_values(repair=True, prepared_format='int8')) self.page = 'progress' self.model_groups = [] self.started = time.monotonic() @@ -944,7 +1018,7 @@ def _update_engine_when_idle(self): self.port_notice_url = None self.browser_wait = None self.persist() - self.controller.run('inspect', dict(self.form, token=self.token)) + self.controller.run('inspect', self.setup_values()) self.page = 'progress' self.model_groups = [] self.started = time.monotonic() @@ -1632,6 +1706,10 @@ def snapshot(self): done=min(group['download_bytes'], group.get('downloaded_bytes') or 0), rate=group.get('bytes_per_second') or progress.get('rate')) file_text = progress_text(current, zh, decimal_sizes=sys.platform == 'darwin') + if self.importer.busy and self.importer.state.get('activity'): + imported = self.importer.state + file_text = progress_text(dict(activity=imported['activity'], unit='bytes', + name=display('Package files', zh), done=imported.get('activity_done'), total=imported.get('activity_total')), zh) # The time row's estimate: only while downloading, from what is left at the current speed. from .setup_progress import number left, rate = (current.get('total') or 0) - (current.get('done') or 0), current.get('rate') @@ -1708,7 +1786,16 @@ def snapshot(self): self.archives_prompt_offer if self.archives_retry_dialog else self.archive_offer()) archives_idle = bool(archive_offer and archive_offer['paths']) and self.archives_idle() archive_dialog = self.archives_dialog_view(archive_offer, archives_idle, error, update) - return plain_data(dict(sampling_cache_bytes=total_bytes(), version=__version__, + downloads = self.downloads_view() + inspection_read_error = (row.get('status') == 'failed' and row.get('action') == 'inspect' + and 'Download list error: read' in row.get('error', '').splitlines()) + download_state = dict(checking=bool(self.controller.busy), error=self.download_error, + needs_check=self.download_error_key in ('unchecked', 'changed'), folder=self.download_export, + inspection_error=inspection_read_error, + notice=downloads['copy']['exported'].format(folder=self.download_export) if self.download_export else '') + if inspection_read_error: + download_state.update(error=self.t(*DOWNLOAD_COPY['error_read']), needs_check=True) + return plain_data(dict(download_list=download_state, sampling_cache_bytes=total_bytes(), version=__version__, disk=disk, decimal_sizes=sys.platform == 'darwin', zh=self.language.startswith('zh'), form=dict(self.form), page=self.page, status=row.get('status', 'idle'), busy=self.controller.busy or self.importer.busy or self.cleaner.busy, cleanup=cleanup, diff --git a/freevideo_engine/media_encoding.py b/freevideo_engine/media_encoding.py index a67d4e34..a81c7798 100644 --- a/freevideo_engine/media_encoding.py +++ b/freevideo_engine/media_encoding.py @@ -34,7 +34,7 @@ def reference_size(width, height, canvas): return max(32, round(width * scale / 32) * 32), max(32, round(height * scale / 32) * 32) -def encode_visual(vae, pixels, *, device='cuda'): +def encode_visual(vae, pixels, *, device='cuda', posterior='sample'): """The pinned VAE chunk/padding/posterior recipe with one live pixel chunk. Posterior sampling still occurs once on the original device with seed 42; @@ -58,7 +58,9 @@ def encode_visual(vae, pixels, *, device='cuda'): moments = torch.cat(moments, dim=2).to(device) if len(pixels) > 1 and vae.config.token_drop: moments = moments[:, :, :-vae.config.token_drop] - latent = DiagonalGaussianDistribution(moments).sample(generator=torch.Generator().manual_seed(42)) + distribution = DiagonalGaussianDistribution(moments) + # References keep the pipeline's seeded sample; a continuation context wants the picture itself. + latent = distribution.mode() if posterior == 'mode' else distribution.sample(generator=torch.Generator().manual_seed(42)) latent = latent.to(torch.float16).float().cpu() mean = torch.tensor(vae.config.latents_mean).view(1, -1, 1, 1, 1) std = torch.tensor(vae.config.latents_std).view(1, -1, 1, 1, 1) diff --git a/freevideo_engine/model_files.json b/freevideo_engine/model_files.json index 51a4513a..9e215d8e 100644 --- a/freevideo_engine/model_files.json +++ b/freevideo_engine/model_files.json @@ -5,7 +5,8 @@ "file": "LICENSE", "bytes": 17604, "sha256": null, - "git_blob": "c389b45855337ccec8ddeb389f4fe902abcd0b19" + "git_blob": "c389b45855337ccec8ddeb389f4fe902abcd0b19", + "download_sha256": "59b99642b95ea21630e311198ddbfffbfe05aadba0c2f5d884cbdf4efcc90f44" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -13,7 +14,8 @@ "file": "h3-base/LICENSE", "bytes": 17604, "sha256": null, - "git_blob": "c389b45855337ccec8ddeb389f4fe902abcd0b19" + "git_blob": "c389b45855337ccec8ddeb389f4fe902abcd0b19", + "download_sha256": "59b99642b95ea21630e311198ddbfffbfe05aadba0c2f5d884cbdf4efcc90f44" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -21,7 +23,8 @@ "file": "h3-base/README.md", "bytes": 38391, "sha256": null, - "git_blob": "aa9bba6233ed8d0b486fb43e1a35b4fb5ede2a60" + "git_blob": "aa9bba6233ed8d0b486fb43e1a35b4fb5ede2a60", + "download_sha256": "89a98af9e91fcb54c46cbcb88f8b8aa7aaf3a461714dbf21d2d3ef4b39312ee3" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -29,7 +32,8 @@ "file": "h3-base/audio_scheduler/scheduler_config.json", "bytes": 96, "sha256": null, - "git_blob": "28a3d4ed602d925031c7e5d6eeb07df8958ff0f1" + "git_blob": "28a3d4ed602d925031c7e5d6eeb07df8958ff0f1", + "download_sha256": "804780f7133477067bd6bbfbc02dc8b3cf9feeb400f97c08f5b1d5f6cbab3840" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -37,7 +41,8 @@ "file": "h3-base/audio_vae/config.json", "bytes": 2271, "sha256": null, - "git_blob": "15df6fe0b288b260dca0f2aebf1cd81334d77e96" + "git_blob": "15df6fe0b288b260dca0f2aebf1cd81334d77e96", + "download_sha256": "9a3c645ff892b376c6f5f4c8685964cd75474731af594ff058492a0000caabb6" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -53,7 +58,8 @@ "file": "h3-base/model_index.json", "bytes": 2936, "sha256": null, - "git_blob": "454be19164878173a962c39c81633e4377880fca" + "git_blob": "454be19164878173a962c39c81633e4377880fca", + "download_sha256": "5a587fe13b2371427415ac892463142683aefcd8d322e274a3a095eac37ac7d2" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -61,7 +67,8 @@ "file": "h3-base/modular_model_index.json", "bytes": 2935, "sha256": null, - "git_blob": "5b69f1e801bd0a7e9733c48704c732339f2002fe" + "git_blob": "5b69f1e801bd0a7e9733c48704c732339f2002fe", + "download_sha256": "a2b6a210e482ffb78e613b553f570c44e101afce6741bd4ed91429d0559af031" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -69,7 +76,8 @@ "file": "h3-base/scheduler/scheduler_config.json", "bytes": 97, "sha256": null, - "git_blob": "1d44f95b955a648ad4fab455e4fe11053888c30b" + "git_blob": "1d44f95b955a648ad4fab455e4fe11053888c30b", + "download_sha256": "8fa6c3aa70dc9e691e1a6df899fd1b6f75f70481a27cee6e18a303817075c304" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -77,7 +85,8 @@ "file": "h3-base/transformer/config.json", "bytes": 546, "sha256": null, - "git_blob": "c251a4133c81852a150c6668a570e142dc776b6e" + "git_blob": "c251a4133c81852a150c6668a570e142dc776b6e", + "download_sha256": "74c11bff524336576096993cbfcdcdc2ef4fa2fa4409df693bdcbc6c666282ae" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -197,7 +206,8 @@ "file": "h3-base/transformer/diffusion_pytorch_model.safetensors.index.json", "bytes": 64488, "sha256": null, - "git_blob": "da038def25abb5a5e4be509816ee55a0b7105e09" + "git_blob": "da038def25abb5a5e4be509816ee55a0b7105e09", + "download_sha256": "ac30a3b58963f2e735d493475fbb81853a5735ec947619648b3e045acda6783e" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -205,7 +215,8 @@ "file": "h3-base/vae/config.json", "bytes": 2011, "sha256": null, - "git_blob": "ed4874a5886972f4029fcd99cbbe72de0177b2c9" + "git_blob": "ed4874a5886972f4029fcd99cbbe72de0177b2c9", + "download_sha256": "78f67deec3d63aae807f2bfe7154bc1e26f6372cb20b63265fcbae1b62bb5745" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -237,7 +248,8 @@ "file": "h3-base/vae/diffusion_pytorch_model.safetensors.index.json", "bytes": 74228, "sha256": null, - "git_blob": "05dfb255c44daa4f1ac487f57f18210fa5c19439" + "git_blob": "05dfb255c44daa4f1ac487f57f18210fa5c19439", + "download_sha256": "15f6d44553c3c616b0dc999920aa784f92ecee7e4201f1f99ac405cfbf3061ca" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -245,7 +257,8 @@ "file": "stage-dmd-step-250/adapters/default/adapter_config.json", "bytes": 415, "sha256": null, - "git_blob": "ef3c876ab5a596a1b6b4b28f3443bbe30c5455d8" + "git_blob": "ef3c876ab5a596a1b6b4b28f3443bbe30c5455d8", + "download_sha256": "89e0ff8920b9629b826eb99ab6150cce6924a53aa445186d1b493225fd091b96" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -261,7 +274,8 @@ "file": "stage-dmd-step-250/adapters/turbo/adapter_config.json", "bytes": 22264, "sha256": null, - "git_blob": "dbd7be544c9f3092b229f5cb6487f8da2f4ccc05" + "git_blob": "dbd7be544c9f3092b229f5cb6487f8da2f4ccc05", + "download_sha256": "627968f670747c29cd7a0d3f8c75166e501d70f9e829b5cd3242a8f14cefbc18" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -277,7 +291,8 @@ "file": "stage-dmd-step-250/linear_branch/config.json", "bytes": 465, "sha256": null, - "git_blob": "314502a7c9d403f28493ac1cb5054032c8c7b545" + "git_blob": "314502a7c9d403f28493ac1cb5054032c8c7b545", + "download_sha256": "decb06ac7e664610f677fb445318502b3c51f9c1b8603a2cdd00b16042be5bc8" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -293,7 +308,8 @@ "file": "stage-dmd-step-250/metadata.json", "bytes": 463, "sha256": null, - "git_blob": "27ec152079222e3f2a83b8ee0bc634a77da0df01" + "git_blob": "27ec152079222e3f2a83b8ee0bc634a77da0df01", + "download_sha256": "54054ceb1c91b3fdf7fa0278e4a8841c127e8cf666b5e240d69613661f9d3e9e" }, { "repo": "OpenVDN/vdn-minimax-h3", @@ -301,7 +317,8 @@ "file": "stage-dmd-step-250/model_spec.json", "bytes": 25705, "sha256": null, - "git_blob": "e039c13987797f91e11a3477c38fa8fd1f6a506f" + "git_blob": "e039c13987797f91e11a3477c38fa8fd1f6a506f", + "download_sha256": "4171f4384e952f1f73467981a893440c03298af4956b947e2c8a857ba9f5a62b" }, { "repo": "t8star/Vdn-Minimax-H3-Comfy", diff --git a/freevideo_engine/model_guidance.py b/freevideo_engine/model_guidance.py index 2c50da50..99dc560c 100644 --- a/freevideo_engine/model_guidance.py +++ b/freevideo_engine/model_guidance.py @@ -1,9 +1,302 @@ """User-facing model download destinations, from the installation manifests.""" +from contextlib import ExitStack +import errno +import hashlib import json -from pathlib import Path +import os +from pathlib import Path, PurePosixPath +import re +import shutil import sys -from urllib.parse import quote, urlsplit +import tempfile +import time +import uuid +from urllib.parse import urlsplit + +from . import network from .long_paths import io_path, plain_errors +from .model_status import FAMILIES, NAMES, family + +# Iris can replace the third-party download copy here, including export guidance. +DOWNLOAD_COPY = { + 'title': ('Download list', '下载清单'), + 'guide': ('The list is filtered for this GPU, so the whole model repository is not needed. ' + 'When the download finishes, choose "Use downloaded folder"; FreeVideo checks every file before using it.', + '清单已按显卡筛选，无需下载整个模型仓库。下载完成后点“使用已下载的文件夹”，FreeVideo 会逐个校验后使用。'), + 'export': ('Export download list', '导出下载清单'), + 'use_folder': ('Use downloaded folder', '使用已下载的文件夹'), + 'checking': ('Checking the hardware and the downloaded files…', '正在检查硬件和已下载的文件…'), + 'check': ('Check again', '重新检查'), + 'unavailable': ('The download list is not ready yet. Choose "Check again".', '下载清单尚未生成，请点“重新检查”。'), + 'total': ('To download: {count} files ({size})', '需要下载：{count} 个文件（{size}）'), + 'total_remaining': ('Still to download: {count} files ({size})', '还需下载：{count} 个文件（{size}）'), + 'group': ('{count} files · {size}', '{count} 个文件 · {size}'), + 'reused': ('Checked and ready to use: {count} files ({size})', '已校验、可直接使用：{count} 个文件（{size}）'), + 'exported': ('Download list exported to {folder}', '下载清单已导出到 {folder}'), + 'open_folder': ('Open folder', '打开文件夹'), + 'close': ('Close', '关闭'), + 'missing_mirror': ('Not on ModelScope', '魔搭暂无'), + 'error_folder': ('Choose an existing folder to save the download list in.', '请选择一个已有的文件夹来保存下载清单。'), + 'error_unchecked': ('Choose "Check again" first, then export the download list.', '请先点“重新检查”，再导出下载清单。'), + 'error_changed': ('The required files have changed. Choose "Check again", then export the download list.', + '所需文件有变化。请点“重新检查”，再导出下载清单。'), + 'error_write': ('The download list could not be saved in this folder: {reason}', '下载清单未能保存到这个文件夹：{reason}'), + 'error_read': ('FreeVideo does not have permission to read some files in the selected folder. Choose another folder, or download the files again without administrator rights.', + 'FreeVideo 没有权限读取所选文件夹里的部分文件。请选择其他文件夹，或不使用管理员身份重新下载。'), + 'write_permission': ('this account cannot write to it. Choose another folder.', '当前账户没有写入权限，请换一个文件夹。'), + 'write_disk_full': ('the drive is full.', '磁盘空间不足。'), + 'write_path_long': ('the path is too long. Choose a folder closer to the top of the drive.', '路径太长，请选择层级更浅的文件夹。'), + 'write_other': ('Choose another folder and try again.', '请换一个文件夹再试一次。'), + 'video': NAMES['video'], 'text': NAMES['text'], 'decoder': NAMES['decoder'], 'sampling': NAMES['sampling'], + 'readme': ( + 'FreeVideo download list\n' + '\n' + 'Run this command in this folder:\n' + 'aria2c --input-file=aria2.txt --continue=true --check-integrity=true\n' + '\n' + 'You can also import urls.txt into another download tool. urls.txt uses the download source chosen in Settings (Automatic uses Hugging Face); a few files are only available from Hugging Face. Keep all downloaded files in this folder or its subfolders.\n' + '\n' + 'When the download finishes, return to the launcher, choose "Use downloaded folder", then "Check & continue". FreeVideo recognizes files by size and full checksum, so different file names are fine; missing or damaged files are downloaded as usual.\n' + '\n' + 'The paths in SHA256SUMS are where aria2 saves the files. With urls.txt the file names may differ, which does not matter.\n' + '\n' + 'There is no need to download the whole model repository. This folder is not an offline package; do not import it with "Choose offline packages".\n', + 'FreeVideo 下载清单\n' + '\n' + '在这个文件夹里运行下面的命令：\n' + 'aria2c --input-file=aria2.txt --continue=true --check-integrity=true\n' + '\n' + '也可以把 urls.txt 导入其他下载工具。urls.txt 用的是设置里选中的下载源（选“自动”时用 Hugging Face）；个别文件只能从 Hugging Face 下载。请把下载的文件都放在这个文件夹或它的子文件夹里。\n' + '\n' + '下载完成后，回到启动器点“使用已下载的文件夹”，再点“检查并继续”。FreeVideo 按文件大小和完整校验值识别文件，文件名不同也可以；缺少或损坏的文件会照常下载。\n' + '\n' + 'SHA256SUMS 里的路径是 aria2 的存放路径。用 urls.txt 下载时文件名可能不同，不影响使用。\n' + '\n' + '不必下载整个模型仓库。这个文件夹不是离线包，不要用“选择离线包”导入。\n' + '\n'), +} + + +# The export goes into a folder of its own inside the chosen one, so files the +# user keeps there (a README.txt, say) are never overwritten. +EXPORT_FOLDER = 'FreeVideo-download-list' +GENERATOR = 'FreeVideo download list' +EXPORT_FILES = ('manifest.json', 'aria2.txt', 'SHA256SUMS', 'urls.txt', 'README.txt') + + +class DownloadListError(ValueError): + """key names the DOWNLOAD_COPY 'error_' text the launcher shows.""" + def __init__(self, key, reason='', original_error=''): + self.key, self.reason = key, reason + self.original_error = original_error + super().__init__(DOWNLOAD_COPY['error_' + key][0].format(reason=reason)) + + +def download_write_reason(error, folder): + """Stable reason codes; OS messages are retained only for diagnostics.""" + if isinstance(error, PermissionError) or error.errno in (errno.EACCES, errno.EPERM): + return 'permission' + winerror = getattr(error, 'winerror', None) + if error.errno == errno.ENOSPC or winerror in (112, 39): + return 'disk_full' + paths = (error.filename, error.filename2, folder) + if (error.errno == errno.ENAMETOOLONG or winerror == 206 + or winerror == 3 and any(path is not None and len(str(path)) > 260 for path in paths)): + return 'path_long' + return 'other' + + +def download_copy(zh=False): + return {key: value[bool(zh)] for key, value in DOWNLOAD_COPY.items()} + + +def download_files(entries): + """Project the install plan's placements; never select another GPU variant.""" + result = [] + for row, status in entries: + if status != 'download': + continue + mirror = None if row.get('official_only') else network.modelscope_mapping(row) + result.append(dict(group=family(row), repo=row['repo'], hf_revision=row['revision'], + modelscope_repo=mirror['repo'] if mirror else '', modelscope_revision=mirror['revision'] if mirror else '', + file=row['file'], bytes=row['bytes'], sha256=row.get('sha256') or row.get('download_sha256'), + relative_path='/'.join(('payload', row['repo'], row['file'])), + hf_url=network.model_url(row, 'official', env={}), + modelscope_url=network.model_url(row, 'modelscope', env={}) if mirror else '')) + return result + + +def download_view(plan, zh=False, *, decimal_sizes=False): + from .launcher_copy import size_text + def size(value): + if value < (10**6 if decimal_sizes else 2**20): + unit = 'MB' if decimal_sizes else 'MiB' + return ('不到 1 ' if zh else 'under 1 ') + unit + return size_text(value, decimal_sizes=decimal_sizes) + copy = download_copy(zh) + rows = plan.get('download_files', []) + groups = [] + for group in FAMILIES: + files = [dict(row, name=row['file'], size=size(row['bytes'])) for row in rows if row['group'] == group] + amount = sum(row['bytes'] for row in files) + groups.append(dict(id=group, title=copy[group], files=files, count=len(files), bytes=amount, + summary=copy['group'].format(count=len(files), size=size(amount)))) + reused = (plan.get('local_models') or {}).get('matches', {}) + return dict(available='download_files' in plan, groups=groups, copy=copy, + total_bytes=sum(row['bytes'] for row in rows), count=len(rows), + total=copy['total_remaining' if reused else 'total'].format(size=size(sum(row['bytes'] for row in rows)), count=len(rows)), + reused=copy['reused'].format(count=len(reused), size=size(sum(r['bytes'] for r in reused.values()))) if reused else '') + + +def export_download_list(plan, folder, source='auto'): + """Offline export. The imported manifest is never trusted as a hash source.""" + folder = Path(folder).expanduser().resolve() + if not folder.is_dir(): + raise DownloadListError('folder') + if 'download_files' not in plan or not plan.get('download_files_verified', True): + raise DownloadListError('unchecked') + if source not in ('auto', 'official', 'hf-mirror', 'modelscope'): + raise DownloadListError('changed') + rows = plan['download_files'] + outputs = set() + aria, sums, urls = [], [], [] + for row in rows: + relative = row['relative_path'] + path = PurePosixPath(relative) + if (path.is_absolute() or any(part in ('', '.', '..') for part in relative.split('/')) + or any(char in relative for char in '\\:\r\n\t\x00') or relative in outputs + or not re.fullmatch('[0-9a-f]{64}', row.get('sha256') or '') + or not re.fullmatch('[0-9a-f]{40}', row['hf_revision']) + or type(row['bytes']) is not int or row['bytes'] <= 0): + raise DownloadListError('changed') + outputs.add(relative) + # Use only catalog pins and URLs generated by network.model_url. + original = dict(repo=row['repo'], revision=row['hf_revision'], file=row['file']) + hf = network.model_url(original, 'official', env={}) + mirror = network.model_url(original, 'modelscope', env={}) if row['modelscope_url'] else '' + if row['hf_url'] != hf or row['modelscope_url'] != mirror: + raise DownloadListError('changed') + aria.extend(['\t'.join(url for url in (hf, mirror) if url), + ' dir=' + path.parent.as_posix(), ' out=' + path.name, + ' checksum=sha-256=' + row['sha256']]) + sums.append(row['sha256'] + ' ' + relative) + chosen = 'official' if source == 'auto' or (source == 'modelscope' and not mirror) else source + urls.append(network.model_url(original, chosen, env={})) + prepared = plan.get('prepared_model') or {} + manifest = dict(schema_version=1, generator=GENERATOR, engine_version=plan.get('engine_version'), + video_model=dict(format=prepared.get('scale_granularity') or plan.get('model_scale_granularity') or plan.get('prepared_format'), + reason=plan.get('model_source_reason', ''), source=plan.get('model_source')), + sampling_caches=bool(plan.get('sampling_caches')), source=source, + total_bytes=sum(row['bytes'] for row in rows), files=rows) + documents = {'manifest.json': json.dumps(manifest, ensure_ascii=False, indent=2) + '\n', + 'aria2.txt': '\n'.join(aria) + ('\n' if aria else ''), + 'SHA256SUMS': '\n'.join(sums) + ('\n' if sums else ''), + 'urls.txt': '\n'.join(urls) + ('\n' if urls else ''), + 'README.txt': DOWNLOAD_COPY['readme'][1] + '\n' + DOWNLOAD_COPY['readme'][0]} + # Finish every file before anything appears or is replaced. No payload is downloaded. + stage = None + try: + # Not mkdtemp: Python 3.12.4 protects its Windows ACL from inheritance. + # Downloads made by an administrator must remain readable by the account + # that chose this folder, as with desktop_shortcut's launcher staging. + temporary = folder / ('.freevideo-list-' + uuid.uuid4().hex) + temporary.mkdir() + stage = temporary + for name, content in documents.items(): + (stage / name).write_text(content, encoding='utf-8', newline='\n') + # Two launchers exporting into one folder take turns: the five files + # always come from the same export. + with export_lock(folder): + target, earlier = export_target(folder) + if earlier: + replace_export(stage, target) + else: + os.rename(stage, target) # One step: the folder appears complete or not at all. + stage = None + except OSError as error: + raise DownloadListError('write', download_write_reason(error, folder), str(error)) from error + finally: + if stage is not None: + shutil.rmtree(stage, ignore_errors=True) + return str(target) + + +def export_lock(folder, wait=30.): + """Hold the cross-process lock of one export folder; waits for another export. + + The lock file lives in the temporary directory, not in the user's folder. + """ + from .locking import runtime_lock + key = hashlib.sha256(os.path.normcase(str(folder)).encode('utf-8')).hexdigest()[:16] + path = Path(tempfile.gettempdir()) / ('freevideo-download-list-%s.lock' % key) + deadline = time.monotonic() + wait + while True: + stack = ExitStack() + try: + stack.enter_context(runtime_lock(path, inherit=False)) + return stack + except BlockingIOError: + stack.close() + if time.monotonic() > deadline: + raise OSError(errno.EBUSY, 'Another download list export into this folder did not finish') + time.sleep(.1) + + +def earlier_export(path): + """A folder this launcher exported before, identified by its manifest.""" + try: + if path.is_symlink() or not path.is_dir(): + return False + value = json.loads((path / 'manifest.json').read_text(encoding='utf-8')) + except (OSError, ValueError): + return False + return isinstance(value, dict) and value.get('generator') == GENERATOR + + +def export_target(folder): + """(folder, earlier): our earlier export is refreshed; anything else is left alone.""" + for index in range(1, 100): + path = folder / (EXPORT_FOLDER if index == 1 else '%s-%d' % (EXPORT_FOLDER, index)) + if not os.path.lexists(path): + return path, False + if earlier_export(path): + return path, True + raise OSError(errno.EEXIST, 'Too many folders named ' + EXPORT_FOLDER) + + +def replace_export(stage, target): + """Swap in the five list files; downloads already in the folder stay. + + The files always belong to one export: if any move fails, the earlier ones + are put back before the error is raised. + """ + previous, placed = [], [] + try: + for name in EXPORT_FILES: + current = target / name + if os.path.lexists(current): + kept = target / ('.%s.previous-%s' % (name, os.urandom(4).hex())) + os.replace(current, kept) + previous.append((kept, current)) + os.replace(stage / name, current) + placed.append(current) + except OSError: + for current in placed: + try: + os.unlink(current) + except OSError: + pass + for kept, current in previous: + try: + os.replace(kept, current) + except OSError: + pass + raise + for kept, _ in previous: + try: + os.unlink(kept) + except OSError: + pass def runtime_packages_supported(): @@ -48,22 +341,12 @@ def video_instructions(zh=False): '按模型页说明选择显卡版本，保留下载的目录结构。')[bool(zh)] -@plain_errors -def links(): - root = io_path(Path(__file__).parent) - spec = json.loads((root / 'dependencies.json').read_text(encoding='utf-8'))['models'] - edge = json.loads((root / 'prepared_models.json').read_text(encoding='utf-8')) - return { - 'video': [('Hugging Face', 'https://huggingface.co/' + edge['repo']), - ('ModelScope', 'https://modelscope.ai/models/' + spec['edge_modelscope']['repo'])], - 'encoder': [('Hugging Face', 'https://huggingface.co/' + spec['encoder_repo'] + '/blob/' + - spec['encoder_revision'] + '/' + quote(spec['encoder_file'], safe='/'))], - 'decoder': [('Hugging Face · Video', 'https://huggingface.co/' + spec['vdn_repo'] + '/tree/' + - spec['vdn_revision'] + '/h3-base/vae'), - ('Hugging Face · Audio', 'https://huggingface.co/' + spec['vdn_repo'] + '/tree/' + - spec['vdn_revision'] + '/h3-base/audio_vae'), - ('ModelScope', 'https://modelscope.ai/models/' + spec['vdn_modelscope']['repo'])], - } +def links(plan=None): + """Compatibility for the Tk launcher: fixed files from a checked plan only.""" + return {group: [(row['file'] + ' · ' + source, row[key]) + for row in (plan or {}).get('download_files', []) if row['group'] == group + for source, key in (('Hugging Face', 'hf_url'), ('ModelScope', 'modelscope_url')) if row[key]] + for group in FAMILIES} @plain_errors @@ -103,7 +386,8 @@ def show_component(app, component): body = app.ttk.Frame(dialog, padding=24); body.pack(fill='both', expand=True) app.ttk.Label(body, text=app.t(*titles[component]), style='Heading.TLabel').pack(anchor='w') app.ttk.Label(body, text=app.t(*instructions[component]), wraplength=490).pack(fill='x', pady=12) - for title, url in links()[component]: + checked = app.controller.state.get('plan', {}) + for title, url in links(checked)['text' if component == 'encoder' else component]: app.ttk.Button(body, text=display(title, app.zh) + ' ↗', command=lambda u=url: webbrowser.open(u)).pack(fill='x', pady=4) for row in cloud_models(): app.ttk.Button(body, text=app.t('Quark model pack · ', '夸克模型包 · ') + row['label'] + ' ↗', diff --git a/freevideo_engine/model_transfer.py b/freevideo_engine/model_transfer.py index d1a9ff30..328e519d 100644 --- a/freevideo_engine/model_transfer.py +++ b/freevideo_engine/model_transfer.py @@ -135,10 +135,11 @@ def update(self, amount, *, transfer=False): self.last = now -def verify(path, row): +def verify(path, row, progress=None): if path.stat().st_size != row['bytes']: raise ValueError('Downloaded size differs from the pinned manifest') - return network.hash_file(path, 'sha256' if row.get('sha256') else 'sha1', not row.get('sha256'), discard_cache=True) == (row.get('sha256') or row['git_blob']) + return network.hash_file(path, 'sha256' if row.get('sha256') else 'sha1', not row.get('sha256'), + discard_cache=True, progress=progress) == (row.get('sha256') or row['git_blob']) def checked_range_response(response, requested, total, progress): @@ -187,10 +188,13 @@ def sdk_worker(request): progress = Progress(row['bytes']) backend = [] started = time.monotonic() + from .install_activity import byte_progress + verifying = byte_progress('verify_progress', target, + report=lambda **row: emit(phase='verify', **row)) try: if target.is_file(): - emit(phase='verify') - if verify(target, row): + emit(phase='verify', done_bytes=0, total_bytes=row['bytes']) + if verify(target, row, progress=verifying): emit(phase='ready', backend='verified-stage', seconds=time.monotonic()-started, fingerprint=fingerprint(target)) return 0 network.retain_partial(target, 'hash-rejected') @@ -213,7 +217,14 @@ def __init__(self, filename, total): def update(self, amount): progress.update(0) def end(self): - emit(phase='reconstruct', done_bytes=row['bytes'], total_bytes=row['bytes']) + emit(phase='reconstruct', done_bytes=0, total_bytes=row['bytes']) + def merge_progress(self, done, total): + merging(done, total) + def verify_progress(self, done, total): + verifying(done, total) + + merging = byte_progress('assemble_progress', target, + report=lambda **row: emit(phase='reconstruct', **row)) original_get, original_parallel = _download.requests.get, getattr(_download, '_parallel_download', None) @@ -318,8 +329,8 @@ def report(done, total, rate): # count alone does not prove that the file descriptor is closed. emit(phase='transfer_complete', done_bytes=row['bytes'], total_bytes=row['bytes']) downloaded = time.monotonic() - emit(phase='verify') - if not verify(target, row): + emit(phase='verify', done_bytes=0, total_bytes=row['bytes']) + if not verify(target, row, progress=verifying): raise ValueError('Downloaded hash differs from the pinned manifest') emit(phase='ready', backend=backend[-1] if backend else 'hf-cache', download_seconds=downloaded-started, verification_seconds=time.monotonic()-downloaded, @@ -524,6 +535,12 @@ def read_output(): network_bytes_per_second=current.get('transfer_bytes', 0) / max(.001, current.get('elapsed_seconds', time.monotonic()-started)), method=current.get('backend')) + if current['phase'] in ('verify', 'reconstruct') and current.get('total_bytes'): + from .model_status import NAMES, family + sys.stdout.write(json.dumps(dict(event='verify_progress' if current['phase'] == 'verify' else 'assemble_progress', + file=str(path), name=NAMES[family(row)][0], done_bytes=current.get('done_bytes', 0), + total_bytes=current['total_bytes'])) + '\n') + sys.stdout.flush() prior_phase = current['phase'] last_emit = time.monotonic() # The SDK reports the transfer complete before it starts its final diff --git a/freevideo_engine/modern_launcher.py b/freevideo_engine/modern_launcher.py index 5122321f..a651d0c3 100644 --- a/freevideo_engine/modern_launcher.py +++ b/freevideo_engine/modern_launcher.py @@ -77,10 +77,12 @@ def create_ui(session, *, show=True): class Bridge(QObject): changed = Signal() + downloadsChanged = Signal() def __init__(self): super().__init__(engine) self.alerted = None self.value = session.snapshot() + self.download_value = session.downloads_view() self.timer = QTimer(self) self.timer.setInterval(250) self.timer.timeout.connect(self.refresh) @@ -90,6 +92,10 @@ def __init__(self): def state(self): return self.value + @Property('QVariantMap', notify=downloadsChanged) + def downloadList(self): + return self.download_value + def invoke(self, function, *args, **kwargs): try: function(*args, **kwargs) @@ -105,6 +111,10 @@ def refresh(self): if value != self.value: self.value = value self.changed.emit() + downloads = session.downloads_view() + if downloads is not self.download_value: + self.download_value = downloads + self.downloadsChanged.emit() update = value.get('update') or {} if update.get('remind') and update.get('key') != self.alerted and engine.rootObjects(): # Flash the taskbar button once per new reminder when the @@ -166,6 +176,26 @@ def browse(self, key): if path: self.invoke(session.add_folder, path) if key == 'model_dirs' else self.invoke(session.edit, key, path) + @Slot() + def prepareDownloadList(self): + self.invoke(session.prepare_download_list) + + @Slot() + def exportDownloadList(self): + if session.controller.busy: + return + path = QFileDialog.getExistingDirectory(None, session.downloads_view()['copy']['export'], '', QFileDialog.ShowDirsOnly) + if path: + self.invoke(session.export_downloads, path) + + @Slot() + def useDownloadedFolder(self): + if session.controller.busy or session.importer.busy: + return + path = QFileDialog.getExistingDirectory(None, session.downloads_view()['copy']['use_folder'], '', QFileDialog.ShowDirsOnly) + if path: + self.invoke(session.use_downloaded_folder, path) + @Slot() def browsePackages(self): if session.controller.busy or session.importer.busy: @@ -237,6 +267,13 @@ def link(self, value): else: self.invoke(open_browser, value) + @Slot() + def openDownloadFolder(self): + from PySide6.QtGui import QDesktopServices + folder = session.download_export + if folder and Path(folder).is_dir(): + self.invoke(QDesktopServices.openUrl, QUrl.fromLocalFile(str(Path(folder).resolve()))) + @Slot(str) def selectSource(self, value): self.invoke(session.select_source, value) @@ -292,17 +329,11 @@ def shutdown(self): self.timer.stop() session.close() - from .model_guidance import links, cloud_models - from .launcher_copy import display + from .model_guidance import cloud_models engine = QQmlApplicationEngine() bridge = Bridge() engine.rootContext().setContextProperty('backend', bridge) engine.rootContext().setContextProperty('initialState', bridge.value) - # Nested Python tuples remain opaque objects in QML. Named maps become - # QVariantMaps, so both the displayed label and clicked URL are available. - engine.rootContext().setContextProperty('modelLinks', { - component: [dict(label=label, label_zh=display(label, True), url=url) for label, url in rows] - for component, rows in links().items()}) engine.rootContext().setContextProperty('cloudLinks', cloud_models()) engine.rootContext().setContextProperty('theme', theme()) engine.load(QUrl.fromLocalFile(str(Path(__file__).parent / 'launcher/Main.qml'))) diff --git a/freevideo_engine/monitoring.py b/freevideo_engine/monitoring.py index b3d92c0c..2b4ce75a 100644 --- a/freevideo_engine/monitoring.py +++ b/freevideo_engine/monitoring.py @@ -93,18 +93,21 @@ def thermal_slowdown(measured, *, share=.5, clock_ratio=.7, minimum_samples=200) busy = measured.get('busy_activity') if isinstance(measured, dict) else None if not isinstance(busy, dict): return None - samples = busy.get('clock_reason_samples') or 0 - counts = busy.get('clock_reason_counts') or {} - clock = busy.get('sm_clock_mhz') or {} - mean, peak = clock.get('mean'), clock.get('maximum') + samples = _count(busy.get('clock_reason_samples')) + counts = busy.get('clock_reason_counts') + counts = counts if isinstance(counts, dict) else {} + clock = busy.get('sm_clock_mhz') + clock = clock if isinstance(clock, dict) else {} + mean, peak = _count(clock.get('mean')), _count(clock.get('maximum')) if samples < minimum_samples or not mean or not peak: return None - thermal = min(samples, max(counts.get('sw_thermal') or 0, counts.get('hw_thermal') or 0)) + thermal = min(samples, max(_count(counts.get('sw_thermal')), _count(counts.get('hw_thermal')))) if thermal < share * samples or mean > clock_ratio * peak: return None - temperature = (busy.get('temperature_c') or {}).get('mean') + temperature = busy.get('temperature_c') + temperature = temperature.get('mean') if isinstance(temperature, dict) else None return dict(share=round(thermal / samples, 3), sm_clock_mean_mhz=round(mean), sm_clock_max_mhz=round(peak), - temperature_mean_c=round(temperature, 1) if isinstance(temperature, (int, float)) else None) + temperature_mean_c=round(temperature, 1) if type(temperature) in (int, float) and 0 <= temperature <= 1000 else None) class OtherPrograms: @@ -231,14 +234,33 @@ def shared_gpu(measured, *, share=.5, minimum_checks=30): if not isinstance(programs, dict): return None checks, shared, mean = (programs.get(key) for key in ('checks', 'shared_checks', 'other_mean_percent')) - if (any(type(value) not in (int, float) or not value >= 0 for value in (checks, shared, mean)) - or checks < minimum_checks or shared < share * checks): + if (any(type(value) not in (int, float) for value in (checks, shared, mean)) + or not minimum_checks <= checks <= 1e9 or not 0 <= shared <= checks or not 0 <= mean <= 100 + or shared < share * checks): return None return dict(share=round(shared / checks, 3), other_mean_percent=round(mean)) +# Other programs' mean share of the GPU, in percent, from which the result says so: +# the run took at least about a quarter longer than with the GPU to itself. +SHARED_NOTE_PERCENT = 20 + + +def shared_gpu_note(measured): + """{other_mean_percent} for the note under a result, or None. + + Shown when other programs averaged at least SHARED_NOTE_PERCENT of the + GPU, and shared_gpu holds: at least 30 checks, with other programs using + at least 20% in at least half of them. + """ + if shared_gpu(measured) is None: + return None + mean = measured['other_programs']['other_mean_percent'] + return dict(other_mean_percent=mean) if mean >= SHARED_NOTE_PERCENT else None + + def _count(value): - return value if type(value) in (int, float) and 0 <= value < float('inf') else 0 + return value if type(value) in (int, float) and 0 <= value <= 1e12 else 0 def combined(rows): diff --git a/freevideo_engine/network.py b/freevideo_engine/network.py index f72c702e..452dbb37 100644 --- a/freevideo_engine/network.py +++ b/freevideo_engine/network.py @@ -586,8 +586,8 @@ def download_environment(environ, retry_mode='fast'): r"\b(?:invalid|bad|wrong|incorrect|mismatched?)\s+checksums?\b" r"|\bchecksums?\s+(?:mismatch|error|failed|failure)\b" r"|\bchecksums?\s+(?:verification|validation)\s+(?:failed|failure|error)\b" - r"|\bchecksums?\s+(?:does not|doesn't|did not|didn't|do not|don't)\s+match\b" - r"|\b(?:does not|doesn't|did not|didn't|do not|don't)\s+match\s+(?:the\s+|its\s+|expected\s+)?(?:checksums?|hash(?:es)?)\b" + r"|\bchecksums?\s+(?:does\s+not|doesn't|did\s+not|didn't|do\s+not|don't)\s+match\b" + r"|\b(?:does\s+not|doesn't|did\s+not|didn't|do\s+not|don't)\s+match\s+(?:the\s+|its\s+|expected\s+)?(?:checksums?|hash(?:es)?)\b" r"|\bhash\s+(?:mismatch|check\s+failed|does\s+not\s+match)\b|\bhashes\s+(?:do\s+not|didn't)\s+match\b" r"|\bsha-?256\s+mismatch\b|\bdigest\s+mismatch\b" r"|\bintegrity\s+(?:check|verification)\s+failed\b") @@ -772,8 +772,16 @@ def matches(candidate, source='local'): return False event(network, category=category, source=source, file=path.name, action='verifying', algorithm=algorithm) check = network.get('resource_check') if keep_partial else None + from .install_activity import byte_progress + report = byte_progress('verify_progress', path.name, report=lambda **row: + event(network, category=category, source=source, action='verify-progress', **row)) + def verification(done, total): + if check: + check() + if model: + report(done, total) return hash_file(candidate, algorithm, git_blob, discard_cache=category in ('models', 'vdn-models'), - progress=(lambda done, total: check()) if check else None) == expected + progress=verification if model or check else None) == expected if path.is_file(): if matches(path): return diff --git a/freevideo_engine/offline_packages.py b/freevideo_engine/offline_packages.py index e02e93e0..4f6c5267 100644 --- a/freevideo_engine/offline_packages.py +++ b/freevideo_engine/offline_packages.py @@ -220,7 +220,7 @@ def stamp(path): return [info.st_size, info.st_mtime_ns] -def unchanged(path, row, known, current=None): +def unchanged(path, row, known, current=None, progress=None): """Still the file its import checked: the size matches, and the recorded time too. Only a file whose record says it was rewritten since is hashed. `current` @@ -235,7 +235,24 @@ def unchanged(path, row, known, current=None): if known.get(row['path']) in (None, current): return True from .network import hash_file - return hash_file(path) == row['sha256'] + return hash_file(path, **({'progress': progress} if progress else {})) == row['sha256'] + + +def file_progress(progress, activity, total): + from .install_activity import byte_progress + return byte_progress('', '', report=lambda **row: + progress(activity=activity, activity_done=row['done_bytes'], activity_total=row['total_bytes'])) + + +def copy_progress(source, target, progress): + """The existing cross-disk copy, with a bounded buffer and actual bytes.""" + total, done = source.stat().st_size, 0 + progress(0, total) + with source.open('rb') as input_file, target.open('wb') as output: + for chunk in iter(lambda: input_file.read(4 * 2**20), b''): + output.write(chunk) + done += len(chunk) + progress(done, total) def import_archive(path, destination, progress=lambda **kw: None, cancelled=lambda: False): @@ -265,18 +282,19 @@ def import_archive(path, destination, progress=lambda **kw: None, cancelled=lamb current = present.get(row['path']) if current is None: continue - if current[0] != row['bytes'] or not unchanged(root / row['path'], row, known, current): + if current[0] != row['bytes'] or not unchanged(root / row['path'], row, known, current, + progress=file_progress(progress, 'verify', row['bytes'])): raise ValueError('已有导入文件已改变，未覆盖：' + plain(root / row['path'])) verified[row['path']] = current marker = root / value['marker'] if not missing and marker.is_file() and marker.read_bytes() == value['raw']: # Imported before and complete: nothing to read again. save(root / VERIFIED, verified) - progress(done=total, total=total, detail=path.name) + progress(done=total, total=total, detail=path.name, activity='') return dict(root=plain(root), kind=value['kind'], variant=value['variant'], name=path.name, **origin) done = sum(r['bytes'] for r in value['files'] if r['path'] in verified) root.mkdir(parents=True, exist_ok=True) - progress(done=done, total=total, detail=path.name) + progress(done=done, total=total, detail=path.name, activity='') with zipfile.ZipFile(path) as archive: for row in value['files']: if cancelled(): @@ -374,7 +392,8 @@ def assemble(runtime, models, source, progress=lambda **kw: None): def source_for(row): for root in models: current = walks[root].get(row['path']) - if current and unchanged(io_path(root) / row['path'], row, known[root], current): + if current and unchanged(io_path(root) / row['path'], row, known[root], current, + progress=file_progress(progress, 'verify', row['bytes'])): return io_path(root) / row['path'] return None @@ -383,19 +402,26 @@ def place(src, target): try: os.link(src, target) except OSError: - shutil.copyfile(src, target) + copy_progress(src, target, file_progress(progress, 'copy_bytes', src.stat().st_size)) + included = [] for row in model['files']: target, src = runtime / row['path'], source_for(row) if src is None: + if pack_name(row['path']) == '采样缓存': + total -= row['bytes'] + progress(done=done, total=total, detail='检查配套包', activity='') + continue raise ValueError('还需导入「%s」包，缺少文件：%s' % (pack_name(row['path']), row['path'])) if row['path'] in here: - if not os.path.samefile(target, src) and hash_file(target) != row['sha256']: + if not os.path.samefile(target, src) and hash_file(target, + progress=file_progress(progress, 'verify', row['bytes'])) != row['sha256']: raise ValueError('离线安装已有不同模型，未覆盖：' + plain(target)) else: place(src, target) + included.append(row) done += row['bytes'] - progress(done=done, total=total, detail='检查配套包') + progress(done=done, total=total, detail='检查配套包', activity='') listed = {row['path'] for row in model['files']} for root in models: package = io_path(root) / 'offline-package.json' @@ -405,11 +431,14 @@ def place(src, target): if row['path'] in listed or not row['path'].startswith(OPTIONAL_PREFIX) or row['path'] in here: continue current = walks[root].get(row['path']) - if not current or not unchanged(io_path(root) / row['path'], row, known[root], current): + if not current or not unchanged(io_path(root) / row['path'], row, known[root], current, + progress=file_progress(progress, 'verify', row['bytes'])): continue # The engine fetches a table it cannot find when a request needs it. place(io_path(root) / row['path'], runtime / row['path']) listed.add(row['path']) - return finish(runtime, config, model, source, here) + progress(done=done, total=total, detail='检查配套包', activity='') + # Only placed rows are recorded: a skipped Sampling caches table is fetched when needed. + return finish(runtime, config, dict(model, files=included), source, here) def runtime_config(runtime): diff --git a/freevideo_engine/paths.py b/freevideo_engine/paths.py index f5bdbe5c..b7ce2b25 100644 --- a/freevideo_engine/paths.py +++ b/freevideo_engine/paths.py @@ -33,8 +33,8 @@ def installed_model_root(): return model_root() -def add_vdn(): - root = vdn_root().resolve() +def add_vdn(root=None): + root = (root or vdn_root()).resolve() if not (root / 'src/models/hybrid_attention.py').is_file(): raise RuntimeError('VDN dependency is missing. Run freevideo setup or set FREEVIDEO_VDN_ROOT.') if str(root) not in sys.path: diff --git a/freevideo_engine/policy.py b/freevideo_engine/policy.py index b2bdceb2..1c1720d7 100644 --- a/freevideo_engine/policy.py +++ b/freevideo_engine/policy.py @@ -427,8 +427,23 @@ def int8_gpu_output_need(tokens, windows=False, full_tail=False): FF_STASH_WIDTH = 14336 # BF16 SwiGLU output channels kept for the per-tensor FP8 scale +# Int8 GPU-output workspace in GiB, second transfer slot included, for +# canvases below the envelope's anchors: (video tokens, GiB). A two-pass +# 960x544 / 4.5 s request on a Windows RTX 5060 Ti 16 GB peaked 2.48 GiB at +# 4320 tokens and 3.46 GiB at 16320 tokens beyond its sixteen resident +# blocks, against 7.14 and 7.82 from the envelope, so the plan held sixteen +# and eighteen blocks where the budget held about twenty-five and streamed +# the rest from disk on every step. Each figure carries one GiB of margin. +# The curve joins the envelope at its lowest anchor (896 square, 243 +# frames) and never exceeds it. It rests on one card. +SMALL_CANVAS_INT8_WORKSPACE = ( + (4320, 3.48), + (16320, 4.46), + (56448, 6.5 + 6. * 56448 / 72576 + BLOCK_BYTES / GiB - 56448 * FF_STASH_WIDTH * 2 / GiB), +) -def windows_gpu_output_workspace(video_tokens, *, prefetch=False, ff_stash=True): + +def windows_gpu_output_workspace(video_tokens, *, prefetch=False, ff_stash=True, small_canvas=False): """Starting workspace for head-8 GPU outputs with the full per-tensor FF stash. Int8 projections scale each activation row on its own, so every FF row tile @@ -442,8 +457,17 @@ def windows_gpu_output_workspace(video_tokens, *, prefetch=False, ff_stash=True) The 6.5 + 6 * token-ratio envelope covers both anchors. It is a cold-start estimate, not a fitted throughput model or proof of capacity elsewhere. The OS growth reserve has already been subtracted from the budget. + + small_canvas=True counts resident blocks below the anchors from the + measured int8 curve (SMALL_CANVAS_INT8_WORKSPACE) instead of the envelope. + Head-group choices keep using the envelope. """ tokens = 72576 if video_tokens is None else video_tokens + if small_canvas and not ff_stash and tokens < SMALL_CANVAS_INT8_WORKSPACE[-1][0]: + for (low, at_low), (high, at_high) in zip(SMALL_CANVAS_INT8_WORKSPACE, SMALL_CANVAS_INT8_WORKSPACE[1:]): + if tokens <= high: + at = at_low + (at_high - at_low) * max(0, tokens - low) / (high - low) + return int(at * GiB) - (0 if prefetch else BLOCK_BYTES) workspace = int((6.5 + 6. * tokens / 72576) * GiB) + (BLOCK_BYTES if prefetch else 0) return workspace if ff_stash else workspace - tokens * FF_STASH_WIDTH * 2 @@ -1220,7 +1244,8 @@ def held(group): # FF stash. In addition, crediting a shorter geometry against its old # 345-frame anchor added five blocks to a request that already OOMed. # Budget the current execution path before host placement is planned. - windows_workspace = windows_gpu_output_workspace(effective_tokens, prefetch=prefetch, ff_stash=not int8) + windows_workspace = windows_gpu_output_workspace(effective_tokens, prefetch=prefetch, ff_stash=not int8, + small_canvas=True) windows_workspace += max(0, activation_bytes(head, effective_tokens) - activation_bytes(8, effective_tokens)) if prefetch and gpu_budget < 14 * GiB: # Saving a transfer slot must not turn fully retained host weights diff --git a/freevideo_engine/progress_timeline.py b/freevideo_engine/progress_timeline.py index bb2bab87..7fb3870e 100644 --- a/freevideo_engine/progress_timeline.py +++ b/freevideo_engine/progress_timeline.py @@ -290,7 +290,7 @@ SEGMENT_KEYS = ('phase', 'stage', 'timing_phase', 'label_key') ENUMS = dict(phase=PHASES, stage=STAGES, timing_phase=TIMING_PHASES, label_key=LABEL_KEYS, unit=UNITS, warning=WARNINGS) -OVERALL_ENUMS = dict(status=STATUSES) +OVERALL_ENUMS = dict(status=STATUSES, remaining_source=frozenset(('local', 'similar', 'measured', 'other'))) RETRY_ENUMS = dict(kind=RETRY_KINDS, failed_phase=PHASES) COUNTERS = frozenset(('done', 'total', 'block', 'blocks', 'reference_trimmed')) FLAGS = frozenset(('low_memory', 'kernel_cache_note', 'estimated', 'new_request', 'reset', 'result')) @@ -349,7 +349,7 @@ def _fields(value, *, strict=False, nested=None): (nested is None and (key in COUNTERS or key in FLAGS or key in SECONDS or key in FRACTIONS or key in ('t', 'bytes_per_second', 'overall', 'retry'))) or (nested == 'overall' and (key in OVERALL_SECONDS or key in OVERALL_FRACTIONS - or key in ('estimated', 'sampling_time_weighted'))) or + or key in ('estimated', 'sampling_time_weighted', 'at_ceiling'))) or (nested == 'retry' and key in ('attempt', 'max_attempts', 'reuse_sampling'))) if not known: # Heartbeat and the bridge's extra overall fields are intentionally omitted. @@ -370,8 +370,9 @@ def _fields(value, *, strict=False, nested=None): raise ValueError('Invalid progress mapping') result[key], count = _fields(raw, strict=strict, nested=key) errors += count - elif (nested is None and key in FLAGS) or key in ('estimated', 'reuse_sampling', 'sampling_time_weighted'): - if (strict or key == 'sampling_time_weighted') and type(raw) is not bool: + elif (nested is None and key in FLAGS) or key in ('estimated', 'reuse_sampling', 'sampling_time_weighted', + 'at_ceiling'): + if (strict or key in ('sampling_time_weighted', 'at_ceiling')) and type(raw) is not bool: raise ValueError('Invalid progress flag') result[key] = bool(raw) elif (nested is None and key in COUNTERS) or key in ('attempt', 'max_attempts'): diff --git a/freevideo_engine/provider_resume.py b/freevideo_engine/provider_resume.py index c8ed27de..4c5a6f25 100644 --- a/freevideo_engine/provider_resume.py +++ b/freevideo_engine/provider_resume.py @@ -46,6 +46,11 @@ def ms_parallel(module, url, target, file_size, headers, cookies, progress_callb offset = merged.stat().st_size if merged.exists() else 0 if offset > file_size: raise ValueError('Retained merge exceeds the pinned model size') + def verifying(done, total): + for callback in callbacks: + report = getattr(callback, 'verify_progress', None) + if report: + report(done, total) if offset % module.DOWNLOAD_PART_SIZE: # A switched HTTP download may end in the middle of an SDK part. # Seed that part so its first Range starts after the retained prefix. @@ -59,11 +64,14 @@ def ms_parallel(module, url, target, file_size, headers, cookies, progress_callb with merged.open('rb') as prefix, part.open('a+b') as output: prefix.seek(start); output.seek(0) remaining=min(existing,offset-start) + total = remaining + verifying(0, total) while remaining: count=min(BLOCK,remaining) if prefix.read(count)!=output.read(count): raise ValueError('Retained part differs from switched prefix') remaining-=count + verifying(total-remaining, total) if existing 0, row.get('unit') == 'bytes'), reverse=True) current = active[0] if active else self._view(self.last, now) + if self.download_notices: + notice = next(reversed(self.download_notices.values())) + # A temporary activity line, with the underlying stage and counters + # retained so clearing it restores the current operation immediately. + current = dict(current, label=current.get('label') or notice['label'], + detail=notice['detail'], activity='', name='') return dict(phase_progress=dict(self.phase), active_tasks=active, progress=current, model_groups=self.model_groups) @staticmethod @@ -86,6 +100,20 @@ def progress_text(event, zh=False, *, decimal_sizes=False): """One activity line for the launcher; step counts and ETAs remain in worker events/logs.""" t = lambda en, cn: cn if zh else en activity, name = event.get('activity'), event.get('name') or '' + if activity == 'count': + done, total = event.get('done'), event.get('total') + if number(done) and number(total) and total > 0 and done <= total: + return name + t(' · Completed ', ' · 已完成 ') + '%d / %d' % (done, total) + return name + if activity in ('verify', 'assemble', 'copy_bytes'): + verb = {'verify': ('Verifying ', '正在校验'), 'assemble': ('Assembling ', '正在合并'), + 'copy_bytes': ('Copying ', '正在复制')}[activity] + # MiB rather than tenths of GiB: even slow disks change the visible text. + parts = [t(*verb) + name] + done, total = event.get('done'), event.get('total') + if event.get('unit') == 'bytes' and number(done) and number(total) and total > 0 and done <= total: + parts.append('%d / %d %s' % (done // 2**20, total // 2**20, 'MiB')) + return ' · '.join(parts) if activity == 'resume': return t('The connection dropped; resuming the download and keeping what was already downloaded', '连接中断，正在继续下载，已下载的部分会保留') diff --git a/freevideo_engine/stage_estimate.py b/freevideo_engine/stage_estimate.py index c06f1cbd..5b5dded9 100644 --- a/freevideo_engine/stage_estimate.py +++ b/freevideo_engine/stage_estimate.py @@ -124,6 +124,11 @@ def canvases(r): first = dict(full, width=32*uw*m, height=32*uh*m) else: first = dict(full) + # two_pass.plan() runs a first pass under SMALL_FIRST_PASS_PIXELS (138,240) as one pass at the + # target size, except when the first pass itself is wanted (a preview or the upscale of one). + if (not sg.get('first') and two and r.get('flow') not in ('preview', 'upscale') + and first != full and first['width']*first['height'] < 138240): + return full, full, full second = dict(full, **(sg.get('second') or {})) decode = first if r.get('flow') == 'preview' else full return first, second, decode diff --git a/freevideo_engine/storage.py b/freevideo_engine/storage.py index 9d58d050..3b878c8e 100644 --- a/freevideo_engine/storage.py +++ b/freevideo_engine/storage.py @@ -83,7 +83,7 @@ def record_download(root, path, row, *, origin='download'): save(ledger, {'root': plain(resolved(root)), 'files': files}) -def compact_sources(plan, rows, report): +def compact_sources(plan, rows, report, progress=None): """Caller must first verify the complete FP8 cache and pass GPU setup probes. Deletion uses an allowlist of pinned conversion-only filenames and a matching @@ -138,9 +138,11 @@ def compact_sources(plan, rows, report): # Destructive cleanup cannot trust a metadata cache hit. Equal-size # edits may share a filesystem timestamp tick with the receipt. if row.get('sha256'): - matches = hash_file(path, discard_cache=True) == row['sha256'] + matches = hash_file(path, discard_cache=True, progress=( + lambda done, total: progress(path, done, total)) if progress else None) == row['sha256'] elif row.get('git_blob'): - matches = hash_file(path, 'sha1', git_blob=True, discard_cache=True) == row['git_blob'] + matches = hash_file(path, 'sha1', git_blob=True, discard_cache=True, progress=( + lambda done, total: progress(path, done, total)) if progress else None) == row['git_blob'] else: raise OSError('No pinned content hash; source retained') if not matches: diff --git a/freevideo_engine/terminal_ui.py b/freevideo_engine/terminal_ui.py index 593c4141..730e16f8 100644 --- a/freevideo_engine/terminal_ui.py +++ b/freevideo_engine/terminal_ui.py @@ -322,6 +322,7 @@ class LogProgress: def __init__(self, path): self.path, self.offset, self.pending = Path(path), 0, '' self.packages = {} + self.downloaded_packages = set() self.installing_packages = False self.model_source = '' self.model_backend = '' @@ -382,23 +383,48 @@ def update(**values): self.packages[name] = float(size) * {'KiB': 2**10, 'MiB': 2**20, 'GiB': 2**30}[unit] amount = sum(self.packages.values()) / 2**30 size_text = '%.1f GiB' % amount if amount >= 1 else '%.0f MiB' % (amount * 1024) - update(done=0, total=0, scope='package-download', activity='note', name='Downloading runtime packages', + update(done=len(self.downloaded_packages), total=len(self.packages), unit='items', + scope='package-download', activity='count', name='Downloading runtime packages', detail='Downloading packages · about %s across %d files' % (size_text, len(self.packages))) + elif re.fullmatch(r'Downloaded [\w.+-]+', text): + name = text.split(' ', 1)[1] + if name in self.packages: + self.downloaded_packages.add(name) + update(done=len(self.downloaded_packages), total=len(self.packages), unit='items', + scope='package-download', activity='count', name='Downloading runtime packages', + detail='Downloading runtime packages') elif re.match(r'Resolved \d+ packages? in ', text): self.packages.clear() + self.downloaded_packages.clear() self.installing_packages = False - update(done=0, total=0, scope='package-resolve', detail='Dependencies resolved · preparing downloads') + update(done=0, total=0, scope='package-resolve', activity='note', name='Packages resolving', + detail='Dependencies resolved · preparing downloads') elif re.match(r'Prepared \d+ packages? in ', text): self.installing_packages = True - update(done=0, total=0, scope='package-install', detail='Downloads ready · installing packages') + update(done=0, total=0, scope='package-install', activity='note', name='Packages installing', + detail='Downloads ready · installing packages') elif re.match(r'Installed \d+ packages? in ', text): - update(done=0, total=0, scope='package-finish', detail='Packages installed · finishing checks') + update(done=0, total=0, scope='package-finish', activity='note', name='Packages ready', + detail='Packages installed · finishing checks') + elif re.match(r'Uninstalled \d+ packages? in ', text): + update(done=0, total=0, scope='package-remove', activity='note', name='Packages installing', + detail='Packages installing') + elif text.startswith('Creating virtual environment'): + update(done=0, total=0, scope='venv-create', activity='note', name='Environment creating', detail='Environment creating') + elif text.startswith('Installing seed packages'): + update(done=0, total=0, scope='venv-seed', activity='note', name='Base packages installing', detail='Base packages installing') + elif text.startswith('Installed Python '): + update(done=0, total=0, scope='python-ready', activity='note', name='Python ready', detail='Python ready') match = re.search(r'\[(\d+)/(\d+)\]', line) if match: self.installing_packages |= 'Installing wheels' in text + package_phase = ('remove', 'Packages removing') if 'Uninstalling' in text else ( + ('prepare', 'Packages preparing') if 'Preparing packages' in text else ( + ('install', 'Packages installing') if self.installing_packages else ('compile', 'GPU acceleration building'))) update(done=int(match[1]), total=int(match[2]), + activity='count', name=package_phase[1], detail='Installing packages' if self.installing_packages else 'Compiling CUDA kernels', - scope='package-install' if self.installing_packages else 'compile', unit='items') + scope='package-' + package_phase[0], unit='items') try: event = json.loads(line) except (ValueError, TypeError): @@ -406,7 +432,23 @@ def update(**values): if not isinstance(event, dict): continue kind = event.get('event') - if kind == 'model_groups': + if kind == 'installation_step': + update(done=event.get('done', 0), total=event.get('total', 0), unit='items', + activity='count' if event.get('total') else 'note', name=event['name'], + detail=event['name'], scope='install-' + event['name']) + elif kind == 'freevideo_ui' and event.get('kind') in ('task', 'progress'): + # A nested Comfy environment has its own phases. Forward work, + # never its percentage into the outer installation's one bar. + update(done=event.get('done', 0), total=event.get('total', 0), unit=event.get('unit'), + activity=event.get('activity') or 'note', name=event.get('name') or event.get('label', ''), + detail=event.get('detail', ''), scope='comfy-' + str(event.get('key', ''))) + elif kind in ('verify_progress', 'assemble_progress') or (kind == 'network' and event.get('action') == 'verify-progress'): + update(done=event['done_bytes'], total=event['total_bytes'], unit='bytes', + activity='assemble' if kind == 'assemble_progress' else 'verify', + name=event.get('name') or model_download_name(event['file']).split(' · ')[0], + detail='Model files verifying' if kind != 'assemble_progress' else 'Model files assembling', + scope=kind + '-' + event['file']) + elif kind == 'model_groups': self.model_groups = event['groups'] continue if kind in ('loaded', 'sampling_start', 'sampling_progress', 'step', 'sample_finalize'): @@ -441,9 +483,11 @@ def update(**values): elif kind == 'prepare_fp8_phase': work = 'Verify cached weights' if event['phase'] == 'verify_cache' else 'Read and convert source weights' update(done=max(0, event['index'] - 1), total=event['total'], + activity='count', name='Model files verifying' if event['phase'] == 'verify_cache' else 'Model files preparing', detail=work + ' · ' + event['group'], scope='prepare-fp8-' + event['phase'], unit='items') elif kind == 'prepared_fp8_group': update(done=event.get('index', 0), total=event.get('total', 0), + activity='count', name='Model files preparing', detail=('Reuse verified FP8 ' if event.get('reused') else 'Prepare FP8 ') + event['group'], scope='prepare-fp8', unit='items') elif kind == 'storage_compacted': update(done=0, total=0, activity='', name='', @@ -513,7 +557,7 @@ def update(**values): state = event.get('state', 'running') active = state == 'running' self.parallel_tasks['verify-models'] = dict(label='Verify / reuse local models', state=state, - activity='check' if active else '', + activity='verify' if active else '', name=(event.get('name') or model_download_name(event.get('file', '')).split(' · ')[0]) if active else '', done=event.get('done_bytes', 0) if active else event.get('files_done', 0), total=event.get('total_bytes', 0) if active else event.get('files_total', 0), @@ -540,7 +584,7 @@ def update(**values): speed = event.get('network_bytes_per_second') shown_speed = speed if speed is not None else event['bytes_per_second'] update(done=done, total=total, scope=description + self.model_source, rate=event['bytes_per_second'], unit='bytes', - activity='download', name=event.get('name') or ('PyTorch' if description == 'torch' + activity='download', name=event.get('name') or ('Python' if description.startswith('cpython-') else 'PyTorch' if description == 'torch' else model_download_name(description).split(' · ')[0]), detail='%s%s%.1f MiB/s · %.1f / %.1f MiB · %s' % (source + ' · ' if source else '', diff --git a/freevideo_engine/windows_bootstrap.ps1 b/freevideo_engine/windows_bootstrap.ps1 index b1bcb68b..b7cc3e9f 100644 --- a/freevideo_engine/windows_bootstrap.ps1 +++ b/freevideo_engine/windows_bootstrap.ps1 @@ -67,9 +67,11 @@ function Invoke-FreeVideoBootstrapCommand { function Test-FreeVideoSlowDownloadFailure { param([string]$Detail) # Keep the patient-round gate aligned with network.slow_download_failure. + # URL paths may contain validation words without a validation failure. + $Detail = $Detail -replace 'https?://\S+', '' return ($Detail -match 'timed out|timeout|connection|could not resolve|failed to download|failed to fetch|network is unreachable|ssl|tls|http status|status code|requested url returned error|rpc failed|early eof|unable to access|remote end hung up|couldn''t connect|proxy' -and $Detail -notmatch '\b(401|403|404|410)\b' -and - $Detail -notmatch 'no solution found|no matching distribution|could not find a version|checksum|hash mismatch|hashes do not match|hashes didn''t match|hash does not match|hash check failed|sha256 mismatch|sha-256 mismatch|digest mismatch|integrity|certificate verify failed|certificate_verify_failed|invalid peer certificate|unknown issuer|not found|resolutionimpossible|failed to build|invalid wheel|permission denied|no space left on device|disk full') + $Detail -notmatch 'no solution found|no matching distribution|could not find a version|\b(?:invalid|bad|wrong|incorrect|mismatched?)\s+checksums?\b|\bchecksums?\s+(?:mismatch|error|failed|failure)\b|\bchecksums?\s+(?:verification|validation)\s+(?:failed|failure|error)\b|\bchecksums?\s+(?:does\s+not|doesn''t|did\s+not|didn''t|do\s+not|don''t)\s+match\b|\b(?:does\s+not|doesn''t|did\s+not|didn''t|do\s+not|don''t)\s+match\s+(?:the\s+|its\s+|expected\s+)?(?:checksums?|hash(?:es)?)\b|\bhash\s+(?:mismatch|check\s+failed|does\s+not\s+match)\b|\bhashes\s+(?:do\s+not|didn''t)\s+match\b|\bsha-?256\s+mismatch\b|\bdigest\s+mismatch\b|\bintegrity\s+(?:check|verification)\s+failed\b|certificate verify failed|certificate_verify_failed|invalid peer certificate|unknown issuer|not found|resolutionimpossible|failed to build|invalid wheel|permission denied|no space left on device|disk full') } function Test-FreeVideoUvArchive { diff --git a/freevideo_engine/worker.py b/freevideo_engine/worker.py index ddb2d773..5b25e9ff 100644 --- a/freevideo_engine/worker.py +++ b/freevideo_engine/worker.py @@ -164,6 +164,13 @@ def release_idle(): resident.gpu_budget = min(resident.gpu_budget, limit) save(request['metrics'], metrics) manifest = json.loads((Path(request['cache']) / 'manifest.json').read_text(encoding='utf-8')) + if continuation is not None and 'tail' in continuation: + # Nothing was stored for the previous video: its ending is encoded here, before any model of + # this run is loaded, and every later attempt reads the same record. + from .continuation_tail import complete + continuation, metrics['continuation_encode'] = complete( + continuation, request['geometry'], manifest, request.get('sampling_plan'), resident=resident) + request['continuation'] = continuation if manifest.get('precision') not in ('fp8', 'int8', 'bf16'): raise ValueError('The Engine requires an official FP8 or int8 cache') options, decoder = request['engine_options'], request['decoder_options'] @@ -204,9 +211,10 @@ def fit_decoder(): # A two-pass request that runs as one pass (small canvases) continues # too: its first and final canvases are the same one, so one tail # serves both stages. An explicit single-pass request keeps nothing. + # Whatever media conditioned the request, the sampler returns only the + # generated picture and sound rows, so the tail is the same. plan_flags = request.get('sampling_plan') or {} continuable = (bool(plan_flags.get('enabled') or plan_flags.get('requested')) - and (options.get('task', 't2va') == 't2va' or bool(continuation and continuation.get('anchor'))) and not request.get('preview')) first_tail = continuation_data = None def tail_failure(error): diff --git a/scripts/test_freevideo_entry.mjs b/scripts/test_freevideo_entry.mjs index 0b867324..d386fb3a 100644 --- a/scripts/test_freevideo_entry.mjs +++ b/scripts/test_freevideo_entry.mjs @@ -53,6 +53,7 @@ test('main browser entry registers both views and preserves node hooks', async t }, referenceTrimText: row => `trimmed ${row.kind} ${row.number}`, thermalText: thermal => `thermal ${thermal.sm_clock_mean_mhz}`, + sharedGpuText: shared => `shared GPU ${shared.other_mean_percent}%`, encoderFileText: file => `encoder ${file.sizes.join(' / ')} GiB`, encoderFilePath: file => file?.offline_installation ? file.path || '' : '', createProgressConnection: () => ({start() {}, refresh() {}, reset() {}}), @@ -82,7 +83,7 @@ test('main browser entry registers both views and preserves node hooks', async t .replace('import { app } from "../../scripts/app.js";', 'const {app} = globalThis.__freevideoEntryTest;') .replace('import { api } from "../../scripts/api.js";', 'const {api} = globalThis.__freevideoEntryTest;') .replace("import { openStudio, loraPanel, loraWarning, promptGuide } from \"./studio.js\";", 'const {openStudio,loraPanel,loraWarning,promptGuide} = globalThis.__freevideoEntryTest;') - .replace("import { createGenerationProgress, referenceTrimText, thermalText, encoderFileText, encoderFilePath } from './generation_progress.js';", 'const {createGenerationProgress,referenceTrimText,thermalText,encoderFileText,encoderFilePath} = globalThis.__freevideoEntryTest;') + .replace("import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, encoderFileText, encoderFilePath } from './generation_progress.js';", 'const {createGenerationProgress,referenceTrimText,thermalText,sharedGpuText,encoderFileText,encoderFilePath} = globalThis.__freevideoEntryTest;') .replace("import { createProgressConnection } from './progress_connection.js';", 'const {createProgressConnection} = globalThis.__freevideoEntryTest;') .replace("import { notifyCompatibility } from './compatibility.js';", 'const {notifyCompatibility} = globalThis.__freevideoEntryTest;') .replace("import { startUpdateChecks } from './updates.js';", 'const {startUpdateChecks} = globalThis.__freevideoEntryTest;') @@ -191,12 +192,14 @@ test('main browser entry registers both views and preserves node hooks', async t launcher: true, offline_installation: false}; const resultWidth = node.size[0]; node.setSize([resultWidth, 100]); - node.freevideoShowResult({freevideo_summary: [{video, result_cache_hit: true, sample_seconds: 99, encoder_file}]}); + node.freevideoShowResult({freevideo_summary: [{video, result_cache_hit: true, sample_seconds: 99, encoder_file, + thermal: {sm_clock_mean_mhz: 900}, shared_gpu: {other_mean_percent: 42}}]}); const reused = node.dom.find(row => row.name === 'freevideo_result').element; assert.equal(reused.children[0].hidden, true, 'Do not show the old sampling time as current work'); assert.ok(reused.children[1].children.some(e => e.textContent === 'Reused previous result')); assert.ok(reused.children.some(e => e.className === 'fv-note' && e.textContent === 'encoder 25.3 / 14.6 GiB'), 'A cached result still came from the same encoder file'); + assert.ok(!reused.children.some(e => e.textContent === 'shared GPU 42%'), 'A cached result did not share the GPU this time'); assert.deepEqual(node.size, [resultWidth, node.computeSize()[1]], 'Grow the node after appending the encoder notice'); const again = reused.children[1].children.at(-1); assert.equal(again.textContent, 'Regenerate'); @@ -218,11 +221,12 @@ test('main browser entry registers both views and preserves node hooks', async t assert.ok(node.computeSize()[1] < progressHeight); assert.deepEqual(node.size, [resultWidth, progressHeight], 'Shorter results must not shrink the node'); node.freevideoShowResult({freevideo_summary: [{video, sample_seconds: 5, encoder_file, thermal: {sm_clock_mean_mhz: 900}, + shared_gpu: {other_mean_percent: 42}, reference_trims: [{kind: 'video', number: 1, seconds: 20.3, used_seconds: 5.17}]}]}); assert.deepEqual(reused.children.filter(e => e.className === 'fv-note fv-trim-note').map(e => e.textContent), ['trimmed video 1'], 'Say which reference was shortened, never cut silently'); assert.deepEqual(reused.children.filter(e => e.className === 'fv-note' && !e.hidden).map(e => e.textContent), - ['thermal 900', 'encoder 25.3 / 14.6 GiB'], 'Show the encoder notice after the thermal notice'); + ['thermal 900', 'shared GPU 42%', 'encoder 25.3 / 14.6 GiB'], 'Show thermal, shared GPU and encoder notices in order'); assert.ok(node.computeSize()[1] > progressHeight); assert.deepEqual(node.size, [resultWidth, node.computeSize()[1]], 'Measure the result height after all notices are appended'); @@ -265,8 +269,9 @@ test('main browser entry registers both views and preserves node hooks', async t node.freevideoShowFailure({exception_message: 'Failed'}); assert.equal(options.getMinHeight(), 420, 'Failure keeps its existing height'); node.freevideoShowResult({freevideo_summary: [{video, low_memory: true, thermal: {sm_clock_mean_mhz: 900}, + shared_gpu: {other_mean_percent: 42}, reference_trims: [{kind: 'video', number: 1}]}]}); - assert.equal(options.getMinHeight(), 461, 'Measure memory, thermal and reference trim notices'); + assert.equal(options.getMinHeight(), 543, 'Add 82 px for the shared GPU notice: 66 px text height and two 8 px margins'); }); class MediaNode extends GenerateNode { diff --git a/web/agent_activity.css b/web/agent_activity.css index 094555cb..3b2efcac 100644 --- a/web/agent_activity.css +++ b/web/agent_activity.css @@ -47,9 +47,3 @@ @media(max-width:820px){.fv-studio .fv-header:has(>.fv-view-switch):has(.fv-agent-pill:not([hidden])){height:auto;min-height:98px;grid-template-columns:1fr auto;padding-block:10px}.fv-studio .fv-header:has(.fv-agent-pill:not([hidden]))>.fv-view-switch{grid-column:1/-1;grid-row:2;justify-self:center}.fv-studio .fv-header:has(.fv-agent-pill:not([hidden]))>.fv-header-actions{grid-column:2;grid-row:1}} @media(max-width:480px){.fv-agent-panel{top:104px;left:16px;right:16px;width:auto}} .fv-agent-panel:focus{outline:none} -/* The same rows as an always-open list inside a view. */ -.fv-agent-list{display:grid;gap:10px;min-width:0} -.fv-agent-list[hidden],.fv-agent-list [hidden]{display:none} -.fv-agent-list h2{margin:0;font-size:var(--fv-strong);font-weight:var(--fv-semibold)} -.fv-agent-list ol{list-style:none;margin:0 -12px;padding:0;display:grid;gap:2px} -.fv-agent-empty{margin:0;font-size:var(--fv-body);line-height:1.5;color:var(--fv-muted)} diff --git a/web/agent_activity.js b/web/agent_activity.js index 811c7bd5..5785391a 100644 --- a/web/agent_activity.js +++ b/web/agent_activity.js @@ -1,4 +1,4 @@ -import {agentGet, agentPost} from './agent_session.js'; +import {agentPost} from './agent_session.js'; // What agents connected to this FreeVideo are doing. Their requests share the // studio's queue, so people can see them, follow progress and stop them. const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./agent_activity.css', import.meta.url).href; document.head.append(css); @@ -32,7 +32,8 @@ const left = (s, t) => s >= 60 ? t(`About ${Math.ceil(s / 60)} min`, `约 ${Math // The engine's phases in the reader's language. function phaseText(p, t) { const step = p.step; - // The first step is still getting ready: the line does not count from zero. + // The first step is still getting ready: the line does not count from zero. A row here does not know which + // part of a long video it is on, and a later part loads nothing, so it does not say the model is loading. if (p.phase === 'sampling' && step?.total) return step.done >= 1 ? t(`Step ${step.done} / ${step.total}`, `生成 ${step.done} / ${step.total}`) : t('Preparing', '准备中'); return {encoding: t('Reading prompt and media', '读取提示词和素材'), load: t('Loading the model', '加载模型'), decode: t('Saving video and audio', '保存视频与音频'), starting: t('Starting', '启动中'), @@ -130,58 +131,6 @@ async function showWork(t, job) { (await import('./library.js')).openLibrary(t, {selectedWork: job.work}); } -// The same jobs as a list that is always open, for a view of its own: what the agents are doing, how far -// along, where the result is, and a stop on each; open(job) can replace where a finished video opens. It shows -// once an agent is connected or has jobs, and asks nothing while it is out of sight. -export function agentActivityList(t, api, {interval = [1500, 8000], open = job => showWork(t, job)} = {}) { - const element = el('section', 'fv-agent-list'); element.hidden = true; - const heading = el('h2', null, t('Jobs', '任务')), list = el('ol'); - const empty = el('p', 'fv-agent-empty', t('Videos the agent makes will appear here.', 'Agent 生成的视频会显示在这里。')); - element.append(heading, list, empty); - let timer = null, stopped = false, on = null, failures = 0, active = 0; - // Hidden itself, the list is in sight when the view that holds it is. - const inSight = () => element.isConnected && (element.hidden ? element.parentElement : element)?.getClientRects().length > 0; - async function connected() { - try { - const response = await agentGet(api, '/freevideo/agent/v1/connect/status'); - return response.ok && (await response.json()).connected === true; - } catch { return false; } - } - async function poll() { - clearTimeout(timer); timer = null; - if (stopped) return; - const shown = inSight(); - if (shown && on === null) { - // Agent features can be off in this FreeVideo: ask once. - try { - const response = await api.fetchApi('/freevideo/agent', {cache: 'no-store'}); - on = response.ok && (await response.json()).enabled === true; - } catch { on = false; } - } - if (stopped || on === false) return; - if (shown && on) { - try { - const response = await api.fetchApi('/freevideo/agent/activity', {cache: 'no-store'}); - if (response.ok) { - const data = await response.json(), jobs = data.jobs || []; - const show = jobs.length > 0 || await connected(); - if (stopped) return; - failures = 0; active = data.active || 0; - const key = refill(element, list, rows(jobs, t, api, poll, open)); - list.hidden = !jobs.length; empty.hidden = !!jobs.length; element.hidden = !show; - if (key) list.querySelector(`[data-job="${key[0]}"] .${key[1].split(' ')[0]}`)?.focus?.(); - } else failures++; - } catch { failures++; } - } - if (stopped) return; - // Fast while agents work; every 3 s while empty, like the connection card, so connecting shows at once. - const wait = !shown ? 500 : active ? interval[0] : list.hidden ? Math.min(3000, interval[1]) : interval[1] * Math.min(4, 1 + failures); - timer = setTimeout(poll, wait); - } - poll(); - return {element, refresh: poll, stop() { stopped = true; clearTimeout(timer); }}; -} - export function createAgentActivity(t, api, host, {interval = [1500, 8000], open = job => showWork(t, job)} = {}) { const element = el('button', 'fv-agent-pill'); element.type = 'button'; element.hidden = true; element.setAttribute('aria-haspopup', 'dialog'); element.setAttribute('aria-expanded', 'false'); diff --git a/web/autopilot.js b/web/autopilot.js index 4cfe7d03..c3829b46 100644 --- a/web/autopilot.js +++ b/web/autopilot.js @@ -5,7 +5,7 @@ import { about, agentChat, jobName } from './autopilot_chat.js'; import { openEditorView } from './editor_loader.js'; import { closeDialog } from './motion.js'; import { openSetup } from './setup.js'; -import { extendVideo, openStudio, showNodes } from './studio.js'; +import { openExtendEditor, openStudio, showNodes } from './studio.js'; import { viewChanged, viewSwitch } from './view_navigation.js'; import { wordmark } from './wordmark.js'; @@ -45,9 +45,9 @@ export function openAutopilotView(node, {studio = null} = {}) { // Leaving for another view waits until this page has closed, so the view underneath is never covered twice. const closed = () => dialog.open ? new Promise(done => { dialog.addEventListener('close', done, {once: true}); closeDialog(dialog); }) : null; const chat = agentChat(t, api, {shown: inSight, - // A finished video opens as its work in Creations. Continuing it there belongs to the studio, so this page gives way first. + // Creations continues the work on the editor timeline; this page gives way first. open: async job => (await import('./library.js')).openLibrary(t, {selectedWork: job.work, back: t('Back to chat', '返回对话'), - extend: async record => { await closed(); return extendVideo(record, node); }}), + extend: async (record, options) => { await closed(); return openExtendEditor(record, node, {studio, ...options}); }}), edit: job => leave(() => openEditorView(node, {studio: studio?.open ? studio : undefined, work: {id: job.work, video: `FreeVideo/${job.work}/video.mp4`, geometry: {width: job.width, height: job.height, seconds: job.seconds}}})), changed: view => { last = view; show(); }}); // With no agent the card stands alone in the middle. Once one has connected the conversation stays, also after diff --git a/web/autopilot_chat.css b/web/autopilot_chat.css index eea7bf42..ea0c8026 100644 --- a/web/autopilot_chat.css +++ b/web/autopilot_chat.css @@ -1,6 +1,19 @@ /* The conversation with an agent: one column, the box to type in fixed under it. */ -.fv-chat{box-sizing:border-box;position:relative;min-height:0;min-width:0;display:grid;grid-template-rows:minmax(0,1fr) auto;grid-template-columns:minmax(0,1fr);gap:16px;padding:0 40px 22px} +.fv-chat{--fv-chats-width:232px;--fv-chats-pad:12px;--fv-chats-gap:6px;--fv-chats-row:40px;--fv-chats-radius:9px;--fv-chats-font:14px;--fv-chats-dot:6px;--fv-chats-toggle:34px;box-sizing:border-box;position:relative;min-height:0;min-width:0;display:grid;grid-template-columns:minmax(0,1fr)} +.fv-chat-main{position:relative;min-height:0;min-width:0;display:grid;grid-template-rows:minmax(0,1fr) auto;grid-template-columns:minmax(0,1fr);gap:16px;padding:0 40px 22px} .fv-chat[hidden],.fv-chat [hidden]{display:none} +/* List dimensions are kept here for art review; the breakpoint below is shared with the drawer's keyboard behavior. */ +.fv-chat[data-conversations=true]{grid-template-columns:var(--fv-chats-width) minmax(0,1fr)} +.fv-chat-conversations{box-sizing:border-box;min-height:0;min-width:0;display:flex;flex-direction:column;gap:var(--fv-chats-gap);padding:16px var(--fv-chats-pad);border-right:1px solid var(--fv-border);background:var(--fv-bg)} +.fv-chat-conversation-list{min-height:0;overflow:auto;display:grid;align-content:start;gap:var(--fv-chats-gap)} +.fv-studio .fv-chat-conversations button{box-sizing:border-box;flex:none;justify-content:flex-start;min-width:0;width:100%;min-height:var(--fv-chats-row);padding:8px var(--fv-chats-pad);border-radius:var(--fv-chats-radius);border:1px solid transparent;background:transparent;font-size:var(--fv-chats-font);font-weight:var(--fv-regular);color:var(--fv-ink);text-align:left;gap:10px} +.fv-studio .fv-chat-conversations .fv-chat-new{border-color:var(--fv-border);margin-bottom:6px} +.fv-studio .fv-chat-conversations button:hover:not(:disabled){background:var(--fv-raised)} +.fv-studio .fv-chat-conversations button:focus-visible{outline-offset:-2px} +.fv-studio .fv-chat-conversation[aria-current]{background:var(--fv-raised);border-color:var(--fv-accent-dim)} +.fv-chat-conversation span{min-width:0;overflow:hidden;text-overflow:ellipsis;white-space:nowrap} +.fv-chat-running{flex:none;width:var(--fv-chats-dot);height:var(--fv-chats-dot);border-radius:50%;background:var(--fv-accent)} +.fv-chat-tools{display:none} .fv-chat-scroll{min-height:0;overflow:auto;margin:0 -40px;padding:24px 40px 2px;scrollbar-gutter:stable both-edges} .fv-chat-log{list-style:none;margin:0 auto;padding:0;width:100%;max-width:720px;display:grid;grid-template-columns:minmax(0,1fr);align-content:start;gap:18px} .fv-chat-log>li{min-width:0}.fv-chat-log p{margin:0;line-height:1.6} @@ -69,6 +82,15 @@ .fv-studio .fv-chat-send:hover:not(:disabled){background:var(--fv-accent-hover)} .fv-studio .fv-chat-send:disabled{opacity:1;background:#22344a;color:#9fb3c8} .fv-studio .fv-chat-send:focus-visible{outline-offset:2px} -@media(max-width:480px){.fv-chat{padding:0 14px 14px}.fv-chat-scroll{margin:0 -14px;padding:16px 14px 2px}.fv-chat-hello{padding-top:32px}.fv-chat-job{max-width:none;gap:12px}.fv-chat-confirm{max-width:none} - .fv-chat-preview,.fv-chat-preview[data-shape=wide]{width:98px}.fv-chat-percent{font-size:22px}.fv-studio.fv-autopilot .fv-chat-input textarea{font-size:16px}.fv-studio .fv-chat-down{right:14px;bottom:84px}} +@media(max-width:720px){ + .fv-chat[data-conversations=true]{grid-template-columns:minmax(0,1fr)} + .fv-chat[data-conversations=true] .fv-chat-main{grid-template-rows:auto minmax(0,1fr) auto} + .fv-chat-tools:not([hidden]){display:flex;padding-top:12px;margin-bottom:-12px} + .fv-studio .fv-chat-chats{position:relative;width:var(--fv-chats-toggle);height:var(--fv-chats-toggle);min-height:0;padding:0;border-radius:var(--fv-chats-radius);background:var(--fv-card)} + .fv-chat-chats .fv-chat-running{position:absolute;top:5px;right:5px} + .fv-chat-conversations{display:none;position:absolute;inset:0 auto 0 0;z-index:3;width:var(--fv-chats-width);box-shadow:16px 0 32px #0005} + .fv-chat[data-chats-open=true] .fv-chat-conversations:not([hidden]){display:flex} +} +@media(max-width:480px){.fv-chat-main{padding:0 14px 14px}.fv-chat-scroll{margin:0 -14px;padding:16px 14px 2px}.fv-chat-hello{padding-top:32px}.fv-chat-job{max-width:none;gap:12px}.fv-chat-confirm{max-width:none} + .fv-chat-preview,.fv-chat-preview[data-shape=wide]{width:98px}.fv-chat-length{display:none}.fv-chat-percent{font-size:22px}.fv-studio.fv-autopilot .fv-chat-input textarea{font-size:16px}.fv-studio .fv-chat-down{right:14px;bottom:84px}} @media(prefers-reduced-motion:reduce){.fv-chat-water,.fv-chat-percent,.fv-chat-parts>i::after{transition:none}.fv-chat-water::after,.fv-chat-dots>i,.fv-chat-preview>img{animation:none}} diff --git a/web/autopilot_chat.js b/web/autopilot_chat.js index f39a5ca9..533ac560 100644 --- a/web/autopilot_chat.js +++ b/web/autopilot_chat.js @@ -1,7 +1,8 @@ // The conversation with the connected agent: what the user says, what the agent answers, and the agent's jobs as // cards at the place they happened. Reads and sends use the page session; the agent's token never enters the page. import {agentGet, agentPost} from './agent_session.js'; -import { estimatedFraction, remainingShown } from './generation_progress.js'; +import { estimatedFraction, remainingAbove, remainingShown } from './generation_progress.js'; +import {emptyConversations, hasConversations, mergeConversations, runningElsewhere, selectConversation} from './autopilot_conversations.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./autopilot_chat.css', import.meta.url).href; document.head.append(css); @@ -13,6 +14,7 @@ function glyph(path, width = 2.2) { return svg; } const press = (cls, text, action) => { const b = el('button', cls, text); b.type = 'button'; b.onclick = action; return b; }; +let chatNumber = 0; const WORKING = ['running', 'queued', 'waiting'], VIDEOS = ['generate', 'assemble']; // What a job with no picture of its own is doing, in the reader's language. @@ -22,30 +24,75 @@ export const about = (seconds, t) => seconds >= 60 ? t(`About ${Math.ceil(second export function clock(seconds) { const s = Math.max(0, Math.round(seconds)); return `${Math.floor(s / 60)}:${String(s % 60).padStart(2, '0')}`; } export const jobName = (job, t) => job.label || t('Video', '视频'); export const jobFraction = job => Math.max(0, Math.min(1, Number(job.progress?.fraction) || 0)); -// What a job's last progress stands for `age` seconds after it was made, by the creation panel's rule. While the -// engine has no timing for what it is doing (a model loading for the first time, a video in several parts), the -// level still rises over the seconds the reported fraction has stood (`still`): at the pace of the time the engine -// said is left, slowing down, and by no more than a quarter of what remains. -const DRIFT = .25; -export function jobEstimate(job, age, still = age) { - const p = job.progress || {}, e = p.estimate, overall = e?.overall, left = p.remaining_seconds; - const sent = overall ? estimatedFraction(e, overall, age, age) : null; - if (overall?.estimated === true && overall.phase_weight > 0) return sent ?? jobFraction(job); - const base = Math.max(jobFraction(job), sent ?? 0); - return left > 0 ? base + (1 - base) * DRIFT * (1 - Math.exp(-Math.max(0, still) / (DRIFT * left))) : base; +// What a job's last progress stands for `age` seconds after it was made, by the creation panel's rule. +export function jobEstimate(job, age) { + const e = job.progress?.estimate, sent = e?.overall ? estimatedFraction(e, e.overall, age, age) : null; + return Math.max(jobFraction(job), sent ?? 0); +} +// The level a job shows, given the engine's number (`base`), the time it says is left and the clock. The engine's +// number can stand for a long while: a model loading for the first time runs far past the time the engine gave it, +// and a video in several parts has no timing at all. While it stands the level still rises, at the pace of the time +// left, slowing down, and by no more than a quarter of what remains. When the number moves again, what was added is +// kept only as far as the number has not caught up with it: the rest of the way is then drawn shorter, so the level +// neither waits nor goes back and is full at the end, and a run that goes as the engine said shows the engine's number. +const DRIFT = .25, RISE = .0005; +export const newLevel = () => ({value: 0, top: null, since: 0, scale: 1}); +// The time left a running job shows, by the creation panel's rule: once measured never raised, and gone once its last +// minute has passed. `row` is what the job showed before (none yet: it starts at the engine's number); `seconds` is null +// from the moment the time has run out. +export function jobLeft(row, sent, source, time) { + if (!row) { + if (!(sent > 0)) return null; + row = {seconds: sent, at: time, minute: null, last: 0, source: null, above: 0}; + } + if (row.seconds === null) return row; + // Before the run is measured the engine's number is an estimate from earlier runs: follow it up as well as down. + const named = typeof source === 'string', initial = source === 'similar'; + // The first number that is not that estimate replaces it once, up or down, and its last minute starts over. + if (named && !initial && row.source === 'similar' && sent > 0) { row.seconds = sent; row.minute = null; } + if (named) row.source = source; + const target = sent > 0 ? sent : null; row.above = remainingAbove(row.above, row.seconds, target, time - row.at); + row.seconds = remainingShown(row.seconds, target, time - row.at, initial, row.above); row.at = time; + if (row.seconds < 60 && row.minute === null) { row.minute = time; row.last = row.seconds; } + // An estimate that rises out of its last minute starts that minute over. + else if (row.seconds >= 60 && initial) row.minute = null; + if (row.seconds <= 0 || row.minute !== null && time - row.minute >= row.last) row.seconds = null; + return row; +} +export function jobLevel(row, base, left, time) { + if (row.top === null || base > row.top + RISE) { + if (row.top !== null && base < 1) row.scale = Math.min(row.scale, (1 - row.value) / (1 - base)); + row.top = base; row.since = time; + } + // The number itself, or where standing since it last moved has brought the level: whichever is further on. + const from = 1 - row.scale * (1 - row.top), level = 1 - row.scale * (1 - Math.max(base, row.top)); + const more = left > 0 ? (1 - from) * DRIFT * (1 - Math.exp(-Math.max(0, time - row.since) / (DRIFT * left))) : 0; + return row.value = Math.max(row.value, Math.min(.99, Math.max(level, from + more))); } // working, done, stopped or failed: which card a job shows. export const jobMode = job => WORKING.includes(job.status) ? 'working' : job.status === 'succeeded' ? 'done' : job.status === 'cancelled' ? 'stopped' : 'failed'; +const SAVING = ['decode', 'finishing', 'assembling'], stepped = p => p.phase === 'sampling' && p.step?.total && p.step.done >= 1; // The one line that says where a job is. No step count is written unless the engine gave one. export function jobLine(job, t) { const p = job.progress || {}; if (job.status !== 'running') return t('Queued', '排队中'); - if (p.phase === 'sampling' && p.step?.total && p.step.done >= 1) return t(`Step ${p.step.done} / ${p.step.total}`, `生成 ${p.step.done} / ${p.step.total}`); - if (['decode', 'finishing', 'assembling'].includes(p.phase)) return t('Saving', '保存中'); + if (stepped(p)) return t(`Step ${p.step.done} / ${p.step.total}`, `生成 ${p.step.done} / ${p.step.total}`); + if (SAVING.includes(p.phase)) return t('Saving', '保存中'); if (p.phase === 'encoding') return t('Reading the prompt', '读取提示词'); - if (['load', 'loading'].includes(p.phase)) return t('Loading the model', '加载模型'); + // Before the first step of the first part the model is still settling in: the line does not turn back to "Preparing". + if (['load', 'loading'].includes(p.phase) || p.phase === 'sampling' && !(p.block?.done > 0)) return t('Loading the model', '加载模型'); return t('Preparing', '准备中'); } +// The line a card shows. After the last step, and again after saving, the engine has a moment with no step and no +// stage to report: a card that has counted steps or said "Saving" for a part keeps that line until there is a newer +// one, and does not go back to the lines from before the first step. `said` is the card's own memory. +export function cardLine(said, job, t) { + const p = job.progress || {}, part = p.block?.index ?? 0; + if (job.status !== 'running' || said.part !== part) said.last = null; + said.part = part; + if (job.status === 'running' && (stepped(p) || SAVING.includes(p.phase))) said.last = job; + return jobLine(said.last || job, t); +} export function jobMeta(job, t) { const parts = []; if (job.width > 0 && job.height > 0) parts.push(`${job.width} × ${job.height}`); @@ -91,16 +138,110 @@ function newId() { // user can see the conversation; the page calls sync() whenever that may have changed. changed(view) follows every // answer: {known, agent, current, active, any}. export function agentChat(t, api, {open, edit, shown = () => true, changed = () => {}, interval = 1500} = {}) { - const element = el('section', 'fv-chat'), scroller = el('div', 'fv-chat-scroll'), log = el('ol', 'fv-chat-log'); + const element = el('section', 'fv-chat'), main = el('div', 'fv-chat-main'), scroller = el('div', 'fv-chat-scroll'), log = el('ol', 'fv-chat-log'); log.setAttribute('role', 'log'); log.setAttribute('aria-label', t('Conversation', '对话')); const down = press('fv-chat-down', null, () => { scroller.scrollTo({top: scroller.scrollHeight, behavior: 'smooth'}); }); down.append(glyph('M12 5v14M5 12l7 7 7-7')); down.setAttribute('aria-label', t('Latest', '最新')); down.hidden = true; const form = el('form', 'fv-chat-input'), input = el('textarea'), sendButton = el('button', 'fv-chat-send'); input.rows = 1; input.maxLength = 8000; input.setAttribute('aria-label', t('Message the agent', '对 Agent 说')); sendButton.type = 'submit'; sendButton.setAttribute('aria-label', t('Send', '发送')); sendButton.append(glyph('M12 19V5M5 12l7-7 7 7')); - form.append(input, sendButton); scroller.append(log); element.append(scroller, down, form); + form.append(input, sendButton); scroller.append(log); main.append(scroller, down, form); element.append(main); let state = emptyChat(), pending = [], known = false, stopped = false, timer = null, busy = false, failures = 0; + let conversations = emptyConversations(), selectedSession, selection = 0, reread = false, focusSession = null, creating = false; + // Work under way in another conversation while this one has none: {session, job, ids}. The page's header keeps it in sight. + let away = null; + let listBusy = false, listAgain = false, listAt = -Infinity; + const listInterval = 10000; + const sidebar = el('nav', 'fv-chat-conversations'), list = el('div', 'fv-chat-conversation-list'), tools = el('div', 'fv-chat-tools'); + sidebar.id = `fv-chat-conversations-${++chatNumber}`; sidebar.setAttribute('aria-label', t('Chats', '对话')); sidebar.hidden = true; + const newChat = press('fv-chat-new', t('New chat', '新对话'), () => void startChat()); + newChat.prepend(glyph('M12 5v14M5 12h14')); newChat.title = t('New chat', '新对话'); + const chats = press('fv-chat-chats', null, () => drawer(chats.getAttribute('aria-expanded') !== 'true')); + const chatsDot = el('i', 'fv-chat-running'); chatsDot.setAttribute('aria-hidden', 'true'); chatsDot.hidden = true; + chats.append(glyph('M4 5h16v12H9l-5 4V5M8 9h8M8 13h5'), chatsDot); + chats.title = t('Chats', '对话'); chats.setAttribute('aria-label', chats.title); + chats.setAttribute('aria-controls', sidebar.id); chats.setAttribute('aria-expanded', 'false'); + tools.hidden = true; tools.append(chats); main.prepend(tools); sidebar.append(newChat, list); element.prepend(sidebar); + const listNodes = new Map(), narrow = () => window.matchMedia('(max-width: 720px)').matches; + function drawer(on, restore = false) { + element.dataset.chatsOpen = String(on); chats.setAttribute('aria-expanded', String(on)); + // Only the overlay needs to keep keyboard focus in its list. + main.inert = on && narrow(); + if (on) (list.querySelector('[aria-current]') || newChat).focus(); + else if (restore) chats.focus(); + } + function renderConversations() { + const visible = hasConversations(conversations); + element.dataset.conversations = String(visible); sidebar.hidden = tools.hidden = !visible; + if (!visible) drawer(false); + newChat.disabled = creating; chatsDot.hidden = !conversations.conversations.some(row => row.running > 0); + const keep = new Set(conversations.conversations.map(row => row.id)); + for (const [id, node] of listNodes) if (!keep.has(id)) { node.button.remove(); listNodes.delete(id); } + conversations.conversations.forEach((row, index) => { + let node = listNodes.get(row.id); + if (!node) { + const button = press('fv-chat-conversation', null, () => chooseChat(row.id)), dot = el('i', 'fv-chat-running'), title = el('span'); + button.dataset.session = row.id; dot.setAttribute('aria-hidden', 'true'); button.append(dot, title); + listNodes.set(row.id, node = {button, dot, title}); + } + node.title.textContent = row.title; node.button.title = row.title; node.dot.hidden = row.running <= 0; + if (row.id === conversations.current) node.button.setAttribute('aria-current', 'true'); else node.button.removeAttribute('aria-current'); + if (list.children[index] !== node.button) list.insertBefore(node.button, list.children[index] || null); + }); + } + function chooseChat(session, focus = false) { + selectedSession = session; selection++; pinned = true; away = null; + conversations = selectConversation(conversations, session); renderConversations(); + input.value = ''; grow(); drawer(false); + focusSession = focus ? session : null; + if (focus && session === state.session) input.focus(); else if (narrow()) chats.focus(); + render(); void readConversations(true); void poll(); + } + async function startChat() { + if (creating || !active() || conversations.supported !== true) return; + creating = true; renderConversations(); render(); + try { + const response = await agentPost(api, '/freevideo/agent/chat/new', {}); + if (stopped) return; + if (response.ok) { + const data = await response.json(); + if (typeof data.session === 'string' && data.session) chooseChat(data.session, true); + } else if (response.status === 409) { state = {...state, agent: {connected: false, name: null}}; report(); } + } catch { /* Keep the current conversation available for another try. */ } + finally { creating = false; if (!stopped) { renderConversations(); render(); } } + } + async function readConversations(force = false) { + if (!active() || conversations.supported === false) return; + if (listBusy) { if (force) listAgain = true; return; } + if (!force && performance.now() - listAt < listInterval) return; + listBusy = true; listAt = performance.now(); + try { + const response = await agentGet(api, '/freevideo/agent/chat/list', {signal: closed.signal}); + if (stopped) return; + conversations = mergeConversations(conversations, response.ok ? await response.json() : null, response.status); + renderConversations(); + if (conversations.supported === false && selectedSession !== undefined) { selectedSession = undefined; selection++; void poll(); } + } catch { /* List failures do not interrupt reading or sending. */ } + finally { listBusy = false; } + if (listAgain) { listAgain = false; void readConversations(true); } + } + const outside = event => { + if (chats.getAttribute('aria-expanded') === 'true' && !sidebar.contains(event.target) && !chats.contains(event.target)) drawer(false, true); + }; + const keys = event => { + if (chats.getAttribute('aria-expanded') !== 'true') return; + if (event.key === 'Escape') { event.preventDefault(); event.stopPropagation(); drawer(false, true); } + else if (event.key === 'Tab' && narrow()) { + const buttons = [...sidebar.querySelectorAll('button')].filter(button => !button.disabled); + if (!buttons.length) return; + if (event.shiftKey && document.activeElement === buttons[0]) { event.preventDefault(); buttons.at(-1).focus(); } + else if (!event.shiftKey && document.activeElement === buttons.at(-1)) { event.preventDefault(); buttons[0].focus(); } + } + }; + const resize = () => { if (!narrow()) drawer(false); }; + document.addEventListener('pointerdown', outside, true); document.addEventListener('keydown', keys, true); + window.addEventListener('resize', resize); // Whether the reader is at the newest entry. It holds while the conversation is out of sight, which forgets where it was scrolled. let pinned = true; const closed = new AbortController(); @@ -110,32 +251,25 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () function fractionOf(job) { const p = job.progress || {}, {age_seconds: sent, ...sample} = p.estimate || {fraction: p.fraction}; const key = JSON.stringify(sample), time = performance.now() / 1000; + // A job that stopped or failed and is tried again starts from the beginning, not from where the last try got to. + if (job.status !== 'running' && job.status !== 'succeeded') levels.delete(job.id); let row = levels.get(job.id); - if (!row) levels.set(job.id, row = {key: null, at: 0, job: null, value: 0, raw: null, since: time}); + if (!row) levels.set(job.id, row = {key: null, at: 0, job: null, level: newLevel()}); // remaining_seconds counts down between samples; the one that came with the sample is the one to move by. if (row.key !== key) { row.key = key; row.at = time - (sent > 0 ? sent : 0); row.job = job; } - // A loading model sends its own counts on and on while the fraction stands: that standing time is its own clock. - if (row.raw !== p.fraction) { row.raw = p.fraction; row.since = time; } - if (job.status === 'running') row.value = Math.max(row.value, Math.min(.99, jobEstimate(row.job, time - row.at, time - row.since))); - return row.value; + if (job.status === 'running') jobLevel(row.level, jobEstimate(row.job, time - row.at), row.job.progress?.remaining_seconds, time); + return row.level.value; } - // The time left each job shows, by the creation panel's rule: never raised, and gone once its last minute has passed. const times = new Map(); function leftOf(job) { - const time = performance.now() / 1000, sent = job.progress?.remaining_seconds; if (job.status !== 'running') { times.delete(job.id); return null; } - let row = times.get(job.id); - if (!row) { - if (!(sent > 0)) return null; - times.set(job.id, row = {seconds: sent, at: time, minute: null, last: 0}); - } - if (row.seconds === null) return null; - row.seconds = remainingShown(row.seconds, sent > 0 ? sent : null, time - row.at); row.at = time; - if (row.seconds < 60 && row.minute === null) { row.minute = time; row.last = row.seconds; } - if (row.seconds <= 0 || row.minute !== null && time - row.minute >= row.last) row.seconds = null; - return row.seconds; + const row = jobLeft(times.get(job.id), job.progress?.remaining_seconds, + job.progress?.estimate?.overall?.remaining_source, performance.now() / 1000); + if (row) times.set(job.id, row); + return row ? row.seconds : null; } const active = () => !stopped && element.isConnected && !document.hidden && shown(); + const sendingAllowed = () => !creating && (!selectedSession || selectedSession === state.session) && canSend(state); // ---- one entry of the conversation each ---- function message(role) { @@ -160,6 +294,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () poll(); }); foot.append(left, stop); info.append(head, middle, foot); item.replaceChildren(box, info); + const said = {}; // The number rides on the water, and goes under the line once there is no room left above it. const level = next => { const fraction = fractionOf(next); @@ -180,8 +315,9 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () cells.forEach((value, index) => parts.children[index].style.setProperty('--w', `${value * 100}%`)); parts.hidden = !cells.length; const block = next.progress?.block; - part.textContent = cells.length ? t(`Part ${block.index} / ${block.total}`, `第 ${block.index} / ${block.total} 段`) : jobLine(next, t); - step.textContent = cells.length ? jobLine(next, t) : ''; + const where = cardLine(said, next, t); + part.textContent = cells.length ? t(`Part ${block.index} / ${block.total}`, `第 ${block.index} / ${block.total} 段`) : where; + step.textContent = cells.length ? where : ''; const going = stopping.has(next.id); stop.disabled = going; stop.textContent = going ? t('Stopping', '正在停止') : t('Stop', '停止'); stop.setAttribute('aria-label', t('Stop ', '停止 ') + jobName(next, t)); @@ -252,7 +388,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } else { item.replaceChildren(); apply = null; } } apply?.(job, latest); - const again = item.querySelector('.fv-chat-retry'); if (again) again.disabled = !canSend(state); + const again = item.querySelector('.fv-chat-retry'); if (again) again.disabled = !sendingAllowed(); }}; } function confirmCard() { @@ -290,7 +426,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () t('Make a 1-minute vertical short film: a day in a coffee shop', '做一条 1 分钟的竖屏短片：咖啡店的一天')]; list.append(...lines.map(line => press(null, line, () => send(line)))); item.append(el('p', 'fv-chat-meta', t('Try saying:', '可以这样说：')), list); - return {element: item, update() { list.querySelectorAll('button').forEach(button => { button.disabled = !canSend(state); }); }}; + return {element: item, update() { list.querySelectorAll('button').forEach(button => { button.disabled = !sendingAllowed(); }); }}; } function dots() { const item = el('li', 'fv-chat-thinking'), mark = el('span', 'fv-chat-dots'); mark.append(el('i'), el('i'), el('i')); @@ -333,10 +469,11 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () node.update(value, extra || {}); if (log.children[index] !== node.element) log.insertBefore(node.element, log.children[index] || null); }); - const sendable = canSend(state); - input.disabled = state.agent.connected !== true; + const sendable = sendingAllowed(); + input.disabled = state.agent.connected !== true || creating || !!selectedSession && selectedSession !== state.session; input.placeholder = state.agent.connected === true && !sendable ? t('Ask your agent to keep listening', '回到 Agent 那边让它继续') : t('Message the agent…', '对 Agent 说…'); sendButton.disabled = !sendable || !input.value.trim(); + if (focusSession === state.session && focusSession && !input.disabled && active()) { focusSession = null; input.focus(); } if (stick) { pinned = true; scroller.scrollTop = scroller.scrollHeight; down.hidden = true; } else if (log.children.length > before) down.hidden = false; } @@ -346,18 +483,39 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () if (pinned) down.hidden = true; }, {passive: true}); + async function readAway() { + const row = state.current ? null : runningElsewhere(conversations, state.session); + if (!row) { away = null; return; } + try { + // Past the last message: only that conversation's jobs come back. + const query = new URLSearchParams({session: row.id, after: String(Number.MAX_SAFE_INTEGER)}); + const response = await agentGet(api, `/freevideo/agent/chat?${query}`, {signal: closed.signal}); + if (!response.ok) { away = null; return; } + const data = await response.json(), jobs = Object.values(data.jobs || {}).filter(job => WORKING.includes(job?.status)); + const job = data.jobs?.[data.current] || jobs[0]; + away = job ? {session: row.id, job, ids: jobs.map(job => job.id)} : null; + } catch { /* The last reading stays until the next one. */ } + } function report() { - const current = state.current ? state.jobs[state.current] || null : null; + const current = (state.current ? state.jobs[state.current] : away?.job) || null; changed({known, agent: state.agent, current, fraction: current ? fractionOf(current) : 0, left: current ? leftOf(current) : null, active: state.active, any: state.items.size > 0 || pending.length > 0}); } async function send(text) { text = String(text).trim(); - if (!text || !canSend(state)) return false; + if (!text || !sendingAllowed()) return false; + const sentSelection = selection; const entry = {id: newId(), text}; pending.push(entry); render(true); report(); let response = null; - try { response = await agentPost(api, '/freevideo/agent/chat', entry); } catch { /* Handled below as not sent. */ } - if (response?.ok) { const value = await response.json().catch(() => ({})); if (value.agent) state.agent = value.agent; poll(); return true; } + const body = conversations.supported === true && state.session ? {...entry, session: state.session} : entry; + try { response = await agentPost(api, '/freevideo/agent/chat', body); } catch { /* Handled below as not sent. */ } + if (stopped) return false; + if (response?.ok) { + const value = await response.json().catch(() => ({})); + if (sentSelection === selection) { if (value.agent) state.agent = value.agent; void poll(); } + void readConversations(true); return true; + } + if (sentSelection !== selection) return false; // Not sent: the sentence goes back to the box. An agent that was never connected leads back to the card. pending = pending.filter(item => item !== entry); if (!input.value) { input.value = text; grow(); } @@ -370,7 +528,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () // At most five lines; the box scrolls after that. const line = parseFloat(getComputedStyle(input).lineHeight) || 22; input.style.height = `${Math.min(input.scrollHeight, Math.round(line * 5) + 14)}px`; - sendButton.disabled = !canSend(state) || !input.value.trim(); + sendButton.disabled = !sendingAllowed() || !input.value.trim(); } input.addEventListener('input', grow); // Enter sends, Shift+Enter starts a new line; Enter that confirms an input-method candidate does neither. @@ -381,33 +539,53 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () form.addEventListener('submit', event => { event.preventDefault(); const text = input.value; - if (!text.trim() || !canSend(state)) return; + if (!text.trim() || !sendingAllowed()) return; input.value = ''; grow(); void send(text); }); + // Both a server-side session change and a list selection reset the same reader. Job clocks stay keyed by job id. + function resetChat() { + state = emptyChat(); pending = []; answers.clear(); nodes.clear(); + log.querySelectorAll('video').forEach(video => video.pause()); log.replaceChildren(); pinned = true; + } async function poll() { clearTimeout(timer); timer = null; - if (stopped || busy) return; + if (stopped) return; + if (busy) { reread = true; return; } if (!active()) return; - busy = true; + busy = true; reread = false; + void readConversations(); + const readSelection = selection, requested = selectedSession; let again = false; try { - const after = state.session ? state.last : 0; - const response = await agentGet(api, '/freevideo/agent/chat' + (after ? `?after=${after}` : ''), {signal: closed.signal}); + const after = state.session && (!requested || requested === state.session) ? state.last : 0; + const query = new URLSearchParams(); + if (requested) query.set('session', requested); + if (after) query.set('after', String(after)); + const response = await agentGet(api, '/freevideo/agent/chat' + (query.size ? `?${query}` : ''), {signal: closed.signal}); if (stopped) return; - if (response.ok) { + if (readSelection !== selection) again = true; + else if (response.ok) { const data = await response.json(); + if (stopped) return; + if (readSelection !== selection) { again = true; } // Another conversation began: forget this one and read the new one whole. - if (after && (data.session ?? null) !== state.session) { state = emptyChat(); again = true; } + else if (after && (data.session ?? null) !== state.session) { resetChat(); again = true; } else { - if ((data.session ?? null) !== state.session) state = emptyChat(); + if ((data.session ?? null) !== state.session) resetChat(); state = chatMerge(state, data); failures = 0; + conversations = selectConversation(conversations, state.session); renderConversations(); + await readAway(); + if (stopped) return; + if (readSelection !== selection) { away = null; again = true; } } + } else if (response.status === 404 && requested) { + selectedSession = undefined; selection++; resetChat(); again = true; void readConversations(true); } else failures++; } catch { failures++; } finally { busy = false; } if (stopped) return; - if (again) return poll(); + if (again || reread) return poll(); known = true; // The next read is booked before the page hears of this one, so the page's sync() does not start another. const idle = state.agent.connected !== true && !state.active; @@ -416,7 +594,7 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } // Between reads the estimate keeps moving, on the cards and in the page's header alike. const moving = setInterval(() => { - if (!active() || !Object.values(state.jobs).some(job => job.status === 'running')) return; + if (!active() || ![...Object.values(state.jobs), ...(away ? [away.job] : [])].some(job => job.status === 'running')) return; for (const item of log.children) item.tick?.(); report(); }, 250); @@ -428,12 +606,19 @@ export function agentChat(t, api, {open, edit, shown = () => true, changed = () } render(); return {element, input, sync, refresh: poll, - scrollToJob(id) { log.querySelector(`[data-job="${CSS.escape(id)}"]`)?.scrollIntoView({block: 'center', behavior: 'smooth'}); }, + scrollToJob(id) { + // A job of another conversation: go to that conversation, which opens at its end where the job is. + if (away?.job.id === id && !state.jobs[id]) chooseChat(away.session); + else log.querySelector(`[data-job="${CSS.escape(id)}"]`)?.scrollIntoView({block: 'center', behavior: 'smooth'}); + }, async stopAll() { - const ids = Object.values(state.jobs).filter(job => WORKING.includes(job.status)).map(job => job.id); + const ids = [...new Set([...Object.values(state.jobs).filter(job => WORKING.includes(job.status)).map(job => job.id), ...(away?.ids || [])])]; ids.forEach(id => stopping.add(id)); render(); await Promise.allSettled(ids.map(id => agentPost(api, '/freevideo/agent/activity/cancel', {id}))); poll(); }, - stop() { stopped = true; closed.abort(); clearInterval(moving); clearTimeout(timer); timer = null; document.removeEventListener('visibilitychange', visible); log.querySelectorAll('video').forEach(video => video.pause()); }}; + stop() { stopped = true; closed.abort(); clearInterval(moving); clearTimeout(timer); timer = null; + document.removeEventListener('visibilitychange', visible); document.removeEventListener('pointerdown', outside, true); + document.removeEventListener('keydown', keys, true); window.removeEventListener('resize', resize); + log.querySelectorAll('video').forEach(video => video.pause()); }}; } diff --git a/web/autopilot_conversations.js b/web/autopilot_conversations.js new file mode 100644 index 00000000..ce2baca9 --- /dev/null +++ b/web/autopilot_conversations.js @@ -0,0 +1,21 @@ +// The list is a server snapshot, in the server's order. An unsaid current conversation need not be in it. +export const emptyConversations = () => ({supported: null, current: null, conversations: []}); +export const selectConversation = (state, session) => ({...state, current: session ?? null}); +export const hasConversations = state => state.supported === true && state.conversations.length > 0; +// The first other conversation with work under way; null when there is none. +export const runningElsewhere = (state, session) => state.conversations.find(row => row.running > 0 && row.id !== session) || null; + +export function mergeConversations(state, data, status = 200) { + // A legacy server is probed once per page; ordinary errors retain the last good snapshot. + if (status === 404) return {...state, supported: false, conversations: []}; + if (state.supported === false || status !== 200 || !Array.isArray(data?.conversations)) return state; + const seen = new Set(), conversations = []; + for (const row of data.conversations) { + if (typeof row?.id !== 'string' || !row.id || typeof row.title !== 'string' || !row.title.trim() || seen.has(row.id)) continue; + seen.add(row.id); + conversations.push({id: row.id, title: row.title, updated_at: row.updated_at, + running: Math.max(0, Math.floor(Number(row.running) || 0))}); + if (conversations.length === 50) break; + } + return {...state, supported: true, conversations}; +} diff --git a/web/editor.js b/web/editor.js index c5d2f9fc..a119d602 100644 --- a/web/editor.js +++ b/web/editor.js @@ -6,7 +6,7 @@ import {openSetup} from './setup.js'; import * as model from './editor_model.js'; import {createEditorState} from './editor_state.js'; import {createEditorStore, sourceKey, mediaURL} from './editor_store.js'; -import {generationSize} from './editor_generation.js'; +import {generationSize, continuedSize} from './editor_generation.js'; import {createEditorLibrary} from './editor_library.js'; import {createEditorPlayer} from './editor_player.js'; import {createEditorTransport, registerEditorShortcuts} from './editor_transport.js'; @@ -141,7 +141,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given tracksHost.inert = genHost.inert = true; title.disabled = true; // CSS delays the pending indication; fast opens stay at full opacity. tracksHost.classList.add('fv-ed-pending'); genHost.classList.add('fv-ed-pending'); - let discardOnSwitch = false, initialSelection = null, offerInitial = null; + let discardOnSwitch = false, initialSelection = null, offerInitial = null, extendEntry = null; let viewKey = null, viewReady = false; let library = null, closing = false, closePromise = null, exportSnapshot = null, lastSeed = null, projectEpoch = 0; let disposed = false, suppressSave = false, savedContent = '', initializing = true, tracks = null, genbar = null, adjustPanel = null, exportStatus = null, pointerEditing = false, deferredStamp = null; @@ -199,7 +199,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given const store = createEditorStore(api, {t, notify, standIn: given?.generationStandIn, onGenerationReady: ready => { if (disposed) return; generationAsked = true; genHost.hidden = !ready; element.dataset.generationReady = String(ready); - settleGenbar(); if (ready) offerInitial?.(); + settleGenbar(); if (ready) { offerInitial?.(); extendEntry?.(); } tracks?.refresh?.(); genbar?.update(mostRecentJob ? barJob(jobStatuses.get(mostRecentJob)) : null); updateTransport(); adjustPanel?.update(); }}); @@ -332,7 +332,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given viewKey = record.id || key || 'draft:' + (globalThis.crypto?.randomUUID?.() || `${Date.now()}-${Math.random()}`); let remembered = editorViews.get(viewKey); if (!remembered) try { remembered = JSON.parse(localStorage.getItem(viewPrefix + viewKey)); } catch {} - initialSelection = offerInitial = null; + initialSelection = offerInitial = extendEntry = null; projectEpoch++; player.pause(); for (const stop of jobWatchers.values()) stop(); jobWatchers.clear(); jobStatuses.clear(); progress.clear(); runProgress.clear(); progressConnection?.reset(); mostRecentJob = null; genbar?.update(null); queue.reset(record); stopExport?.(); stopExport = null; exportStatus = exportSnapshot = null; exportButton.textContent = t('Export', '导出'); @@ -435,7 +435,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given } const metadata = store.sources.get(sourceKey(value.source)); try { state.apply(t('Insert source', '放入素材'), record => model.insertSource(record, { - ...value, width: metadata?.width ?? value.width, height: metadata?.height ?? value.height, + ...value, width: metadata?.width ?? value.width, height: metadata?.height ?? value.height, text: value.text ?? metadata?.text, at: value.at ?? model.duration(record)}), {clearRange: state.selection !== initialSelection}); return {ok: true}; } catch (valueError) { @@ -506,7 +506,12 @@ export function openEditor({api, app, from = null, panel = {}, components: given submittedSelection = state.select({kind: 'clip', ids}); await completeSources(); const paths = Object.fromEntries([...store.sources].map(([key, value]) => [key, value.path])); const request = model.generationRequest(prepared, ids, {text, paths}); - const tooLong = request.blocks.filter(block => capability.maxRequestFrames != null && block.frames > capability.maxRequestFrames); + const follow = continuedSize(model.generationBefore(prepared, ids), store.sources), edit = capability.generationSize || generationSize(state.record); + Object.assign(frozen.prompt[frozen.node].inputs, follow || {}); + // Parts made at another size than the edit's have that size's limit. + const limit = (follow && (follow.width !== edit.width || follow.height !== edit.height) + ? await store.requestFrames(follow, state.record.settings) : null) ?? capability.maxRequestFrames; + const tooLong = request.blocks.filter(block => limit != null && block.frames > limit); if (tooLong.length) { state.settle(record => tooLong.reduce((next, block) => model.fail(next, block.id, 'too_long', null), record)); return; @@ -605,10 +610,10 @@ export function openEditor({api, app, from = null, panel = {}, components: given // A sound-only export is not a work: it opens as a file the browser plays and can save. if (value.only === 'sound') { if (value.url) window.open(mediaURL(api, value.url), '_blank', 'noopener'); return; } const module = await import('./library.js'); - module.openLibrary(t, {selectedWork: value.work, extend: async record => { + module.openLibrary(t, {selectedWork: value.work, extend: async (record, options) => { if (await handle.close() === false) throw new Error(saveWarning.textContent); await onView('studio'); - return (await import('./studio.js')).extendVideo(record); + return (await import('./studio.js')).openExtendEditor(record, undefined, options); }}); } function updateExport(value) { @@ -856,7 +861,7 @@ export function openEditor({api, app, from = null, panel = {}, components: given record = model.emptyProject({width: even(metadata?.width || g.width || panel.width || 1344), height: even(metadata?.height || g.height || panel.height || 768), settings: panel.settings || null, name: t('Untitled edit', '未命名剪辑')}); record = model.insertSource(record, {source: {kind: 'work', id}, frames: metadata?.frames || g.frames || Math.round((g.seconds || 15) * 24), - hasVideo: true, hasAudio: metadata?.has_audio ?? work.has_audio !== false, at: 0}); + hasVideo: true, hasAudio: metadata?.has_audio ?? work.has_audio !== false, text: metadata?.text, at: 0}); if (!metadata) store.cache({[key]: {video: mediaURL(api, work.video), audio: mediaURL(api, work.video), frames: g.frames, has_video: true, has_audio: true, path: work.video, missing: false, thumbnail: '/freevideo/library/thumbnail?' + new URLSearchParams({id})}}); } @@ -909,6 +914,29 @@ export function openEditor({api, app, from = null, panel = {}, components: given } for (const [id, value] of progress) tracks.setProgress(id, value); tracks.setScrollLeft(editorViews.get(viewKey)?.scrollLeft); + if (from?.work && from.extend) { + const id = from.work.id || from.work.video?.replace(/^FreeVideo\//, '').replace(/\/video\.mp4$/, ''); + const epoch = projectEpoch; + // Selects the work and, when nothing follows it yet, adds one minimum draft. Runs once; when the + // generation probe failed this time it runs after a later probe succeeds. + const enter = (select = true) => { + if (closing || disposed || epoch !== projectEpoch) return true; + const clip = state.record.tracks[0].clips.findLast(clip => clip.source?.kind === 'work' && clip.source.id === id); + if (!clip) return true; + // Coming back to a work that already has parts after it selects the waiting draft instead of adding another. + const edge = clip.start + clip.frames; + const after = state.record.tracks[0].clips.filter(other => other.id !== clip.id && other.start >= edge); + const waiting = after.find(other => other.pending === 'draft'); + if (select) state.select({kind: 'clip', ids: model.pair(state.record, (waiting || clip).id).map(member => member.id)}); + if (after.length || !clip.extend || !clip.source || store.sources.get(sourceKey(clip.source))?.missing) return true; + if (!store.generationAvailable) return false; + const spare = Math.min(...model.pair(state.record, clip.id).map(member => member.length - member.in - member.frames)); + tracks.extendClip(clip.id, spare + model.RULES.minSeconds * model.FPS, {silent: true}); + return true; + }; + await store.generationReady(); + if (!enter() && !closing && !disposed) extendEntry = () => { if (enter(false) || closing || disposed) extendEntry = null; }; + } })().catch(error).finally(() => { // Every exit (including close and partial failure) settles readiness. // After a completed start the saved baseline is already set; edits made since must stay unsaved. diff --git a/web/editor_entry.js b/web/editor_entry.js index 82081776..9d2926e9 100644 --- a/web/editor_entry.js +++ b/web/editor_entry.js @@ -15,7 +15,7 @@ function editorPanel(node) { return {node: String(node.id), prompt: value.output, extra_data: {extra_pnginfo: {workflow: value.workflow || {}}}}; }}; } -export function openEditorView(node, {studio, work = null} = {}) { +export function openEditorView(node, {studio, work = null, extend = false} = {}) { studio?.querySelectorAll('video,audio').forEach(media => media.pause()); const panel = editorPanel(node); const progressElement = studio?.querySelector('.fv-generation-progress'); @@ -27,7 +27,7 @@ export function openEditorView(node, {studio, work = null} = {}) { return number ? {percent: Number(number[1])} : {percent: 0}; }; } - const handle = openEditor({api, app, panel, from: work ? {work} : null, onView(view) { + const handle = openEditor({api, app, panel, from: work ? {work, ...(extend ? {extend: true} : {})} : null, onView(view) { if (view === 'nodes') showNodes(node, studio); else if (view === 'autopilot') openAutopilot(node, {studio}); else if (studio?.open) viewChanged('studio', node); diff --git a/web/editor_generation.js b/web/editor_generation.js index 65512c22..4bf818da 100644 --- a/web/editor_generation.js +++ b/web/editor_generation.js @@ -13,3 +13,12 @@ export function generationSize(project, sizes = []) { || Math.abs(a.width * a.height - area) - Math.abs(b.width * b.height - area)); return {width: candidates[0].width, height: candidates[0].height}; } +// A part that continues finished footage is made at that footage's size, whatever the edit's own: the size all the +// footage a request continues has in common. null when a video in the request starts fresh or the sizes differ; the +// edit's size then stands. +export function continuedSize(before, sources) { + const sizes = before.map(key => key && sources.get(key)); + if (!sizes.length || !sizes.every(valid)) return null; + const {width, height} = sizes[0]; + return sizes.every(size => size.width === width && size.height === height) ? {width, height} : null; +} diff --git a/web/editor_loader.js b/web/editor_loader.js index 8221aea1..3b2bc61e 100644 --- a/web/editor_loader.js +++ b/web/editor_loader.js @@ -5,8 +5,10 @@ import {app} from '../../scripts/app.js'; export async function openEditorView(node, options) { try { const entry = await import('./editor_entry.js'); + if (options?.isCurrent?.() === false) return null; return entry.openEditorView(node, options); } catch (error) { + if (options?.isCurrent?.() === false) return null; console.warn('FreeVideo edit view:', error); const language = new URLSearchParams(location.search).get('freevideo_lang'); const cn = language === 'zh' || (language !== 'en' && String(navigator.language || '').toLowerCase().startsWith('zh')); diff --git a/web/editor_model.js b/web/editor_model.js index 67031600..0c80ecf3 100644 --- a/web/editor_model.js +++ b/web/editor_model.js @@ -24,6 +24,10 @@ const check = (yes, code = 'invalid', message = code) => { if (!yes) reject(code const ID = /^c[1-9]\d{0,8}$/; // Who last wrote a block's description (its note on the timeline), and an outside reference (a shot id). const TEXT_BY = ['user', 'agent'], REF = /^[A-Za-z0-9_-]{1,128}$/; +// A short line about a block for whoever looks at the timeline; regenerating never reads it. +// One line: no control or line-break character anywhere, nothing at its ends that either side would strip. +const NOTE_BREAK = /[\u0000-\u001f\u007f-\u009f\u2028\u2029]/; +const validNote = text => validText(text) && text === text.trim() && !NOTE_BREAK.test(text) && Array.from(text).length >= 1 && Array.from(text).length <= 120; const WORK = /^\d{4}-\d{2}-\d{2}\/[0-9a-f]{32}$/; const PENDING = [null, 'draft', 'queued', 'running', 'failed', 'stopped']; const key = source => `${source.kind}:${source.id}`; @@ -168,6 +172,7 @@ export function parse(value) { // Who last wrote the description, and an outside reference (a shot id) for callers. if (item.text_by !== undefined) { check(TEXT_BY.includes(item.text_by), 'text'); clip.text_by = item.text_by; } if (item.ref !== undefined) { check(typeof item.ref === 'string' && REF.test(item.ref), 'ref'); clip.ref = item.ref; } + if (item.note !== undefined) { check(validNote(item.note), 'text'); clip.note = item.note; } // The part this one was pulled out of, or the part before it in the same range. if (item.follows !== undefined) { check(kind === 'video' && typeof item.follows === 'string' && ID.test(item.follows) && item.follows !== item.id, 'follows'); clip.follows = item.follows; } check(source !== null || (item.take === undefined && item.made_text === undefined), 'versions'); madeFields(item, clip); @@ -335,14 +340,31 @@ export function setRef(record, ids, ref) { check(typeof ref === 'string' && (!ref || REF.test(ref)), 'ref'); return onBlocks(record, ids, clip => { if (ref) clip.ref = ref; else delete clip.ref; }, false); } +// A note kept with the block; an empty value removes it. A block being generated can still be noted. +export function setNote(record, ids, text) { + check(typeof text === 'string' && (!text.trim() || validNote(text.trim())), 'text'); + return onBlocks(record, ids, clip => { if (text.trim()) clip.note = text.trim(); else delete clip.note; }, false, {picture: true, kept: true}); +} // Only an agent's description is marked; no mark means the user wrote it, as in every earlier edit. -function written(clip, text, by) { clip.text = text; if (text && by === 'agent') clip.text_by = by; else delete clip.text_by; } -function onBlocks(record, ids, change, refuseBusy) { +// A note was written about the description it came with, so it goes when the user writes another one. +function written(clip, text, by) { + if (by === 'user' && text !== clip.text) delete clip.note; + clip.text = text; if (text && by === 'agent') clip.text_by = by; else delete clip.text_by; +} +function onBlocks(record, ids, change, refuseBusy, {picture = false, kept = false} = {}) { check(Array.isArray(ids) && ids.length > 0 && ids.every(id => typeof id === 'string'), 'id'); for (const id of ids) check(find(record, id), 'missing'); const selected = linkedIds(record, ids), clips = all(record).filter(clip => selected.has(clip.id)); - check(!clips.some(isText), 'text'); assertUnlocked(record, [...selected]); if (refuseBusy) busy(clips); + check(!clips.some(isText), 'text'); + // A sound block takes the change only together with the picture it is linked to. + check(!picture || clips.every(clip => trackOf(record, clip.id).kind === 'video' || clip.link), 'text'); + assertUnlocked(record, [...selected]); if (refuseBusy) busy(clips); const next = copy(record); for (const clip of all(next)) if (selected.has(clip.id)) change(clip); + // The blocks set aside while a part is being replaced take the change too: whichever of them comes back shows it. + if (kept) for (const tx of next.replacements || []) for (const row of [...tx.originals, ...tx.clips]) { + const shown = clips.filter(clip => trackOf(record, clip.id).id === row.id); + for (const clip of row.clips) if (shown.some(v => v.id === clip.id || (clip.start >= v.start && end(clip) <= end(v)))) change(clip); + } return finish(next); } export function linkedIds(record, ids) { @@ -411,9 +433,19 @@ function sourceAudioTrack(record, start, frames, allowRoom = false) { } throw new EditError('audio_full'); } -export function insertSource(record, {source, frames, hasVideo, hasAudio, width, height, at: position = 0}) { - source = sourceData(source); check(whole(frames, 1) && whole(position), 'range'); +// A saved project file holds 8 MiB; every character outside plain ASCII is written there as six. +const STORED_ROOM = 8 * 1024 * 1024 - 64 * 1024; +function storedSize(record) { + const written = JSON.stringify(record); let size = written.length; + for (let i = 0; i < written.length; i++) { const code = written.charCodeAt(i); size += code > 126 ? 5 : code === 44 || code === 58 ? 1 : 0; } + return size; +} +// A saved video brings the description it was generated from; one that cannot be kept is left out. +export function insertSource(record, {source, frames, hasVideo, hasAudio, width, height, at: position = 0, text = '', by = 'user'}) { + const given = source; source = sourceData(source); check(whole(frames, 1) && whole(position), 'range'); check(typeof hasVideo === 'boolean' && typeof hasAudio === 'boolean' && (hasVideo || hasAudio), 'source'); + check(TEXT_BY.includes(by), 'text'); + const words = hasVideo && typeof text === 'string' && validText(text.trim()) ? text.trim() : ''; const next = copy(record); if (hasVideo) followFirstSize(next, width, height); let track = trackBy(next, hasVideo ? 'v1' : 'a1'); @@ -423,11 +455,16 @@ export function insertSource(record, {source, frames, hasVideo, hasAudio, width, const sound = available ? audioTrack(next, sourceAudioTrack(available, start, frames, true).id) : null; applyRoom(next, start, frames, {tracks: [track.id, ...(sound ? [sound.id] : [])]}); const primary = newClip(next, track, {source, frames, start}); + if (words) written(primary, words, by); if (hasVideo && hasAudio) { const mate = newClip(next, sound, {source, frames, start}); primary.link = mate.id; mate.link = primary.id; + if (words) written(mate, words, by); } - return finish(next); + const result = finish(next); + // The description is a convenience: a project it would make too large to save takes the video without it. + if (words && storedSize(result) > STORED_ROOM) return insertSource(record, {source: given, frames, hasVideo, hasAudio, width, height, at: position}); + return result; } export function move(record, id, start) { if (isText(find(record, id))) return changeText(record, id, {start}); @@ -896,6 +933,12 @@ export function prepare(record, ids) { } return finish(next); } +// The finished footage each video of a generation request continues, as source keys; null for one that starts fresh. +export function generationBefore(record, ids) { + const clips = generationClips(record, ids); + return generationPlan(record, clips).filter(row => !row.before || !clips.some(clip => clip.id === row.before.id)) + .map(row => row.before ? key(row.before.source) : null); +} export function generationRequest(record, ids, {text, paths = {}} = {}) { check(object(paths), 'source'); if (text !== undefined) check(validText(text), 'text'); diff --git a/web/editor_notes.css b/web/editor_notes.css index e61a0063..48bdd29c 100644 --- a/web/editor_notes.css +++ b/web/editor_notes.css @@ -4,7 +4,7 @@ .fv-editor .fv-ed-notes-again{display:inline-flex;align-items:center;gap:4px;height:26px;padding:0 8px;font-size:12px} .fv-ed-notes-again .fv-ed-glyph{width:12px;height:12px} .fv-editor .fv-ed-notes-again:disabled{opacity:.4} -.fv-editor .fv-ed-notes-text{display:block;height:auto;box-sizing:border-box;width:100%;min-height:34px;max-height:94px;padding:6px 10px;resize:none;overflow-y:auto;font:13px/20px var(--fv-font);color:var(--fv-ink);background:var(--fv-bg);border:1px solid var(--fv-border);border-radius:8px} +.fv-editor .fv-ed-notes-text{display:block;height:auto;box-sizing:border-box;width:100%;min-height:34px;max-height:98px;padding:6px 10px;resize:none;overflow-y:auto;font:13px/20px var(--fv-font);color:var(--fv-ink);background:var(--fv-bg);border:1px solid var(--fv-border);border-radius:8px} .fv-editor .fv-ed-notes-text:focus-visible{outline:1px solid var(--fv-accent);outline-offset:-1px;border-color:var(--fv-accent)} .fv-editor .fv-ed-notes-text:disabled{opacity:.4} .fv-ed-notes-versions{display:flex;flex-direction:column;gap:6px;margin-top:6px} diff --git a/web/editor_notes.js b/web/editor_notes.js index 54bdc670..46e71322 100644 --- a/web/editor_notes.js +++ b/web/editor_notes.js @@ -12,8 +12,10 @@ function glyph(d) { const AGAIN = 'M13.5 2.5v3.5H10M13.2 6A5.5 5.5 0 1 0 13.5 9.5', PENCIL = 'M10.5 2.5l3 3-8 8h-3v-3z'; const generating = clip => ['queued', 'running'].includes(clip?.pending); const firstLine = text => (text || '').trim().split('\n')[0]; -// Only the picture track grows a description strip, and only once one block has a description. -export const describedTrack = track => track.kind === 'video' && track.clips.some(clip => !isText(clip) && clip.text?.trim()); +// A note written about a block (by an agent working on the project) is what the strip shows; without one, the description. +export const stripText = clip => clip.note || firstLine(clip.text); +// Only the picture track grows a description strip, and only once one block has something to show. +export const describedTrack = track => track.kind === 'video' && track.clips.some(clip => !isText(clip) && stripText(clip)); const agentWrote = record => record.tracks.some(track => track.clips.some(clip => clip.text_by === 'agent')); // Status words appear only in the hover tip and the panels (Iris 10-11). export function blockStatus(clip, t, fraction = 0) { @@ -101,7 +103,7 @@ export function createBlockNotes({state, t, actions}) { function showTip(item) { const view = item.element.closest('.fv-ed-track-view'), clip = item.clip; if (!view || !item.element.isConnected) return; const fraction = Number(item.element.style.getPropertyValue('--fv-ed-progress')) || 0; - const text = [clip.text.trim(), blockStatus(clip, t, fraction)].filter(Boolean).join(' · '); + const text = [clip.note || clip.text.trim(), blockStatus(clip, t, fraction)].filter(Boolean).join(' · '); if (!text) return; if (!tip) { tip = el('div', 'fv-ed-note-tip'); tip.setAttribute('role', 'tooltip'); } if (tip.parentElement !== view) view.append(tip); @@ -120,7 +122,7 @@ export function createBlockNotes({state, t, actions}) { items.set(item.clip.id, item); item.element.dataset.noted = String(noted); item.note.hidden = item.number.hidden = !noted; - item.note.textContent = firstLine(item.clip.text); item.number.textContent = String(index + 1); + item.note.textContent = stripText(item.clip); item.number.textContent = String(index + 1); // The pencil marks blocks an agent leaves alone; a project nobody else wrote in has none. item.mark.hidden = !agentWrote(record) || !!item.clip.text_by; } diff --git a/web/editor_store.js b/web/editor_store.js index c27ba232..eae650d1 100644 --- a/web/editor_store.js +++ b/web/editor_store.js @@ -391,6 +391,18 @@ export function createEditorStore(api, {t = en => en, notify = () => {}, onGener // these limits; only the callback publishes a measured capability. read(); return fallback; }, + // The longest single part this computer makes at a size other than the edit's; null when it cannot be told. + async requestFrames(size, settings) { + try { + for (let tries = 0; tries < 5 && !disposed; tries++) { + const reply = await json('/freevideo/capabilities?' + new URLSearchParams(size)); + if (reply?.status !== 'computing') return !reply || (reply.status && reply.status !== 'ready') ? null + : capabilityPoints(reply, {...size, settings}).maxRequestFrames; + await new Promise(done => clock.setTimeout(done, 1000)); + } + } catch { /* not known */ } + return null; + }, async startChain(payload) { if (standIn) return standIn.startChain(payload); if (!generationAvailable) return null; diff --git a/web/editor_text_model.js b/web/editor_text_model.js index 58626532..cf421837 100644 --- a/web/editor_text_model.js +++ b/web/editor_text_model.js @@ -48,7 +48,7 @@ export function readTextProject(raw, parseMedia) { check(typeof c.text === 'string' && Array.from(c.text).length <= TEXT_LIMIT && !/[\u0000-\u0008\u000b-\u001f\u007f\ud800-\udfff]/u.test(c.text)); check(['source','pending','link','part','job','reason'].every(k => c[k] == null) && (c.extend === undefined || c.extend === false) && (c.versions === undefined || (Array.isArray(c.versions) && c.versions.length === 0))); check((c.in === undefined || c.in === 0) && (c.length === undefined || c.length === c.frames)); - check(!['volume', 'fit', 'splice', 'target_frames', 'text_by', 'ref', 'take', 'made_text'].some(k => c[k] !== undefined)); + check(!['volume', 'fit', 'splice', 'target_frames', 'text_by', 'ref', 'note', 'take', 'made_text'].some(k => c[k] !== undefined)); const clip = {id: c.id, start: c.start, frames: c.frames, in: 0, length: c.frames, source: null, link: null, text: c.text, part: null, pending: null, reason: null, job: null, extend: false, versions: [], text_kind: track.kind}; check(track.kind !== 'subtitle' || c.text_style === undefined); diff --git a/web/editor_tracks.js b/web/editor_tracks.js index efde6c30..c0a821c5 100644 --- a/web/editor_tracks.js +++ b/web/editor_tracks.js @@ -212,6 +212,19 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = actions.editError?.(error); return false; } } + function extendClip(id, frames, {silent = false} = {}) { + if (disposed || actions.generationAvailable?.() === false) return false; + let next; + try { next = model.extendClip(state.record, id, frames, actions.generationLimits?.() || {}); } + catch (error) { + if (!(error instanceof model.EditError)) throw error; + if (!silent) actions.editError?.(error); + return false; + } + const ids = next.tracks[0].clips.filter(clip => clip.pending === 'draft' && !model.find(state.record, clip.id)) + .flatMap(clip => model.pair(next, clip.id).map(member => member.id)); + return attempt(t('Extend', '拉长'), () => next, ids.length ? {selectionAfter: {kind: 'clip', ids}} : undefined); + } function position(node, start, frames) { node.style.left = `${pixels(start)}px`; node.style.width = `${Math.max(2, pixels(frames))}px`; } @@ -840,18 +853,8 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = else if (finished.type === 'plannedBoundary') attempt(t('Resize parts', '调整段长'), record => model.resizePlannedBoundary(record, finished.id, finished.neighbor.clip.id, result.frame, actions.generationLimits?.())); else if (result.kind === 'planned') attempt(t('Resize', '改长短'), record => model.resizePlanned(record, finished.id, finished.edge, result.frame, actions.generationLimits?.())); else if (result.kind === 'extend') { - if (actions.generationAvailable?.() === false) return; const frames = result.frame - finished.clip.start - finished.clip.frames + result.frames; - let next = null; - try { next = model.extendClip(state.record, finished.id, frames, actions.generationLimits?.() || {}); } - catch (error) { if (!(error instanceof model.EditError)) throw error; actions.editError?.(error); } - if (next) { - // The new parts stay selected through undo and redo, so Generate stays usable. - const clip = model.find(next, finished.id), start = clip.start + clip.frames; - const ids = next.tracks[0].clips.filter(c => c.pending === 'draft' && c.start >= start && c.start < start + result.frames) - .flatMap(c => model.pair(next, c.id).map(member => member.id)); - attempt(t('Extend', '拉长'), () => next, ids.length ? {selectionAfter: {kind: 'clip', ids}} : undefined); - } + extendClip(finished.id, frames); } else attempt(t('Trim', '裁短'), record => model.trim(record, finished.id, finished.edge, result.frame)); renderSelection(); } finally { actions.gestureEnd?.(); } @@ -1097,7 +1100,7 @@ export function createTracks(host, {state, sources = {}, t = en => en, actions = showInsertion(null); insertion.classList.remove('fv-ed-blocked-insertion'); }, 700); } - return {element, keyboard, menu, dismissPopup, flashBlocked, seekPlayhead, + return {element, keyboard, menu, dismissPopup, flashBlocked, seekPlayhead, extendClip, setSnapping(enabled) { if (typeof enabled !== 'boolean') return {ok: false, reason: 'snapping'}; setTransportSnapping(state, enabled); cachedPoints = allPoints(gesture?.ids || []); diff --git a/web/error_panel.js b/web/error_panel.js index 201756ed..ef625d68 100644 --- a/web/error_panel.js +++ b/web/error_panel.js @@ -11,7 +11,13 @@ const record = value => value && typeof value === 'object' && !Array.isArray(val const PLAIN_ERRORS = [ {key: 'continuation-missing', message: 'The previous video has no usable continuation data. Generate it again with this version of FreeVideo.', title: ['This video cannot be continued', '这个视频暂时不能接续'], - summary: ['Generate the previous video again with this version of FreeVideo. Older videos, replaced files and expired continuation data cannot be used.', '请用当前版本的 FreeVideo 重新生成上一段视频。旧版本的视频、被替换的文件或已清理接续数据的视频不能接续。']}, + summary: ['Generate the previous video again, then continue it.', '请重新生成上一段视频，再接续。']}, + {key: 'continuation-short', message: 'The previous video is too short to continue; it needs to be at least 2 s long.', + title: ['The previous video is too short', '上一段视频太短'], + summary: ['Choose a video that is at least 2 seconds long.', '请选择至少 2 秒的视频。']}, + {key: 'continuation-unreadable', message: 'The previous video cannot be read.', + title: ['The previous video cannot be read', '无法读取上一段视频'], + summary: ['Choose an MP4 video that plays normally.', '请选择能正常播放的 MP4 视频。']}, {key: 'continuation-size', message: 'Use the same width and height as the previous video.', title: ['The video sizes differ', '两段视频的画面大小不一致'], summary: ['Set the width and height to the previous video’s size.', '请把宽度和高度设为与上一段视频相同。']}, @@ -50,7 +56,18 @@ const PLAIN_ERRORS = [ function knownError(text) { const lines = String(text).slice(0, 65536).split(/\r?\n/).map(line => line.trim()).filter(Boolean); const index = /^FreeVideo generation failed(?: \(exit -?\d+\))?$/.test(lines[0] ?? '') ? 1 : 0; - const line = (lines[index] ?? '').replace(/^[A-Za-z_][\w.]*(?:Error|Exception):\s*/, ''); + const line = (lines[index] ?? '').replace(/^(?:[A-Za-z_][\w.]*(?:Error|Exception)|(?:[\w.]+\.)?SamplingTableUnavailable):\s*/, ''); + if (line.startsWith('SAMPLING_TABLE_UNAVAILABLE: ')) { + try { + const d = JSON.parse(line.slice('SAMPLING_TABLE_UNAVAILABLE: '.length)); + if (Number.isInteger(d.base_steps) && d.base_steps >= 1 && d.base_steps <= 32 + && typeof d.two_pass === 'boolean' && Number.isInteger(d.refine_steps) && d.refine_steps >= 0 && d.refine_steps <= 31 + && Number.isSafeInteger(d.missing_bytes) && d.missing_bytes > 0 && typeof d.decimal_sizes === 'boolean' + && Array.isArray(d.available) && d.available.every(i => Number.isInteger(i) && i >= 0 && i <= 3)) + return {kind: 'sampling-table', base_steps: d.base_steps, two_pass: d.two_pass, refine_steps: d.refine_steps, + missing_bytes: d.missing_bytes, decimal_sizes: d.decimal_sizes, available: [...new Set(d.available)].sort()}; + } catch { /* Malformed metadata stays in technical details. */ } + } return PLAIN_ERRORS.find(error => line === error.message || error.aliases?.includes(line))?.key ?? null; } @@ -230,6 +247,37 @@ export function createErrorReport(detail, context = {}) { export function failureAdvice(value, t, known = knownError(value)) { const text = String(value).slice(0, 65536); const row = (kind, title, summary, action) => ({kind, title: t(...title), summary: t(...summary), action: t(...action)}); + if (known?.kind === 'sampling-table') { + // Names and plan matching follow sampling_effort.js; metadata carries no user text. + const levels = [['Light', '轻量'], ['Medium', '标准'], ['High', '精细'], ['Max', '极致']]; + const index = [8, 12, 16, 20].findIndex((steps, i) => known.base_steps === steps + && known.two_pass === (i === 0) && (i !== 0 || known.refine_steps === 3)); + const custom = index < 0; + // Iris: a level name and its 档 never break apart. The summary is set as text, + // so word joiners (U+2060) keep each group on one line instead of markup. + const keep = value => [...value].join('\u2060'); + const join = language => { + const names = known.available.map(i => language ? `“${levels[i][1]}”` : levels[i][0]); + if (language) names[names.length - 1] += '档'; + const kept = language ? names.map(keep) : names; + return kept.length < 2 ? kept[0] || '' : kept.slice(0, -1).join(language ? '、' : ', ') + (language ? '或' : ' or ') + kept.at(-1); + }; + const bytes = known.missing_bytes; + // As launcher_copy.size_text: one decimal for GiB, whole MiB; decimal units on macOS. + let size = bytes >= 2**30 ? (bytes / 2**30).toFixed(1) + ' GiB' : Math.floor(bytes / 2**20) + ' MiB'; + if (known.decimal_sizes) { + const unit = [[1e12, 'TB', 2], [1e9, 'GB', 2], [1e6, 'MB', 1]].find(([scale]) => bytes >= scale); + size = unit ? Number((bytes / unit[0]).toFixed(unit[2])) + ' ' + unit[1] : Math.round(bytes / 1e3) + ' KB'; + } + // A custom step count is not a quality level: no "档" / "quality level" for it. + const title = custom + ? [`The sampling table for Custom (${known.base_steps} steps) is not downloaded yet`, `“自定义（${known.base_steps} 步）”的采样表还没有下载`] + : [`The ${levels[index][0]} quality level's sampling table is not downloaded yet`, `${keep(`“${levels[index][1]}”档`)}的采样表还没有下载`]; + const what = custom ? ['the sampling table for this step count', '这个步数的采样表'] : ["that level's sampling table", '这一档的采样表']; + return {...row('sampling-table', title, + [`This video first needs ${what[0]} (about ${size}), and it could not be downloaded this time. Check the network connection and generate again; FreeVideo downloads it automatically.${known.available.length ? ` You can also switch to a quality level that is already available: ${join(0)}.` : ''}`, + `这次生成需要先下载${what[1]}（约 ${size}），但这次没能下载。请确认网络连接后再生成一次，FreeVideo 会自动下载。${known.available.length ? `也可以改用已经可以使用的${join(1)}。` : ''}`], ['', '']), plain: true}; + } // ComfyUI and worker tracebacks carry English exceptions. Translate these // actionable failures through the existing signature/advice path. for (const {key, title, summary} of PLAIN_ERRORS) { @@ -471,7 +519,7 @@ export function createErrorPanel(t) { ? t('Use Download report in the preview or node panel for the diagnostic report.', '完整诊断请使用预览区或节点面板中的“下载报告”。') : t('Known input text, paths and common credentials are removed. Export saves a local error summary; review it before sharing.', '已去除已知输入文本、路径和常见凭据。导出会保存本地错误摘要，分享前可查看内容。'); - copy.textContent = t('Copy error', '复制错误'); + copy.textContent = advice.kind === 'sampling-table' ? t('Copy full details', '复制完整详情') : t('Copy error', '复制错误'); if (popup) showFailureDialog(report, t); return report; }, clear() { currentReport = null; details.value = ''; element.hidden = true; }}; diff --git a/web/first_run_guide.js b/web/first_run_guide.js index 4efc7c76..41d452e9 100644 --- a/web/first_run_guide.js +++ b/web/first_run_guide.js @@ -12,7 +12,7 @@ export const REPLAY_TEXT = { export const CHAPTERS = [ {id: 'describe', anchor: '[data-fv-guide="describe"]', added: false, en: 'Type your prompt here. You can have an agent follow the Skill at the top right, or turn on FreeToken enhancement.', - zh: '在这里输入提示词。可以让 Agent 参考右⁠上⁠角的 Skill，或开启 FreeToken 提示词增强。'}, + zh: '在这里输入提示词。可以让 Agent 参⁠考 Skill，或开启 FreeToken 提示词增强。'}, {id: 'range', anchor: '[data-fv-guide="range"]', added: true, en: 'See the available sizes and lengths here, and how long each takes.', zh: '在这里查看可生成的尺\u2060寸、时长和预计用时。'}, {id: 'generate', anchor: '[data-fv-guide="generate"]', added: false, @@ -320,10 +320,18 @@ export function createFirstRunGuide({dialog, api, cn, isBusy = () => false}) { if (current === handle) current = null; }}; current = handle; - const startTimer = setTimeout(() => { - if (!automatic || attempted || manual || layer) return; - attempted = true; - show().then(shown => { if (!shown) stopped = true; }); - }, 1000); + // Steps are fixed when the guide starts, so give an anchor that is still waiting for its data up to 5 s. + // The Agent tab is not waited for: its discovery has already finished when the guide is created. + const pending = () => CHAPTERS.some(c => c.id !== 'agent' && target(c) && !visible(target(c))); + function begin(waited = 0) { + startTimer = setTimeout(() => { + if (!automatic || attempted || manual || layer || disposed) return; + if (pending() && waited < 5000) { begin(waited + 250); return; } + attempted = true; + show().then(shown => { if (!shown) stopped = true; }); + }, waited ? 250 : 1000); + } + let startTimer = null; + begin(); return handle; } diff --git a/web/freevideo.js b/web/freevideo.js index 8a962606..7678efee 100644 --- a/web/freevideo.js +++ b/web/freevideo.js @@ -3,7 +3,7 @@ import { app } from "../../scripts/app.js"; import { api } from "../../scripts/api.js"; import { openEditorView } from './editor_loader.js'; import { openStudio, loraPanel, loraWarning, promptGuide } from "./studio.js"; -import { createGenerationProgress, referenceTrimText, thermalText, encoderFileText, encoderFilePath } from './generation_progress.js'; +import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, encoderFileText, encoderFilePath } from './generation_progress.js'; import { createProgressConnection } from './progress_connection.js'; import { notifyCompatibility } from './compatibility.js'; import { startUpdateChecks } from './updates.js'; @@ -26,7 +26,7 @@ const style = document.createElement("style"); style.textContent = ` .fv-panel{box-sizing:border-box;width:100%;height:100%;padding:16px;background:var(--fv-card);color:var(--fv-ink);border:1px solid var(--fv-border);border-radius:var(--fv-r-md,12px);font:13px/1.6 var(--fv-font);overflow:auto;color-scheme:dark} .fv-panel *{box-sizing:border-box}.fv-toolbar,.fv-row,.fv-links{display:flex;align-items:center;gap:8px;flex-wrap:wrap} -.fv-panel .fv-links{justify-content:center}.fv-panel[data-freevideo=result]>.fv-note{text-align:center} +.fv-panel .fv-links{justify-content:center}.fv-panel[data-freevideo=result]>.fv-note{text-align:center;text-wrap:balance} .fv-panel .fv-toolbar{padding:4px 0 8px}.fv-panel .fv-stats{text-align:center}.fv-panel .fv-stat{padding:10px 3px;border-radius:var(--fv-r-md,12px);background:var(--fv-raised)} .fv-panel .fv-stats[hidden]{display:none} .fv-panel .fv-connected{display:flex;align-items:center;justify-content:center;gap:6px;flex-wrap:wrap;margin:10px 0}.fv-panel .fv-connection{border:1px solid var(--fv-border);border-radius:var(--fv-r-sm,8px);padding:5px 9px;background:var(--fv-selected);color:var(--fv-accent);font-size:var(--fv-micro,12px)} @@ -504,6 +504,8 @@ function resultPanel(node) { for (const row of value.reference_trims || []) panel.append(el("div", referenceTrimText(row, text), "fv-note fv-trim-note")); if (value.low_memory && !value.result_cache_hit) resultNotes.push(el("div", text('This video used the low-memory mode for this GPU, so it took longer than usual. A lower resolution or a shorter video runs faster.', '这段视频使用了省显存方式，所以比平时慢。降低分辨率或缩短时长会更快。'), "fv-note")); if (value.thermal && !value.result_cache_hit) resultNotes.push(el("div", thermalText(value.thermal, text), "fv-note")); + const shared = value.shared_gpu && !value.result_cache_hit ? sharedGpuText(value.shared_gpu, text) : ''; + if (shared) resultNotes.push(el("div", shared, "fv-note")); if (value.encoder_file) { const note = el("div", encoderFileText(value.encoder_file, text), "fv-note"); const path = encoderFilePath(value.encoder_file); diff --git a/web/generation_progress.js b/web/generation_progress.js index 2fee372b..6c32e8a7 100644 --- a/web/generation_progress.js +++ b/web/generation_progress.js @@ -177,11 +177,21 @@ function trimStatus(rows, t) { export function thermalText(thermal, t) { const share = thermal.sm_clock_max_mhz ? Math.round(100 * thermal.sm_clock_mean_mhz / thermal.sm_clock_max_mhz) : null; const heat = Number.isFinite(thermal.temperature_mean_c) ? Math.round(thermal.temperature_mean_c) : null; - const detail = [heat !== null ? `${heat} °C` : '', share !== null ? t(`${share}% of its peak clock`, `频率只有峰值的 ${share}%`) : ''].filter(Boolean).join(t(', ', '，')); + const detail = [heat !== null ? `${heat}\u00a0°C` : '', share !== null ? t(`${share}% of its peak clock`, `频率只有峰值的 ${share}%`) : ''].filter(Boolean).join(t(', ', '，')); return t(`The GPU spent most of this video in thermal slowdown${detail ? ` (${detail})` : ''}, so it took much longer than usual. Improve cooling; on a laptop, plug in and choose a high-performance power mode.`, `生成这段视频时，显卡大部分时间因过热降频${detail ? `（${detail}）` : ''}，所以比平时慢很多。请改善散热；笔记本请接通电源并选择高性能模式。`); } +// Other programs held a substantial share of the GPU during this run (summary.shared_gpu). +// The share is rounded to 5% and written as at most 95%; without a usable number there is no note. +export function sharedGpuText(shared, t) { + const mean = shared?.other_mean_percent; + if (typeof mean !== 'number' || !Number.isFinite(mean)) return ''; + const percent = Math.min(95, Math.round(mean / 5) * 5); + return t(`Other programs were using about ${percent}% of the GPU.`, + `其他程序占用了约 ${percent}% 的显卡。`); +} + export function encoderFilePath(file) { return file?.offline_installation === true && typeof file.path === 'string' ? file.path : ''; } @@ -216,17 +226,24 @@ export function referenceTrimText(row, t) { `${name} 比生成的视频长，已截取前 ${used} 秒${rule}`); } -// The time left on show `dt` seconds on, given the engine's newest estimate (null when it sent none). Never raised: -// an estimate at or above the shown time holds it, so a slower run does not count down to "less than a minute" -// early. A lower one is eased into. -export function remainingShown(shown, target, dt) { +// The time left on show `dt` seconds on, given the engine's newest estimate (null when it sent none). Once measured, an +// estimate at or above the shown time holds it, so a slower run does not count down to "less than a minute" +// early; one that has stayed well above it for a minute (`above` seconds so far) raises it gently, or a run +// measured much slower would leave a low number standing for minutes. A lower one is eased into. An initial +// estimate (made before this run was measured) is followed both ways: a slow machine must not keep a low number. +export function remainingShown(shown, target, dt, initial = false, above = 0) { if (target === null) return Math.max(0, shown - dt); + if (initial) return shown + (target - shown) * (1 - Math.exp(-dt / 15)); if (target < shown - dt) { const counted = Math.max(0, shown - dt); return counted + (target - counted) * (1 - Math.exp(-dt / 15)); } - return target < shown ? target : shown; + if (target < shown) return target; + return above >= 60 ? Math.min(target, shown + Math.max(1, shown * .003) * dt) : shown; } +// The seconds the engine's estimate has stayed well above the shown time (a quarter more), `dt` seconds on. +export const remainingAbove = (above, shown, target, dt) => + target !== null && target > Math.max(shown * 1.25, shown + 30) ? above + dt : 0; // The whole-video fraction the engine's last message stands for now: its estimate moved on by the seconds since the // step sample (stateAge) and the whole-video sample (overallAge) arrived. Null when the message carries none. export function estimatedFraction(state, overall, stateAge, overallAge) { @@ -294,6 +311,13 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f // Mac: the unified-memory notice of this request, and whether its first pass has ended. let memoryTight = null, firstPassOver = false; let state = {}, overall = {}, started = 0, sampledAt = 0, overallAt = 0, timer = null, elapsedShown = 0; + // Where the bridge's remaining time comes from: 'similar' (no record of these + // settings here yet), 'local' (this computer's record) or 'measured' (this run). + // Absent from older bridges, which keep the two-steps rule. + let estimateSource = null, etaChangedAt = -Infinity; + // The page shows its own "preparing" row at the click; the engine's first + // message for that request follows and must not restart the elapsed time. + let placeholder = false, elapsedLead = 0; let lastProgressAt = 0, previousProgress = null, segmentStartedAt = 0, segmentKey = ''; let pauseAnnounced = false, lastExplanation = '', announcedMessages = new Set(); let announcementRecord = null; @@ -301,7 +325,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f let completedSteps = 0, previousRemaining = null, remainingAt = 0, lastActivity = null; // Once the shown time reaches zero before the video is saved, it stays hidden for this request. // The last minute runs on real time even while an estimate holds the number. - let remainingRetired = false, lastMinuteAt = null, lastMinute = 0; + let remainingRetired = false, lastMinuteAt = null, lastMinute = 0, aboveSeconds = 0; let shownFraction = null; // The visible percentage counts toward its value with the bar instead of // jumping. Accessible values are set exactly and immediately elsewhere. @@ -600,7 +624,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f } // Preserve the server's heartbeat/floor policy for the raw target only. - // The displayed estimate has its own clock and never increases. + // The displayed estimate has its own clock; once it is no longer initial it never increases. function rawRemaining() { if (!valid(overall.remaining_seconds)) return null; return hasHeartbeat ? overall.remaining_seconds : Math.max( @@ -608,8 +632,25 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f overall.remaining_seconds - Math.max(0, (now() - overallAt) / 1000)); } function observeRemaining(message) { - if (completedSteps < 2 || remainingSeconds !== null || remainingRetired || !message.overall) return; + if (!message.overall) return; + const source = typeof message.overall.remaining_source === 'string' ? message.overall.remaining_source : null; const raw = valid(message.overall.remaining_seconds) ? rawRemaining() : null; + if (source) { + // The bridge estimates the whole request from the first message: + // show it at once. Leaving the initial (similar-settings) estimate replaces + // the number once, up or down; after that it only falls. + const leaving = estimateSource === 'similar' && source !== 'similar'; + estimateSource = source; + // The correction starts the last minute over. Once the shown time has run + // out it stays hidden for this attempt, as without an initial estimate. + if (leaving && raw !== null && !remainingRetired) { lastMinuteAt = null; lastMinute = 0; } + if (raw === null || remainingRetired) return; + // A retry's fresh estimate still waits for the 5 s spacing of the shown + // minutes; only a new request starts the line over (in update()). + if (remainingSeconds === null || leaving) { remainingSeconds = raw; remainingAt = now(); } + return; + } + if (completedSteps < 2 || remainingSeconds !== null || remainingRetired) return; // Compare distinct received messages, never repeated timer ticks. // Zero is stable only against another zero. if (raw !== null && previousRemaining !== null @@ -620,7 +661,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f } function resetRemaining() { completedSteps = 0; previousRemaining = null; remainingSeconds = null; remainingAt = now(); - remainingRetired = false; lastMinuteAt = null; lastMinute = 0; + remainingRetired = false; lastMinuteAt = null; lastMinute = 0; aboveSeconds = 0; } function tick() { if (element.hidden) { drawStall(0, true); return; } @@ -631,12 +672,16 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f // The page counts from its first message until the engine's own elapsed time // arrives; that switch never moves the shown time back within a request. const seconds = elapsedShown = Math.max(elapsedShown, - valid(overall.elapsed_seconds) ? overall.elapsed_seconds + overallAge : (now()-started)/1000); + valid(overall.elapsed_seconds) ? overall.elapsed_seconds + overallAge + elapsedLead : (now()-started)/1000); elapsed.textContent = `${t('Elapsed', '已用时')} ${duration(Math.max(0, seconds))}`; if (ended) remainingSeconds = null; if (remainingSeconds !== null) { - remainingSeconds = remainingShown(remainingSeconds, rawRemaining(), Math.max(0, (now() - remainingAt) / 1000)); + const sent = rawRemaining(), passed = Math.max(0, (now() - remainingAt) / 1000); + aboveSeconds = remainingAbove(aboveSeconds, remainingSeconds, sent, passed); + remainingSeconds = remainingShown(remainingSeconds, sent, passed, estimateSource === 'similar', aboveSeconds); if (remainingSeconds < 60 && lastMinuteAt === null) { lastMinuteAt = now(); lastMinute = remainingSeconds; } + // An initial estimate may rise again: its last minute starts over. + else if (remainingSeconds >= 60 && estimateSource === 'similar') lastMinuteAt = null; // A run slower than estimated would leave "less than a minute" up for minutes: // once that minute has passed, or the time reaches zero, stop showing it. if (!ended && (remainingSeconds <= 0 @@ -646,9 +691,14 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f } remainingAt = now(); eta.hidden = remainingSeconds === null; - eta.textContent = remainingSeconds === null ? '' : remainingSeconds >= 60 + const etaText = remainingSeconds === null ? '' : remainingSeconds >= 60 ? t(`About ${Math.ceil(remainingSeconds / 60)}\u00a0min left`, `预计还需约 ${Math.ceil(remainingSeconds / 60)}\u00a0分钟`) : t('Less than a minute left', '预计还需不到 1\u00a0分钟'); + const initial = remainingSeconds !== null && estimateSource === 'similar'; + // While it follows the bridge, the shown minutes change at most every 5 s. + if (eta.textContent !== etaText && (!initial || !eta.textContent || now() - etaChangedAt >= 5000)) { + eta.textContent = etaText; etaChangedAt = now(); + } elapsed.hidden = false; times.hidden = elapsed.hidden && eta.hidden; drawStall(age, ended); drawActivity(age, ended); @@ -662,8 +712,21 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f // not a stage change or something the user needs to act on. if (message.warning === 'ram_budget_warning' && !message.phase) return; const reset = newRequest(message); - if (element.hidden || reset) { - state = {}; overall = {}; started = now(); overallAt = now(); shownFraction = null; elapsedShown = 0; + const placeholderReset = element.hidden || reset; + const terminal = ['complete', 'failed', 'cancelled'].includes(message.phase); + if (placeholderReset) { + // The engine's first message after the page's own placeholder is the + // same click: keep counting from it. The placeholder lasts from this + // node's own start until any engine message, its end, or hide(). + const sameClick = placeholder && message.stream_id !== undefined; + // From then on the engine's elapsed time is offset by the wait before it began. + const firstAt = valid(message.received_at) ? message.received_at : now(); + elapsedLead = sameClick ? Math.max(0, (firstAt - started) / 1000 + - (valid(message.overall?.elapsed_seconds) ? message.overall.elapsed_seconds : 0)) : 0; + if (!sameClick) { started = now(); elapsedShown = 0; } + placeholder = message.stream_id === undefined && !terminal; + state = {}; overall = {}; overallAt = now(); shownFraction = null; estimateSource = null; + eta.textContent = ''; etaChangedAt = -Infinity; lastProgressAt = valid(message.received_at) ? message.received_at : now(); previousProgress = null; segmentKey = ''; segmentStartedAt = lastProgressAt; pauseAnnounced = false; lastExplanation = ''; announcedMessages.clear(); @@ -674,21 +737,32 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f samplingPlan = {}; secondPass = false; hasHeartbeat = false; announcementRecord = null; } + if (!placeholderReset && (message.stream_id !== undefined || terminal)) placeholder = false; const record = announcementsFor(message, reset); if (record) { announcementRecord = record; for (const text of record.messages) announcedMessages.add(text); } const receivedAt = valid(message.received_at) ? message.received_at : now(); - const heartbeat = message.overall?.heartbeat === true; + // The estimate in `overall` can be newer than the engine message it rides + // on: a heartbeat refreshes only that (overall_received_at, from publish). + const overallReceivedAt = valid(message.overall_received_at) ? message.overall_received_at : receivedAt; + const beat = message.overall?.heartbeat === true; + // A heartbeat repeats the last engine message. A page that has not seen + // that message yet (reopened, or opened mid-run) takes it as the message. + const heartbeat = beat && previousProgress !== null; if (message.reset === true) { - hasHeartbeat = false; resetRemaining(); lastActivity = null; + hasHeartbeat = false; lastActivity = null; + // A stage restarting its own count (the sampler's start) leaves the + // bridge's whole-request estimate standing; a retry starts afresh. + if (estimateSource && !message.retry && message.phase !== 'recovery') { completedSteps = 0; previousRemaining = null; } + else resetRemaining(); // A reset that would move a shown share back starts its stage afresh // (hidden, 5 s again). One before anything was counted, such as the // sampler's own start, keeps the stage's clock and the 2 s rule. if (miniFraction > 0) resetMini(); } - if (heartbeat) hasHeartbeat = true; + if (beat) hasHeartbeat = true; if (!heartbeat && message.phase === 'sampling' && valid(message.done)) completedSteps = Math.max(completedSteps, message.done); if (message.low_memory && typeof message.low_memory === 'object') lowMemory = true; @@ -714,8 +788,8 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f samplingPlan = {...message.plan_context}; // A heartbeat carries a newer estimate, never a newer engine event. // Keep sampledAt paired with the sampling counters/elapsed sample. - if (heartbeat && previousProgress !== null) { - overall = {...overall, ...message.overall}; overallAt = receivedAt; + if (heartbeat) { + overall = {...overall, ...message.overall}; overallAt = overallReceivedAt; observeRemaining(message); tick(); return; } @@ -724,7 +798,7 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f || message.prediction || message.sampling_plan || reset; if (!activityEvent && previousProgress !== null) { if (message.overall && typeof message.overall === 'object') { - overall = {...message.overall}; overallAt = receivedAt; observeRemaining(message); + overall = {...message.overall}; overallAt = overallReceivedAt; observeRemaining(message); } tick(); return; } @@ -759,14 +833,14 @@ export function createGenerationProgress(t, now = () => Date.now(), {compact = f } element.dataset.phase = message.phase || message.timing_phase || ''; if (message.overall && typeof message.overall === 'object') { - overall = {...message.overall}; overallAt = sampledAt; + overall = {...message.overall}; overallAt = overallReceivedAt; observeRemaining(message); } element.hidden = false; tick(); if (timer === null) timer = setInterval(tick, 250); } function hide() { - element.hidden = true; status.hidden = true; recovery = ''; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false; + element.hidden = true; status.hidden = true; recovery = ''; placeholder = false; elapsedLead = 0; trims.clear(); lowMemory = false; compatibility = null; references = false; hasHeartbeat = false; compatibilityNote.hidden = true; compatibilityNote.textContent = ''; memoryTight = null; firstPassOver = false; elapsedShown = 0; diff --git a/web/health.js b/web/health.js index 39870fe3..111fe4b4 100644 --- a/web/health.js +++ b/web/health.js @@ -13,7 +13,7 @@ const t = (en, zh) => cn ? zh : en; // launcher.js removes this flag during setup; read it before that. const launchVisit = query.get('freevideo') === 'launch'; // Every script this folder ships; a test keeps the list in step with web/*.js. -export const MODULES = ['agent_activity.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'browser_extension.js', 'chain_support.js', 'capability_view.js', 'compatibility.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js', +export const MODULES = ['agent_activity.js', 'agent_connect.js', 'agent_session.js', 'autopilot.js', 'autopilot_chat.js', 'autopilot_conversations.js', 'browser_extension.js', 'chain_support.js', 'capability_view.js', 'compatibility.js', 'error_panel.js', 'first_run_guide.js', 'fonts.js', 'freevideo.js', 'generation_progress.js', 'editor.js', 'editor_actions.js', 'editor_adjust.js', 'editor_adjust_picture.js', 'editor_picture_fields.js', 'editor_picture_session.js', 'editor_preview_picture.js', 'editor_entry.js', 'editor_export_dialog.js', 'editor_first_frame.js', 'editor_genbar.js', 'editor_geometry.js', 'editor_library.js', 'editor_menu.js', 'editor_model.js', 'editor_frame.js', 'editor_player.js', 'editor_room.js', 'editor_select.js', 'editor_state.js', 'editor_store.js', 'editor_tracks.js', 'editor_generation.js', 'editor_transport.js', 'editor_transport_model.js', 'editor_audio.js', 'editor_audio_model.js', 'editor_audio_notes.js', 'editor_audio_replace.js', 'editor_audio_rules.js', 'editor_speed.js', 'editor_speed_model.js', 'editor_loader.js', 'editor_text_coverage.js', 'editor_text.js', 'editor_text_model.js', 'editor_text_layout.js', 'editor_notes.js', 'editor_icons.js', 'dialog_keys.js', 'frame_sizes.js', 'health.js', 'image_commands.js', 'image_editor.js', 'image_editor_engine.js', 'launcher.js', 'length_plan.js', 'library.js', 'motion.js', 'output_download.js', 'port_notice.js', 'preview_scene.js', 'progress_connection.js', 'prompt_draft.js', 'prompt_enhance.js', 'prompt_references.js', 'report_issue.js', 'studio_recent.js', @@ -599,12 +599,26 @@ async function check() { // Opening the workspace waits for an animation frame. await sleep(1500); const problems = []; - const ui = window.FreeVideoUI || {}; - if (!ui.ready) { + if (!window.FreeVideoUI?.ready) { problems.push(...await diagnoseModules()); if (problems.length && problems.every(p => p.retry && !extensionProblem(p)) && await reloadOnce(problems)) return; } - if (!ui.ready && !problems.length) problems.push({text: t('FreeVideo’s scripts loaded, but ComfyUI did not start them.', 'FreeVideo 的脚本已下载，但 ComfyUI 没有启动它们。'), advice: 'reload'}); + // Every file loads, so ComfyUI is most likely still working through other + // extensions: it starts them one after another, and a slow one ahead of + // FreeVideo's delays it. Give it the rest of the minute before naming it. + if (!window.FreeVideoUI?.ready && !problems.length) await waitFor(() => window.FreeVideoUI?.ready, 50000); + const notStarted = {text: t('FreeVideo’s scripts loaded, but ComfyUI did not start them.', 'FreeVideo 的脚本已下载，但 ComfyUI 没有启动它们。'), advice: 'reload'}; + if (!window.FreeVideoUI?.ready && !problems.length) { + problems.push(notStarted); + // A start later than that clears the card instead of leaving it up. + const timer = setInterval(() => { + if (!window.FreeVideoUI?.ready) return; + clearInterval(timer); + if (!card || !lastProblems.length || !lastProblems.every(p => p === notStarted || p.advice === 'plugin')) return; + card.remove(); card = null; lastProblems = []; + send('ready', []); + }, 1000); + } if (!problems.length) problems.push(...await diagnoseStyles()); let checkStudio = false; if (launchVisit && !problems.length) { @@ -637,6 +651,8 @@ async function check() { && !reports.some(item => item.type === 'error' && item.where === 'studio')) problems.unshift({text: t('The creative workspace did not open.', '创作面板没有打开。'), advice: 'reload'}); problems.push(...lastProblems); + // Started while this check was waiting for the page to be in front. + if (window.FreeVideoUI?.ready && problems.includes(notStarted)) problems.splice(problems.indexOf(notStarted), 1); if (problems.length) { problems.push(...pluginErrors()); show(distinct(problems)); send('error', distinct(problems)); diff --git a/web/library.js b/web/library.js index adf7d66d..b79f229a 100644 --- a/web/library.js +++ b/web/library.js @@ -2,6 +2,7 @@ import { api } from '../../scripts/api.js'; import { closeDialog } from './motion.js'; import { resultActions, previewBadge } from './result_actions.js'; import { extendSourceExists } from './studio_extend.js'; +import { topDialog } from './dialog_keys.js'; const style = document.createElement('link'); style.rel = 'stylesheet'; style.href = new URL('./library.css', import.meta.url).href; document.head.append(style); @@ -152,9 +153,11 @@ export function openLibrary(t, {upscale = null, selectedWork = null, extend = nu date.textContent = timestamp(row); geometry.textContent = dimensions(row); links.replaceChildren(resultActions(row, t, { extend: async saved => { + const isCurrent = () => !disposed && dialog.open && dialog.dataset.closing !== 'true' && topDialog() === dialog && selected === saved.video; + if (!isCurrent()) return; player?.pause(); - const run = extend || (async record => (await import('./studio.js')).extendVideo(record)); - try { const studio = await run(saved); dialog.close(); studio?.querySelector('.fv-prompt-editor textarea')?.focus({preventScroll: true}); } + const run = extend || (async (record, options) => (await import('./studio.js')).openExtendEditor(record, undefined, options)); + try { const editor = await run(saved, {isCurrent}); if (editor && !disposed) { dialog.close(); editor.element?.focus({preventScroll: true}); } } catch (error) { notice.textContent = error.message; } }, secondPass: row.preview && upscale ? () => secondPass(row) : null, diff --git a/web/progress_connection.js b/web/progress_connection.js index 078c3ee9..04e6ad10 100644 --- a/web/progress_connection.js +++ b/web/progress_connection.js @@ -10,7 +10,9 @@ export function createProgressConnection(api, consume, now = () => Date.now()) { // The graph may still be loading. Do not acknowledge an undelivered // snapshot; the next poll restores it once the node exists. const age = Number.isFinite(message.age_seconds) ? Math.max(0, message.age_seconds) : 0; - if (consume({...message, received_at: now() - age * 1000}) !== false) + // A heartbeat refreshes only the estimate: it is younger than the engine message. + const overallAge = Number.isFinite(message.overall_age_seconds) ? Math.max(0, message.overall_age_seconds) : age; + if (consume({...message, received_at: now() - age * 1000, overall_received_at: now() - overallAge * 1000}) !== false) seen.set(key, message); } async function refresh() { diff --git a/web/result_actions.js b/web/result_actions.js index 12433ab0..2bbe5f2e 100644 --- a/web/result_actions.js +++ b/web/result_actions.js @@ -1,7 +1,7 @@ import { api } from '../../scripts/api.js'; import { outputDownloadURL } from './output_download.js'; import { shareButton } from './share.js'; -import { EXTEND_ENTRY_OPEN, canExtend, extendText, extendSourceExists } from './studio_extend.js'; +import { canExtend, extendText, extendSourceExists } from './studio_extend.js'; const css = document.createElement('link'); css.rel = 'stylesheet'; css.href = new URL('./result_actions.css', import.meta.url).href; document.head.append(css); @@ -115,7 +115,7 @@ export function resultActions(record, t, {diagnostic = null, secondPass = null, download.href = outputDownloadURL(api, record.video); download.download = ''; bar.append(download); } - if (EXTEND_ENTRY_OPEN && extend && canExtend(record)) { + if (extend && canExtend(record)) { const button = action('button', extendText(t, 'action'), 'extend', 'fv-action-secondary fv-action-extend'); button.hidden = true; extendSourceExists(api, record).then(exists => { button.hidden = !exists; }); diff --git a/web/studio.css b/web/studio.css index 86cf8b23..b20eb021 100644 --- a/web/studio.css +++ b/web/studio.css @@ -119,6 +119,8 @@ .fv-studio .fv-progress{width:100%;height:4px;accent-color:var(--fv-accent);border:0;margin:0 0 10px}.fv-studio .fv-budget,.fv-studio .fv-prewarm{text-align:center;font-size:var(--fv-micro);color:var(--fv-muted)} .fv-studio .fv-panel{height:auto;padding:0;border:none;border-radius:0;background:transparent;color:var(--fv-ink);font-size:var(--fv-micro);overflow:visible} .fv-studio .fv-panel .fv-toolbar{justify-content:flex-start}.fv-studio .fv-panel .fv-note{font-size:var(--fv-micro);margin:8px 0;text-align:left} +.fv-studio .fv-preview-column>.fv-note{text-align:center;text-wrap:balance} +.fv-studio .fv-preview-column>.fv-note:not([hidden])+.fv-note{margin-top:calc(0px - 19px)} .fv-studio .fv-panel .fv-empty{display:grid;place-items:center;padding:22px 15px;min-height:92px;margin-top:12px;font-size:var(--fv-micro);white-space:pre-line;border-color:var(--fv-border);background:var(--fv-card)} .fv-studio .fv-panel .fv-preview{max-height:170px;margin:10px auto;object-fit:contain}.fv-studio .fv-panel .fv-card{background:var(--fv-card);border-color:var(--fv-border)} .fv-studio .fv-panel select{min-height:var(--fv-h-sm);padding:5px 8px;max-width:145px;width:auto}.fv-studio .fv-panel button{font-size:var(--fv-micro);padding:5px 10px;min-height:var(--fv-h-sm)} diff --git a/web/studio.js b/web/studio.js index a2731268..eced36b2 100644 --- a/web/studio.js +++ b/web/studio.js @@ -1,9 +1,10 @@ import { createErrorPanel, errorText, createErrorReport } from './error_panel.js'; +import {topDialog} from './dialog_keys.js'; import { app } from '../../scripts/app.js'; import { api } from '../../scripts/api.js'; import { openSetup } from './setup.js'; import { wordmark } from './wordmark.js'; -import { createGenerationProgress, referenceTrimText, thermalText, encoderFileText, encoderFilePath } from './generation_progress.js'; +import { createGenerationProgress, referenceTrimText, thermalText, sharedGpuText, encoderFileText, encoderFilePath } from './generation_progress.js'; import { viewSwitch, viewChanged, agentTabReady, openAutopilot } from './view_navigation.js'; import { agentInterfaceEnabled } from './agent_connect.js'; import { createUpdateNotice, createVersionInfo, productVersion } from './updates.js'; @@ -190,6 +191,14 @@ export async function extendVideo(record, node = current?.node || app.graph?._no return openStudio(node, {extendFrom: await readExtendRecord(api, record)}); } +export async function openExtendEditor(record, node = current?.node || app.graph?._nodes?.find(row => row.type === 'FreeVideoGenerate'), + {studio = current?.node === node ? current?.dialog : undefined, isCurrent} = {}) { + if (!node || isCurrent?.() === false) return null; + const handle = await openEditorView(node, {studio, work: record, extend: true, isCurrent}); + await handle?.ready; + return handle; +} + export function openStudio(node, {extendFrom = null} = {}) { if (current?.node === node && current.dialog.open) { if (extendFrom) current.enterExtend(extendFrom); else current.dialog.focus(); @@ -541,7 +550,8 @@ export function openStudio(node, {extendFrom = null} = {}) { const previewCaption = el('span'), previewTools = el('div', null, 'fv-preview-tools'); previewTools.append(previewCaption); previewTools.append(button(t('Creations', '作品'), () => { - stage.querySelector('video')?.pause(); openLibrary(t, {upscale: value => node.freevideoUpscaleResult(value), extend: saved => extendVideo(saved, node)}); + stage.querySelector('video')?.pause(); openLibrary(t, {upscale: value => node.freevideoUpscaleResult(value), + extend: (saved, options) => openExtendEditor(saved, node, {studio: dialog, ...options})}); })); previewHead.append(previewTools); output.append(previewHead); const previewSpace = el('div', null, 'fv-preview-space'); output.append(previewSpace); const stage = el('div', null, 'fv-stage'); @@ -647,8 +657,9 @@ export function openStudio(node, {extendFrom = null} = {}) { }; const memoryNote = el('div', t('This video used the low-memory mode for this GPU, so it took longer than usual. A lower resolution or a shorter video runs faster.', '这段视频使用了省显存方式，所以比平时慢。降低分辨率或缩短时长会更快。'), 'fv-note'); memoryNote.hidden = true; const thermalNote = el('div', '', 'fv-note'); thermalNote.hidden = true; + const sharedGpuNote = el('div', '', 'fv-note'); sharedGpuNote.hidden = true; const encoderNote = el('div', '', 'fv-note'); encoderNote.hidden = true; - output.append(status, reuseRow, trimNote, memoryNote, thermalNote, encoderNote, links, stats, budget, progress.stall, progress.report, prewarm); + output.append(status, reuseRow, trimNote, memoryNote, thermalNote, sharedGpuNote, encoderNote, links, stats, budget, progress.stall, progress.report, prewarm); const failure = createErrorPanel(t); output.append(failure.element); // With the picture, the result's actions and numbers form one block, centred below the header. output.append(el('div', null, 'fv-preview-end')); @@ -680,6 +691,7 @@ export function openStudio(node, {extendFrom = null} = {}) { trimNote.textContent = trims.map(row => referenceTrimText(row, t)).join('\n'); trimNote.hidden = !trims.length; memoryNote.hidden = !r.low_memory || !!r.result_cache_hit; thermalNote.textContent = r.thermal ? thermalText(r.thermal, t) : ''; thermalNote.hidden = !r.thermal || !!r.result_cache_hit; + sharedGpuNote.textContent = r.shared_gpu ? sharedGpuText(r.shared_gpu, t) : ''; sharedGpuNote.hidden = !sharedGpuNote.textContent || !!r.result_cache_hit; encoderNote.textContent = r.encoder_file ? encoderFileText(r.encoder_file, t) : ''; encoderNote.hidden = !r.encoder_file; const encoderPath = encoderFilePath(r.encoder_file); if (encoderPath) encoderNote.append(el('pre', encoderPath, 'fv-encoder-path')); @@ -696,7 +708,8 @@ export function openStudio(node, {extendFrom = null} = {}) { const reported = progress.hasReport(); const tipShowing = !!links.querySelector('.fv-report-tip:not(.fv-report-tip-leaving)'); links.replaceChildren(resultActions(r, t, {diagnostic: reported ? downloadReport : null, - extend: async saved => { try { await extendVideo(saved, node); } catch (error) { resultActionFailed(error); } }, + extend: saved => openExtendEditor(saved, node, {studio: dialog, + isCurrent: () => dialog.open && dialog.dataset.closing !== 'true' && topDialog() === dialog && result === r}), secondPass: r.preview ? secondPass(r) : null})); if (runButtonReady) updateExtendButton(); // After a video finished just now, point at the report menu until the diff --git a/web/studio_extend.js b/web/studio_extend.js index c1e555dc..840a5e41 100644 --- a/web/studio_extend.js +++ b/web/studio_extend.js @@ -8,8 +8,8 @@ export const EXTEND_TEXT = Object.freeze({ }); export const extendText = (t, key) => t(...EXTEND_TEXT[key]); -// The saved video of a continuation holds only what it adds, so the result card and the library keep this entry -// closed until the joined whole video is made; the editor's own extension is unaffected. +// Only the legacy in-studio continuation entry waits for a joined whole video. +// Result cards and Creations continue on the editor timeline independently. export const EXTEND_ENTRY_OPEN = false; export function canExtend(record) { return Boolean(record?.video && /\.mp4$/i.test(record.video) && !record.preview