From 7479102bf101d407947027949cee8b3b1449abc8 Mon Sep 17 00:00:00 2001 From: sensitron Date: Thu, 17 Sep 2026 18:14:48 +0200 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20feat(benchmark):=20compare=20mo?= =?UTF-8?q?dels=20on=20one=20range?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ✨ Features: - "Copy render settings" in the Render button's context menu (Ctrl/⌘+Shift+C) copies the current request as versioned JSON. It is registered on the window, so it also works while Render is disabled, e.g. while a model is still downloading. - `core/request_json.py` round-trips sampling (exact fractions), crop, segmentation, framing, transform and `max_bytes`. Location and rebuild flags stay with the caller. - `scripts/benchmark_models.py` renders that request once per model through `ProductionPreviewRuntime`, so weights still download through the Qt transport. Defaults to `V1_MODEL_IDS`. `bria-rmbg` and `u2net_cloth_seg` run only when named, and bria prints its licence note. - Per model it records prepare and render time, the provider actually used and any fallback notice. It flags cut-cache reuse and mismatched frame counts or durations. Results are rewritten after every model, so Ctrl+C keeps the finished ones. - `scripts/benchmark_report.py` writes a self-contained `index.html` grid with a background toggle and a synchronised restart. - 🔄 Export and presenter share `_try_export_request`, which does not validate the output location, so a bad output setting cannot disable the export. - 🌐 German and English catalogues carry the three new strings. - 🧪 Tests: JSON round trip including transform, export reachability while Render is disabled, requests identical across models except model and provider, partial results on interruption, report escaping. - 📝 README: "Comparing models" under "Run from source". Refs #170 --- README.md | 21 + resources/matteloop_de.qm | Bin 69659 -> 70254 bytes resources/matteloop_de.ts | 178 +++--- resources/matteloop_en.qm | Bin 61703 -> 62186 bytes resources/matteloop_en.ts | 178 +++--- scripts/benchmark_models.py | 597 +++++++++++++++++++++ scripts/benchmark_report.py | 270 ++++++++++ src/matteloop/core/errors.py | 1 + src/matteloop/core/request_json.py | 314 +++++++++++ src/matteloop/ui/action_shelf.py | 74 ++- src/matteloop/ui/controller.py | 30 +- src/matteloop/ui/copy.py | 6 + src/matteloop/ui/main_window.py | 10 + src/matteloop/ui/main_window_render.py | 10 + src/matteloop/ui/ports.py | 6 + src/matteloop/ui/presentation_model.py | 1 + src/matteloop/ui/presenter.py | 21 + src/matteloop/ui/render_settings_export.py | 72 +++ src/matteloop/ui/request_builder.py | 52 ++ tests/core/test_request_json.py | 324 +++++++++++ tests/scripts/__init__.py | 1 + tests/scripts/test_benchmark_models.py | 485 +++++++++++++++++ tests/scripts/test_benchmark_report.py | 148 +++++ tests/ui/test_render_settings_export.py | 279 ++++++++++ 24 files changed, 2912 insertions(+), 166 deletions(-) create mode 100644 scripts/benchmark_models.py create mode 100644 scripts/benchmark_report.py create mode 100644 src/matteloop/core/request_json.py create mode 100644 src/matteloop/ui/render_settings_export.py create mode 100644 tests/core/test_request_json.py create mode 100644 tests/scripts/__init__.py create mode 100644 tests/scripts/test_benchmark_models.py create mode 100644 tests/scripts/test_benchmark_report.py create mode 100644 tests/ui/test_render_settings_export.py diff --git a/README.md b/README.md index 62512d7..665dad4 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,27 @@ launch warnings, and current platform qualification status. Notes for people writing decoder tests live in [docs/testing-fixtures.md](docs/testing-fixtures.md). +### Comparing models + +To see how every model handles the same clip, set up the range, crop and +settings in the app, then choose **Copy render settings** from the Render +button's context menu (Ctrl/⌘+Shift+C). The benchmark renders those exact +settings once per model and writes one WebP each, a `results.json` with +timings, and an `index.html` that shows all of them side by side: + +```sh +uv run python scripts/benchmark_models.py --out benchmark-run +``` + +- It reads the settings from the clipboard. On Linux or a headless machine, + save them to a file and pass `--request `. +- By default it renders the 13 models the app offers, about 6.8 GB of weights + on a first run. `bria-rmbg` (own licence terms) and `u2net_cloth_seg` (needs + a clothing-category input) run only when named with `--models`. +- Renders share the app's cut cache. A model whose cuts already exist for these + settings is reused rather than segmented again, and flagged in the report — + its tile may show hand-edited cuts, and its time is not a real measurement. + ## License MatteLoop's original source code, documentation, and visual assets are licensed diff --git a/resources/matteloop_de.qm b/resources/matteloop_de.qm index d309ea027c25ea2291bb5c3c1d2091d0982278d3..52ff8ea10d6e3535edb9e4681a193e545556b5a9 100644 GIT binary patch delta 4729 zcmZWs4OoqL|Nor(ocliS_wf)Tp_DO&A(}>P6*7xS9(ie^)FG$V3t=2q-YuyZvPd#Y znU|Kg#fp~Kp`y%NWD|RuP0Xg+|C95-p6hy^=ju9r&iUQHm(Ta}effRwjGd~6-KvzX z)|~(*1D#&kux2wc2(WjsVa-WE=mA`OL5$_+8Q^{uxV#t`)QVpLgIloz@Zjh2-@xcV zAmaou)(6l{1sT08EK+^La?dNf`KHCeZ$(4WsVcaBVe)x<028 zk?`p=5P0E)(WL=^^M@FHDT%1Z7}w7L)fX5ucoS9VXv4+9Hhg{$9|rXYx{k(#wI#rv zI!v5$(hA7uF=;&)?0<#HMaO`4LlCr`<0qEeu!R)##RA4${dbrSl1vZQd#KPA7j=N)B{(dUE4Vj;`0HP;$z)F7yj6n8= zYM`}u!jmi zvm<@`onox$OrNh+Om2z+n$i@(zM+67RuQ&5br(y=1xA1<>IZC=%2z&Y` zpzbDEexf1=>V)_lx^iZXu%JN%mevbapF{>-qmY!)6*!nDB$Y5mLN5zR*N81KR$)T} zEt*p-WURjhd@x$r75N-+J1!JFlYzfh318>50EK6SqPDcm+(jr}5(jKG31?f==WO@a zsKEQeMFue1RtlF=NhCJHhGz~7HwW|sz7G@bZt4bfP7xlJpXR{@Lj9n=G|1XPcrxHD z(B-KOEiE=&-9h-((V1ECo>KYl17O7%rD_tTeq*ZAeo-h;H$>@B@&+ycUD+jtL?fC1~3{f0BBZ}eCCTYZ?0(aPC-y?}?kmEji{v{t{VN=qXp zEg7gxxt_#q_*1zpn`vcES7x4D2Yfh2S#*F4K2|6%r3jR6oASEvLm>N_^47b-K+jC& zy&!L9O@s|Ci#VZtP{E?rqH^%$_%~51@8D48|Aksr zu#pprD^%vGyMaV|l|`n+m7%JYdg8Rhs=BSfHw7_ zWdR9nEfnLIal+ZDV!}C&SEkuFn!VK2}`6-<@gKRovh>9Qb*i zxIOd^iPehP(L89wX)$*p34P-!9?1#>J}VWA?lK1U6^j=dXxRAPqV->CTwvWU;Zl1p z>@KMU79uCDq^qm~Vj$^HHnC(@NOF5B;?`EOTSnhIZ;(2DK+BYGNcNt0fs-91#|@k4 z+k;ZioW87hzf13ql!4^Gq@lhnWC?!K&~QJ_|I>!Y&PyY=aKEQzR%ui(TC#YXG*;kM z8tyBN{|A}*&X*#-=7tsbq&cyZc&SX2<}OkJQ%j|U782H-loox>^lNLDmP(Y`W1h5p zV-j6BK}snnF^>W%wG~ZQrOjF6c^m#DZNm^idrQh#!Sm#P(yjy&%(AMb?2Fv6S37C< z$P(6p**080(uPew(h1!WD6mvI9YBe1?vcK)i$h#fr%Z|o zJViZh3oZOdklL%1r7|#6J-ppkDsfHin?(YRE^6OrjFDNl)FU%V;Km*G2Sv%imp#>E z)4Q>v&Q?#zZ=fsv)zK4M4Jc8^9k2k$oYae!X#wvD^^Tq1%>N}9)cZbP0_Ypn`9p$9 zq*z_hjT@$ap)M(4MN8_VF75e%L^|7W`7L$XHa-vPtiBXD4yeyl-`l~BqswhrTA}`R z$qIHjqiq-~t6!)YBU1)xjF)6!_dA+4dwI8v_(NmY-Uxe8wvn9sFk6S?q$gSPxGEj)3~hTnJ9KFH@m?sK#cI~N14 zX6>W?jZi2g?c?GeK=l>vuY(&{&|J068~ka=8J$o;$&b9JQ=a^dw_=d4fAu%)yb7#3 z-#>zZw*qt#@zYpnLUm>%gROA3&OD?m(8EcWFzXP|txC6e(F#_|dRE@caQ zxg{U#Hg`{C=agi_s}Z_weM$kp23U+*WF$f zOBY)mbw3^FK=hk>dr!LH{de@8e|BfPeMsN?^gtlhqVMBCX(K}PeTPtq41fL59);}n zy6e4~3xJUk`j0j-cwZFjgU(d|k#_oN8|!#kRp`Ul^Wgi{dNW&Y^g5t7@AC!laU4%KZPQgWfuWOoCGN%ge5_@;TaYVL$!K1iHrgwm!3z77q5)?}H5N+OE%C zb&t(uCw)<48u0ZC{mFJDoEfAqSrI}wVf$WHK4D!DMng8dU4Nlogpv=MWw#WycEjGB@abc-!@K8GgfA27ObeaXMNi_`n zP6b>~G5EjDZu!tkLqL~I2Gv8u^htca#NH6n<0%`FUktNa(Qbrc&RP;YmTFkEl5aoW zhYX2rzhs4cYDiLe0q+zTR_||M{%gDq8)vZeHtaNHS5dk@h8YeGNn+Fa)KEBo0)x}d z@XcQ&6gJC-2PYfKYbD^7o8h7y%3jXS@I&V?UN$ogw@YXUEH*6cYWVpSUpP+AF#L5b z9hh|9sC(rPoc1#s9g=}x3XJw8R&Lxk#^_;xheS>pr-slc{j!YVE(+fJvN58y+4M;_ z#%yM^8W$L29Ve3c6l46p*=)(;jZ1Db+W!=d%i40oSNX=3o-;|jr*ZWNHCwn48(L=? z*T?eUDL)$1BS>U_sBwp$hPqra?y%N#ATQULm72&G#&yQ*UF=k*jW+JyM)x``!ukI^PTbA;?y_C^_^+R+;OtC;|*_q*(cjS@nDDZSbi&P1MmcL??+^)+#|c* ze92OqB)j)=VWq5=J>M&3>AWxdcPG&wKazvCH!!#!$W!<7fEk%`@Y!~3<1ffFitJdR z{N*`?BCqu(8=kO}=iLpUiy-ft`2aZc*ed7m?8eSzfLu7Bn04TpeDvES_Hyy^cYZmX zci4t=1-T@Pr8u{b{KF@{T%IIf85RocIVxW*aON!-D%ZS{f!;^tJA0|fJ9@dc%$?6C z$@O!dv1#osKk4WXe3E05E?R)^J~yeS9R|i&x0|$rk~My=seM;knDNkL-$WuOPnddU z&1ZV8G!2N1XSDxe8kJE<>6V+uzhF&2T5Sq%A-`oWO%Y?`*zb=q#nfJA!MbZ&c+4EQTNET7o^9H;nQ1v?;~7(Cb1J*hSEl?3`Yk&TFa2l99u=9?JUg#JTW(f_8af}U%i!)oo)9$wEkd{{J zez>m9!On`KX{Rm^{3wiTEnGbhjtEB+A`!=t@Ym-!;{E^M*Nj<+fE(w;zn%+xPHtx5>$-(EmM_@7k?&LSM__h$&dWZ?Uhbw-OD(Oqv(++PMED7x!;s v&Xmm@Mi&1eT==`Q6Vv*%zSj6@OnakP1so9gy^W4xWng30O|)`@X;PopZkZd_ys#vZclvA`C-}iisvmSrUyUkullVCSOLfXGtboBpF3z zgzQX_ELjWL6S8Lf#n@_ocmLzMTzBVu=Y8J$+3xpwE4GUpw~DJ8*cJmA3e;Pn;*DTp z5T7$uyfGgTT!C{(h|~CgA>eTZxX=UWP=gbJjy0GFbh<`71^hS&*w6*=^9BsBfahz0 zl@C`cAx&KZ?3w^++Gb!&5Tt`uKyd`>^@;;lze2s};lTPK zX#8a&Z203fT-?e4YaO)mehe7vqWw=vK%Ed3BSxvXWC*&pe8)(*!`rPr@b(FQ%nJY- zU%-!rF~p_l)ADyfY>qx16M$ZSs5s}Yif>c#Q;<90bOr;L<^p%p5IF3p4N!(-@NzCl zKZ&6^hk)AW5wwZ#5A{;cVG(ioX_NPK~{sk64zm8`xM6NmHsx^a8fPwhkC@0x2tR0#}~n;4FT4X%db& z^Mj?a$S({B($3&|t9d|!Be>z~33N@w?H73@+6>Qb`2$~<;C&Ub)Lny;n7ehJn&~e@U`&*Xlb>s1Mm+-hSu}~BHN`nuvp&!VSQw_+J~{;W`Isi- z;SRtOtjTWJ0k|(|j(oR>H8x6fc@G`LT1~ObcA(usjjcpy2mJm<^T@pz7+Xsak~^|m z+Y5$W5x`6bq2&}Byz3`)YFy)4uFx~}EO4-gF!rC1B>tBW@r04cNE2qHGKHgt3bQIC zAZol|^Iia~nI^={ZUAIW7GiQ)8)JSFVy+UubP`rp(4p{dHeuuPJAm&QVcW!aK+DcT z=352$;3w=!tp>7}2syRs*|<%@@u=y*A92FznzqR;SFu_rT(EMZ+F?TB8WNdOu43+N z;a0mg!08sk{e*^qqqFd|;3PMmCOqrVn#Ffpc+u`Oi_tbo{o~gt6=M^G*NvL6D6eU? z-9DS|mR zY1?#XF&Dkp`rEvLq}ST8ou0tM*V=LCS**QpX(K8bQQL_R+Ev$MxZyJGh7`8bcsp&< z*=0cAv)Y_2E*KH5EnFoqboI2?eI5hb259em9|C+^U;7})i=KZ{apFg9X;lSq?V$E) zOcHS9v-X3Q8{B;$+Sk2C(TWnCy72uz578@xj-1^ghFFfX|Boh$;lbO1`Rm09g(1G$ zT#PjlgWHKqf8qPWFfq=Xo=*KBCi)%)dKRfTYl@27>{YxqS=^PHN+B95?#izVOiWer zl()Eh*9dmi4RQA^7j{z@F>8N3u=u8!yHH^@-nb;5s#*BkAL8lXt67|xVu7%a&qdswx4~Z51y+6JHiH=T{uWF9Cc%&RxP|EAUfy zNk7aV7=Bc8TEKRU`9^a2Ga6`rOln?Xi(`Aul3H|UXk4yHZWk{yL>Hwt-93TLmn3(& zDJA(Isl)%>U=56ry2iNz1>K|`dM$8fh}1LG8%TL6_1?wzwziU=Ku6v^k@`(80^;9D zLua%7YGq1eo@Y`q?WHgeuFLN)h0W*$+&d#pwB?)vzKWD0W|2UmhcqLaALKQXW}oHz zs~4om;`3x$DJ@$X!v4;cmM2ThaV=^2ZVzC0nY6M|ci?eP*KxjpcA1JfE9G8^v{$iA?%j-z%&ljW{e(X$Jg4OTwaCotojiUI4PHr= zCrlg6g%$EIb46fSPkDAV85{lOxqH}twLZ)9WQMxKOL_k47$D(_ysCi2I=abgYS21H zUYFdTr{FGm13L2@1RP3-!F4#n3SJ%onC*`p_oK&pJl3&EmV`z)z%3fETx|be?&X%E0=% z?sn^WT7~L-l1bprGo8;{*2w5!U9Th(C{EG!&52{GcIf=pHl%($)(y<4U@8lBlLKoU zSfrbt6#*Q0qMI9S0DdUfZQ1I@?&_mUTM@ZZ$VNQ3cpb-9_-cqZ)D zz5YUCJ@YfY`VC!*?0#(RcNStBAS* z`uu|FtolRxGfhvkeX8|mx{_G=XZ_{ZjD&-mzF4M5ekb%V-5H4^ZS+-9p^RXpzFOnN z!ze(-RYrp@iv&IU8}#-SoEMrI^yzWHYhQ!Kmd+o1&Kka+MCPtP8=3`f;5rvn7cw`Ook}{U(6@w}ydp*vI~^hCv;fg4B3Ja1K3L_}MVAX=@gjonc10Gw|}A zVQ$$P3Xz{-acL#cKgF=5tbocIZdfr~OTwOpgng;NZGVGpy&ajf4=`-0s81iG?06h#{?zc)o!?cTH#|S?3S3VxyzW@R z_B&>%TItVI&dVrVW61Z)M(xozK--%}_nQZRfo?{he?x$#S8c}eGlp~8Ei;B&SZvuX zjp3aeFxOj+v&Zb^$Q5XeoV$>8E$7A$Hk-oZiA`IZ!qdDc)eTH@9>-C5UYjBV=K+H@nrtIU#5S~+ zX@357D&H0rXJwjVXEQg>!%azf^w6Wglm-Pzb~L3gdO)GtWy-0H2QrVCj@ps%rkkeR zWrKkK>`nRJt$}~Xnf~dQ352XPy|27VN4A;`MRkB%X=Y0|es_9;S$Rdwe`;==qGkW* zm73d1d=dT0>|xJ^C$E`1>6!plq2^BY#_*i3W$yO3$Vq09*}pBP&>HF{{iggnaBJ;UNuixN`g5<%yVN$ab9pYFQ~nl3OUIfqw(Y|$<4fYcLlqv z*t~jVF8lxGSaV7-L-$TF@9iAJLt~P8|IC48{KkCX6A6uZsA5)qb3v)h+3=0|f)YyI zC^27h91AQjHQ&wEg8x+9TV^gl!HdRWSM#T;PF7ks8Y-FY1}wym}TvF5=n1v*dlN4N= z#fw9(qK)A9gF6$qQ#rF0?G8rbx~PaFygA!7RK!RUdwfn2W9LxH0~M1a9SFaud~?H# z#oA19c+rWYU6x{NK6WMW^t96ADVb@%Q(E4lHlO!ZJes+%NH!~7e#is1k5c@dNwlP{ z611s;m&nmd@J?>ni=k1{gHo}*TwGGV{Ou8CCfV2P48x{Tv`mXfixA@Jy5 zWq-TlK(ndJ!DBH@n9ipB?VHNf`KTB^U&&3T^zPiPT$lvk5O4 zWlG6sg~HQbxwn&%=(toV&G#U|V&&O{w=A+W<;6Gt9AI3m@`VWC#6YWVcsB1Bj#h)9 zrCwB7>olNa8$zuPUr6MzE!x^Nc_!PbueII88LaN3*4`WUGi0##e@_iRu-Q7Un*40D ztmFNrQ}gdwrW`iG0Hi5e0QbGB|+ zSJQ@G)}*R696sk-GsZJ@aTV5_t0d%cf21|{#?QdBFzdM#8lEj!AD$<+yksqNPT|hC otdG9G$GhIPa5njPE`dXQX|_Z5fLiAqvs<_Jj>~R!tmmNr0hvra?f?J) diff --git a/resources/matteloop_de.ts b/resources/matteloop_de.ts index ba2f33e..0b76166 100644 --- a/resources/matteloop_de.ts +++ b/resources/matteloop_de.ts @@ -4,20 +4,25 @@ ActionShelf - - + + Preview Frame Vorschau anzeigen - - + + Render Video Video rendern - - + + Copy render settings + Render-Einstellungen kopieren + + + + Preferences Einstellungen @@ -38,142 +43,142 @@ Inspector - + Segmentation Segmentierung - + Time & Sampling Zeit & Abtastung - + Crop & Cleanup Zuschnitt und Bereinigung - + Transform Transformation - + Output Ausgabe - + Workspace Arbeitsbereich - - - + + + X X - - - + + + Y Y - - - + + + Width Breite - - - + + + Height Höhe - + Ready Bereit - + Downloading Wird heruntergeladen - - + + Not cached Nicht zwischengespeichert - + Model Modell - + Edge treatment Kantenbehandlung - + Output FPS Ausgabe-FPS - - + + Start Start - - + + End Ende - - + + Duration Dauer - + Alpha threshold Alpha-Schwellenwert - + Padding Rand - + Horizontal stretch Horizontale Streckung - + Directory Ordner - + Filename Dateiname - + Maximum size Maximale Größe @@ -448,7 +453,12 @@ Ordner öffnen - + + Open a video with a valid range to copy its render settings. + Öffnen Sie ein Video mit einem gültigen Bereich, um dessen Render-Einstellungen zu kopieren. + + + Segmentation is unavailable because ONNX Runtime could not enumerate any providers. Repair the installation with: %1 Segmentierung ist nicht verfügbar, weil ONNX Runtime keine Provider ermitteln konnte. Reparieren Sie die Installation mit: %1 @@ -1263,12 +1273,12 @@ Freed %s. Geben Sie Speicherplatz frei und rendern Sie dann erneut. - + Rebuilding from edited cuts Aus bearbeiteten Schnitten wird neu aufgebaut - + Rendering video Video wird gerendert @@ -1790,96 +1800,104 @@ Freed %s. RenderController - + Preview recommended Vorschau empfohlen - + Preview this frame before rendering? Vorschau dieses Bildes vor dem Rendern anzeigen? - + A preview lets you verify the cutout before processing the whole video. Mit einer Vorschau können Sie die Freistellung prüfen, bevor das gesamte Video verarbeitet wird. - + Preview first Zuerst Vorschau anzeigen - + Render anyway Trotzdem rendern - + Cancel Abbrechen - + Matching cut set found Passendes Schnittset gefunden - + A validated cut set matches the current source and settings. Ein validiertes Schnittset passt zur aktuellen Quelle und den Einstellungen. - + Rebuild reuses the cuts and only reruns framing and encoding. Regenerate removes backgrounds again for every selected frame. Der Neuaufbau verwendet die Schnitte wieder und führt nur Framing und Kodierung erneut aus. Die erneute Generierung entfernt die Hintergründe für jedes ausgewählte Bild erneut. - + Reuse cuts and rebuild Schnitte wiederverwenden und neu aufbauen - + Regenerate backgrounds Hintergründe neu entfernen - + Output already exists Ausgabe ist bereits vorhanden - + %s already exists. %s ist bereits vorhanden. - + Choose how to handle the existing output. Wählen Sie, wie mit der vorhandenen Ausgabe verfahren werden soll. - + Replace Ersetzen - + Choose another name Anderen Namen auswählen - + Choose output name Ausgabenamen auswählen - + WebP files (*.webp) WebP-Dateien (*.webp) + + RenderSettingsExportController + + + Copied render settings to clipboard + Render-Einstellungen in die Zwischenablage kopiert + + ResultPlayerCanvas @@ -2044,17 +2062,17 @@ Freed %s. SourceController - + Open video Video öffnen - + Video files (*.mp4 *.mov *.webm *.mkv) Videodateien (*.mp4 *.mov *.webm *.mkv) - + Choose output directory Ausgabeordner auswählen @@ -2090,57 +2108,57 @@ Freed %s. SourceErrors - + Open a video stored on this Mac. Öffnen Sie ein auf diesem Mac gespeichertes Video. - + Open a video file that can be opened and read. Öffnen Sie eine Videodatei, die geöffnet und gelesen werden kann. - + Open a file that contains a video track. Öffnen Sie eine Datei mit einer Videospur. - + Open another video file; this one appears damaged. Öffnen Sie eine andere Videodatei; diese scheint beschädigt zu sein. - + Open a video with a positive duration. Öffnen Sie ein Video mit positiver Dauer. - + Convert to 8-bit SDR and try again. Konvertieren Sie das Video in 8-Bit-SDR und versuchen Sie es erneut. - + Resize to between 8×8 and 3840×2160. Ändern Sie die Größe auf einen Wert zwischen 8×8 und 3840×2160. - + Convert the video to 60 fps or less. Konvertieren Sie das Video auf höchstens 60 Bilder/s. - + Open a video under 10 minutes. Öffnen Sie ein Video mit weniger als 10 Minuten Dauer. - + Open an MP4, MOV, WebM, or MKV video. Öffnen Sie ein MP4-, MOV-, WebM- oder MKV-Video. - + This video could not be read. Open another video. Dieses Video konnte nicht gelesen werden. Öffnen Sie ein anderes Video. @@ -2523,8 +2541,8 @@ Freed %s. UpdateBanner - - + + diff --git a/resources/matteloop_en.qm b/resources/matteloop_en.qm index 38ce644c187f16d2b45806f223b0e090e9d193ab..13356ec6051d73ea83ce2b987c247a7681efebb0 100644 GIT binary patch delta 4675 zcmZ8l30RHk-@ea#&ikJIJ*F~9D$3ZI5`DHXDw;92Vnm{pI%z7WlA7U2*_%+9#!`~( zm9b?FA^DSii6Of`ntx_KX2w{$zPmZ|&2@d><*K{)d7tO^+`s+zTKG`;;)!y(vvmW2 z@j#0h z+!xTAffp+QEB7?vu?-LcuTV}pgs-{RD(rwzH5DjTKpAk0f{#PBIGq9(LzPtstnnb0 z0p(ku+BO9c-`X(vq74u9gKFm<;OjY1)$RihmO->E2J$XKOk59qcOT+`55UnAXyu;` zq_0D(xgo$JceH&U3o8~pg?qP0fcz1<`aU7!<>>ikK45RLVR(oQ)BMrD%WuG*pW)lB zCs5l7!%hYQF5NJ!CKGV3$LC$<0ji<+y!R?#&>|bg#@g`aFZeRp1904nF-t0elNS&) z;fNJ5PQbV<9?0v7@#P1Bk0KGgj_bu9HY}~hgsn84gAYRLo&!mLVA8v-z{lk_Jk}0j zEu2VrFQz@dL<@9-rM)|!|HiNfNNti=)QF`eJ89XUkw2r6M6aO$*44o08Wb(71rD9Y zffQ~$un~vbal<#0P*p?!t**e0u5$pVIk*|%LqFfdPxU8(W>t88dj!yYAAbKpJX5Q{ ztqLIauA+I*e4tfx#V3J7fG-XzI=}c9@R+Xfa%=}&NmO|KNO|3dD*~*PkKjKQ0cKY~ z`bsfebfeFwD8|3H0C&?Bp?(p7`ne)9g+81=QIS-y;(mh-Bc>}d!yW@e^A&kMO+ng0 z@k5XwFtbLnDLfPyTcs#@xEV0aQ0#H(1)Q!^9R9C`K*@f^rCn4Mj}_MI?wiQu55+CD zJx#k&@z~=!Fd<723VSm;Jq7KKIKb>JbeTbcr@j`v+g32xItiZ@p9MAz79xN7lf(;z zxTmzpn*Bm@FLt8WG{&q2tXA?ZhQ6s%7TS^t6YGV7U5a-ab7$~c;2f!gV5S2)O((0R7Tq{ zY_$zj&I>PH-58t~mCAmPfaGtLs&O>6!%L-OS_DvCsC24qLDwWH+bk#1fGJA1MiNR} zrtD~cjrHTDva3@Z&@D&VV<3b2=yc@>t1r{hqMWwX2RIv{j5^1lwGPfz#=WLVw_jH- zzmdrc7b@2jF|8(bSLUBx3iwtj%gcBmNL1D=7ihYz${T)9faMnDoqnN!o2T-A@DOUA zW5cjdl@C5N07phEpJnC)TP(^qO3FX=tIFZytH7sQRZe}mzM()hBs7Bgzb9E0YUGBk zdX;(NCLnQwDo&<}4@0#;PaLCGEt$^sa`i+ExJ{2q#$2_t~OsiIQ`(=qw9=)T1|7wGX`?9`CW^jax)9!S%)vlF{r zxBwJ(5qk{u0kXaoJ){n-?FC}5|Gvo>m@f9uZV&AC5eI3Mz>jg_XA!=@igV)79bBK$ zS{yD=kz4n~kqOtR$RKh2ET&(xVdB&mrNGT(ahey;?Me}+S(Ck)ms`cy@>77Rix`(e z0t>r{$#c1(Bt@Kcmg|RCh~Hd4N8j}nmo5nfQe4EWLXqoLV%APC;D-BJFBT>6qB$eP?WrWRAx7L=IF_yz#qxWMfmJc$xdtjW)JL@bCzl5jJSEgLrzUSD zmB2#O#!b>)yADJ+Ncto1fhFxFxj8M;VT|N3m%ew2ms)*6&4fQC$G-Pisw*VdWvf_v zr%N4*yR-9!O8xw0VD@pTzaI;k`J&W6DgaQH*|2D(;`GLumzad zO&awPnfXkUqIXf?p-d?zaU5&4oisg71&r<`&1xiJZIzU^i|N-aSDGWy)E@7o^c9&v z<_l?gHHmrlm2#R;o-C~{97PKW(i-&PIXh|XeBNg`BW;{Tg4tH1RCJyK-AbfQ{*|oR z&ulm&&xUtxK@j#mRxUY2P8U?kOY(tM-QuR6#I~*+4#-C*2DYoJLIa2+C zIkfC?>9zk#7MdjK^`S;)(>%4qEn28+y1L7Ew19QnZnejIXEvcj>b^hG*Nz=+IHOKI zAdecFzE}I4WT_n9Og+&42U;RY?N>+wH>RllUNJ_#x})~bCxN5CsJ|%BrmIrb!&kbn ze$=YRlr+$l(dvYtCIv>Rlgi?NjrHoZxmuvVMP0CA2=hNKUA^tQ*?{hXx};AiyVW># zsS5?p%TQOAvZ5tTR-f$fh(spZ(BiJHTEq7v)#{qDBY~QM>iY#0Jk{BTJ6o$?&YsWM z_{4^hcIw~NjFFLlXpA*73)dx0v#snceFHQO?cZ3Lw^uc-NBIJEqcrZjy92YbH9blq z8NItTo)hYTs&g9eEe=eV0h+$g7^JS@8vg*AdhdG8&@N2J@J}=W+6v(OK25-_Mj-dH zCb(@W3)+57aEX!W`Zpf!rZE@V1C}UFLOp|fMS><{rj-l31kI9btOI#wP4=r>B;c$$ zbeEiQ&pYBiybv*I-F+ZYNI*RpTueoX)e8_C0cCJT$iYk-(1bl9<;>f zXPOVQBWS_fnnr~av#8dF)^}QU8430ot<^X*aBk?R)mXP@1J?wtaXViOtk$-OC-Zho zv>k&sa)heTcJA{VC!NPypDrYpR;L}@#tpDb(2hxE9{V5Hj_pksLP11>je(;nrp5KYuBdhnV}%Aj5RsG0?7r}n!L#*SYbt##GjVxY1}`-44+ z^vKZ`G_+wX%+_vK_%W*AX{&;GaQAs_^*w4>oNU9gkF}3Vcu|jz+9$0mfKLu;pLuZK zgDmZfiuOQ7srF^>2BzOy?T2L}fWgi>;VMm@U!qeU`ITL9vd*J+KZ8EPs`L9J6mUPI zi%t&VY_?x#HZs@>HtWoNoPo9q-K?p*IePWbeUmnyu`pYgS>jHgYIMuAUx#hKus_{oC*3thZe6^P!Y3 zbJUMs#o(>oqz^uO6_`9sAF|>xo7Gx05Da#Wvxk{W*cO1}*nl`bsX=?KIdG!UU>v}G zW$O*{f5tNZe>54|7Ag4w;$`?$vj?#MEDtKES6Jq_Nirm{_UH4HeR;$U&# zFyd2A%j=sN0^8&>sLBnK#_|2vPYq%1f8ikV#xS)B?Ys;zOGvON#E`Zij0NYAA;WGx zE96^4rosp4X=hlpvw``q-eXuXrIOLQz)*CZrn_Zt*xe_SO(Vy!Z{`>Vr@vwUTM`OR zw&6OBq56RY+!^EF&9m!_%x#4al6+ws%3&IWmKE@A@%^JhoD=UF74MyF& z5x_QYqtPiFxcJoQSZSr;P8*Hhj&*>>!#FXFKIweW80D^D-|uUTZW=ZR?vZ;V-qym-_? zu_372dLHk}p5mRI=(D z5I^Pi)?Lqa|r)~X1c^M9FPFNp3(k{oy_0$Vys&8W{)ZfXv$ZglMAL_JUYg3UkWf`G zQ{~Ocbj5F`e-%-1%q`QybHt7o)1!7pfTE-6albnLXM>9wc<90t=!hYRM*>pe3JW3; zhY0?*@F|g)#D8Y4MIlkc&0q$>VTgNtLaM7JGA<(0;+hzllw^*JN@VClKX_`AIX*5h zIx=>eV$aTIM_Vdp<^HzVrWiE{QoSw z=eF(au*clVYqsA1Z^g}s%pIL#pPN%=H|c*FG_gxSIGKde6cPWZ?lvSo!5qnZ{;%$T k-56_52#*i5L^K(~dCXrM0{^y+->igqOVXa~&8eUN4^TmK=Kufz delta 4295 zcmX9>30#i(8a>~;ef#^N?E8`}Wht7-HZ5eI%95o)wrH{>#PDS*OQ##pTb%-5~nk2Xa3`TDcThF&fh9G+^@-B1z;uf|M~HkW)1r zwM)Zevmj;V0^vg;mF5BYFCfpM6PqN+kvo9Zhan%Y1WtWIeZNE?<_FY|84E1?h~{6z zVMlyETDg<~);x6Zc}d3Za2uQoIMmT_f{TXHb>Zdwk&$Q)9~U>^ZV38c2m~5eq5t(b z;%N+U{s|DvF`!!t&^uVe89g<;>x99NfYPF3z#KEkV9ORH|;z#1J58lP>%D&x~ws zU2fB^z(oh$sc)782g-Ff579xa(3P~>Ln1{w`+dCwLmH`j(Yb_$KTZ(Ry0KbC!FV77 zm}U^1XVYN8Zo#8@wP&A&Z}(ndDCY^2-~A0J-a^Db8Hvs1LeyTSaQJIseuWG~1PFGY zSQgt$A#OnvVC!}vu7I^MGFyndMSScjBvsI%v6XgV+v-O^-#5bU@Q*<2Y$4~p3OuMG z9NJq2q3mY88rYJgCMI;rcofnbcRq1CsEdOGh9( zQFxlt6sT86cvWi8;P2HY%vg)YKCjCzELZ&SJ|d5{e!#wsp7z)Lk!jXcPf-2o*c_W-*`I zB@VIsP=?;(l>NPc%Ui^$*I2B5CW#Rr7*TtMNld;Q#{-?kt?880nE#5YS5^Vuo5cKs z{2<`2cs*HQ=%U5DzAu3!Z}Cx&5TJ!9{ub;_wRF>PyiR;pSpl3}CccVG1@`U}{}Oq? zh1QZ|o!daiXsJ7h&8fKl8vJQ-+EM26m z2d${4w$j18^}vE-QbD}RYCKaZU9A4_hZO13PgTHA1yYf4l-I@5jd}4jBx)GtDwW1E z#}VtKN9&gW+ZIdZl^+1pVGWNvNPm_v=g0d?Ujuo6jGK&?HozxWHjEy^NV?07VkyVD z8|7B}VgT3ca=Qw9BIPw%Zr_`sX;vn?+_(Yk94~k5-3wSXSnjN}VNbs!cm4N0)__g+ zN^}AaMaq2)B5*KW{&tcNki1>)cYyb2{wWU<=t!xfJapC_I#N>}wSe-o{V9jO&H+j? z8!72ZElC@(6x#@tErs)P{g z`<=WxP3HYC^6D&i%FabjYStULvQ^$b=?RGy%jvUt(Bh_YW;6+Q=xDt272%?~>&lE6mP@PlHyQvytUshCfHWuN$@sI?i9)-4ss z80Nl7AEo|4dM4~p8gzfk7X4mnmXyNW?o!(9bp-}?RC@TSz>jMcFJCsYX}c7!ss5Y^ zff}Y=QT#U3-mghYzqWK_#%#MXNZ7~5Q>6^6NoGAglxc@(@c0R3dgKUxcubkGP+~DE z%7Q8~HcnR-9-{necqmI0hPqR+vUF`65PM2VE+VncW@TM9s-m(njiY&!hq4u(fZ>?3 zEuIHjW+=NCkl^YkN_sKvwXiQy_V^XBL7dQVR+xqlXDFvkM}epQ%DF&>_-wRt`3?=$ z6*cU%R4Lj{VkZ)n(wP_7c;;#NBv&b4wuGTwt9o z7=eAi>pOpKLjC)%(szHzT-W}&)ZQw@pl?*sGR8csfDs7HDjPF50?r-s6!xvctf!{s)YD4z<$ zWiJxD_SSIoEhABPprJ&eM?T*f{_M<1>}_nQTs(;pJYcBOH3C92G+a__)E^|lZ(0}) zjuj+UY&2vh0=GPjR(mEN^x9&qJCn?tk1@6l+RfD|$=KfWBR8BC#$L`O7Jbmzw{c5u zG>*oAXzJK^t#NoareIx7yp1c+e!ej(vjrCoH{-&xb-=x3t+cr@Q|HSe(#cpD#19Yaj73lB zVS1*9*#nJb**wVQZ{y2`X91@K$6TQ#xmpEZWXMV-p0zLA#~)eNx03B zZ+~tQPydTkv6HED=`p~+tI792AwbK`cGI+|v1~NUO<`6R+m45(FwZ8;^)1ta&?9U} zFfCdb&kg2?DK5Jecf2f9@@DRE;h#+#Tf}m6D$?-SPSaMG3mmJrP20i)SyOFIC+rcl z=VL1Qn!*P5*7Ue3iElTX9xsoidfuA;d6L%?>@&>`x-$jc>zfZLZU%3s0)Hjb^`+~zN+&pzP z55Bb09JZd=(qInD@Ma_0YL0%H2(0lmFA7@1-v-WR`*;$u59wiET6mXZ^^JyM6V1yO zFgHyS&8ZjYVaJQ+45&b&n>lm&Z|oCxbN+|*z?R?5ryWQ*A;4U)YB=zlVlMP?1?~?v zzZ;qZ1YRTEY_ZUFZ-57eM3CxV`-i)QvX?DmQFHn%ucqrJMzPf z^A-<%OW?^gi%0!X%CgeZ^Ss1OrnY5BCvM5xd@X^EQ@Q!9u#6kQ=i&L52~O|0h-|ln z{vS&%(-TNAZLMYDvI%TBt1Pj#cCbTUv&89ou|=0zR%BIhgBodBJHCMW|K*e=y@a8= z*WYr)Gmb+e)RH$ZfQ%Pej(sMfQGOb3?P)1`rf@fmw-l?BnB#KGFAXPi*fg{}=8p;f z({RTo%kTf-4~;!NEuU{~;81#OHGLVvmK$!hHcA9;G_*D-pxw5SR*weuCuCA$9W#MB zX_sJ~+DgZ{Z?;aWUN*f#t#dZATFuw2kS^vMzqiYA-!!jj2V058hdq zb)N`K-E3XaN6*EhSi^`b*42?bc<3nWhG`^{^2WN;Oh=n8wWh6$1-|{unr`3C4P{uA zbx#HfxEZWBTlD5?G|Bq>`9=D-hk=RwJ(z0l-c287uLpm^Sfm-jLH}Ra>pxlG2-E7s~ZgLV( zyh&~Uip)fV>imGcIk&6o-nJE6{ zy-ZzS4ad2u1!-)(8+NO|%=TqQPpdb3P9lL^_1B!1)NP=8|BFfk*VQNc8HvsV)Mth6 zd_GnEWBPk8sj^yLZwMEdT{fjS0@%OMrXQP2^W$trL1ashx7BV!$5u$T246{Jk3HGe zCT$+&6l?1e9>wZDV(Ygpk0D!T8}^AkeAjK;)GG3u)y_6;&|LQXyS6#cZg5wOutgtk z$1$C1i=X;|`v1YU!monMsc2h$mET|f*0xzk;-dp>TQ^qAP`0I3uH*9g%9cHisat%` zmVb+cI$oS^E4cSPaJ{|lYB~*vW!j!!BewCjm9 ActionShelf - - + + Preview Frame Preview Frame - - + + Render Video Render Video - - + + Copy render settings + Copy render settings + + + + Preferences Preferences @@ -38,142 +43,142 @@ Inspector - + Segmentation Segmentation - + Time & Sampling Time & Sampling - + Crop & Cleanup Crop & Cleanup - + Transform Transform - + Output Output - + Workspace Workspace - - - + + + X X - - - + + + Y Y - - - + + + Width Width - - - + + + Height Height - + Ready Ready - + Downloading Downloading - - + + Not cached Not cached - + Model Model - + Edge treatment Edge treatment - + Output FPS Output FPS - - + + Start Start - - + + End End - - + + Duration Duration - + Alpha threshold Alpha threshold - + Padding Padding - + Horizontal stretch Horizontal stretch - + Directory Directory - + Filename Filename - + Maximum size Maximum size @@ -448,7 +453,12 @@ Open folder - + + Open a video with a valid range to copy its render settings. + Open a video with a valid range to copy its render settings. + + + Segmentation is unavailable because ONNX Runtime could not enumerate any providers. Repair the installation with: %1 Segmentation is unavailable because ONNX Runtime could not enumerate any providers. Repair the installation with: %1 @@ -1263,12 +1273,12 @@ Freed %s. Free disk space, then render again. - + Rebuilding from edited cuts Rebuilding from edited cuts - + Rendering video Rendering video @@ -1790,96 +1800,104 @@ Freed %s. RenderController - + Preview recommended Preview recommended - + Preview this frame before rendering? Preview this frame before rendering? - + A preview lets you verify the cutout before processing the whole video. A preview lets you verify the cutout before processing the whole video. - + Preview first Preview first - + Render anyway Render anyway - + Cancel Cancel - + Matching cut set found Matching cut set found - + A validated cut set matches the current source and settings. A validated cut set matches the current source and settings. - + Rebuild reuses the cuts and only reruns framing and encoding. Regenerate removes backgrounds again for every selected frame. Rebuild reuses the cuts and only reruns framing and encoding. Regenerate removes backgrounds again for every selected frame. - + Reuse cuts and rebuild Reuse cuts and rebuild - + Regenerate backgrounds Regenerate backgrounds - + Output already exists Output already exists - + %s already exists. %s already exists. - + Choose how to handle the existing output. Choose how to handle the existing output. - + Replace Replace - + Choose another name Choose another name - + Choose output name Choose output name - + WebP files (*.webp) WebP files (*.webp) + + RenderSettingsExportController + + + Copied render settings to clipboard + Copied render settings to clipboard + + ResultPlayerCanvas @@ -2044,17 +2062,17 @@ Freed %s. SourceController - + Open video Open video - + Video files (*.mp4 *.mov *.webm *.mkv) Video files (*.mp4 *.mov *.webm *.mkv) - + Choose output directory Choose output directory @@ -2090,57 +2108,57 @@ Freed %s. SourceErrors - + Open a video stored on this Mac. Open a video stored on this Mac. - + Open a video file that can be opened and read. Open a video file that can be opened and read. - + Open a file that contains a video track. Open a file that contains a video track. - + Open another video file; this one appears damaged. Open another video file; this one appears damaged. - + Open a video with a positive duration. Open a video with a positive duration. - + Convert to 8-bit SDR and try again. Convert to 8-bit SDR and try again. - + Resize to between 8×8 and 3840×2160. Resize to between 8×8 and 3840×2160. - + Convert the video to 60 fps or less. Convert the video to 60 fps or less. - + Open a video under 10 minutes. Open a video under 10 minutes. - + Open an MP4, MOV, WebM, or MKV video. Open an MP4, MOV, WebM, or MKV video. - + This video could not be read. Open another video. This video could not be read. Open another video. @@ -2523,8 +2541,8 @@ Freed %s. UpdateBanner - - + + diff --git a/scripts/benchmark_models.py b/scripts/benchmark_models.py new file mode 100644 index 0000000..5df5af4 --- /dev/null +++ b/scripts/benchmark_models.py @@ -0,0 +1,597 @@ +"""Render one clip with every catalog model and produce a comparison report. + +Renders the request exported through the Render button's "Copy render +settings" context-menu action (`ui/render_settings_export.py`, +`core/request_json.py`) through every -- or a chosen subset of -- V1 +segmentation model, holding source, sampling, crop, framing, transform, and +output size budget constant. + +This reuses the application's own wiring rather than a copy of it: +`ui.preview_controller.runtime.ProductionPreviewRuntime` is the same class +`ui/main_window.py` builds for the real preview and render jobs, so model +download, session preparation, and rendering behave identically to a run +started from the GUI. In particular, model weights still download through +`ui/download_transport.py`'s Qt transport -- see CLAUDE.md, "One network +path" -- this script never opens a socket of its own. + +Cut-workspace caveat (issue #170 review, item 7): every render -- benchmark +or otherwise -- promotes its segmented cuts into the single durable +workspace cache all of MatteLoop shares (`paths.cut_workspace_root()`, +fixed, not overridable). There is no supported way to give this script its +own isolated workspace root without changing `jobs/render.py` (frozen) and +the workspace-root validation in `jobs/workspace/_platform.py` and +`_models.py`, which treats the shared root as a canonical, checked +invariant, not a parameter. Short of that change, this script instead +minimises the damage: it never forces re-segmentation over an existing cut +set (there is no `--regenerate`; a previous CLI version's flag detected the +destructive case reviewers actually hit -- overwriting a hand-edited cut +workspace with fresh, unedited segmentation output -- and removing it +removes that risk), and it detects and loudly flags a cache hit instead of +reporting it as if it were the raw model's output. See the module docstring +note in `_predict_cache_hit` below and the "STOP" discussion in the review +for the options that would need a maintainer decision to isolate this fully. + +This is a developer tool under `scripts/`, not a V1 product feature (see +docs/v1-scope.md). It does not download models or render anything by itself +when imported for its argument-parsing and reporting logic; a real run is a +deliberate, separate act driven from the command line. +""" + +from __future__ import annotations + +import argparse +import json +import sys +import time +from collections.abc import Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import Protocol + +from matteloop.core.errors import AppError +from matteloop.core.execution_providers import ( + ALLOWED_EXECUTION_PROVIDERS, + is_allowed_provider, + provider_options_from_runtime, + select_provider, +) +from matteloop.core.parameters import V1_MODEL_IDS +from matteloop.core.request_json import render_settings_from_json +from matteloop.core.specs import CollisionPolicy, OutputSpec, RenderRequest +from matteloop.core.state import JobKind +from matteloop.jobs.context import CancellationState, JobContext +from matteloop.jobs.models.catalog import ModelCatalog +from matteloop.jobs.render import ( + FilesystemWorkspacePort, + LocalSourcePort, + RenderArtifact, + find_matching_cut_workspace, +) + +# `pytest` imports this module as `scripts.benchmark_models` (repository root +# on sys.path, see tests/scripts/__init__.py); running it directly as +# `python scripts/benchmark_models.py` instead puts `scripts/` itself on +# sys.path, the same dual-mode split already used by scripts/build.py and +# scripts/qt_source.py. Both spellings have to resolve. +try: + from scripts.benchmark_report import render_report_html +except ImportError: + from benchmark_report import render_report_html # type: ignore[no-redef] + +_LICENSED_MODEL_ID = "bria-rmbg" + + +class BenchmarkError(Exception): + """A user-facing problem with the benchmark's arguments or input.""" + + +class Clipboard(Protocol): + def text(self) -> str: ... + + +class BenchmarkRuntime(Protocol): + """The subset of `ProductionPreviewRuntime` this script depends on.""" + + def prepare( + self, model_id: str, extras: dict[str, object], context: JobContext + ) -> object: ... + + def render(self, request: RenderRequest, context: JobContext) -> RenderArtifact: ... + + +@dataclass(frozen=True) +class ModelResult: + """One model's benchmark outcome, ready for `results.json` and the report.""" + + model_id: str + display_name: str + status: str # "ok" or "error" + error: str | None = None + weight_size_bytes: int | None = None + prepare_seconds: float | None = None + render_seconds: float | None = None + cache_hit: bool | None = None + active_provider: str | None = None + fallback_notice: str | None = None + file_size_bytes: int | None = None + frame_count: int | None = None + duration_ms: int | None = None + output_relative_path: str | None = None + + +@dataclass(frozen=True) +class BenchmarkRun: + """Everything the report and `results.json` need about one run.""" + + generated_at: str + provider: str + request_settings: Mapping[str, object] + results: tuple[ModelResult, ...] + warnings: tuple[str, ...] = () + + +def select_models(catalog: ModelCatalog, requested: str | None) -> tuple[str, ...]: + """Resolve `--models`, defaulting to the app's own V1 model set. + + `bria-rmbg` (model-specific licence terms) and `u2net_cloth_seg` (needs + a clothing-category input the UI cannot provide) are excluded from that + default, matching docs/v1-scope.md; both still work when named + explicitly via `--models`. + """ + if requested is None: + return V1_MODEL_IDS + ids = tuple(item.strip() for item in requested.split(",") if item.strip()) + if not ids: + raise BenchmarkError("--models must name at least one model id") + unknown = sorted(set(ids) - set(catalog.ids)) + if unknown: + valid = ", ".join(sorted(catalog.ids)) + raise BenchmarkError( + f"unknown model id(s): {', '.join(unknown)}; valid ids are: {valid}" + ) + return ids + + +def resolve_provider(requested: str | None) -> str: + """Resolve `--provider`, defaulting to the app's own recommendation.""" + if requested is not None: + if not is_allowed_provider(requested): + valid = ", ".join(ALLOWED_EXECUTION_PROVIDERS) + raise BenchmarkError( + f"unknown execution provider {requested!r}; " + f"valid providers are: {valid}" + ) + return requested + return select_provider(None, provider_options_from_runtime()) + + +def load_request_text(path: Path | None, clipboard: Clipboard) -> str: + """Read the exported render-settings JSON from a file or the clipboard.""" + if path is not None: + try: + return path.read_text(encoding="utf-8") + except OSError as error: + raise BenchmarkError( + f"could not read request file {path}: {error}" + ) from error + text = clipboard.text() + if not text.strip(): + raise BenchmarkError( + "the clipboard is empty; use the Render button's \"Copy render " + 'settings" context-menu action first, or pass --request' + ) + return text + + +def build_base_request(request_text: str, out_dir: Path) -> RenderRequest: + """Parse the exported settings once, with a placeholder output. + + `_model_request` replaces `output` for every model with a real filename + under `out_dir`, carrying over this base request's `output.max_bytes` -- + the only part of the placeholder output that is ever read. + """ + return render_settings_from_json( + request_text, + directory=out_dir, + filename="_placeholder.webp", + collision_policy=CollisionPolicy.REPLACE, + ) + + +def run_benchmark( + *, + base_request: RenderRequest, + request_settings: Mapping[str, object], + model_ids: Sequence[str], + provider: str, + out_dir: Path, + catalog: ModelCatalog, + runtime: BenchmarkRuntime, + clock: Callable[[], float] = time.perf_counter, + log: Callable[[str], None] = print, +) -> BenchmarkRun: + """Render `model_ids` sequentially through `runtime` and collect results. + + Writes `results.json` and `index.html` after every model, not just at + the end, so a Ctrl+C partway through keeps every tile already rendered + (issue #170 review, item 8) instead of losing the whole run. + """ + if not model_ids: + raise BenchmarkError("model_ids must not be empty") + out_dir.mkdir(parents=True, exist_ok=True) + generated_at = datetime.now(UTC).isoformat(timespec="seconds") + results: list[ModelResult] = [] + run = BenchmarkRun(generated_at, provider, request_settings, ()) + for index, model_id in enumerate(model_ids, start=1): + results.append( + _run_one_model( + base_request=base_request, + model_id=model_id, + index=index, + total=len(model_ids), + provider=provider, + out_dir=out_dir, + catalog=catalog, + runtime=runtime, + clock=clock, + log=log, + ) + ) + run = BenchmarkRun( + generated_at=generated_at, + provider=provider, + request_settings=request_settings, + results=tuple(results), + warnings=_consistency_warnings(results), + ) + write_results(out_dir, run) + write_report(out_dir, run) + return run + + +@dataclass(frozen=True) +class _ModelTiming: + prepare_seconds: float + render_seconds: float + cache_hit: bool | None + active_provider: str | None + fallback_notice: str | None + + +def _model_request( + base_request: RenderRequest, model_id: str, provider: str, out_dir: Path +) -> RenderRequest: + return replace( + base_request, + segmentation=replace( + base_request.segmentation, model_id=model_id, execution_provider=provider + ), + output=OutputSpec( + out_dir, + f"{model_id}.webp", + max_bytes=base_request.output.max_bytes, + collision_policy=CollisionPolicy.REPLACE, + ), + ) + + +def _execute_model( + runtime: BenchmarkRuntime, + request: RenderRequest, + model_id: str, + context: JobContext, + clock: Callable[[], float], +) -> tuple[_ModelTiming, RenderArtifact]: + prepare_start = clock() + runtime.prepare( + model_id, + {"execution_provider": request.segmentation.execution_provider}, + context, + ) + prepare_seconds = clock() - prepare_start + cache_hit = _predict_cache_hit(runtime, request, context) + render_start = clock() + artifact = runtime.render(request, context) + render_seconds = clock() - render_start + timing = _ModelTiming( + prepare_seconds=prepare_seconds, + render_seconds=render_seconds, + cache_hit=cache_hit, + active_provider=getattr(runtime, "active_provider", None), + fallback_notice=getattr(runtime, "fallback_notice", None), + ) + return timing, artifact + + +def _run_one_model( + *, + base_request: RenderRequest, + model_id: str, + index: int, + total: int, + provider: str, + out_dir: Path, + catalog: ModelCatalog, + runtime: BenchmarkRuntime, + clock: Callable[[], float], + log: Callable[[str], None], +) -> ModelResult: + log(f"[{index}/{total}] {model_id} ...") + spec = catalog.get(model_id) + weight_size = spec.artifact.size_bytes if spec.artifact is not None else None + request = _model_request(base_request, model_id, provider, out_dir) + context = _new_job_context(out_dir, model_id) + try: + timing, artifact = _execute_model(runtime, request, model_id, context, clock) + except Exception as error: # noqa: BLE001 - recorded per model, loop continues + log(f" {model_id}: failed -- {error}") + return ModelResult( + model_id=model_id, + display_name=spec.display_name, + status="error", + error=str(error), + weight_size_bytes=weight_size, + ) + log( + f" {model_id}: ok -- prepare {timing.prepare_seconds:.1f}s, " + f"render {timing.render_seconds:.1f}s" + ) + return ModelResult( + model_id=model_id, + display_name=spec.display_name, + status="ok", + weight_size_bytes=weight_size, + prepare_seconds=timing.prepare_seconds, + render_seconds=timing.render_seconds, + cache_hit=timing.cache_hit, + active_provider=timing.active_provider, + fallback_notice=timing.fallback_notice, + file_size_bytes=artifact.file_size, + frame_count=artifact.frame_count, + duration_ms=artifact.duration_ms, + output_relative_path=f"{model_id}.webp", + ) + + +def _new_job_context(out_dir: Path, model_id: str) -> JobContext: + return JobContext( + job_id=f"benchmark-{model_id}", + kind=JobKind.RENDER, + workspace=out_dir, + progress_sink=lambda _event: None, + cancellation=CancellationState(), + ) + + +def _predict_cache_hit( + runtime: BenchmarkRuntime, request: RenderRequest, context: JobContext +) -> bool | None: + """Predict, without changing `jobs/render.py`, whether segmentation will + be skipped in favour of an already-promoted cut workspace. + + `RenderService.render()` makes exactly this check internally + (`find_matching_cut_workspace`, frozen per docs/engineering-guardrails.md) + before deciding to segment or reuse; calling the same public function + first lets the script report the outcome instead of guessing at it. A + hit means this tile may show a previously hand-edited cut set rather + than the model's raw output (issue #170 review, item 7) -- surfaced as a + run-level warning by `_consistency_warnings`, since there is no isolated + workspace to render into instead. Returns ``None`` when the runtime does + not expose a `ModelCatalog` to look the model's weight hash up in -- + true for a test double that has no reason to carry one. + """ + catalog = getattr(runtime, "catalog", None) + if not isinstance(catalog, ModelCatalog): + return None + spec = catalog.get(request.segmentation.model_id) + if spec.artifact is None: + return None + workspace = find_matching_cut_workspace( + LocalSourcePort(), + FilesystemWorkspacePort(), + request, + model_weight_sha256=spec.artifact.sha256, + rembg_version=catalog.rembg_version, + context=context, + ) + return workspace is not None + + +def _consistency_warnings(results: Sequence[ModelResult]) -> tuple[str, ...]: + """Flag anything that would make the comparison misleading. + + Every model renders the same sampling, crop, and framing, so successful + models should agree on frame count and duration; a mismatch usually + means a model dropped or duplicated frames. A cache hit means the tile + may not be this model's raw output at all (see `_predict_cache_hit`). + """ + warnings: list[str] = [] + ok = [result for result in results if result.status == "ok"] + frame_counts = {result.frame_count for result in ok} + if len(frame_counts) > 1: + pairs = ", ".join(f"{result.model_id}={result.frame_count}" for result in ok) + warnings.append(f"frame_count differs across models: {pairs}") + durations = {result.duration_ms for result in ok} + if len(durations) > 1: + pairs = ", ".join(f"{result.model_id}={result.duration_ms}" for result in ok) + warnings.append(f"duration_ms differs across models: {pairs}") + reused = [result.model_id for result in ok if result.cache_hit] + if reused: + warnings.append( + "reused an existing durable cut workspace for: " + + ", ".join(reused) + + " -- these tiles may show a previously hand-edited cut set, " + "not the model's raw output" + ) + return tuple(warnings) + + +def write_results(out_dir: Path, run: BenchmarkRun) -> Path: + path = out_dir / "results.json" + payload = { + "generated_at": run.generated_at, + "provider": run.provider, + "request": run.request_settings, + "warnings": list(run.warnings), + "models": [asdict(result) for result in run.results], + } + path.write_text( + json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return path + + +def write_report(out_dir: Path, run: BenchmarkRun) -> Path: + path = out_dir / "index.html" + path.write_text(render_report_html(run), encoding="utf-8") + return path + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Render one clip with every V1 segmentation model, using " + "settings copied from the Render button, and produce a " + "side-by-side comparison report." + ) + ) + parser.add_argument( + "--request", + type=Path, + default=None, + help=( + "path to exported render-settings JSON; default: read the " + "clipboard. On Linux or a headless machine, prefer --request: " + "clipboard access there depends on a running desktop session " + "and is not reliable otherwise." + ), + ) + parser.add_argument( + "--models", + type=str, + default=None, + help=( + "comma-separated model ids; default: the app's own 13-model V1 " + "set. 'bria-rmbg' and 'u2net_cloth_seg' are excluded from that " + "default but work if named explicitly; selecting 'bria-rmbg' " + "prints its licence note before rendering." + ), + ) + parser.add_argument( + "--provider", + type=str, + default=None, + help="execution provider; default: this machine's recommended provider", + ) + parser.add_argument( + "--out", + type=Path, + default=None, + help="output directory; default: ./benchmark-", + ) + return parser + + +def _default_out_dir() -> Path: + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + return Path.cwd() / f"benchmark-{stamp}" + + +def _print_license_note_if_selected( + catalog: ModelCatalog, + model_ids: Sequence[str], + log: Callable[[str], None] = print, +) -> None: + """Print `bria-rmbg`'s licence note before rendering it (issue #170, item 3). + + Split out from `main()` so the decision -- and the note it prints -- is + testable without building a real runtime. + """ + if _LICENSED_MODEL_ID in model_ids: + log(f"licence: {catalog.get(_LICENSED_MODEL_ID).license_note}") + + +class _TextClipboard: + """Adapts Qt's clipboard to the `Clipboard` protocol, importing Qt lazily + so argument-parsing and reporting logic can be exercised without it.""" + + def text(self) -> str: + from PySide6.QtGui import QGuiApplication + + app = QGuiApplication.instance() + if app is None: + app = QGuiApplication(sys.argv[:1]) + clipboard = app.clipboard() + if clipboard is None: + raise BenchmarkError("no clipboard is available on this platform") + return clipboard.text() + + +def _ensure_qt_application() -> object: + """Guarantee a Qt application instance exists before any network I/O. + + `QtNetworkDownloadTransport` (ui/download_transport.py) pumps a + `QEventLoop` while waiting for each downloaded chunk; without a + constructed `QCoreApplication`, that loop never dispatches, so the + first uncached model hangs forever instead of downloading (issue #170 + review, item 1). This has to run before the runtime is built, + regardless of whether the request comes from `--request` or the + clipboard -- previously only the clipboard path created one, lazily, + inside `_TextClipboard.text()`. + """ + from PySide6.QtCore import QCoreApplication + from PySide6.QtGui import QGuiApplication + + return QCoreApplication.instance() or QGuiApplication(sys.argv[:1]) + + +def main(argv: Sequence[str] | None = None) -> int: + args = build_arg_parser().parse_args(argv) + _ensure_qt_application() + try: + request_text = load_request_text(args.request, _TextClipboard()) + catalog = ModelCatalog.load_resource() + model_ids = select_models(catalog, args.models) + provider = resolve_provider(args.provider) + out_dir = args.out if args.out is not None else _default_out_dir() + base_request = build_base_request(request_text, out_dir) + request_settings = json.loads(request_text) + except (BenchmarkError, AppError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + + _print_license_note_if_selected(catalog, model_ids) + + from matteloop.ui.preview_controller.runtime import ProductionPreviewRuntime + + runtime = ProductionPreviewRuntime() + try: + run = run_benchmark( + base_request=base_request, + request_settings=request_settings, + model_ids=model_ids, + provider=provider, + out_dir=out_dir, + catalog=catalog, + runtime=runtime, + ) + except KeyboardInterrupt: + print( + "interrupted; results for models completed so far were already " + f"written to {out_dir}", + file=sys.stderr, + ) + return 130 + finally: + runtime.close() + + for warning in run.warnings: + print(f"warning: {warning}", file=sys.stderr) + succeeded = sum(1 for result in run.results if result.status == "ok") + print(f"{succeeded}/{len(run.results)} models succeeded; wrote {out_dir}") + return 0 if succeeded == len(run.results) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/benchmark_report.py b/scripts/benchmark_report.py new file mode 100644 index 0000000..b4d65d1 --- /dev/null +++ b/scripts/benchmark_report.py @@ -0,0 +1,270 @@ +"""Self-contained HTML report for a `scripts/benchmark_models.py` run. + +Kept separate from `benchmark_models.py` so that module stays comfortably +under the size that would call for a package split. This file only turns +already-computed results into one static page: inline CSS and JavaScript, +relative image paths, no external resources, so the report keeps working if +the output directory is copied or zipped elsewhere. +""" + +from __future__ import annotations + +from html import escape +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from scripts.benchmark_models import BenchmarkRun, ModelResult + +_BACKGROUNDS = ("checkerboard", "black", "white", "green") + +_CSS = """ +:root { color-scheme: light dark; } +* { box-sizing: border-box; } +body { + margin: 0; + font-family: -apple-system, "Segoe UI", Roboto, sans-serif; + background: #14161a; + color: #f2f2f2; +} +header { + position: sticky; + top: 0; + z-index: 2; + padding: 12px 16px; + background: #1e2128; + border-bottom: 1px solid #2a2e37; + display: flex; + align-items: center; + gap: 16px; + flex-wrap: wrap; +} +h1 { font-size: 16px; margin: 0; } +.controls { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; } +.meta { color: #9aa0aa; font-size: 12px; } +.grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); + gap: 16px; + padding: 16px; +} +.tile { + background: #1e2128; + border: 1px solid #2a2e37; + border-radius: 8px; + padding: 12px; +} +.tile-error { border-color: #a33; } +.image-wrap { + aspect-ratio: 4 / 3; + border-radius: 4px; + overflow: hidden; + display: flex; + align-items: center; + justify-content: center; + background-color: #808080; +} +.grid.bg-checkerboard .image-wrap { + background-image: + linear-gradient(45deg, #666 25%, transparent 25%), + linear-gradient(-45deg, #666 25%, transparent 25%), + linear-gradient(45deg, transparent 75%, #666 75%), + linear-gradient(-45deg, transparent 75%, #666 75%); + background-size: 20px 20px; + background-position: 0 0, 0 10px, 10px -10px, -10px 0px; + background-color: #999; +} +.grid.bg-black .image-wrap { background-color: #000; } +.grid.bg-white .image-wrap { background-color: #fff; } +.grid.bg-green .image-wrap { background-color: #0f0; } +.result-image { max-width: 100%; max-height: 100%; } +.model-id { color: #9aa0aa; font-size: 12px; margin: 2px 0 8px; } +.stats { + display: grid; + grid-template-columns: auto auto; + gap: 2px 8px; + font-size: 12px; + margin: 8px 0 0; +} +.stats dt { color: #9aa0aa; } +.stats dd { margin: 0; } +.error { color: #f88; font-size: 13px; } +.warnings { + margin: 0; + padding: 8px 16px; + background: #3a2f14; + color: #f0c674; + font-size: 12px; +} +.warnings ul { margin: 4px 0 0; padding-left: 18px; } +.cache-hit { color: #f0c674; } +button, select { + font: inherit; + background: #14161a; + color: #f2f2f2; + border: 1px solid #2a2e37; + border-radius: 4px; + padding: 4px 8px; +} +""" + +_SCRIPT = """ +(function () { + var grid = document.getElementById("grid"); + var bgSelect = document.getElementById("bg-select"); + var restartButton = document.getElementById("restart-all"); + + function applyBackground() { + grid.className = "grid bg-" + bgSelect.value; + } + + function restartAll() { + // Preload every image off-DOM first, then swap all visible tags + // to the reloaded sources together, so the animations start back in + // step (best effort -- network timing can still drift once playing). + var stamp = Date.now(); + var images = Array.prototype.slice.call( + grid.querySelectorAll("img.result-image") + ); + if (images.length === 0) { + return; + } + var remaining = images.length; + var settle = function () { + remaining -= 1; + if (remaining === 0) { + images.forEach(function (img) { + img.src = img.getAttribute("data-restart-src"); + }); + } + }; + images.forEach(function (img) { + var url = img.getAttribute("data-src") + "?t=" + stamp; + img.setAttribute("data-restart-src", url); + var preload = new Image(); + preload.onload = settle; + preload.onerror = settle; + preload.src = url; + }); + } + + bgSelect.addEventListener("change", applyBackground); + restartButton.addEventListener("click", restartAll); + applyBackground(); +})(); +""" + + +def render_report_html(run: BenchmarkRun) -> str: + """Render the full self-contained results page for one benchmark run.""" + tiles = "\n".join(_tile_html(result) for result in run.results) + header = ( + "

MatteLoop model benchmark

" + '
' + "" + '' + f'Provider: {escape(run.provider)}' + f" · Generated {escape(run.generated_at)}" + "
" + ) + warnings = _warnings_html(run.warnings) + return ( + "\n" + '\n\n\n' + "MatteLoop model benchmark\n" + f"\n\n\n" + f"
{header}
\n" + f"{warnings}" + f'
\n{tiles}\n
\n' + f"\n\n\n" + ) + + +def _warnings_html(warnings: tuple[str, ...]) -> str: + if not warnings: + return "" + items = "".join(f"
  • {escape(warning)}
    • " for warning in warnings) + return f'
    Warnings
      {items}
    \n' + + +def _background_options_html() -> str: + options = [] + for index, name in enumerate(_BACKGROUNDS): + selected = ' selected="selected"' if index == 0 else "" + options.append(f'') + return "".join(options) + + +def _tile_html(result: ModelResult) -> str: + title = escape(result.display_name or result.model_id) + model_id = escape(result.model_id) + if result.status != "ok": + error = escape(result.error or "unknown error") + return ( + f'
    ' + f"

    {title}

    " + f'

    {model_id}

    ' + f'

    Failed: {error}

    ' + "
    " + ) + src = escape(result.output_relative_path or "") + provider_row = _provider_row(result.active_provider, result.fallback_notice) + cache_class = ' class="cache-hit"' if result.cache_hit else "" + return ( + f'
    ' + f'
    {title} result
    ' + f"

    {title}

    " + f'

    {model_id}

    ' + f'
    ' + f"
    Weight
    {_format_mb(result.weight_size_bytes)}
    " + f"
    Render
    {_format_seconds(result.render_seconds)}
    " + f"
    Prepare
    {_format_seconds(result.prepare_seconds)}
    " + f"
    File size
    {_format_file_size(result.file_size_bytes)}
    " + f"
    Frames
    {_format_int(result.frame_count)}
    " + f"
    Duration
    {_format_ms(result.duration_ms)}
    " + f"
    Cache
    {_cache_hit_label(result.cache_hit)}" + f"{provider_row}" + "
    " + "
    " + ) + + +def _provider_row(active_provider: str | None, fallback_notice: str | None) -> str: + if active_provider is None: + return "" + value = escape(active_provider) + if fallback_notice: + value += f" ({escape(fallback_notice)})" + return f"
    Provider
    {value}
    " + + +def _cache_hit_label(cache_hit: bool | None) -> str: + if cache_hit is None: + return "unknown" + return "hit — may be edited" if cache_hit else "miss" + + +def _format_seconds(value: float | None) -> str: + return "–" if value is None else f"{value:.1f}s" + + +def _format_ms(value: int | None) -> str: + return "–" if value is None else f"{value} ms" + + +def _format_int(value: int | None) -> str: + return "–" if value is None else str(value) + + +def _format_mb(value: int | None) -> str: + return "–" if value is None else f"{value / (1024 * 1024):.1f} MB" + + +def _format_file_size(value: int | None) -> str: + if value is None: + return "–" + if value >= 1024 * 1024: + return f"{value / (1024 * 1024):.2f} MB" + return f"{value / 1024:.1f} KB" diff --git a/src/matteloop/core/errors.py b/src/matteloop/core/errors.py index 280b758..79cd50c 100644 --- a/src/matteloop/core/errors.py +++ b/src/matteloop/core/errors.py @@ -16,6 +16,7 @@ class ErrorCode(StrEnum): INVALID_TRANSFORM = "invalid_transform" INVALID_OUTPUT = "invalid_output" INVALID_RENDER_REQUEST = "invalid_render_request" + RENDER_SETTINGS_INVALID = "render_settings_invalid" INVALID_FINAL_DIMENSIONS = "invalid_final_dimensions" INVALID_ERROR = "invalid_error" IMPOSSIBLE_SIZE = "impossible_size" diff --git a/src/matteloop/core/request_json.py b/src/matteloop/core/request_json.py new file mode 100644 index 0000000..54b8e87 --- /dev/null +++ b/src/matteloop/core/request_json.py @@ -0,0 +1,314 @@ +"""Versioned JSON codec for exporting and re-importing render settings. + +Deliberately excludes the output *location* (directory, filename) and the +job flags ``rebuild``/``regenerate``: those are the caller's decision (where +to write the file, whether to reuse cached frames), not part of the settings +a user wants to copy or replay elsewhere. ``render_settings_from_json`` +always rebuilds with ``rebuild=regenerate=False``. + +The transform (trim, crop, resize) and the output size budget +(``output.max_bytes``) *are* part of the settings a user is comparing -- +they are not tied to a location -- so both round-trip exactly. +``render_settings_from_json`` applies the stored ``max_bytes`` onto the +caller-supplied directory and filename. + +Fractions round-trip exactly as ``{"numerator", "denominator"}`` objects, +matching the convention already used for persisted fingerprints in +``core/fingerprints.py``. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from decimal import Decimal, InvalidOperation +from fractions import Fraction +from pathlib import Path + +from matteloop.core.errors import ErrorCode, ValidationError +from matteloop.core.specs import ( + AlphaMattingSpec, + CollisionPolicy, + CropSpec, + EdgeMode, + FramingSpec, + MismatchMode, + OutputSpec, + RenderRequest, + ResizeSpec, + SamplingSpec, + SegmentationSpec, + TransformSpec, +) + +FORMAT = "matteloop.render-settings" +VERSION = 1 + +_MALFORMED_EXCEPTIONS = ( + KeyError, + TypeError, + ValueError, + ZeroDivisionError, + InvalidOperation, +) + + +def render_settings_to_json(request: RenderRequest) -> str: + """Serialise everything about a request except its output location.""" + payload = { + "format": FORMAT, + "version": VERSION, + "source": str(request.source), + "sampling": _sampling_payload(request.sampling), + "crop": _crop_payload(request.crop), + "segmentation": _segmentation_payload(request.segmentation), + "framing": _framing_payload(request.framing), + "transform": _transform_payload(request.transform), + "output": {"max_bytes": request.output.max_bytes}, + } + return json.dumps(payload, indent=2, sort_keys=True) + "\n" + + +def render_settings_from_json( + text: str, + *, + directory: Path, + filename: str, + collision_policy: CollisionPolicy = CollisionPolicy.CANCEL, +) -> RenderRequest: + """Rebuild a request from exported settings, at the given output location. + + ``directory`` and ``filename`` are the caller's decision, matching + ``render_settings_to_json``'s exclusion of the output location; the + exported ``output.max_bytes`` setting is applied on top of them. The + rebuilt request always carries ``rebuild=regenerate=False``; the export + never carried those. + """ + payload = _decode_envelope(text) + try: + source = Path(_require_str(payload, "source")) + sampling = _sampling_from_payload(_require_mapping(payload, "sampling")) + crop = _crop_from_payload(_require_mapping(payload, "crop")) + segmentation = _segmentation_from_payload( + _require_mapping(payload, "segmentation") + ) + framing = _framing_from_payload(_require_mapping(payload, "framing")) + transform = _transform_from_payload(_require_mapping(payload, "transform")) + max_bytes = _max_bytes_from_payload(_require_mapping(payload, "output")) + except _MALFORMED_EXCEPTIONS as error: + raise _invalid(f"render settings payload is malformed: {error}") from error + output = OutputSpec( + directory=directory, + filename=filename, + max_bytes=max_bytes, + collision_policy=collision_policy, + ) + return RenderRequest( + source=source, + sampling=sampling, + crop=crop, + segmentation=segmentation, + framing=framing, + output=output, + transform=transform, + rebuild=False, + regenerate=False, + ) + + +def _decode_envelope(text: str) -> Mapping[str, object]: + try: + payload = json.loads(text) + except json.JSONDecodeError as error: + raise _invalid(f"render settings are not valid JSON: {error}") from error + if not isinstance(payload, Mapping): + raise _invalid("render settings payload must be a JSON object") + if payload.get("format") != FORMAT: + raise _invalid(f"render settings format must be {FORMAT!r}") + version = payload.get("version") + if version != VERSION: + raise _invalid( + f"render settings version {version!r} is not supported; " + f"expected {VERSION}" + ) + return payload + + +def _fraction_payload(value: Fraction) -> dict[str, int]: + return {"numerator": value.numerator, "denominator": value.denominator} + + +def _fraction_from_payload(payload: Mapping[str, object], key: str) -> Fraction: + inner = _require_mapping(payload, key) + numerator = _require_int(inner, "numerator") + denominator = _require_int(inner, "denominator") + return Fraction(numerator, denominator) + + +def _sampling_payload(sampling: SamplingSpec) -> dict[str, object]: + return { + "start": _fraction_payload(sampling.start), + "end": _fraction_payload(sampling.end), + "fps": sampling.fps, + } + + +def _sampling_from_payload(payload: Mapping[str, object]) -> SamplingSpec: + return SamplingSpec( + _fraction_from_payload(payload, "start"), + _fraction_from_payload(payload, "end"), + _require_int(payload, "fps"), + ) + + +def _crop_payload(crop: CropSpec) -> dict[str, int]: + return {"x": crop.x, "y": crop.y, "width": crop.width, "height": crop.height} + + +def _crop_from_payload(payload: Mapping[str, object]) -> CropSpec: + return CropSpec( + _require_int(payload, "x"), + _require_int(payload, "y"), + _require_int(payload, "width"), + _require_int(payload, "height"), + ) + + +def _segmentation_payload(segmentation: SegmentationSpec) -> dict[str, object]: + matting = segmentation.alpha_matting + return { + "model_id": segmentation.model_id, + "edge_mode": segmentation.edge_mode.value, + "alpha_matting": { + "foreground_threshold": matting.foreground_threshold, + "background_threshold": matting.background_threshold, + "erode_size": matting.erode_size, + }, + "execution_provider": segmentation.execution_provider, + } + + +def _segmentation_from_payload(payload: Mapping[str, object]) -> SegmentationSpec: + matting_payload = _require_mapping(payload, "alpha_matting") + matting = AlphaMattingSpec( + foreground_threshold=_require_int(matting_payload, "foreground_threshold"), + background_threshold=_require_int(matting_payload, "background_threshold"), + erode_size=_require_int(matting_payload, "erode_size"), + ) + return SegmentationSpec( + model_id=_require_str(payload, "model_id"), + edge_mode=EdgeMode(_require_str(payload, "edge_mode")), + alpha_matting=matting, + execution_provider=_require_str(payload, "execution_provider"), + ) + + +def _framing_payload(framing: FramingSpec) -> dict[str, object]: + return { + "trim": framing.trim, + "alpha_threshold": str(framing.alpha_threshold), + "padding": framing.padding, + "stretch_x": str(framing.stretch_x), + } + + +def _framing_from_payload(payload: Mapping[str, object]) -> FramingSpec: + return FramingSpec( + trim=_require_bool(payload, "trim"), + alpha_threshold=Decimal(_require_str(payload, "alpha_threshold")), + padding=_require_int(payload, "padding"), + stretch_x=Decimal(_require_str(payload, "stretch_x")), + ) + + +def _transform_payload(transform: TransformSpec) -> dict[str, object]: + return { + "first_frame": transform.first_frame, + "last_frame": transform.last_frame, + "crop": None if transform.crop is None else _crop_payload(transform.crop), + "resize": ( + None if transform.resize is None else _resize_payload(transform.resize) + ), + } + + +def _transform_from_payload(payload: Mapping[str, object]) -> TransformSpec: + crop_payload = payload.get("crop") + resize_payload = payload.get("resize") + return TransformSpec( + first_frame=_require_int(payload, "first_frame"), + last_frame=_require_int_or_none(payload, "last_frame"), + crop=( + None + if crop_payload is None + else _crop_from_payload(_require_mapping(payload, "crop")) + ), + resize=( + None + if resize_payload is None + else _resize_from_payload(_require_mapping(payload, "resize")) + ), + ) + + +def _resize_payload(resize: ResizeSpec) -> dict[str, object]: + return { + "width": resize.width, + "height": resize.height, + "mismatch": resize.mismatch.value, + } + + +def _resize_from_payload(payload: Mapping[str, object]) -> ResizeSpec: + return ResizeSpec( + width=_require_int_or_none(payload, "width"), + height=_require_int_or_none(payload, "height"), + mismatch=MismatchMode(_require_str(payload, "mismatch")), + ) + + +def _max_bytes_from_payload(payload: Mapping[str, object]) -> int | None: + return _require_int_or_none(payload, "max_bytes") + + +def _require_mapping(payload: Mapping[str, object], key: str) -> Mapping[str, object]: + value = payload.get(key) + if not isinstance(value, Mapping): + raise ValueError(f"{key} must be an object") + return value + + +def _require_str(payload: Mapping[str, object], key: str) -> str: + value = payload.get(key) + if not isinstance(value, str): + raise ValueError(f"{key} must be a string") + return value + + +def _require_int(payload: Mapping[str, object], key: str) -> int: + value = payload.get(key) + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{key} must be an integer") + return value + + +def _require_int_or_none(payload: Mapping[str, object], key: str) -> int | None: + if key not in payload: + raise ValueError(f"{key} is required") + value = payload[key] + if value is None: + return None + if not isinstance(value, int) or isinstance(value, bool): + raise ValueError(f"{key} must be an integer or null") + return value + + +def _require_bool(payload: Mapping[str, object], key: str) -> bool: + value = payload.get(key) + if not isinstance(value, bool): + raise ValueError(f"{key} must be a boolean") + return value + + +def _invalid(detail: str) -> ValidationError: + return ValidationError(ErrorCode.RENDER_SETTINGS_INVALID, "request-json", detail) diff --git a/src/matteloop/ui/action_shelf.py b/src/matteloop/ui/action_shelf.py index 12e5bdc..0f3f24c 100644 --- a/src/matteloop/ui/action_shelf.py +++ b/src/matteloop/ui/action_shelf.py @@ -2,11 +2,29 @@ from __future__ import annotations -from PySide6.QtCore import QCoreApplication, QRectF, QSettings, QSize, Qt -from PySide6.QtGui import QColor, QIcon, QKeySequence, QPainter, QPen, QPixmap +from PySide6.QtCore import ( + QCoreApplication, + QEvent, + QObject, + QRectF, + QSettings, + QSize, + Qt, +) +from PySide6.QtGui import ( + QAction, + QColor, + QContextMenuEvent, + QIcon, + QKeySequence, + QPainter, + QPen, + QPixmap, +) from PySide6.QtWidgets import ( QFrame, QHBoxLayout, + QMenu, QPushButton, QVBoxLayout, QWidget, @@ -125,6 +143,8 @@ def __init__( self.render_button.setAccessibleName( QCoreApplication.translate("ActionShelf", "Render Video") ) + self._render_menu: QMenu | None = None + self.copy_render_settings_action = self._build_copy_render_settings_action() self.preferences_button = _preferences_button() self.update_button = _update_button() for button in (self.preview_button, self.render_button): @@ -145,6 +165,56 @@ def __init__( ) self.preferences_button.clicked.connect(self.open_preferences) + def _build_copy_render_settings_action(self) -> QAction: + """Build the export action, reachable while Render is disabled (issue #170). + + Qt drops both a disabled widget's own shortcuts and the `ContextMenu` + events it would otherwise turn into `customContextMenuRequested` + (verified offscreen, with the Render button enabled and disabled) -- + so a `QAction` or a context-menu route that lives *on* the Render + button goes unreachable exactly when this command is most useful: + while a model download or a job is running and the button is + disabled. + + The fix has two parts. The shortcut is a `WindowShortcut`-context + action that `MainWindow` registers on the window itself, not the + button, so its availability never depends on the button's enabled + state. The right-click route is delivered through an event filter + installed on the button (`eventFilter` below): Qt still calls an + installed filter for a disabled widget's events, even though it + never calls the widget's own `contextMenuEvent()` or emits its + `customContextMenuRequested` -- also verified offscreen. + """ + action = QAction( + QCoreApplication.translate("ActionShelf", "Copy render settings"), self + ) + action.setObjectName("copy_render_settings_action") + action.setShortcut(QKeySequence("Ctrl+Shift+C")) + action.setShortcutContext(Qt.ShortcutContext.WindowShortcut) + self.render_button.installEventFilter(self) + return action + + def _render_context_menu(self) -> QMenu: + """Return the Render button's popup menu, built once and reused. + + Kept as one instance (issue #170 review) rather than a fresh `QMenu` + per right-click. Still a method, not inlined, so tests can inspect + the menu without triggering a real popup -- `QMenu.exec` blocks + until the menu closes, which a headless test has no way to do. + """ + if self._render_menu is None: + menu = QMenu(self.render_button) + menu.setToolTipsVisible(True) + menu.addAction(self.copy_render_settings_action) + self._render_menu = menu + return self._render_menu + + def eventFilter(self, watched: QObject, event: QEvent) -> bool: + if watched is self.render_button and isinstance(event, QContextMenuEvent): + self._render_context_menu().exec(event.globalPos()) + return True + return super().eventFilter(watched, event) + def _build_preferences_dialog( self, store: StateStore, diff --git a/src/matteloop/ui/controller.py b/src/matteloop/ui/controller.py index 724b3b4..0b93660 100644 --- a/src/matteloop/ui/controller.py +++ b/src/matteloop/ui/controller.py @@ -55,6 +55,7 @@ from matteloop.ui.model_manager import ModelManagerController, ModelRemovalService from matteloop.ui.ports import ( ChooseVideoRequested, + CopyRenderSettingsRequested, ManageModelsRequested, ManageWorkspacesRequested, OpenOutputFolderRequested, @@ -70,6 +71,7 @@ from matteloop.ui.preferences import persist_parameters from matteloop.ui.preview_controller import PreviewController, PreviewRuntime from matteloop.ui.render_controller import RenderController +from matteloop.ui.render_settings_export import RenderSettingsExportController from matteloop.ui.timeline import SourceFrameWorker from matteloop.ui.transform_group import TransformGroup from matteloop.ui.transform_stage import TransformStageController @@ -188,6 +190,9 @@ def __init__( dialog_parent=dialog_parent, parent=self, ) + self._render_settings_export = self._build_render_settings_export( + store, dialog_parent + ) self._model_manager = self._build_model_manager() self._transform_stage = TransformStageController(store, parent=self) self._render_controller.artifact_ready.connect( @@ -210,18 +215,33 @@ def __init__( self._decode_request_ids = count(1) self._threads: dict[str, tuple[QThread, _SourceLoadWorker]] = {} self._frame_threads: list[tuple[QThread, SourceFrameWorker]] = [] - self._frame_timer = QTimer(self) - self._frame_timer.setSingleShot(True) - self._frame_timer.setInterval(125) - self._frame_timer.timeout.connect(self._decode_current_playhead) + self._frame_timer = self._build_frame_timer() self._closed = False self._shutdown_complete = True + def _build_render_settings_export( + self, store: StateStore, dialog_parent: QWidget | None + ) -> RenderSettingsExportController: + return RenderSettingsExportController( + store, dialog_parent=dialog_parent, parent=self + ) + + def _build_frame_timer(self) -> QTimer: + timer = QTimer(self) + timer.setSingleShot(True) + timer.setInterval(125) + timer.timeout.connect(self._decode_current_playhead) + return timer + def set_dialog_parent(self, parent: QWidget) -> None: """Set the window used as the parent for native source dialogs.""" self._dialog_parent = parent self._preview_controller.set_dialog_parent(parent) self._render_controller.set_dialog_parent(parent) + self._render_settings_export.set_dialog_parent(parent) + self._render_settings_export.set_anchor_widget( + getattr(parent, "render_button", None) + ) self._model_manager.set_dialog_parent(parent) @property @@ -300,6 +320,8 @@ def dispatch(self, command: WindowCommand) -> None: self._preview_controller.dispatch(command) elif isinstance(command, RenderVideoRequested): self._render_controller.dispatch(command) + elif isinstance(command, CopyRenderSettingsRequested): + self._render_settings_export.dispatch(command) elif isinstance(command, CropEvent): self._store.dispatch(command) elif isinstance(command, ParameterEvent): diff --git a/src/matteloop/ui/copy.py b/src/matteloop/ui/copy.py index a17b9bd..0e7f37e 100644 --- a/src/matteloop/ui/copy.py +++ b/src/matteloop/ui/copy.py @@ -452,6 +452,12 @@ def main_window_copy(value: str) -> str: "Render complete": QCoreApplication.translate("MainWindow", "Render complete"), "Open output": QCoreApplication.translate("MainWindow", "Open output"), "Open folder": QCoreApplication.translate("MainWindow", "Open folder"), + "Open a video with a valid range to copy its render settings.": ( + QCoreApplication.translate( + "MainWindow", + "Open a video with a valid range to copy its render settings.", + ) + ), } return exact.get(value, value) diff --git a/src/matteloop/ui/main_window.py b/src/matteloop/ui/main_window.py index 3e6e0a7..6e7969b 100644 --- a/src/matteloop/ui/main_window.py +++ b/src/matteloop/ui/main_window.py @@ -26,6 +26,7 @@ from matteloop.ui.main_window_render import render_source_error, render_window from matteloop.ui.ports import ( ChooseVideoRequested, + CopyRenderSettingsRequested, ManageModelsRequested, ManageWorkspacesRequested, OpenOutputFolderRequested, @@ -146,6 +147,12 @@ def _build(self) -> None: self.update_dialog = UpdateDialog(self) self.preview_button = self.action_shelf.preview_button self.render_button = self.action_shelf.render_button + self.copy_render_settings_action = self.action_shelf.copy_render_settings_action + # Registered on the window, not the Render button: a WindowShortcut + # action added to a disabled widget is itself unreachable, and the + # export command must stay reachable while Render is disabled (issue + # #170 review) -- verified offscreen, see action_shelf.py. + self.addAction(self.copy_render_settings_action) self.rebuild_button = self.inspector.rebuild_button runtime_message = runtime_banner_copy() self.runtime_container = QFrame() @@ -226,6 +233,9 @@ def _build(self) -> None: self.render_button.clicked.connect( lambda: self._services.dispatch(RenderVideoRequested()) ) + self.copy_render_settings_action.triggered.connect( + lambda: self._services.dispatch(CopyRenderSettingsRequested()) + ) self.rebuild_button.clicked.connect( lambda: self._services.dispatch(RebuildEditedCutsRequested()) ) diff --git a/src/matteloop/ui/main_window_render.py b/src/matteloop/ui/main_window_render.py index c851d30..04c120d 100644 --- a/src/matteloop/ui/main_window_render.py +++ b/src/matteloop/ui/main_window_render.py @@ -81,6 +81,16 @@ def _render_actions(window: MainWindow, model: PresentationModel) -> None: window.preview_button.setEnabled(model.preview_enabled) window.preview_button.setText(presented_copy(model.preview_label)) window.render_button.setEnabled(model.render_enabled) + window.copy_render_settings_action.setEnabled( + model.render_settings_export_enabled + ) + window.copy_render_settings_action.setToolTip( + "" + if model.render_settings_export_enabled + else main_window_copy( + "Open a video with a valid range to copy its render settings." + ) + ) window.rebuild_button.setEnabled(model.rebuild_enabled) window.rebuild_button.setVisible(model.show_rebuild) window.edited_cut_recovery.setVisible(model.recovery_visible) diff --git a/src/matteloop/ui/ports.py b/src/matteloop/ui/ports.py index f762970..8dfb7f8 100644 --- a/src/matteloop/ui/ports.py +++ b/src/matteloop/ui/ports.py @@ -48,6 +48,11 @@ class RenderVideoRequested: pass +@dataclass(frozen=True) +class CopyRenderSettingsRequested: + pass + + @dataclass(frozen=True) class RebuildEditedCutsRequested: pass @@ -83,6 +88,7 @@ class ManageWorkspacesRequested: | VideoDropped | PreviewFrameRequested | RenderVideoRequested + | CopyRenderSettingsRequested | RebuildEditedCutsRequested | OpenOutputRequested | OpenOutputFolderRequested diff --git a/src/matteloop/ui/presentation_model.py b/src/matteloop/ui/presentation_model.py index 8e8e888..f7174a4 100644 --- a/src/matteloop/ui/presentation_model.py +++ b/src/matteloop/ui/presentation_model.py @@ -65,6 +65,7 @@ class PresentationModel: model_available: bool preview_label: str render_enabled: bool + render_settings_export_enabled: bool rebuild_enabled: bool open_output_enabled: bool open_folder_enabled: bool diff --git a/src/matteloop/ui/presenter.py b/src/matteloop/ui/presenter.py index 041ffb4..dd3c1e2 100644 --- a/src/matteloop/ui/presenter.py +++ b/src/matteloop/ui/presenter.py @@ -20,6 +20,7 @@ source_surface_copy, success_copy, ) +from matteloop.ui.request_builder import _try_export_request from matteloop.ui.source_error_copy import source_error_copy from matteloop.ui.source_presentation import present_source_metadata as s from matteloop.ui.timeline_presentation import present_timeline @@ -35,6 +36,25 @@ def _failure_message(error: object | None, retry: str) -> str: return f"Preview failed — {retry}" +def _can_export_render_settings(state: AppState) -> bool: + """Ask the shared export-request builder (issue #170). + + Shares its predicate with ``RenderSettingsExportController.dispatch`` so + "can we build a request to export" has one implementation. It does not + reuse the Render button's own builder, because that one also validates + the real output directory/filename -- settings this action never + exports and that should not be able to disable it. + """ + if state.source is not Source.READY or state.source_value is None: + return False + return ( + _try_export_request( + state.source_value, state.timeline, state.crop, state.parameters + ) + is not None + ) + + def present(state: AppState) -> PresentationModel: """Present reducer state without importing Qt, jobs, or runtime services.""" allowed = capabilities(state) @@ -138,6 +158,7 @@ def present(state: AppState) -> PresentationModel: "Download & Preview" if not state.model_available else "Preview Frame" ), render_enabled=allowed.can_render, + render_settings_export_enabled=_can_export_render_settings(state), rebuild_enabled=allowed.can_rebuild, open_output_enabled=allowed.can_open_output, open_folder_enabled=allowed.can_open_folder, diff --git a/src/matteloop/ui/render_settings_export.py b/src/matteloop/ui/render_settings_export.py new file mode 100644 index 0000000..0d81feb --- /dev/null +++ b/src/matteloop/ui/render_settings_export.py @@ -0,0 +1,72 @@ +"""GUI-thread export of the current render request as versioned JSON. + +Kept as its own module rather than a `RenderController` responsibility: +`render_controller.py` is already close to the 800-line module budget in +docs/engineering-guardrails.md G6, and this command has nothing to do with +running a job. +""" + +from __future__ import annotations + +from PySide6.QtCore import QObject, QPoint +from PySide6.QtGui import QGuiApplication +from PySide6.QtWidgets import QToolTip, QWidget + +from matteloop.core.request_json import render_settings_to_json +from matteloop.ui.ports import CopyRenderSettingsRequested, StateStore +from matteloop.ui.request_builder import _try_export_request + + +class RenderSettingsExportController(QObject): + """Copies the current render request to the clipboard as versioned JSON.""" + + def __init__( + self, + store: StateStore, + *, + dialog_parent: QWidget | None = None, + parent: QObject | None = None, + ) -> None: + super().__init__(parent) + self._store = store + self._dialog_parent = dialog_parent + self._anchor_widget: QWidget | None = None + + def set_dialog_parent(self, parent: QWidget) -> None: + """Follow the window used for anchoring the copy confirmation.""" + self._dialog_parent = parent + + def set_anchor_widget(self, widget: QWidget | None) -> None: + """Anchor the copy confirmation near the Render button, if known.""" + self._anchor_widget = widget + + def dispatch(self, command: CopyRenderSettingsRequested) -> None: + del command + state = self._store.state + if state.source_value is None: + return + request = _try_export_request( + state.source_value, state.timeline, state.crop, state.parameters + ) + if request is None: + return + text = render_settings_to_json(request) + clipboard = QGuiApplication.clipboard() + if clipboard is None: + return + clipboard.setText(text) + # Confirm only what actually happened: some platforms (headless + # clipboards, a clipboard manager intercepting the change) can leave + # the clipboard holding something other than what was just set. + if clipboard.text() != text: + return + self._show_confirmation() + + def _show_confirmation(self) -> None: + message = self.tr("Copied render settings to clipboard") + anchor = self._anchor_widget or self._dialog_parent + if anchor is not None: + position = anchor.mapToGlobal(QPoint(0, anchor.height())) + QToolTip.showText(position, message, anchor) + else: + QToolTip.showText(QPoint(0, 0), message) diff --git a/src/matteloop/ui/request_builder.py b/src/matteloop/ui/request_builder.py index 1ad5ee2..74c6763 100644 --- a/src/matteloop/ui/request_builder.py +++ b/src/matteloop/ui/request_builder.py @@ -6,6 +6,7 @@ from fractions import Fraction from pathlib import Path +from matteloop.core.errors import AppError from matteloop.core.parameters import ( ParameterState, output_directory_for_source, @@ -110,3 +111,54 @@ def _render_request( ), transform=parameters.transform, ) + + +_EXPORT_PLACEHOLDER_DIRECTORY = Path(".") +_EXPORT_PLACEHOLDER_FILENAME = "export.webp" + + +def _try_export_request( + metadata: object, + timeline: TimelineState | None, + crop: CropSpec | None, + parameters: ParameterState, +) -> RenderRequest | None: + """Build the request "Copy render settings" would export, or ``None``. + + Shared by the presenter's export-enabled check and the export command + itself (issue #170), so "can we build a request to export" has exactly + one implementation. Unlike ``_render_request``, this never resolves or + validates the *real* output directory/filename -- only + ``render_settings_to_json``'s ``output.max_bytes`` needs an ``OutputSpec`` + at all, so a placeholder stands in for the location. That keeps the + export action's availability (checked on every ``present()``) independent + of whether the user's chosen output directory happens to be valid right + now, and avoids rebuilding the full, real ``OutputSpec`` for a value that + is discarded either way. + """ + try: + inputs = _preview_inputs(metadata, timeline, crop, parameters) + return RenderRequest( + source=inputs.source, + sampling=SamplingSpec(inputs.start, inputs.end, fps=parameters.fps), + crop=inputs.crop, + segmentation=SegmentationSpec( + model_id=parameters.model_id, + edge_mode=parameters.edge_mode, + execution_provider=parameters.execution_provider, + ), + framing=FramingSpec( + trim=parameters.trim, + alpha_threshold=parameters.alpha_threshold, + padding=parameters.padding, + stretch_x=parameters.stretch_x, + ), + output=OutputSpec.from_mib( + _EXPORT_PLACEHOLDER_DIRECTORY, + _EXPORT_PLACEHOLDER_FILENAME, + parameters.max_mib, + ), + transform=parameters.transform, + ) + except (AppError, ValueError): + return None diff --git a/tests/core/test_request_json.py b/tests/core/test_request_json.py new file mode 100644 index 0000000..e10d012 --- /dev/null +++ b/tests/core/test_request_json.py @@ -0,0 +1,324 @@ +from __future__ import annotations + +import json +from decimal import Decimal +from fractions import Fraction +from pathlib import Path + +import pytest + +from matteloop.core.errors import ErrorCode, ValidationError +from matteloop.core.request_json import ( + FORMAT, + VERSION, + render_settings_from_json, + render_settings_to_json, +) +from matteloop.core.specs import ( + AlphaMattingSpec, + CollisionPolicy, + CropSpec, + EdgeMode, + FramingSpec, + MismatchMode, + OutputSpec, + RenderRequest, + ResizeSpec, + SamplingSpec, + SegmentationSpec, + TransformSpec, +) + + +def _request(tmp_path: Path, **overrides: object) -> RenderRequest: + defaults: dict[str, object] = { + "source": tmp_path / "clip.mp4", + "sampling": SamplingSpec( + start=Fraction(7, 1001), end=Fraction(1001, 30000), fps=24 + ), + "crop": CropSpec(x=4, y=8, width=640, height=360), + "segmentation": SegmentationSpec( + model_id="birefnet-general", + edge_mode=EdgeMode.ALPHA_MATTING, + alpha_matting=AlphaMattingSpec( + foreground_threshold=200, + background_threshold=20, + erode_size=5, + ), + execution_provider="CPUExecutionProvider", + ), + "framing": FramingSpec( + trim=True, + alpha_threshold=Decimal("12.5"), + padding=3, + stretch_x=Decimal("1.25"), + ), + "output": OutputSpec( + directory=tmp_path / "out", + filename="original.webp", + collision_policy=CollisionPolicy.REPLACE, + ), + "transform": TransformSpec(first_frame=2, last_frame=9), + "rebuild": True, + } + defaults.update(overrides) + return RenderRequest(**defaults) # type: ignore[arg-type] + + +def _rebuild(text: str, tmp_path: Path, **overrides: object) -> RenderRequest: + kwargs: dict[str, object] = { + "directory": tmp_path / "elsewhere", + "filename": "new.webp", + } + kwargs.update(overrides) + return render_settings_from_json(text, **kwargs) # type: ignore[arg-type] + + +def test_round_trip_preserves_sampling_crop_segmentation_and_framing( + tmp_path: Path, +) -> None: + request = _request(tmp_path) + + text = render_settings_to_json(request) + rebuilt = _rebuild(text, tmp_path) + + assert rebuilt.source == request.source + assert rebuilt.sampling == request.sampling + assert rebuilt.crop == request.crop + assert rebuilt.segmentation == request.segmentation + assert rebuilt.framing == request.framing + + +def test_round_trip_preserves_a_non_identity_transform(tmp_path: Path) -> None: + """Trim, crop, and resize all survive the round trip (issue #170 review).""" + request = _request( + tmp_path, + transform=TransformSpec( + first_frame=3, + last_frame=40, + crop=CropSpec(x=10, y=20, width=200, height=150), + resize=ResizeSpec(width=480, height=None, mismatch=MismatchMode.COVER), + ), + ) + + rebuilt = _rebuild(render_settings_to_json(request), tmp_path) + + assert rebuilt.transform == request.transform + + +def test_round_trip_preserves_a_default_identity_transform(tmp_path: Path) -> None: + request = _request(tmp_path, transform=TransformSpec()) + + rebuilt = _rebuild(render_settings_to_json(request), tmp_path) + + assert rebuilt.transform == TransformSpec() + + +def test_round_trip_preserves_the_output_size_budget(tmp_path: Path) -> None: + request = _request( + tmp_path, + output=OutputSpec( + directory=tmp_path / "out", filename="original.webp", max_bytes=5_000_000 + ), + ) + + rebuilt = _rebuild( + render_settings_to_json(request), + tmp_path, + directory=tmp_path / "chosen", + filename="chosen.webp", + ) + + assert rebuilt.output.max_bytes == 5_000_000 + assert rebuilt.output.directory == tmp_path / "chosen" + assert rebuilt.output.filename == "chosen.webp" + + +def test_round_trip_preserves_no_size_budget(tmp_path: Path) -> None: + request = _request( + tmp_path, + output=OutputSpec( + directory=tmp_path / "out", filename="original.webp", max_bytes=None + ), + ) + + rebuilt = _rebuild(render_settings_to_json(request), tmp_path) + + assert rebuilt.output.max_bytes is None + + +def test_round_trip_preserves_non_trivial_fraction_denominators( + tmp_path: Path, +) -> None: + request = _request( + tmp_path, + sampling=SamplingSpec( + start=Fraction(1, 3000), end=Fraction(1001, 30000), fps=15 + ), + ) + + rebuilt = _rebuild(render_settings_to_json(request), tmp_path) + + assert rebuilt.sampling.start == Fraction(1, 3000) + assert rebuilt.sampling.end == Fraction(1001, 30000) + assert rebuilt.sampling.start.denominator == 3000 + assert rebuilt.sampling.end.denominator == 30000 + + +def test_serialised_fractions_are_exact_numerator_denominator_pairs( + tmp_path: Path, +) -> None: + request = _request( + tmp_path, sampling=SamplingSpec(start=Fraction(0), end=Fraction(1001, 30000)) + ) + + payload = json.loads(render_settings_to_json(request)) + + assert payload["sampling"]["end"] == {"numerator": 1001, "denominator": 30000} + + +def test_output_location_and_job_flags_are_not_present_in_the_json( + tmp_path: Path, +) -> None: + """The output location and rebuild/regenerate flags are the caller's call; + transform and the size budget are settings, so they stay in the JSON.""" + request = _request(tmp_path) + + payload = json.loads(render_settings_to_json(request)) + + assert "directory" not in payload["output"] + assert "filename" not in payload["output"] + assert "collision_policy" not in payload["output"] + assert "rebuild" not in payload + assert "regenerate" not in payload + assert "transform" in payload + assert "max_bytes" in payload["output"] + + +def test_rebuilt_request_uses_the_given_output_location_and_flags( + tmp_path: Path, +) -> None: + request = _request(tmp_path) + + rebuilt = _rebuild( + render_settings_to_json(request), + tmp_path, + directory=tmp_path / "chosen", + filename="chosen.webp", + collision_policy=CollisionPolicy.REPLACE, + ) + + assert rebuilt.output.directory == tmp_path / "chosen" + assert rebuilt.output.filename == "chosen.webp" + assert rebuilt.output.collision_policy is CollisionPolicy.REPLACE + assert rebuilt.rebuild is False + assert rebuilt.regenerate is False + + +def test_envelope_carries_the_stable_format_and_version(tmp_path: Path) -> None: + request = _request(tmp_path) + + payload = json.loads(render_settings_to_json(request)) + + assert payload["format"] == FORMAT == "matteloop.render-settings" + assert payload["version"] == VERSION == 1 + + +def test_rejects_wrong_format(tmp_path: Path) -> None: + text = json.dumps({"format": "something-else", "version": 1}) + + with pytest.raises(ValidationError) as excinfo: + _rebuild(text, tmp_path) + + assert excinfo.value.code is ErrorCode.RENDER_SETTINGS_INVALID + + +def test_rejects_unsupported_version(tmp_path: Path) -> None: + request = _request(tmp_path) + payload = json.loads(render_settings_to_json(request)) + payload["version"] = 2 + + with pytest.raises(ValidationError) as excinfo: + _rebuild(json.dumps(payload), tmp_path) + + assert excinfo.value.code is ErrorCode.RENDER_SETTINGS_INVALID + + +def test_rejects_malformed_json(tmp_path: Path) -> None: + with pytest.raises(ValidationError) as excinfo: + _rebuild("{not json", tmp_path) + + assert excinfo.value.code is ErrorCode.RENDER_SETTINGS_INVALID + + +def test_rejects_a_json_array_envelope(tmp_path: Path) -> None: + with pytest.raises(ValidationError): + _rebuild("[]", tmp_path) + + +def test_rejects_missing_required_fields(tmp_path: Path) -> None: + text = json.dumps({"format": FORMAT, "version": VERSION}) + + with pytest.raises(ValidationError) as excinfo: + _rebuild(text, tmp_path) + + assert excinfo.value.code is ErrorCode.RENDER_SETTINGS_INVALID + + +def test_rejects_a_missing_transform(tmp_path: Path) -> None: + request = _request(tmp_path) + payload = json.loads(render_settings_to_json(request)) + del payload["transform"] + + with pytest.raises(ValidationError) as excinfo: + _rebuild(json.dumps(payload), tmp_path) + + assert excinfo.value.code is ErrorCode.RENDER_SETTINGS_INVALID + + +def test_rejects_a_missing_max_bytes(tmp_path: Path) -> None: + request = _request(tmp_path) + payload = json.loads(render_settings_to_json(request)) + del payload["output"]["max_bytes"] + + with pytest.raises(ValidationError) as excinfo: + _rebuild(json.dumps(payload), tmp_path) + + assert excinfo.value.code is ErrorCode.RENDER_SETTINGS_INVALID + + +def test_rejects_a_zero_denominator_fraction(tmp_path: Path) -> None: + request = _request(tmp_path) + payload = json.loads(render_settings_to_json(request)) + payload["sampling"]["start"] = {"numerator": 0, "denominator": 0} + + with pytest.raises(ValidationError) as excinfo: + _rebuild(json.dumps(payload), tmp_path) + + assert excinfo.value.code is ErrorCode.RENDER_SETTINGS_INVALID + + +def test_domain_validation_still_applies_to_rebuilt_specs(tmp_path: Path) -> None: + """An in-envelope but domain-invalid crop still raises, via CropSpec itself.""" + request = _request(tmp_path) + payload = json.loads(render_settings_to_json(request)) + payload["crop"]["width"] = -1 + + with pytest.raises(ValidationError) as excinfo: + _rebuild(json.dumps(payload), tmp_path) + + assert excinfo.value.code is ErrorCode.INVALID_CROP + + +def test_domain_validation_still_applies_to_the_rebuilt_transform( + tmp_path: Path, +) -> None: + request = _request(tmp_path) + payload = json.loads(render_settings_to_json(request)) + payload["transform"]["last_frame"] = 0 + payload["transform"]["first_frame"] = 5 + + with pytest.raises(ValidationError) as excinfo: + _rebuild(json.dumps(payload), tmp_path) + + assert excinfo.value.code is ErrorCode.INVALID_TRANSFORM diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 0000000..c54774f --- /dev/null +++ b/tests/scripts/__init__.py @@ -0,0 +1 @@ +"""Developer-tooling script tests (`scripts/`).""" diff --git a/tests/scripts/test_benchmark_models.py b/tests/scripts/test_benchmark_models.py new file mode 100644 index 0000000..f0613cd --- /dev/null +++ b/tests/scripts/test_benchmark_models.py @@ -0,0 +1,485 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from fractions import Fraction +from pathlib import Path + +import pytest + +from matteloop.core.errors import ValidationError +from matteloop.core.execution_providers import CPU_EXECUTION_PROVIDER +from matteloop.core.parameters import V1_MODEL_IDS +from matteloop.core.request_json import render_settings_to_json +from matteloop.core.specs import ( + CropSpec, + EdgeMode, + FramingSpec, + OutputSpec, + RenderRequest, + SamplingSpec, + SegmentationSpec, +) +from matteloop.jobs.models.catalog import ModelCatalog +from matteloop.jobs.render import RenderArtifact +from scripts.benchmark_models import ( + BenchmarkError, + _print_license_note_if_selected, + build_base_request, + load_request_text, + resolve_provider, + run_benchmark, + select_models, +) + + +class _StubClipboard: + def __init__(self, text: str) -> None: + self._text = text + + def text(self) -> str: + return self._text + + +class _FakeRuntime: + """A `BenchmarkRuntime` double with no network, model, or Qt dependency.""" + + def __init__( + self, + out_dir: Path, + *, + fail_models: frozenset[str] = frozenset(), + active_provider: str = CPU_EXECUTION_PROVIDER, + fallback_notice: str | None = None, + ): + self._out_dir = out_dir + self._fail_models = fail_models + self.active_provider = active_provider + self.fallback_notice = fallback_notice + self.prepared: list[str] = [] + self.rendered: list[str] = [] + self.rendered_requests: list[RenderRequest] = [] + + def prepare(self, model_id, extras, context): # noqa: ANN001, ANN201 + del extras, context + self.prepared.append(model_id) + return None + + def render(self, request, context): # noqa: ANN001, ANN201 + del context + model_id = request.segmentation.model_id + self.rendered.append(model_id) + self.rendered_requests.append(request) + if model_id in self._fail_models: + raise RuntimeError(f"boom: {model_id}") + return RenderArtifact( + output_path=self._out_dir / f"{model_id}.webp", + fingerprint="fake-fingerprint", + cut_workspace=None, # type: ignore[arg-type] + manifest=None, # type: ignore[arg-type] + frame_count=3, + width=64, + height=48, + file_size=1024, + duration_ms=200, + requested_timestamps=(), + actual_pts=None, + delays_ms=(), + ownership_peak=0, + ownership_current=0, + rebuilt=False, + ) + + def close(self) -> None: + pass + + +def _request(tmp_path: Path) -> RenderRequest: + return RenderRequest( + source=tmp_path / "clip.mp4", + sampling=SamplingSpec(start=Fraction(0), end=Fraction(1), fps=15), + crop=CropSpec(0, 0, 64, 48), + segmentation=SegmentationSpec( + model_id="birefnet-portrait", edge_mode=EdgeMode.STANDARD + ), + framing=FramingSpec(), + output=OutputSpec(tmp_path / "out", "out.webp"), + ) + + +def _request_text(tmp_path: Path) -> str: + return render_settings_to_json(_request(tmp_path)) + + +def _base_request(tmp_path: Path, out_dir: Path) -> tuple[RenderRequest, dict]: + text = _request_text(tmp_path) + return build_base_request(text, out_dir), json.loads(text) + + +def test_default_model_selection_is_the_apps_own_v1_set() -> None: + catalog = ModelCatalog.load_resource() + + selected = select_models(catalog, None) + + assert selected == V1_MODEL_IDS + assert "u2net_cloth_seg" not in selected + assert "bria-rmbg" not in selected + assert len(selected) == 13 + + +def test_unknown_model_id_is_rejected_with_the_valid_ids_listed() -> None: + catalog = ModelCatalog.load_resource() + + with pytest.raises(BenchmarkError) as excinfo: + select_models(catalog, "birefnet-general,not-a-model") + + message = str(excinfo.value) + unknown_part, _, valid_part = message.partition(";") + assert "not-a-model" in unknown_part + assert "birefnet-general" not in unknown_part + for model_id in catalog.ids: + assert model_id in valid_part + + +def test_explicit_models_may_include_the_default_exclusions() -> None: + catalog = ModelCatalog.load_resource() + + selected = select_models(catalog, "u2net_cloth_seg,bria-rmbg") + + assert selected == ("u2net_cloth_seg", "bria-rmbg") + + +def test_empty_models_argument_is_rejected() -> None: + catalog = ModelCatalog.load_resource() + + with pytest.raises(BenchmarkError): + select_models(catalog, " ") + + +def test_ensure_qt_application_guarantees_an_instance_without_network() -> None: + """Issue #170 review, item 1: main()'s first act must guarantee a Qt + application exists before the runtime (and its download transport) is + built, regardless of whether the request came from --request or the + clipboard. + + Run in a subprocess, not in-process: this repository's own test session + shares one Qt application across every test file, and creating a bare + ``QGuiApplication`` here (instead of the ``QApplication`` the widget + tests need) would leave that singleton behind for every test that runs + afterwards -- which is exactly the kind of cross-test pollution that + made later UI tests in this file hang when this test was first added + in-process. + """ + script = ( + "from PySide6.QtCore import QCoreApplication\n" + "from scripts.benchmark_models import _ensure_qt_application\n" + "app = _ensure_qt_application()\n" + "assert QCoreApplication.instance() is app\n" + "print('ok')\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + check=False, + env=os.environ | {"QT_QPA_PLATFORM": "offscreen"}, + text=True, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "ok" + + +def test_print_license_note_when_bria_rmbg_is_selected() -> None: + catalog = ModelCatalog.load_resource() + logged: list[str] = [] + + _print_license_note_if_selected(catalog, ("u2net", "bria-rmbg"), log=logged.append) + + assert len(logged) == 1 + assert catalog.get("bria-rmbg").license_note in logged[0] + + +def test_no_license_note_when_bria_rmbg_is_not_selected() -> None: + catalog = ModelCatalog.load_resource() + logged: list[str] = [] + + _print_license_note_if_selected(catalog, ("u2net", "u2netp"), log=logged.append) + + assert logged == [] + + +def test_resolve_provider_rejects_an_unknown_provider() -> None: + with pytest.raises(BenchmarkError) as excinfo: + resolve_provider("TotallyMadeUpExecutionProvider") + + assert "TotallyMadeUpExecutionProvider" in str(excinfo.value) + + +def test_resolve_provider_accepts_an_allowlisted_provider() -> None: + assert resolve_provider(CPU_EXECUTION_PROVIDER) == CPU_EXECUTION_PROVIDER + + +def test_load_request_text_reads_a_file(tmp_path: Path) -> None: + path = tmp_path / "settings.json" + path.write_text("stand-in text", encoding="utf-8") + + assert load_request_text(path, _StubClipboard("unused")) == "stand-in text" + + +def test_load_request_text_falls_back_to_the_clipboard(tmp_path: Path) -> None: + del tmp_path + text = load_request_text(None, _StubClipboard("clipboard payload")) + + assert text == "clipboard payload" + + +def test_load_request_text_rejects_an_empty_clipboard() -> None: + with pytest.raises(BenchmarkError): + load_request_text(None, _StubClipboard(" ")) + + +def test_load_request_text_reports_a_missing_file_clearly(tmp_path: Path) -> None: + missing = tmp_path / "does-not-exist.json" + + with pytest.raises(BenchmarkError) as excinfo: + load_request_text(missing, _StubClipboard("unused")) + + assert str(missing) in str(excinfo.value) + + +def test_build_base_request_rejects_a_malformed_request_with_a_clear_error( + tmp_path: Path, +) -> None: + with pytest.raises(ValidationError) as excinfo: + build_base_request("{not json", tmp_path / "out") + + assert "JSON" in excinfo.value.technical_detail + + +def test_run_benchmark_rejects_empty_model_ids(tmp_path: Path) -> None: + out_dir = tmp_path / "out" + base_request, request_settings = _base_request(tmp_path, out_dir) + + with pytest.raises(BenchmarkError): + run_benchmark( + base_request=base_request, + request_settings=request_settings, + model_ids=(), + provider=CPU_EXECUTION_PROVIDER, + out_dir=out_dir, + catalog=ModelCatalog.load_resource(), + runtime=_FakeRuntime(out_dir), + log=lambda _message: None, + ) + + +def test_run_benchmark_records_a_failing_model_and_keeps_going( + tmp_path: Path, +) -> None: + out_dir = tmp_path / "out" + runtime = _FakeRuntime(out_dir, fail_models=frozenset({"u2net"})) + base_request, request_settings = _base_request(tmp_path, out_dir) + + run = run_benchmark( + base_request=base_request, + request_settings=request_settings, + model_ids=("u2net", "u2netp"), + provider=CPU_EXECUTION_PROVIDER, + out_dir=out_dir, + catalog=ModelCatalog.load_resource(), + runtime=runtime, + log=lambda _message: None, + ) + + by_id = {result.model_id: result for result in run.results} + assert runtime.rendered == ["u2net", "u2netp"] + assert by_id["u2net"].status == "error" + assert "boom: u2net" in (by_id["u2net"].error or "") + assert by_id["u2net"].file_size_bytes is None + assert by_id["u2netp"].status == "ok" + assert by_id["u2netp"].file_size_bytes == 1024 + assert by_id["u2netp"].frame_count == 3 + assert by_id["u2netp"].duration_ms == 200 + assert by_id["u2netp"].output_relative_path == "u2netp.webp" + assert by_id["u2netp"].active_provider == CPU_EXECUTION_PROVIDER + + +def test_run_benchmark_writes_results_after_every_model_not_only_at_the_end( + tmp_path: Path, +) -> None: + """Issue #170 review, item 8: a Ctrl+C partway through must not lose + tiles already rendered, so results.json/index.html are written + incrementally rather than only once at the end of the loop.""" + out_dir = tmp_path / "out" + base_request, request_settings = _base_request(tmp_path, out_dir) + written_after_first: list[int] = [] + + class _RecordingRuntime(_FakeRuntime): + def render(self, request, context): # noqa: ANN001, ANN201 + artifact = super().render(request, context) + results_path = out_dir / "results.json" + if results_path.exists(): + written_after_first.append( + len(json.loads(results_path.read_text())["models"]) + ) + else: + written_after_first.append(0) + return artifact + + runtime = _RecordingRuntime(out_dir) + + run_benchmark( + base_request=base_request, + request_settings=request_settings, + model_ids=("u2net", "u2netp"), + provider=CPU_EXECUTION_PROVIDER, + out_dir=out_dir, + catalog=ModelCatalog.load_resource(), + runtime=runtime, + log=lambda _message: None, + ) + + # Before the second model rendered, the first model's result was + # already on disk. + assert written_after_first == [0, 1] + assert (out_dir / "results.json").exists() + assert (out_dir / "index.html").exists() + + +def test_run_benchmark_carries_the_output_size_budget_to_every_model( + tmp_path: Path, +) -> None: + out_dir = tmp_path / "out" + request = RenderRequest( + source=tmp_path / "clip.mp4", + sampling=SamplingSpec(start=Fraction(0), end=Fraction(1), fps=15), + crop=CropSpec(0, 0, 64, 48), + segmentation=SegmentationSpec( + model_id="birefnet-portrait", edge_mode=EdgeMode.STANDARD + ), + framing=FramingSpec(), + output=OutputSpec(tmp_path / "out", "out.webp", max_bytes=5_000_000), + ) + text = render_settings_to_json(request) + base_request = build_base_request(text, out_dir) + runtime = _FakeRuntime(out_dir) + + run_benchmark( + base_request=base_request, + request_settings=json.loads(text), + model_ids=("u2net", "u2netp"), + provider=CPU_EXECUTION_PROVIDER, + out_dir=out_dir, + catalog=ModelCatalog.load_resource(), + runtime=runtime, + log=lambda _message: None, + ) + + assert [r.output.max_bytes for r in runtime.rendered_requests] == [ + 5_000_000, + 5_000_000, + ] + + +def test_run_benchmark_applies_the_same_settings_to_every_model_except_selection( + tmp_path: Path, +) -> None: + """Only model id, provider, and output may vary between models; every + other setting (sampling, crop, framing, edge mode, transform) must be + identical, and the requested provider must actually be applied.""" + out_dir = tmp_path / "out" + base_request, request_settings = _base_request(tmp_path, out_dir) + runtime = _FakeRuntime(out_dir) + + run_benchmark( + base_request=base_request, + request_settings=request_settings, + model_ids=("u2net", "u2netp", "silueta"), + provider=CPU_EXECUTION_PROVIDER, + out_dir=out_dir, + catalog=ModelCatalog.load_resource(), + runtime=runtime, + log=lambda _message: None, + ) + + requests = runtime.rendered_requests + assert len(requests) == 3 + for request in requests: + assert request.sampling == base_request.sampling + assert request.crop == base_request.crop + assert request.framing == base_request.framing + assert request.transform == base_request.transform + assert request.segmentation.edge_mode == base_request.segmentation.edge_mode + assert request.segmentation.execution_provider == CPU_EXECUTION_PROVIDER + assert request.output.max_bytes == base_request.output.max_bytes + assert [r.segmentation.model_id for r in requests] == [ + "u2net", + "u2netp", + "silueta", + ] + assert [r.output.filename for r in requests] == [ + "u2net.webp", + "u2netp.webp", + "silueta.webp", + ] + + +def test_run_benchmark_flags_a_frame_count_mismatch(tmp_path: Path) -> None: + out_dir = tmp_path / "out" + base_request, request_settings = _base_request(tmp_path, out_dir) + + class _MismatchedRuntime(_FakeRuntime): + def render(self, request, context): # noqa: ANN001, ANN201 + artifact = super().render(request, context) + if request.segmentation.model_id == "u2netp": + from dataclasses import replace + + artifact = replace(artifact, frame_count=99) + return artifact + + run = run_benchmark( + base_request=base_request, + request_settings=request_settings, + model_ids=("u2net", "u2netp"), + provider=CPU_EXECUTION_PROVIDER, + out_dir=out_dir, + catalog=ModelCatalog.load_resource(), + runtime=_MismatchedRuntime(out_dir), + log=lambda _message: None, + ) + + assert any("frame_count differs" in warning for warning in run.warnings) + + +def test_results_json_has_the_documented_shape(tmp_path: Path) -> None: + out_dir = tmp_path / "out" + base_request, request_settings = _base_request(tmp_path, out_dir) + runtime = _FakeRuntime(out_dir) + + run_benchmark( + base_request=base_request, + request_settings=request_settings, + model_ids=("u2net",), + provider=CPU_EXECUTION_PROVIDER, + out_dir=out_dir, + catalog=ModelCatalog.load_resource(), + runtime=runtime, + log=lambda _message: None, + ) + + payload = json.loads((out_dir / "results.json").read_text(encoding="utf-8")) + + assert payload["provider"] == CPU_EXECUTION_PROVIDER + assert "regenerate" not in payload + assert payload["request"]["segmentation"]["model_id"] == "birefnet-portrait" + assert payload["warnings"] == [] + assert len(payload["models"]) == 1 + model_payload = payload["models"][0] + assert model_payload["model_id"] == "u2net" + assert model_payload["status"] == "ok" + assert model_payload["file_size_bytes"] == 1024 + assert model_payload["active_provider"] == CPU_EXECUTION_PROVIDER diff --git a/tests/scripts/test_benchmark_report.py b/tests/scripts/test_benchmark_report.py new file mode 100644 index 0000000..cf00c02 --- /dev/null +++ b/tests/scripts/test_benchmark_report.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from scripts.benchmark_models import BenchmarkRun, ModelResult +from scripts.benchmark_report import render_report_html + +_MALICIOUS_ERROR = '' + + +def _run(*results: ModelResult, warnings: tuple[str, ...] = ()) -> BenchmarkRun: + return BenchmarkRun( + generated_at="2026-09-17T12:00:00+00:00", + provider="CPUExecutionProvider", + request_settings={"source": "clip.mp4"}, + results=results, + warnings=warnings, + ) + + +def _ok_result(model_id: str, **overrides: object) -> ModelResult: + defaults: dict[str, object] = dict( + model_id=model_id, + display_name=model_id.replace("-", " ").title(), + status="ok", + weight_size_bytes=176 * 1024 * 1024, + prepare_seconds=4.2, + render_seconds=11.8, + cache_hit=False, + active_provider="CPUExecutionProvider", + fallback_notice=None, + file_size_bytes=612_000, + frame_count=30, + duration_ms=2000, + output_relative_path=f"{model_id}.webp", + ) + defaults.update(overrides) + return ModelResult(**defaults) # type: ignore[arg-type] + + +def test_report_contains_one_tile_per_model() -> None: + run = _run(_ok_result("birefnet-general"), _ok_result("birefnet-portrait")) + + html = render_report_html(run) + + assert html.count('class="tile') == 2 + assert 'data-model="birefnet-general"' in html + assert 'data-model="birefnet-portrait"' in html + + +def test_report_references_relative_webp_paths() -> None: + run = _run(_ok_result("isnet-anime")) + + html = render_report_html(run) + + assert 'data-src="isnet-anime.webp"' in html + # Never an absolute filesystem path or a path outside the output directory. + assert "C:\\" not in html + assert "/isnet-anime.webp" not in html + + +def test_report_puts_the_image_src_in_markup_so_it_loads_without_js() -> None: + run = _run(_ok_result("isnet-anime")) + + html = render_report_html(run) + + assert 'src="isnet-anime.webp" data-src="isnet-anime.webp"' in html + + +def test_report_shows_the_duration() -> None: + run = _run(_ok_result("silueta", duration_ms=1234)) + + html = render_report_html(run) + + assert "1234 ms" in html + + +def test_report_shows_the_active_provider_and_fallback_notice() -> None: + run = _run( + _ok_result( + "silueta", + active_provider="CPUExecutionProvider", + fallback_notice="CUDA unavailable; using CPU", + ) + ) + + html = render_report_html(run) + + assert "CPUExecutionProvider" in html + assert "CUDA unavailable; using CPU" in html + + +def test_report_flags_a_cache_hit_as_possibly_edited() -> None: + run = _run(_ok_result("silueta", cache_hit=True)) + + html = render_report_html(run) + + assert "cache-hit" in html + assert "may be edited" in html + + +def test_report_shows_run_level_warnings() -> None: + run = _run( + _ok_result("silueta"), + warnings=("frame_count differs across models: silueta=30, u2net=29",), + ) + + html = render_report_html(run) + + assert "frame_count differs across models" in html + + +def test_report_omits_the_warnings_block_when_there_are_none() -> None: + run = _run(_ok_result("silueta")) + + html = render_report_html(run) + + assert '
    None: + failed = ModelResult( + model_id="u2net", + display_name="U2Net", + status="error", + error=_MALICIOUS_ERROR, + ) + + html = render_report_html(_run(failed)) + + assert _MALICIOUS_ERROR not in html + assert "<script>" in html + assert 'data-model="u2net"' in html + + +def test_report_has_a_background_toggle_and_a_restart_button() -> None: + html = render_report_html(_run(_ok_result("silueta"))) + + assert 'id="bg-select"' in html + assert 'id="restart-all"' in html + for background in ("checkerboard", "black", "white", "green"): + assert f'value="{background}"' in html + + +def test_report_is_self_contained_with_no_external_resources() -> None: + html = render_report_html(_run(_ok_result("bria-rmbg"))) + + assert "http://" not in html + assert "https://" not in html + assert " QSettings: + settings = QSettings( + QSettings.IniFormat, QSettings.UserScope, "matteloop-test", name + ) + settings.clear() + return settings + + +def _ready_store(path: Path) -> ReducerStore: + loading = reduce(AppState(), SourceLoadRequested("source", "load")) + return ReducerStore( + reduce(loading, SourceLoaded("source", "load", _Metadata(path))) + ) + + +def _window( + qtbot, store: ReducerStore, name: str +) -> tuple[MainWindow, SourceController]: + controller = SourceController(store) + window = MainWindow(store, controller, _settings(name)) + qtbot.addWidget(window) + window.show() + return window, controller + + +_DISABLED_REASON = "Open a video with a valid range to copy its render settings." + + +def test_copy_render_settings_action_is_disabled_without_a_source(qtbot) -> None: + store = ReducerStore() + window, _controller = _window(qtbot, store, "export-no-source") + + assert window.copy_render_settings_action.isEnabled() is False + assert window.copy_render_settings_action.toolTip() == _DISABLED_REASON + + +def test_copy_render_settings_action_is_enabled_with_a_loaded_source( + qtbot, tmp_path: Path +) -> None: + source = tmp_path / "clip.mp4" + source.write_bytes(b"not a real video, just a path for the request builder") + store = _ready_store(source) + window, _controller = _window(qtbot, store, "export-ready") + + assert window.copy_render_settings_action.isEnabled() is True + assert window.copy_render_settings_action.toolTip() != _DISABLED_REASON + + +def test_render_button_exposes_a_context_menu_with_the_copy_action( + qtbot, tmp_path: Path +) -> None: + source = tmp_path / "clip.mp4" + source.write_bytes(b"stand-in bytes") + store = _ready_store(source) + window, _controller = _window(qtbot, store, "export-menu") + + menu = window.action_shelf._render_context_menu() + try: + assert window.copy_render_settings_action in menu.actions() + assert menu.toolTipsVisible() is True + finally: + menu.close() + + +def test_triggering_the_action_copies_json_that_decodes_back_to_the_same_settings( + qtbot, tmp_path: Path +) -> None: + source = tmp_path / "clip.mp4" + source.write_bytes(b"stand-in bytes") + store = _ready_store(source) + window, controller = _window(qtbot, store, "export-copy") + + controller.dispatch(CopyRenderSettingsRequested()) + + clipboard = QGuiApplication.clipboard() + assert clipboard is not None + text = clipboard.text() + rebuilt = render_settings_from_json(text, directory=tmp_path, filename="out.webp") + + assert rebuilt.source == source + assert rebuilt.sampling.start == Fraction(0) + assert rebuilt.sampling.end == Fraction(2) + assert rebuilt.sampling.fps == 15 + assert rebuilt.crop == CropSpec(0, 0, 128, 64) + assert rebuilt.segmentation == SegmentationSpec( + model_id="birefnet-portrait", + edge_mode=EdgeMode.STANDARD, + execution_provider=CPU_EXECUTION_PROVIDER, + ) + + +def test_triggering_the_action_without_a_source_does_not_touch_the_clipboard( + qtbot, +) -> None: + store = ReducerStore() + _unused_window, controller = _window(qtbot, store, "export-noop") + clipboard = QGuiApplication.clipboard() + assert clipboard is not None + clipboard.setText("sentinel") + + controller.dispatch(CopyRenderSettingsRequested()) + + assert clipboard.text() == "sentinel" + + +def _disabled_render_store(source: Path) -> ReducerStore: + """A ready source whose Render button is disabled (model not available).""" + return ReducerStore(replace(_ready_store(source).state, model_available=False)) + + +def test_render_button_is_disabled_but_the_export_action_stays_enabled( + qtbot, tmp_path: Path +) -> None: + source = tmp_path / "clip.mp4" + source.write_bytes(b"stand-in bytes") + store = _disabled_render_store(source) + window, _controller = _window(qtbot, store, "export-render-disabled") + + assert window.render_button.isEnabled() is False + assert window.copy_render_settings_action.isEnabled() is True + + +def test_shortcut_triggers_the_export_while_render_is_disabled( + qtbot, tmp_path: Path +) -> None: + """Issue #170 review: a disabled Render button must not swallow the + export shortcut. The action lives on the window (WindowShortcut), not + on the button, precisely so this keeps working.""" + source = tmp_path / "clip.mp4" + source.write_bytes(b"stand-in bytes") + store = _disabled_render_store(source) + window, _controller = _window(qtbot, store, "export-shortcut-disabled") + window.activateWindow() + QTest.qWaitForWindowExposed(window) + assert window.render_button.isEnabled() is False + clipboard = QGuiApplication.clipboard() + assert clipboard is not None + clipboard.setText("sentinel") + + QTest.keyClick( + window, + Qt.Key.Key_C, + Qt.KeyboardModifier.ControlModifier | Qt.KeyboardModifier.ShiftModifier, + ) + QApplication.processEvents() + + assert clipboard.text() != "sentinel" + assert json.loads(clipboard.text())["source"] == str(source) + + +class _StubMenu: + """A duck-typed popup that records `.exec()` without blocking. + + `QMenu.exec()` opens a real nested event loop that never returns + without a selection or dismissal, and monkeypatching `QMenu.exec` at + the class level does not reach PySide6's C++-backed method dispatch -- + so the stub replaces `_render_context_menu()`'s return value instead of + trying to intercept the real popup. + """ + + def __init__(self) -> None: + self.shown: list[QPoint] = [] + + def exec(self, pos: QPoint) -> None: + self.shown.append(pos) + + +def test_right_click_context_menu_reaches_the_action_while_render_is_disabled( + qtbot, tmp_path: Path, monkeypatch +) -> None: + """Issue #170 review: Qt drops a disabled widget's own ContextMenu event + and customContextMenuRequested signal, so the right-click route has to + go through an event filter instead -- verified here by feeding the + button's installed filter the same event Qt would deliver.""" + source = tmp_path / "clip.mp4" + source.write_bytes(b"stand-in bytes") + store = _disabled_render_store(source) + window, _controller = _window(qtbot, store, "export-context-menu-disabled") + assert window.render_button.isEnabled() is False + + stub = _StubMenu() + monkeypatch.setattr(window.action_shelf, "_render_context_menu", lambda: stub) + + event = QContextMenuEvent( + QContextMenuEvent.Reason.Mouse, QPoint(5, 5), QPoint(100, 100) + ) + handled = window.action_shelf.eventFilter(window.render_button, event) + + assert handled is True + assert stub.shown == [QPoint(100, 100)] + + +def test_confirmation_is_anchored_to_the_given_widget_not_the_cursor( + qtbot, tmp_path: Path, monkeypatch +) -> None: + """Issue #170 review, item 12: position the confirmation near the + Render button/window, not wherever the cursor happens to be.""" + source = tmp_path / "clip.mp4" + source.write_bytes(b"stand-in bytes") + store = _ready_store(source) + anchor = QWidget() + anchor.resize(80, 40) + qtbot.addWidget(anchor) + anchor.show() + controller = RenderSettingsExportController(store) + controller.set_anchor_widget(anchor) + + shown_positions: list[QPoint] = [] + monkeypatch.setattr( + "matteloop.ui.render_settings_export.QToolTip.showText", + staticmethod(lambda pos, *_args, **_kwargs: shown_positions.append(pos)), + ) + + controller.dispatch(CopyRenderSettingsRequested()) + + assert shown_positions == [anchor.mapToGlobal(QPoint(0, anchor.height()))] + + +def test_no_confirmation_when_the_clipboard_does_not_actually_match( + qtbot, tmp_path: Path, monkeypatch +) -> None: + """Issue #170 review, item 12: only confirm what actually happened -- + not every platform's clipboard reliably holds what was just set.""" + source = tmp_path / "clip.mp4" + source.write_bytes(b"stand-in bytes") + store = _ready_store(source) + controller = RenderSettingsExportController(store) + + class _LossyClipboard: + def setText(self, _text: str) -> None: + pass + + def text(self) -> str: + return "something else entirely" + + monkeypatch.setattr( + QGuiApplication, "clipboard", staticmethod(lambda: _LossyClipboard()) + ) + shown: list[object] = [] + monkeypatch.setattr( + "matteloop.ui.render_settings_export.QToolTip.showText", + staticmethod(lambda *args, **kwargs: shown.append(args)), + ) + + controller.dispatch(CopyRenderSettingsRequested()) + + assert shown == [] From 3476e1d28ea23592d3f5a9b80f5eea22884a8f41 Mon Sep 17 00:00:00 2001 From: sensitron Date: Thu, 17 Sep 2026 18:51:11 +0200 Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=90=9B=20fix(benchmark):=20run=20--re?= =?UTF-8?q?quest=20without=20a=20display?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 🐛 `_ensure_qt_application` always built a `QGuiApplication`. That needs a display backend, so on a headless machine the documented `--request` path could abort before reading the file. It now builds a plain `QCoreApplication` for `--request`, which is enough for the download transport's event loop, and a `QGuiApplication` only when the settings come from the clipboard. An existing instance is reused. - 🧪 Two subprocess tests pin the application class per input source. Trigger: Codex review of #171 (P2). Both new tests failed before the fix. Refs #170 --- scripts/benchmark_models.py | 28 ++++++++------- tests/scripts/test_benchmark_models.py | 47 ++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/scripts/benchmark_models.py b/scripts/benchmark_models.py index 5df5af4..52922b8 100644 --- a/scripts/benchmark_models.py +++ b/scripts/benchmark_models.py @@ -528,27 +528,31 @@ def text(self) -> str: return clipboard.text() -def _ensure_qt_application() -> object: +def _ensure_qt_application(*, needs_clipboard: bool) -> object: """Guarantee a Qt application instance exists before any network I/O. - `QtNetworkDownloadTransport` (ui/download_transport.py) pumps a - `QEventLoop` while waiting for each downloaded chunk; without a - constructed `QCoreApplication`, that loop never dispatches, so the - first uncached model hangs forever instead of downloading (issue #170 - review, item 1). This has to run before the runtime is built, - regardless of whether the request comes from `--request` or the - clipboard -- previously only the clipboard path created one, lazily, - inside `_TextClipboard.text()`. + `QtNetworkDownloadTransport` pumps a `QEventLoop` while it waits for a + download; without an application that loop never dispatches and the + first uncached model hangs. Only reading the clipboard needs a + `QGuiApplication`, which requires a display backend, so a `--request` + run gets a plain `QCoreApplication` and stays usable on a headless + machine. An existing instance is reused. """ from PySide6.QtCore import QCoreApplication - from PySide6.QtGui import QGuiApplication - return QCoreApplication.instance() or QGuiApplication(sys.argv[:1]) + existing = QCoreApplication.instance() + if existing is not None: + return existing + if needs_clipboard: + from PySide6.QtGui import QGuiApplication + + return QGuiApplication(sys.argv[:1]) + return QCoreApplication(sys.argv[:1]) def main(argv: Sequence[str] | None = None) -> int: args = build_arg_parser().parse_args(argv) - _ensure_qt_application() + _ensure_qt_application(needs_clipboard=args.request is None) try: request_text = load_request_text(args.request, _TextClipboard()) catalog = ModelCatalog.load_resource() diff --git a/tests/scripts/test_benchmark_models.py b/tests/scripts/test_benchmark_models.py index f0613cd..c3687ec 100644 --- a/tests/scripts/test_benchmark_models.py +++ b/tests/scripts/test_benchmark_models.py @@ -158,11 +158,15 @@ def test_empty_models_argument_is_rejected() -> None: select_models(catalog, " ") -def test_ensure_qt_application_guarantees_an_instance_without_network() -> None: - """Issue #170 review, item 1: main()'s first act must guarantee a Qt - application exists before the runtime (and its download transport) is - built, regardless of whether the request came from --request or the - clipboard. +def test_ensure_qt_application_creates_a_core_application_for_a_file_request() -> None: + """Issue #170 review, item 1 and PR #171 Codex P2: main()'s first act + must guarantee a Qt application exists before the runtime (and its + download transport) is built -- but a `--request ` run never + touches the clipboard, so it must not require a display/GUI backend, only + the event loop `QtNetworkDownloadTransport` pumps while waiting for a + downloaded chunk. A plain `QCoreApplication` provides that; forcing a + `QGuiApplication` here is what made a headless `--request` run on Linux + contradict the README's documented headless workflow. Run in a subprocess, not in-process: this repository's own test session shares one Qt application across every test file, and creating a bare @@ -174,9 +178,40 @@ def test_ensure_qt_application_guarantees_an_instance_without_network() -> None: """ script = ( "from PySide6.QtCore import QCoreApplication\n" + "from PySide6.QtGui import QGuiApplication\n" "from scripts.benchmark_models import _ensure_qt_application\n" - "app = _ensure_qt_application()\n" + "app = _ensure_qt_application(needs_clipboard=False)\n" "assert QCoreApplication.instance() is app\n" + "assert not isinstance(app, QGuiApplication)\n" + "print('ok')\n" + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, + check=False, + env=os.environ | {"QT_QPA_PLATFORM": "offscreen"}, + text=True, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "ok" + + +def test_ensure_qt_application_creates_a_gui_application_for_the_clipboard() -> None: + """The clipboard path (`_TextClipboard.text()`) calls `QGuiApplication + .instance().clipboard()`, which plain `QCoreApplication` does not have, + so reading from the clipboard still needs the GUI application -- only the + `--request ` path may downgrade to `QCoreApplication`. + """ + script = ( + "from PySide6.QtCore import QCoreApplication\n" + "from PySide6.QtGui import QGuiApplication\n" + "from scripts.benchmark_models import _ensure_qt_application\n" + "app = _ensure_qt_application(needs_clipboard=True)\n" + "assert QCoreApplication.instance() is app\n" + "assert isinstance(app, QGuiApplication)\n" "print('ok')\n" ) result = subprocess.run( From c5d642036445e9d4235513cee13ec1fe2de193d1 Mon Sep 17 00:00:00 2001 From: sensitron Date: Thu, 17 Sep 2026 19:02:10 +0200 Subject: [PATCH 3/5] =?UTF-8?q?=E2=9C=A8=20feat(benchmark):=20summarise=20?= =?UTF-8?q?and=20sort=20model=20timings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ✨ Features: - `format_summary_table` prints a table after every run, including an interrupted one: model, prepare and render seconds, frames, size, provider actually used, and status. Rows are sorted by render time with failed models last. A provider fallback shows as `CPU (fallback)`, and a cut-cache hit as `reused cuts`, because that time is not a real measurement. - `index.html` can sort tiles by render time, model name or file size. The tiles are generated in render-time order, so the page is already sorted without JavaScript. - `sort_by_render_time` is shared by the table and the report, so both use the same order. - `run_benchmark` raises `BenchmarkInterrupted` carrying the models finished before Ctrl+C. - 📝 Code comments no longer refer to review item numbers. - 🧪 Tests cover table order, the fallback and reused-cuts markers, the sort control and data attributes, tile order, and the interrupted run. Refs #170 --- scripts/benchmark_models.py | 155 ++++++++++++++++++++----- scripts/benchmark_report.py | 93 ++++++++++++++- src/matteloop/ui/action_shelf.py | 4 +- src/matteloop/ui/main_window.py | 3 +- src/matteloop/ui/presenter.py | 2 +- src/matteloop/ui/request_builder.py | 2 +- tests/scripts/test_benchmark_models.py | 113 +++++++++++++++++- tests/scripts/test_benchmark_report.py | 50 ++++++++ 8 files changed, 382 insertions(+), 40 deletions(-) diff --git a/scripts/benchmark_models.py b/scripts/benchmark_models.py index 52922b8..ff6f253 100644 --- a/scripts/benchmark_models.py +++ b/scripts/benchmark_models.py @@ -14,22 +14,11 @@ `ui/download_transport.py`'s Qt transport -- see CLAUDE.md, "One network path" -- this script never opens a socket of its own. -Cut-workspace caveat (issue #170 review, item 7): every render -- benchmark -or otherwise -- promotes its segmented cuts into the single durable -workspace cache all of MatteLoop shares (`paths.cut_workspace_root()`, -fixed, not overridable). There is no supported way to give this script its -own isolated workspace root without changing `jobs/render.py` (frozen) and -the workspace-root validation in `jobs/workspace/_platform.py` and -`_models.py`, which treats the shared root as a canonical, checked -invariant, not a parameter. Short of that change, this script instead -minimises the damage: it never forces re-segmentation over an existing cut -set (there is no `--regenerate`; a previous CLI version's flag detected the -destructive case reviewers actually hit -- overwriting a hand-edited cut -workspace with fresh, unedited segmentation output -- and removing it -removes that risk), and it detects and loudly flags a cache hit instead of -reporting it as if it were the raw model's output. See the module docstring -note in `_predict_cache_hit` below and the "STOP" discussion in the review -for the options that would need a maintainer decision to isolate this fully. +Renders share the application's durable cut-workspace cache +(`paths.cut_workspace_root()`), which cannot be redirected without changing +the frozen render and workspace code. The script therefore never forces +re-segmentation over an existing cut set, and it flags a reused one: that +tile may show hand-edited cuts, and its time is not a real measurement. This is a developer tool under `scripts/`, not a V1 product feature (see docs/v1-scope.md). It does not download models or render anything by itself @@ -53,6 +42,7 @@ from matteloop.core.execution_providers import ( ALLOWED_EXECUTION_PROVIDERS, is_allowed_provider, + provider_base_label, provider_options_from_runtime, select_provider, ) @@ -75,9 +65,17 @@ # sys.path, the same dual-mode split already used by scripts/build.py and # scripts/qt_source.py. Both spellings have to resolve. try: - from scripts.benchmark_report import render_report_html + from scripts.benchmark_report import ( + _format_file_size, + render_report_html, + sort_by_render_time, + ) except ImportError: - from benchmark_report import render_report_html # type: ignore[no-redef] + from benchmark_report import ( # type: ignore[no-redef] + _format_file_size, + render_report_html, + sort_by_render_time, + ) _LICENSED_MODEL_ID = "bria-rmbg" @@ -131,6 +129,18 @@ class BenchmarkRun: warnings: tuple[str, ...] = () +class BenchmarkInterrupted(Exception): + """Ctrl+C reached `run_benchmark` mid-loop. + + Carries the run as of the last finished model, so `main()` can still + print the summary table for it. + """ + + def __init__(self, run: BenchmarkRun) -> None: + super().__init__("interrupted") + self.run = run + + def select_models(catalog: ModelCatalog, requested: str | None) -> tuple[str, ...]: """Resolve `--models`, defaulting to the app's own V1 model set. @@ -215,7 +225,7 @@ def run_benchmark( Writes `results.json` and `index.html` after every model, not just at the end, so a Ctrl+C partway through keeps every tile already rendered - (issue #170 review, item 8) instead of losing the whole run. + instead of losing the whole run. """ if not model_ids: raise BenchmarkError("model_ids must not be empty") @@ -224,8 +234,8 @@ def run_benchmark( results: list[ModelResult] = [] run = BenchmarkRun(generated_at, provider, request_settings, ()) for index, model_id in enumerate(model_ids, start=1): - results.append( - _run_one_model( + try: + result = _run_one_model( base_request=base_request, model_id=model_id, index=index, @@ -237,7 +247,12 @@ def run_benchmark( clock=clock, log=log, ) - ) + except KeyboardInterrupt as error: + # `run` still holds every model that finished before this one; + # `_run_one_model` only catches `Exception`, not `BaseException`, + # so Ctrl+C during prepare/render reaches here uncaught. + raise BenchmarkInterrupted(run) from error + results.append(result) run = BenchmarkRun( generated_at=generated_at, provider=provider, @@ -375,7 +390,7 @@ def _predict_cache_hit( before deciding to segment or reuse; calling the same public function first lets the script report the outcome instead of guessing at it. A hit means this tile may show a previously hand-edited cut set rather - than the model's raw output (issue #170 review, item 7) -- surfaced as a + than the model's raw output -- surfaced as a run-level warning by `_consistency_warnings`, since there is no isolated workspace to render into instead. Returns ``None`` when the runtime does not expose a `ModelCatalog` to look the model's weight hash up in -- @@ -427,6 +442,76 @@ def _consistency_warnings(results: Sequence[ModelResult]) -> tuple[str, ...]: return tuple(warnings) +_SUMMARY_COLUMNS = ( + "model", + "prepare (s)", + "render (s)", + "frames", + "size", + "provider", + "status", +) + + +def format_summary_table(results: Sequence[ModelResult]) -> str: + """Render a plain-text table of `results`, fastest render first. + + Reuses `sort_by_render_time` -- the same ordering the HTML report's + tiles use server-side -- so the console table and the report agree on + what "fastest first, failed last" means instead of each having its own + notion of it. A cache hit is labelled "reused cuts" rather than folded + into "ok", since that render time is not a fresh measurement of the + model (see `_predict_cache_hit`). + """ + rows = [_summary_row(result) for result in sort_by_render_time(results)] + widths = [ + max(len(header), *(len(row[column]) for row in rows)) + if rows + else len(header) + for column, header in enumerate(_SUMMARY_COLUMNS) + ] + lines = [ + _summary_line(_SUMMARY_COLUMNS, widths), + _summary_line(tuple("-" * width for width in widths), widths), + ] + lines.extend(_summary_line(row, widths) for row in rows) + return "\n".join(lines) + + +def _summary_line(cells: Sequence[str], widths: Sequence[int]) -> str: + padded = (cell.ljust(width) for cell, width in zip(cells, widths, strict=True)) + return " ".join(padded) + + +def _summary_row(result: ModelResult) -> tuple[str, ...]: + return ( + result.model_id, + _dash_or(result.prepare_seconds, "{:.1f}"), + _dash_or(result.render_seconds, "{:.1f}"), + _dash_or(result.frame_count, "{}"), + _format_file_size(result.file_size_bytes), + _summary_provider(result), + _summary_status(result), + ) + + +def _dash_or(value: float | int | None, template: str) -> str: + return "–" if value is None else template.format(value) + + +def _summary_provider(result: ModelResult) -> str: + if result.active_provider is None: + return "–" + label = provider_base_label(result.active_provider) + return f"{label} (fallback)" if result.fallback_notice else label + + +def _summary_status(result: ModelResult) -> str: + if result.status != "ok": + return "failed" + return "reused cuts" if result.cache_hit else "ok" + + def write_results(out_dir: Path, run: BenchmarkRun) -> Path: path = out_dir / "results.json" payload = { @@ -498,12 +583,19 @@ def _default_out_dir() -> Path: return Path.cwd() / f"benchmark-{stamp}" +def _interrupted_message(out_dir: Path) -> str: + return ( + "interrupted; results for models completed so far were already " + f"written to {out_dir}" + ) + + def _print_license_note_if_selected( catalog: ModelCatalog, model_ids: Sequence[str], log: Callable[[str], None] = print, ) -> None: - """Print `bria-rmbg`'s licence note before rendering it (issue #170, item 3). + """Print `bria-rmbg`'s licence note before rendering it. Split out from `main()` so the decision -- and the note it prints -- is testable without building a real runtime. @@ -580,18 +672,23 @@ def main(argv: Sequence[str] | None = None) -> int: catalog=catalog, runtime=runtime, ) + except BenchmarkInterrupted as error: + print(_interrupted_message(out_dir), file=sys.stderr) + if error.run.results: + print(format_summary_table(error.run.results)) + return 130 except KeyboardInterrupt: - print( - "interrupted; results for models completed so far were already " - f"written to {out_dir}", - file=sys.stderr, - ) + # Defensive fallback: an interrupt landing outside the per-model + # try/except in run_benchmark (e.g. during results.json/index.html + # I/O between models) has no partial `run` to summarise. + print(_interrupted_message(out_dir), file=sys.stderr) return 130 finally: runtime.close() for warning in run.warnings: print(f"warning: {warning}", file=sys.stderr) + print(format_summary_table(run.results)) succeeded = sum(1 for result in run.results if result.status == "ok") print(f"{succeeded}/{len(run.results)} models succeeded; wrote {out_dir}") return 0 if succeeded == len(run.results) else 1 diff --git a/scripts/benchmark_report.py b/scripts/benchmark_report.py index b4d65d1..2f9927e 100644 --- a/scripts/benchmark_report.py +++ b/scripts/benchmark_report.py @@ -9,6 +9,7 @@ from __future__ import annotations +from collections.abc import Sequence from html import escape from typing import TYPE_CHECKING @@ -16,6 +17,11 @@ from scripts.benchmark_models import BenchmarkRun, ModelResult _BACKGROUNDS = ("checkerboard", "black", "white", "green") +_SORT_OPTIONS = ( + ("render", "Render time (fastest first)"), + ("name", "Model name"), + ("size", "File size"), +) _CSS = """ :root { color-scheme: light dark; } @@ -111,12 +117,46 @@ (function () { var grid = document.getElementById("grid"); var bgSelect = document.getElementById("bg-select"); + var sortSelect = document.getElementById("sort-select"); var restartButton = document.getElementById("restart-all"); function applyBackground() { grid.className = "grid bg-" + bgSelect.value; } + function sortValue(tile, key) { + if (key === "name") { + return (tile.getAttribute("data-model") || "").toLowerCase(); + } + var attribute = key === "size" ? "data-file-size" : "data-render-seconds"; + if (!tile.hasAttribute(attribute)) { + // No measurement (a failed model) always sorts last, regardless of + // which numeric column is selected. + return Infinity; + } + return parseFloat(tile.getAttribute(attribute)); + } + + function applySort() { + var key = sortSelect.value; + var tiles = Array.prototype.slice.call(grid.querySelectorAll(".tile")); + tiles.sort(function (a, b) { + var va = sortValue(a, key); + var vb = sortValue(b, key); + if (va < vb) { + return -1; + } + if (va > vb) { + return 1; + } + return 0; + }); + // Reorders the existing tile nodes in place -- no re-rendering. + tiles.forEach(function (tile) { + grid.appendChild(tile); + }); + } + function restartAll() { // Preload every image off-DOM first, then swap all visible tags // to the reloaded sources together, so the animations start back in @@ -148,18 +188,39 @@ } bgSelect.addEventListener("change", applyBackground); + sortSelect.addEventListener("change", applySort); restartButton.addEventListener("click", restartAll); applyBackground(); })(); """ +def sort_by_render_time(results: Sequence[ModelResult]) -> list[ModelResult]: + """Sort `results` by render time ascending, failed models last. + + Shared by this module's server-side tile order -- so the no-JS view + already matches the report's default sort -- and by the console summary + table in `benchmark_models.py`, which reuses this instead of sorting a + second way. + """ + + def key(result: ModelResult) -> tuple[int, float]: + if result.status != "ok" or result.render_seconds is None: + return (1, 0.0) + return (0, result.render_seconds) + + return sorted(results, key=key) + + def render_report_html(run: BenchmarkRun) -> str: """Render the full self-contained results page for one benchmark run.""" - tiles = "\n".join(_tile_html(result) for result in run.results) + tiles = "\n".join(_tile_html(result) for result in sort_by_render_time(run.results)) header = ( "

    MatteLoop model benchmark

    " '
    ' + "" "" @@ -196,13 +257,39 @@ def _background_options_html() -> str: return "".join(options) +def _sort_options_html() -> str: + options = [] + for index, (value, label) in enumerate(_SORT_OPTIONS): + selected = ' selected="selected"' if index == 0 else "" + options.append(f'') + return "".join(options) + + +def _sort_attributes_html(result: ModelResult) -> str: + """`data-render-seconds`/`data-file-size` for the report's sort control. + + Left out when the underlying value is unknown -- a failed model has no + render time or file size -- so the script can treat a missing attribute + as "sort last" (see `sortValue` in `_SCRIPT`) instead of inventing a + numeric sentinel here that the script would have to know about too. + """ + attributes = "" + if result.render_seconds is not None: + attributes += f' data-render-seconds="{escape(str(result.render_seconds))}"' + if result.file_size_bytes is not None: + attributes += f' data-file-size="{escape(str(result.file_size_bytes))}"' + return attributes + + def _tile_html(result: ModelResult) -> str: title = escape(result.display_name or result.model_id) model_id = escape(result.model_id) + sort_attributes = _sort_attributes_html(result) if result.status != "ok": error = escape(result.error or "unknown error") return ( - f'
    ' + f'
    " f"

    {title}

    " f'

    {model_id}

    ' f'

    Failed: {error}

    ' @@ -212,7 +299,7 @@ def _tile_html(result: ModelResult) -> str: provider_row = _provider_row(result.active_provider, result.fallback_notice) cache_class = ' class="cache-hit"' if result.cache_hit else "" return ( - f'
    ' + f'
    ' f'
    {title} result
    ' f"

    {title}

    " diff --git a/src/matteloop/ui/action_shelf.py b/src/matteloop/ui/action_shelf.py index 0f3f24c..b63dcc7 100644 --- a/src/matteloop/ui/action_shelf.py +++ b/src/matteloop/ui/action_shelf.py @@ -166,7 +166,7 @@ def __init__( self.preferences_button.clicked.connect(self.open_preferences) def _build_copy_render_settings_action(self) -> QAction: - """Build the export action, reachable while Render is disabled (issue #170). + """Build the export action, reachable while Render is disabled. Qt drops both a disabled widget's own shortcuts and the `ContextMenu` events it would otherwise turn into `customContextMenuRequested` @@ -197,7 +197,7 @@ def _build_copy_render_settings_action(self) -> QAction: def _render_context_menu(self) -> QMenu: """Return the Render button's popup menu, built once and reused. - Kept as one instance (issue #170 review) rather than a fresh `QMenu` + Kept as one instance rather than a fresh `QMenu` per right-click. Still a method, not inlined, so tests can inspect the menu without triggering a real popup -- `QMenu.exec` blocks until the menu closes, which a headless test has no way to do. diff --git a/src/matteloop/ui/main_window.py b/src/matteloop/ui/main_window.py index 6e7969b..492910f 100644 --- a/src/matteloop/ui/main_window.py +++ b/src/matteloop/ui/main_window.py @@ -150,8 +150,7 @@ def _build(self) -> None: self.copy_render_settings_action = self.action_shelf.copy_render_settings_action # Registered on the window, not the Render button: a WindowShortcut # action added to a disabled widget is itself unreachable, and the - # export command must stay reachable while Render is disabled (issue - # #170 review) -- verified offscreen, see action_shelf.py. + # export command must stay reachable while Render is disabled. self.addAction(self.copy_render_settings_action) self.rebuild_button = self.inspector.rebuild_button runtime_message = runtime_banner_copy() diff --git a/src/matteloop/ui/presenter.py b/src/matteloop/ui/presenter.py index dd3c1e2..99b9ba5 100644 --- a/src/matteloop/ui/presenter.py +++ b/src/matteloop/ui/presenter.py @@ -37,7 +37,7 @@ def _failure_message(error: object | None, retry: str) -> str: def _can_export_render_settings(state: AppState) -> bool: - """Ask the shared export-request builder (issue #170). + """Ask the shared export-request builder. Shares its predicate with ``RenderSettingsExportController.dispatch`` so "can we build a request to export" has one implementation. It does not diff --git a/src/matteloop/ui/request_builder.py b/src/matteloop/ui/request_builder.py index 74c6763..bb3184e 100644 --- a/src/matteloop/ui/request_builder.py +++ b/src/matteloop/ui/request_builder.py @@ -126,7 +126,7 @@ def _try_export_request( """Build the request "Copy render settings" would export, or ``None``. Shared by the presenter's export-enabled check and the export command - itself (issue #170), so "can we build a request to export" has exactly + itself, so "can we build a request to export" has exactly one implementation. Unlike ``_render_request``, this never resolves or validates the *real* output directory/filename -- only ``render_settings_to_json``'s ``output.max_bytes`` needs an ``OutputSpec`` diff --git a/tests/scripts/test_benchmark_models.py b/tests/scripts/test_benchmark_models.py index c3687ec..efbc764 100644 --- a/tests/scripts/test_benchmark_models.py +++ b/tests/scripts/test_benchmark_models.py @@ -26,8 +26,11 @@ from matteloop.jobs.render import RenderArtifact from scripts.benchmark_models import ( BenchmarkError, + BenchmarkInterrupted, + ModelResult, _print_license_note_if_selected, build_base_request, + format_summary_table, load_request_text, resolve_provider, run_benchmark, @@ -159,7 +162,7 @@ def test_empty_models_argument_is_rejected() -> None: def test_ensure_qt_application_creates_a_core_application_for_a_file_request() -> None: - """Issue #170 review, item 1 and PR #171 Codex P2: main()'s first act + """main()'s first act must guarantee a Qt application exists before the runtime (and its download transport) is built -- but a `--request ` run never touches the clipboard, so it must not require a display/GUI backend, only @@ -346,7 +349,7 @@ def test_run_benchmark_records_a_failing_model_and_keeps_going( def test_run_benchmark_writes_results_after_every_model_not_only_at_the_end( tmp_path: Path, ) -> None: - """Issue #170 review, item 8: a Ctrl+C partway through must not lose + """A Ctrl+C partway through must not lose tiles already rendered, so results.json/index.html are written incrementally rather than only once at the end of the loop.""" out_dir = tmp_path / "out" @@ -518,3 +521,109 @@ def test_results_json_has_the_documented_shape(tmp_path: Path) -> None: assert model_payload["status"] == "ok" assert model_payload["file_size_bytes"] == 1024 assert model_payload["active_provider"] == CPU_EXECUTION_PROVIDER + + +def _summary_result(model_id: str, **overrides: object) -> ModelResult: + defaults: dict[str, object] = dict( + model_id=model_id, + display_name=model_id, + status="ok", + prepare_seconds=1.0, + render_seconds=2.0, + frame_count=30, + file_size_bytes=1024, + active_provider=CPU_EXECUTION_PROVIDER, + ) + defaults.update(overrides) + return ModelResult(**defaults) # type: ignore[arg-type] + + +def test_format_summary_table_sorts_by_render_time_with_failed_last() -> None: + results = ( + _summary_result("slow", render_seconds=20.0), + ModelResult( + model_id="broken", display_name="Broken", status="error", error="boom" + ), + _summary_result("fast", render_seconds=1.0), + ) + + table = format_summary_table(results) + + lines = table.splitlines() + data_rows = lines[2:] + assert [line.split()[0] for line in data_rows] == ["fast", "slow", "broken"] + assert "failed" in data_rows[2] + + +def test_format_summary_table_marks_a_fallback_provider() -> None: + result = _summary_result("silueta", fallback_notice="CUDA unavailable; using CPU") + + table = format_summary_table((result,)) + + assert "CPU (fallback)" in table + + +def test_format_summary_table_marks_a_cache_hit_as_reused_cuts() -> None: + result = _summary_result("silueta", cache_hit=True) + + table = format_summary_table((result,)) + + assert "reused cuts" in table + + +def test_format_summary_table_uses_the_reports_file_size_formatting() -> None: + result = _summary_result("silueta", file_size_bytes=1024) + + table = format_summary_table((result,)) + + assert "1.0 KB" in table + + +def test_format_summary_table_has_column_headers() -> None: + table = format_summary_table((_summary_result("silueta"),)) + + header = table.splitlines()[0] + columns = ( + "model", + "prepare (s)", + "render (s)", + "frames", + "size", + "provider", + "status", + ) + for column in columns: + assert column in header + + +def test_run_benchmark_raises_benchmark_interrupted_with_completed_results( + tmp_path: Path, +) -> None: + out_dir = tmp_path / "out" + base_request, request_settings = _base_request(tmp_path, out_dir) + + class _InterruptingRuntime(_FakeRuntime): + def render(self, request, context): # noqa: ANN001, ANN201 + if request.segmentation.model_id == "u2netp": + raise KeyboardInterrupt + return super().render(request, context) + + runtime = _InterruptingRuntime(out_dir) + + with pytest.raises(BenchmarkInterrupted) as excinfo: + run_benchmark( + base_request=base_request, + request_settings=request_settings, + model_ids=("u2net", "u2netp"), + provider=CPU_EXECUTION_PROVIDER, + out_dir=out_dir, + catalog=ModelCatalog.load_resource(), + runtime=runtime, + log=lambda _message: None, + ) + + partial = excinfo.value.run + assert [result.model_id for result in partial.results] == ["u2net"] + assert partial.results[0].status == "ok" + # The results completed before the interrupt were still written to disk. + assert (out_dir / "results.json").exists() diff --git a/tests/scripts/test_benchmark_report.py b/tests/scripts/test_benchmark_report.py index cf00c02..4399b11 100644 --- a/tests/scripts/test_benchmark_report.py +++ b/tests/scripts/test_benchmark_report.py @@ -146,3 +146,53 @@ def test_report_is_self_contained_with_no_external_resources() -> None: assert "http://" not in html assert "https://" not in html assert " None: + html = render_report_html(_run(_ok_result("silueta"))) + + assert 'id="sort-select"' in html + assert 'value="render"' in html + assert 'value="name"' in html + assert 'value="size"' in html + + +def test_report_tiles_carry_sortable_data_attributes() -> None: + run = _run(_ok_result("silueta", render_seconds=5.5, file_size_bytes=2048)) + + html = render_report_html(run) + + assert 'data-render-seconds="5.5"' in html + assert 'data-file-size="2048"' in html + + +def test_report_omits_sort_attributes_for_a_failed_model() -> None: + failed = ModelResult( + model_id="u2net", display_name="U2Net", status="error", error="boom" + ) + + html = render_report_html(_run(failed)) + + tile_start = html.index('data-model="u2net"') + tile_end = html.index("
    ", tile_start) + tile_html = html[tile_start:tile_end] + assert "data-render-seconds" not in tile_html + assert "data-file-size" not in tile_html + + +def test_report_emits_tiles_in_render_time_order_with_failed_last() -> None: + failed = ModelResult( + model_id="broken", display_name="Broken", status="error", error="boom" + ) + run = _run( + _ok_result("slow", render_seconds=20.0), + failed, + _ok_result("fast", render_seconds=1.0), + ) + + html = render_report_html(run) + + fast_index = html.index('data-model="fast"') + slow_index = html.index('data-model="slow"') + broken_index = html.index('data-model="broken"') + assert fast_index < slow_index < broken_index From 958568f794c4c51ef25f8f515f09fac29cd39af8 Mon Sep 17 00:00:00 2001 From: sensitron Date: Thu, 17 Sep 2026 20:09:43 +0200 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=90=9B=20fix(theme):=20stop=20a=20men?= =?UTF-8?q?u=20shortcut=20overlapping=20its=20label?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 🐛 The Render button's context menu drew "Ctrl+Shift+C" on top of the "Copy render settings" label. Qt's windows11 style, the default on Windows 11 since Qt 6.7, under-sizes `CT_MenuItem` for this label, shortcut and font unless a stylesheet rule owns the item's box model. `_control_stylesheet` now styles `QMenu` and `QMenu::item`, which moves the sizing onto Qt's stylesheet path. The palette follows the existing combo-box popup. - 🧪 The test renders the menu offscreen and measures the gap between the label's ink and the shortcut's ink, because `sizeHint()` does not expose the defect: the buggy hint (220px) is already wider than the text it holds (193px). Measured gap 4px before the fix, 12px after, and the test skips where the windows11 style is absent. Trigger: repro — right-click the Render button on Windows 11; the shortcut sits on the label. The test fails without the stylesheet rule. Refs #170 --- src/matteloop/ui/theme.py | 9 ++ tests/ui/test_theme_menu_sizing.py | 147 +++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 tests/ui/test_theme_menu_sizing.py diff --git a/src/matteloop/ui/theme.py b/src/matteloop/ui/theme.py index 4571445..dadfa8e 100644 --- a/src/matteloop/ui/theme.py +++ b/src/matteloop/ui/theme.py @@ -88,6 +88,15 @@ def _control_stylesheet() -> str: selection-background-color: {ACCENT_COLOR}; selection-color: {PRIMARY_ACTION_TEXT_COLOR}; }} QComboBox::drop-down {{ border: 0; width: 28px; }} + /* The explicit QMenu::item padding is required, not cosmetic: Qt's + windows11 style (the Windows 11 default since Qt 6.7) under-sizes + CT_MenuItem for this font, so the shortcut column overlaps the + label unless a stylesheet rule takes over the item's box model. */ + QMenu {{ background: {CONTROL_COLOR}; border: 1px solid {DIVIDER_COLOR}; }} + QMenu::item {{ padding: 6px 16px; }} + QMenu::item:selected {{ background: {ACCENT_COLOR}; + color: {PRIMARY_ACTION_TEXT_COLOR}; }} + QMenu::item:disabled {{ color: {DISABLED_COLOR}; }} QAbstractSpinBox::up-button, QAbstractSpinBox::down-button {{ width: 24px; border: 0; background: transparent; subcontrol-origin: border; subcontrol-position: top right; diff --git a/tests/ui/test_theme_menu_sizing.py b/tests/ui/test_theme_menu_sizing.py new file mode 100644 index 0000000..45315e7 --- /dev/null +++ b/tests/ui/test_theme_menu_sizing.py @@ -0,0 +1,147 @@ +"""Regression test for issue #170. + +Right-clicking the Render button opened the "Copy render settings" context +menu with its shortcut text ("Ctrl+Shift+C") rendered overlapping the label. +Qt's `windows11` style (the Windows 11 default since Qt 6.7) under-sizes +`CT_MenuItem` for this label/shortcut/font combination when no stylesheet +rule targets `QMenu::item` -- the reserved item width comes out narrower +than the text it must hold, so the right-aligned shortcut is drawn over the +tail of the left-aligned label. Styling `QMenu::item` explicitly (theme.py) +hands the item's box model to Qt's stylesheet-aware style, which measures it +correctly. + +A plain `sizeHint().width() >= label_width + shortcut_width` check does not +catch this: measured on this repository, the *buggy* sizeHint (220px) is +already wider than the raw label+shortcut text sum (193px) -- the deficit is +in an internal reservation (icon/checkbox column) that a text-sum comparison +never sees, even though the rendered text visibly overlaps. So this test +renders the real menu offscreen and looks for the gap between the label's +ink and the shortcut's ink directly, which is what a person looking at the +menu actually judges. Measured gaps on this repository: 4px without the fix +(letter-spacing noise only, no separating gap) vs. 12px with it. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager + +import pytest +from PySide6.QtCore import QSettings +from PySide6.QtGui import QFont, QImage +from PySide6.QtWidgets import QApplication, QStyleFactory + +from matteloop.ui.controller import SourceController +from matteloop.ui.main_window import MainWindow +from matteloop.ui.store import ReducerStore +from matteloop.ui.theme import install_theme + +# Between the measured 4px (defect present) and 12px (fixed) gap -- see the +# module docstring for how those numbers were obtained. +_MIN_LABEL_SHORTCUT_GAP_PX = 8 +# The menu's 1px frame border and any anti-aliasing bleed sit in these +# outermost columns; excluding them keeps the border out of the ink scan. +_BORDER_MARGIN_PX = 4 + + +def _settings(name: str) -> QSettings: + settings = QSettings( + QSettings.IniFormat, QSettings.UserScope, "matteloop-test", name + ) + settings.clear() + return settings + + +def _window(qtbot, name: str) -> MainWindow: + store = ReducerStore() + controller = SourceController(store) + window = MainWindow(store, controller, _settings(name)) + qtbot.addWidget(window) + window.show() + return window + + +@contextmanager +def _themed_app_on_windows11() -> Iterator[QApplication]: + """Apply the real production theme under the style that exposes the + defect, then put the shared session `QApplication` back the way this + test found it -- other tests share the same instance.""" + app = QApplication.instance() + assert app is not None + if "windows11" not in {name.lower() for name in QStyleFactory.keys()}: + pytest.skip("the windows11 style, which exposes the defect, is not installed") + original_style_name = app.style().objectName() + original_stylesheet = app.styleSheet() + original_font = QFont(app.font()) + app.setStyle("windows11") + install_theme(app) + try: + yield app + finally: + app.setStyleSheet(original_stylesheet) + app.setFont(original_font) + if original_style_name: + app.setStyle(original_style_name) + + +def _max_gap_between_ink_clusters(image: QImage) -> int: + """Return the widest run of background-only columns between two runs of + non-background ("ink") columns, scanned across the item's vertical + centre. A correctly laid out "label shortcut" item has one wide gap + between the two words; an overlapping one only has narrow inter-letter + gaps.""" + width, height = image.width(), image.height() + background = image.pixelColor(2, 2) + + def is_ink(x: int, y: int) -> bool: + pixel = image.pixelColor(x, y) + diff = ( + abs(pixel.red() - background.red()) + + abs(pixel.green() - background.green()) + + abs(pixel.blue() - background.blue()) + ) + return diff > 30 + + mid = height // 2 + band = range(max(0, mid - 6), min(height, mid + 6)) + scan_end = max(_BORDER_MARGIN_PX, width - _BORDER_MARGIN_PX) + scan_range = range(_BORDER_MARGIN_PX, scan_end) + ink_columns = [any(is_ink(x, y) for y in band) for x in scan_range] + + clusters: list[tuple[int, int]] = [] + cluster_start: int | None = None + for offset, ink in enumerate(ink_columns): + if ink and cluster_start is None: + cluster_start = offset + elif not ink and cluster_start is not None: + clusters.append((cluster_start, offset - 1)) + cluster_start = None + if cluster_start is not None: + clusters.append((cluster_start, len(ink_columns) - 1)) + + if len(clusters) < 2: + return 0 + return max(b[0] - a[1] for a, b in zip(clusters, clusters[1:])) + + +def test_render_context_menu_does_not_overlap_label_and_shortcut(qtbot) -> None: + """A visible gap must separate the label from its shortcut -- see the + module docstring for why this is checked by rendering rather than by + comparing `sizeHint()` against a text-width sum.""" + with _themed_app_on_windows11(): + window = _window(qtbot, "theme-menu-sizing") + menu = window.action_shelf._render_context_menu() + try: + menu.resize(menu.sizeHint()) + image = menu.grab().toImage() + + gap = _max_gap_between_ink_clusters(image) + + assert gap >= _MIN_LABEL_SHORTCUT_GAP_PX, ( + f"widest gap between ink clusters in the render context menu " + f"is {gap}px (need >= {_MIN_LABEL_SHORTCUT_GAP_PX}px) -- the " + "shortcut text is overlapping the label instead of sitting " + "in its own column" + ) + finally: + menu.close() From 225f7db6a73703aa877768161f61e55d204340ba Mon Sep 17 00:00:00 2001 From: sensitron Date: Thu, 17 Sep 2026 20:09:43 +0200 Subject: [PATCH 5/5] =?UTF-8?q?=E2=9C=A8=20feat(benchmark):=20play=20only?= =?UTF-8?q?=20the=20visible=20comparison=20tiles?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ✨ Features: - Thirteen lossless animated WebPs playing at once made the report lag. An `IntersectionObserver` now keeps the animation on visible tiles and swaps the rest to a still frame, a click toggles one tile, and the column count is selectable and remembered. - `benchmark_stills.py` extracts frame 0 of each WebP as a PNG for that fallback. A tile whose still cannot be extracted keeps its animation. - `--report-only DIR` rebuilds the report and the stills from an existing `results.json` and the WebPs beside it, without a render, a model or a Qt GUI application. - Without JavaScript the page keeps the animated sources, as before. - ⚙️ `/benchmark-*/` is ignored, so a run's WebPs cannot reach a commit. - 🧪 Tests cover the still filenames, a tile without a still, the new controls, the rebuild from `results.json` and its error on a missing file. - 📝 README names `--report-only`. Refs #170 --- .gitignore | 3 + README.md | 3 + scripts/benchmark_models.py | 73 +++++++++ scripts/benchmark_report.py | 196 +++++++++++++++++++++++-- scripts/benchmark_stills.py | 77 ++++++++++ tests/scripts/test_benchmark_models.py | 102 +++++++++++++ tests/scripts/test_benchmark_report.py | 50 +++++++ tests/scripts/test_benchmark_stills.py | 57 +++++++ 8 files changed, 551 insertions(+), 10 deletions(-) create mode 100644 scripts/benchmark_stills.py create mode 100644 tests/scripts/test_benchmark_stills.py diff --git a/.gitignore b/.gitignore index 43c7049..507337f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,6 @@ qemu_*.core # A stray SQLite session file some tool writes when a database name of # ":memory:" reaches a path. Its colon makes every Windows checkout fail. :memory:* +# Model-benchmark output: one directory per run, holding a lossless WebP per +# model. The script's own default is benchmark-. +/benchmark-*/ diff --git a/README.md b/README.md index 665dad4..e54c482 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,9 @@ uv run python scripts/benchmark_models.py --out benchmark-run - Renders share the app's cut cache. A model whose cuts already exist for these settings is reused rather than segmented again, and flagged in the report — its tile may show hand-edited cuts, and its time is not a real measurement. +- To rebuild `index.html` and each model's still-image fallback from an + existing run's `results.json` and WebPs — after a report change, without + rendering again — pass `--report-only ` instead of `--out`. ## License diff --git a/scripts/benchmark_models.py b/scripts/benchmark_models.py index ff6f253..6305eea 100644 --- a/scripts/benchmark_models.py +++ b/scripts/benchmark_models.py @@ -70,12 +70,17 @@ render_report_html, sort_by_render_time, ) + from scripts.benchmark_stills import rebuild_stills, write_still_frame except ImportError: from benchmark_report import ( # type: ignore[no-redef] _format_file_size, render_report_html, sort_by_render_time, ) + from benchmark_stills import ( # type: ignore[no-redef] + rebuild_stills, + write_still_frame, + ) _LICENSED_MODEL_ID = "bria-rmbg" @@ -116,6 +121,7 @@ class ModelResult: frame_count: int | None = None duration_ms: int | None = None output_relative_path: str | None = None + still_relative_path: str | None = None @dataclass(frozen=True) @@ -352,6 +358,7 @@ def _run_one_model( f" {model_id}: ok -- prepare {timing.prepare_seconds:.1f}s, " f"render {timing.render_seconds:.1f}s" ) + still_relative_path = write_still_frame(out_dir, model_id, log) return ModelResult( model_id=model_id, display_name=spec.display_name, @@ -366,6 +373,7 @@ def _run_one_model( frame_count=artifact.frame_count, duration_ms=artifact.duration_ms, output_relative_path=f"{model_id}.webp", + still_relative_path=still_relative_path, ) @@ -533,6 +541,57 @@ def write_report(out_dir: Path, run: BenchmarkRun) -> Path: return path +def load_run_from_results_json(path: Path) -> BenchmarkRun: + """Load a `BenchmarkRun` back from a `results.json` written by + `write_results`, for `--report-only`. + + Raises `BenchmarkError` -- never a bare exception -- when the file is + missing, unreadable, not JSON, or missing an expected field, so + `run_report_only` can report it clearly instead of a traceback. + """ + if not path.exists(): + raise BenchmarkError( + f"{path} does not exist; run a benchmark into this directory first" + ) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except OSError as error: + raise BenchmarkError(f"could not read {path}: {error}") from error + except json.JSONDecodeError as error: + raise BenchmarkError(f"{path} is not valid JSON: {error}") from error + try: + results = tuple(ModelResult(**model) for model in payload["models"]) + return BenchmarkRun( + generated_at=payload["generated_at"], + provider=payload["provider"], + request_settings=payload["request"], + results=results, + warnings=tuple(payload.get("warnings", ())), + ) + except (KeyError, TypeError) as error: + raise BenchmarkError( + f"{path} is missing an expected field: {error}" + ) from error + + +def run_report_only(out_dir: Path, log: Callable[[str], None] = print) -> int: + """Rebuild `index.html` and every model's still image from an existing + `results.json` and the WebPs already in `out_dir`. + + No render, no model download, no runtime, and no Qt application -- + see `--report-only` in `build_arg_parser`. + """ + try: + run = load_run_from_results_json(out_dir / "results.json") + except BenchmarkError as error: + print(f"error: {error}", file=sys.stderr) + return 1 + run = rebuild_stills(out_dir, run, log=log) + write_report(out_dir, run) + log(f"rebuilt {out_dir / 'index.html'}") + return 0 + + def build_arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description=( @@ -575,6 +634,18 @@ def build_arg_parser() -> argparse.ArgumentParser: default=None, help="output directory; default: ./benchmark-", ) + parser.add_argument( + "--report-only", + type=Path, + default=None, + metavar="DIR", + help=( + "rebuild index.html and each model's still image from an " + "existing DIR/results.json and the WebPs already there -- no " + "render, no model download, no --request; every other option " + "is ignored" + ), + ) return parser @@ -644,6 +715,8 @@ def _ensure_qt_application(*, needs_clipboard: bool) -> object: def main(argv: Sequence[str] | None = None) -> int: args = build_arg_parser().parse_args(argv) + if args.report_only is not None: + return run_report_only(args.report_only) _ensure_qt_application(needs_clipboard=args.request is None) try: request_text = load_request_text(args.request, _TextClipboard()) diff --git a/scripts/benchmark_report.py b/scripts/benchmark_report.py index 2f9927e..f1eadfa 100644 --- a/scripts/benchmark_report.py +++ b/scripts/benchmark_report.py @@ -22,6 +22,12 @@ ("name", "Model name"), ("size", "File size"), ) +_COLUMNS_OPTIONS = ( + ("auto", "Auto"), + ("1", "1"), + ("2", "2"), + ("3", "3"), +) _CSS = """ :root { color-scheme: light dark; } @@ -53,14 +59,19 @@ gap: 16px; padding: 16px; } +.grid.cols-1 { grid-template-columns: 1fr; } +.grid.cols-2 { grid-template-columns: repeat(2, 1fr); } +.grid.cols-3 { grid-template-columns: repeat(3, 1fr); } .tile { background: #1e2128; border: 1px solid #2a2e37; border-radius: 8px; padding: 12px; + cursor: pointer; } -.tile-error { border-color: #a33; } +.tile-error { border-color: #a33; cursor: default; } .image-wrap { + position: relative; aspect-ratio: 4 / 3; border-radius: 4px; overflow: hidden; @@ -69,6 +80,23 @@ justify-content: center; background-color: #808080; } +.playback-badge { + position: absolute; + right: 6px; + bottom: 6px; + font-size: 11px; + padding: 2px 6px; + border-radius: 3px; + background: rgba(0, 0, 0, 0.6); + color: #fff; + pointer-events: none; +} +.tile[data-playback="playing"] .playback-badge::after { + content: "Playing \\2014 click to pause"; +} +.tile[data-playback="paused"] .playback-badge::after { + content: "Still \\2014 click to play"; +} .grid.bg-checkerboard .image-wrap { background-image: linear-gradient(45deg, #666 25%, transparent 25%), @@ -118,10 +146,110 @@ var grid = document.getElementById("grid"); var bgSelect = document.getElementById("bg-select"); var sortSelect = document.getElementById("sort-select"); + var columnsSelect = document.getElementById("columns-select"); var restartButton = document.getElementById("restart-all"); + var COLUMNS_STORAGE_KEY = "matteloop-benchmark-columns"; function applyBackground() { - grid.className = "grid bg-" + bgSelect.value; + Array.prototype.forEach.call(bgSelect.options, function (option) { + grid.classList.remove("bg-" + option.value); + }); + grid.classList.add("bg-" + bgSelect.value); + } + + function loadColumnsPreference() { + try { + return window.localStorage.getItem(COLUMNS_STORAGE_KEY) || "auto"; + } catch (error) { + // Private browsing / blocked storage -- fall back to "auto". + return "auto"; + } + } + + function saveColumnsPreference(value) { + try { + window.localStorage.setItem(COLUMNS_STORAGE_KEY, value); + } catch (error) { + // Ignore: a remembered column count is a convenience, not a + // requirement, so a blocked store must not break the page. + } + } + + function applyColumns() { + Array.prototype.forEach.call(columnsSelect.options, function (option) { + grid.classList.remove("cols-" + option.value); + }); + if (columnsSelect.value !== "auto") { + grid.classList.add("cols-" + columnsSelect.value); + } + saveColumnsPreference(columnsSelect.value); + } + + function setImageSrc(img, url) { + if (url && img.getAttribute("src") !== url) { + img.src = url; + } + } + + function applyAutoPlayback(tile) { + // The click handler (toggleManualPlayback) takes a tile out of this + // automatic, visibility-driven mode until the page reloads. + if (tile.getAttribute("data-manual") === "true") { + return; + } + var img = tile.querySelector("img.result-image"); + var still = img && img.getAttribute("data-still"); + if (!img || !still) { + return; + } + var visible = tile.getAttribute("data-visible") !== "false"; + if (visible) { + setImageSrc(img, img.getAttribute("data-src")); + tile.setAttribute("data-playback", "playing"); + } else { + setImageSrc(img, still); + tile.setAttribute("data-playback", "paused"); + } + } + + function toggleManualPlayback(tile) { + var img = tile.querySelector("img.result-image"); + var still = img && img.getAttribute("data-still"); + if (!img || !still) { + // No still was extracted for this model -- degrade to always + // animated, nothing to toggle to (see engineering-guardrails.md G4). + return; + } + tile.setAttribute("data-manual", "true"); + if (tile.getAttribute("data-playback") === "paused") { + setImageSrc(img, img.getAttribute("data-src")); + tile.setAttribute("data-playback", "playing"); + } else { + setImageSrc(img, still); + tile.setAttribute("data-playback", "paused"); + } + } + + function initPlaybackObserver() { + var tiles = Array.prototype.slice.call(grid.querySelectorAll(".tile")); + if (typeof IntersectionObserver === "undefined") { + // No observer support -- every tile keeps the animated src already + // in the markup rather than losing the fallback entirely. + return; + } + var observer = new IntersectionObserver( + function (entries) { + entries.forEach(function (entry) { + var tile = entry.target; + tile.setAttribute("data-visible", entry.isIntersecting ? "true" : "false"); + applyAutoPlayback(tile); + }); + }, + { rootMargin: "200px 0px" } + ); + tiles.forEach(function (tile) { + observer.observe(tile); + }); } function sortValue(tile, key) { @@ -158,13 +286,18 @@ } function restartAll() { - // Preload every image off-DOM first, then swap all visible tags - // to the reloaded sources together, so the animations start back in - // step (best effort -- network timing can still drift once playing). + // Preload every currently-playing image off-DOM first, then swap those + // tags to the reloaded sources together, so the animations start + // back in step (best effort -- network timing can still drift once + // playing). A tile currently showing its still is left alone: nothing + // is animating there to restart. var stamp = Date.now(); - var images = Array.prototype.slice.call( - grid.querySelectorAll("img.result-image") - ); + var images = Array.prototype.slice + .call(grid.querySelectorAll("img.result-image")) + .filter(function (img) { + var tile = img.closest(".tile"); + return !tile || tile.getAttribute("data-playback") !== "paused"; + }); if (images.length === 0) { return; } @@ -189,8 +322,18 @@ bgSelect.addEventListener("change", applyBackground); sortSelect.addEventListener("change", applySort); + columnsSelect.addEventListener("change", applyColumns); restartButton.addEventListener("click", restartAll); + grid.addEventListener("click", function (event) { + var tile = event.target.closest(".tile"); + if (tile) { + toggleManualPlayback(tile); + } + }); applyBackground(); + columnsSelect.value = loadColumnsPreference(); + applyColumns(); + initPlaybackObserver(); })(); """ @@ -224,6 +367,9 @@ def render_report_html(run: BenchmarkRun) -> str: "" + "" '' f'Provider: {escape(run.provider)}' f" · Generated {escape(run.generated_at)}" @@ -265,6 +411,17 @@ def _sort_options_html() -> str: return "".join(options) +def _columns_options_html() -> str: + """Options for the columns `