diff --git a/agent.md b/agent.md index b88b0af..082daa1 100644 --- a/agent.md +++ b/agent.md @@ -154,7 +154,9 @@ and on a heavy day it refuses the files of a whole folder for a while. It refuses the whole file but still serves pieces of it, so a refused file of up to 30 GB comes in a piece at a time instead, in the background steps only: `status: "importing"`, with `received` saying how much is in. A big file takes -a while this way. A clip Drive won't serve even a piece of, or one over 30 GB, +a while this way. A file over 30 GB, the most the video host takes, fails at +once with that reason, before anything is pulled: retry it if the limit is +raised. A clip Drive won't serve even a piece of is not failed: it waits (`status: "waiting"` with `retry_at`, counted in `counts.drive_waiting`) and is tried again by itself, further apart each time, for about a day, after which it fails with the reason. One that was coming in diff --git a/src/server/footage.ts b/src/server/footage.ts index 577ed3d..fb1340c 100644 --- a/src/server/footage.ts +++ b/src/server/footage.ts @@ -670,6 +670,9 @@ const DRIVE_WAIT_ALL = /** How long a clip Drive refused waits before each new try: about a day in all. */ const DRIVE_RETRY_MS = [30, 60, 120, 240, 240, 240, 240, 240].map((m) => m * 60_000); +/** Why a file over the video host's limit can't come in, in the size people see. */ +const overHostLimit = (bytes: number) => `${(bytes / 1e9).toFixed(1)} GB is over the video host's 30 GB limit`; + /** * Drive refused the clip. It lifts its limit within a day, so the clip waits * and is tried again by itself, further apart each time, then gives up. A clip @@ -712,14 +715,16 @@ async function dropUpload(cfg: MediaConfig, r: WorkRow): Promise { /** * Drive won't hand the file over whole: open an upload for it to come in a - * piece at a time instead. Null when that can't help, so the clip waits on - * Drive: it won't serve even a piece, the file is over the video host's limit, - * or the clip is marked for a re-encode, which reads the whole file. + * piece at a time instead. A file over the video host's limit fails with that + * reason instead: waiting on Drive can't get it in. Null when pieces can't + * help, so the clip waits on Drive: it won't serve even a piece, or the clip + * is marked for a re-encode, which reads the whole file. */ async function openRelay(cfg: MediaConfig, r: WorkRow): Promise { - if (r.transcode) return null; const size = await rangedSize(r.drive_file_id); - if (!size || size > MAX_RELAY_BYTES) return null; + if (!size) return null; + if (size > MAX_RELAY_BYTES) return { orgWide: false, detail: overHostLimit(size) }; + if (r.transcode) return null; await dropFailedCopy(cfg, r); const opened = await openMediaUpload(cfg, size, r.name); if ("failure" in opened) return ORG_LIMITS.has(opened.failure.error) ? { orgWide: true, detail: opened.failure.detail } : null; @@ -833,6 +838,8 @@ async function startImport( } return { orgWide: false, detail: verdict.reason ?? "that file isn't a video" }; } + // The video host would refuse it after pulling it ("MaxFileSizeError"): say so first. + if (verdict.size && verdict.size > MAX_RELAY_BYTES) return { orgWide: false, detail: overHostLimit(verdict.size) }; return importFrom(cfg, r, url, verdict.contentType?.startsWith("video/") ? verdict.contentType : "video/mp4", verdict.size ?? 0); } diff --git a/src/server/relay.ts b/src/server/relay.ts index 4e53e90..95a5e28 100644 --- a/src/server/relay.ts +++ b/src/server/relay.ts @@ -20,7 +20,11 @@ import { directDownloadUrl, judgeLinkResponse } from "./drive-link"; */ export const PIECE_BYTES = 200 * 1024 * 1024; -/** The largest file the video host takes. A larger one waits on Drive, as before. */ +/** + * The largest file the video host takes ("30 GB"), whichever way it comes in. + * A larger one fails with that reason before anything is pulled: no wait can + * get it in. + */ export const MAX_RELAY_BYTES = 30 * 1024 ** 3; /** diff --git a/test/e2e/run.mjs b/test/e2e/run.mjs index 601ee80..725f8b2 100644 --- a/test/e2e/run.mjs +++ b/test/e2e/run.mjs @@ -115,6 +115,8 @@ globalThis.fetch = async (input, init = {}) => { if (big.refuseFrom !== undefined && start >= big.refuseFrom) return quotaPage(); return new Response(zeros(end - start + 1), { status: 206, headers: { "content-type": "video/mp4", "content-range": `bytes ${start}-${end}/${big.size}` } }); } + const served = world.servedSizes?.[id]; + if (served && !ranged) return new Response("x".repeat(64), { status: 200, headers: { "content-type": "video/mp4", "content-length": String(served) } }); // Over the limit and refusing pieces too: nothing gets it in but waiting. if (id === fid("q1") && !world.quotaLifted) return quotaPage(); if (id.startsWith("conn") && id !== "conn_big___________________" && !ranged) { @@ -473,7 +475,7 @@ world.quotaLifted = true; // piece at a time, on deliveries only, into an upload on the media service, // and carries on from where the upload stands after a piece that didn't land // or a wait on Drive. An upload about to expire starts again on a new one; a -// file over the video host's 30 GB limit waits on Drive as before. +// file over the video host's 30 GB limit fails at once, saying so. { const P = 200 * 2 ** 20; const A = 2 * P + 3 * 2 ** 20; @@ -504,8 +506,8 @@ world.quotaLifted = true; assert.equal(pieces(), 0, "a read sends no pieces"); const a1 = uploadOf("INTERVIEW_A.MP4"); assert.deepEqual([world.uploads.get(a1).size, world.uploads.get(a1).name], [A, "INTERVIEW_A.MP4"]); - assert.equal(it["STATIC.MP4"].status, "waiting", "over 30 GB: it waits on Drive, as before"); - assert.match(it["STATIC.MP4"].error, /Waiting for Google Drive/); + assert.equal(it["STATIC.MP4"].status, "failed", "over 30 GB: no wait can get it in"); + assert.equal(it["STATIC.MP4"].error, "33.3 GB is over the video host's 30 GB limit"); assert.ok(![...world.uploads.values()].some((x) => x.name === "STATIC.MP4"), "and no upload is opened for it"); // First delivery: INTERVIEW_A's second piece doesn't land, and INTERVIEW_B's @@ -555,7 +557,7 @@ world.quotaLifted = true; r = await call("DELETE", `/api/projects/${p4}`); assert.equal(r.status, 200, JSON.stringify(r.data)); assert.ok(world.media.get(a1).deleted && world.media.get(b2).deleted); - console.log("7c ok: a file Drive won't hand over whole comes in by pieces on deliveries, carries on after a failed piece and a wait on Drive, restarts an expiring upload; over 30 GB still waits"); + console.log("7c ok: a file Drive won't hand over whole comes in by pieces on deliveries, carries on after a failed piece and a wait on Drive, restarts an expiring upload; over 30 GB fails at once, saying so"); } // 7d. Deleting a project deletes an upload still under way; one the service @@ -602,6 +604,23 @@ world.quotaLifted = true; console.log("7e ok: a piece the video host refuses for good fails the clip with its reason, and deletes the upload"); } +// 7f. A file Drive serves whole but over the video host's 30 GB limit is not +// handed over to be refused after the pull: it fails at once, saying so. +{ + world.servedSizes = { [fid("stage")]: 34_000_000_000 }; + world.drive[fid("day8")] = page("Day 8", [fileEntry(fid("stage"), "STAGE.MP4")]); + r = await call("POST", "/api/projects", { folder: `https://drive.google.com/drive/folders/${fid("day8")}` }); + const p8 = r.data.id; + const before = (world.importUrls ?? []).length; + const item = (await call("GET", `/api/projects/${p8}/footage?logs=0`)).data.items[0]; + assert.equal(item.status, "failed", JSON.stringify(item)); + assert.equal(item.error, "34.0 GB is over the video host's 30 GB limit"); + assert.equal((world.importUrls ?? []).length, before, "nothing was handed to the video host"); + r = await call("DELETE", `/api/projects/${p8}`); + assert.equal(r.status, 200, JSON.stringify(r.data)); + console.log("7f ok: a file over the video host's 30 GB limit fails at once with the reason, never pulled"); +} + // 8. Retry puts failures back in line. db.prepare("UPDATE project_footage SET status = 'failed', error = 'x' WHERE project_id = ? AND name = 'B_0002.MP4'").run(pid); r = await call("POST", `/api/projects/${pid}/footage/retry`);