Drive: Google's own Drive API through the connection, not its broker's actions

agent.md

@@ -164,10 +164,12 @@ has its own limit.
 
 When the org has a Google Drive connection whose account can open the
 folder, clips are downloaded through it instead, as that account, which
-Drive's limit on shared links doesn't touch. Those downloads run in the
-background steps only. A file too big for the connection's temporary storage
-(seen at 9 GB; 3.4 GB passed) comes by the original's shared link, as above:
-whole, or a piece at a time when Drive refuses it whole. It is never copied inside Google first: Drive answers a header
+Drive's limit on shared links doesn't touch: Google's own Drive API, signed
+by the connection. Those downloads run in the background steps only. A file
+over 250 MB, more than one answer through the connection carries, comes by
+the original's shared link, as above: whole, or a piece at a time when Drive
+refuses it whole. A piece Drive refuses on the shared link (it can, after a
+few GB of one file) comes through the connection instead. It is never copied inside Google first: Drive answers a header
 check on a fresh copy of a big file with an empty page, and the video host
 refuses such a download. Copies an earlier version made are deleted once
 their import is over (or with the project).

package.json

@@ -13,7 +13,7 @@
   },
   "dependencies": {
     "@clawnify/app": "^0.2.1",
-    "@clawnify/connections": "^0.5.0",
+    "@clawnify/connections": "^0.7.0",
     "@clawnify/db": "^0.4.2",
     "@clawnify/queue": "^0.1.1",
     "@fontsource-variable/inter": "^5.3.0",

pnpm-workspace.yaml

@@ -2,3 +2,5 @@ allowBuilds:
   esbuild: true
   sharp: true
   workerd: true
+minimumReleaseAgeExclude:
+  - '@clawnify/connections@0.7.0'

src/server/drive.ts

@@ -4,16 +4,47 @@
 // exactly like an upload, so the editor and the export never read from Drive
 // themselves.
 //
-// The connection's broker never hands this app a raw Google token, so the
-// bytes come the one way it allows: the download action parks the file behind
-// a short-lived signed link, and the import streams that link into storage.
+// Every call is Google's own Drive API, sent through the connection's broker,
+// which signs it with the org's credential: the app never holds a Google
+// token, and the code is the same whoever holds the credential. A file comes
+// back as a short-lived link to its bytes, at most 250 MB in one answer.
 
 import { connect, describe, type ConnectionsEnv } from "@clawnify/connections";
 
-// Both toolkits carry the same Drive actions, prefixed with their own name.
 // Google Drive comes first: it is the connection a studio makes for its files.
 const SERVICES = ["googledrive", "googlesuper"] as const;
-type Service = (typeof SERVICES)[number];
+export type DriveService = (typeof SERVICES)[number];
+type Service = DriveService;
+
+/**
+ * Where the Drive API sits under each connection's base URL: Google Drive's
+ * already ends in `/drive/v3`, Google Super's is the googleapis.com root.
+ */
+const DRIVE_API: Record<Service, string> = { googledrive: "", googlesuper: "/drive/v3" };
+
+/** One request to the Drive API through the connection. Query values are sent as given. */
+function driveCall(
+  env: ConnectionsEnv,
+  service: Service,
+  method: "GET" | "DELETE",
+  path: string,
+  query: Record<string, string> = {},
+) {
+  return connect(service, env).rawRequest({
+    method,
+    endpoint: `${DRIVE_API[service]}${path}`,
+    parameters: Object.entries(query).map(([name, value]) => ({ name, value, in: "query" as const })),
+  });
+}
+
+/** The broker's words for a provider 404: "<service> 404: …". */
+const notFound = (e: unknown) => /^\w+ 404:/.test(e instanceof Error ? e.message : String(e));
+
+/**
+ * The most one answer through the connection can carry: the broker refuses a
+ * bigger file ("maximum file size limit of 250MB").
+ */
+export const PROXY_FILE_BYTES = 250_000_000;
 
 /** What the picker lists. `duration` is seconds, for videos Drive has probed. */
 export interface DriveFile {
@@ -85,18 +116,25 @@ export async function listDriveFiles(
   const search = opts.search?.replace(/['\\]/g, " ").trim();
   if (search) q += ` and name contains '${search}'`;
   const shared = opts.folderId === SHARED_WITH_ME;
+  // Shared items are found by the query, not by a parent folder.
   if (shared) q += " and sharedWithMe = true";
+  else {
+    const parent = opts.folderId || "root";
+    if (parent !== "root" && !DRIVE_FILE_ID.test(parent)) throw new Error("not a Drive folder id");
+    q += ` and '${parent}' in parents`;
+  }
 
   const service = await requireDriveService(env);
-  const data = (await connect(service, env).run(`${service.toUpperCase()}_FIND_FILE`, {
+  const data = (await driveCall(env, service, "GET", "/files", {
     q,
     // `folder` sorts folders ahead of files, so the picker needs no re-sort.
     orderBy: "folder,modifiedTime desc",
-    pageSize: 50,
+    pageSize: "50",
     fields:
       "nextPageToken,files(id,name,mimeType,size,modifiedTime,thumbnailLink,videoMediaMetadata(durationMillis))",
-    // Shared items are found by the query, not by a parent folder.
-    ...(shared ? {} : { folder_id: opts.folderId || "root" }),
+    // Folders on a shared drive, as well as the account's own.
+    supportsAllDrives: "true",
+    includeItemsFromAllDrives: "true",
     ...(opts.pageToken ? { pageToken: opts.pageToken } : {}),
   })) as {
     files?: {
@@ -138,11 +176,16 @@ async function driveItem(
   id: string,
 ): Promise<{ name: string; parents: string[] } | null> {
   const service = await requireDriveService(env);
-  const data = (await connect(service, env).run(`${service.toUpperCase()}_GET_FILE_METADATA`, {
-    fileId: id,
-    fields: "name,parents",
-  })) as { name?: string; parents?: string[] };
-  return data.name ? { name: data.name, parents: data.parents ?? [] } : null;
+  try {
+    const data = (await driveCall(env, service, "GET", `/files/${encodeURIComponent(id)}`, {
+      fields: "name,parents",
+      supportsAllDrives: "true",
+    })) as { name?: string; parents?: string[] };
+    return data.name ? { name: data.name, parents: data.parents ?? [] } : null;
+  } catch (e) {
+    if (notFound(e)) return null;
+    throw e;
+  }
 }
 
 export async function driveFolderName(env: ConnectionsEnv, id: string): Promise<string | null> {
@@ -165,8 +208,103 @@ export async function withinFolder(env: ConnectionsEnv, itemId: string, folderId
   return false;
 }
 
-/** A signed, short-lived link to one Drive file's original bytes. */
-export async function driveDownloadLink(
+export interface DriveFileInfo {
+  name: string;
+  mimeType: string;
+  /** Bytes. Null for a file Drive keeps no size for (a Google Doc, say). */
+  size: number | null;
+}
+
+/**
+ * A short-lived link to one Drive file's original bytes, through the
+ * connection, or `tooBig` (with what the file is) when it is over what one
+ * answer carries. Then it comes by its shared link instead (footage, a piece
+ * at a time when need be), or by {@link driveActionLink}.
+ */
+export async function driveFileLink(
+  env: ConnectionsEnv,
+  fileId: string,
+): Promise<(DriveFileInfo & { url: string }) | (DriveFileInfo & { tooBig: true })> {
+  const service = await requireDriveService(env);
+  const path = `/files/${encodeURIComponent(fileId)}`;
+  const meta = (await driveCall(env, service, "GET", path, { fields: "name,mimeType,size", supportsAllDrives: "true" })) as {
+    name?: string;
+    mimeType?: string;
+    size?: string;
+  };
+  const info: DriveFileInfo = {
+    name: meta.name || fileId,
+    mimeType: meta.mimeType || "application/octet-stream",
+    size: meta.size ? Number(meta.size) : null,
+  };
+  // Asked for whole, a bigger file is fetched into the broker's storage before
+  // it is refused: ask only for what can come back.
+  if (info.size === null || info.size > PROXY_FILE_BYTES) return { ...info, tooBig: true };
+  const file = await connect(service, env).rawFile({
+    method: "GET",
+    endpoint: `${DRIVE_API[service]}${path}`,
+    parameters: [
+      { name: "alt", value: "media", in: "query" },
+      { name: "supportsAllDrives", value: "true", in: "query" },
+    ],
+  });
+  return { ...info, url: file.url };
+}
+
+/**
+ * One piece of a Drive file through the connection, as a Response carrying
+ * Drive's own status and headers and the bytes behind the broker's link. A
+ * refusal comes back as its status, so the caller can tell it from a hiccup;
+ * null when the broker or its link couldn't be reached.
+ *
+ * Drive serves these as the connected account even when its limit refuses
+ * the shared link's pieces: on 2026-10-10 a 15 GB file's shared link refused
+ * ranges past about 4 GB read, while this served the same range.
+ */
+export async function driveFilePiece(
+  env: ConnectionsEnv,
+  service: DriveService,
+  fileId: string,
+  start: number,
+  end: number,
+  signal: AbortSignal,
+): Promise<Response | null> {
+  let file;
+  try {
+    file = await connect(service, env).rawFile({
+      method: "GET",
+      endpoint: `${DRIVE_API[service]}/files/${encodeURIComponent(fileId)}`,
+      parameters: [
+        { name: "alt", value: "media", in: "query" },
+        { name: "supportsAllDrives", value: "true", in: "query" },
+        { name: "Range", value: `bytes=${start}-${end}`, in: "header" },
+      ],
+    });
+  } catch (e) {
+    const status = /^\w+ (\d{3}):/.exec(e instanceof Error ? e.message : String(e))?.[1];
+    return status ? new Response(null, { status: Number(status) }) : null;
+  }
+  const bytes = await fetch(file.url, { signal }).catch(() => null);
+  if (!bytes?.ok || !bytes.body) {
+    await bytes?.body?.cancel().catch(() => {});
+    return null;
+  }
+  const range = file.headers["content-range"] ?? file.headers["Content-Range"];
+  return new Response(bytes.body, {
+    status: file.status ?? 206,
+    headers: { "content-type": file.contentType, ...(range ? { "content-range": range } : {}) },
+  });
+}
+
+/**
+ * A whole file of any size up to the broker's own storage (3.4 GB passed, 9 GB
+ * did not) through its download action, which parks the file behind a
+ * short-lived link. The one Drive call still made as an action: for a file
+ * over what one proxy answer carries, where nothing reads it in pieces (an
+ * import into the media library). See the platform's
+ * docs/internal/connections-architecture.md §19.5 and §23.
+ */
+export async function driveActionLink(
   env: ConnectionsEnv,
   fileId: string,
 ): Promise<{ url: string; name: string; mimeType: string }> {
@@ -182,5 +320,5 @@ export async function driveDownloadLink(
 /** Delete a file the app made in the connected account (a copy an earlier version made for an import). */
 export async function driveRemove(env: ConnectionsEnv, fileId: string): Promise<void> {
   const service = await requireDriveService(env);
-  await connect(service, env).run(`${service.toUpperCase()}_GOOGLE_DRIVE_DELETE_FOLDER_OR_FILE_ACTION`, { fileId, supportsAllDrives: true });
+  await driveCall(env, service, "DELETE", `/files/${encodeURIComponent(fileId)}`, { supportsAllDrives: "true" });
 }

src/server/footage.ts

@@ -21,7 +21,16 @@ import { query, get, run } from "./db";
 import { deleteMedia, importMedia, mediaState, openMediaUpload, prepareMedia, startTranscode, transcodeState, type MediaConfig } from "./media";
 import { refusalDetail } from "./refusal";
 import { directDownloadUrl, folderListingUrl, judgeLinkResponse, listFolderVideos, type FolderVideo } from "./drive-link";
-import { MAX_RELAY_BYTES, RELAY_AT_ONCE, RELAY_BUDGET_MS, RELAY_EXPIRY_MARGIN_MS, rangedSize, relayPieces } from "./relay";
+import {
+  MAX_RELAY_BYTES,
+  RELAY_AT_ONCE,
+  RELAY_BUDGET_MS,
+  RELAY_EXPIRY_MARGIN_MS,
+  rangedSize,
+  relayPieces,
+  sharedLinkReader,
+  type PieceReader,
+} from "./relay";
 
 const DEFAULT_SERVICES_URL = "https://services.clawnify.com";
 
@@ -342,6 +351,11 @@ export interface DriveSource {
   download(fileId: string): Promise<{ url: string; mimeType: string } | { error: string }>;
   /** Delete a copy an earlier version made in the connected account for an import. */
   remove(fileId: string): Promise<void>;
+  /**
+   * One piece of a file through the connection: Drive's answer, with its own
+   * status and headers. It serves pieces the shared link refuses.
+   */
+  piece?(fileId: string, start: number, end: number, signal: AbortSignal): Promise<Response | null>;
 }
 
 /**
@@ -505,7 +519,7 @@ export async function stepFootage(cfg: MediaConfig, projectId: string, opts: Ste
   //     counts from here, so it doesn't matter what ran before.
   if (opts.relay && relaying.length) {
     const until = Date.now() + RELAY_BUDGET_MS;
-    await Promise.all(relaying.slice(0, RELAY_AT_ONCE).map((r) => relayClip(cfg, r, until)));
+    await Promise.all(relaying.slice(0, RELAY_AT_ONCE).map((r) => relayClip(cfg, r, until, opts.drive)));
   }
 
   // 2. Start imports while there is room, together. A refusal that is about
@@ -719,13 +733,19 @@ async function openRelay(cfg: MediaConfig, r: WorkRow): Promise<true | { orgWide
   return true;
 }
 
-/** Move a clip coming in a piece at a time on, and settle it once it is all in. */
-async function relayClip(cfg: MediaConfig, r: WorkRow, until: number): Promise<void> {
+/**
+ * Move a clip coming in a piece at a time on, and settle it once it is all in.
+ * Pieces come from the shared link, or, where Drive refuses those, through the
+ * org's Drive connection when it has one.
+ */
+async function relayClip(cfg: MediaConfig, r: WorkRow, until: number, drive?: DriveSource): Promise<void> {
   const uid = r.upload_uid!;
   const expiring = r.upload_expires !== null && Date.parse(r.upload_expires) - Date.now() < RELAY_EXPIRY_MARGIN_MS;
+  const readers: PieceReader[] = [sharedLinkReader(r.drive_file_id)];
+  if (drive?.piece) readers.push((start, end, signal) => drive.piece!(r.drive_file_id, start, end, signal));
   const result = expiring
     ? ({ state: "gone" } as const)
-    : await relayPieces({ url: r.upload_url!, fileId: r.drive_file_id, size: r.upload_size! }, until, async (received) => {
+    : await relayPieces({ url: r.upload_url!, size: r.upload_size! }, readers, until, async (received) => {
         await setRow(r.id, { upload_done: received });
       });
   if (result.state === "moving") return;
@@ -787,8 +807,8 @@ async function startImport(
   // Already coming in a piece at a time, back from a wait on Drive: it
   // carries on from where its upload got to, on the next delivery.
   if (r.upload_uid) return true;
-  // Through the org's connection first. A file it can't hand over (too big for
-  // the connector's temporary storage) comes by the original's shared link,
+  // Through the org's connection first. A file it can't hand over (over the
+  // 250 MB one answer through it carries) comes by the original's shared link,
   // with its waits. Not by a copy in the connected account: Drive answers a
   // header check on a fresh copy of a big file with an empty page while the
   // file itself downloads, and the video host, which checks first, refuses