From d11a6767143a45735be432830a3ff88ce0b357c4 Mon Sep 17 00:00:00 2001 From: Om Gate Date: Thu, 23 Jul 2026 14:23:29 +0530 Subject: [PATCH 1/5] feat: indexing v2 --- src/constants.ts | 60 +++ src/core/collection.ts | 491 +++++++++++++++++++++-- src/core/connection.ts | 158 ++++++++ src/core/editor.ts | 35 ++ src/core/indexManifest.ts | 330 +++++++++++++++ src/core/job.ts | 179 +++++++++ src/core/rtstream.ts | 683 +++++++++++++++++++++++++++++++- src/core/sandbox.ts | 156 ++++++++ src/core/sandboxModels.ts | 27 ++ src/core/scene.ts | 8 +- src/core/search/responses.ts | 127 ++++++ src/core/search/searchResult.ts | 54 ++- src/core/shot.ts | 36 +- src/core/timeline.ts | 37 +- src/core/understanding.ts | 406 +++++++++++++++++++ src/core/video.ts | 602 +++++++++++++++++++++++++++- src/core/voiceClone.ts | 93 +++++ src/index.ts | 49 ++- src/interfaces/core.ts | 6 +- src/types/config.ts | 1 + src/types/response.ts | 1 + src/utils/error.ts | 27 +- src/utils/httpClient.ts | 110 +++-- src/utils/index.ts | 111 ++++-- src/utils/upload.ts | 37 ++ 25 files changed, 3691 insertions(+), 133 deletions(-) create mode 100644 src/core/indexManifest.ts create mode 100644 src/core/job.ts create mode 100644 src/core/sandbox.ts create mode 100644 src/core/sandboxModels.ts create mode 100644 src/core/search/responses.ts create mode 100644 src/core/understanding.ts create mode 100644 src/core/voiceClone.ts diff --git a/src/constants.ts b/src/constants.ts index 28c33ef..545ee06 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -70,12 +70,68 @@ export const ApiPath = { start: 'start', stop: 'stop', export: 'export', + // indexing-v2 additions + indexes: 'indexes', + records: 'records', + understand: 'understand', + ask: 'ask', + semantic_search: 'semantic-search', + query: 'query', + aggregate: 'aggregate', + job: 'job', + sandbox: 'sandbox', + voice_clone: 'voice_clone', + identities: 'identities', + merge: 'merge', + split: 'split', + async_response: 'async-response', +} as const; + +/** + * Retrieval capabilities an index can be built for (the `use_for` values). + */ +export const IndexCapability = { + semantic: 'semantic', + query: 'query', + aggregate: 'aggregate', +} as const; + +/** + * Field groups that map artifact fields to retrieval capabilities. + */ +export const FieldGroup = { + semantic: 'semantic', + fts: 'fts', + filter: 'filter', + aggregate: 'aggregate', + sort: 'sort', +} as const; + +/** + * Compute tier a sandbox can be provisioned at. + */ +export const SandboxTier = { + small: 'small', + medium: 'medium', +} as const; + +/** + * Lifecycle status of a sandbox. + */ +export const SandboxStatus = { + provisioning: 'provisioning', + active: 'active', + stopping: 'stopping', + stopped: 'stopped', + failed: 'failed', + alert: 'alert', } as const; export const ResponseStatus = { processing: 'processing', in_progress: 'in progress', success: 'success', + complete: 'complete', } as const; export const MeetingStatus = { @@ -190,6 +246,10 @@ export const HttpClientDefaultValues = { max_retries: 3, timeout: 60 * 1000, backoff_factor: 0.1, + /** Max time (ms) to poll an async output URL before timing out. */ + maxPollTime: 500 * 1000, + /** Interval (ms) between polls of an async output URL. */ + pollInterval: 5 * 1000, } as const; export const MaxSupported = { diff --git a/src/core/collection.ts b/src/core/collection.ts index ea9bcef..b154e9e 100644 --- a/src/core/collection.ts +++ b/src/core/collection.ts @@ -25,8 +25,9 @@ import type { AudioResponse, ImageResponse, } from '@/types/response'; +import type { SearchResponse as SearchApiResponse } from '@/types/response'; import { HttpClient } from '@/utils/httpClient'; -import { uploadToServer } from '@/utils/upload'; +import { uploadToServer, uploadBytes } from '@/utils/upload'; import { DefaultSearchType, DefaultIndexType, @@ -34,14 +35,25 @@ import { } from '@/core/config'; import { SearchFactory } from './search'; import { SearchResult } from './search/searchResult'; +import { + AskResponse, + SearchResponse, + warnLegacySearchOnce, + type AskResponseData, + type SearchResponseData, +} from './search/responses'; import { Video } from './video'; import { Audio } from './audio'; import { Image } from './image'; import { Meeting } from './meeting'; import { CaptureSession } from './captureSession'; import { RTStream, RTStreamSearchResult, RTStreamShot } from './rtstream'; +import { GenerationJob, type JobData } from './job'; +import { VoiceClone, type VoiceCloneBase } from './voiceClone'; import { VideodbError } from '@/utils/error'; +const MAX_GENERATE_TEXT_PAYLOAD_SIZE = 250 * 1024; + const { video, audio, @@ -55,8 +67,46 @@ const { title, meeting, record, + ask, + semantic_search, + query: queryPath, + aggregate, + voice_clone, } = ApiPath; +/** + * Options for {@link Collection.search}. Combines new-style (v2) params with + * legacy-shaped params; passing any legacy param (or `namespace: "rtstream"`) + * routes the call to {@link Collection.legacySearch}. + */ +export interface CollectionSearchOptions { + // legacy-shaped + searchType?: SearchType; + indexType?: IndexType; + resultThreshold?: number; + scoreThreshold?: number; + dynamicScorePercentage?: number; + filter?: Array>; + sortDocsOn?: string; + namespace?: string; + sceneIndexId?: string; + indexId?: string; + algorithm?: string; + // new (v2) + topK?: number; + mode?: string; + returnFields?: unknown[] | Record | string; + includeClip?: boolean; + sessionId?: string; + config?: Record; + // unsupported in search() + indexName?: string; + indexNames?: string[] | string; + indexIds?: string[] | string; + // internal — rejected + deepsearchConfig?: unknown; +} + /** * The base VideoDB class * @remarks @@ -243,7 +293,227 @@ export class Collection implements ICollection { * @param sceneIndexId - [optional] Filter by specific scene index * @returns SearchResult or RTStreamSearchResult object */ + /** + * Search this collection. + * + * New search is used by default. Calls that pass legacy-shaped parameters + * (or `namespace: "rtstream"`) are routed to {@link legacySearch} with a + * one-time warning. + * @param query - Search query + * @param options - Search options (new-style or legacy-shaped) + */ public search = async ( + query: string, + options: CollectionSearchOptions = {} + ): Promise => { + const oldParams: (keyof CollectionSearchOptions)[] = [ + 'searchType', + 'indexType', + 'resultThreshold', + 'dynamicScorePercentage', + 'sceneIndexId', + 'indexId', + 'algorithm', + 'sortDocsOn', + 'namespace', + ]; + const newParams: (keyof CollectionSearchOptions)[] = [ + 'topK', + 'mode', + 'returnFields', + 'includeClip', + 'sessionId', + 'config', + ]; + const unsupported: (keyof CollectionSearchOptions)[] = [ + 'indexName', + 'indexNames', + 'indexId', + 'indexIds', + ]; + + const hasOld = oldParams.some( + k => options[k] !== undefined && options[k] !== null + ); + const hasNew = newParams.some( + k => options[k] !== undefined && options[k] !== null + ); + const hasUnsupported = unsupported.some( + k => options[k] !== undefined && options[k] !== null + ); + + if (options.deepsearchConfig != null) { + throw new VideodbError( + 'deepsearchConfig is internal and cannot be passed to search().' + ); + } + if (hasOld && (hasNew || hasUnsupported)) { + throw new VideodbError( + 'Cannot mix legacy search params with new search params. ' + + 'Use search(...) for new search or legacySearch(...) for legacy search.' + ); + } + if (hasUnsupported) { + throw new VideodbError( + 'indexName/indexNames/indexId/indexIds are not supported in search(). ' + + 'Use semanticSearch(), query(), or aggregate() for index-specific calls.' + ); + } + + if (hasOld) { + warnLegacySearchOnce(); + return this.legacySearch( + query, + options.searchType, + options.indexType, + options.resultThreshold, + options.scoreThreshold, + options.dynamicScorePercentage, + options.filter, + options.sortDocsOn, + options.namespace, + options.sceneIndexId + ); + } + + return this.#newSearch(query, options); + }; + + #newSearch = async ( + query: string, + options: CollectionSearchOptions + ): Promise => { + const payload: Record = { query }; + if (options.topK != null) payload.top_k = options.topK; + if (options.mode != null) payload.mode = options.mode; + if (options.returnFields != null) + payload.return_fields = options.returnFields; + if (options.includeClip != null) payload.include_clip = options.includeClip; + if (options.sessionId != null) payload.session_id = options.sessionId; + if (options.config != null) payload.config = options.config; + const res = await this.#vhttp.post, typeof payload>( + [collection, this.id, search, 'v2'], + payload + ); + return new SearchResponse(this.#vhttp, res.data as SearchResponseData); + }; + + /** + * Ask a question and get an answer generated from retrieved collection context. + */ + public ask = async ( + question: string, + topK: number = 15, + mode: string = 'default', + includeSources: boolean = false + ): Promise => { + const res = await this.#vhttp.post, object>( + [collection, this.id, ask], + { question, top_k: topK, mode, include_sources: includeSources } + ); + return new AskResponse(this.#vhttp, res.data as AskResponseData); + }; + + /** Semantic search across one or more indexes in this collection. */ + public semanticSearch = async ( + query: string, + options: { + indexNames?: string[] | string; + topK?: number; + scoreThreshold?: number; + filter?: unknown[] | Record; + returnFields?: unknown[] | Record | string; + indexIds?: string[] | string; + } = {} + ): Promise => { + const res = await this.#vhttp.post( + [collection, this.id, semantic_search], + { + query, + index_names: options.indexNames, + index_ids: options.indexIds, + top_k: options.topK ?? 10, + score_threshold: options.scoreThreshold, + filter: options.filter, + return_fields: options.returnFields, + } + ); + return new SearchResult(this.#vhttp, res.data); + }; + + /** Structured query against an index in this collection. */ + public query = async ( + options: { + indexName?: string; + filter?: unknown[] | Record; + limit?: number; + returnFields?: unknown[] | Record | string; + sort?: string | [string, string][]; + indexId?: string; + } = {} + ): Promise => { + const res = await this.#vhttp.post( + [collection, this.id, queryPath], + { + index_name: options.indexName, + index_id: options.indexId, + filter: options.filter, + limit: options.limit ?? 100, + return_fields: options.returnFields, + sort: options.sort, + } + ); + return new SearchResult(this.#vhttp, res.data); + }; + + /** Aggregate over an index in this collection. */ + public aggregate = async ( + options: { + indexName?: string; + filter?: unknown[] | Record; + groupBy?: string; + metric?: string; + limit?: number; + sort?: string | [string, string][]; + indexId?: string; + } = {} + ): Promise | Record[]> => { + // convert:false — aggregate buckets are keyed by user field names/values. + const res = await this.#vhttp.post< + Record | Record[], + object + >( + [collection, this.id, aggregate], + { + index_name: options.indexName, + index_id: options.indexId, + filter: options.filter, + group_by: options.groupBy, + metric: options.metric ?? 'count', + limit: options.limit ?? 100, + sort: options.sort, + }, + undefined, + { convert: false } + ); + return res.data; + }; + + /** + * Legacy search (pre-v2), including RTStream namespace search. + * @param query - Search query + * @param searchType - [optional] Type of search to be performed + * @param indexType - [optional] Index Type + * @param resultThreshold - [optional] Result Threshold + * @param scoreThreshold - [optional] Score Threshold + * @param dynamicScorePercentage - [optional] Percentage of dynamic score to consider + * @param filter - [optional] Additional metadata filters + * @param sortDocsOn - [optional] Sort docs within each video by "score" or "start" + * @param namespace - [optional] Search namespace ("rtstream" to search RTStreams) + * @param sceneIndexId - [optional] Filter by specific scene index + * @returns SearchResult or RTStreamSearchResult object + */ + public legacySearch = async ( query: string, searchType?: SearchType, indexType?: IndexType, @@ -259,7 +529,8 @@ export class Collection implements ICollection { if (namespace === 'rtstream') { const data: Record = { query }; if (sceneIndexId !== undefined) data.scene_index_id = sceneIndexId; - if (resultThreshold !== undefined) data.result_threshold = resultThreshold; + if (resultThreshold !== undefined) + data.result_threshold = resultThreshold; if (scoreThreshold !== undefined) data.score_threshold = scoreThreshold; if (dynamicScorePercentage !== undefined) data.dynamic_score_percentage = dynamicScorePercentage; @@ -414,15 +685,41 @@ export class Collection implements ICollection { public generateImage = async ( prompt: string, aspectRatio: '1:1' | '9:16' | '16:9' | '4:3' | '3:4' = '1:1', - callbackUrl?: string - ): Promise => { - const res = await this.#vhttp.post( + callbackUrl?: string, + options: { + modelName?: string; + config?: Record; + sandboxId?: string; + wait?: boolean; + pollInterval?: number; + timeout?: number; + } = {} + ): Promise => { + const body: Record = { + prompt, + aspect_ratio: aspectRatio, + callback_url: callbackUrl, + }; + if (options.modelName != null) body.model_name = options.modelName; + if (options.config != null) body.config = options.config; + if (options.sandboxId != null) body.sandbox_id = options.sandboxId; + + const res = await this.#vhttp.post( [collection, this.id, generate, image], - { prompt, aspect_ratio: aspectRatio, callback_url: callbackUrl } + body ); - if (res.data) { - return new Image(this.#vhttp, res.data); + const data = res.data; + if (!data) return undefined; + if (data.jobId || data.job_id) { + const job = GenerationJob.fromData(this.#vhttp, data, 'image'); + return options.wait + ? ((await job.wait( + options.timeout ?? 600, + options.pollInterval ?? 5 + )) as Image | JobData) + : job; } + return new Image(this.#vhttp, data as unknown as ImageBase); }; /** @@ -462,7 +759,13 @@ export class Collection implements ICollection { ): Promise