diff --git a/CHANGELOG.md b/CHANGELOG.md index c29c650..ab176c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,34 @@ # Changelog +## [0.3.0] (2026-07-25) + +Indexing v2 — a new retrieval architecture plus a generation/compute stack, ported from `videodb-python`'s `feat/add-indexing-v2`. + +### Added + +- **Search v2 on `Video` and `Collection`** — new `search()` dispatcher with dedicated `ask()`, `semanticSearch()`, `query()`, and `aggregate()` methods, returning the v2 `SearchResponse` / `AskResponse` envelopes (iterable `SearchResult`). `legacySearch()` preserves the old behavior behind a one-time deprecation warning +- **Understanding runs** — `Video.understand()` / `getUnderstanding()` / `listUnderstandings()` / `deleteUnderstanding()`, backed by new `Understanding` and `UnderstandingAnalyzer` classes and analyzer-normalization helpers (`normalizeUnderstandingAnalyzers`, `normalizeAnalyzerType`, `defaultAnalyzerName`) +- **Index manifest** — `Video.index()` / `getIndex()` / `listIndexes()` / `deleteIndex()`, backed by new `Index`, `IndexRecord`, `FieldSchema`, and `RecordPage` classes with `IndexCapability` and `FieldGroup` constants +- **Sandboxes (persistent GPU compute pools)** — `Connection.createSandbox()` / `getSandbox()` / `listSandboxes()`, a `Sandbox` class with `waitForReady()` / `stop()` / `waitForStop()` lifecycle polling, and `SandboxTier` / `SandboxStatus` constants +- **Async generation jobs** — `GenerationJob` with `wait()` polling, plus `Connection.getJobStatus()` and `waitForJob()`. `Collection.generateImage()` / `generateVoice()` / `generateText()` gain self-inference model routing (`SandboxModel`), `sandboxId`, and `wait` options, returning a `GenerationJob` when the server dispatches async work +- **Voice clones** — `VoiceClone` class and CRUD on both `Connection` and `Collection` (`createVoiceClone` / `getVoiceClone` / `listVoiceClones` / `deleteVoiceClone`), plus `voiceCloneId` support in `generateVoice()` +- **Real-time stream understanding & indexing** — new `RTStreamUnderstanding` and `RTStreamIndex` classes with `understand()` / `index()` on `RTStream`, alert management (`createAlert` / `listAlerts` / `enableAlert` / `disableAlert`), and record retrieval +- **Embed-code helpers** — `getEmbedCode()` on `Video`, `Shot`, `Timeline`, and RTStream types, plus `buildIframeEmbedCode()` / `playerUrlToEmbedUrl()` utilities. `Video`, `Shot`, and RTStream types now carry `playerUrl` +- **`sandboxId` threading** into `Video.indexScenes()` / `indexVisuals()`, `Scene.describe()` (also gains `modelConfig`), and all `RTStream` indexing methods, to route inference to a specific sandbox +- **`Connection.getAsyncResponse()`** to fetch a stored async response without polling + +### Changed + +- `HttpClient` `get`/`post` accept a `wait` flag and per-call poll overrides (`maxPollTime` / `pollInterval`); the async-output poll loop was rewritten from `backoff` to a monotonic-clock loop that throws `RequestTimeoutError` on timeout +- `uploadBytes()` helper added for in-memory uploads; `upload()` refactored to reuse it, and large `generateText()` prompts are now offloaded to a presigned URL (`prompt_url`) above ~250 KB to avoid gateway payload limits +- `ResponseStatus` gains `complete`; new `ApiPath` segments and polling defaults added + +### Fixed + +- `Sandbox.isReady` and `waitForReady()` now treat `alert` as a ready state (matching `videodb-python`'s `READY_STATUSES`) and no longer report `provisioning` as ready + +--- + ## [0.2.7] (2026-06-03) ### Added diff --git a/docs/PR-feat-indexing-v2.md b/docs/PR-feat-indexing-v2.md new file mode 100644 index 0000000..c3c95da --- /dev/null +++ b/docs/PR-feat-indexing-v2.md @@ -0,0 +1,45 @@ +## Pull Request + +**Description:** + +Ports `videodb-python`'s `feat/add-indexing-v2` drop to the NodeJS SDK (`feat/indexing-v2` → `main`), bringing the SDK to parity across two related workstreams: + +1. **Indexing v2** — a new retrieval architecture (Search v2 dispatcher, Understanding runs, and an Index manifest) that supersedes the legacy single-shot search. +2. **Generation / compute stack** — persistent GPU **sandboxes**, async **generation jobs** for self-inference TTS/image models, and reusable **voice clones**, wired into the `generate*` methods. + +Also adds embed-code helpers and threads `sandboxId` through the inference-bearing methods. Bodies are authored in snake_case at the call site and responses are auto-camelCased by `HttpClient`, per existing SDK conventions; every new public symbol is re-exported from the `src/index.ts` barrel. Bumps the package `0.2.7` → `0.3.0` (minor) and updates the changelog. + +Scope: 25 files changed, ~+3,700 / −133. New source modules: `understanding.ts`, `indexManifest.ts`, `job.ts`, `sandbox.ts`, `sandboxModels.ts`, `voiceClone.ts`, `search/responses.ts`. + +**Changes:** +- [x] Search v2 on `Video` & `Collection` — `search()` dispatcher + `ask()` / `semanticSearch()` / `query()` / `aggregate()`, v2 `SearchResponse` / `AskResponse` envelopes, iterable `SearchResult`; `legacySearch()` behind a one-time deprecation warning +- [x] Understanding runs — `Understanding` + `UnderstandingAnalyzer` classes and `Video.understand()` / `getUnderstanding()` / `listUnderstandings()` / `deleteUnderstanding()` +- [x] Index manifest — `Index` / `IndexRecord` / `FieldSchema` / `RecordPage` and `Video.index()` / `getIndex()` / `listIndexes()` / `deleteIndex()`, with `IndexCapability` / `FieldGroup` constants +- [x] Sandboxes — `Sandbox` (lifecycle polling) + `Connection.createSandbox()` / `getSandbox()` / `listSandboxes()`, `SandboxTier` / `SandboxStatus` / `SandboxModel` +- [x] Generation jobs — `GenerationJob` + `Connection.getJobStatus()` / `waitForJob()`; `generateImage()` / `generateVoice()` / `generateText()` gain self-inference routing, `sandboxId`, and `wait` +- [x] Voice clones — `VoiceClone` CRUD on `Connection` and `Collection`; `voiceCloneId` in `generateVoice()` +- [x] RTStream understanding & indexing — `RTStreamUnderstanding` / `RTStreamIndex`, `understand()` / `index()`, alert management, record retrieval +- [x] Embed-code helpers — `getEmbedCode()` on `Video` / `Shot` / `Timeline` / RTStream, `buildIframeEmbedCode()` / `playerUrlToEmbedUrl()`, `playerUrl` fields +- [x] `sandboxId` threading into `Video.indexScenes()` / `indexVisuals()`, `Scene.describe()` (+ `modelConfig`), and RTStream indexing; `Connection.getAsyncResponse()` +- [x] Infra — `wait` flag + per-call poll overrides on `HttpClient`, monotonic-clock poll loop w/ `RequestTimeoutError`, `uploadBytes()` helper, `prompt_url` offload for large `generateText()` prompts +- [x] Bugfix: `Sandbox.isReady` / `waitForReady()` treat `alert` as ready (matching Python `READY_STATUSES`) and no longer report `provisioning` as ready +- [x] Version bump `0.2.7` → `0.3.0` + `CHANGELOG.md` + +**Related Issues:** +- N/A + +**Testing:** + +Validated against the companion **`nodejs-test-suite`** (symlinked to this package's built `dist/`), across pure, mocked, and live-API layers: + +- **Offline (pure + mock):** full suite green — sandbox constant/getter behavior, poll-loop branches (terminal-state, timeout, `alert`-resolves-ready), and `createSandbox` / `listSandboxes` wire payloads. `npm run typecheck` passes. +- **Live API:** `Sandbox` provision → wait-ready → stop lifecycle; `VoiceClone` CRUD (via `Connection` and `Collection`); `GenerationJob` polling; and all `generate*` methods (text, image, music, sound-effect, voice, video). The only live failures observed were pre-existing backend/account flakiness (semantic/title search, one upload stream-link generation, billing `getInvoices`) — unrelated to this change. + +Re-test after merge: because this rebuilds the SDK `dist/`, downstream consumers of `search()` should confirm the v2 dispatcher / legacy fallback behaves as expected, and the sandbox-routed `generate*` paths should be smoke-tested once a sandbox is provisioned. + +**Checklist:** +- [x] Code follows project coding standards +- [x] Tests have been added or updated +- [ ] Code Review +- [ ] Manual test after merge +- [ ] All checks passed diff --git a/package-lock.json b/package-lock.json index 9e47052..eead51a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "videodb", - "version": "0.2.7", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "videodb", - "version": "0.2.7", + "version": "0.3.0", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index 0e07e6e..e1ea320 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "videodb", - "version": "0.2.7", + "version": "0.3.0", "description": "A NodeJS wrapper for VideoDB's API written in TypeScript", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/src/constants.ts b/src/constants.ts index 28c33ef..545ee06 100644 --- a/src/constants.ts +++ b/src/constants.ts @@ -70,12 +70,68 @@ export const ApiPath = { start: 'start', stop: 'stop', export: 'export', + // indexing-v2 additions + indexes: 'indexes', + records: 'records', + understand: 'understand', + ask: 'ask', + semantic_search: 'semantic-search', + query: 'query', + aggregate: 'aggregate', + job: 'job', + sandbox: 'sandbox', + voice_clone: 'voice_clone', + identities: 'identities', + merge: 'merge', + split: 'split', + async_response: 'async-response', +} as const; + +/** + * Retrieval capabilities an index can be built for (the `use_for` values). + */ +export const IndexCapability = { + semantic: 'semantic', + query: 'query', + aggregate: 'aggregate', +} as const; + +/** + * Field groups that map artifact fields to retrieval capabilities. + */ +export const FieldGroup = { + semantic: 'semantic', + fts: 'fts', + filter: 'filter', + aggregate: 'aggregate', + sort: 'sort', +} as const; + +/** + * Compute tier a sandbox can be provisioned at. + */ +export const SandboxTier = { + small: 'small', + medium: 'medium', +} as const; + +/** + * Lifecycle status of a sandbox. + */ +export const SandboxStatus = { + provisioning: 'provisioning', + active: 'active', + stopping: 'stopping', + stopped: 'stopped', + failed: 'failed', + alert: 'alert', } as const; export const ResponseStatus = { processing: 'processing', in_progress: 'in progress', success: 'success', + complete: 'complete', } as const; export const MeetingStatus = { @@ -190,6 +246,10 @@ export const HttpClientDefaultValues = { max_retries: 3, timeout: 60 * 1000, backoff_factor: 0.1, + /** Max time (ms) to poll an async output URL before timing out. */ + maxPollTime: 500 * 1000, + /** Interval (ms) between polls of an async output URL. */ + pollInterval: 5 * 1000, } as const; export const MaxSupported = { diff --git a/src/core/collection.ts b/src/core/collection.ts index ea9bcef..2c28db1 100644 --- a/src/core/collection.ts +++ b/src/core/collection.ts @@ -25,8 +25,9 @@ import type { AudioResponse, ImageResponse, } from '@/types/response'; +import type { SearchResponse as SearchApiResponse } from '@/types/response'; import { HttpClient } from '@/utils/httpClient'; -import { uploadToServer } from '@/utils/upload'; +import { uploadToServer, uploadBytes } from '@/utils/upload'; import { DefaultSearchType, DefaultIndexType, @@ -34,14 +35,25 @@ import { } from '@/core/config'; import { SearchFactory } from './search'; import { SearchResult } from './search/searchResult'; +import { + AskResponse, + SearchResponse, + warnLegacySearchOnce, + type AskResponseData, + type SearchResponseData, +} from './search/responses'; import { Video } from './video'; import { Audio } from './audio'; import { Image } from './image'; import { Meeting } from './meeting'; import { CaptureSession } from './captureSession'; import { RTStream, RTStreamSearchResult, RTStreamShot } from './rtstream'; +import { GenerationJob, type JobData } from './job'; +import { VoiceClone, type VoiceCloneBase } from './voiceClone'; import { VideodbError } from '@/utils/error'; +const MAX_GENERATE_TEXT_PAYLOAD_SIZE = 250 * 1024; + const { video, audio, @@ -55,8 +67,46 @@ const { title, meeting, record, + ask, + semantic_search, + query: queryPath, + aggregate, + voice_clone, } = ApiPath; +/** + * Options for {@link Collection.search}. Combines new-style (v2) params with + * legacy-shaped params; passing any legacy param (or `namespace: "rtstream"`) + * routes the call to {@link Collection.legacySearch}. + */ +export interface CollectionSearchOptions { + // legacy-shaped + searchType?: SearchType; + indexType?: IndexType; + resultThreshold?: number; + scoreThreshold?: number; + dynamicScorePercentage?: number; + filter?: Array>; + sortDocsOn?: string; + namespace?: string; + sceneIndexId?: string; + indexId?: string; + algorithm?: string; + // new (v2) + topK?: number; + mode?: string; + returnFields?: unknown[] | Record | string; + includeClip?: boolean; + sessionId?: string; + config?: Record; + // unsupported in search() + indexName?: string; + indexNames?: string[] | string; + indexIds?: string[] | string; + // internal — rejected + deepsearchConfig?: unknown; +} + /** * The base VideoDB class * @remarks @@ -243,7 +293,270 @@ export class Collection implements ICollection { * @param sceneIndexId - [optional] Filter by specific scene index * @returns SearchResult or RTStreamSearchResult object */ + /** + * Search this collection. + * + * New search is used by default. Calls that pass legacy-shaped parameters + * (or `namespace: "rtstream"`) are routed to {@link legacySearch} with a + * one-time warning. + * @param query - Search query + * @param options - Search options (new-style or legacy-shaped) + */ public search = async ( + query: string, + optionsOrSearchType: CollectionSearchOptions | SearchType = {}, + indexType?: IndexType, + resultThreshold?: number, + scoreThreshold?: number, + dynamicScorePercentage?: number, + filter?: Array>, + sortDocsOn?: string, + namespace?: string, + sceneIndexId?: string + ): Promise => { + // Back-compat: the pre-v2 signature was fully positional — + // search(query, searchType, indexType, resultThreshold, scoreThreshold, + // dynamicScorePercentage, filter, sortDocsOn, namespace, sceneIndexId) + // Detect it (2nd arg is a SearchType string, or any trailing positional arg + // is present) and fold it into an options object, forcing legacy routing — + // mirroring videodb-python's `has_old = bool(args)`. + const positionalLegacy = + typeof optionsOrSearchType === 'string' || + indexType !== undefined || + resultThreshold !== undefined || + scoreThreshold !== undefined || + dynamicScorePercentage !== undefined || + filter !== undefined || + sortDocsOn !== undefined || + namespace !== undefined || + sceneIndexId !== undefined; + + const options: CollectionSearchOptions = positionalLegacy + ? { + searchType: + typeof optionsOrSearchType === 'string' + ? (optionsOrSearchType as SearchType) + : undefined, + indexType, + resultThreshold, + scoreThreshold, + dynamicScorePercentage, + filter, + sortDocsOn, + namespace, + sceneIndexId, + } + : (optionsOrSearchType as CollectionSearchOptions); + + const oldParams: (keyof CollectionSearchOptions)[] = [ + 'searchType', + 'indexType', + 'resultThreshold', + 'scoreThreshold', + 'dynamicScorePercentage', + 'sceneIndexId', + 'indexId', + 'algorithm', + 'sortDocsOn', + 'namespace', + ]; + const newParams: (keyof CollectionSearchOptions)[] = [ + 'topK', + 'mode', + 'returnFields', + 'includeClip', + 'sessionId', + 'config', + ]; + const unsupported: (keyof CollectionSearchOptions)[] = [ + 'indexName', + 'indexNames', + 'indexId', + 'indexIds', + ]; + + const hasOld = + positionalLegacy || + oldParams.some(k => options[k] !== undefined && options[k] !== null); + const hasNew = newParams.some( + k => options[k] !== undefined && options[k] !== null + ); + const hasUnsupported = unsupported.some( + k => options[k] !== undefined && options[k] !== null + ); + + if (options.deepsearchConfig != null) { + throw new VideodbError( + 'deepsearchConfig is internal and cannot be passed to search().' + ); + } + if (hasOld && (hasNew || hasUnsupported)) { + throw new VideodbError( + 'Cannot mix legacy search params with new search params. ' + + 'Use search(...) for new search or legacySearch(...) for legacy search.' + ); + } + if (hasUnsupported) { + throw new VideodbError( + 'indexName/indexNames/indexId/indexIds are not supported in search(). ' + + 'Use semanticSearch(), query(), or aggregate() for index-specific calls.' + ); + } + + if (hasOld) { + warnLegacySearchOnce(); + return this.legacySearch( + query, + options.searchType, + options.indexType, + options.resultThreshold, + options.scoreThreshold, + options.dynamicScorePercentage, + options.filter, + options.sortDocsOn, + options.namespace, + options.sceneIndexId + ); + } + + return this.#newSearch(query, options); + }; + + #newSearch = async ( + query: string, + options: CollectionSearchOptions + ): Promise => { + const payload: Record = { query }; + if (options.topK != null) payload.top_k = options.topK; + if (options.mode != null) payload.mode = options.mode; + if (options.returnFields != null) + payload.return_fields = options.returnFields; + if (options.includeClip != null) payload.include_clip = options.includeClip; + if (options.sessionId != null) payload.session_id = options.sessionId; + if (options.config != null) payload.config = options.config; + const res = await this.#vhttp.post, typeof payload>( + [collection, this.id, search, 'v2'], + payload + ); + return new SearchResponse(this.#vhttp, res.data as SearchResponseData); + }; + + /** + * Ask a question and get an answer generated from retrieved collection context. + */ + public ask = async ( + question: string, + topK: number = 15, + mode: string = 'default', + includeSources: boolean = false + ): Promise => { + const res = await this.#vhttp.post, object>( + [collection, this.id, ask], + { question, top_k: topK, mode, include_sources: includeSources } + ); + return new AskResponse(this.#vhttp, res.data as AskResponseData); + }; + + /** Semantic search across one or more indexes in this collection. */ + public semanticSearch = async ( + query: string, + options: { + indexNames?: string[] | string; + topK?: number; + scoreThreshold?: number; + filter?: unknown[] | Record; + returnFields?: unknown[] | Record | string; + indexIds?: string[] | string; + } = {} + ): Promise => { + const res = await this.#vhttp.post( + [collection, this.id, semantic_search], + { + query, + index_names: options.indexNames, + index_ids: options.indexIds, + top_k: options.topK ?? 10, + score_threshold: options.scoreThreshold, + filter: options.filter, + return_fields: options.returnFields, + } + ); + return new SearchResult(this.#vhttp, res.data); + }; + + /** Structured query against an index in this collection. */ + public query = async ( + options: { + indexName?: string; + filter?: unknown[] | Record; + limit?: number; + returnFields?: unknown[] | Record | string; + sort?: string | [string, string][]; + indexId?: string; + } = {} + ): Promise => { + const res = await this.#vhttp.post( + [collection, this.id, queryPath], + { + index_name: options.indexName, + index_id: options.indexId, + filter: options.filter, + limit: options.limit ?? 100, + return_fields: options.returnFields, + sort: options.sort, + } + ); + return new SearchResult(this.#vhttp, res.data); + }; + + /** Aggregate over an index in this collection. */ + public aggregate = async ( + options: { + indexName?: string; + filter?: unknown[] | Record; + groupBy?: string; + metric?: string; + limit?: number; + sort?: string | [string, string][]; + indexId?: string; + } = {} + ): Promise | Record[]> => { + // convert:false — aggregate buckets are keyed by user field names/values. + const res = await this.#vhttp.post< + Record | Record[], + object + >( + [collection, this.id, aggregate], + { + index_name: options.indexName, + index_id: options.indexId, + filter: options.filter, + group_by: options.groupBy, + metric: options.metric ?? 'count', + limit: options.limit ?? 100, + sort: options.sort, + }, + undefined, + { convert: false } + ); + return res.data; + }; + + /** + * Legacy search (pre-v2), including RTStream namespace search. + * @param query - Search query + * @param searchType - [optional] Type of search to be performed + * @param indexType - [optional] Index Type + * @param resultThreshold - [optional] Result Threshold + * @param scoreThreshold - [optional] Score Threshold + * @param dynamicScorePercentage - [optional] Percentage of dynamic score to consider + * @param filter - [optional] Additional metadata filters + * @param sortDocsOn - [optional] Sort docs within each video by "score" or "start" + * @param namespace - [optional] Search namespace ("rtstream" to search RTStreams) + * @param sceneIndexId - [optional] Filter by specific scene index + * @returns SearchResult or RTStreamSearchResult object + */ + public legacySearch = async ( query: string, searchType?: SearchType, indexType?: IndexType, @@ -259,7 +572,8 @@ export class Collection implements ICollection { if (namespace === 'rtstream') { const data: Record = { query }; if (sceneIndexId !== undefined) data.scene_index_id = sceneIndexId; - if (resultThreshold !== undefined) data.result_threshold = resultThreshold; + if (resultThreshold !== undefined) + data.result_threshold = resultThreshold; if (scoreThreshold !== undefined) data.score_threshold = scoreThreshold; if (dynamicScorePercentage !== undefined) data.dynamic_score_percentage = dynamicScorePercentage; @@ -414,15 +728,41 @@ export class Collection implements ICollection { public generateImage = async ( prompt: string, aspectRatio: '1:1' | '9:16' | '16:9' | '4:3' | '3:4' = '1:1', - callbackUrl?: string - ): Promise => { - const res = await this.#vhttp.post( + callbackUrl?: string, + options: { + modelName?: string; + config?: Record; + sandboxId?: string; + wait?: boolean; + pollInterval?: number; + timeout?: number; + } = {} + ): Promise => { + const body: Record = { + prompt, + aspect_ratio: aspectRatio, + callback_url: callbackUrl, + }; + if (options.modelName != null) body.model_name = options.modelName; + if (options.config != null) body.config = options.config; + if (options.sandboxId != null) body.sandbox_id = options.sandboxId; + + const res = await this.#vhttp.post( [collection, this.id, generate, image], - { prompt, aspect_ratio: aspectRatio, callback_url: callbackUrl } + body ); - if (res.data) { - return new Image(this.#vhttp, res.data); + const data = res.data; + if (!data) return undefined; + if (data.jobId || data.job_id) { + const job = GenerationJob.fromData(this.#vhttp, data, 'image'); + return options.wait + ? ((await job.wait( + options.timeout ?? 600, + options.pollInterval ?? 5 + )) as Image | JobData) + : job; } + return new Image(this.#vhttp, data as unknown as ImageBase); }; /** @@ -462,7 +802,13 @@ export class Collection implements ICollection { ): Promise