From 4434e8010fc0406c3e7e9b3a19771bbb94660316 Mon Sep 17 00:00:00 2001 From: Duda Nogueira Date: Mon, 3 Aug 2026 18:27:36 -0300 Subject: [PATCH 1/7] feat(backup): support incremental base backup ID in the v2 creator Adds .withIncrementalBaseBackupId() to BackupCreator so the incremental_base_backup_id field is sent on backup creation, plus validation rejecting an empty base ID or one equal to the backup being created. --- src/backup/backupCreator.ts | 18 ++++++++++++++++++ src/backup/validation.ts | 15 +++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/backup/backupCreator.ts b/src/backup/backupCreator.ts index 5690f73e..0d60c57c 100644 --- a/src/backup/backupCreator.ts +++ b/src/backup/backupCreator.ts @@ -14,6 +14,7 @@ import { validateBackupId, validateExcludeClassNames, validateIncludeClassNames, + validateIncrementalBaseBackupId, } from './validation.js'; const WAIT_INTERVAL = 1000; @@ -26,6 +27,7 @@ export default class BackupCreator extends CommandBase { private statusGetter: BackupCreateStatusGetter; private waitForCompletion!: boolean; private config?: BackupConfig; + private incrementalBaseBackupId?: string; constructor(client: Connection, statusGetter: BackupCreateStatusGetter) { super(client); @@ -70,12 +72,27 @@ export default class BackupCreator extends CommandBase { return this; } + /** + * The ID of an existing backup to use as the base for a file-based incremental backup. + * Only the files that changed since the base backup are included in the new backup. + * + * `backupId` is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the + * `id` returned by a previous backup creation. + * + * Requires Weaviate v1.37.0 or higher. + */ + withIncrementalBaseBackupId(backupId: string) { + this.incrementalBaseBackupId = backupId; + return this; + } + validate = (): void => { this.addErrors([ ...validateIncludeClassNames(this.includeClassNames), ...validateExcludeClassNames(this.excludeClassNames), ...validateBackend(this.backend), ...validateBackupId(this.backupId), + ...validateIncrementalBaseBackupId(this.incrementalBaseBackupId, this.backupId), ]); }; @@ -90,6 +107,7 @@ export default class BackupCreator extends CommandBase { config: this.config, include: this.includeClassNames, exclude: this.excludeClassNames, + incremental_base_backup_id: this.incrementalBaseBackupId, } as BackupCreateRequest; if (this.waitForCompletion) { diff --git a/src/backup/validation.ts b/src/backup/validation.ts index 778ecb87..4a9ceffb 100644 --- a/src/backup/validation.ts +++ b/src/backup/validation.ts @@ -45,3 +45,18 @@ export function validateBackupId(backupId?: string) { } return []; } + +export function validateIncrementalBaseBackupId(incrementalBaseBackupId?: string, backupId?: string) { + if (incrementalBaseBackupId === undefined || incrementalBaseBackupId === null) { + return []; + } + if (!isValidStringProperty(incrementalBaseBackupId)) { + return [ + 'string incrementalBaseBackupId must be a non-empty string - set with .withIncrementalBaseBackupId(backupId)', + ]; + } + if (incrementalBaseBackupId === backupId) { + return ['incrementalBaseBackupId must be different from the ID of the backup being created']; + } + return []; +} From e87bed04d061a1aa54dd668ab9e5c54c54010677 Mon Sep 17 00:00:00 2001 From: Duda Nogueira Date: Mon, 3 Aug 2026 18:28:37 -0300 Subject: [PATCH 2/7] feat(backup): expose incremental backups in the collections client client.backup.create() and collection.backup.create() now accept incrementalBaseBackupId. The request is gated client-side on Weaviate >=1.37.0 via DbVersionSupport.supportsIncrementalBackups() and throws WeaviateUnsupportedFeatureError on older servers. The base backup ID is also surfaced on getCreateStatus() and list(), which Weaviate only returns to root users. --- src/collections/backup/client.ts | 45 +++++++++++++++++++++++----- src/collections/backup/collection.ts | 33 ++++++++++++++++---- src/collections/backup/index.ts | 10 +++++-- src/collections/backup/types.ts | 16 ++++++++++ src/collections/collection/index.ts | 2 +- src/index.ts | 2 +- src/openapi/types.ts | 1 + src/utils/dbVersion.ts | 7 +++++ 8 files changed, 100 insertions(+), 16 deletions(-) diff --git a/src/collections/backup/client.ts b/src/collections/backup/client.ts index 4a655f10..edbb87b0 100644 --- a/src/collections/backup/client.ts +++ b/src/collections/backup/client.ts @@ -5,7 +5,11 @@ import { BackupRestoreStatusGetter, BackupRestorer, } from '../../backup/index.js'; -import { validateBackend, validateBackupId } from '../../backup/validation.js'; +import { + validateBackend, + validateBackupId, + validateIncrementalBaseBackupId, +} from '../../backup/validation.js'; import Connection from '../../connection/index.js'; import { WeaviateBackupCanceled, @@ -14,24 +18,27 @@ import { WeaviateInvalidInputError, WeaviateUnexpectedResponseError, WeaviateUnexpectedStatusCodeError, + WeaviateUnsupportedFeatureError, } from '../../errors.js'; import { BackupCreateResponse, BackupCreateStatusResponse, + BackupListResponse, BackupRestoreResponse, } from '../../openapi/types.js'; +import { DbVersionSupport } from '../../utils/dbVersion.js'; import { BackupArgs, BackupCancelArgs, - BackupConfigCreate, BackupConfigRestore, + BackupCreateArgs, BackupReturn, BackupStatusArgs, BackupStatusReturn, ListBackupOptions, } from './types.js'; -export const backup = (connection: Connection): Backup => { +export const backup = (connection: Connection, dbVersionSupport: DbVersionSupport): Backup => { const parseStatus = (res: BackupCreateStatusResponse | BackupRestoreResponse): BackupStatusReturn => { if (res.id === undefined) { throw new WeaviateUnexpectedResponseError('Backup ID is undefined in response'); @@ -47,6 +54,9 @@ export const backup = (connection: Connection): Backup => { error: res.error, path: res.path, status: res.status, + // Only returned by Weaviate >=1.37, for incremental backups, and only to root users + incrementalBaseBackupId: + 'incremental_base_backup_id' in res ? res.incremental_base_backup_id || undefined : undefined, }; }; const parseResponse = (res: BackupCreateResponse | BackupRestoreResponse): BackupReturn => { @@ -109,10 +119,22 @@ export const backup = (connection: Connection): Backup => { return true; }, - create: async (args: BackupArgs): Promise => { + create: async (args: BackupCreateArgs): Promise => { let builder = new BackupCreator(connection, new BackupCreateStatusGetter(connection)) .withBackupId(args.backupId) .withBackend(args.backend); + if (args.incrementalBaseBackupId !== undefined) { + const baseBackupId = args.incrementalBaseBackupId.toLowerCase(); + const errors = validateIncrementalBaseBackupId(baseBackupId, args.backupId.toLowerCase()); + if (errors.length > 0) { + throw new WeaviateInvalidInputError(errors.join(', ')); + } + const check = await dbVersionSupport.supportsIncrementalBackups(); + if (!check.supports) { + throw new WeaviateUnsupportedFeatureError(check.message); + } + builder = builder.withIncrementalBaseBackupId(baseBackupId); + } if (args.includeCollections) { builder = builder.withIncludeClassNames(...args.includeCollections); } @@ -213,7 +235,12 @@ export const backup = (connection: Connection): Backup => { if (opts?.startedAtAsc) { url += '?order=asc'; } - return connection.get(url); + return connection.get(url).then((res) => + res.map(({ incremental_base_backup_id: baseBackupId, ...rest }) => ({ + ...rest, + incrementalBaseBackupId: baseBackupId || undefined, + })) + ) as Promise; }, }; }; @@ -231,13 +258,17 @@ export interface Backup { /** * Create a backup of the database. * - * @param {BackupArgs} args The arguments for the request. + * Pass `incrementalBaseBackupId` to create a file-based incremental backup, which only + * contains the files that changed since the given base backup. Requires Weaviate `v1.37.0` or higher. + * + * @param {BackupCreateArgs} args The arguments for the request. * @returns {Promise} The response from Weaviate. * @throws {WeaviateInvalidInputError} If the input is invalid. + * @throws {WeaviateUnsupportedFeatureError} If `incrementalBaseBackupId` is used with Weaviate <1.37.0. * @throws {WeaviateBackupFailed} If the backup creation fails. * @throws {WeaviateBackupCanceled} If the backup creation is canceled. */ - create(args: BackupArgs): Promise; + create(args: BackupCreateArgs): Promise; /** * Get the status of a backup creation. * diff --git a/src/collections/backup/collection.ts b/src/collections/backup/collection.ts index 377d8009..04c1a528 100644 --- a/src/collections/backup/collection.ts +++ b/src/collections/backup/collection.ts @@ -1,5 +1,6 @@ import { Backend } from '../../backup/index.js'; import Connection from '../../connection/index.js'; +import { DbVersionSupport } from '../../utils/dbVersion.js'; import { backup } from './client.js'; import { BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js'; @@ -13,10 +14,28 @@ export type BackupCollectionArgs = { waitForCompletion?: boolean; }; -export const backupCollection = (connection: Connection, name: string) => { - const handler = backup(connection); +/** The arguments required to create a backup of a collection. */ +export type BackupCollectionCreateArgs = BackupCollectionArgs & { + /** + * The ID of an existing backup to use as the base for a file-based incremental backup. + * If set, only the files that have changed since the base backup are included in the new backup. + * + * This is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the `id` + * returned by a previous backup creation. + * + * Requires Weaviate `v1.37.0` or higher. + */ + incrementalBaseBackupId?: string; +}; + +export const backupCollection = ( + connection: Connection, + name: string, + dbVersionSupport: DbVersionSupport +) => { + const handler = backup(connection, dbVersionSupport); return { - create: (args: BackupCollectionArgs) => + create: (args: BackupCollectionCreateArgs) => handler.create({ ...args, includeCollections: [name], @@ -35,13 +54,17 @@ export interface BackupCollection { /** * Create a backup of this collection. * - * @param {BackupArgs} args The arguments for the request. + * Pass `incrementalBaseBackupId` to create a file-based incremental backup, which only + * contains the files that changed since the given base backup. Requires Weaviate `v1.37.0` or higher. + * + * @param {BackupCollectionCreateArgs} args The arguments for the request. * @returns {Promise} The response from Weaviate. * @throws {WeaviateInvalidInputError} If the input is invalid. + * @throws {WeaviateUnsupportedFeatureError} If `incrementalBaseBackupId` is used with Weaviate <1.37.0. * @throws {WeaviateBackupFailed} If the backup creation fails. * @throws {WeaviateBackupCanceled} If the backup creation is canceled. */ - create(args: BackupCollectionArgs): Promise; + create(args: BackupCollectionCreateArgs): Promise; /** * Get the status of a backup. * diff --git a/src/collections/backup/index.ts b/src/collections/backup/index.ts index e4c6f1b1..baea10f6 100644 --- a/src/collections/backup/index.ts +++ b/src/collections/backup/index.ts @@ -1,3 +1,9 @@ export type { Backup } from './client.js'; -export type { BackupCollection, BackupCollectionArgs } from './collection.js'; -export type { BackupArgs, BackupConfigCreate, BackupConfigRestore, BackupStatusArgs } from './types.js'; +export type { BackupCollection, BackupCollectionArgs, BackupCollectionCreateArgs } from './collection.js'; +export type { + BackupArgs, + BackupConfigCreate, + BackupConfigRestore, + BackupCreateArgs, + BackupStatusArgs, +} from './types.js'; diff --git a/src/collections/backup/types.ts b/src/collections/backup/types.ts index c76e5a25..546694a6 100644 --- a/src/collections/backup/types.ts +++ b/src/collections/backup/types.ts @@ -16,6 +16,8 @@ export type BackupStatusReturn = { status: BackupStatus; /** Size of the backup in Gibs */ size?: number; + /** The ID of the base backup this incremental backup was built on; undefined if the backup is not incremental. */ + incrementalBaseBackupId?: string; }; /** The return type of a backup creation or restoration operation */ @@ -64,6 +66,20 @@ export type BackupArgs = { config?: C; }; +/** The arguments required to create a backup. */ +export type BackupCreateArgs = BackupArgs & { + /** + * The ID of an existing backup to use as the base for a file-based incremental backup. + * If set, only the files that have changed since the base backup are included in the new backup. + * + * This is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the `id` + * returned by a previous backup creation. + * + * Requires Weaviate `v1.37.0` or higher. + */ + incrementalBaseBackupId?: string; +}; + /** The arguments required to get the status of a backup. */ export type BackupStatusArgs = { /** The ID of the backup. */ diff --git a/src/collections/collection/index.ts b/src/collections/collection/index.ts index c4164352..c0676216 100644 --- a/src/collections/collection/index.ts +++ b/src/collections/collection/index.ts @@ -138,7 +138,7 @@ const collection = ( ); return { aggregate: aggregateCollection, - backup: backupCollection(connection, capitalizedName), + backup: backupCollection(connection, capitalizedName, dbVersionSupport), config: config(connection, capitalizedName, dbVersionSupport, tenant), data: data(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant), filter: filter(), diff --git a/src/index.ts b/src/index.ts index b3fa9e7d..b9cf3926 100644 --- a/src/index.ts +++ b/src/index.ts @@ -233,7 +233,7 @@ async function client(params: ClientParams): Promise { const ifc: WeaviateClient = { alias: alias(connection), - backup: backup(connection), + backup: backup(connection, dbVersionSupport), batch: batch(connection, dbVersionSupport), cluster: cluster(connection), collections: collections(connection, dbVersionSupport), diff --git a/src/openapi/types.ts b/src/openapi/types.ts index 4cecdcfc..edf9e939 100644 --- a/src/openapi/types.ts +++ b/src/openapi/types.ts @@ -15,6 +15,7 @@ export type DataObject = definitions['Object']; export type BackupCreateRequest = definitions['BackupCreateRequest']; export type BackupCreateResponse = definitions['BackupCreateResponse']; export type BackupCreateStatusResponse = definitions['BackupCreateStatusResponse']; +export type BackupListResponse = definitions['BackupListResponse']; export type BackupRestoreRequest = definitions['BackupRestoreRequest']; export type BackupRestoreResponse = definitions['BackupRestoreResponse']; export type BackupRestoreStatusResponse = definitions['BackupRestoreStatusResponse']; diff --git a/src/utils/dbVersion.ts b/src/utils/dbVersion.ts index b37c328f..3bdf8c95 100644 --- a/src/utils/dbVersion.ts +++ b/src/utils/dbVersion.ts @@ -163,6 +163,13 @@ export class DbVersionSupport { message: this.errorMessage('Tokenize endpoint stopwords / stopwordPresets', version.show(), '1.37.2'), })); + supportsIncrementalBackups = () => + this.dbVersionProvider.getVersion().then((version) => ({ + version, + supports: version.isAtLeast(1, 37, 0), + message: this.errorMessage('Incremental backups', version.show(), '1.37.0'), + })); + supportsServerSideDefaultVectorIndexType = () => this.dbVersionProvider.getVersion().then((version) => ({ version, From d4d816580d271865efe91c14f7193c5888fbfc45 Mon Sep 17 00:00:00 2001 From: Duda Nogueira Date: Mon, 3 Aug 2026 18:33:13 -0300 Subject: [PATCH 3/7] test(backup): cover incremental backup creation and version gating Mock tests assert the payload sent on create, the lowercasing of the base ID, the >=1.37.0 gate and the parsing of the base ID in list() and getCreateStatus(). Integration tests create an incremental backup on top of a base backup and restore it. --- test/collections/backup/integration.test.ts | 61 +++++++- test/collections/backup/mock.test.ts | 163 +++++++++++++++++++- 2 files changed, 222 insertions(+), 2 deletions(-) diff --git a/test/collections/backup/integration.test.ts b/test/collections/backup/integration.test.ts index db90067d..4d1dff33 100644 --- a/test/collections/backup/integration.test.ts +++ b/test/collections/backup/integration.test.ts @@ -2,7 +2,7 @@ /* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */ /* eslint-disable no-await-in-loop */ import { afterAll, beforeAll, describe, expect, it, test } from 'vitest'; -import { WeaviateBackupFailed } from '../../../src/errors.js'; +import { WeaviateBackupFailed, WeaviateInvalidInputError } from '../../../src/errors.js'; import weaviate, { Backend, Collection, WeaviateClient } from '../../../src/index.js'; import { requireAtLeast } from '../../../test/version.js'; @@ -258,6 +258,65 @@ describe('Integration testing of backups', () => { }); }); + requireAtLeast(1, 37, 0).describe('incremental backups', () => { + it('creates an incremental backup on top of a base backup', async () => { + const client = await clientPromise; + const collection = await client.collections + .create({ name: 'TestIncrementalBackup' }) + .then((col) => col.data.insert().then(() => col)); + + const base = await client.backup.create({ + backupId: randomBackupId(), + backend: 'filesystem', + includeCollections: [collection.name], + waitForCompletion: true, + }); + expect(base.status).toBe('SUCCESS'); + + // Add data so that the incremental backup has something to pick up + await collection.data.insert(); + + const incremental = await client.backup.create({ + backupId: randomBackupId(), + backend: 'filesystem', + includeCollections: [collection.name], + incrementalBaseBackupId: base.id, + waitForCompletion: true, + }); + expect(incremental.status).toBe('SUCCESS'); + + // Weaviate only reports the base backup ID to root users, so treat it as optional + if (incremental.incrementalBaseBackupId !== undefined) { + expect(incremental.incrementalBaseBackupId).toBe(base.id); + } + + // The incremental backup must be restorable + await client.collections.delete(collection.name); + const restored = await client.backup.restore({ + backupId: incremental.id, + backend: 'filesystem', + includeCollections: [collection.name], + waitForCompletion: true, + }); + expect(restored.status).toBe('SUCCESS'); + await expect(collection.length()).resolves.toBe(2); + + await client.collections.delete(collection.name); + }); + + it('rejects an incremental backup based on itself', async () => { + const client = await clientPromise; + const backupId = randomBackupId(); + await expect( + client.backup.create({ + backupId, + backend: 'filesystem', + incrementalBaseBackupId: backupId, + }) + ).rejects.toThrow(WeaviateInvalidInputError); + }); + }); + function randomBackupId() { return 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); } diff --git a/test/collections/backup/mock.test.ts b/test/collections/backup/mock.test.ts index ab4ca53a..d7a1e0e7 100644 --- a/test/collections/backup/mock.test.ts +++ b/test/collections/backup/mock.test.ts @@ -3,7 +3,11 @@ import { Server as HttpServer } from 'http'; import { Server as GrpcServer, createServer } from 'nice-grpc'; import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { BackupStatus } from '../../../src/collections/backup/types.js'; -import { WeaviateBackupCanceled } from '../../../src/errors.js'; +import { + WeaviateBackupCanceled, + WeaviateInvalidInputError, + WeaviateUnsupportedFeatureError, +} from '../../../src/errors.js'; import weaviate, { WeaviateClient } from '../../../src/index.js'; import { HealthCheckRequest, @@ -13,8 +17,10 @@ import { HealthServiceImplementation, } from '../../../src/proto/google/health/v1/health.js'; import { + BackupCreateRequest, BackupCreateResponse, BackupCreateStatusResponse, + BackupListResponse, BackupRestoreResponse, } from '../../../src/v2/index.js'; @@ -186,3 +192,158 @@ describe('Mock testing of backup cancellation', () => { afterAll(() => mock.close()); }); + +const BASE_BACKUP_ID = 'test-backup-base'; + +/** Mocks the backup endpoints, recording the payload sent by the client on creation. */ +class IncrementalMock { + private grpc: GrpcServer; + private http: HttpServer; + static lastCreateRequest: BackupCreateRequest; + + constructor(grpc: GrpcServer, http: HttpServer) { + this.grpc = grpc; + this.http = http; + } + + public static use = async (version: string, httpPort: number, grpcPort: number) => { + const httpApp = express(); + httpApp.use(express.json()); + httpApp.get('/v1/meta', (req, res) => res.send({ version })); + + httpApp.post(`/v1/backups/${BACKEND}`, (req, res: Response) => { + IncrementalMock.lastCreateRequest = req.body; + res.send({ + id: req.body.id, + backend: BACKEND, + classes: ['Article'], + path: 'path/to/backup', + status: 'STARTED', + }); + }); + httpApp.get(`/v1/backups/${BACKEND}/:id`, (req, res: Response) => + res.send({ + id: req.params.id, + backend: BACKEND, + path: 'path/to/backup', + status: 'SUCCESS', + incremental_base_backup_id: IncrementalMock.lastCreateRequest?.incremental_base_backup_id, + }) + ); + httpApp.get(`/v1/backups/${BACKEND}`, (req, res: Response) => + res.send([ + { id: BASE_BACKUP_ID, classes: ['Article'], status: 'SUCCESS', incremental_base_backup_id: '' }, + { + id: BACKUP_ID, + classes: ['Article'], + status: 'SUCCESS', + incremental_base_backup_id: BASE_BACKUP_ID, + }, + ]) + ); + + const healthMockImpl: HealthServiceImplementation = { + check: (request: HealthCheckRequest): Promise => + Promise.resolve(HealthCheckResponse.create({ status: HealthCheckResponse_ServingStatus.SERVING })), + watch: vi.fn(), + }; + + const grpc = createServer(); + grpc.add(HealthDefinition, healthMockImpl); + + httpApp.on('error', (error) => console.error('HTTP Server Error:', error)); + + await grpc.listen(`localhost:${grpcPort}`); + const http = await httpApp.listen(httpPort); + return new IncrementalMock(grpc, http); + }; + + public close = () => Promise.all([this.http.close(), this.grpc.shutdown()]); +} + +describe('Mock testing of incremental backups', () => { + describe('with a supported Weaviate version', () => { + let client: WeaviateClient; + let mock: IncrementalMock; + + beforeAll(async () => { + mock = await IncrementalMock.use('1.37.0', 8914, 8915); + client = await weaviate.connectToLocal({ port: 8914, grpcPort: 8915 }); + }); + + it('should send the base backup ID when creating an incremental backup', async () => { + const res = await client.backup.create({ + backupId: BACKUP_ID, + backend: BACKEND, + incrementalBaseBackupId: BASE_BACKUP_ID, + waitForCompletion: true, + }); + expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBe(BASE_BACKUP_ID); + expect(res.status).toBe('SUCCESS'); + expect(res.incrementalBaseBackupId).toBe(BASE_BACKUP_ID); + }); + + it('should lowercase the base backup ID', async () => { + await client.backup.create({ + backupId: BACKUP_ID, + backend: BACKEND, + incrementalBaseBackupId: 'Test-Backup-BASE', + waitForCompletion: true, + }); + expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBe(BASE_BACKUP_ID); + }); + + it('should not send the field for a regular backup', async () => { + await client.backup.create({ backupId: BACKUP_ID, backend: BACKEND }); + expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBeUndefined(); + }); + + it('should throw if the base backup is the backup being created', async () => { + const promise = client.backup.create({ + backupId: BACKUP_ID, + backend: BACKEND, + incrementalBaseBackupId: BACKUP_ID, + }); + await expect(promise).rejects.toThrow(WeaviateInvalidInputError); + }); + + it('should surface the base backup ID when listing backups', async () => { + const backups = await client.backup.list(BACKEND); + expect(backups[0].incrementalBaseBackupId).toBeUndefined(); + expect(backups[1].incrementalBaseBackupId).toBe(BASE_BACKUP_ID); + }); + + it('should surface the base backup ID when getting the creation status', async () => { + const status = await client.backup.getCreateStatus({ backupId: BACKUP_ID, backend: BACKEND }); + expect(status.incrementalBaseBackupId).toBeUndefined(); // last create was a regular backup + }); + + afterAll(() => mock.close()); + }); + + describe('with an unsupported Weaviate version', () => { + let client: WeaviateClient; + let mock: IncrementalMock; + + beforeAll(async () => { + mock = await IncrementalMock.use('1.36.0', 8916, 8917); + client = await weaviate.connectToLocal({ port: 8916, grpcPort: 8917 }); + }); + + it('should throw when requesting an incremental backup', async () => { + const promise = client.backup.create({ + backupId: BACKUP_ID, + backend: BACKEND, + incrementalBaseBackupId: BASE_BACKUP_ID, + }); + await expect(promise).rejects.toThrow(WeaviateUnsupportedFeatureError); + }); + + it('should still allow regular backups', async () => { + const res = await client.backup.create({ backupId: BACKUP_ID, backend: BACKEND }); + expect(res.status).toBe('STARTED'); + }); + + afterAll(() => mock.close()); + }); +}); From 58507b918a84fd4cd6933b2361737d5a402b2d01 Mon Sep 17 00:00:00 2001 From: Duda Nogueira Date: Fri, 14 Aug 2026 12:35:22 -0300 Subject: [PATCH 4/7] fix(backup): address review on incremental backups - Gate the v2 builder on the server version too. `weaviateV2` is public API and Weaviate below 1.37.0 silently writes a full backup when it sees `incremental_base_backup_id`, so the builder now throws WeaviateUnsupportedFeatureError instead. - Lowercase the base backup ID in `withIncrementalBaseBackupId` and compare IDs case-insensitively in `validateIncrementalBaseBackupId`, so `backupId: 'B1'` with base `'b1'` is rejected on both the v2 and v3 paths. - Surface `size` from `parseStatus`; it was declared on BackupStatusReturn but never populated. - Document that `incrementalBaseBackupId` is undefined for non-root callers, on servers below 1.37.6 (when the read-back was added), and on a create without `waitForCompletion`. - Collapse the duplicated `incrementalBaseBackupId` doc block to one line and derive BackupCollectionCreateArgs from BackupCreateArgs. - Make the integration coverage honest: assert the field is undefined on the anonymous instance, and add a root-user round-trip against the RBAC instance, which now enables backup-filesystem. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01F65TAaZWcf7unm86QQ3FA8 --- ci/docker-compose-rbac.yml | 6 +- src/backup/backupCreator.ts | 45 ++++++++--- src/backup/index.ts | 5 +- src/backup/validation.ts | 3 +- src/collections/backup/client.ts | 13 ++-- src/collections/backup/collection.ts | 19 +---- src/collections/backup/types.ts | 18 ++--- src/v2/index.ts | 2 +- test/collections/backup/integration.test.ts | 63 ++++++++++++++-- test/collections/backup/mock.test.ts | 82 ++++++++++++++++++++- 10 files changed, 202 insertions(+), 54 deletions(-) diff --git a/ci/docker-compose-rbac.yml b/ci/docker-compose-rbac.yml index 1f44d3a3..59d80ed5 100644 --- a/ci/docker-compose-rbac.yml +++ b/ci/docker-compose-rbac.yml @@ -16,7 +16,11 @@ services: - 50062:50051 restart: on-failure:0 environment: - ENABLE_MODULES: "generative-dummy,reranker-dummy" + # backup-filesystem is enabled here so that backup behaviour which Weaviate only exposes to + # root users (e.g. incremental_base_backup_id) can be tested; ci/docker-compose-backup.yml + # is anonymous and therefore never has a root caller. + ENABLE_MODULES: "generative-dummy,reranker-dummy,backup-filesystem" + BACKUP_FILESYSTEM_PATH: "/tmp/backups" PERSISTENCE_DATA_PATH: "./data-weaviate-0" CLUSTER_IN_LOCALHOST: "true" CLUSTER_GOSSIP_BIND_PORT: "7100" diff --git a/src/backup/backupCreator.ts b/src/backup/backupCreator.ts index 0d60c57c..448a5120 100644 --- a/src/backup/backupCreator.ts +++ b/src/backup/backupCreator.ts @@ -1,11 +1,12 @@ import Connection from '../connection/index.js'; -import { WeaviateInvalidInputError } from '../errors.js'; +import { WeaviateInvalidInputError, WeaviateUnsupportedFeatureError } from '../errors.js'; import { BackupConfig, BackupCreateRequest, BackupCreateResponse, BackupCreateStatusResponse, } from '../openapi/types.js'; +import { DbVersionSupport } from '../utils/dbVersion.js'; import { CommandBase } from '../validation/commandBase.js'; import BackupCreateStatusGetter from './backupCreateStatusGetter.js'; import { Backend } from './index.js'; @@ -28,10 +29,16 @@ export default class BackupCreator extends CommandBase { private waitForCompletion!: boolean; private config?: BackupConfig; private incrementalBaseBackupId?: string; + private dbVersionSupport?: DbVersionSupport; - constructor(client: Connection, statusGetter: BackupCreateStatusGetter) { + constructor( + client: Connection, + statusGetter: BackupCreateStatusGetter, + dbVersionSupport?: DbVersionSupport + ) { super(client); this.statusGetter = statusGetter; + this.dbVersionSupport = dbVersionSupport; } withIncludeClassNames(...classNames: string[]) { @@ -73,16 +80,16 @@ export default class BackupCreator extends CommandBase { } /** - * The ID of an existing backup to use as the base for a file-based incremental backup. - * Only the files that changed since the base backup are included in the new backup. - * - * `backupId` is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the - * `id` returned by a previous backup creation. + * The ID of an existing backup to use as the base for a file-based incremental backup. Files + * that are identical to the base backup are not copied and are restored from the base instead, + * so deleting a base backup breaks every incremental backup built on it. * * Requires Weaviate v1.37.0 or higher. */ withIncrementalBaseBackupId(backupId: string) { - this.incrementalBaseBackupId = backupId; + // Weaviate lowercases backup IDs, so normalize here: the payload matches what the server + // stores, and validate() compares like for like. + this.incrementalBaseBackupId = typeof backupId === 'string' ? backupId.toLowerCase() : backupId; return this; } @@ -110,10 +117,26 @@ export default class BackupCreator extends CommandBase { incremental_base_backup_id: this.incrementalBaseBackupId, } as BackupCreateRequest; - if (this.waitForCompletion) { - return this._createAndWaitForCompletion(payload); + return this.checkIncrementalSupport().then(() => + this.waitForCompletion ? this._createAndWaitForCompletion(payload) : this._create(payload) + ); + }; + + /** + * Weaviate below v1.37.0 ignores `incremental_base_backup_id` and silently writes a full backup, + * so fail loudly rather than hand back something other than what was asked for. + * + * No-op when the creator was built without a version provider, or for non-incremental backups. + */ + private checkIncrementalSupport = (): Promise => { + if (this.incrementalBaseBackupId === undefined || this.dbVersionSupport === undefined) { + return Promise.resolve(); } - return this._create(payload); + return this.dbVersionSupport.supportsIncrementalBackups().then((check) => { + if (!check.supports) { + throw new WeaviateUnsupportedFeatureError(check.message); + } + }); }; _create = (payload: BackupCreateRequest): Promise => { diff --git a/src/backup/index.ts b/src/backup/index.ts index d66736fa..2abd899d 100644 --- a/src/backup/index.ts +++ b/src/backup/index.ts @@ -1,4 +1,5 @@ import Connection from '../connection/index.js'; +import { DbVersionSupport } from '../utils/dbVersion.js'; import BackupCreateStatusGetter from './backupCreateStatusGetter.js'; import BackupCreator from './backupCreator.js'; import BackupRestoreStatusGetter from './backupRestoreStatusGetter.js'; @@ -22,9 +23,9 @@ export interface Backup { restoreStatusGetter: () => BackupRestoreStatusGetter; } -const backup = (client: Connection): Backup => { +const backup = (client: Connection, dbVersionSupport: DbVersionSupport): Backup => { return { - creator: () => new BackupCreator(client, new BackupCreateStatusGetter(client)), + creator: () => new BackupCreator(client, new BackupCreateStatusGetter(client), dbVersionSupport), createStatusGetter: () => new BackupCreateStatusGetter(client), restorer: () => new BackupRestorer(client, new BackupRestoreStatusGetter(client)), restoreStatusGetter: () => new BackupRestoreStatusGetter(client), diff --git a/src/backup/validation.ts b/src/backup/validation.ts index 4a9ceffb..88bfd255 100644 --- a/src/backup/validation.ts +++ b/src/backup/validation.ts @@ -55,7 +55,8 @@ export function validateIncrementalBaseBackupId(incrementalBaseBackupId?: string 'string incrementalBaseBackupId must be a non-empty string - set with .withIncrementalBaseBackupId(backupId)', ]; } - if (incrementalBaseBackupId === backupId) { + // Weaviate treats backup IDs as case-insensitive, so 'B1' and 'b1' name the same backup. + if (incrementalBaseBackupId.toLowerCase() === backupId?.toLowerCase()) { return ['incrementalBaseBackupId must be different from the ID of the backup being created']; } return []; diff --git a/src/collections/backup/client.ts b/src/collections/backup/client.ts index edbb87b0..5a90f9a0 100644 --- a/src/collections/backup/client.ts +++ b/src/collections/backup/client.ts @@ -54,7 +54,9 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor error: res.error, path: res.path, status: res.status, - // Only returned by Weaviate >=1.37, for incremental backups, and only to root users + // Restore responses carry neither of these fields; see BackupStatusReturn for when Weaviate + // omits `incremental_base_backup_id` from a create status. + size: 'size' in res ? res.size : undefined, incrementalBaseBackupId: 'incremental_base_backup_id' in res ? res.incremental_base_backup_id || undefined : undefined, }; @@ -124,8 +126,7 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor .withBackupId(args.backupId) .withBackend(args.backend); if (args.incrementalBaseBackupId !== undefined) { - const baseBackupId = args.incrementalBaseBackupId.toLowerCase(); - const errors = validateIncrementalBaseBackupId(baseBackupId, args.backupId.toLowerCase()); + const errors = validateIncrementalBaseBackupId(args.incrementalBaseBackupId, args.backupId); if (errors.length > 0) { throw new WeaviateInvalidInputError(errors.join(', ')); } @@ -133,7 +134,8 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor if (!check.supports) { throw new WeaviateUnsupportedFeatureError(check.message); } - builder = builder.withIncrementalBaseBackupId(baseBackupId); + // The builder lowercases the ID to match how Weaviate stores it. + builder = builder.withIncrementalBaseBackupId(args.incrementalBaseBackupId); } if (args.includeCollections) { builder = builder.withIncludeClassNames(...args.includeCollections); @@ -258,8 +260,7 @@ export interface Backup { /** * Create a backup of the database. * - * Pass `incrementalBaseBackupId` to create a file-based incremental backup, which only - * contains the files that changed since the given base backup. Requires Weaviate `v1.37.0` or higher. + * Set `incrementalBaseBackupId` for a file-based incremental backup (Weaviate `v1.37.0` or higher). * * @param {BackupCreateArgs} args The arguments for the request. * @returns {Promise} The response from Weaviate. diff --git a/src/collections/backup/collection.ts b/src/collections/backup/collection.ts index 04c1a528..56d7b185 100644 --- a/src/collections/backup/collection.ts +++ b/src/collections/backup/collection.ts @@ -2,7 +2,7 @@ import { Backend } from '../../backup/index.js'; import Connection from '../../connection/index.js'; import { DbVersionSupport } from '../../utils/dbVersion.js'; import { backup } from './client.js'; -import { BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js'; +import { BackupCreateArgs, BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js'; /** The arguments required to create and restore backups. */ export type BackupCollectionArgs = { @@ -15,18 +15,8 @@ export type BackupCollectionArgs = { }; /** The arguments required to create a backup of a collection. */ -export type BackupCollectionCreateArgs = BackupCollectionArgs & { - /** - * The ID of an existing backup to use as the base for a file-based incremental backup. - * If set, only the files that have changed since the base backup are included in the new backup. - * - * This is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the `id` - * returned by a previous backup creation. - * - * Requires Weaviate `v1.37.0` or higher. - */ - incrementalBaseBackupId?: string; -}; +export type BackupCollectionCreateArgs = BackupCollectionArgs & + Pick; export const backupCollection = ( connection: Connection, @@ -54,8 +44,7 @@ export interface BackupCollection { /** * Create a backup of this collection. * - * Pass `incrementalBaseBackupId` to create a file-based incremental backup, which only - * contains the files that changed since the given base backup. Requires Weaviate `v1.37.0` or higher. + * Set `incrementalBaseBackupId` for a file-based incremental backup (Weaviate `v1.37.0` or higher). * * @param {BackupCollectionCreateArgs} args The arguments for the request. * @returns {Promise} The response from Weaviate. diff --git a/src/collections/backup/types.ts b/src/collections/backup/types.ts index 546694a6..6f8129ca 100644 --- a/src/collections/backup/types.ts +++ b/src/collections/backup/types.ts @@ -16,7 +16,13 @@ export type BackupStatusReturn = { status: BackupStatus; /** Size of the backup in Gibs */ size?: number; - /** The ID of the base backup this incremental backup was built on; undefined if the backup is not incremental. */ + /** + * The ID of the base backup an incremental backup was built on. + * + * Undefined when the backup is not incremental. Also undefined for an incremental backup when the + * caller is not a root user, when the server is older than `v1.37.6`, and on the return of + * `create()` without `waitForCompletion` — the create response carries no such field. + */ incrementalBaseBackupId?: string; }; @@ -68,15 +74,7 @@ export type BackupArgs = { /** The arguments required to create a backup. */ export type BackupCreateArgs = BackupArgs & { - /** - * The ID of an existing backup to use as the base for a file-based incremental backup. - * If set, only the files that have changed since the base backup are included in the new backup. - * - * This is a plain backup ID string: either a literal, e.g. `'my-base-backup'`, or the `id` - * returned by a previous backup creation. - * - * Requires Weaviate `v1.37.0` or higher. - */ + /** The ID of an existing backup to build a file-based incremental backup on. Files identical to the base are not copied and are restored from the base instead, so deleting a base backup breaks every incremental built on it. Requires Weaviate `v1.37.0` or higher. */ incrementalBaseBackupId?: string; }; diff --git a/src/v2/index.ts b/src/v2/index.ts index 57c0e51b..13502d5f 100644 --- a/src/v2/index.ts +++ b/src/v2/index.ts @@ -51,7 +51,7 @@ const app = { batch: batch(conn, dbVersionSupport), misc: misc(conn, dbVersionProvider), c11y: c11y(conn), - backup: backup(conn), + backup: backup(conn, dbVersionSupport), cluster: cluster(conn), }; diff --git a/test/collections/backup/integration.test.ts b/test/collections/backup/integration.test.ts index 4d1dff33..2ed379c6 100644 --- a/test/collections/backup/integration.test.ts +++ b/test/collections/backup/integration.test.ts @@ -3,7 +3,7 @@ /* eslint-disable no-await-in-loop */ import { afterAll, beforeAll, describe, expect, it, test } from 'vitest'; import { WeaviateBackupFailed, WeaviateInvalidInputError } from '../../../src/errors.js'; -import weaviate, { Backend, Collection, WeaviateClient } from '../../../src/index.js'; +import weaviate, { ApiKey, Backend, Collection, WeaviateClient } from '../../../src/index.js'; import { requireAtLeast } from '../../../test/version.js'; // These must run sequentially because Weaviate is not capable of running multiple backups at the same time @@ -285,10 +285,9 @@ describe('Integration testing of backups', () => { }); expect(incremental.status).toBe('SUCCESS'); - // Weaviate only reports the base backup ID to root users, so treat it as optional - if (incremental.incrementalBaseBackupId !== undefined) { - expect(incremental.incrementalBaseBackupId).toBe(base.id); - } + // This instance is anonymous, so the caller is never a root user and Weaviate withholds the + // base backup ID. The root-user round-trip is covered by the RBAC suite below. + expect(incremental.incrementalBaseBackupId).toBeUndefined(); // The incremental backup must be restorable await client.collections.delete(collection.name); @@ -321,3 +320,57 @@ describe('Integration testing of backups', () => { return 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); } }); + +// Weaviate only returns incremental_base_backup_id to root users, and only from v1.37.6 onwards. +// ci/docker-compose-rbac.yml is the only instance with a root user (AUTHORIZATION_ADMIN_USERS +// populates the RBAC root user list), so the read-back can only be asserted there. +requireAtLeast(1, 37, 6).describe('Integration testing of incremental backups as a root user', () => { + const clientPromise = weaviate.connectToLocal({ + port: 8091, + grpcPort: 50062, + authCredentials: new ApiKey('admin-key'), + }); + + const collectionName = 'TestIncrementalBackupRoot'; + const randomBackupId = () => 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); + + afterAll(() => clientPromise.then((client) => client.collections.delete(collectionName))); + + it('reports the base backup ID on create, status and list', async () => { + const client = await clientPromise; + const collection = await client.collections + .create({ name: collectionName }) + .then((col) => col.data.insert().then(() => col)); + + const base = await client.backup.create({ + backupId: randomBackupId(), + backend: 'filesystem', + includeCollections: [collection.name], + waitForCompletion: true, + }); + expect(base.status).toBe('SUCCESS'); + expect(base.incrementalBaseBackupId).toBeUndefined(); + + await collection.data.insert(); + + const incremental = await client.backup.create({ + backupId: randomBackupId(), + backend: 'filesystem', + includeCollections: [collection.name], + incrementalBaseBackupId: base.id, + waitForCompletion: true, + }); + expect(incremental.status).toBe('SUCCESS'); + expect(incremental.incrementalBaseBackupId).toBe(base.id); + + const status = await client.backup.getCreateStatus({ + backupId: incremental.id, + backend: 'filesystem', + }); + expect(status.incrementalBaseBackupId).toBe(base.id); + + const listed = await client.backup.list('filesystem'); + expect(listed.find((b) => b.id === incremental.id)?.incrementalBaseBackupId).toBe(base.id); + expect(listed.find((b) => b.id === base.id)?.incrementalBaseBackupId).toBeUndefined(); + }); +}); diff --git a/test/collections/backup/mock.test.ts b/test/collections/backup/mock.test.ts index d7a1e0e7..8faafc36 100644 --- a/test/collections/backup/mock.test.ts +++ b/test/collections/backup/mock.test.ts @@ -8,7 +8,7 @@ import { WeaviateInvalidInputError, WeaviateUnsupportedFeatureError, } from '../../../src/errors.js'; -import weaviate, { WeaviateClient } from '../../../src/index.js'; +import weaviate, { WeaviateClient, weaviateV2 } from '../../../src/index.js'; import { HealthCheckRequest, HealthCheckResponse, @@ -227,6 +227,7 @@ class IncrementalMock { backend: BACKEND, path: 'path/to/backup', status: 'SUCCESS', + size: 1.5, incremental_base_backup_id: IncrementalMock.lastCreateRequest?.incremental_base_backup_id, }) ); @@ -307,6 +308,27 @@ describe('Mock testing of incremental backups', () => { await expect(promise).rejects.toThrow(WeaviateInvalidInputError); }); + it('should throw if the base backup only differs from the backup being created in case', async () => { + // Weaviate lowercases backup IDs, so these name the same backup. + const promise = client.backup.create({ + backupId: BACKUP_ID.toUpperCase(), + backend: BACKEND, + incrementalBaseBackupId: BACKUP_ID, + }); + await expect(promise).rejects.toThrow(WeaviateInvalidInputError); + }); + + it('should send the base backup ID when creating a collection-scoped backup', async () => { + await client.collections.use('Article').backup.create({ + backupId: BACKUP_ID, + backend: BACKEND, + incrementalBaseBackupId: BASE_BACKUP_ID, + waitForCompletion: true, + }); + expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBe(BASE_BACKUP_ID); + expect(IncrementalMock.lastCreateRequest.include).toEqual(['Article']); + }); + it('should surface the base backup ID when listing backups', async () => { const backups = await client.backup.list(BACKEND); expect(backups[0].incrementalBaseBackupId).toBeUndefined(); @@ -314,8 +336,53 @@ describe('Mock testing of incremental backups', () => { }); it('should surface the base backup ID when getting the creation status', async () => { + await client.backup.create({ backupId: BACKUP_ID, backend: BACKEND }); // resets the recorded payload + const regular = await client.backup.getCreateStatus({ backupId: BACKUP_ID, backend: BACKEND }); + expect(regular.incrementalBaseBackupId).toBeUndefined(); + + await client.backup.create({ + backupId: BACKUP_ID, + backend: BACKEND, + incrementalBaseBackupId: BASE_BACKUP_ID, + }); + const incremental = await client.backup.getCreateStatus({ backupId: BACKUP_ID, backend: BACKEND }); + expect(incremental.incrementalBaseBackupId).toBe(BASE_BACKUP_ID); + }); + + it('should surface the backup size reported by Weaviate', async () => { const status = await client.backup.getCreateStatus({ backupId: BACKUP_ID, backend: BACKEND }); - expect(status.incrementalBaseBackupId).toBeUndefined(); // last create was a regular backup + expect(status.size).toBe(1.5); + }); + + afterAll(() => mock.close()); + }); + + describe('with the v2 builder', () => { + let mock: IncrementalMock; + const clientV2 = weaviateV2.client({ scheme: 'http', host: 'localhost:8918' }); + + beforeAll(async () => { + mock = await IncrementalMock.use('1.37.0', 8918, 8919); + }); + + it('should lowercase the base backup ID', async () => { + await clientV2.backup + .creator() + .withBackupId(BACKUP_ID) + .withBackend(BACKEND) + .withIncrementalBaseBackupId('Test-Backup-BASE') + .do(); + expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBe(BASE_BACKUP_ID); + }); + + it('should throw if the base backup only differs from the backup being created in case', async () => { + const promise = clientV2.backup + .creator() + .withBackupId(BACKUP_ID.toUpperCase()) + .withBackend(BACKEND) + .withIncrementalBaseBackupId(BACKUP_ID) + .do(); + await expect(promise).rejects.toThrow(WeaviateInvalidInputError); }); afterAll(() => mock.close()); @@ -344,6 +411,17 @@ describe('Mock testing of incremental backups', () => { expect(res.status).toBe('STARTED'); }); + it('should throw from the v2 builder too', async () => { + const promise = weaviateV2 + .client({ scheme: 'http', host: 'localhost:8916' }) + .backup.creator() + .withBackupId(BACKUP_ID) + .withBackend(BACKEND) + .withIncrementalBaseBackupId(BASE_BACKUP_ID) + .do(); + await expect(promise).rejects.toThrow(WeaviateUnsupportedFeatureError); + }); + afterAll(() => mock.close()); }); }); From dd7399276c5db44ea8f248fe5945ff0fd3968373 Mon Sep 17 00:00:00 2001 From: Duda Nogueira Date: Fri, 28 Aug 2026 11:06:12 -0300 Subject: [PATCH 5/7] fix(backup): trim incremental backup support per review Drop client-side base backup ID validation and lowercasing: resolving backup IDs is the server's job. Move incrementalBaseBackupId onto BackupConfigCreate instead of declaring new argument types, read the response fields directly rather than guarding with `in`, and replace the mock suite and the root-user suite with two integration tests. Reverts the backup-filesystem module added to the RBAC compose file. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014t3DrgYbqzyQyiKjQ1UfwW --- ci/docker-compose-rbac.yml | 6 +- src/backup/backupCreator.ts | 18 +- src/backup/validation.ts | 16 -- src/collections/backup/client.ts | 43 ++-- src/collections/backup/collection.ts | 15 +- src/collections/backup/index.ts | 10 +- src/collections/backup/types.ts | 16 +- test/collections/backup/integration.test.ts | 78 +------ test/collections/backup/mock.test.ts | 243 +------------------- 9 files changed, 39 insertions(+), 406 deletions(-) diff --git a/ci/docker-compose-rbac.yml b/ci/docker-compose-rbac.yml index 59d80ed5..1f44d3a3 100644 --- a/ci/docker-compose-rbac.yml +++ b/ci/docker-compose-rbac.yml @@ -16,11 +16,7 @@ services: - 50062:50051 restart: on-failure:0 environment: - # backup-filesystem is enabled here so that backup behaviour which Weaviate only exposes to - # root users (e.g. incremental_base_backup_id) can be tested; ci/docker-compose-backup.yml - # is anonymous and therefore never has a root caller. - ENABLE_MODULES: "generative-dummy,reranker-dummy,backup-filesystem" - BACKUP_FILESYSTEM_PATH: "/tmp/backups" + ENABLE_MODULES: "generative-dummy,reranker-dummy" PERSISTENCE_DATA_PATH: "./data-weaviate-0" CLUSTER_IN_LOCALHOST: "true" CLUSTER_GOSSIP_BIND_PORT: "7100" diff --git a/src/backup/backupCreator.ts b/src/backup/backupCreator.ts index 448a5120..e204830e 100644 --- a/src/backup/backupCreator.ts +++ b/src/backup/backupCreator.ts @@ -15,7 +15,6 @@ import { validateBackupId, validateExcludeClassNames, validateIncludeClassNames, - validateIncrementalBaseBackupId, } from './validation.js'; const WAIT_INTERVAL = 1000; @@ -80,16 +79,12 @@ export default class BackupCreator extends CommandBase { } /** - * The ID of an existing backup to use as the base for a file-based incremental backup. Files - * that are identical to the base backup are not copied and are restored from the base instead, - * so deleting a base backup breaks every incremental backup built on it. - * - * Requires Weaviate v1.37.0 or higher. + * The ID of an existing backup to build a file-based incremental backup on. Unchanged files are + * restored from the base, so deleting a base backup breaks every incremental built on it. + * Requires Weaviate `v1.37.0` or higher. */ withIncrementalBaseBackupId(backupId: string) { - // Weaviate lowercases backup IDs, so normalize here: the payload matches what the server - // stores, and validate() compares like for like. - this.incrementalBaseBackupId = typeof backupId === 'string' ? backupId.toLowerCase() : backupId; + this.incrementalBaseBackupId = backupId; return this; } @@ -99,7 +94,6 @@ export default class BackupCreator extends CommandBase { ...validateExcludeClassNames(this.excludeClassNames), ...validateBackend(this.backend), ...validateBackupId(this.backupId), - ...validateIncrementalBaseBackupId(this.incrementalBaseBackupId, this.backupId), ]); }; @@ -124,9 +118,7 @@ export default class BackupCreator extends CommandBase { /** * Weaviate below v1.37.0 ignores `incremental_base_backup_id` and silently writes a full backup, - * so fail loudly rather than hand back something other than what was asked for. - * - * No-op when the creator was built without a version provider, or for non-incremental backups. + * so fail loudly instead. No-op without a version provider, or for non-incremental backups. */ private checkIncrementalSupport = (): Promise => { if (this.incrementalBaseBackupId === undefined || this.dbVersionSupport === undefined) { diff --git a/src/backup/validation.ts b/src/backup/validation.ts index 88bfd255..778ecb87 100644 --- a/src/backup/validation.ts +++ b/src/backup/validation.ts @@ -45,19 +45,3 @@ export function validateBackupId(backupId?: string) { } return []; } - -export function validateIncrementalBaseBackupId(incrementalBaseBackupId?: string, backupId?: string) { - if (incrementalBaseBackupId === undefined || incrementalBaseBackupId === null) { - return []; - } - if (!isValidStringProperty(incrementalBaseBackupId)) { - return [ - 'string incrementalBaseBackupId must be a non-empty string - set with .withIncrementalBaseBackupId(backupId)', - ]; - } - // Weaviate treats backup IDs as case-insensitive, so 'B1' and 'b1' name the same backup. - if (incrementalBaseBackupId.toLowerCase() === backupId?.toLowerCase()) { - return ['incrementalBaseBackupId must be different from the ID of the backup being created']; - } - return []; -} diff --git a/src/collections/backup/client.ts b/src/collections/backup/client.ts index 5a90f9a0..9e886221 100644 --- a/src/collections/backup/client.ts +++ b/src/collections/backup/client.ts @@ -5,11 +5,7 @@ import { BackupRestoreStatusGetter, BackupRestorer, } from '../../backup/index.js'; -import { - validateBackend, - validateBackupId, - validateIncrementalBaseBackupId, -} from '../../backup/validation.js'; +import { validateBackend, validateBackupId } from '../../backup/validation.js'; import Connection from '../../connection/index.js'; import { WeaviateBackupCanceled, @@ -30,8 +26,8 @@ import { DbVersionSupport } from '../../utils/dbVersion.js'; import { BackupArgs, BackupCancelArgs, + BackupConfigCreate, BackupConfigRestore, - BackupCreateArgs, BackupReturn, BackupStatusArgs, BackupStatusReturn, @@ -39,7 +35,7 @@ import { } from './types.js'; export const backup = (connection: Connection, dbVersionSupport: DbVersionSupport): Backup => { - const parseStatus = (res: BackupCreateStatusResponse | BackupRestoreResponse): BackupStatusReturn => { + const parseStatus = (res: BackupCreateStatusResponse & BackupRestoreResponse): BackupStatusReturn => { if (res.id === undefined) { throw new WeaviateUnexpectedResponseError('Backup ID is undefined in response'); } @@ -54,11 +50,8 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor error: res.error, path: res.path, status: res.status, - // Restore responses carry neither of these fields; see BackupStatusReturn for when Weaviate - // omits `incremental_base_backup_id` from a create status. - size: 'size' in res ? res.size : undefined, - incrementalBaseBackupId: - 'incremental_base_backup_id' in res ? res.incremental_base_backup_id || undefined : undefined, + size: res.size, + incrementalBaseBackupId: res.incremental_base_backup_id || undefined, }; }; const parseResponse = (res: BackupCreateResponse | BackupRestoreResponse): BackupReturn => { @@ -121,21 +114,12 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor return true; }, - create: async (args: BackupCreateArgs): Promise => { - let builder = new BackupCreator(connection, new BackupCreateStatusGetter(connection)) + create: async (args: BackupArgs): Promise => { + let builder = new BackupCreator(connection, new BackupCreateStatusGetter(connection), dbVersionSupport) .withBackupId(args.backupId) .withBackend(args.backend); - if (args.incrementalBaseBackupId !== undefined) { - const errors = validateIncrementalBaseBackupId(args.incrementalBaseBackupId, args.backupId); - if (errors.length > 0) { - throw new WeaviateInvalidInputError(errors.join(', ')); - } - const check = await dbVersionSupport.supportsIncrementalBackups(); - if (!check.supports) { - throw new WeaviateUnsupportedFeatureError(check.message); - } - // The builder lowercases the ID to match how Weaviate stores it. - builder = builder.withIncrementalBaseBackupId(args.incrementalBaseBackupId); + if (args.config?.incrementalBaseBackupId !== undefined) { + builder = builder.withIncrementalBaseBackupId(args.config.incrementalBaseBackupId); } if (args.includeCollections) { builder = builder.withIncludeClassNames(...args.includeCollections); @@ -153,6 +137,8 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor try { res = await builder.do(); } catch (err) { + // The version gate rejects before anything is sent, so it is not a backup failure. + if (err instanceof WeaviateUnsupportedFeatureError) throw err; throw new WeaviateBackupFailed(`Backup creation failed: ${err}`, 'creation'); } if (res.status === 'FAILED') { @@ -260,16 +246,13 @@ export interface Backup { /** * Create a backup of the database. * - * Set `incrementalBaseBackupId` for a file-based incremental backup (Weaviate `v1.37.0` or higher). - * - * @param {BackupCreateArgs} args The arguments for the request. + * @param {BackupArgs} args The arguments for the request. * @returns {Promise} The response from Weaviate. * @throws {WeaviateInvalidInputError} If the input is invalid. - * @throws {WeaviateUnsupportedFeatureError} If `incrementalBaseBackupId` is used with Weaviate <1.37.0. * @throws {WeaviateBackupFailed} If the backup creation fails. * @throws {WeaviateBackupCanceled} If the backup creation is canceled. */ - create(args: BackupCreateArgs): Promise; + create(args: BackupArgs): Promise; /** * Get the status of a backup creation. * diff --git a/src/collections/backup/collection.ts b/src/collections/backup/collection.ts index 56d7b185..de4b32ee 100644 --- a/src/collections/backup/collection.ts +++ b/src/collections/backup/collection.ts @@ -2,7 +2,7 @@ import { Backend } from '../../backup/index.js'; import Connection from '../../connection/index.js'; import { DbVersionSupport } from '../../utils/dbVersion.js'; import { backup } from './client.js'; -import { BackupCreateArgs, BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js'; +import { BackupConfigCreate, BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js'; /** The arguments required to create and restore backups. */ export type BackupCollectionArgs = { @@ -14,10 +14,6 @@ export type BackupCollectionArgs = { waitForCompletion?: boolean; }; -/** The arguments required to create a backup of a collection. */ -export type BackupCollectionCreateArgs = BackupCollectionArgs & - Pick; - export const backupCollection = ( connection: Connection, name: string, @@ -25,7 +21,7 @@ export const backupCollection = ( ) => { const handler = backup(connection, dbVersionSupport); return { - create: (args: BackupCollectionCreateArgs) => + create: (args: BackupCollectionArgs & { config?: BackupConfigCreate }) => handler.create({ ...args, includeCollections: [name], @@ -44,16 +40,13 @@ export interface BackupCollection { /** * Create a backup of this collection. * - * Set `incrementalBaseBackupId` for a file-based incremental backup (Weaviate `v1.37.0` or higher). - * - * @param {BackupCollectionCreateArgs} args The arguments for the request. + * @param {BackupArgs} args The arguments for the request. * @returns {Promise} The response from Weaviate. * @throws {WeaviateInvalidInputError} If the input is invalid. - * @throws {WeaviateUnsupportedFeatureError} If `incrementalBaseBackupId` is used with Weaviate <1.37.0. * @throws {WeaviateBackupFailed} If the backup creation fails. * @throws {WeaviateBackupCanceled} If the backup creation is canceled. */ - create(args: BackupCollectionCreateArgs): Promise; + create(args: BackupCollectionArgs & { config?: BackupConfigCreate }): Promise; /** * Get the status of a backup. * diff --git a/src/collections/backup/index.ts b/src/collections/backup/index.ts index baea10f6..e4c6f1b1 100644 --- a/src/collections/backup/index.ts +++ b/src/collections/backup/index.ts @@ -1,9 +1,3 @@ export type { Backup } from './client.js'; -export type { BackupCollection, BackupCollectionArgs, BackupCollectionCreateArgs } from './collection.js'; -export type { - BackupArgs, - BackupConfigCreate, - BackupConfigRestore, - BackupCreateArgs, - BackupStatusArgs, -} from './types.js'; +export type { BackupCollection, BackupCollectionArgs } from './collection.js'; +export type { BackupArgs, BackupConfigCreate, BackupConfigRestore, BackupStatusArgs } from './types.js'; diff --git a/src/collections/backup/types.ts b/src/collections/backup/types.ts index 6f8129ca..0f18eccd 100644 --- a/src/collections/backup/types.ts +++ b/src/collections/backup/types.ts @@ -16,13 +16,7 @@ export type BackupStatusReturn = { status: BackupStatus; /** Size of the backup in Gibs */ size?: number; - /** - * The ID of the base backup an incremental backup was built on. - * - * Undefined when the backup is not incremental. Also undefined for an incremental backup when the - * caller is not a root user, when the server is older than `v1.37.6`, and on the return of - * `create()` without `waitForCompletion` — the create response carries no such field. - */ + /** The ID of the base backup an incremental backup was built on. Only returned to root users, from Weaviate `v1.37.6`, and never by `create()` without `waitForCompletion`. */ incrementalBaseBackupId?: string; }; @@ -46,6 +40,8 @@ export type BackupConfigCreate = { compressionLevel?: BackupCompressionLevel; /** The percentage of CPU to use for the backup creation job. */ cpuPercentage?: number; + /** The ID of an existing backup to build a file-based incremental backup on. Unchanged files are restored from the base, so deleting a base backup breaks every incremental built on it. Requires Weaviate `v1.37.0` or higher. */ + incrementalBaseBackupId?: string; }; /** Configuration options available when restoring a backup */ @@ -72,12 +68,6 @@ export type BackupArgs = { config?: C; }; -/** The arguments required to create a backup. */ -export type BackupCreateArgs = BackupArgs & { - /** The ID of an existing backup to build a file-based incremental backup on. Files identical to the base are not copied and are restored from the base instead, so deleting a base backup breaks every incremental built on it. Requires Weaviate `v1.37.0` or higher. */ - incrementalBaseBackupId?: string; -}; - /** The arguments required to get the status of a backup. */ export type BackupStatusArgs = { /** The ID of the backup. */ diff --git a/test/collections/backup/integration.test.ts b/test/collections/backup/integration.test.ts index 2ed379c6..f047cf2a 100644 --- a/test/collections/backup/integration.test.ts +++ b/test/collections/backup/integration.test.ts @@ -2,8 +2,8 @@ /* eslint-disable @typescript-eslint/no-non-null-asserted-optional-chain */ /* eslint-disable no-await-in-loop */ import { afterAll, beforeAll, describe, expect, it, test } from 'vitest'; -import { WeaviateBackupFailed, WeaviateInvalidInputError } from '../../../src/errors.js'; -import weaviate, { ApiKey, Backend, Collection, WeaviateClient } from '../../../src/index.js'; +import { WeaviateBackupFailed } from '../../../src/errors.js'; +import weaviate, { Backend, Collection, WeaviateClient } from '../../../src/index.js'; import { requireAtLeast } from '../../../test/version.js'; // These must run sequentially because Weaviate is not capable of running multiple backups at the same time @@ -259,7 +259,7 @@ describe('Integration testing of backups', () => { }); requireAtLeast(1, 37, 0).describe('incremental backups', () => { - it('creates an incremental backup on top of a base backup', async () => { + it('creates and restores an incremental backup on top of a base backup', async () => { const client = await clientPromise; const collection = await client.collections .create({ name: 'TestIncrementalBackup' }) @@ -273,23 +273,17 @@ describe('Integration testing of backups', () => { }); expect(base.status).toBe('SUCCESS'); - // Add data so that the incremental backup has something to pick up await collection.data.insert(); const incremental = await client.backup.create({ backupId: randomBackupId(), backend: 'filesystem', includeCollections: [collection.name], - incrementalBaseBackupId: base.id, + config: { incrementalBaseBackupId: base.id }, waitForCompletion: true, }); expect(incremental.status).toBe('SUCCESS'); - // This instance is anonymous, so the caller is never a root user and Weaviate withholds the - // base backup ID. The root-user round-trip is covered by the RBAC suite below. - expect(incremental.incrementalBaseBackupId).toBeUndefined(); - - // The incremental backup must be restorable await client.collections.delete(collection.name); const restored = await client.backup.restore({ backupId: incremental.id, @@ -303,16 +297,16 @@ describe('Integration testing of backups', () => { await client.collections.delete(collection.name); }); - it('rejects an incremental backup based on itself', async () => { + it('sends the base backup ID to the server', async () => { const client = await clientPromise; - const backupId = randomBackupId(); await expect( client.backup.create({ - backupId, + backupId: randomBackupId(), backend: 'filesystem', - incrementalBaseBackupId: backupId, + includeCollections: ['TestBackupCollection'], + config: { incrementalBaseBackupId: 'does-not-exist' }, }) - ).rejects.toThrow(WeaviateInvalidInputError); + ).rejects.toThrow(/could not fetch base backup/); }); }); @@ -320,57 +314,3 @@ describe('Integration testing of backups', () => { return 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); } }); - -// Weaviate only returns incremental_base_backup_id to root users, and only from v1.37.6 onwards. -// ci/docker-compose-rbac.yml is the only instance with a root user (AUTHORIZATION_ADMIN_USERS -// populates the RBAC root user list), so the read-back can only be asserted there. -requireAtLeast(1, 37, 6).describe('Integration testing of incremental backups as a root user', () => { - const clientPromise = weaviate.connectToLocal({ - port: 8091, - grpcPort: 50062, - authCredentials: new ApiKey('admin-key'), - }); - - const collectionName = 'TestIncrementalBackupRoot'; - const randomBackupId = () => 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); - - afterAll(() => clientPromise.then((client) => client.collections.delete(collectionName))); - - it('reports the base backup ID on create, status and list', async () => { - const client = await clientPromise; - const collection = await client.collections - .create({ name: collectionName }) - .then((col) => col.data.insert().then(() => col)); - - const base = await client.backup.create({ - backupId: randomBackupId(), - backend: 'filesystem', - includeCollections: [collection.name], - waitForCompletion: true, - }); - expect(base.status).toBe('SUCCESS'); - expect(base.incrementalBaseBackupId).toBeUndefined(); - - await collection.data.insert(); - - const incremental = await client.backup.create({ - backupId: randomBackupId(), - backend: 'filesystem', - includeCollections: [collection.name], - incrementalBaseBackupId: base.id, - waitForCompletion: true, - }); - expect(incremental.status).toBe('SUCCESS'); - expect(incremental.incrementalBaseBackupId).toBe(base.id); - - const status = await client.backup.getCreateStatus({ - backupId: incremental.id, - backend: 'filesystem', - }); - expect(status.incrementalBaseBackupId).toBe(base.id); - - const listed = await client.backup.list('filesystem'); - expect(listed.find((b) => b.id === incremental.id)?.incrementalBaseBackupId).toBe(base.id); - expect(listed.find((b) => b.id === base.id)?.incrementalBaseBackupId).toBeUndefined(); - }); -}); diff --git a/test/collections/backup/mock.test.ts b/test/collections/backup/mock.test.ts index 8faafc36..ab4ca53a 100644 --- a/test/collections/backup/mock.test.ts +++ b/test/collections/backup/mock.test.ts @@ -3,12 +3,8 @@ import { Server as HttpServer } from 'http'; import { Server as GrpcServer, createServer } from 'nice-grpc'; import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { BackupStatus } from '../../../src/collections/backup/types.js'; -import { - WeaviateBackupCanceled, - WeaviateInvalidInputError, - WeaviateUnsupportedFeatureError, -} from '../../../src/errors.js'; -import weaviate, { WeaviateClient, weaviateV2 } from '../../../src/index.js'; +import { WeaviateBackupCanceled } from '../../../src/errors.js'; +import weaviate, { WeaviateClient } from '../../../src/index.js'; import { HealthCheckRequest, HealthCheckResponse, @@ -17,10 +13,8 @@ import { HealthServiceImplementation, } from '../../../src/proto/google/health/v1/health.js'; import { - BackupCreateRequest, BackupCreateResponse, BackupCreateStatusResponse, - BackupListResponse, BackupRestoreResponse, } from '../../../src/v2/index.js'; @@ -192,236 +186,3 @@ describe('Mock testing of backup cancellation', () => { afterAll(() => mock.close()); }); - -const BASE_BACKUP_ID = 'test-backup-base'; - -/** Mocks the backup endpoints, recording the payload sent by the client on creation. */ -class IncrementalMock { - private grpc: GrpcServer; - private http: HttpServer; - static lastCreateRequest: BackupCreateRequest; - - constructor(grpc: GrpcServer, http: HttpServer) { - this.grpc = grpc; - this.http = http; - } - - public static use = async (version: string, httpPort: number, grpcPort: number) => { - const httpApp = express(); - httpApp.use(express.json()); - httpApp.get('/v1/meta', (req, res) => res.send({ version })); - - httpApp.post(`/v1/backups/${BACKEND}`, (req, res: Response) => { - IncrementalMock.lastCreateRequest = req.body; - res.send({ - id: req.body.id, - backend: BACKEND, - classes: ['Article'], - path: 'path/to/backup', - status: 'STARTED', - }); - }); - httpApp.get(`/v1/backups/${BACKEND}/:id`, (req, res: Response) => - res.send({ - id: req.params.id, - backend: BACKEND, - path: 'path/to/backup', - status: 'SUCCESS', - size: 1.5, - incremental_base_backup_id: IncrementalMock.lastCreateRequest?.incremental_base_backup_id, - }) - ); - httpApp.get(`/v1/backups/${BACKEND}`, (req, res: Response) => - res.send([ - { id: BASE_BACKUP_ID, classes: ['Article'], status: 'SUCCESS', incremental_base_backup_id: '' }, - { - id: BACKUP_ID, - classes: ['Article'], - status: 'SUCCESS', - incremental_base_backup_id: BASE_BACKUP_ID, - }, - ]) - ); - - const healthMockImpl: HealthServiceImplementation = { - check: (request: HealthCheckRequest): Promise => - Promise.resolve(HealthCheckResponse.create({ status: HealthCheckResponse_ServingStatus.SERVING })), - watch: vi.fn(), - }; - - const grpc = createServer(); - grpc.add(HealthDefinition, healthMockImpl); - - httpApp.on('error', (error) => console.error('HTTP Server Error:', error)); - - await grpc.listen(`localhost:${grpcPort}`); - const http = await httpApp.listen(httpPort); - return new IncrementalMock(grpc, http); - }; - - public close = () => Promise.all([this.http.close(), this.grpc.shutdown()]); -} - -describe('Mock testing of incremental backups', () => { - describe('with a supported Weaviate version', () => { - let client: WeaviateClient; - let mock: IncrementalMock; - - beforeAll(async () => { - mock = await IncrementalMock.use('1.37.0', 8914, 8915); - client = await weaviate.connectToLocal({ port: 8914, grpcPort: 8915 }); - }); - - it('should send the base backup ID when creating an incremental backup', async () => { - const res = await client.backup.create({ - backupId: BACKUP_ID, - backend: BACKEND, - incrementalBaseBackupId: BASE_BACKUP_ID, - waitForCompletion: true, - }); - expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBe(BASE_BACKUP_ID); - expect(res.status).toBe('SUCCESS'); - expect(res.incrementalBaseBackupId).toBe(BASE_BACKUP_ID); - }); - - it('should lowercase the base backup ID', async () => { - await client.backup.create({ - backupId: BACKUP_ID, - backend: BACKEND, - incrementalBaseBackupId: 'Test-Backup-BASE', - waitForCompletion: true, - }); - expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBe(BASE_BACKUP_ID); - }); - - it('should not send the field for a regular backup', async () => { - await client.backup.create({ backupId: BACKUP_ID, backend: BACKEND }); - expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBeUndefined(); - }); - - it('should throw if the base backup is the backup being created', async () => { - const promise = client.backup.create({ - backupId: BACKUP_ID, - backend: BACKEND, - incrementalBaseBackupId: BACKUP_ID, - }); - await expect(promise).rejects.toThrow(WeaviateInvalidInputError); - }); - - it('should throw if the base backup only differs from the backup being created in case', async () => { - // Weaviate lowercases backup IDs, so these name the same backup. - const promise = client.backup.create({ - backupId: BACKUP_ID.toUpperCase(), - backend: BACKEND, - incrementalBaseBackupId: BACKUP_ID, - }); - await expect(promise).rejects.toThrow(WeaviateInvalidInputError); - }); - - it('should send the base backup ID when creating a collection-scoped backup', async () => { - await client.collections.use('Article').backup.create({ - backupId: BACKUP_ID, - backend: BACKEND, - incrementalBaseBackupId: BASE_BACKUP_ID, - waitForCompletion: true, - }); - expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBe(BASE_BACKUP_ID); - expect(IncrementalMock.lastCreateRequest.include).toEqual(['Article']); - }); - - it('should surface the base backup ID when listing backups', async () => { - const backups = await client.backup.list(BACKEND); - expect(backups[0].incrementalBaseBackupId).toBeUndefined(); - expect(backups[1].incrementalBaseBackupId).toBe(BASE_BACKUP_ID); - }); - - it('should surface the base backup ID when getting the creation status', async () => { - await client.backup.create({ backupId: BACKUP_ID, backend: BACKEND }); // resets the recorded payload - const regular = await client.backup.getCreateStatus({ backupId: BACKUP_ID, backend: BACKEND }); - expect(regular.incrementalBaseBackupId).toBeUndefined(); - - await client.backup.create({ - backupId: BACKUP_ID, - backend: BACKEND, - incrementalBaseBackupId: BASE_BACKUP_ID, - }); - const incremental = await client.backup.getCreateStatus({ backupId: BACKUP_ID, backend: BACKEND }); - expect(incremental.incrementalBaseBackupId).toBe(BASE_BACKUP_ID); - }); - - it('should surface the backup size reported by Weaviate', async () => { - const status = await client.backup.getCreateStatus({ backupId: BACKUP_ID, backend: BACKEND }); - expect(status.size).toBe(1.5); - }); - - afterAll(() => mock.close()); - }); - - describe('with the v2 builder', () => { - let mock: IncrementalMock; - const clientV2 = weaviateV2.client({ scheme: 'http', host: 'localhost:8918' }); - - beforeAll(async () => { - mock = await IncrementalMock.use('1.37.0', 8918, 8919); - }); - - it('should lowercase the base backup ID', async () => { - await clientV2.backup - .creator() - .withBackupId(BACKUP_ID) - .withBackend(BACKEND) - .withIncrementalBaseBackupId('Test-Backup-BASE') - .do(); - expect(IncrementalMock.lastCreateRequest.incremental_base_backup_id).toBe(BASE_BACKUP_ID); - }); - - it('should throw if the base backup only differs from the backup being created in case', async () => { - const promise = clientV2.backup - .creator() - .withBackupId(BACKUP_ID.toUpperCase()) - .withBackend(BACKEND) - .withIncrementalBaseBackupId(BACKUP_ID) - .do(); - await expect(promise).rejects.toThrow(WeaviateInvalidInputError); - }); - - afterAll(() => mock.close()); - }); - - describe('with an unsupported Weaviate version', () => { - let client: WeaviateClient; - let mock: IncrementalMock; - - beforeAll(async () => { - mock = await IncrementalMock.use('1.36.0', 8916, 8917); - client = await weaviate.connectToLocal({ port: 8916, grpcPort: 8917 }); - }); - - it('should throw when requesting an incremental backup', async () => { - const promise = client.backup.create({ - backupId: BACKUP_ID, - backend: BACKEND, - incrementalBaseBackupId: BASE_BACKUP_ID, - }); - await expect(promise).rejects.toThrow(WeaviateUnsupportedFeatureError); - }); - - it('should still allow regular backups', async () => { - const res = await client.backup.create({ backupId: BACKUP_ID, backend: BACKEND }); - expect(res.status).toBe('STARTED'); - }); - - it('should throw from the v2 builder too', async () => { - const promise = weaviateV2 - .client({ scheme: 'http', host: 'localhost:8916' }) - .backup.creator() - .withBackupId(BACKUP_ID) - .withBackend(BACKEND) - .withIncrementalBaseBackupId(BASE_BACKUP_ID) - .do(); - await expect(promise).rejects.toThrow(WeaviateUnsupportedFeatureError); - }); - - afterAll(() => mock.close()); - }); -}); From 2e14362d75f91fbcf553b0210d288874f03f14d3 Mon Sep 17 00:00:00 2001 From: Duda Nogueira Date: Mon, 21 Sep 2026 09:05:18 -0300 Subject: [PATCH 6/7] fix(backup): drop redundant comments and null-coalescing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the review nits on incremental backup support: - `parseStatus` and `list` no longer coalesce `incremental_base_backup_id` with `|| undefined`. The server declares the field as a non-pointer string with `omitempty`, so a non-incremental backup omits the key entirely rather than sending `""` — the value is already `undefined`. - Drop the comment on the rethrow in the v3 `create` try/catch. - Reduce the `checkIncrementalSupport` doc block to the one fact that is not readable from the guard below it: the server ignores the field silently rather than rejecting it. Co-Authored-By: Claude Opus 5 (1M context) --- src/backup/backupCreator.ts | 5 +---- src/collections/backup/client.ts | 5 ++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/backup/backupCreator.ts b/src/backup/backupCreator.ts index e204830e..5dd3fa6d 100644 --- a/src/backup/backupCreator.ts +++ b/src/backup/backupCreator.ts @@ -116,10 +116,7 @@ export default class BackupCreator extends CommandBase { ); }; - /** - * Weaviate below v1.37.0 ignores `incremental_base_backup_id` and silently writes a full backup, - * so fail loudly instead. No-op without a version provider, or for non-incremental backups. - */ + /** Weaviate below v1.37.0 ignores the field and silently writes a full backup, so fail loudly instead. */ private checkIncrementalSupport = (): Promise => { if (this.incrementalBaseBackupId === undefined || this.dbVersionSupport === undefined) { return Promise.resolve(); diff --git a/src/collections/backup/client.ts b/src/collections/backup/client.ts index 9e886221..5b67fe7c 100644 --- a/src/collections/backup/client.ts +++ b/src/collections/backup/client.ts @@ -51,7 +51,7 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor path: res.path, status: res.status, size: res.size, - incrementalBaseBackupId: res.incremental_base_backup_id || undefined, + incrementalBaseBackupId: res.incremental_base_backup_id, }; }; const parseResponse = (res: BackupCreateResponse | BackupRestoreResponse): BackupReturn => { @@ -137,7 +137,6 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor try { res = await builder.do(); } catch (err) { - // The version gate rejects before anything is sent, so it is not a backup failure. if (err instanceof WeaviateUnsupportedFeatureError) throw err; throw new WeaviateBackupFailed(`Backup creation failed: ${err}`, 'creation'); } @@ -226,7 +225,7 @@ export const backup = (connection: Connection, dbVersionSupport: DbVersionSuppor return connection.get(url).then((res) => res.map(({ incremental_base_backup_id: baseBackupId, ...rest }) => ({ ...rest, - incrementalBaseBackupId: baseBackupId || undefined, + incrementalBaseBackupId: baseBackupId, })) ) as Promise; }, From dec2460542ff8ff927355e1c7aeaa1aecb3631de Mon Sep 17 00:00:00 2001 From: Duda Nogueira Date: Tue, 29 Sep 2026 16:45:59 -0300 Subject: [PATCH 7/7] fix(backup): trim docstrings and fold config into BackupCollectionArgs Version floors and base-deletion caveats move to the backup docs. BackupCollectionArgs takes the config type as a parameter, like BackupArgs, so create() no longer intersects an ad-hoc config field. --- src/backup/backupCreator.ts | 5 +---- src/collections/backup/collection.ts | 16 ++++++++++++---- src/collections/backup/types.ts | 4 ++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/backup/backupCreator.ts b/src/backup/backupCreator.ts index 5dd3fa6d..18a602b5 100644 --- a/src/backup/backupCreator.ts +++ b/src/backup/backupCreator.ts @@ -79,9 +79,7 @@ export default class BackupCreator extends CommandBase { } /** - * The ID of an existing backup to build a file-based incremental backup on. Unchanged files are - * restored from the base, so deleting a base backup breaks every incremental built on it. - * Requires Weaviate `v1.37.0` or higher. + * The ID of an existing backup to build a file-based incremental backup on. */ withIncrementalBaseBackupId(backupId: string) { this.incrementalBaseBackupId = backupId; @@ -116,7 +114,6 @@ export default class BackupCreator extends CommandBase { ); }; - /** Weaviate below v1.37.0 ignores the field and silently writes a full backup, so fail loudly instead. */ private checkIncrementalSupport = (): Promise => { if (this.incrementalBaseBackupId === undefined || this.dbVersionSupport === undefined) { return Promise.resolve(); diff --git a/src/collections/backup/collection.ts b/src/collections/backup/collection.ts index de4b32ee..a242dfb6 100644 --- a/src/collections/backup/collection.ts +++ b/src/collections/backup/collection.ts @@ -2,16 +2,24 @@ import { Backend } from '../../backup/index.js'; import Connection from '../../connection/index.js'; import { DbVersionSupport } from '../../utils/dbVersion.js'; import { backup } from './client.js'; -import { BackupConfigCreate, BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js'; +import { + BackupConfigCreate, + BackupConfigRestore, + BackupReturn, + BackupStatusArgs, + BackupStatusReturn, +} from './types.js'; /** The arguments required to create and restore backups. */ -export type BackupCollectionArgs = { +export type BackupCollectionArgs = { /** The ID of the backup. */ backupId: string; /** The backend to use for the backup. */ backend: Backend; /** The collections to include in the backup. */ waitForCompletion?: boolean; + /** The configuration options for the backup. */ + config?: C; }; export const backupCollection = ( @@ -21,7 +29,7 @@ export const backupCollection = ( ) => { const handler = backup(connection, dbVersionSupport); return { - create: (args: BackupCollectionArgs & { config?: BackupConfigCreate }) => + create: (args: BackupCollectionArgs) => handler.create({ ...args, includeCollections: [name], @@ -46,7 +54,7 @@ export interface BackupCollection { * @throws {WeaviateBackupFailed} If the backup creation fails. * @throws {WeaviateBackupCanceled} If the backup creation is canceled. */ - create(args: BackupCollectionArgs & { config?: BackupConfigCreate }): Promise; + create(args: BackupCollectionArgs): Promise; /** * Get the status of a backup. * diff --git a/src/collections/backup/types.ts b/src/collections/backup/types.ts index 0f18eccd..155e66d1 100644 --- a/src/collections/backup/types.ts +++ b/src/collections/backup/types.ts @@ -16,7 +16,7 @@ export type BackupStatusReturn = { status: BackupStatus; /** Size of the backup in Gibs */ size?: number; - /** The ID of the base backup an incremental backup was built on. Only returned to root users, from Weaviate `v1.37.6`, and never by `create()` without `waitForCompletion`. */ + /** The ID of the base backup an incremental backup was built on. */ incrementalBaseBackupId?: string; }; @@ -40,7 +40,7 @@ export type BackupConfigCreate = { compressionLevel?: BackupCompressionLevel; /** The percentage of CPU to use for the backup creation job. */ cpuPercentage?: number; - /** The ID of an existing backup to build a file-based incremental backup on. Unchanged files are restored from the base, so deleting a base backup breaks every incremental built on it. Requires Weaviate `v1.37.0` or higher. */ + /** The ID of an existing backup to build a file-based incremental backup on. */ incrementalBaseBackupId?: string; };