diff --git a/src/backup/backupCreator.ts b/src/backup/backupCreator.ts index 5690f73e..5dd3fa6d 100644 --- a/src/backup/backupCreator.ts +++ b/src/backup/backupCreator.ts @@ -1,11 +1,12 @@ import Connection from '../connection/index.js'; -import { WeaviateInvalidInputError } from '../errors.js'; +import { WeaviateInvalidInputError, WeaviateUnsupportedFeatureError } from '../errors.js'; import { BackupConfig, BackupCreateRequest, BackupCreateResponse, BackupCreateStatusResponse, } from '../openapi/types.js'; +import { DbVersionSupport } from '../utils/dbVersion.js'; import { CommandBase } from '../validation/commandBase.js'; import BackupCreateStatusGetter from './backupCreateStatusGetter.js'; import { Backend } from './index.js'; @@ -26,10 +27,17 @@ export default class BackupCreator extends CommandBase { private statusGetter: BackupCreateStatusGetter; private waitForCompletion!: boolean; private config?: BackupConfig; - - constructor(client: Connection, statusGetter: BackupCreateStatusGetter) { + private incrementalBaseBackupId?: string; + private dbVersionSupport?: DbVersionSupport; + + constructor( + client: Connection, + statusGetter: BackupCreateStatusGetter, + dbVersionSupport?: DbVersionSupport + ) { super(client); this.statusGetter = statusGetter; + this.dbVersionSupport = dbVersionSupport; } withIncludeClassNames(...classNames: string[]) { @@ -70,6 +78,16 @@ export default class BackupCreator extends CommandBase { return this; } + /** + * The ID of an existing backup to build a file-based incremental backup on. Unchanged files are + * restored from the base, so deleting a base backup breaks every incremental built on it. + * Requires Weaviate `v1.37.0` or higher. + */ + withIncrementalBaseBackupId(backupId: string) { + this.incrementalBaseBackupId = backupId; + return this; + } + validate = (): void => { this.addErrors([ ...validateIncludeClassNames(this.includeClassNames), @@ -90,12 +108,24 @@ export default class BackupCreator extends CommandBase { config: this.config, include: this.includeClassNames, exclude: this.excludeClassNames, + incremental_base_backup_id: this.incrementalBaseBackupId, } as BackupCreateRequest; - if (this.waitForCompletion) { - return this._createAndWaitForCompletion(payload); + return this.checkIncrementalSupport().then(() => + this.waitForCompletion ? this._createAndWaitForCompletion(payload) : this._create(payload) + ); + }; + + /** Weaviate below v1.37.0 ignores the field and silently writes a full backup, so fail loudly instead. */ + private checkIncrementalSupport = (): Promise => { + if (this.incrementalBaseBackupId === undefined || this.dbVersionSupport === undefined) { + return Promise.resolve(); } - return this._create(payload); + return this.dbVersionSupport.supportsIncrementalBackups().then((check) => { + if (!check.supports) { + throw new WeaviateUnsupportedFeatureError(check.message); + } + }); }; _create = (payload: BackupCreateRequest): Promise => { diff --git a/src/backup/index.ts b/src/backup/index.ts index d66736fa..2abd899d 100644 --- a/src/backup/index.ts +++ b/src/backup/index.ts @@ -1,4 +1,5 @@ import Connection from '../connection/index.js'; +import { DbVersionSupport } from '../utils/dbVersion.js'; import BackupCreateStatusGetter from './backupCreateStatusGetter.js'; import BackupCreator from './backupCreator.js'; import BackupRestoreStatusGetter from './backupRestoreStatusGetter.js'; @@ -22,9 +23,9 @@ export interface Backup { restoreStatusGetter: () => BackupRestoreStatusGetter; } -const backup = (client: Connection): Backup => { +const backup = (client: Connection, dbVersionSupport: DbVersionSupport): Backup => { return { - creator: () => new BackupCreator(client, new BackupCreateStatusGetter(client)), + creator: () => new BackupCreator(client, new BackupCreateStatusGetter(client), dbVersionSupport), createStatusGetter: () => new BackupCreateStatusGetter(client), restorer: () => new BackupRestorer(client, new BackupRestoreStatusGetter(client)), restoreStatusGetter: () => new BackupRestoreStatusGetter(client), diff --git a/src/collections/backup/client.ts b/src/collections/backup/client.ts index 4a655f10..5b67fe7c 100644 --- a/src/collections/backup/client.ts +++ b/src/collections/backup/client.ts @@ -14,12 +14,15 @@ import { WeaviateInvalidInputError, WeaviateUnexpectedResponseError, WeaviateUnexpectedStatusCodeError, + WeaviateUnsupportedFeatureError, } from '../../errors.js'; import { BackupCreateResponse, BackupCreateStatusResponse, + BackupListResponse, BackupRestoreResponse, } from '../../openapi/types.js'; +import { DbVersionSupport } from '../../utils/dbVersion.js'; import { BackupArgs, BackupCancelArgs, @@ -31,8 +34,8 @@ import { ListBackupOptions, } from './types.js'; -export const backup = (connection: Connection): Backup => { - const parseStatus = (res: BackupCreateStatusResponse | BackupRestoreResponse): BackupStatusReturn => { +export const backup = (connection: Connection, dbVersionSupport: DbVersionSupport): Backup => { + const parseStatus = (res: BackupCreateStatusResponse & BackupRestoreResponse): BackupStatusReturn => { if (res.id === undefined) { throw new WeaviateUnexpectedResponseError('Backup ID is undefined in response'); } @@ -47,6 +50,8 @@ export const backup = (connection: Connection): Backup => { error: res.error, path: res.path, status: res.status, + size: res.size, + incrementalBaseBackupId: res.incremental_base_backup_id, }; }; const parseResponse = (res: BackupCreateResponse | BackupRestoreResponse): BackupReturn => { @@ -110,9 +115,12 @@ export const backup = (connection: Connection): Backup => { return true; }, create: async (args: BackupArgs): Promise => { - let builder = new BackupCreator(connection, new BackupCreateStatusGetter(connection)) + let builder = new BackupCreator(connection, new BackupCreateStatusGetter(connection), dbVersionSupport) .withBackupId(args.backupId) .withBackend(args.backend); + if (args.config?.incrementalBaseBackupId !== undefined) { + builder = builder.withIncrementalBaseBackupId(args.config.incrementalBaseBackupId); + } if (args.includeCollections) { builder = builder.withIncludeClassNames(...args.includeCollections); } @@ -129,6 +137,7 @@ export const backup = (connection: Connection): Backup => { try { res = await builder.do(); } catch (err) { + if (err instanceof WeaviateUnsupportedFeatureError) throw err; throw new WeaviateBackupFailed(`Backup creation failed: ${err}`, 'creation'); } if (res.status === 'FAILED') { @@ -213,7 +222,12 @@ export const backup = (connection: Connection): Backup => { if (opts?.startedAtAsc) { url += '?order=asc'; } - return connection.get(url); + return connection.get(url).then((res) => + res.map(({ incremental_base_backup_id: baseBackupId, ...rest }) => ({ + ...rest, + incrementalBaseBackupId: baseBackupId, + })) + ) as Promise; }, }; }; diff --git a/src/collections/backup/collection.ts b/src/collections/backup/collection.ts index 377d8009..de4b32ee 100644 --- a/src/collections/backup/collection.ts +++ b/src/collections/backup/collection.ts @@ -1,7 +1,8 @@ import { Backend } from '../../backup/index.js'; import Connection from '../../connection/index.js'; +import { DbVersionSupport } from '../../utils/dbVersion.js'; import { backup } from './client.js'; -import { BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js'; +import { BackupConfigCreate, BackupReturn, BackupStatusArgs, BackupStatusReturn } from './types.js'; /** The arguments required to create and restore backups. */ export type BackupCollectionArgs = { @@ -13,10 +14,14 @@ export type BackupCollectionArgs = { waitForCompletion?: boolean; }; -export const backupCollection = (connection: Connection, name: string) => { - const handler = backup(connection); +export const backupCollection = ( + connection: Connection, + name: string, + dbVersionSupport: DbVersionSupport +) => { + const handler = backup(connection, dbVersionSupport); return { - create: (args: BackupCollectionArgs) => + create: (args: BackupCollectionArgs & { config?: BackupConfigCreate }) => handler.create({ ...args, includeCollections: [name], @@ -41,7 +46,7 @@ export interface BackupCollection { * @throws {WeaviateBackupFailed} If the backup creation fails. * @throws {WeaviateBackupCanceled} If the backup creation is canceled. */ - create(args: BackupCollectionArgs): Promise; + create(args: BackupCollectionArgs & { config?: BackupConfigCreate }): Promise; /** * Get the status of a backup. * diff --git a/src/collections/backup/types.ts b/src/collections/backup/types.ts index c76e5a25..0f18eccd 100644 --- a/src/collections/backup/types.ts +++ b/src/collections/backup/types.ts @@ -16,6 +16,8 @@ export type BackupStatusReturn = { status: BackupStatus; /** Size of the backup in Gibs */ size?: number; + /** The ID of the base backup an incremental backup was built on. Only returned to root users, from Weaviate `v1.37.6`, and never by `create()` without `waitForCompletion`. */ + incrementalBaseBackupId?: string; }; /** The return type of a backup creation or restoration operation */ @@ -38,6 +40,8 @@ export type BackupConfigCreate = { compressionLevel?: BackupCompressionLevel; /** The percentage of CPU to use for the backup creation job. */ cpuPercentage?: number; + /** The ID of an existing backup to build a file-based incremental backup on. Unchanged files are restored from the base, so deleting a base backup breaks every incremental built on it. Requires Weaviate `v1.37.0` or higher. */ + incrementalBaseBackupId?: string; }; /** Configuration options available when restoring a backup */ diff --git a/src/collections/collection/index.ts b/src/collections/collection/index.ts index c4164352..c0676216 100644 --- a/src/collections/collection/index.ts +++ b/src/collections/collection/index.ts @@ -138,7 +138,7 @@ const collection = ( ); return { aggregate: aggregateCollection, - backup: backupCollection(connection, capitalizedName), + backup: backupCollection(connection, capitalizedName, dbVersionSupport), config: config(connection, capitalizedName, dbVersionSupport, tenant), data: data(connection, capitalizedName, dbVersionSupport, consistencyLevel, tenant), filter: filter(), diff --git a/src/index.ts b/src/index.ts index b3fa9e7d..b9cf3926 100644 --- a/src/index.ts +++ b/src/index.ts @@ -233,7 +233,7 @@ async function client(params: ClientParams): Promise { const ifc: WeaviateClient = { alias: alias(connection), - backup: backup(connection), + backup: backup(connection, dbVersionSupport), batch: batch(connection, dbVersionSupport), cluster: cluster(connection), collections: collections(connection, dbVersionSupport), diff --git a/src/openapi/types.ts b/src/openapi/types.ts index 4cecdcfc..edf9e939 100644 --- a/src/openapi/types.ts +++ b/src/openapi/types.ts @@ -15,6 +15,7 @@ export type DataObject = definitions['Object']; export type BackupCreateRequest = definitions['BackupCreateRequest']; export type BackupCreateResponse = definitions['BackupCreateResponse']; export type BackupCreateStatusResponse = definitions['BackupCreateStatusResponse']; +export type BackupListResponse = definitions['BackupListResponse']; export type BackupRestoreRequest = definitions['BackupRestoreRequest']; export type BackupRestoreResponse = definitions['BackupRestoreResponse']; export type BackupRestoreStatusResponse = definitions['BackupRestoreStatusResponse']; diff --git a/src/utils/dbVersion.ts b/src/utils/dbVersion.ts index b37c328f..3bdf8c95 100644 --- a/src/utils/dbVersion.ts +++ b/src/utils/dbVersion.ts @@ -163,6 +163,13 @@ export class DbVersionSupport { message: this.errorMessage('Tokenize endpoint stopwords / stopwordPresets', version.show(), '1.37.2'), })); + supportsIncrementalBackups = () => + this.dbVersionProvider.getVersion().then((version) => ({ + version, + supports: version.isAtLeast(1, 37, 0), + message: this.errorMessage('Incremental backups', version.show(), '1.37.0'), + })); + supportsServerSideDefaultVectorIndexType = () => this.dbVersionProvider.getVersion().then((version) => ({ version, diff --git a/src/v2/index.ts b/src/v2/index.ts index 57c0e51b..13502d5f 100644 --- a/src/v2/index.ts +++ b/src/v2/index.ts @@ -51,7 +51,7 @@ const app = { batch: batch(conn, dbVersionSupport), misc: misc(conn, dbVersionProvider), c11y: c11y(conn), - backup: backup(conn), + backup: backup(conn, dbVersionSupport), cluster: cluster(conn), }; diff --git a/test/collections/backup/integration.test.ts b/test/collections/backup/integration.test.ts index db90067d..f047cf2a 100644 --- a/test/collections/backup/integration.test.ts +++ b/test/collections/backup/integration.test.ts @@ -258,6 +258,58 @@ describe('Integration testing of backups', () => { }); }); + requireAtLeast(1, 37, 0).describe('incremental backups', () => { + it('creates and restores an incremental backup on top of a base backup', async () => { + const client = await clientPromise; + const collection = await client.collections + .create({ name: 'TestIncrementalBackup' }) + .then((col) => col.data.insert().then(() => col)); + + const base = await client.backup.create({ + backupId: randomBackupId(), + backend: 'filesystem', + includeCollections: [collection.name], + waitForCompletion: true, + }); + expect(base.status).toBe('SUCCESS'); + + await collection.data.insert(); + + const incremental = await client.backup.create({ + backupId: randomBackupId(), + backend: 'filesystem', + includeCollections: [collection.name], + config: { incrementalBaseBackupId: base.id }, + waitForCompletion: true, + }); + expect(incremental.status).toBe('SUCCESS'); + + await client.collections.delete(collection.name); + const restored = await client.backup.restore({ + backupId: incremental.id, + backend: 'filesystem', + includeCollections: [collection.name], + waitForCompletion: true, + }); + expect(restored.status).toBe('SUCCESS'); + await expect(collection.length()).resolves.toBe(2); + + await client.collections.delete(collection.name); + }); + + it('sends the base backup ID to the server', async () => { + const client = await clientPromise; + await expect( + client.backup.create({ + backupId: randomBackupId(), + backend: 'filesystem', + includeCollections: ['TestBackupCollection'], + config: { incrementalBaseBackupId: 'does-not-exist' }, + }) + ).rejects.toThrow(/could not fetch base backup/); + }); + }); + function randomBackupId() { return 'backup-id-' + Math.floor(Math.random() * Number.MAX_SAFE_INTEGER); }