diff --git a/crates/vchordrq/src/build.rs b/crates/vchordrq/src/build.rs index a7371353..d622fbc6 100644 --- a/crates/vchordrq/src/build.rs +++ b/crates/vchordrq/src/build.rs @@ -119,6 +119,7 @@ pub fn build( height_of_root: structures.len() as u32, is_residual, rerank_in_heap: vchordrq_options.rerank_in_table, + indexed_vectors: Some(0), centroids_first: centroids.first(), vectors_first: vectors, centroid_prefetch: pointer_of_centroids diff --git a/crates/vchordrq/src/bulkdelete.rs b/crates/vchordrq/src/bulkdelete.rs index ae54ec54..0aae88fe 100644 --- a/crates/vchordrq/src/bulkdelete.rs +++ b/crates/vchordrq/src/bulkdelete.rs @@ -25,7 +25,8 @@ pub fn bulkdelete( index: &R, check: impl Fn(), callback: impl Fn(NonZero) -> bool, -) where +) -> u64 +where R::Page: Page, { let meta_guard = index.read(0); @@ -38,6 +39,8 @@ pub fn bulkdelete( drop(meta_guard); + let mut live = 0_u64; + let step = |state: State| { let mut results = Vec::new(); for first in state { @@ -64,19 +67,21 @@ pub fn bulkdelete( while current != u32::MAX { check(); let read = index.read(current); - let flag = 'flag: { + let (flag, page_live) = 'scan: { + let mut page_live = 0_u64; for i in 1..=read.len() { let bytes = read.get(i).expect("data corruption"); let tuple = FrozenTuple::deserialize_ref(bytes); if let FrozenTupleReader::_0(tuple) = tuple { for p in tuple.payload().iter() { if Some(true) == p.map(&callback) { - break 'flag true; + break 'scan (true, 0); } + page_live += u64::from(p.is_some()); } } } - false + (false, page_live) }; if flag { drop(read); @@ -89,9 +94,12 @@ pub fn bulkdelete( if Some(true) == p.map(&callback) { *p = None; } + live += u64::from(p.is_some()); } } } + } else { + live += page_live; } current = directory.next().unwrap_or(u32::MAX); } @@ -101,16 +109,18 @@ pub fn bulkdelete( while current != u32::MAX { check(); let read = index.read(current); - let flag = 'flag: { + let (flag, page_live) = 'scan: { + let mut page_live = 0_u64; for i in 1..=read.len() { let bytes = read.get(i).expect("data corruption"); let tuple = AppendableTuple::deserialize_ref(bytes); let p = tuple.payload(); if Some(true) == p.map(&callback) { - break 'flag true; + break 'scan (true, 0); } + page_live += u64::from(p.is_some()); } - false + (false, page_live) }; if flag { drop(read); @@ -122,14 +132,18 @@ pub fn bulkdelete( if Some(true) == p.map(&callback) { *p = None; } + live += u64::from(p.is_some()); } current = write.get_opaque().next; } else { + live += page_live; current = read.get_opaque().next; } } } } + + live } pub fn bulkdelete_vectors( diff --git a/crates/vchordrq/src/cost.rs b/crates/vchordrq/src/cost.rs index 60579401..c8e40252 100644 --- a/crates/vchordrq/src/cost.rs +++ b/crates/vchordrq/src/cost.rs @@ -18,6 +18,7 @@ use index::relation::{Page, RelationRead}; pub struct Cost { pub dim: u32, pub cells: Vec, + pub indexed_vectors: Option, } #[must_use] @@ -27,8 +28,13 @@ pub fn cost(index: &R) -> Cost { let meta_tuple = MetaTuple::deserialize_ref(meta_bytes); let dim = meta_tuple.dim(); let cells = meta_tuple.cells().to_vec(); + let indexed_vectors = meta_tuple.indexed_vectors(); drop(meta_guard); - Cost { dim, cells } + Cost { + dim, + cells, + indexed_vectors, + } } diff --git a/crates/vchordrq/src/lib.rs b/crates/vchordrq/src/lib.rs index d0efef05..1d6679f8 100644 --- a/crates/vchordrq/src/lib.rs +++ b/crates/vchordrq/src/lib.rs @@ -24,9 +24,11 @@ mod freepages; mod insert; mod linked_vec; mod maintain; +mod maxsim_cost; mod prewarm; mod rerank; mod search; +mod statistics; mod tape; mod tape_writer; mod tuples; @@ -43,9 +45,11 @@ pub use cost::cost; pub use fast_heap::FastHeap; pub use insert::{InsertChooser, insert, insert_vector}; pub use maintain::{MaintainChooser, maintain}; +pub use maxsim_cost::{MaxsimCostEstimate, MaxsimCostInput, estimate_maxsim_cost}; pub use prewarm::prewarm; pub use rerank::{how, rerank_heap, rerank_index}; pub use search::{default_search, maxsim_search}; +pub use statistics::set_indexed_vectors; use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout}; diff --git a/crates/vchordrq/src/maxsim_cost.rs b/crates/vchordrq/src/maxsim_cost.rs new file mode 100644 index 00000000..68580ce5 --- /dev/null +++ b/crates/vchordrq/src/maxsim_cost.rs @@ -0,0 +1,128 @@ +// This software is licensed under a dual license model: +// +// GNU Affero General Public License v3 (AGPLv3): You may use, modify, and +// distribute this software under the terms of the AGPLv3. +// +// Elastic License v2 (ELv2): You may also use, modify, and distribute this +// software under the Elastic License v2, which has specific restrictions. +// +// We welcome any commercial collaboration or support. For inquiries +// regarding the licenses, please contact us at: +// vectorchord-inquiry@tensorchord.ai +// +// Copyright (c) 2025-2026 TensorChord Inc. + +#[derive(Clone, Copy, Debug)] +pub struct MaxsimCostInput { + pub heap_rows: f64, + pub index_tokens: f64, + pub token_nodes_per_query: f64, + pub base_index_pages: f64, + pub query_tokens: u32, + pub limit_tuples: Option, + pub filter_selectivity: f64, +} + +#[derive(Clone, Copy, Debug)] +pub struct MaxsimCostEstimate { + pub startup_cost: f64, + pub total_cost: f64, + pub selectivity: f64, + pub index_pages: f64, +} + +/// Estimate the eager token search and page aggregation performed by MaxSim. +/// +/// The constants are deliberately conservative rather than hardware-specific; +/// an optional exact reranker can add its own cost in a later layer. +pub fn estimate_maxsim_cost(input: MaxsimCostInput) -> MaxsimCostEstimate { + let heap_rows = input.heap_rows.max(1.0); + let index_tokens = input.index_tokens.max(heap_rows); + let query_tokens = f64::from(input.query_tokens.max(1)); + let average_document_tokens = (index_tokens / heap_rows).clamp(1.0, 65_536.0); + let token_visits = input.token_nodes_per_query.max(1.0) * query_tokens; + + // Until page-level candidate statistics exist, approximate the chance that + // at least one token from a document is visited by the token index. + let token_visit_fraction = (token_visits / index_tokens).clamp(0.0, 1.0); + let candidate_probability = 1.0 - (1.0 - token_visit_fraction).powf(average_document_tokens); + let generated_pages = (heap_rows * candidate_probability).clamp(1.0, heap_rows); + + let returned_pages = input + .limit_tuples + .map(|limit| limit.max(1.0) / input.filter_selectivity.clamp(1e-9, 1.0)) + .unwrap_or(generated_pages) + .min(generated_pages); + + // Search and aggregation are eager in the current scanner, so LIMIT does + // not remove this work from startup cost. + let startup_cost = 0.001 * token_visits + 0.01 * token_visits + 0.05 * generated_pages; + let total_cost = startup_cost + returned_pages; + let selectivity = (returned_pages / heap_rows).clamp(1e-9, 1.0); + let index_pages = + input.base_index_pages.max(1.0) * (1.0 + 0.25 * (query_tokens - 1.0).max(0.0)); + + MaxsimCostEstimate { + startup_cost, + total_cost, + selectivity, + index_pages, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn input() -> MaxsimCostInput { + MaxsimCostInput { + heap_rows: 10_000.0, + index_tokens: 1_000_000.0, + token_nodes_per_query: 2_000.0, + base_index_pages: 1_000.0, + query_tokens: 16, + limit_tuples: Some(20.0), + filter_selectivity: 1.0, + } + } + + #[test] + fn maxsim_cost_is_finite_and_nonzero() { + let estimate = estimate_maxsim_cost(input()); + assert!(estimate.startup_cost > 0.0); + assert!(estimate.total_cost >= estimate.startup_cost); + assert!((1e-9..=1.0).contains(&estimate.selectivity)); + assert!(estimate.index_pages >= 1.0); + } + + #[test] + fn query_token_count_increases_eager_work() { + let one = estimate_maxsim_cost(MaxsimCostInput { + query_tokens: 1, + ..input() + }); + let many = estimate_maxsim_cost(MaxsimCostInput { + query_tokens: 64, + ..input() + }); + assert!(many.startup_cost > one.startup_cost); + assert!(many.index_pages > one.index_pages); + } + + #[test] + fn missing_statistics_remain_finite() { + let estimate = estimate_maxsim_cost(MaxsimCostInput { + heap_rows: -1.0, + index_tokens: 0.0, + token_nodes_per_query: 0.0, + base_index_pages: 0.0, + filter_selectivity: 0.0, + limit_tuples: None, + ..input() + }); + assert!(estimate.startup_cost.is_finite()); + assert!(estimate.total_cost.is_finite()); + assert!(estimate.selectivity.is_finite()); + assert!(estimate.index_pages.is_finite()); + } +} diff --git a/crates/vchordrq/src/statistics.rs b/crates/vchordrq/src/statistics.rs new file mode 100644 index 00000000..80d850e2 --- /dev/null +++ b/crates/vchordrq/src/statistics.rs @@ -0,0 +1,30 @@ +// This software is licensed under a dual license model: +// +// GNU Affero General Public License v3 (AGPLv3): You may use, modify, and +// distribute this software under the terms of the AGPLv3. +// +// Elastic License v2 (ELv2): You may also use, modify, and distribute this +// software under the Elastic License v2, which has specific restrictions. +// +// We welcome any commercial collaboration or support. For inquiries +// regarding the licenses, please contact us at: +// vectorchord-inquiry@tensorchord.ai +// +// Copyright (c) 2025-2026 TensorChord Inc. + +use crate::tuples::{MetaTuple, WithWriter}; +use index::relation::{Page, RelationWrite}; + +/// Store the number of live vector nodes observed by the latest complete +/// build or vacuum pass. +/// +/// This is deliberately refreshed in bulk instead of on every insert. A +/// per-insert update would serialize all writers on the metapage, which is a +/// poor tradeoff for a planner statistic. Like PostgreSQL's relation +/// statistics, the value may be stale between maintenance passes. +pub fn set_indexed_vectors(index: &R, indexed_vectors: u64) { + let mut meta_guard = index.write(0, false); + let meta_bytes = meta_guard.get_mut(1).expect("data corruption"); + let mut meta_tuple = MetaTuple::deserialize_mut(meta_bytes); + meta_tuple.set_indexed_vectors(indexed_vectors); +} diff --git a/crates/vchordrq/src/tuples.rs b/crates/vchordrq/src/tuples.rs index 8438c1ae..85dee786 100644 --- a/crates/vchordrq/src/tuples.rs +++ b/crates/vchordrq/src/tuples.rs @@ -21,6 +21,8 @@ pub const ALIGN: usize = 8; pub type Tag = u64; const MAGIC: Tag = Tag::from_ne_bytes(*b"vchordrq"); const VERSION: u64 = 1001; +const STATISTICS_VERSION: u16 = 1; +const MAX_INDEXED_VECTORS: u64 = (1_u64 << 48) - 1; #[inline(always)] fn tag(source: &[u8]) -> Tag { @@ -55,12 +57,13 @@ struct MetaTupleHeader { rerank_in_heap: Bool, cells_s: u16, cells_e: u16, - _padding_0: [Padding; 2], + statistics_version: u16, centroids_first: u32, vectors_first_s: u16, vectors_first_e: u16, freepages_first: u32, - _padding_1: [Padding; 6], + indexed_vectors_low: u32, + indexed_vectors_high: u16, // tree centroid_prefetch_s: u16, centroid_prefetch_e: u16, @@ -69,11 +72,24 @@ struct MetaTupleHeader { first: u32, } +// Statistics deliberately replace the old 2-byte and 6-byte padding regions. +// Keep these assertions in non-test builds: changing any offset would require +// an index format version bump and REINDEX instead of the compatibility path. +const _: () = { + assert!(size_of::() == 56); + assert!(std::mem::offset_of!(MetaTupleHeader, statistics_version) == 22); + assert!(std::mem::offset_of!(MetaTupleHeader, centroids_first) == 24); + assert!(std::mem::offset_of!(MetaTupleHeader, indexed_vectors_low) == 36); + assert!(std::mem::offset_of!(MetaTupleHeader, indexed_vectors_high) == 40); + assert!(std::mem::offset_of!(MetaTupleHeader, centroid_prefetch_s) == 42); +}; + pub struct MetaTuple { pub dim: u32, pub height_of_root: u32, pub is_residual: bool, pub rerank_in_heap: bool, + pub indexed_vectors: Option, pub cells: Vec, pub centroids_first: u32, pub vectors_first: Vec, @@ -94,6 +110,7 @@ impl Tuple for MetaTuple { height_of_root, is_residual, rerank_in_heap, + indexed_vectors, cells, centroids_first, vectors_first, @@ -103,6 +120,12 @@ impl Tuple for MetaTuple { centroid_norm, first, } => { + if let Some(indexed_vectors) = indexed_vectors { + assert!( + *indexed_vectors <= MAX_INDEXED_VECTORS, + "indexed vector count exceeds the on-disk 48-bit limit" + ); + } buffer.extend((MAGIC as Tag).to_ne_bytes()); buffer.extend(std::iter::repeat_n(0, size_of::())); // cells @@ -136,17 +159,20 @@ impl Tuple for MetaTuple { rerank_in_heap: (*rerank_in_heap).into(), cells_s, cells_e, + statistics_version: indexed_vectors + .map(|_| STATISTICS_VERSION) + .unwrap_or(0), centroids_first: *centroids_first, vectors_first_s, vectors_first_e, freepages_first: *freepages_first, + indexed_vectors_low: indexed_vectors.unwrap_or(0) as u32, + indexed_vectors_high: (indexed_vectors.unwrap_or(0) >> 32) as u16, centroid_prefetch_s, centroid_prefetch_e, centroid_head: *centroid_head, centroid_norm: *centroid_norm, first: *first, - _padding_0: Default::default(), - _padding_1: Default::default(), } .as_bytes(), ); @@ -186,6 +212,28 @@ impl WithReader for MetaTuple { } } +impl WithWriter for MetaTuple { + type Writer<'a> = MetaTupleWriter<'a>; + + fn deserialize_mut(source: &mut [u8]) -> MetaTupleWriter<'_> { + let tag = tag(source); + match tag { + MAGIC => { + let mut checker = MutChecker::new(source); + let header: &mut MetaTupleHeader = checker.prefix(size_of::()); + if VERSION != header.version { + panic!( + "deserialization: bad version number; {}", + "after upgrading VectorChord, please use REINDEX to rebuild the index." + ); + } + MetaTupleWriter { header } + } + _ => panic!("deserialization: bad magic number"), + } + } +} + #[derive(Debug, Clone, Copy)] pub struct MetaTupleReader<'a> { header: &'a MetaTupleHeader, @@ -207,6 +255,16 @@ impl<'a> MetaTupleReader<'a> { pub fn rerank_in_heap(self) -> bool { self.header.rerank_in_heap.into() } + pub fn indexed_vectors(self) -> Option { + match self.header.statistics_version { + 0 => None, + STATISTICS_VERSION => Some( + u64::from(self.header.indexed_vectors_low) + | (u64::from(self.header.indexed_vectors_high) << 32), + ), + _ => panic!("deserialization: unsupported statistics version"), + } + } pub fn cells(self) -> &'a [u32] { self.cells } @@ -233,6 +291,23 @@ impl<'a> MetaTupleReader<'a> { } } +#[derive(Debug)] +pub struct MetaTupleWriter<'a> { + header: &'a mut MetaTupleHeader, +} + +impl MetaTupleWriter<'_> { + pub fn set_indexed_vectors(&mut self, indexed_vectors: u64) { + assert!( + indexed_vectors <= MAX_INDEXED_VECTORS, + "indexed vector count exceeds the on-disk 48-bit limit" + ); + self.header.statistics_version = STATISTICS_VERSION; + self.header.indexed_vectors_low = indexed_vectors as u32; + self.header.indexed_vectors_high = (indexed_vectors >> 32) as u16; + } +} + #[repr(C, align(8))] #[derive(Debug, Clone, FromBytes, IntoBytes, Immutable, KnownLayout)] struct FreepagesTupleHeader { @@ -1485,3 +1560,71 @@ impl AppendableTupleWriter<'_> { &mut self.header.payload } } + +#[cfg(test)] +mod tests { + use super::*; + + fn meta(indexed_vectors: Option) -> MetaTuple { + MetaTuple { + dim: 3, + height_of_root: 1, + is_residual: false, + rerank_in_heap: false, + indexed_vectors, + cells: vec![4], + centroids_first: 1, + vectors_first: vec![2], + freepages_first: 3, + centroid_prefetch: vec![4], + centroid_head: 0, + centroid_norm: 1.0, + first: 5, + } + } + + #[test] + fn meta_statistics_reuse_padding_without_moving_fields() { + assert_eq!(size_of::(), 56); + assert_eq!( + std::mem::offset_of!(MetaTupleHeader, statistics_version), + 22 + ); + assert_eq!(std::mem::offset_of!(MetaTupleHeader, centroids_first), 24); + assert_eq!( + std::mem::offset_of!(MetaTupleHeader, indexed_vectors_low), + 36 + ); + assert_eq!( + std::mem::offset_of!(MetaTupleHeader, indexed_vectors_high), + 40 + ); + assert_eq!( + std::mem::offset_of!(MetaTupleHeader, centroid_prefetch_s), + 42 + ); + } + + #[test] + fn meta_statistics_roundtrip_full_48_bit_range() { + for expected in [0, 1, u32::MAX as u64 + 1, MAX_INDEXED_VECTORS] { + let bytes = meta(Some(expected)).serialize(); + assert_eq!( + MetaTuple::deserialize_ref(&bytes).indexed_vectors(), + Some(expected) + ); + } + } + + #[test] + fn old_meta_padding_reads_as_missing_statistics_and_can_be_upgraded() { + let mut bytes = meta(None).serialize(); + assert_eq!(MetaTuple::deserialize_ref(&bytes).indexed_vectors(), None); + + MetaTuple::deserialize_mut(&mut bytes).set_indexed_vectors(42); + assert_eq!( + MetaTuple::deserialize_ref(&bytes).indexed_vectors(), + Some(42) + ); + } +} diff --git a/crates/vchordrq/src/types.rs b/crates/vchordrq/src/types.rs index c41ba501..d9b85cbc 100644 --- a/crates/vchordrq/src/types.rs +++ b/crates/vchordrq/src/types.rs @@ -12,12 +12,14 @@ // // Copyright (c) 2025-2026 TensorChord Inc. +use distance::Distance; use serde::{Deserialize, Serialize}; use simd::f16; use validator::{Validate, ValidationError}; use vector::rabitq4::{Rabitq4Borrowed, Rabitq4Owned}; use vector::rabitq8::{Rabitq8Borrowed, Rabitq8Owned}; use vector::vect::{VectBorrowed, VectOwned}; +use vector::{VectorBorrowed, VectorOwned}; #[derive(Debug, Clone, Serialize, Deserialize, Validate)] #[serde(deny_unknown_fields)] @@ -61,6 +63,38 @@ pub enum OwnedVector { Rabitq4(Rabitq4Owned), } +impl OwnedVector { + pub fn dim(&self) -> u32 { + match self { + Self::Vecf32(vector) => vector.as_borrowed().dim(), + Self::Vecf16(vector) => vector.as_borrowed().dim(), + Self::Rabitq8(vector) => vector.as_borrowed().dim(), + Self::Rabitq4(vector) => vector.as_borrowed().dim(), + } + } + + pub fn operator_dot(&self, rhs: &Self) -> Option { + if self.dim() != rhs.dim() { + return None; + } + match (self, rhs) { + (Self::Vecf32(lhs), Self::Vecf32(rhs)) => { + Some(lhs.as_borrowed().operator_dot(rhs.as_borrowed())) + } + (Self::Vecf16(lhs), Self::Vecf16(rhs)) => { + Some(lhs.as_borrowed().operator_dot(rhs.as_borrowed())) + } + (Self::Rabitq8(lhs), Self::Rabitq8(rhs)) => { + Some(lhs.as_borrowed().operator_dot(rhs.as_borrowed())) + } + (Self::Rabitq4(lhs), Self::Rabitq4(rhs)) => { + Some(lhs.as_borrowed().operator_dot(rhs.as_borrowed())) + } + _ => None, + } + } +} + #[derive(Debug, Clone, Copy)] pub enum BorrowedVector<'a> { Vecf32(VectBorrowed<'a, f32>), diff --git a/src/index/gucs.rs b/src/index/gucs.rs index 8c0e9fd3..2dc08d26 100644 --- a/src/index/gucs.rs +++ b/src/index/gucs.rs @@ -78,6 +78,10 @@ static VCHORDRQ_MAXSIM_THRESHOLD: GucSetting = GucSetting::::new(0); static mut VCHORDRQ_MAXSIM_THRESHOLD_CONFIG: *mut pgrx::pg_sys::config_generic = core::ptr::null_mut(); +static VCHORDRQ_MAXSIM_PLANNER_QUERY_TOKENS: GucSetting = GucSetting::::new(32); + +static VCHORDRQ_MAXSIM_PLANNER_DOCUMENT_TOKENS: GucSetting = GucSetting::::new(256); + static VCHORDRQ_PREFILTER: GucSetting = GucSetting::::new(false); static VCHORDRQ_IO_SEARCH: GucSetting = GucSetting::::new( @@ -151,6 +155,26 @@ pub fn init() { GucContext::Userset, GucFlags::default(), ); + GucRegistry::define_int_guc( + c"vchordrq.maxsim_planner_query_tokens", + c"Expected MaxSim query-token count used by the planner.", + c"Set this to the measured deployment average until expression statistics are available.", + &VCHORDRQ_MAXSIM_PLANNER_QUERY_TOKENS, + 1, + 65_536, + GucContext::Userset, + GucFlags::default(), + ); + GucRegistry::define_int_guc( + c"vchordrq.maxsim_planner_document_tokens", + c"Fallback MaxSim document-token count used by the planner.", + c"Used for indexes that predate the indexed-vector statistic.", + &VCHORDRQ_MAXSIM_PLANNER_DOCUMENT_TOKENS, + 1, + 65_536, + GucContext::Userset, + GucFlags::default(), + ); GucRegistry::define_bool_guc( c"vchordrq.prefilter", c"`prefilter` argument of vchordrq.", @@ -472,6 +496,14 @@ pub fn vchordrq_maxsim_threshold(index: pgrx::pg_sys::Relation) -> u32 { } } +pub fn vchordrq_maxsim_planner_query_tokens() -> u32 { + VCHORDRQ_MAXSIM_PLANNER_QUERY_TOKENS.get() as u32 +} + +pub fn vchordrq_maxsim_planner_document_tokens() -> u32 { + VCHORDRQ_MAXSIM_PLANNER_DOCUMENT_TOKENS.get() as u32 +} + pub fn vchordrq_prefilter() -> bool { VCHORDRQ_PREFILTER.get() } diff --git a/src/index/vchordrq/am/am_build.rs b/src/index/vchordrq/am/am_build.rs index 647d2478..95b4bc45 100644 --- a/src/index/vchordrq/am/am_build.rs +++ b/src/index/vchordrq/am/am_build.rs @@ -349,9 +349,10 @@ pub unsafe extern "C-unwind" fn ambuild( reporter.phase(BuildPhase::from_code(BuildPhaseCode::Inserting)); order }, - |indtuples| { + |indtuples, indexed_vectors| { reporter.tuples_done(indtuples); reporter.tuples_total(indtuples); + vchordrq::set_indexed_vectors(&index, indexed_vectors); // enter the barrier let shared = leader.vchordrqshared; pgrx::pg_sys::SpinLockAcquire(&raw mut (*shared).mutex); @@ -429,9 +430,10 @@ pub unsafe extern "C-unwind" fn ambuild( || { reporter.phase(BuildPhase::from_code(BuildPhaseCode::Inserting)); }, - |indtuples| { + |indtuples, indexed_vectors| { reporter.tuples_done(indtuples); reporter.tuples_total(indtuples); + vchordrq::set_indexed_vectors(&index, indexed_vectors); reporter.phase(BuildPhase::from_code(BuildPhaseCode::Compacting)); }, || {}, @@ -460,6 +462,7 @@ struct VchordrqShared { barrier_enter_0: i32, nparticipants: u32, indtuples: u64, + indexed_vectors: u64, barrier_leave_0: bool, barrier_enter_1: i32, barrier_leave_1: bool, @@ -721,6 +724,7 @@ impl VchordrqLeader { barrier_leave_2: false, mutex: std::mem::zeroed(), indtuples: 0, + indexed_vectors: 0, }); pgrx::pg_sys::ConditionVariableInit(&raw mut (*vchordrqshared).condvar_barrier_enter_0); pgrx::pg_sys::ConditionVariableInit(&raw mut (*vchordrqshared).condvar_barrier_leave_0); @@ -875,7 +879,7 @@ pub unsafe extern "C-unwind" fn vchordrq_parallel_build_main( pgrx::pg_sys::ConditionVariableCancelSleep(); order }, - |_| { + |_, _| { // enter the barrier let shared = vchordrqshared; pgrx::pg_sys::SpinLockAcquire(&raw mut (*shared).mutex); @@ -941,7 +945,7 @@ unsafe fn parallel_build( vchordrqcached: *const u8, mut callback: impl FnMut(u64), sync_0: impl FnOnce() -> u32, - sync_1: impl FnOnce(u64), + sync_1: impl FnOnce(u64, u64), sync_2: impl FnOnce(), ) { use vchordrq_cached::VchordrqCachedReader; @@ -989,6 +993,7 @@ unsafe fn parallel_build( let store = value .and_then(|x| unsafe { opfamily.store(x) }) .unwrap_or_default(); + let indexed_vectors = store.len() as u64; for (vector, extra) in store { let key = ctid_to_key(ctid); let payload = kv_to_pointer((key, extra)); @@ -1010,6 +1015,7 @@ unsafe fn parallel_build( { pgrx::pg_sys::SpinLockAcquire(&raw mut (*vchordrqshared).mutex); (*vchordrqshared).indtuples += 1; + (*vchordrqshared).indexed_vectors += indexed_vectors; indtuples = (*vchordrqshared).indtuples; pgrx::pg_sys::SpinLockRelease(&raw mut (*vchordrqshared).mutex); } @@ -1029,6 +1035,7 @@ unsafe fn parallel_build( let store = value .and_then(|x| unsafe { opfamily.store(x) }) .unwrap_or_default(); + let indexed_vectors = store.len() as u64; for (vector, extra) in store { let key = ctid_to_key(ctid); let payload = kv_to_pointer((key, extra)); @@ -1050,6 +1057,7 @@ unsafe fn parallel_build( { pgrx::pg_sys::SpinLockAcquire(&raw mut (*vchordrqshared).mutex); (*vchordrqshared).indtuples += 1; + (*vchordrqshared).indexed_vectors += indexed_vectors; indtuples = (*vchordrqshared).indtuples; pgrx::pg_sys::SpinLockRelease(&raw mut (*vchordrqshared).mutex); } @@ -1065,7 +1073,13 @@ unsafe fn parallel_build( drop(index); - sync_1(unsafe { (*vchordrqshared).indtuples }); + let (indtuples, indexed_vectors) = unsafe { + ( + (*vchordrqshared).indtuples, + (*vchordrqshared).indexed_vectors, + ) + }; + sync_1(indtuples, indexed_vectors); let index = unsafe { PostgresRelation::new(index_relation) }; @@ -1085,7 +1099,7 @@ unsafe fn sequential_build( vchordrqcached: &[u8], mut callback: impl FnMut(u64), sync_0: impl FnOnce(), - sync_1: impl FnOnce(u64), + sync_1: impl FnOnce(u64, u64), sync_2: impl FnOnce(), ) { use vchordrq_cached::VchordrqCachedReader; @@ -1125,6 +1139,7 @@ unsafe fn sequential_build( let index = unsafe { BufferedPostgresRelation::new(index_relation) }; let mut indtuples = 0; + let mut indexed_vectors = 0_u64; match cached { VchordrqCachedReader::_0(_) => { traverser.traverse(true, |tuple: &mut dyn crate::index::traverse::Tuple| { @@ -1134,6 +1149,7 @@ unsafe fn sequential_build( let store = value .and_then(|x| unsafe { opfamily.store(x) }) .unwrap_or_default(); + indexed_vectors += store.len() as u64; for (vector, extra) in store { let key = ctid_to_key(ctid); let payload = kv_to_pointer((key, extra)); @@ -1166,6 +1182,7 @@ unsafe fn sequential_build( let store = value .and_then(|x| unsafe { opfamily.store(x) }) .unwrap_or_default(); + indexed_vectors += store.len() as u64; for (vector, extra) in store { let key = ctid_to_key(ctid); let payload = kv_to_pointer((key, extra)); @@ -1194,7 +1211,7 @@ unsafe fn sequential_build( drop(index); - sync_1(indtuples); + sync_1(indtuples, indexed_vectors); let index = unsafe { PostgresRelation::new(index_relation) }; diff --git a/src/index/vchordrq/am/mod.rs b/src/index/vchordrq/am/mod.rs index 5040b587..07c2c71b 100644 --- a/src/index/vchordrq/am/mod.rs +++ b/src/index/vchordrq/am/mod.rs @@ -307,32 +307,22 @@ pub unsafe extern "C-unwind" fn amcostestimate( std::ptr::null_mut(), ) }; + // PostgreSQL's relation estimate is the planner's current heap-row + // cardinality after base restrictions. Keep the raw clause + // selectivity separately because the MaxSim cost model needs both. + let total_rows = (*(*index_opt_info).rel).rows.max(1.0); + let filter_selectivity = selectivity.clamp(0.0, 1.0); // index exists if !(*index_opt_info).hypothetical { let relation = Index::open((*index_opt_info).indexoid, pgrx::pg_sys::NoLock as _); let opfamily = opfamily(relation.raw()); - if !matches!( + let is_maxsim = matches!( opfamily, - Opfamily::HalfvecCosine - | Opfamily::HalfvecIp - | Opfamily::HalfvecL2 - | Opfamily::VectorCosine - | Opfamily::VectorIp - | Opfamily::VectorL2 - | Opfamily::Rabitq8Cosine - | Opfamily::Rabitq8Ip - | Opfamily::Rabitq8L2 - | Opfamily::Rabitq4Cosine - | Opfamily::Rabitq4Ip - | Opfamily::Rabitq4L2 - ) { - *index_startup_cost = 0.0; - *index_total_cost = 0.0; - *index_selectivity = 1.0; - *index_correlation = 0.0; - *index_pages = 1.0; - return; - } + Opfamily::VectorMaxsim + | Opfamily::HalfvecMaxsim + | Opfamily::Rabitq8Maxsim + | Opfamily::Rabitq4Maxsim + ); let index = PostgresRelation::::new(relation.raw()); let probes = gucs::vchordrq_probes(relation.raw()); let cost = vchordrq::cost(&index); @@ -343,15 +333,25 @@ pub unsafe extern "C-unwind" fn amcostestimate( probes.len() ); } + let estimated_index_vectors = if is_maxsim { + cost.indexed_vectors.map_or_else( + || { + (*index_opt_info).tuples.max(total_rows).max(1.0) + * f64::from(gucs::vchordrq_maxsim_planner_document_tokens()) + }, + |indexed_vectors| indexed_vectors as f64, + ) + } else { + (*index_opt_info).tuples.max(0.0) + }; let node_count = { - let tuples = (*index_opt_info).tuples as u32; let mut count = 0.0; - let r = cost.cells.iter().copied().rev(); - let numerator = std::iter::once(1).chain(probes.clone()); + let r = cost.cells.iter().copied().rev().map(f64::from); + let numerator = std::iter::once(1.0).chain(probes.iter().copied().map(f64::from)); let denumerator = r.clone(); - let scale = r.skip(1).chain(std::iter::once(tuples)); + let scale = r.skip(1).chain(std::iter::once(estimated_index_vectors)); for (scale, (numerator, denumerator)) in scale.zip(numerator.zip(denumerator)) { - count += (scale as f64) * 1.0f64.min((numerator as f64) / (denumerator as f64)); + count += scale * 1.0f64.min(numerator / denumerator); } count }; @@ -367,6 +367,27 @@ pub unsafe extern "C-unwind" fn amcostestimate( pages += cost.cells[0] as f64; pages }; + if is_maxsim { + let estimate = vchordrq::estimate_maxsim_cost(vchordrq::MaxsimCostInput { + heap_rows: total_rows, + index_tokens: estimated_index_vectors, + token_nodes_per_query: node_count, + base_index_pages: page_count, + query_tokens: gucs::vchordrq_maxsim_planner_query_tokens(), + limit_tuples: ((*root).limit_tuples > 0.0).then_some((*root).limit_tuples), + filter_selectivity, + }); + *index_startup_cost = estimate.startup_cost; + *index_total_cost = estimate.total_cost; + *index_selectivity = estimate.selectivity; + *index_correlation = 0.0; + *index_pages = estimate.index_pages; + return; + } + // Preserve upstream's cost/selectivity behavior for every + // non-MaxSim opfamily. TileMaxSim-specific estimates above are + // intentionally isolated so enabling the optional backend cannot + // perturb the normal vector scan planner path. let next_count = f64::max(1.0, (*root).limit_tuples) * f64::min(1000.0, 1.0 / selectivity); *index_startup_cost = 0.001 * node_count; @@ -458,7 +479,9 @@ pub unsafe extern "C-unwind" fn ambulkdelete( pg_guard_ffi_boundary(|| callback(&mut ctid, callback_state)) } }; - crate::index::vchordrq::dispatch::bulkdelete(opfamily, &index, check, callback); + let indexed_vectors = + crate::index::vchordrq::dispatch::bulkdelete(opfamily, &index, check, callback); + vchordrq::set_indexed_vectors(&index, indexed_vectors); stats } diff --git a/src/index/vchordrq/dispatch.rs b/src/index/vchordrq/dispatch.rs index 01d70a87..36baee63 100644 --- a/src/index/vchordrq/dispatch.rs +++ b/src/index/vchordrq/dispatch.rs @@ -71,42 +71,51 @@ pub fn bulkdelete( index: &R, check: impl Fn(), callback: impl Fn(NonZero) -> bool, -) where +) -> u64 +where R: RelationRead + RelationWrite, R::Page: Page, { match (opfamily.vector_kind(), opfamily.distance_kind()) { (VectorKind::Vecf32, DistanceKind::L2S) => { - vchordrq::bulkdelete::<_, Op, L2S>>(index, &check, &callback); + let live = vchordrq::bulkdelete::<_, Op, L2S>>(index, &check, &callback); vchordrq::bulkdelete_vectors::<_, Op, L2S>>(index, &check, &callback); + live } (VectorKind::Vecf32, DistanceKind::Dot) => { - vchordrq::bulkdelete::<_, Op, Dot>>(index, &check, &callback); + let live = vchordrq::bulkdelete::<_, Op, Dot>>(index, &check, &callback); vchordrq::bulkdelete_vectors::<_, Op, Dot>>(index, &check, &callback); + live } (VectorKind::Vecf16, DistanceKind::L2S) => { - vchordrq::bulkdelete::<_, Op, L2S>>(index, &check, &callback); + let live = vchordrq::bulkdelete::<_, Op, L2S>>(index, &check, &callback); vchordrq::bulkdelete_vectors::<_, Op, L2S>>(index, &check, &callback); + live } (VectorKind::Vecf16, DistanceKind::Dot) => { - vchordrq::bulkdelete::<_, Op, Dot>>(index, &check, &callback); + let live = vchordrq::bulkdelete::<_, Op, Dot>>(index, &check, &callback); vchordrq::bulkdelete_vectors::<_, Op, Dot>>(index, &check, &callback); + live } (VectorKind::Rabitq8, DistanceKind::L2S) => { - vchordrq::bulkdelete::<_, Op>(index, &check, &callback); + let live = vchordrq::bulkdelete::<_, Op>(index, &check, &callback); vchordrq::bulkdelete_vectors::<_, Op>(index, &check, &callback); + live } (VectorKind::Rabitq8, DistanceKind::Dot) => { - vchordrq::bulkdelete::<_, Op>(index, &check, &callback); + let live = vchordrq::bulkdelete::<_, Op>(index, &check, &callback); vchordrq::bulkdelete_vectors::<_, Op>(index, &check, &callback); + live } (VectorKind::Rabitq4, DistanceKind::L2S) => { - vchordrq::bulkdelete::<_, Op>(index, &check, &callback); + let live = vchordrq::bulkdelete::<_, Op>(index, &check, &callback); vchordrq::bulkdelete_vectors::<_, Op>(index, &check, &callback); + live } (VectorKind::Rabitq4, DistanceKind::Dot) => { - vchordrq::bulkdelete::<_, Op>(index, &check, &callback); + let live = vchordrq::bulkdelete::<_, Op>(index, &check, &callback); vchordrq::bulkdelete_vectors::<_, Op>(index, &check, &callback); + live } } }