Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions arrow-array/src/array/byte_view_array.rs
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,23 @@ impl<T: ByteViewType + ?Sized> GenericByteViewArray<T> {
&self.buffers
}

/// Returns a cloned `Arc` of the buffers storing non-inline string or binary data.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we should just change the other APIs be consistent -- like make data_buffers return &Arc<[Buffer]> and change new_unchecked to take Arc<[Buffer]> directly (rather than Into<Arc<[Buffer]>> )

That way we could find more places that unecessairly copy these buffers 🤔

///
/// This is useful when needing to construct a new byte view array from this existing
/// array, but [`into_parts`] is not feasible (e.g. need to keep both arrays around),
/// and trying to reconstruct the buffers from [`data_buffers`] would require a
/// `Vec` allocation and cloning of each buffer element, which can be expensive
/// if there is a large number of buffers.
///
/// This operation is `O(1)` and clones only the collection's `Arc`.
///
/// [`into_parts`]: Self::into_parts
/// [`data_buffers`]: Self::data_buffers
#[inline]
pub fn data_buffers_cloned(&self) -> Arc<[Buffer]> {
Arc::clone(&self.buffers)
Comment thread
YimingQiao marked this conversation as resolved.
}

/// Returns the element at index `i`
///
/// Note: This method does not check for nulls and the value is arbitrary
Expand Down
5 changes: 4 additions & 1 deletion arrow-select/src/filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -934,7 +934,7 @@ fn filter_byte_view<T: ByteViewType>(
) -> GenericByteViewArray<T> {
let new_view_buffer = filter_native(array.views(), predicate);
let views = ScalarBuffer::new(new_view_buffer, 0, predicate.count);
let buffers = array.data_buffers().to_vec();
let buffers = array.data_buffers_cloned();
let nulls = predicate.filter_nulls(array.nulls());

// SAFETY: each view is copied unchanged from `array.views()` and `buffers`
Expand Down Expand Up @@ -1297,6 +1297,9 @@ mod tests {
let actual = filter(&array, &predicate).unwrap();

assert_eq!(actual.len(), 3);
let actual_buffers = actual.as_byte_view::<T>().data_buffers_cloned();
let input_buffers = array.data_buffers_cloned();
assert!(Arc::ptr_eq(&actual_buffers, &input_buffers));

let expected = {
// ["hello", null, "large payload over 12 bytes"]
Expand Down
8 changes: 5 additions & 3 deletions arrow-select/src/take.rs
Original file line number Diff line number Diff line change
Expand Up @@ -636,10 +636,9 @@ fn take_byte_view<T: ByteViewType, IndexType: ArrowPrimitiveType>(
) -> Result<GenericByteViewArray<T>, ArrowError> {
let new_views = take_native(array.views(), indices);
let new_nulls = take_nulls(array.nulls(), indices);
let buffers = array.data_buffers_cloned();
// Safety: array.views was valid, and take_native copies only valid values, and verifies bounds
Ok(unsafe {
GenericByteViewArray::new_unchecked(new_views, array.data_buffers().to_vec(), new_nulls)
})
Ok(unsafe { GenericByteViewArray::new_unchecked(new_views, buffers, new_nulls) })
}

/// `take` implementation for list arrays
Expand Down Expand Up @@ -1806,6 +1805,9 @@ mod tests {
let actual = take(&array, &index, None).unwrap();

assert_eq!(actual.len(), index.len());
let actual_buffers = actual.as_byte_view::<T>().data_buffers_cloned();
let input_buffers = array.data_buffers_cloned();
assert!(Arc::ptr_eq(&actual_buffers, &input_buffers));

let expected = {
// ["large payload over 12 bytes", null, "world", "large payload over 12 bytes", "lulu", null]
Expand Down
Loading