From 1b341acbecadf3b3bc4495a6989c3404ae6485cf Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Thu, 9 Jul 2026 13:06:09 -0400 Subject: [PATCH 1/7] add `variant_to_arrow` union builder Casting Variant to dense or sparse Union dispatches each value to the union field that most exactly represents its runtime type (lossless widening allowed, declaration order breaks ties). Null rows land in a Null-typed child if declared, otherwise the first child, since unions have no top-level null buffer. Co-Authored-By: Claude Fable 5 --- parquet-variant-compute/src/variant_get.rs | 282 +++++++++++++++++- .../src/variant_to_arrow.rs | 228 +++++++++++++- 2 files changed, 503 insertions(+), 7 deletions(-) diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index fc01ca8d3b20..809dd43ce631 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -493,10 +493,10 @@ mod test { Array, ArrayRef, AsArray, BinaryArray, BinaryViewArray, BooleanArray, Date32Array, Date64Array, Decimal32Array, Decimal64Array, Decimal128Array, Decimal256Array, FixedSizeListArray, Float32Array, Float64Array, Int8Array, Int16Array, Int32Array, - Int64Array, LargeBinaryArray, LargeListArray, LargeListViewArray, LargeStringArray, - ListArray, ListViewArray, NullArray, NullBuilder, StringArray, StringViewArray, - StructArray, Time32MillisecondArray, Time32SecondArray, Time64MicrosecondArray, - Time64NanosecondArray, + Int64Array, Int64Builder, LargeBinaryArray, LargeListArray, LargeListViewArray, + LargeStringArray, ListArray, ListBuilder, ListViewArray, NullArray, NullBuilder, + StringArray, StringViewArray, StructArray, Time32MillisecondArray, Time32SecondArray, + Time64MicrosecondArray, Time64NanosecondArray, UnionArray, }; use arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow::compute::{CastOptions, cast}; @@ -505,7 +505,9 @@ mod test { use arrow::util::display::FormatOptions; use arrow_schema::ArrowError; use arrow_schema::DataType::{Boolean, Float32, Float64, Int8}; - use arrow_schema::{DataType, Field, FieldRef, Fields, IntervalUnit, TimeUnit}; + use arrow_schema::{ + DataType, Field, FieldRef, Fields, IntervalUnit, TimeUnit, UnionFields, UnionMode, + }; use chrono::DateTime; use parquet_variant::{ EMPTY_VARIANT_METADATA_BYTES, Variant, VariantDecimal4, VariantDecimal8, VariantDecimal16, @@ -5037,4 +5039,274 @@ mod test { .with_precision_and_scale(20, 3) .unwrap() ); + + fn union_get_options(fields: &UnionFields, mode: UnionMode) -> GetOptions<'static> { + let field = Field::new("union", DataType::Union(fields.clone(), mode), true); + GetOptions::new().with_as_type(Some(FieldRef::from(field))) + } + + fn int_str_bool_union_fields() -> UnionFields { + UnionFields::try_new( + vec![0, 1, 2], + vec![ + Field::new("int", DataType::Int64, true), + Field::new("str", DataType::Utf8, true), + Field::new("bool", DataType::Boolean, true), + ], + ) + .unwrap() + } + + /// int8, string, bool, array-level null, `Variant::Null`, double (no matching field), int64 + fn mixed_variant_array() -> ArrayRef { + let mut builder = VariantArrayBuilder::new(7); + builder.append_variant(Variant::Int8(1)); + builder.append_variant(Variant::from("hello")); + builder.append_variant(Variant::from(true)); + builder.append_null(); + builder.append_variant(Variant::Null); + builder.append_variant(Variant::Double(2.5)); + builder.append_variant(Variant::Int64(5_000_000_000)); + ArrayRef::from(builder.build()) + } + + #[test] + fn get_variant_as_dense_union() { + let fields = int_str_bool_union_fields(); + let array = mixed_variant_array(); + let result = variant_get(&array, union_get_options(&fields, UnionMode::Dense)).unwrap(); + + // nulls, `Variant::Null`, and the unmatched Double all land as nulls in the first child + let expected: ArrayRef = Arc::new( + UnionArray::try_new( + fields, + ScalarBuffer::from(vec![0i8, 1, 2, 0, 0, 0, 0]), + Some(ScalarBuffer::from(vec![0i32, 0, 0, 1, 2, 3, 4])), + vec![ + Arc::new(Int64Array::from(vec![ + Some(1), + None, + None, + None, + Some(5_000_000_000), + ])), + Arc::new(StringArray::from(vec!["hello"])), + Arc::new(BooleanArray::from(vec![true])), + ], + ) + .unwrap(), + ); + assert_eq!(&result, &expected); + } + + #[test] + fn get_variant_as_sparse_union() { + let fields = int_str_bool_union_fields(); + let array = mixed_variant_array(); + let result = variant_get(&array, union_get_options(&fields, UnionMode::Sparse)).unwrap(); + + let expected: ArrayRef = Arc::new( + UnionArray::try_new( + fields, + ScalarBuffer::from(vec![0i8, 1, 2, 0, 0, 0, 0]), + None, + vec![ + Arc::new(Int64Array::from(vec![ + Some(1), + None, + None, + None, + None, + None, + Some(5_000_000_000), + ])), + Arc::new(StringArray::from(vec![ + None, + Some("hello"), + None, + None, + None, + None, + None, + ])), + Arc::new(BooleanArray::from(vec![ + None, + None, + Some(true), + None, + None, + None, + None, + ])), + ], + ) + .unwrap(), + ); + assert_eq!(&result, &expected); + } + + #[test] + fn get_variant_as_union_prefers_most_exact_field() { + // Int8 picks the later-declared Int32 over Int64: exactness wins over declaration order + let fields = UnionFields::try_new( + vec![0, 1], + vec![ + Field::new("big", DataType::Int64, true), + Field::new("small", DataType::Int32, true), + ], + ) + .unwrap(); + let mut builder = VariantArrayBuilder::new(3); + builder.append_variant(Variant::Int8(1)); + builder.append_variant(Variant::Int32(2)); + builder.append_variant(Variant::Int64(3)); + let array = ArrayRef::from(builder.build()); + + let result = variant_get(&array, union_get_options(&fields, UnionMode::Dense)).unwrap(); + + let expected: ArrayRef = Arc::new( + UnionArray::try_new( + fields, + ScalarBuffer::from(vec![1i8, 1, 0]), + Some(ScalarBuffer::from(vec![0i32, 1, 0])), + vec![ + Arc::new(Int64Array::from(vec![3])), + Arc::new(Int32Array::from(vec![1, 2])), + ], + ) + .unwrap(), + ); + assert_eq!(&result, &expected); + } + + #[test] + fn get_variant_as_union_with_null_field() { + // nulls and unmatched values land in the Null-typed field instead of the first one + let fields = UnionFields::try_new( + vec![0, 1], + vec![ + Field::new("null", DataType::Null, true), + Field::new("int", DataType::Int64, true), + ], + ) + .unwrap(); + let mut builder = VariantArrayBuilder::new(4); + builder.append_variant(Variant::Int8(1)); + builder.append_null(); + builder.append_variant(Variant::Null); + builder.append_variant(Variant::from("no matching field")); + let array = ArrayRef::from(builder.build()); + + let result = variant_get(&array, union_get_options(&fields, UnionMode::Dense)).unwrap(); + + let expected: ArrayRef = Arc::new( + UnionArray::try_new( + fields, + ScalarBuffer::from(vec![1i8, 0, 0, 0]), + Some(ScalarBuffer::from(vec![0i32, 0, 1, 2])), + vec![ + Arc::new(NullArray::new(3)), + Arc::new(Int64Array::from(vec![1])), + ], + ) + .unwrap(), + ); + assert_eq!(&result, &expected); + } + + #[test] + fn get_variant_as_union_of_nested_types() { + let fields = UnionFields::try_new( + vec![0, 1, 2], + vec![ + Field::new( + "struct", + DataType::Struct(Fields::from(vec![Field::new("a", DataType::Int64, true)])), + true, + ), + Field::new( + "list", + DataType::List(Arc::new(Field::new("item", DataType::Int64, true))), + true, + ), + Field::new("str", DataType::Utf8, true), + ], + ) + .unwrap(); + let json = StringArray::from(vec![r#"{"a": 1}"#, "[1, 2, 3]", "\"s\""]); + let array = ArrayRef::from(json_to_variant(&(Arc::new(json) as ArrayRef)).unwrap()); + + let result = variant_get(&array, union_get_options(&fields, UnionMode::Dense)).unwrap(); + + let mut list_builder = ListBuilder::new(Int64Builder::new()); + list_builder.append_value([Some(1), Some(2), Some(3)]); + let expected: ArrayRef = Arc::new( + UnionArray::try_new( + fields, + ScalarBuffer::from(vec![0i8, 1, 2]), + Some(ScalarBuffer::from(vec![0i32, 0, 0])), + vec![ + Arc::new(StructArray::from(vec![( + Arc::new(Field::new("a", DataType::Int64, true)), + Arc::new(Int64Array::from(vec![1])) as ArrayRef, + )])), + Arc::new(list_builder.finish()), + Arc::new(StringArray::from(vec!["s"])), + ], + ) + .unwrap(), + ); + assert_eq!(&result, &expected); + } + + #[test] + fn get_variant_as_union_no_matching_field() { + let fields = + UnionFields::try_new(vec![0], vec![Field::new("str", DataType::Utf8, true)]).unwrap(); + let mut builder = VariantArrayBuilder::new(2); + builder.append_variant(Variant::from("kept")); + builder.append_variant(Variant::Int8(1)); + let array = ArrayRef::from(builder.build()); + + // Safe mode: the Int8 row becomes a null in the first (only) child. + let result = variant_get(&array, union_get_options(&fields, UnionMode::Dense)).unwrap(); + let expected: ArrayRef = Arc::new( + UnionArray::try_new( + fields.clone(), + ScalarBuffer::from(vec![0i8, 0]), + Some(ScalarBuffer::from(vec![0i32, 1])), + vec![Arc::new(StringArray::from(vec![Some("kept"), None]))], + ) + .unwrap(), + ); + assert_eq!(&result, &expected); + + // Strict mode: the same row is a cast error. + let options = union_get_options(&fields, UnionMode::Dense).with_cast_options(CastOptions { + safe: false, + ..Default::default() + }); + let err = variant_get(&array, options).unwrap_err(); + assert!( + err.to_string().contains("no field can represent it"), + "unexpected error: {err}" + ); + } + + #[test] + fn get_variant_as_union_empty_fields_errors() { + let mut builder = VariantArrayBuilder::new(1); + builder.append_variant(Variant::Int8(1)); + let array = ArrayRef::from(builder.build()); + + let err = variant_get( + &array, + union_get_options(&UnionFields::empty(), UnionMode::Dense), + ) + .unwrap_err(); + assert!( + err.to_string().contains("at least one union field"), + "unexpected error: {err}" + ); + } } diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs index 9841da555da0..12b6e51fc780 100644 --- a/parquet-variant-compute/src/variant_to_arrow.rs +++ b/parquet-variant-compute/src/variant_to_arrow.rs @@ -30,13 +30,13 @@ use arrow::array::{ BooleanBuilder, FixedSizeBinaryBuilder, FixedSizeListArray, GenericListArray, GenericListViewArray, LargeBinaryBuilder, LargeStringBuilder, NullArray, NullBufferBuilder, OffsetSizeTrait, PrimitiveBuilder, StringBuilder, StringLikeArrayBuilder, StringViewBuilder, - StructArray, + StructArray, UnionArray, }; use arrow::buffer::{OffsetBuffer, ScalarBuffer}; use arrow::compute::{CastOptions, DecimalCast, cast_with_options}; use arrow::datatypes::{self, DataType, DecimalType}; use arrow::error::{ArrowError, Result}; -use arrow_schema::{FieldRef, Fields, TimeUnit}; +use arrow_schema::{FieldRef, Fields, TimeUnit, UnionFields, UnionMode}; use parquet_variant::{Variant, VariantPath}; use std::sync::Arc; @@ -48,6 +48,7 @@ pub(crate) enum VariantToArrowRowBuilder<'a> { Primitive(PrimitiveVariantToArrowRowBuilder<'a>), Array(ArrayVariantToArrowRowBuilder<'a>), Struct(StructVariantToArrowRowBuilder<'a>), + Union(UnionVariantToArrowRowBuilder<'a>), Encoded(EncodedVariantToArrowRowBuilder<'a>), BinaryVariant(VariantToBinaryVariantArrowRowBuilder), @@ -62,6 +63,7 @@ impl<'a> VariantToArrowRowBuilder<'a> { Primitive(b) => b.append_null(), Array(b) => b.append_null(), Struct(b) => b.append_null(), + Union(b) => b.append_null(), Encoded(b) => b.append_null(), BinaryVariant(b) => b.append_null(), WithPath(path_builder) => path_builder.append_null(), @@ -74,6 +76,7 @@ impl<'a> VariantToArrowRowBuilder<'a> { Primitive(b) => b.append_value(&value), Array(b) => b.append_value(&value), Struct(b) => b.append_value(&value), + Union(b) => b.append_value(&value), Encoded(b) => b.append_value(value), BinaryVariant(b) => b.append_value(value), WithPath(path_builder) => path_builder.append_value(value), @@ -86,6 +89,7 @@ impl<'a> VariantToArrowRowBuilder<'a> { Primitive(b) => b.finish(), Array(b) => b.finish(), Struct(b) => b.finish(), + Union(b) => b.finish(), Encoded(b) => b.finish(), BinaryVariant(b) => b.finish(), WithPath(path_builder) => path_builder.finish(), @@ -114,6 +118,15 @@ fn make_typed_variant_to_arrow_row_builder<'a>( ArrayVariantToArrowRowBuilder::try_new(data_type, cast_options, capacity, false)?; Ok(Array(builder)) } + DataType::Union(union_fields, mode) => { + let builder = UnionVariantToArrowRowBuilder::try_new( + union_fields, + *mode, + cast_options, + capacity, + )?; + Ok(Union(builder)) + } DataType::Dictionary(_, value_type) => { let builder = EncodedVariantToArrowRowBuilder::try_new( data_type, @@ -648,6 +661,217 @@ impl<'a> StructVariantToArrowRowBuilder<'a> { } } +/// Builder for converting variant values into a [`UnionArray`]. +/// +/// Each value is dispatched to the union field that most exactly represents its runtime type +/// (see [`union_child_rank`]), with ties broken by declaration order. Unions have no top-level +/// null buffer, so null rows -- and, in safe mode, values no field can represent -- become a +/// null in the [`DataType::Null`] child if the union declares one, otherwise in the first child. +pub(crate) struct UnionVariantToArrowRowBuilder<'a> { + fields: &'a UnionFields, + mode: UnionMode, + children: Vec>, + type_ids: Vec, + /// Dense mode only + offsets: Vec, + null_child: usize, + cast_options: &'a CastOptions<'a>, +} + +struct UnionChildBuilder<'a> { + type_id: i8, + builder: VariantToArrowRowBuilder<'a>, + len: i32, +} + +impl<'a> UnionVariantToArrowRowBuilder<'a> { + fn try_new( + fields: &'a UnionFields, + mode: UnionMode, + cast_options: &'a CastOptions<'a>, + capacity: usize, + ) -> Result { + // null rows need a child to land in + if fields.is_empty() { + return Err(ArrowError::InvalidArgumentError( + "Casting Variant to a union requires at least one union field".to_string(), + )); + } + let mut children = Vec::with_capacity(fields.len()); + for (type_id, field) in fields.iter() { + children.push(UnionChildBuilder { + type_id, + builder: make_typed_variant_to_arrow_row_builder( + field.data_type(), + cast_options, + capacity, + )?, + len: 0, + }); + } + let null_child = fields + .iter() + .position(|(_, field)| field.data_type() == &DataType::Null) + .unwrap_or(0); + let offsets = match mode { + UnionMode::Dense => Vec::with_capacity(capacity), + UnionMode::Sparse => Vec::new(), + }; + Ok(Self { + fields, + mode, + children, + type_ids: Vec::with_capacity(capacity), + offsets, + null_child, + cast_options, + }) + } + + fn append_null(&mut self) -> Result<()> { + self.append_to_child(self.null_child, None)?; + Ok(()) + } + + fn append_value(&mut self, value: &Variant<'_, '_>) -> Result { + // `Variant::Null` becomes null even in strict mode, like in the other builders + if matches!(value, Variant::Null) { + self.append_null()?; + return Ok(false); + } + match self.select_child(value) { + Some(index) => self.append_to_child(index, Some(value)), + None if self.cast_options.safe => { + self.append_null()?; + Ok(false) + } + None => Err(ArrowError::CastError(format!( + "Failed to cast variant {value:?} to union: no field can represent it" + ))), + } + } + + fn select_child(&self, value: &Variant<'_, '_>) -> Option { + let mut best: Option<(u8, usize)> = None; + for (index, (_, field)) in self.fields.iter().enumerate() { + let Some(rank) = union_child_rank(value, field.data_type()) else { + continue; + }; + if best.is_none_or(|(best_rank, _)| rank < best_rank) { + best = Some((rank, index)); + } + } + best.map(|(_, index)| index) + } + + fn append_to_child(&mut self, index: usize, value: Option<&Variant<'_, '_>>) -> Result { + self.type_ids.push(self.children[index].type_id); + match self.mode { + UnionMode::Dense => { + let child = &mut self.children[index]; + self.offsets.push(child.len); + child.len = child.len.add_checked(1)?; + match value { + Some(value) => child.builder.append_value(value.clone()), + None => { + child.builder.append_null()?; + Ok(false) + } + } + } + UnionMode::Sparse => { + let mut appended = false; + for (child_index, child) in self.children.iter_mut().enumerate() { + match value { + Some(value) if child_index == index => { + appended = child.builder.append_value(value.clone())?; + } + _ => child.builder.append_null()?, + } + } + Ok(appended) + } + } + } + + fn finish(self) -> Result { + let mut type_ids = Vec::with_capacity(self.children.len()); + let mut fields = Vec::with_capacity(self.children.len()); + let mut arrays = Vec::with_capacity(self.children.len()); + for (child, (_, field)) in self.children.into_iter().zip(self.fields.iter()) { + let array = child.builder.finish()?; + type_ids.push(child.type_id); + fields.push( + field + .as_ref() + .clone() + .with_data_type(array.data_type().clone()), + ); + arrays.push(array); + } + let fields = UnionFields::try_new(type_ids, fields)?; + let offsets = (self.mode == UnionMode::Dense).then(|| ScalarBuffer::from(self.offsets)); + let array = + UnionArray::try_new(fields, ScalarBuffer::from(self.type_ids), offsets, arrays)?; + Ok(Arc::new(array)) + } +} + +/// Ranks how exactly a union child of type `data_type` can represent a variant value's runtime +/// type: 0 is the value's natural Arrow type, higher ranks are lossless widenings, and `None` +/// means the child cannot represent the value losslessly. Every pair admitted here must be +/// convertible by the corresponding row builder. +fn union_child_rank(value: &Variant<'_, '_>, data_type: &DataType) -> Option { + use DataType::*; + let rank = match (value, data_type) { + (Variant::BooleanTrue | Variant::BooleanFalse, Boolean) => 0, + (Variant::Int8(_), Int8) => 0, + (Variant::Int8(_), Int16) => 1, + (Variant::Int8(_), Int32) => 2, + (Variant::Int8(_), Int64) => 3, + (Variant::Int16(_), Int16) => 0, + (Variant::Int16(_), Int32) => 1, + (Variant::Int16(_), Int64) => 2, + (Variant::Int32(_), Int32) => 0, + (Variant::Int32(_), Int64) => 1, + (Variant::Int64(_), Int64) => 0, + (Variant::Float(_), Float32) => 0, + (Variant::Float(_), Float64) => 1, + (Variant::Double(_), Float64) => 0, + (Variant::Decimal4(_), Decimal32(..)) => 0, + (Variant::Decimal4(_), Decimal64(..)) => 1, + (Variant::Decimal4(_), Decimal128(..)) => 2, + (Variant::Decimal4(_), Decimal256(..)) => 3, + (Variant::Decimal8(_), Decimal64(..)) => 0, + (Variant::Decimal8(_), Decimal128(..)) => 1, + (Variant::Decimal8(_), Decimal256(..)) => 2, + (Variant::Decimal16(_), Decimal128(..)) => 0, + (Variant::Decimal16(_), Decimal256(..)) => 1, + (Variant::Date(_), Date32) => 0, + (Variant::Date(_), Date64) => 1, + (Variant::TimestampMicros(_), Timestamp(TimeUnit::Microsecond, Some(_))) => 0, + (Variant::TimestampMicros(_), Timestamp(TimeUnit::Nanosecond, Some(_))) => 1, + (Variant::TimestampNanos(_), Timestamp(TimeUnit::Nanosecond, Some(_))) => 0, + (Variant::TimestampNtzMicros(_), Timestamp(TimeUnit::Microsecond, None)) => 0, + (Variant::TimestampNtzMicros(_), Timestamp(TimeUnit::Nanosecond, None)) => 1, + (Variant::TimestampNtzNanos(_), Timestamp(TimeUnit::Nanosecond, None)) => 0, + (Variant::Time(_), Time64(TimeUnit::Microsecond)) => 0, + (Variant::Time(_), Time64(TimeUnit::Nanosecond)) => 1, + (Variant::String(_) | Variant::ShortString(_), Utf8 | LargeUtf8 | Utf8View) => 0, + (Variant::Binary(_), Binary | LargeBinary | BinaryView) => 0, + (Variant::Uuid(_), FixedSizeBinary(16)) => 0, + (Variant::Object(_), Struct(_)) => 0, + // unreachable until casting Variant to Map lands (#10012) + (Variant::Object(_), Map(..)) => 1, + (Variant::List(_), List(_)) => 0, + (Variant::List(_), LargeList(_)) => 1, + (Variant::List(_), ListView(_)) => 2, + (Variant::List(_), LargeListView(_)) => 3, + _ => return None, + }; + Some(rank) +} + impl<'a> ArrayVariantToArrowRowBuilder<'a> { /// Creates a new list builder for the given data type. /// From 97ddd9331506da15c5050e714fba9e4421610496 Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Mon, 20 Jul 2026 18:12:32 -0400 Subject: [PATCH 2/7] add map support tests --- parquet-variant-compute/src/variant_get.rs | 62 +++++++++++++++++++ .../src/variant_to_arrow.rs | 1 - 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index ced251d55f28..cc6ce2d2947f 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -5418,6 +5418,68 @@ mod test { assert_eq!(&result, &expected); } + #[test] + fn get_variant_as_union_with_map_field() { + // With no Struct field in the union, an object routes to the Map child. + let fields = UnionFields::try_new( + vec![0, 1], + vec![ + Field::new("map", map_data_type(DataType::Int64), true), + Field::new("str", DataType::Utf8, true), + ], + ) + .unwrap(); + let json = StringArray::from(vec![r#"{"a": 1, "b": 2}"#, "\"hi\""]); + let array = ArrayRef::from(json_to_variant(&(Arc::new(json) as ArrayRef)).unwrap()); + + let result = variant_get(&array, union_get_options(&fields, UnionMode::Dense)).unwrap(); + + let mut map_builder = MapBuilder::new(None, StringBuilder::new(), Int64Builder::new()); + map_builder.keys().append_value("a"); + map_builder.values().append_value(1); + map_builder.keys().append_value("b"); + map_builder.values().append_value(2); + map_builder.append(true).unwrap(); + let expected: ArrayRef = Arc::new( + UnionArray::try_new( + fields, + ScalarBuffer::from(vec![0i8, 1]), + Some(ScalarBuffer::from(vec![0i32, 0])), + vec![ + Arc::new(map_builder.finish()), + Arc::new(StringArray::from(vec!["hi"])), + ], + ) + .unwrap(), + ); + assert_eq!(&result, &expected); + } + + #[test] + fn get_variant_as_union_prefers_struct_over_map() { + // Both a Struct and a Map field can hold an object; the object routes to Struct because + // it represents the object more exactly (rank 0 vs 1). + let fields = UnionFields::try_new( + vec![0, 1], + vec![ + Field::new("map", map_data_type(DataType::Int64), true), + Field::new( + "struct", + DataType::Struct(Fields::from(vec![Field::new("a", DataType::Int64, true)])), + true, + ), + ], + ) + .unwrap(); + let json = StringArray::from(vec![r#"{"a": 1}"#]); + let array = ArrayRef::from(json_to_variant(&(Arc::new(json) as ArrayRef)).unwrap()); + + let result = variant_get(&array, union_get_options(&fields, UnionMode::Dense)).unwrap(); + let union = result.as_any().downcast_ref::().unwrap(); + // type_id 1 == the struct child + assert_eq!(union.type_ids(), &[1i8]); + } + #[test] fn get_variant_as_union_no_matching_field() { let fields = diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs index 02241283af2d..624f3c41bccc 100644 --- a/parquet-variant-compute/src/variant_to_arrow.rs +++ b/parquet-variant-compute/src/variant_to_arrow.rs @@ -874,7 +874,6 @@ fn union_child_rank(value: &Variant<'_, '_>, data_type: &DataType) -> Option (Variant::Binary(_), Binary | LargeBinary | BinaryView) => 0, (Variant::Uuid(_), FixedSizeBinary(16)) => 0, (Variant::Object(_), Struct(_)) => 0, - // unreachable until casting Variant to Map lands (#10012) (Variant::Object(_), Map(..)) => 1, (Variant::List(_), List(_)) => 0, (Variant::List(_), LargeList(_)) => 1, From 4bac07b29b6e9aa9968623ac3436430bedf335cc Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Wed, 5 Aug 2026 17:16:35 -0400 Subject: [PATCH 3/7] address @Jefffrey's review --- parquet-variant-compute/src/variant_get.rs | 43 ++++++++++++++++--- .../src/variant_to_arrow.rs | 2 + 2 files changed, 38 insertions(+), 7 deletions(-) diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index 4931b898b281..37136881c639 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -3534,8 +3534,8 @@ mod test { } #[test] - fn test_unshredded_struct_safe_cast_non_object_rows_are_null() { - let json_strings = vec![r#"{"a": 1, "b": 2}"#, "123", "{}"]; + fn test_unshredded_struct_safe_cast_and_field_mismatches() { + let json_strings = vec![r#"{"a": 1, "b": 2, "extra": 3}"#, "123", "{}"]; let string_array: Arc = Arc::new(StringArray::from(json_strings)); let variant_array_ref = ArrayRef::from(json_to_variant(&string_array).unwrap()); @@ -3562,7 +3562,8 @@ mod test { .column(1) .as_primitive::(); - // Row 0 is an object, so the struct row is valid with extracted fields. + // Row 0 is an object, so the struct row is valid with extracted fields. Object fields + // that aren't present in the requested struct are ignored. assert!(!struct_result.is_null(0)); assert_eq!(field_a.value(0), 1); assert_eq!(field_b.value(0), 2); @@ -3578,6 +3579,33 @@ mod test { assert!(field_b.is_null(2)); } + #[test] + fn test_unshredded_struct_missing_non_nullable_field_errors() { + let string_array: Arc = Arc::new(StringArray::from(vec![r#"{"a": 1}"#])); + let variant_array_ref = ArrayRef::from(json_to_variant(&string_array).unwrap()); + + let struct_fields = Fields::from(vec![ + Field::new("a", DataType::Int32, false), + Field::new("missing", DataType::Int32, false), + ]); + let options = GetOptions { + path: VariantPath::default(), + as_type: Some(Arc::new(Field::new( + "result", + DataType::Struct(struct_fields), + true, + ))), + cast_options: CastOptions::default(), + }; + + let err = variant_get(&variant_array_ref, options).unwrap_err(); + assert!( + err.to_string() + .contains("unmasked nulls for non-nullable StructArray field \"missing\""), + "unexpected error: {err}" + ); + } + #[test] fn test_unshredded_struct_strict_cast_non_object_errors() { let json_strings = vec![r#"{"a": 1, "b": 2}"#, "123"]; @@ -5320,8 +5348,8 @@ mod test { let fields = UnionFields::try_new( vec![0, 1], vec![ - Field::new("null", DataType::Null, true), Field::new("int", DataType::Int64, true), + Field::new("null", DataType::Null, true), ], ) .unwrap(); @@ -5337,11 +5365,11 @@ mod test { let expected: ArrayRef = Arc::new( UnionArray::try_new( fields, - ScalarBuffer::from(vec![1i8, 0, 0, 0]), + ScalarBuffer::from(vec![0i8, 1, 1, 1]), Some(ScalarBuffer::from(vec![0i32, 0, 1, 2])), vec![ - Arc::new(NullArray::new(3)), Arc::new(Int64Array::from(vec![1])), + Arc::new(NullArray::new(3)), ], ) .unwrap(), @@ -5458,8 +5486,9 @@ mod test { #[test] fn get_variant_as_union_no_matching_field() { + // Like other requested fields, union child nullability does not override safe casting. let fields = - UnionFields::try_new(vec![0], vec![Field::new("str", DataType::Utf8, true)]).unwrap(); + UnionFields::try_new(vec![0], vec![Field::new("str", DataType::Utf8, false)]).unwrap(); let mut builder = VariantArrayBuilder::new(2); builder.append_variant(Variant::from("kept")); builder.append_variant(Variant::Int8(1)); diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs index e692f67f6534..79263bee2af3 100644 --- a/parquet-variant-compute/src/variant_to_arrow.rs +++ b/parquet-variant-compute/src/variant_to_arrow.rs @@ -712,6 +712,8 @@ impl<'a> UnionVariantToArrowRowBuilder<'a> { } let mut children = Vec::with_capacity(fields.len()); for (type_id, field) in fields.iter() { + // Match the other typed builders: nullability is schema metadata and does not + // override safe-cast behavior, which may append null for an unrepresentable value. children.push(UnionChildBuilder { type_id, builder: make_typed_variant_to_arrow_row_builder( From b708d3f9f0b48a568fbca662d0a572a36223432e Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Thu, 6 Aug 2026 11:00:14 -0400 Subject: [PATCH 4/7] ensure variant fits into the union decimal field --- parquet-variant-compute/src/variant_get.rs | 32 +++++++++++++++ .../src/variant_to_arrow.rs | 40 ++++++++++++++----- 2 files changed, 63 insertions(+), 9 deletions(-) diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index 37136881c639..31621260c733 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -5342,6 +5342,38 @@ mod test { assert_eq!(&result, &expected); } + #[test] + fn get_variant_as_union_skips_decimal_that_cannot_fit() { + let fields = UnionFields::try_new( + vec![0, 1], + vec![ + Field::new("too_narrow", DataType::Decimal32(3, 2), true), + Field::new("fits", DataType::Decimal32(5, 2), true), + ], + ) + .unwrap(); + let mut builder = VariantArrayBuilder::new(1); + builder.append_variant(VariantDecimal4::try_new(12_345, 2).unwrap().into()); + let array = ArrayRef::from(builder.build()); + + for safe in [true, false] { + let options = + union_get_options(&fields, UnionMode::Dense).with_cast_options(CastOptions { + safe, + ..Default::default() + }); + let result = variant_get(&array, options).unwrap(); + let union = result.as_any().downcast_ref::().unwrap(); + assert_eq!(union.type_ids(), &[1i8]); + let decimal = union + .child(1) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!(decimal.value(0), 12_345); + } + } + #[test] fn get_variant_as_union_with_null_field() { // nulls and unmatched values land in the Null-typed field instead of the first one diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs index 79263bee2af3..d161d7234c84 100644 --- a/parquet-variant-compute/src/variant_to_arrow.rs +++ b/parquet-variant-compute/src/variant_to_arrow.rs @@ -853,15 +853,15 @@ fn union_child_rank(value: &Variant<'_, '_>, data_type: &DataType) -> Option (Variant::Float(_), Float32) => 0, (Variant::Float(_), Float64) => 1, (Variant::Double(_), Float64) => 0, - (Variant::Decimal4(_), Decimal32(..)) => 0, - (Variant::Decimal4(_), Decimal64(..)) => 1, - (Variant::Decimal4(_), Decimal128(..)) => 2, - (Variant::Decimal4(_), Decimal256(..)) => 3, - (Variant::Decimal8(_), Decimal64(..)) => 0, - (Variant::Decimal8(_), Decimal128(..)) => 1, - (Variant::Decimal8(_), Decimal256(..)) => 2, - (Variant::Decimal16(_), Decimal128(..)) => 0, - (Variant::Decimal16(_), Decimal256(..)) => 1, + (Variant::Decimal4(_), Decimal32(..)) if variant_fits_decimal(value, data_type) => 0, + (Variant::Decimal4(_), Decimal64(..)) if variant_fits_decimal(value, data_type) => 1, + (Variant::Decimal4(_), Decimal128(..)) if variant_fits_decimal(value, data_type) => 2, + (Variant::Decimal4(_), Decimal256(..)) if variant_fits_decimal(value, data_type) => 3, + (Variant::Decimal8(_), Decimal64(..)) if variant_fits_decimal(value, data_type) => 0, + (Variant::Decimal8(_), Decimal128(..)) if variant_fits_decimal(value, data_type) => 1, + (Variant::Decimal8(_), Decimal256(..)) if variant_fits_decimal(value, data_type) => 2, + (Variant::Decimal16(_), Decimal128(..)) if variant_fits_decimal(value, data_type) => 0, + (Variant::Decimal16(_), Decimal256(..)) if variant_fits_decimal(value, data_type) => 1, (Variant::Date(_), Date32) => 0, (Variant::Date(_), Date64) => 1, (Variant::TimestampMicros(_), Timestamp(TimeUnit::Microsecond, Some(_))) => 0, @@ -886,6 +886,28 @@ fn union_child_rank(value: &Variant<'_, '_>, data_type: &DataType) -> Option Some(rank) } +fn variant_fits_decimal(value: &Variant<'_, '_>, data_type: &DataType) -> bool { + match data_type { + DataType::Decimal32(precision, scale) => { + variant_to_unscaled_decimal::(value, *precision, *scale) + .is_some() + } + DataType::Decimal64(precision, scale) => { + variant_to_unscaled_decimal::(value, *precision, *scale) + .is_some() + } + DataType::Decimal128(precision, scale) => { + variant_to_unscaled_decimal::(value, *precision, *scale) + .is_some() + } + DataType::Decimal256(precision, scale) => { + variant_to_unscaled_decimal::(value, *precision, *scale) + .is_some() + } + _ => false, + } +} + /// Builder for converting variant objects into Arrow `MapArray`s. /// /// Each variant object field becomes one map entry: the field name is the map key and the field From 4e05c2bc0df5245c1dc9829a0f230cea25c5d0b0 Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Thu, 6 Aug 2026 11:00:39 -0400 Subject: [PATCH 5/7] add encoded fields to the union child ranking --- parquet-variant-compute/src/variant_get.rs | 37 +++++++++++++++++++ .../src/variant_to_arrow.rs | 4 ++ 2 files changed, 41 insertions(+) diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index 31621260c733..177f0c3efa00 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -5342,6 +5342,43 @@ mod test { assert_eq!(&result, &expected); } + #[test] + fn get_variant_as_union_with_encoded_children() { + let encoded_types = [ + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + DataType::RunEndEncoded( + Arc::new(Field::new("run_ends", DataType::Int32, false)), + Arc::new(Field::new("values", DataType::Utf8, true)), + ), + ]; + + for data_type in encoded_types { + let fields = UnionFields::try_new( + vec![0], + vec![Field::new("encoded", data_type.clone(), true)], + ) + .unwrap(); + let mut builder = VariantArrayBuilder::new(2); + builder.append_variant(Variant::from("apple")); + builder.append_variant(Variant::from("banana")); + let array = ArrayRef::from(builder.build()); + let options = + union_get_options(&fields, UnionMode::Dense).with_cast_options(CastOptions { + safe: false, + ..Default::default() + }); + + let result = variant_get(&array, options).unwrap(); + let union = result.as_any().downcast_ref::().unwrap(); + assert_eq!(union.type_ids(), &[0i8, 0]); + assert_eq!(union.child(0).data_type(), &data_type); + + let decoded = cast(union.child(0).as_ref(), &DataType::Utf8).unwrap(); + let expected = StringArray::from(vec!["apple", "banana"]); + assert_eq!(decoded.as_ref(), &expected); + } + } + #[test] fn get_variant_as_union_skips_decimal_that_cannot_fit() { let fields = UnionFields::try_new( diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs index d161d7234c84..8ed22b9dac4e 100644 --- a/parquet-variant-compute/src/variant_to_arrow.rs +++ b/parquet-variant-compute/src/variant_to_arrow.rs @@ -839,6 +839,10 @@ impl<'a> UnionVariantToArrowRowBuilder<'a> { fn union_child_rank(value: &Variant<'_, '_>, data_type: &DataType) -> Option { use DataType::*; let rank = match (value, data_type) { + (_, Dictionary(_, value_type)) => return union_child_rank(value, value_type), + (_, RunEndEncoded(_, value_field)) => { + return union_child_rank(value, value_field.data_type()); + } (Variant::BooleanTrue | Variant::BooleanFalse, Boolean) => 0, (Variant::Int8(_), Int8) => 0, (Variant::Int8(_), Int16) => 1, From c81c1e7c916d375e5ea9adcfb5319cc626621c7a Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Thu, 6 Aug 2026 16:06:06 -0400 Subject: [PATCH 6/7] Add fixed size list to the union child rank --- parquet-variant-compute/src/variant_get.rs | 34 +++++++++++++++++++ .../src/variant_to_arrow.rs | 1 + 2 files changed, 35 insertions(+) diff --git a/parquet-variant-compute/src/variant_get.rs b/parquet-variant-compute/src/variant_get.rs index 177f0c3efa00..110a3f7d0deb 100644 --- a/parquet-variant-compute/src/variant_get.rs +++ b/parquet-variant-compute/src/variant_get.rs @@ -5379,6 +5379,40 @@ mod test { } } + #[test] + fn get_variant_as_union_with_fixed_size_list_child() { + let item = Arc::new(Field::new("item", DataType::Int64, true)); + let fields = UnionFields::try_new( + vec![0], + vec![Field::new("fixed", DataType::FixedSizeList(item, 2), true)], + ) + .unwrap(); + let json = StringArray::from(vec!["[1, 2]"]); + let array = ArrayRef::from(json_to_variant(&(Arc::new(json) as ArrayRef)).unwrap()); + + for safe in [true, false] { + let options = + union_get_options(&fields, UnionMode::Dense).with_cast_options(CastOptions { + safe, + ..Default::default() + }); + let result = variant_get(&array, options).unwrap(); + let union = result.as_any().downcast_ref::().unwrap(); + assert_eq!(union.type_ids(), &[0i8]); + let list = union + .child(0) + .as_any() + .downcast_ref::() + .unwrap(); + assert_eq!( + list.value(0) + .as_primitive::() + .values(), + &[1, 2] + ); + } + } + #[test] fn get_variant_as_union_skips_decimal_that_cannot_fit() { let fields = UnionFields::try_new( diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs index 8ed22b9dac4e..463cb380d560 100644 --- a/parquet-variant-compute/src/variant_to_arrow.rs +++ b/parquet-variant-compute/src/variant_to_arrow.rs @@ -882,6 +882,7 @@ fn union_child_rank(value: &Variant<'_, '_>, data_type: &DataType) -> Option (Variant::Object(_), Struct(_)) => 0, (Variant::Object(_), Map(..)) => 1, (Variant::List(_), List(_)) => 0, + (Variant::List(list), FixedSizeList(_, size)) if list.len() == *size as usize => 0, (Variant::List(_), LargeList(_)) => 1, (Variant::List(_), ListView(_)) => 2, (Variant::List(_), LargeListView(_)) => 3, From 38ade1022c4a4423c77449a3993f83bdf54b0517 Mon Sep 17 00:00:00 2001 From: sdf-jkl Date: Thu, 13 Aug 2026 11:16:41 -0400 Subject: [PATCH 7/7] merge fix --- parquet-variant-compute/src/variant_to_arrow.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/parquet-variant-compute/src/variant_to_arrow.rs b/parquet-variant-compute/src/variant_to_arrow.rs index c50668e0e7d6..eaaab1336554 100644 --- a/parquet-variant-compute/src/variant_to_arrow.rs +++ b/parquet-variant-compute/src/variant_to_arrow.rs @@ -760,6 +760,7 @@ impl<'a> UnionVariantToArrowRowBuilder<'a> { field.data_type(), cast_options, capacity, + false, )?, len: 0, });