use std::fmt::{Debug, Display};
use std::ptr;
use std::sync::Arc;
mod accessor;
use arrow_buffer::{ArrowNativeType, BooleanBufferBuilder, Buffer as ArrowBuffer};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use vortex_buffer::Buffer;
use vortex_dtype::{match_each_native_ptype, DType, NativePType, PType};
use vortex_error::{VortexExpect as _, VortexResult};
use crate::array::BoolArray;
use crate::encoding::ids;
use crate::iter::Accessor;
use crate::stats::StatsSet;
use crate::validity::{ArrayValidity, LogicalValidity, Validity, ValidityMetadata, ValidityVTable};
use crate::variants::{PrimitiveArrayTrait, VariantsVTable};
use crate::visitor::{ArrayVisitor, VisitorVTable};
use crate::{
impl_encoding, ArrayData, ArrayLen, ArrayTrait, Canonical, IntoArrayData, IntoCanonical,
};
mod compute;
mod patch;
mod stats;
impl_encoding!("vortex.primitive", ids::PRIMITIVE, Primitive);
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct PrimitiveMetadata {
validity: ValidityMetadata,
}
impl Display for PrimitiveMetadata {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
Debug::fmt(self, f)
}
}
impl PrimitiveArray {
pub fn new(buffer: Buffer, ptype: PType, validity: Validity) -> Self {
let length = match_each_native_ptype!(ptype, |$P| {
let (prefix, values, suffix) = unsafe { buffer.align_to::<$P>() };
assert!(
prefix.is_empty() && suffix.is_empty() && (buffer.as_ptr() as usize) % std::mem::size_of::<$P>() == 0,
"buffer is not aligned: {:?}",
buffer.as_ptr()
);
values.len()
});
ArrayData::try_new_owned(
&PrimitiveEncoding,
DType::from(ptype).with_nullability(validity.nullability()),
length,
Arc::new(PrimitiveMetadata {
validity: validity
.to_metadata(length)
.vortex_expect("Invalid validity"),
}),
Some(buffer),
validity.into_array().into_iter().collect_vec().into(),
StatsSet::default(),
)
.and_then(|data| data.try_into())
.vortex_expect("Should not fail to create PrimitiveArray")
}
pub fn from_vec<T: NativePType>(values: Vec<T>, validity: Validity) -> Self {
match_each_native_ptype!(T::PTYPE, |$P| {
PrimitiveArray::new(
Buffer::from(
ArrowBuffer::from(unsafe { std::mem::transmute::<Vec<T>, Vec<$P>>(values) })
),
T::PTYPE,
validity,
)
})
}
pub fn from_nullable_vec<T: NativePType>(values: Vec<Option<T>>) -> Self {
let elems: Vec<T> = values.iter().map(|v| v.unwrap_or_default()).collect();
let validity = Validity::from_iter(values.iter().map(|v| v.is_some()));
Self::from_vec(elems, validity)
}
pub fn validity(&self) -> Validity {
self.metadata().validity.to_validity(|| {
self.as_ref()
.child(0, &Validity::DTYPE, self.len())
.vortex_expect("PrimitiveArray: validity child")
})
}
pub fn buffer(&self) -> &Buffer {
self.as_ref()
.buffer()
.vortex_expect("Missing buffer in PrimitiveArray")
}
pub fn maybe_null_slice<T: NativePType>(&self) -> &[T] {
assert_eq!(
T::PTYPE,
self.ptype(),
"Attempted to get slice of type {} from array of type {}",
T::PTYPE,
self.ptype(),
);
let raw_slice = self.buffer().as_slice();
let typed_len = raw_slice.len() / size_of::<T>();
unsafe { std::slice::from_raw_parts(raw_slice.as_ptr().cast(), typed_len) }
}
pub fn into_maybe_null_slice<T: NativePType + ArrowNativeType>(self) -> Vec<T> {
assert_eq!(
T::PTYPE,
self.ptype(),
"Attempted to get maybe_null_slice of type {} from array of type {}",
T::PTYPE,
self.ptype(),
);
self.into_buffer().into_vec::<T>().unwrap_or_else(|b| {
let (prefix, values, suffix) = unsafe { b.as_ref().align_to::<T>() };
assert!(prefix.is_empty() && suffix.is_empty());
Vec::from(values)
})
}
pub fn get_as_cast<T: NativePType>(&self, idx: usize) -> T {
match_each_native_ptype!(self.ptype(), |$P| {
T::from(self.maybe_null_slice::<$P>()[idx]).expect("failed to cast")
})
}
pub fn reinterpret_cast(&self, ptype: PType) -> Self {
if self.ptype() == ptype {
return self.clone();
}
assert_eq!(
self.ptype().byte_width(),
ptype.byte_width(),
"can't reinterpret cast between integers of two different widths"
);
PrimitiveArray::new(self.buffer().clone(), ptype, self.validity())
}
pub fn into_buffer(self) -> Buffer {
self.into_array()
.into_buffer()
.vortex_expect("PrimitiveArray must have a buffer")
}
}
impl ArrayTrait for PrimitiveArray {}
impl VariantsVTable<PrimitiveArray> for PrimitiveEncoding {
fn as_primitive_array<'a>(
&self,
array: &'a PrimitiveArray,
) -> Option<&'a dyn PrimitiveArrayTrait> {
Some(array)
}
}
impl<T: NativePType> Accessor<T> for PrimitiveArray {
fn array_len(&self) -> usize {
self.len()
}
fn is_valid(&self, index: usize) -> bool {
ArrayValidity::is_valid(self, index)
}
#[inline]
fn value_unchecked(&self, index: usize) -> T {
self.maybe_null_slice::<T>()[index]
}
fn array_validity(&self) -> Validity {
self.validity()
}
#[inline]
fn decode_batch(&self, start_idx: usize) -> Vec<T> {
let batch_size = <Self as Accessor<T>>::batch_size(self, start_idx);
let mut v = Vec::<T>::with_capacity(batch_size);
let null_slice = self.maybe_null_slice::<T>();
unsafe {
v.set_len(batch_size);
ptr::copy_nonoverlapping(
null_slice.as_ptr().add(start_idx),
v.as_mut_ptr(),
batch_size,
);
}
v
}
}
impl PrimitiveArrayTrait for PrimitiveArray {}
impl<T: NativePType> FromIterator<Option<T>> for PrimitiveArray {
fn from_iter<I: IntoIterator<Item = Option<T>>>(iter: I) -> Self {
let iter = iter.into_iter();
let mut values = Vec::with_capacity(iter.size_hint().0);
let mut validity = BooleanBufferBuilder::new(values.capacity());
for i in iter {
match i {
None => {
validity.append(false);
values.push(T::default());
}
Some(e) => {
validity.append(true);
values.push(e);
}
}
}
Self::from_vec(
values,
Validity::Array(BoolArray::from(validity.finish()).into_array()),
)
}
}
impl<T: NativePType> From<Vec<T>> for PrimitiveArray {
fn from(values: Vec<T>) -> Self {
Self::from_vec(values, Validity::NonNullable)
}
}
impl<T: NativePType> IntoArrayData for Vec<T> {
fn into_array(self) -> ArrayData {
PrimitiveArray::from(self).into_array()
}
}
impl IntoCanonical for PrimitiveArray {
fn into_canonical(self) -> VortexResult<Canonical> {
Ok(Canonical::Primitive(self))
}
}
impl ValidityVTable<PrimitiveArray> for PrimitiveEncoding {
fn is_valid(&self, array: &PrimitiveArray, index: usize) -> bool {
array.validity().is_valid(index)
}
fn logical_validity(&self, array: &PrimitiveArray) -> LogicalValidity {
array.validity().to_logical(array.len())
}
}
impl VisitorVTable<PrimitiveArray> for PrimitiveEncoding {
fn accept(&self, array: &PrimitiveArray, visitor: &mut dyn ArrayVisitor) -> VortexResult<()> {
visitor.visit_buffer(array.buffer())?;
visitor.visit_validity(&array.validity())
}
}