Struct SorbetBatch

pub struct SorbetBatch {
    schema: SorbetSchema,
    batch: RecordBatch,
}

Any rerun-compatible [ArrowRecordBatch].

This is a wrapper around a SorbetSchema and a [ArrowRecordBatch].

Fields

schema: SorbetSchema

batch: RecordBatch

Implementations

impl SorbetBatch

pub fn try_new( batch_type: BatchType, schema: SorbetSchema, row_ids: Option<ArrowArrayRef>, index_arrays: Vec<ArrowArrayRef>, data_arrays: Vec<ArrowArrayRef>, ) -> Result<Self, ArrowError>

pub fn drop_all_rows(self) -> Self

Returns self but with all rows removed.

impl SorbetBatch

pub fn sorbet_schema(&self) -> &SorbetSchema

The parsed rerun schema of this batch.

pub fn heap_size_bytes(&self) -> Option<u64>

The heap size of this batch in bytes, if known.

pub fn fields(&self) -> &ArrowFields

pub fn arrow_batch_metadata(&self) -> &ArrowBatchMetadata

pub fn row_id_column( &self, ) -> Option<(&RowIdColumnDescriptor, &ArrowStructArray)>

The RowId column, if any.

pub fn all_columns( &self, ) -> impl Iterator<Item = (ColumnDescriptorRef<'_>, &ArrowArrayRef)>

All the columns along with their descriptors.

pub fn index_columns( &self, ) -> impl Iterator<Item = (&IndexColumnDescriptor, &ArrowArrayRef)>

The columns of the indices (timelines).

pub fn component_columns( &self, ) -> impl Iterator<Item = (&ComponentColumnDescriptor, &ArrowArrayRef)>

The columns of the components.

impl SorbetBatch

pub fn try_from_record_batch( batch: &ArrowRecordBatch, batch_type: BatchType, ) -> Result<Self, SorbetError>

Will automatically wrap data columns in ListArrays if they are not already.

Will also migrate old types to new types.

Methods from Deref<Target = ArrowRecordBatch>

pub fn schema(&self) -> Arc<Schema>

Returns the [Schema] of the record batch.

pub fn schema_ref(&self) -> &Arc<Schema>

Returns a reference to the [Schema] of the record batch.

pub fn project(&self, indices: &[usize]) -> Result<RecordBatch, ArrowError>

Projects the schema onto the specified columns

pub fn normalize( &self, separator: &str, max_level: Option<usize>, ) -> Result<RecordBatch, ArrowError>

Normalize a semi-structured [RecordBatch] into a flat table.

Nested [Field]s will generate names separated by separator, up to a depth of max_level (unlimited if None).

e.g. given a [RecordBatch] with schema:

    "foo": StructArray<"bar": Utf8>

A separator of "." would generate a batch with the schema:

    "foo.bar": Utf8

Note that giving a depth of Some(0) to max_level is the same as passing in None; it will be treated as unlimited.

Example

let animals: ArrayRef = Arc::new(StringArray::from(vec!["Parrot", ""]));
let n_legs: ArrayRef = Arc::new(Int64Array::from(vec![Some(2), Some(4)]));

let animals_field = Arc::new(Field::new("animals", DataType::Utf8, true));
let n_legs_field = Arc::new(Field::new("n_legs", DataType::Int64, true));

let a = Arc::new(StructArray::from(vec![
    (animals_field.clone(), Arc::new(animals.clone()) as ArrayRef),
    (n_legs_field.clone(), Arc::new(n_legs.clone()) as ArrayRef),
]));

let schema = Schema::new(vec![
    Field::new(
        "a",
        DataType::Struct(Fields::from(vec![animals_field, n_legs_field])),
        false,
    )
]);

let normalized = RecordBatch::try_new(Arc::new(schema), vec![a])
    .expect("valid conversion")
    .normalize(".", None)
    .expect("valid normalization");

let expected = RecordBatch::try_from_iter_with_nullable(vec![
    ("a.animals", animals.clone(), true),
    ("a.n_legs", n_legs.clone(), true),
])
.expect("valid conversion");

assert_eq!(expected, normalized);

pub fn num_columns(&self) -> usize

Returns the number of columns in the record batch.

Example

let id_array = Int32Array::from(vec![1, 2, 3, 4, 5]);
let schema = Schema::new(vec![
    Field::new("id", DataType::Int32, false)
]);

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(id_array)]).unwrap();

assert_eq!(batch.num_columns(), 1);

pub fn num_rows(&self) -> usize

Returns the number of rows in each column.

Example

let id_array = Int32Array::from(vec![1, 2, 3, 4, 5]);
let schema = Schema::new(vec![
    Field::new("id", DataType::Int32, false)
]);

let batch = RecordBatch::try_new(Arc::new(schema), vec![Arc::new(id_array)]).unwrap();

assert_eq!(batch.num_rows(), 5);

pub fn column(&self, index: usize) -> &Arc<dyn Array>

Get a reference to a column’s array by index.

Panics

Panics if index is outside of 0..num_columns.

pub fn column_by_name(&self, name: &str) -> Option<&Arc<dyn Array>>

Get a reference to a column’s array by name.

pub fn columns(&self) -> &[Arc<dyn Array>]

Get a reference to all columns in the record batch.

pub fn slice(&self, offset: usize, length: usize) -> RecordBatch

Return a new RecordBatch where each column is sliced according to offset and length

Panics

Panics if offset with length is greater than column length.

pub fn get_array_memory_size(&self) -> usize

Returns the total number of bytes of memory occupied physically by this batch.

Note that this does not always correspond to the exact memory usage of a RecordBatch (might overestimate), since multiple columns can share the same buffers or slices thereof, the memory used by the shared buffers might be counted multiple times.

Trait Implementations

impl AsRef<RecordBatch> for SorbetBatch

fn as_ref(&self) -> &ArrowRecordBatch

Converts this type into a shared reference of the (usually inferred) input type.

impl AsRef<SorbetBatch> for ChunkBatch

fn as_ref(&self) -> &SorbetBatch

Converts this type into a shared reference of the (usually inferred) input type.

impl Clone for SorbetBatch

fn clone(&self) -> SorbetBatch

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for SorbetBatch

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Deref for SorbetBatch

type Target = RecordBatch

The resulting type after dereferencing.

fn deref(&self) -> &ArrowRecordBatch

Dereferences the value.

impl Display for SorbetBatch

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl From<&SorbetBatch> for RecordBatch

fn from(batch: &SorbetBatch) -> Self

Converts to this type from the input type.

impl From<SorbetBatch> for RecordBatch

fn from(batch: SorbetBatch) -> Self

Converts to this type from the input type.

impl TryFrom<SorbetBatch> for ChunkBatch

fn try_from(sorbet_batch: SorbetBatch) -> Result<Self, Self::Error>

Will automatically wrap data columns in ListArrays if they are not already.

type Error = SorbetError

The type returned in the event of a conversion error.