Struct ReadFormat

pub struct ReadFormat {
    metadata: RegionMetadataRef,
    arrow_schema: SchemaRef,
    field_id_to_index: HashMap<ColumnId, usize>,
    projection_indices: Vec<usize>,
    field_id_to_projected_index: HashMap<ColumnId, usize>,
}

Helper for reading the SST format.

Fields

metadata: RegionMetadataRef

The metadata stored in the SST.

arrow_schema: SchemaRef

SST file schema.

field_id_to_index: HashMap<ColumnId, usize>

Field column id to its index in schema (SST schema). In SST schema, fields are stored in the front of the schema.

projection_indices: Vec<usize>

Indices of columns to read from the SST. It contains all internal columns.

field_id_to_projected_index: HashMap<ColumnId, usize>

Field column id to their index in the projected schema ( the schema of Batch).

Implementations

impl ReadFormat

pub fn new( metadata: RegionMetadataRef, column_ids: impl Iterator<Item = ColumnId>, ) -> ReadFormat

Creates a helper with existing metadata and column_ids to read.

pub(crate) fn arrow_schema(&self) -> &SchemaRef

Gets the arrow schema of the SST file.

This schema is computed from the region metadata but should be the same as the arrow schema decoded from the file metadata.

pub(crate) fn metadata(&self) -> &RegionMetadataRef

Gets the metadata of the SST.

pub(crate) fn projection_indices(&self) -> &[usize]

Gets sorted projection indices to read.

pub fn convert_record_batch( &self, record_batch: &RecordBatch, batches: &mut VecDeque<Batch>, ) -> Result<()>

Convert a arrow record batch into batches.

Note that the record_batch may only contains a subset of columns if it is projected.

pub fn min_values( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column_id: ColumnId, ) -> StatValues

Returns min values of specific column in row groups.

pub fn max_values( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column_id: ColumnId, ) -> StatValues

Returns max values of specific column in row groups.

pub fn null_counts( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column_id: ColumnId, ) -> StatValues

Returns null counts of specific column in row groups.

fn get_field_batch_columns( &self, record_batch: &RecordBatch, ) -> Result<Vec<BatchColumn>>

Get fields from record_batch.

fn tag_values( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column: &ColumnMetadata, is_min: bool, ) -> StatValues

Returns min/max values of specific tag.

fn first_tag_values( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column: &ColumnMetadata, is_min: bool, ) -> Option<ArrayRef>

Returns min/max values of the first tag. Returns None if the tag does not have statistics.

fn column_values( row_groups: &[impl Borrow<RowGroupMetaData>], column: &ColumnMetadata, column_index: usize, is_min: bool, ) -> Option<ArrayRef>

Returns min/max values of specific non-tag columns. Returns None if the column does not have statistics.

fn column_null_counts( row_groups: &[impl Borrow<RowGroupMetaData>], column_index: usize, ) -> Option<ArrayRef>

Returns null counts of specific non-tag columns.

fn primary_key_position(&self) -> usize

Index in SST of the primary key.

fn time_index_position(&self) -> usize

Index in SST of the time index.

pub fn field_index_by_id(&self, column_id: ColumnId) -> Option<usize>

Index of a field column by its column id.