Struct PrimaryKeyReadFormat 

pub struct PrimaryKeyReadFormat {
    metadata: RegionMetadataRef,
    arrow_schema: SchemaRef,
    field_id_to_index: HashMap<ColumnId, usize>,
    projection_indices: Vec<usize>,
    field_id_to_projected_index: HashMap<ColumnId, usize>,
    override_sequence: Option<SequenceNumber>,
}

Helper for reading the SST format.

Fields

metadata: RegionMetadataRef

The metadata stored in the SST.

arrow_schema: SchemaRef

SST file schema.

field_id_to_index: HashMap<ColumnId, usize>

Field column id to its index in schema (SST schema). In SST schema, fields are stored in the front of the schema.

projection_indices: Vec<usize>

Indices of columns to read from the SST. It contains all internal columns.

field_id_to_projected_index: HashMap<ColumnId, usize>

Field column id to their index in the projected schema ( the schema of Batch).

override_sequence: Option<SequenceNumber>

Sequence number to override the sequence read from the SST.

Implementations

impl PrimaryKeyReadFormat

pub fn new( metadata: RegionMetadataRef, column_ids: impl Iterator<Item = ColumnId>, ) -> PrimaryKeyReadFormat

Creates a helper with existing metadata and column_ids to read.

pub(crate) fn set_override_sequence(&mut self, sequence: Option<SequenceNumber>)

Sets the sequence number to override.

pub(crate) fn arrow_schema(&self) -> &SchemaRef

Gets the arrow schema of the SST file.

This schema is computed from the region metadata but should be the same as the arrow schema decoded from the file metadata.

pub(crate) fn metadata(&self) -> &RegionMetadataRef

Gets the metadata of the SST.

pub(crate) fn projection_indices(&self) -> &[usize]

Gets sorted projection indices to read.

pub(crate) fn new_override_sequence_array( &self, length: usize, ) -> Option<ArrayRef>

Creates a sequence array to override.

pub fn convert_record_batch( &self, record_batch: &RecordBatch, override_sequence_array: Option<&ArrayRef>, batches: &mut VecDeque<Batch>, ) -> Result<()>

Convert a arrow record batch into batches.

The length of override_sequence_array must be larger than the length of the record batch. Note that the record_batch may only contains a subset of columns if it is projected.

pub fn min_values( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column_id: ColumnId, ) -> StatValues

Returns min values of specific column in row groups.

pub fn max_values( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column_id: ColumnId, ) -> StatValues

Returns max values of specific column in row groups.

pub fn null_counts( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column_id: ColumnId, ) -> StatValues

Returns null counts of specific column in row groups.

fn get_field_batch_columns( &self, record_batch: &RecordBatch, ) -> Result<Vec<BatchColumn>>

Get fields from record_batch.

fn tag_values( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column: &ColumnMetadata, is_min: bool, ) -> StatValues

Returns min/max values of specific tag.

fn first_tag_values( &self, row_groups: &[impl Borrow<RowGroupMetaData>], column: &ColumnMetadata, is_min: bool, ) -> Option<ArrayRef>

Returns min/max values of the first tag. Returns None if the tag does not have statistics.

fn primary_key_position(&self) -> usize

Index in SST of the primary key.

fn time_index_position(&self) -> usize

Index in SST of the time index.

pub fn field_index_by_id(&self, column_id: ColumnId) -> Option<usize>

Index of a field column by its column id.