Struct ParquetReaderBuilder 

pub struct ParquetReaderBuilder {

    table_dir: String,
    path_type: PathType,
    file_handle: FileHandle,
    object_store: ObjectStore,
    predicate: Option<Predicate>,
    projection: Option<Vec<ColumnId>>,
    cache_strategy: CacheStrategy,
    inverted_index_appliers: [Option<Arc<InvertedIndexApplier>>; 2],
    bloom_filter_index_appliers: [Option<Arc<BloomFilterIndexApplier>>; 2],
    fulltext_index_appliers: [Option<Arc<FulltextIndexApplier>>; 2],
    expected_metadata: Option<RegionMetadataRef>,
    flat_format: bool,
    compaction: bool,
    pre_filter_mode: PreFilterMode,
}

Parquet SST reader builder.

Fields

table_dir: String

SST directory.

path_type: PathType

Path type for generating file paths.

file_handle: FileHandle

object_store: ObjectStore

predicate: Option<Predicate>

Predicate to push down.

projection: Option<Vec<ColumnId>>

Metadata of columns to read.

None reads all columns. Due to schema change, the projection can contain columns not in the parquet file.

cache_strategy: CacheStrategy

Strategy to cache SST data.

inverted_index_appliers: [Option<Arc<InvertedIndexApplier>>; 2]

Index appliers.

bloom_filter_index_appliers: [Option<Arc<BloomFilterIndexApplier>>; 2]

fulltext_index_appliers: [Option<Arc<FulltextIndexApplier>>; 2]

expected_metadata: Option<RegionMetadataRef>

Expected metadata of the region while reading the SST. This is usually the latest metadata of the region. The reader use it get the correct column id of a column by name.

flat_format: bool

Whether to use flat format for reading.

compaction: bool

Whether this reader is for compaction.

pre_filter_mode: PreFilterMode

Mode to pre-filter columns.

Implementations

impl ParquetReaderBuilder

pub fn new( table_dir: String, path_type: PathType, file_handle: FileHandle, object_store: ObjectStore, ) -> ParquetReaderBuilder

Returns a new ParquetReaderBuilder to read specific SST.

pub fn predicate(self, predicate: Option<Predicate>) -> ParquetReaderBuilder

Attaches the predicate to the builder.

pub fn projection( self, projection: Option<Vec<ColumnId>>, ) -> ParquetReaderBuilder

Attaches the projection to the builder.

The reader only applies the projection to fields.

pub fn cache(self, cache: CacheStrategy) -> ParquetReaderBuilder

Attaches the cache to the builder.

pub(crate) fn inverted_index_appliers( self, index_appliers: [Option<Arc<InvertedIndexApplier>>; 2], ) -> Self

Attaches the inverted index appliers to the builder.

pub(crate) fn bloom_filter_index_appliers( self, index_appliers: [Option<Arc<BloomFilterIndexApplier>>; 2], ) -> Self

Attaches the bloom filter index appliers to the builder.

pub(crate) fn fulltext_index_appliers( self, index_appliers: [Option<Arc<FulltextIndexApplier>>; 2], ) -> Self

Attaches the fulltext index appliers to the builder.

pub fn expected_metadata( self, expected_metadata: Option<RegionMetadataRef>, ) -> Self

Attaches the expected metadata to the builder.

pub fn flat_format(self, flat_format: bool) -> Self

Sets the flat format flag.

pub fn compaction(self, compaction: bool) -> Self

Sets the compaction flag.

pub(crate) fn pre_filter_mode(self, pre_filter_mode: PreFilterMode) -> Self

Sets the pre-filter mode.

pub async fn build(&self) -> Result<ParquetReader>

Builds a ParquetReader.

This needs to perform IO operation.

pub(crate) async fn build_reader_input( &self, metrics: &mut ReaderMetrics, ) -> Result<(FileRangeContext, RowGroupSelection)>

Builds a FileRangeContext and collects row groups to read.

This needs to perform IO operation.

fn get_region_metadata( file_path: &str, key_value_meta: Option<&Vec<KeyValue>>, ) -> Result<RegionMetadata>

Decodes region metadata from key value.

async fn read_parquet_metadata( &self, file_path: &str, file_size: u64, ) -> Result<Arc<ParquetMetaData>>

Reads parquet metadata of specific file.

async fn row_groups_to_read( &self, read_format: &ReadFormat, parquet_meta: &ParquetMetaData, metrics: &mut ReaderFilterMetrics, ) -> RowGroupSelection

Computes row groups to read, along with their respective row selections.

async fn prune_row_groups_by_fulltext_index( &self, row_group_size: usize, num_row_groups: usize, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

Prunes row groups by fulltext index. Returns true if the row groups are pruned.

async fn prune_row_groups_by_inverted_index( &self, row_group_size: usize, num_row_groups: usize, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

Applies index to prune row groups.

TODO(zhongzc): Devise a mechanism to enforce the non-use of indices as an escape route in case of index issues, and it can be used to test the correctness of the index.

async fn prune_row_groups_by_bloom_filter( &self, row_group_size: usize, parquet_meta: &ParquetMetaData, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

async fn prune_row_groups_by_fulltext_bloom( &self, row_group_size: usize, parquet_meta: &ParquetMetaData, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

fn compute_skip_fields(&self, parquet_meta: &ParquetMetaData) -> bool

Computes whether to skip field columns when building statistics based on PreFilterMode.

fn prune_row_groups_by_minmax( &self, read_format: &ReadFormat, parquet_meta: &ParquetMetaData, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

Prunes row groups by min-max index.

fn apply_index_result_and_update_cache( &self, predicate_key: &PredicateKey, file_id: FileId, result: RowGroupSelection, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, index_type: &str, )