Struct ParquetReaderBuilder 

pub struct ParquetReaderBuilder {

    table_dir: String,
    path_type: PathType,
    file_handle: FileHandle,
    object_store: ObjectStore,
    predicate: Option<Predicate>,
    projection: Option<Vec<ColumnId>>,
    cache_strategy: CacheStrategy,
    inverted_index_appliers: [Option<Arc<InvertedIndexApplier>>; 2],
    bloom_filter_index_appliers: [Option<Arc<BloomFilterIndexApplier>>; 2],
    fulltext_index_appliers: [Option<Arc<FulltextIndexApplier>>; 2],
    expected_metadata: Option<RegionMetadataRef>,
    compaction: bool,
    pre_filter_mode: PreFilterMode,
    decode_primary_key_values: bool,
    page_index_policy: PageIndexPolicy,
}

Parquet SST reader builder.

Fields

table_dir: String

SST directory.

path_type: PathType

Path type for generating file paths.

file_handle: FileHandle

object_store: ObjectStore

predicate: Option<Predicate>

Predicate to push down.

projection: Option<Vec<ColumnId>>

Metadata of columns to read.

None reads all columns. Due to schema change, the projection can contain columns not in the parquet file.

cache_strategy: CacheStrategy

Strategy to cache SST data.

inverted_index_appliers: [Option<Arc<InvertedIndexApplier>>; 2]

Index appliers.

bloom_filter_index_appliers: [Option<Arc<BloomFilterIndexApplier>>; 2]

fulltext_index_appliers: [Option<Arc<FulltextIndexApplier>>; 2]

expected_metadata: Option<RegionMetadataRef>

Expected metadata of the region while reading the SST. This is usually the latest metadata of the region. The reader use it get the correct column id of a column by name.

compaction: bool

Whether this reader is for compaction.

pre_filter_mode: PreFilterMode

Mode to pre-filter columns.

decode_primary_key_values: bool

Whether to decode primary key values eagerly when reading primary key format SSTs.

page_index_policy: PageIndexPolicy

Implementations

impl ParquetReaderBuilder

pub fn new( table_dir: String, path_type: PathType, file_handle: FileHandle, object_store: ObjectStore, ) -> ParquetReaderBuilder

Returns a new ParquetReaderBuilder to read specific SST.

pub fn predicate(self, predicate: Option<Predicate>) -> ParquetReaderBuilder

Attaches the predicate to the builder.

pub fn projection( self, projection: Option<Vec<ColumnId>>, ) -> ParquetReaderBuilder

Attaches the projection to the builder.

The reader only applies the projection to fields.

pub fn cache(self, cache: CacheStrategy) -> ParquetReaderBuilder

Attaches the cache to the builder.

pub(crate) fn inverted_index_appliers( self, index_appliers: [Option<Arc<InvertedIndexApplier>>; 2], ) -> Self

Attaches the inverted index appliers to the builder.

pub(crate) fn bloom_filter_index_appliers( self, index_appliers: [Option<Arc<BloomFilterIndexApplier>>; 2], ) -> Self

Attaches the bloom filter index appliers to the builder.

pub(crate) fn fulltext_index_appliers( self, index_appliers: [Option<Arc<FulltextIndexApplier>>; 2], ) -> Self

Attaches the fulltext index appliers to the builder.

pub fn expected_metadata( self, expected_metadata: Option<RegionMetadataRef>, ) -> Self

Attaches the expected metadata to the builder.

pub fn compaction(self, compaction: bool) -> Self

Sets the compaction flag.

pub(crate) fn pre_filter_mode(self, pre_filter_mode: PreFilterMode) -> Self

Sets the pre-filter mode.

pub(crate) fn decode_primary_key_values(self, decode: bool) -> Self

Decodes primary key values eagerly when reading primary key format SSTs.

pub fn page_index_policy(self, page_index_policy: PageIndexPolicy) -> Self

pub async fn build(&self) -> Result<Option<ParquetReader>>

Builds a ParquetReader.

This needs to perform IO operation.

pub(crate) async fn build_reader_input( &self, metrics: &mut ReaderMetrics, ) -> Result<Option<(FileRangeContext, RowGroupSelection)>>

Builds a FileRangeContext and collects row groups to read.

This needs to perform IO operation.

async fn build_reader_input_inner( &self, metrics: &mut ReaderMetrics, ) -> Result<Option<(FileRangeContext, RowGroupSelection)>>

fn is_same_region_partition( region_partition_expr_str: Option<&str>, file_partition_expr: Option<&PartitionExpr>, ) -> Result<(Option<PartitionExpr>, bool)>

fn build_partition_filter( &self, read_format: &ReadFormat, prune_schema: &Arc<Schema>, ) -> Result<Option<PartitionFilterContext>>

Compare partition expressions from expected metadata and file metadata, and build a partition filter if they differ.

async fn read_parquet_metadata( &self, file_path: &str, file_size: u64, cache_metrics: &mut MetadataCacheMetrics, page_index_policy: PageIndexPolicy, ) -> Result<(Arc<CachedSstMeta>, bool)>

Reads parquet metadata of specific file. Returns (fused metadata, cache_miss_flag).

async fn row_groups_to_read( &self, read_format: &ReadFormat, parquet_meta: &ParquetMetaData, metrics: &mut ReaderFilterMetrics, ) -> RowGroupSelection

Computes row groups to read, along with their respective row selections.

async fn prune_row_groups_by_fulltext_index( &self, row_group_size: usize, num_row_groups: usize, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

Prunes row groups by fulltext index. Returns true if the row groups are pruned.

async fn prune_row_groups_by_inverted_index( &self, row_group_size: usize, num_row_groups: usize, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

Applies index to prune row groups.

TODO(zhongzc): Devise a mechanism to enforce the non-use of indices as an escape route in case of index issues, and it can be used to test the correctness of the index.

async fn prune_row_groups_by_bloom_filter( &self, row_group_size: usize, parquet_meta: &ParquetMetaData, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

async fn prune_row_groups_by_fulltext_bloom( &self, row_group_size: usize, parquet_meta: &ParquetMetaData, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> bool

fn row_groups_by_minmax( &self, read_format: &ReadFormat, parquet_meta: &ParquetMetaData, row_group_size: usize, total_row_count: usize, metrics: &mut ReaderFilterMetrics, skip_fields: bool, ) -> RowGroupSelection

Computes row groups selection after min-max pruning.

fn apply_index_result_and_update_cache( &self, predicate_key: &PredicateKey, file_id: FileId, result: RowGroupSelection, output: &mut RowGroupSelection, metrics: &mut ReaderFilterMetrics, index_type: &str, )