Struct ParquetWriter

pub struct ParquetWriter<F: WriterFactory, I: IndexerBuilder, P: FilePathProvider> {
    path_provider: P,
    writer: Option<AsyncArrowWriter<SizeAwareWriter<F::Writer>>>,
    current_file: FileId,
    writer_factory: F,
    metadata: RegionMetadataRef,
    indexer_builder: I,
    current_indexer: Option<Indexer>,
    bytes_written: Arc<AtomicUsize>,
    file_cleaner: Option<TempFileCleaner>,
    metrics: Metrics,
}

Parquet SST writer.

Fields

path_provider: P

Path provider that creates SST and index file paths according to file id.

writer: Option<AsyncArrowWriter<SizeAwareWriter<F::Writer>>>

current_file: FileId

Current active file id.

writer_factory: F

metadata: RegionMetadataRef

Region metadata of the source and the target SST.

indexer_builder: I

Indexer build that can create indexer for multiple files.

current_indexer: Option<Indexer>

Current active indexer.

bytes_written: Arc<AtomicUsize>

file_cleaner: Option<TempFileCleaner>

Cleaner to remove temp files on failure.

metrics: Metrics

Write metrics

Implementations

impl<I, P> ParquetWriter<ObjectStoreWriterFactory, I, P>

where P: FilePathProvider, I: IndexerBuilder,

pub async fn new_with_object_store( object_store: ObjectStore, metadata: RegionMetadataRef, indexer_builder: I, path_provider: P, metrics: Metrics, ) -> ParquetWriter<ObjectStoreWriterFactory, I, P>

pub(crate) fn with_file_cleaner(self, cleaner: TempFileCleaner) -> Self

impl<F, I, P> ParquetWriter<F, I, P>

where F: WriterFactory, I: IndexerBuilder, P: FilePathProvider,

pub async fn new( factory: F, metadata: RegionMetadataRef, indexer_builder: I, path_provider: P, metrics: Metrics, ) -> ParquetWriter<F, I, P>

Creates a new parquet SST writer.

async fn finish_current_file( &mut self, ssts: &mut SstInfoArray, stats: &mut SourceStats, ) -> Result<()>

Finishes current SST file and index file.

pub async fn write_all( &mut self, source: Source, override_sequence: Option<SequenceNumber>, opts: &WriteOptions, ) -> Result<SstInfoArray>

Iterates source and writes all rows to Parquet file.

Returns the SstInfo if the SST is written.

async fn write_all_without_cleaning( &mut self, source: Source, override_sequence: Option<SequenceNumber>, opts: &WriteOptions, ) -> Result<SstInfoArray>

pub async fn write_all_flat( &mut self, source: FlatSource, opts: &WriteOptions, ) -> Result<SstInfoArray>

Iterates FlatSource and writes all RecordBatch in flat format to Parquet file.

Returns the SstInfo if the SST is written.

async fn write_all_flat_without_cleaning( &mut self, source: FlatSource, opts: &WriteOptions, ) -> Result<SstInfoArray>

fn customize_column_config( builder: WriterPropertiesBuilder, region_metadata: &RegionMetadataRef, ) -> WriterPropertiesBuilder

Customizes per-column config according to schema and maybe column cardinality.

async fn write_next_batch( &mut self, source: &mut Source, write_format: &PrimaryKeyWriteFormat, opts: &WriteOptions, ) -> Result<Option<Batch>>

async fn write_next_flat_batch( &mut self, source: &mut FlatSource, flat_format: &FlatWriteFormat, opts: &WriteOptions, ) -> Result<Option<RecordBatch>>

async fn maybe_init_writer( &mut self, schema: &SchemaRef, opts: &WriteOptions, ) -> Result<&mut AsyncArrowWriter<SizeAwareWriter<F::Writer>>>

pub fn into_metrics(self) -> Metrics

Consumes write and return the collected metrics.