Struct FileCache 

pub(crate) struct FileCache {
    local_store: ObjectStore,
    parquet_index: Cache<IndexKey, IndexValue>,
    puffin_index: Cache<IndexKey, IndexValue>,
    puffin_capacity: u64,
}

A file cache manages files on local store and evict files based on size.

Fields

local_store: ObjectStore

Local store to cache files.

parquet_index: Cache<IndexKey, IndexValue>

Index to track cached Parquet files.

puffin_index: Cache<IndexKey, IndexValue>

Index to track cached Puffin files.

puffin_capacity: u64

Capacity of the puffin (index) cache in bytes.

Implementations

impl FileCache

pub(crate) fn new( local_store: ObjectStore, capacity: ReadableSize, ttl: Option<Duration>, index_cache_percent: Option<u8>, ) -> FileCache

Creates a new file cache.

fn build_cache( local_store: ObjectStore, capacity: u64, ttl: Option<Duration>, label: &'static str, ) -> Cache<IndexKey, IndexValue>

Builds a cache for a specific file type.

fn memory_index(&self, file_type: FileType) -> &Cache<IndexKey, IndexValue>

Returns the appropriate memory index for the given file type.

pub(crate) async fn put(&self, key: IndexKey, value: IndexValue)

Puts a file into the cache index.

The WriteCache should ensure the file is in the correct path.

pub(crate) async fn get(&self, key: IndexKey) -> Option<IndexValue>

pub(crate) async fn reader(&self, key: IndexKey) -> Option<Reader>

Reads a file from the cache.

pub(crate) async fn read_ranges( &self, key: IndexKey, ranges: &[Range<u64>], ) -> Option<Vec<Bytes>>

Reads ranges from the cache.

pub(crate) async fn remove(&self, key: IndexKey)

Removes a file from the cache explicitly. It always tries to remove the file from the local store because we may not have the file in the memory index if upload is failed.

async fn recover_inner(&self) -> Result<()>

pub(crate) async fn recover( self: &Arc<Self>, sync: bool, task_receiver: Option<UnboundedReceiver<RegionLoadCacheTask>>, )

Recovers the index from local store.

If task_receiver is provided, spawns a background task after recovery to process RegionLoadCacheTask messages for loading files into the cache.

pub(crate) fn cache_file_path(&self, key: IndexKey) -> String

Returns the cache file path for the key.

pub(crate) fn local_store(&self) -> ObjectStore

Returns the local store of the file cache.

pub(crate) async fn get_parquet_meta_data( &self, key: IndexKey, ) -> Option<ParquetMetaData>

Get the parquet metadata in file cache. If the file is not in the cache or fail to load metadata, return None.

async fn get_reader(&self, file_path: &str) -> Result<Option<Reader>>

pub(crate) fn contains_key(&self, key: &IndexKey) -> bool

Checks if the key is in the file cache.

pub(crate) fn puffin_cache_capacity(&self) -> u64

Returns the capacity of the puffin (index) cache in bytes.

pub(crate) fn puffin_cache_size(&self) -> u64

Returns the current weighted size (used bytes) of the puffin (index) cache.

pub(crate) async fn download( &self, index_key: IndexKey, remote_path: &str, remote_store: &ObjectStore, file_size: u64, ) -> Result<()>

Downloads a file in remote_path from the remote object store to the local cache (specified by index_key).

async fn download_without_cleaning( &self, index_key: IndexKey, remote_path: &str, remote_store: &ObjectStore, file_size: u64, ) -> Result<()>

Trait Implementations

impl Debug for FileCache

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.