1pub(crate) mod cache_size;
18pub(crate) mod columnar;
19
20pub(crate) mod file_cache;
21pub(crate) mod index;
22pub(crate) mod manifest_cache;
23pub(crate) mod parquet_page;
24pub(crate) mod range_cache;
25#[cfg(test)]
26pub(crate) mod test_util;
27pub(crate) mod write_cache;
28
29use std::sync::{Arc, Weak};
30use std::{io, mem};
31
32use bytes::Bytes;
33use common_base::readable_size::ReadableSize;
34use common_cache::metrics::{CacheSizeMetric, CacheSizeRegistration};
35use common_datasource::compression::CompressionType;
36use common_runtime::Runtime;
37use common_runtime::runtime::RuntimeTrait;
38use common_telemetry::warn;
39use datatypes::arrow::buffer::BooleanBuffer;
40use datatypes::arrow::record_batch::RecordBatch;
41use datatypes::types::json_type::JsonNativeType;
42use datatypes::value::Value;
43use datatypes::vectors::VectorRef;
44use index::bloom_filter_index::{BloomFilterIndexCache, BloomFilterIndexCacheRef};
45use index::result_cache::IndexResultCache;
46use moka::notification::RemovalCause;
47use moka::sync::Cache;
48use object_store::ObjectStore;
49use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
50use parquet::file::metadata::{
51 FileMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, ParquetMetaDataWriter,
52};
53use puffin::puffin_manager::cache::{PuffinMetadataCache, PuffinMetadataCacheRef};
54use smallvec::SmallVec;
55use snafu::{OptionExt, ResultExt};
56use store_api::metadata::{RegionMetadata, RegionMetadataRef};
57use store_api::storage::{ColumnId, ConcreteDataType, FileId, RegionId, TimeSeriesRowSelector};
58pub use write_cache::{WriteCacheUploadStoreWrapper, WriteCacheUploadStoreWrapperRef};
59
60use crate::cache::cache_size::parquet_meta_size;
61use crate::cache::columnar::{DecodedKey, DecodedValue, columnar_meta_weight};
62use crate::cache::file_cache::{FileType, IndexKey};
63use crate::cache::index::inverted_index::{InvertedIndexCache, InvertedIndexCacheRef};
64use crate::cache::parquet_page::ParquetPageCache;
65use crate::cache::range_cache::RangeScanCacheValue;
66use crate::cache::write_cache::WriteCacheRef;
67use crate::error::{
68 CompressObjectSnafu, DecompressObjectSnafu, InvalidMetadataSnafu, InvalidParquetSnafu,
69 JoinSnafu, ReadParquetSnafu, Result, UnexpectedSnafu, WriteParquetSnafu,
70};
71use crate::memtable::record_batch_estimated_size;
72use crate::metrics::{CACHE_BYTES, CACHE_EVICTION, CACHE_HIT, CACHE_MISS};
73use crate::read::Batch;
74use crate::read::range_cache::RangeScanCacheKey;
75use crate::read::read_columns::JsonTargetTypes;
76use crate::sst::file::{RegionFileId, RegionIndexId};
77use crate::sst::parquet::PARQUET_METADATA_KEY;
78use crate::sst::parquet::read_columns::ParquetReadColumns;
79use crate::sst::parquet::reader::MetadataCacheMetrics;
80
81pub(crate) fn find_shared_page_source<E: std::error::Error + Send + Sync + 'static>(
83 mut error: &io::Error,
84) -> Option<Arc<E>> {
85 loop {
86 let source = error.get_ref()?;
87 if let Some(source) = source.downcast_ref::<Arc<E>>() {
88 return Some(source.clone());
89 }
90 if let Some(nested) = source.downcast_ref::<Arc<io::Error>>() {
91 error = nested;
92 } else {
93 error = source.downcast_ref::<io::Error>()?;
94 }
95 }
96}
97
98const SST_META_TYPE: &str = "sst_meta";
100const SST_META_DECODED_TYPE: &str = "sst_meta_decoded";
102const VECTOR_TYPE: &str = "vector";
104const FILE_TYPE: &str = "file";
106const INDEX_TYPE: &str = "index";
108const SELECTOR_RESULT_TYPE: &str = "selector_result";
110const RANGE_RESULT_TYPE: &str = "range_result";
112const PREFILTER_RESULT_TYPE: &str = "prefilter_result";
114const RANGE_RESULT_CONCAT_MEMORY_LIMIT: ReadableSize = ReadableSize::mb(512);
115const RANGE_RESULT_CONCAT_MEMORY_PERMIT: ReadableSize = ReadableSize::kb(1);
116
117#[derive(Debug)]
118pub(crate) struct RangeResultMemoryLimiter {
119 semaphore: Arc<tokio::sync::Semaphore>,
120 permit_bytes: usize,
121 total_permits: usize,
122 #[cfg(test)]
125 waited_acquires: std::sync::atomic::AtomicUsize,
126}
127
128impl Default for RangeResultMemoryLimiter {
129 fn default() -> Self {
130 Self::new(
131 RANGE_RESULT_CONCAT_MEMORY_LIMIT.as_bytes() as usize,
132 RANGE_RESULT_CONCAT_MEMORY_PERMIT.as_bytes() as usize,
133 )
134 }
135}
136
137impl RangeResultMemoryLimiter {
138 pub(crate) fn new(limit_bytes: usize, permit_bytes: usize) -> Self {
139 let permit_bytes = permit_bytes.max(1);
140 let total_permits = limit_bytes
141 .div_ceil(permit_bytes)
142 .clamp(1, tokio::sync::Semaphore::MAX_PERMITS);
143 Self {
144 semaphore: Arc::new(tokio::sync::Semaphore::new(total_permits)),
145 permit_bytes,
146 total_permits,
147 #[cfg(test)]
148 waited_acquires: std::sync::atomic::AtomicUsize::new(0),
149 }
150 }
151
152 #[cfg(test)]
153 pub(crate) fn permit_bytes(&self) -> usize {
154 self.permit_bytes
155 }
156
157 #[cfg(test)]
158 pub(crate) fn waited_acquires(&self) -> usize {
159 self.waited_acquires
160 .load(std::sync::atomic::Ordering::Acquire)
161 }
162
163 #[cfg(test)]
164 pub(crate) fn available_permits(&self) -> usize {
165 self.semaphore.available_permits()
166 }
167
168 pub(crate) fn try_acquire(&self, bytes: usize) -> Option<tokio::sync::OwnedSemaphorePermit> {
171 let permits = bytes.div_ceil(self.permit_bytes).max(1);
172 if permits > self.total_permits {
173 return None;
174 }
175 self.semaphore
176 .clone()
177 .try_acquire_many_owned(permits.try_into().ok()?)
178 .ok()
179 }
180
181 pub(crate) async fn acquire(&self, bytes: usize) -> Result<tokio::sync::SemaphorePermit<'_>> {
182 let permits = bytes.div_ceil(self.permit_bytes).max(1);
183 if permits > self.total_permits {
184 return UnexpectedSnafu {
185 reason: format!(
186 "range result memory request of {bytes} bytes exceeds limiter capacity of {} bytes",
187 self.total_permits.saturating_mul(self.permit_bytes)
188 ),
189 }
190 .fail();
191 }
192 #[cfg(test)]
195 if self.semaphore.available_permits() < permits {
196 self.waited_acquires
197 .fetch_add(1, std::sync::atomic::Ordering::Release);
198 }
199
200 self.semaphore
201 .acquire_many(permits as u32)
202 .await
203 .map_err(|_| {
204 UnexpectedSnafu {
205 reason: "range result memory limiter is unexpectedly closed",
206 }
207 .build()
208 })
209 }
210}
211
212#[derive(Debug)]
217pub(crate) struct CachedSstMeta {
218 parquet_metadata: Arc<ParquetMetaData>,
219 parquet_metadata_size: usize,
220 region_metadata: RegionMetadataRef,
221 region_metadata_weight: usize,
222 page_index_policy: PageIndexPolicy,
223}
224
225#[derive(Debug)]
231pub(crate) struct CompactSstMeta {
232 encoded_metadata: Bytes,
233 decoded_size: usize,
234 region_metadata: Weak<RegionMetadata>,
235 page_index_policy: PageIndexPolicy,
236 decoded: tokio::sync::Mutex<Weak<CachedSstMeta>>,
237}
238
239#[derive(Debug)]
241pub(crate) struct PreparedSstMeta {
242 pub(crate) compact: Arc<CompactSstMeta>,
243 pub(crate) decoded: Arc<CachedSstMeta>,
244}
245
246impl PreparedSstMeta {
247 pub(crate) fn decoded(&self) -> Arc<CachedSstMeta> {
248 self.decoded.clone()
249 }
250}
251
252#[derive(Debug)]
254pub(crate) enum SstMetaPreparation {
255 Prepared(PreparedSstMeta),
257 DecodedOnly {
259 decoded: Arc<CachedSstMeta>,
260 encoding_error: crate::error::Error,
261 },
262}
263
264impl SstMetaPreparation {
265 pub(crate) fn decoded(&self) -> Arc<CachedSstMeta> {
266 match self {
267 Self::Prepared(metadata) => metadata.decoded(),
268 Self::DecodedOnly { decoded, .. } => decoded.clone(),
269 }
270 }
271}
272
273impl CompactSstMeta {
274 async fn decode(&self, runtime: &Runtime) -> Result<Arc<CachedSstMeta>> {
275 let mut decoded_guard = self.decoded.lock().await;
276 if let Some(decoded) = decoded_guard.upgrade() {
277 return Ok(decoded);
278 }
279
280 let encoded_metadata = self.encoded_metadata.clone();
281 let decoded_size = self.decoded_size;
282 let region_metadata = self.region_metadata.upgrade();
283 let page_index_policy = self.page_index_policy;
284 let decoded = runtime
285 .spawn_blocking(move || {
286 let bytes = zstd::bulk::decompress(&encoded_metadata, decoded_size).context(
287 DecompressObjectSnafu {
288 compress_type: CompressionType::Zstd,
289 path: "cached SST metadata",
290 },
291 )?;
292 let bytes = Bytes::from(bytes);
293 let mut reader = ParquetMetaDataReader::new()
294 .with_column_index_policy(PageIndexPolicy::Skip)
295 .with_offset_index_policy(page_index_policy);
296 reader.try_parse(&bytes).context(ReadParquetSnafu {
297 path: "cached SST metadata",
298 })?;
299 let metadata = reader.finish().context(ReadParquetSnafu {
300 path: "cached SST metadata",
301 })?;
302 CachedSstMeta::try_new_with_page_index_policy(
303 "cached SST metadata",
304 metadata,
305 region_metadata,
306 page_index_policy,
307 )
308 .map(Arc::new)
309 })
310 .await
311 .context(JoinSnafu)??;
312
313 *decoded_guard = Arc::downgrade(&decoded);
314 Ok(decoded)
315 }
316
317 fn satisfies_page_index_policy(&self, requested: PageIndexPolicy) -> bool {
318 satisfies_page_index_policy(self.page_index_policy, requested)
319 }
320}
321
322pub(crate) async fn decode_sst_meta(
324 file_path: &str,
325 parquet_metadata: ParquetMetaData,
326 region_metadata: Option<RegionMetadataRef>,
327 page_index_policy: PageIndexPolicy,
328 runtime: &Runtime,
329) -> Result<Arc<CachedSstMeta>> {
330 let file_path = file_path.to_string();
331 runtime
332 .spawn_blocking(move || {
333 let parquet_metadata = strip_column_indexes(parquet_metadata);
334 CachedSstMeta::try_new_with_page_index_policy(
335 &file_path,
336 parquet_metadata,
337 region_metadata,
338 page_index_policy,
339 )
340 .map(Arc::new)
341 })
342 .await
343 .context(JoinSnafu)?
344}
345
346pub(crate) async fn prepare_sst_meta(
348 file_path: &str,
349 parquet_metadata: ParquetMetaData,
350 region_metadata: Option<RegionMetadataRef>,
351 page_index_policy: PageIndexPolicy,
352 runtime: &Runtime,
353) -> Result<SstMetaPreparation> {
354 let file_path = file_path.to_string();
355 runtime
356 .spawn_blocking(move || {
357 prepare_sst_meta_sync(
358 &file_path,
359 parquet_metadata,
360 region_metadata,
361 page_index_policy,
362 )
363 })
364 .await
365 .context(JoinSnafu)?
366}
367
368pub(crate) fn prepare_sst_meta_sync(
370 file_path: &str,
371 parquet_metadata: ParquetMetaData,
372 region_metadata: Option<RegionMetadataRef>,
373 page_index_policy: PageIndexPolicy,
374) -> Result<SstMetaPreparation> {
375 let parquet_metadata = strip_column_indexes(parquet_metadata);
376 let cache_encoding = encode_compact_sst_meta(file_path, &parquet_metadata);
377 finish_sst_meta_preparation(
378 file_path,
379 parquet_metadata,
380 region_metadata,
381 page_index_policy,
382 cache_encoding,
383 )
384}
385
386fn strip_column_indexes(parquet_metadata: ParquetMetaData) -> ParquetMetaData {
387 let mut builder = parquet_metadata.into_builder();
390 builder.take_column_index();
391 builder.build()
392}
393
394fn encode_compact_sst_meta(
395 file_path: &str,
396 parquet_metadata: &ParquetMetaData,
397) -> Result<(Bytes, usize)> {
398 let mut encoded = Vec::new();
399 ParquetMetaDataWriter::new(&mut encoded, parquet_metadata)
400 .finish()
401 .context(WriteParquetSnafu)?;
402 let decoded_size = encoded.len();
403 let encoded_metadata = zstd::bulk::compress(&encoded, 3).context(CompressObjectSnafu {
404 compress_type: CompressionType::Zstd,
405 path: file_path,
406 })?;
407
408 Ok((
412 Bytes::from(encoded_metadata.into_boxed_slice()),
413 decoded_size,
414 ))
415}
416
417fn finish_sst_meta_preparation(
418 file_path: &str,
419 parquet_metadata: ParquetMetaData,
420 region_metadata: Option<RegionMetadataRef>,
421 page_index_policy: PageIndexPolicy,
422 cache_encoding: Result<(Bytes, usize)>,
423) -> Result<SstMetaPreparation> {
424 let decoded = Arc::new(CachedSstMeta::try_new_with_page_index_policy(
425 file_path,
426 parquet_metadata,
427 region_metadata,
428 page_index_policy,
429 )?);
430 let (encoded_metadata, decoded_size) = match cache_encoding {
431 Ok(encoded) => encoded,
432 Err(encoding_error) => {
433 return Ok(SstMetaPreparation::DecodedOnly {
434 decoded,
435 encoding_error,
436 });
437 }
438 };
439 let compact = Arc::new(CompactSstMeta {
440 encoded_metadata,
441 decoded_size,
442 region_metadata: Arc::downgrade(&decoded.region_metadata),
443 page_index_policy,
444 decoded: tokio::sync::Mutex::new(Arc::downgrade(&decoded)),
445 });
446
447 Ok(SstMetaPreparation::Prepared(PreparedSstMeta {
448 compact,
449 decoded,
450 }))
451}
452
453impl CachedSstMeta {
454 #[cfg(test)]
455 pub(crate) fn try_new(file_path: &str, parquet_metadata: ParquetMetaData) -> Result<Self> {
456 let page_index_policy = infer_loaded_page_index_policy(&parquet_metadata);
457 Self::try_new_with_page_index_policy(file_path, parquet_metadata, None, page_index_policy)
458 }
459
460 pub(crate) fn try_new_with_region_metadata(
461 file_path: &str,
462 parquet_metadata: ParquetMetaData,
463 region_metadata: Option<RegionMetadataRef>,
464 ) -> Result<Self> {
465 let page_index_policy = infer_loaded_page_index_policy(&parquet_metadata);
466 Self::try_new_with_page_index_policy(
467 file_path,
468 parquet_metadata,
469 region_metadata,
470 page_index_policy,
471 )
472 }
473
474 pub(crate) fn try_new_with_page_index_policy(
475 file_path: &str,
476 parquet_metadata: ParquetMetaData,
477 region_metadata: Option<RegionMetadataRef>,
478 page_index_policy: PageIndexPolicy,
479 ) -> Result<Self> {
480 let file_metadata = parquet_metadata.file_metadata();
481 let key_values = file_metadata
482 .key_value_metadata()
483 .context(InvalidParquetSnafu {
484 file: file_path,
485 reason: "missing key value meta",
486 })?;
487 let meta_value = key_values
488 .iter()
489 .find(|kv| kv.key == PARQUET_METADATA_KEY)
490 .with_context(|| InvalidParquetSnafu {
491 file: file_path,
492 reason: format!("key {} not found", PARQUET_METADATA_KEY),
493 })?;
494 let json = meta_value
495 .value
496 .as_ref()
497 .with_context(|| InvalidParquetSnafu {
498 file: file_path,
499 reason: format!("No value for key {}", PARQUET_METADATA_KEY),
500 })?;
501 let region_metadata = match region_metadata {
502 Some(region_metadata) => region_metadata,
503 None => Arc::new(
504 store_api::metadata::RegionMetadata::from_json(json)
505 .context(InvalidMetadataSnafu)?,
506 ),
507 };
508 let region_metadata_weight = region_metadata.estimated_size().max(json.len());
510 let parquet_metadata = Arc::new(strip_region_metadata_from_parquet(parquet_metadata));
511 let parquet_metadata_size = parquet_meta_size(&parquet_metadata);
512
513 Ok(Self {
514 parquet_metadata,
515 parquet_metadata_size,
516 region_metadata,
517 region_metadata_weight,
518 page_index_policy,
519 })
520 }
521
522 pub(crate) fn parquet_metadata(&self) -> Arc<ParquetMetaData> {
523 self.parquet_metadata.clone()
524 }
525
526 pub(crate) fn parquet_metadata_size(&self) -> usize {
528 self.parquet_metadata_size
529 }
530
531 pub(crate) fn region_metadata(&self) -> RegionMetadataRef {
532 self.region_metadata.clone()
533 }
534
535 fn satisfies_page_index_policy(&self, requested: PageIndexPolicy) -> bool {
536 satisfies_page_index_policy(self.page_index_policy, requested)
537 }
538}
539
540fn satisfies_page_index_policy(cached: PageIndexPolicy, requested: PageIndexPolicy) -> bool {
541 match requested {
542 PageIndexPolicy::Skip => true,
543 PageIndexPolicy::Optional => cached != PageIndexPolicy::Skip,
544 PageIndexPolicy::Required => cached == PageIndexPolicy::Required,
545 }
546}
547
548fn infer_loaded_page_index_policy(parquet_metadata: &ParquetMetaData) -> PageIndexPolicy {
549 if parquet_metadata.offset_index().is_some() {
550 PageIndexPolicy::Optional
551 } else {
552 PageIndexPolicy::Skip
553 }
554}
555
556fn strip_region_metadata_from_parquet(parquet_metadata: ParquetMetaData) -> ParquetMetaData {
557 let file_metadata = parquet_metadata.file_metadata();
558 let filtered_key_values = file_metadata.key_value_metadata().and_then(|key_values| {
559 let filtered = key_values
560 .iter()
561 .filter(|kv| kv.key != PARQUET_METADATA_KEY)
562 .cloned()
563 .collect::<Vec<_>>();
564 (!filtered.is_empty()).then_some(filtered)
565 });
566 let stripped_file_metadata = FileMetaData::new(
567 file_metadata.version(),
568 file_metadata.num_rows(),
569 file_metadata.created_by().map(ToString::to_string),
570 filtered_key_values,
571 file_metadata.schema_descr_ptr(),
572 file_metadata.column_orders().cloned(),
573 );
574
575 let mut builder = parquet_metadata.into_builder();
576 let row_groups = builder.take_row_groups();
577 let offset_index = builder.take_offset_index();
578
579 parquet::file::metadata::ParquetMetaDataBuilder::new(stripped_file_metadata)
580 .set_row_groups(row_groups)
581 .set_offset_index(offset_index)
582 .build()
583}
584
585fn removal_cause_str(cause: RemovalCause) -> &'static str {
586 match cause {
587 RemovalCause::Expired => "expired",
588 RemovalCause::Explicit => "explicit",
589 RemovalCause::Replaced => "replaced",
590 RemovalCause::Size => "size",
591 }
592}
593
594#[derive(Debug, Clone, PartialEq, Eq, Hash)]
595pub(crate) struct PrefilterRowSelector {
596 row_count: usize,
597 skip: bool,
598}
599
600impl From<&RowSelector> for PrefilterRowSelector {
605 fn from(selector: &RowSelector) -> Self {
606 Self {
607 row_count: selector.row_count,
608 skip: selector.skip,
609 }
610 }
611}
612
613#[derive(Debug, Clone, PartialEq, Eq, Hash)]
615pub(crate) struct PrefilterKey {
616 file_id: FileId,
617 row_group_idx: u32,
618 row_selection: Option<Arc<Vec<PrefilterRowSelector>>>,
619 schema_version: u64,
620 filter_exprs: SmallVec<[String; 1]>,
621 mem_usage: usize,
622}
623
624impl PrefilterKey {
625 pub(crate) fn row_selection_snapshot(
626 row_selection: Option<&RowSelection>,
627 ) -> Option<Arc<Vec<PrefilterRowSelector>>> {
628 row_selection.map(|selection| {
629 Arc::new(
630 selection
631 .iter()
632 .map(PrefilterRowSelector::from)
633 .collect::<Vec<_>>(),
634 )
635 })
636 }
637
638 pub(crate) fn new(
639 file_id: FileId,
640 row_group_idx: u32,
641 row_selection: Option<Arc<Vec<PrefilterRowSelector>>>,
642 schema_version: u64,
643 filter_exprs: SmallVec<[String; 1]>,
644 ) -> Self {
645 let row_selection_bytes = row_selection
646 .as_ref()
647 .map(|selection| selection.len() * mem::size_of::<PrefilterRowSelector>())
648 .unwrap_or(0);
649 let spilled_expr_bytes = if filter_exprs.spilled() {
650 filter_exprs.capacity() * mem::size_of::<String>()
651 } else {
652 0
653 };
654 let expr_bytes = filter_exprs.iter().map(|s| s.capacity()).sum::<usize>();
655
656 Self {
657 file_id,
658 row_group_idx,
659 row_selection,
660 schema_version,
661 filter_exprs,
662 mem_usage: mem::size_of::<Self>()
663 + row_selection_bytes
664 + spilled_expr_bytes
665 + expr_bytes,
666 }
667 }
668
669 fn mem_usage(&self) -> usize {
670 self.mem_usage
671 }
672}
673
674type PrefilterResultCache = Cache<PrefilterKey, Arc<BooleanBuffer>>;
675
676fn new_prefilter_result_cache(capacity: u64) -> PrefilterResultCache {
677 Cache::builder()
678 .max_capacity(capacity)
679 .weigher(prefilter_result_cache_weight)
680 .eviction_listener(|k, v, cause| {
681 let size = prefilter_result_cache_weight(&k, &v);
682 CACHE_BYTES
683 .with_label_values(&[PREFILTER_RESULT_TYPE])
684 .sub(size.into());
685 CACHE_EVICTION
686 .with_label_values(&[PREFILTER_RESULT_TYPE, removal_cause_str(cause)])
687 .inc();
688 })
689 .build()
690}
691
692fn prefilter_result_cache_weight(k: &PrefilterKey, v: &Arc<BooleanBuffer>) -> u32 {
693 (k.mem_usage() + mem::size_of::<BooleanBuffer>() + v.values().len()) as u32
694}
695
696#[derive(Clone)]
698pub enum CacheStrategy {
699 EnableAll(CacheManagerRef),
702 Compaction(CacheManagerRef),
707 Disabled,
709}
710
711impl CacheStrategy {
712 pub(crate) fn sst_meta_runtime(&self) -> Runtime {
714 match self {
715 CacheStrategy::Compaction(_) => common_runtime::compact_runtime(),
716 CacheStrategy::EnableAll(_) | CacheStrategy::Disabled => {
717 common_runtime::global_runtime()
718 }
719 }
720 }
721
722 pub(crate) fn sst_meta_cache_enabled(&self) -> bool {
724 match self {
725 CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => {
726 cache_manager.sst_meta_cache_enabled()
727 }
728 CacheStrategy::Disabled => false,
729 }
730 }
731
732 #[cfg(test)]
733 pub(crate) async fn get_or_load_sst_meta<F>(
734 &self,
735 file_id: RegionFileId,
736 page_index_policy: PageIndexPolicy,
737 load: F,
738 ) -> Result<(Arc<CachedSstMeta>, bool)>
739 where
740 F: std::future::Future<Output = Result<(Arc<CachedSstMeta>, bool)>>,
741 {
742 self.get_or_load_observed_sst_meta(file_id, page_index_policy, async {
743 (load.await, SstMetaResolution::default())
744 })
745 .await
746 .0
747 }
748
749 pub(crate) async fn get_or_load_observed_sst_meta<F>(
751 &self,
752 file_id: RegionFileId,
753 page_index_policy: PageIndexPolicy,
754 load: F,
755 ) -> (Result<(Arc<CachedSstMeta>, bool)>, SstMetaResolution)
756 where
757 F: std::future::Future<Output = (Result<(Arc<CachedSstMeta>, bool)>, SstMetaResolution)>,
758 {
759 let cache = match self {
760 Self::EnableAll(manager) | Self::Compaction(manager) => {
761 manager.sst_decoded_meta_cache.as_ref()
762 }
763 Self::Disabled => None,
764 };
765 let Some(cache) = cache else {
766 return load.await;
767 };
768 let key = SstDecodedMetaKey::new(file_id, page_index_policy);
769 let result = cache
770 .try_get_with(&key, async {
771 let (result, resolution) = load.await;
772 let (metadata, cache_miss) = result.map_err(|source| SstMetaLoadError {
773 source: Arc::new(source),
774 resolution,
775 })?;
776 let value = SstMetaLoadResult::new(metadata, cache_miss, resolution);
777 Ok::<_, SstMetaLoadError>(value)
778 })
779 .await;
780 match result {
781 Ok(value) => (
782 Ok((value.metadata.clone(), value.cache_miss)),
783 value.resolution,
784 ),
785 Err(error) => (
786 Err(error.source.clone()).context(crate::error::LoadSstMetaSnafu),
787 error.resolution,
788 ),
789 }
790 }
791
792 pub(crate) async fn observe_sst_meta(
794 &self,
795 file_id: RegionFileId,
796 page_index_policy: PageIndexPolicy,
797 metrics: &mut MetadataCacheMetrics,
798 ) -> SstMetaObservation {
799 match self {
800 Self::EnableAll(manager) | Self::Compaction(manager) => {
801 manager
802 .observe_sst_meta(file_id, page_index_policy, metrics)
803 .await
804 }
805 Self::Disabled => SstMetaObservation::Miss,
806 }
807 }
808
809 pub(crate) async fn resolve_sst_meta(
811 &self,
812 observation: SstMetaObservation,
813 file_id: RegionFileId,
814 metrics: &mut MetadataCacheMetrics,
815 page_index_policy: PageIndexPolicy,
816 ) -> Option<Arc<CachedSstMeta>> {
817 match self {
818 Self::EnableAll(manager) | Self::Compaction(manager) => {
819 manager
820 .resolve_sst_meta(
821 observation,
822 file_id,
823 metrics,
824 page_index_policy,
825 &self.sst_meta_runtime(),
826 )
827 .await
828 }
829 Self::Disabled => {
830 metrics.cache_miss += 1;
831 None
832 }
833 }
834 }
835
836 pub(crate) async fn get_sst_meta_data_from_mem_cache(
838 &self,
839 file_id: RegionFileId,
840 page_index_policy: PageIndexPolicy,
841 ) -> Option<Arc<CachedSstMeta>> {
842 match self {
843 CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => {
844 cache_manager
845 .get_sst_meta_data_from_mem_cache(file_id, page_index_policy)
846 .await
847 }
848 CacheStrategy::Disabled => None,
849 }
850 }
851
852 pub async fn get_parquet_meta_data_from_mem_cache(
854 &self,
855 file_id: RegionFileId,
856 ) -> Option<Arc<ParquetMetaData>> {
857 self.get_sst_meta_data_from_mem_cache(file_id, PageIndexPolicy::Skip)
858 .await
859 .map(|metadata| metadata.parquet_metadata())
860 }
861
862 pub(crate) fn put_compact_sst_meta(
864 &self,
865 file_id: RegionFileId,
866 metadata: Arc<CompactSstMeta>,
867 ) {
868 match self {
869 CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => {
870 cache_manager.put_compact_sst_meta(file_id, metadata);
871 }
872 CacheStrategy::Disabled => {}
873 }
874 }
875
876 pub async fn put_parquet_meta_data(
878 &self,
879 file_id: RegionFileId,
880 metadata: Arc<ParquetMetaData>,
881 region_metadata: Option<RegionMetadataRef>,
882 ) {
883 match self {
884 CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => {
885 cache_manager
886 .put_parquet_meta_data(file_id, metadata, region_metadata)
887 .await;
888 }
889 CacheStrategy::Disabled => {}
890 }
891 }
892
893 pub(crate) fn get_prefilter_result(&self, key: &PrefilterKey) -> Option<Arc<BooleanBuffer>> {
896 match self {
897 CacheStrategy::EnableAll(cache_manager) => cache_manager.get_prefilter_result(key),
898 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
899 }
900 }
901
902 pub(crate) fn put_prefilter_result(&self, key: PrefilterKey, result: Arc<BooleanBuffer>) {
905 if let CacheStrategy::EnableAll(cache_manager) = self {
906 cache_manager.put_prefilter_result(key, result);
907 }
908 }
909
910 pub async fn remove_parquet_meta_data(&self, file_id: RegionFileId) {
912 match self {
913 CacheStrategy::EnableAll(cache_manager) => {
914 cache_manager.remove_parquet_meta_data(file_id).await;
915 }
916 CacheStrategy::Compaction(cache_manager) => {
917 cache_manager.remove_parquet_meta_data(file_id).await;
918 }
919 CacheStrategy::Disabled => {}
920 }
921 }
922
923 pub fn get_repeated_vector(
926 &self,
927 data_type: &ConcreteDataType,
928 value: &Value,
929 ) -> Option<VectorRef> {
930 match self {
931 CacheStrategy::EnableAll(cache_manager) => {
932 cache_manager.get_repeated_vector(data_type, value)
933 }
934 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
935 }
936 }
937
938 pub fn put_repeated_vector(&self, value: Value, vector: VectorRef) {
941 if let CacheStrategy::EnableAll(cache_manager) = self {
942 cache_manager.put_repeated_vector(value, vector);
943 }
944 }
945
946 pub(crate) fn parquet_page_cache(&self) -> Option<&Arc<ParquetPageCache>> {
948 match self {
949 Self::EnableAll(manager) => manager.page_cache.as_ref(),
950 Self::Compaction(_) | Self::Disabled => None,
951 }
952 }
953
954 pub async fn evict_puffin_cache(&self, file_id: RegionIndexId) {
956 match self {
957 CacheStrategy::EnableAll(cache_manager) => {
958 cache_manager.evict_puffin_cache(file_id).await
959 }
960 CacheStrategy::Compaction(cache_manager) => {
961 cache_manager.evict_puffin_cache(file_id).await
962 }
963 CacheStrategy::Disabled => {}
964 }
965 }
966
967 pub fn get_selector_result(
970 &self,
971 selector_key: &SelectorResultKey,
972 ) -> Option<Arc<SelectorResultValue>> {
973 match self {
974 CacheStrategy::EnableAll(cache_manager) => {
975 cache_manager.get_selector_result(selector_key)
976 }
977 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
978 }
979 }
980
981 pub fn put_selector_result(
984 &self,
985 selector_key: SelectorResultKey,
986 result: Arc<SelectorResultValue>,
987 ) {
988 if let CacheStrategy::EnableAll(cache_manager) = self {
989 cache_manager.put_selector_result(selector_key, result);
990 }
991 }
992
993 #[cfg(test)]
996 pub(crate) fn get_range_result(
997 &self,
998 key: &RangeScanCacheKey,
999 ) -> Option<Arc<RangeScanCacheValue>> {
1000 match self {
1001 CacheStrategy::EnableAll(cache_manager) => cache_manager.get_range_result(key),
1002 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1003 }
1004 }
1005
1006 #[cfg(test)]
1009 pub(crate) fn put_range_result(
1010 &self,
1011 key: RangeScanCacheKey,
1012 result: Arc<RangeScanCacheValue>,
1013 ) {
1014 if let CacheStrategy::EnableAll(cache_manager) = self {
1015 cache_manager.put_range_result(key, result);
1016 }
1017 }
1018
1019 pub(crate) fn range_result_cache(&self) -> Option<&RangeResultCacheRef> {
1020 match self {
1021 Self::EnableAll(manager) => manager.range_result_cache.as_ref(),
1022 Self::Compaction(_) | Self::Disabled => None,
1023 }
1024 }
1025
1026 pub(crate) fn has_range_result_cache(&self) -> bool {
1028 match self {
1029 CacheStrategy::EnableAll(cache_manager) => cache_manager.has_range_result_cache(),
1030 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => false,
1031 }
1032 }
1033
1034 #[cfg(test)]
1035 pub(crate) fn range_concat_memory(&self) -> Option<&Arc<RangeResultMemoryLimiter>> {
1036 self.range_result_cache().map(|cache| cache.concat_memory())
1037 }
1038
1039 pub fn write_cache(&self) -> Option<&WriteCacheRef> {
1042 match self {
1043 CacheStrategy::EnableAll(cache_manager) => cache_manager.write_cache(),
1044 CacheStrategy::Compaction(cache_manager) => cache_manager.write_cache(),
1045 CacheStrategy::Disabled => None,
1046 }
1047 }
1048
1049 pub fn inverted_index_cache(&self) -> Option<&InvertedIndexCacheRef> {
1052 match self {
1053 CacheStrategy::EnableAll(cache_manager) => cache_manager.inverted_index_cache(),
1054 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1055 }
1056 }
1057
1058 pub fn bloom_filter_index_cache(&self) -> Option<&BloomFilterIndexCacheRef> {
1061 match self {
1062 CacheStrategy::EnableAll(cache_manager) => cache_manager.bloom_filter_index_cache(),
1063 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1064 }
1065 }
1066
1067 pub fn puffin_metadata_cache(&self) -> Option<&PuffinMetadataCacheRef> {
1070 match self {
1071 CacheStrategy::EnableAll(cache_manager) => cache_manager.puffin_metadata_cache(),
1072 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1073 }
1074 }
1075
1076 pub fn index_result_cache(&self) -> Option<&IndexResultCache> {
1079 match self {
1080 CacheStrategy::EnableAll(cache_manager) => cache_manager.index_result_cache(),
1081 CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1082 }
1083 }
1084
1085 pub fn maybe_download_background(
1087 &self,
1088 index_key: IndexKey,
1089 remote_path: String,
1090 remote_store: ObjectStore,
1091 file_size: u64,
1092 ) {
1093 if let CacheStrategy::EnableAll(cache_manager) = self
1094 && let Some(write_cache) = cache_manager.write_cache()
1095 {
1096 write_cache.file_cache().maybe_download_background(
1097 index_key,
1098 remote_path,
1099 remote_store,
1100 file_size,
1101 );
1102 }
1103 }
1104}
1105
1106#[derive(Default)]
1110pub struct CacheManager {
1111 sst_meta_cache: Option<SstMetaCache>,
1113 _sst_meta_size_registration: Option<CacheSizeRegistration>,
1115 disable_sst_meta_cache_on_write: bool,
1117 sst_decoded_meta_cache: Option<SstDecodedMetaCache>,
1119 columnar_meta_cache: Option<ColumnarMetaCache>,
1123 vector_cache: Option<VectorCache>,
1125 page_cache: Option<Arc<ParquetPageCache>>,
1127 write_cache: Option<WriteCacheRef>,
1129 inverted_index_cache: Option<InvertedIndexCacheRef>,
1131 bloom_filter_index_cache: Option<BloomFilterIndexCacheRef>,
1133 puffin_metadata_cache: Option<PuffinMetadataCacheRef>,
1135 selector_result_cache: Option<SelectorResultCache>,
1137 range_result_cache: Option<RangeResultCacheRef>,
1139 #[cfg(test)]
1141 range_result_cache_size: u64,
1142 index_result_cache: Option<IndexResultCache>,
1144 prefilter_result_cache: Option<PrefilterResultCache>,
1146}
1147
1148pub type CacheManagerRef = Arc<CacheManager>;
1149
1150impl CacheManager {
1151 pub fn builder() -> CacheManagerBuilder {
1153 CacheManagerBuilder::default()
1154 }
1155
1156 #[cfg(test)]
1158 pub(crate) async fn get_sst_meta_data(
1159 &self,
1160 file_id: RegionFileId,
1161 metrics: &mut MetadataCacheMetrics,
1162 page_index_policy: PageIndexPolicy,
1163 runtime: &Runtime,
1164 ) -> Option<Arc<CachedSstMeta>> {
1165 let observation = self
1166 .observe_sst_meta(file_id, page_index_policy, metrics)
1167 .await;
1168 self.resolve_sst_meta(observation, file_id, metrics, page_index_policy, runtime)
1169 .await
1170 }
1171
1172 async fn observe_sst_meta(
1173 &self,
1174 file_id: RegionFileId,
1175 page_index_policy: PageIndexPolicy,
1176 metrics: &mut MetadataCacheMetrics,
1177 ) -> SstMetaObservation {
1178 let cache_key = SstMetaKey(file_id.region_id(), file_id.file_id());
1179 let compact = self
1180 .get_compact_sst_meta(&cache_key)
1181 .filter(|metadata| metadata.satisfies_page_index_policy(page_index_policy));
1182 let decoded = self
1183 .get_decoded_sst_meta(file_id, page_index_policy)
1184 .await
1185 .filter(|metadata| metadata.satisfies_page_index_policy(page_index_policy));
1186 let observation = match (compact, decoded) {
1187 (_, Some(decoded)) => SstMetaObservation::Decoded(decoded),
1188 (Some(compact), None) => SstMetaObservation::Compact(compact),
1189 (None, None) => SstMetaObservation::Miss,
1190 };
1191 observation.record(metrics);
1192 observation
1193 }
1194
1195 async fn resolve_sst_meta(
1196 &self,
1197 observation: SstMetaObservation,
1198 file_id: RegionFileId,
1199 metrics: &mut MetadataCacheMetrics,
1200 page_index_policy: PageIndexPolicy,
1201 runtime: &Runtime,
1202 ) -> Option<Arc<CachedSstMeta>> {
1203 match observation {
1204 SstMetaObservation::Decoded(decoded) => return Some(decoded),
1205 SstMetaObservation::Compact(compact) => match compact.decode(runtime).await {
1206 Ok(decoded) => return Some(decoded),
1207 Err(err) => {
1208 warn!(err; "Failed to decode compact SST metadata, region_id: {}, file_id: {}", file_id.region_id(), file_id.file_id());
1209 self.remove_parquet_meta_data(file_id).await;
1210 }
1211 },
1212 SstMetaObservation::Miss => {}
1213 }
1214
1215 let key = IndexKey::new(file_id.region_id(), file_id.file_id(), FileType::Parquet);
1216 if let Some(write_cache) = &self.write_cache {
1217 let file_cache = write_cache.file_cache();
1218 if self.sst_meta_cache_enabled() {
1219 if let Some(metadata) = file_cache
1220 .get_sst_meta_data(key, metrics, page_index_policy, runtime)
1221 .await
1222 {
1223 metrics.file_cache_hit += 1;
1224 let decoded = metadata.decoded();
1225 match metadata {
1226 SstMetaPreparation::Prepared(metadata) => {
1227 self.put_compact_sst_meta(file_id, metadata.compact);
1228 }
1229 SstMetaPreparation::DecodedOnly { encoding_error, .. } => {
1230 warn!(
1231 encoding_error;
1232 "Failed to encode file-cached SST metadata for memory cache, region_id: {}, file_id: {}",
1233 file_id.region_id(),
1234 file_id.file_id()
1235 );
1236 }
1237 }
1238 return Some(decoded);
1239 }
1240 } else if let Some(decoded) = file_cache
1241 .get_decoded_sst_meta_data(key, metrics, page_index_policy, runtime)
1242 .await
1243 {
1244 metrics.file_cache_hit += 1;
1245 return Some(decoded);
1246 }
1247 }
1248
1249 metrics.cache_miss += 1;
1250 None
1251 }
1252
1253 pub(crate) async fn get_sst_meta_data_from_mem_cache(
1256 &self,
1257 file_id: RegionFileId,
1258 page_index_policy: PageIndexPolicy,
1259 ) -> Option<Arc<CachedSstMeta>> {
1260 let value = self
1261 .get_sst_meta_data_from_mem_cache_inner(file_id, page_index_policy)
1262 .await;
1263 update_hit_miss(value, SST_META_DECODED_TYPE)
1264 }
1265
1266 async fn get_sst_meta_data_from_mem_cache_inner(
1267 &self,
1268 file_id: RegionFileId,
1269 page_index_policy: PageIndexPolicy,
1270 ) -> Option<Arc<CachedSstMeta>> {
1271 self.get_decoded_sst_meta(file_id, page_index_policy).await
1272 }
1273
1274 pub async fn get_parquet_meta_data_from_mem_cache(
1277 &self,
1278 file_id: RegionFileId,
1279 ) -> Option<Arc<ParquetMetaData>> {
1280 self.get_sst_meta_data_from_mem_cache(file_id, PageIndexPolicy::Skip)
1281 .await
1282 .map(|metadata| metadata.parquet_metadata())
1283 }
1284
1285 pub(crate) async fn put_sst_meta_data(
1287 &self,
1288 file_id: RegionFileId,
1289 metadata: Arc<CachedSstMeta>,
1290 ) {
1291 if let Some(cache) = &self.sst_decoded_meta_cache {
1292 let key = SstDecodedMetaKey::new(file_id, metadata.page_index_policy);
1293 let value = SstMetaLoadResult::new(metadata, false, SstMetaResolution::default());
1294 cache.insert(key, value).await;
1295 }
1296 }
1297
1298 pub(crate) fn put_compact_sst_meta(
1300 &self,
1301 file_id: RegionFileId,
1302 metadata: Arc<CompactSstMeta>,
1303 ) {
1304 let key = SstMetaKey(file_id.region_id(), file_id.file_id());
1305 if let Some(cache) = &self.sst_meta_cache {
1306 cache.insert(key, metadata);
1307 }
1308 }
1309
1310 fn get_compact_sst_meta(&self, key: &SstMetaKey) -> Option<Arc<CompactSstMeta>> {
1311 self.sst_meta_cache.as_ref()?.get(key)
1312 }
1313
1314 async fn get_decoded_sst_meta(
1315 &self,
1316 file_id: RegionFileId,
1317 requested: PageIndexPolicy,
1318 ) -> Option<Arc<CachedSstMeta>> {
1319 let cache = self.sst_decoded_meta_cache.as_ref()?;
1320 for policy in [
1321 PageIndexPolicy::Skip,
1322 PageIndexPolicy::Optional,
1323 PageIndexPolicy::Required,
1324 ] {
1325 if let Some(value) = cache.get(&SstDecodedMetaKey::new(file_id, policy)).await
1328 && value.metadata.satisfies_page_index_policy(requested)
1329 {
1330 return Some(value.metadata.clone());
1331 }
1332 }
1333 None
1334 }
1335
1336 pub(crate) async fn get_compact_sst_meta_data(
1341 &self,
1342 file_id: RegionFileId,
1343 page_index_policy: PageIndexPolicy,
1344 ) -> Option<Arc<CachedSstMeta>> {
1345 let key = SstMetaKey(file_id.region_id(), file_id.file_id());
1346 let compact = self
1347 .get_compact_sst_meta(&key)
1348 .filter(|metadata| metadata.satisfies_page_index_policy(page_index_policy))?;
1349 match compact.decode(&common_runtime::global_runtime()).await {
1350 Ok(metadata) => Some(metadata),
1351 Err(err) => {
1352 warn!(err; "Failed to decode compact SST metadata, region_id: {}, file_id: {}", file_id.region_id(), file_id.file_id());
1353 self.remove_parquet_meta_data(file_id).await;
1354 None
1355 }
1356 }
1357 }
1358
1359 pub async fn put_parquet_meta_data(
1361 &self,
1362 file_id: RegionFileId,
1363 metadata: Arc<ParquetMetaData>,
1364 region_metadata: Option<RegionMetadataRef>,
1365 ) {
1366 if self.sst_decoded_meta_cache.is_some() {
1367 let file_path = format!(
1368 "region_id={}, file_id={}",
1369 file_id.region_id(),
1370 file_id.file_id()
1371 );
1372 match CachedSstMeta::try_new_with_region_metadata(
1373 &file_path,
1374 Arc::unwrap_or_clone(metadata),
1375 region_metadata,
1376 ) {
1377 Ok(metadata) => self.put_sst_meta_data(file_id, Arc::new(metadata)).await,
1378 Err(err) => warn!(
1379 err; "Failed to decode region metadata while caching parquet metadata, region_id: {}, file_id: {}",
1380 file_id.region_id(),
1381 file_id.file_id()
1382 ),
1383 }
1384 }
1385 }
1386
1387 pub async fn remove_parquet_meta_data(&self, file_id: RegionFileId) {
1389 let key = SstMetaKey(file_id.region_id(), file_id.file_id());
1390 if let Some(cache) = &self.sst_meta_cache {
1391 cache.remove(&key);
1392 }
1393 if let Some(cache) = &self.sst_decoded_meta_cache {
1394 for policy in [
1395 PageIndexPolicy::Skip,
1396 PageIndexPolicy::Optional,
1397 PageIndexPolicy::Required,
1398 ] {
1399 cache
1400 .invalidate(&SstDecodedMetaKey::new(file_id, policy))
1401 .await;
1402 }
1403 }
1404 }
1405
1406 pub(crate) fn sst_meta_cache_is_full(&self) -> bool {
1408 let Some(cache) = &self.sst_meta_cache else {
1409 return true;
1410 };
1411 cache
1412 .policy()
1413 .max_capacity()
1414 .is_some_and(|capacity| cache.weighted_size() >= capacity)
1415 }
1416
1417 pub(crate) fn sst_meta_cache_enabled(&self) -> bool {
1419 self.sst_meta_cache.is_some()
1420 }
1421
1422 pub(crate) fn sst_meta_cache_on_write_enabled(&self) -> bool {
1424 self.sst_meta_cache_enabled() && !self.disable_sst_meta_cache_on_write
1425 }
1426
1427 pub fn get_repeated_vector(
1429 &self,
1430 data_type: &ConcreteDataType,
1431 value: &Value,
1432 ) -> Option<VectorRef> {
1433 self.vector_cache.as_ref().and_then(|vector_cache| {
1434 let value = vector_cache.get(&(data_type.clone(), value.clone()));
1435 update_hit_miss(value, VECTOR_TYPE)
1436 })
1437 }
1438
1439 pub fn put_repeated_vector(&self, value: Value, vector: VectorRef) {
1441 if let Some(cache) = &self.vector_cache {
1442 let key = (vector.data_type(), value);
1443 CACHE_BYTES
1444 .with_label_values(&[VECTOR_TYPE])
1445 .add(vector_cache_weight(&key, &vector).into());
1446 cache.insert(key, vector);
1447 }
1448 }
1449
1450 pub async fn evict_puffin_cache(&self, file_id: RegionIndexId) {
1452 if let Some(cache) = &self.bloom_filter_index_cache {
1453 cache.invalidate_file(file_id.file_id());
1454 }
1455
1456 if let Some(cache) = &self.inverted_index_cache {
1457 cache.invalidate_file(file_id.file_id());
1458 }
1459
1460 if let Some(cache) = &self.index_result_cache {
1461 cache.invalidate_file(file_id.file_id());
1462 }
1463
1464 if let Some(cache) = &self.puffin_metadata_cache {
1465 if let Err(err) = cache.invalidate_file(&file_id.to_string()) {
1467 warn!(err; "Failed to invalidate Puffin metadata, file_id: {file_id}");
1468 }
1469 }
1470
1471 if let Some(write_cache) = &self.write_cache {
1472 write_cache
1473 .remove(IndexKey::new(
1474 file_id.region_id(),
1475 file_id.file_id(),
1476 FileType::Puffin(file_id.version),
1477 ))
1478 .await;
1479 }
1480 }
1481
1482 pub fn get_selector_result(
1484 &self,
1485 selector_key: &SelectorResultKey,
1486 ) -> Option<Arc<SelectorResultValue>> {
1487 self.selector_result_cache
1488 .as_ref()
1489 .and_then(|selector_result_cache| selector_result_cache.get(selector_key))
1490 }
1491
1492 pub fn put_selector_result(
1494 &self,
1495 selector_key: SelectorResultKey,
1496 result: Arc<SelectorResultValue>,
1497 ) {
1498 if let Some(cache) = &self.selector_result_cache {
1499 CACHE_BYTES
1500 .with_label_values(&[SELECTOR_RESULT_TYPE])
1501 .add(selector_result_cache_weight(&selector_key, &result).into());
1502 cache.insert(selector_key, result);
1503 }
1504 }
1505
1506 pub(crate) fn invalidate_range_results(&self, file_id: FileId) {
1508 if let Some(worker) = &self.range_result_cache {
1509 worker.invalidate_file(file_id);
1510 }
1511 }
1512
1513 #[cfg(test)]
1515 pub(crate) fn get_range_result(
1516 &self,
1517 key: &RangeScanCacheKey,
1518 ) -> Option<Arc<RangeScanCacheValue>> {
1519 self.range_result_cache
1520 .as_ref()
1521 .and_then(|cache| update_hit_miss(cache.get(key), RANGE_RESULT_TYPE))
1522 }
1523
1524 #[cfg(test)]
1526 pub(crate) fn put_range_result(
1527 &self,
1528 key: RangeScanCacheKey,
1529 result: Arc<RangeScanCacheValue>,
1530 ) {
1531 if let Some(cache) = &self.range_result_cache {
1532 cache.insert(key, result);
1533 }
1534 }
1535
1536 pub(crate) fn has_range_result_cache(&self) -> bool {
1538 self.range_result_cache.is_some()
1539 }
1540
1541 #[cfg(test)]
1542 pub(crate) fn range_result_cache_size(&self) -> usize {
1543 self.range_result_cache_size as usize
1544 }
1545
1546 pub(crate) fn write_cache(&self) -> Option<&WriteCacheRef> {
1548 self.write_cache.as_ref()
1549 }
1550
1551 pub(crate) fn inverted_index_cache(&self) -> Option<&InvertedIndexCacheRef> {
1552 self.inverted_index_cache.as_ref()
1553 }
1554
1555 pub(crate) fn bloom_filter_index_cache(&self) -> Option<&BloomFilterIndexCacheRef> {
1556 self.bloom_filter_index_cache.as_ref()
1557 }
1558
1559 pub(crate) fn puffin_metadata_cache(&self) -> Option<&PuffinMetadataCacheRef> {
1560 self.puffin_metadata_cache.as_ref()
1561 }
1562
1563 pub(crate) fn index_result_cache(&self) -> Option<&IndexResultCache> {
1564 self.index_result_cache.as_ref()
1565 }
1566
1567 pub(crate) fn get_prefilter_result(&self, key: &PrefilterKey) -> Option<Arc<BooleanBuffer>> {
1568 self.prefilter_result_cache
1569 .as_ref()
1570 .and_then(|cache| update_hit_miss(cache.get(key), PREFILTER_RESULT_TYPE))
1571 }
1572
1573 pub(crate) fn put_prefilter_result(&self, key: PrefilterKey, result: Arc<BooleanBuffer>) {
1574 if let Some(cache) = &self.prefilter_result_cache {
1575 CACHE_BYTES
1576 .with_label_values(&[PREFILTER_RESULT_TYPE])
1577 .add(prefilter_result_cache_weight(&key, &result).into());
1578 cache.insert(key, result);
1579 }
1580 }
1581}
1582
1583pub fn selector_result_cache_miss() {
1585 CACHE_MISS.with_label_values(&[SELECTOR_RESULT_TYPE]).inc()
1586}
1587
1588pub fn selector_result_cache_hit() {
1590 CACHE_HIT.with_label_values(&[SELECTOR_RESULT_TYPE]).inc()
1591}
1592
1593#[derive(Default)]
1595pub struct CacheManagerBuilder {
1596 sst_meta_cache_size: u64,
1597 enable_columnar_meta_cache: bool,
1601 disable_sst_meta_cache_on_write: bool,
1602 vector_cache_size: u64,
1603 page_cache_size: u64,
1604 index_metadata_size: u64,
1605 index_content_size: u64,
1606 index_content_page_size: u64,
1607 index_result_cache_size: u64,
1608 prefilter_result_cache_size: u64,
1609 puffin_metadata_size: u64,
1610 write_cache: Option<WriteCacheRef>,
1611 selector_result_cache_size: u64,
1612 range_result_cache_size: u64,
1613}
1614
1615impl CacheManagerBuilder {
1616 pub fn sst_meta_cache_size(mut self, bytes: u64) -> Self {
1618 self.sst_meta_cache_size = bytes;
1619 self
1620 }
1621
1622 pub fn enable_columnar_meta_cache(mut self, enable: bool) -> Self {
1624 self.enable_columnar_meta_cache = enable;
1625 self
1626 }
1627
1628 pub(crate) fn disable_sst_meta_cache_on_write(mut self) -> Self {
1630 self.disable_sst_meta_cache_on_write = true;
1631 self
1632 }
1633
1634 pub fn vector_cache_size(mut self, bytes: u64) -> Self {
1636 self.vector_cache_size = bytes;
1637 self
1638 }
1639
1640 pub fn page_cache_size(mut self, bytes: u64) -> Self {
1642 self.page_cache_size = bytes;
1643 self
1644 }
1645
1646 pub fn write_cache(mut self, cache: Option<WriteCacheRef>) -> Self {
1648 self.write_cache = cache;
1649 self
1650 }
1651
1652 pub fn index_metadata_size(mut self, bytes: u64) -> Self {
1654 self.index_metadata_size = bytes;
1655 self
1656 }
1657
1658 pub fn index_content_size(mut self, bytes: u64) -> Self {
1660 self.index_content_size = bytes;
1661 self
1662 }
1663
1664 pub fn index_content_page_size(mut self, bytes: u64) -> Self {
1666 self.index_content_page_size = bytes;
1667 self
1668 }
1669
1670 pub fn index_result_cache_size(mut self, bytes: u64) -> Self {
1672 self.index_result_cache_size = bytes;
1673 self
1674 }
1675
1676 pub fn prefilter_result_cache_size(mut self, bytes: u64) -> Self {
1678 self.prefilter_result_cache_size = bytes;
1679 self
1680 }
1681
1682 pub fn puffin_metadata_size(mut self, bytes: u64) -> Self {
1684 self.puffin_metadata_size = bytes;
1685 self
1686 }
1687
1688 pub fn selector_result_cache_size(mut self, bytes: u64) -> Self {
1690 self.selector_result_cache_size = bytes;
1691 self
1692 }
1693
1694 pub fn range_result_cache_size(mut self, bytes: u64) -> Self {
1696 self.range_result_cache_size = bytes;
1697 self
1698 }
1699
1700 pub fn build(self) -> CacheManager {
1702 self.build_with_sst_meta_size_metric(CACHE_BYTES.bind(&[SST_META_TYPE]))
1703 }
1704
1705 fn build_with_sst_meta_size_metric(
1706 self,
1707 sst_meta_size_metric: CacheSizeMetric,
1708 ) -> CacheManager {
1709 let compact_meta_capacity = self.sst_meta_cache_size.div_ceil(2);
1713 let columnar_meta_capacity = if self.enable_columnar_meta_cache {
1718 self.sst_meta_cache_size / 4
1719 } else {
1720 0
1721 };
1722 let decoded_meta_capacity = self.sst_meta_cache_size / 2 - columnar_meta_capacity;
1723 let sst_meta_cache = (compact_meta_capacity != 0).then(|| {
1724 Cache::builder()
1725 .max_capacity(compact_meta_capacity)
1726 .weigher(meta_cache_weight)
1727 .eviction_listener(|_k, _v, cause| {
1728 CACHE_EVICTION
1729 .with_label_values(&[SST_META_TYPE, removal_cause_str(cause)])
1730 .inc();
1731 })
1732 .build()
1733 });
1734 let sst_meta_size_registration = sst_meta_cache
1735 .clone()
1736 .map(|cache| sst_meta_size_metric.register(move || cache.weighted_size()));
1737 let sst_decoded_meta_cache = (decoded_meta_capacity != 0).then(|| {
1738 common_cache::Cache::new(
1739 "sst_decoded_metadata",
1740 decoded_meta_capacity,
1741 decoded_meta_cache_weight,
1742 |_k, _v, cause| {
1743 CACHE_EVICTION
1744 .with_label_values(&[SST_META_DECODED_TYPE, cause.as_str()])
1745 .inc();
1746 },
1747 )
1748 .with_size_metric(CACHE_BYTES.bind(&[SST_META_DECODED_TYPE]))
1749 .with_metrics(crate::metrics::cache_loading_metrics(SST_META_DECODED_TYPE))
1750 });
1751 let columnar_meta_cache = (columnar_meta_capacity != 0).then(|| {
1752 Cache::builder()
1753 .max_capacity(columnar_meta_capacity)
1754 .weigher(columnar_meta_weight)
1755 .eviction_listener(|k, v, cause| {
1756 let size = columnar_meta_weight(&k, &v);
1757 CACHE_BYTES.with_label_values(&[v.label()]).sub(size.into());
1758 CACHE_EVICTION
1759 .with_label_values(&[v.label(), removal_cause_str(cause)])
1760 .inc();
1761 })
1762 .build()
1763 });
1764 let vector_cache = (self.vector_cache_size != 0).then(|| {
1765 Cache::builder()
1766 .max_capacity(self.vector_cache_size)
1767 .weigher(vector_cache_weight)
1768 .eviction_listener(|k, v, cause| {
1769 let size = vector_cache_weight(&k, &v);
1770 CACHE_BYTES
1771 .with_label_values(&[VECTOR_TYPE])
1772 .sub(size.into());
1773 CACHE_EVICTION
1774 .with_label_values(&[VECTOR_TYPE, removal_cause_str(cause)])
1775 .inc();
1776 })
1777 .build()
1778 });
1779 let page_cache =
1780 (self.page_cache_size != 0).then(|| ParquetPageCache::from_env(self.page_cache_size));
1781 let inverted_index_cache = InvertedIndexCache::new(
1782 self.index_metadata_size,
1783 self.index_content_size,
1784 self.index_content_page_size,
1785 );
1786 let bloom_filter_index_cache = BloomFilterIndexCache::new(
1788 self.index_metadata_size,
1789 self.index_content_size,
1790 self.index_content_page_size,
1791 );
1792 let index_result_cache = (self.index_result_cache_size != 0)
1793 .then(|| IndexResultCache::new(self.index_result_cache_size));
1794 let prefilter_result_cache = (self.prefilter_result_cache_size != 0)
1795 .then(|| new_prefilter_result_cache(self.prefilter_result_cache_size));
1796 let puffin_metadata_cache =
1797 PuffinMetadataCache::new(self.puffin_metadata_size, &CACHE_BYTES)
1798 .with_load_metrics(crate::metrics::cache_loading_metrics("puffin_metadata"));
1799 let selector_result_cache = (self.selector_result_cache_size != 0).then(|| {
1800 Cache::builder()
1801 .max_capacity(self.selector_result_cache_size)
1802 .weigher(selector_result_cache_weight)
1803 .eviction_listener(|k, v, cause| {
1804 let size = selector_result_cache_weight(&k, &v);
1805 CACHE_BYTES
1806 .with_label_values(&[SELECTOR_RESULT_TYPE])
1807 .sub(size.into());
1808 CACHE_EVICTION
1809 .with_label_values(&[SELECTOR_RESULT_TYPE, removal_cause_str(cause)])
1810 .inc();
1811 })
1812 .build()
1813 });
1814 let range_result_cache = (self.range_result_cache_size != 0)
1815 .then(|| range_cache::RangeResultCache::new(self.range_result_cache_size));
1816 CacheManager {
1817 sst_meta_cache,
1818 _sst_meta_size_registration: sst_meta_size_registration,
1819 disable_sst_meta_cache_on_write: self.disable_sst_meta_cache_on_write,
1820 sst_decoded_meta_cache,
1821 columnar_meta_cache,
1822 vector_cache,
1823 page_cache,
1824 write_cache: self.write_cache,
1825 inverted_index_cache: Some(Arc::new(inverted_index_cache)),
1826 bloom_filter_index_cache: Some(Arc::new(bloom_filter_index_cache)),
1827 puffin_metadata_cache: Some(Arc::new(puffin_metadata_cache)),
1828 selector_result_cache,
1829 range_result_cache,
1830 #[cfg(test)]
1831 range_result_cache_size: self.range_result_cache_size,
1832 index_result_cache,
1833 prefilter_result_cache,
1834 }
1835 }
1836}
1837
1838fn meta_cache_weight(k: &SstMetaKey, v: &Arc<CompactSstMeta>) -> u32 {
1839 let size = k.estimated_size() + mem::size_of::<CompactSstMeta>() + v.encoded_metadata.len();
1842 u32::try_from(size).unwrap_or(u32::MAX)
1843}
1844
1845fn decoded_meta_cache_weight(k: &SstDecodedMetaKey, v: &Arc<SstMetaLoadResult>) -> u32 {
1846 SstMetaLoadResult::weight(k, &v.metadata)
1847}
1848
1849fn vector_cache_weight(_k: &(ConcreteDataType, Value), v: &VectorRef) -> u32 {
1850 (mem::size_of::<ConcreteDataType>() + mem::size_of::<Value>() + v.memory_size()) as u32
1852}
1853
1854fn selector_result_cache_weight(k: &SelectorResultKey, v: &Arc<SelectorResultValue>) -> u32 {
1855 (mem::size_of_val(k) + v.estimated_size()) as u32
1856}
1857
1858fn range_result_cache_weight(k: &RangeScanCacheKey, v: &Arc<RangeScanCacheValue>) -> u32 {
1859 (k.estimated_size() + v.estimated_size()) as u32
1860}
1861
1862fn update_hit_miss<T>(value: Option<T>, cache_type: &str) -> Option<T> {
1864 if value.is_some() {
1865 CACHE_HIT.with_label_values(&[cache_type]).inc();
1866 } else {
1867 CACHE_MISS.with_label_values(&[cache_type]).inc();
1868 }
1869 value
1870}
1871
1872#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1874pub(crate) struct SstMetaKey(RegionId, FileId);
1875
1876impl SstMetaKey {
1877 fn estimated_size(&self) -> usize {
1879 mem::size_of::<Self>()
1880 }
1881}
1882
1883#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1885pub struct SelectorResultKey {
1886 pub file_id: FileId,
1888 pub row_group_idx: usize,
1890 pub selector: TimeSeriesRowSelector,
1892}
1893
1894pub enum SelectorResult {
1896 PrimaryKey(Vec<Batch>),
1898 Flat(Vec<RecordBatch>),
1900}
1901
1902pub struct SelectorResultValue {
1904 pub result: SelectorResult,
1906 pub read_cols: ParquetReadColumns,
1908 pub json_target_types: JsonTargetTypes,
1913}
1914
1915impl SelectorResultValue {
1916 pub fn new(result: Vec<Batch>, read_cols: ParquetReadColumns) -> SelectorResultValue {
1918 SelectorResultValue {
1919 result: SelectorResult::PrimaryKey(result),
1920 read_cols,
1921 json_target_types: Arc::default(),
1922 }
1923 }
1924
1925 pub fn new_flat(
1927 result: Vec<RecordBatch>,
1928 read_cols: ParquetReadColumns,
1929 json_target_types: JsonTargetTypes,
1930 ) -> SelectorResultValue {
1931 SelectorResultValue {
1932 result: SelectorResult::Flat(result),
1933 read_cols,
1934 json_target_types,
1935 }
1936 }
1937
1938 fn estimated_size(&self) -> usize {
1940 let result_size: usize = match &self.result {
1941 SelectorResult::PrimaryKey(batches) => {
1942 batches.iter().map(|batch| batch.memory_size()).sum()
1943 }
1944 SelectorResult::Flat(batches) => batches.iter().map(record_batch_estimated_size).sum(),
1945 };
1946 result_size
1947 + self.json_target_types.len() * (size_of::<ColumnId>() + size_of::<JsonNativeType>())
1948 }
1949}
1950
1951pub(crate) enum SstMetaObservation {
1953 Decoded(Arc<CachedSstMeta>),
1954 Compact(Arc<CompactSstMeta>),
1955 Miss,
1956}
1957
1958impl SstMetaObservation {
1959 fn record(&self, metrics: &mut MetadataCacheMetrics) {
1960 match self {
1961 Self::Decoded(_) | Self::Compact(_) => {
1962 metrics.mem_cache_hit += 1;
1963 CACHE_HIT.with_label_values(&[SST_META_TYPE]).inc();
1964 match self {
1965 Self::Decoded(_) => CACHE_HIT.with_label_values(&[SST_META_DECODED_TYPE]).inc(),
1966 _ => CACHE_MISS.with_label_values(&[SST_META_DECODED_TYPE]).inc(),
1967 }
1968 }
1969 Self::Miss => CACHE_MISS.with_label_values(&[SST_META_TYPE]).inc(),
1970 }
1971 }
1972}
1973
1974#[derive(Clone, Copy, Default)]
1981pub(crate) struct SstMetaResolution {
1982 pub(crate) file_cache_hit: usize,
1983 pub(crate) cache_miss: usize,
1984}
1985
1986struct SstMetaLoadError {
1987 source: Arc<crate::error::Error>,
1988 resolution: SstMetaResolution,
1989}
1990
1991struct SstMetaLoadResult {
1993 metadata: Arc<CachedSstMeta>,
1994 cache_miss: bool,
1996 resolution: SstMetaResolution,
1997}
1998
1999impl SstMetaLoadResult {
2000 fn weight(key: &SstDecodedMetaKey, metadata: &CachedSstMeta) -> u32 {
2001 let size = size_of_val(key)
2002 + size_of::<Self>()
2003 + metadata.parquet_metadata_size
2004 + metadata.region_metadata_weight;
2005 u32::try_from(size).unwrap_or(u32::MAX)
2006 }
2007
2008 fn new(
2009 metadata: Arc<CachedSstMeta>,
2010 cache_miss: bool,
2011 resolution: SstMetaResolution,
2012 ) -> Arc<Self> {
2013 Arc::new(Self {
2014 metadata,
2015 cache_miss,
2016 resolution,
2017 })
2018 }
2019}
2020
2021#[derive(Clone, Debug, PartialEq, Eq, Hash)]
2023struct SstDecodedMetaKey {
2024 file: SstMetaKey,
2025 policy: mem::Discriminant<PageIndexPolicy>,
2026}
2027
2028impl SstDecodedMetaKey {
2029 fn new(file: RegionFileId, policy: PageIndexPolicy) -> Self {
2030 Self {
2031 file: SstMetaKey(file.region_id(), file.file_id()),
2032 policy: mem::discriminant(&policy),
2033 }
2034 }
2035}
2036
2037type SstMetaCache = Cache<SstMetaKey, Arc<CompactSstMeta>>;
2039type SstDecodedMetaCache = common_cache::Cache<SstDecodedMetaKey, Arc<SstMetaLoadResult>>;
2041pub(crate) type ColumnarMetaCache = Cache<DecodedKey, DecodedValue>;
2043type VectorCache = Cache<(ConcreteDataType, Value), VectorRef>;
2047type SelectorResultCache = Cache<SelectorResultKey, Arc<SelectorResultValue>>;
2049type RangeResultCacheRef = Arc<range_cache::RangeResultCache>;
2051
2052#[cfg(test)]
2053mod tests {
2054 use std::sync::Arc;
2055
2056 use api::v1::SemanticType;
2057 use api::v1::index::{BloomFilterMeta, InvertedIndexMetas};
2058 use datatypes::schema::ColumnSchema;
2059 use datatypes::vectors::Int64Vector;
2060 use parquet::file::page_index::offset_index::{OffsetIndexMetaData, PageLocation};
2061 use puffin::file_metadata::FileMetadata;
2062 use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder};
2063 use store_api::storage::ColumnId;
2064
2065 use super::*;
2066 use crate::cache::index::bloom_filter_index::Tag;
2067 use crate::cache::index::result_cache::PredicateKey;
2068 use crate::cache::test_util::{
2069 parquet_meta, sst_parquet_meta, sst_parquet_meta_with_region_metadata,
2070 };
2071 use crate::read::range_cache::{RangeScanCacheKey, ScanRequestFingerprintBuilder};
2072 use crate::read::read_columns::ReadColumns;
2073 use crate::sst::parquet::row_selection::RowGroupSelection;
2074
2075 #[test]
2076 fn test_find_shared_page_source() {
2077 let source = Arc::new(
2078 UnexpectedSnafu {
2079 reason: "page read failed",
2080 }
2081 .build(),
2082 );
2083 let direct = io::Error::other(source.clone());
2084 let nested = io::Error::other(Arc::new(io::Error::other(direct)));
2085 let retained = find_shared_page_source::<crate::error::Error>(&nested).unwrap();
2086 assert!(Arc::ptr_eq(&source, &retained));
2087 assert!(
2088 find_shared_page_source::<crate::error::Error>(&io::Error::other("untyped error"))
2089 .is_none()
2090 );
2091 }
2092
2093 #[tokio::test]
2094 async fn test_disable_cache() {
2095 let cache = CacheManager::default();
2096 assert!(cache.sst_meta_cache.is_none());
2097 assert!(cache.sst_decoded_meta_cache.is_none());
2098 assert!(cache.vector_cache.is_none());
2099 assert!(cache.page_cache.is_none());
2100
2101 let region_id = RegionId::new(1, 1);
2102 let file_id = RegionFileId::new(region_id, FileId::random());
2103 let metadata = parquet_meta();
2104 let mut metrics = MetadataCacheMetrics::default();
2105 cache.put_parquet_meta_data(file_id, metadata, None).await;
2106 assert!(
2107 cache
2108 .get_sst_meta_data(
2109 file_id,
2110 &mut metrics,
2111 Default::default(),
2112 &common_runtime::global_runtime()
2113 )
2114 .await
2115 .is_none()
2116 );
2117
2118 let value = Value::Int64(10);
2119 let vector: VectorRef = Arc::new(Int64Vector::from_slice([10, 10, 10, 10]));
2120 cache.put_repeated_vector(value.clone(), vector.clone());
2121 assert!(
2122 cache
2123 .get_repeated_vector(&ConcreteDataType::int64_datatype(), &value)
2124 .is_none()
2125 );
2126
2127 assert!(cache.page_cache.is_none());
2128
2129 assert!(cache.write_cache().is_none());
2130 }
2131
2132 #[tokio::test]
2133 async fn test_sst_meta_load_coalescing() {
2134 use std::sync::atomic::{AtomicUsize, Ordering};
2135
2136 let manager = Arc::new(CacheManager::builder().sst_meta_cache_size(2).build());
2137 let strategy = CacheStrategy::EnableAll(manager.clone());
2138 let (metadata, region) = sst_parquet_meta();
2139 let metadata = Arc::new(
2140 CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap(),
2141 );
2142 let file_id = RegionFileId::new(region.region_id, FileId::random());
2143 let calls = AtomicUsize::new(0);
2144 let gate = tokio::sync::Notify::new();
2145 let load = || async {
2146 calls.fetch_add(1, Ordering::Relaxed);
2147 gate.notified().await;
2148 Ok((metadata.clone(), true))
2149 };
2150 let mut first =
2151 Box::pin(strategy.get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, load()));
2152 let mut second =
2153 Box::pin(strategy.get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, load()));
2154 assert!(futures::poll!(&mut first).is_pending());
2155 assert!(futures::poll!(&mut second).is_pending());
2156 assert_eq!(1, calls.load(Ordering::Relaxed));
2157 gate.notify_waiters();
2158 let (first, second) = tokio::join!(first, second);
2159 let first = first.unwrap();
2160 let second = second.unwrap();
2161 assert!(Arc::ptr_eq(&first.0, &second.0));
2162 assert!(
2163 first.1 && second.1,
2164 "all waiters preserve the remote miss flag"
2165 );
2166
2167 manager
2169 .sst_decoded_meta_cache
2170 .as_ref()
2171 .unwrap()
2172 .run_pending_tasks()
2173 .await;
2174 let mut next =
2176 Box::pin(strategy.get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, load()));
2177 assert!(futures::poll!(&mut next).is_pending());
2178 assert_eq!(2, calls.load(Ordering::Relaxed));
2179 drop(next);
2180 manager
2181 .sst_decoded_meta_cache
2182 .as_ref()
2183 .unwrap()
2184 .run_pending_tasks()
2185 .await;
2186 assert_eq!(
2187 0,
2188 manager
2189 .sst_decoded_meta_cache
2190 .as_ref()
2191 .unwrap()
2192 .entry_count()
2193 );
2194 }
2195
2196 #[tokio::test]
2197 async fn test_compact_sst_meta_stronger_policy_reused_from_skip_key() {
2198 for policy in [PageIndexPolicy::Optional, PageIndexPolicy::Required] {
2199 let manager = Arc::new(
2200 CacheManager::builder()
2201 .sst_meta_cache_size(1024 * 1024)
2202 .build(),
2203 );
2204 let strategy = CacheStrategy::EnableAll(manager.clone());
2205 let (metadata, region) = sst_parquet_meta();
2206 let file_id = RegionFileId::new(region.region_id, FileId::random());
2207 let prepared = prepare_sst_meta(
2208 "test.parquet",
2209 with_offset_indexes(Arc::unwrap_or_clone(metadata)),
2210 None,
2211 policy,
2212 &common_runtime::global_runtime(),
2213 )
2214 .await
2215 .unwrap();
2216 let SstMetaPreparation::Prepared(prepared) = prepared else {
2217 panic!("valid metadata should produce a compact entry");
2218 };
2219 manager.put_compact_sst_meta(file_id, prepared.compact);
2220 let mut metrics = MetadataCacheMetrics::default();
2221 let observation = strategy
2222 .observe_sst_meta(file_id, PageIndexPolicy::Skip, &mut metrics)
2223 .await;
2224 assert!(matches!(&observation, SstMetaObservation::Compact(_)));
2225 let (decoded, _) = strategy
2226 .get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, async {
2227 let value = strategy
2228 .resolve_sst_meta(
2229 observation,
2230 file_id,
2231 &mut MetadataCacheMetrics::default(),
2232 PageIndexPolicy::Skip,
2233 )
2234 .await
2235 .expect("compact entry must decode without storage I/O");
2236 Ok((value, false))
2237 })
2238 .await
2239 .unwrap();
2240 assert!(decoded.satisfies_page_index_policy(policy));
2241 let cache = manager.sst_decoded_meta_cache.as_ref().unwrap();
2242 cache.run_pending_tasks().await;
2243 let weight = cache.weighted_size();
2244 assert_eq!(cache.entry_count(), 1);
2245
2246 manager
2248 .sst_meta_cache
2249 .as_ref()
2250 .unwrap()
2251 .invalidate(&SstMetaKey(file_id.region_id(), file_id.file_id()));
2252 for requested in [PageIndexPolicy::Optional, policy] {
2253 let mut metrics = MetadataCacheMetrics::default();
2254 let observation = strategy
2255 .observe_sst_meta(file_id, requested, &mut metrics)
2256 .await;
2257 let SstMetaObservation::Decoded(cached) = observation else {
2258 panic!("the completed Skip-key entry already satisfies this request");
2259 };
2260 assert!(Arc::ptr_eq(&decoded, &cached));
2261 assert_eq!(metrics.mem_cache_hit, 1);
2262 assert_eq!(metrics.cache_miss, 0);
2263 }
2264 cache.run_pending_tasks().await;
2265 assert_eq!(cache.entry_count(), 1);
2266 assert_eq!(cache.weighted_size(), weight);
2267 assert!(
2268 cache
2269 .get(&SstDecodedMetaKey::new(file_id, PageIndexPolicy::Optional))
2270 .await
2271 .is_none()
2272 );
2273 assert!(
2274 cache
2275 .get(&SstDecodedMetaKey::new(file_id, PageIndexPolicy::Required))
2276 .await
2277 .is_none()
2278 );
2279 }
2280 }
2281
2282 #[tokio::test]
2283 async fn test_async_sst_meta_policy_reuse_and_invalidation() {
2284 let manager = Arc::new(
2285 CacheManager::builder()
2286 .sst_meta_cache_size(1024 * 1024)
2287 .build(),
2288 );
2289 let strategy = CacheStrategy::EnableAll(manager.clone());
2290 let (parquet_metadata, region) = sst_parquet_meta();
2291 let file_id = RegionFileId::new(region.region_id, FileId::random());
2292 let policies = [
2293 PageIndexPolicy::Skip,
2294 PageIndexPolicy::Optional,
2295 PageIndexPolicy::Required,
2296 ];
2297 for loaded in policies {
2298 let metadata = Arc::new(
2299 CachedSstMeta::try_new_with_page_index_policy(
2300 "test.parquet",
2301 (*parquet_metadata).clone(),
2302 None,
2303 loaded,
2304 )
2305 .unwrap(),
2306 );
2307 let (result, miss) = strategy
2308 .get_or_load_sst_meta(file_id, loaded, async { Ok((metadata.clone(), true)) })
2309 .await
2310 .unwrap();
2311 assert!(Arc::ptr_eq(&metadata, &result));
2312 assert!(miss);
2313 for requested in policies {
2314 let mut metrics = MetadataCacheMetrics::default();
2315 let cached = strategy
2316 .observe_sst_meta(file_id, requested, &mut metrics)
2317 .await;
2318 let cached = match cached {
2319 SstMetaObservation::Decoded(value) => Some(value),
2320 _ => None,
2321 };
2322 assert_eq!(
2323 satisfies_page_index_policy(loaded, requested),
2324 cached.is_some()
2325 );
2326 if let Some(cached) = cached {
2327 assert!(Arc::ptr_eq(&metadata, &cached));
2328 assert_eq!(1, metrics.mem_cache_hit);
2329 }
2330 }
2331 manager.remove_parquet_meta_data(file_id).await;
2332 }
2333
2334 for loaded in policies {
2336 let metadata = Arc::new(
2337 CachedSstMeta::try_new_with_page_index_policy(
2338 "test.parquet",
2339 (*parquet_metadata).clone(),
2340 None,
2341 loaded,
2342 )
2343 .unwrap(),
2344 );
2345 manager.put_sst_meta_data(file_id, metadata).await;
2346 }
2347 manager.remove_parquet_meta_data(file_id).await;
2348 for requested in policies {
2349 assert!(
2350 manager
2351 .get_sst_meta_data_from_mem_cache(file_id, requested)
2352 .await
2353 .is_none()
2354 );
2355 }
2356 }
2357
2358 #[tokio::test]
2359 async fn test_sst_meta_load_cancellation_and_policy_isolation() {
2360 let strategy = CacheStrategy::EnableAll(Arc::new(
2361 CacheManager::builder().sst_meta_cache_size(2000).build(),
2362 ));
2363 let (metadata, region) = sst_parquet_meta();
2364 let metadata = Arc::new(
2365 CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap(),
2366 );
2367 let file_id = RegionFileId::new(region.region_id, FileId::random());
2368 let mut first = Box::pin(strategy.get_or_load_sst_meta(
2369 file_id,
2370 PageIndexPolicy::Skip,
2371 std::future::pending(),
2372 ));
2373 assert!(futures::poll!(&mut first).is_pending());
2374 for (id, policy) in [
2376 (file_id, PageIndexPolicy::Optional),
2377 (file_id, PageIndexPolicy::Required),
2378 (
2379 RegionFileId::new(region.region_id, FileId::random()),
2380 PageIndexPolicy::Skip,
2381 ),
2382 ] {
2383 let mut independent = Box::pin(
2384 strategy.get_or_load_sst_meta(id, policy, async { Ok((metadata.clone(), false)) }),
2385 );
2386 assert!(futures::poll!(&mut independent).is_ready());
2387 }
2388 let mut waiter = Box::pin(strategy.get_or_load_sst_meta(
2389 file_id,
2390 PageIndexPolicy::Skip,
2391 async { Ok((metadata.clone(), false)) },
2392 ));
2393 assert!(futures::poll!(&mut waiter).is_pending());
2394 drop(first);
2395 let result = tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
2396 .await
2397 .unwrap()
2398 .unwrap();
2399 assert!(Arc::ptr_eq(&metadata, &result.0));
2400 }
2401
2402 #[tokio::test]
2403 async fn test_sst_meta_load_failure_retry_and_disabled_cache() {
2404 use common_error::ext::ErrorExt;
2405
2406 let manager = Arc::new(CacheManager::builder().sst_meta_cache_size(2000).build());
2407 let strategy = CacheStrategy::EnableAll(manager);
2408 let (metadata, region) = sst_parquet_meta();
2409 let metadata = Arc::new(
2410 CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap(),
2411 );
2412 let file_id = RegionFileId::new(region.region_id, FileId::random());
2413 let gate = tokio::sync::Notify::new();
2414 let mut first = Box::pin(strategy.get_or_load_observed_sst_meta(
2415 file_id,
2416 PageIndexPolicy::Skip,
2417 async {
2418 gate.notified().await;
2419 (
2420 UnexpectedSnafu {
2421 reason: "load failed",
2422 }
2423 .fail(),
2424 SstMetaResolution {
2425 file_cache_hit: 0,
2426 cache_miss: 1,
2427 },
2428 )
2429 },
2430 ));
2431 let mut waiter = Box::pin(strategy.get_or_load_observed_sst_meta(
2432 file_id,
2433 PageIndexPolicy::Skip,
2434 async { panic!("waiter must share the failure") },
2435 ));
2436 assert!(futures::poll!(&mut first).is_pending());
2437 assert!(futures::poll!(&mut waiter).is_pending());
2438 gate.notify_waiters();
2439 let (first, first_resolution) = first.await;
2440 let (second, second_resolution) = waiter.await;
2441 for resolution in [first_resolution, second_resolution] {
2442 assert_eq!(1, resolution.cache_miss);
2443 assert_eq!(0, resolution.file_cache_hit);
2444 }
2445 let first = first.unwrap_err();
2446 let second = second.unwrap_err();
2447 assert_eq!(first.status_code(), second.status_code());
2448 match (first, second) {
2449 (
2450 crate::error::Error::LoadSstMeta { source: a, .. },
2451 crate::error::Error::LoadSstMeta { source: b, .. },
2452 ) => assert!(Arc::ptr_eq(&a, &b)),
2453 errors => panic!("unexpected errors: {errors:?}"),
2454 }
2455 assert!(
2456 strategy
2457 .get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, async {
2458 Ok((metadata.clone(), true))
2459 })
2460 .await
2461 .is_ok()
2462 );
2463
2464 for strategy in [
2465 CacheStrategy::Disabled,
2466 CacheStrategy::EnableAll(Arc::new(CacheManager::default())),
2467 CacheStrategy::EnableAll(Arc::new(
2468 CacheManager::builder().sst_meta_cache_size(1).build(),
2469 )),
2470 ] {
2471 let mut blocked = Box::pin(strategy.get_or_load_sst_meta(
2472 file_id,
2473 PageIndexPolicy::Skip,
2474 std::future::pending(),
2475 ));
2476 assert!(futures::poll!(&mut blocked).is_pending());
2477 let mut independent = Box::pin(strategy.get_or_load_sst_meta(
2478 file_id,
2479 PageIndexPolicy::Skip,
2480 async { Ok((metadata.clone(), true)) },
2481 ));
2482 assert!(futures::poll!(&mut independent).is_ready());
2483 }
2484 }
2485
2486 #[test]
2487 fn test_compact_sst_meta_bytes_follow_cache_lifetime() {
2488 use common_cache::metrics::CacheSizeMetrics;
2489 use prometheus::Opts;
2490 use prometheus::core::Collector;
2491
2492 let metrics =
2493 CacheSizeMetrics::new(Opts::new("test_compact_sst_meta_bytes", "test"), &["type"])
2494 .unwrap();
2495 let (metadata, region) = sst_parquet_meta();
2496 let SstMetaPreparation::Prepared(prepared) = prepare_sst_meta_sync(
2497 "test.parquet",
2498 Arc::unwrap_or_clone(metadata),
2499 None,
2500 PageIndexPolicy::Skip,
2501 )
2502 .unwrap() else {
2503 panic!("valid metadata should produce a compact entry");
2504 };
2505 let file_id = RegionFileId::new(region.region_id, FileId::random());
2506 let expected_bytes = f64::from(meta_cache_weight(
2507 &SstMetaKey(file_id.region_id(), file_id.file_id()),
2508 &prepared.compact,
2509 ));
2510 let make_cache = || {
2511 let manager = Arc::new(
2512 CacheManager::builder()
2513 .sst_meta_cache_size(1024 * 1024)
2514 .build_with_sst_meta_size_metric(metrics.bind(&[SST_META_TYPE])),
2515 );
2516 manager.put_compact_sst_meta(file_id, prepared.compact.clone());
2517 manager.sst_meta_cache.as_ref().unwrap().run_pending_tasks();
2518 manager
2519 };
2520 let sample = || {
2521 metrics.collect()[0].get_metric()[0]
2522 .get_gauge()
2523 .as_ref()
2524 .unwrap()
2525 .value()
2526 };
2527 let resident = make_cache();
2528 assert!(expected_bytes > 0.0);
2529 assert_eq!(expected_bytes, sample());
2530
2531 for _ in 0..3 {
2533 let transient = make_cache();
2534 assert_eq!(2.0 * expected_bytes, sample());
2535 let reader = transient.clone();
2536 drop(transient);
2537 assert_eq!(2.0 * expected_bytes, sample());
2538 drop(reader);
2539 assert_eq!(expected_bytes, sample());
2540 }
2541 drop(resident);
2542 assert_eq!(0.0, sample());
2543 }
2544
2545 #[test]
2546 fn test_sst_meta_cache_splits_capacity() {
2547 let cache = CacheManager::builder().sst_meta_cache_size(101).build();
2549 assert_eq!(
2550 50,
2551 cache.sst_decoded_meta_cache.as_ref().unwrap().capacity()
2552 );
2553 assert!(cache.columnar_meta_cache.is_none());
2554
2555 let cache = CacheManager::builder()
2557 .sst_meta_cache_size(101)
2558 .enable_columnar_meta_cache(true)
2559 .build();
2560
2561 assert_eq!(
2562 Some(51),
2563 cache
2564 .sst_meta_cache
2565 .as_ref()
2566 .unwrap()
2567 .policy()
2568 .max_capacity()
2569 );
2570 assert_eq!(
2572 25,
2573 cache.sst_decoded_meta_cache.as_ref().unwrap().capacity()
2574 );
2575 assert_eq!(
2576 Some(25),
2577 cache
2578 .columnar_meta_cache
2579 .as_ref()
2580 .unwrap()
2581 .policy()
2582 .max_capacity()
2583 );
2584 }
2585
2586 #[tokio::test]
2587 async fn test_parquet_meta_cache() {
2588 let cache = CacheManager::builder().sst_meta_cache_size(2000).build();
2589 let mut metrics = MetadataCacheMetrics::default();
2590 let region_id = RegionId::new(1, 1);
2591 let file_id = RegionFileId::new(region_id, FileId::random());
2592 assert!(
2593 cache
2594 .get_sst_meta_data(
2595 file_id,
2596 &mut metrics,
2597 Default::default(),
2598 &common_runtime::global_runtime()
2599 )
2600 .await
2601 .is_none()
2602 );
2603 let (metadata, region_metadata) = sst_parquet_meta();
2604 cache.put_parquet_meta_data(file_id, metadata, None).await;
2605 let cached = cache
2606 .get_sst_meta_data(
2607 file_id,
2608 &mut metrics,
2609 Default::default(),
2610 &common_runtime::global_runtime(),
2611 )
2612 .await
2613 .unwrap();
2614 assert_eq!(region_metadata, cached.region_metadata());
2615 assert!(
2616 cached
2617 .parquet_metadata()
2618 .file_metadata()
2619 .key_value_metadata()
2620 .is_none_or(|key_values| {
2621 key_values
2622 .iter()
2623 .all(|key_value| key_value.key != PARQUET_METADATA_KEY)
2624 })
2625 );
2626 cache.remove_parquet_meta_data(file_id).await;
2627 assert!(
2628 cache
2629 .get_sst_meta_data(
2630 file_id,
2631 &mut metrics,
2632 Default::default(),
2633 &common_runtime::global_runtime()
2634 )
2635 .await
2636 .is_none()
2637 );
2638 }
2639
2640 #[tokio::test]
2641 async fn test_parquet_meta_cache_with_provided_region_metadata() {
2642 let cache = CacheManager::builder().sst_meta_cache_size(2000).build();
2643 let mut metrics = MetadataCacheMetrics::default();
2644 let region_id = RegionId::new(1, 1);
2645 let file_id = RegionFileId::new(region_id, FileId::random());
2646 let (metadata, region_metadata) = sst_parquet_meta();
2647
2648 cache
2649 .put_parquet_meta_data(file_id, metadata, Some(region_metadata.clone()))
2650 .await;
2651
2652 let cached = cache
2653 .get_sst_meta_data(
2654 file_id,
2655 &mut metrics,
2656 Default::default(),
2657 &common_runtime::global_runtime(),
2658 )
2659 .await
2660 .unwrap();
2661 assert!(Arc::ptr_eq(®ion_metadata, &cached.region_metadata()));
2662 }
2663
2664 fn with_offset_indexes(metadata: ParquetMetaData) -> ParquetMetaData {
2665 let offset_indexes = metadata
2666 .row_groups()
2667 .iter()
2668 .map(|row_group| {
2669 row_group
2670 .columns()
2671 .iter()
2672 .map(|_| OffsetIndexMetaData {
2673 page_locations: vec![PageLocation {
2674 offset: 42,
2675 compressed_page_size: 128,
2676 first_row_index: 0,
2677 }],
2678 unencoded_byte_array_data_bytes: None,
2679 })
2680 .collect()
2681 })
2682 .collect();
2683 metadata
2684 .into_builder()
2685 .set_offset_index(Some(offset_indexes))
2686 .build()
2687 }
2688
2689 #[tokio::test]
2690 async fn test_compact_sst_meta_round_trip() {
2691 let cache = CacheManager::builder()
2692 .sst_meta_cache_size(1024 * 1024)
2693 .build();
2694 let region_id = RegionId::new(1, 1);
2695 let file_id = RegionFileId::new(region_id, FileId::random());
2696 let (metadata, expected_region_metadata) = sst_parquet_meta();
2697 let metadata = Arc::unwrap_or_clone(metadata);
2698 let metadata = with_offset_indexes(metadata);
2699 let expected_rows = metadata.file_metadata().num_rows();
2700 let prepared = prepare_sst_meta(
2701 "test.parquet",
2702 metadata,
2703 None,
2704 PageIndexPolicy::Required,
2705 &common_runtime::global_runtime(),
2706 )
2707 .await
2708 .unwrap();
2709 let SstMetaPreparation::Prepared(prepared) = prepared else {
2710 panic!("valid metadata should produce a compact cache entry");
2711 };
2712
2713 cache.put_compact_sst_meta(file_id, prepared.compact);
2715 let mut metrics = MetadataCacheMetrics::default();
2716 let cached = cache
2717 .get_sst_meta_data(
2718 file_id,
2719 &mut metrics,
2720 PageIndexPolicy::Required,
2721 &common_runtime::global_runtime(),
2722 )
2723 .await
2724 .unwrap();
2725
2726 assert_eq!(1, metrics.mem_cache_hit);
2727 assert_eq!(0, metrics.cache_miss);
2728 assert_eq!(
2729 expected_rows,
2730 cached.parquet_metadata.file_metadata().num_rows()
2731 );
2732 assert!(cached.parquet_metadata.offset_index().is_some());
2733 assert!(
2734 cache
2735 .sst_decoded_meta_cache
2736 .as_ref()
2737 .unwrap()
2738 .get(&SstDecodedMetaKey::new(file_id, PageIndexPolicy::Required))
2739 .await
2740 .is_none(),
2741 "compact inspection must not fill the decoded reservation"
2742 );
2743 assert_eq!(expected_region_metadata, cached.region_metadata());
2744 assert!(
2745 cached
2746 .parquet_metadata
2747 .file_metadata()
2748 .key_value_metadata()
2749 .is_none_or(|key_values| key_values
2750 .iter()
2751 .all(|key_value| key_value.key != PARQUET_METADATA_KEY))
2752 );
2753 }
2754
2755 #[tokio::test]
2756 async fn test_cache_encoding_failure_preserves_decoded_metadata() {
2757 let (metadata, expected_region_metadata) = sst_parquet_meta();
2758 let expected_rows = metadata.file_metadata().num_rows();
2759 let cache_encoding: Result<(Bytes, usize)> = Err(UnexpectedSnafu {
2760 reason: "injected compact metadata encoding failure",
2761 }
2762 .build());
2763
2764 let preparation = finish_sst_meta_preparation(
2765 "test.parquet",
2766 Arc::unwrap_or_clone(metadata),
2767 None,
2768 PageIndexPolicy::Skip,
2769 cache_encoding,
2770 )
2771 .unwrap();
2772 let SstMetaPreparation::DecodedOnly {
2773 decoded,
2774 encoding_error,
2775 } = preparation
2776 else {
2777 panic!("cache encoding failure should return decoded-only metadata");
2778 };
2779
2780 assert_eq!(
2781 expected_rows,
2782 decoded.parquet_metadata.file_metadata().num_rows()
2783 );
2784 assert_eq!(expected_region_metadata, decoded.region_metadata());
2785 assert!(
2786 encoding_error
2787 .to_string()
2788 .contains("injected compact metadata encoding failure")
2789 );
2790 let file_id = RegionFileId::new(expected_region_metadata.region_id, FileId::random());
2791 let manager = Arc::new(
2792 CacheManager::builder()
2793 .sst_meta_cache_size(1024 * 1024)
2794 .build(),
2795 );
2796 let strategy = CacheStrategy::EnableAll(manager.clone());
2797 let (shared, miss) = strategy
2798 .get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, async {
2799 Ok((decoded.clone(), true))
2800 })
2801 .await
2802 .unwrap();
2803 assert!(miss);
2804 assert!(Arc::ptr_eq(&decoded, &shared));
2805 assert!(
2806 manager
2807 .get_compact_sst_meta(&SstMetaKey(file_id.region_id(), file_id.file_id()))
2808 .is_none()
2809 );
2810 let mut metrics = MetadataCacheMetrics::default();
2811 let SstMetaObservation::Decoded(cached) = strategy
2812 .observe_sst_meta(file_id, PageIndexPolicy::Skip, &mut metrics)
2813 .await
2814 else {
2815 panic!("decoded fallback must be retained without a compact entry");
2816 };
2817 assert!(Arc::ptr_eq(&decoded, &cached));
2818 assert_eq!(1, metrics.mem_cache_hit);
2819 assert_eq!(0, metrics.cache_miss);
2820 }
2821
2822 #[tokio::test]
2823 async fn test_parquet_meta_cache_respects_page_index_policy() {
2824 let cache = CacheManager::builder().sst_meta_cache_size(2000).build();
2825 let region_id = RegionId::new(1, 1);
2826 let file_id = RegionFileId::new(region_id, FileId::random());
2827 let (metadata, _) = sst_parquet_meta();
2828
2829 let skip_metadata = Arc::new(
2830 CachedSstMeta::try_new_with_page_index_policy(
2831 "test.parquet",
2832 Arc::unwrap_or_clone(metadata.clone()),
2833 None,
2834 PageIndexPolicy::Skip,
2835 )
2836 .unwrap(),
2837 );
2838 cache.put_sst_meta_data(file_id, skip_metadata).await;
2839
2840 let mut metrics = MetadataCacheMetrics::default();
2841 assert!(
2842 cache
2843 .get_sst_meta_data(
2844 file_id,
2845 &mut metrics,
2846 PageIndexPolicy::Optional,
2847 &common_runtime::global_runtime()
2848 )
2849 .await
2850 .is_none()
2851 );
2852 assert_eq!(1, metrics.cache_miss);
2853
2854 let optional_metadata = Arc::new(
2855 CachedSstMeta::try_new_with_page_index_policy(
2856 "test.parquet",
2857 Arc::unwrap_or_clone(metadata),
2858 None,
2859 PageIndexPolicy::Optional,
2860 )
2861 .unwrap(),
2862 );
2863 cache.put_sst_meta_data(file_id, optional_metadata).await;
2864
2865 let mut metrics = MetadataCacheMetrics::default();
2866 assert!(
2867 cache
2868 .get_sst_meta_data(
2869 file_id,
2870 &mut metrics,
2871 PageIndexPolicy::Optional,
2872 &common_runtime::global_runtime()
2873 )
2874 .await
2875 .is_some()
2876 );
2877 assert_eq!(1, metrics.mem_cache_hit);
2878
2879 let mut metrics = MetadataCacheMetrics::default();
2880 assert!(
2881 cache
2882 .get_sst_meta_data(
2883 file_id,
2884 &mut metrics,
2885 PageIndexPolicy::Skip,
2886 &common_runtime::global_runtime()
2887 )
2888 .await
2889 .is_some()
2890 );
2891 assert_eq!(1, metrics.mem_cache_hit);
2892 }
2893
2894 #[test]
2895 fn test_decoded_meta_cache_weight_accounts_for_region_metadata() {
2896 let region_metadata = Arc::new(wide_region_metadata(128));
2897 let json_len = region_metadata.to_json().unwrap().len();
2898 let metadata = sst_parquet_meta_with_region_metadata(region_metadata.clone());
2899 let cached = Arc::new(
2900 CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap(),
2901 );
2902 let key = SstDecodedMetaKey::new(
2903 RegionFileId::new(region_metadata.region_id, FileId::random()),
2904 PageIndexPolicy::Skip,
2905 );
2906 let value = SstMetaLoadResult::new(cached.clone(), false, SstMetaResolution::default());
2907
2908 assert!(cached.region_metadata_weight > json_len);
2909 assert_eq!(
2910 decoded_meta_cache_weight(&key, &value) as usize,
2911 size_of_val(&key)
2912 + size_of::<SstMetaLoadResult>()
2913 + cached.parquet_metadata_size
2914 + cached.region_metadata_weight
2915 );
2916 assert_eq!(
2917 cached.parquet_metadata_size,
2918 parquet_meta_size(&cached.parquet_metadata)
2919 );
2920 }
2921
2922 #[test]
2923 fn test_decoded_meta_cache_weight_saturates_on_overflow() {
2924 let region_metadata = Arc::new(wide_region_metadata(1));
2925 let metadata = sst_parquet_meta_with_region_metadata(region_metadata.clone());
2926 let mut cached =
2927 CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap();
2928 cached.region_metadata_weight = u32::MAX as usize + 1;
2929 let cached = Arc::new(cached);
2930 let key = SstDecodedMetaKey::new(
2931 RegionFileId::new(region_metadata.region_id, FileId::random()),
2932 PageIndexPolicy::Skip,
2933 );
2934 let value = SstMetaLoadResult::new(cached.clone(), false, SstMetaResolution::default());
2935
2936 assert_eq!(u32::MAX, decoded_meta_cache_weight(&key, &value));
2937 }
2938
2939 #[test]
2940 fn test_repeated_vector_cache() {
2941 let cache = CacheManager::builder().vector_cache_size(4096).build();
2942 let value = Value::Int64(10);
2943 assert!(
2944 cache
2945 .get_repeated_vector(&ConcreteDataType::int64_datatype(), &value)
2946 .is_none()
2947 );
2948 let vector: VectorRef = Arc::new(Int64Vector::from_slice([10, 10, 10, 10]));
2949 cache.put_repeated_vector(value.clone(), vector.clone());
2950 let cached = cache
2951 .get_repeated_vector(&ConcreteDataType::int64_datatype(), &value)
2952 .unwrap();
2953 assert_eq!(vector, cached);
2954 }
2955
2956 #[test]
2957 fn test_selector_result_cache() {
2958 let cache = CacheManager::builder()
2959 .selector_result_cache_size(1000)
2960 .build();
2961 let file_id = FileId::random();
2962 let key = SelectorResultKey {
2963 file_id,
2964 row_group_idx: 0,
2965 selector: TimeSeriesRowSelector::LastRow { after_merge: false },
2966 };
2967 assert!(cache.get_selector_result(&key).is_none());
2968 let result = Arc::new(SelectorResultValue::new(
2969 Vec::new(),
2970 ParquetReadColumns::from_deduped(Vec::new()),
2971 ));
2972 cache.put_selector_result(key, result);
2973 assert!(cache.get_selector_result(&key).is_some());
2974 }
2975
2976 #[test]
2977 fn test_prefilter_result_cache() {
2978 let disabled = CacheManager::builder().build();
2979 let file_id = FileId::random();
2980 let key = PrefilterKey::new(
2981 file_id,
2982 0,
2983 None,
2984 1,
2985 SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()]),
2986 );
2987 let selection = Arc::new(BooleanBuffer::new_set(3));
2988
2989 disabled.put_prefilter_result(key.clone(), selection.clone());
2990 assert!(disabled.get_prefilter_result(&key).is_none());
2991
2992 let cache = Arc::new(
2993 CacheManager::builder()
2994 .prefilter_result_cache_size(1000)
2995 .build(),
2996 );
2997 assert!(cache.get_prefilter_result(&key).is_none());
2998 cache.put_prefilter_result(key.clone(), selection.clone());
2999 assert_eq!(
3000 cache.get_prefilter_result(&key).unwrap().as_ref(),
3001 selection.as_ref()
3002 );
3003
3004 let enable_all = CacheStrategy::EnableAll(cache.clone());
3005 assert!(enable_all.get_prefilter_result(&key).is_some());
3006
3007 let compaction = CacheStrategy::Compaction(cache.clone());
3008 assert!(compaction.get_prefilter_result(&key).is_none());
3009 compaction.put_prefilter_result(key.clone(), selection.clone());
3010 assert!(cache.get_prefilter_result(&key).is_some());
3011
3012 let disabled_strategy = CacheStrategy::Disabled;
3013 assert!(disabled_strategy.get_prefilter_result(&key).is_none());
3014 disabled_strategy.put_prefilter_result(key.clone(), selection);
3015 assert!(cache.get_prefilter_result(&key).is_some());
3016 }
3017
3018 #[test]
3019 fn test_prefilter_key_distinguishes_dimensions() {
3020 let file_id = FileId::random();
3021 let row_selection = RowSelection::from(vec![RowSelector::skip(1), RowSelector::select(3)]);
3022 let other_row_selection =
3023 RowSelection::from(vec![RowSelector::skip(2), RowSelector::select(2)]);
3024 let row_selection = PrefilterKey::row_selection_snapshot(Some(&row_selection));
3025 let other_row_selection = PrefilterKey::row_selection_snapshot(Some(&other_row_selection));
3026 let base = PrefilterKey::new(
3027 file_id,
3028 0,
3029 row_selection.clone(),
3030 1,
3031 SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()]),
3032 );
3033
3034 assert_ne!(
3035 base,
3036 PrefilterKey::new(
3037 FileId::random(),
3038 0,
3039 row_selection.clone(),
3040 1,
3041 SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()])
3042 )
3043 );
3044 assert_ne!(
3045 base,
3046 PrefilterKey::new(
3047 file_id,
3048 1,
3049 row_selection.clone(),
3050 1,
3051 SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()])
3052 )
3053 );
3054 assert_ne!(
3055 base,
3056 PrefilterKey::new(
3057 file_id,
3058 0,
3059 other_row_selection,
3060 1,
3061 SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()])
3062 )
3063 );
3064 assert_ne!(
3065 base,
3066 PrefilterKey::new(
3067 file_id,
3068 0,
3069 row_selection.clone(),
3070 1,
3071 SmallVec::from_vec(vec!["tag_0 IN ([b])".to_string()])
3072 )
3073 );
3074 assert_ne!(
3075 base,
3076 PrefilterKey::new(
3077 file_id,
3078 0,
3079 row_selection.clone(),
3080 2,
3081 SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()])
3082 )
3083 );
3084 let pk_group = PrefilterKey::new(
3085 file_id,
3086 0,
3087 row_selection,
3088 1,
3089 SmallVec::from_vec(vec![
3090 "tag_0 IN ([a])".to_string(),
3091 "tag_1 IN ([x])".to_string(),
3092 ]),
3093 );
3094 assert_ne!(base, pk_group);
3095 }
3096
3097 #[test]
3098 fn test_range_result_cache() {
3099 let cache = Arc::new(
3100 CacheManager::builder()
3101 .range_result_cache_size(1024 * 1024)
3102 .build(),
3103 );
3104
3105 let key = RangeScanCacheKey {
3106 region_id: RegionId::new(1, 1),
3107 row_groups: vec![(FileId::random(), 0)],
3108 scan: ScanRequestFingerprintBuilder {
3109 read_columns: ReadColumns::new(std::iter::empty()),
3110 read_column_types: vec![],
3111 filters: vec!["tag_0 = 1".to_string()],
3112 time_filters: vec![],
3113 series_row_selector: None,
3114 append_mode: false,
3115 filter_deleted: true,
3116 merge_mode: crate::region::options::MergeMode::LastRow,
3117 sequence_range: None,
3118 partition_expr_version: 0,
3119 }
3120 .build(),
3121 };
3122 let value = Arc::new(RangeScanCacheValue::empty());
3123
3124 assert!(cache.get_range_result(&key).is_none());
3125 cache.put_range_result(key.clone(), value.clone());
3126 assert!(cache.get_range_result(&key).is_some());
3127
3128 let enable_all = CacheStrategy::EnableAll(cache.clone());
3129 assert!(enable_all.get_range_result(&key).is_some());
3130
3131 let compaction = CacheStrategy::Compaction(cache.clone());
3132 assert!(compaction.get_range_result(&key).is_none());
3133 compaction.put_range_result(key.clone(), value.clone());
3134 assert!(cache.get_range_result(&key).is_some());
3135
3136 let disabled = CacheStrategy::Disabled;
3137 assert!(disabled.get_range_result(&key).is_none());
3138 disabled.put_range_result(key.clone(), value);
3139 assert!(cache.get_range_result(&key).is_some());
3140 }
3141
3142 #[test]
3143 fn test_range_result_cache_size_configures_limiter() {
3144 let cache_size = 3 * 1024_u64;
3145 let cache = CacheManager::builder()
3146 .range_result_cache_size(cache_size)
3147 .build();
3148
3149 assert_eq!(cache.range_result_cache_size(), cache_size as usize);
3150 assert_eq!(
3151 cache
3152 .range_result_cache
3153 .as_ref()
3154 .unwrap()
3155 .concat_memory()
3156 .permit_bytes(),
3157 RANGE_RESULT_CONCAT_MEMORY_PERMIT.as_bytes() as usize
3158 );
3159 assert_eq!(
3160 cache
3161 .range_result_cache
3162 .as_ref()
3163 .unwrap()
3164 .concat_memory()
3165 .available_permits(),
3166 (cache_size as usize).div_ceil(RANGE_RESULT_CONCAT_MEMORY_PERMIT.as_bytes() as usize)
3167 );
3168 }
3169
3170 #[tokio::test]
3171 async fn test_range_concat_memory_rejects_oversized_request() {
3172 let limiter = RangeResultMemoryLimiter::new(2 * 1024, 1024);
3173 assert_eq!(limiter.available_permits(), 2);
3174
3175 let err = limiter.acquire(10 * 1024).await.unwrap_err();
3176 assert!(
3177 err.to_string().contains("exceeds limiter capacity"),
3178 "unexpected error: {err}"
3179 );
3180 assert_eq!(limiter.available_permits(), 2);
3181 }
3182
3183 #[tokio::test]
3184 async fn test_range_concat_memory_allows_request_up_to_capacity() {
3185 let limiter = RangeResultMemoryLimiter::new(2 * 1024, 1024);
3186 let permit = limiter.acquire(2 * 1024).await.unwrap();
3187 assert_eq!(limiter.available_permits(), 0);
3188 drop(permit);
3189 assert_eq!(limiter.available_permits(), 2);
3190 }
3191
3192 #[rstest::rstest]
3193 #[case::empty_result(0, Some(1))]
3194 #[case::rounded_result(1025, Some(2))]
3195 #[case::exhausted(4096, None)]
3196 #[case::oversized(usize::MAX, None)]
3197 #[test]
3198 fn test_range_result_memory_reservation_is_nonblocking(
3199 #[case] result_size: usize,
3200 #[case] expected_permits: Option<usize>,
3201 ) {
3202 let limiter = RangeResultMemoryLimiter::new(4096, 1024);
3203 let blocker = limiter.try_acquire(1024).unwrap();
3204 let result_permit = limiter.try_acquire(result_size);
3205 assert_eq!(
3206 result_permit.as_ref().map(|permit| permit.num_permits()),
3207 expected_permits
3208 );
3209 assert_eq!(
3210 limiter.available_permits(),
3211 3 - expected_permits.unwrap_or(0)
3212 );
3213 drop(result_permit);
3214 assert_eq!(limiter.available_permits(), 3);
3215 drop(blocker);
3216 assert_eq!(limiter.available_permits(), 4);
3217 }
3218
3219 #[tokio::test]
3220 async fn test_evict_puffin_cache_clears_all_entries() {
3221 use std::collections::{BTreeMap, HashMap};
3222
3223 use puffin::puffin_manager::file_key::{PuffinFileKey, PuffinStorageScope};
3224
3225 let region_id = RegionId::new(1, 1);
3226 let index_id = RegionIndexId::new(RegionFileId::new(region_id, FileId::random()), 0);
3227 let file_key = PuffinFileKey::new(
3228 Arc::new(PuffinStorageScope::new(
3229 "memory".into(),
3230 "cache-test".into(),
3231 "/".into(),
3232 )),
3233 "table/index.puffin".into(),
3234 index_id,
3235 );
3236 let metadata_key = file_key.clone().map_handle(|id| id.to_string());
3237 let other_scope_key = PuffinFileKey::new(
3238 Arc::new(PuffinStorageScope::new(
3239 "memory".into(),
3240 "other-store".into(),
3241 "/".into(),
3242 )),
3243 "table/index.puffin".into(),
3244 index_id.to_string(),
3245 );
3246 let newer_metadata_key = file_key.clone().map_handle(|mut id| {
3247 id.version += 1;
3248 id.to_string()
3249 });
3250 let column_id: ColumnId = 1;
3251
3252 let index_metadata_size = file_key.estimated_size()
3255 + ::index::target::IndexTarget::ColumnId(column_id).approx_mem_size()
3256 + std::mem::size_of::<Tag>()
3257 + std::mem::size_of::<BloomFilterMeta>()
3258 + std::mem::size_of::<InvertedIndexMetas>();
3259 let puffin_metadata_size: usize = [&metadata_key, &other_scope_key, &newer_metadata_key]
3260 .into_iter()
3261 .map(|key| {
3262 key.estimated_size()
3263 + key.handle().capacity()
3264 + FileMetadata {
3265 blobs: Vec::new(),
3266 properties: HashMap::new(),
3267 }
3268 .memory_usage()
3269 })
3270 .sum();
3271 let cache = Arc::new(
3272 CacheManager::builder()
3273 .index_metadata_size(index_metadata_size as u64)
3274 .index_content_size(128)
3275 .index_content_page_size(64)
3276 .index_result_cache_size(128)
3277 .puffin_metadata_size(puffin_metadata_size as u64)
3278 .build(),
3279 );
3280
3281 let bloom_cache = cache.bloom_filter_index_cache().unwrap().clone();
3282 let inverted_cache = cache.inverted_index_cache().unwrap().clone();
3283 let result_cache = cache.index_result_cache().unwrap();
3284 let puffin_metadata_cache = cache.puffin_metadata_cache().unwrap().clone();
3285
3286 let bloom_key = (
3287 file_key.clone(),
3288 Arc::new(::index::target::IndexTarget::ColumnId(column_id)),
3289 Tag::Skipping,
3290 );
3291 bloom_cache
3292 .put_metadata(bloom_key.clone(), Arc::new(BloomFilterMeta::default()))
3293 .await;
3294 inverted_cache
3295 .put_metadata(file_key.clone(), Arc::new(InvertedIndexMetas::default()))
3296 .await;
3297 let predicate = PredicateKey::new_bloom(Arc::new(BTreeMap::new()));
3298 let selection = Arc::new(RowGroupSelection::default());
3299 result_cache.put(predicate.clone(), index_id.file_id(), selection);
3300 let metadata = Arc::new(FileMetadata {
3301 blobs: Vec::new(),
3302 properties: HashMap::new(),
3303 });
3304 puffin_metadata_cache
3305 .put_metadata(metadata_key.clone(), metadata)
3306 .await;
3307
3308 assert!(bloom_cache.get_metadata(bloom_key.clone()).await.is_some());
3309 assert!(
3310 inverted_cache
3311 .get_metadata(file_key.clone())
3312 .await
3313 .is_some()
3314 );
3315 assert!(result_cache.get(&predicate, index_id.file_id()).is_some());
3316 assert!(
3317 puffin_metadata_cache
3318 .get_metadata(&metadata_key)
3319 .await
3320 .is_some()
3321 );
3322
3323 for key in [&other_scope_key, &newer_metadata_key] {
3326 puffin_metadata_cache
3327 .put_metadata(
3328 key.clone(),
3329 Arc::new(FileMetadata {
3330 blobs: Vec::new(),
3331 properties: HashMap::new(),
3332 }),
3333 )
3334 .await;
3335 assert!(puffin_metadata_cache.get_metadata(key).await.is_some());
3336 }
3337 cache.evict_puffin_cache(index_id).await;
3338 assert!(
3339 puffin_metadata_cache
3340 .get_metadata(&other_scope_key)
3341 .await
3342 .is_none()
3343 );
3344 assert!(
3345 puffin_metadata_cache
3346 .get_metadata(&newer_metadata_key)
3347 .await
3348 .is_some()
3349 );
3350
3351 assert!(bloom_cache.get_metadata(bloom_key.clone()).await.is_none());
3352 assert!(
3353 inverted_cache
3354 .get_metadata(file_key.clone())
3355 .await
3356 .is_none()
3357 );
3358 assert!(result_cache.get(&predicate, index_id.file_id()).is_none());
3359 assert!(
3360 puffin_metadata_cache
3361 .get_metadata(&metadata_key)
3362 .await
3363 .is_none()
3364 );
3365
3366 bloom_cache
3368 .put_metadata(bloom_key.clone(), Arc::new(BloomFilterMeta::default()))
3369 .await;
3370 inverted_cache
3371 .put_metadata(file_key.clone(), Arc::new(InvertedIndexMetas::default()))
3372 .await;
3373 result_cache.put(
3374 predicate.clone(),
3375 index_id.file_id(),
3376 Arc::new(RowGroupSelection::default()),
3377 );
3378 puffin_metadata_cache
3379 .put_metadata(
3380 metadata_key.clone(),
3381 Arc::new(FileMetadata {
3382 blobs: Vec::new(),
3383 properties: HashMap::new(),
3384 }),
3385 )
3386 .await;
3387
3388 let strategy = CacheStrategy::EnableAll(cache.clone());
3389 strategy.evict_puffin_cache(index_id).await;
3390
3391 assert!(bloom_cache.get_metadata(bloom_key.clone()).await.is_none());
3392 assert!(
3393 inverted_cache
3394 .get_metadata(file_key.clone())
3395 .await
3396 .is_none()
3397 );
3398 assert!(result_cache.get(&predicate, index_id.file_id()).is_none());
3399 assert!(
3400 puffin_metadata_cache
3401 .get_metadata(&metadata_key)
3402 .await
3403 .is_none()
3404 );
3405 }
3406
3407 fn wide_region_metadata(column_count: u32) -> RegionMetadata {
3408 let region_id = RegionId::new(1024, 7);
3409 let mut builder = RegionMetadataBuilder::new(region_id);
3410 let mut primary_key = Vec::new();
3411
3412 for column_id in 0..column_count {
3413 let semantic_type = if column_id < 32 {
3414 primary_key.push(column_id);
3415 SemanticType::Tag
3416 } else {
3417 SemanticType::Field
3418 };
3419 let mut column_schema = ColumnSchema::new(
3420 format!("wide_column_{column_id}"),
3421 ConcreteDataType::string_datatype(),
3422 true,
3423 );
3424 column_schema
3425 .mut_metadata()
3426 .insert(format!("cache_key_{column_id}"), "cache_value".repeat(4));
3427 builder.push_column_metadata(ColumnMetadata {
3428 column_schema,
3429 semantic_type,
3430 column_id,
3431 });
3432 }
3433
3434 builder.push_column_metadata(ColumnMetadata {
3435 column_schema: ColumnSchema::new(
3436 "ts",
3437 ConcreteDataType::timestamp_millisecond_datatype(),
3438 false,
3439 ),
3440 semantic_type: SemanticType::Timestamp,
3441 column_id: column_count,
3442 });
3443 builder.primary_key(primary_key);
3444
3445 builder.build().unwrap()
3446 }
3447}