Skip to main content

mito2/
cache.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Cache for the engine.
16
17pub(crate) mod cache_size;
18pub(crate) mod columnar;
19
20pub(crate) mod file_cache;
21pub(crate) mod index;
22pub(crate) mod manifest_cache;
23pub(crate) mod parquet_page;
24pub(crate) mod range_cache;
25#[cfg(test)]
26pub(crate) mod test_util;
27pub(crate) mod write_cache;
28
29use std::sync::{Arc, Weak};
30use std::{io, mem};
31
32use bytes::Bytes;
33use common_base::readable_size::ReadableSize;
34use common_cache::metrics::{CacheSizeMetric, CacheSizeRegistration};
35use common_datasource::compression::CompressionType;
36use common_runtime::Runtime;
37use common_runtime::runtime::RuntimeTrait;
38use common_telemetry::warn;
39use datatypes::arrow::buffer::BooleanBuffer;
40use datatypes::arrow::record_batch::RecordBatch;
41use datatypes::types::json_type::JsonNativeType;
42use datatypes::value::Value;
43use datatypes::vectors::VectorRef;
44use index::bloom_filter_index::{BloomFilterIndexCache, BloomFilterIndexCacheRef};
45use index::result_cache::IndexResultCache;
46use moka::notification::RemovalCause;
47use moka::sync::Cache;
48use object_store::ObjectStore;
49use parquet::arrow::arrow_reader::{RowSelection, RowSelector};
50use parquet::file::metadata::{
51    FileMetaData, PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader, ParquetMetaDataWriter,
52};
53use puffin::puffin_manager::cache::{PuffinMetadataCache, PuffinMetadataCacheRef};
54use smallvec::SmallVec;
55use snafu::{OptionExt, ResultExt};
56use store_api::metadata::{RegionMetadata, RegionMetadataRef};
57use store_api::storage::{ColumnId, ConcreteDataType, FileId, RegionId, TimeSeriesRowSelector};
58pub use write_cache::{WriteCacheUploadStoreWrapper, WriteCacheUploadStoreWrapperRef};
59
60use crate::cache::cache_size::parquet_meta_size;
61use crate::cache::columnar::{DecodedKey, DecodedValue, columnar_meta_weight};
62use crate::cache::file_cache::{FileType, IndexKey};
63use crate::cache::index::inverted_index::{InvertedIndexCache, InvertedIndexCacheRef};
64use crate::cache::parquet_page::ParquetPageCache;
65use crate::cache::range_cache::RangeScanCacheValue;
66use crate::cache::write_cache::WriteCacheRef;
67use crate::error::{
68    CompressObjectSnafu, DecompressObjectSnafu, InvalidMetadataSnafu, InvalidParquetSnafu,
69    JoinSnafu, ReadParquetSnafu, Result, UnexpectedSnafu, WriteParquetSnafu,
70};
71use crate::memtable::record_batch_estimated_size;
72use crate::metrics::{CACHE_BYTES, CACHE_EVICTION, CACHE_HIT, CACHE_MISS};
73use crate::read::Batch;
74use crate::read::range_cache::RangeScanCacheKey;
75use crate::read::read_columns::JsonTargetTypes;
76use crate::sst::file::{RegionFileId, RegionIndexId};
77use crate::sst::parquet::PARQUET_METADATA_KEY;
78use crate::sst::parquet::read_columns::ParquetReadColumns;
79use crate::sst::parquet::reader::MetadataCacheMetrics;
80
81/// Finds a typed shared error through the range-reader I/O wrappers.
82pub(crate) fn find_shared_page_source<E: std::error::Error + Send + Sync + 'static>(
83    mut error: &io::Error,
84) -> Option<Arc<E>> {
85    loop {
86        let source = error.get_ref()?;
87        if let Some(source) = source.downcast_ref::<Arc<E>>() {
88            return Some(source.clone());
89        }
90        if let Some(nested) = source.downcast_ref::<Arc<io::Error>>() {
91            error = nested;
92        } else {
93            error = source.downcast_ref::<io::Error>()?;
94        }
95    }
96}
97
98/// Metrics type key for sst meta.
99const SST_META_TYPE: &str = "sst_meta";
100/// Metrics type key for the optional decoded SST metadata acceleration tier.
101const SST_META_DECODED_TYPE: &str = "sst_meta_decoded";
102/// Metrics type key for vector.
103const VECTOR_TYPE: &str = "vector";
104/// Metrics type key for files on the local store.
105const FILE_TYPE: &str = "file";
106/// Metrics type key for index files (puffin) on the local store.
107const INDEX_TYPE: &str = "index";
108/// Metrics type key for selector result cache.
109const SELECTOR_RESULT_TYPE: &str = "selector_result";
110/// Metrics type key for range scan result cache.
111const RANGE_RESULT_TYPE: &str = "range_result";
112/// Metrics type key for prefilter result cache.
113const PREFILTER_RESULT_TYPE: &str = "prefilter_result";
114const RANGE_RESULT_CONCAT_MEMORY_LIMIT: ReadableSize = ReadableSize::mb(512);
115const RANGE_RESULT_CONCAT_MEMORY_PERMIT: ReadableSize = ReadableSize::kb(1);
116
117#[derive(Debug)]
118pub(crate) struct RangeResultMemoryLimiter {
119    semaphore: Arc<tokio::sync::Semaphore>,
120    permit_bytes: usize,
121    total_permits: usize,
122    /// Number of acquisitions that found too few permits and parked. Test-only
123    /// signal to synchronize with a caller waiting inside [Self::acquire].
124    #[cfg(test)]
125    waited_acquires: std::sync::atomic::AtomicUsize,
126}
127
128impl Default for RangeResultMemoryLimiter {
129    fn default() -> Self {
130        Self::new(
131            RANGE_RESULT_CONCAT_MEMORY_LIMIT.as_bytes() as usize,
132            RANGE_RESULT_CONCAT_MEMORY_PERMIT.as_bytes() as usize,
133        )
134    }
135}
136
137impl RangeResultMemoryLimiter {
138    pub(crate) fn new(limit_bytes: usize, permit_bytes: usize) -> Self {
139        let permit_bytes = permit_bytes.max(1);
140        let total_permits = limit_bytes
141            .div_ceil(permit_bytes)
142            .clamp(1, tokio::sync::Semaphore::MAX_PERMITS);
143        Self {
144            semaphore: Arc::new(tokio::sync::Semaphore::new(total_permits)),
145            permit_bytes,
146            total_permits,
147            #[cfg(test)]
148            waited_acquires: std::sync::atomic::AtomicUsize::new(0),
149        }
150    }
151
152    #[cfg(test)]
153    pub(crate) fn permit_bytes(&self) -> usize {
154        self.permit_bytes
155    }
156
157    #[cfg(test)]
158    pub(crate) fn waited_acquires(&self) -> usize {
159        self.waited_acquires
160            .load(std::sync::atomic::Ordering::Acquire)
161    }
162
163    #[cfg(test)]
164    pub(crate) fn available_permits(&self) -> usize {
165        self.semaphore.available_permits()
166    }
167
168    /// Never wait while holding a partial load: several partial loads could
169    /// otherwise exhaust the budget and wait on each other indefinitely.
170    pub(crate) fn try_acquire(&self, bytes: usize) -> Option<tokio::sync::OwnedSemaphorePermit> {
171        let permits = bytes.div_ceil(self.permit_bytes).max(1);
172        if permits > self.total_permits {
173            return None;
174        }
175        self.semaphore
176            .clone()
177            .try_acquire_many_owned(permits.try_into().ok()?)
178            .ok()
179    }
180
181    pub(crate) async fn acquire(&self, bytes: usize) -> Result<tokio::sync::SemaphorePermit<'_>> {
182        let permits = bytes.div_ceil(self.permit_bytes).max(1);
183        if permits > self.total_permits {
184            return UnexpectedSnafu {
185                reason: format!(
186                    "range result memory request of {bytes} bytes exceeds limiter capacity of {} bytes",
187                    self.total_permits.saturating_mul(self.permit_bytes)
188                ),
189            }
190            .fail();
191        }
192        // Nothing awaits between this check and the parking below, so an observed
193        // increment means the caller is about to wait for the missing permits.
194        #[cfg(test)]
195        if self.semaphore.available_permits() < permits {
196            self.waited_acquires
197                .fetch_add(1, std::sync::atomic::Ordering::Release);
198        }
199
200        self.semaphore
201            .acquire_many(permits as u32)
202            .await
203            .map_err(|_| {
204                UnexpectedSnafu {
205                    reason: "range result memory limiter is unexpectedly closed",
206                }
207                .build()
208            })
209    }
210}
211
212/// Cached SST metadata combines the parquet footer with the decoded region metadata.
213///
214/// The cached parquet footer strips the `greptime:metadata` JSON payload and stores the decoded
215/// [RegionMetadata] separately so readers can skip repeated deserialization work.
216#[derive(Debug)]
217pub(crate) struct CachedSstMeta {
218    parquet_metadata: Arc<ParquetMetaData>,
219    parquet_metadata_size: usize,
220    region_metadata: RegionMetadataRef,
221    region_metadata_weight: usize,
222    page_index_policy: PageIndexPolicy,
223}
224
225/// Compact, authoritative form of one SST's metadata.
226///
227/// The entry contains a zstd-compressed, self-contained Parquet metadata stream. The decoded
228/// representation is held weakly to coalesce concurrent decodes without retaining memory outside
229/// the cache capacity.
230#[derive(Debug)]
231pub(crate) struct CompactSstMeta {
232    encoded_metadata: Bytes,
233    decoded_size: usize,
234    region_metadata: Weak<RegionMetadata>,
235    page_index_policy: PageIndexPolicy,
236    decoded: tokio::sync::Mutex<Weak<CachedSstMeta>>,
237}
238
239/// Both forms produced by decoding metadata after a cache miss.
240#[derive(Debug)]
241pub(crate) struct PreparedSstMeta {
242    pub(crate) compact: Arc<CompactSstMeta>,
243    pub(crate) decoded: Arc<CachedSstMeta>,
244}
245
246impl PreparedSstMeta {
247    pub(crate) fn decoded(&self) -> Arc<CachedSstMeta> {
248        self.decoded.clone()
249    }
250}
251
252/// Result of decoding SST metadata and attempting to encode its compact cache entry.
253#[derive(Debug)]
254pub(crate) enum SstMetaPreparation {
255    /// Both cache representations are ready for admission.
256    Prepared(PreparedSstMeta),
257    /// The metadata is usable by the reader, but its compact cache encoding failed.
258    DecodedOnly {
259        decoded: Arc<CachedSstMeta>,
260        encoding_error: crate::error::Error,
261    },
262}
263
264impl SstMetaPreparation {
265    pub(crate) fn decoded(&self) -> Arc<CachedSstMeta> {
266        match self {
267            Self::Prepared(metadata) => metadata.decoded(),
268            Self::DecodedOnly { decoded, .. } => decoded.clone(),
269        }
270    }
271}
272
273impl CompactSstMeta {
274    async fn decode(&self, runtime: &Runtime) -> Result<Arc<CachedSstMeta>> {
275        let mut decoded_guard = self.decoded.lock().await;
276        if let Some(decoded) = decoded_guard.upgrade() {
277            return Ok(decoded);
278        }
279
280        let encoded_metadata = self.encoded_metadata.clone();
281        let decoded_size = self.decoded_size;
282        let region_metadata = self.region_metadata.upgrade();
283        let page_index_policy = self.page_index_policy;
284        let decoded = runtime
285            .spawn_blocking(move || {
286                let bytes = zstd::bulk::decompress(&encoded_metadata, decoded_size).context(
287                    DecompressObjectSnafu {
288                        compress_type: CompressionType::Zstd,
289                        path: "cached SST metadata",
290                    },
291                )?;
292                let bytes = Bytes::from(bytes);
293                let mut reader = ParquetMetaDataReader::new()
294                    .with_column_index_policy(PageIndexPolicy::Skip)
295                    .with_offset_index_policy(page_index_policy);
296                reader.try_parse(&bytes).context(ReadParquetSnafu {
297                    path: "cached SST metadata",
298                })?;
299                let metadata = reader.finish().context(ReadParquetSnafu {
300                    path: "cached SST metadata",
301                })?;
302                CachedSstMeta::try_new_with_page_index_policy(
303                    "cached SST metadata",
304                    metadata,
305                    region_metadata,
306                    page_index_policy,
307                )
308                .map(Arc::new)
309            })
310            .await
311            .context(JoinSnafu)??;
312
313        *decoded_guard = Arc::downgrade(&decoded);
314        Ok(decoded)
315    }
316
317    fn satisfies_page_index_policy(&self, requested: PageIndexPolicy) -> bool {
318        satisfies_page_index_policy(self.page_index_policy, requested)
319    }
320}
321
322/// Decodes SST metadata on the given blocking runtime without preparing a compact cache entry.
323pub(crate) async fn decode_sst_meta(
324    file_path: &str,
325    parquet_metadata: ParquetMetaData,
326    region_metadata: Option<RegionMetadataRef>,
327    page_index_policy: PageIndexPolicy,
328    runtime: &Runtime,
329) -> Result<Arc<CachedSstMeta>> {
330    let file_path = file_path.to_string();
331    runtime
332        .spawn_blocking(move || {
333            let parquet_metadata = strip_column_indexes(parquet_metadata);
334            CachedSstMeta::try_new_with_page_index_policy(
335                &file_path,
336                parquet_metadata,
337                region_metadata,
338                page_index_policy,
339            )
340            .map(Arc::new)
341        })
342        .await
343        .context(JoinSnafu)?
344}
345
346/// Decodes SST metadata and attempts to encode both cache representations on the given blocking runtime.
347pub(crate) async fn prepare_sst_meta(
348    file_path: &str,
349    parquet_metadata: ParquetMetaData,
350    region_metadata: Option<RegionMetadataRef>,
351    page_index_policy: PageIndexPolicy,
352    runtime: &Runtime,
353) -> Result<SstMetaPreparation> {
354    let file_path = file_path.to_string();
355    runtime
356        .spawn_blocking(move || {
357            prepare_sst_meta_sync(
358                &file_path,
359                parquet_metadata,
360                region_metadata,
361                page_index_policy,
362            )
363        })
364        .await
365        .context(JoinSnafu)?
366}
367
368/// Synchronously prepares SST metadata. Callers must run this on a blocking runtime.
369pub(crate) fn prepare_sst_meta_sync(
370    file_path: &str,
371    parquet_metadata: ParquetMetaData,
372    region_metadata: Option<RegionMetadataRef>,
373    page_index_policy: PageIndexPolicy,
374) -> Result<SstMetaPreparation> {
375    let parquet_metadata = strip_column_indexes(parquet_metadata);
376    let cache_encoding = encode_compact_sst_meta(file_path, &parquet_metadata);
377    finish_sst_meta_preparation(
378        file_path,
379        parquet_metadata,
380        region_metadata,
381        page_index_policy,
382        cache_encoding,
383    )
384}
385
386fn strip_column_indexes(parquet_metadata: ParquetMetaData) -> ParquetMetaData {
387    // Defensively discard column indexes supplied by external metadata producers. Mito only
388    // consumes offset indexes.
389    let mut builder = parquet_metadata.into_builder();
390    builder.take_column_index();
391    builder.build()
392}
393
394fn encode_compact_sst_meta(
395    file_path: &str,
396    parquet_metadata: &ParquetMetaData,
397) -> Result<(Bytes, usize)> {
398    let mut encoded = Vec::new();
399    ParquetMetaDataWriter::new(&mut encoded, parquet_metadata)
400        .finish()
401        .context(WriteParquetSnafu)?;
402    let decoded_size = encoded.len();
403    let encoded_metadata = zstd::bulk::compress(&encoded, 3).context(CompressObjectSnafu {
404        compress_type: CompressionType::Zstd,
405        path: file_path,
406    })?;
407
408    // `zstd::bulk::compress` allocates for the compression upper bound and leaves the excess
409    // capacity in its `Vec`. Convert through a boxed slice so the retained allocation matches the
410    // cache weight.
411    Ok((
412        Bytes::from(encoded_metadata.into_boxed_slice()),
413        decoded_size,
414    ))
415}
416
417fn finish_sst_meta_preparation(
418    file_path: &str,
419    parquet_metadata: ParquetMetaData,
420    region_metadata: Option<RegionMetadataRef>,
421    page_index_policy: PageIndexPolicy,
422    cache_encoding: Result<(Bytes, usize)>,
423) -> Result<SstMetaPreparation> {
424    let decoded = Arc::new(CachedSstMeta::try_new_with_page_index_policy(
425        file_path,
426        parquet_metadata,
427        region_metadata,
428        page_index_policy,
429    )?);
430    let (encoded_metadata, decoded_size) = match cache_encoding {
431        Ok(encoded) => encoded,
432        Err(encoding_error) => {
433            return Ok(SstMetaPreparation::DecodedOnly {
434                decoded,
435                encoding_error,
436            });
437        }
438    };
439    let compact = Arc::new(CompactSstMeta {
440        encoded_metadata,
441        decoded_size,
442        region_metadata: Arc::downgrade(&decoded.region_metadata),
443        page_index_policy,
444        decoded: tokio::sync::Mutex::new(Arc::downgrade(&decoded)),
445    });
446
447    Ok(SstMetaPreparation::Prepared(PreparedSstMeta {
448        compact,
449        decoded,
450    }))
451}
452
453impl CachedSstMeta {
454    #[cfg(test)]
455    pub(crate) fn try_new(file_path: &str, parquet_metadata: ParquetMetaData) -> Result<Self> {
456        let page_index_policy = infer_loaded_page_index_policy(&parquet_metadata);
457        Self::try_new_with_page_index_policy(file_path, parquet_metadata, None, page_index_policy)
458    }
459
460    pub(crate) fn try_new_with_region_metadata(
461        file_path: &str,
462        parquet_metadata: ParquetMetaData,
463        region_metadata: Option<RegionMetadataRef>,
464    ) -> Result<Self> {
465        let page_index_policy = infer_loaded_page_index_policy(&parquet_metadata);
466        Self::try_new_with_page_index_policy(
467            file_path,
468            parquet_metadata,
469            region_metadata,
470            page_index_policy,
471        )
472    }
473
474    pub(crate) fn try_new_with_page_index_policy(
475        file_path: &str,
476        parquet_metadata: ParquetMetaData,
477        region_metadata: Option<RegionMetadataRef>,
478        page_index_policy: PageIndexPolicy,
479    ) -> Result<Self> {
480        let file_metadata = parquet_metadata.file_metadata();
481        let key_values = file_metadata
482            .key_value_metadata()
483            .context(InvalidParquetSnafu {
484                file: file_path,
485                reason: "missing key value meta",
486            })?;
487        let meta_value = key_values
488            .iter()
489            .find(|kv| kv.key == PARQUET_METADATA_KEY)
490            .with_context(|| InvalidParquetSnafu {
491                file: file_path,
492                reason: format!("key {} not found", PARQUET_METADATA_KEY),
493            })?;
494        let json = meta_value
495            .value
496            .as_ref()
497            .with_context(|| InvalidParquetSnafu {
498                file: file_path,
499                reason: format!("No value for key {}", PARQUET_METADATA_KEY),
500            })?;
501        let region_metadata = match region_metadata {
502            Some(region_metadata) => region_metadata,
503            None => Arc::new(
504                store_api::metadata::RegionMetadata::from_json(json)
505                    .context(InvalidMetadataSnafu)?,
506            ),
507        };
508        // Keep the previous JSON-byte floor and charge the decoded structures as well.
509        let region_metadata_weight = region_metadata.estimated_size().max(json.len());
510        let parquet_metadata = Arc::new(strip_region_metadata_from_parquet(parquet_metadata));
511        let parquet_metadata_size = parquet_meta_size(&parquet_metadata);
512
513        Ok(Self {
514            parquet_metadata,
515            parquet_metadata_size,
516            region_metadata,
517            region_metadata_weight,
518            page_index_policy,
519        })
520    }
521
522    pub(crate) fn parquet_metadata(&self) -> Arc<ParquetMetaData> {
523        self.parquet_metadata.clone()
524    }
525
526    /// Returns the immutable parquet metadata size computed when it was decoded.
527    pub(crate) fn parquet_metadata_size(&self) -> usize {
528        self.parquet_metadata_size
529    }
530
531    pub(crate) fn region_metadata(&self) -> RegionMetadataRef {
532        self.region_metadata.clone()
533    }
534
535    fn satisfies_page_index_policy(&self, requested: PageIndexPolicy) -> bool {
536        satisfies_page_index_policy(self.page_index_policy, requested)
537    }
538}
539
540fn satisfies_page_index_policy(cached: PageIndexPolicy, requested: PageIndexPolicy) -> bool {
541    match requested {
542        PageIndexPolicy::Skip => true,
543        PageIndexPolicy::Optional => cached != PageIndexPolicy::Skip,
544        PageIndexPolicy::Required => cached == PageIndexPolicy::Required,
545    }
546}
547
548fn infer_loaded_page_index_policy(parquet_metadata: &ParquetMetaData) -> PageIndexPolicy {
549    if parquet_metadata.offset_index().is_some() {
550        PageIndexPolicy::Optional
551    } else {
552        PageIndexPolicy::Skip
553    }
554}
555
556fn strip_region_metadata_from_parquet(parquet_metadata: ParquetMetaData) -> ParquetMetaData {
557    let file_metadata = parquet_metadata.file_metadata();
558    let filtered_key_values = file_metadata.key_value_metadata().and_then(|key_values| {
559        let filtered = key_values
560            .iter()
561            .filter(|kv| kv.key != PARQUET_METADATA_KEY)
562            .cloned()
563            .collect::<Vec<_>>();
564        (!filtered.is_empty()).then_some(filtered)
565    });
566    let stripped_file_metadata = FileMetaData::new(
567        file_metadata.version(),
568        file_metadata.num_rows(),
569        file_metadata.created_by().map(ToString::to_string),
570        filtered_key_values,
571        file_metadata.schema_descr_ptr(),
572        file_metadata.column_orders().cloned(),
573    );
574
575    let mut builder = parquet_metadata.into_builder();
576    let row_groups = builder.take_row_groups();
577    let offset_index = builder.take_offset_index();
578
579    parquet::file::metadata::ParquetMetaDataBuilder::new(stripped_file_metadata)
580        .set_row_groups(row_groups)
581        .set_offset_index(offset_index)
582        .build()
583}
584
585fn removal_cause_str(cause: RemovalCause) -> &'static str {
586    match cause {
587        RemovalCause::Expired => "expired",
588        RemovalCause::Explicit => "explicit",
589        RemovalCause::Replaced => "replaced",
590        RemovalCause::Size => "size",
591    }
592}
593
594#[derive(Debug, Clone, PartialEq, Eq, Hash)]
595pub(crate) struct PrefilterRowSelector {
596    row_count: usize,
597    skip: bool,
598}
599
600// `parquet::arrow::arrow_reader::RowSelector` does not implement `Hash`, but
601// prefilter cache keys must hash the upstream row-selection snapshot. Keep a
602// local hashable mirror of the two fields that define selector semantics.
603// TODO(yingwen): Remove this mirror if upstream `RowSelector` implements `Hash`.
604impl From<&RowSelector> for PrefilterRowSelector {
605    fn from(selector: &RowSelector) -> Self {
606        Self {
607            row_count: selector.row_count,
608            skip: selector.skip,
609        }
610    }
611}
612
613/// Key for a cached prefilter result.
614#[derive(Debug, Clone, PartialEq, Eq, Hash)]
615pub(crate) struct PrefilterKey {
616    file_id: FileId,
617    row_group_idx: u32,
618    row_selection: Option<Arc<Vec<PrefilterRowSelector>>>,
619    schema_version: u64,
620    filter_exprs: SmallVec<[String; 1]>,
621    mem_usage: usize,
622}
623
624impl PrefilterKey {
625    pub(crate) fn row_selection_snapshot(
626        row_selection: Option<&RowSelection>,
627    ) -> Option<Arc<Vec<PrefilterRowSelector>>> {
628        row_selection.map(|selection| {
629            Arc::new(
630                selection
631                    .iter()
632                    .map(PrefilterRowSelector::from)
633                    .collect::<Vec<_>>(),
634            )
635        })
636    }
637
638    pub(crate) fn new(
639        file_id: FileId,
640        row_group_idx: u32,
641        row_selection: Option<Arc<Vec<PrefilterRowSelector>>>,
642        schema_version: u64,
643        filter_exprs: SmallVec<[String; 1]>,
644    ) -> Self {
645        let row_selection_bytes = row_selection
646            .as_ref()
647            .map(|selection| selection.len() * mem::size_of::<PrefilterRowSelector>())
648            .unwrap_or(0);
649        let spilled_expr_bytes = if filter_exprs.spilled() {
650            filter_exprs.capacity() * mem::size_of::<String>()
651        } else {
652            0
653        };
654        let expr_bytes = filter_exprs.iter().map(|s| s.capacity()).sum::<usize>();
655
656        Self {
657            file_id,
658            row_group_idx,
659            row_selection,
660            schema_version,
661            filter_exprs,
662            mem_usage: mem::size_of::<Self>()
663                + row_selection_bytes
664                + spilled_expr_bytes
665                + expr_bytes,
666        }
667    }
668
669    fn mem_usage(&self) -> usize {
670        self.mem_usage
671    }
672}
673
674type PrefilterResultCache = Cache<PrefilterKey, Arc<BooleanBuffer>>;
675
676fn new_prefilter_result_cache(capacity: u64) -> PrefilterResultCache {
677    Cache::builder()
678        .max_capacity(capacity)
679        .weigher(prefilter_result_cache_weight)
680        .eviction_listener(|k, v, cause| {
681            let size = prefilter_result_cache_weight(&k, &v);
682            CACHE_BYTES
683                .with_label_values(&[PREFILTER_RESULT_TYPE])
684                .sub(size.into());
685            CACHE_EVICTION
686                .with_label_values(&[PREFILTER_RESULT_TYPE, removal_cause_str(cause)])
687                .inc();
688        })
689        .build()
690}
691
692fn prefilter_result_cache_weight(k: &PrefilterKey, v: &Arc<BooleanBuffer>) -> u32 {
693    (k.mem_usage() + mem::size_of::<BooleanBuffer>() + v.values().len()) as u32
694}
695
696/// Cache strategies that may only enable a subset of caches.
697#[derive(Clone)]
698pub enum CacheStrategy {
699    /// Strategy for normal operations.
700    /// Doesn't disable any cache.
701    EnableAll(CacheManagerRef),
702    /// Strategy for compaction.
703    /// Disables some caches during compaction to avoid affecting queries.
704    /// Enables the write cache so that the compaction can read files cached
705    /// in the write cache and write the compacted files back to the write cache.
706    Compaction(CacheManagerRef),
707    /// Do not use any cache.
708    Disabled,
709}
710
711impl CacheStrategy {
712    /// Returns the runtime for CPU-bound SST metadata work for this request.
713    pub(crate) fn sst_meta_runtime(&self) -> Runtime {
714        match self {
715            CacheStrategy::Compaction(_) => common_runtime::compact_runtime(),
716            CacheStrategy::EnableAll(_) | CacheStrategy::Disabled => {
717                common_runtime::global_runtime()
718            }
719        }
720    }
721
722    /// Returns whether the SST metadata cache is enabled for this strategy.
723    pub(crate) fn sst_meta_cache_enabled(&self) -> bool {
724        match self {
725            CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => {
726                cache_manager.sst_meta_cache_enabled()
727            }
728            CacheStrategy::Disabled => false,
729        }
730    }
731
732    #[cfg(test)]
733    pub(crate) async fn get_or_load_sst_meta<F>(
734        &self,
735        file_id: RegionFileId,
736        page_index_policy: PageIndexPolicy,
737        load: F,
738    ) -> Result<(Arc<CachedSstMeta>, bool)>
739    where
740        F: std::future::Future<Output = Result<(Arc<CachedSstMeta>, bool)>>,
741    {
742        self.get_or_load_observed_sst_meta(file_id, page_index_policy, async {
743            (load.await, SstMetaResolution::default())
744        })
745        .await
746        .0
747    }
748
749    /// Shares the resolved source on success and failure, without sharing I/O accounting.
750    pub(crate) async fn get_or_load_observed_sst_meta<F>(
751        &self,
752        file_id: RegionFileId,
753        page_index_policy: PageIndexPolicy,
754        load: F,
755    ) -> (Result<(Arc<CachedSstMeta>, bool)>, SstMetaResolution)
756    where
757        F: std::future::Future<Output = (Result<(Arc<CachedSstMeta>, bool)>, SstMetaResolution)>,
758    {
759        let cache = match self {
760            Self::EnableAll(manager) | Self::Compaction(manager) => {
761                manager.sst_decoded_meta_cache.as_ref()
762            }
763            Self::Disabled => None,
764        };
765        let Some(cache) = cache else {
766            return load.await;
767        };
768        let key = SstDecodedMetaKey::new(file_id, page_index_policy);
769        let result = cache
770            .try_get_with(&key, async {
771                let (result, resolution) = load.await;
772                let (metadata, cache_miss) = result.map_err(|source| SstMetaLoadError {
773                    source: Arc::new(source),
774                    resolution,
775                })?;
776                let value = SstMetaLoadResult::new(metadata, cache_miss, resolution);
777                Ok::<_, SstMetaLoadError>(value)
778            })
779            .await;
780        match result {
781            Ok(value) => (
782                Ok((value.metadata.clone(), value.cache_miss)),
783                value.resolution,
784            ),
785            Err(error) => (
786                Err(error.source.clone()).context(crate::error::LoadSstMetaSnafu),
787                error.resolution,
788            ),
789        }
790    }
791
792    /// Observes memory residency for this caller before sharing decode or I/O work.
793    pub(crate) async fn observe_sst_meta(
794        &self,
795        file_id: RegionFileId,
796        page_index_policy: PageIndexPolicy,
797        metrics: &mut MetadataCacheMetrics,
798    ) -> SstMetaObservation {
799        match self {
800            Self::EnableAll(manager) | Self::Compaction(manager) => {
801                manager
802                    .observe_sst_meta(file_id, page_index_policy, metrics)
803                    .await
804            }
805            Self::Disabled => SstMetaObservation::Miss,
806        }
807    }
808
809    /// Resolves an observed metadata entry and updates this caller's cache metrics.
810    pub(crate) async fn resolve_sst_meta(
811        &self,
812        observation: SstMetaObservation,
813        file_id: RegionFileId,
814        metrics: &mut MetadataCacheMetrics,
815        page_index_policy: PageIndexPolicy,
816    ) -> Option<Arc<CachedSstMeta>> {
817        match self {
818            Self::EnableAll(manager) | Self::Compaction(manager) => {
819                manager
820                    .resolve_sst_meta(
821                        observation,
822                        file_id,
823                        metrics,
824                        page_index_policy,
825                        &self.sst_meta_runtime(),
826                    )
827                    .await
828            }
829            Self::Disabled => {
830                metrics.cache_miss += 1;
831                None
832            }
833        }
834    }
835
836    /// Calls [CacheManager::get_sst_meta_data_from_mem_cache()].
837    pub(crate) async fn get_sst_meta_data_from_mem_cache(
838        &self,
839        file_id: RegionFileId,
840        page_index_policy: PageIndexPolicy,
841    ) -> Option<Arc<CachedSstMeta>> {
842        match self {
843            CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => {
844                cache_manager
845                    .get_sst_meta_data_from_mem_cache(file_id, page_index_policy)
846                    .await
847            }
848            CacheStrategy::Disabled => None,
849        }
850    }
851
852    /// Calls [CacheManager::get_parquet_meta_data_from_mem_cache()].
853    pub async fn get_parquet_meta_data_from_mem_cache(
854        &self,
855        file_id: RegionFileId,
856    ) -> Option<Arc<ParquetMetaData>> {
857        self.get_sst_meta_data_from_mem_cache(file_id, PageIndexPolicy::Skip)
858            .await
859            .map(|metadata| metadata.parquet_metadata())
860    }
861
862    /// Calls [CacheManager::put_compact_sst_meta()].
863    pub(crate) fn put_compact_sst_meta(
864        &self,
865        file_id: RegionFileId,
866        metadata: Arc<CompactSstMeta>,
867    ) {
868        match self {
869            CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => {
870                cache_manager.put_compact_sst_meta(file_id, metadata);
871            }
872            CacheStrategy::Disabled => {}
873        }
874    }
875
876    /// Calls [CacheManager::put_parquet_meta_data()].
877    pub async fn put_parquet_meta_data(
878        &self,
879        file_id: RegionFileId,
880        metadata: Arc<ParquetMetaData>,
881        region_metadata: Option<RegionMetadataRef>,
882    ) {
883        match self {
884            CacheStrategy::EnableAll(cache_manager) | CacheStrategy::Compaction(cache_manager) => {
885                cache_manager
886                    .put_parquet_meta_data(file_id, metadata, region_metadata)
887                    .await;
888            }
889            CacheStrategy::Disabled => {}
890        }
891    }
892
893    /// Calls [CacheManager::get_prefilter_result()].
894    /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
895    pub(crate) fn get_prefilter_result(&self, key: &PrefilterKey) -> Option<Arc<BooleanBuffer>> {
896        match self {
897            CacheStrategy::EnableAll(cache_manager) => cache_manager.get_prefilter_result(key),
898            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
899        }
900    }
901
902    /// Calls [CacheManager::put_prefilter_result()].
903    /// It does nothing if the strategy isn't [CacheStrategy::EnableAll].
904    pub(crate) fn put_prefilter_result(&self, key: PrefilterKey, result: Arc<BooleanBuffer>) {
905        if let CacheStrategy::EnableAll(cache_manager) = self {
906            cache_manager.put_prefilter_result(key, result);
907        }
908    }
909
910    /// Calls [CacheManager::remove_parquet_meta_data()].
911    pub async fn remove_parquet_meta_data(&self, file_id: RegionFileId) {
912        match self {
913            CacheStrategy::EnableAll(cache_manager) => {
914                cache_manager.remove_parquet_meta_data(file_id).await;
915            }
916            CacheStrategy::Compaction(cache_manager) => {
917                cache_manager.remove_parquet_meta_data(file_id).await;
918            }
919            CacheStrategy::Disabled => {}
920        }
921    }
922
923    /// Calls [CacheManager::get_repeated_vector()].
924    /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
925    pub fn get_repeated_vector(
926        &self,
927        data_type: &ConcreteDataType,
928        value: &Value,
929    ) -> Option<VectorRef> {
930        match self {
931            CacheStrategy::EnableAll(cache_manager) => {
932                cache_manager.get_repeated_vector(data_type, value)
933            }
934            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
935        }
936    }
937
938    /// Calls [CacheManager::put_repeated_vector()].
939    /// It does nothing if the strategy isn't [CacheStrategy::EnableAll].
940    pub fn put_repeated_vector(&self, value: Value, vector: VectorRef) {
941        if let CacheStrategy::EnableAll(cache_manager) = self {
942            cache_manager.put_repeated_vector(value, vector);
943        }
944    }
945
946    /// Returns the Parquet byte cache for normal query reads.
947    pub(crate) fn parquet_page_cache(&self) -> Option<&Arc<ParquetPageCache>> {
948        match self {
949            Self::EnableAll(manager) => manager.page_cache.as_ref(),
950            Self::Compaction(_) | Self::Disabled => None,
951        }
952    }
953
954    /// Calls [CacheManager::evict_puffin_cache()].
955    pub async fn evict_puffin_cache(&self, file_id: RegionIndexId) {
956        match self {
957            CacheStrategy::EnableAll(cache_manager) => {
958                cache_manager.evict_puffin_cache(file_id).await
959            }
960            CacheStrategy::Compaction(cache_manager) => {
961                cache_manager.evict_puffin_cache(file_id).await
962            }
963            CacheStrategy::Disabled => {}
964        }
965    }
966
967    /// Calls [CacheManager::get_selector_result()].
968    /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
969    pub fn get_selector_result(
970        &self,
971        selector_key: &SelectorResultKey,
972    ) -> Option<Arc<SelectorResultValue>> {
973        match self {
974            CacheStrategy::EnableAll(cache_manager) => {
975                cache_manager.get_selector_result(selector_key)
976            }
977            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
978        }
979    }
980
981    /// Calls [CacheManager::put_selector_result()].
982    /// It does nothing if the strategy isn't [CacheStrategy::EnableAll].
983    pub fn put_selector_result(
984        &self,
985        selector_key: SelectorResultKey,
986        result: Arc<SelectorResultValue>,
987    ) {
988        if let CacheStrategy::EnableAll(cache_manager) = self {
989            cache_manager.put_selector_result(selector_key, result);
990        }
991    }
992
993    /// Calls [CacheManager::get_range_result()].
994    /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
995    #[cfg(test)]
996    pub(crate) fn get_range_result(
997        &self,
998        key: &RangeScanCacheKey,
999    ) -> Option<Arc<RangeScanCacheValue>> {
1000        match self {
1001            CacheStrategy::EnableAll(cache_manager) => cache_manager.get_range_result(key),
1002            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1003        }
1004    }
1005
1006    /// Calls [CacheManager::put_range_result()].
1007    /// It does nothing if the strategy isn't [CacheStrategy::EnableAll].
1008    #[cfg(test)]
1009    pub(crate) fn put_range_result(
1010        &self,
1011        key: RangeScanCacheKey,
1012        result: Arc<RangeScanCacheValue>,
1013    ) {
1014        if let CacheStrategy::EnableAll(cache_manager) = self {
1015            cache_manager.put_range_result(key, result);
1016        }
1017    }
1018
1019    pub(crate) fn range_result_cache(&self) -> Option<&RangeResultCacheRef> {
1020        match self {
1021            Self::EnableAll(manager) => manager.range_result_cache.as_ref(),
1022            Self::Compaction(_) | Self::Disabled => None,
1023        }
1024    }
1025
1026    /// Returns true if the range result cache is enabled.
1027    pub(crate) fn has_range_result_cache(&self) -> bool {
1028        match self {
1029            CacheStrategy::EnableAll(cache_manager) => cache_manager.has_range_result_cache(),
1030            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => false,
1031        }
1032    }
1033
1034    #[cfg(test)]
1035    pub(crate) fn range_concat_memory(&self) -> Option<&Arc<RangeResultMemoryLimiter>> {
1036        self.range_result_cache().map(|cache| cache.concat_memory())
1037    }
1038
1039    /// Calls [CacheManager::write_cache()].
1040    /// It returns None if the strategy is [CacheStrategy::Disabled].
1041    pub fn write_cache(&self) -> Option<&WriteCacheRef> {
1042        match self {
1043            CacheStrategy::EnableAll(cache_manager) => cache_manager.write_cache(),
1044            CacheStrategy::Compaction(cache_manager) => cache_manager.write_cache(),
1045            CacheStrategy::Disabled => None,
1046        }
1047    }
1048
1049    /// Calls [CacheManager::index_cache()].
1050    /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
1051    pub fn inverted_index_cache(&self) -> Option<&InvertedIndexCacheRef> {
1052        match self {
1053            CacheStrategy::EnableAll(cache_manager) => cache_manager.inverted_index_cache(),
1054            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1055        }
1056    }
1057
1058    /// Calls [CacheManager::bloom_filter_index_cache()].
1059    /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
1060    pub fn bloom_filter_index_cache(&self) -> Option<&BloomFilterIndexCacheRef> {
1061        match self {
1062            CacheStrategy::EnableAll(cache_manager) => cache_manager.bloom_filter_index_cache(),
1063            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1064        }
1065    }
1066
1067    /// Calls [CacheManager::puffin_metadata_cache()].
1068    /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
1069    pub fn puffin_metadata_cache(&self) -> Option<&PuffinMetadataCacheRef> {
1070        match self {
1071            CacheStrategy::EnableAll(cache_manager) => cache_manager.puffin_metadata_cache(),
1072            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1073        }
1074    }
1075
1076    /// Calls [CacheManager::index_result_cache()].
1077    /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled].
1078    pub fn index_result_cache(&self) -> Option<&IndexResultCache> {
1079        match self {
1080            CacheStrategy::EnableAll(cache_manager) => cache_manager.index_result_cache(),
1081            CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None,
1082        }
1083    }
1084
1085    /// Triggers download if the strategy is [CacheStrategy::EnableAll] and write cache is available.
1086    pub fn maybe_download_background(
1087        &self,
1088        index_key: IndexKey,
1089        remote_path: String,
1090        remote_store: ObjectStore,
1091        file_size: u64,
1092    ) {
1093        if let CacheStrategy::EnableAll(cache_manager) = self
1094            && let Some(write_cache) = cache_manager.write_cache()
1095        {
1096            write_cache.file_cache().maybe_download_background(
1097                index_key,
1098                remote_path,
1099                remote_store,
1100                file_size,
1101            );
1102        }
1103    }
1104}
1105
1106/// Manages cached data for the engine.
1107///
1108/// All caches are disabled by default.
1109#[derive(Default)]
1110pub struct CacheManager {
1111    /// Cache for compact, authoritative SST metadata.
1112    sst_meta_cache: Option<SstMetaCache>,
1113    /// Unregisters compact metadata bytes when this manager is dropped.
1114    _sst_meta_size_registration: Option<CacheSizeRegistration>,
1115    /// Skips output metadata prewarming for caches used only by the current compaction.
1116    disable_sst_meta_cache_on_write: bool,
1117    /// Cache for decoded SST metadata, used only as an acceleration tier.
1118    sst_decoded_meta_cache: Option<SstDecodedMetaCache>,
1119    /// Immutable native-reader (columnar series/samples) metadata tier. It cannot share
1120    /// `sst_decoded_meta_cache` because that tier stores `SstMetaLoadResult` under a
1121    /// page-index-aware key.
1122    columnar_meta_cache: Option<ColumnarMetaCache>,
1123    /// Cache for vectors.
1124    vector_cache: Option<VectorCache>,
1125    /// Cache for SST byte ranges.
1126    page_cache: Option<Arc<ParquetPageCache>>,
1127    /// A Cache for writing files to object stores.
1128    write_cache: Option<WriteCacheRef>,
1129    /// Cache for inverted index.
1130    inverted_index_cache: Option<InvertedIndexCacheRef>,
1131    /// Cache for bloom filter index.
1132    bloom_filter_index_cache: Option<BloomFilterIndexCacheRef>,
1133    /// Puffin metadata cache.
1134    puffin_metadata_cache: Option<PuffinMetadataCacheRef>,
1135    /// Cache for time series selectors.
1136    selector_result_cache: Option<SelectorResultCache>,
1137    /// Cache for range scan outputs in flat format.
1138    range_result_cache: Option<RangeResultCacheRef>,
1139    /// Configured capacity for range scan outputs in flat format.
1140    #[cfg(test)]
1141    range_result_cache_size: u64,
1142    /// Cache for index result.
1143    index_result_cache: Option<IndexResultCache>,
1144    /// Cache for prefilter result.
1145    prefilter_result_cache: Option<PrefilterResultCache>,
1146}
1147
1148pub type CacheManagerRef = Arc<CacheManager>;
1149
1150impl CacheManager {
1151    /// Returns a builder to build the cache.
1152    pub fn builder() -> CacheManagerBuilder {
1153        CacheManagerBuilder::default()
1154    }
1155
1156    /// Gets fused SST metadata with metrics tracking for direct cache tests.
1157    #[cfg(test)]
1158    pub(crate) async fn get_sst_meta_data(
1159        &self,
1160        file_id: RegionFileId,
1161        metrics: &mut MetadataCacheMetrics,
1162        page_index_policy: PageIndexPolicy,
1163        runtime: &Runtime,
1164    ) -> Option<Arc<CachedSstMeta>> {
1165        let observation = self
1166            .observe_sst_meta(file_id, page_index_policy, metrics)
1167            .await;
1168        self.resolve_sst_meta(observation, file_id, metrics, page_index_policy, runtime)
1169            .await
1170    }
1171
1172    async fn observe_sst_meta(
1173        &self,
1174        file_id: RegionFileId,
1175        page_index_policy: PageIndexPolicy,
1176        metrics: &mut MetadataCacheMetrics,
1177    ) -> SstMetaObservation {
1178        let cache_key = SstMetaKey(file_id.region_id(), file_id.file_id());
1179        let compact = self
1180            .get_compact_sst_meta(&cache_key)
1181            .filter(|metadata| metadata.satisfies_page_index_policy(page_index_policy));
1182        let decoded = self
1183            .get_decoded_sst_meta(file_id, page_index_policy)
1184            .await
1185            .filter(|metadata| metadata.satisfies_page_index_policy(page_index_policy));
1186        let observation = match (compact, decoded) {
1187            (_, Some(decoded)) => SstMetaObservation::Decoded(decoded),
1188            (Some(compact), None) => SstMetaObservation::Compact(compact),
1189            (None, None) => SstMetaObservation::Miss,
1190        };
1191        observation.record(metrics);
1192        observation
1193    }
1194
1195    async fn resolve_sst_meta(
1196        &self,
1197        observation: SstMetaObservation,
1198        file_id: RegionFileId,
1199        metrics: &mut MetadataCacheMetrics,
1200        page_index_policy: PageIndexPolicy,
1201        runtime: &Runtime,
1202    ) -> Option<Arc<CachedSstMeta>> {
1203        match observation {
1204            SstMetaObservation::Decoded(decoded) => return Some(decoded),
1205            SstMetaObservation::Compact(compact) => match compact.decode(runtime).await {
1206                Ok(decoded) => return Some(decoded),
1207                Err(err) => {
1208                    warn!(err; "Failed to decode compact SST metadata, region_id: {}, file_id: {}", file_id.region_id(), file_id.file_id());
1209                    self.remove_parquet_meta_data(file_id).await;
1210                }
1211            },
1212            SstMetaObservation::Miss => {}
1213        }
1214
1215        let key = IndexKey::new(file_id.region_id(), file_id.file_id(), FileType::Parquet);
1216        if let Some(write_cache) = &self.write_cache {
1217            let file_cache = write_cache.file_cache();
1218            if self.sst_meta_cache_enabled() {
1219                if let Some(metadata) = file_cache
1220                    .get_sst_meta_data(key, metrics, page_index_policy, runtime)
1221                    .await
1222                {
1223                    metrics.file_cache_hit += 1;
1224                    let decoded = metadata.decoded();
1225                    match metadata {
1226                        SstMetaPreparation::Prepared(metadata) => {
1227                            self.put_compact_sst_meta(file_id, metadata.compact);
1228                        }
1229                        SstMetaPreparation::DecodedOnly { encoding_error, .. } => {
1230                            warn!(
1231                                encoding_error;
1232                                "Failed to encode file-cached SST metadata for memory cache, region_id: {}, file_id: {}",
1233                                file_id.region_id(),
1234                                file_id.file_id()
1235                            );
1236                        }
1237                    }
1238                    return Some(decoded);
1239                }
1240            } else if let Some(decoded) = file_cache
1241                .get_decoded_sst_meta_data(key, metrics, page_index_policy, runtime)
1242                .await
1243            {
1244                metrics.file_cache_hit += 1;
1245                return Some(decoded);
1246            }
1247        }
1248
1249        metrics.cache_miss += 1;
1250        None
1251    }
1252
1253    /// Gets cached fused SST metadata from in-memory cache.
1254    /// This method does not perform I/O.
1255    pub(crate) async fn get_sst_meta_data_from_mem_cache(
1256        &self,
1257        file_id: RegionFileId,
1258        page_index_policy: PageIndexPolicy,
1259    ) -> Option<Arc<CachedSstMeta>> {
1260        let value = self
1261            .get_sst_meta_data_from_mem_cache_inner(file_id, page_index_policy)
1262            .await;
1263        update_hit_miss(value, SST_META_DECODED_TYPE)
1264    }
1265
1266    async fn get_sst_meta_data_from_mem_cache_inner(
1267        &self,
1268        file_id: RegionFileId,
1269        page_index_policy: PageIndexPolicy,
1270    ) -> Option<Arc<CachedSstMeta>> {
1271        self.get_decoded_sst_meta(file_id, page_index_policy).await
1272    }
1273
1274    /// Gets cached [ParquetMetaData] from in-memory cache.
1275    /// This method does not perform I/O.
1276    pub async fn get_parquet_meta_data_from_mem_cache(
1277        &self,
1278        file_id: RegionFileId,
1279    ) -> Option<Arc<ParquetMetaData>> {
1280        self.get_sst_meta_data_from_mem_cache(file_id, PageIndexPolicy::Skip)
1281            .await
1282            .map(|metadata| metadata.parquet_metadata())
1283    }
1284
1285    /// Puts fused SST metadata into the cache.
1286    pub(crate) async fn put_sst_meta_data(
1287        &self,
1288        file_id: RegionFileId,
1289        metadata: Arc<CachedSstMeta>,
1290    ) {
1291        if let Some(cache) = &self.sst_decoded_meta_cache {
1292            let key = SstDecodedMetaKey::new(file_id, metadata.page_index_policy);
1293            let value = SstMetaLoadResult::new(metadata, false, SstMetaResolution::default());
1294            cache.insert(key, value).await;
1295        }
1296    }
1297
1298    /// Puts a compact SST metadata entry into the cache.
1299    pub(crate) fn put_compact_sst_meta(
1300        &self,
1301        file_id: RegionFileId,
1302        metadata: Arc<CompactSstMeta>,
1303    ) {
1304        let key = SstMetaKey(file_id.region_id(), file_id.file_id());
1305        if let Some(cache) = &self.sst_meta_cache {
1306            cache.insert(key, metadata);
1307        }
1308    }
1309
1310    fn get_compact_sst_meta(&self, key: &SstMetaKey) -> Option<Arc<CompactSstMeta>> {
1311        self.sst_meta_cache.as_ref()?.get(key)
1312    }
1313
1314    async fn get_decoded_sst_meta(
1315        &self,
1316        file_id: RegionFileId,
1317        requested: PageIndexPolicy,
1318    ) -> Option<Arc<CachedSstMeta>> {
1319        let cache = self.sst_decoded_meta_cache.as_ref()?;
1320        for policy in [
1321            PageIndexPolicy::Skip,
1322            PageIndexPolicy::Optional,
1323            PageIndexPolicy::Required,
1324        ] {
1325            // The key describes the loading request, while compact metadata may
1326            // already contain stronger page-index guarantees.
1327            if let Some(value) = cache.get(&SstDecodedMetaKey::new(file_id, policy)).await
1328                && value.metadata.satisfies_page_index_policy(requested)
1329            {
1330                return Some(value.metadata.clone());
1331            }
1332        }
1333        None
1334    }
1335
1336    /// Gets and decodes only the compact tier without promoting into the decoded cache.
1337    ///
1338    /// This is used by startup preloading so inspecting already-cached metadata cannot consume the
1339    /// decoded reservation and prematurely stop compact preloading.
1340    pub(crate) async fn get_compact_sst_meta_data(
1341        &self,
1342        file_id: RegionFileId,
1343        page_index_policy: PageIndexPolicy,
1344    ) -> Option<Arc<CachedSstMeta>> {
1345        let key = SstMetaKey(file_id.region_id(), file_id.file_id());
1346        let compact = self
1347            .get_compact_sst_meta(&key)
1348            .filter(|metadata| metadata.satisfies_page_index_policy(page_index_policy))?;
1349        match compact.decode(&common_runtime::global_runtime()).await {
1350            Ok(metadata) => Some(metadata),
1351            Err(err) => {
1352                warn!(err; "Failed to decode compact SST metadata, region_id: {}, file_id: {}", file_id.region_id(), file_id.file_id());
1353                self.remove_parquet_meta_data(file_id).await;
1354                None
1355            }
1356        }
1357    }
1358
1359    /// Puts [ParquetMetaData] into the cache.
1360    pub async fn put_parquet_meta_data(
1361        &self,
1362        file_id: RegionFileId,
1363        metadata: Arc<ParquetMetaData>,
1364        region_metadata: Option<RegionMetadataRef>,
1365    ) {
1366        if self.sst_decoded_meta_cache.is_some() {
1367            let file_path = format!(
1368                "region_id={}, file_id={}",
1369                file_id.region_id(),
1370                file_id.file_id()
1371            );
1372            match CachedSstMeta::try_new_with_region_metadata(
1373                &file_path,
1374                Arc::unwrap_or_clone(metadata),
1375                region_metadata,
1376            ) {
1377                Ok(metadata) => self.put_sst_meta_data(file_id, Arc::new(metadata)).await,
1378                Err(err) => warn!(
1379                    err; "Failed to decode region metadata while caching parquet metadata, region_id: {}, file_id: {}",
1380                    file_id.region_id(),
1381                    file_id.file_id()
1382                ),
1383            }
1384        }
1385    }
1386
1387    /// Removes [ParquetMetaData] from the cache.
1388    pub async fn remove_parquet_meta_data(&self, file_id: RegionFileId) {
1389        let key = SstMetaKey(file_id.region_id(), file_id.file_id());
1390        if let Some(cache) = &self.sst_meta_cache {
1391            cache.remove(&key);
1392        }
1393        if let Some(cache) = &self.sst_decoded_meta_cache {
1394            for policy in [
1395                PageIndexPolicy::Skip,
1396                PageIndexPolicy::Optional,
1397                PageIndexPolicy::Required,
1398            ] {
1399                cache
1400                    .invalidate(&SstDecodedMetaKey::new(file_id, policy))
1401                    .await;
1402            }
1403        }
1404    }
1405
1406    /// Returns whether the authoritative SST metadata tier has reached its reservation.
1407    pub(crate) fn sst_meta_cache_is_full(&self) -> bool {
1408        let Some(cache) = &self.sst_meta_cache else {
1409            return true;
1410        };
1411        cache
1412            .policy()
1413            .max_capacity()
1414            .is_some_and(|capacity| cache.weighted_size() >= capacity)
1415    }
1416
1417    /// Returns true if the in-memory SST meta cache is enabled.
1418    pub(crate) fn sst_meta_cache_enabled(&self) -> bool {
1419        self.sst_meta_cache.is_some()
1420    }
1421
1422    /// Whether newly written SST metadata should be prepared for later readers.
1423    pub(crate) fn sst_meta_cache_on_write_enabled(&self) -> bool {
1424        self.sst_meta_cache_enabled() && !self.disable_sst_meta_cache_on_write
1425    }
1426
1427    /// Gets a vector with repeated value for specific `key`.
1428    pub fn get_repeated_vector(
1429        &self,
1430        data_type: &ConcreteDataType,
1431        value: &Value,
1432    ) -> Option<VectorRef> {
1433        self.vector_cache.as_ref().and_then(|vector_cache| {
1434            let value = vector_cache.get(&(data_type.clone(), value.clone()));
1435            update_hit_miss(value, VECTOR_TYPE)
1436        })
1437    }
1438
1439    /// Puts a vector with repeated value into the cache.
1440    pub fn put_repeated_vector(&self, value: Value, vector: VectorRef) {
1441        if let Some(cache) = &self.vector_cache {
1442            let key = (vector.data_type(), value);
1443            CACHE_BYTES
1444                .with_label_values(&[VECTOR_TYPE])
1445                .add(vector_cache_weight(&key, &vector).into());
1446            cache.insert(key, vector);
1447        }
1448    }
1449
1450    /// Evicts every puffin-related cache entry for the given file.
1451    pub async fn evict_puffin_cache(&self, file_id: RegionIndexId) {
1452        if let Some(cache) = &self.bloom_filter_index_cache {
1453            cache.invalidate_file(file_id.file_id());
1454        }
1455
1456        if let Some(cache) = &self.inverted_index_cache {
1457            cache.invalidate_file(file_id.file_id());
1458        }
1459
1460        if let Some(cache) = &self.index_result_cache {
1461            cache.invalidate_file(file_id.file_id());
1462        }
1463
1464        if let Some(cache) = &self.puffin_metadata_cache {
1465            // Evict this exact generation across all storage scopes and paths.
1466            if let Err(err) = cache.invalidate_file(&file_id.to_string()) {
1467                warn!(err; "Failed to invalidate Puffin metadata, file_id: {file_id}");
1468            }
1469        }
1470
1471        if let Some(write_cache) = &self.write_cache {
1472            write_cache
1473                .remove(IndexKey::new(
1474                    file_id.region_id(),
1475                    file_id.file_id(),
1476                    FileType::Puffin(file_id.version),
1477                ))
1478                .await;
1479        }
1480    }
1481
1482    /// Gets result of for the selector.
1483    pub fn get_selector_result(
1484        &self,
1485        selector_key: &SelectorResultKey,
1486    ) -> Option<Arc<SelectorResultValue>> {
1487        self.selector_result_cache
1488            .as_ref()
1489            .and_then(|selector_result_cache| selector_result_cache.get(selector_key))
1490    }
1491
1492    /// Puts result of the selector into the cache.
1493    pub fn put_selector_result(
1494        &self,
1495        selector_key: SelectorResultKey,
1496        result: Arc<SelectorResultValue>,
1497    ) {
1498        if let Some(cache) = &self.selector_result_cache {
1499            CACHE_BYTES
1500                .with_label_values(&[SELECTOR_RESULT_TYPE])
1501                .add(selector_result_cache_weight(&selector_key, &result).into());
1502            cache.insert(selector_key, result);
1503        }
1504    }
1505
1506    /// Retires decoded range results when an SST leaves the current version.
1507    pub(crate) fn invalidate_range_results(&self, file_id: FileId) {
1508        if let Some(worker) = &self.range_result_cache {
1509            worker.invalidate_file(file_id);
1510        }
1511    }
1512
1513    /// Gets cached result for range scan.
1514    #[cfg(test)]
1515    pub(crate) fn get_range_result(
1516        &self,
1517        key: &RangeScanCacheKey,
1518    ) -> Option<Arc<RangeScanCacheValue>> {
1519        self.range_result_cache
1520            .as_ref()
1521            .and_then(|cache| update_hit_miss(cache.get(key), RANGE_RESULT_TYPE))
1522    }
1523
1524    /// Puts range scan result into cache.
1525    #[cfg(test)]
1526    pub(crate) fn put_range_result(
1527        &self,
1528        key: RangeScanCacheKey,
1529        result: Arc<RangeScanCacheValue>,
1530    ) {
1531        if let Some(cache) = &self.range_result_cache {
1532            cache.insert(key, result);
1533        }
1534    }
1535
1536    /// Returns true if the range result cache is enabled.
1537    pub(crate) fn has_range_result_cache(&self) -> bool {
1538        self.range_result_cache.is_some()
1539    }
1540
1541    #[cfg(test)]
1542    pub(crate) fn range_result_cache_size(&self) -> usize {
1543        self.range_result_cache_size as usize
1544    }
1545
1546    /// Gets the write cache.
1547    pub(crate) fn write_cache(&self) -> Option<&WriteCacheRef> {
1548        self.write_cache.as_ref()
1549    }
1550
1551    pub(crate) fn inverted_index_cache(&self) -> Option<&InvertedIndexCacheRef> {
1552        self.inverted_index_cache.as_ref()
1553    }
1554
1555    pub(crate) fn bloom_filter_index_cache(&self) -> Option<&BloomFilterIndexCacheRef> {
1556        self.bloom_filter_index_cache.as_ref()
1557    }
1558
1559    pub(crate) fn puffin_metadata_cache(&self) -> Option<&PuffinMetadataCacheRef> {
1560        self.puffin_metadata_cache.as_ref()
1561    }
1562
1563    pub(crate) fn index_result_cache(&self) -> Option<&IndexResultCache> {
1564        self.index_result_cache.as_ref()
1565    }
1566
1567    pub(crate) fn get_prefilter_result(&self, key: &PrefilterKey) -> Option<Arc<BooleanBuffer>> {
1568        self.prefilter_result_cache
1569            .as_ref()
1570            .and_then(|cache| update_hit_miss(cache.get(key), PREFILTER_RESULT_TYPE))
1571    }
1572
1573    pub(crate) fn put_prefilter_result(&self, key: PrefilterKey, result: Arc<BooleanBuffer>) {
1574        if let Some(cache) = &self.prefilter_result_cache {
1575            CACHE_BYTES
1576                .with_label_values(&[PREFILTER_RESULT_TYPE])
1577                .add(prefilter_result_cache_weight(&key, &result).into());
1578            cache.insert(key, result);
1579        }
1580    }
1581}
1582
1583/// Increases selector cache miss metrics.
1584pub fn selector_result_cache_miss() {
1585    CACHE_MISS.with_label_values(&[SELECTOR_RESULT_TYPE]).inc()
1586}
1587
1588/// Increases selector cache hit metrics.
1589pub fn selector_result_cache_hit() {
1590    CACHE_HIT.with_label_values(&[SELECTOR_RESULT_TYPE]).inc()
1591}
1592
1593/// Builder to construct a [CacheManager].
1594#[derive(Default)]
1595pub struct CacheManagerBuilder {
1596    sst_meta_cache_size: u64,
1597    /// Reserves a quarter of `sst_meta_cache_size` for the native-reader metadata tier.
1598    /// Only enabled when the columnar series-index format is configured; otherwise the
1599    /// decoded tier keeps main's full half of the budget.
1600    enable_columnar_meta_cache: bool,
1601    disable_sst_meta_cache_on_write: bool,
1602    vector_cache_size: u64,
1603    page_cache_size: u64,
1604    index_metadata_size: u64,
1605    index_content_size: u64,
1606    index_content_page_size: u64,
1607    index_result_cache_size: u64,
1608    prefilter_result_cache_size: u64,
1609    puffin_metadata_size: u64,
1610    write_cache: Option<WriteCacheRef>,
1611    selector_result_cache_size: u64,
1612    range_result_cache_size: u64,
1613}
1614
1615impl CacheManagerBuilder {
1616    /// Sets meta cache size.
1617    pub fn sst_meta_cache_size(mut self, bytes: u64) -> Self {
1618        self.sst_meta_cache_size = bytes;
1619        self
1620    }
1621
1622    /// Enables the native-reader metadata tier, funded by a quarter of `sst_meta_cache_size`.
1623    pub fn enable_columnar_meta_cache(mut self, enable: bool) -> Self {
1624        self.enable_columnar_meta_cache = enable;
1625        self
1626    }
1627
1628    /// Keeps read-side metadata caching without prewarming newly written SSTs.
1629    pub(crate) fn disable_sst_meta_cache_on_write(mut self) -> Self {
1630        self.disable_sst_meta_cache_on_write = true;
1631        self
1632    }
1633
1634    /// Sets vector cache size.
1635    pub fn vector_cache_size(mut self, bytes: u64) -> Self {
1636        self.vector_cache_size = bytes;
1637        self
1638    }
1639
1640    /// Sets page cache size.
1641    pub fn page_cache_size(mut self, bytes: u64) -> Self {
1642        self.page_cache_size = bytes;
1643        self
1644    }
1645
1646    /// Sets write cache.
1647    pub fn write_cache(mut self, cache: Option<WriteCacheRef>) -> Self {
1648        self.write_cache = cache;
1649        self
1650    }
1651
1652    /// Sets cache size for index metadata.
1653    pub fn index_metadata_size(mut self, bytes: u64) -> Self {
1654        self.index_metadata_size = bytes;
1655        self
1656    }
1657
1658    /// Sets cache size for index content.
1659    pub fn index_content_size(mut self, bytes: u64) -> Self {
1660        self.index_content_size = bytes;
1661        self
1662    }
1663
1664    /// Sets page size for index content.
1665    pub fn index_content_page_size(mut self, bytes: u64) -> Self {
1666        self.index_content_page_size = bytes;
1667        self
1668    }
1669
1670    /// Sets cache size for index result.
1671    pub fn index_result_cache_size(mut self, bytes: u64) -> Self {
1672        self.index_result_cache_size = bytes;
1673        self
1674    }
1675
1676    /// Sets cache size for prefilter result.
1677    pub fn prefilter_result_cache_size(mut self, bytes: u64) -> Self {
1678        self.prefilter_result_cache_size = bytes;
1679        self
1680    }
1681
1682    /// Sets cache size for puffin metadata.
1683    pub fn puffin_metadata_size(mut self, bytes: u64) -> Self {
1684        self.puffin_metadata_size = bytes;
1685        self
1686    }
1687
1688    /// Sets selector result cache size.
1689    pub fn selector_result_cache_size(mut self, bytes: u64) -> Self {
1690        self.selector_result_cache_size = bytes;
1691        self
1692    }
1693
1694    /// Sets range result cache size.
1695    pub fn range_result_cache_size(mut self, bytes: u64) -> Self {
1696        self.range_result_cache_size = bytes;
1697        self
1698    }
1699
1700    /// Builds the [CacheManager].
1701    pub fn build(self) -> CacheManager {
1702        self.build_with_sst_meta_size_metric(CACHE_BYTES.bind(&[SST_META_TYPE]))
1703    }
1704
1705    fn build_with_sst_meta_size_metric(
1706        self,
1707        sst_meta_size_metric: CacheSizeMetric,
1708    ) -> CacheManager {
1709        // Reserve half the configured capacity for the compact authoritative tier. This prevents
1710        // decoded acceleration entries from evicting metadata that would require storage I/O to
1711        // recover. Both reservations together equal the configured limit.
1712        let compact_meta_capacity = self.sst_meta_cache_size.div_ceil(2);
1713        // TODO(narthex-merge): native-reader metadata is a second acceleration tier that main's
1714        // typed decoded cache cannot hold; it takes a quarter of the same budget so the total
1715        // still matches the configured size. Re-unify the tiers once the loader accounting is
1716        // shared, and move this to a dedicated config knob instead of splitting the reservation.
1717        let columnar_meta_capacity = if self.enable_columnar_meta_cache {
1718            self.sst_meta_cache_size / 4
1719        } else {
1720            0
1721        };
1722        let decoded_meta_capacity = self.sst_meta_cache_size / 2 - columnar_meta_capacity;
1723        let sst_meta_cache = (compact_meta_capacity != 0).then(|| {
1724            Cache::builder()
1725                .max_capacity(compact_meta_capacity)
1726                .weigher(meta_cache_weight)
1727                .eviction_listener(|_k, _v, cause| {
1728                    CACHE_EVICTION
1729                        .with_label_values(&[SST_META_TYPE, removal_cause_str(cause)])
1730                        .inc();
1731                })
1732                .build()
1733        });
1734        let sst_meta_size_registration = sst_meta_cache
1735            .clone()
1736            .map(|cache| sst_meta_size_metric.register(move || cache.weighted_size()));
1737        let sst_decoded_meta_cache = (decoded_meta_capacity != 0).then(|| {
1738            common_cache::Cache::new(
1739                "sst_decoded_metadata",
1740                decoded_meta_capacity,
1741                decoded_meta_cache_weight,
1742                |_k, _v, cause| {
1743                    CACHE_EVICTION
1744                        .with_label_values(&[SST_META_DECODED_TYPE, cause.as_str()])
1745                        .inc();
1746                },
1747            )
1748            .with_size_metric(CACHE_BYTES.bind(&[SST_META_DECODED_TYPE]))
1749            .with_metrics(crate::metrics::cache_loading_metrics(SST_META_DECODED_TYPE))
1750        });
1751        let columnar_meta_cache = (columnar_meta_capacity != 0).then(|| {
1752            Cache::builder()
1753                .max_capacity(columnar_meta_capacity)
1754                .weigher(columnar_meta_weight)
1755                .eviction_listener(|k, v, cause| {
1756                    let size = columnar_meta_weight(&k, &v);
1757                    CACHE_BYTES.with_label_values(&[v.label()]).sub(size.into());
1758                    CACHE_EVICTION
1759                        .with_label_values(&[v.label(), removal_cause_str(cause)])
1760                        .inc();
1761                })
1762                .build()
1763        });
1764        let vector_cache = (self.vector_cache_size != 0).then(|| {
1765            Cache::builder()
1766                .max_capacity(self.vector_cache_size)
1767                .weigher(vector_cache_weight)
1768                .eviction_listener(|k, v, cause| {
1769                    let size = vector_cache_weight(&k, &v);
1770                    CACHE_BYTES
1771                        .with_label_values(&[VECTOR_TYPE])
1772                        .sub(size.into());
1773                    CACHE_EVICTION
1774                        .with_label_values(&[VECTOR_TYPE, removal_cause_str(cause)])
1775                        .inc();
1776                })
1777                .build()
1778        });
1779        let page_cache =
1780            (self.page_cache_size != 0).then(|| ParquetPageCache::from_env(self.page_cache_size));
1781        let inverted_index_cache = InvertedIndexCache::new(
1782            self.index_metadata_size,
1783            self.index_content_size,
1784            self.index_content_page_size,
1785        );
1786        // TODO(ruihang): check if it's ok to reuse the same param with inverted index
1787        let bloom_filter_index_cache = BloomFilterIndexCache::new(
1788            self.index_metadata_size,
1789            self.index_content_size,
1790            self.index_content_page_size,
1791        );
1792        let index_result_cache = (self.index_result_cache_size != 0)
1793            .then(|| IndexResultCache::new(self.index_result_cache_size));
1794        let prefilter_result_cache = (self.prefilter_result_cache_size != 0)
1795            .then(|| new_prefilter_result_cache(self.prefilter_result_cache_size));
1796        let puffin_metadata_cache =
1797            PuffinMetadataCache::new(self.puffin_metadata_size, &CACHE_BYTES)
1798                .with_load_metrics(crate::metrics::cache_loading_metrics("puffin_metadata"));
1799        let selector_result_cache = (self.selector_result_cache_size != 0).then(|| {
1800            Cache::builder()
1801                .max_capacity(self.selector_result_cache_size)
1802                .weigher(selector_result_cache_weight)
1803                .eviction_listener(|k, v, cause| {
1804                    let size = selector_result_cache_weight(&k, &v);
1805                    CACHE_BYTES
1806                        .with_label_values(&[SELECTOR_RESULT_TYPE])
1807                        .sub(size.into());
1808                    CACHE_EVICTION
1809                        .with_label_values(&[SELECTOR_RESULT_TYPE, removal_cause_str(cause)])
1810                        .inc();
1811                })
1812                .build()
1813        });
1814        let range_result_cache = (self.range_result_cache_size != 0)
1815            .then(|| range_cache::RangeResultCache::new(self.range_result_cache_size));
1816        CacheManager {
1817            sst_meta_cache,
1818            _sst_meta_size_registration: sst_meta_size_registration,
1819            disable_sst_meta_cache_on_write: self.disable_sst_meta_cache_on_write,
1820            sst_decoded_meta_cache,
1821            columnar_meta_cache,
1822            vector_cache,
1823            page_cache,
1824            write_cache: self.write_cache,
1825            inverted_index_cache: Some(Arc::new(inverted_index_cache)),
1826            bloom_filter_index_cache: Some(Arc::new(bloom_filter_index_cache)),
1827            puffin_metadata_cache: Some(Arc::new(puffin_metadata_cache)),
1828            selector_result_cache,
1829            range_result_cache,
1830            #[cfg(test)]
1831            range_result_cache_size: self.range_result_cache_size,
1832            index_result_cache,
1833            prefilter_result_cache,
1834        }
1835    }
1836}
1837
1838fn meta_cache_weight(k: &SstMetaKey, v: &Arc<CompactSstMeta>) -> u32 {
1839    // We ignore the size of `Arc`. Region metadata is already present in the compressed Parquet
1840    // stream and is materialized only in the decoded acceleration tier.
1841    let size = k.estimated_size() + mem::size_of::<CompactSstMeta>() + v.encoded_metadata.len();
1842    u32::try_from(size).unwrap_or(u32::MAX)
1843}
1844
1845fn decoded_meta_cache_weight(k: &SstDecodedMetaKey, v: &Arc<SstMetaLoadResult>) -> u32 {
1846    SstMetaLoadResult::weight(k, &v.metadata)
1847}
1848
1849fn vector_cache_weight(_k: &(ConcreteDataType, Value), v: &VectorRef) -> u32 {
1850    // We ignore the heap size of `Value`.
1851    (mem::size_of::<ConcreteDataType>() + mem::size_of::<Value>() + v.memory_size()) as u32
1852}
1853
1854fn selector_result_cache_weight(k: &SelectorResultKey, v: &Arc<SelectorResultValue>) -> u32 {
1855    (mem::size_of_val(k) + v.estimated_size()) as u32
1856}
1857
1858fn range_result_cache_weight(k: &RangeScanCacheKey, v: &Arc<RangeScanCacheValue>) -> u32 {
1859    (k.estimated_size() + v.estimated_size()) as u32
1860}
1861
1862/// Updates cache hit/miss metrics.
1863fn update_hit_miss<T>(value: Option<T>, cache_type: &str) -> Option<T> {
1864    if value.is_some() {
1865        CACHE_HIT.with_label_values(&[cache_type]).inc();
1866    } else {
1867        CACHE_MISS.with_label_values(&[cache_type]).inc();
1868    }
1869    value
1870}
1871
1872/// Cache key (region id, file id) for SST meta.
1873#[derive(Debug, Clone, PartialEq, Eq, Hash)]
1874pub(crate) struct SstMetaKey(RegionId, FileId);
1875
1876impl SstMetaKey {
1877    /// Returns memory used by the key (estimated).
1878    fn estimated_size(&self) -> usize {
1879        mem::size_of::<Self>()
1880    }
1881}
1882
1883/// Cache key for time series row selector result.
1884#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1885pub struct SelectorResultKey {
1886    /// Id of the SST file.
1887    pub file_id: FileId,
1888    /// Index of the row group.
1889    pub row_group_idx: usize,
1890    /// Time series row selector.
1891    pub selector: TimeSeriesRowSelector,
1892}
1893
1894/// Result stored in the selector result cache.
1895pub enum SelectorResult {
1896    /// Batches in the primary key format.
1897    PrimaryKey(Vec<Batch>),
1898    /// Record batches in the flat format.
1899    Flat(Vec<RecordBatch>),
1900}
1901
1902/// Cached result for time series row selector.
1903pub struct SelectorResultValue {
1904    /// Batches of rows selected by the selector.
1905    pub result: SelectorResult,
1906    /// The read columns of rows.
1907    pub read_cols: ParquetReadColumns,
1908    /// JSON2 target types used by flat-format reads.
1909    ///
1910    /// JSON2 projection is query-driven; the same parquet columns can produce
1911    /// different cached batches under different type hints.
1912    pub json_target_types: JsonTargetTypes,
1913}
1914
1915impl SelectorResultValue {
1916    /// Creates a new selector result value with primary key format.
1917    pub fn new(result: Vec<Batch>, read_cols: ParquetReadColumns) -> SelectorResultValue {
1918        SelectorResultValue {
1919            result: SelectorResult::PrimaryKey(result),
1920            read_cols,
1921            json_target_types: Arc::default(),
1922        }
1923    }
1924
1925    /// Creates a new selector result value with flat format.
1926    pub fn new_flat(
1927        result: Vec<RecordBatch>,
1928        read_cols: ParquetReadColumns,
1929        json_target_types: JsonTargetTypes,
1930    ) -> SelectorResultValue {
1931        SelectorResultValue {
1932            result: SelectorResult::Flat(result),
1933            read_cols,
1934            json_target_types,
1935        }
1936    }
1937
1938    /// Returns memory used by the value (estimated).
1939    fn estimated_size(&self) -> usize {
1940        let result_size: usize = match &self.result {
1941            SelectorResult::PrimaryKey(batches) => {
1942                batches.iter().map(|batch| batch.memory_size()).sum()
1943            }
1944            SelectorResult::Flat(batches) => batches.iter().map(record_batch_estimated_size).sum(),
1945        };
1946        result_size
1947            + self.json_target_types.len() * (size_of::<ColumnId>() + size_of::<JsonNativeType>())
1948    }
1949}
1950
1951/// A caller's memory observation. Holding the entry avoids a second lookup after coordination.
1952pub(crate) enum SstMetaObservation {
1953    Decoded(Arc<CachedSstMeta>),
1954    Compact(Arc<CompactSstMeta>),
1955    Miss,
1956}
1957
1958impl SstMetaObservation {
1959    fn record(&self, metrics: &mut MetadataCacheMetrics) {
1960        match self {
1961            Self::Decoded(_) | Self::Compact(_) => {
1962                metrics.mem_cache_hit += 1;
1963                CACHE_HIT.with_label_values(&[SST_META_TYPE]).inc();
1964                match self {
1965                    Self::Decoded(_) => CACHE_HIT.with_label_values(&[SST_META_DECODED_TYPE]).inc(),
1966                    _ => CACHE_MISS.with_label_values(&[SST_META_DECODED_TYPE]).inc(),
1967                }
1968            }
1969            Self::Miss => CACHE_MISS.with_label_values(&[SST_META_TYPE]).inc(),
1970        }
1971    }
1972}
1973
1974/// Resolved cache source, not physical I/O. Also returned when the subsequent load fails.
1975/// A caller that observed a memory miss can find a completed value at the native recheck;
1976/// it reuses that value's source without executing its initializer. This does not prove
1977/// that the caller waited for an in-flight load or performed a file-cache lookup.
1978/// Ordinary memory hits return before this resolution is applied.
1979/// A default value means no file-cache resolution was needed (or known).
1980#[derive(Clone, Copy, Default)]
1981pub(crate) struct SstMetaResolution {
1982    pub(crate) file_cache_hit: usize,
1983    pub(crate) cache_miss: usize,
1984}
1985
1986struct SstMetaLoadError {
1987    source: Arc<crate::error::Error>,
1988    resolution: SstMetaResolution,
1989}
1990
1991/// Metadata returned by a shared load. Cache hits ignore the original load's miss flag.
1992struct SstMetaLoadResult {
1993    metadata: Arc<CachedSstMeta>,
1994    /// Preserves background-download eligibility for all requests sharing a storage load.
1995    cache_miss: bool,
1996    resolution: SstMetaResolution,
1997}
1998
1999impl SstMetaLoadResult {
2000    fn weight(key: &SstDecodedMetaKey, metadata: &CachedSstMeta) -> u32 {
2001        let size = size_of_val(key)
2002            + size_of::<Self>()
2003            + metadata.parquet_metadata_size
2004            + metadata.region_metadata_weight;
2005        u32::try_from(size).unwrap_or(u32::MAX)
2006    }
2007
2008    fn new(
2009        metadata: Arc<CachedSstMeta>,
2010        cache_miss: bool,
2011        resolution: SstMetaResolution,
2012    ) -> Arc<Self> {
2013        Arc::new(Self {
2014            metadata,
2015            cache_miss,
2016            resolution,
2017        })
2018    }
2019}
2020
2021/// Separates loads that require different page-index guarantees.
2022#[derive(Clone, Debug, PartialEq, Eq, Hash)]
2023struct SstDecodedMetaKey {
2024    file: SstMetaKey,
2025    policy: mem::Discriminant<PageIndexPolicy>,
2026}
2027
2028impl SstDecodedMetaKey {
2029    fn new(file: RegionFileId, policy: PageIndexPolicy) -> Self {
2030        Self {
2031            file: SstMetaKey(file.region_id(), file.file_id()),
2032            policy: mem::discriminant(&policy),
2033        }
2034    }
2035}
2036
2037/// Maps (region id, file id) to compact SST metadata.
2038type SstMetaCache = Cache<SstMetaKey, Arc<CompactSstMeta>>;
2039/// Maps (region id, file id) to decoded SST metadata retained as an acceleration tier.
2040type SstDecodedMetaCache = common_cache::Cache<SstDecodedMetaKey, Arc<SstMetaLoadResult>>;
2041/// Maps native-reader metadata identity to validated immutable structures.
2042pub(crate) type ColumnarMetaCache = Cache<DecodedKey, DecodedValue>;
2043/// Maps [Value] to a vector that holds this value repeatedly.
2044///
2045/// e.g. `"hello" => ["hello", "hello", "hello"]`
2046type VectorCache = Cache<(ConcreteDataType, Value), VectorRef>;
2047/// Maps (file id, row group id, time series row selector) to [SelectorResultValue].
2048type SelectorResultCache = Cache<SelectorResultKey, Arc<SelectorResultValue>>;
2049/// Maps partition-range scan key to cached flat batches.
2050type RangeResultCacheRef = Arc<range_cache::RangeResultCache>;
2051
2052#[cfg(test)]
2053mod tests {
2054    use std::sync::Arc;
2055
2056    use api::v1::SemanticType;
2057    use api::v1::index::{BloomFilterMeta, InvertedIndexMetas};
2058    use datatypes::schema::ColumnSchema;
2059    use datatypes::vectors::Int64Vector;
2060    use parquet::file::page_index::offset_index::{OffsetIndexMetaData, PageLocation};
2061    use puffin::file_metadata::FileMetadata;
2062    use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder};
2063    use store_api::storage::ColumnId;
2064
2065    use super::*;
2066    use crate::cache::index::bloom_filter_index::Tag;
2067    use crate::cache::index::result_cache::PredicateKey;
2068    use crate::cache::test_util::{
2069        parquet_meta, sst_parquet_meta, sst_parquet_meta_with_region_metadata,
2070    };
2071    use crate::read::range_cache::{RangeScanCacheKey, ScanRequestFingerprintBuilder};
2072    use crate::read::read_columns::ReadColumns;
2073    use crate::sst::parquet::row_selection::RowGroupSelection;
2074
2075    #[test]
2076    fn test_find_shared_page_source() {
2077        let source = Arc::new(
2078            UnexpectedSnafu {
2079                reason: "page read failed",
2080            }
2081            .build(),
2082        );
2083        let direct = io::Error::other(source.clone());
2084        let nested = io::Error::other(Arc::new(io::Error::other(direct)));
2085        let retained = find_shared_page_source::<crate::error::Error>(&nested).unwrap();
2086        assert!(Arc::ptr_eq(&source, &retained));
2087        assert!(
2088            find_shared_page_source::<crate::error::Error>(&io::Error::other("untyped error"))
2089                .is_none()
2090        );
2091    }
2092
2093    #[tokio::test]
2094    async fn test_disable_cache() {
2095        let cache = CacheManager::default();
2096        assert!(cache.sst_meta_cache.is_none());
2097        assert!(cache.sst_decoded_meta_cache.is_none());
2098        assert!(cache.vector_cache.is_none());
2099        assert!(cache.page_cache.is_none());
2100
2101        let region_id = RegionId::new(1, 1);
2102        let file_id = RegionFileId::new(region_id, FileId::random());
2103        let metadata = parquet_meta();
2104        let mut metrics = MetadataCacheMetrics::default();
2105        cache.put_parquet_meta_data(file_id, metadata, None).await;
2106        assert!(
2107            cache
2108                .get_sst_meta_data(
2109                    file_id,
2110                    &mut metrics,
2111                    Default::default(),
2112                    &common_runtime::global_runtime()
2113                )
2114                .await
2115                .is_none()
2116        );
2117
2118        let value = Value::Int64(10);
2119        let vector: VectorRef = Arc::new(Int64Vector::from_slice([10, 10, 10, 10]));
2120        cache.put_repeated_vector(value.clone(), vector.clone());
2121        assert!(
2122            cache
2123                .get_repeated_vector(&ConcreteDataType::int64_datatype(), &value)
2124                .is_none()
2125        );
2126
2127        assert!(cache.page_cache.is_none());
2128
2129        assert!(cache.write_cache().is_none());
2130    }
2131
2132    #[tokio::test]
2133    async fn test_sst_meta_load_coalescing() {
2134        use std::sync::atomic::{AtomicUsize, Ordering};
2135
2136        let manager = Arc::new(CacheManager::builder().sst_meta_cache_size(2).build());
2137        let strategy = CacheStrategy::EnableAll(manager.clone());
2138        let (metadata, region) = sst_parquet_meta();
2139        let metadata = Arc::new(
2140            CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap(),
2141        );
2142        let file_id = RegionFileId::new(region.region_id, FileId::random());
2143        let calls = AtomicUsize::new(0);
2144        let gate = tokio::sync::Notify::new();
2145        let load = || async {
2146            calls.fetch_add(1, Ordering::Relaxed);
2147            gate.notified().await;
2148            Ok((metadata.clone(), true))
2149        };
2150        let mut first =
2151            Box::pin(strategy.get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, load()));
2152        let mut second =
2153            Box::pin(strategy.get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, load()));
2154        assert!(futures::poll!(&mut first).is_pending());
2155        assert!(futures::poll!(&mut second).is_pending());
2156        assert_eq!(1, calls.load(Ordering::Relaxed));
2157        gate.notify_waiters();
2158        let (first, second) = tokio::join!(first, second);
2159        let first = first.unwrap();
2160        let second = second.unwrap();
2161        assert!(Arc::ptr_eq(&first.0, &second.0));
2162        assert!(
2163            first.1 && second.1,
2164            "all waiters preserve the remote miss flag"
2165        );
2166
2167        // Moka applies capacity limits during maintenance, not synchronously on insertion.
2168        manager
2169            .sst_decoded_meta_cache
2170            .as_ref()
2171            .unwrap()
2172            .run_pending_tasks()
2173            .await;
2174        // An entry larger than the decoded reservation must not be retained.
2175        let mut next =
2176            Box::pin(strategy.get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, load()));
2177        assert!(futures::poll!(&mut next).is_pending());
2178        assert_eq!(2, calls.load(Ordering::Relaxed));
2179        drop(next);
2180        manager
2181            .sst_decoded_meta_cache
2182            .as_ref()
2183            .unwrap()
2184            .run_pending_tasks()
2185            .await;
2186        assert_eq!(
2187            0,
2188            manager
2189                .sst_decoded_meta_cache
2190                .as_ref()
2191                .unwrap()
2192                .entry_count()
2193        );
2194    }
2195
2196    #[tokio::test]
2197    async fn test_compact_sst_meta_stronger_policy_reused_from_skip_key() {
2198        for policy in [PageIndexPolicy::Optional, PageIndexPolicy::Required] {
2199            let manager = Arc::new(
2200                CacheManager::builder()
2201                    .sst_meta_cache_size(1024 * 1024)
2202                    .build(),
2203            );
2204            let strategy = CacheStrategy::EnableAll(manager.clone());
2205            let (metadata, region) = sst_parquet_meta();
2206            let file_id = RegionFileId::new(region.region_id, FileId::random());
2207            let prepared = prepare_sst_meta(
2208                "test.parquet",
2209                with_offset_indexes(Arc::unwrap_or_clone(metadata)),
2210                None,
2211                policy,
2212                &common_runtime::global_runtime(),
2213            )
2214            .await
2215            .unwrap();
2216            let SstMetaPreparation::Prepared(prepared) = prepared else {
2217                panic!("valid metadata should produce a compact entry");
2218            };
2219            manager.put_compact_sst_meta(file_id, prepared.compact);
2220            let mut metrics = MetadataCacheMetrics::default();
2221            let observation = strategy
2222                .observe_sst_meta(file_id, PageIndexPolicy::Skip, &mut metrics)
2223                .await;
2224            assert!(matches!(&observation, SstMetaObservation::Compact(_)));
2225            let (decoded, _) = strategy
2226                .get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, async {
2227                    let value = strategy
2228                        .resolve_sst_meta(
2229                            observation,
2230                            file_id,
2231                            &mut MetadataCacheMetrics::default(),
2232                            PageIndexPolicy::Skip,
2233                        )
2234                        .await
2235                        .expect("compact entry must decode without storage I/O");
2236                    Ok((value, false))
2237                })
2238                .await
2239                .unwrap();
2240            assert!(decoded.satisfies_page_index_policy(policy));
2241            let cache = manager.sst_decoded_meta_cache.as_ref().unwrap();
2242            cache.run_pending_tasks().await;
2243            let weight = cache.weighted_size();
2244            assert_eq!(cache.entry_count(), 1);
2245
2246            // Losing the compact copy must not force another decode or storage load.
2247            manager
2248                .sst_meta_cache
2249                .as_ref()
2250                .unwrap()
2251                .invalidate(&SstMetaKey(file_id.region_id(), file_id.file_id()));
2252            for requested in [PageIndexPolicy::Optional, policy] {
2253                let mut metrics = MetadataCacheMetrics::default();
2254                let observation = strategy
2255                    .observe_sst_meta(file_id, requested, &mut metrics)
2256                    .await;
2257                let SstMetaObservation::Decoded(cached) = observation else {
2258                    panic!("the completed Skip-key entry already satisfies this request");
2259                };
2260                assert!(Arc::ptr_eq(&decoded, &cached));
2261                assert_eq!(metrics.mem_cache_hit, 1);
2262                assert_eq!(metrics.cache_miss, 0);
2263            }
2264            cache.run_pending_tasks().await;
2265            assert_eq!(cache.entry_count(), 1);
2266            assert_eq!(cache.weighted_size(), weight);
2267            assert!(
2268                cache
2269                    .get(&SstDecodedMetaKey::new(file_id, PageIndexPolicy::Optional))
2270                    .await
2271                    .is_none()
2272            );
2273            assert!(
2274                cache
2275                    .get(&SstDecodedMetaKey::new(file_id, PageIndexPolicy::Required))
2276                    .await
2277                    .is_none()
2278            );
2279        }
2280    }
2281
2282    #[tokio::test]
2283    async fn test_async_sst_meta_policy_reuse_and_invalidation() {
2284        let manager = Arc::new(
2285            CacheManager::builder()
2286                .sst_meta_cache_size(1024 * 1024)
2287                .build(),
2288        );
2289        let strategy = CacheStrategy::EnableAll(manager.clone());
2290        let (parquet_metadata, region) = sst_parquet_meta();
2291        let file_id = RegionFileId::new(region.region_id, FileId::random());
2292        let policies = [
2293            PageIndexPolicy::Skip,
2294            PageIndexPolicy::Optional,
2295            PageIndexPolicy::Required,
2296        ];
2297        for loaded in policies {
2298            let metadata = Arc::new(
2299                CachedSstMeta::try_new_with_page_index_policy(
2300                    "test.parquet",
2301                    (*parquet_metadata).clone(),
2302                    None,
2303                    loaded,
2304                )
2305                .unwrap(),
2306            );
2307            let (result, miss) = strategy
2308                .get_or_load_sst_meta(file_id, loaded, async { Ok((metadata.clone(), true)) })
2309                .await
2310                .unwrap();
2311            assert!(Arc::ptr_eq(&metadata, &result));
2312            assert!(miss);
2313            for requested in policies {
2314                let mut metrics = MetadataCacheMetrics::default();
2315                let cached = strategy
2316                    .observe_sst_meta(file_id, requested, &mut metrics)
2317                    .await;
2318                let cached = match cached {
2319                    SstMetaObservation::Decoded(value) => Some(value),
2320                    _ => None,
2321                };
2322                assert_eq!(
2323                    satisfies_page_index_policy(loaded, requested),
2324                    cached.is_some()
2325                );
2326                if let Some(cached) = cached {
2327                    assert!(Arc::ptr_eq(&metadata, &cached));
2328                    assert_eq!(1, metrics.mem_cache_hit);
2329                }
2330            }
2331            manager.remove_parquet_meta_data(file_id).await;
2332        }
2333
2334        // Populate all policy keys before removing the file, not just the last inserted key.
2335        for loaded in policies {
2336            let metadata = Arc::new(
2337                CachedSstMeta::try_new_with_page_index_policy(
2338                    "test.parquet",
2339                    (*parquet_metadata).clone(),
2340                    None,
2341                    loaded,
2342                )
2343                .unwrap(),
2344            );
2345            manager.put_sst_meta_data(file_id, metadata).await;
2346        }
2347        manager.remove_parquet_meta_data(file_id).await;
2348        for requested in policies {
2349            assert!(
2350                manager
2351                    .get_sst_meta_data_from_mem_cache(file_id, requested)
2352                    .await
2353                    .is_none()
2354            );
2355        }
2356    }
2357
2358    #[tokio::test]
2359    async fn test_sst_meta_load_cancellation_and_policy_isolation() {
2360        let strategy = CacheStrategy::EnableAll(Arc::new(
2361            CacheManager::builder().sst_meta_cache_size(2000).build(),
2362        ));
2363        let (metadata, region) = sst_parquet_meta();
2364        let metadata = Arc::new(
2365            CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap(),
2366        );
2367        let file_id = RegionFileId::new(region.region_id, FileId::random());
2368        let mut first = Box::pin(strategy.get_or_load_sst_meta(
2369            file_id,
2370            PageIndexPolicy::Skip,
2371            std::future::pending(),
2372        ));
2373        assert!(futures::poll!(&mut first).is_pending());
2374        // A different policy or file does not wait for this load.
2375        for (id, policy) in [
2376            (file_id, PageIndexPolicy::Optional),
2377            (file_id, PageIndexPolicy::Required),
2378            (
2379                RegionFileId::new(region.region_id, FileId::random()),
2380                PageIndexPolicy::Skip,
2381            ),
2382        ] {
2383            let mut independent = Box::pin(
2384                strategy.get_or_load_sst_meta(id, policy, async { Ok((metadata.clone(), false)) }),
2385            );
2386            assert!(futures::poll!(&mut independent).is_ready());
2387        }
2388        let mut waiter = Box::pin(strategy.get_or_load_sst_meta(
2389            file_id,
2390            PageIndexPolicy::Skip,
2391            async { Ok((metadata.clone(), false)) },
2392        ));
2393        assert!(futures::poll!(&mut waiter).is_pending());
2394        drop(first);
2395        let result = tokio::time::timeout(std::time::Duration::from_secs(5), waiter)
2396            .await
2397            .unwrap()
2398            .unwrap();
2399        assert!(Arc::ptr_eq(&metadata, &result.0));
2400    }
2401
2402    #[tokio::test]
2403    async fn test_sst_meta_load_failure_retry_and_disabled_cache() {
2404        use common_error::ext::ErrorExt;
2405
2406        let manager = Arc::new(CacheManager::builder().sst_meta_cache_size(2000).build());
2407        let strategy = CacheStrategy::EnableAll(manager);
2408        let (metadata, region) = sst_parquet_meta();
2409        let metadata = Arc::new(
2410            CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap(),
2411        );
2412        let file_id = RegionFileId::new(region.region_id, FileId::random());
2413        let gate = tokio::sync::Notify::new();
2414        let mut first = Box::pin(strategy.get_or_load_observed_sst_meta(
2415            file_id,
2416            PageIndexPolicy::Skip,
2417            async {
2418                gate.notified().await;
2419                (
2420                    UnexpectedSnafu {
2421                        reason: "load failed",
2422                    }
2423                    .fail(),
2424                    SstMetaResolution {
2425                        file_cache_hit: 0,
2426                        cache_miss: 1,
2427                    },
2428                )
2429            },
2430        ));
2431        let mut waiter = Box::pin(strategy.get_or_load_observed_sst_meta(
2432            file_id,
2433            PageIndexPolicy::Skip,
2434            async { panic!("waiter must share the failure") },
2435        ));
2436        assert!(futures::poll!(&mut first).is_pending());
2437        assert!(futures::poll!(&mut waiter).is_pending());
2438        gate.notify_waiters();
2439        let (first, first_resolution) = first.await;
2440        let (second, second_resolution) = waiter.await;
2441        for resolution in [first_resolution, second_resolution] {
2442            assert_eq!(1, resolution.cache_miss);
2443            assert_eq!(0, resolution.file_cache_hit);
2444        }
2445        let first = first.unwrap_err();
2446        let second = second.unwrap_err();
2447        assert_eq!(first.status_code(), second.status_code());
2448        match (first, second) {
2449            (
2450                crate::error::Error::LoadSstMeta { source: a, .. },
2451                crate::error::Error::LoadSstMeta { source: b, .. },
2452            ) => assert!(Arc::ptr_eq(&a, &b)),
2453            errors => panic!("unexpected errors: {errors:?}"),
2454        }
2455        assert!(
2456            strategy
2457                .get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, async {
2458                    Ok((metadata.clone(), true))
2459                })
2460                .await
2461                .is_ok()
2462        );
2463
2464        for strategy in [
2465            CacheStrategy::Disabled,
2466            CacheStrategy::EnableAll(Arc::new(CacheManager::default())),
2467            CacheStrategy::EnableAll(Arc::new(
2468                CacheManager::builder().sst_meta_cache_size(1).build(),
2469            )),
2470        ] {
2471            let mut blocked = Box::pin(strategy.get_or_load_sst_meta(
2472                file_id,
2473                PageIndexPolicy::Skip,
2474                std::future::pending(),
2475            ));
2476            assert!(futures::poll!(&mut blocked).is_pending());
2477            let mut independent = Box::pin(strategy.get_or_load_sst_meta(
2478                file_id,
2479                PageIndexPolicy::Skip,
2480                async { Ok((metadata.clone(), true)) },
2481            ));
2482            assert!(futures::poll!(&mut independent).is_ready());
2483        }
2484    }
2485
2486    #[test]
2487    fn test_compact_sst_meta_bytes_follow_cache_lifetime() {
2488        use common_cache::metrics::CacheSizeMetrics;
2489        use prometheus::Opts;
2490        use prometheus::core::Collector;
2491
2492        let metrics =
2493            CacheSizeMetrics::new(Opts::new("test_compact_sst_meta_bytes", "test"), &["type"])
2494                .unwrap();
2495        let (metadata, region) = sst_parquet_meta();
2496        let SstMetaPreparation::Prepared(prepared) = prepare_sst_meta_sync(
2497            "test.parquet",
2498            Arc::unwrap_or_clone(metadata),
2499            None,
2500            PageIndexPolicy::Skip,
2501        )
2502        .unwrap() else {
2503            panic!("valid metadata should produce a compact entry");
2504        };
2505        let file_id = RegionFileId::new(region.region_id, FileId::random());
2506        let expected_bytes = f64::from(meta_cache_weight(
2507            &SstMetaKey(file_id.region_id(), file_id.file_id()),
2508            &prepared.compact,
2509        ));
2510        let make_cache = || {
2511            let manager = Arc::new(
2512                CacheManager::builder()
2513                    .sst_meta_cache_size(1024 * 1024)
2514                    .build_with_sst_meta_size_metric(metrics.bind(&[SST_META_TYPE])),
2515            );
2516            manager.put_compact_sst_meta(file_id, prepared.compact.clone());
2517            manager.sst_meta_cache.as_ref().unwrap().run_pending_tasks();
2518            manager
2519        };
2520        let sample = || {
2521            metrics.collect()[0].get_metric()[0]
2522                .get_gauge()
2523                .as_ref()
2524                .unwrap()
2525                .value()
2526        };
2527        let resident = make_cache();
2528        assert!(expected_bytes > 0.0);
2529        assert_eq!(expected_bytes, sample());
2530
2531        // Repeated remote compactions must not leave byte counts after releasing their caches.
2532        for _ in 0..3 {
2533            let transient = make_cache();
2534            assert_eq!(2.0 * expected_bytes, sample());
2535            let reader = transient.clone();
2536            drop(transient);
2537            assert_eq!(2.0 * expected_bytes, sample());
2538            drop(reader);
2539            assert_eq!(expected_bytes, sample());
2540        }
2541        drop(resident);
2542        assert_eq!(0.0, sample());
2543    }
2544
2545    #[test]
2546    fn test_sst_meta_cache_splits_capacity() {
2547        // Default (columnar format off): main's split, the native-reader tier is not created.
2548        let cache = CacheManager::builder().sst_meta_cache_size(101).build();
2549        assert_eq!(
2550            50,
2551            cache.sst_decoded_meta_cache.as_ref().unwrap().capacity()
2552        );
2553        assert!(cache.columnar_meta_cache.is_none());
2554
2555        // Columnar format on: a quarter of the budget funds the native-reader tier.
2556        let cache = CacheManager::builder()
2557            .sst_meta_cache_size(101)
2558            .enable_columnar_meta_cache(true)
2559            .build();
2560
2561        assert_eq!(
2562            Some(51),
2563            cache
2564                .sst_meta_cache
2565                .as_ref()
2566                .unwrap()
2567                .policy()
2568                .max_capacity()
2569        );
2570        // The acceleration half is split between the decoded and native-reader tiers.
2571        assert_eq!(
2572            25,
2573            cache.sst_decoded_meta_cache.as_ref().unwrap().capacity()
2574        );
2575        assert_eq!(
2576            Some(25),
2577            cache
2578                .columnar_meta_cache
2579                .as_ref()
2580                .unwrap()
2581                .policy()
2582                .max_capacity()
2583        );
2584    }
2585
2586    #[tokio::test]
2587    async fn test_parquet_meta_cache() {
2588        let cache = CacheManager::builder().sst_meta_cache_size(2000).build();
2589        let mut metrics = MetadataCacheMetrics::default();
2590        let region_id = RegionId::new(1, 1);
2591        let file_id = RegionFileId::new(region_id, FileId::random());
2592        assert!(
2593            cache
2594                .get_sst_meta_data(
2595                    file_id,
2596                    &mut metrics,
2597                    Default::default(),
2598                    &common_runtime::global_runtime()
2599                )
2600                .await
2601                .is_none()
2602        );
2603        let (metadata, region_metadata) = sst_parquet_meta();
2604        cache.put_parquet_meta_data(file_id, metadata, None).await;
2605        let cached = cache
2606            .get_sst_meta_data(
2607                file_id,
2608                &mut metrics,
2609                Default::default(),
2610                &common_runtime::global_runtime(),
2611            )
2612            .await
2613            .unwrap();
2614        assert_eq!(region_metadata, cached.region_metadata());
2615        assert!(
2616            cached
2617                .parquet_metadata()
2618                .file_metadata()
2619                .key_value_metadata()
2620                .is_none_or(|key_values| {
2621                    key_values
2622                        .iter()
2623                        .all(|key_value| key_value.key != PARQUET_METADATA_KEY)
2624                })
2625        );
2626        cache.remove_parquet_meta_data(file_id).await;
2627        assert!(
2628            cache
2629                .get_sst_meta_data(
2630                    file_id,
2631                    &mut metrics,
2632                    Default::default(),
2633                    &common_runtime::global_runtime()
2634                )
2635                .await
2636                .is_none()
2637        );
2638    }
2639
2640    #[tokio::test]
2641    async fn test_parquet_meta_cache_with_provided_region_metadata() {
2642        let cache = CacheManager::builder().sst_meta_cache_size(2000).build();
2643        let mut metrics = MetadataCacheMetrics::default();
2644        let region_id = RegionId::new(1, 1);
2645        let file_id = RegionFileId::new(region_id, FileId::random());
2646        let (metadata, region_metadata) = sst_parquet_meta();
2647
2648        cache
2649            .put_parquet_meta_data(file_id, metadata, Some(region_metadata.clone()))
2650            .await;
2651
2652        let cached = cache
2653            .get_sst_meta_data(
2654                file_id,
2655                &mut metrics,
2656                Default::default(),
2657                &common_runtime::global_runtime(),
2658            )
2659            .await
2660            .unwrap();
2661        assert!(Arc::ptr_eq(&region_metadata, &cached.region_metadata()));
2662    }
2663
2664    fn with_offset_indexes(metadata: ParquetMetaData) -> ParquetMetaData {
2665        let offset_indexes = metadata
2666            .row_groups()
2667            .iter()
2668            .map(|row_group| {
2669                row_group
2670                    .columns()
2671                    .iter()
2672                    .map(|_| OffsetIndexMetaData {
2673                        page_locations: vec![PageLocation {
2674                            offset: 42,
2675                            compressed_page_size: 128,
2676                            first_row_index: 0,
2677                        }],
2678                        unencoded_byte_array_data_bytes: None,
2679                    })
2680                    .collect()
2681            })
2682            .collect();
2683        metadata
2684            .into_builder()
2685            .set_offset_index(Some(offset_indexes))
2686            .build()
2687    }
2688
2689    #[tokio::test]
2690    async fn test_compact_sst_meta_round_trip() {
2691        let cache = CacheManager::builder()
2692            .sst_meta_cache_size(1024 * 1024)
2693            .build();
2694        let region_id = RegionId::new(1, 1);
2695        let file_id = RegionFileId::new(region_id, FileId::random());
2696        let (metadata, expected_region_metadata) = sst_parquet_meta();
2697        let metadata = Arc::unwrap_or_clone(metadata);
2698        let metadata = with_offset_indexes(metadata);
2699        let expected_rows = metadata.file_metadata().num_rows();
2700        let prepared = prepare_sst_meta(
2701            "test.parquet",
2702            metadata,
2703            None,
2704            PageIndexPolicy::Required,
2705            &common_runtime::global_runtime(),
2706        )
2707        .await
2708        .unwrap();
2709        let SstMetaPreparation::Prepared(prepared) = prepared else {
2710            panic!("valid metadata should produce a compact cache entry");
2711        };
2712
2713        // Retain only the compact form so the lookup must exercise decompression and decoding.
2714        cache.put_compact_sst_meta(file_id, prepared.compact);
2715        let mut metrics = MetadataCacheMetrics::default();
2716        let cached = cache
2717            .get_sst_meta_data(
2718                file_id,
2719                &mut metrics,
2720                PageIndexPolicy::Required,
2721                &common_runtime::global_runtime(),
2722            )
2723            .await
2724            .unwrap();
2725
2726        assert_eq!(1, metrics.mem_cache_hit);
2727        assert_eq!(0, metrics.cache_miss);
2728        assert_eq!(
2729            expected_rows,
2730            cached.parquet_metadata.file_metadata().num_rows()
2731        );
2732        assert!(cached.parquet_metadata.offset_index().is_some());
2733        assert!(
2734            cache
2735                .sst_decoded_meta_cache
2736                .as_ref()
2737                .unwrap()
2738                .get(&SstDecodedMetaKey::new(file_id, PageIndexPolicy::Required))
2739                .await
2740                .is_none(),
2741            "compact inspection must not fill the decoded reservation"
2742        );
2743        assert_eq!(expected_region_metadata, cached.region_metadata());
2744        assert!(
2745            cached
2746                .parquet_metadata
2747                .file_metadata()
2748                .key_value_metadata()
2749                .is_none_or(|key_values| key_values
2750                    .iter()
2751                    .all(|key_value| key_value.key != PARQUET_METADATA_KEY))
2752        );
2753    }
2754
2755    #[tokio::test]
2756    async fn test_cache_encoding_failure_preserves_decoded_metadata() {
2757        let (metadata, expected_region_metadata) = sst_parquet_meta();
2758        let expected_rows = metadata.file_metadata().num_rows();
2759        let cache_encoding: Result<(Bytes, usize)> = Err(UnexpectedSnafu {
2760            reason: "injected compact metadata encoding failure",
2761        }
2762        .build());
2763
2764        let preparation = finish_sst_meta_preparation(
2765            "test.parquet",
2766            Arc::unwrap_or_clone(metadata),
2767            None,
2768            PageIndexPolicy::Skip,
2769            cache_encoding,
2770        )
2771        .unwrap();
2772        let SstMetaPreparation::DecodedOnly {
2773            decoded,
2774            encoding_error,
2775        } = preparation
2776        else {
2777            panic!("cache encoding failure should return decoded-only metadata");
2778        };
2779
2780        assert_eq!(
2781            expected_rows,
2782            decoded.parquet_metadata.file_metadata().num_rows()
2783        );
2784        assert_eq!(expected_region_metadata, decoded.region_metadata());
2785        assert!(
2786            encoding_error
2787                .to_string()
2788                .contains("injected compact metadata encoding failure")
2789        );
2790        let file_id = RegionFileId::new(expected_region_metadata.region_id, FileId::random());
2791        let manager = Arc::new(
2792            CacheManager::builder()
2793                .sst_meta_cache_size(1024 * 1024)
2794                .build(),
2795        );
2796        let strategy = CacheStrategy::EnableAll(manager.clone());
2797        let (shared, miss) = strategy
2798            .get_or_load_sst_meta(file_id, PageIndexPolicy::Skip, async {
2799                Ok((decoded.clone(), true))
2800            })
2801            .await
2802            .unwrap();
2803        assert!(miss);
2804        assert!(Arc::ptr_eq(&decoded, &shared));
2805        assert!(
2806            manager
2807                .get_compact_sst_meta(&SstMetaKey(file_id.region_id(), file_id.file_id()))
2808                .is_none()
2809        );
2810        let mut metrics = MetadataCacheMetrics::default();
2811        let SstMetaObservation::Decoded(cached) = strategy
2812            .observe_sst_meta(file_id, PageIndexPolicy::Skip, &mut metrics)
2813            .await
2814        else {
2815            panic!("decoded fallback must be retained without a compact entry");
2816        };
2817        assert!(Arc::ptr_eq(&decoded, &cached));
2818        assert_eq!(1, metrics.mem_cache_hit);
2819        assert_eq!(0, metrics.cache_miss);
2820    }
2821
2822    #[tokio::test]
2823    async fn test_parquet_meta_cache_respects_page_index_policy() {
2824        let cache = CacheManager::builder().sst_meta_cache_size(2000).build();
2825        let region_id = RegionId::new(1, 1);
2826        let file_id = RegionFileId::new(region_id, FileId::random());
2827        let (metadata, _) = sst_parquet_meta();
2828
2829        let skip_metadata = Arc::new(
2830            CachedSstMeta::try_new_with_page_index_policy(
2831                "test.parquet",
2832                Arc::unwrap_or_clone(metadata.clone()),
2833                None,
2834                PageIndexPolicy::Skip,
2835            )
2836            .unwrap(),
2837        );
2838        cache.put_sst_meta_data(file_id, skip_metadata).await;
2839
2840        let mut metrics = MetadataCacheMetrics::default();
2841        assert!(
2842            cache
2843                .get_sst_meta_data(
2844                    file_id,
2845                    &mut metrics,
2846                    PageIndexPolicy::Optional,
2847                    &common_runtime::global_runtime()
2848                )
2849                .await
2850                .is_none()
2851        );
2852        assert_eq!(1, metrics.cache_miss);
2853
2854        let optional_metadata = Arc::new(
2855            CachedSstMeta::try_new_with_page_index_policy(
2856                "test.parquet",
2857                Arc::unwrap_or_clone(metadata),
2858                None,
2859                PageIndexPolicy::Optional,
2860            )
2861            .unwrap(),
2862        );
2863        cache.put_sst_meta_data(file_id, optional_metadata).await;
2864
2865        let mut metrics = MetadataCacheMetrics::default();
2866        assert!(
2867            cache
2868                .get_sst_meta_data(
2869                    file_id,
2870                    &mut metrics,
2871                    PageIndexPolicy::Optional,
2872                    &common_runtime::global_runtime()
2873                )
2874                .await
2875                .is_some()
2876        );
2877        assert_eq!(1, metrics.mem_cache_hit);
2878
2879        let mut metrics = MetadataCacheMetrics::default();
2880        assert!(
2881            cache
2882                .get_sst_meta_data(
2883                    file_id,
2884                    &mut metrics,
2885                    PageIndexPolicy::Skip,
2886                    &common_runtime::global_runtime()
2887                )
2888                .await
2889                .is_some()
2890        );
2891        assert_eq!(1, metrics.mem_cache_hit);
2892    }
2893
2894    #[test]
2895    fn test_decoded_meta_cache_weight_accounts_for_region_metadata() {
2896        let region_metadata = Arc::new(wide_region_metadata(128));
2897        let json_len = region_metadata.to_json().unwrap().len();
2898        let metadata = sst_parquet_meta_with_region_metadata(region_metadata.clone());
2899        let cached = Arc::new(
2900            CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap(),
2901        );
2902        let key = SstDecodedMetaKey::new(
2903            RegionFileId::new(region_metadata.region_id, FileId::random()),
2904            PageIndexPolicy::Skip,
2905        );
2906        let value = SstMetaLoadResult::new(cached.clone(), false, SstMetaResolution::default());
2907
2908        assert!(cached.region_metadata_weight > json_len);
2909        assert_eq!(
2910            decoded_meta_cache_weight(&key, &value) as usize,
2911            size_of_val(&key)
2912                + size_of::<SstMetaLoadResult>()
2913                + cached.parquet_metadata_size
2914                + cached.region_metadata_weight
2915        );
2916        assert_eq!(
2917            cached.parquet_metadata_size,
2918            parquet_meta_size(&cached.parquet_metadata)
2919        );
2920    }
2921
2922    #[test]
2923    fn test_decoded_meta_cache_weight_saturates_on_overflow() {
2924        let region_metadata = Arc::new(wide_region_metadata(1));
2925        let metadata = sst_parquet_meta_with_region_metadata(region_metadata.clone());
2926        let mut cached =
2927            CachedSstMeta::try_new("test.parquet", Arc::unwrap_or_clone(metadata)).unwrap();
2928        cached.region_metadata_weight = u32::MAX as usize + 1;
2929        let cached = Arc::new(cached);
2930        let key = SstDecodedMetaKey::new(
2931            RegionFileId::new(region_metadata.region_id, FileId::random()),
2932            PageIndexPolicy::Skip,
2933        );
2934        let value = SstMetaLoadResult::new(cached.clone(), false, SstMetaResolution::default());
2935
2936        assert_eq!(u32::MAX, decoded_meta_cache_weight(&key, &value));
2937    }
2938
2939    #[test]
2940    fn test_repeated_vector_cache() {
2941        let cache = CacheManager::builder().vector_cache_size(4096).build();
2942        let value = Value::Int64(10);
2943        assert!(
2944            cache
2945                .get_repeated_vector(&ConcreteDataType::int64_datatype(), &value)
2946                .is_none()
2947        );
2948        let vector: VectorRef = Arc::new(Int64Vector::from_slice([10, 10, 10, 10]));
2949        cache.put_repeated_vector(value.clone(), vector.clone());
2950        let cached = cache
2951            .get_repeated_vector(&ConcreteDataType::int64_datatype(), &value)
2952            .unwrap();
2953        assert_eq!(vector, cached);
2954    }
2955
2956    #[test]
2957    fn test_selector_result_cache() {
2958        let cache = CacheManager::builder()
2959            .selector_result_cache_size(1000)
2960            .build();
2961        let file_id = FileId::random();
2962        let key = SelectorResultKey {
2963            file_id,
2964            row_group_idx: 0,
2965            selector: TimeSeriesRowSelector::LastRow { after_merge: false },
2966        };
2967        assert!(cache.get_selector_result(&key).is_none());
2968        let result = Arc::new(SelectorResultValue::new(
2969            Vec::new(),
2970            ParquetReadColumns::from_deduped(Vec::new()),
2971        ));
2972        cache.put_selector_result(key, result);
2973        assert!(cache.get_selector_result(&key).is_some());
2974    }
2975
2976    #[test]
2977    fn test_prefilter_result_cache() {
2978        let disabled = CacheManager::builder().build();
2979        let file_id = FileId::random();
2980        let key = PrefilterKey::new(
2981            file_id,
2982            0,
2983            None,
2984            1,
2985            SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()]),
2986        );
2987        let selection = Arc::new(BooleanBuffer::new_set(3));
2988
2989        disabled.put_prefilter_result(key.clone(), selection.clone());
2990        assert!(disabled.get_prefilter_result(&key).is_none());
2991
2992        let cache = Arc::new(
2993            CacheManager::builder()
2994                .prefilter_result_cache_size(1000)
2995                .build(),
2996        );
2997        assert!(cache.get_prefilter_result(&key).is_none());
2998        cache.put_prefilter_result(key.clone(), selection.clone());
2999        assert_eq!(
3000            cache.get_prefilter_result(&key).unwrap().as_ref(),
3001            selection.as_ref()
3002        );
3003
3004        let enable_all = CacheStrategy::EnableAll(cache.clone());
3005        assert!(enable_all.get_prefilter_result(&key).is_some());
3006
3007        let compaction = CacheStrategy::Compaction(cache.clone());
3008        assert!(compaction.get_prefilter_result(&key).is_none());
3009        compaction.put_prefilter_result(key.clone(), selection.clone());
3010        assert!(cache.get_prefilter_result(&key).is_some());
3011
3012        let disabled_strategy = CacheStrategy::Disabled;
3013        assert!(disabled_strategy.get_prefilter_result(&key).is_none());
3014        disabled_strategy.put_prefilter_result(key.clone(), selection);
3015        assert!(cache.get_prefilter_result(&key).is_some());
3016    }
3017
3018    #[test]
3019    fn test_prefilter_key_distinguishes_dimensions() {
3020        let file_id = FileId::random();
3021        let row_selection = RowSelection::from(vec![RowSelector::skip(1), RowSelector::select(3)]);
3022        let other_row_selection =
3023            RowSelection::from(vec![RowSelector::skip(2), RowSelector::select(2)]);
3024        let row_selection = PrefilterKey::row_selection_snapshot(Some(&row_selection));
3025        let other_row_selection = PrefilterKey::row_selection_snapshot(Some(&other_row_selection));
3026        let base = PrefilterKey::new(
3027            file_id,
3028            0,
3029            row_selection.clone(),
3030            1,
3031            SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()]),
3032        );
3033
3034        assert_ne!(
3035            base,
3036            PrefilterKey::new(
3037                FileId::random(),
3038                0,
3039                row_selection.clone(),
3040                1,
3041                SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()])
3042            )
3043        );
3044        assert_ne!(
3045            base,
3046            PrefilterKey::new(
3047                file_id,
3048                1,
3049                row_selection.clone(),
3050                1,
3051                SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()])
3052            )
3053        );
3054        assert_ne!(
3055            base,
3056            PrefilterKey::new(
3057                file_id,
3058                0,
3059                other_row_selection,
3060                1,
3061                SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()])
3062            )
3063        );
3064        assert_ne!(
3065            base,
3066            PrefilterKey::new(
3067                file_id,
3068                0,
3069                row_selection.clone(),
3070                1,
3071                SmallVec::from_vec(vec!["tag_0 IN ([b])".to_string()])
3072            )
3073        );
3074        assert_ne!(
3075            base,
3076            PrefilterKey::new(
3077                file_id,
3078                0,
3079                row_selection.clone(),
3080                2,
3081                SmallVec::from_vec(vec!["tag_0 IN ([a])".to_string()])
3082            )
3083        );
3084        let pk_group = PrefilterKey::new(
3085            file_id,
3086            0,
3087            row_selection,
3088            1,
3089            SmallVec::from_vec(vec![
3090                "tag_0 IN ([a])".to_string(),
3091                "tag_1 IN ([x])".to_string(),
3092            ]),
3093        );
3094        assert_ne!(base, pk_group);
3095    }
3096
3097    #[test]
3098    fn test_range_result_cache() {
3099        let cache = Arc::new(
3100            CacheManager::builder()
3101                .range_result_cache_size(1024 * 1024)
3102                .build(),
3103        );
3104
3105        let key = RangeScanCacheKey {
3106            region_id: RegionId::new(1, 1),
3107            row_groups: vec![(FileId::random(), 0)],
3108            scan: ScanRequestFingerprintBuilder {
3109                read_columns: ReadColumns::new(std::iter::empty()),
3110                read_column_types: vec![],
3111                filters: vec!["tag_0 = 1".to_string()],
3112                time_filters: vec![],
3113                series_row_selector: None,
3114                append_mode: false,
3115                filter_deleted: true,
3116                merge_mode: crate::region::options::MergeMode::LastRow,
3117                sequence_range: None,
3118                partition_expr_version: 0,
3119            }
3120            .build(),
3121        };
3122        let value = Arc::new(RangeScanCacheValue::empty());
3123
3124        assert!(cache.get_range_result(&key).is_none());
3125        cache.put_range_result(key.clone(), value.clone());
3126        assert!(cache.get_range_result(&key).is_some());
3127
3128        let enable_all = CacheStrategy::EnableAll(cache.clone());
3129        assert!(enable_all.get_range_result(&key).is_some());
3130
3131        let compaction = CacheStrategy::Compaction(cache.clone());
3132        assert!(compaction.get_range_result(&key).is_none());
3133        compaction.put_range_result(key.clone(), value.clone());
3134        assert!(cache.get_range_result(&key).is_some());
3135
3136        let disabled = CacheStrategy::Disabled;
3137        assert!(disabled.get_range_result(&key).is_none());
3138        disabled.put_range_result(key.clone(), value);
3139        assert!(cache.get_range_result(&key).is_some());
3140    }
3141
3142    #[test]
3143    fn test_range_result_cache_size_configures_limiter() {
3144        let cache_size = 3 * 1024_u64;
3145        let cache = CacheManager::builder()
3146            .range_result_cache_size(cache_size)
3147            .build();
3148
3149        assert_eq!(cache.range_result_cache_size(), cache_size as usize);
3150        assert_eq!(
3151            cache
3152                .range_result_cache
3153                .as_ref()
3154                .unwrap()
3155                .concat_memory()
3156                .permit_bytes(),
3157            RANGE_RESULT_CONCAT_MEMORY_PERMIT.as_bytes() as usize
3158        );
3159        assert_eq!(
3160            cache
3161                .range_result_cache
3162                .as_ref()
3163                .unwrap()
3164                .concat_memory()
3165                .available_permits(),
3166            (cache_size as usize).div_ceil(RANGE_RESULT_CONCAT_MEMORY_PERMIT.as_bytes() as usize)
3167        );
3168    }
3169
3170    #[tokio::test]
3171    async fn test_range_concat_memory_rejects_oversized_request() {
3172        let limiter = RangeResultMemoryLimiter::new(2 * 1024, 1024);
3173        assert_eq!(limiter.available_permits(), 2);
3174
3175        let err = limiter.acquire(10 * 1024).await.unwrap_err();
3176        assert!(
3177            err.to_string().contains("exceeds limiter capacity"),
3178            "unexpected error: {err}"
3179        );
3180        assert_eq!(limiter.available_permits(), 2);
3181    }
3182
3183    #[tokio::test]
3184    async fn test_range_concat_memory_allows_request_up_to_capacity() {
3185        let limiter = RangeResultMemoryLimiter::new(2 * 1024, 1024);
3186        let permit = limiter.acquire(2 * 1024).await.unwrap();
3187        assert_eq!(limiter.available_permits(), 0);
3188        drop(permit);
3189        assert_eq!(limiter.available_permits(), 2);
3190    }
3191
3192    #[rstest::rstest]
3193    #[case::empty_result(0, Some(1))]
3194    #[case::rounded_result(1025, Some(2))]
3195    #[case::exhausted(4096, None)]
3196    #[case::oversized(usize::MAX, None)]
3197    #[test]
3198    fn test_range_result_memory_reservation_is_nonblocking(
3199        #[case] result_size: usize,
3200        #[case] expected_permits: Option<usize>,
3201    ) {
3202        let limiter = RangeResultMemoryLimiter::new(4096, 1024);
3203        let blocker = limiter.try_acquire(1024).unwrap();
3204        let result_permit = limiter.try_acquire(result_size);
3205        assert_eq!(
3206            result_permit.as_ref().map(|permit| permit.num_permits()),
3207            expected_permits
3208        );
3209        assert_eq!(
3210            limiter.available_permits(),
3211            3 - expected_permits.unwrap_or(0)
3212        );
3213        drop(result_permit);
3214        assert_eq!(limiter.available_permits(), 3);
3215        drop(blocker);
3216        assert_eq!(limiter.available_permits(), 4);
3217    }
3218
3219    #[tokio::test]
3220    async fn test_evict_puffin_cache_clears_all_entries() {
3221        use std::collections::{BTreeMap, HashMap};
3222
3223        use puffin::puffin_manager::file_key::{PuffinFileKey, PuffinStorageScope};
3224
3225        let region_id = RegionId::new(1, 1);
3226        let index_id = RegionIndexId::new(RegionFileId::new(region_id, FileId::random()), 0);
3227        let file_key = PuffinFileKey::new(
3228            Arc::new(PuffinStorageScope::new(
3229                "memory".into(),
3230                "cache-test".into(),
3231                "/".into(),
3232            )),
3233            "table/index.puffin".into(),
3234            index_id,
3235        );
3236        let metadata_key = file_key.clone().map_handle(|id| id.to_string());
3237        let other_scope_key = PuffinFileKey::new(
3238            Arc::new(PuffinStorageScope::new(
3239                "memory".into(),
3240                "other-store".into(),
3241                "/".into(),
3242            )),
3243            "table/index.puffin".into(),
3244            index_id.to_string(),
3245        );
3246        let newer_metadata_key = file_key.clone().map_handle(|mut id| {
3247            id.version += 1;
3248            id.to_string()
3249        });
3250        let column_id: ColumnId = 1;
3251
3252        // Budget for the actual keys and empty values used below, including all
3253        // three footer entries. Shared scopes are conservatively charged per key.
3254        let index_metadata_size = file_key.estimated_size()
3255            + ::index::target::IndexTarget::ColumnId(column_id).approx_mem_size()
3256            + std::mem::size_of::<Tag>()
3257            + std::mem::size_of::<BloomFilterMeta>()
3258            + std::mem::size_of::<InvertedIndexMetas>();
3259        let puffin_metadata_size: usize = [&metadata_key, &other_scope_key, &newer_metadata_key]
3260            .into_iter()
3261            .map(|key| {
3262                key.estimated_size()
3263                    + key.handle().capacity()
3264                    + FileMetadata {
3265                        blobs: Vec::new(),
3266                        properties: HashMap::new(),
3267                    }
3268                    .memory_usage()
3269            })
3270            .sum();
3271        let cache = Arc::new(
3272            CacheManager::builder()
3273                .index_metadata_size(index_metadata_size as u64)
3274                .index_content_size(128)
3275                .index_content_page_size(64)
3276                .index_result_cache_size(128)
3277                .puffin_metadata_size(puffin_metadata_size as u64)
3278                .build(),
3279        );
3280
3281        let bloom_cache = cache.bloom_filter_index_cache().unwrap().clone();
3282        let inverted_cache = cache.inverted_index_cache().unwrap().clone();
3283        let result_cache = cache.index_result_cache().unwrap();
3284        let puffin_metadata_cache = cache.puffin_metadata_cache().unwrap().clone();
3285
3286        let bloom_key = (
3287            file_key.clone(),
3288            Arc::new(::index::target::IndexTarget::ColumnId(column_id)),
3289            Tag::Skipping,
3290        );
3291        bloom_cache
3292            .put_metadata(bloom_key.clone(), Arc::new(BloomFilterMeta::default()))
3293            .await;
3294        inverted_cache
3295            .put_metadata(file_key.clone(), Arc::new(InvertedIndexMetas::default()))
3296            .await;
3297        let predicate = PredicateKey::new_bloom(Arc::new(BTreeMap::new()));
3298        let selection = Arc::new(RowGroupSelection::default());
3299        result_cache.put(predicate.clone(), index_id.file_id(), selection);
3300        let metadata = Arc::new(FileMetadata {
3301            blobs: Vec::new(),
3302            properties: HashMap::new(),
3303        });
3304        puffin_metadata_cache
3305            .put_metadata(metadata_key.clone(), metadata)
3306            .await;
3307
3308        assert!(bloom_cache.get_metadata(bloom_key.clone()).await.is_some());
3309        assert!(
3310            inverted_cache
3311                .get_metadata(file_key.clone())
3312                .await
3313                .is_some()
3314        );
3315        assert!(result_cache.get(&predicate, index_id.file_id()).is_some());
3316        assert!(
3317            puffin_metadata_cache
3318                .get_metadata(&metadata_key)
3319                .await
3320                .is_some()
3321        );
3322
3323        // Retirement only has the immutable index handle, so matching scopes are
3324        // all invalidated while a different generation remains available.
3325        for key in [&other_scope_key, &newer_metadata_key] {
3326            puffin_metadata_cache
3327                .put_metadata(
3328                    key.clone(),
3329                    Arc::new(FileMetadata {
3330                        blobs: Vec::new(),
3331                        properties: HashMap::new(),
3332                    }),
3333                )
3334                .await;
3335            assert!(puffin_metadata_cache.get_metadata(key).await.is_some());
3336        }
3337        cache.evict_puffin_cache(index_id).await;
3338        assert!(
3339            puffin_metadata_cache
3340                .get_metadata(&other_scope_key)
3341                .await
3342                .is_none()
3343        );
3344        assert!(
3345            puffin_metadata_cache
3346                .get_metadata(&newer_metadata_key)
3347                .await
3348                .is_some()
3349        );
3350
3351        assert!(bloom_cache.get_metadata(bloom_key.clone()).await.is_none());
3352        assert!(
3353            inverted_cache
3354                .get_metadata(file_key.clone())
3355                .await
3356                .is_none()
3357        );
3358        assert!(result_cache.get(&predicate, index_id.file_id()).is_none());
3359        assert!(
3360            puffin_metadata_cache
3361                .get_metadata(&metadata_key)
3362                .await
3363                .is_none()
3364        );
3365
3366        // Refill caches and evict via CacheStrategy to ensure delegation works.
3367        bloom_cache
3368            .put_metadata(bloom_key.clone(), Arc::new(BloomFilterMeta::default()))
3369            .await;
3370        inverted_cache
3371            .put_metadata(file_key.clone(), Arc::new(InvertedIndexMetas::default()))
3372            .await;
3373        result_cache.put(
3374            predicate.clone(),
3375            index_id.file_id(),
3376            Arc::new(RowGroupSelection::default()),
3377        );
3378        puffin_metadata_cache
3379            .put_metadata(
3380                metadata_key.clone(),
3381                Arc::new(FileMetadata {
3382                    blobs: Vec::new(),
3383                    properties: HashMap::new(),
3384                }),
3385            )
3386            .await;
3387
3388        let strategy = CacheStrategy::EnableAll(cache.clone());
3389        strategy.evict_puffin_cache(index_id).await;
3390
3391        assert!(bloom_cache.get_metadata(bloom_key.clone()).await.is_none());
3392        assert!(
3393            inverted_cache
3394                .get_metadata(file_key.clone())
3395                .await
3396                .is_none()
3397        );
3398        assert!(result_cache.get(&predicate, index_id.file_id()).is_none());
3399        assert!(
3400            puffin_metadata_cache
3401                .get_metadata(&metadata_key)
3402                .await
3403                .is_none()
3404        );
3405    }
3406
3407    fn wide_region_metadata(column_count: u32) -> RegionMetadata {
3408        let region_id = RegionId::new(1024, 7);
3409        let mut builder = RegionMetadataBuilder::new(region_id);
3410        let mut primary_key = Vec::new();
3411
3412        for column_id in 0..column_count {
3413            let semantic_type = if column_id < 32 {
3414                primary_key.push(column_id);
3415                SemanticType::Tag
3416            } else {
3417                SemanticType::Field
3418            };
3419            let mut column_schema = ColumnSchema::new(
3420                format!("wide_column_{column_id}"),
3421                ConcreteDataType::string_datatype(),
3422                true,
3423            );
3424            column_schema
3425                .mut_metadata()
3426                .insert(format!("cache_key_{column_id}"), "cache_value".repeat(4));
3427            builder.push_column_metadata(ColumnMetadata {
3428                column_schema,
3429                semantic_type,
3430                column_id,
3431            });
3432        }
3433
3434        builder.push_column_metadata(ColumnMetadata {
3435            column_schema: ColumnSchema::new(
3436                "ts",
3437                ConcreteDataType::timestamp_millisecond_datatype(),
3438                false,
3439            ),
3440            semantic_type: SemanticType::Timestamp,
3441            column_id: column_count,
3442        });
3443        builder.primary_key(primary_key);
3444
3445        builder.build().unwrap()
3446    }
3447}