Skip to main content

mito2/read/
scan_region.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Scans a region according to the scan request.
16
17use std::collections::{BTreeMap, HashSet};
18use std::fmt;
19use std::num::NonZeroU64;
20use std::sync::{Arc, OnceLock};
21use std::time::Instant;
22
23use api::v1::SemanticType;
24use common_error::ext::BoxedError;
25use common_recordbatch::adapter::RegionQueryStatCounters;
26use common_recordbatch::filter::SimpleFilterEvaluator;
27use common_recordbatch::{QueryMemoryTracker, SendableRecordBatchStream};
28use common_telemetry::tracing::Instrument;
29use common_telemetry::{debug, error, tracing, warn};
30use common_time::range::TimestampRange;
31use datafusion::optimizer::simplify_expressions::ExprSimplifier;
32use datafusion::physical_plan::expressions::DynamicFilterPhysicalExpr;
33use datafusion_common::pruning::PruningStatistics;
34use datafusion_common::{Column, ScalarValue, ToDFSchema};
35use datafusion_expr::Expr;
36use datafusion_expr::simplify::SimplifyContext;
37use datafusion_expr::utils::expr_to_columns;
38use datatypes::arrow::array::{ArrayRef, BooleanArray, UInt64Array};
39use datatypes::extension::json::is_json2_extension_type;
40use datatypes::prelude::ConcreteDataType;
41use datatypes::types::json_type::JsonNativeType;
42use datatypes::value::timestamp_to_scalar_value;
43use futures::StreamExt;
44use partition::expr::PartitionExpr;
45use smallvec::SmallVec;
46use snafu::{OptionExt, ResultExt, ensure};
47use store_api::metadata::{RegionMetadata, RegionMetadataRef};
48use store_api::region_engine::{PartitionRange, RegionScannerRef};
49use store_api::storage::{
50    ColumnId, RegionId, ScanRequest, SequenceNumber, SequenceRange, TimeSeriesDistribution,
51    TimeSeriesRowSelector,
52};
53use table::predicate::{Predicate, build_time_range_predicate, extract_time_range_from_expr};
54use tokio::sync::{Semaphore, mpsc};
55use tokio_stream::wrappers::ReceiverStream;
56
57use crate::access_layer::AccessLayerRef;
58use crate::cache::CacheStrategy;
59use crate::config::DEFAULT_MAX_CONCURRENT_SCAN_FILES;
60use crate::error::{
61    EvalPartitionFilterSnafu, InvalidPartitionExprSnafu, InvalidRequestSnafu,
62    RegionSequenceDomainBrokenSnafu, Result, SequenceRangeUnsupportedSnafu,
63};
64#[cfg(feature = "enterprise")]
65use crate::extension::{BoxedExtensionRange, BoxedExtensionRangeProvider};
66use crate::memtable::{MemtableRange, RangesOptions};
67use crate::metrics::READ_SST_COUNT;
68use crate::read::compat::{self, FlatCompatBatch};
69use crate::read::flat_projection::FlatProjectionMapper;
70use crate::read::range::{FileRangeBuilder, MemRangeBuilder, RangeMeta, RowGroupIndex};
71use crate::read::range_cache::{ScanRequestFingerprint, implied_time_range_from_exprs};
72use crate::read::read_columns::ReadColumns;
73use crate::read::seq_scan::SeqScan;
74use crate::read::series_scan::SeriesScan;
75use crate::read::stream::ScanBatchStream;
76use crate::read::unordered_scan::UnorderedScan;
77use crate::read::{BoxedRecordBatchStream, RecordBatch};
78use crate::region::options::MergeMode;
79use crate::region::version::VersionRef;
80use crate::series_index::SeriesIndexReadContext;
81use crate::sst::file::FileHandle;
82use crate::sst::index::bloom_filter::applier::{
83    BloomFilterIndexApplierBuilder, BloomFilterIndexApplierRef,
84};
85use crate::sst::index::fulltext_index::applier::FulltextIndexApplierRef;
86use crate::sst::index::fulltext_index::applier::builder::FulltextIndexApplierBuilder;
87use crate::sst::index::inverted_index::applier::InvertedIndexApplierRef;
88use crate::sst::index::inverted_index::applier::builder::InvertedIndexApplierBuilder;
89use crate::sst::parquet::Json2RewriteTargets;
90use crate::sst::parquet::file_range::PreFilterMode;
91use crate::sst::parquet::reader::ReaderMetrics;
92use crate::sst::primary_key::PrimaryKeyRangeMapper;
93
94/// A scanner scans a region and returns a [SendableRecordBatchStream].
95pub(crate) enum Scanner {
96    /// Sequential scan.
97    Seq(SeqScan),
98    /// Unordered scan.
99    Unordered(UnorderedScan),
100    /// Per-series scan.
101    Series(SeriesScan),
102}
103
104impl Scanner {
105    /// Returns a [SendableRecordBatchStream] to retrieve scan results from all partitions.
106    #[tracing::instrument(level = tracing::Level::DEBUG, skip_all)]
107    pub(crate) async fn scan(&self) -> Result<SendableRecordBatchStream, BoxedError> {
108        match self {
109            Scanner::Seq(seq_scan) => seq_scan.build_stream(),
110            Scanner::Unordered(unordered_scan) => unordered_scan.build_stream().await,
111            Scanner::Series(series_scan) => series_scan.build_stream().await,
112        }
113    }
114
115    /// Create a stream of [`Batch`] by this scanner.
116    pub(crate) fn scan_batch(&self) -> Result<ScanBatchStream> {
117        match self {
118            Scanner::Seq(x) => x.scan_all_partitions(),
119            Scanner::Unordered(x) => x.scan_all_partitions(),
120            Scanner::Series(x) => x.scan_all_partitions(),
121        }
122    }
123}
124
125#[cfg(test)]
126impl Scanner {
127    /// Returns number of files to scan.
128    pub(crate) fn num_files(&self) -> usize {
129        match self {
130            Scanner::Seq(seq_scan) => seq_scan.input().num_files(),
131            Scanner::Unordered(unordered_scan) => unordered_scan.input().num_files(),
132            Scanner::Series(series_scan) => series_scan.input().num_files(),
133        }
134    }
135
136    /// Returns number of memtables to scan.
137    pub(crate) fn num_memtables(&self) -> usize {
138        match self {
139            Scanner::Seq(seq_scan) => seq_scan.input().num_memtables(),
140            Scanner::Unordered(unordered_scan) => unordered_scan.input().num_memtables(),
141            Scanner::Series(series_scan) => series_scan.input().num_memtables(),
142        }
143    }
144
145    /// Returns SST file ids to scan.
146    pub(crate) fn file_ids(&self) -> Vec<crate::sst::file::RegionFileId> {
147        match self {
148            Scanner::Seq(seq_scan) => seq_scan.input().file_ids(),
149            Scanner::Unordered(unordered_scan) => unordered_scan.input().file_ids(),
150            Scanner::Series(series_scan) => series_scan.input().file_ids(),
151        }
152    }
153
154    pub(crate) fn index_ids(&self) -> Vec<crate::sst::file::RegionIndexId> {
155        match self {
156            Scanner::Seq(seq_scan) => seq_scan.input().index_ids(),
157            Scanner::Unordered(unordered_scan) => unordered_scan.input().index_ids(),
158            Scanner::Series(series_scan) => series_scan.input().index_ids(),
159        }
160    }
161
162    pub(crate) fn snapshot_sequence(&self) -> Option<SequenceNumber> {
163        match self {
164            Scanner::Seq(seq_scan) => seq_scan.input().snapshot_sequence,
165            Scanner::Unordered(unordered_scan) => unordered_scan.input().snapshot_sequence,
166            Scanner::Series(series_scan) => series_scan.input().snapshot_sequence,
167        }
168    }
169
170    /// Sets the target partitions for the scanner. It can controls the parallelism of the scanner.
171    pub(crate) fn set_target_partitions(&mut self, target_partitions: usize) {
172        use store_api::region_engine::{PrepareRequest, RegionScanner};
173
174        let request = PrepareRequest::default().with_target_partitions(target_partitions);
175        match self {
176            Scanner::Seq(seq_scan) => seq_scan.prepare(request).unwrap(),
177            Scanner::Unordered(unordered_scan) => unordered_scan.prepare(request).unwrap(),
178            Scanner::Series(series_scan) => series_scan.prepare(request).unwrap(),
179        }
180    }
181}
182
183#[cfg_attr(doc, aquamarine::aquamarine)]
184/// Helper to scans a region by [ScanRequest].
185///
186/// [ScanRegion] collects SSTs and memtables to scan without actually reading them. It
187/// creates a [Scanner] to actually scan these targets in [Scanner::scan()].
188///
189/// ```mermaid
190/// classDiagram
191/// class ScanRegion {
192///     -VersionRef version
193///     -ScanRequest request
194///     ~scanner() Scanner
195///     ~seq_scan() SeqScan
196/// }
197/// class Scanner {
198///     <<enumeration>>
199///     SeqScan
200///     UnorderedScan
201///     +scan() SendableRecordBatchStream
202/// }
203/// class SeqScan {
204///     -ScanInput input
205///     +build() SendableRecordBatchStream
206/// }
207/// class UnorderedScan {
208///     -ScanInput input
209///     +build() SendableRecordBatchStream
210/// }
211/// class ScanInput {
212///     -ProjectionMapper mapper
213///     -Option~TimeRange~ time_range
214///     -Option~Predicate~ predicate
215///     -Vec~MemtableRef~ memtables
216///     -Vec~FileHandle~ files
217/// }
218/// class ProjectionMapper {
219///     ~output_schema() SchemaRef
220///     ~convert(Batch) RecordBatch
221/// }
222/// ScanRegion -- Scanner
223/// ScanRegion o-- ScanRequest
224/// Scanner o-- SeqScan
225/// Scanner o-- UnorderedScan
226/// SeqScan o-- ScanInput
227/// UnorderedScan o-- ScanInput
228/// Scanner -- SendableRecordBatchStream
229/// ScanInput o-- ProjectionMapper
230/// SeqScan -- SendableRecordBatchStream
231/// UnorderedScan -- SendableRecordBatchStream
232/// ```
233pub(crate) struct ScanRegion {
234    /// Version of the region at scan.
235    version: VersionRef,
236    /// Pinned index snapshot and its storage for candidate discovery.
237    series_index: Option<SeriesIndexReadContext>,
238    /// Access layer of the region.
239    access_layer: AccessLayerRef,
240    /// Scan request.
241    request: ScanRequest,
242    /// Cache.
243    cache_strategy: CacheStrategy,
244    /// Maximum number of SST files to scan concurrently.
245    max_concurrent_scan_files: usize,
246    scan_memory_tracker: Option<QueryMemoryTracker>,
247    /// Whether to enable the experimental two-phase metric series scan.
248    experimental_series_scan_v2: bool,
249    /// Whether to ignore range indexes during scans.
250    ignore_range_index: bool,
251    /// Whether to ignore inverted index.
252    ignore_inverted_index: bool,
253    /// Whether to ignore fulltext index.
254    ignore_fulltext_index: bool,
255    /// Whether to ignore bloom filter.
256    ignore_bloom_filter: bool,
257    /// Start time of the scan task.
258    start_time: Option<Instant>,
259    /// Whether to filter out the deleted rows.
260    /// Usually true for normal read, and false for scan for compaction.
261    filter_deleted: bool,
262    /// Files and exact sequence capability selected together by the engine.
263    exact_selection: Option<(Vec<FileHandle>, Option<SequenceRange>)>,
264    /// Counters that should receive query-load metrics.
265    query_stat_counters: Option<RegionQueryStatCounters>,
266    #[cfg(feature = "enterprise")]
267    extension_range_provider: Option<BoxedExtensionRangeProvider>,
268}
269
270impl ScanRegion {
271    /// Creates a [ScanRegion].
272    pub(crate) fn new(
273        version: VersionRef,
274        access_layer: AccessLayerRef,
275        request: ScanRequest,
276        cache_strategy: CacheStrategy,
277    ) -> ScanRegion {
278        ScanRegion {
279            version,
280            series_index: None,
281            access_layer,
282            request,
283            cache_strategy,
284            max_concurrent_scan_files: DEFAULT_MAX_CONCURRENT_SCAN_FILES,
285            scan_memory_tracker: None,
286            experimental_series_scan_v2: false,
287            ignore_range_index: false,
288            ignore_inverted_index: false,
289            ignore_fulltext_index: false,
290            ignore_bloom_filter: false,
291            start_time: None,
292            filter_deleted: true,
293            exact_selection: None,
294            query_stat_counters: None,
295            #[cfg(feature = "enterprise")]
296            extension_range_provider: None,
297        }
298    }
299
300    /// Pins the series-index snapshot used by candidate discovery.
301    #[must_use]
302    pub(crate) fn with_series_index(
303        mut self,
304        series_index: Option<SeriesIndexReadContext>,
305    ) -> Self {
306        self.series_index = series_index.map(|mut context| {
307            // A daily artifact cannot be partially selected by source SST identity. Pin only
308            // generations whose complete source set is still visible in this data snapshot.
309            let visible = self
310                .version
311                .ssts
312                .levels()
313                .iter()
314                .flat_map(|l| l.files())
315                .map(|f| f.file_id().file_id())
316                .collect();
317            let previous = &context.version;
318            context.version = Arc::new(crate::series_index::SeriesIndexVersion {
319                samples: previous
320                    .samples
321                    .iter()
322                    .filter(|(_, h)| {
323                        h.entry().sources_visible(&visible)
324                            && (h.entry().window.is_none()
325                                || (self.request.sst_min_sequence.is_none()
326                                    && !self.request.exact_sequence_range))
327                    })
328                    .map(|(id, h)| (*id, h.clone()))
329                    .collect(),
330                range_indexes: previous.range_indexes.clone(),
331                series_indexes: previous.series_indexes.clone(),
332                index_buckets: previous.index_buckets.clone(),
333            });
334            context
335        });
336        self
337    }
338
339    /// Sets counters that should receive query-load metrics.
340    #[must_use]
341    pub(crate) fn with_query_stat_counters(mut self, counters: RegionQueryStatCounters) -> Self {
342        self.query_stat_counters = Some(counters);
343        self
344    }
345
346    /// Sets maximum number of SST files to scan concurrently.
347    #[must_use]
348    pub(crate) fn with_max_concurrent_scan_files(
349        mut self,
350        max_concurrent_scan_files: usize,
351    ) -> Self {
352        self.max_concurrent_scan_files = max_concurrent_scan_files;
353        self
354    }
355
356    pub(crate) fn with_scan_memory_tracker(mut self, tracker: QueryMemoryTracker) -> Self {
357        self.scan_memory_tracker = Some(tracker);
358        self
359    }
360
361    /// Sets whether eligible metric series scans use the two-phase implementation.
362    #[must_use]
363    pub(crate) fn with_experimental_series_scan_v2(mut self, enabled: bool) -> Self {
364        self.experimental_series_scan_v2 = enabled;
365        self
366    }
367
368    /// Sets whether to ignore range indexes during scans.
369    #[must_use]
370    pub(crate) fn with_ignore_range_index(mut self, ignore: bool) -> Self {
371        self.ignore_range_index = ignore;
372        self
373    }
374
375    /// Sets whether to ignore inverted index.
376    #[must_use]
377    pub(crate) fn with_ignore_inverted_index(mut self, ignore: bool) -> Self {
378        self.ignore_inverted_index = ignore;
379        self
380    }
381
382    /// Sets whether to ignore fulltext index.
383    #[must_use]
384    pub(crate) fn with_ignore_fulltext_index(mut self, ignore: bool) -> Self {
385        self.ignore_fulltext_index = ignore;
386        self
387    }
388
389    /// Sets whether to ignore bloom filter.
390    #[must_use]
391    pub(crate) fn with_ignore_bloom_filter(mut self, ignore: bool) -> Self {
392        self.ignore_bloom_filter = ignore;
393        self
394    }
395
396    #[must_use]
397    pub(crate) fn with_start_time(mut self, now: Instant) -> Self {
398        self.start_time = Some(now);
399        self
400    }
401
402    pub(crate) fn set_filter_deleted(&mut self, filter_deleted: bool) {
403        self.filter_deleted = filter_deleted;
404    }
405
406    pub(crate) fn with_exact_selection(
407        mut self,
408        exact_selection: (Vec<FileHandle>, Option<SequenceRange>),
409    ) -> Self {
410        self.exact_selection = Some(exact_selection);
411        self
412    }
413
414    #[cfg(feature = "enterprise")]
415    pub(crate) fn set_extension_range_provider(
416        &mut self,
417        extension_range_provider: BoxedExtensionRangeProvider,
418    ) {
419        self.extension_range_provider = Some(extension_range_provider);
420    }
421
422    /// Returns a [Scanner] to scan the region.
423    #[tracing::instrument(skip_all, fields(region_id = %self.region_id()))]
424    pub(crate) async fn scanner(self) -> Result<Scanner> {
425        if self.use_series_scan() {
426            self.series_scan().await.map(Scanner::Series)
427        } else if self.use_unordered_scan() {
428            // If table is append only and there is no series row selector, we use unordered scan in query.
429            // We still use seq scan in compaction.
430            self.unordered_scan().await.map(Scanner::Unordered)
431        } else {
432            self.seq_scan().await.map(Scanner::Seq)
433        }
434    }
435
436    /// Returns a [RegionScanner] to scan the region.
437    #[tracing::instrument(
438        level = tracing::Level::DEBUG,
439        skip_all,
440        fields(region_id = %self.region_id())
441    )]
442    pub(crate) async fn region_scanner(self) -> Result<RegionScannerRef> {
443        if self.use_series_scan() {
444            self.series_scan()
445                .await
446                .map(|scanner| Box::new(scanner) as _)
447        } else if self.use_unordered_scan() {
448            self.unordered_scan()
449                .await
450                .map(|scanner| Box::new(scanner) as _)
451        } else {
452            self.seq_scan().await.map(|scanner| Box::new(scanner) as _)
453        }
454    }
455
456    /// Scan sequentially.
457    #[tracing::instrument(skip_all, fields(region_id = %self.region_id()))]
458    pub(crate) async fn seq_scan(self) -> Result<SeqScan> {
459        let input = self.scan_input().await?;
460        Ok(SeqScan::new(input))
461    }
462
463    /// Unordered scan.
464    #[tracing::instrument(skip_all, fields(region_id = %self.region_id()))]
465    pub(crate) async fn unordered_scan(self) -> Result<UnorderedScan> {
466        let input = self.scan_input().await?;
467        let query_start = input.query_start.unwrap_or_else(Instant::now);
468        let ranges = RangeMeta::unordered_scan_ranges(&input).await;
469        let stream_ctx = StreamContext::unordered_scan_ctx(input, ranges, query_start);
470        Ok(UnorderedScan::new(stream_ctx))
471    }
472
473    /// Scans by series.
474    #[tracing::instrument(skip_all, fields(region_id = %self.region_id()))]
475    pub(crate) async fn series_scan(self) -> Result<SeriesScan> {
476        let experimental_series_scan_v2 = self.experimental_series_scan_v2;
477        let input = self.scan_input().await?;
478        Ok(SeriesScan::new(input, experimental_series_scan_v2))
479    }
480
481    /// Returns true if the region can use unordered scan for current request.
482    fn use_unordered_scan(&self) -> bool {
483        // We use unordered scan when:
484        // 1. The region is in append mode.
485        // 2. There is no series row selector.
486        // 3. The required distribution is None or TimeSeriesDistribution::TimeWindowed.
487        //
488        // We still use seq scan in compaction.
489        self.version.options.append_mode
490            && self.request.series_row_selector.is_none()
491            && (self.request.distribution.is_none()
492                || self.request.distribution == Some(TimeSeriesDistribution::TimeWindowed))
493    }
494
495    /// Returns true if the region can use series scan for current request.
496    fn use_series_scan(&self) -> bool {
497        self.request.distribution == Some(TimeSeriesDistribution::PerSeries)
498    }
499
500    /// Creates a scan input.
501    #[tracing::instrument(skip_all, fields(region_id = %self.region_id()))]
502    async fn scan_input(mut self) -> Result<ScanInput> {
503        let metadata = &self.version.metadata;
504        let sst_min_sequence = self.request.sst_min_sequence.and_then(NonZeroU64::new);
505        let time_range = self.build_time_range_predicate();
506        let predicate = PredicateGroup::new(metadata, &self.request.filters)?;
507
508        let read_col_ids =
509            self.build_read_col_ids(self.request.projection.as_deref(), &predicate)?;
510        let read_cols = self.build_read_columns(&read_col_ids)?;
511
512        // The mapper always computes projected column ids as the schema of SSTs may change.
513        let projection = self
514            .request
515            .projection
516            .clone()
517            .unwrap_or_else(|| (0..metadata.column_metadatas.len()).collect());
518        let mapper = FlatProjectionMapper::new_with_read_columns(metadata, projection, read_cols)?;
519        let mapper = if self.request.preserve_pk_dictionary_encoding {
520            mapper.with_pk_dictionary_encoding()
521        } else {
522            mapper
523        };
524
525        // Only time-pruned sources from this pinned version may accompany an assigned SST
526        // in a grouped sample artifact. Explicit selections and sequence pruning stay strict.
527        let time_pruned_files = if self.exact_selection.is_none()
528            && !self.request.skip_sst_files
529            && self.request.sst_min_sequence.is_none()
530            && !self.request.exact_sequence_range
531        {
532            self.version
533                .ssts
534                .levels()
535                .iter()
536                .flat_map(|level| level.files.values())
537                .filter(|file| !file_in_range(file, &time_range))
538                .map(|file| file.file_id().file_id())
539                .collect()
540        } else {
541            HashSet::new()
542        };
543        let (files, sequence_range) =
544            if let Some((files, sequence_range)) = self.exact_selection.take() {
545                (files, sequence_range)
546            } else {
547                exact_sequence_range(&self.request, &self.version)?
548            };
549        if sst_min_sequence.is_some() && sequence_range.is_some() {
550            return SequenceRangeUnsupportedSnafu {
551                region_id: self.region_id(),
552                min_seq: self.request.memtable_min_sequence.unwrap_or_default(),
553                max_seq: self.request.memtable_max_sequence.unwrap_or_default(),
554                reason:
555                    "sst_min_sequence pruning hint is incompatible with exact sequence-range reads"
556                        .to_string(),
557            }
558            .fail();
559        }
560        let memtables = self.version.memtables.list_memtables();
561        // Skip empty memtables and memtables out of time range.
562        let mut mem_range_builders = Vec::new();
563        let filter_mode = pre_filter_mode(
564            self.version.options.append_mode,
565            self.version.options.merge_mode(),
566        );
567
568        for m in memtables {
569            // check if memtable is empty by reading stats.
570            let Some((start, end)) = m.stats().time_range() else {
571                continue;
572            };
573            // The time range of the memtable is inclusive.
574            let memtable_range = TimestampRange::new_inclusive(Some(start), Some(end));
575            if !memtable_range.intersects(&time_range) {
576                continue;
577            }
578            let ranges_in_memtable = m.ranges(
579                Some(&read_col_ids),
580                RangesOptions::default()
581                    .with_predicate(predicate.clone())
582                    .with_sequence(SequenceRange::new(
583                        self.request.memtable_min_sequence,
584                        self.request.memtable_max_sequence,
585                    ))
586                    .with_pre_filter_mode(filter_mode),
587            )?;
588            mem_range_builders.extend(ranges_in_memtable.ranges.into_values().map(|v| {
589                let stats = v.stats().clone();
590                MemRangeBuilder::new(v, stats)
591            }));
592        }
593
594        let region_id = self.region_id();
595        debug!(
596            "Scan region {}, request: {:?}, time range: {:?}, memtables: {}, ssts_to_read: {}, append_mode: {}",
597            region_id,
598            self.request,
599            time_range,
600            mem_range_builders.len(),
601            files.len(),
602            self.version.options.append_mode,
603        );
604
605        let (non_field_filters, field_filters) = self.partition_by_field_filters();
606        let inverted_index_appliers = [
607            self.build_invereted_index_applier(&non_field_filters),
608            self.build_invereted_index_applier(&field_filters),
609        ];
610        let bloom_filter_appliers = [
611            self.build_bloom_filter_applier(&non_field_filters),
612            self.build_bloom_filter_applier(&field_filters),
613        ];
614        let fulltext_index_appliers = [
615            self.build_fulltext_index_applier(&non_field_filters),
616            self.build_fulltext_index_applier(&field_filters),
617        ];
618
619        let input = ScanInput::builder(self.access_layer, mapper)
620            .with_series_index(self.series_index)
621            .with_ignore_range_index(self.ignore_range_index)
622            .with_time_range(Some(time_range))
623            .with_predicate(predicate)
624            .with_memtables(mem_range_builders)
625            .with_files(files)
626            .with_time_pruned_files(time_pruned_files)
627            .with_primary_key_mapper(self.version.ssts.primary_key_mapper())
628            .with_cache(self.cache_strategy)
629            .with_inverted_index_appliers(inverted_index_appliers)
630            .with_bloom_filter_index_appliers(bloom_filter_appliers)
631            .with_fulltext_index_appliers(fulltext_index_appliers)
632            .with_max_concurrent_scan_files(self.max_concurrent_scan_files)
633            .with_scan_memory_tracker(self.scan_memory_tracker)
634            .with_start_time(self.start_time)
635            .with_append_mode(self.version.options.append_mode)
636            .with_filter_deleted(self.filter_deleted)
637            .with_merge_mode(self.version.options.merge_mode())
638            .with_series_row_selector(self.request.series_row_selector)
639            .with_distribution(self.request.distribution)
640            .with_explain_flat_format(
641                self.version.options.sst_format == Some(crate::sst::FormatType::Flat),
642            )
643            .with_snapshot_sequence(
644                self.request
645                    .snapshot_on_scan
646                    .then_some(self.request.memtable_max_sequence)
647                    .flatten(),
648            )
649            .with_sequence_range(sequence_range)
650            .with_query_stat_counters(self.query_stat_counters);
651        #[cfg(feature = "enterprise")]
652        let input = if !self.request.skip_sst_files
653            && let Some(provider) = self.extension_range_provider
654        {
655            if sequence_range.is_some() {
656                // Defense in depth: the engine already rejects exact
657                // sequence-range reads on follower regions with an extension
658                // provider, but if one ever reaches the reader, fail closed
659                // here rather than letting unfiltered extension streams bypass
660                // the row-level sequence filter and emit out-of-range rows.
661                return SequenceRangeUnsupportedSnafu {
662                    region_id,
663                    min_seq: self.request.memtable_min_sequence.unwrap_or_default(),
664                    max_seq: self.request.memtable_max_sequence.unwrap_or_default(),
665                    reason:
666                        "exact sequence-range reads are unsupported when an extension range provider is present"
667                            .to_string(),
668                }
669                .fail();
670            }
671            let ranges = provider
672                .find_extension_ranges(self.version.flushed_sequence, time_range, &self.request)
673                .await?;
674            debug!("Find extension ranges: {ranges:?}");
675            input.with_extension_ranges(ranges)
676        } else {
677            input
678        };
679        Ok(input.build())
680    }
681
682    /// Builds the deduplicated root column ids required by the projection and
683    /// predicate.
684    fn build_read_col_ids(
685        &self,
686        projection: Option<&[usize]>,
687        predicate: &PredicateGroup,
688    ) -> Result<Vec<ColumnId>> {
689        let metadata = &self.version.metadata;
690        let Some(projection) = projection else {
691            return Ok(metadata
692                .column_metadatas
693                .iter()
694                .map(|col| col.column_id)
695                .collect());
696        };
697
698        let mut read_col_ids = Vec::new();
699        let mut seen = HashSet::new();
700        for idx in projection {
701            let col_id = metadata
702                .column_metadatas
703                .get(*idx)
704                .with_context(|| InvalidRequestSnafu {
705                    region_id: metadata.region_id,
706                    reason: format!("projection index {} is out of bounds", idx),
707                })?
708                .column_id;
709            let inserted = seen.insert(col_id);
710            // DataFusion's logical `OptimizeProjections` rule deduplicates pushed-down
711            // projection indices via `RequiredIndices::compact()`.
712            debug_assert!(
713                inserted,
714                "projection contains duplicate column id: {}",
715                col_id
716            );
717            // Keep the projection order.
718            read_col_ids.push(col_id);
719        }
720
721        if projection.is_empty() {
722            let time_index = metadata.time_index_column().column_id;
723            if seen.insert(time_index) {
724                read_col_ids.push(time_index);
725            }
726        }
727
728        let mut extra_col_names = HashSet::new();
729        let mut cols = HashSet::new();
730
731        if let Some(p) = predicate.predicate_without_region() {
732            for expr in p.exprs() {
733                cols.clear();
734                if expr_to_columns(expr, &mut cols).is_err() {
735                    continue;
736                }
737                extra_col_names.extend(cols.iter().map(|col| col.name.clone()));
738            }
739        }
740
741        if let Some(expr) = predicate.region_partition_expr() {
742            expr.collect_column_names(&mut extra_col_names);
743        }
744
745        if !extra_col_names.is_empty() {
746            for col in &metadata.column_metadatas {
747                if extra_col_names.remove(&col.column_schema.name) && !seen.contains(&col.column_id)
748                {
749                    read_col_ids.push(col.column_id);
750                }
751            }
752            if !extra_col_names.is_empty() {
753                warn!(
754                    "Some columns in filters are not found in region {}: {:?}",
755                    metadata.region_id, extra_col_names
756                );
757            }
758        }
759        Ok(read_col_ids)
760    }
761
762    /// Builds logical read columns and attaches JSON2 target types when needed.
763    ///
764    /// The behavior about JSON2 is as follows:
765    /// - JSON2 columns without hints use Variant to read the whole column.
766    /// - Hints targeting non-JSON2 read columns are rejected.
767    fn build_read_columns(&self, col_ids: &[ColumnId]) -> Result<ReadColumns> {
768        let metadata = &self.version.metadata;
769        let json_type_hint = &self.request.json_type_hint;
770
771        let has_json2 = metadata
772            .schema
773            .arrow_schema()
774            .fields()
775            .iter()
776            .any(is_json2_extension_type);
777
778        if !has_json2 && json_type_hint.is_empty() {
779            return Ok(ReadColumns::new(col_ids.iter().copied()));
780        }
781
782        let mut json_target_types = BTreeMap::new();
783        for &col_id in col_ids {
784            let Some(col) = metadata.column_by_id(col_id) else {
785                continue;
786            };
787            let col_name = &col.column_schema.name;
788            let hint = json_type_hint.get(col_name);
789            if !col.column_schema.data_type.is_json2() {
790                ensure!(
791                    hint.is_none(),
792                    InvalidRequestSnafu {
793                        region_id: metadata.region_id,
794                        reason: format!(
795                            "JSON type hint targets non-JSON2 column {} (id: {}, type: {})",
796                            col_name, col_id, col.column_schema.data_type
797                        ),
798                    }
799                );
800                continue;
801            }
802            let target_type = hint.cloned().unwrap_or(JsonNativeType::Variant);
803            json_target_types.insert(col_id, target_type);
804        }
805        Ok(ReadColumns::new(col_ids.iter().copied()).with_json_target_types(json_target_types))
806    }
807
808    fn region_id(&self) -> RegionId {
809        self.version.metadata.region_id
810    }
811
812    /// Build time range predicate from filters.
813    fn build_time_range_predicate(&self) -> TimestampRange {
814        let time_index = self.version.metadata.time_index_column();
815        let unit = time_index
816            .column_schema
817            .data_type
818            .as_timestamp()
819            .expect("Time index must have timestamp-compatible type")
820            .unit();
821        build_time_range_predicate(&time_index.column_schema.name, unit, &self.request.filters)
822    }
823
824    /// Partitions filters into two groups: non-field filters and field filters.
825    /// Returns `(non_field_filters, field_filters)`.
826    fn partition_by_field_filters(&self) -> (Vec<Expr>, Vec<Expr>) {
827        let field_columns = self
828            .version
829            .metadata
830            .field_columns()
831            .map(|col| &col.column_schema.name)
832            .collect::<HashSet<_>>();
833
834        let mut columns = HashSet::new();
835
836        self.request.filters.iter().cloned().partition(|expr| {
837            columns.clear();
838            // `expr_to_columns` won't return error.
839            if expr_to_columns(expr, &mut columns).is_err() {
840                // If we can't extract columns, treat it as non-field filter
841                return true;
842            }
843            // Return true for non-field filters (partition puts true cases in first vec)
844            !columns
845                .iter()
846                .any(|column| field_columns.contains(&column.name))
847        })
848    }
849
850    /// Use the latest schema to build the inverted index applier.
851    fn build_invereted_index_applier(&self, filters: &[Expr]) -> Option<InvertedIndexApplierRef> {
852        if self.ignore_inverted_index {
853            return None;
854        }
855
856        let file_cache = self.cache_strategy.write_cache().map(|w| w.file_cache());
857        let inverted_index_cache = self.cache_strategy.inverted_index_cache().cloned();
858
859        let puffin_metadata_cache = self.cache_strategy.puffin_metadata_cache().cloned();
860
861        InvertedIndexApplierBuilder::new(
862            self.access_layer.table_dir().to_string(),
863            self.access_layer.path_type(),
864            self.access_layer.object_store().clone(),
865            self.version.metadata.as_ref(),
866            self.version.metadata.inverted_indexed_column_ids(
867                self.version
868                    .options
869                    .index_options
870                    .inverted_index
871                    .ignore_column_ids
872                    .iter(),
873            ),
874            self.access_layer.puffin_manager_factory().clone(),
875        )
876        .with_file_cache(file_cache)
877        .with_inverted_index_cache(inverted_index_cache)
878        .with_puffin_metadata_cache(puffin_metadata_cache)
879        .build(filters)
880        .inspect_err(|err| warn!(err; "Failed to build invereted index applier"))
881        .ok()
882        .flatten()
883        .map(Arc::new)
884    }
885
886    /// Use the latest schema to build the bloom filter index applier.
887    fn build_bloom_filter_applier(&self, filters: &[Expr]) -> Option<BloomFilterIndexApplierRef> {
888        if self.ignore_bloom_filter {
889            return None;
890        }
891
892        let file_cache = self.cache_strategy.write_cache().map(|w| w.file_cache());
893        let bloom_filter_index_cache = self.cache_strategy.bloom_filter_index_cache().cloned();
894        let puffin_metadata_cache = self.cache_strategy.puffin_metadata_cache().cloned();
895
896        BloomFilterIndexApplierBuilder::new(
897            self.access_layer.table_dir().to_string(),
898            self.access_layer.path_type(),
899            self.access_layer.object_store().clone(),
900            self.version.metadata.as_ref(),
901            self.access_layer.puffin_manager_factory().clone(),
902        )
903        .with_file_cache(file_cache)
904        .with_bloom_filter_index_cache(bloom_filter_index_cache)
905        .with_puffin_metadata_cache(puffin_metadata_cache)
906        .build(filters)
907        .inspect_err(|err| warn!(err; "Failed to build bloom filter index applier"))
908        .ok()
909        .flatten()
910        .map(Arc::new)
911    }
912
913    /// Use the latest schema to build the fulltext index applier.
914    fn build_fulltext_index_applier(&self, filters: &[Expr]) -> Option<FulltextIndexApplierRef> {
915        if self.ignore_fulltext_index {
916            return None;
917        }
918
919        let file_cache = self.cache_strategy.write_cache().map(|w| w.file_cache());
920        let puffin_metadata_cache = self.cache_strategy.puffin_metadata_cache().cloned();
921        let bloom_filter_index_cache = self.cache_strategy.bloom_filter_index_cache().cloned();
922        FulltextIndexApplierBuilder::new(
923            self.access_layer.table_dir().to_string(),
924            self.access_layer.path_type(),
925            self.access_layer.object_store().clone(),
926            self.access_layer.puffin_manager_factory().clone(),
927            self.version.metadata.as_ref(),
928        )
929        .with_file_cache(file_cache)
930        .with_puffin_metadata_cache(puffin_metadata_cache)
931        .with_bloom_filter_cache(bloom_filter_index_cache)
932        .build(filters)
933        .inspect_err(|err| warn!(err; "Failed to build fulltext index applier"))
934        .ok()
935        .flatten()
936        .map(Arc::new)
937    }
938}
939
940/// Returns true if the time range of a SST `file` matches the `predicate`.
941fn file_in_range(file: &FileHandle, predicate: &TimestampRange) -> bool {
942    if predicate == &TimestampRange::min_to_max() {
943        return true;
944    }
945    // end timestamp of a SST is inclusive.
946    let (start, end) = file.time_range();
947    let file_ts_range = TimestampRange::new_inclusive(Some(start), Some(end));
948    file_ts_range.intersects(predicate)
949}
950
951/// Returns true if `time_range` contains the inclusive time range of `file`.
952fn time_range_covers_file(time_range: Option<&TimestampRange>, file: &FileHandle) -> bool {
953    let Some(time_range) = time_range else {
954        return false;
955    };
956    let (start, end) = file.time_range();
957    time_range.contains(&start) && time_range.contains(&end)
958}
959
960/// Common input for different scanners.
961pub struct ScanInput {
962    /// Pinned series-index snapshot and its storage, when configured.
963    pub(crate) series_index: Option<SeriesIndexReadContext>,
964    /// Whether to ignore range indexes while retaining series-index candidate discovery.
965    ignore_range_index: bool,
966    /// Region SST access layer.
967    access_layer: AccessLayerRef,
968    /// Maps projected Batches to RecordBatches.
969    pub(crate) mapper: Arc<FlatProjectionMapper>,
970    /// The columns to read from memtables and SSTs.
971    /// Notice this is different from the columns in `mapper` which are projected columns.
972    /// But this read columns might also include non-projected columns needed for filtering.
973    pub(crate) read_cols: ReadColumns,
974    /// Time range filter for time index.
975    pub(crate) time_range: Option<TimestampRange>,
976    /// Cached analysis of the finalized scan request.
977    scan_analysis: Option<ScanAnalysis>,
978    /// Predicate to push down.
979    pub(crate) predicate: PredicateGroup,
980    /// Region partition expr applied at read time.
981    region_partition_expr: Option<PartitionExpr>,
982    /// Memtable range builders for memtables in the time range..
983    pub(crate) memtables: Vec<MemRangeBuilder>,
984    /// Handles to SST files to scan.
985    pub(crate) files: Vec<FileHandle>,
986    /// Visible sources excluded solely by the query time range.
987    pub(crate) time_pruned_files: HashSet<store_api::storage::FileId>,
988    /// Shares the pinned schema's encoded defaults across parallel range readers.
989    primary_key_mapper: OnceLock<Arc<PrimaryKeyRangeMapper>>,
990    /// Scan-wide hint for rows in an execution batch.
991    batch_size: usize,
992    /// Cache.
993    pub(crate) cache_strategy: CacheStrategy,
994    /// Ignores file not found error.
995    ignore_file_not_found: bool,
996    /// Maximum number of SST files to scan concurrently.
997    pub(crate) max_concurrent_scan_files: usize,
998    /// Shared scan quota for projected SST row groups; absent during compaction.
999    scan_memory_tracker: Option<QueryMemoryTracker>,
1000    /// Index appliers.
1001    inverted_index_appliers: [Option<InvertedIndexApplierRef>; 2],
1002    bloom_filter_index_appliers: [Option<BloomFilterIndexApplierRef>; 2],
1003    fulltext_index_appliers: [Option<FulltextIndexApplierRef>; 2],
1004    /// Start time of the query.
1005    pub(crate) query_start: Option<Instant>,
1006    /// The region is using append mode.
1007    pub(crate) append_mode: bool,
1008    /// Whether to remove deletion markers.
1009    pub(crate) filter_deleted: bool,
1010    /// Mode to merge duplicate rows.
1011    pub(crate) merge_mode: MergeMode,
1012    /// Hint to select rows from time series.
1013    pub(crate) series_row_selector: Option<TimeSeriesRowSelector>,
1014    /// Hint for the required distribution of the scanner.
1015    pub(crate) distribution: Option<TimeSeriesDistribution>,
1016    /// Whether the region's configured SST format is flat.
1017    explain_flat_format: bool,
1018    /// Snapshot upper bound bound at scan open and propagated back to the caller.
1019    pub(crate) snapshot_sequence: Option<SequenceNumber>,
1020    /// Set only when the region preserves per-row sequences
1021    /// (`preserve_row_sequence` on an append-only table) and the scan
1022    /// carries an explicit exact range. When set, SST readers apply the same
1023    /// `(checkpoint, upper_bound]` row-level filter as memtables.
1024    pub(crate) sequence_range: Option<SequenceRange>,
1025    /// Whether this scan is for compaction.
1026    pub(crate) compaction: bool,
1027    /// Max number of compaction inputs decoded concurrently ahead of the merge.
1028    /// Values below 2 decode inputs on the merging task.
1029    pub(crate) compaction_read_parallelism: usize,
1030    /// Compaction-only JSON2 physical rewrite targets.
1031    json2_rewrite_targets: Json2RewriteTargets,
1032    /// Counters that should receive query-load metrics.
1033    pub(crate) query_stat_counters: Option<RegionQueryStatCounters>,
1034    #[cfg(feature = "enterprise")]
1035    extension_ranges: Vec<BoxedExtensionRange>,
1036}
1037
1038/// Cached output of scan request analysis: the optional range-cache fingerprint
1039/// plus the derived implied time range used by range cache and prefilter.
1040struct ScanAnalysis {
1041    /// `None` when the scan is not eligible for range caching.
1042    fingerprint: Option<ScanRequestFingerprint>,
1043    /// `Some(r)` = all time-only predicates are guaranteed true on `r` (in the
1044    /// column's `TimeUnit`).
1045    /// `None`    = there is no analyzable time-only predicate or at least one
1046    /// predicate could not be proven (e.g. `OR`), so the time-filter
1047    /// optimization is disabled for this scan.
1048    implied_time_range: Option<TimestampRange>,
1049}
1050
1051/// Builder for a finalized [ScanInput].
1052pub(crate) struct ScanInputBuilder {
1053    input: ScanInput,
1054}
1055
1056impl ScanInput {
1057    /// Creates a new [ScanInputBuilder].
1058    #[must_use]
1059    pub(crate) fn builder(
1060        access_layer: AccessLayerRef,
1061        mapper: FlatProjectionMapper,
1062    ) -> ScanInputBuilder {
1063        ScanInputBuilder {
1064            input: ScanInput {
1065                series_index: None,
1066                ignore_range_index: false,
1067                access_layer,
1068                read_cols: mapper.read_columns().clone(),
1069                mapper: Arc::new(mapper),
1070                time_range: None,
1071                scan_analysis: None,
1072                predicate: PredicateGroup::default(),
1073                region_partition_expr: None,
1074                memtables: Vec::new(),
1075                files: Vec::new(),
1076                time_pruned_files: HashSet::new(),
1077                primary_key_mapper: OnceLock::new(),
1078                batch_size: crate::sst::parquet::DEFAULT_READ_BATCH_SIZE,
1079                cache_strategy: CacheStrategy::Disabled,
1080                ignore_file_not_found: false,
1081                max_concurrent_scan_files: DEFAULT_MAX_CONCURRENT_SCAN_FILES,
1082                scan_memory_tracker: None,
1083                inverted_index_appliers: [None, None],
1084                bloom_filter_index_appliers: [None, None],
1085                fulltext_index_appliers: [None, None],
1086                query_start: None,
1087                append_mode: false,
1088                filter_deleted: true,
1089                merge_mode: MergeMode::default(),
1090                series_row_selector: None,
1091                distribution: None,
1092                explain_flat_format: false,
1093                snapshot_sequence: None,
1094                sequence_range: None,
1095                compaction: false,
1096                compaction_read_parallelism: 1,
1097                json2_rewrite_targets: Arc::default(),
1098                query_stat_counters: None,
1099                #[cfg(feature = "enterprise")]
1100                extension_ranges: Vec::new(),
1101            },
1102        }
1103    }
1104
1105    /// Returns the scan-wide hint for rows in an execution batch.
1106    pub(crate) fn batch_size(&self) -> usize {
1107        self.batch_size
1108    }
1109
1110    /// Interprets file statistics using this scan's pinned schema.
1111    pub(crate) fn primary_key_mapper(&self) -> &PrimaryKeyRangeMapper {
1112        self.primary_key_mapper
1113            .get_or_init(|| Arc::new(PrimaryKeyRangeMapper::new(self.region_metadata().clone())))
1114    }
1115
1116    /// Returns the range implied by the range-cache time filters.
1117    pub(crate) fn implied_time_range(&self) -> Option<&TimestampRange> {
1118        self.scan_analysis
1119            .as_ref()
1120            .expect("ScanInput must be built")
1121            .implied_time_range
1122            .as_ref()
1123    }
1124
1125    /// Returns the fingerprint for partition range caching.
1126    pub(crate) fn scan_fingerprint(&self) -> Option<&ScanRequestFingerprint> {
1127        self.scan_analysis
1128            .as_ref()
1129            .expect("ScanInput must be built")
1130            .fingerprint
1131            .as_ref()
1132    }
1133}
1134
1135impl ScanInputBuilder {
1136    fn with_primary_key_mapper(mut self, mapper: Arc<PrimaryKeyRangeMapper>) -> Self {
1137        self.input.primary_key_mapper = OnceLock::from(mapper);
1138        self
1139    }
1140
1141    /// Sets whether to ignore range indexes during scans.
1142    #[must_use]
1143    pub(crate) fn with_ignore_range_index(mut self, ignore: bool) -> Self {
1144        self.input.ignore_range_index = ignore;
1145        self
1146    }
1147
1148    /// Sets the pinned series-index context for candidate discovery.
1149    #[must_use]
1150    pub(crate) fn with_series_index(
1151        mut self,
1152        series_index: Option<SeriesIndexReadContext>,
1153    ) -> Self {
1154        self.input.series_index = series_index;
1155        self
1156    }
1157
1158    /// Sets time range filter for time index.
1159    #[must_use]
1160    pub(crate) fn with_time_range(mut self, time_range: Option<TimestampRange>) -> Self {
1161        self.input.time_range = time_range;
1162        self
1163    }
1164
1165    /// Sets predicate to push down.
1166    #[must_use]
1167    pub(crate) fn with_predicate(mut self, predicate: PredicateGroup) -> Self {
1168        self.input.region_partition_expr = predicate.region_partition_expr().cloned();
1169        self.input.predicate = predicate;
1170        self
1171    }
1172
1173    /// Builds a finalized [ScanInput] and computes its scan analysis.
1174    #[must_use]
1175    pub(crate) fn build(mut self) -> ScanInput {
1176        if self.input.compaction {
1177            self.input.scan_memory_tracker = None;
1178        }
1179        let input = &self.input;
1180        let eligible = !input.compaction
1181            && !input.files.is_empty()
1182            && matches!(input.cache_strategy, CacheStrategy::EnableAll(_));
1183
1184        let metadata = input.region_metadata();
1185        let tag_names: HashSet<&str> = metadata
1186            .column_metadatas
1187            .iter()
1188            .filter(|col| col.semantic_type == SemanticType::Tag)
1189            .map(|col| col.column_schema.name.as_str())
1190            .collect();
1191
1192        let time_index = metadata.time_index_column();
1193        let time_index_name = time_index.column_schema.name.clone();
1194        let ts_col_unit = time_index
1195            .column_schema
1196            .data_type
1197            .as_timestamp()
1198            .expect("Time index must have timestamp-compatible type")
1199            .unit();
1200
1201        let exprs = input
1202            .predicate_group()
1203            .predicate_without_region()
1204            .map(|predicate| predicate.exprs())
1205            .unwrap_or_default();
1206
1207        let mut filters = Vec::new();
1208        let mut time_only_exprs: Vec<&Expr> = Vec::new();
1209        let mut has_tag_filter = false;
1210        let mut columns = HashSet::new();
1211
1212        for expr in exprs {
1213            columns.clear();
1214            let is_time_only = match expr_to_columns(expr, &mut columns) {
1215                Ok(()) if !columns.is_empty() => {
1216                    has_tag_filter |= columns
1217                        .iter()
1218                        .any(|col| tag_names.contains(col.name.as_str()));
1219                    columns.iter().all(|col| col.name == time_index_name)
1220                }
1221                _ => false,
1222            };
1223
1224            // Route time-only exprs that the legacy extractor recognizes into
1225            // `time_only_exprs` so the implication walker
1226            // (`implied_time_range_from_exprs`, called below) can attempt to drop
1227            // them from the cache key when the partition's `FileTimeRange` is fully
1228            // covered, then stringify them into the fingerprint's `time_filters`
1229            // bucket. Time-only exprs that the extractor doesn't recognize stay in
1230            // `filters` and never get stripped — conservatively correct.
1231            if is_time_only
1232                && extract_time_range_from_expr(&time_index_name, ts_col_unit, expr).is_some()
1233            {
1234                time_only_exprs.push(expr);
1235            } else {
1236                filters.push(expr.to_string());
1237            }
1238        }
1239
1240        let implied_time_range =
1241            implied_time_range_from_exprs(&time_index_name, ts_col_unit, &time_only_exprs);
1242
1243        // We only cache requests that have tag filters to avoid caching all series.
1244        let fingerprint = if eligible && has_tag_filter {
1245            let mut time_filters: Vec<String> =
1246                time_only_exprs.iter().map(|e| e.to_string()).collect();
1247
1248            // Ensure the filters are sorted for consistent fingerprinting.
1249            filters.sort_unstable();
1250            time_filters.sort_unstable();
1251            let read_columns = input.read_cols.clone();
1252            let fingerprint = crate::read::range_cache::ScanRequestFingerprintBuilder {
1253                read_column_types: read_columns
1254                    .column_ids_iter()
1255                    .map(|id| {
1256                        read_columns
1257                            .json_target_type(id)
1258                            .cloned()
1259                            .map(ConcreteDataType::json2)
1260                            .or_else(|| {
1261                                metadata
1262                                    .column_by_id(id)
1263                                    .map(|col| col.column_schema.data_type.clone())
1264                            })
1265                    })
1266                    .collect(),
1267                read_columns,
1268                filters,
1269                time_filters,
1270                series_row_selector: input.series_row_selector,
1271                append_mode: input.append_mode,
1272                filter_deleted: input.filter_deleted,
1273                merge_mode: input.merge_mode,
1274                sequence_range: input.sequence_range,
1275                partition_expr_version: metadata.partition_expr_version,
1276            }
1277            .build();
1278            Some(fingerprint)
1279        } else {
1280            None
1281        };
1282
1283        self.input.scan_analysis = Some(ScanAnalysis {
1284            fingerprint,
1285            implied_time_range,
1286        });
1287        self.input
1288    }
1289
1290    /// Sets memtable range builders.
1291    #[must_use]
1292    pub(crate) fn with_memtables(mut self, memtables: Vec<MemRangeBuilder>) -> Self {
1293        self.input.memtables = memtables;
1294        self
1295    }
1296
1297    /// Sets visible sources excluded solely by time pruning.
1298    #[must_use]
1299    pub(crate) fn with_time_pruned_files(
1300        mut self,
1301        files: HashSet<store_api::storage::FileId>,
1302    ) -> Self {
1303        self.input.time_pruned_files = files;
1304        self
1305    }
1306
1307    /// Sets files to read.
1308    #[must_use]
1309    pub(crate) fn with_files(mut self, files: Vec<FileHandle>) -> Self {
1310        self.input.files = files;
1311        self
1312    }
1313
1314    /// Sets the scan-wide hint for rows in an execution batch.
1315    #[must_use]
1316    pub(crate) fn with_batch_size(mut self, batch_size: usize) -> Self {
1317        self.input.batch_size = batch_size;
1318        self
1319    }
1320
1321    /// Sets cache for this query.
1322    #[must_use]
1323    pub(crate) fn with_cache(mut self, cache: CacheStrategy) -> Self {
1324        self.input.cache_strategy = cache;
1325        self
1326    }
1327
1328    /// Ignores file not found error.
1329    #[must_use]
1330    pub(crate) fn with_ignore_file_not_found(mut self, ignore: bool) -> Self {
1331        self.input.ignore_file_not_found = ignore;
1332        self
1333    }
1334
1335    /// Sets maximum number of SST files to scan concurrently.
1336    #[must_use]
1337    pub(crate) fn with_max_concurrent_scan_files(
1338        mut self,
1339        max_concurrent_scan_files: usize,
1340    ) -> Self {
1341        self.input.max_concurrent_scan_files = max_concurrent_scan_files;
1342        self
1343    }
1344
1345    pub(crate) fn with_scan_memory_tracker(mut self, tracker: Option<QueryMemoryTracker>) -> Self {
1346        self.input.scan_memory_tracker = tracker;
1347        self
1348    }
1349
1350    /// Sets inverted index appliers.
1351    #[must_use]
1352    pub(crate) fn with_inverted_index_appliers(
1353        mut self,
1354        appliers: [Option<InvertedIndexApplierRef>; 2],
1355    ) -> Self {
1356        self.input.inverted_index_appliers = appliers;
1357        self
1358    }
1359
1360    /// Sets bloom filter appliers.
1361    #[must_use]
1362    pub(crate) fn with_bloom_filter_index_appliers(
1363        mut self,
1364        appliers: [Option<BloomFilterIndexApplierRef>; 2],
1365    ) -> Self {
1366        self.input.bloom_filter_index_appliers = appliers;
1367        self
1368    }
1369
1370    /// Sets fulltext index appliers.
1371    #[must_use]
1372    pub(crate) fn with_fulltext_index_appliers(
1373        mut self,
1374        appliers: [Option<FulltextIndexApplierRef>; 2],
1375    ) -> Self {
1376        self.input.fulltext_index_appliers = appliers;
1377        self
1378    }
1379
1380    /// Sets start time of the query.
1381    #[must_use]
1382    pub(crate) fn with_start_time(mut self, now: Option<Instant>) -> Self {
1383        self.input.query_start = now;
1384        self
1385    }
1386
1387    #[must_use]
1388    pub(crate) fn with_append_mode(mut self, is_append_mode: bool) -> Self {
1389        self.input.append_mode = is_append_mode;
1390        self
1391    }
1392
1393    pub(crate) fn with_query_stat_counters(
1394        mut self,
1395        counters: Option<RegionQueryStatCounters>,
1396    ) -> Self {
1397        self.input.query_stat_counters = counters;
1398        self
1399    }
1400
1401    /// Sets whether to remove deletion markers during scan.
1402    #[must_use]
1403    pub(crate) fn with_filter_deleted(mut self, filter_deleted: bool) -> Self {
1404        self.input.filter_deleted = filter_deleted;
1405        self
1406    }
1407
1408    /// Sets the merge mode.
1409    #[must_use]
1410    pub(crate) fn with_merge_mode(mut self, merge_mode: MergeMode) -> Self {
1411        self.input.merge_mode = merge_mode;
1412        self
1413    }
1414
1415    /// Sets the distribution hint.
1416    #[must_use]
1417    pub(crate) fn with_distribution(
1418        mut self,
1419        distribution: Option<TimeSeriesDistribution>,
1420    ) -> Self {
1421        self.input.distribution = distribution;
1422        self
1423    }
1424
1425    /// Sets whether the region's configured SST format is flat for explain output.
1426    #[must_use]
1427    pub(crate) fn with_explain_flat_format(mut self, explain_flat_format: bool) -> Self {
1428        self.input.explain_flat_format = explain_flat_format;
1429        self
1430    }
1431
1432    /// Sets the time series row selector.
1433    #[must_use]
1434    pub(crate) fn with_series_row_selector(
1435        mut self,
1436        series_row_selector: Option<TimeSeriesRowSelector>,
1437    ) -> Self {
1438        self.input.series_row_selector = series_row_selector;
1439        self
1440    }
1441
1442    #[must_use]
1443    pub(crate) fn with_snapshot_sequence(
1444        mut self,
1445        snapshot_sequence: Option<SequenceNumber>,
1446    ) -> Self {
1447        self.input.snapshot_sequence = snapshot_sequence;
1448        self
1449    }
1450
1451    #[must_use]
1452    pub(crate) fn with_sequence_range(mut self, sequence_range: Option<SequenceRange>) -> Self {
1453        self.input.sequence_range = sequence_range;
1454        self
1455    }
1456
1457    /// Sets whether this scan is for compaction.
1458    #[must_use]
1459    pub(crate) fn with_compaction(mut self, compaction: bool) -> Self {
1460        self.input.compaction = compaction;
1461        self
1462    }
1463
1464    /// Sets the max number of compaction inputs decoded concurrently.
1465    #[must_use]
1466    pub(crate) fn with_compaction_read_parallelism(mut self, parallelism: usize) -> Self {
1467        self.input.compaction_read_parallelism = parallelism;
1468        self
1469    }
1470
1471    /// Sets compaction-only JSON2 physical rewrite targets.
1472    #[must_use]
1473    pub(crate) fn with_json2_rewrite_targets(mut self, targets: Json2RewriteTargets) -> Self {
1474        self.input.json2_rewrite_targets = targets;
1475        self
1476    }
1477
1478    #[cfg(feature = "enterprise")]
1479    #[must_use]
1480    pub(crate) fn with_extension_ranges(
1481        mut self,
1482        extension_ranges: Vec<BoxedExtensionRange>,
1483    ) -> Self {
1484        self.input.extension_ranges = extension_ranges;
1485        self
1486    }
1487}
1488
1489impl ScanInput {
1490    pub(crate) fn uses_scan_memory_limit(&self) -> bool {
1491        let covered = self
1492            .scan_memory_tracker
1493            .as_ref()
1494            .is_some_and(|tracker| tracker.limit() > 0);
1495        #[cfg(feature = "enterprise")]
1496        let covered = covered && self.extension_ranges.is_empty();
1497        covered
1498    }
1499
1500    /// Builds memtable ranges to scan by `index`.
1501    pub(crate) fn build_mem_ranges(&self, index: RowGroupIndex) -> SmallVec<[MemtableRange; 2]> {
1502        let memtable = &self.memtables[index.index];
1503        let mut ranges = SmallVec::new();
1504        memtable.build_ranges(index.row_group_index, &mut ranges);
1505        ranges
1506    }
1507
1508    pub(crate) fn predicate_for_file(&self, file: &FileHandle) -> Option<Predicate> {
1509        if self.should_skip_region_partition(file) {
1510            self.predicate.predicate_without_region().cloned()
1511        } else {
1512            self.predicate.predicate().cloned()
1513        }
1514    }
1515
1516    fn should_skip_region_partition(&self, file: &FileHandle) -> bool {
1517        match (
1518            self.region_partition_expr.as_ref(),
1519            file.meta_ref().partition_expr.as_ref(),
1520        ) {
1521            (Some(region_expr), Some(file_expr)) => region_expr == file_expr.as_ref(),
1522            _ => false,
1523        }
1524    }
1525
1526    /// Tries to build file-level pruning statistics using only the [FileHandle]'s manifest-level
1527    /// time range, without reading any parquet metadata.
1528    ///
1529    /// Returns `None` if timestamp unit conversion overflows (conservative: keep the file).
1530    fn try_file_level_pruning_stats(&self, file: &FileHandle) -> Option<FileLevelPruningStats> {
1531        let (ts_min, ts_max) = file.time_range();
1532        let time_index = self.mapper.metadata().time_index_column();
1533        let time_index_unit = time_index.column_schema.data_type.as_timestamp()?.unit();
1534
1535        // Convert file timestamps to the time index column's unit. Use `convert_to_ceil` for
1536        // the upper bound to avoid accidentally shrinking the manifest range.
1537        let min_ts = ts_min.convert_to(time_index_unit)?;
1538        let max_ts = ts_max.convert_to_ceil(time_index_unit)?;
1539
1540        Some(FileLevelPruningStats {
1541            min_scalar: timestamp_to_scalar_value(time_index_unit, Some(min_ts.value())),
1542            max_scalar: timestamp_to_scalar_value(time_index_unit, Some(max_ts.value())),
1543            time_index_col_name: time_index.column_schema.name.clone(),
1544        })
1545    }
1546
1547    /// Checks whether a file can be definitively pruned using only its manifest-level
1548    /// time range and the current predicate, without reading any parquet metadata.
1549    ///
1550    /// Returns `true` if [PruningStatistics] proves the file cannot contain matching rows.
1551    #[inline]
1552    pub(crate) fn can_manifest_prune_file(&self, file: &FileHandle) -> bool {
1553        let predicate = self.predicate_for_file(file);
1554        self.manifest_prunes_file(file, predicate.as_ref())
1555    }
1556
1557    fn manifest_prunes_file(&self, file: &FileHandle, predicate: Option<&Predicate>) -> bool {
1558        if let Some(pred) = predicate
1559            && !pred.is_empty()
1560            && let Some(file_level_stats) = self.try_file_level_pruning_stats(file)
1561        {
1562            let pruning_results = pred.prune_with_stats(
1563                &file_level_stats,
1564                self.mapper.metadata().schema.arrow_schema(),
1565            );
1566            pruning_results.first() == Some(&false)
1567        } else {
1568            false
1569        }
1570    }
1571
1572    /// Prunes a file to scan and returns the builder to build readers.
1573    ///
1574    /// This is the public entry point used by direct tests and non-pruner callers.
1575    /// It performs its own manifest-level pruning check internally.
1576    #[tracing::instrument(
1577        skip_all,
1578        fields(
1579            region_id = %self.region_metadata().region_id,
1580            file_id = %file.file_id()
1581        )
1582    )]
1583    pub async fn prune_file(
1584        &self,
1585        file: &FileHandle,
1586        pre_filter_mode: PreFilterMode,
1587        reader_metrics: &mut ReaderMetrics,
1588    ) -> Result<FileRangeBuilder> {
1589        let predicate = self.predicate_for_file(file);
1590
1591        // Early file-level pruning using manifest time range before any parquet metadata access.
1592        if self.manifest_prunes_file(file, predicate.as_ref()) {
1593            reader_metrics.filter_metrics.files_time_range_pruned += 1;
1594            return Ok(FileRangeBuilder::default());
1595        }
1596
1597        self.prune_file_after_manifest_check(file, pre_filter_mode, true, predicate, reader_metrics)
1598            .await
1599    }
1600
1601    /// Second half of `prune_file` — performs the actual parquet metadata /
1602    /// reader setup. Callers that already performed manifest-level pruning
1603    /// (e.g. the `Pruner` via its shared `manifest_pruned_files` cache) should
1604    /// call this directly to avoid a redundant manifest check.
1605    ///
1606    /// `predicate` is the result of `self.predicate_for_file(file)` computed
1607    /// externally so the caller can reuse it if needed.
1608    pub(crate) async fn prune_file_after_manifest_check(
1609        &self,
1610        file: &FileHandle,
1611        pre_filter_mode: PreFilterMode,
1612        enable_predicate_prefilter: bool,
1613        predicate: Option<Predicate>,
1614        reader_metrics: &mut ReaderMetrics,
1615    ) -> Result<FileRangeBuilder> {
1616        let may_build_selective_row_selection = predicate.is_some();
1617        let postpone_time_index_filter = time_range_covers_file(self.implied_time_range(), file);
1618        let decode_pk_values = !self.compaction
1619            && self
1620                .mapper
1621                .read_columns()
1622                .column_ids_iter()
1623                .any(|column_id| self.mapper.metadata().primary_key.contains(&column_id));
1624        let reader = self
1625            .access_layer
1626            .read_sst(file.clone())
1627            .series_index(
1628                self.series_index
1629                    .clone()
1630                    .filter(|_| !self.ignore_range_index),
1631            )
1632            .memory_tracker(self.scan_memory_tracker.clone())
1633            .predicate(predicate)
1634            .projection(Some(self.read_cols.clone()))
1635            .json2_rewrite_targets(self.json2_rewrite_targets.clone())
1636            .cache(self.cache_strategy.clone())
1637            .inverted_index_appliers(self.inverted_index_appliers.clone())
1638            .bloom_filter_index_appliers(self.bloom_filter_index_appliers.clone())
1639            .fulltext_index_appliers(self.fulltext_index_appliers.clone());
1640        let reader = reader.batch_size(self.batch_size);
1641        let reader = if !self.compaction && may_build_selective_row_selection {
1642            reader.deferred_optional_page_index()
1643        } else {
1644            reader
1645        };
1646        let res = reader
1647            .expected_metadata(Some(self.mapper.metadata().clone()))
1648            .compaction(self.compaction)
1649            .pre_filter_mode(pre_filter_mode)
1650            .enable_predicate_prefilter(enable_predicate_prefilter)
1651            .postpone_time_index_filter(postpone_time_index_filter)
1652            .decode_primary_key_values(decode_pk_values)
1653            .build_reader_input(reader_metrics)
1654            .await;
1655        let read_input = match res {
1656            Ok(x) => x,
1657            Err(e) => {
1658                if e.is_object_not_found() && self.ignore_file_not_found {
1659                    error!(e; "File to scan does not exist, region_id: {}, file: {}", file.region_id(), file.file_id());
1660                    return Ok(FileRangeBuilder::default());
1661                } else {
1662                    return Err(e);
1663                }
1664            }
1665        };
1666
1667        let Some((mut file_range_ctx, selection)) = read_input else {
1668            return Ok(FileRangeBuilder::default());
1669        };
1670
1671        let need_compat = !compat::has_same_columns_and_pk_encoding(
1672            &self.mapper,
1673            file_range_ctx.read_format(),
1674            self.compaction,
1675        );
1676        if need_compat {
1677            // They have different schema. We need to adapt the batch first so the
1678            // mapper can convert it.
1679            let compat = FlatCompatBatch::try_new(
1680                &self.mapper,
1681                file_range_ctx.read_format(),
1682                self.compaction,
1683            )?;
1684            file_range_ctx.set_compat_batch(compat);
1685        }
1686        Ok(FileRangeBuilder::new(Arc::new(file_range_ctx), selection))
1687    }
1688
1689    /// Scans flat sources (RecordBatch streams) in parallel.
1690    ///
1691    /// # Panics if the input doesn't allow parallel scan.
1692    #[tracing::instrument(
1693        skip(self, sources, semaphore),
1694        fields(
1695            region_id = %self.region_metadata().region_id,
1696            source_count = sources.len()
1697        )
1698    )]
1699    pub(crate) fn create_parallel_flat_sources(
1700        &self,
1701        sources: Vec<BoxedRecordBatchStream>,
1702        semaphore: Arc<Semaphore>,
1703        channel_size: usize,
1704    ) -> Result<Vec<BoxedRecordBatchStream>> {
1705        if sources.len() <= 1 {
1706            return Ok(sources);
1707        }
1708
1709        // Spawn a task for each source.
1710        let sources = sources
1711            .into_iter()
1712            .map(|source| {
1713                let (sender, receiver) = mpsc::channel(channel_size);
1714                self.spawn_flat_scan_task(source, semaphore.clone(), sender);
1715                let stream = Box::pin(ReceiverStream::new(receiver));
1716                Box::pin(stream) as _
1717            })
1718            .collect();
1719        Ok(sources)
1720    }
1721
1722    /// Spawns a task to scan a flat source (RecordBatch stream) asynchronously.
1723    #[tracing::instrument(
1724        skip(self, input, semaphore, sender),
1725        fields(region_id = %self.region_metadata().region_id)
1726    )]
1727    pub(crate) fn spawn_flat_scan_task(
1728        &self,
1729        mut input: BoxedRecordBatchStream,
1730        semaphore: Arc<Semaphore>,
1731        sender: mpsc::Sender<Result<RecordBatch>>,
1732    ) {
1733        let region_id = self.region_metadata().region_id;
1734        let span = tracing::info_span!(
1735            "ScanInput::parallel_scan_task",
1736            region_id = %region_id,
1737            stream_kind = "flat"
1738        );
1739        common_runtime::spawn_query(
1740            async move {
1741                loop {
1742                    // We release the permit before sending result to avoid the task waiting on
1743                    // the channel with the permit held.
1744                    let maybe_batch = {
1745                        tokio::select! {
1746                            // Cancellation or early completion can drop the receiver while
1747                            // we wait for a permit or I/O. Exit and drop the
1748                            // input to release its reservations without waiting for a batch.
1749                            _ = sender.closed() => break,
1750                            batch = async {
1751                                // Safety: We never close the semaphore.
1752                                let _permit = semaphore.acquire().await.unwrap();
1753                                input.next().await
1754                            } => batch,
1755                        }
1756                    };
1757                    match maybe_batch {
1758                        Some(Ok(batch)) => {
1759                            // The receiver is gone when the query is cancelled or finishes early,
1760                            // so stop reading the source.
1761                            if let Err(e) = sender.send(Ok(batch)).await {
1762                                debug!(
1763                                    "Stop parallel scan task, receiver dropped, region_id: {}, error: {}",
1764                                    region_id, e
1765                                );
1766                                break;
1767                            }
1768                        }
1769                        Some(Err(e)) => {
1770                            let _ = sender.send(Err(e)).await;
1771                            break;
1772                        }
1773                        None => break,
1774                    }
1775                }
1776            }
1777            .instrument(span),
1778        );
1779    }
1780
1781    /// Whether physical source row counts are known and equal the rows visible
1782    /// to this region, before applying query predicates.
1783    pub(crate) fn total_rows_is_exact(&self) -> bool {
1784        // Legacy SST metadata uses zero for unknown row counts.
1785        if self.files.iter().any(|file| file.num_rows() == 0) {
1786            return false;
1787        }
1788
1789        if self.region_partition_expr.is_none() {
1790            return true;
1791        }
1792
1793        // Extension ranges do not expose their partition expressions.
1794        #[cfg(feature = "enterprise")]
1795        if !self.extension_ranges.is_empty() {
1796            return false;
1797        }
1798
1799        // Repartition flushes existing memtables before entering staging. New
1800        // writes are routed under the target partition rules, whereas historical
1801        // SSTs can be shared by regions and still need partition filtering.
1802        self.files
1803            .iter()
1804            .all(|file| self.should_skip_region_partition(file))
1805    }
1806
1807    pub(crate) fn total_rows(&self) -> usize {
1808        let rows_in_files: usize = self.files.iter().map(|f| f.num_rows()).sum();
1809        let rows_in_memtables: usize = self.memtables.iter().map(|m| m.stats().num_rows()).sum();
1810
1811        let rows = rows_in_files + rows_in_memtables;
1812        #[cfg(feature = "enterprise")]
1813        let rows = rows
1814            + self
1815                .extension_ranges
1816                .iter()
1817                .map(|x| x.num_rows())
1818                .sum::<u64>() as usize;
1819        rows
1820    }
1821
1822    pub(crate) fn predicate_group(&self) -> &PredicateGroup {
1823        &self.predicate
1824    }
1825
1826    /// Returns number of memtables to scan.
1827    pub(crate) fn num_memtables(&self) -> usize {
1828        self.memtables.len()
1829    }
1830
1831    /// Returns number of SST files to scan.
1832    pub(crate) fn num_files(&self) -> usize {
1833        self.files.len()
1834    }
1835
1836    /// Gets the file handle from a row group index.
1837    pub(crate) fn file_from_index(&self, index: RowGroupIndex) -> &FileHandle {
1838        let file_index = index.index - self.num_memtables();
1839        &self.files[file_index]
1840    }
1841
1842    pub fn region_metadata(&self) -> &RegionMetadataRef {
1843        self.mapper.metadata()
1844    }
1845
1846    fn range_pre_filter_mode(&self, source_count: usize) -> PreFilterMode {
1847        if source_count <= 1 {
1848            // Duplicated rows in the same source is not a normal case and we don't provide
1849            // strict dedup semantic (last_row/last_non_null) for it. We expect the duplicated rows
1850            // are exactly identical in the same source so we use PreFilterMode::All for
1851            // performance reason.
1852            return PreFilterMode::All;
1853        }
1854
1855        pre_filter_mode(self.append_mode, self.merge_mode)
1856    }
1857}
1858
1859#[cfg(feature = "enterprise")]
1860impl ScanInput {
1861    #[cfg(feature = "enterprise")]
1862    pub(crate) fn extension_ranges(&self) -> &[BoxedExtensionRange] {
1863        &self.extension_ranges
1864    }
1865
1866    /// Get a boxed [ExtensionRange] by the index in all ranges.
1867    #[cfg(feature = "enterprise")]
1868    pub(crate) fn extension_range(&self, i: usize) -> &BoxedExtensionRange {
1869        &self.extension_ranges[i - self.num_memtables() - self.num_files()]
1870    }
1871}
1872
1873/// Lightweight [PruningStatistics] that only uses the file-level time range from manifest
1874/// metadata, avoiding any parquet metadata reads. Used for early file-level pruning before
1875/// accessing row-group-level statistics.
1876pub(crate) struct FileLevelPruningStats {
1877    /// Scalar value for the file's minimum timestamp in the time index column's unit.
1878    pub(crate) min_scalar: ScalarValue,
1879    /// Scalar value for the file's maximum timestamp in the time index column's unit.
1880    pub(crate) max_scalar: ScalarValue,
1881    /// Name of the time index column.
1882    pub(crate) time_index_col_name: String,
1883}
1884
1885impl PruningStatistics for FileLevelPruningStats {
1886    fn min_values(&self, column: &Column) -> Option<ArrayRef> {
1887        if column.name == self.time_index_col_name {
1888            ScalarValue::iter_to_array(std::iter::once(self.min_scalar.clone())).ok()
1889        } else {
1890            None
1891        }
1892    }
1893
1894    fn max_values(&self, column: &Column) -> Option<ArrayRef> {
1895        if column.name == self.time_index_col_name {
1896            ScalarValue::iter_to_array(std::iter::once(self.max_scalar.clone())).ok()
1897        } else {
1898            None
1899        }
1900    }
1901
1902    fn num_containers(&self) -> usize {
1903        1
1904    }
1905
1906    fn null_counts(&self, column: &Column) -> Option<ArrayRef> {
1907        if column.name == self.time_index_col_name {
1908            // The time index column is NOT NULL.
1909            Some(Arc::new(UInt64Array::from(vec![0u64])))
1910        } else {
1911            None
1912        }
1913    }
1914
1915    fn row_counts(&self) -> Option<ArrayRef> {
1916        None
1917    }
1918
1919    fn contained(&self, _column: &Column, _values: &HashSet<ScalarValue>) -> Option<BooleanArray> {
1920        None
1921    }
1922}
1923
1924#[cfg(test)]
1925impl ScanInput {
1926    /// Returns SST file ids to scan.
1927    pub(crate) fn file_ids(&self) -> Vec<crate::sst::file::RegionFileId> {
1928        self.files.iter().map(|file| file.file_id()).collect()
1929    }
1930
1931    pub(crate) fn index_ids(&self) -> Vec<crate::sst::file::RegionIndexId> {
1932        self.files.iter().map(|file| file.index_id()).collect()
1933    }
1934}
1935
1936fn pre_filter_mode(append_mode: bool, merge_mode: MergeMode) -> PreFilterMode {
1937    if append_mode {
1938        return PreFilterMode::All;
1939    }
1940
1941    match merge_mode {
1942        MergeMode::LastRow => PreFilterMode::SkipFields,
1943        MergeMode::LastNonNull => PreFilterMode::SkipFields,
1944    }
1945}
1946
1947/// Selects the SST files for a scan and, when requested, determines whether
1948/// the selected files support an exact sequence-range scan.
1949///
1950/// Files excluded by the request time range are not selected: `(C, H]` rows are
1951/// a subset of the query's time-range rows, so a time-pruned file cannot
1952/// contribute a row to `(C, H]`.
1953///
1954/// Unmarked local files use `FileMeta.sequence` as an admission barrier rather
1955/// than a row maximum. Region edits allocate this barrier; compaction inherits
1956/// the maximum input bound without assigning a new barrier. `C >= barrier`
1957/// proves Flow has already consumed the entire file, so such a file is excluded
1958/// before the capability check. An unknown bound cannot prove this exclusion.
1959///
1960/// A foreign file is different: the parquet reader virtualizes every row to its
1961/// target-local `FileMeta.sequence`. Consequently, a present sequence is the
1962/// only trust requirement for a foreign file; its source marker is irrelevant.
1963/// A foreign file without that barrier is retained as a failed-closed error so
1964/// it cannot be mistaken for a local sequence domain. This check is performed
1965/// even when another exact-range capability condition would return `None`.
1966pub(crate) fn exact_sequence_range(
1967    request: &ScanRequest,
1968    version: &crate::region::version::Version,
1969) -> Result<(Vec<FileHandle>, Option<SequenceRange>)> {
1970    if request.skip_sst_files {
1971        return Ok((Vec::new(), None));
1972    }
1973
1974    let time_index = version.metadata.time_index_column();
1975    let unit = time_index
1976        .column_schema
1977        .data_type
1978        .as_timestamp()
1979        .expect("Time index must have timestamp-compatible type")
1980        .unit();
1981    let time_range =
1982        build_time_range_predicate(&time_index.column_schema.name, unit, &request.filters);
1983    let min = request.memtable_min_sequence;
1984    let mut check_capability = request.exact_sequence_range && min.is_some();
1985    let sst_min_sequence = request.sst_min_sequence.and_then(NonZeroU64::new);
1986    let mut files = Vec::new();
1987    let mut files_allow_exact_range = true;
1988
1989    for file in version
1990        .ssts
1991        .levels()
1992        .iter()
1993        .flat_map(|level| level.files.values())
1994        .filter(|file| file_in_range(file, &time_range))
1995    {
1996        let meta = file.meta_ref();
1997        let selected = (!request.exact_sequence_range
1998            || min.is_none_or(|min| meta.sequence.is_none_or(|sequence| sequence.get() > min)))
1999            && match (sst_min_sequence, meta.sequence) {
2000                (Some(min_sequence), Some(file_sequence)) => file_sequence > min_sequence,
2001                // A missing file sequence is treated as newer than the SST
2002                // pruning hint, matching scan input construction.
2003                (Some(_), None) | (None, _) => true,
2004            };
2005        if !selected {
2006            continue;
2007        }
2008
2009        if let Some(min) = min
2010            && check_capability
2011        {
2012            if meta.region_id != version.metadata.region_id && meta.sequence.is_none() {
2013                return RegionSequenceDomainBrokenSnafu {
2014                    region_id: version.metadata.region_id,
2015                    file_region_id: meta.region_id,
2016                    file_id: meta.file_id,
2017                }
2018                .fail();
2019            }
2020            if meta.region_id == version.metadata.region_id
2021                && !file.is_effective_target_sequence_trusted(version.metadata.region_id)
2022                && meta.sequence.is_none_or(|barrier| barrier.get() > min)
2023            {
2024                // Match the capability helper's short-circuit behavior: once
2025                // an unadmitted local file is found, later files cannot change
2026                // the result or expose a foreign-domain error.
2027                files_allow_exact_range = false;
2028                check_capability = false;
2029            }
2030        }
2031        files.push(file.clone());
2032    }
2033
2034    let sequence_range = match (
2035        request.exact_sequence_range,
2036        min,
2037        version.options.preserve_row_sequence,
2038        request.memtable_max_sequence,
2039        files_allow_exact_range,
2040    ) {
2041        (true, Some(min), true, Some(max), true) => Some(SequenceRange::GtLtEq { min, max }),
2042        _ => None,
2043    };
2044    Ok((files, sequence_range))
2045}
2046
2047/// Metadata totals for distinct SST files, not post-filter rows or actual read bytes.
2048/// Older files may have unknown row and row-group counts represented by zero.
2049#[derive(Default, serde::Serialize)]
2050struct FileSummary {
2051    files: usize,
2052    rows: u64,
2053    row_groups: u64,
2054    size: u64,
2055    index_size: u64,
2056}
2057
2058impl FileSummary {
2059    fn new<'a>(files: impl IntoIterator<Item = &'a FileHandle>) -> Self {
2060        let mut summary = Self::default();
2061        for file in files {
2062            summary.files += 1;
2063            summary.rows += file.meta_ref().num_rows;
2064            summary.row_groups += file.meta_ref().num_row_groups;
2065            summary.size += file.size();
2066            summary.index_size += file.index_size();
2067        }
2068        summary
2069    }
2070}
2071
2072/// Keeps the existing debug-list spacing while escaping strings as JSON, including
2073/// control characters for which Rust's Debug escaping is not valid JSON.
2074struct ExplainString<'a>(&'a str);
2075
2076impl fmt::Debug for ExplainString<'_> {
2077    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2078        f.write_str(&serde_json::to_string(self.0).map_err(|_| fmt::Error)?)
2079    }
2080}
2081
2082/// Context shared by different streams from a scanner.
2083/// It contains the input and ranges to scan.
2084pub struct StreamContext {
2085    /// Input memtables and files.
2086    pub input: ScanInput,
2087    /// Metadata for partition ranges.
2088    pub(crate) ranges: Vec<RangeMeta>,
2089    // Metrics:
2090    /// The start time of the query.
2091    pub(crate) query_start: Instant,
2092}
2093
2094impl StreamContext {
2095    /// Creates a new [StreamContext] for [SeqScan].
2096    pub(crate) fn seq_scan_ctx(input: ScanInput) -> Self {
2097        let query_start = input.query_start.unwrap_or_else(Instant::now);
2098        let ranges = RangeMeta::seq_scan_ranges(&input);
2099        READ_SST_COUNT.observe(input.num_files() as f64);
2100        Self {
2101            input,
2102            ranges,
2103            query_start,
2104        }
2105    }
2106
2107    /// Creates a new [StreamContext] for [UnorderedScan].
2108    pub(crate) fn unordered_scan_ctx(
2109        input: ScanInput,
2110        ranges: Vec<RangeMeta>,
2111        query_start: Instant,
2112    ) -> Self {
2113        READ_SST_COUNT.observe(input.num_files() as f64);
2114        Self {
2115            input,
2116            ranges,
2117            query_start,
2118        }
2119    }
2120
2121    /// Returns true if the index refers to a memtable.
2122    pub(crate) fn is_mem_range_index(&self, index: RowGroupIndex) -> bool {
2123        self.input.num_memtables() > index.index
2124    }
2125
2126    pub(crate) fn is_file_range_index(&self, index: RowGroupIndex) -> bool {
2127        !self.is_mem_range_index(index)
2128            && index.index < self.input.num_files() + self.input.num_memtables()
2129    }
2130
2131    pub(crate) fn range_pre_filter_mode(&self, part_range: &PartitionRange) -> PreFilterMode {
2132        let range_meta = &self.ranges[part_range.identifier];
2133        let source_count = range_meta.indices.len();
2134
2135        self.input.range_pre_filter_mode(source_count)
2136    }
2137
2138    /// Retrieves the partition ranges.
2139    pub(crate) fn partition_ranges(&self) -> Vec<PartitionRange> {
2140        self.ranges
2141            .iter()
2142            .enumerate()
2143            .map(|(idx, range_meta)| range_meta.new_partition_range(idx))
2144            .collect()
2145    }
2146
2147    /// Format the context for explain.
2148    pub(crate) fn format_for_explain(&self, verbose: bool, f: &mut fmt::Formatter) -> fmt::Result {
2149        let (mut num_mem_ranges, mut num_file_ranges, mut num_other_ranges) = (0, 0, 0);
2150        for range_meta in &self.ranges {
2151            for idx in &range_meta.row_group_indices {
2152                if self.is_mem_range_index(*idx) {
2153                    num_mem_ranges += 1;
2154                } else if self.is_file_range_index(*idx) {
2155                    num_file_ranges += 1;
2156                } else {
2157                    num_other_ranges += 1;
2158                }
2159            }
2160        }
2161        if verbose {
2162            write!(f, "{{")?;
2163        }
2164        write!(
2165            f,
2166            r#""partition_count":{{"count":{}, "mem_ranges":{}, "files":{}, "file_ranges":{}"#,
2167            self.ranges.len(),
2168            num_mem_ranges,
2169            self.input.num_files(),
2170            num_file_ranges,
2171        )?;
2172        if num_other_ranges > 0 {
2173            write!(f, r#", "other_ranges":{}"#, num_other_ranges)?;
2174        }
2175        write!(f, "}}")?;
2176
2177        if let Some(selector) = &self.input.series_row_selector {
2178            write!(f, ", \"selector\":\"{}\"", selector)?;
2179        }
2180        if let Some(distribution) = &self.input.distribution {
2181            write!(f, ", \"distribution\":\"{}\"", distribution)?;
2182        }
2183
2184        self.format_scan_details(f)?;
2185        if verbose {
2186            self.format_verbose_content(f)?;
2187        }
2188
2189        Ok(())
2190    }
2191
2192    fn format_scan_details(&self, f: &mut fmt::Formatter) -> fmt::Result {
2193        let output_schema = self.input.mapper.output_schema();
2194        if !output_schema.is_empty() {
2195            write!(f, ", \"projection\": ")?;
2196            f.debug_list()
2197                .entries(
2198                    output_schema
2199                        .column_schemas()
2200                        .iter()
2201                        .map(|col| ExplainString(&col.name)),
2202                )
2203                .finish()?;
2204        }
2205        if let Some(predicate) = self.input.predicate.predicate() {
2206            if !predicate.exprs().is_empty() {
2207                let exprs: Vec<_> = predicate.exprs().iter().map(|e| e.to_string()).collect();
2208                write!(f, ", \"filters\": ")?;
2209                f.debug_list()
2210                    .entries(exprs.iter().map(|e| ExplainString(e)))
2211                    .finish()?;
2212            }
2213            if !predicate.dyn_filters().is_empty() {
2214                let filters: Vec<_> = predicate
2215                    .dyn_filters()
2216                    .iter()
2217                    .map(|filter| filter.to_string())
2218                    .collect();
2219                write!(f, ", \"dyn_filters\": ")?;
2220                f.debug_list()
2221                    .entries(filters.iter().map(|filter| ExplainString(filter)))
2222                    .finish()?;
2223            }
2224        }
2225        if let Some(time_range) = &self.input.time_range {
2226            write!(
2227                f,
2228                ", \"time_range\": {:?}",
2229                ExplainString(&time_range.to_string())
2230            )?;
2231        }
2232        let summary = FileSummary::new(&self.input.files);
2233        write!(
2234            f,
2235            ", \"file_summary\": {}",
2236            serde_json::to_string(&summary).map_err(|_| fmt::Error)?
2237        )?;
2238
2239        // Source indices are unique within a range even when a file is split into
2240        // row groups. Memtables and extension sources must not count as files.
2241        let file_offset = self.input.num_memtables();
2242        let file_end = file_offset + self.input.num_files();
2243        let mut max_range = None;
2244        let mut max_files = 0;
2245        for range in &self.ranges {
2246            let files = range
2247                .indices
2248                .iter()
2249                .filter(|source| (file_offset..file_end).contains(&source.index))
2250                .count();
2251            if files > max_files {
2252                max_files = files;
2253                max_range = Some(range);
2254            }
2255        }
2256        if let Some(range) = max_range {
2257            let summary = FileSummary::new(range.indices.iter().filter_map(|source| {
2258                source
2259                    .index
2260                    .checked_sub(file_offset)
2261                    .and_then(|index| self.input.files.get(index))
2262            }));
2263            let (start, end) = range.time_range;
2264            // RangeMeta bounds are inclusive, just like the verbose file list.
2265            let summary = serde_json::json!({
2266                "time_range_start": format!("{}::{}", start.value(), start.unit()),
2267                "time_range_end": format!("{}::{}", end.value(), end.unit()),
2268                "files": summary.files,
2269                "rows": summary.rows,
2270                "row_groups": summary.row_groups,
2271                "size": summary.size,
2272                "index_size": summary.index_size,
2273                "estimated_rows": range.num_rows,
2274            });
2275            write!(f, ", \"max_part_range\": {summary}")?;
2276        }
2277        Ok(())
2278    }
2279
2280    fn format_verbose_content(&self, f: &mut fmt::Formatter) -> fmt::Result {
2281        struct FileWrapper<'a> {
2282            file: &'a FileHandle,
2283        }
2284
2285        impl fmt::Debug for FileWrapper<'_> {
2286            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2287                let (start, end) = self.file.time_range();
2288                write!(
2289                    f,
2290                    r#"{{"file_id":"{}","time_range_start":"{}::{}","time_range_end":"{}::{}","rows":{},"size":{},"index_size":{}}}"#,
2291                    self.file.file_id(),
2292                    start.value(),
2293                    start.unit(),
2294                    end.value(),
2295                    end.unit(),
2296                    self.file.num_rows(),
2297                    self.file.size(),
2298                    self.file.index_size()
2299                )
2300            }
2301        }
2302
2303        struct InputWrapper<'a> {
2304            input: &'a ScanInput,
2305        }
2306
2307        #[cfg(feature = "enterprise")]
2308        impl InputWrapper<'_> {
2309            fn format_extension_ranges(&self, f: &mut fmt::Formatter) -> fmt::Result {
2310                if self.input.extension_ranges.is_empty() {
2311                    return Ok(());
2312                }
2313
2314                let mut delimiter = "";
2315                write!(f, ", extension_ranges: [")?;
2316                for range in self.input.extension_ranges() {
2317                    write!(f, "{}{:?}", delimiter, range)?;
2318                    delimiter = ", ";
2319                }
2320                write!(f, "]")?;
2321                Ok(())
2322            }
2323        }
2324
2325        impl fmt::Debug for InputWrapper<'_> {
2326            fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2327                if !self.input.files.is_empty() {
2328                    write!(f, ", \"files\": ")?;
2329                    f.debug_list()
2330                        .entries(self.input.files.iter().map(|file| FileWrapper { file }))
2331                        .finish()?;
2332                }
2333                write!(f, ", \"flat_format\": {}", self.input.explain_flat_format)?;
2334                #[cfg(feature = "enterprise")]
2335                self.format_extension_ranges(f)?;
2336
2337                Ok(())
2338            }
2339        }
2340
2341        write!(f, "{:?}", InputWrapper { input: &self.input })
2342    }
2343
2344    /// Add new dynamic filters to the predicates.
2345    /// Safe after stream creation; in-flight reads may still observe an older snapshot.
2346    pub(crate) fn add_dyn_filter_to_predicate(
2347        self: &Arc<Self>,
2348        filter_exprs: Vec<Arc<dyn datafusion::physical_plan::PhysicalExpr>>,
2349    ) -> Vec<bool> {
2350        let mut supported = Vec::with_capacity(filter_exprs.len());
2351        let filter_expr = filter_exprs
2352            .into_iter()
2353            .filter_map(|expr| {
2354                if let Ok(dyn_filter) = (expr as Arc<dyn std::any::Any + Send + Sync + 'static>)
2355                .downcast::<datafusion::physical_plan::expressions::DynamicFilterPhysicalExpr>()
2356            {
2357                supported.push(true);
2358                Some(dyn_filter)
2359            } else {
2360                supported.push(false);
2361                None
2362            }
2363            })
2364            .collect();
2365        self.input.predicate.add_dyn_filters(filter_expr);
2366        supported
2367    }
2368}
2369
2370/// Predicates to evaluate.
2371/// It only keeps filters that [SimpleFilterEvaluator] supports.
2372#[derive(Clone, Default)]
2373pub struct PredicateGroup {
2374    time_filters: Option<Arc<Vec<SimpleFilterEvaluator>>>,
2375    /// Predicate that includes request filters and region partition expr (if any).
2376    predicate_all: Predicate,
2377    /// Predicate that only includes request filters.
2378    predicate_without_region: Predicate,
2379    /// Region partition expression restored from metadata.
2380    region_partition_expr: Option<PartitionExpr>,
2381}
2382
2383impl PredicateGroup {
2384    /// Creates a new `PredicateGroup` from exprs according to the metadata.
2385    pub fn new(metadata: &RegionMetadata, exprs: &[Expr]) -> Result<Self> {
2386        let mut combined_exprs = exprs.to_vec();
2387        let mut region_partition_expr = None;
2388
2389        if let Some(expr_json) = metadata.partition_expr.as_ref()
2390            && !expr_json.is_empty()
2391            && let Some(expr) = PartitionExpr::from_json_str(expr_json)
2392                .context(InvalidPartitionExprSnafu { expr: expr_json })?
2393        {
2394            let logical_expr = expr
2395                .try_as_logical_expr()
2396                .context(InvalidPartitionExprSnafu {
2397                    expr: expr_json.clone(),
2398                })?;
2399
2400            // Persisted partition bounds retain their original types after schema changes.
2401            let schema = metadata
2402                .schema
2403                .arrow_schema()
2404                .clone()
2405                .to_dfschema_ref()
2406                .context(EvalPartitionFilterSnafu)?;
2407            let simplifier = ExprSimplifier::new(
2408                SimplifyContext::builder()
2409                    .with_schema(schema.clone())
2410                    .build(),
2411            );
2412            let logical_expr = simplifier
2413                .coerce(logical_expr, &schema)
2414                .and_then(|expr| simplifier.simplify(expr))
2415                .context(EvalPartitionFilterSnafu)?;
2416            combined_exprs.push(logical_expr);
2417            region_partition_expr = Some(expr);
2418        }
2419
2420        let mut time_filters = Vec::with_capacity(combined_exprs.len());
2421        // Columns in the expr.
2422        let mut columns = HashSet::new();
2423        for expr in &combined_exprs {
2424            columns.clear();
2425            let Some(filter) = Self::expr_to_filter(expr, metadata, &mut columns) else {
2426                continue;
2427            };
2428            time_filters.push(filter);
2429        }
2430        let time_filters = if time_filters.is_empty() {
2431            None
2432        } else {
2433            Some(Arc::new(time_filters))
2434        };
2435
2436        let predicate_all = Predicate::new(combined_exprs);
2437        let predicate_without_region = Predicate::new(exprs.to_vec());
2438
2439        Ok(Self {
2440            time_filters,
2441            predicate_all,
2442            predicate_without_region,
2443            region_partition_expr,
2444        })
2445    }
2446
2447    /// Returns time filters.
2448    pub(crate) fn time_filters(&self) -> Option<Arc<Vec<SimpleFilterEvaluator>>> {
2449        self.time_filters.clone()
2450    }
2451
2452    /// Returns predicate of all exprs (including region partition expr if present).
2453    pub(crate) fn predicate(&self) -> Option<&Predicate> {
2454        if self.predicate_all.is_empty() {
2455            None
2456        } else {
2457            Some(&self.predicate_all)
2458        }
2459    }
2460
2461    /// Returns predicate that excludes region partition expr.
2462    pub(crate) fn predicate_without_region(&self) -> Option<&Predicate> {
2463        if self.predicate_without_region.is_empty() {
2464            None
2465        } else {
2466            Some(&self.predicate_without_region)
2467        }
2468    }
2469
2470    /// Add dynamic filters in the predicates.
2471    pub(crate) fn add_dyn_filters(&self, dyn_filters: Vec<Arc<DynamicFilterPhysicalExpr>>) {
2472        self.predicate_all.add_dyn_filters(dyn_filters.clone());
2473        self.predicate_without_region.add_dyn_filters(dyn_filters);
2474    }
2475
2476    /// Removes dynamic filters while preserving the static and region predicates.
2477    pub(crate) fn clear_dyn_filters(&self) {
2478        self.predicate_all.clear_dyn_filters();
2479        self.predicate_without_region.clear_dyn_filters();
2480    }
2481
2482    /// Returns the region partition expr from metadata, if any.
2483    pub(crate) fn region_partition_expr(&self) -> Option<&PartitionExpr> {
2484        self.region_partition_expr.as_ref()
2485    }
2486
2487    fn expr_to_filter(
2488        expr: &Expr,
2489        metadata: &RegionMetadata,
2490        columns: &mut HashSet<Column>,
2491    ) -> Option<SimpleFilterEvaluator> {
2492        columns.clear();
2493        // `expr_to_columns` won't return error.
2494        // We still ignore these expressions for safety.
2495        expr_to_columns(expr, columns).ok()?;
2496        if columns.len() > 1 {
2497            // Simple filter doesn't support multiple columns.
2498            return None;
2499        }
2500        let column = columns.iter().next()?;
2501        let column_meta = metadata.column_by_name(&column.name)?;
2502        if column_meta.semantic_type == SemanticType::Timestamp {
2503            SimpleFilterEvaluator::try_new(expr)
2504        } else {
2505            None
2506        }
2507    }
2508}
2509
2510#[cfg(test)]
2511mod tests {
2512    use std::collections::BTreeMap;
2513    use std::sync::Arc;
2514
2515    use common_time::timestamp::{TimeUnit, Timestamp};
2516    use datafusion::physical_plan::expressions::{
2517        binary as physical_binary, col as physical_col, lit as physical_lit,
2518    };
2519    use datafusion_common::ScalarValue;
2520    use datafusion_expr::{Operator, col, lit};
2521    use datatypes::arrow::datatypes::{
2522        DataType as ArrowDataType, Field, Schema as ArrowSchema, TimeUnit as ArrowTimeUnit,
2523    };
2524    use datatypes::prelude::ConcreteDataType;
2525    use datatypes::schema::ColumnSchema;
2526    use datatypes::types::json_type::JsonObjectType;
2527    use datatypes::value::Value;
2528    use partition::expr::col as partition_col;
2529    use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
2530    use store_api::storage::{RegionId, TimeSeriesDistribution, TimeSeriesRowSelector};
2531
2532    use super::*;
2533    use crate::cache::CacheManager;
2534    use crate::read::range_cache::ScanRequestFingerprintBuilder;
2535    use crate::sst::file::FileMeta;
2536    use crate::test_util::memtable_util::metadata_with_primary_key;
2537    use crate::test_util::scheduler_util::SchedulerEnv;
2538
2539    async fn new_scan_input(metadata: RegionMetadataRef, filters: Vec<Expr>) -> ScanInputBuilder {
2540        let env = SchedulerEnv::new().await;
2541        let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
2542        let predicate = PredicateGroup::new(metadata.as_ref(), &filters).unwrap();
2543        let file = FileHandle::new(
2544            crate::sst::file::FileMeta::default(),
2545            Arc::new(crate::sst::file_purger::NoopFilePurger),
2546        );
2547
2548        ScanInput::builder(env.access_layer.clone(), mapper)
2549            .with_predicate(predicate)
2550            .with_cache(CacheStrategy::EnableAll(Arc::new(
2551                CacheManager::builder()
2552                    .range_result_cache_size(1024)
2553                    .build(),
2554            )))
2555            .with_files(vec![file])
2556    }
2557
2558    #[test]
2559    fn test_partition_time_filter_after_unit_widening() {
2560        use datatypes::arrow::array::TimestampMicrosecondArray;
2561
2562        let bound = Value::Timestamp(Timestamp::new_millisecond(1));
2563        for (expr, expected) in [
2564            (
2565                partition_col("ts").lt(bound.clone()),
2566                vec![true, false, false],
2567            ),
2568            (partition_col("ts").gt_eq(bound), vec![false, true, true]),
2569        ] {
2570            let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 1));
2571            builder.push_column_metadata(ColumnMetadata {
2572                column_schema: ColumnSchema::new(
2573                    "ts",
2574                    ConcreteDataType::timestamp_microsecond_datatype(),
2575                    false,
2576                ),
2577                semantic_type: SemanticType::Timestamp,
2578                column_id: 0,
2579            });
2580            builder.primary_key(vec![]);
2581            builder.partition_expr_json(Some(expr.as_json_str().unwrap()));
2582            let metadata = builder.build().unwrap();
2583            let group = PredicateGroup::new(&metadata, &[]).unwrap();
2584            let filters = group.time_filters().unwrap();
2585            let input: ArrayRef = Arc::new(TimestampMicrosecondArray::from(vec![999, 1000, 1001]));
2586            assert_eq!(filters.len(), 1);
2587            assert_eq!(
2588                filters[0].evaluate_array(&input).unwrap(),
2589                datafusion_common::arrow::buffer::BooleanBuffer::from(expected),
2590            );
2591        }
2592    }
2593
2594    #[test]
2595    fn test_partition_time_filter_overflow_after_unit_widening() {
2596        // Year 3000 fits milliseconds but overflows nanoseconds.
2597        let bound = Value::Timestamp(Timestamp::new_millisecond(32_503_680_000_000));
2598        let expr = partition_col("ts").lt(bound);
2599        let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 1));
2600        builder.push_column_metadata(ColumnMetadata {
2601            column_schema: ColumnSchema::new(
2602                "ts",
2603                ConcreteDataType::timestamp_nanosecond_datatype(),
2604                false,
2605            ),
2606            semantic_type: SemanticType::Timestamp,
2607            column_id: 0,
2608        });
2609        builder.primary_key(vec![]);
2610        builder.partition_expr_json(Some(expr.as_json_str().unwrap()));
2611        let metadata = builder.build().unwrap();
2612
2613        let error = PredicateGroup::new(&metadata, &[])
2614            .err()
2615            .expect("an overflowing partition bound must fail scan construction");
2616        let crate::error::Error::EvalPartitionFilter { error, .. } = error else {
2617            panic!("unexpected error: {error:?}");
2618        };
2619        assert!(
2620            error
2621                .to_string()
2622                .contains("converted value exceeds the representable i64 range"),
2623            "{error}"
2624        );
2625    }
2626
2627    #[tokio::test]
2628    async fn test_total_rows_is_exact_with_legacy_file() {
2629        let file_meta = FileMeta {
2630            num_rows: 10,
2631            ..Default::default()
2632        };
2633        let mut json = serde_json::to_value(&file_meta).unwrap();
2634        json.as_object_mut().unwrap().remove("num_rows");
2635        let legacy_meta = serde_json::from_value::<FileMeta>(json).unwrap();
2636        assert_eq!(legacy_meta.num_rows, 0);
2637
2638        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
2639        for (file_metas, expected_rows, exact) in [
2640            (vec![file_meta.clone()], 10, true),
2641            (vec![legacy_meta.clone()], 0, false),
2642            (vec![file_meta, legacy_meta], 10, false),
2643        ] {
2644            let files = file_metas
2645                .into_iter()
2646                .map(|meta| {
2647                    FileHandle::new(meta, Arc::new(crate::sst::file_purger::NoopFilePurger))
2648                })
2649                .collect();
2650            let input = new_scan_input(metadata.clone(), vec![])
2651                .await
2652                .with_append_mode(true)
2653                .with_files(files)
2654                .build();
2655            assert_eq!(input.total_rows(), expected_rows);
2656            assert_eq!(input.total_rows_is_exact(), exact);
2657        }
2658    }
2659
2660    #[tokio::test]
2661    async fn test_total_rows_is_exact_after_partition_filter() {
2662        let expr = partition_col("k0").gt_eq(Value::String("foo".into()));
2663        let other = partition_col("k0").gt_eq(Value::String("bar".into()));
2664        for (region_expr, file_exprs, exact) in [
2665            (None, vec![None], true),
2666            (None, vec![Some(expr.clone())], true),
2667            (Some(expr.clone()), vec![], true),
2668            (Some(expr.clone()), vec![Some(expr.clone())], true),
2669            (Some(expr.clone()), vec![None], false),
2670            (Some(expr.clone()), vec![Some(other)], false),
2671            (Some(expr.clone()), vec![Some(expr), None], false),
2672        ] {
2673            let mut builder =
2674                RegionMetadataBuilder::from_existing(metadata_with_primary_key(vec![0, 1], false));
2675            builder.partition_expr_json(region_expr.map(|expr| expr.as_json_str().unwrap()));
2676            let metadata = Arc::new(builder.build_without_validation().unwrap());
2677            let files = file_exprs
2678                .into_iter()
2679                .map(|partition_expr| {
2680                    FileHandle::new(
2681                        FileMeta {
2682                            num_rows: 1,
2683                            partition_expr: partition_expr.map(Into::into),
2684                            ..Default::default()
2685                        },
2686                        Arc::new(crate::sst::file_purger::NoopFilePurger),
2687                    )
2688                })
2689                .collect();
2690            let input = new_scan_input(metadata, vec![])
2691                .await
2692                .with_files(files)
2693                .build();
2694            assert_eq!(input.total_rows_is_exact(), exact);
2695        }
2696    }
2697
2698    /// Helper to create a timestamp millisecond literal.
2699    fn ts_lit(val: i64) -> datafusion_expr::Expr {
2700        lit(ScalarValue::TimestampMillisecond(Some(val), None))
2701    }
2702
2703    fn metadata_with_time_index_unit(unit: TimeUnit) -> RegionMetadataRef {
2704        let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456));
2705        builder
2706            .push_column_metadata(ColumnMetadata {
2707                column_schema: ColumnSchema::new(
2708                    "k0".to_string(),
2709                    ConcreteDataType::string_datatype(),
2710                    false,
2711                ),
2712                semantic_type: SemanticType::Tag,
2713                column_id: 0,
2714            })
2715            .push_column_metadata(ColumnMetadata {
2716                column_schema: ColumnSchema::new(
2717                    "k1".to_string(),
2718                    ConcreteDataType::uint32_datatype(),
2719                    false,
2720                ),
2721                semantic_type: SemanticType::Tag,
2722                column_id: 1,
2723            })
2724            .push_column_metadata(ColumnMetadata {
2725                column_schema: ColumnSchema::new(
2726                    "ts".to_string(),
2727                    ConcreteDataType::timestamp_datatype(unit),
2728                    false,
2729                ),
2730                semantic_type: SemanticType::Timestamp,
2731                column_id: 2,
2732            })
2733            .push_column_metadata(ColumnMetadata {
2734                column_schema: ColumnSchema::new(
2735                    "v0".to_string(),
2736                    ConcreteDataType::int64_datatype(),
2737                    true,
2738                ),
2739                semantic_type: SemanticType::Field,
2740                column_id: 3,
2741            })
2742            .primary_key(vec![0, 1]);
2743
2744        Arc::new(builder.build().unwrap())
2745    }
2746
2747    fn file_handle_with_time_range(start: Timestamp, end: Timestamp) -> FileHandle {
2748        FileHandle::new(
2749            FileMeta {
2750                time_range: (start, end),
2751                ..Default::default()
2752            },
2753            Arc::new(crate::sst::file_purger::NoopFilePurger),
2754        )
2755    }
2756
2757    #[test]
2758    fn test_time_range_covers_file() {
2759        let file = file_handle_with_time_range(
2760            Timestamp::new_millisecond(1000),
2761            Timestamp::new_millisecond(2000),
2762        );
2763
2764        assert!(!time_range_covers_file(None, &file));
2765        assert!(time_range_covers_file(
2766            Some(&TimestampRange::min_to_max()),
2767            &file
2768        ));
2769        assert!(time_range_covers_file(
2770            Some(&TimestampRange::new_inclusive(
2771                Some(Timestamp::new_millisecond(1000)),
2772                Some(Timestamp::new_millisecond(2000)),
2773            )),
2774            &file
2775        ));
2776        assert!(time_range_covers_file(
2777            TimestampRange::with_unit(500, 3000, TimeUnit::Millisecond).as_ref(),
2778            &file
2779        ));
2780        assert!(!time_range_covers_file(
2781            TimestampRange::with_unit(1000, 2000, TimeUnit::Millisecond).as_ref(),
2782            &file
2783        ));
2784        assert!(!time_range_covers_file(
2785            TimestampRange::with_unit(1001, 3000, TimeUnit::Millisecond).as_ref(),
2786            &file
2787        ));
2788        assert!(!time_range_covers_file(
2789            Some(&TimestampRange::empty()),
2790            &file
2791        ));
2792
2793        let seconds_file = file_handle_with_time_range(
2794            Timestamp::new(1, TimeUnit::Second),
2795            Timestamp::new(2, TimeUnit::Second),
2796        );
2797        assert!(time_range_covers_file(
2798            TimestampRange::with_unit(1000, 2001, TimeUnit::Millisecond).as_ref(),
2799            &seconds_file
2800        ));
2801    }
2802
2803    #[tokio::test]
2804    async fn test_scan_input_builder_computes_scan_analysis() {
2805        let metadata = metadata_with_time_index_unit(TimeUnit::Millisecond);
2806        let env = SchedulerEnv::new().await;
2807        let file = file_handle_with_time_range(
2808            Timestamp::new_millisecond(1000),
2809            Timestamp::new_millisecond(2000),
2810        );
2811
2812        let make_input = |filters: &[Expr]| {
2813            let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
2814            ScanInput::builder(env.access_layer.clone(), mapper)
2815                .with_predicate(PredicateGroup::new(&metadata, filters).unwrap())
2816                .build()
2817        };
2818
2819        let covered = make_input(&[col("ts").gt_eq(ts_lit(500)), col("ts").lt(ts_lit(3000))]);
2820        assert!(time_range_covers_file(covered.implied_time_range(), &file));
2821
2822        // The convex hull of this disjunction covers the file, but most rows do
2823        // not satisfy it. The strict implied-range analysis must reject it.
2824        let disjoint = make_input(&[col("ts").eq(ts_lit(1000)).or(col("ts").eq(ts_lit(2000)))]);
2825        assert!(disjoint.implied_time_range().is_none());
2826        assert!(!time_range_covers_file(
2827            disjoint.implied_time_range(),
2828            &file
2829        ));
2830
2831        let replaced = ScanInput::builder(
2832            env.access_layer.clone(),
2833            FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap(),
2834        )
2835        .with_predicate(PredicateGroup::new(&metadata, &[col("ts").gt(ts_lit(3000))]).unwrap())
2836        .build();
2837        assert!(!time_range_covers_file(
2838            replaced.implied_time_range(),
2839            &file
2840        ));
2841    }
2842
2843    #[tokio::test]
2844    async fn test_scan_input_uses_explicit_batch_size() {
2845        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
2846        let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
2847        let env = SchedulerEnv::new().await;
2848        let input = ScanInput::builder(env.access_layer.clone(), mapper).build();
2849        assert_eq!(
2850            crate::sst::parquet::DEFAULT_READ_BATCH_SIZE,
2851            input.batch_size()
2852        );
2853
2854        let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
2855        let input = ScanInput::builder(env.access_layer.clone(), mapper)
2856            .with_compaction(true)
2857            .with_batch_size(256)
2858            .build();
2859        assert_eq!(256, input.batch_size());
2860
2861        let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
2862        let input = ScanInput::builder(env.access_layer.clone(), mapper)
2863            .with_batch_size(256)
2864            .with_compaction(true)
2865            .with_compaction(false)
2866            .build();
2867        assert_eq!(256, input.batch_size());
2868    }
2869
2870    #[tokio::test]
2871    async fn test_build_scan_fingerprint_for_eligible_scan() {
2872        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
2873        let input = new_scan_input(
2874            metadata.clone(),
2875            vec![
2876                col("ts").gt_eq(ts_lit(1000)),
2877                col("k0").eq(lit("foo")),
2878                col("v0").gt(lit(1)),
2879            ],
2880        )
2881        .await
2882        .with_distribution(Some(TimeSeriesDistribution::PerSeries))
2883        .with_series_row_selector(Some(TimeSeriesRowSelector::LastRow { after_merge: true }))
2884        .with_merge_mode(MergeMode::LastNonNull)
2885        .with_filter_deleted(false)
2886        .build();
2887
2888        let fingerprint = input.scan_fingerprint().unwrap();
2889
2890        let expected = ScanRequestFingerprintBuilder {
2891            read_columns: input.read_cols.clone(),
2892            read_column_types: vec![
2893                metadata
2894                    .column_by_id(0)
2895                    .map(|col| col.column_schema.data_type.clone()),
2896                metadata
2897                    .column_by_id(2)
2898                    .map(|col| col.column_schema.data_type.clone()),
2899                metadata
2900                    .column_by_id(3)
2901                    .map(|col| col.column_schema.data_type.clone()),
2902            ],
2903            filters: vec![
2904                col("k0").eq(lit("foo")).to_string(),
2905                col("v0").gt(lit(1)).to_string(),
2906            ],
2907            time_filters: vec![col("ts").gt_eq(ts_lit(1000)).to_string()],
2908            series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: true }),
2909            append_mode: false,
2910            filter_deleted: false,
2911            merge_mode: MergeMode::LastNonNull,
2912            sequence_range: None,
2913            partition_expr_version: 0,
2914        }
2915        .build();
2916        assert_eq!(&expected, fingerprint);
2917        assert_eq!(
2918            input.series_row_selector,
2919            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
2920        );
2921    }
2922
2923    #[tokio::test]
2924    async fn test_build_scan_fingerprint_requires_tag_filter() {
2925        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
2926        let input = new_scan_input(
2927            metadata,
2928            vec![col("ts").gt_eq(lit(1000)), col("v0").gt(lit(1))],
2929        )
2930        .await
2931        .build();
2932
2933        assert!(input.scan_fingerprint().is_none());
2934    }
2935
2936    #[tokio::test]
2937    async fn test_build_scan_fingerprint_respects_scan_eligibility() {
2938        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
2939        let filters = vec![col("k0").eq(lit("foo"))];
2940
2941        let disabled = ScanInput::builder(
2942            SchedulerEnv::new().await.access_layer.clone(),
2943            FlatProjectionMapper::new(&metadata, [0, 2, 3].into_iter()).unwrap(),
2944        )
2945        .with_predicate(PredicateGroup::new(metadata.as_ref(), &filters).unwrap())
2946        .build();
2947        assert!(disabled.scan_fingerprint().is_none());
2948
2949        let compaction = new_scan_input(metadata.clone(), filters.clone())
2950            .await
2951            .with_compaction(true)
2952            .build();
2953        assert!(compaction.scan_fingerprint().is_none());
2954
2955        // No files to read.
2956        let no_files = new_scan_input(metadata, filters)
2957            .await
2958            .with_files(vec![])
2959            .build();
2960        assert!(no_files.scan_fingerprint().is_none());
2961    }
2962
2963    #[tokio::test]
2964    async fn test_build_scan_fingerprint_tracks_schema_and_partition_expr_changes() {
2965        let base = metadata_with_primary_key(vec![0, 1], false);
2966        let mut builder = RegionMetadataBuilder::from_existing(base);
2967        let partition_expr = partition_col("k0")
2968            .gt_eq(Value::String("foo".into()))
2969            .as_json_str()
2970            .unwrap();
2971        builder.partition_expr_json(Some(partition_expr));
2972        let metadata = Arc::new(builder.build_without_validation().unwrap());
2973
2974        let input = new_scan_input(metadata.clone(), vec![col("k0").eq(lit("foo"))])
2975            .await
2976            .build();
2977        let fingerprint = input.scan_fingerprint().unwrap();
2978
2979        let expected = ScanRequestFingerprintBuilder {
2980            read_columns: input.read_cols.clone(),
2981            read_column_types: vec![
2982                metadata
2983                    .column_by_id(0)
2984                    .map(|col| col.column_schema.data_type.clone()),
2985                metadata
2986                    .column_by_id(2)
2987                    .map(|col| col.column_schema.data_type.clone()),
2988                metadata
2989                    .column_by_id(3)
2990                    .map(|col| col.column_schema.data_type.clone()),
2991            ],
2992            filters: vec![col("k0").eq(lit("foo")).to_string()],
2993            time_filters: vec![],
2994            series_row_selector: None,
2995            append_mode: false,
2996            filter_deleted: true,
2997            merge_mode: MergeMode::LastRow,
2998            sequence_range: None,
2999            partition_expr_version: metadata.partition_expr_version,
3000        }
3001        .build();
3002        assert_eq!(&expected, fingerprint);
3003        assert_ne!(0, metadata.partition_expr_version);
3004    }
3005
3006    #[tokio::test]
3007    async fn test_build_scan_fingerprint_uses_json_target_types() {
3008        let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456));
3009        builder
3010            .push_column_metadata(ColumnMetadata {
3011                column_schema: ColumnSchema::new(
3012                    "k0".to_string(),
3013                    ConcreteDataType::string_datatype(),
3014                    false,
3015                ),
3016                semantic_type: SemanticType::Tag,
3017                column_id: 0,
3018            })
3019            .push_column_metadata(ColumnMetadata {
3020                column_schema: ColumnSchema::new(
3021                    "ts".to_string(),
3022                    ConcreteDataType::timestamp_millisecond_datatype(),
3023                    false,
3024                ),
3025                semantic_type: SemanticType::Timestamp,
3026                column_id: 1,
3027            })
3028            .push_column_metadata(ColumnMetadata {
3029                column_schema: ColumnSchema::new(
3030                    "j".to_string(),
3031                    ConcreteDataType::json2(JsonNativeType::Variant),
3032                    true,
3033                ),
3034                semantic_type: SemanticType::Field,
3035                column_id: 2,
3036            })
3037            .primary_key(vec![0]);
3038        let metadata = Arc::new(builder.build().unwrap());
3039
3040        let make_input = |target_type| async {
3041            let env = SchedulerEnv::new().await;
3042            let read_cols = ReadColumns::new([0, 1, 2])
3043                .with_json_target_types(BTreeMap::from([(2, target_type)]));
3044            let mapper =
3045                FlatProjectionMapper::new_with_read_columns(&metadata, vec![0, 1, 2], read_cols)
3046                    .unwrap();
3047            let predicate =
3048                PredicateGroup::new(metadata.as_ref(), &[col("k0").eq(lit("foo"))]).unwrap();
3049            let file = FileHandle::new(
3050                FileMeta::default(),
3051                Arc::new(crate::sst::file_purger::NoopFilePurger),
3052            );
3053            ScanInput::builder(env.access_layer.clone(), mapper)
3054                .with_predicate(predicate)
3055                .with_cache(CacheStrategy::EnableAll(Arc::new(
3056                    CacheManager::builder()
3057                        .range_result_cache_size(1024)
3058                        .build(),
3059                )))
3060                .with_files(vec![file])
3061                .build()
3062        };
3063
3064        let int_target = JsonNativeType::i64();
3065        let string_target = JsonNativeType::String;
3066        let int_input = make_input(int_target.clone()).await;
3067        let string_input = make_input(string_target).await;
3068        let int_fingerprint = int_input.scan_fingerprint().unwrap();
3069        let string_fingerprint = string_input.scan_fingerprint().unwrap();
3070
3071        assert_ne!(int_fingerprint, string_fingerprint);
3072        assert_eq!(
3073            Some(&Some(ConcreteDataType::json2(int_target))),
3074            int_fingerprint.read_column_types().get(2)
3075        );
3076    }
3077
3078    #[tokio::test]
3079    async fn test_scan_input_rejects_json_type_hint_for_non_json2_column() {
3080        let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456));
3081        builder
3082            .push_column_metadata(ColumnMetadata {
3083                column_schema: ColumnSchema::new(
3084                    "k0".to_string(),
3085                    ConcreteDataType::string_datatype(),
3086                    false,
3087                ),
3088                semantic_type: SemanticType::Tag,
3089                column_id: 0,
3090            })
3091            .push_column_metadata(ColumnMetadata {
3092                column_schema: ColumnSchema::new(
3093                    "ts".to_string(),
3094                    ConcreteDataType::timestamp_millisecond_datatype(),
3095                    false,
3096                ),
3097                semantic_type: SemanticType::Timestamp,
3098                column_id: 1,
3099            })
3100            .push_column_metadata(ColumnMetadata {
3101                column_schema: ColumnSchema::new(
3102                    "j".to_string(),
3103                    ConcreteDataType::json2(JsonNativeType::Object(JsonObjectType::from([(
3104                        "a".to_string(),
3105                        JsonNativeType::i64(),
3106                    )]))),
3107                    true,
3108                ),
3109                semantic_type: SemanticType::Field,
3110                column_id: 2,
3111            })
3112            .push_column_metadata(ColumnMetadata {
3113                column_schema: ColumnSchema::new(
3114                    "v0".to_string(),
3115                    ConcreteDataType::int64_datatype(),
3116                    true,
3117                ),
3118                semantic_type: SemanticType::Field,
3119                column_id: 3,
3120            })
3121            .primary_key(vec![0]);
3122        let metadata = Arc::new(builder.build().unwrap());
3123        let mutable = crate::memtable::time_partition::TimePartitions::new(
3124            metadata.clone(),
3125            Arc::new(crate::test_util::memtable_util::EmptyMemtableBuilder::default()),
3126            0,
3127            None,
3128        );
3129        let version = Arc::new(
3130            crate::region::version::VersionBuilder::new(metadata.clone(), mutable).build(),
3131        );
3132        let env = SchedulerEnv::new().await;
3133        let request = ScanRequest {
3134            projection: Some(vec![0, 1, 2, 3]),
3135            json_type_hint: std::collections::HashMap::from([(
3136                "v0".to_string(),
3137                JsonNativeType::i64(),
3138            )]),
3139            ..Default::default()
3140        };
3141
3142        let err = ScanRegion::new(
3143            version,
3144            env.access_layer.clone(),
3145            request,
3146            CacheStrategy::Disabled,
3147        )
3148        .scan_input()
3149        .await;
3150        let Err(err) = err else {
3151            panic!("scan input should reject JSON type hint for non-JSON2 column");
3152        };
3153
3154        assert!(err.to_string().contains("non-JSON2 column v0"));
3155    }
3156
3157    #[test]
3158    fn test_update_dyn_filters_with_empty_base_predicates() {
3159        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
3160        let predicate_group = PredicateGroup::new(metadata.as_ref(), &[]).unwrap();
3161        assert!(predicate_group.predicate().is_none());
3162        assert!(predicate_group.predicate_without_region().is_none());
3163
3164        let dyn_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], physical_lit(false)));
3165        predicate_group.add_dyn_filters(vec![dyn_filter]);
3166
3167        let predicate_all = predicate_group.predicate().unwrap();
3168        assert!(predicate_all.exprs().is_empty());
3169        assert_eq!(1, predicate_all.dyn_filters().len());
3170
3171        let predicate_without_region = predicate_group.predicate_without_region().unwrap();
3172        assert!(predicate_without_region.exprs().is_empty());
3173        assert_eq!(1, predicate_without_region.dyn_filters().len());
3174    }
3175
3176    #[test]
3177    fn test_clear_dyn_filters_preserves_predicate_group_static_filters() {
3178        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
3179        let static_filters = vec![col("k0").eq(lit("foo"))];
3180        let predicate_group = PredicateGroup::new(metadata.as_ref(), &static_filters).unwrap();
3181        let dynamic_filter = Arc::new(DynamicFilterPhysicalExpr::new(vec![], physical_lit(true)));
3182        predicate_group.add_dyn_filters(vec![dynamic_filter.clone()]);
3183
3184        predicate_group.clear_dyn_filters();
3185        // Updating the old producer cannot add its wrapper to a new execution.
3186        dynamic_filter.update(physical_lit(false)).unwrap();
3187
3188        for predicate in [
3189            predicate_group.predicate().unwrap(),
3190            predicate_group.predicate_without_region().unwrap(),
3191        ] {
3192            assert_eq!(predicate.exprs(), static_filters);
3193            assert!(predicate.dyn_filters().is_empty());
3194        }
3195    }
3196
3197    #[test]
3198    fn test_file_level_pruning_stats_prunes_old_file() {
3199        let ts_col_name = "ts";
3200        let predicate = Predicate::new(vec![col(ts_col_name).gt(ts_lit(1000))]);
3201        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
3202            ts_col_name,
3203            ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None),
3204            false,
3205        )]));
3206
3207        // File with time range [0ms, 500ms] is completely before `ts > 1000ms`.
3208        let stats = FileLevelPruningStats {
3209            min_scalar: ScalarValue::TimestampMillisecond(Some(0), None),
3210            max_scalar: ScalarValue::TimestampMillisecond(Some(500), None),
3211            time_index_col_name: ts_col_name.to_string(),
3212        };
3213        assert_eq!(
3214            vec![false],
3215            predicate.prune_with_stats(&stats, &arrow_schema)
3216        );
3217
3218        // File with time range [0ms, 2000ms] overlaps `ts > 1000ms`, so keep it.
3219        let stats = FileLevelPruningStats {
3220            min_scalar: ScalarValue::TimestampMillisecond(Some(0), None),
3221            max_scalar: ScalarValue::TimestampMillisecond(Some(2000), None),
3222            time_index_col_name: ts_col_name.to_string(),
3223        };
3224        assert_eq!(
3225            vec![true],
3226            predicate.prune_with_stats(&stats, &arrow_schema)
3227        );
3228    }
3229
3230    #[test]
3231    fn test_file_level_pruning_stats_no_predicate_keeps_all() {
3232        let predicate = Predicate::new(vec![]);
3233        assert!(predicate.is_empty());
3234
3235        let stats = FileLevelPruningStats {
3236            min_scalar: ScalarValue::TimestampMillisecond(Some(0), None),
3237            max_scalar: ScalarValue::TimestampMillisecond(Some(500), None),
3238            time_index_col_name: "ts".to_string(),
3239        };
3240        let arrow_schema = Arc::new(ArrowSchema::new(Vec::<Field>::new()));
3241        assert_eq!(
3242            vec![true],
3243            predicate.prune_with_stats(&stats, &arrow_schema)
3244        );
3245    }
3246
3247    #[tokio::test]
3248    async fn test_file_level_pruning_stats_ceil_max_unit_conversion() {
3249        let metadata = metadata_with_time_index_unit(TimeUnit::Millisecond);
3250        let input = new_scan_input(metadata, vec![]).await.build();
3251        let file = file_handle_with_time_range(
3252            Timestamp::new(1_000_001, TimeUnit::Nanosecond),
3253            Timestamp::new(1_000_001, TimeUnit::Nanosecond),
3254        );
3255
3256        let stats = input.try_file_level_pruning_stats(&file).unwrap();
3257        assert_eq!(
3258            ScalarValue::TimestampMillisecond(Some(1), None),
3259            stats.min_scalar
3260        );
3261        assert_eq!(
3262            ScalarValue::TimestampMillisecond(Some(2), None),
3263            stats.max_scalar
3264        );
3265
3266        // The actual max timestamp is slightly greater than 1ms. It must be kept for `ts > 1ms`.
3267        let predicate = Predicate::new(vec![col("ts").gt(ts_lit(1))]);
3268        assert_eq!(
3269            vec![true],
3270            predicate.prune_with_stats(&stats, input.mapper.metadata().schema.arrow_schema())
3271        );
3272    }
3273
3274    #[tokio::test]
3275    async fn test_file_level_pruning_stats_overflow_keeps_file() {
3276        let metadata = metadata_with_time_index_unit(TimeUnit::Nanosecond);
3277        let input = new_scan_input(metadata, vec![]).await.build();
3278        let file = file_handle_with_time_range(
3279            Timestamp::new(0, TimeUnit::Second),
3280            Timestamp::new(i64::MAX, TimeUnit::Second),
3281        );
3282
3283        assert!(input.try_file_level_pruning_stats(&file).is_none());
3284    }
3285
3286    #[test]
3287    fn test_file_level_pruning_stats_keeps_inclusive_boundary() {
3288        let ts_col_name = "ts";
3289        let predicate = Predicate::new(vec![col(ts_col_name).gt_eq(ts_lit(1000))]);
3290        let arrow_schema = Arc::new(ArrowSchema::new(vec![Field::new(
3291            ts_col_name,
3292            ArrowDataType::Timestamp(ArrowTimeUnit::Millisecond, None),
3293            false,
3294        )]));
3295        let stats = FileLevelPruningStats {
3296            min_scalar: ScalarValue::TimestampMillisecond(Some(0), None),
3297            max_scalar: ScalarValue::TimestampMillisecond(Some(1000), None),
3298            time_index_col_name: ts_col_name.to_string(),
3299        };
3300
3301        assert_eq!(
3302            vec![true],
3303            predicate.prune_with_stats(&stats, &arrow_schema)
3304        );
3305    }
3306
3307    #[tokio::test]
3308    async fn test_file_level_pruning_with_dyn_filter_only_predicate() {
3309        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
3310        let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
3311        let predicate_group = PredicateGroup::new(metadata.as_ref(), &[]).unwrap();
3312        predicate_group.add_dyn_filters(vec![Arc::new(DynamicFilterPhysicalExpr::new(
3313            vec![],
3314            physical_lit(false),
3315        ))]);
3316        let input = ScanInput::builder(SchedulerEnv::new().await.access_layer.clone(), mapper)
3317            .with_predicate(predicate_group)
3318            .build();
3319        let file = file_handle_with_time_range(
3320            Timestamp::new_millisecond(0),
3321            Timestamp::new_millisecond(1000),
3322        );
3323        let mut reader_metrics = ReaderMetrics::default();
3324
3325        let builder = input
3326            .prune_file(&file, PreFilterMode::SkipFields, &mut reader_metrics)
3327            .await
3328            .unwrap();
3329
3330        assert_eq!(1, reader_metrics.filter_metrics.files_time_range_pruned);
3331        let mut ranges = SmallVec::new();
3332        builder.build_ranges(-1, &mut ranges);
3333        assert!(ranges.is_empty());
3334    }
3335
3336    #[tokio::test]
3337    async fn test_manifest_pruning_observes_dynamic_filter_update() {
3338        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
3339        let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
3340        let predicate_group = PredicateGroup::new(metadata.as_ref(), &[]).unwrap();
3341        let arrow_schema = metadata.schema.arrow_schema();
3342        let ts_expr = physical_col("ts", arrow_schema.as_ref()).unwrap();
3343        let dyn_filter = Arc::new(DynamicFilterPhysicalExpr::new(
3344            vec![ts_expr.clone()],
3345            physical_lit(true),
3346        ));
3347        predicate_group.add_dyn_filters(vec![dyn_filter.clone()]);
3348        let input = ScanInput::builder(SchedulerEnv::new().await.access_layer.clone(), mapper)
3349            .with_predicate(predicate_group)
3350            .build();
3351        let file = file_handle_with_time_range(
3352            Timestamp::new_millisecond(0),
3353            Timestamp::new_millisecond(1000),
3354        );
3355
3356        assert!(!input.can_manifest_prune_file(&file));
3357
3358        let updated = physical_binary(
3359            ts_expr,
3360            Operator::Gt,
3361            physical_lit(ScalarValue::TimestampMillisecond(Some(1000), None)),
3362            arrow_schema.as_ref(),
3363        )
3364        .unwrap();
3365        dyn_filter.update(updated).unwrap();
3366
3367        assert!(input.can_manifest_prune_file(&file));
3368        input.predicate.clear_dyn_filters();
3369        assert!(!input.can_manifest_prune_file(&file));
3370    }
3371
3372    #[tokio::test]
3373    async fn test_range_pre_filter_mode() {
3374        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
3375        let cases = [
3376            (true, MergeMode::LastRow, 1, PreFilterMode::All),
3377            (false, MergeMode::LastNonNull, 1, PreFilterMode::All),
3378            (false, MergeMode::LastRow, 2, PreFilterMode::SkipFields),
3379            (true, MergeMode::LastRow, 2, PreFilterMode::All),
3380        ];
3381
3382        for (append_mode, merge_mode, source_count, expected_mode) in cases {
3383            let input = new_scan_input(metadata.clone(), vec![])
3384                .await
3385                .with_append_mode(append_mode)
3386                .with_merge_mode(merge_mode)
3387                .build();
3388
3389            assert_eq!(expected_mode, input.range_pre_filter_mode(source_count));
3390        }
3391    }
3392
3393    #[test]
3394    fn test_exact_sequence_selection_preserves_range_and_fallback() {
3395        let request = ScanRequest {
3396            exact_sequence_range: true,
3397            memtable_min_sequence: Some(1),
3398            memtable_max_sequence: Some(2),
3399            ..Default::default()
3400        };
3401        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
3402        let mutable = crate::memtable::time_partition::TimePartitions::new(
3403            metadata.clone(),
3404            Arc::new(crate::test_util::memtable_util::EmptyMemtableBuilder::default()),
3405            0,
3406            None,
3407        );
3408        let version = Arc::new(
3409            crate::region::version::VersionBuilder::new(metadata, mutable)
3410                .options(crate::region::options::RegionOptions {
3411                    preserve_row_sequence: true,
3412                    ..Default::default()
3413                })
3414                .build(),
3415        );
3416        let (files, range) = exact_sequence_range(&request, &version).unwrap();
3417        assert!(files.is_empty());
3418        assert_eq!(Some(SequenceRange::GtLtEq { min: 1, max: 2 }), range);
3419
3420        let skip_sst_request = ScanRequest {
3421            skip_sst_files: true,
3422            ..request
3423        };
3424        let (files, range) = exact_sequence_range(&skip_sst_request, &version).unwrap();
3425        assert!(files.is_empty());
3426        assert_eq!(None, range);
3427    }
3428
3429    #[test]
3430    fn test_exact_sequence_selection_checks_selected_files() {
3431        use std::num::NonZeroU64;
3432
3433        use crate::test_util::new_noop_file_purger;
3434
3435        let request = ScanRequest {
3436            exact_sequence_range: true,
3437            memtable_min_sequence: Some(5),
3438            memtable_max_sequence: Some(10),
3439            ..Default::default()
3440        };
3441        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
3442        let mutable = crate::memtable::time_partition::TimePartitions::new(
3443            metadata.clone(),
3444            Arc::new(crate::test_util::memtable_util::EmptyMemtableBuilder::default()),
3445            0,
3446            None,
3447        );
3448        let target_region_id = metadata.region_id;
3449
3450        // Files at or below the cursor are excluded. Among the remaining files,
3451        // marked locals are exact-capable while unmarked locals deny exactness.
3452        for (sequence, marked, expected_files, expected_range) in [
3453            (NonZeroU64::new(5), false, false, true),
3454            (NonZeroU64::new(5), true, false, true),
3455            (NonZeroU64::new(100), false, true, false),
3456            (None, false, true, false),
3457            (None, true, true, true),
3458        ] {
3459            let version = crate::region::version::VersionBuilder::new(
3460                metadata.clone(),
3461                mutable.new_with_part_duration(None, None),
3462            )
3463            .options(crate::region::options::RegionOptions {
3464                preserve_row_sequence: true,
3465                ..Default::default()
3466            })
3467            .add_files(
3468                new_noop_file_purger(),
3469                [FileMeta {
3470                    region_id: target_region_id,
3471                    sequence,
3472                    preserve_row_sequence: marked,
3473                    ..Default::default()
3474                }]
3475                .into_iter(),
3476            )
3477            .build();
3478            let (files, range) = exact_sequence_range(&request, &version).unwrap();
3479            assert_eq!(expected_files, !files.is_empty());
3480            assert_eq!(expected_range, range.is_some());
3481        }
3482
3483        // Foreign files use the target-local barrier, regardless of their
3484        // source marker. A missing barrier fails closed.
3485        for (sequence, marked, expected_count, expected_error) in [
3486            (NonZeroU64::new(5), false, 0, false),
3487            (NonZeroU64::new(6), false, 1, false),
3488            (NonZeroU64::new(11), true, 1, false),
3489            (None, true, 1, true),
3490        ] {
3491            let file_id = store_api::storage::FileId::random();
3492            let version = crate::region::version::VersionBuilder::new(
3493                metadata.clone(),
3494                mutable.new_with_part_duration(None, None),
3495            )
3496            .options(crate::region::options::RegionOptions {
3497                preserve_row_sequence: true,
3498                ..Default::default()
3499            })
3500            .add_files(
3501                new_noop_file_purger(),
3502                [FileMeta {
3503                    region_id: RegionId::new(1, 2),
3504                    file_id,
3505                    sequence,
3506                    preserve_row_sequence: marked,
3507                    ..Default::default()
3508                }]
3509                .into_iter(),
3510            )
3511            .build();
3512            let result = exact_sequence_range(&request, &version);
3513            if expected_error {
3514                assert!(result.is_err());
3515            } else {
3516                let (files, range) = result.unwrap();
3517                assert_eq!(expected_count, files.len());
3518                assert!(range.is_some());
3519            }
3520        }
3521
3522        // Foreign-domain validation still takes precedence over preserve mode
3523        // and a missing upper bound once the file is selected.
3524        let request = ScanRequest {
3525            memtable_max_sequence: None,
3526            ..request
3527        };
3528        let version = crate::region::version::VersionBuilder::new(metadata, mutable)
3529            .options(crate::region::options::RegionOptions {
3530                preserve_row_sequence: false,
3531                ..Default::default()
3532            })
3533            .add_files(
3534                new_noop_file_purger(),
3535                [FileMeta {
3536                    region_id: RegionId::new(1, 2),
3537                    ..Default::default()
3538                }]
3539                .into_iter(),
3540            )
3541            .build();
3542        assert!(exact_sequence_range(&request, &version).is_err());
3543    }
3544
3545    #[tokio::test]
3546    async fn test_parallel_flat_source_stops_after_receiver_dropped() {
3547        use std::sync::atomic::{AtomicUsize, Ordering};
3548
3549        use datatypes::arrow::array::Int64Array;
3550
3551        const SOURCE_BATCHES: usize = 10_000;
3552
3553        let metadata = Arc::new(metadata_with_primary_key(vec![0, 1], false));
3554        let mapper = FlatProjectionMapper::new(&metadata, [0, 2, 3]).unwrap();
3555        let env = SchedulerEnv::new().await;
3556        let input = ScanInput::builder(env.access_layer.clone(), mapper).build();
3557
3558        let schema = Arc::new(ArrowSchema::new(vec![Field::new(
3559            "v",
3560            ArrowDataType::Int64,
3561            false,
3562        )]));
3563        let batch =
3564            RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![1]))]).unwrap();
3565        let pulled = Arc::new(AtomicUsize::new(0));
3566        let mut released = Vec::new();
3567        let sources = (0..2)
3568            .map(|_| {
3569                let (release_tx, release_rx) = tokio::sync::oneshot::channel::<()>();
3570                released.push(release_rx);
3571                let batch = batch.clone();
3572                let pulled = pulled.clone();
3573                futures::stream::iter(0..SOURCE_BATCHES)
3574                    .map(move |_| {
3575                        // Dropped together with the source stream.
3576                        let _ = &release_tx;
3577                        pulled.fetch_add(1, Ordering::Relaxed);
3578                        Ok(batch.clone())
3579                    })
3580                    .boxed()
3581            })
3582            .collect();
3583
3584        let streams = input
3585            .create_parallel_flat_sources(sources, Arc::new(Semaphore::new(2)), 1)
3586            .unwrap();
3587        drop(streams);
3588
3589        for release_rx in released {
3590            tokio::time::timeout(std::time::Duration::from_secs(10), release_rx)
3591                .await
3592                .unwrap()
3593                .unwrap_err();
3594        }
3595        // Each task may have read a few batches ahead into its channel before the
3596        // receiver went away, but must not drain the rest of its source.
3597        assert!(pulled.load(Ordering::Relaxed) < 100);
3598    }
3599}