Skip to main content

mito2/read/
series_scan.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Per-series scan implementation.
16
17mod prepared;
18mod raw;
19
20use std::collections::HashSet;
21use std::fmt;
22use std::sync::atomic::{AtomicUsize, Ordering};
23use std::sync::{Arc, Mutex};
24use std::time::Instant;
25
26use api::v1::SemanticType;
27use async_stream::try_stream;
28use common_error::ext::BoxedError;
29use common_query::native_histogram::is_native_histogram_value_type;
30use common_recordbatch::util::ChainedRecordBatchStream;
31use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
32use common_telemetry::tracing::{self, Instrument};
33use common_telemetry::{debug, warn};
34use datafusion::execution::memory_pool::MemoryPool;
35use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
36use datafusion::physical_plan::{DisplayAs, DisplayFormatType, PhysicalExpr};
37use datafusion_common::tree_node::TreeNode;
38use datafusion_expr::{Expr, Volatility};
39use datatypes::arrow::array::BinaryArray;
40use datatypes::arrow::datatypes::DataType as ArrowDataType;
41use datatypes::arrow::record_batch::RecordBatch;
42use datatypes::prelude::ConcreteDataType;
43use datatypes::schema::SchemaRef;
44use datatypes::series::SeriesOrdering;
45use datatypes::series::samples::{FieldSpec, SeriesSchema};
46use datatypes::types::TimestampType;
47use futures::{StreamExt, TryStreamExt};
48use smallvec::SmallVec;
49use snafu::{OptionExt, ResultExt, ensure};
50use store_api::metadata::RegionMetadataRef;
51use store_api::metric_engine_consts::is_metric_engine_internal_column;
52use store_api::region_engine::{
53    PartitionRange, PrepareRequest, QueryScanContext, RawLabelSource, RegionScanner,
54    ScannerProperties, SendableRawSeriesStream,
55};
56use store_api::storage::consts::{ReservedColumnId, is_internal_column};
57use table::predicate::Predicate;
58use tokio::sync::Semaphore;
59use tokio::sync::mpsc::error::TrySendError;
60use tokio::sync::mpsc::{self, Receiver, Sender};
61
62use crate::error::{
63    Error, InvalidRecordBatchSnafu, InvalidSenderSnafu, JoinSnafu, PartitionOutOfRangeSnafu,
64    Result, ScanMultiTimesSnafu, ScanSeriesSnafu, TooManyFilesToReadSnafu,
65};
66use crate::metrics::SCAN_MAX_CONCURRENT_FILES;
67use crate::read::ScannerMetrics;
68use crate::read::pruner::{PartitionPruner, Pruner, PrunerOptions};
69use crate::read::scan_region::{ScanInput, StreamContext};
70use crate::read::scan_util::{
71    PartitionMetrics, PartitionMetricsList, SeriesDistributorMetrics, compute_average_batch_size,
72    compute_parallel_channel_size,
73};
74use crate::read::seq_scan::{FlatReadMode, SeqScan};
75use crate::read::series_candidate::{SeriesCandidateScanner, is_sparse_metric_metadata};
76use crate::read::series_reader::{AssignedSeriesBatch, SeriesBatchCollector, SeriesReader};
77use crate::read::stream::{ConvertBatchStream, ScanBatch, ScanBatchStream};
78use crate::sst::parquet::flat_format::primary_key_column_index;
79use crate::sst::parquet::format::PrimaryKeyArray;
80
81/// Maximum number of candidate series retained before distributing assignments.
82const CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD: usize = 1_000_000;
83
84/// Test-only override of [`CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD`], where `0` means "use the
85/// production constant".
86///
87/// The production build compiles this static, the setter below and the private accessor out, so
88/// the shipped threshold is always the constant above.
89#[cfg(test)]
90static CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD_OVERRIDE: AtomicUsize = AtomicUsize::new(0);
91
92/// Test-only number of candidate assignment rounds sent since the override was last set.
93///
94/// A round is one `send_assignments` call, so a partition that received more than one assignment
95/// sees `>= 2` rounds here. Increments are compiled out of production builds.
96#[cfg(test)]
97static CANDIDATE_ASSIGNMENT_ROUNDS: AtomicUsize = AtomicUsize::new(0);
98
99/// Returns the candidate-series assignment threshold, honouring the test-only override.
100fn candidate_series_assignment_threshold() -> usize {
101    #[cfg(test)]
102    {
103        let threshold = CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD_OVERRIDE.load(Ordering::Relaxed);
104        if threshold > 0 {
105            return threshold;
106        }
107    }
108    CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD
109}
110
111/// Test-only: overrides the candidate-series assignment threshold and resets the round counter.
112///
113/// Pass `0` to restore the production threshold. Tests must reset the override before they
114/// return; a scope guard in `engine::scan_test` does that even on panic.
115#[cfg(test)]
116pub(crate) fn set_candidate_series_assignment_threshold(threshold: usize) {
117    CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD_OVERRIDE.store(threshold, Ordering::Relaxed);
118    CANDIDATE_ASSIGNMENT_ROUNDS.store(0, Ordering::Relaxed);
119}
120
121/// Test-only: returns the number of assignment rounds sent since the override was last set.
122#[cfg(test)]
123pub(crate) fn candidate_assignment_rounds() -> usize {
124    CANDIDATE_ASSIGNMENT_ROUNDS.load(Ordering::Relaxed)
125}
126
127/// Legacy data receivers for output partitions.
128type LegacyReceiverList = Vec<Option<Receiver<Result<SeriesBatch>>>>;
129
130/// Candidate assignment receivers for output partitions.
131type CandidateReceiverList = Vec<Option<CandidateReceiver>>;
132
133/// Candidate assignment senders for output partitions.
134type CandidateSenderList = Vec<Option<mpsc::UnboundedSender<Result<SeriesReaderInput>>>>;
135
136#[derive(Clone)]
137struct RawScanPreparation {
138    schema: Arc<SeriesSchema>,
139    fields: Vec<(usize, ConcreteDataType)>,
140    predicates: Vec<Arc<dyn PhysicalExpr>>,
141}
142
143/// Memoized raw-scan preflight: the label binding it was computed for, and its answer.
144///
145/// The answer carries the preparation together with whether its data comes from the columnar
146/// prepared source; `None` means the raw path is unavailable for this scan.
147type PreflightCache = Option<(RawLabelSource, Option<(RawScanPreparation, bool)>)>;
148
149struct CandidateReceiver {
150    receiver: mpsc::UnboundedReceiver<Result<SeriesReaderInput>>,
151    active_receivers: Arc<AtomicUsize>,
152}
153
154impl CandidateReceiver {
155    async fn recv(&mut self) -> Option<Result<SeriesReaderInput>> {
156        self.receiver.recv().await
157    }
158}
159
160impl Drop for CandidateReceiver {
161    fn drop(&mut self) {
162        let previous = self.active_receivers.fetch_sub(1, Ordering::Relaxed);
163        debug_assert!(previous > 0);
164    }
165}
166
167/// Input required by a partition-local series reader.
168struct SeriesReaderInput {
169    assigned_series: AssignedSeriesBatch,
170    partition_pruner: Arc<PartitionPruner>,
171    range_semaphore: Arc<Semaphore>,
172}
173
174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175enum SeriesScanMode {
176    Legacy,
177    TwoPhase,
178}
179
180impl SeriesScanMode {
181    fn as_str(self) -> &'static str {
182        match self {
183            Self::Legacy => "legacy",
184            Self::TwoPhase => "two_phase",
185        }
186    }
187}
188
189/// Scans a region and returns sorted rows of a series in the same partition.
190///
191/// The output order is always order by `(primary key, time index)` inside every
192/// partition.
193/// Always returns the same series (primary key) to the same partition.
194pub struct SeriesScan {
195    /// Implementation used by this scan.
196    mode: SeriesScanMode,
197    prepared: Arc<prepared::Preparation>,
198    /// Properties of the scanner.
199    properties: ScannerProperties,
200    /// Context of streams.
201    stream_ctx: Arc<StreamContext>,
202    /// Shared pruner for file range building.
203    pruner: Arc<Pruner>,
204    /// Legacy data receivers for each partition.
205    legacy_receivers: Mutex<LegacyReceiverList>,
206    /// Candidate assignment receivers for each partition.
207    candidate_receivers: Mutex<CandidateReceiverList>,
208    /// Metrics for each partition.
209    /// The scanner only sets in query and keeps it empty during compaction.
210    metrics_list: Arc<PartitionMetricsList>,
211    /// Memoized result of [`Self::prepare_raw_scan`] for the label binding it was computed for.
212    ///
213    /// Preflight only reads pinned scanner state, and one execution asks for it once per output
214    /// partition, so the first answer serves the rest. Errors are never stored, and every entry
215    /// that mutates the state the answer depends on clears the entry. The guard is taken and
216    /// released inside a synchronous call, so it never spans an await.
217    preflight: Mutex<PreflightCache>,
218    /// Test-only number of preflight computations performed by this scanner.
219    ///
220    /// A cached answer does not increment it, so a test can prove that a repeated preflight over
221    /// the same pinned state is served from [`Self::preflight`].
222    #[cfg(test)]
223    preflight_runs: AtomicUsize,
224}
225
226impl SeriesScan {
227    /// Creates a new [SeriesScan].
228    pub(crate) fn new(input: ScanInput, experimental_series_scan_v2: bool) -> Self {
229        let mode = if experimental_series_scan_v2 && Self::supports_two_phase(&input) {
230            SeriesScanMode::TwoPhase
231        } else {
232            SeriesScanMode::Legacy
233        };
234        let mut properties = ScannerProperties::default()
235            .with_append_mode(input.append_mode)
236            .with_total_rows(input.total_rows())
237            .with_total_rows_is_exact(input.append_mode && input.total_rows_is_exact());
238        if let Some(counters) = input.query_stat_counters.clone() {
239            properties.set_query_stat_counters(counters);
240        }
241        let stream_ctx = Arc::new(StreamContext::seq_scan_ctx(input));
242        properties.partitions = vec![stream_ctx.partition_ranges()];
243
244        // Create the shared pruner with number of workers equal to CPU cores.
245        let num_workers = common_stat::get_total_cpu_cores().max(1);
246        let pruner = match mode {
247            SeriesScanMode::Legacy => Arc::new(Pruner::new(stream_ctx.clone(), num_workers)),
248            SeriesScanMode::TwoPhase => Arc::new(Pruner::new_with_options(
249                stream_ctx.clone(),
250                num_workers,
251                PrunerOptions {
252                    retain_builders: true,
253                    enable_predicate_prefilter: false,
254                },
255            )),
256        };
257
258        Self {
259            mode,
260            prepared: Arc::new(prepared::Preparation::default()),
261            properties,
262            stream_ctx,
263            pruner,
264            legacy_receivers: Mutex::new(Vec::new()),
265            candidate_receivers: Mutex::new(Vec::new()),
266            metrics_list: Arc::new(PartitionMetricsList::default()),
267            preflight: Mutex::new(None),
268            #[cfg(test)]
269            preflight_runs: AtomicUsize::new(0),
270        }
271    }
272
273    /// Returns the number of preflight computations this scanner has run.
274    ///
275    /// Test-only: a memoized preflight does not count, so a test can distinguish a cache hit from a
276    /// recomputation.
277    #[cfg(test)]
278    pub(crate) fn preflight_runs(&self) -> usize {
279        self.preflight_runs.load(Ordering::Relaxed)
280    }
281
282    /// Drops the memoized preflight.
283    ///
284    /// Every entry point that mutates state the preflight reads calls this, so a stale answer can
285    /// never outlive the state it was derived from.
286    fn clear_preflight_cache(&self) {
287        *self.preflight.lock().unwrap() = None;
288    }
289
290    /// Returns the memoized preflight, if it was computed for the same label binding.
291    ///
292    /// A binding only matches when its immutable identity matches as well: the preflight validates
293    /// the pinned metadata snapshot and the label binding, so an answer for another binding must
294    /// never be reused even when the labels look alike.
295    fn cached_preflight(
296        &self,
297        label_source: &RawLabelSource,
298    ) -> Option<Option<(RawScanPreparation, bool)>> {
299        let cache = self.preflight.lock().unwrap();
300        let (cached_source, cached) = cache.as_ref()?;
301        if !prepared::same_labels(cached_source, label_source) {
302            return None;
303        }
304        Some(cached.clone())
305    }
306
307    fn uses_columnar(&self) -> bool {
308        self.stream_ctx
309            .input
310            .series_index
311            .as_ref()
312            .is_some_and(|context| context.format == crate::config::SeriesIndexFormat::Columnar)
313    }
314
315    /// Whether every selected *field* is a `Float64`, which is all the columnar prepared reader
316    /// serves.
317    ///
318    /// Only field columns count: the projection also carries tags and the time index, and those are
319    /// no part of the samples the prepared reader hands over. Checking them here would send an
320    /// ordinary `Float64` query down the ordinary scan just because it projects a timestamp.
321    fn columnar_fields_supported(&self) -> bool {
322        let metadata = self.stream_ctx.input.mapper.metadata();
323        let selected = self.stream_ctx.input.mapper.output_schema();
324        selected
325            .column_schemas()
326            .iter()
327            .filter(|output| {
328                metadata.column_metadatas.iter().any(|column| {
329                    column.column_schema.name == output.name
330                        && column.semantic_type == SemanticType::Field
331                })
332            })
333            .all(|output| output.data_type == ConcreteDataType::float64_datatype())
334    }
335
336    /// Returns whether raw series come from the columnar prepared source.
337    ///
338    /// Derived only from the pinned series index version and every configured partition range, so
339    /// preflight, the partition reader and scanner-wide preparation agree.
340    /// `false` means the stored sources are read by the ordinary scan.
341    fn uses_prepared_columnar(&self) -> Result<bool, BoxedError> {
342        if !self.uses_columnar() {
343            return Ok(false);
344        }
345        let ranges = self
346            .properties
347            .partitions
348            .iter()
349            .flatten()
350            .copied()
351            .collect::<Vec<_>>();
352        prepared::validate_inputs(&self.stream_ctx, &ranges).map_err(BoxedError::new)
353    }
354
355    /// Prepares the raw scan and reports whether its data comes from the columnar prepared source.
356    ///
357    /// The answer only depends on pinned scanner state plus the label binding, and an execution
358    /// asks for it once per output partition, so it is memoized per binding. Only successful
359    /// answers are stored: a rejected binding must fail on every call, not replay a rejection or
360    /// hide a later one.
361    fn prepare_raw_scan(
362        &self,
363        label_source: &RawLabelSource,
364    ) -> Result<Option<(RawScanPreparation, bool)>, BoxedError> {
365        if let Some(cached) = self.cached_preflight(label_source) {
366            return Ok(cached);
367        }
368        let computed = self.prepare_raw_scan_uncached(label_source)?;
369        // Both sides of the cache are cheap to clone, and callers own their copy: the merge schema
370        // helper rewrites the field indices and predicates of the preparation it is given, which
371        // must never reach the stored answer.
372        *self.preflight.lock().unwrap() = Some((label_source.clone(), computed.clone()));
373        Ok(computed)
374    }
375
376    /// Computes a fresh [`Self::prepare_raw_scan`] answer from the pinned scanner state.
377    fn prepare_raw_scan_uncached(
378        &self,
379        label_source: &RawLabelSource,
380    ) -> Result<Option<(RawScanPreparation, bool)>, BoxedError> {
381        #[cfg(test)]
382        self.preflight_runs.fetch_add(1, Ordering::Relaxed);
383        let input = &self.stream_ctx.input;
384        let metadata = input.mapper.metadata();
385        match label_source {
386            RawLabelSource::PrimaryKey {
387                physical_metadata,
388                logical_tags,
389                ..
390            } => {
391                if !Arc::ptr_eq(metadata, physical_metadata) {
392                    debug!(
393                        "Raw series scan preflight rejected metadata snapshot: region_id={}, physical_region_id={}, schema_version={}, physical_schema_version={}",
394                        metadata.region_id,
395                        physical_metadata.region_id,
396                        metadata.schema_version,
397                        physical_metadata.schema_version,
398                    );
399                    return Err(BoxedError::new(
400                        InvalidRecordBatchSnafu {
401                            reason: "raw label source metadata does not match scanner snapshot"
402                                .to_string(),
403                        }
404                        .build(),
405                    ));
406                }
407                let mut ids = HashSet::new();
408                let mut names = HashSet::new();
409                for tag in logical_tags.iter() {
410                    let column = metadata.column_by_id(tag.column_id).filter(|column| {
411                        column.semantic_type == SemanticType::Tag
412                            && metadata.primary_key.contains(&tag.column_id)
413                    });
414                    let valid = column.is_some_and(|column| {
415                        column.column_schema.name == tag.name.as_ref()
416                            && tag.name.as_ref() != "__name__"
417                            && !is_internal_column(tag.name.as_ref())
418                            && !is_metric_engine_internal_column(tag.name.as_ref())
419                            && !ReservedColumnId::is_reserved(tag.column_id)
420                    });
421                    let unique_id = valid && ids.insert(tag.column_id);
422                    let unique_name = unique_id && names.insert(tag.name.as_ref());
423                    if !valid || !unique_id || !unique_name {
424                        debug!(
425                            "Raw series scan preflight rejected label binding: region_id={}, column_id={}, binding_valid={}, unique_id={}, unique_name={}",
426                            metadata.region_id, tag.column_id, valid, unique_id, unique_name,
427                        );
428                        return Err(BoxedError::new(
429                            InvalidRecordBatchSnafu {
430                                reason: "invalid raw label binding".to_string(),
431                            }
432                            .build(),
433                        ));
434                    }
435                }
436            }
437            RawLabelSource::Explicit => {
438                debug!(
439                    "Raw series scan preflight rejected explicit labels: region_id={}",
440                    metadata.region_id,
441                );
442                return Ok(None);
443            }
444        }
445        if self.uses_columnar() && !is_sparse_metric_metadata(metadata) {
446            return Ok(None);
447        }
448        let selected = input.mapper.output_schema();
449        let mut fields = Vec::new();
450        let mut specs = Vec::new();
451        // The columnar prepared reader serves `Float64` fields only. A selection carrying another
452        // kind the raw path supports - a native histogram struct, today - is read by the ordinary
453        // scan instead: the columnar samples are an artifact beside the SST rows, so the same
454        // sources are still reachable, and the frame schema and ordering follow the reader that
455        // actually answers.
456        let columnar_fields_supported = self.columnar_fields_supported();
457        for output in selected.column_schemas() {
458            let Some(column) = metadata.column_metadatas.iter().find(|column| {
459                column.column_schema.name == output.name
460                    && column.semantic_type == SemanticType::Field
461            }) else {
462                continue;
463            };
464            let Some((index, (_, data_type))) = input
465                .mapper
466                .batch_schema()
467                .iter()
468                .enumerate()
469                .find(|(_, (id, _))| *id == column.column_id)
470            else {
471                debug!(
472                    "Raw series scan preflight missing mapper field: region_id={}, column_id={}",
473                    metadata.region_id, column.column_id,
474                );
475                return Err(BoxedError::new(
476                    InvalidRecordBatchSnafu {
477                        reason: format!(
478                            "selected field {} is missing from mapper batch schema",
479                            output.name
480                        ),
481                    }
482                    .build(),
483                ));
484            };
485            // The raw converter casts a numeric field to `Float64` and converts a native histogram
486            // struct into histogram samples; nothing else has a raw representation.
487            if !data_type.is_numeric() && !is_native_histogram_value_type(data_type) {
488                debug!(
489                    "Raw series scan preflight rejected field type: region_id={}, column_id={}",
490                    metadata.region_id, column.column_id,
491                );
492                return Ok(None);
493            }
494            fields.push((index, data_type.clone()));
495            specs.push(FieldSpec {
496                name: Arc::from(output.name.as_str()),
497            });
498        }
499        let time_is_millis = matches!(metadata.time_index_type(), TimestampType::Millisecond(_));
500        if fields.is_empty()
501            || input.append_mode
502            || input.series_row_selector.is_some()
503            || !time_is_millis
504        {
505            debug!(
506                "Raw series scan preflight rejected scan options: region_id={}, selected_fields={}, append_mode={}, row_selector={}, time_is_millis={}",
507                metadata.region_id,
508                fields.len(),
509                input.append_mode,
510                input.series_row_selector.is_some(),
511                time_is_millis,
512            );
513            return Ok(None);
514        }
515        // Structural eligibility errors still reject the raw path; ordinary scans cover sources
516        // without columnar artifacts, so missing coverage is not an error here.
517        let prepared_columnar = self.uses_prepared_columnar()? && columnar_fields_supported;
518        let schema = Arc::new(SeriesSchema {
519            fields: specs.into(),
520            eval_grid: None,
521        });
522        let typed_schema = input.mapper.input_arrow_schema(false);
523        let mut predicates = Vec::new();
524        let predicate = if prepared_columnar {
525            input.predicate_group().predicate()
526        } else {
527            // The ordinary scan applies region predicates before this residual filtering.
528            input.predicate_group().predicate_without_region()
529        };
530        if let Some(predicate) = predicate {
531            if !predicate.dyn_filters().is_empty() {
532                debug!(
533                    "Raw series scan preflight rejected dynamic predicate filters: region_id={}",
534                    metadata.region_id,
535                );
536                return Ok(None);
537            }
538            for expr in predicate.exprs() {
539                // Fresh execution properties cannot preserve Stable or query-dependent semantics.
540                if expr
541                    .exists(|node| {
542                        Ok(matches!(node, Expr::ScalarFunction(function)
543                            if function.func.signature().volatility != Volatility::Immutable)
544                            || matches!(node, Expr::ScalarVariable(..)))
545                    })
546                    .unwrap_or(true)
547                {
548                    debug!(
549                        "Raw series scan preflight rejected predicate volatility or variable: region_id={}",
550                        metadata.region_id,
551                    );
552                    return Ok(None);
553                }
554                // Evaluate residuals with original types before raw conversion to Float64.
555                let Ok(physical) = Predicate::to_physical_expr(expr, &typed_schema) else {
556                    debug!(
557                        "Raw series scan preflight failed predicate compilation: region_id={}",
558                        metadata.region_id,
559                    );
560                    return Ok(None);
561                };
562                if !matches!(
563                    physical.data_type(&typed_schema),
564                    Ok(ArrowDataType::Boolean | ArrowDataType::Null)
565                ) {
566                    debug!(
567                        "Raw series scan preflight rejected predicate type: region_id={}",
568                        metadata.region_id,
569                    );
570                    return Ok(None);
571                }
572                predicates.push(physical);
573            }
574        }
575        Ok(Some((
576            RawScanPreparation {
577                schema,
578                fields,
579                predicates,
580            },
581            prepared_columnar,
582        )))
583    }
584
585    /// Starts scanner-wide preparation from the preflight already performed by the caller.
586    fn prepare_series_with_raw(
587        &self,
588        ctx: &QueryScanContext,
589        metrics_set: &ExecutionPlanMetricsSet,
590        label_source: &RawLabelSource,
591        preparation: Result<Option<(RawScanPreparation, bool)>, BoxedError>,
592    ) -> futures::future::BoxFuture<
593        'static,
594        Result<Option<Arc<dyn store_api::region_engine::PreparedSeriesSource>>, BoxedError>,
595    > {
596        let state = self.prepared.clone();
597        let stream_ctx = self.stream_ctx.clone();
598        let ranges = self
599            .properties
600            .partitions
601            .iter()
602            .flatten()
603            .copied()
604            .collect();
605        let labels = label_source.clone();
606        let metrics = new_partition_metrics(
607            &self.stream_ctx,
608            ctx.explain_verbose,
609            metrics_set,
610            0,
611            &self.metrics_list,
612        );
613        Box::pin(async move {
614            let Some((preparation, prepared_columnar)) = preparation? else {
615                return Ok(None);
616            };
617            if !prepared_columnar {
618                // Known unsupported coverage: the caller keeps the ordinary scan instead.
619                return Ok(None);
620            }
621            state
622                .prepare(stream_ctx, ranges, labels, preparation, metrics)
623                .await
624                .map(Some)
625        })
626    }
627
628    fn supports_two_phase(input: &ScanInput) -> bool {
629        if !is_sparse_metric_metadata(input.region_metadata()) {
630            return false;
631        }
632        #[cfg(feature = "enterprise")]
633        if !input.extension_ranges().is_empty() {
634            return false;
635        }
636        true
637    }
638
639    #[tracing::instrument(
640        skip_all,
641        fields(
642            region_id = %self.stream_ctx.input.mapper.metadata().region_id,
643            partition = partition
644        )
645    )]
646    fn scan_partition_impl(
647        &self,
648        ctx: &QueryScanContext,
649        metrics_set: &ExecutionPlanMetricsSet,
650        partition: usize,
651    ) -> Result<SendableRecordBatchStream> {
652        let metrics = new_partition_metrics(
653            &self.stream_ctx,
654            ctx.explain_verbose,
655            metrics_set,
656            partition,
657            &self.metrics_list,
658        );
659
660        let batch_stream =
661            self.scan_batch_in_partition(ctx, partition, metrics.clone(), metrics_set)?;
662
663        let input = &self.stream_ctx.input;
664        let record_batch_stream = ConvertBatchStream::new(
665            batch_stream,
666            input.mapper.clone(),
667            input.cache_strategy.clone(),
668            metrics,
669        );
670
671        Ok(Box::pin(RecordBatchStreamWrapper::new(
672            input.mapper.output_schema(),
673            Box::pin(record_batch_stream),
674        )))
675    }
676
677    #[tracing::instrument(
678        skip_all,
679        fields(
680            region_id = %self.stream_ctx.input.mapper.metadata().region_id,
681            partition = partition
682        )
683    )]
684    fn scan_batch_in_partition(
685        &self,
686        ctx: &QueryScanContext,
687        partition: usize,
688        part_metrics: PartitionMetrics,
689        metrics_set: &ExecutionPlanMetricsSet,
690    ) -> Result<ScanBatchStream> {
691        if ctx.explain_verbose {
692            common_telemetry::info!(
693                "SeriesScan partition {}, region_id: {}",
694                partition,
695                self.stream_ctx.input.region_metadata().region_id
696            );
697        }
698
699        ensure!(
700            partition < self.properties.num_partitions(),
701            PartitionOutOfRangeSnafu {
702                given: partition,
703                all: self.properties.num_partitions(),
704            }
705        );
706
707        match self.mode {
708            SeriesScanMode::Legacy => self.scan_legacy_batch_in_partition(
709                partition,
710                part_metrics,
711                metrics_set,
712                ctx.explain_verbose,
713            ),
714            SeriesScanMode::TwoPhase => {
715                self.scan_two_phase_batch_in_partition(ctx, partition, part_metrics, metrics_set)
716            }
717        }
718    }
719
720    fn scan_legacy_batch_in_partition(
721        &self,
722        partition: usize,
723        part_metrics: PartitionMetrics,
724        metrics_set: &ExecutionPlanMetricsSet,
725        explain_verbose: bool,
726    ) -> Result<ScanBatchStream> {
727        self.maybe_start_legacy_distributor(metrics_set, explain_verbose);
728        let mut receiver = self.legacy_receivers.lock().unwrap()[partition]
729            .take()
730            .context(ScanMultiTimesSnafu { partition })?;
731        let stream = try_stream! {
732            part_metrics.on_first_poll();
733
734            let mut fetch_start = Instant::now();
735            let mut metrics = ScannerMetrics::default();
736            while let Some(series) = receiver.recv().await {
737                let series = series?;
738                metrics.scan_cost += fetch_start.elapsed();
739                metrics.num_batches += series.num_batches();
740                metrics.num_rows += series.num_rows();
741
742                let yield_start = Instant::now();
743                yield ScanBatch::Series(series);
744                metrics.yield_cost += yield_start.elapsed();
745                fetch_start = Instant::now();
746            }
747
748            part_metrics.merge_metrics(&metrics);
749            part_metrics.on_finish();
750        };
751        Ok(Box::pin(stream))
752    }
753
754    fn scan_two_phase_batch_in_partition(
755        &self,
756        ctx: &QueryScanContext,
757        partition: usize,
758        part_metrics: PartitionMetrics,
759        metrics_set: &ExecutionPlanMetricsSet,
760    ) -> Result<ScanBatchStream> {
761        self.maybe_start_candidate_distributor(ctx, metrics_set);
762        // Safety: `scan_batch_in_partition` validates the index, and the receiver
763        // list always contains one entry for each output partition.
764        let mut receiver = self.candidate_receivers.lock().unwrap()[partition]
765            .take()
766            .context(ScanMultiTimesSnafu { partition })?;
767        let stream_ctx = self.stream_ctx.clone();
768        let partition_ranges = self
769            .properties
770            .partitions
771            .iter()
772            .flatten()
773            .copied()
774            .collect::<Vec<_>>();
775        let stream = try_stream! {
776            part_metrics.on_first_poll();
777
778            let mut fetch_start = Instant::now();
779            let mut metrics = ScannerMetrics::default();
780            while let Some(input) = receiver.recv().await {
781                let input = input?;
782                metrics.scan_cost += fetch_start.elapsed();
783
784                let build_start = Instant::now();
785                let reader = SeriesReader::try_new(
786                    stream_ctx.clone(),
787                    partition_ranges.clone(),
788                    input.assigned_series,
789                    input.partition_pruner,
790                    input.range_semaphore,
791                    part_metrics.clone(),
792                )?;
793                let mut reader_stream = reader.build_stream().await?;
794                metrics.scan_cost += build_start.elapsed();
795                fetch_start = Instant::now();
796                while let Some(record_batch) = reader_stream.try_next().await? {
797                    metrics.scan_cost += fetch_start.elapsed();
798                    metrics.num_batches += 1;
799                    metrics.num_rows += record_batch.num_rows();
800
801                    let yield_start = Instant::now();
802                    yield ScanBatch::RecordBatch(record_batch);
803                    metrics.yield_cost += yield_start.elapsed();
804                    fetch_start = Instant::now();
805                }
806            }
807            metrics.scan_cost += fetch_start.elapsed();
808
809            part_metrics.merge_metrics(&metrics);
810            part_metrics.on_finish();
811        };
812        Ok(Box::pin(stream))
813    }
814
815    fn maybe_start_legacy_distributor(
816        &self,
817        metrics_set: &ExecutionPlanMetricsSet,
818        explain_verbose: bool,
819    ) {
820        let mut rx_list = self.legacy_receivers.lock().unwrap();
821        if !rx_list.is_empty() {
822            return;
823        }
824
825        let (senders, receivers) = new_legacy_channel_list(self.properties.num_partitions());
826        let mut distributor = SeriesDistributor {
827            stream_ctx: self.stream_ctx.clone(),
828            range_semaphore: Some(Arc::new(Semaphore::new(self.properties.num_partitions()))),
829            final_merge_semaphore: Some(Arc::new(Semaphore::new(self.properties.num_partitions()))),
830            partitions: self.properties.partitions.clone(),
831            pruner: self.pruner.clone(),
832            senders,
833            metrics_set: metrics_set.clone(),
834            metrics_list: self.metrics_list.clone(),
835            explain_verbose,
836        };
837        let region_id = distributor.stream_ctx.input.mapper.metadata().region_id;
838        let span = tracing::info_span!("SeriesScan::distributor", region_id = %region_id);
839        common_runtime::spawn_query(
840            async move {
841                distributor.execute().await;
842            }
843            .instrument(span),
844        );
845
846        *rx_list = receivers;
847    }
848
849    fn maybe_start_candidate_distributor(
850        &self,
851        ctx: &QueryScanContext,
852        metrics_set: &ExecutionPlanMetricsSet,
853    ) {
854        let mut rx_list = self.candidate_receivers.lock().unwrap();
855        if !rx_list.is_empty() {
856            return;
857        }
858
859        let (senders, receivers, active_receivers) =
860            new_candidate_channel_list(self.properties.num_partitions());
861        // Capture the first partition's runtime pool once for the shared distributor.
862        // reset_state clears the receivers so the next execution captures a fresh pool.
863        let mut distributor = SeriesCandidateDistributor {
864            memory_pool: ctx.memory_pool.clone(),
865            stream_ctx: self.stream_ctx.clone(),
866            range_semaphore: Arc::new(Semaphore::new(self.properties.num_partitions())),
867            partitions: self.properties.partitions.clone(),
868            pruner: self.pruner.clone(),
869            senders,
870            active_receivers,
871            metrics_set: metrics_set.clone(),
872            metrics_list: self.metrics_list.clone(),
873            explain_verbose: ctx.explain_verbose,
874        };
875        let region_id = distributor.stream_ctx.input.mapper.metadata().region_id;
876        let span = tracing::info_span!("SeriesScan::candidate_distributor", region_id = %region_id);
877        common_runtime::spawn_query(
878            async move {
879                distributor.execute().await;
880            }
881            .instrument(span),
882        );
883
884        *rx_list = receivers;
885    }
886
887    /// Scans the region and returns a stream.
888    #[tracing::instrument(
889        skip_all,
890        fields(region_id = %self.stream_ctx.input.mapper.metadata().region_id)
891    )]
892    pub(crate) async fn build_stream(&self) -> Result<SendableRecordBatchStream, BoxedError> {
893        let part_num = self.properties.num_partitions();
894        let metrics_set = ExecutionPlanMetricsSet::default();
895        let ctx = QueryScanContext::default();
896        let streams = (0..part_num)
897            .map(|i| self.scan_partition(&ctx, &metrics_set, i))
898            .collect::<Result<Vec<_>, BoxedError>>()?;
899        let chained_stream = ChainedRecordBatchStream::new(streams).map_err(BoxedError::new)?;
900        Ok(Box::pin(chained_stream))
901    }
902
903    /// Scan [`Batch`] in all partitions one by one.
904    pub(crate) fn scan_all_partitions(&self) -> Result<ScanBatchStream> {
905        let metrics_set = ExecutionPlanMetricsSet::new();
906
907        let ctx = QueryScanContext::default();
908        let streams = (0..self.properties.partitions.len())
909            .map(|partition| {
910                let metrics = new_partition_metrics(
911                    &self.stream_ctx,
912                    false,
913                    &metrics_set,
914                    partition,
915                    &self.metrics_list,
916                );
917
918                self.scan_batch_in_partition(&ctx, partition, metrics, &metrics_set)
919            })
920            .collect::<Result<Vec<_>>>()?;
921
922        Ok(Box::pin(futures::stream::iter(streams).flatten()))
923    }
924
925    /// Checks resource limit for the scanner.
926    pub(crate) fn check_scan_limit(&self) -> Result<()> {
927        // Sum the total number of files across all partitions
928        let total_files: usize = self
929            .properties
930            .partitions
931            .iter()
932            .flat_map(|partition| partition.iter())
933            .map(|part_range| {
934                let range_meta = &self.stream_ctx.ranges[part_range.identifier];
935                range_meta.indices.len()
936            })
937            .sum();
938
939        if !self.stream_ctx.input.compaction {
940            SCAN_MAX_CONCURRENT_FILES
941                .with_label_values(&["query"])
942                .observe(total_files as f64);
943        }
944        if self.stream_ctx.input.uses_scan_memory_limit() {
945            return Ok(());
946        }
947
948        let max_concurrent_files = self.stream_ctx.input.max_concurrent_scan_files;
949        if total_files > max_concurrent_files {
950            return TooManyFilesToReadSnafu {
951                actual: total_files,
952                max: max_concurrent_files,
953            }
954            .fail();
955        }
956
957        Ok(())
958    }
959}
960
961fn new_legacy_channel_list(num_partitions: usize) -> (SenderList, LegacyReceiverList) {
962    let (senders, receivers): (Vec<_>, Vec<_>) = (0..num_partitions)
963        .map(|_| {
964            let (sender, receiver) = mpsc::channel(1);
965            (Some(sender), Some(receiver))
966        })
967        .unzip();
968    (SenderList::new(senders), receivers)
969}
970
971fn new_candidate_channel_list(
972    num_partitions: usize,
973) -> (CandidateSenderList, CandidateReceiverList, Arc<AtomicUsize>) {
974    let active_receivers = Arc::new(AtomicUsize::new(num_partitions));
975    let (senders, receivers) = (0..num_partitions)
976        .map(|_| {
977            // Partition streams can be consumed sequentially. A bounded channel for an unpolled
978            // partition could block the distributor and prevent it from sending more assignments
979            // to the partition currently being consumed, causing a deadlock.
980            let (sender, receiver) = mpsc::unbounded_channel();
981            let receiver = CandidateReceiver {
982                receiver,
983                active_receivers: active_receivers.clone(),
984            };
985            (Some(sender), Some(receiver))
986        })
987        .unzip();
988    (senders, receivers, active_receivers)
989}
990
991impl RegionScanner for SeriesScan {
992    fn name(&self) -> &str {
993        "SeriesScan"
994    }
995
996    fn properties(&self) -> &ScannerProperties {
997        &self.properties
998    }
999
1000    fn schema(&self) -> SchemaRef {
1001        self.stream_ctx.input.mapper.output_schema()
1002    }
1003
1004    fn metadata(&self) -> RegionMetadataRef {
1005        self.stream_ctx.input.mapper.metadata().clone()
1006    }
1007
1008    fn scan_partition(
1009        &self,
1010        ctx: &QueryScanContext,
1011        metrics_set: &ExecutionPlanMetricsSet,
1012        partition: usize,
1013    ) -> Result<SendableRecordBatchStream, BoxedError> {
1014        self.scan_partition_impl(ctx, metrics_set, partition)
1015            .map_err(BoxedError::new)
1016    }
1017
1018    fn raw_series_ordering(&self) -> Result<SeriesOrdering, BoxedError> {
1019        // Both prepared-columnar and ordinary (legacy or two-phase) scans guarantee key/time order.
1020        Ok(SeriesOrdering::KeyThenTime)
1021    }
1022
1023    fn raw_series_schema(
1024        &self,
1025        label_source: &RawLabelSource,
1026    ) -> Result<Option<Arc<SeriesSchema>>, BoxedError> {
1027        Ok(self
1028            .prepare_raw_scan(label_source)?
1029            .map(|(preparation, _)| preparation.schema))
1030    }
1031
1032    fn raw_series_has_explicit_labels(&self, labels: &RawLabelSource) -> Result<bool, BoxedError> {
1033        let Some((mut raw, columnar)) = self.prepare_raw_scan(labels)? else {
1034            return Ok(false);
1035        };
1036        Ok(columnar
1037            && prepared::sample_merge_schema(&self.stream_ctx, &mut raw)
1038                .map_err(BoxedError::new)?
1039                .is_some())
1040    }
1041
1042    fn scan_series_partition(
1043        &self,
1044        ctx: &QueryScanContext,
1045        metrics_set: &ExecutionPlanMetricsSet,
1046        partition: usize,
1047        label_source: &RawLabelSource,
1048    ) -> Result<Option<SendableRawSeriesStream>, BoxedError> {
1049        let Some((preparation, prepared_columnar)) = self.prepare_raw_scan(label_source)? else {
1050            return Ok(None);
1051        };
1052        if prepared_columnar {
1053            let partitions = self.properties.num_partitions().max(1);
1054            if partition >= partitions {
1055                return Err(BoxedError::new(
1056                    PartitionOutOfRangeSnafu {
1057                        given: partition,
1058                        all: partitions,
1059                    }
1060                    .build(),
1061                ));
1062            }
1063            // Preflight needs the declaration before the stream is polled, while preparation
1064            // needs the original field indices and predicates to derive its own merge schema.
1065            let explicit_labels =
1066                prepared::sample_merge_schema(&self.stream_ctx, &mut preparation.clone())
1067                    .map_err(BoxedError::new)?
1068                    .is_some();
1069            let schema = preparation.schema.clone();
1070            let preparation = self
1071                .check_scan_limit()
1072                .map_err(BoxedError::new)
1073                .map(|_| Some((preparation, prepared_columnar)));
1074            let future = self.prepare_series_with_raw(ctx, metrics_set, label_source, preparation);
1075            let labels = label_source.clone();
1076            let stream = async_stream::try_stream! {
1077                let source = future.await?.ok_or_else(|| BoxedError::new(InvalidRecordBatchSnafu { reason: "columnar prepared capability disappeared" }.build()))?;
1078                let n = source.num_series() as u128;
1079                let start = (n * partition as u128 / partitions as u128) as usize;
1080                let end = (n * (partition + 1) as u128 / partitions as u128) as usize;
1081                let mut reader = source.read_series(start..end)?;
1082                while let Some(chunk) = reader.try_next().await? { yield chunk; }
1083            };
1084            return Ok(Some(raw::from_stream(
1085                schema,
1086                labels,
1087                SeriesOrdering::KeyThenTime,
1088                explicit_labels,
1089                Box::pin(stream),
1090            )));
1091        }
1092        // Stored sources without columnar artifacts (memtables, unindexed SSTs) are read by the
1093        // ordinary scan, so the raw path stays available.
1094        let metrics = new_partition_metrics(
1095            &self.stream_ctx,
1096            ctx.explain_verbose,
1097            metrics_set,
1098            partition,
1099            &self.metrics_list,
1100        );
1101        let batches = self
1102            .scan_batch_in_partition(ctx, partition, metrics.clone(), metrics_set)
1103            .map_err(BoxedError::new)?;
1104        Ok(Some(raw::make_stream(
1105            batches,
1106            preparation.schema,
1107            label_source.clone(),
1108            preparation.fields,
1109            metrics,
1110            preparation.predicates,
1111            SeriesOrdering::KeyThenTime,
1112        )))
1113    }
1114
1115    fn prepare_series(
1116        &self,
1117        ctx: &QueryScanContext,
1118        metrics_set: &ExecutionPlanMetricsSet,
1119        label_source: &RawLabelSource,
1120    ) -> futures::future::BoxFuture<
1121        'static,
1122        Result<Option<Arc<dyn store_api::region_engine::PreparedSeriesSource>>, BoxedError>,
1123    > {
1124        if !self.uses_columnar() {
1125            return Box::pin(async { Ok(None) });
1126        }
1127        let preparation = self
1128            .check_scan_limit()
1129            .map_err(BoxedError::new)
1130            .and_then(|_| self.prepare_raw_scan(label_source));
1131        self.prepare_series_with_raw(ctx, metrics_set, label_source, preparation)
1132    }
1133
1134    fn prepare(&mut self, request: PrepareRequest) -> Result<(), BoxedError> {
1135        self.prepared = Arc::new(prepared::Preparation::default());
1136        self.clear_preflight_cache();
1137        self.properties.prepare(request);
1138
1139        self.check_scan_limit().map_err(BoxedError::new)?;
1140
1141        Ok(())
1142    }
1143
1144    fn has_predicate_without_region(&self) -> bool {
1145        let predicate = self
1146            .stream_ctx
1147            .input
1148            .predicate_group()
1149            .predicate_without_region();
1150        predicate.is_some()
1151    }
1152
1153    fn add_dyn_filter_to_predicate(
1154        &mut self,
1155        filter_exprs: Vec<Arc<dyn datafusion::physical_plan::PhysicalExpr>>,
1156    ) -> Vec<bool> {
1157        let matched = self.stream_ctx.add_dyn_filter_to_predicate(filter_exprs);
1158        // The added filters change the predicate group the preflight inspects.
1159        self.clear_preflight_cache();
1160        matched
1161    }
1162
1163    fn reset_state(&mut self) {
1164        self.prepared = Arc::new(prepared::Preparation::default());
1165        self.clear_preflight_cache();
1166        self.stream_ctx.input.predicate.clear_dyn_filters();
1167        let num_workers = common_stat::get_total_cpu_cores().max(1);
1168        self.pruner = match self.mode {
1169            SeriesScanMode::Legacy => Arc::new(Pruner::new(self.stream_ctx.clone(), num_workers)),
1170            SeriesScanMode::TwoPhase => Arc::new(Pruner::new_with_options(
1171                self.stream_ctx.clone(),
1172                num_workers,
1173                PrunerOptions {
1174                    retain_builders: true,
1175                    enable_predicate_prefilter: false,
1176                },
1177            )),
1178        };
1179        self.legacy_receivers.lock().unwrap().clear();
1180        self.candidate_receivers.lock().unwrap().clear();
1181        self.metrics_list = Arc::new(PartitionMetricsList::default());
1182    }
1183
1184    fn set_logical_region(&mut self, logical_region: bool) {
1185        self.properties.set_logical_region(logical_region);
1186    }
1187
1188    fn set_query_load_region_id(&mut self, region_id: store_api::storage::RegionId) {
1189        self.properties.set_query_load_region_id(region_id);
1190    }
1191
1192    fn snapshot_sequence(&self) -> Option<u64> {
1193        self.stream_ctx.input.snapshot_sequence
1194    }
1195}
1196
1197impl DisplayAs for SeriesScan {
1198    fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
1199        write!(
1200            f,
1201            "SeriesScan: region={}, ",
1202            self.stream_ctx.input.mapper.metadata().region_id
1203        )?;
1204        match t {
1205            DisplayFormatType::Default | DisplayFormatType::TreeRender => {
1206                self.stream_ctx.format_for_explain(false, f)?;
1207            }
1208            DisplayFormatType::Verbose => {
1209                self.stream_ctx.format_for_explain(true, f)?;
1210            }
1211        }
1212        write!(f, ", \"mode\":\"{}\"", self.mode.as_str())?;
1213        if matches!(t, DisplayFormatType::Verbose) {
1214            self.metrics_list.format_verbose_metrics(f)?;
1215        }
1216        Ok(())
1217    }
1218}
1219
1220impl fmt::Debug for SeriesScan {
1221    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1222        f.debug_struct("SeriesScan")
1223            .field("mode", &self.mode)
1224            .field("num_ranges", &self.stream_ctx.ranges.len())
1225            .finish()
1226    }
1227}
1228
1229#[cfg(test)]
1230impl SeriesScan {
1231    /// Returns the input.
1232    pub(crate) fn input(&self) -> &ScanInput {
1233        &self.stream_ctx.input
1234    }
1235
1236    /// Returns the scan mode for tests.
1237    pub(crate) fn mode(&self) -> &'static str {
1238        self.mode.as_str()
1239    }
1240}
1241
1242/// The distributor scans series and distributes them to different partitions.
1243struct SeriesDistributor {
1244    /// Context for the scan stream.
1245    stream_ctx: Arc<StreamContext>,
1246    /// Semaphore for file scanning and range-level merging.
1247    range_semaphore: Option<Arc<Semaphore>>,
1248    /// Semaphore for the final merge across all range streams.
1249    /// Must be separate from `range_semaphore` to avoid deadlock: final merge tasks
1250    /// hold a permit while waiting for data from range-level merge tasks, which also
1251    /// need permits to produce data.
1252    final_merge_semaphore: Option<Arc<Semaphore>>,
1253    /// Partition ranges to scan.
1254    partitions: Vec<Vec<PartitionRange>>,
1255    /// Shared pruner for file range building.
1256    pruner: Arc<Pruner>,
1257    /// Senders of all partitions.
1258    senders: SenderList,
1259    /// Metrics set to report.
1260    /// The distributor report the metrics as an additional partition.
1261    /// This may double the scan cost of the [SeriesScan] metrics. We can
1262    /// get per-partition metrics in verbose mode to see the metrics of the
1263    /// distributor.
1264    metrics_set: ExecutionPlanMetricsSet,
1265    metrics_list: Arc<PartitionMetricsList>,
1266    /// Whether to use verbose logging and collect detailed metrics.
1267    explain_verbose: bool,
1268}
1269
1270impl SeriesDistributor {
1271    /// Executes the distributor.
1272    #[tracing::instrument(
1273        skip_all,
1274        fields(region_id = %self.stream_ctx.input.mapper.metadata().region_id)
1275    )]
1276    async fn execute(&mut self) {
1277        if let Err(e) = self.scan_partitions_flat().await {
1278            self.senders.send_error(e).await;
1279        }
1280    }
1281
1282    /// Scans all parts in flat format using FlatSeriesBatchDivider.
1283    #[tracing::instrument(
1284        skip_all,
1285        fields(region_id = %self.stream_ctx.input.mapper.metadata().region_id)
1286    )]
1287    async fn scan_partitions_flat(&mut self) -> Result<()> {
1288        // Initialize reference counts for all partition ranges.
1289        for partition_ranges in &self.partitions {
1290            self.pruner.add_partition_ranges(partition_ranges);
1291        }
1292
1293        // Create PartitionPruner covering all partitions
1294        let all_partition_ranges: Vec<_> = self.partitions.iter().flatten().cloned().collect();
1295        let needs_merge = all_partition_ranges.len() > 1;
1296        let partition_pruner = Arc::new(PartitionPruner::new(
1297            self.pruner.clone(),
1298            &all_partition_ranges,
1299        ));
1300
1301        let part_metrics = new_partition_metrics(
1302            &self.stream_ctx,
1303            self.explain_verbose,
1304            &self.metrics_set,
1305            self.partitions.len(),
1306            &self.metrics_list,
1307        );
1308        part_metrics.on_first_poll();
1309        // Start fetch time before building sources so scan cost contains
1310        // build part cost.
1311        let mut fetch_start = Instant::now();
1312
1313        // Builds one deduped stream per partition range, then merges across ranges.
1314        let build_start = Instant::now();
1315        let mut tasks = Vec::new();
1316        for partition in &self.partitions {
1317            for part_range in partition {
1318                let stream_ctx = self.stream_ctx.clone();
1319                let part_range = *part_range;
1320                let part_metrics = part_metrics.clone();
1321                let partition_pruner = partition_pruner.clone();
1322                let file_scan_semaphore = self.range_semaphore.clone();
1323                let merge_semaphore = self.range_semaphore.clone();
1324                tasks.push(common_runtime::spawn_query(async move {
1325                    SeqScan::build_flat_partition_range_read(
1326                        &stream_ctx,
1327                        &part_range,
1328                        if needs_merge {
1329                            FlatReadMode::CrossRangeMerge
1330                        } else {
1331                            FlatReadMode::Sequential
1332                        },
1333                        &part_metrics,
1334                        partition_pruner,
1335                        file_scan_semaphore,
1336                        merge_semaphore,
1337                    )
1338                    .await
1339                }));
1340            }
1341        }
1342        let mut range_streams = Vec::with_capacity(tasks.len());
1343        let mut estimated_batch_sizes = Vec::with_capacity(tasks.len());
1344        for task in tasks {
1345            let (stream, estimated_batch_size) = task.await.context(JoinSnafu)??;
1346            range_streams.push(stream);
1347            estimated_batch_sizes.push(estimated_batch_size);
1348        }
1349        let channel_size =
1350            compute_parallel_channel_size(compute_average_batch_size(estimated_batch_sizes));
1351        common_telemetry::debug!(
1352            "SeriesDistributor built {} range_streams, region: {}, build cost: {:?}, channel_size: {}",
1353            range_streams.len(),
1354            self.stream_ctx.input.region_metadata().region_id,
1355            build_start.elapsed(),
1356            channel_size,
1357        );
1358
1359        // Each partition range stream is already deduped, so skip dedup here.
1360        // Use a separate semaphore for the final merge to avoid deadlock with
1361        // range-level merge tasks that share the range_semaphore.
1362        let mut reader = SeqScan::build_flat_reader_from_sources(
1363            &self.stream_ctx,
1364            range_streams,
1365            self.final_merge_semaphore.clone(),
1366            Some(&part_metrics),
1367            true,
1368            channel_size,
1369        )
1370        .await?;
1371        let mut metrics = SeriesDistributorMetrics::default();
1372
1373        let mut divider = FlatSeriesBatchDivider::default();
1374        while let Some(record_batch) = reader.try_next().await? {
1375            metrics.scan_cost += fetch_start.elapsed();
1376            metrics.num_batches += 1;
1377            metrics.num_rows += record_batch.num_rows();
1378
1379            debug_assert!(record_batch.num_rows() > 0);
1380            if record_batch.num_rows() == 0 {
1381                fetch_start = Instant::now();
1382                continue;
1383            }
1384
1385            // Use divider to split series
1386            let divider_start = Instant::now();
1387            let series_batch = divider.push(record_batch);
1388            metrics.divider_cost += divider_start.elapsed();
1389            if let Some(series_batch) = series_batch {
1390                let yield_start = Instant::now();
1391                self.senders
1392                    .send_batch(SeriesBatch::Flat(series_batch))
1393                    .await?;
1394                metrics.yield_cost += yield_start.elapsed();
1395            }
1396            fetch_start = Instant::now();
1397        }
1398
1399        // Send any remaining batch in the divider
1400        let divider_start = Instant::now();
1401        let series_batch = divider.finish();
1402        metrics.divider_cost += divider_start.elapsed();
1403        if let Some(series_batch) = series_batch {
1404            let yield_start = Instant::now();
1405            self.senders
1406                .send_batch(SeriesBatch::Flat(series_batch))
1407                .await?;
1408            metrics.yield_cost += yield_start.elapsed();
1409        }
1410
1411        metrics.scan_cost += fetch_start.elapsed();
1412        metrics.num_series_send_full = self.senders.num_full;
1413        part_metrics.set_distributor_metrics(&metrics);
1414
1415        part_metrics.on_finish();
1416
1417        Ok(())
1418    }
1419}
1420
1421/// Discovers metric series and sends one reader assignment to each partition.
1422struct SeriesCandidateDistributor {
1423    memory_pool: Arc<dyn MemoryPool>,
1424    stream_ctx: Arc<StreamContext>,
1425    range_semaphore: Arc<Semaphore>,
1426    partitions: Vec<Vec<PartitionRange>>,
1427    pruner: Arc<Pruner>,
1428    senders: CandidateSenderList,
1429    active_receivers: Arc<AtomicUsize>,
1430    metrics_set: ExecutionPlanMetricsSet,
1431    metrics_list: Arc<PartitionMetricsList>,
1432    explain_verbose: bool,
1433}
1434
1435impl SeriesCandidateDistributor {
1436    async fn execute(&mut self) {
1437        if let Err(e) = self.distribute().await {
1438            self.send_error(e);
1439        }
1440    }
1441
1442    async fn distribute(&mut self) -> Result<()> {
1443        let part_metrics = new_partition_metrics(
1444            &self.stream_ctx,
1445            self.explain_verbose,
1446            &self.metrics_set,
1447            self.partitions.len(),
1448            &self.metrics_list,
1449        );
1450        part_metrics.on_first_poll();
1451
1452        let candidate_scanner = SeriesCandidateScanner::try_new(
1453            self.stream_ctx.clone(),
1454            self.partitions.clone(),
1455            self.pruner.clone(),
1456            self.range_semaphore.clone(),
1457            self.memory_pool.clone(),
1458            self.metrics_set.clone(),
1459            part_metrics.clone(),
1460        )?;
1461        let partition_pruner = candidate_scanner.partition_pruner();
1462        let mut candidates = candidate_scanner.build_stream().await?;
1463        let mut collector =
1464            SeriesBatchCollector::new(self.partitions.len()).context(InvalidSenderSnafu)?;
1465        let mut chunked = false;
1466        while let Some(batch) = candidates.try_next().await? {
1467            if !self.should_fetch_candidates() {
1468                part_metrics.on_finish();
1469                return Ok(());
1470            }
1471            collector.push(batch);
1472            if collector.len() >= candidate_series_assignment_threshold() {
1473                chunked = true;
1474                self.send_assignments(collector.finish(false), &partition_pruner);
1475                if !self.should_fetch_candidates() {
1476                    part_metrics.on_finish();
1477                    return Ok(());
1478                }
1479                collector =
1480                    SeriesBatchCollector::new(self.partitions.len()).context(InvalidSenderSnafu)?;
1481            }
1482        }
1483
1484        if collector.len() > 0 {
1485            self.send_assignments(collector.finish(!chunked), &partition_pruner);
1486        }
1487        part_metrics.on_finish();
1488        Ok(())
1489    }
1490
1491    fn send_assignments(
1492        &mut self,
1493        assignments: Vec<AssignedSeriesBatch>,
1494        partition_pruner: &Arc<PartitionPruner>,
1495    ) {
1496        // One round of candidate collection; a partition that receives two assignments sees two.
1497        #[cfg(test)]
1498        CANDIDATE_ASSIGNMENT_ROUNDS.fetch_add(1, Ordering::Relaxed);
1499        for (partition, assigned_series) in assignments.into_iter().enumerate() {
1500            if assigned_series.series().is_empty() {
1501                continue;
1502            }
1503            let Some(sender) = self.senders[partition].as_ref() else {
1504                continue;
1505            };
1506            let sent = sender
1507                .send(Ok(SeriesReaderInput {
1508                    assigned_series,
1509                    partition_pruner: partition_pruner.clone(),
1510                    range_semaphore: self.range_semaphore.clone(),
1511                }))
1512                .is_ok();
1513            if !sent {
1514                self.senders[partition] = None;
1515            }
1516        }
1517    }
1518
1519    fn should_fetch_candidates(&self) -> bool {
1520        self.active_receivers.load(Ordering::Relaxed) > 0
1521    }
1522
1523    fn send_error(&mut self, error: Error) {
1524        let error = Arc::new(error);
1525        for sender in self.senders.iter_mut().filter_map(Option::take) {
1526            let result = Err(error.clone()).context(ScanSeriesSnafu);
1527            let _ = sender.send(result);
1528        }
1529    }
1530}
1531
1532/// Batches of complete series.
1533///
1534/// A package may hold rows of more than one series, and every series in it is complete: no later
1535/// package of the same stream holds more rows of those series.
1536#[derive(Debug)]
1537pub enum SeriesBatch {
1538    Flat(FlatSeriesBatch),
1539}
1540
1541impl SeriesBatch {
1542    /// Returns the number of batches.
1543    pub fn num_batches(&self) -> usize {
1544        match self {
1545            SeriesBatch::Flat(flat_batch) => flat_batch.batches.len(),
1546        }
1547    }
1548
1549    /// Returns the total number of rows across all batches.
1550    pub fn num_rows(&self) -> usize {
1551        match self {
1552            SeriesBatch::Flat(flat_batch) => flat_batch.batches.iter().map(|x| x.num_rows()).sum(),
1553        }
1554    }
1555}
1556
1557/// Batches of the same series in flat format.
1558#[derive(Default, Debug)]
1559pub struct FlatSeriesBatch {
1560    pub batches: SmallVec<[RecordBatch; 4]>,
1561}
1562
1563/// List of senders.
1564struct SenderList {
1565    senders: Vec<Option<Sender<Result<SeriesBatch>>>>,
1566    /// Number of None senders.
1567    num_nones: usize,
1568    /// Index of the current partition to send.
1569    sender_idx: usize,
1570    /// Number of full senders.
1571    num_full: usize,
1572}
1573
1574impl SenderList {
1575    fn new(senders: Vec<Option<Sender<Result<SeriesBatch>>>>) -> Self {
1576        let num_nones = senders.iter().filter(|sender| sender.is_none()).count();
1577        Self {
1578            senders,
1579            num_nones,
1580            sender_idx: 0,
1581            num_full: 0,
1582        }
1583    }
1584
1585    /// Finds a partition and tries to send the batch to the partition.
1586    /// Returns None if it sends successfully.
1587    fn try_send_batch(&mut self, mut batch: SeriesBatch) -> Result<Option<SeriesBatch>> {
1588        for _ in 0..self.senders.len() {
1589            ensure!(self.num_nones < self.senders.len(), InvalidSenderSnafu);
1590
1591            let sender_idx = self.fetch_add_sender_idx();
1592            let Some(sender) = &self.senders[sender_idx] else {
1593                continue;
1594            };
1595
1596            match sender.try_send(Ok(batch)) {
1597                Ok(()) => return Ok(None),
1598                Err(TrySendError::Full(res)) => {
1599                    self.num_full += 1;
1600                    // Safety: we send Ok.
1601                    batch = res.unwrap();
1602                }
1603                Err(TrySendError::Closed(res)) => {
1604                    self.senders[sender_idx] = None;
1605                    self.num_nones += 1;
1606                    // Safety: we send Ok.
1607                    batch = res.unwrap();
1608                }
1609            }
1610        }
1611
1612        Ok(Some(batch))
1613    }
1614
1615    /// Finds a partition and sends the batch to the partition.
1616    async fn send_batch(&mut self, batch: SeriesBatch) -> Result<()> {
1617        let Some(batch) = self.try_send_batch(batch)? else {
1618            return Ok(());
1619        };
1620
1621        loop {
1622            ensure!(self.num_nones < self.senders.len(), InvalidSenderSnafu);
1623
1624            // Waits on every partition at once and sends to whichever frees a slot first. A
1625            // consumer may leave partitions unpolled for a long time, e.g. one that drains them one
1626            // after another, and waiting on such a partition would stall the scan.
1627            let closed = {
1628                let reservations = self.senders.iter().enumerate().filter_map(|(idx, sender)| {
1629                    sender
1630                        .as_ref()
1631                        .map(|sender| Box::pin(async move { (idx, sender.reserve().await) }))
1632                });
1633                let ((idx, reserved), _, _) = futures::future::select_all(reservations).await;
1634                match reserved {
1635                    Ok(permit) => {
1636                        permit.send(Ok(batch));
1637                        return Ok(());
1638                    }
1639                    Err(_) => idx,
1640                }
1641            };
1642            self.senders[closed] = None;
1643            self.num_nones += 1;
1644        }
1645    }
1646
1647    async fn send_error(&self, error: Error) {
1648        let error = Arc::new(error);
1649        for sender in self.senders.iter().flatten() {
1650            let result = Err(error.clone()).context(ScanSeriesSnafu);
1651            let _ = sender.send(result).await;
1652        }
1653    }
1654
1655    fn fetch_add_sender_idx(&mut self) -> usize {
1656        let sender_idx = self.sender_idx;
1657        self.sender_idx = (self.sender_idx + 1) % self.senders.len();
1658        sender_idx
1659    }
1660}
1661
1662fn new_partition_metrics(
1663    stream_ctx: &StreamContext,
1664    explain_verbose: bool,
1665    metrics_set: &ExecutionPlanMetricsSet,
1666    partition: usize,
1667    metrics_list: &PartitionMetricsList,
1668) -> PartitionMetrics {
1669    let metrics = PartitionMetrics::new(
1670        stream_ctx.input.mapper.metadata().region_id,
1671        partition,
1672        "SeriesScan",
1673        stream_ctx.query_start,
1674        explain_verbose,
1675        metrics_set,
1676    );
1677
1678    metrics_list.set(partition, metrics.clone());
1679    metrics
1680}
1681
1682/// A divider to split flat record batches by time series.
1683///
1684/// It only ensures rows of the same series are returned in the same [FlatSeriesBatch].
1685/// However, a [FlatSeriesBatch] may contain rows from multiple series.
1686#[derive(Default)]
1687struct FlatSeriesBatchDivider {
1688    buffer: FlatSeriesBatch,
1689}
1690
1691impl FlatSeriesBatchDivider {
1692    /// Pushes a record batch into the divider.
1693    ///
1694    /// Returns a [FlatSeriesBatch] if we ensure the batch contains all rows of the series in it.
1695    fn push(&mut self, batch: RecordBatch) -> Option<FlatSeriesBatch> {
1696        // If buffer is empty
1697        if self.buffer.batches.is_empty() {
1698            self.buffer.batches.push(batch);
1699            return None;
1700        }
1701
1702        // Gets the primary key column from the incoming batch.
1703        let pk_column_idx = primary_key_column_index(batch.num_columns());
1704        let batch_pk_column = batch.column(pk_column_idx);
1705        let batch_pk_array = batch_pk_column
1706            .as_any()
1707            .downcast_ref::<PrimaryKeyArray>()
1708            .unwrap();
1709        let batch_pk_values = batch_pk_array
1710            .values()
1711            .as_any()
1712            .downcast_ref::<BinaryArray>()
1713            .unwrap();
1714        // Gets the last primary key of the incoming batch.
1715        let batch_last_pk =
1716            primary_key_at(batch_pk_array, batch_pk_values, batch_pk_array.len() - 1);
1717        // Gets the last primary key of the buffer.
1718        // Safety: the buffer is not empty.
1719        let buffer_last_batch = self.buffer.batches.last().unwrap();
1720        let buffer_pk_column = buffer_last_batch.column(pk_column_idx);
1721        let buffer_pk_array = buffer_pk_column
1722            .as_any()
1723            .downcast_ref::<PrimaryKeyArray>()
1724            .unwrap();
1725        let buffer_pk_values = buffer_pk_array
1726            .values()
1727            .as_any()
1728            .downcast_ref::<BinaryArray>()
1729            .unwrap();
1730        let buffer_last_pk =
1731            primary_key_at(buffer_pk_array, buffer_pk_values, buffer_pk_array.len() - 1);
1732
1733        // If last primary key in the batch is the same as last primary key in the buffer.
1734        if batch_last_pk == buffer_last_pk {
1735            self.buffer.batches.push(batch);
1736            return None;
1737        }
1738        // Otherwise, the batch must have a different primary key, we find the first offset of the
1739        // changed primary key.
1740        let batch_pk_keys = batch_pk_array.keys();
1741        let pk_indices = batch_pk_keys.values();
1742        let mut change_offset = 0;
1743        for (i, &key) in pk_indices.iter().enumerate() {
1744            let batch_pk = batch_pk_values.value(key as usize);
1745
1746            if buffer_last_pk != batch_pk {
1747                change_offset = i;
1748                break;
1749            }
1750        }
1751
1752        // Splits the batch at the change offset
1753        let (first_part, remaining_part) = if change_offset > 0 {
1754            let first_part = batch.slice(0, change_offset);
1755            let remaining_part = batch.slice(change_offset, batch.num_rows() - change_offset);
1756            (Some(first_part), Some(remaining_part))
1757        } else {
1758            (None, Some(batch))
1759        };
1760
1761        // Creates the result from current buffer + first part of new batch
1762        let mut result = std::mem::take(&mut self.buffer);
1763        if let Some(first_part) = first_part {
1764            result.batches.push(first_part);
1765        }
1766
1767        // Pushes remaining part to the buffer if it exists
1768        if let Some(remaining_part) = remaining_part {
1769            self.buffer.batches.push(remaining_part);
1770        }
1771
1772        Some(result)
1773    }
1774
1775    /// Returns the final [FlatSeriesBatch].
1776    fn finish(&mut self) -> Option<FlatSeriesBatch> {
1777        if self.buffer.batches.is_empty() {
1778            None
1779        } else {
1780            Some(std::mem::take(&mut self.buffer))
1781        }
1782    }
1783}
1784
1785/// Helper function to extract primary key bytes at a specific index from [PrimaryKeyArray].
1786fn primary_key_at<'a>(
1787    primary_key: &PrimaryKeyArray,
1788    primary_key_values: &'a BinaryArray,
1789    index: usize,
1790) -> &'a [u8] {
1791    let key = primary_key.keys().value(index);
1792    primary_key_values.value(key as usize)
1793}
1794
1795#[cfg(test)]
1796mod tests {
1797    use std::sync::Arc;
1798
1799    use api::v1::OpType;
1800    use datatypes::arrow::array::{
1801        ArrayRef, BinaryDictionaryBuilder, Int64Array, StringDictionaryBuilder,
1802        TimestampMillisecondArray, UInt8Array, UInt64Array,
1803    };
1804    use datatypes::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit, UInt32Type};
1805    use datatypes::arrow::record_batch::RecordBatch;
1806
1807    use super::*;
1808    use crate::read::flat_projection::FlatProjectionMapper;
1809    use crate::read::scan_region::PredicateGroup;
1810    use crate::test_util::scheduler_util::SchedulerEnv;
1811    use crate::test_util::sst_util::sst_region_metadata_with_encoding;
1812
1813    #[tokio::test]
1814    async fn two_phase_eligibility_allows_exact_sequence_range() {
1815        let env = SchedulerEnv::new().await;
1816        let metadata = Arc::new(sst_region_metadata_with_encoding(
1817            store_api::codec::PrimaryKeyEncoding::Sparse,
1818        ));
1819        let predicate = PredicateGroup::default();
1820
1821        let eligible = ScanInput::builder(
1822            env.access_layer.clone(),
1823            FlatProjectionMapper::new(&metadata, [0]).unwrap(),
1824        )
1825        .with_predicate(predicate.clone())
1826        .build();
1827        assert!(SeriesScan::supports_two_phase(&eligible));
1828
1829        let exact_sequence = ScanInput::builder(
1830            env.access_layer.clone(),
1831            FlatProjectionMapper::new(&metadata, [0]).unwrap(),
1832        )
1833        .with_predicate(predicate)
1834        .with_sequence_range(Some(store_api::storage::SequenceRange::GtLtEq {
1835            min: 1,
1836            max: 2,
1837        }))
1838        .build();
1839        assert!(SeriesScan::supports_two_phase(&exact_sequence));
1840    }
1841
1842    #[test]
1843    fn candidate_distributor_stops_after_all_receivers_close() {
1844        let (_senders, mut receivers, active_receivers) = new_candidate_channel_list(2);
1845        assert_eq!(2, active_receivers.load(Ordering::Relaxed));
1846
1847        drop(receivers[0].take());
1848        assert_eq!(1, active_receivers.load(Ordering::Relaxed));
1849
1850        drop(receivers[1].take());
1851        assert_eq!(0, active_receivers.load(Ordering::Relaxed));
1852    }
1853
1854    #[tokio::test]
1855    async fn distributor_does_not_stall_on_partitions_read_one_after_another() {
1856        const NUM_BATCHES: usize = 2000;
1857        let (mut senders, receivers) = new_legacy_channel_list(16);
1858        let producer = tokio::spawn(async move {
1859            for _ in 0..NUM_BATCHES {
1860                senders
1861                    .send_batch(SeriesBatch::Flat(FlatSeriesBatch::default()))
1862                    .await
1863                    .unwrap();
1864            }
1865        });
1866        // Reads partition 0 to its end before touching the next one, so all other partitions stay
1867        // full while the producer still has batches to send.
1868        let consume = async {
1869            let mut received = 0;
1870            for mut receiver in receivers.into_iter().flatten() {
1871                while let Some(batch) = receiver.recv().await {
1872                    batch.unwrap();
1873                    received += 1;
1874                }
1875            }
1876            received
1877        };
1878        let received = tokio::time::timeout(std::time::Duration::from_secs(1), consume)
1879            .await
1880            .expect("the distributor stalled on partitions the consumer was not reading");
1881        producer.await.unwrap();
1882        assert_eq!(NUM_BATCHES, received);
1883    }
1884
1885    fn new_test_record_batch(
1886        primary_keys: &[&[u8]],
1887        timestamps: &[i64],
1888        sequences: &[u64],
1889        op_types: &[OpType],
1890        fields: &[u64],
1891    ) -> RecordBatch {
1892        let num_rows = timestamps.len();
1893        debug_assert_eq!(sequences.len(), num_rows);
1894        debug_assert_eq!(op_types.len(), num_rows);
1895        debug_assert_eq!(fields.len(), num_rows);
1896        debug_assert_eq!(primary_keys.len(), num_rows);
1897
1898        let columns: Vec<ArrayRef> = vec![
1899            build_test_pk_string_dict_array(primary_keys),
1900            Arc::new(Int64Array::from_iter(
1901                fields.iter().map(|v| Some(*v as i64)),
1902            )),
1903            Arc::new(TimestampMillisecondArray::from_iter_values(
1904                timestamps.iter().copied(),
1905            )),
1906            build_test_pk_array(primary_keys),
1907            Arc::new(UInt64Array::from_iter_values(sequences.iter().copied())),
1908            Arc::new(UInt8Array::from_iter_values(
1909                op_types.iter().map(|v| *v as u8),
1910            )),
1911        ];
1912
1913        RecordBatch::try_new(build_test_flat_schema(), columns).unwrap()
1914    }
1915
1916    fn build_test_pk_string_dict_array(primary_keys: &[&[u8]]) -> ArrayRef {
1917        let mut builder = StringDictionaryBuilder::<UInt32Type>::new();
1918        for &pk in primary_keys {
1919            let pk_str = std::str::from_utf8(pk).unwrap();
1920            builder.append(pk_str).unwrap();
1921        }
1922        Arc::new(builder.finish())
1923    }
1924
1925    fn build_test_pk_array(primary_keys: &[&[u8]]) -> ArrayRef {
1926        let mut builder = BinaryDictionaryBuilder::<UInt32Type>::new();
1927        for &pk in primary_keys {
1928            builder.append(pk).unwrap();
1929        }
1930        Arc::new(builder.finish())
1931    }
1932
1933    fn build_test_flat_schema() -> SchemaRef {
1934        let fields = vec![
1935            Field::new(
1936                "k0",
1937                DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
1938                false,
1939            ),
1940            Field::new("field0", DataType::Int64, true),
1941            Field::new(
1942                "ts",
1943                DataType::Timestamp(TimeUnit::Millisecond, None),
1944                false,
1945            ),
1946            Field::new(
1947                "__primary_key",
1948                DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Binary)),
1949                false,
1950            ),
1951            Field::new("__sequence", DataType::UInt64, false),
1952            Field::new("__op_type", DataType::UInt8, false),
1953        ];
1954        Arc::new(Schema::new(fields))
1955    }
1956
1957    #[test]
1958    fn test_empty_buffer_first_push() {
1959        let mut divider = FlatSeriesBatchDivider::default();
1960        let result = divider.finish();
1961        assert!(result.is_none());
1962
1963        let mut divider = FlatSeriesBatchDivider::default();
1964        let batch = new_test_record_batch(
1965            &[b"series1", b"series1"],
1966            &[1000, 2000],
1967            &[1, 2],
1968            &[OpType::Put, OpType::Put],
1969            &[10, 20],
1970        );
1971        let result = divider.push(batch);
1972        assert!(result.is_none());
1973        assert_eq!(divider.buffer.batches.len(), 1);
1974    }
1975
1976    #[test]
1977    fn test_same_series_accumulation() {
1978        let mut divider = FlatSeriesBatchDivider::default();
1979
1980        let batch1 = new_test_record_batch(
1981            &[b"series1", b"series1"],
1982            &[1000, 2000],
1983            &[1, 2],
1984            &[OpType::Put, OpType::Put],
1985            &[10, 20],
1986        );
1987
1988        let batch2 = new_test_record_batch(
1989            &[b"series1", b"series1"],
1990            &[3000, 4000],
1991            &[3, 4],
1992            &[OpType::Put, OpType::Put],
1993            &[30, 40],
1994        );
1995
1996        divider.push(batch1);
1997        let result = divider.push(batch2);
1998        assert!(result.is_none());
1999        let series_batch = divider.finish().unwrap();
2000        assert_eq!(series_batch.batches.len(), 2);
2001    }
2002
2003    #[test]
2004    fn test_series_boundary_detection() {
2005        let mut divider = FlatSeriesBatchDivider::default();
2006
2007        let batch1 = new_test_record_batch(
2008            &[b"series1", b"series1"],
2009            &[1000, 2000],
2010            &[1, 2],
2011            &[OpType::Put, OpType::Put],
2012            &[10, 20],
2013        );
2014
2015        let batch2 = new_test_record_batch(
2016            &[b"series2", b"series2"],
2017            &[3000, 4000],
2018            &[3, 4],
2019            &[OpType::Put, OpType::Put],
2020            &[30, 40],
2021        );
2022
2023        divider.push(batch1);
2024        let series_batch = divider.push(batch2).unwrap();
2025        assert_eq!(series_batch.batches.len(), 1);
2026
2027        assert_eq!(divider.buffer.batches.len(), 1);
2028    }
2029
2030    #[test]
2031    fn test_series_boundary_within_batch() {
2032        let mut divider = FlatSeriesBatchDivider::default();
2033
2034        let batch1 = new_test_record_batch(
2035            &[b"series1", b"series1"],
2036            &[1000, 2000],
2037            &[1, 2],
2038            &[OpType::Put, OpType::Put],
2039            &[10, 20],
2040        );
2041
2042        let batch2 = new_test_record_batch(
2043            &[b"series1", b"series2"],
2044            &[3000, 4000],
2045            &[3, 4],
2046            &[OpType::Put, OpType::Put],
2047            &[30, 40],
2048        );
2049
2050        divider.push(batch1);
2051        let series_batch = divider.push(batch2).unwrap();
2052        assert_eq!(series_batch.batches.len(), 2);
2053        assert_eq!(series_batch.batches[0].num_rows(), 2);
2054        assert_eq!(series_batch.batches[1].num_rows(), 1);
2055
2056        assert_eq!(divider.buffer.batches.len(), 1);
2057        assert_eq!(divider.buffer.batches[0].num_rows(), 1);
2058    }
2059
2060    #[test]
2061    fn test_series_splitting() {
2062        let mut divider = FlatSeriesBatchDivider::default();
2063
2064        let batch1 = new_test_record_batch(&[b"series1"], &[1000], &[1], &[OpType::Put], &[10]);
2065
2066        let batch2 = new_test_record_batch(
2067            &[b"series1", b"series2", b"series2", b"series3"],
2068            &[2000, 3000, 4000, 5000],
2069            &[2, 3, 4, 5],
2070            &[OpType::Put, OpType::Put, OpType::Put, OpType::Put],
2071            &[20, 30, 40, 50],
2072        );
2073
2074        divider.push(batch1);
2075        let series_batch = divider.push(batch2).unwrap();
2076        assert_eq!(series_batch.batches.len(), 2);
2077
2078        let total_rows: usize = series_batch.batches.iter().map(|b| b.num_rows()).sum();
2079        assert_eq!(total_rows, 2);
2080
2081        let final_batch = divider.finish().unwrap();
2082        assert_eq!(final_batch.batches.len(), 1);
2083        assert_eq!(final_batch.batches[0].num_rows(), 3);
2084    }
2085}