1mod prepared;
18mod raw;
19
20use std::collections::HashSet;
21use std::fmt;
22use std::sync::atomic::{AtomicUsize, Ordering};
23use std::sync::{Arc, Mutex};
24use std::time::Instant;
25
26use api::v1::SemanticType;
27use async_stream::try_stream;
28use common_error::ext::BoxedError;
29use common_query::native_histogram::is_native_histogram_value_type;
30use common_recordbatch::util::ChainedRecordBatchStream;
31use common_recordbatch::{RecordBatchStreamWrapper, SendableRecordBatchStream};
32use common_telemetry::tracing::{self, Instrument};
33use common_telemetry::{debug, warn};
34use datafusion::execution::memory_pool::MemoryPool;
35use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
36use datafusion::physical_plan::{DisplayAs, DisplayFormatType, PhysicalExpr};
37use datafusion_common::tree_node::TreeNode;
38use datafusion_expr::{Expr, Volatility};
39use datatypes::arrow::array::BinaryArray;
40use datatypes::arrow::datatypes::DataType as ArrowDataType;
41use datatypes::arrow::record_batch::RecordBatch;
42use datatypes::prelude::ConcreteDataType;
43use datatypes::schema::SchemaRef;
44use datatypes::series::SeriesOrdering;
45use datatypes::series::samples::{FieldSpec, SeriesSchema};
46use datatypes::types::TimestampType;
47use futures::{StreamExt, TryStreamExt};
48use smallvec::SmallVec;
49use snafu::{OptionExt, ResultExt, ensure};
50use store_api::metadata::RegionMetadataRef;
51use store_api::metric_engine_consts::is_metric_engine_internal_column;
52use store_api::region_engine::{
53 PartitionRange, PrepareRequest, QueryScanContext, RawLabelSource, RegionScanner,
54 ScannerProperties, SendableRawSeriesStream,
55};
56use store_api::storage::consts::{ReservedColumnId, is_internal_column};
57use table::predicate::Predicate;
58use tokio::sync::Semaphore;
59use tokio::sync::mpsc::error::TrySendError;
60use tokio::sync::mpsc::{self, Receiver, Sender};
61
62use crate::error::{
63 Error, InvalidRecordBatchSnafu, InvalidSenderSnafu, JoinSnafu, PartitionOutOfRangeSnafu,
64 Result, ScanMultiTimesSnafu, ScanSeriesSnafu, TooManyFilesToReadSnafu,
65};
66use crate::metrics::SCAN_MAX_CONCURRENT_FILES;
67use crate::read::ScannerMetrics;
68use crate::read::pruner::{PartitionPruner, Pruner, PrunerOptions};
69use crate::read::scan_region::{ScanInput, StreamContext};
70use crate::read::scan_util::{
71 PartitionMetrics, PartitionMetricsList, SeriesDistributorMetrics, compute_average_batch_size,
72 compute_parallel_channel_size,
73};
74use crate::read::seq_scan::{FlatReadMode, SeqScan};
75use crate::read::series_candidate::{SeriesCandidateScanner, is_sparse_metric_metadata};
76use crate::read::series_reader::{AssignedSeriesBatch, SeriesBatchCollector, SeriesReader};
77use crate::read::stream::{ConvertBatchStream, ScanBatch, ScanBatchStream};
78use crate::sst::parquet::flat_format::primary_key_column_index;
79use crate::sst::parquet::format::PrimaryKeyArray;
80
81const CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD: usize = 1_000_000;
83
84#[cfg(test)]
90static CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD_OVERRIDE: AtomicUsize = AtomicUsize::new(0);
91
92#[cfg(test)]
97static CANDIDATE_ASSIGNMENT_ROUNDS: AtomicUsize = AtomicUsize::new(0);
98
99fn candidate_series_assignment_threshold() -> usize {
101 #[cfg(test)]
102 {
103 let threshold = CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD_OVERRIDE.load(Ordering::Relaxed);
104 if threshold > 0 {
105 return threshold;
106 }
107 }
108 CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD
109}
110
111#[cfg(test)]
116pub(crate) fn set_candidate_series_assignment_threshold(threshold: usize) {
117 CANDIDATE_SERIES_ASSIGNMENT_THRESHOLD_OVERRIDE.store(threshold, Ordering::Relaxed);
118 CANDIDATE_ASSIGNMENT_ROUNDS.store(0, Ordering::Relaxed);
119}
120
121#[cfg(test)]
123pub(crate) fn candidate_assignment_rounds() -> usize {
124 CANDIDATE_ASSIGNMENT_ROUNDS.load(Ordering::Relaxed)
125}
126
127type LegacyReceiverList = Vec<Option<Receiver<Result<SeriesBatch>>>>;
129
130type CandidateReceiverList = Vec<Option<CandidateReceiver>>;
132
133type CandidateSenderList = Vec<Option<mpsc::UnboundedSender<Result<SeriesReaderInput>>>>;
135
136#[derive(Clone)]
137struct RawScanPreparation {
138 schema: Arc<SeriesSchema>,
139 fields: Vec<(usize, ConcreteDataType)>,
140 predicates: Vec<Arc<dyn PhysicalExpr>>,
141}
142
143type PreflightCache = Option<(RawLabelSource, Option<(RawScanPreparation, bool)>)>;
148
149struct CandidateReceiver {
150 receiver: mpsc::UnboundedReceiver<Result<SeriesReaderInput>>,
151 active_receivers: Arc<AtomicUsize>,
152}
153
154impl CandidateReceiver {
155 async fn recv(&mut self) -> Option<Result<SeriesReaderInput>> {
156 self.receiver.recv().await
157 }
158}
159
160impl Drop for CandidateReceiver {
161 fn drop(&mut self) {
162 let previous = self.active_receivers.fetch_sub(1, Ordering::Relaxed);
163 debug_assert!(previous > 0);
164 }
165}
166
167struct SeriesReaderInput {
169 assigned_series: AssignedSeriesBatch,
170 partition_pruner: Arc<PartitionPruner>,
171 range_semaphore: Arc<Semaphore>,
172}
173
174#[derive(Debug, Clone, Copy, PartialEq, Eq)]
175enum SeriesScanMode {
176 Legacy,
177 TwoPhase,
178}
179
180impl SeriesScanMode {
181 fn as_str(self) -> &'static str {
182 match self {
183 Self::Legacy => "legacy",
184 Self::TwoPhase => "two_phase",
185 }
186 }
187}
188
189pub struct SeriesScan {
195 mode: SeriesScanMode,
197 prepared: Arc<prepared::Preparation>,
198 properties: ScannerProperties,
200 stream_ctx: Arc<StreamContext>,
202 pruner: Arc<Pruner>,
204 legacy_receivers: Mutex<LegacyReceiverList>,
206 candidate_receivers: Mutex<CandidateReceiverList>,
208 metrics_list: Arc<PartitionMetricsList>,
211 preflight: Mutex<PreflightCache>,
218 #[cfg(test)]
223 preflight_runs: AtomicUsize,
224}
225
226impl SeriesScan {
227 pub(crate) fn new(input: ScanInput, experimental_series_scan_v2: bool) -> Self {
229 let mode = if experimental_series_scan_v2 && Self::supports_two_phase(&input) {
230 SeriesScanMode::TwoPhase
231 } else {
232 SeriesScanMode::Legacy
233 };
234 let mut properties = ScannerProperties::default()
235 .with_append_mode(input.append_mode)
236 .with_total_rows(input.total_rows())
237 .with_total_rows_is_exact(input.append_mode && input.total_rows_is_exact());
238 if let Some(counters) = input.query_stat_counters.clone() {
239 properties.set_query_stat_counters(counters);
240 }
241 let stream_ctx = Arc::new(StreamContext::seq_scan_ctx(input));
242 properties.partitions = vec![stream_ctx.partition_ranges()];
243
244 let num_workers = common_stat::get_total_cpu_cores().max(1);
246 let pruner = match mode {
247 SeriesScanMode::Legacy => Arc::new(Pruner::new(stream_ctx.clone(), num_workers)),
248 SeriesScanMode::TwoPhase => Arc::new(Pruner::new_with_options(
249 stream_ctx.clone(),
250 num_workers,
251 PrunerOptions {
252 retain_builders: true,
253 enable_predicate_prefilter: false,
254 },
255 )),
256 };
257
258 Self {
259 mode,
260 prepared: Arc::new(prepared::Preparation::default()),
261 properties,
262 stream_ctx,
263 pruner,
264 legacy_receivers: Mutex::new(Vec::new()),
265 candidate_receivers: Mutex::new(Vec::new()),
266 metrics_list: Arc::new(PartitionMetricsList::default()),
267 preflight: Mutex::new(None),
268 #[cfg(test)]
269 preflight_runs: AtomicUsize::new(0),
270 }
271 }
272
273 #[cfg(test)]
278 pub(crate) fn preflight_runs(&self) -> usize {
279 self.preflight_runs.load(Ordering::Relaxed)
280 }
281
282 fn clear_preflight_cache(&self) {
287 *self.preflight.lock().unwrap() = None;
288 }
289
290 fn cached_preflight(
296 &self,
297 label_source: &RawLabelSource,
298 ) -> Option<Option<(RawScanPreparation, bool)>> {
299 let cache = self.preflight.lock().unwrap();
300 let (cached_source, cached) = cache.as_ref()?;
301 if !prepared::same_labels(cached_source, label_source) {
302 return None;
303 }
304 Some(cached.clone())
305 }
306
307 fn uses_columnar(&self) -> bool {
308 self.stream_ctx
309 .input
310 .series_index
311 .as_ref()
312 .is_some_and(|context| context.format == crate::config::SeriesIndexFormat::Columnar)
313 }
314
315 fn columnar_fields_supported(&self) -> bool {
322 let metadata = self.stream_ctx.input.mapper.metadata();
323 let selected = self.stream_ctx.input.mapper.output_schema();
324 selected
325 .column_schemas()
326 .iter()
327 .filter(|output| {
328 metadata.column_metadatas.iter().any(|column| {
329 column.column_schema.name == output.name
330 && column.semantic_type == SemanticType::Field
331 })
332 })
333 .all(|output| output.data_type == ConcreteDataType::float64_datatype())
334 }
335
336 fn uses_prepared_columnar(&self) -> Result<bool, BoxedError> {
342 if !self.uses_columnar() {
343 return Ok(false);
344 }
345 let ranges = self
346 .properties
347 .partitions
348 .iter()
349 .flatten()
350 .copied()
351 .collect::<Vec<_>>();
352 prepared::validate_inputs(&self.stream_ctx, &ranges).map_err(BoxedError::new)
353 }
354
355 fn prepare_raw_scan(
362 &self,
363 label_source: &RawLabelSource,
364 ) -> Result<Option<(RawScanPreparation, bool)>, BoxedError> {
365 if let Some(cached) = self.cached_preflight(label_source) {
366 return Ok(cached);
367 }
368 let computed = self.prepare_raw_scan_uncached(label_source)?;
369 *self.preflight.lock().unwrap() = Some((label_source.clone(), computed.clone()));
373 Ok(computed)
374 }
375
376 fn prepare_raw_scan_uncached(
378 &self,
379 label_source: &RawLabelSource,
380 ) -> Result<Option<(RawScanPreparation, bool)>, BoxedError> {
381 #[cfg(test)]
382 self.preflight_runs.fetch_add(1, Ordering::Relaxed);
383 let input = &self.stream_ctx.input;
384 let metadata = input.mapper.metadata();
385 match label_source {
386 RawLabelSource::PrimaryKey {
387 physical_metadata,
388 logical_tags,
389 ..
390 } => {
391 if !Arc::ptr_eq(metadata, physical_metadata) {
392 debug!(
393 "Raw series scan preflight rejected metadata snapshot: region_id={}, physical_region_id={}, schema_version={}, physical_schema_version={}",
394 metadata.region_id,
395 physical_metadata.region_id,
396 metadata.schema_version,
397 physical_metadata.schema_version,
398 );
399 return Err(BoxedError::new(
400 InvalidRecordBatchSnafu {
401 reason: "raw label source metadata does not match scanner snapshot"
402 .to_string(),
403 }
404 .build(),
405 ));
406 }
407 let mut ids = HashSet::new();
408 let mut names = HashSet::new();
409 for tag in logical_tags.iter() {
410 let column = metadata.column_by_id(tag.column_id).filter(|column| {
411 column.semantic_type == SemanticType::Tag
412 && metadata.primary_key.contains(&tag.column_id)
413 });
414 let valid = column.is_some_and(|column| {
415 column.column_schema.name == tag.name.as_ref()
416 && tag.name.as_ref() != "__name__"
417 && !is_internal_column(tag.name.as_ref())
418 && !is_metric_engine_internal_column(tag.name.as_ref())
419 && !ReservedColumnId::is_reserved(tag.column_id)
420 });
421 let unique_id = valid && ids.insert(tag.column_id);
422 let unique_name = unique_id && names.insert(tag.name.as_ref());
423 if !valid || !unique_id || !unique_name {
424 debug!(
425 "Raw series scan preflight rejected label binding: region_id={}, column_id={}, binding_valid={}, unique_id={}, unique_name={}",
426 metadata.region_id, tag.column_id, valid, unique_id, unique_name,
427 );
428 return Err(BoxedError::new(
429 InvalidRecordBatchSnafu {
430 reason: "invalid raw label binding".to_string(),
431 }
432 .build(),
433 ));
434 }
435 }
436 }
437 RawLabelSource::Explicit => {
438 debug!(
439 "Raw series scan preflight rejected explicit labels: region_id={}",
440 metadata.region_id,
441 );
442 return Ok(None);
443 }
444 }
445 if self.uses_columnar() && !is_sparse_metric_metadata(metadata) {
446 return Ok(None);
447 }
448 let selected = input.mapper.output_schema();
449 let mut fields = Vec::new();
450 let mut specs = Vec::new();
451 let columnar_fields_supported = self.columnar_fields_supported();
457 for output in selected.column_schemas() {
458 let Some(column) = metadata.column_metadatas.iter().find(|column| {
459 column.column_schema.name == output.name
460 && column.semantic_type == SemanticType::Field
461 }) else {
462 continue;
463 };
464 let Some((index, (_, data_type))) = input
465 .mapper
466 .batch_schema()
467 .iter()
468 .enumerate()
469 .find(|(_, (id, _))| *id == column.column_id)
470 else {
471 debug!(
472 "Raw series scan preflight missing mapper field: region_id={}, column_id={}",
473 metadata.region_id, column.column_id,
474 );
475 return Err(BoxedError::new(
476 InvalidRecordBatchSnafu {
477 reason: format!(
478 "selected field {} is missing from mapper batch schema",
479 output.name
480 ),
481 }
482 .build(),
483 ));
484 };
485 if !data_type.is_numeric() && !is_native_histogram_value_type(data_type) {
488 debug!(
489 "Raw series scan preflight rejected field type: region_id={}, column_id={}",
490 metadata.region_id, column.column_id,
491 );
492 return Ok(None);
493 }
494 fields.push((index, data_type.clone()));
495 specs.push(FieldSpec {
496 name: Arc::from(output.name.as_str()),
497 });
498 }
499 let time_is_millis = matches!(metadata.time_index_type(), TimestampType::Millisecond(_));
500 if fields.is_empty()
501 || input.append_mode
502 || input.series_row_selector.is_some()
503 || !time_is_millis
504 {
505 debug!(
506 "Raw series scan preflight rejected scan options: region_id={}, selected_fields={}, append_mode={}, row_selector={}, time_is_millis={}",
507 metadata.region_id,
508 fields.len(),
509 input.append_mode,
510 input.series_row_selector.is_some(),
511 time_is_millis,
512 );
513 return Ok(None);
514 }
515 let prepared_columnar = self.uses_prepared_columnar()? && columnar_fields_supported;
518 let schema = Arc::new(SeriesSchema {
519 fields: specs.into(),
520 eval_grid: None,
521 });
522 let typed_schema = input.mapper.input_arrow_schema(false);
523 let mut predicates = Vec::new();
524 let predicate = if prepared_columnar {
525 input.predicate_group().predicate()
526 } else {
527 input.predicate_group().predicate_without_region()
529 };
530 if let Some(predicate) = predicate {
531 if !predicate.dyn_filters().is_empty() {
532 debug!(
533 "Raw series scan preflight rejected dynamic predicate filters: region_id={}",
534 metadata.region_id,
535 );
536 return Ok(None);
537 }
538 for expr in predicate.exprs() {
539 if expr
541 .exists(|node| {
542 Ok(matches!(node, Expr::ScalarFunction(function)
543 if function.func.signature().volatility != Volatility::Immutable)
544 || matches!(node, Expr::ScalarVariable(..)))
545 })
546 .unwrap_or(true)
547 {
548 debug!(
549 "Raw series scan preflight rejected predicate volatility or variable: region_id={}",
550 metadata.region_id,
551 );
552 return Ok(None);
553 }
554 let Ok(physical) = Predicate::to_physical_expr(expr, &typed_schema) else {
556 debug!(
557 "Raw series scan preflight failed predicate compilation: region_id={}",
558 metadata.region_id,
559 );
560 return Ok(None);
561 };
562 if !matches!(
563 physical.data_type(&typed_schema),
564 Ok(ArrowDataType::Boolean | ArrowDataType::Null)
565 ) {
566 debug!(
567 "Raw series scan preflight rejected predicate type: region_id={}",
568 metadata.region_id,
569 );
570 return Ok(None);
571 }
572 predicates.push(physical);
573 }
574 }
575 Ok(Some((
576 RawScanPreparation {
577 schema,
578 fields,
579 predicates,
580 },
581 prepared_columnar,
582 )))
583 }
584
585 fn prepare_series_with_raw(
587 &self,
588 ctx: &QueryScanContext,
589 metrics_set: &ExecutionPlanMetricsSet,
590 label_source: &RawLabelSource,
591 preparation: Result<Option<(RawScanPreparation, bool)>, BoxedError>,
592 ) -> futures::future::BoxFuture<
593 'static,
594 Result<Option<Arc<dyn store_api::region_engine::PreparedSeriesSource>>, BoxedError>,
595 > {
596 let state = self.prepared.clone();
597 let stream_ctx = self.stream_ctx.clone();
598 let ranges = self
599 .properties
600 .partitions
601 .iter()
602 .flatten()
603 .copied()
604 .collect();
605 let labels = label_source.clone();
606 let metrics = new_partition_metrics(
607 &self.stream_ctx,
608 ctx.explain_verbose,
609 metrics_set,
610 0,
611 &self.metrics_list,
612 );
613 Box::pin(async move {
614 let Some((preparation, prepared_columnar)) = preparation? else {
615 return Ok(None);
616 };
617 if !prepared_columnar {
618 return Ok(None);
620 }
621 state
622 .prepare(stream_ctx, ranges, labels, preparation, metrics)
623 .await
624 .map(Some)
625 })
626 }
627
628 fn supports_two_phase(input: &ScanInput) -> bool {
629 if !is_sparse_metric_metadata(input.region_metadata()) {
630 return false;
631 }
632 #[cfg(feature = "enterprise")]
633 if !input.extension_ranges().is_empty() {
634 return false;
635 }
636 true
637 }
638
639 #[tracing::instrument(
640 skip_all,
641 fields(
642 region_id = %self.stream_ctx.input.mapper.metadata().region_id,
643 partition = partition
644 )
645 )]
646 fn scan_partition_impl(
647 &self,
648 ctx: &QueryScanContext,
649 metrics_set: &ExecutionPlanMetricsSet,
650 partition: usize,
651 ) -> Result<SendableRecordBatchStream> {
652 let metrics = new_partition_metrics(
653 &self.stream_ctx,
654 ctx.explain_verbose,
655 metrics_set,
656 partition,
657 &self.metrics_list,
658 );
659
660 let batch_stream =
661 self.scan_batch_in_partition(ctx, partition, metrics.clone(), metrics_set)?;
662
663 let input = &self.stream_ctx.input;
664 let record_batch_stream = ConvertBatchStream::new(
665 batch_stream,
666 input.mapper.clone(),
667 input.cache_strategy.clone(),
668 metrics,
669 );
670
671 Ok(Box::pin(RecordBatchStreamWrapper::new(
672 input.mapper.output_schema(),
673 Box::pin(record_batch_stream),
674 )))
675 }
676
677 #[tracing::instrument(
678 skip_all,
679 fields(
680 region_id = %self.stream_ctx.input.mapper.metadata().region_id,
681 partition = partition
682 )
683 )]
684 fn scan_batch_in_partition(
685 &self,
686 ctx: &QueryScanContext,
687 partition: usize,
688 part_metrics: PartitionMetrics,
689 metrics_set: &ExecutionPlanMetricsSet,
690 ) -> Result<ScanBatchStream> {
691 if ctx.explain_verbose {
692 common_telemetry::info!(
693 "SeriesScan partition {}, region_id: {}",
694 partition,
695 self.stream_ctx.input.region_metadata().region_id
696 );
697 }
698
699 ensure!(
700 partition < self.properties.num_partitions(),
701 PartitionOutOfRangeSnafu {
702 given: partition,
703 all: self.properties.num_partitions(),
704 }
705 );
706
707 match self.mode {
708 SeriesScanMode::Legacy => self.scan_legacy_batch_in_partition(
709 partition,
710 part_metrics,
711 metrics_set,
712 ctx.explain_verbose,
713 ),
714 SeriesScanMode::TwoPhase => {
715 self.scan_two_phase_batch_in_partition(ctx, partition, part_metrics, metrics_set)
716 }
717 }
718 }
719
720 fn scan_legacy_batch_in_partition(
721 &self,
722 partition: usize,
723 part_metrics: PartitionMetrics,
724 metrics_set: &ExecutionPlanMetricsSet,
725 explain_verbose: bool,
726 ) -> Result<ScanBatchStream> {
727 self.maybe_start_legacy_distributor(metrics_set, explain_verbose);
728 let mut receiver = self.legacy_receivers.lock().unwrap()[partition]
729 .take()
730 .context(ScanMultiTimesSnafu { partition })?;
731 let stream = try_stream! {
732 part_metrics.on_first_poll();
733
734 let mut fetch_start = Instant::now();
735 let mut metrics = ScannerMetrics::default();
736 while let Some(series) = receiver.recv().await {
737 let series = series?;
738 metrics.scan_cost += fetch_start.elapsed();
739 metrics.num_batches += series.num_batches();
740 metrics.num_rows += series.num_rows();
741
742 let yield_start = Instant::now();
743 yield ScanBatch::Series(series);
744 metrics.yield_cost += yield_start.elapsed();
745 fetch_start = Instant::now();
746 }
747
748 part_metrics.merge_metrics(&metrics);
749 part_metrics.on_finish();
750 };
751 Ok(Box::pin(stream))
752 }
753
754 fn scan_two_phase_batch_in_partition(
755 &self,
756 ctx: &QueryScanContext,
757 partition: usize,
758 part_metrics: PartitionMetrics,
759 metrics_set: &ExecutionPlanMetricsSet,
760 ) -> Result<ScanBatchStream> {
761 self.maybe_start_candidate_distributor(ctx, metrics_set);
762 let mut receiver = self.candidate_receivers.lock().unwrap()[partition]
765 .take()
766 .context(ScanMultiTimesSnafu { partition })?;
767 let stream_ctx = self.stream_ctx.clone();
768 let partition_ranges = self
769 .properties
770 .partitions
771 .iter()
772 .flatten()
773 .copied()
774 .collect::<Vec<_>>();
775 let stream = try_stream! {
776 part_metrics.on_first_poll();
777
778 let mut fetch_start = Instant::now();
779 let mut metrics = ScannerMetrics::default();
780 while let Some(input) = receiver.recv().await {
781 let input = input?;
782 metrics.scan_cost += fetch_start.elapsed();
783
784 let build_start = Instant::now();
785 let reader = SeriesReader::try_new(
786 stream_ctx.clone(),
787 partition_ranges.clone(),
788 input.assigned_series,
789 input.partition_pruner,
790 input.range_semaphore,
791 part_metrics.clone(),
792 )?;
793 let mut reader_stream = reader.build_stream().await?;
794 metrics.scan_cost += build_start.elapsed();
795 fetch_start = Instant::now();
796 while let Some(record_batch) = reader_stream.try_next().await? {
797 metrics.scan_cost += fetch_start.elapsed();
798 metrics.num_batches += 1;
799 metrics.num_rows += record_batch.num_rows();
800
801 let yield_start = Instant::now();
802 yield ScanBatch::RecordBatch(record_batch);
803 metrics.yield_cost += yield_start.elapsed();
804 fetch_start = Instant::now();
805 }
806 }
807 metrics.scan_cost += fetch_start.elapsed();
808
809 part_metrics.merge_metrics(&metrics);
810 part_metrics.on_finish();
811 };
812 Ok(Box::pin(stream))
813 }
814
815 fn maybe_start_legacy_distributor(
816 &self,
817 metrics_set: &ExecutionPlanMetricsSet,
818 explain_verbose: bool,
819 ) {
820 let mut rx_list = self.legacy_receivers.lock().unwrap();
821 if !rx_list.is_empty() {
822 return;
823 }
824
825 let (senders, receivers) = new_legacy_channel_list(self.properties.num_partitions());
826 let mut distributor = SeriesDistributor {
827 stream_ctx: self.stream_ctx.clone(),
828 range_semaphore: Some(Arc::new(Semaphore::new(self.properties.num_partitions()))),
829 final_merge_semaphore: Some(Arc::new(Semaphore::new(self.properties.num_partitions()))),
830 partitions: self.properties.partitions.clone(),
831 pruner: self.pruner.clone(),
832 senders,
833 metrics_set: metrics_set.clone(),
834 metrics_list: self.metrics_list.clone(),
835 explain_verbose,
836 };
837 let region_id = distributor.stream_ctx.input.mapper.metadata().region_id;
838 let span = tracing::info_span!("SeriesScan::distributor", region_id = %region_id);
839 common_runtime::spawn_query(
840 async move {
841 distributor.execute().await;
842 }
843 .instrument(span),
844 );
845
846 *rx_list = receivers;
847 }
848
849 fn maybe_start_candidate_distributor(
850 &self,
851 ctx: &QueryScanContext,
852 metrics_set: &ExecutionPlanMetricsSet,
853 ) {
854 let mut rx_list = self.candidate_receivers.lock().unwrap();
855 if !rx_list.is_empty() {
856 return;
857 }
858
859 let (senders, receivers, active_receivers) =
860 new_candidate_channel_list(self.properties.num_partitions());
861 let mut distributor = SeriesCandidateDistributor {
864 memory_pool: ctx.memory_pool.clone(),
865 stream_ctx: self.stream_ctx.clone(),
866 range_semaphore: Arc::new(Semaphore::new(self.properties.num_partitions())),
867 partitions: self.properties.partitions.clone(),
868 pruner: self.pruner.clone(),
869 senders,
870 active_receivers,
871 metrics_set: metrics_set.clone(),
872 metrics_list: self.metrics_list.clone(),
873 explain_verbose: ctx.explain_verbose,
874 };
875 let region_id = distributor.stream_ctx.input.mapper.metadata().region_id;
876 let span = tracing::info_span!("SeriesScan::candidate_distributor", region_id = %region_id);
877 common_runtime::spawn_query(
878 async move {
879 distributor.execute().await;
880 }
881 .instrument(span),
882 );
883
884 *rx_list = receivers;
885 }
886
887 #[tracing::instrument(
889 skip_all,
890 fields(region_id = %self.stream_ctx.input.mapper.metadata().region_id)
891 )]
892 pub(crate) async fn build_stream(&self) -> Result<SendableRecordBatchStream, BoxedError> {
893 let part_num = self.properties.num_partitions();
894 let metrics_set = ExecutionPlanMetricsSet::default();
895 let ctx = QueryScanContext::default();
896 let streams = (0..part_num)
897 .map(|i| self.scan_partition(&ctx, &metrics_set, i))
898 .collect::<Result<Vec<_>, BoxedError>>()?;
899 let chained_stream = ChainedRecordBatchStream::new(streams).map_err(BoxedError::new)?;
900 Ok(Box::pin(chained_stream))
901 }
902
903 pub(crate) fn scan_all_partitions(&self) -> Result<ScanBatchStream> {
905 let metrics_set = ExecutionPlanMetricsSet::new();
906
907 let ctx = QueryScanContext::default();
908 let streams = (0..self.properties.partitions.len())
909 .map(|partition| {
910 let metrics = new_partition_metrics(
911 &self.stream_ctx,
912 false,
913 &metrics_set,
914 partition,
915 &self.metrics_list,
916 );
917
918 self.scan_batch_in_partition(&ctx, partition, metrics, &metrics_set)
919 })
920 .collect::<Result<Vec<_>>>()?;
921
922 Ok(Box::pin(futures::stream::iter(streams).flatten()))
923 }
924
925 pub(crate) fn check_scan_limit(&self) -> Result<()> {
927 let total_files: usize = self
929 .properties
930 .partitions
931 .iter()
932 .flat_map(|partition| partition.iter())
933 .map(|part_range| {
934 let range_meta = &self.stream_ctx.ranges[part_range.identifier];
935 range_meta.indices.len()
936 })
937 .sum();
938
939 if !self.stream_ctx.input.compaction {
940 SCAN_MAX_CONCURRENT_FILES
941 .with_label_values(&["query"])
942 .observe(total_files as f64);
943 }
944 if self.stream_ctx.input.uses_scan_memory_limit() {
945 return Ok(());
946 }
947
948 let max_concurrent_files = self.stream_ctx.input.max_concurrent_scan_files;
949 if total_files > max_concurrent_files {
950 return TooManyFilesToReadSnafu {
951 actual: total_files,
952 max: max_concurrent_files,
953 }
954 .fail();
955 }
956
957 Ok(())
958 }
959}
960
961fn new_legacy_channel_list(num_partitions: usize) -> (SenderList, LegacyReceiverList) {
962 let (senders, receivers): (Vec<_>, Vec<_>) = (0..num_partitions)
963 .map(|_| {
964 let (sender, receiver) = mpsc::channel(1);
965 (Some(sender), Some(receiver))
966 })
967 .unzip();
968 (SenderList::new(senders), receivers)
969}
970
971fn new_candidate_channel_list(
972 num_partitions: usize,
973) -> (CandidateSenderList, CandidateReceiverList, Arc<AtomicUsize>) {
974 let active_receivers = Arc::new(AtomicUsize::new(num_partitions));
975 let (senders, receivers) = (0..num_partitions)
976 .map(|_| {
977 let (sender, receiver) = mpsc::unbounded_channel();
981 let receiver = CandidateReceiver {
982 receiver,
983 active_receivers: active_receivers.clone(),
984 };
985 (Some(sender), Some(receiver))
986 })
987 .unzip();
988 (senders, receivers, active_receivers)
989}
990
991impl RegionScanner for SeriesScan {
992 fn name(&self) -> &str {
993 "SeriesScan"
994 }
995
996 fn properties(&self) -> &ScannerProperties {
997 &self.properties
998 }
999
1000 fn schema(&self) -> SchemaRef {
1001 self.stream_ctx.input.mapper.output_schema()
1002 }
1003
1004 fn metadata(&self) -> RegionMetadataRef {
1005 self.stream_ctx.input.mapper.metadata().clone()
1006 }
1007
1008 fn scan_partition(
1009 &self,
1010 ctx: &QueryScanContext,
1011 metrics_set: &ExecutionPlanMetricsSet,
1012 partition: usize,
1013 ) -> Result<SendableRecordBatchStream, BoxedError> {
1014 self.scan_partition_impl(ctx, metrics_set, partition)
1015 .map_err(BoxedError::new)
1016 }
1017
1018 fn raw_series_ordering(&self) -> Result<SeriesOrdering, BoxedError> {
1019 Ok(SeriesOrdering::KeyThenTime)
1021 }
1022
1023 fn raw_series_schema(
1024 &self,
1025 label_source: &RawLabelSource,
1026 ) -> Result<Option<Arc<SeriesSchema>>, BoxedError> {
1027 Ok(self
1028 .prepare_raw_scan(label_source)?
1029 .map(|(preparation, _)| preparation.schema))
1030 }
1031
1032 fn raw_series_has_explicit_labels(&self, labels: &RawLabelSource) -> Result<bool, BoxedError> {
1033 let Some((mut raw, columnar)) = self.prepare_raw_scan(labels)? else {
1034 return Ok(false);
1035 };
1036 Ok(columnar
1037 && prepared::sample_merge_schema(&self.stream_ctx, &mut raw)
1038 .map_err(BoxedError::new)?
1039 .is_some())
1040 }
1041
1042 fn scan_series_partition(
1043 &self,
1044 ctx: &QueryScanContext,
1045 metrics_set: &ExecutionPlanMetricsSet,
1046 partition: usize,
1047 label_source: &RawLabelSource,
1048 ) -> Result<Option<SendableRawSeriesStream>, BoxedError> {
1049 let Some((preparation, prepared_columnar)) = self.prepare_raw_scan(label_source)? else {
1050 return Ok(None);
1051 };
1052 if prepared_columnar {
1053 let partitions = self.properties.num_partitions().max(1);
1054 if partition >= partitions {
1055 return Err(BoxedError::new(
1056 PartitionOutOfRangeSnafu {
1057 given: partition,
1058 all: partitions,
1059 }
1060 .build(),
1061 ));
1062 }
1063 let explicit_labels =
1066 prepared::sample_merge_schema(&self.stream_ctx, &mut preparation.clone())
1067 .map_err(BoxedError::new)?
1068 .is_some();
1069 let schema = preparation.schema.clone();
1070 let preparation = self
1071 .check_scan_limit()
1072 .map_err(BoxedError::new)
1073 .map(|_| Some((preparation, prepared_columnar)));
1074 let future = self.prepare_series_with_raw(ctx, metrics_set, label_source, preparation);
1075 let labels = label_source.clone();
1076 let stream = async_stream::try_stream! {
1077 let source = future.await?.ok_or_else(|| BoxedError::new(InvalidRecordBatchSnafu { reason: "columnar prepared capability disappeared" }.build()))?;
1078 let n = source.num_series() as u128;
1079 let start = (n * partition as u128 / partitions as u128) as usize;
1080 let end = (n * (partition + 1) as u128 / partitions as u128) as usize;
1081 let mut reader = source.read_series(start..end)?;
1082 while let Some(chunk) = reader.try_next().await? { yield chunk; }
1083 };
1084 return Ok(Some(raw::from_stream(
1085 schema,
1086 labels,
1087 SeriesOrdering::KeyThenTime,
1088 explicit_labels,
1089 Box::pin(stream),
1090 )));
1091 }
1092 let metrics = new_partition_metrics(
1095 &self.stream_ctx,
1096 ctx.explain_verbose,
1097 metrics_set,
1098 partition,
1099 &self.metrics_list,
1100 );
1101 let batches = self
1102 .scan_batch_in_partition(ctx, partition, metrics.clone(), metrics_set)
1103 .map_err(BoxedError::new)?;
1104 Ok(Some(raw::make_stream(
1105 batches,
1106 preparation.schema,
1107 label_source.clone(),
1108 preparation.fields,
1109 metrics,
1110 preparation.predicates,
1111 SeriesOrdering::KeyThenTime,
1112 )))
1113 }
1114
1115 fn prepare_series(
1116 &self,
1117 ctx: &QueryScanContext,
1118 metrics_set: &ExecutionPlanMetricsSet,
1119 label_source: &RawLabelSource,
1120 ) -> futures::future::BoxFuture<
1121 'static,
1122 Result<Option<Arc<dyn store_api::region_engine::PreparedSeriesSource>>, BoxedError>,
1123 > {
1124 if !self.uses_columnar() {
1125 return Box::pin(async { Ok(None) });
1126 }
1127 let preparation = self
1128 .check_scan_limit()
1129 .map_err(BoxedError::new)
1130 .and_then(|_| self.prepare_raw_scan(label_source));
1131 self.prepare_series_with_raw(ctx, metrics_set, label_source, preparation)
1132 }
1133
1134 fn prepare(&mut self, request: PrepareRequest) -> Result<(), BoxedError> {
1135 self.prepared = Arc::new(prepared::Preparation::default());
1136 self.clear_preflight_cache();
1137 self.properties.prepare(request);
1138
1139 self.check_scan_limit().map_err(BoxedError::new)?;
1140
1141 Ok(())
1142 }
1143
1144 fn has_predicate_without_region(&self) -> bool {
1145 let predicate = self
1146 .stream_ctx
1147 .input
1148 .predicate_group()
1149 .predicate_without_region();
1150 predicate.is_some()
1151 }
1152
1153 fn add_dyn_filter_to_predicate(
1154 &mut self,
1155 filter_exprs: Vec<Arc<dyn datafusion::physical_plan::PhysicalExpr>>,
1156 ) -> Vec<bool> {
1157 let matched = self.stream_ctx.add_dyn_filter_to_predicate(filter_exprs);
1158 self.clear_preflight_cache();
1160 matched
1161 }
1162
1163 fn reset_state(&mut self) {
1164 self.prepared = Arc::new(prepared::Preparation::default());
1165 self.clear_preflight_cache();
1166 self.stream_ctx.input.predicate.clear_dyn_filters();
1167 let num_workers = common_stat::get_total_cpu_cores().max(1);
1168 self.pruner = match self.mode {
1169 SeriesScanMode::Legacy => Arc::new(Pruner::new(self.stream_ctx.clone(), num_workers)),
1170 SeriesScanMode::TwoPhase => Arc::new(Pruner::new_with_options(
1171 self.stream_ctx.clone(),
1172 num_workers,
1173 PrunerOptions {
1174 retain_builders: true,
1175 enable_predicate_prefilter: false,
1176 },
1177 )),
1178 };
1179 self.legacy_receivers.lock().unwrap().clear();
1180 self.candidate_receivers.lock().unwrap().clear();
1181 self.metrics_list = Arc::new(PartitionMetricsList::default());
1182 }
1183
1184 fn set_logical_region(&mut self, logical_region: bool) {
1185 self.properties.set_logical_region(logical_region);
1186 }
1187
1188 fn set_query_load_region_id(&mut self, region_id: store_api::storage::RegionId) {
1189 self.properties.set_query_load_region_id(region_id);
1190 }
1191
1192 fn snapshot_sequence(&self) -> Option<u64> {
1193 self.stream_ctx.input.snapshot_sequence
1194 }
1195}
1196
1197impl DisplayAs for SeriesScan {
1198 fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter) -> fmt::Result {
1199 write!(
1200 f,
1201 "SeriesScan: region={}, ",
1202 self.stream_ctx.input.mapper.metadata().region_id
1203 )?;
1204 match t {
1205 DisplayFormatType::Default | DisplayFormatType::TreeRender => {
1206 self.stream_ctx.format_for_explain(false, f)?;
1207 }
1208 DisplayFormatType::Verbose => {
1209 self.stream_ctx.format_for_explain(true, f)?;
1210 }
1211 }
1212 write!(f, ", \"mode\":\"{}\"", self.mode.as_str())?;
1213 if matches!(t, DisplayFormatType::Verbose) {
1214 self.metrics_list.format_verbose_metrics(f)?;
1215 }
1216 Ok(())
1217 }
1218}
1219
1220impl fmt::Debug for SeriesScan {
1221 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1222 f.debug_struct("SeriesScan")
1223 .field("mode", &self.mode)
1224 .field("num_ranges", &self.stream_ctx.ranges.len())
1225 .finish()
1226 }
1227}
1228
1229#[cfg(test)]
1230impl SeriesScan {
1231 pub(crate) fn input(&self) -> &ScanInput {
1233 &self.stream_ctx.input
1234 }
1235
1236 pub(crate) fn mode(&self) -> &'static str {
1238 self.mode.as_str()
1239 }
1240}
1241
1242struct SeriesDistributor {
1244 stream_ctx: Arc<StreamContext>,
1246 range_semaphore: Option<Arc<Semaphore>>,
1248 final_merge_semaphore: Option<Arc<Semaphore>>,
1253 partitions: Vec<Vec<PartitionRange>>,
1255 pruner: Arc<Pruner>,
1257 senders: SenderList,
1259 metrics_set: ExecutionPlanMetricsSet,
1265 metrics_list: Arc<PartitionMetricsList>,
1266 explain_verbose: bool,
1268}
1269
1270impl SeriesDistributor {
1271 #[tracing::instrument(
1273 skip_all,
1274 fields(region_id = %self.stream_ctx.input.mapper.metadata().region_id)
1275 )]
1276 async fn execute(&mut self) {
1277 if let Err(e) = self.scan_partitions_flat().await {
1278 self.senders.send_error(e).await;
1279 }
1280 }
1281
1282 #[tracing::instrument(
1284 skip_all,
1285 fields(region_id = %self.stream_ctx.input.mapper.metadata().region_id)
1286 )]
1287 async fn scan_partitions_flat(&mut self) -> Result<()> {
1288 for partition_ranges in &self.partitions {
1290 self.pruner.add_partition_ranges(partition_ranges);
1291 }
1292
1293 let all_partition_ranges: Vec<_> = self.partitions.iter().flatten().cloned().collect();
1295 let needs_merge = all_partition_ranges.len() > 1;
1296 let partition_pruner = Arc::new(PartitionPruner::new(
1297 self.pruner.clone(),
1298 &all_partition_ranges,
1299 ));
1300
1301 let part_metrics = new_partition_metrics(
1302 &self.stream_ctx,
1303 self.explain_verbose,
1304 &self.metrics_set,
1305 self.partitions.len(),
1306 &self.metrics_list,
1307 );
1308 part_metrics.on_first_poll();
1309 let mut fetch_start = Instant::now();
1312
1313 let build_start = Instant::now();
1315 let mut tasks = Vec::new();
1316 for partition in &self.partitions {
1317 for part_range in partition {
1318 let stream_ctx = self.stream_ctx.clone();
1319 let part_range = *part_range;
1320 let part_metrics = part_metrics.clone();
1321 let partition_pruner = partition_pruner.clone();
1322 let file_scan_semaphore = self.range_semaphore.clone();
1323 let merge_semaphore = self.range_semaphore.clone();
1324 tasks.push(common_runtime::spawn_query(async move {
1325 SeqScan::build_flat_partition_range_read(
1326 &stream_ctx,
1327 &part_range,
1328 if needs_merge {
1329 FlatReadMode::CrossRangeMerge
1330 } else {
1331 FlatReadMode::Sequential
1332 },
1333 &part_metrics,
1334 partition_pruner,
1335 file_scan_semaphore,
1336 merge_semaphore,
1337 )
1338 .await
1339 }));
1340 }
1341 }
1342 let mut range_streams = Vec::with_capacity(tasks.len());
1343 let mut estimated_batch_sizes = Vec::with_capacity(tasks.len());
1344 for task in tasks {
1345 let (stream, estimated_batch_size) = task.await.context(JoinSnafu)??;
1346 range_streams.push(stream);
1347 estimated_batch_sizes.push(estimated_batch_size);
1348 }
1349 let channel_size =
1350 compute_parallel_channel_size(compute_average_batch_size(estimated_batch_sizes));
1351 common_telemetry::debug!(
1352 "SeriesDistributor built {} range_streams, region: {}, build cost: {:?}, channel_size: {}",
1353 range_streams.len(),
1354 self.stream_ctx.input.region_metadata().region_id,
1355 build_start.elapsed(),
1356 channel_size,
1357 );
1358
1359 let mut reader = SeqScan::build_flat_reader_from_sources(
1363 &self.stream_ctx,
1364 range_streams,
1365 self.final_merge_semaphore.clone(),
1366 Some(&part_metrics),
1367 true,
1368 channel_size,
1369 )
1370 .await?;
1371 let mut metrics = SeriesDistributorMetrics::default();
1372
1373 let mut divider = FlatSeriesBatchDivider::default();
1374 while let Some(record_batch) = reader.try_next().await? {
1375 metrics.scan_cost += fetch_start.elapsed();
1376 metrics.num_batches += 1;
1377 metrics.num_rows += record_batch.num_rows();
1378
1379 debug_assert!(record_batch.num_rows() > 0);
1380 if record_batch.num_rows() == 0 {
1381 fetch_start = Instant::now();
1382 continue;
1383 }
1384
1385 let divider_start = Instant::now();
1387 let series_batch = divider.push(record_batch);
1388 metrics.divider_cost += divider_start.elapsed();
1389 if let Some(series_batch) = series_batch {
1390 let yield_start = Instant::now();
1391 self.senders
1392 .send_batch(SeriesBatch::Flat(series_batch))
1393 .await?;
1394 metrics.yield_cost += yield_start.elapsed();
1395 }
1396 fetch_start = Instant::now();
1397 }
1398
1399 let divider_start = Instant::now();
1401 let series_batch = divider.finish();
1402 metrics.divider_cost += divider_start.elapsed();
1403 if let Some(series_batch) = series_batch {
1404 let yield_start = Instant::now();
1405 self.senders
1406 .send_batch(SeriesBatch::Flat(series_batch))
1407 .await?;
1408 metrics.yield_cost += yield_start.elapsed();
1409 }
1410
1411 metrics.scan_cost += fetch_start.elapsed();
1412 metrics.num_series_send_full = self.senders.num_full;
1413 part_metrics.set_distributor_metrics(&metrics);
1414
1415 part_metrics.on_finish();
1416
1417 Ok(())
1418 }
1419}
1420
1421struct SeriesCandidateDistributor {
1423 memory_pool: Arc<dyn MemoryPool>,
1424 stream_ctx: Arc<StreamContext>,
1425 range_semaphore: Arc<Semaphore>,
1426 partitions: Vec<Vec<PartitionRange>>,
1427 pruner: Arc<Pruner>,
1428 senders: CandidateSenderList,
1429 active_receivers: Arc<AtomicUsize>,
1430 metrics_set: ExecutionPlanMetricsSet,
1431 metrics_list: Arc<PartitionMetricsList>,
1432 explain_verbose: bool,
1433}
1434
1435impl SeriesCandidateDistributor {
1436 async fn execute(&mut self) {
1437 if let Err(e) = self.distribute().await {
1438 self.send_error(e);
1439 }
1440 }
1441
1442 async fn distribute(&mut self) -> Result<()> {
1443 let part_metrics = new_partition_metrics(
1444 &self.stream_ctx,
1445 self.explain_verbose,
1446 &self.metrics_set,
1447 self.partitions.len(),
1448 &self.metrics_list,
1449 );
1450 part_metrics.on_first_poll();
1451
1452 let candidate_scanner = SeriesCandidateScanner::try_new(
1453 self.stream_ctx.clone(),
1454 self.partitions.clone(),
1455 self.pruner.clone(),
1456 self.range_semaphore.clone(),
1457 self.memory_pool.clone(),
1458 self.metrics_set.clone(),
1459 part_metrics.clone(),
1460 )?;
1461 let partition_pruner = candidate_scanner.partition_pruner();
1462 let mut candidates = candidate_scanner.build_stream().await?;
1463 let mut collector =
1464 SeriesBatchCollector::new(self.partitions.len()).context(InvalidSenderSnafu)?;
1465 let mut chunked = false;
1466 while let Some(batch) = candidates.try_next().await? {
1467 if !self.should_fetch_candidates() {
1468 part_metrics.on_finish();
1469 return Ok(());
1470 }
1471 collector.push(batch);
1472 if collector.len() >= candidate_series_assignment_threshold() {
1473 chunked = true;
1474 self.send_assignments(collector.finish(false), &partition_pruner);
1475 if !self.should_fetch_candidates() {
1476 part_metrics.on_finish();
1477 return Ok(());
1478 }
1479 collector =
1480 SeriesBatchCollector::new(self.partitions.len()).context(InvalidSenderSnafu)?;
1481 }
1482 }
1483
1484 if collector.len() > 0 {
1485 self.send_assignments(collector.finish(!chunked), &partition_pruner);
1486 }
1487 part_metrics.on_finish();
1488 Ok(())
1489 }
1490
1491 fn send_assignments(
1492 &mut self,
1493 assignments: Vec<AssignedSeriesBatch>,
1494 partition_pruner: &Arc<PartitionPruner>,
1495 ) {
1496 #[cfg(test)]
1498 CANDIDATE_ASSIGNMENT_ROUNDS.fetch_add(1, Ordering::Relaxed);
1499 for (partition, assigned_series) in assignments.into_iter().enumerate() {
1500 if assigned_series.series().is_empty() {
1501 continue;
1502 }
1503 let Some(sender) = self.senders[partition].as_ref() else {
1504 continue;
1505 };
1506 let sent = sender
1507 .send(Ok(SeriesReaderInput {
1508 assigned_series,
1509 partition_pruner: partition_pruner.clone(),
1510 range_semaphore: self.range_semaphore.clone(),
1511 }))
1512 .is_ok();
1513 if !sent {
1514 self.senders[partition] = None;
1515 }
1516 }
1517 }
1518
1519 fn should_fetch_candidates(&self) -> bool {
1520 self.active_receivers.load(Ordering::Relaxed) > 0
1521 }
1522
1523 fn send_error(&mut self, error: Error) {
1524 let error = Arc::new(error);
1525 for sender in self.senders.iter_mut().filter_map(Option::take) {
1526 let result = Err(error.clone()).context(ScanSeriesSnafu);
1527 let _ = sender.send(result);
1528 }
1529 }
1530}
1531
1532#[derive(Debug)]
1537pub enum SeriesBatch {
1538 Flat(FlatSeriesBatch),
1539}
1540
1541impl SeriesBatch {
1542 pub fn num_batches(&self) -> usize {
1544 match self {
1545 SeriesBatch::Flat(flat_batch) => flat_batch.batches.len(),
1546 }
1547 }
1548
1549 pub fn num_rows(&self) -> usize {
1551 match self {
1552 SeriesBatch::Flat(flat_batch) => flat_batch.batches.iter().map(|x| x.num_rows()).sum(),
1553 }
1554 }
1555}
1556
1557#[derive(Default, Debug)]
1559pub struct FlatSeriesBatch {
1560 pub batches: SmallVec<[RecordBatch; 4]>,
1561}
1562
1563struct SenderList {
1565 senders: Vec<Option<Sender<Result<SeriesBatch>>>>,
1566 num_nones: usize,
1568 sender_idx: usize,
1570 num_full: usize,
1572}
1573
1574impl SenderList {
1575 fn new(senders: Vec<Option<Sender<Result<SeriesBatch>>>>) -> Self {
1576 let num_nones = senders.iter().filter(|sender| sender.is_none()).count();
1577 Self {
1578 senders,
1579 num_nones,
1580 sender_idx: 0,
1581 num_full: 0,
1582 }
1583 }
1584
1585 fn try_send_batch(&mut self, mut batch: SeriesBatch) -> Result<Option<SeriesBatch>> {
1588 for _ in 0..self.senders.len() {
1589 ensure!(self.num_nones < self.senders.len(), InvalidSenderSnafu);
1590
1591 let sender_idx = self.fetch_add_sender_idx();
1592 let Some(sender) = &self.senders[sender_idx] else {
1593 continue;
1594 };
1595
1596 match sender.try_send(Ok(batch)) {
1597 Ok(()) => return Ok(None),
1598 Err(TrySendError::Full(res)) => {
1599 self.num_full += 1;
1600 batch = res.unwrap();
1602 }
1603 Err(TrySendError::Closed(res)) => {
1604 self.senders[sender_idx] = None;
1605 self.num_nones += 1;
1606 batch = res.unwrap();
1608 }
1609 }
1610 }
1611
1612 Ok(Some(batch))
1613 }
1614
1615 async fn send_batch(&mut self, batch: SeriesBatch) -> Result<()> {
1617 let Some(batch) = self.try_send_batch(batch)? else {
1618 return Ok(());
1619 };
1620
1621 loop {
1622 ensure!(self.num_nones < self.senders.len(), InvalidSenderSnafu);
1623
1624 let closed = {
1628 let reservations = self.senders.iter().enumerate().filter_map(|(idx, sender)| {
1629 sender
1630 .as_ref()
1631 .map(|sender| Box::pin(async move { (idx, sender.reserve().await) }))
1632 });
1633 let ((idx, reserved), _, _) = futures::future::select_all(reservations).await;
1634 match reserved {
1635 Ok(permit) => {
1636 permit.send(Ok(batch));
1637 return Ok(());
1638 }
1639 Err(_) => idx,
1640 }
1641 };
1642 self.senders[closed] = None;
1643 self.num_nones += 1;
1644 }
1645 }
1646
1647 async fn send_error(&self, error: Error) {
1648 let error = Arc::new(error);
1649 for sender in self.senders.iter().flatten() {
1650 let result = Err(error.clone()).context(ScanSeriesSnafu);
1651 let _ = sender.send(result).await;
1652 }
1653 }
1654
1655 fn fetch_add_sender_idx(&mut self) -> usize {
1656 let sender_idx = self.sender_idx;
1657 self.sender_idx = (self.sender_idx + 1) % self.senders.len();
1658 sender_idx
1659 }
1660}
1661
1662fn new_partition_metrics(
1663 stream_ctx: &StreamContext,
1664 explain_verbose: bool,
1665 metrics_set: &ExecutionPlanMetricsSet,
1666 partition: usize,
1667 metrics_list: &PartitionMetricsList,
1668) -> PartitionMetrics {
1669 let metrics = PartitionMetrics::new(
1670 stream_ctx.input.mapper.metadata().region_id,
1671 partition,
1672 "SeriesScan",
1673 stream_ctx.query_start,
1674 explain_verbose,
1675 metrics_set,
1676 );
1677
1678 metrics_list.set(partition, metrics.clone());
1679 metrics
1680}
1681
1682#[derive(Default)]
1687struct FlatSeriesBatchDivider {
1688 buffer: FlatSeriesBatch,
1689}
1690
1691impl FlatSeriesBatchDivider {
1692 fn push(&mut self, batch: RecordBatch) -> Option<FlatSeriesBatch> {
1696 if self.buffer.batches.is_empty() {
1698 self.buffer.batches.push(batch);
1699 return None;
1700 }
1701
1702 let pk_column_idx = primary_key_column_index(batch.num_columns());
1704 let batch_pk_column = batch.column(pk_column_idx);
1705 let batch_pk_array = batch_pk_column
1706 .as_any()
1707 .downcast_ref::<PrimaryKeyArray>()
1708 .unwrap();
1709 let batch_pk_values = batch_pk_array
1710 .values()
1711 .as_any()
1712 .downcast_ref::<BinaryArray>()
1713 .unwrap();
1714 let batch_last_pk =
1716 primary_key_at(batch_pk_array, batch_pk_values, batch_pk_array.len() - 1);
1717 let buffer_last_batch = self.buffer.batches.last().unwrap();
1720 let buffer_pk_column = buffer_last_batch.column(pk_column_idx);
1721 let buffer_pk_array = buffer_pk_column
1722 .as_any()
1723 .downcast_ref::<PrimaryKeyArray>()
1724 .unwrap();
1725 let buffer_pk_values = buffer_pk_array
1726 .values()
1727 .as_any()
1728 .downcast_ref::<BinaryArray>()
1729 .unwrap();
1730 let buffer_last_pk =
1731 primary_key_at(buffer_pk_array, buffer_pk_values, buffer_pk_array.len() - 1);
1732
1733 if batch_last_pk == buffer_last_pk {
1735 self.buffer.batches.push(batch);
1736 return None;
1737 }
1738 let batch_pk_keys = batch_pk_array.keys();
1741 let pk_indices = batch_pk_keys.values();
1742 let mut change_offset = 0;
1743 for (i, &key) in pk_indices.iter().enumerate() {
1744 let batch_pk = batch_pk_values.value(key as usize);
1745
1746 if buffer_last_pk != batch_pk {
1747 change_offset = i;
1748 break;
1749 }
1750 }
1751
1752 let (first_part, remaining_part) = if change_offset > 0 {
1754 let first_part = batch.slice(0, change_offset);
1755 let remaining_part = batch.slice(change_offset, batch.num_rows() - change_offset);
1756 (Some(first_part), Some(remaining_part))
1757 } else {
1758 (None, Some(batch))
1759 };
1760
1761 let mut result = std::mem::take(&mut self.buffer);
1763 if let Some(first_part) = first_part {
1764 result.batches.push(first_part);
1765 }
1766
1767 if let Some(remaining_part) = remaining_part {
1769 self.buffer.batches.push(remaining_part);
1770 }
1771
1772 Some(result)
1773 }
1774
1775 fn finish(&mut self) -> Option<FlatSeriesBatch> {
1777 if self.buffer.batches.is_empty() {
1778 None
1779 } else {
1780 Some(std::mem::take(&mut self.buffer))
1781 }
1782 }
1783}
1784
1785fn primary_key_at<'a>(
1787 primary_key: &PrimaryKeyArray,
1788 primary_key_values: &'a BinaryArray,
1789 index: usize,
1790) -> &'a [u8] {
1791 let key = primary_key.keys().value(index);
1792 primary_key_values.value(key as usize)
1793}
1794
1795#[cfg(test)]
1796mod tests {
1797 use std::sync::Arc;
1798
1799 use api::v1::OpType;
1800 use datatypes::arrow::array::{
1801 ArrayRef, BinaryDictionaryBuilder, Int64Array, StringDictionaryBuilder,
1802 TimestampMillisecondArray, UInt8Array, UInt64Array,
1803 };
1804 use datatypes::arrow::datatypes::{DataType, Field, Schema, SchemaRef, TimeUnit, UInt32Type};
1805 use datatypes::arrow::record_batch::RecordBatch;
1806
1807 use super::*;
1808 use crate::read::flat_projection::FlatProjectionMapper;
1809 use crate::read::scan_region::PredicateGroup;
1810 use crate::test_util::scheduler_util::SchedulerEnv;
1811 use crate::test_util::sst_util::sst_region_metadata_with_encoding;
1812
1813 #[tokio::test]
1814 async fn two_phase_eligibility_allows_exact_sequence_range() {
1815 let env = SchedulerEnv::new().await;
1816 let metadata = Arc::new(sst_region_metadata_with_encoding(
1817 store_api::codec::PrimaryKeyEncoding::Sparse,
1818 ));
1819 let predicate = PredicateGroup::default();
1820
1821 let eligible = ScanInput::builder(
1822 env.access_layer.clone(),
1823 FlatProjectionMapper::new(&metadata, [0]).unwrap(),
1824 )
1825 .with_predicate(predicate.clone())
1826 .build();
1827 assert!(SeriesScan::supports_two_phase(&eligible));
1828
1829 let exact_sequence = ScanInput::builder(
1830 env.access_layer.clone(),
1831 FlatProjectionMapper::new(&metadata, [0]).unwrap(),
1832 )
1833 .with_predicate(predicate)
1834 .with_sequence_range(Some(store_api::storage::SequenceRange::GtLtEq {
1835 min: 1,
1836 max: 2,
1837 }))
1838 .build();
1839 assert!(SeriesScan::supports_two_phase(&exact_sequence));
1840 }
1841
1842 #[test]
1843 fn candidate_distributor_stops_after_all_receivers_close() {
1844 let (_senders, mut receivers, active_receivers) = new_candidate_channel_list(2);
1845 assert_eq!(2, active_receivers.load(Ordering::Relaxed));
1846
1847 drop(receivers[0].take());
1848 assert_eq!(1, active_receivers.load(Ordering::Relaxed));
1849
1850 drop(receivers[1].take());
1851 assert_eq!(0, active_receivers.load(Ordering::Relaxed));
1852 }
1853
1854 #[tokio::test]
1855 async fn distributor_does_not_stall_on_partitions_read_one_after_another() {
1856 const NUM_BATCHES: usize = 2000;
1857 let (mut senders, receivers) = new_legacy_channel_list(16);
1858 let producer = tokio::spawn(async move {
1859 for _ in 0..NUM_BATCHES {
1860 senders
1861 .send_batch(SeriesBatch::Flat(FlatSeriesBatch::default()))
1862 .await
1863 .unwrap();
1864 }
1865 });
1866 let consume = async {
1869 let mut received = 0;
1870 for mut receiver in receivers.into_iter().flatten() {
1871 while let Some(batch) = receiver.recv().await {
1872 batch.unwrap();
1873 received += 1;
1874 }
1875 }
1876 received
1877 };
1878 let received = tokio::time::timeout(std::time::Duration::from_secs(1), consume)
1879 .await
1880 .expect("the distributor stalled on partitions the consumer was not reading");
1881 producer.await.unwrap();
1882 assert_eq!(NUM_BATCHES, received);
1883 }
1884
1885 fn new_test_record_batch(
1886 primary_keys: &[&[u8]],
1887 timestamps: &[i64],
1888 sequences: &[u64],
1889 op_types: &[OpType],
1890 fields: &[u64],
1891 ) -> RecordBatch {
1892 let num_rows = timestamps.len();
1893 debug_assert_eq!(sequences.len(), num_rows);
1894 debug_assert_eq!(op_types.len(), num_rows);
1895 debug_assert_eq!(fields.len(), num_rows);
1896 debug_assert_eq!(primary_keys.len(), num_rows);
1897
1898 let columns: Vec<ArrayRef> = vec![
1899 build_test_pk_string_dict_array(primary_keys),
1900 Arc::new(Int64Array::from_iter(
1901 fields.iter().map(|v| Some(*v as i64)),
1902 )),
1903 Arc::new(TimestampMillisecondArray::from_iter_values(
1904 timestamps.iter().copied(),
1905 )),
1906 build_test_pk_array(primary_keys),
1907 Arc::new(UInt64Array::from_iter_values(sequences.iter().copied())),
1908 Arc::new(UInt8Array::from_iter_values(
1909 op_types.iter().map(|v| *v as u8),
1910 )),
1911 ];
1912
1913 RecordBatch::try_new(build_test_flat_schema(), columns).unwrap()
1914 }
1915
1916 fn build_test_pk_string_dict_array(primary_keys: &[&[u8]]) -> ArrayRef {
1917 let mut builder = StringDictionaryBuilder::<UInt32Type>::new();
1918 for &pk in primary_keys {
1919 let pk_str = std::str::from_utf8(pk).unwrap();
1920 builder.append(pk_str).unwrap();
1921 }
1922 Arc::new(builder.finish())
1923 }
1924
1925 fn build_test_pk_array(primary_keys: &[&[u8]]) -> ArrayRef {
1926 let mut builder = BinaryDictionaryBuilder::<UInt32Type>::new();
1927 for &pk in primary_keys {
1928 builder.append(pk).unwrap();
1929 }
1930 Arc::new(builder.finish())
1931 }
1932
1933 fn build_test_flat_schema() -> SchemaRef {
1934 let fields = vec![
1935 Field::new(
1936 "k0",
1937 DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
1938 false,
1939 ),
1940 Field::new("field0", DataType::Int64, true),
1941 Field::new(
1942 "ts",
1943 DataType::Timestamp(TimeUnit::Millisecond, None),
1944 false,
1945 ),
1946 Field::new(
1947 "__primary_key",
1948 DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Binary)),
1949 false,
1950 ),
1951 Field::new("__sequence", DataType::UInt64, false),
1952 Field::new("__op_type", DataType::UInt8, false),
1953 ];
1954 Arc::new(Schema::new(fields))
1955 }
1956
1957 #[test]
1958 fn test_empty_buffer_first_push() {
1959 let mut divider = FlatSeriesBatchDivider::default();
1960 let result = divider.finish();
1961 assert!(result.is_none());
1962
1963 let mut divider = FlatSeriesBatchDivider::default();
1964 let batch = new_test_record_batch(
1965 &[b"series1", b"series1"],
1966 &[1000, 2000],
1967 &[1, 2],
1968 &[OpType::Put, OpType::Put],
1969 &[10, 20],
1970 );
1971 let result = divider.push(batch);
1972 assert!(result.is_none());
1973 assert_eq!(divider.buffer.batches.len(), 1);
1974 }
1975
1976 #[test]
1977 fn test_same_series_accumulation() {
1978 let mut divider = FlatSeriesBatchDivider::default();
1979
1980 let batch1 = new_test_record_batch(
1981 &[b"series1", b"series1"],
1982 &[1000, 2000],
1983 &[1, 2],
1984 &[OpType::Put, OpType::Put],
1985 &[10, 20],
1986 );
1987
1988 let batch2 = new_test_record_batch(
1989 &[b"series1", b"series1"],
1990 &[3000, 4000],
1991 &[3, 4],
1992 &[OpType::Put, OpType::Put],
1993 &[30, 40],
1994 );
1995
1996 divider.push(batch1);
1997 let result = divider.push(batch2);
1998 assert!(result.is_none());
1999 let series_batch = divider.finish().unwrap();
2000 assert_eq!(series_batch.batches.len(), 2);
2001 }
2002
2003 #[test]
2004 fn test_series_boundary_detection() {
2005 let mut divider = FlatSeriesBatchDivider::default();
2006
2007 let batch1 = new_test_record_batch(
2008 &[b"series1", b"series1"],
2009 &[1000, 2000],
2010 &[1, 2],
2011 &[OpType::Put, OpType::Put],
2012 &[10, 20],
2013 );
2014
2015 let batch2 = new_test_record_batch(
2016 &[b"series2", b"series2"],
2017 &[3000, 4000],
2018 &[3, 4],
2019 &[OpType::Put, OpType::Put],
2020 &[30, 40],
2021 );
2022
2023 divider.push(batch1);
2024 let series_batch = divider.push(batch2).unwrap();
2025 assert_eq!(series_batch.batches.len(), 1);
2026
2027 assert_eq!(divider.buffer.batches.len(), 1);
2028 }
2029
2030 #[test]
2031 fn test_series_boundary_within_batch() {
2032 let mut divider = FlatSeriesBatchDivider::default();
2033
2034 let batch1 = new_test_record_batch(
2035 &[b"series1", b"series1"],
2036 &[1000, 2000],
2037 &[1, 2],
2038 &[OpType::Put, OpType::Put],
2039 &[10, 20],
2040 );
2041
2042 let batch2 = new_test_record_batch(
2043 &[b"series1", b"series2"],
2044 &[3000, 4000],
2045 &[3, 4],
2046 &[OpType::Put, OpType::Put],
2047 &[30, 40],
2048 );
2049
2050 divider.push(batch1);
2051 let series_batch = divider.push(batch2).unwrap();
2052 assert_eq!(series_batch.batches.len(), 2);
2053 assert_eq!(series_batch.batches[0].num_rows(), 2);
2054 assert_eq!(series_batch.batches[1].num_rows(), 1);
2055
2056 assert_eq!(divider.buffer.batches.len(), 1);
2057 assert_eq!(divider.buffer.batches[0].num_rows(), 1);
2058 }
2059
2060 #[test]
2061 fn test_series_splitting() {
2062 let mut divider = FlatSeriesBatchDivider::default();
2063
2064 let batch1 = new_test_record_batch(&[b"series1"], &[1000], &[1], &[OpType::Put], &[10]);
2065
2066 let batch2 = new_test_record_batch(
2067 &[b"series1", b"series2", b"series2", b"series3"],
2068 &[2000, 3000, 4000, 5000],
2069 &[2, 3, 4, 5],
2070 &[OpType::Put, OpType::Put, OpType::Put, OpType::Put],
2071 &[20, 30, 40, 50],
2072 );
2073
2074 divider.push(batch1);
2075 let series_batch = divider.push(batch2).unwrap();
2076 assert_eq!(series_batch.batches.len(), 2);
2077
2078 let total_rows: usize = series_batch.batches.iter().map(|b| b.num_rows()).sum();
2079 assert_eq!(total_rows, 2);
2080
2081 let final_batch = divider.finish().unwrap();
2082 assert_eq!(final_batch.batches.len(), 1);
2083 assert_eq!(final_batch.batches[0].num_rows(), 3);
2084 }
2085}