Skip to main content

query/optimizer/
scan_hint.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashSet;
16
17use api::v1::SemanticType;
18use arrow_schema::SortOptions;
19use common_function::aggrs::aggr_wrapper::aggr_state_func_name;
20use common_recordbatch::OrderOption;
21use common_recordbatch::filter::SimpleFilterEvaluator;
22use datafusion::datasource::DefaultTableSource;
23use datafusion_common::tree_node::{Transformed, TreeNodeRewriter};
24use datafusion_common::{Column, Result};
25use datafusion_expr::expr::{NullTreatment, Sort};
26use datafusion_expr::{Expr, LogicalPlan, utils};
27use datafusion_optimizer::{OptimizerConfig, OptimizerRule};
28use datatypes::arrow::datatypes::{DataType, TimeUnit as ArrowTimeUnit};
29use promql::extension_plan::{
30    FusedSeries, InstantManipulate, RawSeriesSource, SeriesDivide, SeriesNormalize,
31};
32use store_api::metric_engine_consts::DATA_SCHEMA_TSID_COLUMN_NAME;
33use store_api::storage::{TimeSeriesDistribution, TimeSeriesRowSelector};
34
35use crate::dummy_catalog::DummyTableProvider;
36
37/// This rule will traverse the plan to collect necessary hints for leaf
38/// table scan node and set them in [`ScanRequest`]. Hints include:
39/// - the nearest order requirement to the leaf table scan node as ordering hint.
40/// - the group by columns when all aggregate functions are `last_value` as
41///   time series row selector hint.
42/// - the per-series distribution when the leaf table scan node is the input of a
43///   [`FusedSeries`] or [`RawSeriesSource`] node, which read the raw series frames a
44///   region only packs for that distribution.
45///
46/// [`ScanRequest`]: store_api::storage::ScanRequest
47#[derive(Debug)]
48pub struct ScanHintRule;
49
50impl OptimizerRule for ScanHintRule {
51    fn name(&self) -> &str {
52        "ScanHintRule"
53    }
54
55    fn rewrite(
56        &self,
57        plan: LogicalPlan,
58        _config: &dyn OptimizerConfig,
59    ) -> Result<Transformed<LogicalPlan>> {
60        Self::optimize(plan)
61    }
62}
63
64impl ScanHintRule {
65    fn optimize(plan: LogicalPlan) -> Result<Transformed<LogicalPlan>> {
66        let mut rewriter = ScanHintRewriter::default();
67        // The extension's input is included by DataFusion's normal TreeNode
68        // rewrite, so this is one scoped recursive walk (subquery expressions
69        // are included by the dedicated API as well).
70        plan.rewrite_with_subqueries(&mut rewriter)
71    }
72
73    fn set_hints(
74        plan: LogicalPlan,
75        rewriter: &mut ScanHintRewriter,
76        fused_series_input: bool,
77    ) -> Result<Transformed<LogicalPlan>> {
78        let LogicalPlan::TableScan(mut table_scan) = plan else {
79            return Ok(Transformed::no(plan));
80        };
81        let Some(source) = table_scan.source.downcast_ref::<DefaultTableSource>() else {
82            return Ok(Transformed::no(LogicalPlan::TableScan(table_scan)));
83        };
84        // The provider in the region server is [DummyTableProvider].
85        let Some(original) = source.table_provider.downcast_ref::<DummyTableProvider>() else {
86            return Ok(Transformed::no(LogicalPlan::TableScan(table_scan)));
87        };
88
89        // Attached scan filters are checked below; residual Filter nodes are
90        // rejected by the single-evaluation path allowlist.
91        let filters_preserve_last_row = if rewriter.inside_single_evaluation {
92            Self::filters_preserve_last_row(&table_scan, original)
93        } else {
94            true
95        };
96        let use_last_row = rewriter.inside_single_evaluation && filters_preserve_last_row;
97
98        let has_hint = rewriter.order_expr.is_some()
99            || rewriter.ts_row_selector.is_some()
100            || use_last_row
101            || fused_series_input;
102        if !has_hint {
103            return Ok(Transformed::no(LogicalPlan::TableScan(table_scan)));
104        }
105
106        // A provider can be used by several TableScan nodes. Fork its request
107        // for every hinted use-site before applying hints, rather than mutating
108        // the shared catalog provider. This keeps order and aggregate hints
109        // local as well as the new LastRow hint.
110        let adapter = original.clone_for_scan();
111        Self::apply_hints(&adapter, rewriter);
112        if fused_series_input {
113            // The scan below a `FusedSeries` node reads packs of raw series, which a region only
114            // produces for a per-series request; the existing PromQL pipeline asks for the same
115            // distribution through a sort on `(series key, time)`, which the native node does not
116            // have below it. Applying it after the order hints makes the plan shape the deciding
117            // one: the node's own input is what reads the frames.
118            adapter.with_distribution(TimeSeriesDistribution::PerSeries);
119        }
120        if use_last_row {
121            // Apply the instant-derived hint after the aggregate hint. Both
122            // select LastRow today, and this ordering preserves the existing
123            // aggregate selector when the instant guard rejects a scan.
124            adapter.with_time_series_selector_hint(TimeSeriesRowSelector::LastRow {
125                after_merge: true,
126            });
127        }
128        table_scan.source =
129            std::sync::Arc::new(DefaultTableSource::new(std::sync::Arc::new(adapter)));
130        Ok(Transformed::yes(LogicalPlan::TableScan(table_scan)))
131    }
132
133    /// Checks whether attached scan predicates permit instant-derived LastRow selection.
134    ///
135    /// Selecting the newest row too early can discard an older matching sample if a
136    /// predicate later rejects that row. Only recognized tag/time predicates are
137    /// allowed: tags select whole series, and supported time predicates constrain
138    /// the scan window before row selection. Field or unrecognized predicates are
139    /// conservatively rejected.
140    ///
141    /// This checks only attached predicates; the path allowlist separately rejects
142    /// residual Filter nodes between InstantManipulate and the scan.
143    fn filters_preserve_last_row(
144        table_scan: &datafusion_expr::logical_plan::TableScan,
145        provider: &DummyTableProvider,
146    ) -> bool {
147        let metadata = provider.region_metadata();
148        for filter in &table_scan.filters {
149            let Some(filter) = SimpleFilterEvaluator::try_new(filter) else {
150                return false;
151            };
152            let Some(column_metadata) = metadata.column_by_name(filter.column_name()) else {
153                return false;
154            };
155            if !matches!(
156                column_metadata.semantic_type,
157                SemanticType::Tag | SemanticType::Timestamp
158            ) {
159                return false;
160            }
161        }
162        true
163    }
164
165    fn apply_hints(adapter: &DummyTableProvider, rewriter: &mut ScanHintRewriter) {
166        if let Some(order_expr) = &rewriter.order_expr {
167            Self::set_order_hint(adapter, order_expr);
168        }
169        if let Some((group_by_cols, order_by_col)) = &rewriter.ts_row_selector {
170            Self::set_time_series_row_selector_hint(adapter, group_by_cols, order_by_col);
171        }
172    }
173
174    fn set_order_hint(adapter: &DummyTableProvider, order_expr: &Vec<Sort>) {
175        let mut opts = Vec::with_capacity(order_expr.len());
176        for sort in order_expr {
177            let name = match sort.expr.try_as_col() {
178                Some(col) => col.name.clone(),
179                None => return,
180            };
181            opts.push(OrderOption {
182                name,
183                options: SortOptions {
184                    descending: !sort.asc,
185                    nulls_first: sort.nulls_first,
186                },
187            });
188        }
189        adapter.with_ordering_hint(&opts);
190
191        let region_metadata = adapter.region_metadata();
192        let time_index_name = region_metadata
193            .time_index_column()
194            .column_schema
195            .name
196            .as_str();
197        let sort_cols = order_expr
198            .iter()
199            .filter_map(|s| s.expr.try_as_col())
200            .collect::<Vec<_>>();
201
202        // Special-case metric engine: when the nearest sort requirement is `__tsid, <time index>`,
203        // we can safely enable per-series distribution hint so the region can use `SeriesScan`.
204        //
205        // This pattern is produced by promql planning when `__tsid` is available and is used as the
206        // series identifier (instead of expanding to all tag columns).
207        if sort_cols.len() == 2
208            && sort_cols[0].name == DATA_SCHEMA_TSID_COLUMN_NAME
209            && sort_cols[1].name == time_index_name
210        {
211            adapter.with_distribution(TimeSeriesDistribution::PerSeries);
212            return;
213        }
214
215        let mut sort_expr_cursor = sort_cols.into_iter();
216        // ignore table without pk
217        if region_metadata.primary_key.is_empty() {
218            return;
219        }
220        let mut pk_column_iter = region_metadata.primary_key_columns();
221        let mut curr_sort_expr = sort_expr_cursor.next();
222        let mut curr_pk_col = pk_column_iter.next();
223
224        while let (Some(sort_expr), Some(pk_col)) = (curr_sort_expr, curr_pk_col) {
225            if sort_expr.name == pk_col.column_schema.name {
226                curr_sort_expr = sort_expr_cursor.next();
227                curr_pk_col = pk_column_iter.next();
228            } else {
229                return;
230            }
231        }
232
233        let next_remaining = sort_expr_cursor.next();
234        match (curr_sort_expr, next_remaining) {
235            (Some(expr), None)
236                if expr.name == region_metadata.time_index_column().column_schema.name =>
237            {
238                adapter.with_distribution(TimeSeriesDistribution::PerSeries);
239            }
240            (None, _) => adapter.with_distribution(TimeSeriesDistribution::PerSeries),
241            (Some(_), _) => {}
242        }
243    }
244
245    fn set_time_series_row_selector_hint(
246        adapter: &DummyTableProvider,
247        group_by_cols: &HashSet<Column>,
248        order_by_col: &Column,
249    ) {
250        let region_metadata = adapter.region_metadata();
251        let mut should_set_selector_hint = true;
252        // check if order_by column is time index
253        if let Some(column_metadata) = region_metadata.column_by_name(&order_by_col.name) {
254            if column_metadata.semantic_type != SemanticType::Timestamp {
255                should_set_selector_hint = false;
256            }
257        } else {
258            should_set_selector_hint = false;
259        }
260
261        // check if all group_by columns are primary key
262        for col in group_by_cols {
263            let Some(column_metadata) = region_metadata.column_by_name(&col.name) else {
264                should_set_selector_hint = false;
265                break;
266            };
267            if column_metadata.semantic_type != SemanticType::Tag {
268                should_set_selector_hint = false;
269                break;
270            }
271        }
272
273        if should_set_selector_hint {
274            adapter.with_time_series_selector_hint(TimeSeriesRowSelector::LastRow {
275                after_merge: false,
276            });
277        }
278    }
279}
280
281/// Traverse and apply hints with state scoped to the current logical-plan path.
282///
283/// Rewriting the scan while walking down the tree is important: the state then
284/// describes the actual parent path of that scan, and a shared provider is forked
285/// at that exact use-site. No traversal-order identity is involved.
286#[derive(Default)]
287struct ScanHintRewriter {
288    order_expr: Option<Vec<Sort>>,
289    order_stack: Vec<Option<Vec<Sort>>>,
290    ts_row_selector: Option<(HashSet<Column>, Column)>,
291    ts_stack: Vec<Option<(HashSet<Column>, Column)>>,
292    inside_single_evaluation: bool,
293    single_evaluation_stack: Vec<bool>,
294    /// Whether the node being visited is the input of a [`FusedSeries`] or [`RawSeriesSource`] node.
295    ///
296    /// The flag only describes that one edge: it is set while a `FusedSeries` node is visited, so
297    /// the node visited next — that node's own input — sees it, and a node further down does not.
298    fused_series_input: bool,
299    fused_series_input_stack: Vec<bool>,
300}
301
302impl TreeNodeRewriter for ScanHintRewriter {
303    type Node = LogicalPlan;
304
305    fn f_down(&mut self, node: LogicalPlan) -> Result<Transformed<LogicalPlan>> {
306        self.order_stack.push(self.order_expr.clone());
307        self.ts_stack.push(self.ts_row_selector.clone());
308        self.single_evaluation_stack
309            .push(self.inside_single_evaluation);
310        self.fused_series_input_stack.push(self.fused_series_input);
311        // Read the edge this node sits on before recording what this node is, so a `FusedSeries`
312        // node hands the flag to its own input and to nothing below it.
313        let fused_series_input = std::mem::replace(
314            &mut self.fused_series_input,
315            matches!(&node, LogicalPlan::Extension(extension)
316                if extension.node.as_any().is::<FusedSeries>()
317                    || extension.node.as_any().is::<RawSeriesSource>()),
318        );
319
320        if let LogicalPlan::Sort(sort) = &node {
321            self.order_expr = Some(sort.expr.clone());
322        }
323        if let LogicalPlan::Extension(extension) = &node
324            && let Some(instant) = extension.node.as_any().downcast_ref::<InstantManipulate>()
325        {
326            self.inside_single_evaluation = instant.is_single_evaluation();
327        } else if self.inside_single_evaluation {
328            // This allowlist is coupled to the controlled PromQL planner. It
329            // permits only nodes known to preserve the newest row per series;
330            // every other node is a sticky boundary until a nested instant
331            // extension establishes a new evaluation scope.
332            self.inside_single_evaluation = single_evaluation_node_allowed(&node);
333        }
334        if let LogicalPlan::Aggregate(aggregate) = &node {
335            self.ts_row_selector = Self::extract_last_value_selector(aggregate);
336        }
337
338        let is_branching = matches!(
339            node,
340            LogicalPlan::Subquery(_) | LogicalPlan::SubqueryAlias(_)
341        ) || node.inputs().len() > 1;
342        if is_branching {
343            self.ts_row_selector = None;
344        }
345        if let LogicalPlan::Filter(filter) = &node
346            && let Some(group_by_exprs) = &self.ts_row_selector
347        {
348            let mut referenced = HashSet::default();
349            utils::expr_to_columns(&filter.predicate, &mut referenced)?;
350            if !referenced.is_subset(&group_by_exprs.0) {
351                self.ts_row_selector = None;
352            }
353        }
354
355        ScanHintRule::set_hints(node, self, fused_series_input)
356    }
357
358    fn f_up(&mut self, node: LogicalPlan) -> Result<Transformed<LogicalPlan>> {
359        if let Some(previous) = self.order_stack.pop() {
360            self.order_expr = previous;
361        }
362        if let Some(previous) = self.ts_stack.pop() {
363            self.ts_row_selector = previous;
364        }
365        if let Some(previous) = self.single_evaluation_stack.pop() {
366            self.inside_single_evaluation = previous;
367        }
368        if let Some(previous) = self.fused_series_input_stack.pop() {
369            self.fused_series_input = previous;
370        }
371        Ok(Transformed::no(node))
372    }
373}
374
375/// Returns whether a plan node can occur on the scan path of a controlled
376/// PromQL single evaluation without changing which row is newest per series.
377fn single_evaluation_node_allowed(node: &LogicalPlan) -> bool {
378    match node {
379        LogicalPlan::TableScan(_) | LogicalPlan::SubqueryAlias(_) => true,
380        LogicalPlan::Sort(sort) => sort.fetch.is_none(),
381        LogicalPlan::Projection(projection) => projection
382            .expr
383            .iter()
384            .all(|expr| single_evaluation_projection_expr_allowed(expr, projection)),
385        LogicalPlan::Extension(extension) => {
386            let extension = extension.node.as_any();
387            extension.is::<SeriesDivide>()
388                || extension
389                    .downcast_ref::<SeriesNormalize>()
390                    .is_some_and(|normalize| !normalize.filter_stale_markers())
391        }
392        _ => false,
393    }
394}
395
396/// This whitelist assumes the planner preserves time-index and series identity;
397/// it is not a proof that an arbitrary plan does so.
398///
399/// Identity projections preserve the selected samples. Only the planner's named
400/// seconds/milliseconds-to-milliseconds casts are accepted; a microsecond or
401/// nanosecond cast could collapse a future sample onto the evaluation boundary.
402fn single_evaluation_projection_expr_allowed(
403    expr: &Expr,
404    projection: &datafusion_expr::logical_plan::Projection,
405) -> bool {
406    match expr {
407        Expr::Column(_) => true,
408        Expr::Alias(alias) => match alias.expr.as_ref() {
409            Expr::Column(column) => alias.name == column.name,
410            Expr::Cast(cast) => {
411                let Expr::Column(column) = cast.expr.as_ref() else {
412                    return false;
413                };
414                alias.name == column.name
415                    && matches!(
416                        cast.field.data_type(),
417                        DataType::Timestamp(ArrowTimeUnit::Millisecond, None)
418                    )
419                    && matches!(
420                        projection
421                            .input
422                            .schema()
423                            .qualified_field_from_column(column),
424                        Ok((_, field))
425                            if matches!(field.data_type(), DataType::Timestamp(ArrowTimeUnit::Second | ArrowTimeUnit::Millisecond, None))
426                    )
427            }
428            _ => false,
429        },
430        _ => false,
431    }
432}
433
434impl ScanHintRewriter {
435    fn extract_last_value_selector(
436        aggregate: &datafusion_expr::logical_plan::Aggregate,
437    ) -> Option<(HashSet<Column>, Column)> {
438        let mut order_by_expr = None;
439        if aggregate.aggr_expr.is_empty() {
440            return None;
441        }
442        for expr in &aggregate.aggr_expr {
443            let Expr::AggregateFunction(func) = expr else {
444                return None;
445            };
446            if (func.func.name() != "last_value"
447                && func.func.name() != aggr_state_func_name("last_value"))
448                || func.params.filter.is_some()
449                || func.params.distinct
450                // A shared final-row selector would discard earlier nonnull values.
451                || func.params.null_treatment == Some(NullTreatment::IgnoreNulls)
452            {
453                return None;
454            }
455            let order_by = &func.params.order_by;
456            if order_by.len() != 1 || !order_by[0].asc {
457                return None;
458            }
459            if let Some(existing) = &order_by_expr {
460                if existing != &order_by[0] {
461                    return None;
462                }
463            } else {
464                order_by_expr = Some(order_by[0].clone());
465            }
466        }
467        let Expr::Column(order_by_col) = order_by_expr?.expr else {
468            return None;
469        };
470        let mut group_by_cols = HashSet::with_capacity(aggregate.group_expr.len());
471        for expr in &aggregate.group_expr {
472            let Expr::Column(col) = expr else {
473                return None;
474            };
475            group_by_cols.insert(col.clone());
476        }
477        Some((group_by_cols, order_by_col))
478    }
479}
480
481#[cfg(test)]
482mod test {
483    use std::collections::HashMap;
484    use std::sync::Arc;
485    use std::time::{Duration, UNIX_EPOCH};
486
487    use datafusion::functions_aggregate::first_last::last_value_udaf;
488    use datafusion::functions_aggregate::min_max::max_udaf;
489    use datafusion::functions_window::row_number::RowNumber;
490    use datafusion::logical_expr::expr::WindowFunction;
491    use datafusion::logical_expr::{WindowFrame, WindowFunctionDefinition};
492    use datafusion::prelude::JoinType;
493    use datafusion_common::tree_node::TreeNodeRecursion;
494    use datafusion_common::{DFSchema, DFSchemaRef};
495    use datafusion_expr::expr::{
496        AggregateFunction, AggregateFunctionParams, Cast, NullTreatment, WindowFunctionParams,
497    };
498    use datafusion_expr::expr_fn::scalar_subquery;
499    use datafusion_expr::{Extension, LogicalPlan, LogicalPlanBuilder, col, lit};
500    use datafusion_optimizer::OptimizerContext;
501    use datatypes::arrow::datatypes::{DataType, Field, TimeUnit};
502    use datatypes::data_type::ConcreteDataType;
503    use datatypes::schema::ColumnSchema;
504    use datatypes::series::SeriesOrdering;
505    use datatypes::series::samples::{FieldSpec, SeriesSchema};
506    use promql::extension_plan::{FusedSeries, FusedSeriesBinding, RangeManipulate};
507    use promql_parser::parser;
508    use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
509    use store_api::metric_engine_consts::DATA_SCHEMA_TSID_COLUMN_NAME;
510    use store_api::storage::{RegionId, TimeSeriesRowSelector};
511
512    use super::*;
513    use crate::promql::native_compile::{compile, statement_grid};
514
515    fn scan_requests(plan: &LogicalPlan) -> Vec<store_api::storage::ScanRequest> {
516        scan_requests_with_names(plan)
517            .into_iter()
518            .map(|(_, request)| request)
519            .collect()
520    }
521
522    fn scan_requests_with_names(
523        plan: &LogicalPlan,
524    ) -> Vec<(String, store_api::storage::ScanRequest)> {
525        let mut requests = Vec::new();
526        plan.apply_with_subqueries(|node| {
527            if let LogicalPlan::TableScan(scan) = node
528                && let Some(source) = scan.source.downcast_ref::<DefaultTableSource>()
529                && let Some(provider) = source.table_provider.downcast_ref::<DummyTableProvider>()
530            {
531                requests.push((scan.table_name.to_string(), provider.scan_request()));
532            }
533            Ok(TreeNodeRecursion::Continue)
534        })
535        .unwrap();
536        requests
537    }
538
539    fn instant_plan(provider: Arc<DummyTableProvider>, end: i64) -> LogicalPlan {
540        instant_plan_named(provider, "t", end)
541    }
542
543    fn instant_plan_with_filters(
544        provider: Arc<DummyTableProvider>,
545        filters: Vec<Expr>,
546    ) -> LogicalPlan {
547        let scan = scan_plan(provider, "t");
548        let LogicalPlan::TableScan(mut scan) = scan else {
549            unreachable!();
550        };
551        scan.filters = filters;
552        LogicalPlan::Extension(Extension {
553            node: Arc::new(InstantManipulate::new(
554                1000,
555                1000,
556                1000,
557                1000,
558                0,
559                "ts".to_string(),
560                vec![],
561                Some("v0".to_string()),
562                LogicalPlan::TableScan(scan),
563            )),
564        })
565    }
566
567    fn scan_plan(provider: Arc<DummyTableProvider>, table_name: &str) -> LogicalPlan {
568        LogicalPlanBuilder::scan(
569            table_name,
570            Arc::new(DefaultTableSource::new(provider)),
571            None,
572        )
573        .unwrap()
574        .build()
575        .unwrap()
576    }
577
578    fn mock_table_provider_with_timestamp(
579        region_id: RegionId,
580        timestamp_type: ConcreteDataType,
581    ) -> DummyTableProvider {
582        let mut builder = RegionMetadataBuilder::new(region_id);
583        builder
584            .push_column_metadata(ColumnMetadata {
585                column_schema: ColumnSchema::new("k0", ConcreteDataType::string_datatype(), true),
586                semantic_type: SemanticType::Tag,
587                column_id: 1,
588            })
589            .push_column_metadata(ColumnMetadata {
590                column_schema: ColumnSchema::new("ts", timestamp_type, false),
591                semantic_type: SemanticType::Timestamp,
592                column_id: 2,
593            })
594            .push_column_metadata(ColumnMetadata {
595                column_schema: ColumnSchema::new("v0", ConcreteDataType::float64_datatype(), false),
596                semantic_type: SemanticType::Field,
597                column_id: 3,
598            })
599            .primary_key(vec![1]);
600        let metadata = Arc::new(builder.build().unwrap());
601        let engine = Arc::new(MetaRegionEngine::with_metadata(metadata.clone()));
602        DummyTableProvider::new(region_id, engine, metadata)
603    }
604
605    fn last_value_aggregate(input: LogicalPlan) -> LogicalPlan {
606        last_value_aggregate_with_null_treatments(input, [None, None])
607    }
608
609    fn last_value_aggregate_with_null_treatments(
610        input: LogicalPlan,
611        null_treatments: [Option<NullTreatment>; 2],
612    ) -> LogicalPlan {
613        let aggregate = LogicalPlanBuilder::from(input)
614            .aggregate(
615                vec![col("k0")],
616                vec![
617                    Expr::AggregateFunction(AggregateFunction {
618                        func: last_value_udaf(),
619                        params: AggregateFunctionParams {
620                            args: vec![col("v0")],
621                            distinct: false,
622                            filter: None,
623                            order_by: vec![Sort {
624                                expr: col("ts"),
625                                asc: true,
626                                nulls_first: true,
627                            }],
628                            null_treatment: null_treatments[0],
629                        },
630                    }),
631                    Expr::AggregateFunction(AggregateFunction {
632                        func: last_value_udaf(),
633                        params: AggregateFunctionParams {
634                            args: vec![col("ts")],
635                            distinct: false,
636                            filter: None,
637                            order_by: vec![Sort {
638                                expr: col("ts"),
639                                asc: true,
640                                nulls_first: true,
641                            }],
642                            null_treatment: null_treatments[1],
643                        },
644                    }),
645                ],
646            )
647            .unwrap()
648            .build()
649            .unwrap();
650        let timestamp = aggregate.schema().field(2).name().clone();
651
652        LogicalPlanBuilder::from(aggregate)
653            .project(vec![col("k0"), col(timestamp).alias("ts")])
654            .unwrap()
655            .build()
656            .unwrap()
657    }
658
659    fn single_evaluation(input: LogicalPlan) -> LogicalPlan {
660        LogicalPlan::Extension(Extension {
661            node: Arc::new(InstantManipulate::new(
662                1000,
663                1000,
664                1000,
665                1000,
666                0,
667                "ts".to_string(),
668                vec![],
669                Some("v0".to_string()),
670                input,
671            )),
672        })
673    }
674
675    fn instant_with_expression_subquery(
676        outer_provider: Arc<DummyTableProvider>,
677        inner_plan: LogicalPlan,
678        outer_end: i64,
679    ) -> LogicalPlan {
680        let outer_scan = scan_plan(outer_provider, "outer");
681        let input = LogicalPlanBuilder::from(outer_scan)
682            .project(vec![
683                col("ts"),
684                col("v0"),
685                scalar_subquery(Arc::new(inner_plan)),
686            ])
687            .unwrap()
688            .build()
689            .unwrap();
690        LogicalPlan::Extension(Extension {
691            node: Arc::new(InstantManipulate::new(
692                1000,
693                outer_end,
694                1000,
695                1000,
696                0,
697                "ts".to_string(),
698                vec![],
699                Some("v0".to_string()),
700                input,
701            )),
702        })
703    }
704
705    fn instant_plan_named(
706        provider: Arc<DummyTableProvider>,
707        table_name: &str,
708        end: i64,
709    ) -> LogicalPlan {
710        let input = LogicalPlanBuilder::scan(
711            table_name,
712            Arc::new(DefaultTableSource::new(provider)),
713            None,
714        )
715        .unwrap()
716        .build()
717        .unwrap();
718        LogicalPlan::Extension(Extension {
719            node: Arc::new(InstantManipulate::new(
720                1000,
721                end,
722                1000,
723                1000,
724                0,
725                "ts".to_string(),
726                vec![],
727                Some("v0".to_string()),
728                input,
729            )),
730        })
731    }
732    use crate::optimizer::test_util::{
733        MetaRegionEngine, mock_table_provider, mock_table_provider_with_tsid,
734    };
735
736    #[test]
737    fn single_evaluation_sets_last_row_on_the_rewritten_scan() {
738        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
739        let plan = instant_plan(provider.clone(), 1000);
740        let rewritten = ScanHintRule
741            .rewrite(plan, &OptimizerContext::default())
742            .unwrap()
743            .data;
744
745        assert_eq!(
746            scan_requests(&rewritten)[0].series_row_selector,
747            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
748        );
749
750        assert_eq!(provider.scan_request().series_row_selector, None);
751    }
752
753    #[test]
754    fn single_evaluation_limit_sort_does_not_set_last_row_below_limit() {
755        let mut selectors = Vec::new();
756        for input in [
757            // Instant(1000, lookback=1000) -> Limit(0, 1) -> Sort(ts ASC) -> Scan.
758            LogicalPlanBuilder::from(scan_plan(
759                Arc::new(mock_table_provider(RegionId::new(1, 1))),
760                "t",
761            ))
762            .sort(vec![col("ts").sort(true, false)])
763            .unwrap()
764            .limit(0, Some(1))
765            .unwrap()
766            .build()
767            .unwrap(),
768            // Sort.fetch is a limit embedded in the Sort node and has the same boundary.
769            LogicalPlanBuilder::from(scan_plan(
770                Arc::new(mock_table_provider(RegionId::new(1, 1))),
771                "t",
772            ))
773            .sort_with_limit(vec![col("ts").sort(true, false)], Some(1))
774            .unwrap()
775            .build()
776            .unwrap(),
777        ] {
778            let rewritten = ScanHintRule
779                .rewrite(single_evaluation(input), &OptimizerContext::default())
780                .unwrap()
781                .data;
782            selectors.push(scan_requests(&rewritten)[0].series_row_selector);
783        }
784
785        // With samples (ts=100, v=10) and (ts=900, v=1), an unhinted ascending
786        // sort/limit pipeline returns 10. LastRow at the scan instead leaves only
787        // (900, 1) before the limit. A LastRow scan hint cannot cross either limit.
788        assert_eq!(selectors, vec![None, None]);
789    }
790
791    #[test]
792    fn single_evaluation_allows_controlled_promql_selector_chain() {
793        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
794        let projection = LogicalPlanBuilder::from(scan_plan(provider, "t"))
795            .project(vec![
796                col("k0").alias("k0"),
797                Expr::Cast(Cast::new(
798                    Box::new(col("ts")),
799                    DataType::Timestamp(ArrowTimeUnit::Millisecond, None),
800                ))
801                .alias("ts"),
802                col("v0"),
803            ])
804            .unwrap()
805            .sort(vec![col("ts").sort(true, false)])
806            .unwrap()
807            .build()
808            .unwrap();
809        let divide = LogicalPlan::Extension(Extension {
810            node: Arc::new(SeriesDivide::new(
811                vec!["k0".to_string()],
812                "ts".to_string(),
813                projection,
814            )),
815        });
816        let normalize = LogicalPlan::Extension(Extension {
817            node: Arc::new(SeriesNormalize::new(
818                42,
819                "ts",
820                false,
821                vec!["k0".to_string()],
822                divide,
823            )),
824        });
825        let rewritten = ScanHintRule
826            .rewrite(single_evaluation(normalize), &OptimizerContext::default())
827            .unwrap()
828            .data;
829
830        assert_eq!(
831            scan_requests(&rewritten)[0].series_row_selector,
832            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
833        );
834    }
835
836    #[test]
837    fn single_evaluation_filtering_normalize_does_not_set_last_row() {
838        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
839        let normalize = LogicalPlan::Extension(Extension {
840            node: Arc::new(SeriesNormalize::new(
841                42,
842                "ts",
843                true,
844                vec!["k0".to_string()],
845                scan_plan(provider, "t"),
846            )),
847        });
848        let rewritten = ScanHintRule
849            .rewrite(single_evaluation(normalize), &OptimizerContext::default())
850            .unwrap()
851            .data;
852
853        // An older finite sample can precede a newest stale marker. LastRow
854        // would discard that sample before SeriesNormalize filters the marker.
855        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
856    }
857
858    #[test]
859    fn single_evaluation_rejects_row_changing_nodes() {
860        let provider = || Arc::new(mock_table_provider(RegionId::new(1, 1)));
861        let window = LogicalPlanBuilder::from(scan_plan(provider(), "window"))
862            .window(vec![Expr::WindowFunction(Box::new(WindowFunction {
863                fun: WindowFunctionDefinition::WindowUDF(Arc::new(RowNumber::new().into())),
864                params: WindowFunctionParams {
865                    args: vec![],
866                    partition_by: vec![col("k0")],
867                    order_by: vec![col("ts").sort(true, true)],
868                    window_frame: WindowFrame::new(Some(true)),
869                    filter: None,
870                    null_treatment: None,
871                    distinct: false,
872                },
873            }))])
874            .unwrap()
875            .build()
876            .unwrap();
877        let join = LogicalPlanBuilder::from(scan_plan(provider(), "left"))
878            .join(
879                scan_plan(provider(), "right"),
880                JoinType::Inner,
881                (Vec::<Column>::new(), Vec::<Column>::new()),
882                None,
883            )
884            .unwrap()
885            .project(vec![
886                Expr::Column(Column::new(Some("left"), "ts")),
887                Expr::Column(Column::new(Some("left"), "v0")),
888            ])
889            .unwrap()
890            .build()
891            .unwrap();
892        let nonlast_aggregate = LogicalPlanBuilder::from(scan_plan(provider(), "aggregate"))
893            .aggregate(
894                vec![col("k0")],
895                vec![
896                    Expr::AggregateFunction(AggregateFunction {
897                        func: max_udaf(),
898                        params: AggregateFunctionParams {
899                            args: vec![col("v0")],
900                            distinct: false,
901                            filter: None,
902                            order_by: vec![],
903                            null_treatment: None,
904                        },
905                    }),
906                    Expr::AggregateFunction(AggregateFunction {
907                        func: max_udaf(),
908                        params: AggregateFunctionParams {
909                            args: vec![col("ts")],
910                            distinct: false,
911                            filter: None,
912                            order_by: vec![],
913                            null_treatment: None,
914                        },
915                    })
916                    .alias("ts"),
917                ],
918            )
919            .unwrap()
920            .build()
921            .unwrap();
922        let range = LogicalPlan::Extension(Extension {
923            node: Arc::new(
924                RangeManipulate::new(
925                    1000,
926                    1000,
927                    1000,
928                    0,
929                    1000,
930                    "ts".to_string(),
931                    vec!["v0".to_string()],
932                    scan_plan(provider(), "range"),
933                )
934                .unwrap(),
935            ),
936        });
937
938        for (plan, scan_count) in [(window, 1), (join, 2), (nonlast_aggregate, 1), (range, 1)] {
939            let rewritten = ScanHintRule
940                .rewrite(single_evaluation(plan), &OptimizerContext::default())
941                .unwrap()
942                .data;
943            assert_eq!(
944                scan_requests(&rewritten)
945                    .into_iter()
946                    .map(|request| request.series_row_selector)
947                    .collect::<Vec<_>>(),
948                vec![None; scan_count]
949            );
950        }
951    }
952
953    #[test]
954    fn last_value_null_treatment_controls_selector() {
955        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
956        let context = OptimizerContext::default();
957
958        for treatment in [None, Some(NullTreatment::RespectNulls)] {
959            let rewritten = ScanHintRule
960                .rewrite(
961                    last_value_aggregate_with_null_treatments(
962                        scan_plan(provider.clone(), "t"),
963                        [treatment, treatment],
964                    ),
965                    &context,
966                )
967                .unwrap()
968                .data;
969            assert_eq!(
970                scan_requests(&rewritten)[0].series_row_selector,
971                Some(TimeSeriesRowSelector::LastRow { after_merge: false })
972            );
973        }
974
975        // One IGNORE NULLS member prevents the shared scan selector for every
976        // aggregate in the group, including the member that respects nulls.
977        let rewritten = ScanHintRule
978            .rewrite(
979                last_value_aggregate_with_null_treatments(
980                    scan_plan(provider, "t"),
981                    [None, Some(NullTreatment::IgnoreNulls)],
982                ),
983                &context,
984            )
985            .unwrap()
986            .data;
987        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
988    }
989
990    #[test]
991    fn single_evaluation_last_value_aggregate_keeps_legacy_selector() {
992        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
993        let rewritten = ScanHintRule
994            .rewrite(
995                single_evaluation(last_value_aggregate(scan_plan(provider, "t"))),
996                &OptimizerContext::default(),
997            )
998            .unwrap()
999            .data;
1000
1001        assert_eq!(
1002            scan_requests(&rewritten)[0].series_row_selector,
1003            Some(TimeSeriesRowSelector::LastRow { after_merge: false })
1004        );
1005    }
1006
1007    #[test]
1008    fn single_evaluation_allows_second_to_millisecond_time_index_cast() {
1009        let provider = Arc::new(mock_table_provider_with_timestamp(
1010            RegionId::new(1, 1),
1011            ConcreteDataType::timestamp_second_datatype(),
1012        ));
1013        let input = LogicalPlanBuilder::from(scan_plan(provider, "t"))
1014            .project(vec![
1015                Expr::Cast(Cast::new(
1016                    Box::new(Expr::Column(Column::new(Some("t"), "ts"))),
1017                    DataType::Timestamp(ArrowTimeUnit::Millisecond, None),
1018                ))
1019                .alias("ts"),
1020            ])
1021            .unwrap()
1022            .build()
1023            .unwrap();
1024        let rewritten = ScanHintRule
1025            .rewrite(single_evaluation(input), &OptimizerContext::default())
1026            .unwrap()
1027            .data;
1028
1029        assert_eq!(
1030            scan_requests(&rewritten)[0].series_row_selector,
1031            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1032        );
1033    }
1034
1035    #[test]
1036    fn single_evaluation_uses_last_row_for_microsecond_and_nanosecond_time_indexes() {
1037        for timestamp_type in [
1038            ConcreteDataType::timestamp_microsecond_datatype(),
1039            ConcreteDataType::timestamp_nanosecond_datatype(),
1040        ] {
1041            let direct_provider = Arc::new(mock_table_provider_with_timestamp(
1042                RegionId::new(1, 1),
1043                timestamp_type.clone(),
1044            ));
1045            let direct = ScanHintRule
1046                .rewrite(
1047                    single_evaluation(scan_plan(direct_provider, "direct")),
1048                    &OptimizerContext::default(),
1049                )
1050                .unwrap()
1051                .data;
1052            assert_eq!(
1053                scan_requests(&direct)[0].series_row_selector,
1054                Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1055            );
1056
1057            let projection_provider = Arc::new(mock_table_provider_with_timestamp(
1058                RegionId::new(1, 1),
1059                timestamp_type,
1060            ));
1061            let projection = LogicalPlanBuilder::from(scan_plan(projection_provider, "projection"))
1062                .project(vec![col("ts")])
1063                .unwrap()
1064                .build()
1065                .unwrap();
1066            let projected = ScanHintRule
1067                .rewrite(single_evaluation(projection), &OptimizerContext::default())
1068                .unwrap()
1069                .data;
1070            assert_eq!(
1071                scan_requests(&projected)[0].series_row_selector,
1072                Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1073            );
1074        }
1075    }
1076
1077    #[test]
1078    fn single_evaluation_rejects_lossy_microsecond_and_nanosecond_time_index_casts() {
1079        for timestamp_type in [
1080            ConcreteDataType::timestamp_microsecond_datatype(),
1081            ConcreteDataType::timestamp_nanosecond_datatype(),
1082        ] {
1083            let provider = Arc::new(mock_table_provider_with_timestamp(
1084                RegionId::new(1, 1),
1085                timestamp_type,
1086            ));
1087            let input = LogicalPlanBuilder::from(scan_plan(provider, "t"))
1088                .project(vec![
1089                    Expr::Cast(Cast::new(
1090                        Box::new(Expr::Column(Column::new(Some("t"), "ts"))),
1091                        DataType::Timestamp(ArrowTimeUnit::Millisecond, None),
1092                    ))
1093                    .alias("ts"),
1094                ])
1095                .unwrap()
1096                .build()
1097                .unwrap();
1098            let rewritten = ScanHintRule
1099                .rewrite(single_evaluation(input), &OptimizerContext::default())
1100                .unwrap()
1101                .data;
1102
1103            assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1104        }
1105    }
1106
1107    #[test]
1108    fn single_evaluation_rejects_unresolved_time_index_cast_qualifier() {
1109        let provider = Arc::new(mock_table_provider_with_timestamp(
1110            RegionId::new(1, 1),
1111            ConcreteDataType::timestamp_second_datatype(),
1112        ));
1113        let input = LogicalPlanBuilder::from(scan_plan(provider, "t"))
1114            .project(vec![col("ts")])
1115            .unwrap()
1116            .build()
1117            .unwrap();
1118        let LogicalPlan::Projection(projection) = input else {
1119            unreachable!();
1120        };
1121        let unresolved_cast = Expr::Cast(Cast::new(
1122            Box::new(Expr::Column(Column::new(Some("missing"), "ts"))),
1123            DataType::Timestamp(ArrowTimeUnit::Millisecond, None),
1124        ))
1125        .alias("ts");
1126
1127        assert!(!single_evaluation_projection_expr_allowed(
1128            &unresolved_cast,
1129            &projection
1130        ));
1131    }
1132
1133    #[test]
1134    fn single_evaluation_rejects_projection_expressions_that_change_rows() {
1135        let invalid_projections = [
1136            vec![col("ts").alias("renamed"), col("ts")],
1137            vec![
1138                Expr::BinaryExpr(datafusion_expr::expr::BinaryExpr::new(
1139                    Box::new(col("v0")),
1140                    datafusion_expr::Operator::Plus,
1141                    Box::new(lit(1.0_f64)),
1142                ))
1143                .alias("v0"),
1144                col("ts"),
1145            ],
1146            vec![
1147                Expr::Cast(Cast::new(
1148                    Box::new(col("ts")),
1149                    DataType::Timestamp(ArrowTimeUnit::Second, None),
1150                ))
1151                .alias("ts"),
1152            ],
1153            vec![
1154                Expr::Cast(Cast::new(Box::new(col("v0")), DataType::Int64)).alias("v0"),
1155                col("ts"),
1156            ],
1157            vec![
1158                Expr::Cast(Cast::new(
1159                    Box::new(col("ts")),
1160                    DataType::Timestamp(ArrowTimeUnit::Microsecond, None),
1161                ))
1162                .alias("ts"),
1163            ],
1164            vec![
1165                Expr::Cast(Cast::new(
1166                    Box::new(col("ts")),
1167                    DataType::Timestamp(ArrowTimeUnit::Nanosecond, None),
1168                ))
1169                .alias("ts"),
1170            ],
1171        ];
1172
1173        for expressions in invalid_projections {
1174            let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1175            let input = LogicalPlanBuilder::from(scan_plan(provider, "t"))
1176                .project(expressions)
1177                .unwrap()
1178                .build()
1179                .unwrap();
1180            let rewritten = ScanHintRule
1181                .rewrite(single_evaluation(input), &OptimizerContext::default())
1182                .unwrap()
1183                .data;
1184
1185            assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1186        }
1187    }
1188
1189    #[test]
1190    fn range_evaluation_does_not_set_last_row() {
1191        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1192        let plan = instant_plan(provider.clone(), 2000);
1193        let rewritten = ScanHintRule
1194            .rewrite(plan, &OptimizerContext::default())
1195            .unwrap()
1196            .data;
1197
1198        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1199
1200        assert_eq!(provider.scan_request().series_row_selector, None);
1201    }
1202
1203    #[test]
1204    fn residual_field_filter_does_not_set_last_row() {
1205        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1206        let input = LogicalPlanBuilder::from(scan_plan(provider, "t"))
1207            .filter(col("v0").gt(lit(1.0_f64)))
1208            .unwrap()
1209            .build()
1210            .unwrap();
1211        let rewritten = ScanHintRule
1212            .rewrite(single_evaluation(input), &OptimizerContext::default())
1213            .unwrap()
1214            .data;
1215
1216        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1217    }
1218
1219    #[test]
1220    fn outer_residual_filter_does_not_block_last_row() {
1221        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1222        let plan = LogicalPlanBuilder::from(single_evaluation(scan_plan(provider, "t")))
1223            .filter(col("v0").gt(lit(1.0_f64)))
1224            .unwrap()
1225            .build()
1226            .unwrap();
1227        let rewritten = ScanHintRule
1228            .rewrite(plan, &OptimizerContext::default())
1229            .unwrap()
1230            .data;
1231
1232        assert_eq!(
1233            scan_requests(&rewritten)[0].series_row_selector,
1234            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1235        );
1236    }
1237
1238    #[test]
1239    fn branch_local_residual_filter_isolation_in_both_orders() {
1240        for filtered_first in [true, false] {
1241            let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1242            let filtered = single_evaluation(
1243                LogicalPlanBuilder::from(scan_plan(provider.clone(), "filtered"))
1244                    .filter(col("v0").gt(lit(1.0_f64)))
1245                    .unwrap()
1246                    .build()
1247                    .unwrap(),
1248            );
1249            let plain = single_evaluation(scan_plan(provider.clone(), "plain"));
1250            let union = if filtered_first {
1251                LogicalPlanBuilder::from(filtered)
1252                    .union(plain)
1253                    .unwrap()
1254                    .build()
1255                    .unwrap()
1256            } else {
1257                LogicalPlanBuilder::from(plain)
1258                    .union(filtered)
1259                    .unwrap()
1260                    .build()
1261                    .unwrap()
1262            };
1263            let rewritten = ScanHintRule
1264                .rewrite(union, &OptimizerContext::default())
1265                .unwrap()
1266                .data;
1267            let requests = scan_requests_with_names(&rewritten)
1268                .into_iter()
1269                .collect::<HashMap<_, _>>();
1270
1271            assert_eq!(requests["filtered"].series_row_selector, None);
1272            assert_eq!(
1273                requests["plain"].series_row_selector,
1274                Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1275            );
1276            assert_eq!(provider.scan_request().series_row_selector, None);
1277        }
1278    }
1279
1280    #[test]
1281    fn union_inside_single_evaluation_blocks_both_branches() {
1282        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1283        let union = LogicalPlanBuilder::from(scan_plan(provider.clone(), "left"))
1284            .union(scan_plan(provider, "right"))
1285            .unwrap()
1286            .build()
1287            .unwrap();
1288        let rewritten = ScanHintRule
1289            .rewrite(single_evaluation(union), &OptimizerContext::default())
1290            .unwrap()
1291            .data;
1292
1293        assert_eq!(
1294            scan_requests(&rewritten)
1295                .into_iter()
1296                .map(|request| request.series_row_selector)
1297                .collect::<Vec<_>>(),
1298            vec![None, None]
1299        );
1300    }
1301
1302    #[test]
1303    fn residual_time_filters_do_not_set_last_row() {
1304        for predicate in [
1305            col("ts").lt(lit(1_i64)),
1306            Expr::Cast(Cast::new(Box::new(col("ts")), DataType::Int64)).gt(lit(1_i64)),
1307        ] {
1308            let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1309            let input = LogicalPlanBuilder::from(scan_plan(provider, "t"))
1310                .filter(predicate)
1311                .unwrap()
1312                .build()
1313                .unwrap();
1314            let rewritten = ScanHintRule
1315                .rewrite(single_evaluation(input), &OptimizerContext::default())
1316                .unwrap()
1317                .data;
1318
1319            assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1320        }
1321    }
1322
1323    #[test]
1324    fn inner_single_evaluation_resets_residual_filter() {
1325        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1326        let filtered = LogicalPlanBuilder::from(single_evaluation(scan_plan(provider, "t")))
1327            .filter(col("v0").gt(lit(1.0_f64)))
1328            .unwrap()
1329            .build()
1330            .unwrap();
1331        let rewritten = ScanHintRule
1332            .rewrite(single_evaluation(filtered), &OptimizerContext::default())
1333            .unwrap()
1334            .data;
1335
1336        assert_eq!(
1337            scan_requests(&rewritten)[0].series_row_selector,
1338            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1339        );
1340    }
1341
1342    #[test]
1343    fn single_evaluation_with_tag_filter_sets_last_row() {
1344        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1345        let plan = instant_plan_with_filters(provider, vec![col("k0").eq(lit("tag"))]);
1346        let rewritten = ScanHintRule
1347            .rewrite(plan, &OptimizerContext::default())
1348            .unwrap()
1349            .data;
1350
1351        assert_eq!(
1352            scan_requests(&rewritten)[0].series_row_selector,
1353            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1354        );
1355    }
1356
1357    #[test]
1358    fn single_evaluation_with_timestamp_filter_sets_last_row() {
1359        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1360        let plan = instant_plan_with_filters(provider, vec![col("ts").gt_eq(lit(1_i64))]);
1361        let rewritten = ScanHintRule
1362            .rewrite(plan, &OptimizerContext::default())
1363            .unwrap()
1364            .data;
1365
1366        assert_eq!(
1367            scan_requests(&rewritten)[0].series_row_selector,
1368            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1369        );
1370    }
1371
1372    #[test]
1373    fn single_evaluation_with_cast_timestamp_filter_does_not_set_last_row() {
1374        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1375        let filter = Expr::Cast(Cast::new(Box::new(col("ts")), DataType::Int64)).gt(lit(1_i64));
1376        let plan = instant_plan_with_filters(provider, vec![filter]);
1377        let rewritten = ScanHintRule
1378            .rewrite(plan, &OptimizerContext::default())
1379            .unwrap()
1380            .data;
1381
1382        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1383    }
1384
1385    #[test]
1386    fn single_evaluation_with_multi_column_timestamp_filter_does_not_set_last_row() {
1387        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1388        let plan = instant_plan_with_filters(provider, vec![col("ts").gt(col("k0"))]);
1389        let rewritten = ScanHintRule
1390            .rewrite(plan, &OptimizerContext::default())
1391            .unwrap()
1392            .data;
1393
1394        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1395    }
1396
1397    #[test]
1398    fn single_evaluation_with_field_filter_does_not_set_last_row() {
1399        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1400        let plan = instant_plan_with_filters(provider, vec![col("v0").gt(lit(1.0_f64))]);
1401        let rewritten = ScanHintRule
1402            .rewrite(plan, &OptimizerContext::default())
1403            .unwrap()
1404            .data;
1405
1406        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1407    }
1408
1409    #[test]
1410    fn single_evaluation_with_unknown_filter_does_not_set_last_row() {
1411        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1412        let plan = instant_plan_with_filters(provider, vec![col("unknown").eq(lit(1_i64))]);
1413        let rewritten = ScanHintRule
1414            .rewrite(plan, &OptimizerContext::default())
1415            .unwrap()
1416            .data;
1417
1418        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1419    }
1420
1421    #[test]
1422    fn expression_subquery_isolated_from_outer_single_evaluation() {
1423        let outer_provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1424        let inner_provider = Arc::new(mock_table_provider(RegionId::new(2, 1)));
1425        let plan = instant_with_expression_subquery(
1426            outer_provider.clone(),
1427            instant_plan_named(inner_provider, "inner", 2000),
1428            1000,
1429        );
1430        let rewritten = ScanHintRule
1431            .rewrite(plan, &OptimizerContext::default())
1432            .unwrap()
1433            .data;
1434
1435        let requests = scan_requests_with_names(&rewritten)
1436            .into_iter()
1437            .collect::<HashMap<_, _>>();
1438        assert_eq!(requests["outer"].series_row_selector, None);
1439        assert_eq!(requests["inner"].series_row_selector, None);
1440    }
1441
1442    #[test]
1443    fn expression_subquery_can_start_its_own_single_evaluation() {
1444        let outer_provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1445        let inner_provider = Arc::new(mock_table_provider(RegionId::new(2, 1)));
1446        let plan = instant_with_expression_subquery(
1447            outer_provider,
1448            instant_plan_named(inner_provider, "inner", 1000),
1449            1000,
1450        );
1451        let rewritten = ScanHintRule
1452            .rewrite(plan, &OptimizerContext::default())
1453            .unwrap()
1454            .data;
1455
1456        let requests = scan_requests_with_names(&rewritten)
1457            .into_iter()
1458            .collect::<HashMap<_, _>>();
1459        assert_eq!(requests["outer"].series_row_selector, None);
1460        assert_eq!(
1461            requests["inner"].series_row_selector,
1462            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1463        );
1464    }
1465
1466    #[test]
1467    fn nested_single_outer_range_inner_does_not_set_inner_last_row() {
1468        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1469        let plan = instant_plan_named(provider.clone(), "nested", 2000);
1470        let plan = LogicalPlan::Extension(Extension {
1471            node: Arc::new(InstantManipulate::new(
1472                1000,
1473                1000,
1474                1000,
1475                1000,
1476                0,
1477                "ts".to_string(),
1478                vec![],
1479                Some("v0".to_string()),
1480                plan,
1481            )),
1482        });
1483        let rewritten = ScanHintRule
1484            .rewrite(plan, &OptimizerContext::default())
1485            .unwrap()
1486            .data;
1487        assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None);
1488        assert_eq!(provider.scan_request().series_row_selector, None);
1489    }
1490
1491    #[test]
1492    fn nested_range_outer_single_inner_sets_inner_last_row() {
1493        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1494        let plan = instant_plan_named(provider.clone(), "nested", 1000);
1495        let plan = LogicalPlan::Extension(Extension {
1496            node: Arc::new(InstantManipulate::new(
1497                1000,
1498                2000,
1499                1000,
1500                1000,
1501                0,
1502                "ts".to_string(),
1503                vec![],
1504                Some("v0".to_string()),
1505                plan,
1506            )),
1507        });
1508        let rewritten = ScanHintRule
1509            .rewrite(plan, &OptimizerContext::default())
1510            .unwrap()
1511            .data;
1512        assert_eq!(
1513            scan_requests(&rewritten)[0].series_row_selector,
1514            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1515        );
1516    }
1517
1518    #[test]
1519    fn shared_provider_isolated_between_single_and_range_scans() {
1520        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1521        let plan = LogicalPlanBuilder::from(instant_plan(provider.clone(), 1000))
1522            .union(instant_plan(provider.clone(), 2000))
1523            .unwrap()
1524            .build()
1525            .unwrap();
1526        let rewritten = ScanHintRule
1527            .rewrite(plan, &OptimizerContext::default())
1528            .unwrap()
1529            .data;
1530        let requests = scan_requests(&rewritten);
1531
1532        assert_eq!(requests.len(), 2);
1533        assert_eq!(
1534            requests[0].series_row_selector,
1535            Some(TimeSeriesRowSelector::LastRow { after_merge: true })
1536        );
1537        assert_eq!(requests[1].series_row_selector, None);
1538        assert_eq!(provider.scan_request().series_row_selector, None);
1539    }
1540
1541    #[test]
1542    fn set_order_hint() {
1543        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1544        let table_source = Arc::new(DefaultTableSource::new(provider.clone()));
1545        let plan = LogicalPlanBuilder::scan("t", table_source, None)
1546            .unwrap()
1547            .sort(vec![col("ts").sort(true, false)])
1548            .unwrap()
1549            .sort(vec![col("ts").sort(false, true)])
1550            .unwrap()
1551            .build()
1552            .unwrap();
1553
1554        let context = OptimizerContext::default();
1555        let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data;
1556
1557        // should read the first (with `.sort(true, false)`) sort option
1558        let scan_req = scan_requests(&rewritten)[0].clone();
1559        assert_eq!(
1560            OrderOption {
1561                name: "ts".to_string(),
1562                options: SortOptions {
1563                    descending: false,
1564                    nulls_first: false
1565                }
1566            },
1567            scan_req.output_ordering.as_ref().unwrap()[0]
1568        );
1569    }
1570
1571    #[test]
1572    fn set_time_series_row_selector_hint() {
1573        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1574        let plan = last_value_aggregate(scan_plan(provider.clone(), "t"));
1575
1576        let context = OptimizerContext::default();
1577        let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data;
1578
1579        let scan_req = scan_requests(&rewritten)[0].clone();
1580        assert_eq!(
1581            scan_req.series_row_selector,
1582            Some(TimeSeriesRowSelector::LastRow { after_merge: false })
1583        );
1584    }
1585
1586    #[test]
1587    fn set_order_hint_sets_per_series_distribution_for_tsid_sort() {
1588        let provider = Arc::new(mock_table_provider_with_tsid(RegionId::new(1, 1)));
1589        let table_source = Arc::new(DefaultTableSource::new(provider.clone()));
1590        let plan = LogicalPlanBuilder::scan("t", table_source, None)
1591            .unwrap()
1592            .sort(vec![
1593                col(DATA_SCHEMA_TSID_COLUMN_NAME).sort(true, true),
1594                col("ts").sort(true, true),
1595            ])
1596            .unwrap()
1597            .build()
1598            .unwrap();
1599
1600        let context = OptimizerContext::default();
1601        let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data;
1602
1603        let scan_req = scan_requests(&rewritten)[0].clone();
1604        assert_eq!(
1605            scan_req.distribution,
1606            Some(TimeSeriesDistribution::PerSeries)
1607        );
1608    }
1609
1610    /// A `FusedSeries` node over `input`, bound to the tree a bare selector compiles to.
1611    ///
1612    /// The rule reads the plan shape, not what the node computes, so the simplest shape the
1613    /// compiler accepts is enough for these tests.
1614    fn fused_series(input: LogicalPlan) -> LogicalPlan {
1615        let stmt = promql_parser::parser::EvalStmt {
1616            expr: parser::parse("some_metric").unwrap(),
1617            start: UNIX_EPOCH,
1618            end: UNIX_EPOCH
1619                .checked_add(Duration::from_secs(100_000))
1620                .unwrap(),
1621            interval: Duration::from_secs(5),
1622            lookback_delta: Duration::from_secs(1),
1623        };
1624        let series_schema = Arc::new(SeriesSchema {
1625            fields: Arc::from([FieldSpec { name: "v0".into() }]),
1626            eval_grid: None,
1627        });
1628        let tree = compile(
1629            &stmt,
1630            Arc::clone(&series_schema),
1631            SeriesOrdering::KeyThenTime,
1632        )
1633        .unwrap();
1634        let binding = FusedSeriesBinding {
1635            grid: statement_grid(&stmt).unwrap(),
1636            metric_name: Some("some_metric".into()),
1637            time_index: "ts".to_string(),
1638            tags: Vec::new(),
1639        };
1640        let output_schema: DFSchemaRef = Arc::new(
1641            DFSchema::new_with_metadata(
1642                vec![
1643                    (None, Arc::new(Field::new("v0", DataType::Float64, true))),
1644                    (
1645                        None,
1646                        Arc::new(Field::new(
1647                            "ts",
1648                            DataType::Timestamp(TimeUnit::Millisecond, None),
1649                            false,
1650                        )),
1651                    ),
1652                ],
1653                Default::default(),
1654            )
1655            .unwrap(),
1656        );
1657
1658        LogicalPlan::Extension(Extension {
1659            node: Arc::new(
1660                FusedSeries::try_new(
1661                    Arc::from("some_metric"),
1662                    tree,
1663                    binding,
1664                    vec![input],
1665                    output_schema,
1666                )
1667                .unwrap(),
1668            ),
1669        })
1670    }
1671
1672    /// The region only packs raw series frames for a per-series request, and the node below is
1673    /// what reads them, so the node's own input asks for that distribution.
1674    #[test]
1675    fn a_scan_under_fused_series_asks_for_the_per_series_distribution() {
1676        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1677        let plan = fused_series(scan_plan(provider.clone(), "t"));
1678
1679        let rewritten = ScanHintRule
1680            .rewrite(plan, &OptimizerContext::default())
1681            .unwrap()
1682            .data;
1683
1684        assert_eq!(
1685            scan_requests(&rewritten)[0].distribution,
1686            Some(TimeSeriesDistribution::PerSeries)
1687        );
1688        // The hint belongs to this use-site only, never to the shared catalog provider.
1689        assert_eq!(provider.scan_request().distribution, None);
1690    }
1691
1692    /// The ordinary path: a scan with no fused node above it is left exactly as it was.
1693    #[test]
1694    fn a_scan_not_under_fused_series_keeps_its_request() {
1695        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1696        let plan = scan_plan(provider.clone(), "t");
1697
1698        let rewritten = ScanHintRule
1699            .rewrite(plan, &OptimizerContext::default())
1700            .unwrap()
1701            .data;
1702
1703        assert_eq!(scan_requests(&rewritten)[0].distribution, None);
1704        assert_eq!(provider.scan_request().distribution, None);
1705    }
1706
1707    /// Only the node's own input is a raw series reader; a scan further down feeds something else
1708    /// and keeps the request it had.
1709    #[test]
1710    fn a_scan_below_fused_series_input_keeps_its_request() {
1711        let provider = Arc::new(mock_table_provider(RegionId::new(1, 1)));
1712        let scan = scan_plan(provider.clone(), "t");
1713        let input = LogicalPlanBuilder::from(scan)
1714            .project(vec![col("ts"), col("v0")])
1715            .unwrap()
1716            .build()
1717            .unwrap();
1718        let plan = fused_series(input);
1719
1720        let rewritten = ScanHintRule
1721            .rewrite(plan, &OptimizerContext::default())
1722            .unwrap()
1723            .data;
1724
1725        assert_eq!(scan_requests(&rewritten)[0].distribution, None);
1726    }
1727}