Skip to main content

query/
planner.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::borrow::Cow;
17use std::collections::{HashMap, HashSet};
18use std::ops::ControlFlow;
19use std::str::FromStr;
20use std::sync::Arc;
21
22use arrow_schema::DataType;
23use async_trait::async_trait;
24use catalog::table_source::DfTableSourceProvider;
25use common_error::ext::BoxedError;
26use common_query::promql_annotations::promql_annotation_collector;
27use common_telemetry::tracing;
28use datafusion::common::{DFSchema, plan_err};
29use datafusion::execution::SessionStateBuilder;
30use datafusion::execution::context::SessionState;
31use datafusion::sql::planner::PlannerContext;
32use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
33use datafusion_common::{ScalarValue, ToDFSchema};
34use datafusion_expr::expr::{Exists, InSubquery};
35use datafusion_expr::{
36    Analyze, Explain, ExplainFormat, Expr as DfExpr, LogicalPlan, LogicalPlanBuilder, PlanType,
37    ToStringifiedPlan, col,
38};
39use datafusion_sql::parser::Statement as DfStatement;
40use datafusion_sql::planner::{IdentNormalizer, ParserOptions, SqlToRel};
41use log_query::LogQuery;
42use promql_parser::parser::EvalStmt;
43use session::context::QueryContextRef;
44use snafu::{ResultExt, ensure};
45use sql::CteContent;
46use sql::ast::Expr as SqlExpr;
47use sql::statements::explain::ExplainStatement;
48use sql::statements::query::Query;
49use sql::statements::statement::Statement;
50use sql::statements::tql::Tql;
51use sqlparser::ast::{AccessExpr, Value, visit_expressions_mut};
52use store_api::metric_engine_consts::DATA_SCHEMA_TSID_COLUMN_NAME;
53
54use crate::dist_plan::DistPlannerOptions;
55use crate::dist_plan::region_local_node;
56use crate::error::{
57    CteColumnSchemaMismatchSnafu, PlanSqlSnafu, QueryPlanSnafu, Result, SqlSnafu,
58    UnimplementedSnafu,
59};
60use crate::log_query::planner::LogQueryPlanner;
61use crate::optimizer::expr_depth;
62use crate::options::{NativePromqlExtension, ScheduledTimeExtension};
63use crate::parser::{DEFAULT_LOOKBACK_STRING, PromQuery, QueryLanguageParser, QueryStatement};
64use crate::promql::planner::PromPlanner;
65use crate::query_engine::{DefaultPlanDecoder, QueryEngineState};
66use crate::range_select::plan_rewrite::RangePlanRewriter;
67use crate::{DfContextProviderAdapter, QueryEngineContext};
68
69#[async_trait]
70pub trait LogicalPlanner: Send + Sync {
71    async fn plan(&self, stmt: &QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan>;
72
73    async fn plan_logs_query(
74        &self,
75        query: LogQuery,
76        query_ctx: QueryContextRef,
77    ) -> Result<LogicalPlan>;
78
79    fn optimize(&self, plan: LogicalPlan) -> Result<LogicalPlan>;
80
81    fn as_any(&self) -> &dyn Any;
82}
83
84/// A native plan a distributed planner splits over regions may place fused evaluation in one of two
85/// places: above one independent raw source per region, or inside the region whose own scan it
86/// reads.
87///
88/// The second placement is the analyzer's to give. It moves a node into a region only when it
89/// classifies the node as running where its scan is, and a bare scan is never shipped across a
90/// region boundary on its own: nothing below the boundary could read it. So a scan is admitted only
91/// under a node the analyzer would move, and every other input of the plan has to be a raw source
92/// that names the region it reads.
93///
94/// A fused node that reads no region is the one exception: it has nothing to place in a region, so
95/// the frontend evaluates it and the transport this gate protects is never used.
96fn remote_native_raw_plan_supported(plan: &LogicalPlan) -> bool {
97    /// `region_local` says the nearest enclosing node reads its scan where that scan is planned, so
98    /// the subtree below it travels inside the region's own plan.
99    fn walk(
100        plan: &LogicalPlan,
101        region_local: bool,
102        found_raw: &mut bool,
103        found_region_local: &mut bool,
104    ) -> bool {
105        match plan {
106            LogicalPlan::TableScan(_) => region_local,
107            LogicalPlan::Extension(extension) => {
108                if extension
109                    .node
110                    .as_any()
111                    .is::<promql::extension_plan::RawSeriesSource>()
112                {
113                    *found_raw = true;
114                    return matches!(
115                        extension.node.inputs().as_slice(),
116                        [LogicalPlan::TableScan(_)]
117                    );
118                }
119                let region_local = extension
120                    .node
121                    .as_any()
122                    .is::<promql::extension_plan::FusedSeries>()
123                    && region_local_node(extension.node.as_ref());
124                *found_region_local |= region_local;
125                extension
126                    .node
127                    .inputs()
128                    .iter()
129                    .all(|child| walk(child, region_local, found_raw, found_region_local))
130            }
131            _ => plan
132                .inputs()
133                .iter()
134                .all(|child| walk(child, region_local, found_raw, found_region_local)),
135        }
136    }
137    let mut found_raw = false;
138    let mut found_region_local = false;
139    walk(plan, false, &mut found_raw, &mut found_region_local)
140        && (found_raw || found_region_local || is_regionless_fused_plan(plan))
141}
142
143/// Whether `plan` is a native fused plan with no region below it.
144///
145/// Two shapes qualify, and both are already the finished plan: a statement that reads no data -
146/// `time()`, `vector(1)`, a label rewrite over one of them - is one node with no input at all,
147/// and a discovered selector that matched no metric table is planned over the typed empty frame
148/// [`is_empty_discovered_input`](crate::promql::planner::native::is_empty_discovered_input)
149/// builds. Neither has a scan to ship, so neither needs the remote transport.
150///
151/// The route stacks its own nodes above such an input: a split aggregate runs a `Final` half over
152/// the `Partial` half that reads it, and a segmented statement runs its parent over the segment's
153/// own node. So the inputs a node reads are followed through fused nodes only - never through
154/// another extension, and never through a scan - and every leaf that is not one of them rejects
155/// the shape.
156fn is_regionless_fused_plan(plan: &LogicalPlan) -> bool {
157    // `strip_tsid_column` publishes a statement's own columns over a node that reported the
158    // internal series id, which a data-free statement can synthesize with `label_replace`. That
159    // wrapper is this route's own, so it is read through when it drops exactly that one column.
160    if let LogicalPlan::Projection(projection) = plan {
161        return strips_only_tsid(plan, projection.input.as_ref())
162            && is_regionless_fused_chain(projection.input.as_ref());
163    }
164    is_regionless_fused_chain(plan)
165}
166
167/// Whether `plan` is a fused node whose inputs are fused nodes or the typed empty frame.
168fn is_regionless_fused_chain(plan: &LogicalPlan) -> bool {
169    let LogicalPlan::Extension(extension) = plan else {
170        return false;
171    };
172    let Some(fused) = extension
173        .node
174        .as_any()
175        .downcast_ref::<promql::extension_plan::FusedSeries>()
176    else {
177        return false;
178    };
179    fused.children().iter().all(|child| {
180        crate::promql::planner::native::is_empty_discovered_input(child)
181            || is_regionless_fused_chain(child)
182    })
183}
184
185/// Whether `plan` is the projection that drops exactly the internal series-id column of `inner`.
186///
187/// That is the projection `strip_tsid_column` builds: one plain column per field of its input,
188/// in order, except `__tsid__`. A computed expression, a reordering or a second dropped column is
189/// some other plan and is not read through.
190fn strips_only_tsid(plan: &LogicalPlan, inner: &LogicalPlan) -> bool {
191    let LogicalPlan::Projection(projection) = plan else {
192        return false;
193    };
194    if projection
195        .expr
196        .iter()
197        .any(|expr| !matches!(expr, DfExpr::Column(_)))
198    {
199        return false;
200    }
201    let stripped = inner
202        .schema()
203        .fields()
204        .iter()
205        .filter(|field| field.name() != DATA_SCHEMA_TSID_COLUMN_NAME)
206        .map(|field| field.name())
207        .collect::<Vec<_>>();
208    let projected = plan
209        .schema()
210        .fields()
211        .iter()
212        .map(|field| field.name())
213        .collect::<Vec<_>>();
214    projected == stripped && projected.len() + 1 == inner.schema().fields().len()
215}
216
217/// Whether `plan` runs on the native fused-series route.
218fn contains_native_fused(plan: &LogicalPlan) -> bool {
219    if let LogicalPlan::Extension(extension) = plan
220        && extension
221            .node
222            .as_any()
223            .is::<promql::extension_plan::FusedSeries>()
224    {
225        return true;
226    }
227    plan.inputs()
228        .iter()
229        .any(|input| contains_native_fused(input))
230}
231
232/// Whether `schema` is a series frame's declaration, which describes the columns below it.
233fn publishes_series_frame(schema: &DFSchema) -> bool {
234    schema
235        .metadata()
236        .keys()
237        .any(|key| key.starts_with("greptime.series."))
238}
239
240/// Whether `published` publishes `wanted`'s columns: the same names, in the same order, with the
241/// same types, nullability, per-column metadata and schema metadata.
242///
243/// Qualifiers are not compared: a CTE's plan is read through the alias the CTE registers, which
244/// names every column after the CTE on both routes.
245fn publishes_columns(published: &DFSchema, wanted: &DFSchema) -> bool {
246    published.fields().len() == wanted.fields().len()
247        && published.metadata() == wanted.metadata()
248        && published
249            .iter()
250            .zip(wanted.iter())
251            .all(|((_, published), (_, wanted))| {
252                published.name() == wanted.name()
253                    && published.data_type() == wanted.data_type()
254                    && published.is_nullable() == wanted.is_nullable()
255                    && published.metadata() == wanted.metadata()
256            })
257}
258
259/// The type a column names once its dictionary encoding is taken off.
260///
261/// A label column is dictionary-encoded in the existing planner's plan and plain text in the fused
262/// plan's. That is the one difference between the two answers for the same column that an
263/// expression can carry: a cast reproduces the encoding, and the values stay the same strings.
264fn unwrap_dictionary(data_type: &DataType) -> &DataType {
265    let mut data_type = data_type;
266    while let DataType::Dictionary(_, inner) = data_type {
267        data_type = inner.as_ref();
268    }
269    data_type
270}
271
272/// The native plan, wrapped in the projection that publishes `legacy`'s columns over it.
273///
274/// A hybrid CTE registers the plan as a table the outer SQL query reads, so the columns this plan
275/// publishes *are* the CTE's columns: their names, their order, their types, their nullability and
276/// the metadata they carry are what the statement's SQL sees, and a column list the CTE declares is
277/// applied to them by position. The two routes do not agree on that schema - the fused node takes
278/// its names from the compiled tree and publishes the time index first, while the existing planner
279/// names each value column after the expression it planned and publishes the labels first - so a CTE
280/// answered natively would make one statement mean different things depending on which route planned
281/// it. This wraps the native plan in the projection that publishes the existing planner's answer for
282/// the same statement instead.
283///
284/// One column follows the answer rather than the existing planner: a native rows answer declares the
285/// per-row metric name, which the existing planner never publishes, and that column is part of what
286/// the statement answers. It is appended after the existing planner's columns, so those keep their
287/// names, order, types and metadata, while a column list the CTE declares needs one more name - and a
288/// statement whose answer declares no name column (a scalar-typed answer) publishes none here. A CTE
289/// answered by the existing planner publishes exactly the columns it always has.
290///
291/// The projection is made of expressions - an alias per column, and a cast where the two routes carry
292/// the same values in different Arrow types - because that is what survives analysis: the analyzer
293/// rebuilds a projection's schema from its expressions, so anything the projection declared but its
294/// expressions did not carry would be gone by the time the CTE's columns are read.
295///
296/// The mapping is the identity on every column both plans publish alike - the time index and the
297/// label columns, which carry the same metadata on both routes now - and is a cast for a column
298/// whose identity is unambiguous: one native column of the same type once both are unencoded, no
299/// per-column metadata to lose and the same nullability. `None` says the two plans admit no such
300/// mapping - a differently named time index, an ambiguous set of columns left over, or a column a
301/// cast cannot reproduce faithfully - and the caller then keeps the existing plan rather than publish
302/// a column it cannot fill faithfully.
303fn publish_legacy_schema(native: LogicalPlan, legacy: &LogicalPlan) -> Option<LogicalPlan> {
304    let native_schema = native.schema().clone();
305    let legacy_schema = legacy.schema();
306
307    // A frame's declaration is schema metadata describing the columns under it, and a projection
308    // that renames and reorders them cannot keep it. A statement that publishes one keeps the
309    // existing plan.
310    if publishes_series_frame(native_schema.as_ref()) {
311        return None;
312    }
313    if publishes_columns(native_schema.as_ref(), legacy_schema.as_ref()) {
314        return Some(native);
315    }
316
317    let native_fields = native_schema
318        .iter()
319        .map(|(qualifier, field)| (qualifier.cloned(), field))
320        .collect::<Vec<_>>();
321    let mut taken = vec![false; native_fields.len()];
322    let mut source: Vec<Option<usize>> = vec![None; legacy_schema.fields().len()];
323    // First the columns both plans publish alike: the same name and the same column, metadata
324    // included. These are read as they are, so nothing about them is re-declared here.
325    for (legacy_index, (_, legacy_field)) in legacy_schema.iter().enumerate() {
326        let found =
327            native_fields
328                .iter()
329                .enumerate()
330                .find_map(|(native_index, (_, native_field))| {
331                    (!taken[native_index]
332                        && native_field.name() == legacy_field.name()
333                        && native_field.data_type() == legacy_field.data_type()
334                        && native_field.is_nullable() == legacy_field.is_nullable()
335                        && native_field.metadata() == legacy_field.metadata())
336                    .then_some(native_index)
337                });
338        if let Some(native_index) = found {
339            taken[native_index] = true;
340            source[legacy_index] = Some(native_index);
341        }
342    }
343    // Then the columns whose only difference is that the routes carry them in different Arrow
344    // types. A cast reproduces the existing planner's type, so it is only paired when exactly one
345    // native column can be that column: the same type once both are unencoded, the same
346    // nullability, and no metadata a cast would drop.
347    for (legacy_index, source) in source.iter_mut().enumerate() {
348        if source.is_some() {
349            continue;
350        }
351        let legacy_field = legacy_schema.field(legacy_index);
352        if !legacy_field.metadata().is_empty() {
353            continue;
354        }
355        let candidates = native_fields
356            .iter()
357            .enumerate()
358            .filter(|(native_index, (_, native_field))| {
359                !taken[*native_index]
360                    && unwrap_dictionary(native_field.data_type())
361                        == unwrap_dictionary(legacy_field.data_type())
362                    && native_field.is_nullable() == legacy_field.is_nullable()
363                    // A cast reproduces the field it rebuilds, and the native rows layout marks its
364                    // metric-name column with metadata a cast would drop. Such a column is never a
365                    // candidate for a legacy column: the name is the native answer's own and this
366                    // boundary publishes the existing planner's columns.
367                    && native_field.metadata().is_empty()
368            })
369            .map(|(native_index, _)| native_index)
370            .collect::<Vec<_>>();
371        if let [native_index] = candidates.as_slice() {
372            taken[*native_index] = true;
373            *source = Some(*native_index);
374        }
375    }
376    let source = source.into_iter().collect::<Option<Vec<usize>>>()?;
377
378    let mut exprs = Vec::with_capacity(source.len());
379    for ((_, legacy_field), native_index) in legacy_schema.iter().zip(source) {
380        let (native_qualifier, native_field) = &native_fields[native_index];
381        let column = DfExpr::Column(datafusion_common::Column::new(
382            native_qualifier.clone(),
383            native_field.name().clone(),
384        ));
385        // A column the two plans publish alike is read as it is; one they carry in different Arrow
386        // types is cast to the existing planner's type, which keeps the same values under the same
387        // name - and keeps them when the analyzer rebuilds this projection.
388        let value = if native_field.data_type() == legacy_field.data_type() {
389            column
390        } else {
391            DfExpr::Cast(datafusion_expr::Cast::new(
392                Box::new(column),
393                legacy_field.data_type().clone(),
394            ))
395        };
396        exprs.push(value.alias(legacy_field.name().clone()));
397    }
398    // A native rows answer declares the per-row name column, and that is part of what the statement
399    // answers, so the CTE publishes it too - after the columns above, which keep their names and
400    // order. A statement whose answer has no such column (a scalar-typed answer, for one) publishes
401    // none here either: what the CTE exposes is the answer, not a schema chosen for it.
402    let name_source = native_fields
403        .iter()
404        .position(|(_, native_field)| datatypes::schema::is_metric_name_field(native_field));
405    if let Some(native_index) = name_source {
406        let (native_qualifier, native_field) = &native_fields[native_index];
407        exprs.push(
408            DfExpr::Column(datafusion_common::Column::new(
409                native_qualifier.clone(),
410                native_field.name().clone(),
411            ))
412            .alias(native_field.name().clone()),
413        );
414    }
415    let plan = LogicalPlanBuilder::from(native)
416        .project(exprs)
417        .ok()?
418        .build()
419        .ok()?;
420    // The published columns are the ones this set out to publish, or the native plan is not used at
421    // all: what a CTE exposes may not change for a reason this did not predict. The legacy columns
422    // come first and in their order; the name column, when the answer declared one, is last.
423    let published = plan.schema();
424    let legacy_len = legacy_schema.fields().len();
425    let expected_len = legacy_len + usize::from(name_source.is_some());
426    (published.fields().len() == expected_len
427        && published.metadata() == legacy_schema.metadata()
428        && legacy_schema.iter().zip(published.iter()).all(
429            |((_, legacy_field), (_, published_field))| {
430                legacy_field.name() == published_field.name()
431                    && legacy_field.data_type() == published_field.data_type()
432                    && legacy_field.is_nullable() == published_field.is_nullable()
433                    && legacy_field.metadata() == published_field.metadata()
434            },
435        )
436        && name_source.is_none_or(|native_index| {
437            let wanted = &native_schema.fields()[native_index];
438            let published = &published.fields()[legacy_len];
439            published.name() == wanted.name()
440                && published.data_type() == wanted.data_type()
441                && published.is_nullable() == wanted.is_nullable()
442                && published.metadata() == wanted.metadata()
443                && datatypes::schema::is_metric_name_field(published.as_ref())
444        }))
445    .then_some(plan)
446}
447
448pub struct DfLogicalPlanner {
449    engine_state: Arc<QueryEngineState>,
450    session_state: SessionState,
451}
452
453impl DfLogicalPlanner {
454    pub fn new(engine_state: Arc<QueryEngineState>) -> Self {
455        let session_state = engine_state.session_state();
456        Self {
457            engine_state,
458            session_state,
459        }
460    }
461
462    /// Derive a [`SessionState`] whose [`ExecutionProps`] includes
463    /// `query_execution_start_time` if a scheduled time extension is present
464    /// in the query context.
465    fn derive_session_state_with_scheduled_time(
466        &self,
467        query_ctx: QueryContextRef,
468    ) -> Result<QueryEngineContext> {
469        let extensions = query_ctx.extensions();
470        let mut state = match crate::options::parse_scheduled_time_datetime(&extensions)? {
471            Some(dt) => {
472                let execution_props = self
473                    .session_state
474                    .execution_props()
475                    .clone()
476                    .with_query_execution_start_time(dt);
477                SessionStateBuilder::new_from_existing(self.session_state.clone())
478                    .with_execution_props(execution_props)
479                    .build()
480            }
481            None => self.session_state.clone(),
482        };
483        let query_ctx = self
484            .engine_state
485            .apply_query_options(&mut state, query_ctx)?;
486        state.config_mut().set_extension(query_ctx.clone());
487        state.config_mut().set_extension(self.engine_state.clone());
488        state
489            .config_mut()
490            .options_mut()
491            .extensions
492            .insert(ScheduledTimeExtension {
493                scheduled_time: crate::options::scheduled_time_from_ctx(&query_ctx),
494            });
495        let config_options = state.config_options().clone();
496        let _ = state
497            .execution_props_mut()
498            .config_options
499            .insert(config_options);
500        Ok(QueryEngineContext::new(state, query_ctx))
501    }
502
503    /// Basically the same with `explain_to_plan` in DataFusion, but adapted to Greptime's
504    /// `plan_sql` to support Greptime Statements.
505    async fn explain_to_plan(
506        &self,
507        explain: &ExplainStatement,
508        query_ctx: QueryContextRef,
509    ) -> Result<LogicalPlan> {
510        let plan = self.plan_sql(&explain.statement, query_ctx).await?;
511        if matches!(plan, LogicalPlan::Explain(_)) {
512            return plan_err!("Nested EXPLAINs are not supported").context(PlanSqlSnafu);
513        }
514
515        let verbose = explain.verbose;
516        let analyze = explain.analyze;
517        let format = explain.format.map(|f| f.to_string());
518
519        let plan = Arc::new(plan);
520        let schema = LogicalPlan::explain_schema();
521        let schema = ToDFSchema::to_dfschema_ref(schema)?;
522
523        if verbose && format.is_some() {
524            return plan_err!("EXPLAIN VERBOSE with FORMAT is not supported").context(PlanSqlSnafu);
525        }
526
527        if analyze {
528            // notice format is already set in query context, so can be ignore here
529            Ok(LogicalPlan::Analyze(Analyze {
530                verbose,
531                format: ExplainFormat::Indent,
532                input: plan,
533                schema,
534                analyze_level: None,
535                analyze_categories: None,
536            }))
537        } else {
538            let stringified_plans = vec![plan.to_stringified(PlanType::InitialLogicalPlan)];
539
540            let options = self.session_state.config().options();
541            let format = format
542                .map(|x| ExplainFormat::from_str(&x))
543                .transpose()?
544                .unwrap_or_else(|| options.explain.format.clone());
545
546            Ok(LogicalPlan::Explain(Explain {
547                verbose,
548                explain_format: format,
549                plan,
550                stringified_plans,
551                schema,
552                logical_optimization_succeeded: false,
553                show_statistics: None,
554            }))
555        }
556    }
557
558    #[tracing::instrument(skip_all)]
559    #[async_recursion::async_recursion]
560    async fn plan_sql(&self, stmt: &Statement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
561        let mut planner_context = PlannerContext::new();
562        let mut stmt = Cow::Borrowed(stmt);
563        let mut is_tql_cte = false;
564
565        // handle explain before normal processing so we can explain Greptime Statements
566        if let Statement::Explain(explain) = stmt.as_ref() {
567            return self.explain_to_plan(explain, query_ctx).await;
568        }
569
570        // Check for hybrid CTEs before normal processing
571        if self.has_hybrid_ctes(stmt.as_ref()) {
572            let stmt_owned = stmt.into_owned();
573            let mut query = match stmt_owned {
574                Statement::Query(query) => query.as_ref().clone(),
575                _ => unreachable!("has_hybrid_ctes should only return true for Query statements"),
576            };
577            self.plan_query_with_hybrid_ctes(&query, query_ctx.clone(), &mut planner_context)
578                .await?;
579
580            // remove the processed TQL CTEs from the query
581            query.hybrid_cte = None;
582            stmt = Cow::Owned(Statement::Query(Box::new(query)));
583            is_tql_cte = true;
584        }
585
586        let mut df_stmt = stmt.as_ref().try_into().context(SqlSnafu)?;
587        normalize_field_access_after_subscript(
588            &mut df_stmt,
589            self.session_state
590                .config_options()
591                .sql_parser
592                .enable_ident_normalization,
593        );
594
595        // TODO(LFC): Remove this when Datafusion supports **both** the syntax and implementation of "explain with format".
596        if let datafusion::sql::parser::Statement::Statement(stmt) = &mut df_stmt
597            && matches!(
598                stmt.as_ref(),
599                datafusion::sql::sqlparser::ast::Statement::Explain { .. }
600            )
601        {
602            UnimplementedSnafu {
603                operation: "EXPLAIN with FORMAT using raw datafusion planner",
604            }
605            .fail()?;
606        }
607
608        let engine_context = self.derive_session_state_with_scheduled_time(query_ctx)?;
609        let scheduled_state = engine_context.state().clone();
610        let query_ctx = engine_context.query_ctx();
611        let table_provider = DfTableSourceProvider::new(
612            self.engine_state.catalog_manager().clone(),
613            self.engine_state.disallow_cross_catalog_query(),
614            query_ctx.clone(),
615            Arc::new(DefaultPlanDecoder::new(
616                scheduled_state.clone(),
617                &query_ctx,
618            )?),
619            scheduled_state
620                .config_options()
621                .sql_parser
622                .enable_ident_normalization,
623        );
624
625        let context_provider = DfContextProviderAdapter::try_new(
626            self.engine_state.clone(),
627            scheduled_state.clone(),
628            Some(&df_stmt),
629            query_ctx.clone(),
630        )
631        .await?;
632
633        let config_options = scheduled_state.config().options();
634        let parser_options = &config_options.sql_parser;
635        let parser_options = ParserOptions {
636            map_string_types_to_utf8view: false,
637            ..parser_options.into()
638        };
639
640        let sql_to_rel = SqlToRel::new_with_options(&context_provider, parser_options);
641
642        // this IF is to handle different version of ASTs
643        let result = if is_tql_cte {
644            let Statement::Query(query) = stmt.into_owned() else {
645                unreachable!("is_tql_cte should only be true for Query statements");
646            };
647            let sqlparser_stmt = sqlparser::ast::Statement::Query(Box::new(query.inner));
648            sql_to_rel
649                .sql_statement_to_plan_with_context(sqlparser_stmt, &mut planner_context)
650                .context(PlanSqlSnafu)?
651        } else {
652            sql_to_rel
653                .statement_to_plan(df_stmt)
654                .context(PlanSqlSnafu)?
655        };
656
657        expr_depth::validate_plan(&result)?;
658        common_telemetry::debug!("Logical planner, statement to plan result: {result}");
659        let plan = RangePlanRewriter::new(table_provider, query_ctx.clone())
660            .rewrite(result)
661            .await?;
662
663        // Optimize logical plan by extension rules
664        let plan = self
665            .engine_state
666            .optimize_by_extension_rules(plan, &engine_context)?;
667        common_telemetry::debug!("Logical planner, optimize result: {plan}");
668
669        Ok(plan)
670    }
671
672    /// Generate a relational expression from a SQL expression
673    #[tracing::instrument(skip_all)]
674    pub(crate) async fn sql_to_expr(
675        &self,
676        sql: SqlExpr,
677        schema: &DFSchema,
678        normalize_ident: bool,
679        query_ctx: QueryContextRef,
680    ) -> Result<DfExpr> {
681        let engine_context = self.derive_session_state_with_scheduled_time(query_ctx)?;
682        let query_ctx = engine_context.query_ctx();
683        let context_provider = DfContextProviderAdapter::try_new(
684            self.engine_state.clone(),
685            engine_context.state().clone(),
686            None,
687            query_ctx,
688        )
689        .await?;
690
691        let config_options = engine_context.state().config().options();
692        let parser_options = &config_options.sql_parser;
693        let parser_options: ParserOptions = ParserOptions {
694            map_string_types_to_utf8view: false,
695            enable_ident_normalization: normalize_ident,
696            ..parser_options.into()
697        };
698
699        let sql_to_rel = SqlToRel::new_with_options(&context_provider, parser_options);
700
701        Ok(sql_to_rel.sql_to_expr(sql, schema, &mut PlannerContext::new())?)
702    }
703
704    /// Plans a PromQL statement, offering it to the native fused-series route when the statement
705    /// may take it and the query options it is planned with ask for it.
706    ///
707    /// `native_route_allowed` states whether the statement may take the route at all: a PromQL
708    /// statement embedded in a CTE may not (see [`Self::tql_to_logical_plan`]), every other one
709    /// may. Everything else the route depends on is read from the query options snapshot the
710    /// statement is planned with, where every option is already resolved as request > session
711    /// `SET` > engine default.
712    #[tracing::instrument(skip_all)]
713    async fn plan_pql(
714        &self,
715        stmt: &EvalStmt,
716        query_ctx: QueryContextRef,
717        native_route_allowed: bool,
718    ) -> Result<LogicalPlan> {
719        let engine_context = self.derive_session_state_with_scheduled_time(query_ctx)?;
720        let query_ctx = engine_context.query_ctx();
721        let mut scheduled_state = engine_context.state().clone();
722        // Whether the distributed planner may fall back to the fallback plan rewriter. A fallback
723        // rewriter can wrap a native region scan in an unusable MergeScan, so the native route is
724        // only offered to a query that cannot fall back.
725        //
726        // `apply_query_options` above resolved the three sources this used to disjoin - the engine
727        // option, the session `allow_query_fallback` variable and the `query_fallback` hint - into
728        // this extension, so the read happens here instead. One mainline semantic is kept: a stated
729        // `false` overrides the engine default rather than leaving it on.
730        let allow_query_fallback = scheduled_state
731            .config_options()
732            .extensions
733            .get::<DistPlannerOptions>()
734            .is_some_and(|options| options.allow_query_fallback);
735        // Whether the statement is offered to the native fused-series compiler. This is the second,
736        // independent switch: it states what the query asks for, and no source of it is derived
737        // from the fallback option.
738        let native_promql = scheduled_state
739            .config_options()
740            .extensions
741            .get::<NativePromqlExtension>()
742            .is_some_and(|options| options.requested);
743        let promql_annotations = query_ctx.remote_query_id().map(promql_annotation_collector);
744        if let Some(collector) = &promql_annotations {
745            scheduled_state
746                .config_mut()
747                .options_mut()
748                .extensions
749                .insert(collector.clone());
750        }
751        let plan_decoder = Arc::new(DefaultPlanDecoder::new(
752            scheduled_state.clone(),
753            &query_ctx,
754        )?);
755        let table_provider = || {
756            DfTableSourceProvider::new(
757                self.engine_state.catalog_manager().clone(),
758                self.engine_state.disallow_cross_catalog_query(),
759                query_ctx.clone(),
760                plan_decoder.clone(),
761                scheduled_state
762                    .config_options()
763                    .sql_parser
764                    .enable_ident_normalization,
765            )
766        };
767        let native_requested = native_route_allowed && !allow_query_fallback && native_promql;
768        let plan = PromPlanner::stmt_to_plan_with_annotations(
769            table_provider(),
770            stmt,
771            &self.engine_state,
772            promql_annotations.clone(),
773            // A fallback rewriter can wrap a native region scan in an unusable MergeScan.
774            native_requested,
775        )
776        .await
777        .map_err(BoxedError::new)
778        .context(QueryPlanSnafu)?;
779
780        // A distributed planner plans a native statement over per-selector raw sources, and a
781        // statement that reads no table needs none. Anything else keeps the existing plan. Decide
782        // while the original statement is still available: physical planning cannot fall back
783        // safely. An engine with no distributed planner plans no region, so there is nothing to
784        // admit its plan against.
785        let plan = if native_requested
786            && self.engine_state.with_dist_planner()
787            && !remote_native_raw_plan_supported(&plan)
788        {
789            PromPlanner::stmt_to_plan_with_annotations(
790                table_provider(),
791                stmt,
792                &self.engine_state,
793                promql_annotations,
794                false,
795            )
796            .await
797            .map_err(BoxedError::new)
798            .context(QueryPlanSnafu)?
799        } else {
800            plan
801        };
802        let context = QueryEngineContext::new(scheduled_state, query_ctx);
803        Ok(self
804            .engine_state
805            .optimize_by_extension_rules(plan, &context)?)
806    }
807
808    #[tracing::instrument(skip_all)]
809    fn optimize_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
810        Ok(self.engine_state.optimize_logical_plan(plan)?)
811    }
812
813    /// Check if a statement contains hybrid CTEs (mix of SQL and TQL)
814    fn has_hybrid_ctes(&self, stmt: &Statement) -> bool {
815        if let Statement::Query(query) = stmt {
816            query
817                .hybrid_cte
818                .as_ref()
819                .map(|hybrid_cte| !hybrid_cte.cte_tables.is_empty())
820                .unwrap_or(false)
821        } else {
822            false
823        }
824    }
825
826    /// Plan a query with hybrid CTEs using DataFusion's native PlannerContext
827    async fn plan_query_with_hybrid_ctes(
828        &self,
829        query: &Query,
830        query_ctx: QueryContextRef,
831        planner_context: &mut PlannerContext,
832    ) -> Result<()> {
833        let hybrid_cte = query.hybrid_cte.as_ref().unwrap();
834
835        for cte in &hybrid_cte.cte_tables {
836            match &cte.content {
837                CteContent::Tql(tql) => {
838                    // Plan TQL and register in PlannerContext
839                    let mut logical_plan = self.tql_to_logical_plan(tql, query_ctx.clone()).await?;
840                    if !cte.columns.is_empty() {
841                        let schema = logical_plan.schema();
842                        let schema_fields = schema.fields().to_vec();
843                        ensure!(
844                            schema_fields.len() == cte.columns.len(),
845                            CteColumnSchemaMismatchSnafu {
846                                cte_name: cte.name.value.clone(),
847                                original: schema_fields
848                                    .iter()
849                                    .map(|field| field.name().clone())
850                                    .collect::<Vec<_>>(),
851                                expected: cte
852                                    .columns
853                                    .iter()
854                                    .map(|column| column.to_string())
855                                    .collect::<Vec<_>>(),
856                            }
857                        );
858                        let aliases = cte
859                            .columns
860                            .iter()
861                            .zip(schema_fields.iter())
862                            .map(|(column, field)| col(field.name()).alias(column.to_string()));
863                        logical_plan = LogicalPlanBuilder::from(logical_plan)
864                            .project(aliases)
865                            .context(PlanSqlSnafu)?
866                            .build()
867                            .context(PlanSqlSnafu)?;
868                    }
869
870                    // Wrap in SubqueryAlias to ensure proper table qualification for CTE
871                    logical_plan = LogicalPlan::SubqueryAlias(
872                        datafusion_expr::SubqueryAlias::try_new(
873                            Arc::new(logical_plan),
874                            cte.name.value.clone(),
875                        )
876                        .context(PlanSqlSnafu)?,
877                    );
878
879                    planner_context.insert_cte(&cte.name.value, logical_plan);
880                }
881                CteContent::Sql(_) => {
882                    // SQL CTEs should have been moved to the main query's WITH clause
883                    // during parsing, so we shouldn't encounter them here
884                    unreachable!("SQL CTEs should not be in hybrid_cte.cte_tables");
885                }
886            }
887        }
888
889        Ok(())
890    }
891
892    /// Convert TQL to LogicalPlan directly
893    async fn tql_to_logical_plan(
894        &self,
895        tql: &Tql,
896        query_ctx: QueryContextRef,
897    ) -> Result<LogicalPlan> {
898        match tql {
899            Tql::Eval(eval) => {
900                // Convert TqlEval to PromQuery then to QueryStatement::Promql
901                let prom_query = PromQuery {
902                    query: eval.query.clone(),
903                    start: eval.start.clone(),
904                    end: eval.end.clone(),
905                    step: eval.step.clone(),
906                    lookback: eval
907                        .lookback
908                        .clone()
909                        .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()),
910                    alias: eval.alias.clone(),
911                };
912                let stmt = QueryLanguageParser::parse_promql(&prom_query, &query_ctx)?;
913                // A hybrid CTE registers this plan as a table the outer SQL query reads, so the
914                // `FusedSeries` node sits under a SQL alias and projection instead of at the plan
915                // root. Neither decides where a fused subtree runs: placement asks whether the
916                // node's own contract allows a region to run it, and the column requirements a
917                // node above makes are enforced by their lineage rather than by the shape of the
918                // subtree they travel through, so an alias or projection over the CTE does not
919                // block a multi-input fused subtree. The route is therefore offered here like it
920                // is for a top-level statement.
921                let QueryStatement::Promql(eval, _alias) = &stmt else {
922                    unreachable!("a TQL eval parses to a PromQL statement");
923                };
924                let plan = self.plan_pql(eval, query_ctx.clone(), true).await?;
925                if !contains_native_fused(&plan) {
926                    // The route was not taken - it is off for this query, or it declined this
927                    // shape - so this is the existing plan, which already publishes what a CTE has
928                    // always published.
929                    return Ok(plan);
930                }
931                // What the CTE publishes is SQL-visible, and this route names and orders its output
932                // its own way, so its own schema cannot stand in for the existing planner's: plan
933                // the statement for that schema and publish it over the native answer. A statement
934                // whose columns that projection cannot fill faithfully keeps the existing plan
935                // instead of changing the schema the CTE exposes.
936                let existing = self.plan_pql(eval, query_ctx, false).await?;
937                Ok(publish_legacy_schema(plan, &existing).unwrap_or(existing))
938            }
939            Tql::Explain(_) => UnimplementedSnafu {
940                operation: "TQL EXPLAIN in CTEs",
941            }
942            .fail(),
943            Tql::Analyze(_) => UnimplementedSnafu {
944                operation: "TQL ANALYZE in CTEs",
945            }
946            .fail(),
947        }
948    }
949
950    /// Extracts cast types for all placeholders in a logical plan.
951    /// Returns a map where each placeholder ID is mapped to:
952    /// - Some(DataType) if the placeholder is cast to a specific type
953    /// - None if the placeholder exists but has no cast
954    ///
955    /// Example: `$1::TEXT` returns `{"$1": Some(DataType::Utf8)}`
956    ///
957    /// This function walks through all expressions in the logical plan,
958    /// including subqueries, to identify placeholders and their cast types.
959    fn extract_placeholder_cast_types(
960        plan: &LogicalPlan,
961    ) -> Result<HashMap<String, Option<DataType>>> {
962        let mut placeholder_types = HashMap::new();
963        let mut casted_placeholders = HashSet::new();
964
965        Self::extract_from_plan(plan, &mut placeholder_types, &mut casted_placeholders)?;
966
967        Ok(placeholder_types)
968    }
969
970    fn extract_from_plan(
971        plan: &LogicalPlan,
972        placeholder_types: &mut HashMap<String, Option<DataType>>,
973        casted_placeholders: &mut HashSet<String>,
974    ) -> Result<()> {
975        plan.apply(|node| {
976            for expr in node.expressions() {
977                let _ = expr.apply(|e| {
978                    // Handle casted placeholders
979                    if let DfExpr::Cast(cast) = e
980                        && let DfExpr::Placeholder(ph) = &*cast.expr
981                    {
982                        placeholder_types
983                            .insert(ph.id.clone(), Some(cast.field.data_type().clone()));
984                        casted_placeholders.insert(ph.id.clone());
985                    }
986
987                    // Handle arrow_cast(Placeholder, 'type_string') generated by SQL rewriter
988                    if let DfExpr::ScalarFunction(scalar_func) = e
989                        && scalar_func.name() == "arrow_cast"
990                        && scalar_func.args.len() == 2
991                        && let DfExpr::Placeholder(ph) = &scalar_func.args[0]
992                        && let DfExpr::Literal(ScalarValue::Utf8(Some(type_str)), _) =
993                            &scalar_func.args[1]
994                        && let Ok(data_type) = type_str.parse::<DataType>()
995                    {
996                        placeholder_types.insert(ph.id.clone(), Some(data_type));
997                        casted_placeholders.insert(ph.id.clone());
998                    }
999
1000                    // Handle bare (non-casted) placeholders
1001                    if let DfExpr::Placeholder(ph) = e
1002                        && !casted_placeholders.contains(&ph.id)
1003                        && !placeholder_types.contains_key(&ph.id)
1004                    {
1005                        placeholder_types.insert(ph.id.clone(), None);
1006                    }
1007
1008                    // Recurse into subquery plans embedded in expressions
1009                    match e {
1010                        DfExpr::Exists(Exists { subquery, .. })
1011                        | DfExpr::InSubquery(InSubquery { subquery, .. })
1012                        | DfExpr::ScalarSubquery(subquery) => {
1013                            Self::extract_from_plan(
1014                                &subquery.subquery,
1015                                placeholder_types,
1016                                casted_placeholders,
1017                            )?;
1018                        }
1019                        _ => {}
1020                    }
1021
1022                    Ok(TreeNodeRecursion::Continue)
1023                });
1024            }
1025            Ok(TreeNodeRecursion::Continue)
1026        })?;
1027        Ok(())
1028    }
1029
1030    fn infer_limit_placeholder_types(
1031        plan: &LogicalPlan,
1032        placeholder_types: &mut HashMap<String, Option<DataType>>,
1033    ) -> Result<()> {
1034        plan.apply(|node| {
1035            if let LogicalPlan::Limit(limit) = node {
1036                for expr in limit.skip.iter().chain(limit.fetch.iter()) {
1037                    expr.apply(|e| {
1038                        if let DfExpr::Placeholder(ph) = e {
1039                            placeholder_types
1040                                .entry(ph.id.clone())
1041                                .and_modify(|existing| {
1042                                    if existing.is_none() {
1043                                        *existing = Some(DataType::Int64);
1044                                    }
1045                                })
1046                                .or_insert(Some(DataType::Int64));
1047                        }
1048
1049                        Ok(TreeNodeRecursion::Continue)
1050                    })?;
1051                }
1052            }
1053
1054            Ok(TreeNodeRecursion::Continue)
1055        })?;
1056
1057        Ok(())
1058    }
1059
1060    /// Gets inferred parameter types from a logical plan.
1061    /// Returns a map where each parameter ID is mapped to:
1062    /// - Some(DataType) if the parameter type could be inferred
1063    /// - None if the parameter type could not be inferred
1064    ///
1065    /// This function first uses DataFusion's `get_parameter_types()` to infer types.
1066    /// If any parameters have `None` values (i.e., DataFusion couldn't infer their types),
1067    /// it falls back to using `extract_placeholder_cast_types()` to detect explicit casts
1068    /// and applies context-specific inference such as LIMIT/OFFSET placeholders.
1069    ///
1070    /// This is because datafusion can only infer types for a limited cases.
1071    ///
1072    /// Example: For query `WHERE $1::TEXT AND $2`, DataFusion may not infer `$2`'s type,
1073    /// but this function will return `{"$1": Some(DataType::Utf8), "$2": None}`.
1074    pub fn get_inferred_parameter_types(
1075        plan: &LogicalPlan,
1076    ) -> Result<HashMap<String, Option<DataType>>> {
1077        let mut param_types = plan.get_parameter_types().context(PlanSqlSnafu)?;
1078
1079        let has_none = param_types.values().any(|v| v.is_none());
1080
1081        if has_none {
1082            let cast_types = Self::extract_placeholder_cast_types(plan)?;
1083
1084            for (id, opt_type) in cast_types {
1085                param_types
1086                    .entry(id)
1087                    .and_modify(|existing| {
1088                        if existing.is_none() {
1089                            *existing = opt_type.clone();
1090                        }
1091                    })
1092                    .or_insert(opt_type);
1093            }
1094
1095            Self::infer_limit_placeholder_types(plan, &mut param_types)?;
1096        }
1097
1098        Ok(param_types)
1099    }
1100}
1101
1102/// Normalizes dot field accesses that follow a subscript for DataFusion.
1103///
1104/// sqlparser represents `j.o.l[1].inner.l[2]` as a compound field access with
1105/// the following access chain:
1106///
1107/// ```text
1108/// Dot(Identifier("o")),
1109/// Dot(Identifier("l")),
1110/// Subscript(1),
1111/// Dot(Identifier("inner")),
1112/// Dot(Identifier("l")),
1113/// Subscript(2)
1114/// ```
1115///
1116/// DataFusion first resolves the leading `j.o.l` through
1117/// `JsonExprPlanner::plan_compound_identifier`, which produces an untyped
1118/// `json_get` with path `$.o.l`. Before invoking `JsonExprPlanner::plan_field_access`,
1119/// however, DataFusion eagerly converts every remaining access into a
1120/// `GetFieldAccess`. It accepts string values but not [`SqlExpr::Identifier`]s
1121/// in [`AccessExpr::Dot`] after a subscript. Without this normalization, that
1122/// conversion fails at `.inner`, and `plan_field_access` is never called, even
1123/// for the preceding `[1]`.
1124///
1125/// This function converts dot identifiers after the first subscript into
1126/// `Dot(Value(SingleQuotedString(...)))`, applying DataFusion's identifier
1127/// normalization before discarding whether each identifier was quoted. It
1128/// changes neither the SQL text nor the dot accesses into subscript nodes: the
1129/// resulting AST is conceptually `j.o.l[1].'inner'.'l'[2]`. DataFusion converts
1130/// the string-valued dot accesses into named field accesses, which
1131/// `plan_field_access` safely encodes as dot members, quoting names when needed. It can then extend the
1132/// JSON path to `$.o.l[1].inner.l[2]`.
1133///
1134/// This behavior is unchanged in the latest upstream releases checked here:
1135/// DataFusion 55.0.0 and sqlparser 0.62.0.
1136///
1137/// TODO(LFC): Remove this workaround after upstream supports dot identifiers after subscripts.
1138fn normalize_field_access_after_subscript(stmt: &mut DfStatement, normalize_ident: bool) {
1139    let DfStatement::Statement(stmt) = stmt else {
1140        return;
1141    };
1142    let normalizer = IdentNormalizer::new(normalize_ident);
1143
1144    let _ = visit_expressions_mut(stmt.as_mut(), |expr| {
1145        let SqlExpr::CompoundFieldAccess { access_chain, .. } = expr else {
1146            return ControlFlow::<()>::Continue(());
1147        };
1148        let Some(index) = access_chain
1149            .iter()
1150            .position(|x| matches!(x, AccessExpr::Subscript(_)))
1151        else {
1152            return ControlFlow::Continue(());
1153        };
1154
1155        for access in &mut access_chain[index + 1..] {
1156            let AccessExpr::Dot(SqlExpr::Identifier(ident)) = access else {
1157                continue;
1158            };
1159            let value = normalizer.normalize(ident.clone());
1160            *access = AccessExpr::Dot(SqlExpr::Value(
1161                Value::SingleQuotedString(value).with_span(ident.span),
1162            ));
1163        }
1164        ControlFlow::Continue(())
1165    });
1166}
1167
1168#[async_trait]
1169impl LogicalPlanner for DfLogicalPlanner {
1170    #[tracing::instrument(skip_all)]
1171    async fn plan(&self, stmt: &QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
1172        match stmt {
1173            QueryStatement::Sql(stmt) => self.plan_sql(stmt, query_ctx).await,
1174            // The statement's own plan is what a region receives, so the native route is offered.
1175            QueryStatement::Promql(stmt, _alias) => self.plan_pql(stmt, query_ctx, true).await,
1176        }
1177    }
1178
1179    async fn plan_logs_query(
1180        &self,
1181        query: LogQuery,
1182        query_ctx: QueryContextRef,
1183    ) -> Result<LogicalPlan> {
1184        let engine_context = self.derive_session_state_with_scheduled_time(query_ctx)?;
1185        let query_ctx = engine_context.query_ctx();
1186        let state = engine_context.state().clone();
1187        let plan_decoder = Arc::new(DefaultPlanDecoder::new(state.clone(), &query_ctx)?);
1188        let table_provider = DfTableSourceProvider::new(
1189            self.engine_state.catalog_manager().clone(),
1190            self.engine_state.disallow_cross_catalog_query(),
1191            query_ctx,
1192            plan_decoder,
1193            state.config_options().sql_parser.enable_ident_normalization,
1194        );
1195
1196        let mut planner = LogQueryPlanner::new(table_provider, state);
1197        planner
1198            .query_to_plan(query)
1199            .await
1200            .map_err(BoxedError::new)
1201            .context(QueryPlanSnafu)
1202    }
1203
1204    fn optimize(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
1205        self.optimize_logical_plan(plan)
1206    }
1207
1208    fn as_any(&self) -> &dyn Any {
1209        self
1210    }
1211}
1212
1213#[cfg(test)]
1214mod tests {
1215    use std::sync::Arc;
1216
1217    use arrow_schema::DataType;
1218    use catalog::RegisterTableRequest;
1219    use catalog::memory::MemoryCatalogManager;
1220    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
1221    use common_query::OutputData;
1222    use common_recordbatch::RecordBatch;
1223    use common_recordbatch::util::collect_batches;
1224    use common_time::Timezone;
1225    use datatypes::arrow::array::{Array, Float64Array, StringArray};
1226    use datatypes::prelude::ConcreteDataType;
1227    use datatypes::schema::{ColumnSchema, Schema};
1228    use session::context::{
1229        MetricDiscoveryScope, QueryContext, QueryContextBuilder, QueryContextRef,
1230    };
1231    use store_api::metric_engine_consts::{
1232        DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME, LOGICAL_TABLE_METADATA_KEY,
1233        METRIC_ENGINE_NAME,
1234    };
1235    use table::metadata::{TableInfoBuilder, TableMetaBuilder};
1236    use table::test_util::EmptyTable;
1237
1238    use super::*;
1239    use crate::datafusion::QUERY_FALLBACK_HINT;
1240    use crate::parser::{PromQuery, QueryLanguageParser};
1241    use crate::promql::planner::NATIVE_PROMQL_EXTENSION_KEY;
1242    use crate::{QueryEngineFactory, QueryEngineRef};
1243
1244    async fn create_test_engine() -> QueryEngineRef {
1245        let columns = vec![
1246            ColumnSchema::new("id", ConcreteDataType::int32_datatype(), false),
1247            ColumnSchema::new("name", ConcreteDataType::string_datatype(), true),
1248        ];
1249        let schema = Arc::new(Schema::new(columns));
1250        let table_meta = TableMetaBuilder::empty()
1251            .schema(schema)
1252            .primary_key_indices(vec![0])
1253            .value_indices(vec![1])
1254            .next_column_id(1024)
1255            .build()
1256            .unwrap();
1257        let table_info = TableInfoBuilder::new("test", table_meta).build().unwrap();
1258        let table = EmptyTable::from_table_info(&table_info);
1259
1260        crate::tests::new_query_engine_with_table(table)
1261    }
1262
1263    async fn create_timestamp_test_engine() -> QueryEngineRef {
1264        let columns = vec![
1265            ColumnSchema::new(
1266                "ts",
1267                ConcreteDataType::timestamp_millisecond_datatype(),
1268                false,
1269            )
1270            .with_time_index(true),
1271            ColumnSchema::new(
1272                "st",
1273                ConcreteDataType::timestamp_millisecond_datatype(),
1274                false,
1275            ),
1276            ColumnSchema::new("note", ConcreteDataType::string_datatype(), true),
1277            ColumnSchema::new(
1278                "ts_ns",
1279                ConcreteDataType::timestamp_nanosecond_datatype(),
1280                true,
1281            ),
1282        ];
1283        let schema = Arc::new(Schema::new(columns));
1284        let table_meta = TableMetaBuilder::empty()
1285            .schema(schema)
1286            .primary_key_indices(vec![])
1287            .value_indices(vec![0, 1, 2, 3])
1288            .next_column_id(1024)
1289            .build()
1290            .unwrap();
1291        let table_info = TableInfoBuilder::new("timestamps", table_meta)
1292            .build()
1293            .unwrap();
1294        let table = EmptyTable::from_table_info(&table_info);
1295
1296        crate::tests::new_query_engine_with_table(table)
1297    }
1298
1299    fn create_promql_test_engine() -> QueryEngineRef {
1300        create_promql_test_engine_with(crate::options::QueryOptions::default(), false)
1301    }
1302
1303    /// The PromQL fixture, additionally stating the engine options and whether it plans distributed
1304    /// statements.
1305    fn create_promql_test_engine_with(
1306        query_options: crate::options::QueryOptions,
1307        with_dist_planner: bool,
1308    ) -> QueryEngineRef {
1309        let catalog_manager = MemoryCatalogManager::with_default_setup();
1310        let physical_table_name = "phy";
1311        let physical_table_id = 999u32;
1312
1313        let physical_schema = Arc::new(Schema::new(vec![
1314            ColumnSchema::new(
1315                DATA_SCHEMA_TABLE_ID_COLUMN_NAME.to_string(),
1316                ConcreteDataType::uint32_datatype(),
1317                false,
1318            ),
1319            ColumnSchema::new(
1320                DATA_SCHEMA_TSID_COLUMN_NAME.to_string(),
1321                ConcreteDataType::uint64_datatype(),
1322                false,
1323            ),
1324            ColumnSchema::new("tag_0", ConcreteDataType::string_datatype(), false),
1325            ColumnSchema::new("tag_1", ConcreteDataType::string_datatype(), false),
1326            ColumnSchema::new(
1327                "timestamp",
1328                ConcreteDataType::timestamp_millisecond_datatype(),
1329                false,
1330            )
1331            .with_time_index(true),
1332            ColumnSchema::new("field_0", ConcreteDataType::float64_datatype(), true),
1333        ]));
1334        let physical_meta = TableMetaBuilder::empty()
1335            .schema(physical_schema)
1336            .primary_key_indices(vec![0, 1, 2, 3])
1337            .value_indices(vec![4, 5])
1338            .engine(METRIC_ENGINE_NAME.to_string())
1339            .next_column_id(1024)
1340            .build()
1341            .unwrap();
1342        let physical_info = TableInfoBuilder::default()
1343            .table_id(physical_table_id)
1344            .name(physical_table_name)
1345            .meta(physical_meta)
1346            .build()
1347            .unwrap();
1348        catalog_manager
1349            .register_table_sync(RegisterTableRequest {
1350                catalog: DEFAULT_CATALOG_NAME.to_string(),
1351                schema: DEFAULT_SCHEMA_NAME.to_string(),
1352                table_name: physical_table_name.to_string(),
1353                table_id: physical_table_id,
1354                table: EmptyTable::from_table_info(&physical_info),
1355            })
1356            .unwrap();
1357
1358        let mut options = table::requests::TableOptions::default();
1359        options.extra_options.insert(
1360            LOGICAL_TABLE_METADATA_KEY.to_string(),
1361            physical_table_name.to_string(),
1362        );
1363        let logical_schema = Arc::new(Schema::new(vec![
1364            ColumnSchema::new("tag_0", ConcreteDataType::string_datatype(), false),
1365            ColumnSchema::new("tag_1", ConcreteDataType::string_datatype(), false),
1366            ColumnSchema::new(
1367                "timestamp",
1368                ConcreteDataType::timestamp_millisecond_datatype(),
1369                false,
1370            )
1371            .with_time_index(true),
1372            ColumnSchema::new("field_0", ConcreteDataType::float64_datatype(), true),
1373        ]));
1374        let logical_meta = TableMetaBuilder::empty()
1375            .schema(logical_schema)
1376            .primary_key_indices(vec![0, 1])
1377            .value_indices(vec![3])
1378            .engine(METRIC_ENGINE_NAME.to_string())
1379            .options(options)
1380            .next_column_id(1024)
1381            .build()
1382            .unwrap();
1383        let logical_info = TableInfoBuilder::default()
1384            .table_id(1024)
1385            .name("some_metric")
1386            .meta(logical_meta)
1387            .build()
1388            .unwrap();
1389        catalog_manager
1390            .register_table_sync(RegisterTableRequest {
1391                catalog: DEFAULT_CATALOG_NAME.to_string(),
1392                schema: DEFAULT_SCHEMA_NAME.to_string(),
1393                table_name: "some_metric".to_string(),
1394                table_id: 1024,
1395                table: EmptyTable::from_table_info(&logical_info),
1396            })
1397            .unwrap();
1398        let mut other_info = logical_info.clone();
1399        other_info.ident.table_id = 1025;
1400        other_info.name = "other_metric".to_string();
1401        catalog_manager
1402            .register_table_sync(RegisterTableRequest {
1403                catalog: DEFAULT_CATALOG_NAME.to_string(),
1404                schema: DEFAULT_SCHEMA_NAME.to_string(),
1405                table_name: other_info.name.clone(),
1406                table_id: other_info.ident.table_id,
1407                table: EmptyTable::from_table_info(&other_info),
1408            })
1409            .unwrap();
1410
1411        QueryEngineFactory::try_new_with_plugins(
1412            catalog_manager,
1413            None,
1414            None,
1415            None,
1416            None,
1417            None,
1418            with_dist_planner,
1419            Default::default(),
1420            query_options,
1421        )
1422        .unwrap()
1423        .query_engine()
1424    }
1425
1426    async fn parse_sql_to_plan(sql: &str) -> LogicalPlan {
1427        let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
1428        let engine = create_test_engine().await;
1429        engine
1430            .planner()
1431            .plan(&stmt, QueryContext::arc())
1432            .await
1433            .unwrap()
1434    }
1435
1436    async fn parse_promql_to_plan(query: &str) -> LogicalPlan {
1437        let engine = create_promql_test_engine();
1438        let query_ctx = QueryContext::arc();
1439        let stmt = QueryLanguageParser::parse_promql(
1440            &PromQuery {
1441                query: query.to_string(),
1442                start: "0".to_string(),
1443                end: "10".to_string(),
1444                step: "5s".to_string(),
1445                lookback: "300s".to_string(),
1446                alias: None,
1447            },
1448            &query_ctx,
1449        )
1450        .unwrap();
1451
1452        engine.planner().plan(&stmt, query_ctx).await.unwrap()
1453    }
1454
1455    /// Plans `rate(some_metric[5m])`, which the in-process fixture plans natively, stating the
1456    /// query options the caller asks for: the engine-wide defaults, one session `SET` and one
1457    /// request extension. These are the sources the planning snapshot resolves, in that order.
1458    async fn plan_native_promql_with_query_options(
1459        engine_options: crate::options::QueryOptions,
1460        session: Option<(&str, &str)>,
1461        request: Option<(&str, &str)>,
1462    ) -> crate::error::Result<String> {
1463        let engine = create_promql_test_engine_with(
1464            engine_options,
1465            // The route a deployed frontend plans: a distributed planner over its regions.
1466            true,
1467        );
1468        let mut builder = QueryContextBuilder::default();
1469        if let Some((key, value)) = request {
1470            builder = builder.set_extension(key.to_string(), value.to_string());
1471        }
1472        // The session state a query is sent with, as `SET` leaves it.
1473        let query_ctx = builder.build();
1474        if let Some((key, value)) = session {
1475            query_ctx
1476                .configuration_parameter()
1477                .set_query_option(key, value)
1478                .unwrap();
1479        }
1480        let query_ctx = Arc::new(query_ctx);
1481        let stmt = QueryLanguageParser::parse_promql(
1482            &PromQuery {
1483                query: "rate(some_metric[5m])".to_string(),
1484                start: "0".to_string(),
1485                end: "10".to_string(),
1486                step: "5s".to_string(),
1487                lookback: "300s".to_string(),
1488                alias: None,
1489            },
1490            &query_ctx,
1491        )
1492        .unwrap();
1493
1494        engine
1495            .planner()
1496            .plan(&stmt, query_ctx)
1497            .await
1498            .map(|plan| plan.display_indent().to_string())
1499    }
1500
1501    /// The native route follows the query options the snapshot resolves: the request, then the
1502    /// session `SET`, then the engine-wide default. Only that one value decides, so a session that
1503    /// says `true` routes natively with no request hint and the engine default off, and a request
1504    /// that says `false` leaves the route however the other two are set.
1505    #[tokio::test]
1506    async fn test_plan_pql_native_route_follows_query_options() {
1507        // (engine default, session `SET`, request hint, native route admitted)
1508        for (engine_default, session, request, native) in [
1509            // Nothing states the option: the engine default decides.
1510            (false, None, None, false),
1511            (true, None, None, true),
1512            // A session `SET` alone enables the route while the engine default stays off.
1513            (false, Some("true"), None, true),
1514            // A request over the session value, in both directions.
1515            (false, Some("true"), Some("false"), false),
1516            (false, Some("false"), Some("true"), true),
1517            // Over the engine default too, in both directions.
1518            (true, Some("false"), None, false),
1519            (true, Some("true"), Some("false"), false),
1520            (false, None, Some("true"), true),
1521        ] {
1522            let plan = plan_native_promql_with_query_options(
1523                crate::options::QueryOptions {
1524                    experimental_native_promql: engine_default,
1525                    ..Default::default()
1526                },
1527                session.map(|value| (NATIVE_PROMQL_EXTENSION_KEY, value)),
1528                request.map(|value| (NATIVE_PROMQL_EXTENSION_KEY, value)),
1529            )
1530            .await
1531            .unwrap();
1532            assert_eq!(
1533                plan.contains("FusedSeries"),
1534                native,
1535                "engine default={engine_default}, session={session:?}, request={request:?}: {plan}"
1536            );
1537        }
1538    }
1539
1540    /// Plans `rate(some_metric[5m])` with the fallback option the caller states, while the native
1541    /// route is on and requested, so only the fallback option decides.
1542    async fn plan_native_promql_with_fallback(
1543        engine_fallback: bool,
1544        session_fallback: Option<bool>,
1545        request_fallback: Option<&str>,
1546    ) -> crate::error::Result<String> {
1547        plan_native_promql_with_query_options(
1548            crate::options::QueryOptions {
1549                allow_query_fallback: engine_fallback,
1550                experimental_native_promql: true,
1551                ..Default::default()
1552            },
1553            session_fallback.map(|value| {
1554                (
1555                    "query.allow_query_fallback",
1556                    if value { "true" } else { "false" },
1557                )
1558            }),
1559            request_fallback.map(|value| (QUERY_FALLBACK_HINT, value)),
1560        )
1561        .await
1562    }
1563
1564    /// The native route is admitted only when the query cannot fall back, and the fallback option
1565    /// resolves from the same snapshot, so a stated value overrides the engine default in both
1566    /// directions instead of only ever turning the fallback on.
1567    #[tokio::test]
1568    async fn test_plan_pql_declines_native_route_when_query_may_fall_back() {
1569        // (engine option, session variable, hint, native route admitted)
1570        for (engine, session, request, native) in [
1571            // Nothing asks for the fallback, so the route is planned as before.
1572            (false, None, None, true),
1573            (false, None, Some("false"), true),
1574            (false, Some(false), None, true),
1575            // A per-query source turns the fallback on, so the route is declined.
1576            (false, Some(true), None, false),
1577            (false, None, Some("true"), false),
1578            (false, None, Some("TRUE"), false),
1579            // The engine option is the default: a stated `false` overrides it.
1580            (true, None, None, false),
1581            (true, None, Some("false"), true),
1582            (true, Some(true), Some("false"), true),
1583            (true, Some(false), None, true),
1584        ] {
1585            let plan = plan_native_promql_with_fallback(engine, session, request)
1586                .await
1587                .unwrap();
1588            assert_eq!(
1589                plan.contains("FusedSeries"),
1590                native,
1591                "engine={engine}, session={session:?}, request={request:?}: {plan}"
1592            );
1593        }
1594    }
1595
1596    /// The engine-wide option is the default a request or a session overrides: it is not a gate
1597    /// the request also has to pass. A default deployment stays on the legacy route when nothing
1598    /// is stated, and a client that asks for the native route gets it - which is the case that
1599    /// matters when this branch is merged back, because a client can always state the option.
1600    #[tokio::test]
1601    async fn test_engine_option_is_the_default_a_request_or_session_overrides() {
1602        const KEY: &str = "query.experimental_native_promql";
1603        // (engine option, session `SET`, request hint, native route planned)
1604        for (engine, session, request, native) in [
1605            // Nothing states the option, so the engine default answers.
1606            (false, None, None, false),
1607            (true, None, None, true),
1608            // A request that asks for the route gets it on a default deployment.
1609            (false, None, Some((KEY, "true")), true),
1610            // ... and a request that states `false` keeps the legacy route on an enabled one.
1611            (true, None, Some((KEY, "false")), false),
1612            // A session `SET` states it too, and the request outranks it in both directions.
1613            (false, Some((KEY, "true")), None, true),
1614            (true, Some((KEY, "false")), None, false),
1615            (false, Some((KEY, "false")), Some((KEY, "true")), true),
1616            (true, Some((KEY, "true")), Some((KEY, "false")), false),
1617        ] {
1618            let plan = plan_native_promql_with_query_options(
1619                crate::options::QueryOptions {
1620                    experimental_native_promql: engine,
1621                    ..Default::default()
1622                },
1623                session,
1624                request,
1625            )
1626            .await
1627            .unwrap();
1628
1629            assert_eq!(
1630                plan.contains("FusedSeries"),
1631                native,
1632                "engine={engine}, session={session:?}, request={request:?}: {plan}"
1633            );
1634            assert_eq!(
1635                plan.contains("RawSeriesSource"),
1636                native,
1637                "engine={engine}, session={session:?}, request={request:?}: {plan}"
1638            );
1639        }
1640    }
1641
1642    /// The fallback hint names a query option, so a value that is not a boolean is rejected instead
1643    /// of silently leaving the fallback off.
1644    #[tokio::test]
1645    async fn test_plan_pql_rejects_a_fallback_hint_that_is_not_a_boolean() {
1646        let error = plan_native_promql_with_fallback(false, None, Some("yes"))
1647            .await
1648            .unwrap_err();
1649        let message = error.to_string();
1650        assert!(message.contains("query_fallback"), "{message}");
1651        assert!(message.contains("Expected true or false"), "{message}");
1652    }
1653
1654    /// One deployment, one topology: an engine that plans regions reads every selector through one
1655    /// source per region, so the admission gate admits the same shapes wherever those regions are.
1656    /// An engine with no distributed planner plans no region, so its plans keep the shapes a region
1657    /// runs itself and there is nothing for the gate to judge.
1658    #[tokio::test]
1659    async fn test_native_promql_admission_is_the_same_for_both_deployments() {
1660        for (with_dist_planner, query, sourced) in [
1661            (false, "some_metric + other_metric", true),
1662            (false, "rate(some_metric[5m])", false),
1663            (true, "some_metric + other_metric", true),
1664            (true, "rate(some_metric[5m])", true),
1665        ] {
1666            let engine = create_promql_test_engine_with(
1667                crate::options::QueryOptions {
1668                    experimental_native_promql: true,
1669                    ..Default::default()
1670                },
1671                with_dist_planner,
1672            );
1673            let query_ctx = Arc::new(
1674                QueryContextBuilder::default()
1675                    .set_extension(NATIVE_PROMQL_EXTENSION_KEY.to_string(), "true".to_string())
1676                    .build(),
1677            );
1678            let stmt = QueryLanguageParser::parse_promql(
1679                &PromQuery {
1680                    query: query.to_string(),
1681                    start: "0".to_string(),
1682                    end: "10".to_string(),
1683                    step: "5s".to_string(),
1684                    lookback: "300s".to_string(),
1685                    alias: None,
1686                },
1687                &query_ctx,
1688            )
1689            .unwrap();
1690            let plan = engine.planner().plan(&stmt, query_ctx).await.unwrap();
1691            let display = plan.display_indent().to_string();
1692            assert!(
1693                display.contains("FusedSeries"),
1694                "with_dist_planner={with_dist_planner}, query={query}: {display}"
1695            );
1696            if with_dist_planner {
1697                // The gate admits the plan instead of replanning it through the legacy route: one
1698                // source per region under a fused node.
1699                assert!(remote_native_raw_plan_supported(&plan), "{display}");
1700            }
1701            assert_eq!(
1702                display.contains("RawSeriesSource"),
1703                sourced,
1704                "with_dist_planner={with_dist_planner}, query={query}: {display}"
1705            );
1706        }
1707    }
1708
1709    /// Plans `query` on the fixture engine with the native route requested. The engine is one with
1710    /// no distributed planner, so its plans keep the shapes a region runs itself.
1711    async fn plan_remote_native_promql(
1712        engine: &QueryEngineRef,
1713        query_ctx: &QueryContextRef,
1714        query: &str,
1715    ) -> LogicalPlan {
1716        let stmt = QueryLanguageParser::parse_promql(
1717            &PromQuery {
1718                query: query.to_string(),
1719                start: "0".to_string(),
1720                end: "10".to_string(),
1721                step: "5s".to_string(),
1722                lookback: "300s".to_string(),
1723                alias: None,
1724            },
1725            query_ctx,
1726        )
1727        .unwrap();
1728        engine
1729            .planner()
1730            .plan(&stmt, query_ctx.clone())
1731            .await
1732            .unwrap()
1733    }
1734
1735    /// A query context that asks for the native route and states what a nameless selector may
1736    /// discover; the default context declines discovery altogether.
1737    fn native_query_ctx(scope: MetricDiscoveryScope) -> QueryContextRef {
1738        let mut ctx = (*QueryContext::arc()).clone();
1739        ctx.set_metric_discovery_scope(scope);
1740        ctx.set_extension(NATIVE_PROMQL_EXTENSION_KEY.to_string(), "true".to_string());
1741        Arc::new(ctx)
1742    }
1743
1744    fn native_discovery_query_ctx() -> QueryContextRef {
1745        native_query_ctx(MetricDiscoveryScope::Unrestricted)
1746    }
1747
1748    /// The engine's physical plan for `plan` and the batches it answers.
1749    async fn execute_plan(
1750        engine: &QueryEngineRef,
1751        plan: LogicalPlan,
1752        query_ctx: &QueryContextRef,
1753    ) -> (
1754        Arc<dyn datafusion::physical_plan::ExecutionPlan>,
1755        Vec<RecordBatch>,
1756    ) {
1757        let output = engine.execute(plan, query_ctx.clone()).await.unwrap();
1758        let physical = output
1759            .meta
1760            .plan
1761            .expect("the engine reports its physical plan");
1762        let batches = match output.data {
1763            OutputData::Stream(stream) => collect_batches(stream).await.unwrap(),
1764            OutputData::RecordBatches(batches) => batches,
1765            other => panic!("a PromQL query answers rows, not {other:?}"),
1766        }
1767        .take();
1768        (physical, batches)
1769    }
1770
1771    /// Every node name in the tree rooted at `root`, the root included.
1772    fn physical_plan_names(
1773        root: &Arc<dyn datafusion::physical_plan::ExecutionPlan>,
1774    ) -> Vec<String> {
1775        let mut names = Vec::new();
1776        let mut stack = vec![Arc::clone(root)];
1777        while let Some(node) = stack.pop() {
1778            names.push(node.name().to_string());
1779            stack.extend(node.children().into_iter().cloned());
1780        }
1781        names
1782    }
1783
1784    /// Every logical plan node below `root`, the root included.
1785    fn logical_plan_nodes(root: &LogicalPlan) -> Vec<&LogicalPlan> {
1786        let mut nodes = Vec::new();
1787        let mut stack = vec![root];
1788        while let Some(node) = stack.pop() {
1789            nodes.push(node);
1790            stack.extend(node.inputs());
1791        }
1792        nodes
1793    }
1794
1795    fn fused_series_nodes(plan: &LogicalPlan) -> Vec<&promql::extension_plan::FusedSeries> {
1796        logical_plan_nodes(plan)
1797            .into_iter()
1798            .filter_map(|node| match node {
1799                LogicalPlan::Extension(extension) => extension
1800                    .node
1801                    .as_any()
1802                    .downcast_ref::<promql::extension_plan::FusedSeries>(
1803                ),
1804                _ => None,
1805            })
1806            .collect()
1807    }
1808
1809    fn numeric_values(batches: &[RecordBatch], column: &str) -> Vec<f64> {
1810        batches
1811            .iter()
1812            .flat_map(|batch| {
1813                batch
1814                    .column_by_name(column)
1815                    .unwrap_or_else(|| panic!("no `{column}` column"))
1816                    .as_any()
1817                    .downcast_ref::<Float64Array>()
1818                    .unwrap()
1819                    .values()
1820                    .to_vec()
1821            })
1822            .collect()
1823    }
1824
1825    fn label_values(batches: &[RecordBatch], column: &str) -> Vec<String> {
1826        batches
1827            .iter()
1828            .flat_map(|batch| {
1829                let array = batch
1830                    .column_by_name(column)
1831                    .unwrap_or_else(|| panic!("no `{column}` column"))
1832                    .as_any()
1833                    .downcast_ref::<StringArray>()
1834                    .expect("a label column is Utf8");
1835                (0..array.len())
1836                    .map(|row| array.value(row).to_string())
1837                    .collect::<Vec<_>>()
1838            })
1839            .collect()
1840    }
1841
1842    /// Asserts `plan` is a native plan that reads no region, and that every fused node in it binds
1843    /// no metric: a data-free or zero-match statement has no table to name.
1844    fn assert_regionless_native_plan(plan: &LogicalPlan, query: &str) {
1845        let display = plan.display_indent().to_string();
1846        assert!(display.contains("FusedSeries"), "{query}: {display}");
1847        for absent in ["TableScan", "MergeScan", "RawSeriesSource"] {
1848            assert!(
1849                !display.contains(absent),
1850                "{query} contains {absent}: {display}"
1851            );
1852        }
1853        let fused = fused_series_nodes(plan);
1854        assert!(!fused.is_empty(), "{query}: {display}");
1855        for node in fused {
1856            assert_eq!(
1857                None,
1858                node.binding().metric_name,
1859                "{query} bound a metric to an input that reads no table: {display}"
1860            );
1861        }
1862    }
1863
1864    /// Asserts every empty input under `plan` declares the typed empty raw series frame: the
1865    /// planner's own definition of that input, and the frame a scan would have declared, so the
1866    /// physical planner reads it as a frame producer instead of looking for a scan below it.
1867    fn assert_typed_empty_inputs(plan: &LogicalPlan, query: &str) {
1868        let empty = logical_plan_nodes(plan)
1869            .into_iter()
1870            .filter(|node| matches!(node, LogicalPlan::EmptyRelation(_)))
1871            .collect::<Vec<_>>();
1872        assert!(!empty.is_empty(), "{query} has no empty input");
1873        for node in empty {
1874            assert!(
1875                crate::promql::planner::native::is_empty_discovered_input(node),
1876                "{query}: {node:?}"
1877            );
1878            let LogicalPlan::EmptyRelation(relation) = node else {
1879                unreachable!();
1880            };
1881            assert!(!relation.produce_one_row, "{query}: {relation:?}");
1882            let schema = relation.schema.as_arrow();
1883            for (key, expected) in [
1884                ("greptime.series.content", "raw"),
1885                ("greptime.series.labels", "explicit"),
1886                ("greptime.series.ordering", "contiguous"),
1887            ] {
1888                assert_eq!(
1889                    Some(expected),
1890                    schema.metadata().get(key).map(String::as_str),
1891                    "{query}: {key}"
1892                );
1893            }
1894            // The one value field the empty input is bound to, named as the route names a value.
1895            let fields = schema.field_with_name("fields").expect("a series frame");
1896            let arrow_schema::DataType::Struct(payloads) = fields.data_type() else {
1897                panic!("{query}: {fields:?}");
1898            };
1899            assert_eq!(
1900                vec!["value"],
1901                payloads
1902                    .iter()
1903                    .map(|field| field.name().as_str())
1904                    .collect::<Vec<_>>(),
1905                "{query}"
1906            );
1907        }
1908    }
1909
1910    /// Asserts the physical plan of a regionless native statement: the fused node runs on the
1911    /// frontend and no region read is planned.
1912    fn assert_regionless_physical_plan(
1913        physical: &Arc<dyn datafusion::physical_plan::ExecutionPlan>,
1914        query: &str,
1915    ) {
1916        let names = physical_plan_names(physical);
1917        assert!(
1918            names.iter().any(|name| name == "FusedSeriesExec"),
1919            "{query} is evaluated on the frontend by the fused node: {physical:?}"
1920        );
1921        assert!(
1922            !names.iter().any(|name| name == "MergeScanExec"),
1923            "{query} reads no region: {physical:?}"
1924        );
1925    }
1926
1927    /// A statement that reads no table plans natively over the wire, where there is no region to
1928    /// place anything in: the frontend evaluates the fused node with expression-named values.
1929    #[tokio::test]
1930    async fn test_remote_native_promql_plans_a_data_free_statement_natively() {
1931        let engine = create_promql_test_engine_with(
1932            crate::options::QueryOptions {
1933                experimental_native_promql: true,
1934                ..Default::default()
1935            },
1936            false,
1937        );
1938        let query_ctx = native_discovery_query_ctx();
1939        for (query, expected_values, expected_labels, name_column) in [
1940            // `time()` answers the evaluation grid in seconds. It is a scalar root: it publishes
1941            // one value and no series, so its rows layout declares no name column.
1942            ("time()", &[0.0, 5.0, 10.0][..], &[][..], false),
1943            // `vector(1)` is a samples root: its rows layout declares the per-row name column,
1944            // which names no row because the vector carries no series identity.
1945            ("vector(1)", &[1.0, 1.0, 1.0][..], &[][..], true),
1946            (
1947                r#"label_replace(vector(1), "host", "host1", "", "")"#,
1948                &[1.0, 1.0, 1.0][..],
1949                &[("host", "host1")][..],
1950                true,
1951            ),
1952            // A synthesized `__tsid` is published under the statement's own columns: the stripping
1953            // projection above the node is read through rather than mistaken for a remote plan.
1954            (
1955                r#"label_replace(vector(1), "__tsid", "x", "", "")"#,
1956                &[1.0, 1.0, 1.0][..],
1957                &[][..],
1958                true,
1959            ),
1960        ] {
1961            let plan = plan_remote_native_promql(&engine, &query_ctx, query).await;
1962            let display = plan.display_indent().to_string();
1963            assert_regionless_native_plan(&plan, query);
1964            assert!(
1965                !display.contains("greptime_value"),
1966                "{query} keeps the native value column: {display}"
1967            );
1968            // A data-free statement has no input at all, so nothing is scanned or left empty. The
1969            // value is named after the expression, and a samples root's rows keep the name column
1970            // last.
1971            let expression = promql_parser::parser::parse(query).unwrap().to_string();
1972            let mut expected_columns = vec!["time", expression.as_str()];
1973            expected_columns.extend(expected_labels.iter().map(|(name, _)| *name));
1974            if name_column {
1975                expected_columns.push("__name__");
1976            }
1977            assert_eq!(
1978                expected_columns,
1979                plan.schema()
1980                    .fields()
1981                    .iter()
1982                    .map(|field| field.name().as_str())
1983                    .collect::<Vec<_>>(),
1984                "{query}: {display}"
1985            );
1986            let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await;
1987            assert_regionless_physical_plan(&physical, query);
1988            assert_eq!(
1989                expected_values,
1990                numeric_values(&batches, &expression).as_slice(),
1991                "{query}: {batches:?}"
1992            );
1993            for (label, value) in expected_labels {
1994                assert_eq!(
1995                    vec![value.to_string(); batches_row_count(&batches)],
1996                    label_values(&batches, label),
1997                    "{query}: label `{label}`: {batches:?}"
1998                );
1999            }
2000        }
2001    }
2002
2003    /// The rows every batch of `batches` holds.
2004    fn batches_row_count(batches: &[RecordBatch]) -> usize {
2005        batches.iter().map(|batch| batch.num_rows()).sum()
2006    }
2007
2008    /// A nameless selector that matches no metric table has no region to read either, so the typed
2009    /// empty input it is planned as stays that input and the node above it is evaluated on the
2010    /// frontend. Binding a metric to it is what used to turn this statement into an internal error.
2011    #[tokio::test]
2012    async fn test_remote_native_promql_plans_a_zero_match_selector_natively() {
2013        let engine = create_promql_test_engine_with(
2014            crate::options::QueryOptions {
2015                experimental_native_promql: true,
2016                ..Default::default()
2017            },
2018            false,
2019        );
2020        let query_ctx = native_discovery_query_ctx();
2021        for (query, expected_rows, expected_labels) in [
2022            // The selection itself, over the typed empty frame.
2023            (r#"{job="api"}"#, 0, &[][..]),
2024            // A whole-vector operation over it, which the route plans as a segment: its parent node
2025            // reads the segment's own fused node.
2026            (r#"sort({job="api"})"#, 0, &[][..]),
2027            // A fold splits into a `Final` half over the `Partial` half that reads the input.
2028            (r#"sum({job="api"})"#, 0, &[][..]),
2029            // `absent` answers one series per evaluation point over zero input, labelled by the
2030            // selector's matchers. The grid holds three points.
2031            (r#"absent({job="api"})"#, 3, &[("job", "api")][..]),
2032        ] {
2033            let plan = plan_remote_native_promql(&engine, &query_ctx, query).await;
2034            let display = plan.display_indent().to_string();
2035            assert_regionless_native_plan(&plan, query);
2036            assert!(
2037                display.contains("EmptyRelation"),
2038                "{query} keeps the typed empty input: {display}"
2039            );
2040            assert_typed_empty_inputs(&plan, query);
2041            let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await;
2042            assert_regionless_physical_plan(&physical, query);
2043            // No table matched, so the answer is the empty vector, not an error.
2044            assert_eq!(
2045                expected_rows,
2046                batches_row_count(&batches),
2047                "{query} answered rows over a selector that matches no metric: {physical:?}"
2048            );
2049            for (label, value) in expected_labels {
2050                assert_eq!(
2051                    vec![value.to_string(); expected_rows],
2052                    label_values(&batches, label),
2053                    "{query}: label `{label}`: {batches:?}"
2054                );
2055            }
2056        }
2057
2058        // An allowlist that admits no metric at all reaches the same empty input: the answer comes
2059        // from the statement, not from whatever the catalog holds.
2060        let query_ctx = native_query_ctx(MetricDiscoveryScope::allowed(std::iter::empty()));
2061        let query = r#"{job="api"}"#;
2062        let plan = plan_remote_native_promql(&engine, &query_ctx, query).await;
2063        assert_regionless_native_plan(&plan, "denied discovery");
2064        assert_typed_empty_inputs(&plan, "denied discovery");
2065        let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await;
2066        assert_regionless_physical_plan(&physical, "denied discovery");
2067        assert_eq!(
2068            0,
2069            batches_row_count(&batches),
2070            "denied discovery: {physical:?}"
2071        );
2072    }
2073
2074    /// Plans `sql` and runs the DataFusion analyzer, which is where
2075    /// `InsertAssignmentRule` sits. Planning alone stops short of it, so these
2076    /// assertions would not see the assignment rewrite at all.
2077    async fn analyze_insert(
2078        engine: &QueryEngineRef,
2079        sql: &str,
2080        query_ctx: &QueryContextRef,
2081    ) -> String {
2082        let stmt = QueryLanguageParser::parse_sql(sql, query_ctx).unwrap();
2083        let plan = engine
2084            .planner()
2085            .plan(&stmt, query_ctx.clone())
2086            .await
2087            .unwrap();
2088        let context = engine.engine_context(query_ctx.clone()).unwrap();
2089        let state = context.state();
2090        state
2091            .analyzer()
2092            .execute_and_check(plan, state.config_options(), |_, _| {})
2093            .unwrap()
2094            .display_indent_schema()
2095            .to_string()
2096    }
2097
2098    #[tokio::test]
2099    async fn test_insert_timestamp_literals_use_query_timezone() {
2100        let query_ctx = Arc::new(
2101            QueryContextBuilder::default()
2102                .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2103                .build(),
2104        );
2105        let engine = create_timestamp_test_engine().await;
2106
2107        for (sql, expected_timestamps) in [
2108            (
2109                "INSERT INTO timestamps (ts, st) \
2110                 VALUES ('2026-08-02 12:00:00.001', now())",
2111                &[1_785_643_200_001_i64][..],
2112            ),
2113            (
2114                "INSERT INTO timestamps (ts, st) \
2115                 SELECT '2026-08-03 12:00:00.001', now()",
2116                &[1_785_729_600_001_i64][..],
2117            ),
2118            (
2119                "INSERT INTO timestamps (ts, st) \
2120                 SELECT '2026-08-04 12:00:00.001', now() LIMIT 1",
2121                &[1_785_816_000_001_i64][..],
2122            ),
2123            (
2124                "INSERT INTO timestamps (ts, st) \
2125                 SELECT * FROM (\
2126                     SELECT '2026-08-05 12:00:00.001', now()\
2127                 ) AS source",
2128                &[1_785_902_400_001_i64][..],
2129            ),
2130            (
2131                "INSERT INTO timestamps (ts, st) \
2132                 SELECT '2026-08-18 12:00:00.001', max(st) \
2133                 FROM timestamps GROUP BY note",
2134                &[1_787_025_600_001_i64][..],
2135            ),
2136            (
2137                "INSERT INTO timestamps (ts, st) \
2138                 SELECT c, s FROM (\
2139                     SELECT '2026-08-13 12:00:00.001' AS c, now() AS s\
2140                 ) AS t WHERE c > '2026-01-01'",
2141                &[1_786_593_600_001_i64][..],
2142            ),
2143            (
2144                "INSERT INTO timestamps (ts, st) \
2145                 SELECT c, s FROM (\
2146                     SELECT '2026-08-14 12:00:00.001' AS c, now() AS s\
2147                 ) AS t ORDER BY c",
2148                &[1_786_680_000_001_i64][..],
2149            ),
2150            (
2151                "INSERT INTO timestamps (ts, st) \
2152                 SELECT DISTINCT c, s FROM (\
2153                     SELECT '2026-08-15 12:00:00.001' AS c, now() AS s\
2154                 ) AS t",
2155                &[1_786_766_400_001_i64][..],
2156            ),
2157        ] {
2158            let plan = analyze_insert(&engine, sql, &query_ctx).await;
2159
2160            for expected_timestamp in expected_timestamps {
2161                assert!(
2162                    plan.contains(&format!("TimestampMillisecond({expected_timestamp}, None)")),
2163                    "{plan}"
2164                );
2165            }
2166        }
2167    }
2168
2169    #[tokio::test]
2170    async fn test_insert_explicit_timestamp_cast_keeps_datafusion_semantics() {
2171        let query_ctx = Arc::new(
2172            QueryContextBuilder::default()
2173                .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2174                .build(),
2175        );
2176        let engine = create_timestamp_test_engine().await;
2177        let sql = "INSERT INTO timestamps (ts, st) \
2178                   VALUES (CAST('2026-08-08 12:00:00.001' AS TIMESTAMP), now())";
2179        let plan = analyze_insert(&engine, sql, &query_ctx).await;
2180
2181        // An explicit cast reaches the analyzer as an `arrow_cast` call rather
2182        // than an `Expr::Cast`, which is how it stays out of the rewrite.
2183        assert!(
2184            plan.contains("arrow_cast(Utf8(\"2026-08-08 12:00:00.001\")"),
2185            "{plan}"
2186        );
2187        // 12:00:00.001 read as Shanghai local time; the source query keeps UTC.
2188        assert!(
2189            !plan.contains("TimestampMillisecond(1786104000001, None)"),
2190            "{plan}"
2191        );
2192    }
2193
2194    #[tokio::test]
2195    async fn test_insert_converts_source_literal_shared_by_several_columns() {
2196        let query_ctx = Arc::new(
2197            QueryContextBuilder::default()
2198                .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2199                .build(),
2200        );
2201        let engine = create_timestamp_test_engine().await;
2202
2203        // Rewriting `c` in place would also retype `note` and truncate `ts_ns`.
2204        for (sql, expected) in [
2205            (
2206                "INSERT INTO timestamps (ts, note) SELECT a, b FROM (\
2207                     SELECT c AS a, c AS b FROM (\
2208                         SELECT '2026-08-12 12:00:00.001' AS c\
2209                     ) AS t1\
2210                 ) AS t2",
2211                &[
2212                    "TimestampMillisecond(1786507200001, None)",
2213                    "[a:Utf8, b:Utf8]",
2214                ][..],
2215            ),
2216            (
2217                "INSERT INTO timestamps (ts, ts_ns) SELECT a, b FROM (\
2218                     SELECT c AS a, c AS b FROM (\
2219                         SELECT '2026-08-12 12:00:00.123456789' AS c\
2220                     ) AS t1\
2221                 ) AS t2",
2222                &[
2223                    "TimestampMillisecond(1786507200123, None)",
2224                    "TimestampNanosecond(1786507200123456789, None)",
2225                ][..],
2226            ),
2227        ] {
2228            let plan = analyze_insert(&engine, sql, &query_ctx).await;
2229
2230            for expected in expected {
2231                assert!(plan.contains(expected), "{plan}");
2232            }
2233        }
2234    }
2235
2236    #[tokio::test]
2237    async fn test_insert_union_converts_via_assignment_cast() {
2238        let query_ctx = Arc::new(
2239            QueryContextBuilder::default()
2240                .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2241                .build(),
2242        );
2243        let engine = create_timestamp_test_engine().await;
2244        // Branches disagree, so the conversion stays as a cast on the
2245        // assignment instead of folding. One cast covers every branch, which is
2246        // why a NULL branch no longer cancels the conversion for the column and
2247        // why UNION's dedup keys stay on the original strings.
2248        for sql in [
2249            "INSERT INTO timestamps (ts, st) \
2250             SELECT '2026-08-06 12:00:00.001', now() \
2251             UNION ALL \
2252             SELECT '2026-08-07 12:00:00.001', NULL",
2253            "INSERT INTO timestamps (ts, st) \
2254             SELECT '2026-08-16 12:00:00.001', now() \
2255             UNION \
2256             SELECT '2026-08-17 12:00:00.001', now()",
2257        ] {
2258            let plan = analyze_insert(&engine, sql, &query_ctx).await;
2259
2260            assert!(
2261                plan.contains("AS Timestamp(ms, \"Asia/Shanghai\")"),
2262                "{plan}"
2263            );
2264            // The branches themselves are untouched.
2265            assert!(
2266                plan.contains("Utf8(\"2026-08-07 12:00:00.001\")")
2267                    || plan.contains("Utf8(\"2026-08-17 12:00:00.001\")"),
2268                "{plan}"
2269            );
2270        }
2271    }
2272
2273    #[tokio::test]
2274    async fn test_insert_mixed_union_keeps_source_coercion() {
2275        let query_ctx = Arc::new(
2276            QueryContextBuilder::default()
2277                .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2278                .build(),
2279        );
2280        let engine = create_timestamp_test_engine().await;
2281        let sql = "INSERT INTO timestamps (ts, st) \
2282                   SELECT '2026-08-10 12:00:00.001', now() \
2283                   UNION ALL \
2284                   SELECT CAST('2026-08-11 12:00:00.001' AS TIMESTAMP), now()";
2285        let plan = analyze_insert(&engine, sql, &query_ctx).await;
2286
2287        assert!(
2288            !plan.contains("TimestampMillisecond(1786334400001, None)"),
2289            "{plan}"
2290        );
2291        assert!(
2292            plan.contains("arrow_cast(Utf8(\"2026-08-11 12:00:00.001\")"),
2293            "{plan}"
2294        );
2295        // TypeCoercion has already settled this union to timestamp, so the
2296        // assignment has nothing left to reinterpret. Retargeting the cast here
2297        // would leave a Timestamp(None) -> Timestamp(Some(tz)) step behind,
2298        // which shifts the value instead of relabelling it.
2299        assert!(!plan.contains("Asia/Shanghai"), "{plan}");
2300    }
2301
2302    #[tokio::test]
2303    async fn test_extract_placeholder_cast_types_multiple() {
2304        let plan = parse_sql_to_plan(
2305            "SELECT $1::INT, $2::TEXT, $3, $4::INTEGER FROM test WHERE $5::FLOAT > 0",
2306        )
2307        .await;
2308        let types = DfLogicalPlanner::extract_placeholder_cast_types(&plan).unwrap();
2309
2310        assert_eq!(types.len(), 5);
2311        assert_eq!(types.get("$1"), Some(&Some(DataType::Int32)));
2312        assert_eq!(types.get("$2"), Some(&Some(DataType::Utf8)));
2313        assert_eq!(types.get("$3"), Some(&None));
2314        assert_eq!(types.get("$4"), Some(&Some(DataType::Int32)));
2315        assert_eq!(types.get("$5"), Some(&Some(DataType::Float32)));
2316    }
2317
2318    #[tokio::test]
2319    async fn test_get_inferred_parameter_types_fallback_for_udf_args() {
2320        // datafusion is not able to infer type for scalar function arguments
2321        let plan = parse_sql_to_plan(
2322            "SELECT parse_ident($1), parse_ident($2::TEXT) FROM test WHERE id > $3",
2323        )
2324        .await;
2325        let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2326
2327        assert_eq!(types.len(), 3);
2328
2329        let type_1 = types.get("$1").unwrap();
2330        let type_2 = types.get("$2").unwrap();
2331        let type_3 = types.get("$3").unwrap();
2332
2333        assert!(type_1.is_none(), "Expected $1 to be None");
2334        assert_eq!(type_2, &Some(DataType::Utf8));
2335        assert_eq!(type_3, &Some(DataType::Int32));
2336    }
2337
2338    #[tokio::test]
2339    async fn test_get_inferred_parameter_types_limit_offset() {
2340        let plan = parse_sql_to_plan("SELECT id FROM test LIMIT $1 OFFSET $2").await;
2341        let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2342
2343        assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
2344        assert_eq!(types.get("$2"), Some(&Some(DataType::Int64)));
2345    }
2346
2347    #[tokio::test]
2348    async fn test_plan_pql_applies_extension_rules() {
2349        for inner_agg in ["count", "sum", "avg", "min", "max", "stddev", "stdvar"] {
2350            let plan = parse_promql_to_plan(&format!(
2351                "sum(irate(some_metric[1h])) / scalar(count({inner_agg}(some_metric) by (tag_0)))"
2352            ))
2353            .await;
2354            let plan_str = plan.display_indent_schema().to_string();
2355            assert!(plan_str.contains("Distinct:"), "{inner_agg}: {plan_str}");
2356        }
2357    }
2358
2359    #[tokio::test]
2360    async fn test_plan_pql_filters_null_only_groups_for_non_count_inner_aggs() {
2361        let count_plan = parse_promql_to_plan("scalar(count(count(some_metric) by (tag_0)))").await;
2362        let count_plan_str = count_plan.display_indent_schema().to_string();
2363        assert!(
2364            !count_plan_str.contains("field_0 IS NOT NULL"),
2365            "{count_plan_str}"
2366        );
2367
2368        for inner_agg in ["sum", "avg", "min", "max", "stddev", "stdvar"] {
2369            let plan = parse_promql_to_plan(&format!(
2370                "scalar(count({inner_agg}(some_metric) by (tag_0)))"
2371            ))
2372            .await;
2373            let plan_str = plan.display_indent_schema().to_string();
2374            assert!(
2375                plan_str.contains("field_0 IS NOT NULL"),
2376                "{inner_agg}: {plan_str}"
2377            );
2378        }
2379    }
2380
2381    #[tokio::test]
2382    async fn test_plan_pql_skips_extension_rules_for_non_direct_or_unsupported_inner_agg() {
2383        for query in [
2384            "sum(irate(some_metric[1h])) / scalar(count(sum(irate(some_metric[1h])) by (tag_0)))",
2385            "sum(irate(some_metric[1h])) / scalar(count(group(some_metric) by (tag_0)))",
2386        ] {
2387            let plan = parse_promql_to_plan(query).await;
2388            let plan_str = plan.display_indent_schema().to_string();
2389            assert!(!plan_str.contains("Distinct:"), "{query}: {plan_str}");
2390        }
2391    }
2392
2393    #[tokio::test]
2394    async fn test_plan_sql_does_not_apply_nested_count_rule() {
2395        let plan = parse_sql_to_plan(
2396            "SELECT id, count(inner_count) \
2397             FROM ( \
2398                 SELECT id, count(name) AS inner_count \
2399                 FROM test \
2400                 GROUP BY id \
2401                 ORDER BY id \
2402                 LIMIT 1000000 \
2403             ) t \
2404             GROUP BY id \
2405             ORDER BY id",
2406        )
2407        .await;
2408
2409        let plan_str = plan.display_indent_schema().to_string();
2410        assert!(!plan_str.contains("Distinct:"), "{plan_str}");
2411    }
2412
2413    #[tokio::test]
2414    async fn test_get_inferred_parameter_types_subquery() {
2415        let plan = parse_sql_to_plan(
2416            r#"SELECT * FROM test WHERE id = (SELECT id FROM test CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) p LIMIT 1)"#,
2417        ).await;
2418        let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2419
2420        assert_eq!(types.len(), 1);
2421        let type_1 = types.get("$1").unwrap();
2422        assert_eq!(type_1, &Some(DataType::Utf8));
2423    }
2424
2425    #[tokio::test]
2426    async fn test_get_inferred_parameter_types_insert() {
2427        let plan = parse_sql_to_plan("INSERT INTO test (id, name) VALUES ($1, $2), ($3, $4)").await;
2428        let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2429
2430        assert_eq!(types.len(), 4);
2431        assert_eq!(types.get("$1"), Some(&Some(DataType::Int32)));
2432        assert_eq!(types.get("$2"), Some(&Some(DataType::Utf8)));
2433        assert_eq!(types.get("$3"), Some(&Some(DataType::Int32)));
2434        assert_eq!(types.get("$4"), Some(&Some(DataType::Utf8)));
2435    }
2436
2437    /// A plan that publishes `columns` as its output, one literal per column, named and typed as
2438    /// the column is.
2439    fn plan_publishing(columns: &[(&str, ScalarValue)]) -> LogicalPlan {
2440        let exprs = columns
2441            .iter()
2442            .map(|(name, value)| DfExpr::Literal(value.clone(), None).alias(*name))
2443            .collect::<Vec<_>>();
2444        LogicalPlanBuilder::empty(false)
2445            .project(exprs)
2446            .unwrap()
2447            .build()
2448            .unwrap()
2449    }
2450
2451    /// [`plan_publishing`], declaring what a plan of either route declares: the time index marked as
2452    /// one, and the schema version GreptimeDB's schemas carry.
2453    fn plan_publishing_marked(columns: &[(&str, ScalarValue)], time_index: &str) -> LogicalPlan {
2454        let plan = plan_publishing(columns);
2455        let LogicalPlan::Projection(projection) = plan else {
2456            unreachable!("plan_publishing builds a projection")
2457        };
2458        let fields = projection
2459            .schema
2460            .iter()
2461            .map(|(_, field)| {
2462                let field =
2463                    if field.name() == time_index {
2464                        Arc::new(field.as_ref().clone().with_metadata(
2465                            std::collections::HashMap::from([(
2466                                datatypes::schema::TIME_INDEX_KEY.to_string(),
2467                                "true".to_string(),
2468                            )]),
2469                        ))
2470                    } else if field.name() == "__name__" {
2471                        // The native rows layout marks the column that names each row.
2472                        Arc::new(field.as_ref().clone().with_metadata(
2473                            std::collections::HashMap::from([(
2474                                datatypes::schema::METRIC_NAME_KEY.to_string(),
2475                                "true".to_string(),
2476                            )]),
2477                        ))
2478                    } else {
2479                        Arc::clone(field)
2480                    };
2481                (None, field)
2482            })
2483            .collect::<Vec<_>>();
2484        let schema = Arc::new(
2485            DFSchema::new_with_metadata(
2486                fields,
2487                std::collections::HashMap::from([(
2488                    datatypes::schema::VERSION_KEY.to_string(),
2489                    datatypes::schema::Schema::INITIAL_VERSION.to_string(),
2490                )]),
2491            )
2492            .unwrap(),
2493        );
2494        LogicalPlan::Projection(
2495            datafusion_expr::Projection::try_new_with_schema(
2496                projection.expr.clone(),
2497                projection.input.clone(),
2498                schema,
2499            )
2500            .unwrap(),
2501        )
2502    }
2503
2504    /// The label column the existing planner publishes: dictionary-encoded text.
2505    fn dictionary_label(value: &str) -> ScalarValue {
2506        ScalarValue::Dictionary(
2507            Box::new(DataType::UInt32),
2508            Box::new(ScalarValue::Utf8(Some(value.to_string()))),
2509        )
2510    }
2511
2512    /// The names of a plan's output columns, in order.
2513    fn column_names(plan: &LogicalPlan) -> Vec<String> {
2514        plan.schema()
2515            .fields()
2516            .iter()
2517            .map(|field| field.name().clone())
2518            .collect()
2519    }
2520
2521    fn time_value() -> ScalarValue {
2522        ScalarValue::TimestampMillisecond(Some(0), None)
2523    }
2524
2525    /// The native answer for the statement these tests use: time index first, the value under the
2526    /// tree's own field name, the labels last.
2527    fn native_answer() -> LogicalPlan {
2528        plan_publishing_marked(
2529            &[
2530                ("ts", time_value()),
2531                ("val", ScalarValue::Float64(Some(2.0))),
2532                ("host", ScalarValue::Utf8(Some("a".to_string()))),
2533            ],
2534            "ts",
2535        )
2536    }
2537
2538    /// The existing planner's answer for the same statement: labels first, then the time index, then
2539    /// the value named after the expression it planned, with the label dictionary-encoded.
2540    fn existing_answer() -> LogicalPlan {
2541        plan_publishing_marked(
2542            &[
2543                ("host", dictionary_label("a")),
2544                ("ts", time_value()),
2545                ("lhs.val + rhs.val", ScalarValue::Float64(Some(2.0))),
2546            ],
2547            "ts",
2548        )
2549    }
2550
2551    /// A CTE publishes the existing planner's columns over a native plan as well.
2552    ///
2553    /// The fused plan publishes the time index first, the value under the tree's field name and the
2554    /// labels last, in plain text; the existing planner publishes the labels first, then the time
2555    /// index, then the value under the name of the expression it planned, in dictionary-encoded text.
2556    /// What a CTE exposes is the second one, whichever route answered the statement, because a CTE's
2557    /// columns are SQL-visible - and a column list the CTE declares is applied to them by position.
2558    #[test]
2559    fn a_cte_publishes_the_existing_planners_columns_over_a_native_plan() {
2560        let native = native_answer();
2561        let existing = existing_answer();
2562
2563        let published = publish_legacy_schema(native, &existing)
2564            .expect("every column has one answer in the other plan");
2565
2566        assert_eq!(
2567            column_names(&published),
2568            vec!["host", "ts", "lhs.val + rhs.val"],
2569            "the CTE publishes the existing planner's column order and names"
2570        );
2571        assert!(
2572            publishes_columns(published.schema().as_ref(), existing.schema().as_ref()),
2573            "and every column it publishes: name, order, type, nullability and metadata: {}",
2574            published.schema()
2575        );
2576        assert_eq!(
2577            published
2578                .schema()
2579                .field_with_name(None, "host")
2580                .unwrap()
2581                .data_type(),
2582            &DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
2583            "a label is published as the encoding the existing planner publishes"
2584        );
2585        let LogicalPlan::Projection(projection) = &published else {
2586            panic!("the native plan is wrapped in a projection: {published}");
2587        };
2588        assert_eq!(
2589            column_names(projection.input.as_ref()),
2590            vec!["ts", "val", "host"],
2591            "the native plan is kept as the projection's input"
2592        );
2593    }
2594
2595    /// A native answer that declares the per-row name column publishes it after the existing
2596    /// planner's columns.
2597    ///
2598    /// The name column is text like a label, so pairing by type alone would leave `host` with two
2599    /// candidates and no answer at all; the metadata it carries is what tells the two apart, and
2600    /// what keeps it from standing in for a label. The CTE publishes the existing planner's columns
2601    /// first and in its order, then the name the answer carries: the column is part of what the
2602    /// statement answers.
2603    #[test]
2604    fn a_cte_publishes_the_name_an_answer_carries_after_the_existing_columns() {
2605        let native = plan_publishing_marked(
2606            &[
2607                ("ts", time_value()),
2608                ("val", ScalarValue::Float64(Some(2.0))),
2609                ("host", ScalarValue::Utf8(Some("a".to_string()))),
2610                (
2611                    "__name__",
2612                    ScalarValue::Utf8(Some("cte_native".to_string())),
2613                ),
2614            ],
2615            "ts",
2616        );
2617        let existing = existing_answer();
2618
2619        let published = publish_legacy_schema(native, &existing)
2620            .expect("the declared name column does not compete with a label");
2621
2622        assert_eq!(
2623            column_names(&published),
2624            vec!["host", "ts", "lhs.val + rhs.val", "__name__"],
2625            "the CTE publishes the existing planner's columns, then the name the answer carries"
2626        );
2627        assert!(
2628            datatypes::schema::is_metric_name_field(
2629                published
2630                    .schema()
2631                    .field_with_name(None, "__name__")
2632                    .unwrap()
2633            ),
2634            "the name column is marked as one: {}",
2635            published.schema()
2636        );
2637    }
2638
2639    /// What the CTE publishes survives the analyzer, which rebuilds every projection from its
2640    /// expressions.
2641    ///
2642    /// The declared schema of a projection is not what analysis keeps: the analyzer calls
2643    /// `with_new_exprs`, which rebuilds the projection and derives its schema from the expressions
2644    /// and their input. So the columns have to be carried by the expressions themselves - an alias
2645    /// per column and a cast for the encoding - or they are gone by the time the CTE is read.
2646    #[test]
2647    fn what_a_cte_publishes_survives_the_analyzer_rebuilding_the_projection() {
2648        let existing = existing_answer();
2649        let published = publish_legacy_schema(native_answer(), &existing).unwrap();
2650
2651        // The rebuild `DistPlannerAnalyzer` performs on every projection it visits.
2652        let rebuilt = published
2653            .with_new_exprs(
2654                published.expressions(),
2655                published.inputs().into_iter().cloned().collect(),
2656            )
2657            .unwrap();
2658
2659        assert!(
2660            publishes_columns(rebuilt.schema().as_ref(), existing.schema().as_ref()),
2661            "the rebuilt projection publishes the same columns: {}",
2662            rebuilt.schema()
2663        );
2664        assert_eq!(
2665            rebuilt.to_string(),
2666            published.to_string(),
2667            "and the same expressions"
2668        );
2669    }
2670
2671    /// The declared name column survives the analyzer rebuilding the projection as well.
2672    #[test]
2673    fn the_declared_name_column_survives_the_analyzer_rebuilding_the_projection() {
2674        let existing = existing_answer();
2675        let native = plan_publishing_marked(
2676            &[
2677                ("ts", time_value()),
2678                ("val", ScalarValue::Float64(Some(2.0))),
2679                ("host", ScalarValue::Utf8(Some("a".to_string()))),
2680                (
2681                    "__name__",
2682                    ScalarValue::Utf8(Some("cte_native".to_string())),
2683                ),
2684            ],
2685            "ts",
2686        );
2687        let published = publish_legacy_schema(native, &existing)
2688            .expect("the name column does not compete with a label");
2689
2690        // The rebuild `DistPlannerAnalyzer` performs on every projection it visits.
2691        let rebuilt = published
2692            .with_new_exprs(
2693                published.expressions(),
2694                published.inputs().into_iter().cloned().collect(),
2695            )
2696            .expect("the analyzer rebuilds a projection from its expressions");
2697
2698        let field = rebuilt
2699            .schema()
2700            .field_with_name(None, "__name__")
2701            .expect("the name column is still published");
2702        assert!(
2703            datatypes::schema::is_metric_name_field(field),
2704            "and is still marked as one: {}",
2705            rebuilt.schema()
2706        );
2707    }
2708
2709    /// A statement whose columns do not pair keeps the existing plan.
2710    ///
2711    /// Publishing a column this cannot fill would change what the CTE means, so a shape it cannot
2712    /// map faithfully is refused instead: the caller keeps the existing plan and the CTE keeps the
2713    /// columns it has always published.
2714    #[test]
2715    fn a_shape_with_no_faithful_column_mapping_keeps_the_existing_plan() {
2716        // One native value column cannot fill two of the existing plan's value columns.
2717        let two_values = plan_publishing_marked(
2718            &[
2719                ("host", dictionary_label("a")),
2720                ("ts", time_value()),
2721                ("lhs.val + rhs.val", ScalarValue::Float64(Some(2.0))),
2722                ("lhs.val - rhs.val", ScalarValue::Float64(Some(0.0))),
2723            ],
2724            "ts",
2725        );
2726        assert!(publish_legacy_schema(native_answer(), &two_values).is_none());
2727
2728        // A time index the other route had to rename is marked as the time index on both sides, and
2729        // a column that carries metadata is never reproduced by a cast.
2730        let renamed_time = plan_publishing_marked(
2731            &[
2732                ("native_time_0", time_value()),
2733                ("val", ScalarValue::Float64(Some(2.0))),
2734                ("host", ScalarValue::Utf8(Some("a".to_string()))),
2735            ],
2736            "native_time_0",
2737        );
2738        let existing = plan_publishing_marked(
2739            &[
2740                ("host", dictionary_label("a")),
2741                ("ts", time_value()),
2742                ("val", ScalarValue::Float64(Some(2.0))),
2743            ],
2744            "ts",
2745        );
2746        assert!(publish_legacy_schema(renamed_time, &existing).is_none());
2747    }
2748
2749    #[tokio::test]
2750    async fn test_get_inferred_parameter_types_arrow_cast() {
2751        let plan = parse_sql_to_plan("SELECT $1::INT64, $2::FLOAT64, $3::INT16, $4::INT32, $5::UINT8, $6::UINT16, $7::UINT32").await;
2752        let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2753
2754        assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
2755        assert_eq!(types.get("$2"), Some(&Some(DataType::Float64)));
2756        assert_eq!(types.get("$3"), Some(&Some(DataType::Int16)));
2757        assert_eq!(types.get("$4"), Some(&Some(DataType::Int32)));
2758        assert_eq!(types.get("$5"), Some(&Some(DataType::UInt8)));
2759        assert_eq!(types.get("$6"), Some(&Some(DataType::UInt16)));
2760        assert_eq!(types.get("$7"), Some(&Some(DataType::UInt32)));
2761
2762        let plan = parse_sql_to_plan("SELECT $1::INT8, $2::FLOAT8, $3::INT2, $4::INT8").await;
2763        let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2764
2765        assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
2766        assert_eq!(types.get("$2"), Some(&Some(DataType::Float64)));
2767        assert_eq!(types.get("$3"), Some(&Some(DataType::Int16)));
2768    }
2769}