1use std::any::Any;
16use std::borrow::Cow;
17use std::collections::{HashMap, HashSet};
18use std::ops::ControlFlow;
19use std::str::FromStr;
20use std::sync::Arc;
21
22use arrow_schema::DataType;
23use async_trait::async_trait;
24use catalog::table_source::DfTableSourceProvider;
25use common_error::ext::BoxedError;
26use common_query::promql_annotations::promql_annotation_collector;
27use common_telemetry::tracing;
28use datafusion::common::{DFSchema, plan_err};
29use datafusion::execution::SessionStateBuilder;
30use datafusion::execution::context::SessionState;
31use datafusion::sql::planner::PlannerContext;
32use datafusion_common::tree_node::{TreeNode, TreeNodeRecursion};
33use datafusion_common::{ScalarValue, ToDFSchema};
34use datafusion_expr::expr::{Exists, InSubquery};
35use datafusion_expr::{
36 Analyze, Explain, ExplainFormat, Expr as DfExpr, LogicalPlan, LogicalPlanBuilder, PlanType,
37 ToStringifiedPlan, col,
38};
39use datafusion_sql::parser::Statement as DfStatement;
40use datafusion_sql::planner::{IdentNormalizer, ParserOptions, SqlToRel};
41use log_query::LogQuery;
42use promql_parser::parser::EvalStmt;
43use session::context::QueryContextRef;
44use snafu::{ResultExt, ensure};
45use sql::CteContent;
46use sql::ast::Expr as SqlExpr;
47use sql::statements::explain::ExplainStatement;
48use sql::statements::query::Query;
49use sql::statements::statement::Statement;
50use sql::statements::tql::Tql;
51use sqlparser::ast::{AccessExpr, Value, visit_expressions_mut};
52use store_api::metric_engine_consts::DATA_SCHEMA_TSID_COLUMN_NAME;
53
54use crate::dist_plan::DistPlannerOptions;
55use crate::dist_plan::region_local_node;
56use crate::error::{
57 CteColumnSchemaMismatchSnafu, PlanSqlSnafu, QueryPlanSnafu, Result, SqlSnafu,
58 UnimplementedSnafu,
59};
60use crate::log_query::planner::LogQueryPlanner;
61use crate::optimizer::expr_depth;
62use crate::options::{NativePromqlExtension, ScheduledTimeExtension};
63use crate::parser::{DEFAULT_LOOKBACK_STRING, PromQuery, QueryLanguageParser, QueryStatement};
64use crate::promql::planner::PromPlanner;
65use crate::query_engine::{DefaultPlanDecoder, QueryEngineState};
66use crate::range_select::plan_rewrite::RangePlanRewriter;
67use crate::{DfContextProviderAdapter, QueryEngineContext};
68
69#[async_trait]
70pub trait LogicalPlanner: Send + Sync {
71 async fn plan(&self, stmt: &QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan>;
72
73 async fn plan_logs_query(
74 &self,
75 query: LogQuery,
76 query_ctx: QueryContextRef,
77 ) -> Result<LogicalPlan>;
78
79 fn optimize(&self, plan: LogicalPlan) -> Result<LogicalPlan>;
80
81 fn as_any(&self) -> &dyn Any;
82}
83
84fn remote_native_raw_plan_supported(plan: &LogicalPlan) -> bool {
97 fn walk(
100 plan: &LogicalPlan,
101 region_local: bool,
102 found_raw: &mut bool,
103 found_region_local: &mut bool,
104 ) -> bool {
105 match plan {
106 LogicalPlan::TableScan(_) => region_local,
107 LogicalPlan::Extension(extension) => {
108 if extension
109 .node
110 .as_any()
111 .is::<promql::extension_plan::RawSeriesSource>()
112 {
113 *found_raw = true;
114 return matches!(
115 extension.node.inputs().as_slice(),
116 [LogicalPlan::TableScan(_)]
117 );
118 }
119 let region_local = extension
120 .node
121 .as_any()
122 .is::<promql::extension_plan::FusedSeries>()
123 && region_local_node(extension.node.as_ref());
124 *found_region_local |= region_local;
125 extension
126 .node
127 .inputs()
128 .iter()
129 .all(|child| walk(child, region_local, found_raw, found_region_local))
130 }
131 _ => plan
132 .inputs()
133 .iter()
134 .all(|child| walk(child, region_local, found_raw, found_region_local)),
135 }
136 }
137 let mut found_raw = false;
138 let mut found_region_local = false;
139 walk(plan, false, &mut found_raw, &mut found_region_local)
140 && (found_raw || found_region_local || is_regionless_fused_plan(plan))
141}
142
143fn is_regionless_fused_plan(plan: &LogicalPlan) -> bool {
157 if let LogicalPlan::Projection(projection) = plan {
161 return strips_only_tsid(plan, projection.input.as_ref())
162 && is_regionless_fused_chain(projection.input.as_ref());
163 }
164 is_regionless_fused_chain(plan)
165}
166
167fn is_regionless_fused_chain(plan: &LogicalPlan) -> bool {
169 let LogicalPlan::Extension(extension) = plan else {
170 return false;
171 };
172 let Some(fused) = extension
173 .node
174 .as_any()
175 .downcast_ref::<promql::extension_plan::FusedSeries>()
176 else {
177 return false;
178 };
179 fused.children().iter().all(|child| {
180 crate::promql::planner::native::is_empty_discovered_input(child)
181 || is_regionless_fused_chain(child)
182 })
183}
184
185fn strips_only_tsid(plan: &LogicalPlan, inner: &LogicalPlan) -> bool {
191 let LogicalPlan::Projection(projection) = plan else {
192 return false;
193 };
194 if projection
195 .expr
196 .iter()
197 .any(|expr| !matches!(expr, DfExpr::Column(_)))
198 {
199 return false;
200 }
201 let stripped = inner
202 .schema()
203 .fields()
204 .iter()
205 .filter(|field| field.name() != DATA_SCHEMA_TSID_COLUMN_NAME)
206 .map(|field| field.name())
207 .collect::<Vec<_>>();
208 let projected = plan
209 .schema()
210 .fields()
211 .iter()
212 .map(|field| field.name())
213 .collect::<Vec<_>>();
214 projected == stripped && projected.len() + 1 == inner.schema().fields().len()
215}
216
217fn contains_native_fused(plan: &LogicalPlan) -> bool {
219 if let LogicalPlan::Extension(extension) = plan
220 && extension
221 .node
222 .as_any()
223 .is::<promql::extension_plan::FusedSeries>()
224 {
225 return true;
226 }
227 plan.inputs()
228 .iter()
229 .any(|input| contains_native_fused(input))
230}
231
232fn publishes_series_frame(schema: &DFSchema) -> bool {
234 schema
235 .metadata()
236 .keys()
237 .any(|key| key.starts_with("greptime.series."))
238}
239
240fn publishes_columns(published: &DFSchema, wanted: &DFSchema) -> bool {
246 published.fields().len() == wanted.fields().len()
247 && published.metadata() == wanted.metadata()
248 && published
249 .iter()
250 .zip(wanted.iter())
251 .all(|((_, published), (_, wanted))| {
252 published.name() == wanted.name()
253 && published.data_type() == wanted.data_type()
254 && published.is_nullable() == wanted.is_nullable()
255 && published.metadata() == wanted.metadata()
256 })
257}
258
259fn unwrap_dictionary(data_type: &DataType) -> &DataType {
265 let mut data_type = data_type;
266 while let DataType::Dictionary(_, inner) = data_type {
267 data_type = inner.as_ref();
268 }
269 data_type
270}
271
272fn publish_legacy_schema(native: LogicalPlan, legacy: &LogicalPlan) -> Option<LogicalPlan> {
304 let native_schema = native.schema().clone();
305 let legacy_schema = legacy.schema();
306
307 if publishes_series_frame(native_schema.as_ref()) {
311 return None;
312 }
313 if publishes_columns(native_schema.as_ref(), legacy_schema.as_ref()) {
314 return Some(native);
315 }
316
317 let native_fields = native_schema
318 .iter()
319 .map(|(qualifier, field)| (qualifier.cloned(), field))
320 .collect::<Vec<_>>();
321 let mut taken = vec![false; native_fields.len()];
322 let mut source: Vec<Option<usize>> = vec![None; legacy_schema.fields().len()];
323 for (legacy_index, (_, legacy_field)) in legacy_schema.iter().enumerate() {
326 let found =
327 native_fields
328 .iter()
329 .enumerate()
330 .find_map(|(native_index, (_, native_field))| {
331 (!taken[native_index]
332 && native_field.name() == legacy_field.name()
333 && native_field.data_type() == legacy_field.data_type()
334 && native_field.is_nullable() == legacy_field.is_nullable()
335 && native_field.metadata() == legacy_field.metadata())
336 .then_some(native_index)
337 });
338 if let Some(native_index) = found {
339 taken[native_index] = true;
340 source[legacy_index] = Some(native_index);
341 }
342 }
343 for (legacy_index, source) in source.iter_mut().enumerate() {
348 if source.is_some() {
349 continue;
350 }
351 let legacy_field = legacy_schema.field(legacy_index);
352 if !legacy_field.metadata().is_empty() {
353 continue;
354 }
355 let candidates = native_fields
356 .iter()
357 .enumerate()
358 .filter(|(native_index, (_, native_field))| {
359 !taken[*native_index]
360 && unwrap_dictionary(native_field.data_type())
361 == unwrap_dictionary(legacy_field.data_type())
362 && native_field.is_nullable() == legacy_field.is_nullable()
363 && native_field.metadata().is_empty()
368 })
369 .map(|(native_index, _)| native_index)
370 .collect::<Vec<_>>();
371 if let [native_index] = candidates.as_slice() {
372 taken[*native_index] = true;
373 *source = Some(*native_index);
374 }
375 }
376 let source = source.into_iter().collect::<Option<Vec<usize>>>()?;
377
378 let mut exprs = Vec::with_capacity(source.len());
379 for ((_, legacy_field), native_index) in legacy_schema.iter().zip(source) {
380 let (native_qualifier, native_field) = &native_fields[native_index];
381 let column = DfExpr::Column(datafusion_common::Column::new(
382 native_qualifier.clone(),
383 native_field.name().clone(),
384 ));
385 let value = if native_field.data_type() == legacy_field.data_type() {
389 column
390 } else {
391 DfExpr::Cast(datafusion_expr::Cast::new(
392 Box::new(column),
393 legacy_field.data_type().clone(),
394 ))
395 };
396 exprs.push(value.alias(legacy_field.name().clone()));
397 }
398 let name_source = native_fields
403 .iter()
404 .position(|(_, native_field)| datatypes::schema::is_metric_name_field(native_field));
405 if let Some(native_index) = name_source {
406 let (native_qualifier, native_field) = &native_fields[native_index];
407 exprs.push(
408 DfExpr::Column(datafusion_common::Column::new(
409 native_qualifier.clone(),
410 native_field.name().clone(),
411 ))
412 .alias(native_field.name().clone()),
413 );
414 }
415 let plan = LogicalPlanBuilder::from(native)
416 .project(exprs)
417 .ok()?
418 .build()
419 .ok()?;
420 let published = plan.schema();
424 let legacy_len = legacy_schema.fields().len();
425 let expected_len = legacy_len + usize::from(name_source.is_some());
426 (published.fields().len() == expected_len
427 && published.metadata() == legacy_schema.metadata()
428 && legacy_schema.iter().zip(published.iter()).all(
429 |((_, legacy_field), (_, published_field))| {
430 legacy_field.name() == published_field.name()
431 && legacy_field.data_type() == published_field.data_type()
432 && legacy_field.is_nullable() == published_field.is_nullable()
433 && legacy_field.metadata() == published_field.metadata()
434 },
435 )
436 && name_source.is_none_or(|native_index| {
437 let wanted = &native_schema.fields()[native_index];
438 let published = &published.fields()[legacy_len];
439 published.name() == wanted.name()
440 && published.data_type() == wanted.data_type()
441 && published.is_nullable() == wanted.is_nullable()
442 && published.metadata() == wanted.metadata()
443 && datatypes::schema::is_metric_name_field(published.as_ref())
444 }))
445 .then_some(plan)
446}
447
448pub struct DfLogicalPlanner {
449 engine_state: Arc<QueryEngineState>,
450 session_state: SessionState,
451}
452
453impl DfLogicalPlanner {
454 pub fn new(engine_state: Arc<QueryEngineState>) -> Self {
455 let session_state = engine_state.session_state();
456 Self {
457 engine_state,
458 session_state,
459 }
460 }
461
462 fn derive_session_state_with_scheduled_time(
466 &self,
467 query_ctx: QueryContextRef,
468 ) -> Result<QueryEngineContext> {
469 let extensions = query_ctx.extensions();
470 let mut state = match crate::options::parse_scheduled_time_datetime(&extensions)? {
471 Some(dt) => {
472 let execution_props = self
473 .session_state
474 .execution_props()
475 .clone()
476 .with_query_execution_start_time(dt);
477 SessionStateBuilder::new_from_existing(self.session_state.clone())
478 .with_execution_props(execution_props)
479 .build()
480 }
481 None => self.session_state.clone(),
482 };
483 let query_ctx = self
484 .engine_state
485 .apply_query_options(&mut state, query_ctx)?;
486 state.config_mut().set_extension(query_ctx.clone());
487 state.config_mut().set_extension(self.engine_state.clone());
488 state
489 .config_mut()
490 .options_mut()
491 .extensions
492 .insert(ScheduledTimeExtension {
493 scheduled_time: crate::options::scheduled_time_from_ctx(&query_ctx),
494 });
495 let config_options = state.config_options().clone();
496 let _ = state
497 .execution_props_mut()
498 .config_options
499 .insert(config_options);
500 Ok(QueryEngineContext::new(state, query_ctx))
501 }
502
503 async fn explain_to_plan(
506 &self,
507 explain: &ExplainStatement,
508 query_ctx: QueryContextRef,
509 ) -> Result<LogicalPlan> {
510 let plan = self.plan_sql(&explain.statement, query_ctx).await?;
511 if matches!(plan, LogicalPlan::Explain(_)) {
512 return plan_err!("Nested EXPLAINs are not supported").context(PlanSqlSnafu);
513 }
514
515 let verbose = explain.verbose;
516 let analyze = explain.analyze;
517 let format = explain.format.map(|f| f.to_string());
518
519 let plan = Arc::new(plan);
520 let schema = LogicalPlan::explain_schema();
521 let schema = ToDFSchema::to_dfschema_ref(schema)?;
522
523 if verbose && format.is_some() {
524 return plan_err!("EXPLAIN VERBOSE with FORMAT is not supported").context(PlanSqlSnafu);
525 }
526
527 if analyze {
528 Ok(LogicalPlan::Analyze(Analyze {
530 verbose,
531 format: ExplainFormat::Indent,
532 input: plan,
533 schema,
534 analyze_level: None,
535 analyze_categories: None,
536 }))
537 } else {
538 let stringified_plans = vec![plan.to_stringified(PlanType::InitialLogicalPlan)];
539
540 let options = self.session_state.config().options();
541 let format = format
542 .map(|x| ExplainFormat::from_str(&x))
543 .transpose()?
544 .unwrap_or_else(|| options.explain.format.clone());
545
546 Ok(LogicalPlan::Explain(Explain {
547 verbose,
548 explain_format: format,
549 plan,
550 stringified_plans,
551 schema,
552 logical_optimization_succeeded: false,
553 show_statistics: None,
554 }))
555 }
556 }
557
558 #[tracing::instrument(skip_all)]
559 #[async_recursion::async_recursion]
560 async fn plan_sql(&self, stmt: &Statement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
561 let mut planner_context = PlannerContext::new();
562 let mut stmt = Cow::Borrowed(stmt);
563 let mut is_tql_cte = false;
564
565 if let Statement::Explain(explain) = stmt.as_ref() {
567 return self.explain_to_plan(explain, query_ctx).await;
568 }
569
570 if self.has_hybrid_ctes(stmt.as_ref()) {
572 let stmt_owned = stmt.into_owned();
573 let mut query = match stmt_owned {
574 Statement::Query(query) => query.as_ref().clone(),
575 _ => unreachable!("has_hybrid_ctes should only return true for Query statements"),
576 };
577 self.plan_query_with_hybrid_ctes(&query, query_ctx.clone(), &mut planner_context)
578 .await?;
579
580 query.hybrid_cte = None;
582 stmt = Cow::Owned(Statement::Query(Box::new(query)));
583 is_tql_cte = true;
584 }
585
586 let mut df_stmt = stmt.as_ref().try_into().context(SqlSnafu)?;
587 normalize_field_access_after_subscript(
588 &mut df_stmt,
589 self.session_state
590 .config_options()
591 .sql_parser
592 .enable_ident_normalization,
593 );
594
595 if let datafusion::sql::parser::Statement::Statement(stmt) = &mut df_stmt
597 && matches!(
598 stmt.as_ref(),
599 datafusion::sql::sqlparser::ast::Statement::Explain { .. }
600 )
601 {
602 UnimplementedSnafu {
603 operation: "EXPLAIN with FORMAT using raw datafusion planner",
604 }
605 .fail()?;
606 }
607
608 let engine_context = self.derive_session_state_with_scheduled_time(query_ctx)?;
609 let scheduled_state = engine_context.state().clone();
610 let query_ctx = engine_context.query_ctx();
611 let table_provider = DfTableSourceProvider::new(
612 self.engine_state.catalog_manager().clone(),
613 self.engine_state.disallow_cross_catalog_query(),
614 query_ctx.clone(),
615 Arc::new(DefaultPlanDecoder::new(
616 scheduled_state.clone(),
617 &query_ctx,
618 )?),
619 scheduled_state
620 .config_options()
621 .sql_parser
622 .enable_ident_normalization,
623 );
624
625 let context_provider = DfContextProviderAdapter::try_new(
626 self.engine_state.clone(),
627 scheduled_state.clone(),
628 Some(&df_stmt),
629 query_ctx.clone(),
630 )
631 .await?;
632
633 let config_options = scheduled_state.config().options();
634 let parser_options = &config_options.sql_parser;
635 let parser_options = ParserOptions {
636 map_string_types_to_utf8view: false,
637 ..parser_options.into()
638 };
639
640 let sql_to_rel = SqlToRel::new_with_options(&context_provider, parser_options);
641
642 let result = if is_tql_cte {
644 let Statement::Query(query) = stmt.into_owned() else {
645 unreachable!("is_tql_cte should only be true for Query statements");
646 };
647 let sqlparser_stmt = sqlparser::ast::Statement::Query(Box::new(query.inner));
648 sql_to_rel
649 .sql_statement_to_plan_with_context(sqlparser_stmt, &mut planner_context)
650 .context(PlanSqlSnafu)?
651 } else {
652 sql_to_rel
653 .statement_to_plan(df_stmt)
654 .context(PlanSqlSnafu)?
655 };
656
657 expr_depth::validate_plan(&result)?;
658 common_telemetry::debug!("Logical planner, statement to plan result: {result}");
659 let plan = RangePlanRewriter::new(table_provider, query_ctx.clone())
660 .rewrite(result)
661 .await?;
662
663 let plan = self
665 .engine_state
666 .optimize_by_extension_rules(plan, &engine_context)?;
667 common_telemetry::debug!("Logical planner, optimize result: {plan}");
668
669 Ok(plan)
670 }
671
672 #[tracing::instrument(skip_all)]
674 pub(crate) async fn sql_to_expr(
675 &self,
676 sql: SqlExpr,
677 schema: &DFSchema,
678 normalize_ident: bool,
679 query_ctx: QueryContextRef,
680 ) -> Result<DfExpr> {
681 let engine_context = self.derive_session_state_with_scheduled_time(query_ctx)?;
682 let query_ctx = engine_context.query_ctx();
683 let context_provider = DfContextProviderAdapter::try_new(
684 self.engine_state.clone(),
685 engine_context.state().clone(),
686 None,
687 query_ctx,
688 )
689 .await?;
690
691 let config_options = engine_context.state().config().options();
692 let parser_options = &config_options.sql_parser;
693 let parser_options: ParserOptions = ParserOptions {
694 map_string_types_to_utf8view: false,
695 enable_ident_normalization: normalize_ident,
696 ..parser_options.into()
697 };
698
699 let sql_to_rel = SqlToRel::new_with_options(&context_provider, parser_options);
700
701 Ok(sql_to_rel.sql_to_expr(sql, schema, &mut PlannerContext::new())?)
702 }
703
704 #[tracing::instrument(skip_all)]
713 async fn plan_pql(
714 &self,
715 stmt: &EvalStmt,
716 query_ctx: QueryContextRef,
717 native_route_allowed: bool,
718 ) -> Result<LogicalPlan> {
719 let engine_context = self.derive_session_state_with_scheduled_time(query_ctx)?;
720 let query_ctx = engine_context.query_ctx();
721 let mut scheduled_state = engine_context.state().clone();
722 let allow_query_fallback = scheduled_state
731 .config_options()
732 .extensions
733 .get::<DistPlannerOptions>()
734 .is_some_and(|options| options.allow_query_fallback);
735 let native_promql = scheduled_state
739 .config_options()
740 .extensions
741 .get::<NativePromqlExtension>()
742 .is_some_and(|options| options.requested);
743 let promql_annotations = query_ctx.remote_query_id().map(promql_annotation_collector);
744 if let Some(collector) = &promql_annotations {
745 scheduled_state
746 .config_mut()
747 .options_mut()
748 .extensions
749 .insert(collector.clone());
750 }
751 let plan_decoder = Arc::new(DefaultPlanDecoder::new(
752 scheduled_state.clone(),
753 &query_ctx,
754 )?);
755 let table_provider = || {
756 DfTableSourceProvider::new(
757 self.engine_state.catalog_manager().clone(),
758 self.engine_state.disallow_cross_catalog_query(),
759 query_ctx.clone(),
760 plan_decoder.clone(),
761 scheduled_state
762 .config_options()
763 .sql_parser
764 .enable_ident_normalization,
765 )
766 };
767 let native_requested = native_route_allowed && !allow_query_fallback && native_promql;
768 let plan = PromPlanner::stmt_to_plan_with_annotations(
769 table_provider(),
770 stmt,
771 &self.engine_state,
772 promql_annotations.clone(),
773 native_requested,
775 )
776 .await
777 .map_err(BoxedError::new)
778 .context(QueryPlanSnafu)?;
779
780 let plan = if native_requested
786 && self.engine_state.with_dist_planner()
787 && !remote_native_raw_plan_supported(&plan)
788 {
789 PromPlanner::stmt_to_plan_with_annotations(
790 table_provider(),
791 stmt,
792 &self.engine_state,
793 promql_annotations,
794 false,
795 )
796 .await
797 .map_err(BoxedError::new)
798 .context(QueryPlanSnafu)?
799 } else {
800 plan
801 };
802 let context = QueryEngineContext::new(scheduled_state, query_ctx);
803 Ok(self
804 .engine_state
805 .optimize_by_extension_rules(plan, &context)?)
806 }
807
808 #[tracing::instrument(skip_all)]
809 fn optimize_logical_plan(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
810 Ok(self.engine_state.optimize_logical_plan(plan)?)
811 }
812
813 fn has_hybrid_ctes(&self, stmt: &Statement) -> bool {
815 if let Statement::Query(query) = stmt {
816 query
817 .hybrid_cte
818 .as_ref()
819 .map(|hybrid_cte| !hybrid_cte.cte_tables.is_empty())
820 .unwrap_or(false)
821 } else {
822 false
823 }
824 }
825
826 async fn plan_query_with_hybrid_ctes(
828 &self,
829 query: &Query,
830 query_ctx: QueryContextRef,
831 planner_context: &mut PlannerContext,
832 ) -> Result<()> {
833 let hybrid_cte = query.hybrid_cte.as_ref().unwrap();
834
835 for cte in &hybrid_cte.cte_tables {
836 match &cte.content {
837 CteContent::Tql(tql) => {
838 let mut logical_plan = self.tql_to_logical_plan(tql, query_ctx.clone()).await?;
840 if !cte.columns.is_empty() {
841 let schema = logical_plan.schema();
842 let schema_fields = schema.fields().to_vec();
843 ensure!(
844 schema_fields.len() == cte.columns.len(),
845 CteColumnSchemaMismatchSnafu {
846 cte_name: cte.name.value.clone(),
847 original: schema_fields
848 .iter()
849 .map(|field| field.name().clone())
850 .collect::<Vec<_>>(),
851 expected: cte
852 .columns
853 .iter()
854 .map(|column| column.to_string())
855 .collect::<Vec<_>>(),
856 }
857 );
858 let aliases = cte
859 .columns
860 .iter()
861 .zip(schema_fields.iter())
862 .map(|(column, field)| col(field.name()).alias(column.to_string()));
863 logical_plan = LogicalPlanBuilder::from(logical_plan)
864 .project(aliases)
865 .context(PlanSqlSnafu)?
866 .build()
867 .context(PlanSqlSnafu)?;
868 }
869
870 logical_plan = LogicalPlan::SubqueryAlias(
872 datafusion_expr::SubqueryAlias::try_new(
873 Arc::new(logical_plan),
874 cte.name.value.clone(),
875 )
876 .context(PlanSqlSnafu)?,
877 );
878
879 planner_context.insert_cte(&cte.name.value, logical_plan);
880 }
881 CteContent::Sql(_) => {
882 unreachable!("SQL CTEs should not be in hybrid_cte.cte_tables");
885 }
886 }
887 }
888
889 Ok(())
890 }
891
892 async fn tql_to_logical_plan(
894 &self,
895 tql: &Tql,
896 query_ctx: QueryContextRef,
897 ) -> Result<LogicalPlan> {
898 match tql {
899 Tql::Eval(eval) => {
900 let prom_query = PromQuery {
902 query: eval.query.clone(),
903 start: eval.start.clone(),
904 end: eval.end.clone(),
905 step: eval.step.clone(),
906 lookback: eval
907 .lookback
908 .clone()
909 .unwrap_or_else(|| DEFAULT_LOOKBACK_STRING.to_string()),
910 alias: eval.alias.clone(),
911 };
912 let stmt = QueryLanguageParser::parse_promql(&prom_query, &query_ctx)?;
913 let QueryStatement::Promql(eval, _alias) = &stmt else {
922 unreachable!("a TQL eval parses to a PromQL statement");
923 };
924 let plan = self.plan_pql(eval, query_ctx.clone(), true).await?;
925 if !contains_native_fused(&plan) {
926 return Ok(plan);
930 }
931 let existing = self.plan_pql(eval, query_ctx, false).await?;
937 Ok(publish_legacy_schema(plan, &existing).unwrap_or(existing))
938 }
939 Tql::Explain(_) => UnimplementedSnafu {
940 operation: "TQL EXPLAIN in CTEs",
941 }
942 .fail(),
943 Tql::Analyze(_) => UnimplementedSnafu {
944 operation: "TQL ANALYZE in CTEs",
945 }
946 .fail(),
947 }
948 }
949
950 fn extract_placeholder_cast_types(
960 plan: &LogicalPlan,
961 ) -> Result<HashMap<String, Option<DataType>>> {
962 let mut placeholder_types = HashMap::new();
963 let mut casted_placeholders = HashSet::new();
964
965 Self::extract_from_plan(plan, &mut placeholder_types, &mut casted_placeholders)?;
966
967 Ok(placeholder_types)
968 }
969
970 fn extract_from_plan(
971 plan: &LogicalPlan,
972 placeholder_types: &mut HashMap<String, Option<DataType>>,
973 casted_placeholders: &mut HashSet<String>,
974 ) -> Result<()> {
975 plan.apply(|node| {
976 for expr in node.expressions() {
977 let _ = expr.apply(|e| {
978 if let DfExpr::Cast(cast) = e
980 && let DfExpr::Placeholder(ph) = &*cast.expr
981 {
982 placeholder_types
983 .insert(ph.id.clone(), Some(cast.field.data_type().clone()));
984 casted_placeholders.insert(ph.id.clone());
985 }
986
987 if let DfExpr::ScalarFunction(scalar_func) = e
989 && scalar_func.name() == "arrow_cast"
990 && scalar_func.args.len() == 2
991 && let DfExpr::Placeholder(ph) = &scalar_func.args[0]
992 && let DfExpr::Literal(ScalarValue::Utf8(Some(type_str)), _) =
993 &scalar_func.args[1]
994 && let Ok(data_type) = type_str.parse::<DataType>()
995 {
996 placeholder_types.insert(ph.id.clone(), Some(data_type));
997 casted_placeholders.insert(ph.id.clone());
998 }
999
1000 if let DfExpr::Placeholder(ph) = e
1002 && !casted_placeholders.contains(&ph.id)
1003 && !placeholder_types.contains_key(&ph.id)
1004 {
1005 placeholder_types.insert(ph.id.clone(), None);
1006 }
1007
1008 match e {
1010 DfExpr::Exists(Exists { subquery, .. })
1011 | DfExpr::InSubquery(InSubquery { subquery, .. })
1012 | DfExpr::ScalarSubquery(subquery) => {
1013 Self::extract_from_plan(
1014 &subquery.subquery,
1015 placeholder_types,
1016 casted_placeholders,
1017 )?;
1018 }
1019 _ => {}
1020 }
1021
1022 Ok(TreeNodeRecursion::Continue)
1023 });
1024 }
1025 Ok(TreeNodeRecursion::Continue)
1026 })?;
1027 Ok(())
1028 }
1029
1030 fn infer_limit_placeholder_types(
1031 plan: &LogicalPlan,
1032 placeholder_types: &mut HashMap<String, Option<DataType>>,
1033 ) -> Result<()> {
1034 plan.apply(|node| {
1035 if let LogicalPlan::Limit(limit) = node {
1036 for expr in limit.skip.iter().chain(limit.fetch.iter()) {
1037 expr.apply(|e| {
1038 if let DfExpr::Placeholder(ph) = e {
1039 placeholder_types
1040 .entry(ph.id.clone())
1041 .and_modify(|existing| {
1042 if existing.is_none() {
1043 *existing = Some(DataType::Int64);
1044 }
1045 })
1046 .or_insert(Some(DataType::Int64));
1047 }
1048
1049 Ok(TreeNodeRecursion::Continue)
1050 })?;
1051 }
1052 }
1053
1054 Ok(TreeNodeRecursion::Continue)
1055 })?;
1056
1057 Ok(())
1058 }
1059
1060 pub fn get_inferred_parameter_types(
1075 plan: &LogicalPlan,
1076 ) -> Result<HashMap<String, Option<DataType>>> {
1077 let mut param_types = plan.get_parameter_types().context(PlanSqlSnafu)?;
1078
1079 let has_none = param_types.values().any(|v| v.is_none());
1080
1081 if has_none {
1082 let cast_types = Self::extract_placeholder_cast_types(plan)?;
1083
1084 for (id, opt_type) in cast_types {
1085 param_types
1086 .entry(id)
1087 .and_modify(|existing| {
1088 if existing.is_none() {
1089 *existing = opt_type.clone();
1090 }
1091 })
1092 .or_insert(opt_type);
1093 }
1094
1095 Self::infer_limit_placeholder_types(plan, &mut param_types)?;
1096 }
1097
1098 Ok(param_types)
1099 }
1100}
1101
1102fn normalize_field_access_after_subscript(stmt: &mut DfStatement, normalize_ident: bool) {
1139 let DfStatement::Statement(stmt) = stmt else {
1140 return;
1141 };
1142 let normalizer = IdentNormalizer::new(normalize_ident);
1143
1144 let _ = visit_expressions_mut(stmt.as_mut(), |expr| {
1145 let SqlExpr::CompoundFieldAccess { access_chain, .. } = expr else {
1146 return ControlFlow::<()>::Continue(());
1147 };
1148 let Some(index) = access_chain
1149 .iter()
1150 .position(|x| matches!(x, AccessExpr::Subscript(_)))
1151 else {
1152 return ControlFlow::Continue(());
1153 };
1154
1155 for access in &mut access_chain[index + 1..] {
1156 let AccessExpr::Dot(SqlExpr::Identifier(ident)) = access else {
1157 continue;
1158 };
1159 let value = normalizer.normalize(ident.clone());
1160 *access = AccessExpr::Dot(SqlExpr::Value(
1161 Value::SingleQuotedString(value).with_span(ident.span),
1162 ));
1163 }
1164 ControlFlow::Continue(())
1165 });
1166}
1167
1168#[async_trait]
1169impl LogicalPlanner for DfLogicalPlanner {
1170 #[tracing::instrument(skip_all)]
1171 async fn plan(&self, stmt: &QueryStatement, query_ctx: QueryContextRef) -> Result<LogicalPlan> {
1172 match stmt {
1173 QueryStatement::Sql(stmt) => self.plan_sql(stmt, query_ctx).await,
1174 QueryStatement::Promql(stmt, _alias) => self.plan_pql(stmt, query_ctx, true).await,
1176 }
1177 }
1178
1179 async fn plan_logs_query(
1180 &self,
1181 query: LogQuery,
1182 query_ctx: QueryContextRef,
1183 ) -> Result<LogicalPlan> {
1184 let engine_context = self.derive_session_state_with_scheduled_time(query_ctx)?;
1185 let query_ctx = engine_context.query_ctx();
1186 let state = engine_context.state().clone();
1187 let plan_decoder = Arc::new(DefaultPlanDecoder::new(state.clone(), &query_ctx)?);
1188 let table_provider = DfTableSourceProvider::new(
1189 self.engine_state.catalog_manager().clone(),
1190 self.engine_state.disallow_cross_catalog_query(),
1191 query_ctx,
1192 plan_decoder,
1193 state.config_options().sql_parser.enable_ident_normalization,
1194 );
1195
1196 let mut planner = LogQueryPlanner::new(table_provider, state);
1197 planner
1198 .query_to_plan(query)
1199 .await
1200 .map_err(BoxedError::new)
1201 .context(QueryPlanSnafu)
1202 }
1203
1204 fn optimize(&self, plan: LogicalPlan) -> Result<LogicalPlan> {
1205 self.optimize_logical_plan(plan)
1206 }
1207
1208 fn as_any(&self) -> &dyn Any {
1209 self
1210 }
1211}
1212
1213#[cfg(test)]
1214mod tests {
1215 use std::sync::Arc;
1216
1217 use arrow_schema::DataType;
1218 use catalog::RegisterTableRequest;
1219 use catalog::memory::MemoryCatalogManager;
1220 use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
1221 use common_query::OutputData;
1222 use common_recordbatch::RecordBatch;
1223 use common_recordbatch::util::collect_batches;
1224 use common_time::Timezone;
1225 use datatypes::arrow::array::{Array, Float64Array, StringArray};
1226 use datatypes::prelude::ConcreteDataType;
1227 use datatypes::schema::{ColumnSchema, Schema};
1228 use session::context::{
1229 MetricDiscoveryScope, QueryContext, QueryContextBuilder, QueryContextRef,
1230 };
1231 use store_api::metric_engine_consts::{
1232 DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME, LOGICAL_TABLE_METADATA_KEY,
1233 METRIC_ENGINE_NAME,
1234 };
1235 use table::metadata::{TableInfoBuilder, TableMetaBuilder};
1236 use table::test_util::EmptyTable;
1237
1238 use super::*;
1239 use crate::datafusion::QUERY_FALLBACK_HINT;
1240 use crate::parser::{PromQuery, QueryLanguageParser};
1241 use crate::promql::planner::NATIVE_PROMQL_EXTENSION_KEY;
1242 use crate::{QueryEngineFactory, QueryEngineRef};
1243
1244 async fn create_test_engine() -> QueryEngineRef {
1245 let columns = vec![
1246 ColumnSchema::new("id", ConcreteDataType::int32_datatype(), false),
1247 ColumnSchema::new("name", ConcreteDataType::string_datatype(), true),
1248 ];
1249 let schema = Arc::new(Schema::new(columns));
1250 let table_meta = TableMetaBuilder::empty()
1251 .schema(schema)
1252 .primary_key_indices(vec![0])
1253 .value_indices(vec![1])
1254 .next_column_id(1024)
1255 .build()
1256 .unwrap();
1257 let table_info = TableInfoBuilder::new("test", table_meta).build().unwrap();
1258 let table = EmptyTable::from_table_info(&table_info);
1259
1260 crate::tests::new_query_engine_with_table(table)
1261 }
1262
1263 async fn create_timestamp_test_engine() -> QueryEngineRef {
1264 let columns = vec![
1265 ColumnSchema::new(
1266 "ts",
1267 ConcreteDataType::timestamp_millisecond_datatype(),
1268 false,
1269 )
1270 .with_time_index(true),
1271 ColumnSchema::new(
1272 "st",
1273 ConcreteDataType::timestamp_millisecond_datatype(),
1274 false,
1275 ),
1276 ColumnSchema::new("note", ConcreteDataType::string_datatype(), true),
1277 ColumnSchema::new(
1278 "ts_ns",
1279 ConcreteDataType::timestamp_nanosecond_datatype(),
1280 true,
1281 ),
1282 ];
1283 let schema = Arc::new(Schema::new(columns));
1284 let table_meta = TableMetaBuilder::empty()
1285 .schema(schema)
1286 .primary_key_indices(vec![])
1287 .value_indices(vec![0, 1, 2, 3])
1288 .next_column_id(1024)
1289 .build()
1290 .unwrap();
1291 let table_info = TableInfoBuilder::new("timestamps", table_meta)
1292 .build()
1293 .unwrap();
1294 let table = EmptyTable::from_table_info(&table_info);
1295
1296 crate::tests::new_query_engine_with_table(table)
1297 }
1298
1299 fn create_promql_test_engine() -> QueryEngineRef {
1300 create_promql_test_engine_with(crate::options::QueryOptions::default(), false)
1301 }
1302
1303 fn create_promql_test_engine_with(
1306 query_options: crate::options::QueryOptions,
1307 with_dist_planner: bool,
1308 ) -> QueryEngineRef {
1309 let catalog_manager = MemoryCatalogManager::with_default_setup();
1310 let physical_table_name = "phy";
1311 let physical_table_id = 999u32;
1312
1313 let physical_schema = Arc::new(Schema::new(vec![
1314 ColumnSchema::new(
1315 DATA_SCHEMA_TABLE_ID_COLUMN_NAME.to_string(),
1316 ConcreteDataType::uint32_datatype(),
1317 false,
1318 ),
1319 ColumnSchema::new(
1320 DATA_SCHEMA_TSID_COLUMN_NAME.to_string(),
1321 ConcreteDataType::uint64_datatype(),
1322 false,
1323 ),
1324 ColumnSchema::new("tag_0", ConcreteDataType::string_datatype(), false),
1325 ColumnSchema::new("tag_1", ConcreteDataType::string_datatype(), false),
1326 ColumnSchema::new(
1327 "timestamp",
1328 ConcreteDataType::timestamp_millisecond_datatype(),
1329 false,
1330 )
1331 .with_time_index(true),
1332 ColumnSchema::new("field_0", ConcreteDataType::float64_datatype(), true),
1333 ]));
1334 let physical_meta = TableMetaBuilder::empty()
1335 .schema(physical_schema)
1336 .primary_key_indices(vec![0, 1, 2, 3])
1337 .value_indices(vec![4, 5])
1338 .engine(METRIC_ENGINE_NAME.to_string())
1339 .next_column_id(1024)
1340 .build()
1341 .unwrap();
1342 let physical_info = TableInfoBuilder::default()
1343 .table_id(physical_table_id)
1344 .name(physical_table_name)
1345 .meta(physical_meta)
1346 .build()
1347 .unwrap();
1348 catalog_manager
1349 .register_table_sync(RegisterTableRequest {
1350 catalog: DEFAULT_CATALOG_NAME.to_string(),
1351 schema: DEFAULT_SCHEMA_NAME.to_string(),
1352 table_name: physical_table_name.to_string(),
1353 table_id: physical_table_id,
1354 table: EmptyTable::from_table_info(&physical_info),
1355 })
1356 .unwrap();
1357
1358 let mut options = table::requests::TableOptions::default();
1359 options.extra_options.insert(
1360 LOGICAL_TABLE_METADATA_KEY.to_string(),
1361 physical_table_name.to_string(),
1362 );
1363 let logical_schema = Arc::new(Schema::new(vec![
1364 ColumnSchema::new("tag_0", ConcreteDataType::string_datatype(), false),
1365 ColumnSchema::new("tag_1", ConcreteDataType::string_datatype(), false),
1366 ColumnSchema::new(
1367 "timestamp",
1368 ConcreteDataType::timestamp_millisecond_datatype(),
1369 false,
1370 )
1371 .with_time_index(true),
1372 ColumnSchema::new("field_0", ConcreteDataType::float64_datatype(), true),
1373 ]));
1374 let logical_meta = TableMetaBuilder::empty()
1375 .schema(logical_schema)
1376 .primary_key_indices(vec![0, 1])
1377 .value_indices(vec![3])
1378 .engine(METRIC_ENGINE_NAME.to_string())
1379 .options(options)
1380 .next_column_id(1024)
1381 .build()
1382 .unwrap();
1383 let logical_info = TableInfoBuilder::default()
1384 .table_id(1024)
1385 .name("some_metric")
1386 .meta(logical_meta)
1387 .build()
1388 .unwrap();
1389 catalog_manager
1390 .register_table_sync(RegisterTableRequest {
1391 catalog: DEFAULT_CATALOG_NAME.to_string(),
1392 schema: DEFAULT_SCHEMA_NAME.to_string(),
1393 table_name: "some_metric".to_string(),
1394 table_id: 1024,
1395 table: EmptyTable::from_table_info(&logical_info),
1396 })
1397 .unwrap();
1398 let mut other_info = logical_info.clone();
1399 other_info.ident.table_id = 1025;
1400 other_info.name = "other_metric".to_string();
1401 catalog_manager
1402 .register_table_sync(RegisterTableRequest {
1403 catalog: DEFAULT_CATALOG_NAME.to_string(),
1404 schema: DEFAULT_SCHEMA_NAME.to_string(),
1405 table_name: other_info.name.clone(),
1406 table_id: other_info.ident.table_id,
1407 table: EmptyTable::from_table_info(&other_info),
1408 })
1409 .unwrap();
1410
1411 QueryEngineFactory::try_new_with_plugins(
1412 catalog_manager,
1413 None,
1414 None,
1415 None,
1416 None,
1417 None,
1418 with_dist_planner,
1419 Default::default(),
1420 query_options,
1421 )
1422 .unwrap()
1423 .query_engine()
1424 }
1425
1426 async fn parse_sql_to_plan(sql: &str) -> LogicalPlan {
1427 let stmt = QueryLanguageParser::parse_sql(sql, &QueryContext::arc()).unwrap();
1428 let engine = create_test_engine().await;
1429 engine
1430 .planner()
1431 .plan(&stmt, QueryContext::arc())
1432 .await
1433 .unwrap()
1434 }
1435
1436 async fn parse_promql_to_plan(query: &str) -> LogicalPlan {
1437 let engine = create_promql_test_engine();
1438 let query_ctx = QueryContext::arc();
1439 let stmt = QueryLanguageParser::parse_promql(
1440 &PromQuery {
1441 query: query.to_string(),
1442 start: "0".to_string(),
1443 end: "10".to_string(),
1444 step: "5s".to_string(),
1445 lookback: "300s".to_string(),
1446 alias: None,
1447 },
1448 &query_ctx,
1449 )
1450 .unwrap();
1451
1452 engine.planner().plan(&stmt, query_ctx).await.unwrap()
1453 }
1454
1455 async fn plan_native_promql_with_query_options(
1459 engine_options: crate::options::QueryOptions,
1460 session: Option<(&str, &str)>,
1461 request: Option<(&str, &str)>,
1462 ) -> crate::error::Result<String> {
1463 let engine = create_promql_test_engine_with(
1464 engine_options,
1465 true,
1467 );
1468 let mut builder = QueryContextBuilder::default();
1469 if let Some((key, value)) = request {
1470 builder = builder.set_extension(key.to_string(), value.to_string());
1471 }
1472 let query_ctx = builder.build();
1474 if let Some((key, value)) = session {
1475 query_ctx
1476 .configuration_parameter()
1477 .set_query_option(key, value)
1478 .unwrap();
1479 }
1480 let query_ctx = Arc::new(query_ctx);
1481 let stmt = QueryLanguageParser::parse_promql(
1482 &PromQuery {
1483 query: "rate(some_metric[5m])".to_string(),
1484 start: "0".to_string(),
1485 end: "10".to_string(),
1486 step: "5s".to_string(),
1487 lookback: "300s".to_string(),
1488 alias: None,
1489 },
1490 &query_ctx,
1491 )
1492 .unwrap();
1493
1494 engine
1495 .planner()
1496 .plan(&stmt, query_ctx)
1497 .await
1498 .map(|plan| plan.display_indent().to_string())
1499 }
1500
1501 #[tokio::test]
1506 async fn test_plan_pql_native_route_follows_query_options() {
1507 for (engine_default, session, request, native) in [
1509 (false, None, None, false),
1511 (true, None, None, true),
1512 (false, Some("true"), None, true),
1514 (false, Some("true"), Some("false"), false),
1516 (false, Some("false"), Some("true"), true),
1517 (true, Some("false"), None, false),
1519 (true, Some("true"), Some("false"), false),
1520 (false, None, Some("true"), true),
1521 ] {
1522 let plan = plan_native_promql_with_query_options(
1523 crate::options::QueryOptions {
1524 experimental_native_promql: engine_default,
1525 ..Default::default()
1526 },
1527 session.map(|value| (NATIVE_PROMQL_EXTENSION_KEY, value)),
1528 request.map(|value| (NATIVE_PROMQL_EXTENSION_KEY, value)),
1529 )
1530 .await
1531 .unwrap();
1532 assert_eq!(
1533 plan.contains("FusedSeries"),
1534 native,
1535 "engine default={engine_default}, session={session:?}, request={request:?}: {plan}"
1536 );
1537 }
1538 }
1539
1540 async fn plan_native_promql_with_fallback(
1543 engine_fallback: bool,
1544 session_fallback: Option<bool>,
1545 request_fallback: Option<&str>,
1546 ) -> crate::error::Result<String> {
1547 plan_native_promql_with_query_options(
1548 crate::options::QueryOptions {
1549 allow_query_fallback: engine_fallback,
1550 experimental_native_promql: true,
1551 ..Default::default()
1552 },
1553 session_fallback.map(|value| {
1554 (
1555 "query.allow_query_fallback",
1556 if value { "true" } else { "false" },
1557 )
1558 }),
1559 request_fallback.map(|value| (QUERY_FALLBACK_HINT, value)),
1560 )
1561 .await
1562 }
1563
1564 #[tokio::test]
1568 async fn test_plan_pql_declines_native_route_when_query_may_fall_back() {
1569 for (engine, session, request, native) in [
1571 (false, None, None, true),
1573 (false, None, Some("false"), true),
1574 (false, Some(false), None, true),
1575 (false, Some(true), None, false),
1577 (false, None, Some("true"), false),
1578 (false, None, Some("TRUE"), false),
1579 (true, None, None, false),
1581 (true, None, Some("false"), true),
1582 (true, Some(true), Some("false"), true),
1583 (true, Some(false), None, true),
1584 ] {
1585 let plan = plan_native_promql_with_fallback(engine, session, request)
1586 .await
1587 .unwrap();
1588 assert_eq!(
1589 plan.contains("FusedSeries"),
1590 native,
1591 "engine={engine}, session={session:?}, request={request:?}: {plan}"
1592 );
1593 }
1594 }
1595
1596 #[tokio::test]
1601 async fn test_engine_option_is_the_default_a_request_or_session_overrides() {
1602 const KEY: &str = "query.experimental_native_promql";
1603 for (engine, session, request, native) in [
1605 (false, None, None, false),
1607 (true, None, None, true),
1608 (false, None, Some((KEY, "true")), true),
1610 (true, None, Some((KEY, "false")), false),
1612 (false, Some((KEY, "true")), None, true),
1614 (true, Some((KEY, "false")), None, false),
1615 (false, Some((KEY, "false")), Some((KEY, "true")), true),
1616 (true, Some((KEY, "true")), Some((KEY, "false")), false),
1617 ] {
1618 let plan = plan_native_promql_with_query_options(
1619 crate::options::QueryOptions {
1620 experimental_native_promql: engine,
1621 ..Default::default()
1622 },
1623 session,
1624 request,
1625 )
1626 .await
1627 .unwrap();
1628
1629 assert_eq!(
1630 plan.contains("FusedSeries"),
1631 native,
1632 "engine={engine}, session={session:?}, request={request:?}: {plan}"
1633 );
1634 assert_eq!(
1635 plan.contains("RawSeriesSource"),
1636 native,
1637 "engine={engine}, session={session:?}, request={request:?}: {plan}"
1638 );
1639 }
1640 }
1641
1642 #[tokio::test]
1645 async fn test_plan_pql_rejects_a_fallback_hint_that_is_not_a_boolean() {
1646 let error = plan_native_promql_with_fallback(false, None, Some("yes"))
1647 .await
1648 .unwrap_err();
1649 let message = error.to_string();
1650 assert!(message.contains("query_fallback"), "{message}");
1651 assert!(message.contains("Expected true or false"), "{message}");
1652 }
1653
1654 #[tokio::test]
1659 async fn test_native_promql_admission_is_the_same_for_both_deployments() {
1660 for (with_dist_planner, query, sourced) in [
1661 (false, "some_metric + other_metric", true),
1662 (false, "rate(some_metric[5m])", false),
1663 (true, "some_metric + other_metric", true),
1664 (true, "rate(some_metric[5m])", true),
1665 ] {
1666 let engine = create_promql_test_engine_with(
1667 crate::options::QueryOptions {
1668 experimental_native_promql: true,
1669 ..Default::default()
1670 },
1671 with_dist_planner,
1672 );
1673 let query_ctx = Arc::new(
1674 QueryContextBuilder::default()
1675 .set_extension(NATIVE_PROMQL_EXTENSION_KEY.to_string(), "true".to_string())
1676 .build(),
1677 );
1678 let stmt = QueryLanguageParser::parse_promql(
1679 &PromQuery {
1680 query: query.to_string(),
1681 start: "0".to_string(),
1682 end: "10".to_string(),
1683 step: "5s".to_string(),
1684 lookback: "300s".to_string(),
1685 alias: None,
1686 },
1687 &query_ctx,
1688 )
1689 .unwrap();
1690 let plan = engine.planner().plan(&stmt, query_ctx).await.unwrap();
1691 let display = plan.display_indent().to_string();
1692 assert!(
1693 display.contains("FusedSeries"),
1694 "with_dist_planner={with_dist_planner}, query={query}: {display}"
1695 );
1696 if with_dist_planner {
1697 assert!(remote_native_raw_plan_supported(&plan), "{display}");
1700 }
1701 assert_eq!(
1702 display.contains("RawSeriesSource"),
1703 sourced,
1704 "with_dist_planner={with_dist_planner}, query={query}: {display}"
1705 );
1706 }
1707 }
1708
1709 async fn plan_remote_native_promql(
1712 engine: &QueryEngineRef,
1713 query_ctx: &QueryContextRef,
1714 query: &str,
1715 ) -> LogicalPlan {
1716 let stmt = QueryLanguageParser::parse_promql(
1717 &PromQuery {
1718 query: query.to_string(),
1719 start: "0".to_string(),
1720 end: "10".to_string(),
1721 step: "5s".to_string(),
1722 lookback: "300s".to_string(),
1723 alias: None,
1724 },
1725 query_ctx,
1726 )
1727 .unwrap();
1728 engine
1729 .planner()
1730 .plan(&stmt, query_ctx.clone())
1731 .await
1732 .unwrap()
1733 }
1734
1735 fn native_query_ctx(scope: MetricDiscoveryScope) -> QueryContextRef {
1738 let mut ctx = (*QueryContext::arc()).clone();
1739 ctx.set_metric_discovery_scope(scope);
1740 ctx.set_extension(NATIVE_PROMQL_EXTENSION_KEY.to_string(), "true".to_string());
1741 Arc::new(ctx)
1742 }
1743
1744 fn native_discovery_query_ctx() -> QueryContextRef {
1745 native_query_ctx(MetricDiscoveryScope::Unrestricted)
1746 }
1747
1748 async fn execute_plan(
1750 engine: &QueryEngineRef,
1751 plan: LogicalPlan,
1752 query_ctx: &QueryContextRef,
1753 ) -> (
1754 Arc<dyn datafusion::physical_plan::ExecutionPlan>,
1755 Vec<RecordBatch>,
1756 ) {
1757 let output = engine.execute(plan, query_ctx.clone()).await.unwrap();
1758 let physical = output
1759 .meta
1760 .plan
1761 .expect("the engine reports its physical plan");
1762 let batches = match output.data {
1763 OutputData::Stream(stream) => collect_batches(stream).await.unwrap(),
1764 OutputData::RecordBatches(batches) => batches,
1765 other => panic!("a PromQL query answers rows, not {other:?}"),
1766 }
1767 .take();
1768 (physical, batches)
1769 }
1770
1771 fn physical_plan_names(
1773 root: &Arc<dyn datafusion::physical_plan::ExecutionPlan>,
1774 ) -> Vec<String> {
1775 let mut names = Vec::new();
1776 let mut stack = vec![Arc::clone(root)];
1777 while let Some(node) = stack.pop() {
1778 names.push(node.name().to_string());
1779 stack.extend(node.children().into_iter().cloned());
1780 }
1781 names
1782 }
1783
1784 fn logical_plan_nodes(root: &LogicalPlan) -> Vec<&LogicalPlan> {
1786 let mut nodes = Vec::new();
1787 let mut stack = vec![root];
1788 while let Some(node) = stack.pop() {
1789 nodes.push(node);
1790 stack.extend(node.inputs());
1791 }
1792 nodes
1793 }
1794
1795 fn fused_series_nodes(plan: &LogicalPlan) -> Vec<&promql::extension_plan::FusedSeries> {
1796 logical_plan_nodes(plan)
1797 .into_iter()
1798 .filter_map(|node| match node {
1799 LogicalPlan::Extension(extension) => extension
1800 .node
1801 .as_any()
1802 .downcast_ref::<promql::extension_plan::FusedSeries>(
1803 ),
1804 _ => None,
1805 })
1806 .collect()
1807 }
1808
1809 fn numeric_values(batches: &[RecordBatch], column: &str) -> Vec<f64> {
1810 batches
1811 .iter()
1812 .flat_map(|batch| {
1813 batch
1814 .column_by_name(column)
1815 .unwrap_or_else(|| panic!("no `{column}` column"))
1816 .as_any()
1817 .downcast_ref::<Float64Array>()
1818 .unwrap()
1819 .values()
1820 .to_vec()
1821 })
1822 .collect()
1823 }
1824
1825 fn label_values(batches: &[RecordBatch], column: &str) -> Vec<String> {
1826 batches
1827 .iter()
1828 .flat_map(|batch| {
1829 let array = batch
1830 .column_by_name(column)
1831 .unwrap_or_else(|| panic!("no `{column}` column"))
1832 .as_any()
1833 .downcast_ref::<StringArray>()
1834 .expect("a label column is Utf8");
1835 (0..array.len())
1836 .map(|row| array.value(row).to_string())
1837 .collect::<Vec<_>>()
1838 })
1839 .collect()
1840 }
1841
1842 fn assert_regionless_native_plan(plan: &LogicalPlan, query: &str) {
1845 let display = plan.display_indent().to_string();
1846 assert!(display.contains("FusedSeries"), "{query}: {display}");
1847 for absent in ["TableScan", "MergeScan", "RawSeriesSource"] {
1848 assert!(
1849 !display.contains(absent),
1850 "{query} contains {absent}: {display}"
1851 );
1852 }
1853 let fused = fused_series_nodes(plan);
1854 assert!(!fused.is_empty(), "{query}: {display}");
1855 for node in fused {
1856 assert_eq!(
1857 None,
1858 node.binding().metric_name,
1859 "{query} bound a metric to an input that reads no table: {display}"
1860 );
1861 }
1862 }
1863
1864 fn assert_typed_empty_inputs(plan: &LogicalPlan, query: &str) {
1868 let empty = logical_plan_nodes(plan)
1869 .into_iter()
1870 .filter(|node| matches!(node, LogicalPlan::EmptyRelation(_)))
1871 .collect::<Vec<_>>();
1872 assert!(!empty.is_empty(), "{query} has no empty input");
1873 for node in empty {
1874 assert!(
1875 crate::promql::planner::native::is_empty_discovered_input(node),
1876 "{query}: {node:?}"
1877 );
1878 let LogicalPlan::EmptyRelation(relation) = node else {
1879 unreachable!();
1880 };
1881 assert!(!relation.produce_one_row, "{query}: {relation:?}");
1882 let schema = relation.schema.as_arrow();
1883 for (key, expected) in [
1884 ("greptime.series.content", "raw"),
1885 ("greptime.series.labels", "explicit"),
1886 ("greptime.series.ordering", "contiguous"),
1887 ] {
1888 assert_eq!(
1889 Some(expected),
1890 schema.metadata().get(key).map(String::as_str),
1891 "{query}: {key}"
1892 );
1893 }
1894 let fields = schema.field_with_name("fields").expect("a series frame");
1896 let arrow_schema::DataType::Struct(payloads) = fields.data_type() else {
1897 panic!("{query}: {fields:?}");
1898 };
1899 assert_eq!(
1900 vec!["value"],
1901 payloads
1902 .iter()
1903 .map(|field| field.name().as_str())
1904 .collect::<Vec<_>>(),
1905 "{query}"
1906 );
1907 }
1908 }
1909
1910 fn assert_regionless_physical_plan(
1913 physical: &Arc<dyn datafusion::physical_plan::ExecutionPlan>,
1914 query: &str,
1915 ) {
1916 let names = physical_plan_names(physical);
1917 assert!(
1918 names.iter().any(|name| name == "FusedSeriesExec"),
1919 "{query} is evaluated on the frontend by the fused node: {physical:?}"
1920 );
1921 assert!(
1922 !names.iter().any(|name| name == "MergeScanExec"),
1923 "{query} reads no region: {physical:?}"
1924 );
1925 }
1926
1927 #[tokio::test]
1930 async fn test_remote_native_promql_plans_a_data_free_statement_natively() {
1931 let engine = create_promql_test_engine_with(
1932 crate::options::QueryOptions {
1933 experimental_native_promql: true,
1934 ..Default::default()
1935 },
1936 false,
1937 );
1938 let query_ctx = native_discovery_query_ctx();
1939 for (query, expected_values, expected_labels, name_column) in [
1940 ("time()", &[0.0, 5.0, 10.0][..], &[][..], false),
1943 ("vector(1)", &[1.0, 1.0, 1.0][..], &[][..], true),
1946 (
1947 r#"label_replace(vector(1), "host", "host1", "", "")"#,
1948 &[1.0, 1.0, 1.0][..],
1949 &[("host", "host1")][..],
1950 true,
1951 ),
1952 (
1955 r#"label_replace(vector(1), "__tsid", "x", "", "")"#,
1956 &[1.0, 1.0, 1.0][..],
1957 &[][..],
1958 true,
1959 ),
1960 ] {
1961 let plan = plan_remote_native_promql(&engine, &query_ctx, query).await;
1962 let display = plan.display_indent().to_string();
1963 assert_regionless_native_plan(&plan, query);
1964 assert!(
1965 !display.contains("greptime_value"),
1966 "{query} keeps the native value column: {display}"
1967 );
1968 let expression = promql_parser::parser::parse(query).unwrap().to_string();
1972 let mut expected_columns = vec!["time", expression.as_str()];
1973 expected_columns.extend(expected_labels.iter().map(|(name, _)| *name));
1974 if name_column {
1975 expected_columns.push("__name__");
1976 }
1977 assert_eq!(
1978 expected_columns,
1979 plan.schema()
1980 .fields()
1981 .iter()
1982 .map(|field| field.name().as_str())
1983 .collect::<Vec<_>>(),
1984 "{query}: {display}"
1985 );
1986 let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await;
1987 assert_regionless_physical_plan(&physical, query);
1988 assert_eq!(
1989 expected_values,
1990 numeric_values(&batches, &expression).as_slice(),
1991 "{query}: {batches:?}"
1992 );
1993 for (label, value) in expected_labels {
1994 assert_eq!(
1995 vec![value.to_string(); batches_row_count(&batches)],
1996 label_values(&batches, label),
1997 "{query}: label `{label}`: {batches:?}"
1998 );
1999 }
2000 }
2001 }
2002
2003 fn batches_row_count(batches: &[RecordBatch]) -> usize {
2005 batches.iter().map(|batch| batch.num_rows()).sum()
2006 }
2007
2008 #[tokio::test]
2012 async fn test_remote_native_promql_plans_a_zero_match_selector_natively() {
2013 let engine = create_promql_test_engine_with(
2014 crate::options::QueryOptions {
2015 experimental_native_promql: true,
2016 ..Default::default()
2017 },
2018 false,
2019 );
2020 let query_ctx = native_discovery_query_ctx();
2021 for (query, expected_rows, expected_labels) in [
2022 (r#"{job="api"}"#, 0, &[][..]),
2024 (r#"sort({job="api"})"#, 0, &[][..]),
2027 (r#"sum({job="api"})"#, 0, &[][..]),
2029 (r#"absent({job="api"})"#, 3, &[("job", "api")][..]),
2032 ] {
2033 let plan = plan_remote_native_promql(&engine, &query_ctx, query).await;
2034 let display = plan.display_indent().to_string();
2035 assert_regionless_native_plan(&plan, query);
2036 assert!(
2037 display.contains("EmptyRelation"),
2038 "{query} keeps the typed empty input: {display}"
2039 );
2040 assert_typed_empty_inputs(&plan, query);
2041 let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await;
2042 assert_regionless_physical_plan(&physical, query);
2043 assert_eq!(
2045 expected_rows,
2046 batches_row_count(&batches),
2047 "{query} answered rows over a selector that matches no metric: {physical:?}"
2048 );
2049 for (label, value) in expected_labels {
2050 assert_eq!(
2051 vec![value.to_string(); expected_rows],
2052 label_values(&batches, label),
2053 "{query}: label `{label}`: {batches:?}"
2054 );
2055 }
2056 }
2057
2058 let query_ctx = native_query_ctx(MetricDiscoveryScope::allowed(std::iter::empty()));
2061 let query = r#"{job="api"}"#;
2062 let plan = plan_remote_native_promql(&engine, &query_ctx, query).await;
2063 assert_regionless_native_plan(&plan, "denied discovery");
2064 assert_typed_empty_inputs(&plan, "denied discovery");
2065 let (physical, batches) = execute_plan(&engine, plan, &query_ctx).await;
2066 assert_regionless_physical_plan(&physical, "denied discovery");
2067 assert_eq!(
2068 0,
2069 batches_row_count(&batches),
2070 "denied discovery: {physical:?}"
2071 );
2072 }
2073
2074 async fn analyze_insert(
2078 engine: &QueryEngineRef,
2079 sql: &str,
2080 query_ctx: &QueryContextRef,
2081 ) -> String {
2082 let stmt = QueryLanguageParser::parse_sql(sql, query_ctx).unwrap();
2083 let plan = engine
2084 .planner()
2085 .plan(&stmt, query_ctx.clone())
2086 .await
2087 .unwrap();
2088 let context = engine.engine_context(query_ctx.clone()).unwrap();
2089 let state = context.state();
2090 state
2091 .analyzer()
2092 .execute_and_check(plan, state.config_options(), |_, _| {})
2093 .unwrap()
2094 .display_indent_schema()
2095 .to_string()
2096 }
2097
2098 #[tokio::test]
2099 async fn test_insert_timestamp_literals_use_query_timezone() {
2100 let query_ctx = Arc::new(
2101 QueryContextBuilder::default()
2102 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2103 .build(),
2104 );
2105 let engine = create_timestamp_test_engine().await;
2106
2107 for (sql, expected_timestamps) in [
2108 (
2109 "INSERT INTO timestamps (ts, st) \
2110 VALUES ('2026-08-02 12:00:00.001', now())",
2111 &[1_785_643_200_001_i64][..],
2112 ),
2113 (
2114 "INSERT INTO timestamps (ts, st) \
2115 SELECT '2026-08-03 12:00:00.001', now()",
2116 &[1_785_729_600_001_i64][..],
2117 ),
2118 (
2119 "INSERT INTO timestamps (ts, st) \
2120 SELECT '2026-08-04 12:00:00.001', now() LIMIT 1",
2121 &[1_785_816_000_001_i64][..],
2122 ),
2123 (
2124 "INSERT INTO timestamps (ts, st) \
2125 SELECT * FROM (\
2126 SELECT '2026-08-05 12:00:00.001', now()\
2127 ) AS source",
2128 &[1_785_902_400_001_i64][..],
2129 ),
2130 (
2131 "INSERT INTO timestamps (ts, st) \
2132 SELECT '2026-08-18 12:00:00.001', max(st) \
2133 FROM timestamps GROUP BY note",
2134 &[1_787_025_600_001_i64][..],
2135 ),
2136 (
2137 "INSERT INTO timestamps (ts, st) \
2138 SELECT c, s FROM (\
2139 SELECT '2026-08-13 12:00:00.001' AS c, now() AS s\
2140 ) AS t WHERE c > '2026-01-01'",
2141 &[1_786_593_600_001_i64][..],
2142 ),
2143 (
2144 "INSERT INTO timestamps (ts, st) \
2145 SELECT c, s FROM (\
2146 SELECT '2026-08-14 12:00:00.001' AS c, now() AS s\
2147 ) AS t ORDER BY c",
2148 &[1_786_680_000_001_i64][..],
2149 ),
2150 (
2151 "INSERT INTO timestamps (ts, st) \
2152 SELECT DISTINCT c, s FROM (\
2153 SELECT '2026-08-15 12:00:00.001' AS c, now() AS s\
2154 ) AS t",
2155 &[1_786_766_400_001_i64][..],
2156 ),
2157 ] {
2158 let plan = analyze_insert(&engine, sql, &query_ctx).await;
2159
2160 for expected_timestamp in expected_timestamps {
2161 assert!(
2162 plan.contains(&format!("TimestampMillisecond({expected_timestamp}, None)")),
2163 "{plan}"
2164 );
2165 }
2166 }
2167 }
2168
2169 #[tokio::test]
2170 async fn test_insert_explicit_timestamp_cast_keeps_datafusion_semantics() {
2171 let query_ctx = Arc::new(
2172 QueryContextBuilder::default()
2173 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2174 .build(),
2175 );
2176 let engine = create_timestamp_test_engine().await;
2177 let sql = "INSERT INTO timestamps (ts, st) \
2178 VALUES (CAST('2026-08-08 12:00:00.001' AS TIMESTAMP), now())";
2179 let plan = analyze_insert(&engine, sql, &query_ctx).await;
2180
2181 assert!(
2184 plan.contains("arrow_cast(Utf8(\"2026-08-08 12:00:00.001\")"),
2185 "{plan}"
2186 );
2187 assert!(
2189 !plan.contains("TimestampMillisecond(1786104000001, None)"),
2190 "{plan}"
2191 );
2192 }
2193
2194 #[tokio::test]
2195 async fn test_insert_converts_source_literal_shared_by_several_columns() {
2196 let query_ctx = Arc::new(
2197 QueryContextBuilder::default()
2198 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2199 .build(),
2200 );
2201 let engine = create_timestamp_test_engine().await;
2202
2203 for (sql, expected) in [
2205 (
2206 "INSERT INTO timestamps (ts, note) SELECT a, b FROM (\
2207 SELECT c AS a, c AS b FROM (\
2208 SELECT '2026-08-12 12:00:00.001' AS c\
2209 ) AS t1\
2210 ) AS t2",
2211 &[
2212 "TimestampMillisecond(1786507200001, None)",
2213 "[a:Utf8, b:Utf8]",
2214 ][..],
2215 ),
2216 (
2217 "INSERT INTO timestamps (ts, ts_ns) SELECT a, b FROM (\
2218 SELECT c AS a, c AS b FROM (\
2219 SELECT '2026-08-12 12:00:00.123456789' AS c\
2220 ) AS t1\
2221 ) AS t2",
2222 &[
2223 "TimestampMillisecond(1786507200123, None)",
2224 "TimestampNanosecond(1786507200123456789, None)",
2225 ][..],
2226 ),
2227 ] {
2228 let plan = analyze_insert(&engine, sql, &query_ctx).await;
2229
2230 for expected in expected {
2231 assert!(plan.contains(expected), "{plan}");
2232 }
2233 }
2234 }
2235
2236 #[tokio::test]
2237 async fn test_insert_union_converts_via_assignment_cast() {
2238 let query_ctx = Arc::new(
2239 QueryContextBuilder::default()
2240 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2241 .build(),
2242 );
2243 let engine = create_timestamp_test_engine().await;
2244 for sql in [
2249 "INSERT INTO timestamps (ts, st) \
2250 SELECT '2026-08-06 12:00:00.001', now() \
2251 UNION ALL \
2252 SELECT '2026-08-07 12:00:00.001', NULL",
2253 "INSERT INTO timestamps (ts, st) \
2254 SELECT '2026-08-16 12:00:00.001', now() \
2255 UNION \
2256 SELECT '2026-08-17 12:00:00.001', now()",
2257 ] {
2258 let plan = analyze_insert(&engine, sql, &query_ctx).await;
2259
2260 assert!(
2261 plan.contains("AS Timestamp(ms, \"Asia/Shanghai\")"),
2262 "{plan}"
2263 );
2264 assert!(
2266 plan.contains("Utf8(\"2026-08-07 12:00:00.001\")")
2267 || plan.contains("Utf8(\"2026-08-17 12:00:00.001\")"),
2268 "{plan}"
2269 );
2270 }
2271 }
2272
2273 #[tokio::test]
2274 async fn test_insert_mixed_union_keeps_source_coercion() {
2275 let query_ctx = Arc::new(
2276 QueryContextBuilder::default()
2277 .timezone(Timezone::from_tz_string("Asia/Shanghai").unwrap())
2278 .build(),
2279 );
2280 let engine = create_timestamp_test_engine().await;
2281 let sql = "INSERT INTO timestamps (ts, st) \
2282 SELECT '2026-08-10 12:00:00.001', now() \
2283 UNION ALL \
2284 SELECT CAST('2026-08-11 12:00:00.001' AS TIMESTAMP), now()";
2285 let plan = analyze_insert(&engine, sql, &query_ctx).await;
2286
2287 assert!(
2288 !plan.contains("TimestampMillisecond(1786334400001, None)"),
2289 "{plan}"
2290 );
2291 assert!(
2292 plan.contains("arrow_cast(Utf8(\"2026-08-11 12:00:00.001\")"),
2293 "{plan}"
2294 );
2295 assert!(!plan.contains("Asia/Shanghai"), "{plan}");
2300 }
2301
2302 #[tokio::test]
2303 async fn test_extract_placeholder_cast_types_multiple() {
2304 let plan = parse_sql_to_plan(
2305 "SELECT $1::INT, $2::TEXT, $3, $4::INTEGER FROM test WHERE $5::FLOAT > 0",
2306 )
2307 .await;
2308 let types = DfLogicalPlanner::extract_placeholder_cast_types(&plan).unwrap();
2309
2310 assert_eq!(types.len(), 5);
2311 assert_eq!(types.get("$1"), Some(&Some(DataType::Int32)));
2312 assert_eq!(types.get("$2"), Some(&Some(DataType::Utf8)));
2313 assert_eq!(types.get("$3"), Some(&None));
2314 assert_eq!(types.get("$4"), Some(&Some(DataType::Int32)));
2315 assert_eq!(types.get("$5"), Some(&Some(DataType::Float32)));
2316 }
2317
2318 #[tokio::test]
2319 async fn test_get_inferred_parameter_types_fallback_for_udf_args() {
2320 let plan = parse_sql_to_plan(
2322 "SELECT parse_ident($1), parse_ident($2::TEXT) FROM test WHERE id > $3",
2323 )
2324 .await;
2325 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2326
2327 assert_eq!(types.len(), 3);
2328
2329 let type_1 = types.get("$1").unwrap();
2330 let type_2 = types.get("$2").unwrap();
2331 let type_3 = types.get("$3").unwrap();
2332
2333 assert!(type_1.is_none(), "Expected $1 to be None");
2334 assert_eq!(type_2, &Some(DataType::Utf8));
2335 assert_eq!(type_3, &Some(DataType::Int32));
2336 }
2337
2338 #[tokio::test]
2339 async fn test_get_inferred_parameter_types_limit_offset() {
2340 let plan = parse_sql_to_plan("SELECT id FROM test LIMIT $1 OFFSET $2").await;
2341 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2342
2343 assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
2344 assert_eq!(types.get("$2"), Some(&Some(DataType::Int64)));
2345 }
2346
2347 #[tokio::test]
2348 async fn test_plan_pql_applies_extension_rules() {
2349 for inner_agg in ["count", "sum", "avg", "min", "max", "stddev", "stdvar"] {
2350 let plan = parse_promql_to_plan(&format!(
2351 "sum(irate(some_metric[1h])) / scalar(count({inner_agg}(some_metric) by (tag_0)))"
2352 ))
2353 .await;
2354 let plan_str = plan.display_indent_schema().to_string();
2355 assert!(plan_str.contains("Distinct:"), "{inner_agg}: {plan_str}");
2356 }
2357 }
2358
2359 #[tokio::test]
2360 async fn test_plan_pql_filters_null_only_groups_for_non_count_inner_aggs() {
2361 let count_plan = parse_promql_to_plan("scalar(count(count(some_metric) by (tag_0)))").await;
2362 let count_plan_str = count_plan.display_indent_schema().to_string();
2363 assert!(
2364 !count_plan_str.contains("field_0 IS NOT NULL"),
2365 "{count_plan_str}"
2366 );
2367
2368 for inner_agg in ["sum", "avg", "min", "max", "stddev", "stdvar"] {
2369 let plan = parse_promql_to_plan(&format!(
2370 "scalar(count({inner_agg}(some_metric) by (tag_0)))"
2371 ))
2372 .await;
2373 let plan_str = plan.display_indent_schema().to_string();
2374 assert!(
2375 plan_str.contains("field_0 IS NOT NULL"),
2376 "{inner_agg}: {plan_str}"
2377 );
2378 }
2379 }
2380
2381 #[tokio::test]
2382 async fn test_plan_pql_skips_extension_rules_for_non_direct_or_unsupported_inner_agg() {
2383 for query in [
2384 "sum(irate(some_metric[1h])) / scalar(count(sum(irate(some_metric[1h])) by (tag_0)))",
2385 "sum(irate(some_metric[1h])) / scalar(count(group(some_metric) by (tag_0)))",
2386 ] {
2387 let plan = parse_promql_to_plan(query).await;
2388 let plan_str = plan.display_indent_schema().to_string();
2389 assert!(!plan_str.contains("Distinct:"), "{query}: {plan_str}");
2390 }
2391 }
2392
2393 #[tokio::test]
2394 async fn test_plan_sql_does_not_apply_nested_count_rule() {
2395 let plan = parse_sql_to_plan(
2396 "SELECT id, count(inner_count) \
2397 FROM ( \
2398 SELECT id, count(name) AS inner_count \
2399 FROM test \
2400 GROUP BY id \
2401 ORDER BY id \
2402 LIMIT 1000000 \
2403 ) t \
2404 GROUP BY id \
2405 ORDER BY id",
2406 )
2407 .await;
2408
2409 let plan_str = plan.display_indent_schema().to_string();
2410 assert!(!plan_str.contains("Distinct:"), "{plan_str}");
2411 }
2412
2413 #[tokio::test]
2414 async fn test_get_inferred_parameter_types_subquery() {
2415 let plan = parse_sql_to_plan(
2416 r#"SELECT * FROM test WHERE id = (SELECT id FROM test CROSS JOIN (SELECT parse_ident($1::TEXT) AS parts) p LIMIT 1)"#,
2417 ).await;
2418 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2419
2420 assert_eq!(types.len(), 1);
2421 let type_1 = types.get("$1").unwrap();
2422 assert_eq!(type_1, &Some(DataType::Utf8));
2423 }
2424
2425 #[tokio::test]
2426 async fn test_get_inferred_parameter_types_insert() {
2427 let plan = parse_sql_to_plan("INSERT INTO test (id, name) VALUES ($1, $2), ($3, $4)").await;
2428 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2429
2430 assert_eq!(types.len(), 4);
2431 assert_eq!(types.get("$1"), Some(&Some(DataType::Int32)));
2432 assert_eq!(types.get("$2"), Some(&Some(DataType::Utf8)));
2433 assert_eq!(types.get("$3"), Some(&Some(DataType::Int32)));
2434 assert_eq!(types.get("$4"), Some(&Some(DataType::Utf8)));
2435 }
2436
2437 fn plan_publishing(columns: &[(&str, ScalarValue)]) -> LogicalPlan {
2440 let exprs = columns
2441 .iter()
2442 .map(|(name, value)| DfExpr::Literal(value.clone(), None).alias(*name))
2443 .collect::<Vec<_>>();
2444 LogicalPlanBuilder::empty(false)
2445 .project(exprs)
2446 .unwrap()
2447 .build()
2448 .unwrap()
2449 }
2450
2451 fn plan_publishing_marked(columns: &[(&str, ScalarValue)], time_index: &str) -> LogicalPlan {
2454 let plan = plan_publishing(columns);
2455 let LogicalPlan::Projection(projection) = plan else {
2456 unreachable!("plan_publishing builds a projection")
2457 };
2458 let fields = projection
2459 .schema
2460 .iter()
2461 .map(|(_, field)| {
2462 let field =
2463 if field.name() == time_index {
2464 Arc::new(field.as_ref().clone().with_metadata(
2465 std::collections::HashMap::from([(
2466 datatypes::schema::TIME_INDEX_KEY.to_string(),
2467 "true".to_string(),
2468 )]),
2469 ))
2470 } else if field.name() == "__name__" {
2471 Arc::new(field.as_ref().clone().with_metadata(
2473 std::collections::HashMap::from([(
2474 datatypes::schema::METRIC_NAME_KEY.to_string(),
2475 "true".to_string(),
2476 )]),
2477 ))
2478 } else {
2479 Arc::clone(field)
2480 };
2481 (None, field)
2482 })
2483 .collect::<Vec<_>>();
2484 let schema = Arc::new(
2485 DFSchema::new_with_metadata(
2486 fields,
2487 std::collections::HashMap::from([(
2488 datatypes::schema::VERSION_KEY.to_string(),
2489 datatypes::schema::Schema::INITIAL_VERSION.to_string(),
2490 )]),
2491 )
2492 .unwrap(),
2493 );
2494 LogicalPlan::Projection(
2495 datafusion_expr::Projection::try_new_with_schema(
2496 projection.expr.clone(),
2497 projection.input.clone(),
2498 schema,
2499 )
2500 .unwrap(),
2501 )
2502 }
2503
2504 fn dictionary_label(value: &str) -> ScalarValue {
2506 ScalarValue::Dictionary(
2507 Box::new(DataType::UInt32),
2508 Box::new(ScalarValue::Utf8(Some(value.to_string()))),
2509 )
2510 }
2511
2512 fn column_names(plan: &LogicalPlan) -> Vec<String> {
2514 plan.schema()
2515 .fields()
2516 .iter()
2517 .map(|field| field.name().clone())
2518 .collect()
2519 }
2520
2521 fn time_value() -> ScalarValue {
2522 ScalarValue::TimestampMillisecond(Some(0), None)
2523 }
2524
2525 fn native_answer() -> LogicalPlan {
2528 plan_publishing_marked(
2529 &[
2530 ("ts", time_value()),
2531 ("val", ScalarValue::Float64(Some(2.0))),
2532 ("host", ScalarValue::Utf8(Some("a".to_string()))),
2533 ],
2534 "ts",
2535 )
2536 }
2537
2538 fn existing_answer() -> LogicalPlan {
2541 plan_publishing_marked(
2542 &[
2543 ("host", dictionary_label("a")),
2544 ("ts", time_value()),
2545 ("lhs.val + rhs.val", ScalarValue::Float64(Some(2.0))),
2546 ],
2547 "ts",
2548 )
2549 }
2550
2551 #[test]
2559 fn a_cte_publishes_the_existing_planners_columns_over_a_native_plan() {
2560 let native = native_answer();
2561 let existing = existing_answer();
2562
2563 let published = publish_legacy_schema(native, &existing)
2564 .expect("every column has one answer in the other plan");
2565
2566 assert_eq!(
2567 column_names(&published),
2568 vec!["host", "ts", "lhs.val + rhs.val"],
2569 "the CTE publishes the existing planner's column order and names"
2570 );
2571 assert!(
2572 publishes_columns(published.schema().as_ref(), existing.schema().as_ref()),
2573 "and every column it publishes: name, order, type, nullability and metadata: {}",
2574 published.schema()
2575 );
2576 assert_eq!(
2577 published
2578 .schema()
2579 .field_with_name(None, "host")
2580 .unwrap()
2581 .data_type(),
2582 &DataType::Dictionary(Box::new(DataType::UInt32), Box::new(DataType::Utf8)),
2583 "a label is published as the encoding the existing planner publishes"
2584 );
2585 let LogicalPlan::Projection(projection) = &published else {
2586 panic!("the native plan is wrapped in a projection: {published}");
2587 };
2588 assert_eq!(
2589 column_names(projection.input.as_ref()),
2590 vec!["ts", "val", "host"],
2591 "the native plan is kept as the projection's input"
2592 );
2593 }
2594
2595 #[test]
2604 fn a_cte_publishes_the_name_an_answer_carries_after_the_existing_columns() {
2605 let native = plan_publishing_marked(
2606 &[
2607 ("ts", time_value()),
2608 ("val", ScalarValue::Float64(Some(2.0))),
2609 ("host", ScalarValue::Utf8(Some("a".to_string()))),
2610 (
2611 "__name__",
2612 ScalarValue::Utf8(Some("cte_native".to_string())),
2613 ),
2614 ],
2615 "ts",
2616 );
2617 let existing = existing_answer();
2618
2619 let published = publish_legacy_schema(native, &existing)
2620 .expect("the declared name column does not compete with a label");
2621
2622 assert_eq!(
2623 column_names(&published),
2624 vec!["host", "ts", "lhs.val + rhs.val", "__name__"],
2625 "the CTE publishes the existing planner's columns, then the name the answer carries"
2626 );
2627 assert!(
2628 datatypes::schema::is_metric_name_field(
2629 published
2630 .schema()
2631 .field_with_name(None, "__name__")
2632 .unwrap()
2633 ),
2634 "the name column is marked as one: {}",
2635 published.schema()
2636 );
2637 }
2638
2639 #[test]
2647 fn what_a_cte_publishes_survives_the_analyzer_rebuilding_the_projection() {
2648 let existing = existing_answer();
2649 let published = publish_legacy_schema(native_answer(), &existing).unwrap();
2650
2651 let rebuilt = published
2653 .with_new_exprs(
2654 published.expressions(),
2655 published.inputs().into_iter().cloned().collect(),
2656 )
2657 .unwrap();
2658
2659 assert!(
2660 publishes_columns(rebuilt.schema().as_ref(), existing.schema().as_ref()),
2661 "the rebuilt projection publishes the same columns: {}",
2662 rebuilt.schema()
2663 );
2664 assert_eq!(
2665 rebuilt.to_string(),
2666 published.to_string(),
2667 "and the same expressions"
2668 );
2669 }
2670
2671 #[test]
2673 fn the_declared_name_column_survives_the_analyzer_rebuilding_the_projection() {
2674 let existing = existing_answer();
2675 let native = plan_publishing_marked(
2676 &[
2677 ("ts", time_value()),
2678 ("val", ScalarValue::Float64(Some(2.0))),
2679 ("host", ScalarValue::Utf8(Some("a".to_string()))),
2680 (
2681 "__name__",
2682 ScalarValue::Utf8(Some("cte_native".to_string())),
2683 ),
2684 ],
2685 "ts",
2686 );
2687 let published = publish_legacy_schema(native, &existing)
2688 .expect("the name column does not compete with a label");
2689
2690 let rebuilt = published
2692 .with_new_exprs(
2693 published.expressions(),
2694 published.inputs().into_iter().cloned().collect(),
2695 )
2696 .expect("the analyzer rebuilds a projection from its expressions");
2697
2698 let field = rebuilt
2699 .schema()
2700 .field_with_name(None, "__name__")
2701 .expect("the name column is still published");
2702 assert!(
2703 datatypes::schema::is_metric_name_field(field),
2704 "and is still marked as one: {}",
2705 rebuilt.schema()
2706 );
2707 }
2708
2709 #[test]
2715 fn a_shape_with_no_faithful_column_mapping_keeps_the_existing_plan() {
2716 let two_values = plan_publishing_marked(
2718 &[
2719 ("host", dictionary_label("a")),
2720 ("ts", time_value()),
2721 ("lhs.val + rhs.val", ScalarValue::Float64(Some(2.0))),
2722 ("lhs.val - rhs.val", ScalarValue::Float64(Some(0.0))),
2723 ],
2724 "ts",
2725 );
2726 assert!(publish_legacy_schema(native_answer(), &two_values).is_none());
2727
2728 let renamed_time = plan_publishing_marked(
2731 &[
2732 ("native_time_0", time_value()),
2733 ("val", ScalarValue::Float64(Some(2.0))),
2734 ("host", ScalarValue::Utf8(Some("a".to_string()))),
2735 ],
2736 "native_time_0",
2737 );
2738 let existing = plan_publishing_marked(
2739 &[
2740 ("host", dictionary_label("a")),
2741 ("ts", time_value()),
2742 ("val", ScalarValue::Float64(Some(2.0))),
2743 ],
2744 "ts",
2745 );
2746 assert!(publish_legacy_schema(renamed_time, &existing).is_none());
2747 }
2748
2749 #[tokio::test]
2750 async fn test_get_inferred_parameter_types_arrow_cast() {
2751 let plan = parse_sql_to_plan("SELECT $1::INT64, $2::FLOAT64, $3::INT16, $4::INT32, $5::UINT8, $6::UINT16, $7::UINT32").await;
2752 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2753
2754 assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
2755 assert_eq!(types.get("$2"), Some(&Some(DataType::Float64)));
2756 assert_eq!(types.get("$3"), Some(&Some(DataType::Int16)));
2757 assert_eq!(types.get("$4"), Some(&Some(DataType::Int32)));
2758 assert_eq!(types.get("$5"), Some(&Some(DataType::UInt8)));
2759 assert_eq!(types.get("$6"), Some(&Some(DataType::UInt16)));
2760 assert_eq!(types.get("$7"), Some(&Some(DataType::UInt32)));
2761
2762 let plan = parse_sql_to_plan("SELECT $1::INT8, $2::FLOAT8, $3::INT2, $4::INT8").await;
2763 let types = DfLogicalPlanner::get_inferred_parameter_types(&plan).unwrap();
2764
2765 assert_eq!(types.get("$1"), Some(&Some(DataType::Int64)));
2766 assert_eq!(types.get("$2"), Some(&Some(DataType::Float64)));
2767 assert_eq!(types.get("$3"), Some(&Some(DataType::Int16)));
2768 }
2769}