1use std::collections::{HashMap, HashSet};
16use std::sync::Arc;
17use std::time::Duration;
18
19use api::helper::ColumnDataTypeWrapper;
20use api::v1::alter_table_expr::Kind;
21use api::v1::meta::CreateFlowTask as PbCreateFlowTask;
22use api::v1::repartition::Source;
23use api::v1::{
24 AlterDatabaseExpr, AlterTableExpr, CreateFlowExpr, CreateTableExpr, CreateViewExpr,
25 PartitionedSource, Repartition, TargetPartitionColumns, UnpartitionedSource, column_def,
26};
27#[cfg(feature = "enterprise")]
28use api::v1::{
29 CreateTriggerExpr as PbCreateTriggerExpr, meta::CreateTriggerTask as PbCreateTriggerTask,
30};
31use catalog::CatalogManagerRef;
32use chrono::Utc;
33use common_base::regex_pattern::NAME_PATTERN_REG;
34use common_catalog::consts::{
35 DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, is_ddl_reserved_table, is_readonly_schema,
36 is_readonly_table,
37};
38use common_catalog::{format_full_flow_name, format_full_table_name};
39use common_error::ext::BoxedError;
40#[cfg(feature = "enterprise")]
41use common_meta::cache_invalidator::CacheInvalidatorRef;
42use common_meta::cache_invalidator::Context;
43use common_meta::ddl::create_flow::{
44 DEFER_ON_MISSING_SOURCE_KEY, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType,
45 INTERNAL_EVAL_OFFSET_KEY, INTERNAL_EVAL_SCHEDULE_KEY,
46};
47use common_meta::instruction::CacheIdent;
48#[cfg(feature = "enterprise")]
49use common_meta::key::TableMetadataManagerRef;
50use common_meta::key::schema_name::{SchemaName, SchemaNameKey};
51#[cfg(feature = "enterprise")]
52use common_meta::procedure_executor::ProcedureExecutorRef;
53#[cfg(feature = "enterprise")]
54use common_meta::rpc::ddl::trigger::CreateTriggerTask;
55#[cfg(feature = "enterprise")]
56use common_meta::rpc::ddl::trigger::DropTriggerTask;
57use common_meta::rpc::ddl::{
58 ALTER_LOGICAL_TABLE_IDS_EXTENSION_KEY, CreateFlowTask, CreatorGrantIntent, DdlTask,
59 DropFlowTask, DropViewTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse, TriggerReason,
60};
61use common_query::Output;
62use common_recordbatch::{RecordBatch, RecordBatches};
63use common_sql::convert::sql_value_to_value;
64use common_telemetry::{debug, info, tracing, warn};
65use common_time::{TimeToLive, Timestamp, Timezone};
66use datafusion_common::tree_node::TreeNodeVisitor;
67use datafusion_expr::LogicalPlan;
68use datafusion_expr::logical_plan::Distinct;
69use datatypes::prelude::ConcreteDataType;
70use datatypes::schema::{ColumnSchema, Schema};
71use datatypes::value::Value;
72use datatypes::vectors::{StringVector, VectorRef};
73use humantime::parse_duration;
74use partition::expr::{Operand, PartitionExpr, RestrictedOp};
75use partition::function::IcebergHash;
76use partition::multi_dim::MultiDimPartitionRule;
77use query::parser::QueryStatement;
78use query::plan::extract_and_rewrite_full_table_names;
79use query::promql::planner::NATIVE_PROMQL_EXTENSION_KEY;
80use query::query_engine::DefaultSerializer;
81use query::sql::create_table_stmt;
82use session::context::QueryContextRef;
83use session::table_name::table_idents_to_full_name;
84use snafu::{OptionExt, ResultExt, ensure};
85use sql::parser::{ParseOptions, ParserContext};
86use sql::parsers::utils::is_tql;
87use sql::statements::OptionMap;
88#[cfg(feature = "enterprise")]
89use sql::statements::alter::trigger::AlterTrigger;
90use sql::statements::alter::{AlterDatabase, AlterTable, AlterTableOperation};
91#[cfg(feature = "enterprise")]
92use sql::statements::create::trigger::CreateTrigger;
93use sql::statements::create::{
94 CreateExternalTable, CreateFlow, CreateTable, CreateTableLike, CreateView, Partitions,
95};
96use sql::statements::statement::Statement;
97use sqlparser::ast::{
98 Expr, Function, FunctionArg, FunctionArgExpr, FunctionArguments, Ident, UnaryOperator,
99 Value as ParserValue,
100};
101use store_api::metric_engine_consts::{
102 LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY,
103};
104use store_api::mito_engine_options::TTL_KEY;
105use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
106use table::TableRef;
107use table::dist_table::DistTable;
108use table::metadata::{self, TableId, TableInfo, TableMeta, TableType};
109use table::requests::{
110 AlterKind, AlterTableRequest, COMMENT_KEY, DDL_TIMEOUT, DDL_WAIT, INGEST_ROWS_RATE_LIMIT_KEY,
111 TableOptions, validate_and_normalize_annotation_options,
112};
113use table::table_name::TableName;
114use table::table_reference::TableReference;
115
116use crate::error::{
117 self, AlterExprToRequestSnafu, BuildDfLogicalPlanSnafu, CatalogSnafu, ColumnDataTypeSnafu,
118 ColumnNotFoundSnafu, ConvertSchemaSnafu, CreateLogicalTablesSnafu,
119 DeserializePartitionExprSnafu, EmptyDdlExprSnafu, ExternalSnafu, ExtractTableNamesSnafu,
120 FlowNotFoundSnafu, InvalidPartitionRuleSnafu, InvalidPartitionSnafu, InvalidSqlSnafu,
121 InvalidTableNameSnafu, InvalidViewNameSnafu, InvalidViewStmtSnafu, NotSupportedSnafu,
122 PartitionExprToPbSnafu, Result, RetentionTtlOnLogicalTableSnafu, SchemaInUseSnafu,
123 SchemaNotFoundSnafu, SchemaReadOnlySnafu, SerializePartitionExprSnafu, SubstraitCodecSnafu,
124 TableAlreadyExistsSnafu, TableDdlReservedSnafu, TableMetadataManagerSnafu, TableNotFoundSnafu,
125 TableReadOnlySnafu, UnrecognizedTableOptionSnafu, ViewAlreadyExistsSnafu,
126};
127use crate::expr_helper::{self, RepartitionRequest, RepartitionSource};
128use crate::statement::StatementExecutor;
129use crate::statement::show::create_partitions_stmt;
130use crate::utils::{to_executor_context, to_executor_context_with_origin_frontend};
131
132#[derive(Debug, Clone, Copy)]
134pub(crate) struct DdlSubmitOptions {
135 wait: bool,
136 timeout: Duration,
137}
138
139impl Default for DdlSubmitOptions {
140 fn default() -> Self {
141 Self {
142 wait: SubmitDdlTaskRequest::default_wait(),
143 timeout: SubmitDdlTaskRequest::default_timeout(),
144 }
145 }
146}
147
148#[cfg(not(feature = "enterprise"))]
149const ALLOWED_FLOW_OPTIONS: [&str; 2] = [
150 DEFER_ON_MISSING_SOURCE_KEY,
151 FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
152];
153
154fn build_procedure_id_output(procedure_id: &str) -> Result<Output> {
155 let vector: VectorRef = Arc::new(StringVector::from(vec![procedure_id]));
156 let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
157 "Procedure ID",
158 vector.data_type(),
159 false,
160 )]));
161 let batch =
162 RecordBatch::new(schema.clone(), vec![vector]).context(error::BuildRecordBatchSnafu)?;
163 let batches =
164 RecordBatches::try_new(schema, vec![batch]).context(error::BuildRecordBatchSnafu)?;
165 Ok(Output::new_with_record_batches(batches))
166}
167
168pub(crate) fn parse_ddl_options(options: &OptionMap) -> Result<DdlSubmitOptions> {
170 let defaults = DdlSubmitOptions::default();
171 let wait = match options.get(DDL_WAIT) {
172 Some(value) => value.parse::<bool>().map_err(|_| {
173 InvalidSqlSnafu {
174 err_msg: format!("invalid DDL option '{DDL_WAIT}': '{value}'"),
175 }
176 .build()
177 })?,
178 None => defaults.wait,
179 };
180
181 let timeout = match options.get(DDL_TIMEOUT) {
182 Some(value) => parse_duration(value).map_err(|err| {
183 InvalidSqlSnafu {
184 err_msg: format!("invalid DDL option '{DDL_TIMEOUT}': '{value}': {err}"),
185 }
186 .build()
187 })?,
188 None => defaults.timeout,
189 };
190
191 Ok(DdlSubmitOptions { wait, timeout })
192}
193
194#[cfg(not(feature = "enterprise"))]
195fn supported_flow_options() -> String {
196 ALLOWED_FLOW_OPTIONS.join(", ")
197}
198
199fn normalize_flow_bool_option(key: &str, value: &str) -> Result<String> {
200 value
201 .trim()
202 .to_ascii_lowercase()
203 .parse::<bool>()
204 .map(|value| value.to_string())
205 .map_err(|_| {
206 InvalidSqlSnafu {
207 err_msg: format!("invalid flow option '{key}': '{value}'"),
208 }
209 .build()
210 })
211}
212
213fn validate_and_normalize_flow_options(
214 options: HashMap<String, String>,
215 eval_interval: Option<i64>,
216) -> Result<HashMap<String, String>> {
217 if let Some(secs) = eval_interval
219 && secs <= 0
220 {
221 return InvalidSqlSnafu {
222 err_msg: format!("EVAL INTERVAL must be positive, got {secs} seconds"),
223 }
224 .fail();
225 }
226
227 options
228 .into_iter()
229 .map(|(key, value)| {
230 if matches!(
231 key.as_str(),
232 FlowType::FLOW_TYPE_KEY | INTERNAL_EVAL_OFFSET_KEY | INTERNAL_EVAL_SCHEDULE_KEY
233 ) {
234 return InvalidSqlSnafu {
235 err_msg: format!("flow option '{key}' is reserved for internal use"),
236 }
237 .fail();
238 }
239
240 let normalized_value = match key.as_str() {
241 DEFER_ON_MISSING_SOURCE_KEY | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY => {
242 normalize_flow_bool_option(&key, &value)?
243 }
244 #[cfg(feature = "enterprise")]
245 _ => value,
246 #[cfg(not(feature = "enterprise"))]
247 _ => {
248 return InvalidSqlSnafu {
249 err_msg: format!(
250 "unknown flow option '{key}', supported options: {}",
251 supported_flow_options()
252 ),
253 }
254 .fail();
255 }
256 };
257
258 Ok((key, normalized_value))
259 })
260 .collect()
261}
262
263fn validate_eval_offset(
267 eval_offset_secs: Option<i64>,
268 eval_interval_secs: Option<i64>,
269) -> Result<()> {
270 if let Some(offset_secs) = eval_offset_secs {
271 let Some(eval_interval_secs) = eval_interval_secs else {
272 return InvalidSqlSnafu {
273 err_msg: "EVAL OFFSET requires EVAL INTERVAL to be specified".to_string(),
274 }
275 .fail();
276 };
277 if !(0..eval_interval_secs).contains(&offset_secs) {
278 return InvalidSqlSnafu {
279 err_msg: format!(
280 "EVAL OFFSET must be in range [0, EVAL INTERVAL), got {offset_secs} seconds with EVAL INTERVAL {eval_interval_secs} seconds"
281 ),
282 }
283 .fail();
284 }
285 }
286 Ok(())
287}
288
289fn determine_flow_type_for_source_state(
290 flow_name: &str,
291 flow_options: &HashMap<String, String>,
292 has_missing_source_table: bool,
293 has_instant_ttl_source_table: bool,
294 force_batching: bool,
295) -> Result<Option<FlowType>> {
296 if has_instant_ttl_source_table && force_batching {
297 return InvalidSqlSnafu {
300 err_msg: format!(
301 "flow '{}' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode; use a TTL longer than the flush interval",
302 flow_name
303 ),
304 }
305 .fail();
306 }
307
308 if has_missing_source_table {
309 let defer_on_missing_source = flow_options
310 .get(DEFER_ON_MISSING_SOURCE_KEY)
311 .is_some_and(|value| value == "true");
312 ensure!(
313 defer_on_missing_source,
314 InvalidSqlSnafu {
315 err_msg: format!(
316 "missing source tables for flow '{}'; use WITH ({DEFER_ON_MISSING_SOURCE_KEY} = true) to create a pending flow",
317 flow_name
318 )
319 }
320 );
321 info!(
322 "Flow `{}` is created as a pending batching flow because source tables are missing and defer_on_missing_source=true",
323 flow_name
324 );
325 return Ok(Some(FlowType::Batching));
326 }
327
328 Ok(None)
329}
330
331fn is_stateless_flow_plan(plan: &LogicalPlan) -> bool {
335 let mut scans = 0;
336 let mut supported_nodes = true;
337 let result = plan.apply_with_subqueries(|node| {
338 match node {
339 LogicalPlan::TableScan(_) => scans += 1,
340 LogicalPlan::Projection(_) | LogicalPlan::Filter(_) => {}
341 LogicalPlan::Distinct(Distinct::All(_)) => {}
342 _ => supported_nodes = false,
343 }
344 Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)
345 });
346
347 result.is_ok() && supported_nodes && scans == 1
348}
349
350const INSTANT_TTL_FLOW_QUERY_ERROR: &str = "instant-TTL flow sources support only stateless projection/filter queries over one source; non-stateless queries require a persisted source";
351
352const STATELESS_FLOW_QUERY_ERROR: &str =
353 "flow streaming supports only stateless projection/filter queries over one source";
354
355impl StatementExecutor {
356 pub fn catalog_manager(&self) -> CatalogManagerRef {
357 self.catalog_manager.clone()
358 }
359
360 #[tracing::instrument(skip_all)]
361 pub async fn create_table(&self, stmt: CreateTable, ctx: QueryContextRef) -> Result<TableRef> {
362 let mut create_expr = self.prepare_create_table(&stmt, &ctx).await?;
363 self.create_table_inner(
364 &mut create_expr,
365 stmt.partitions,
366 ctx,
367 TriggerReason::Manual,
368 )
369 .await
370 }
371
372 async fn prepare_create_table(
373 &self,
374 stmt: &CreateTable,
375 ctx: &QueryContextRef,
376 ) -> Result<CreateTableExpr> {
377 let (catalog, schema, _table) = table_idents_to_full_name(&stmt.name, ctx)
378 .map_err(BoxedError::new)
379 .context(error::ExternalSnafu)?;
380
381 let schema_options = self
382 .table_metadata_manager
383 .schema_manager()
384 .get(SchemaNameKey {
385 catalog: &catalog,
386 schema: &schema,
387 })
388 .await
389 .context(TableMetadataManagerSnafu)?
390 .map(|v| v.into_inner());
391
392 let mut create_expr = expr_helper::create_to_expr(stmt, ctx)?;
393 if let Some(schema_options) = schema_options {
395 for (key, value) in schema_options.extra_options.iter() {
396 if key == INGEST_ROWS_RATE_LIMIT_KEY || key.starts_with("compaction.") {
397 continue;
398 }
399 create_expr
400 .table_options
401 .entry(key.clone())
402 .or_insert(value.clone());
403 }
404 }
405
406 Ok(create_expr)
407 }
408
409 pub async fn batch_create_logical_tables(
411 &self,
412 statements: Vec<CreateTable>,
413 ctx: QueryContextRef,
414 ) -> Result<()> {
415 let mut expressions = Vec::with_capacity(statements.len());
416 let mut targets = std::collections::HashSet::new();
417 let mut group = None;
418 for statement in statements {
419 ensure!(
420 statement.engine == METRIC_ENGINE_NAME
421 && statement.options.get(LOGICAL_TABLE_METADATA_KEY).is_some()
422 && statement
423 .options
424 .value(PHYSICAL_TABLE_METADATA_KEY)
425 .is_none(),
426 InvalidSqlSnafu {
427 err_msg: "batch requires explicit Metric logical CREATE TABLE statements"
428 }
429 );
430 let expr = self.prepare_create_table(&statement, &ctx).await?;
431 let key = (
432 expr.catalog_name.clone(),
433 expr.schema_name.clone(),
434 expr.table_options.get(LOGICAL_TABLE_METADATA_KEY).cloned(),
435 );
436 ensure!(
437 group.as_ref().is_none_or(|group| group == &key),
438 InvalidSqlSnafu {
439 err_msg: "batch must share catalog, schema and literal on_physical_table"
440 }
441 );
442 group = Some(key);
443 ensure!(
444 targets.insert(expr.table_name.clone()),
445 InvalidSqlSnafu {
446 err_msg: "duplicate target in logical-table batch"
447 }
448 );
449 self.validate_logical_create(&expr, statement.partitions.as_ref(), &ctx)
450 .await?;
451 expressions.push(expr);
452 }
453 self.create_logical_tables(&expressions, ctx, TriggerReason::Manual)
454 .await?;
455 Ok(())
456 }
457
458 async fn validate_logical_create(
459 &self,
460 expr: &CreateTableExpr,
461 partitions: Option<&Partitions>,
462 ctx: &QueryContextRef,
463 ) -> Result<()> {
464 ensure_table_definition_writable(&expr.schema_name, &expr.table_name)?;
465 if let Some(partitions) = partitions
466 && !partitions.exprs.is_empty()
467 {
468 self.validate_logical_table_partition_rule(expr, partitions, ctx)
469 .await?;
470 }
471 Ok(())
472 }
473
474 #[tracing::instrument(skip_all)]
475 pub async fn create_table_like(
476 &self,
477 stmt: CreateTableLike,
478 ctx: QueryContextRef,
479 ) -> Result<TableRef> {
480 let (catalog, schema, table) = table_idents_to_full_name(&stmt.source_name, &ctx)
481 .map_err(BoxedError::new)
482 .context(error::ExternalSnafu)?;
483 let table_ref = self
484 .catalog_manager
485 .table(&catalog, &schema, &table, Some(&ctx))
486 .await
487 .context(CatalogSnafu)?
488 .context(TableNotFoundSnafu { table_name: &table })?;
489 let partition_info = self
490 .partition_manager
491 .find_physical_partition_info(table_ref.table_info().table_id())
492 .await
493 .context(error::FindTablePartitionRuleSnafu { table_name: table })?;
494
495 let schema_options = self
497 .table_metadata_manager
498 .schema_manager()
499 .get(SchemaNameKey {
500 catalog: &catalog,
501 schema: &schema,
502 })
503 .await
504 .context(TableMetadataManagerSnafu)?
505 .map(|v| v.into_inner());
506
507 let quote_style = ctx.quote_style();
508 let mut create_stmt =
509 create_table_stmt(&table_ref.table_info(), schema_options, quote_style)
510 .context(error::ParseQuerySnafu)?;
511 create_stmt.name = stmt.table_name;
512 create_stmt.if_not_exists = false;
513
514 let table_info = table_ref.table_info();
515 let partitions = create_partitions_stmt(&table_info, &partition_info.partitions)?.and_then(
516 |mut partitions| {
517 if !partitions.column_list.is_empty() {
518 partitions.set_quote(quote_style);
519 Some(partitions)
520 } else {
521 None
522 }
523 },
524 );
525
526 let create_expr = &mut expr_helper::create_to_expr(&create_stmt, &ctx)?;
527 self.create_table_inner(create_expr, partitions, ctx, TriggerReason::Manual)
528 .await
529 }
530
531 #[tracing::instrument(skip_all)]
532 pub async fn create_external_table(
533 &self,
534 create_expr: CreateExternalTable,
535 ctx: QueryContextRef,
536 ) -> Result<TableRef> {
537 if create_expr.if_not_exists {
538 let (catalog_name, schema_name, table_name) =
539 table_idents_to_full_name(&create_expr.name, &ctx)
540 .map_err(BoxedError::new)
541 .context(ExternalSnafu)?;
542 if let Some(table) = self
543 .catalog_manager
544 .table(&catalog_name, &schema_name, &table_name, Some(&ctx))
545 .await
546 .context(CatalogSnafu)?
547 {
548 return Ok(table);
549 }
550 }
551
552 let create_expr =
553 &mut expr_helper::create_external_expr(create_expr, &ctx, &self.local_file_access)
554 .await?;
555 self.create_table_inner(create_expr, None, ctx, TriggerReason::Manual)
556 .await
557 }
558
559 pub async fn create_declared_relationships_table(
563 &self,
564 catalog: &str,
565 query_ctx: QueryContextRef,
566 ) -> Result<TableRef> {
567 let mut expr = super::semantic_graph::build_declared_relationships_expr(catalog);
568 self.create_non_logic_table(&mut expr, None, query_ctx, TriggerReason::AutoCreate)
569 .await
570 }
571
572 #[tracing::instrument(skip_all)]
573 pub async fn create_table_inner(
574 &self,
575 create_table: &mut CreateTableExpr,
576 partitions: Option<Partitions>,
577 query_ctx: QueryContextRef,
578 trigger_reason: TriggerReason,
579 ) -> Result<TableRef> {
580 ensure_table_definition_writable(&create_table.schema_name, &create_table.table_name)?;
581
582 if create_table.engine == METRIC_ENGINE_NAME
583 && create_table
584 .table_options
585 .contains_key(LOGICAL_TABLE_METADATA_KEY)
586 {
587 self.validate_logical_create(create_table, partitions.as_ref(), &query_ctx)
588 .await?;
589 self.create_logical_tables(
591 std::slice::from_ref(create_table),
592 query_ctx,
593 trigger_reason,
594 )
595 .await?
596 .into_iter()
597 .next()
598 .context(error::UnexpectedSnafu {
599 violated: "expected to create logical tables",
600 })
601 } else {
602 self.create_non_logic_table(create_table, partitions, query_ctx, trigger_reason)
604 .await
605 }
606 }
607
608 #[tracing::instrument(skip_all)]
609 pub async fn create_non_logic_table(
610 &self,
611 create_table: &mut CreateTableExpr,
612 partitions: Option<Partitions>,
613 query_ctx: QueryContextRef,
614 trigger_reason: TriggerReason,
615 ) -> Result<TableRef> {
616 let _timer = crate::metrics::DIST_CREATE_TABLE.start_timer();
617
618 let schema = self
620 .table_metadata_manager
621 .schema_manager()
622 .get(SchemaNameKey::new(
623 &create_table.catalog_name,
624 &create_table.schema_name,
625 ))
626 .await
627 .context(TableMetadataManagerSnafu)?;
628 ensure!(
629 schema.is_some(),
630 SchemaNotFoundSnafu {
631 schema_info: &create_table.schema_name,
632 }
633 );
634
635 if let Some(table) = self
637 .catalog_manager
638 .table(
639 &create_table.catalog_name,
640 &create_table.schema_name,
641 &create_table.table_name,
642 Some(&query_ctx),
643 )
644 .await
645 .context(CatalogSnafu)?
646 {
647 return if create_table.create_if_not_exists {
648 Ok(table)
649 } else {
650 TableAlreadyExistsSnafu {
651 table: format_full_table_name(
652 &create_table.catalog_name,
653 &create_table.schema_name,
654 &create_table.table_name,
655 ),
656 }
657 .fail()
658 };
659 }
660
661 ensure!(
662 NAME_PATTERN_REG.is_match(&create_table.table_name),
663 InvalidTableNameSnafu {
664 table_name: &create_table.table_name,
665 }
666 );
667
668 let table_name = TableName::new(
669 &create_table.catalog_name,
670 &create_table.schema_name,
671 &create_table.table_name,
672 );
673
674 let (partitions, partition_cols) = parse_partitions(create_table, partitions, &query_ctx)?;
675 let mut table_info = create_table_info(create_table, partition_cols)?;
676
677 let resp = self
678 .create_table_procedure(
679 create_table.clone(),
680 partitions,
681 table_info.clone(),
682 query_ctx,
683 trigger_reason,
684 )
685 .await?;
686
687 let table_id = resp
688 .table_ids
689 .into_iter()
690 .next()
691 .context(error::UnexpectedSnafu {
692 violated: "expected table_id",
693 })?;
694 info!("Successfully created table '{table_name}' with table id {table_id}");
695
696 table_info.ident.table_id = table_id;
697
698 let table_info = Arc::new(table_info);
699 create_table.table_id = Some(api::v1::TableId { id: table_id });
700
701 let table = DistTable::table(table_info);
702
703 Ok(table)
704 }
705
706 #[tracing::instrument(skip_all)]
707 pub async fn create_logical_tables(
708 &self,
709 create_table_exprs: &[CreateTableExpr],
710 query_context: QueryContextRef,
711 trigger_reason: TriggerReason,
712 ) -> Result<Vec<TableRef>> {
713 let _timer = crate::metrics::DIST_CREATE_TABLES.start_timer();
714 ensure!(
715 !create_table_exprs.is_empty(),
716 EmptyDdlExprSnafu {
717 name: "create logic tables"
718 }
719 );
720
721 let create_table_exprs = create_table_exprs
726 .iter()
727 .cloned()
728 .map(|mut create| {
729 if create
730 .table_options
731 .get(TTL_KEY)
732 .is_some_and(|ttl| is_retention_ttl(ttl))
733 {
734 create.table_options.remove(TTL_KEY);
735 }
736 create
737 })
738 .collect::<Vec<_>>();
739
740 for create_table in &create_table_exprs {
742 ensure!(
743 NAME_PATTERN_REG.is_match(&create_table.table_name),
744 InvalidTableNameSnafu {
745 table_name: &create_table.table_name,
746 }
747 );
748 }
749
750 let raw_tables_info = create_table_exprs
751 .iter()
752 .map(|create| create_table_info(create, vec![]))
753 .collect::<Result<Vec<_>>>()?;
754 let tables_data = create_table_exprs
755 .iter()
756 .cloned()
757 .zip(raw_tables_info.iter().cloned())
758 .collect::<Vec<_>>();
759
760 let resp = self
761 .create_logical_tables_procedure(tables_data, query_context.clone(), trigger_reason)
762 .await?;
763
764 let table_ids = resp.table_ids;
765 ensure!(
766 table_ids.len() == raw_tables_info.len(),
767 CreateLogicalTablesSnafu {
768 reason: format!(
769 "The number of tables is inconsistent with the expected number to be created, expected: {}, actual: {}",
770 raw_tables_info.len(),
771 table_ids.len()
772 )
773 }
774 );
775 info!("Successfully created logical tables: {:?}", table_ids);
776
777 let mut tables_info = Vec::with_capacity(table_ids.len());
781 for (table_id, create_table) in table_ids.iter().zip(create_table_exprs.iter()) {
782 let table = self
783 .catalog_manager
784 .table(
785 &create_table.catalog_name,
786 &create_table.schema_name,
787 &create_table.table_name,
788 Some(&query_context),
789 )
790 .await
791 .context(CatalogSnafu)?
792 .with_context(|| TableNotFoundSnafu {
793 table_name: format_full_table_name(
794 &create_table.catalog_name,
795 &create_table.schema_name,
796 &create_table.table_name,
797 ),
798 })?;
799
800 let table_info = table.table_info();
801 ensure!(
803 table_info.table_id() == *table_id,
804 CreateLogicalTablesSnafu {
805 reason: format!(
806 "Table id mismatch after creation, expected {}, got {} for table {}",
807 table_id,
808 table_info.table_id(),
809 format_full_table_name(
810 &create_table.catalog_name,
811 &create_table.schema_name,
812 &create_table.table_name
813 )
814 )
815 }
816 );
817
818 tables_info.push(table_info);
819 }
820
821 Ok(tables_info.into_iter().map(DistTable::table).collect())
822 }
823
824 async fn validate_logical_table_partition_rule(
825 &self,
826 create_table: &CreateTableExpr,
827 partitions: &Partitions,
828 query_ctx: &QueryContextRef,
829 ) -> Result<()> {
830 let (_, mut logical_partition_exprs) =
831 parse_partitions_for_logical_validation(create_table, partitions, query_ctx)?;
832
833 let physical_table_name = create_table
834 .table_options
835 .get(LOGICAL_TABLE_METADATA_KEY)
836 .with_context(|| CreateLogicalTablesSnafu {
837 reason: format!(
838 "expect `{LOGICAL_TABLE_METADATA_KEY}` option on creating logical table"
839 ),
840 })?;
841
842 let physical_table = self
843 .catalog_manager
844 .table(
845 &create_table.catalog_name,
846 &create_table.schema_name,
847 physical_table_name,
848 Some(query_ctx),
849 )
850 .await
851 .context(CatalogSnafu)?
852 .context(TableNotFoundSnafu {
853 table_name: physical_table_name.clone(),
854 })?;
855
856 let physical_table_info = physical_table.table_info();
857 let (partition_rule, _) = self
858 .partition_manager
859 .find_table_partition_rule(&physical_table_info)
860 .await
861 .context(error::FindTablePartitionRuleSnafu {
862 table_name: physical_table_name.clone(),
863 })?;
864
865 let multi_dim_rule = partition_rule
866 .as_ref()
867 .as_any()
868 .downcast_ref::<MultiDimPartitionRule>()
869 .context(InvalidPartitionRuleSnafu {
870 reason: "physical table partition rule is not range-based",
871 })?;
872
873 let mut physical_partition_exprs = multi_dim_rule.exprs().to_vec();
875 logical_partition_exprs.sort_unstable();
876 physical_partition_exprs.sort_unstable();
877
878 ensure!(
879 physical_partition_exprs == logical_partition_exprs,
880 InvalidPartitionRuleSnafu {
881 reason: format!(
882 "logical table partition rule must match the corresponding physical table's\n logical table partition exprs:\t\t {:?}\n physical table partition exprs:\t {:?}",
883 logical_partition_exprs, physical_partition_exprs
884 ),
885 }
886 );
887
888 Ok(())
889 }
890
891 #[cfg(feature = "enterprise")]
892 #[tracing::instrument(skip_all)]
893 pub async fn create_trigger(
894 &self,
895 stmt: CreateTrigger,
896 query_context: QueryContextRef,
897 ) -> Result<Output> {
898 let expr = expr_helper::to_create_trigger_task_expr(stmt, &query_context)?;
899 self.create_trigger_inner(expr, query_context).await
900 }
901
902 #[cfg(feature = "enterprise")]
903 pub async fn create_trigger_inner(
904 &self,
905 expr: PbCreateTriggerExpr,
906 query_context: QueryContextRef,
907 ) -> Result<Output> {
908 self.create_trigger_procedure(expr, query_context).await?;
909 Ok(Output::new_with_affected_rows(0))
910 }
911
912 #[cfg(feature = "enterprise")]
913 async fn create_trigger_procedure(
914 &self,
915 expr: PbCreateTriggerExpr,
916 query_context: QueryContextRef,
917 ) -> Result<SubmitDdlTaskResponse> {
918 let task = CreateTriggerTask::try_from(PbCreateTriggerTask {
919 create_trigger: Some(expr),
920 })
921 .context(error::InvalidExprSnafu)?;
922
923 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
924 let request = SubmitDdlTaskRequest::new(DdlTask::new_create_trigger(task));
925
926 self.procedure_executor
927 .submit_ddl_task(executor_context, request)
928 .await
929 .context(error::ExecuteDdlSnafu)
930 }
931
932 #[tracing::instrument(skip_all)]
933 pub async fn create_flow(
934 &self,
935 stmt: CreateFlow,
936 query_context: QueryContextRef,
937 ) -> Result<Output> {
938 let eval_offset_secs = stmt.eval_offset;
940 let expr = expr_helper::to_create_flow_task_expr(stmt, &query_context)?;
941
942 self.create_flow_procedure(expr, eval_offset_secs, query_context)
945 .await?;
946 Ok(Output::new_with_affected_rows(0))
947 }
948
949 pub async fn create_flow_inner(
951 &self,
952 expr: CreateFlowExpr,
953 query_context: QueryContextRef,
954 ) -> Result<Output> {
955 self.create_flow_procedure(expr, None, query_context)
956 .await?;
957 Ok(Output::new_with_affected_rows(0))
958 }
959
960 async fn create_flow_procedure(
961 &self,
962 mut expr: CreateFlowExpr,
963 eval_offset_secs: Option<i64>,
964 query_context: QueryContextRef,
965 ) -> Result<SubmitDdlTaskResponse> {
966 let eval_interval_secs = expr.eval_interval.as_ref().map(|e| e.seconds);
967
968 if let Some(secs) = eval_interval_secs
970 && secs <= 0
971 {
972 return InvalidSqlSnafu {
973 err_msg: format!("EVAL INTERVAL must be positive, got {secs} seconds"),
974 }
975 .fail();
976 }
977
978 validate_eval_offset(eval_offset_secs, eval_interval_secs)?;
982
983 expr.flow_options =
987 validate_and_normalize_flow_options(expr.flow_options, eval_interval_secs)?;
988
989 let flow_type = self
990 .determine_flow_type(&expr, query_context.clone())
991 .await?;
992 info!("determined flow={} type: {:#?}", expr.flow_name, flow_type);
993
994 expr.flow_options
995 .insert(FlowType::FLOW_TYPE_KEY.to_string(), flow_type.to_string());
996
997 let mut task = CreateFlowTask::try_from(PbCreateFlowTask {
998 create_flow: Some(expr),
999 })
1000 .context(error::InvalidExprSnafu)?;
1001 task.eval_offset_secs = eval_offset_secs;
1002 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1003 let request = SubmitDdlTaskRequest::new(DdlTask::new_create_flow(task));
1004
1005 self.procedure_executor
1006 .submit_ddl_task(executor_context, request)
1007 .await
1008 .context(error::ExecuteDdlSnafu)
1009 }
1010
1011 async fn determine_flow_type(
1017 &self,
1018 expr: &CreateFlowExpr,
1019 query_ctx: QueryContextRef,
1020 ) -> Result<FlowType> {
1021 let has_eval_interval = expr.eval_interval.is_some();
1022
1023 let mut has_missing_source_table = false;
1024 let mut has_instant_ttl_source_table = false;
1025
1026 for src_table_name in &expr.source_table_names {
1027 let table = self
1028 .catalog_manager()
1029 .table(
1030 &src_table_name.catalog_name,
1031 &src_table_name.schema_name,
1032 &src_table_name.table_name,
1033 Some(&query_ctx),
1034 )
1035 .await
1036 .map_err(BoxedError::new)
1037 .context(ExternalSnafu)?;
1038
1039 let Some(table) = table else {
1040 has_missing_source_table = true;
1041 continue;
1042 };
1043
1044 if table.table_info().meta.options.ttl == Some(common_time::TimeToLive::Instant) {
1045 warn!(
1046 "Source table `{}` for flow `{}`'s ttl=instant, {}",
1047 format_full_table_name(
1048 &src_table_name.catalog_name,
1049 &src_table_name.schema_name,
1050 &src_table_name.table_name
1051 ),
1052 expr.flow_name,
1053 if has_eval_interval {
1054 "rejecting flow because EVAL INTERVAL requires the batching scheduler which does not support instant TTL"
1055 } else {
1056 "fallback to streaming mode"
1057 }
1058 );
1059 has_instant_ttl_source_table = true;
1060 }
1061 }
1062
1063 if let Some(flow_type) = determine_flow_type_for_source_state(
1064 &expr.flow_name,
1065 &expr.flow_options,
1066 has_missing_source_table,
1067 has_instant_ttl_source_table,
1068 has_eval_interval,
1069 )? {
1070 return Ok(flow_type);
1071 }
1072
1073 if has_eval_interval {
1079 return Ok(FlowType::Batching);
1080 }
1081
1082 let engine = &self.query_engine;
1083 let stmts = ParserContext::create_with_dialect(
1084 &expr.sql,
1085 query_ctx.sql_dialect(),
1086 ParseOptions::default(),
1087 )
1088 .map_err(BoxedError::new)
1089 .context(ExternalSnafu)?;
1090
1091 ensure!(
1092 stmts.len() == 1,
1093 InvalidSqlSnafu {
1094 err_msg: format!("Expect only one statement, found {}", stmts.len())
1095 }
1096 );
1097 let stmt = &stmts[0];
1098
1099 let is_tql_query = is_tql(query_ctx.sql_dialect(), &expr.sql)
1100 .map_err(BoxedError::new)
1101 .context(ExternalSnafu)?;
1102 if is_tql_query {
1103 if has_instant_ttl_source_table {
1104 return InvalidSqlSnafu {
1105 err_msg: INSTANT_TTL_FLOW_QUERY_ERROR.to_string(),
1106 }
1107 .fail();
1108 }
1109 return Ok(FlowType::Batching);
1110 }
1111
1112 let plan = match stmt {
1115 Statement::Tql(_) => {
1116 if has_instant_ttl_source_table {
1117 return InvalidSqlSnafu {
1118 err_msg: INSTANT_TTL_FLOW_QUERY_ERROR.to_string(),
1119 }
1120 .fail();
1121 }
1122 return Ok(FlowType::Batching);
1123 }
1124 _ => engine
1125 .planner()
1126 .plan(&QueryStatement::Sql(stmt.clone()), query_ctx)
1127 .await
1128 .map_err(BoxedError::new)
1129 .context(ExternalSnafu)?,
1130 };
1131
1132 struct FindQueryShape {
1135 is_aggr: bool,
1136 is_distinct: bool,
1137 has_distinct_on: bool,
1138 }
1139
1140 impl TreeNodeVisitor<'_> for FindQueryShape {
1141 type Node = LogicalPlan;
1142 fn f_down(
1143 &mut self,
1144 node: &Self::Node,
1145 ) -> datafusion_common::Result<datafusion_common::tree_node::TreeNodeRecursion>
1146 {
1147 match node {
1148 LogicalPlan::Aggregate(_) => self.is_aggr = true,
1149 LogicalPlan::Distinct(Distinct::All(_)) => self.is_distinct = true,
1150 LogicalPlan::Distinct(Distinct::On(_)) => self.has_distinct_on = true,
1151 _ => (),
1152 }
1153 Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)
1154 }
1155 }
1156
1157 let mut query_shape = FindQueryShape {
1158 is_aggr: false,
1159 is_distinct: false,
1160 has_distinct_on: false,
1161 };
1162
1163 plan.visit_with_subqueries(&mut query_shape)
1164 .context(BuildDfLogicalPlanSnafu)?;
1165 if query_shape.has_distinct_on {
1166 return InvalidSqlSnafu {
1167 err_msg: if has_instant_ttl_source_table {
1168 INSTANT_TTL_FLOW_QUERY_ERROR.to_string()
1169 } else {
1170 STATELESS_FLOW_QUERY_ERROR.to_string()
1171 },
1172 }
1173 .fail();
1174 }
1175 if query_shape.is_aggr {
1176 return if has_instant_ttl_source_table {
1177 InvalidSqlSnafu {
1178 err_msg: INSTANT_TTL_FLOW_QUERY_ERROR.to_string(),
1179 }
1180 .fail()
1181 } else {
1182 Ok(FlowType::Batching)
1183 };
1184 }
1185 if query_shape.is_distinct && !has_instant_ttl_source_table {
1186 return Ok(FlowType::Batching);
1187 }
1188
1189 ensure!(
1190 is_stateless_flow_plan(&plan),
1191 InvalidSqlSnafu {
1192 err_msg: if has_instant_ttl_source_table {
1193 INSTANT_TTL_FLOW_QUERY_ERROR.to_string()
1194 } else {
1195 STATELESS_FLOW_QUERY_ERROR.to_string()
1196 },
1197 }
1198 );
1199 Ok(FlowType::Streaming)
1200 }
1201
1202 #[tracing::instrument(skip_all)]
1203 pub async fn create_view(
1204 &self,
1205 create_view: CreateView,
1206 ctx: QueryContextRef,
1207 ) -> Result<TableRef> {
1208 let logical_plan = match &*create_view.query {
1210 Statement::Query(query) => {
1211 self.plan(
1212 &QueryStatement::Sql(Statement::Query(query.clone())),
1213 ctx.clone(),
1214 )
1215 .await?
1216 }
1217 Statement::Tql(query) => {
1218 let mut view_ctx = ctx.as_ref().clone();
1220 view_ctx.set_extension(NATIVE_PROMQL_EXTENSION_KEY, "false");
1221 self.plan_tql(query.clone(), &Arc::new(view_ctx)).await?
1222 }
1223 _ => {
1224 return InvalidViewStmtSnafu {}.fail();
1225 }
1226 };
1227 let definition = create_view.to_string();
1229
1230 let schema: Schema = logical_plan
1233 .schema()
1234 .clone()
1235 .try_into()
1236 .context(ConvertSchemaSnafu)?;
1237 let plan_columns: Vec<_> = schema
1238 .column_schemas()
1239 .iter()
1240 .map(|c| c.name.clone())
1241 .collect();
1242
1243 let columns: Vec<_> = create_view
1244 .columns
1245 .iter()
1246 .map(|ident| ident.to_string())
1247 .collect();
1248
1249 if !columns.is_empty() {
1251 ensure!(
1252 columns.len() == plan_columns.len(),
1253 error::ViewColumnsMismatchSnafu {
1254 view_name: create_view.name.to_string(),
1255 expected: plan_columns.len(),
1256 actual: columns.len(),
1257 }
1258 );
1259 }
1260
1261 let (table_names, plan) = extract_and_rewrite_full_table_names(logical_plan, ctx.clone())
1264 .context(ExtractTableNamesSnafu)?;
1265
1266 let table_names = table_names.into_iter().map(|t| t.into()).collect();
1267
1268 let encoded_plan = DFLogicalSubstraitConvertor
1275 .encode(&plan, DefaultSerializer)
1276 .context(SubstraitCodecSnafu)?;
1277
1278 let expr = expr_helper::to_create_view_expr(
1279 create_view,
1280 encoded_plan.to_vec(),
1281 table_names,
1282 columns,
1283 plan_columns,
1284 definition,
1285 ctx.clone(),
1286 )?;
1287
1288 self.create_view_by_expr(expr, ctx).await
1290 }
1291
1292 pub async fn create_view_by_expr(
1293 &self,
1294 expr: CreateViewExpr,
1295 ctx: QueryContextRef,
1296 ) -> Result<TableRef> {
1297 ensure_table_definition_writable(&expr.schema_name, &expr.view_name)?;
1300 ensure! {
1301 !(expr.create_if_not_exists & expr.or_replace),
1302 InvalidSqlSnafu {
1303 err_msg: "syntax error Create Or Replace and If Not Exist cannot be used together",
1304 }
1305 };
1306 let _timer = crate::metrics::DIST_CREATE_VIEW.start_timer();
1307
1308 let schema_exists = self
1309 .table_metadata_manager
1310 .schema_manager()
1311 .exists(SchemaNameKey::new(&expr.catalog_name, &expr.schema_name))
1312 .await
1313 .context(TableMetadataManagerSnafu)?;
1314
1315 ensure!(
1316 schema_exists,
1317 SchemaNotFoundSnafu {
1318 schema_info: &expr.schema_name,
1319 }
1320 );
1321
1322 if let Some(table) = self
1324 .catalog_manager
1325 .table(
1326 &expr.catalog_name,
1327 &expr.schema_name,
1328 &expr.view_name,
1329 Some(&ctx),
1330 )
1331 .await
1332 .context(CatalogSnafu)?
1333 {
1334 let table_type = table.table_info().table_type;
1335
1336 match (table_type, expr.create_if_not_exists, expr.or_replace) {
1337 (TableType::View, true, false) => {
1338 return Ok(table);
1339 }
1340 (TableType::View, false, false) => {
1341 return ViewAlreadyExistsSnafu {
1342 name: format_full_table_name(
1343 &expr.catalog_name,
1344 &expr.schema_name,
1345 &expr.view_name,
1346 ),
1347 }
1348 .fail();
1349 }
1350 (TableType::View, _, true) => {
1351 }
1353 _ => {
1354 return TableAlreadyExistsSnafu {
1355 table: format_full_table_name(
1356 &expr.catalog_name,
1357 &expr.schema_name,
1358 &expr.view_name,
1359 ),
1360 }
1361 .fail();
1362 }
1363 }
1364 }
1365
1366 ensure!(
1367 NAME_PATTERN_REG.is_match(&expr.view_name),
1368 InvalidViewNameSnafu {
1369 name: expr.view_name.clone(),
1370 }
1371 );
1372
1373 let view_name = TableName::new(&expr.catalog_name, &expr.schema_name, &expr.view_name);
1374
1375 let mut view_info = TableInfo {
1376 ident: metadata::TableIdent {
1377 table_id: 0,
1379 version: 0,
1380 },
1381 name: expr.view_name.clone(),
1382 desc: None,
1383 catalog_name: expr.catalog_name.clone(),
1384 schema_name: expr.schema_name.clone(),
1385 meta: TableMeta::empty(),
1387 table_type: TableType::View,
1388 };
1389
1390 let executor_context = to_executor_context(ctx, TriggerReason::Manual);
1391 let request = SubmitDdlTaskRequest::new(DdlTask::new_create_view(expr, view_info.clone()));
1392
1393 let resp = self
1394 .procedure_executor
1395 .submit_ddl_task(executor_context, request)
1396 .await
1397 .context(error::ExecuteDdlSnafu)?;
1398
1399 debug!(
1400 "Submit creating view '{view_name}' task response: {:?}",
1401 resp
1402 );
1403
1404 let view_id = resp
1405 .table_ids
1406 .into_iter()
1407 .next()
1408 .context(error::UnexpectedSnafu {
1409 violated: "expected table_id",
1410 })?;
1411 info!("Successfully created view '{view_name}' with view id {view_id}");
1412
1413 view_info.ident.table_id = view_id;
1414
1415 let view_info = Arc::new(view_info);
1416
1417 let table = DistTable::table(view_info);
1418
1419 self.cache_invalidator
1421 .invalidate(
1422 &Context::default(),
1423 &[
1424 CacheIdent::TableId(view_id),
1425 CacheIdent::TableName(view_name.clone()),
1426 ],
1427 )
1428 .await
1429 .context(error::InvalidateTableCacheSnafu)?;
1430
1431 Ok(table)
1432 }
1433
1434 #[tracing::instrument(skip_all)]
1435 pub async fn drop_flow(
1436 &self,
1437 catalog_name: String,
1438 flow_name: String,
1439 drop_if_exists: bool,
1440 query_context: QueryContextRef,
1441 ) -> Result<Output> {
1442 if let Some(flow) = self
1443 .flow_metadata_manager
1444 .flow_name_manager()
1445 .get(&catalog_name, &flow_name)
1446 .await
1447 .context(error::TableMetadataManagerSnafu)?
1448 {
1449 let flow_id = flow.flow_id();
1450 let task = DropFlowTask {
1451 catalog_name,
1452 flow_name,
1453 flow_id,
1454 drop_if_exists,
1455 };
1456 self.drop_flow_procedure(task, query_context).await?;
1457
1458 Ok(Output::new_with_affected_rows(0))
1459 } else if drop_if_exists {
1460 Ok(Output::new_with_affected_rows(0))
1461 } else {
1462 FlowNotFoundSnafu {
1463 flow_name: format_full_flow_name(&catalog_name, &flow_name),
1464 }
1465 .fail()
1466 }
1467 }
1468
1469 async fn drop_flow_procedure(
1470 &self,
1471 expr: DropFlowTask,
1472 query_context: QueryContextRef,
1473 ) -> Result<SubmitDdlTaskResponse> {
1474 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1475 let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_flow(expr));
1476
1477 self.procedure_executor
1478 .submit_ddl_task(executor_context, request)
1479 .await
1480 .context(error::ExecuteDdlSnafu)
1481 }
1482
1483 #[cfg(feature = "enterprise")]
1484 #[tracing::instrument(skip_all)]
1485 pub(super) async fn drop_trigger(
1486 &self,
1487 catalog_name: String,
1488 trigger_name: String,
1489 drop_if_exists: bool,
1490 query_context: QueryContextRef,
1491 ) -> Result<Output> {
1492 let task = DropTriggerTask {
1493 catalog_name,
1494 trigger_name,
1495 drop_if_exists,
1496 };
1497 self.drop_trigger_procedure(task, query_context).await?;
1498 Ok(Output::new_with_affected_rows(0))
1499 }
1500
1501 #[cfg(feature = "enterprise")]
1502 async fn drop_trigger_procedure(
1503 &self,
1504 expr: DropTriggerTask,
1505 query_context: QueryContextRef,
1506 ) -> Result<SubmitDdlTaskResponse> {
1507 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1508 let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_trigger(expr));
1509
1510 self.procedure_executor
1511 .submit_ddl_task(executor_context, request)
1512 .await
1513 .context(error::ExecuteDdlSnafu)
1514 }
1515
1516 #[tracing::instrument(skip_all)]
1518 pub async fn drop_view(
1519 &self,
1520 catalog: String,
1521 schema: String,
1522 view: String,
1523 drop_if_exists: bool,
1524 query_context: QueryContextRef,
1525 ) -> Result<Output> {
1526 let view_info = if let Some(view) = self
1527 .catalog_manager
1528 .table(&catalog, &schema, &view, None)
1529 .await
1530 .context(CatalogSnafu)?
1531 {
1532 view.table_info()
1533 } else if drop_if_exists {
1534 return Ok(Output::new_with_affected_rows(0));
1536 } else {
1537 return TableNotFoundSnafu {
1538 table_name: format_full_table_name(&catalog, &schema, &view),
1539 }
1540 .fail();
1541 };
1542
1543 ensure!(
1545 view_info.table_type == TableType::View,
1546 error::InvalidViewSnafu {
1547 msg: "not a view",
1548 view_name: format_full_table_name(&catalog, &schema, &view),
1549 }
1550 );
1551
1552 let view_id = view_info.table_id();
1553 let view_name = TableName::new(&catalog, &schema, &view);
1554
1555 let task = DropViewTask {
1556 catalog,
1557 schema,
1558 view,
1559 view_id,
1560 drop_if_exists,
1561 };
1562
1563 self.drop_view_procedure(task, query_context).await?;
1564
1565 self.cache_invalidator
1567 .invalidate(
1568 &Context::default(),
1569 &[
1570 CacheIdent::TableId(view_id),
1571 CacheIdent::TableName(view_name),
1572 ],
1573 )
1574 .await
1575 .context(error::InvalidateTableCacheSnafu)?;
1576
1577 Ok(Output::new_with_affected_rows(0))
1578 }
1579
1580 async fn drop_view_procedure(
1582 &self,
1583 expr: DropViewTask,
1584 query_context: QueryContextRef,
1585 ) -> Result<SubmitDdlTaskResponse> {
1586 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1587 let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_view(expr));
1588
1589 self.procedure_executor
1590 .submit_ddl_task(executor_context, request)
1591 .await
1592 .context(error::ExecuteDdlSnafu)
1593 }
1594
1595 #[tracing::instrument(skip_all)]
1596 pub async fn alter_logical_tables(
1597 &self,
1598 alter_table_exprs: Vec<AlterTableExpr>,
1599 query_context: QueryContextRef,
1600 trigger_reason: TriggerReason,
1601 ) -> Result<Output> {
1602 let mut tables = Vec::with_capacity(alter_table_exprs.len());
1603 for expr in alter_table_exprs {
1604 let catalog = if expr.catalog_name.is_empty() {
1606 query_context.current_catalog()
1607 } else {
1608 &expr.catalog_name
1609 };
1610 let schema = if expr.schema_name.is_empty() {
1611 query_context.current_schema()
1612 } else {
1613 expr.schema_name.clone()
1614 };
1615 let table_name = &expr.table_name;
1616 let table = self
1617 .catalog_manager
1618 .table(catalog, &schema, table_name, Some(&query_context))
1619 .await
1620 .context(CatalogSnafu)?
1621 .with_context(|| TableNotFoundSnafu {
1622 table_name: format_full_table_name(catalog, &schema, table_name),
1623 })?;
1624 let table_id = table.table_info().ident.table_id;
1625 tables.push((table_id, expr));
1626 }
1627 self.alter_logical_tables_with_ids(tables, query_context, trigger_reason)
1628 .await
1629 }
1630
1631 pub(crate) async fn alter_logical_tables_with_ids(
1633 &self,
1634 tables: Vec<(TableId, AlterTableExpr)>,
1635 query_context: QueryContextRef,
1636 trigger_reason: TriggerReason,
1637 ) -> Result<Output> {
1638 let _timer = crate::metrics::DIST_ALTER_TABLES.start_timer();
1639 ensure!(
1640 !tables.is_empty(),
1641 EmptyDdlExprSnafu {
1642 name: "alter logical tables"
1643 }
1644 );
1645 let mut groups: HashMap<TableId, Vec<(TableId, AlterTableExpr)>> = HashMap::new();
1646 for (table_id, expr) in tables {
1647 let physical_table_id = self
1648 .table_metadata_manager
1649 .table_route_manager()
1650 .get_physical_table_id(table_id)
1651 .await
1652 .context(TableMetadataManagerSnafu)?;
1653 groups
1654 .entry(physical_table_id)
1655 .or_default()
1656 .push((table_id, expr));
1657 }
1658
1659 let mut handles = Vec::with_capacity(groups.len());
1661 for (physical_table_id, exprs) in groups {
1662 let fut = self.alter_logical_tables_procedure(
1663 physical_table_id,
1664 exprs,
1665 query_context.clone(),
1666 trigger_reason,
1667 );
1668 handles.push(fut);
1669 }
1670 let _results = futures::future::try_join_all(handles).await?;
1671
1672 Ok(Output::new_with_affected_rows(0))
1673 }
1674
1675 #[tracing::instrument(skip_all)]
1676 pub async fn drop_table(
1677 &self,
1678 table_name: TableName,
1679 drop_if_exists: bool,
1680 query_context: QueryContextRef,
1681 ) -> Result<Output> {
1682 self.drop_tables(&[table_name], drop_if_exists, query_context)
1684 .await
1685 }
1686
1687 #[tracing::instrument(skip_all)]
1688 pub async fn drop_tables(
1689 &self,
1690 table_names: &[TableName],
1691 drop_if_exists: bool,
1692 query_context: QueryContextRef,
1693 ) -> Result<Output> {
1694 let mut tables = Vec::with_capacity(table_names.len());
1695 for table_name in table_names {
1696 ensure_table_writable(&table_name.schema_name, &table_name.table_name)?;
1697
1698 if let Some(table) = self
1699 .catalog_manager
1700 .table(
1701 &table_name.catalog_name,
1702 &table_name.schema_name,
1703 &table_name.table_name,
1704 Some(&query_context),
1705 )
1706 .await
1707 .context(CatalogSnafu)?
1708 {
1709 tables.push(table.table_info().table_id());
1710 } else if drop_if_exists {
1711 continue;
1713 } else {
1714 return TableNotFoundSnafu {
1715 table_name: table_name.to_string(),
1716 }
1717 .fail();
1718 }
1719 }
1720
1721 for (table_name, table_id) in table_names.iter().zip(tables.into_iter()) {
1722 self.drop_table_procedure(table_name, table_id, drop_if_exists, query_context.clone())
1723 .await?;
1724
1725 self.cache_invalidator
1727 .invalidate(
1728 &Context::default(),
1729 &[
1730 CacheIdent::TableId(table_id),
1731 CacheIdent::TableName(table_name.clone()),
1732 ],
1733 )
1734 .await
1735 .context(error::InvalidateTableCacheSnafu)?;
1736 }
1737 Ok(Output::new_with_affected_rows(0))
1738 }
1739
1740 #[cfg(feature = "enterprise")]
1741 #[tracing::instrument(skip_all)]
1742 pub async fn undrop_table(
1743 &self,
1744 table_name: TableName,
1745 query_context: QueryContextRef,
1746 ) -> Result<Output> {
1747 execute_undrop_table(
1748 &self.table_metadata_manager,
1749 &self.procedure_executor,
1750 &self.cache_invalidator,
1751 table_name,
1752 query_context,
1753 )
1754 .await
1755 }
1756
1757 #[tracing::instrument(skip_all)]
1758 pub async fn drop_database(
1759 &self,
1760 catalog: String,
1761 schema: String,
1762 drop_if_exists: bool,
1763 query_context: QueryContextRef,
1764 ) -> Result<Output> {
1765 ensure!(
1766 !is_readonly_schema(&schema),
1767 SchemaReadOnlySnafu { name: schema }
1768 );
1769
1770 if self
1771 .catalog_manager
1772 .schema_exists(&catalog, &schema, None)
1773 .await
1774 .context(CatalogSnafu)?
1775 {
1776 if schema == query_context.current_schema() {
1777 SchemaInUseSnafu { name: schema }.fail()
1778 } else {
1779 self.drop_database_procedure(catalog, schema, drop_if_exists, query_context)
1780 .await?;
1781
1782 Ok(Output::new_with_affected_rows(0))
1783 }
1784 } else if drop_if_exists {
1785 Ok(Output::new_with_affected_rows(0))
1787 } else {
1788 SchemaNotFoundSnafu {
1789 schema_info: schema,
1790 }
1791 .fail()
1792 }
1793 }
1794
1795 pub async fn truncate_table(
1796 &self,
1797 table_name: TableName,
1798 time_ranges: Vec<(Timestamp, Timestamp)>,
1799 query_context: QueryContextRef,
1800 ) -> Result<Output> {
1801 self.truncate_table_with_options(
1802 table_name,
1803 time_ranges,
1804 DdlSubmitOptions::default(),
1805 query_context,
1806 )
1807 .await
1808 }
1809
1810 #[tracing::instrument(name = "truncate_table", skip_all)]
1812 pub(crate) async fn truncate_table_with_options(
1813 &self,
1814 table_name: TableName,
1815 time_ranges: Vec<(Timestamp, Timestamp)>,
1816 ddl_options: DdlSubmitOptions,
1817 query_context: QueryContextRef,
1818 ) -> Result<Output> {
1819 ensure_table_writable(&table_name.schema_name, &table_name.table_name)?;
1820
1821 let table = self
1822 .catalog_manager
1823 .table(
1824 &table_name.catalog_name,
1825 &table_name.schema_name,
1826 &table_name.table_name,
1827 Some(&query_context),
1828 )
1829 .await
1830 .context(CatalogSnafu)?
1831 .with_context(|| TableNotFoundSnafu {
1832 table_name: table_name.to_string(),
1833 })?;
1834 let table_id = table.table_info().table_id();
1835 let response = self
1836 .truncate_table_procedure(
1837 &table_name,
1838 table_id,
1839 time_ranges,
1840 ddl_options,
1841 query_context.clone(),
1842 )
1843 .await?;
1844
1845 if !ddl_options.wait {
1846 let procedure_id = String::from_utf8_lossy(&response.key).into_owned();
1847 let output = build_procedure_id_output(&procedure_id)?;
1848 query_context.set_last_procedure_id(procedure_id);
1849 return Ok(output);
1850 }
1851
1852 Ok(Output::new_with_affected_rows(0))
1853 }
1854
1855 #[tracing::instrument(skip_all)]
1856 pub async fn alter_table(
1857 &self,
1858 alter_table: AlterTable,
1859 query_context: QueryContextRef,
1860 ) -> Result<Output> {
1861 if matches!(
1862 alter_table.alter_operation(),
1863 AlterTableOperation::Repartition { .. } | AlterTableOperation::Partition { .. }
1864 ) {
1865 let request = expr_helper::to_repartition_request(alter_table, &query_context)?;
1866 return self.repartition_table(request, &query_context).await;
1867 }
1868
1869 let expr = expr_helper::to_alter_table_expr(alter_table, &query_context)?;
1870 self.alter_table_inner(expr, query_context, TriggerReason::Manual)
1871 .await
1872 }
1873
1874 #[tracing::instrument(skip_all)]
1875 pub async fn repartition_table(
1876 &self,
1877 request: RepartitionRequest,
1878 query_context: &QueryContextRef,
1879 ) -> Result<Output> {
1880 ensure_table_definition_writable(&request.schema_name, &request.table_name)?;
1882
1883 let table_ref = TableReference::full(
1884 &request.catalog_name,
1885 &request.schema_name,
1886 &request.table_name,
1887 );
1888 let table = self
1890 .catalog_manager
1891 .table(
1892 &request.catalog_name,
1893 &request.schema_name,
1894 &request.table_name,
1895 Some(query_context),
1896 )
1897 .await
1898 .context(CatalogSnafu)?
1899 .with_context(|| TableNotFoundSnafu {
1900 table_name: table_ref.to_string(),
1901 })?;
1902 let table_id = table.table_info().ident.table_id;
1903 let (physical_table_id, physical_table_route) = self
1905 .table_metadata_manager
1906 .table_route_manager()
1907 .get_physical_table_route(table_id)
1908 .await
1909 .context(TableMetadataManagerSnafu)?;
1910
1911 ensure!(
1912 physical_table_id == table_id,
1913 NotSupportedSnafu {
1914 feat: "REPARTITION on logical tables"
1915 }
1916 );
1917
1918 let table_info = table.table_info();
1919 let existing_partition_columns = table_info.meta.partition_columns().collect::<Vec<_>>();
1920 let column_schemas = table_info.meta.schema.column_schemas();
1921 let target_partition_columns = match &request.source {
1927 RepartitionSource::Partitions {
1928 target_partition_columns,
1929 ..
1930 } => {
1931 ensure!(
1932 !existing_partition_columns.is_empty(),
1933 InvalidPartitionRuleSnafu {
1934 reason: format!(
1935 "table {} does not have partition columns, cannot repartition",
1936 table_ref
1937 )
1938 }
1939 );
1940
1941 if let Some(target_partition_columns) = target_partition_columns {
1942 ensure!(
1943 !target_partition_columns.is_empty(),
1944 InvalidPartitionRuleSnafu {
1945 reason: "ON COLUMNS requires at least one partition column"
1946 }
1947 );
1948 validate_and_collect_partition_columns(
1949 target_partition_columns,
1950 column_schemas,
1951 )?
1952 } else {
1953 existing_partition_columns.clone()
1954 }
1955 }
1956 RepartitionSource::Unpartitioned { partition_columns } => {
1957 ensure!(
1958 !partition_columns.is_empty(),
1959 InvalidPartitionRuleSnafu {
1960 reason: "PARTITION ON COLUMNS requires at least one partition column"
1961 }
1962 );
1963 ensure!(
1964 existing_partition_columns.is_empty(),
1965 InvalidPartitionRuleSnafu {
1966 reason: format!("table {} already has partition columns", table_ref)
1967 }
1968 );
1969 partition_columns
1970 .iter()
1971 .map(|column_name| {
1972 column_schemas
1973 .iter()
1974 .find(|column| &column.name == column_name)
1975 .with_context(|| ColumnNotFoundSnafu { msg: column_name })
1976 })
1977 .collect::<Result<Vec<_>>>()?
1978 }
1979 };
1980
1981 let from_column_name_and_type = column_name_and_type(&existing_partition_columns);
1982 let target_column_name_and_type = column_name_and_type(&target_partition_columns);
1983 let target_partition_column_names = target_partition_columns
1984 .iter()
1985 .map(|column| column.name.clone())
1986 .collect::<Vec<_>>();
1987 let timezone = query_context.timezone();
1988 let from_partition_exprs = match &request.source {
1990 RepartitionSource::Partitions { from_exprs, .. } => from_exprs
1991 .iter()
1992 .map(|expr| convert_one_expr(expr, &from_column_name_and_type, &timezone))
1993 .collect::<Result<Vec<_>>>()?,
1994 RepartitionSource::Unpartitioned { .. } => vec![],
1995 };
1996
1997 let mut into_partition_exprs = request
1998 .into_exprs
1999 .iter()
2000 .map(|expr| convert_one_expr(expr, &target_column_name_and_type, &timezone))
2001 .collect::<Result<Vec<_>>>()?;
2002
2003 if matches!(&request.source, RepartitionSource::Partitions { .. })
2006 && from_partition_exprs.len() > 1
2007 && into_partition_exprs.len() == 1
2008 && let Some(expr) = into_partition_exprs.pop()
2009 {
2010 into_partition_exprs.push(partition::simplify::simplify_merged_partition_expr(expr));
2011 }
2012
2013 let mut existing_partition_exprs =
2015 Vec::with_capacity(physical_table_route.region_routes.len());
2016 for route in &physical_table_route.region_routes {
2017 let expr_json = route.region.partition_expr();
2018 if !expr_json.is_empty() {
2019 match PartitionExpr::from_json_str(&expr_json) {
2020 Ok(Some(expr)) => existing_partition_exprs.push(expr),
2021 Ok(None) => {
2022 }
2024 Err(e) => {
2025 return Err(e).context(DeserializePartitionExprSnafu);
2026 }
2027 }
2028 }
2029 }
2030
2031 if matches!(&request.source, RepartitionSource::Partitions { .. }) {
2034 for from_expr in &from_partition_exprs {
2035 ensure!(
2036 existing_partition_exprs.contains(from_expr),
2037 InvalidPartitionRuleSnafu {
2038 reason: format!(
2039 "partition expression '{}' does not exist in table {}",
2040 from_expr, table_ref
2041 )
2042 }
2043 );
2044 }
2045 }
2046
2047 let new_partition_exprs: Vec<PartitionExpr> = match &request.source {
2050 RepartitionSource::Partitions { .. } => existing_partition_exprs
2051 .into_iter()
2052 .filter(|expr| !from_partition_exprs.contains(expr))
2053 .chain(into_partition_exprs.clone().into_iter())
2054 .collect(),
2055 RepartitionSource::Unpartitioned { .. } => into_partition_exprs.clone(),
2056 };
2057 ensure_partition_expr_columns_in_target(
2058 &new_partition_exprs,
2059 &target_partition_column_names.iter().collect(),
2060 )?;
2061 let new_partition_exprs_len = new_partition_exprs.len();
2062 let from_partition_exprs_len = from_partition_exprs.len();
2063
2064 let _ = MultiDimPartitionRule::try_new(
2066 target_partition_column_names,
2067 vec![],
2068 new_partition_exprs,
2069 true,
2070 )
2071 .context(InvalidPartitionSnafu)?;
2072
2073 let ddl_options = parse_ddl_options(&request.options)?;
2074 let serialize_exprs = |exprs: Vec<PartitionExpr>| -> Result<Vec<String>> {
2075 let mut json_exprs = Vec::with_capacity(exprs.len());
2076 for expr in exprs {
2077 json_exprs.push(expr.as_json_str().context(SerializePartitionExprSnafu)?);
2078 }
2079 Ok(json_exprs)
2080 };
2081 let from_partition_exprs_json = serialize_exprs(from_partition_exprs)?;
2082 let into_partition_exprs_json = serialize_exprs(into_partition_exprs)?;
2083 let source = match &request.source {
2084 RepartitionSource::Partitions {
2085 target_partition_columns,
2086 ..
2087 } => Source::PartitionExprs(PartitionedSource {
2088 exprs: from_partition_exprs_json,
2089 target_partition_columns: target_partition_columns
2090 .clone()
2091 .map(|columns| TargetPartitionColumns { columns }),
2092 }),
2093 RepartitionSource::Unpartitioned { partition_columns } => {
2094 Source::Unpartitioned(UnpartitionedSource {
2095 partition_columns: partition_columns.clone(),
2096 })
2097 }
2098 };
2099 let repartition = Repartition {
2100 into_partition_exprs: into_partition_exprs_json,
2101 source: Some(source),
2102 ..Default::default()
2103 };
2104 let executor_context = to_executor_context(query_context.clone(), TriggerReason::Manual);
2105 let mut req = SubmitDdlTaskRequest::new(DdlTask::new_alter_table(AlterTableExpr {
2106 catalog_name: request.catalog_name.clone(),
2107 schema_name: request.schema_name.clone(),
2108 table_name: request.table_name.clone(),
2109 kind: Some(Kind::Repartition(repartition)),
2110 }));
2111 req.wait = ddl_options.wait;
2112 req.timeout = ddl_options.timeout;
2113
2114 info!(
2115 "Submitting repartition task for table {} (table_id={}), from {} to {} partitions, timeout: {:?}, wait: {}",
2116 table_ref,
2117 table_id,
2118 from_partition_exprs_len,
2119 new_partition_exprs_len,
2120 ddl_options.timeout,
2121 ddl_options.wait
2122 );
2123
2124 let response = self
2125 .procedure_executor
2126 .submit_ddl_task(executor_context, req)
2127 .await
2128 .context(error::ExecuteDdlSnafu)?;
2129
2130 if !ddl_options.wait {
2131 let procedure_id = String::from_utf8_lossy(&response.key).into_owned();
2132 let output = build_procedure_id_output(&procedure_id)?;
2133 query_context.set_last_procedure_id(procedure_id);
2134 return Ok(output);
2135 }
2136
2137 let invalidate_keys = vec![
2139 CacheIdent::TableId(table_id),
2140 CacheIdent::TableName(TableName::new(
2141 request.catalog_name,
2142 request.schema_name,
2143 request.table_name,
2144 )),
2145 ];
2146
2147 self.cache_invalidator
2149 .invalidate(&Context::default(), &invalidate_keys)
2150 .await
2151 .context(error::InvalidateTableCacheSnafu)?;
2152
2153 Ok(Output::new_with_affected_rows(0))
2154 }
2155
2156 #[tracing::instrument(skip_all)]
2157 pub async fn alter_table_inner(
2158 &self,
2159 expr: AlterTableExpr,
2160 query_context: QueryContextRef,
2161 trigger_reason: TriggerReason,
2162 ) -> Result<Output> {
2163 ensure_table_definition_writable(&expr.schema_name, &expr.table_name)?;
2164
2165 let catalog_name = if expr.catalog_name.is_empty() {
2166 DEFAULT_CATALOG_NAME.to_string()
2167 } else {
2168 expr.catalog_name.clone()
2169 };
2170
2171 let schema_name = if expr.schema_name.is_empty() {
2172 DEFAULT_SCHEMA_NAME.to_string()
2173 } else {
2174 expr.schema_name.clone()
2175 };
2176
2177 let table_name = expr.table_name.clone();
2178
2179 let table = self
2180 .catalog_manager
2181 .table(
2182 &catalog_name,
2183 &schema_name,
2184 &table_name,
2185 Some(&query_context),
2186 )
2187 .await
2188 .context(CatalogSnafu)?
2189 .with_context(|| TableNotFoundSnafu {
2190 table_name: format_full_table_name(&catalog_name, &schema_name, &table_name),
2191 })?;
2192
2193 let table_id = table.table_info().ident.table_id;
2194 let need_alter = verify_alter(table_id, table.table_info(), expr.clone())?;
2195 if !need_alter {
2196 return Ok(Output::new_with_affected_rows(0));
2197 }
2198 info!(
2199 "Table info before alter is {:?}, expr: {:?}",
2200 table.table_info(),
2201 expr
2202 );
2203
2204 let physical_table_id = self
2205 .table_metadata_manager
2206 .table_route_manager()
2207 .get_physical_table_id(table_id)
2208 .await
2209 .context(TableMetadataManagerSnafu)?;
2210
2211 let executor_context = to_executor_context(query_context, trigger_reason);
2212
2213 let (req, invalidate_keys) = if physical_table_id == table_id {
2214 let req = SubmitDdlTaskRequest::new(DdlTask::new_alter_table(expr));
2216
2217 let invalidate_keys = vec![
2218 CacheIdent::TableId(table_id),
2219 CacheIdent::TableName(TableName::new(catalog_name, schema_name, table_name)),
2220 ];
2221
2222 (req, invalidate_keys)
2223 } else {
2224 if let Some(Kind::SetTableOptions(options)) = &expr.kind {
2228 ensure!(
2229 !options
2230 .table_options
2231 .iter()
2232 .any(|option| option.key == TTL_KEY && is_retention_ttl(&option.value)),
2233 RetentionTtlOnLogicalTableSnafu
2234 );
2235 }
2236 let annotation_alter = match expr.kind.as_ref() {
2239 Some(kind) => common_grpc_expr::annotation_alter_family(kind)
2240 .context(AlterExprToRequestSnafu)?
2241 .is_some_and(|family| family.allows_logical_tables()),
2242 None => false,
2243 };
2244 let task = if annotation_alter {
2245 DdlTask::new_alter_table(expr)
2246 } else {
2247 DdlTask::new_alter_logical_tables(vec![expr])
2248 };
2249 let req = SubmitDdlTaskRequest::new(task);
2250
2251 let mut invalidate_keys = vec![
2252 CacheIdent::TableId(physical_table_id),
2253 CacheIdent::TableId(table_id),
2254 CacheIdent::TableName(TableName::new(catalog_name, schema_name, table_name)),
2255 ];
2256
2257 let physical_table = self
2258 .table_metadata_manager
2259 .table_info_manager()
2260 .get(physical_table_id)
2261 .await
2262 .context(TableMetadataManagerSnafu)?
2263 .map(|x| x.into_inner());
2264 if let Some(physical_table) = physical_table {
2265 let physical_table_name = TableName::new(
2266 physical_table.table_info.catalog_name,
2267 physical_table.table_info.schema_name,
2268 physical_table.table_info.name,
2269 );
2270 invalidate_keys.push(CacheIdent::TableName(physical_table_name));
2271 }
2272
2273 (req, invalidate_keys)
2274 };
2275
2276 self.procedure_executor
2277 .submit_ddl_task(executor_context, req)
2278 .await
2279 .context(error::ExecuteDdlSnafu)?;
2280
2281 self.cache_invalidator
2283 .invalidate(&Context::default(), &invalidate_keys)
2284 .await
2285 .context(error::InvalidateTableCacheSnafu)?;
2286
2287 Ok(Output::new_with_affected_rows(0))
2288 }
2289
2290 #[cfg(feature = "enterprise")]
2291 #[tracing::instrument(skip_all)]
2292 pub async fn alter_trigger(
2293 &self,
2294 _alter_expr: AlterTrigger,
2295 _query_context: QueryContextRef,
2296 ) -> Result<Output> {
2297 crate::error::NotSupportedSnafu {
2298 feat: "alter trigger",
2299 }
2300 .fail()
2301 }
2302
2303 #[tracing::instrument(skip_all)]
2304 pub async fn alter_database(
2305 &self,
2306 alter_expr: AlterDatabase,
2307 query_context: QueryContextRef,
2308 ) -> Result<Output> {
2309 let alter_expr = expr_helper::to_alter_database_expr(alter_expr, &query_context)?;
2310 self.alter_database_inner(alter_expr, query_context).await
2311 }
2312
2313 #[tracing::instrument(skip_all)]
2314 pub async fn alter_database_inner(
2315 &self,
2316 alter_expr: AlterDatabaseExpr,
2317 query_context: QueryContextRef,
2318 ) -> Result<Output> {
2319 ensure!(
2320 !is_readonly_schema(&alter_expr.schema_name),
2321 SchemaReadOnlySnafu {
2322 name: query_context.current_schema().clone()
2323 }
2324 );
2325
2326 let exists = self
2327 .catalog_manager
2328 .schema_exists(&alter_expr.catalog_name, &alter_expr.schema_name, None)
2329 .await
2330 .context(CatalogSnafu)?;
2331 ensure!(
2332 exists,
2333 SchemaNotFoundSnafu {
2334 schema_info: alter_expr.schema_name,
2335 }
2336 );
2337
2338 let cache_ident = [CacheIdent::SchemaName(SchemaName {
2339 catalog_name: alter_expr.catalog_name.clone(),
2340 schema_name: alter_expr.schema_name.clone(),
2341 })];
2342
2343 self.alter_database_procedure(alter_expr, query_context)
2344 .await?;
2345
2346 self.cache_invalidator
2348 .invalidate(&Context::default(), &cache_ident)
2349 .await
2350 .context(error::InvalidateTableCacheSnafu)?;
2351
2352 Ok(Output::new_with_affected_rows(0))
2353 }
2354
2355 async fn create_table_procedure(
2356 &self,
2357 create_table: CreateTableExpr,
2358 partitions: Vec<PartitionExpr>,
2359 table_info: TableInfo,
2360 query_context: QueryContextRef,
2361 trigger_reason: TriggerReason,
2362 ) -> Result<SubmitDdlTaskResponse> {
2363 let partitions = partitions
2364 .into_iter()
2365 .map(|expr| expr.as_pb_partition().context(PartitionExprToPbSnafu))
2366 .collect::<Result<Vec<_>>>()?;
2367
2368 let executor_context = to_executor_context_with_origin_frontend(
2369 query_context,
2370 &self.origin_frontend_addr,
2371 trigger_reason,
2372 );
2373 let request = SubmitDdlTaskRequest::new(DdlTask::new_create_table(
2374 create_table,
2375 partitions,
2376 table_info,
2377 ));
2378
2379 self.procedure_executor
2380 .submit_ddl_task(executor_context, request)
2381 .await
2382 .context(error::ExecuteDdlSnafu)
2383 }
2384
2385 async fn create_logical_tables_procedure(
2386 &self,
2387 tables_data: Vec<(CreateTableExpr, TableInfo)>,
2388 query_context: QueryContextRef,
2389 trigger_reason: TriggerReason,
2390 ) -> Result<SubmitDdlTaskResponse> {
2391 let executor_context = to_executor_context_with_origin_frontend(
2392 query_context,
2393 &self.origin_frontend_addr,
2394 trigger_reason,
2395 );
2396 let request = SubmitDdlTaskRequest::new(DdlTask::new_create_logical_tables(tables_data));
2397
2398 self.procedure_executor
2399 .submit_ddl_task(executor_context, request)
2400 .await
2401 .context(error::ExecuteDdlSnafu)
2402 }
2403
2404 async fn alter_logical_tables_procedure(
2405 &self,
2406 physical_table_id: TableId,
2407 tables_data: Vec<(TableId, AlterTableExpr)>,
2408 query_context: QueryContextRef,
2409 trigger_reason: TriggerReason,
2410 ) -> Result<SubmitDdlTaskResponse> {
2411 let mut invalidate_keys = vec![CacheIdent::TableId(physical_table_id)];
2412 if let Some(physical) = self
2413 .table_metadata_manager
2414 .table_info_manager()
2415 .get(physical_table_id)
2416 .await
2417 .context(TableMetadataManagerSnafu)?
2418 {
2419 invalidate_keys.push(CacheIdent::TableName(physical.table_name()));
2420 }
2421 for (id, expr) in &tables_data {
2422 invalidate_keys.push(CacheIdent::TableId(*id));
2423 invalidate_keys.push(CacheIdent::TableName(TableName::new(
2424 &expr.catalog_name,
2425 &expr.schema_name,
2426 &expr.table_name,
2427 )));
2428 }
2429 let mut executor_context = to_executor_context(query_context, trigger_reason);
2430 let (table_ids, exprs): (Vec<_>, Vec<_>) = tables_data.into_iter().unzip();
2431 if let Some(context) = &mut executor_context.query_context {
2432 context.extensions.insert(
2433 ALTER_LOGICAL_TABLE_IDS_EXTENSION_KEY.to_string(),
2434 table_ids
2435 .iter()
2436 .map(ToString::to_string)
2437 .collect::<Vec<_>>()
2438 .join(","),
2439 );
2440 }
2441 let request = SubmitDdlTaskRequest::new(DdlTask::new_alter_logical_tables(exprs));
2442
2443 let response = self
2444 .procedure_executor
2445 .submit_ddl_task(executor_context, request)
2446 .await
2447 .context(error::ExecuteDdlSnafu)?;
2448 self.cache_invalidator
2450 .invalidate(&Context::default(), &invalidate_keys)
2451 .await
2452 .context(error::InvalidateTableCacheSnafu)?;
2453 Ok(response)
2454 }
2455
2456 async fn drop_table_procedure(
2457 &self,
2458 table_name: &TableName,
2459 table_id: TableId,
2460 drop_if_exists: bool,
2461 query_context: QueryContextRef,
2462 ) -> Result<SubmitDdlTaskResponse> {
2463 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2464 let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_table(
2465 table_name.catalog_name.clone(),
2466 table_name.schema_name.clone(),
2467 table_name.table_name.clone(),
2468 table_id,
2469 drop_if_exists,
2470 ));
2471
2472 self.procedure_executor
2473 .submit_ddl_task(executor_context, request)
2474 .await
2475 .context(error::ExecuteDdlSnafu)
2476 }
2477
2478 async fn drop_database_procedure(
2479 &self,
2480 catalog: String,
2481 schema: String,
2482 drop_if_exists: bool,
2483 query_context: QueryContextRef,
2484 ) -> Result<SubmitDdlTaskResponse> {
2485 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2486 let request =
2487 SubmitDdlTaskRequest::new(DdlTask::new_drop_database(catalog, schema, drop_if_exists));
2488
2489 self.procedure_executor
2490 .submit_ddl_task(executor_context, request)
2491 .await
2492 .context(error::ExecuteDdlSnafu)
2493 }
2494
2495 async fn alter_database_procedure(
2496 &self,
2497 alter_expr: AlterDatabaseExpr,
2498 query_context: QueryContextRef,
2499 ) -> Result<SubmitDdlTaskResponse> {
2500 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2501 let request = SubmitDdlTaskRequest::new(DdlTask::new_alter_database(alter_expr));
2502
2503 self.procedure_executor
2504 .submit_ddl_task(executor_context, request)
2505 .await
2506 .context(error::ExecuteDdlSnafu)
2507 }
2508
2509 async fn truncate_table_procedure(
2510 &self,
2511 table_name: &TableName,
2512 table_id: TableId,
2513 time_ranges: Vec<(Timestamp, Timestamp)>,
2514 ddl_options: DdlSubmitOptions,
2515 query_context: QueryContextRef,
2516 ) -> Result<SubmitDdlTaskResponse> {
2517 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2518 let mut request = SubmitDdlTaskRequest::new(DdlTask::new_truncate_table(
2519 table_name.catalog_name.clone(),
2520 table_name.schema_name.clone(),
2521 table_name.table_name.clone(),
2522 table_id,
2523 time_ranges,
2524 ));
2525 request.wait = ddl_options.wait;
2526 request.timeout = ddl_options.timeout;
2527
2528 self.procedure_executor
2529 .submit_ddl_task(executor_context, request)
2530 .await
2531 .context(error::ExecuteDdlSnafu)
2532 }
2533
2534 #[tracing::instrument(skip_all)]
2535 pub async fn create_database(
2536 &self,
2537 database: &str,
2538 create_if_not_exists: bool,
2539 options: HashMap<String, String>,
2540 query_context: QueryContextRef,
2541 ) -> Result<Output> {
2542 let catalog = query_context.current_catalog();
2543 ensure!(
2544 NAME_PATTERN_REG.is_match(catalog),
2545 error::UnexpectedSnafu {
2546 violated: format!("Invalid catalog name: {}", catalog)
2547 }
2548 );
2549
2550 ensure!(
2551 NAME_PATTERN_REG.is_match(database),
2552 error::UnexpectedSnafu {
2553 violated: format!("Invalid database name: {}", database)
2554 }
2555 );
2556
2557 #[cfg(feature = "enterprise")]
2558 let creator = match &self.create_database_handler {
2559 Some(handler) => handler.creator(&query_context).context(ExternalSnafu)?,
2560 None => None,
2561 };
2562 #[cfg(not(feature = "enterprise"))]
2563 let creator = None;
2564
2565 let output = if !self
2566 .catalog_manager
2567 .schema_exists(catalog, database, None)
2568 .await
2569 .context(CatalogSnafu)?
2570 && !self.catalog_manager.is_reserved_schema_name(database)
2571 {
2572 self.create_database_procedure(
2573 catalog.to_string(),
2574 database.to_string(),
2575 create_if_not_exists,
2576 options,
2577 query_context.clone(),
2578 creator,
2579 )
2580 .await?;
2581
2582 Output::new_with_affected_rows(1)
2583 } else if create_if_not_exists {
2584 Output::new_with_affected_rows(1)
2585 } else {
2586 return error::SchemaExistsSnafu { name: database }.fail();
2587 };
2588
2589 #[cfg(feature = "enterprise")]
2590 if let Some(handler) = &self.create_database_handler {
2591 handler
2592 .refresh_current_user(&query_context)
2593 .await
2594 .context(ExternalSnafu)?;
2595 }
2596
2597 Ok(output)
2598 }
2599
2600 async fn create_database_procedure(
2601 &self,
2602 catalog: String,
2603 database: String,
2604 create_if_not_exists: bool,
2605 options: HashMap<String, String>,
2606 query_context: QueryContextRef,
2607 creator: Option<CreatorGrantIntent>,
2608 ) -> Result<SubmitDdlTaskResponse> {
2609 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2610 let request = SubmitDdlTaskRequest::new(DdlTask::new_create_database(
2611 catalog,
2612 database,
2613 create_if_not_exists,
2614 options,
2615 creator,
2616 ));
2617
2618 self.procedure_executor
2619 .submit_ddl_task(executor_context, request)
2620 .await
2621 .context(error::ExecuteDdlSnafu)
2622 }
2623}
2624
2625pub fn parse_partitions(
2627 create_table: &CreateTableExpr,
2628 partitions: Option<Partitions>,
2629 query_ctx: &QueryContextRef,
2630) -> Result<(Vec<PartitionExpr>, Vec<String>)> {
2631 let partition_columns = find_partition_columns(&partitions)?;
2634 let partition_exprs =
2635 find_partition_entries(create_table, &partitions, &partition_columns, query_ctx)?;
2636
2637 let exprs = partition_exprs.clone();
2639 MultiDimPartitionRule::try_new(partition_columns.clone(), vec![], exprs, true)
2640 .context(InvalidPartitionSnafu)?;
2641
2642 Ok((partition_exprs, partition_columns))
2643}
2644
2645fn parse_partitions_for_logical_validation(
2646 create_table: &CreateTableExpr,
2647 partitions: &Partitions,
2648 query_ctx: &QueryContextRef,
2649) -> Result<(Vec<String>, Vec<PartitionExpr>)> {
2650 let partition_columns = partitions
2651 .column_list
2652 .iter()
2653 .map(|ident| ident.value.clone())
2654 .collect::<Vec<_>>();
2655
2656 let column_name_and_type = partition_columns
2657 .iter()
2658 .map(|pc| {
2659 let column = create_table
2660 .column_defs
2661 .iter()
2662 .find(|c| &c.name == pc)
2663 .context(ColumnNotFoundSnafu { msg: pc.clone() })?;
2664 let column_name = &column.name;
2665 let data_type = ConcreteDataType::from(
2666 ColumnDataTypeWrapper::try_new(column.data_type, column.datatype_extension.clone())
2667 .context(ColumnDataTypeSnafu)?,
2668 );
2669 Ok((column_name, data_type))
2670 })
2671 .collect::<Result<HashMap<_, _>>>()?;
2672
2673 let mut partition_exprs = Vec::with_capacity(partitions.exprs.len());
2674 for expr in &partitions.exprs {
2675 let partition_expr = convert_one_expr(expr, &column_name_and_type, &query_ctx.timezone())?;
2676 partition_exprs.push(partition_expr);
2677 }
2678
2679 MultiDimPartitionRule::try_new(
2680 partition_columns.clone(),
2681 vec![],
2682 partition_exprs.clone(),
2683 true,
2684 )
2685 .context(InvalidPartitionSnafu)?;
2686
2687 Ok((partition_columns, partition_exprs))
2688}
2689
2690pub fn verify_alter(
2696 table_id: TableId,
2697 table_info: Arc<TableInfo>,
2698 expr: AlterTableExpr,
2699) -> Result<bool> {
2700 let request: AlterTableRequest =
2701 common_grpc_expr::alter_expr_to_request(table_id, expr, Some(&table_info.meta))
2702 .context(AlterExprToRequestSnafu)?;
2703
2704 let AlterTableRequest {
2705 table_name,
2706 alter_kind,
2707 ..
2708 } = &request;
2709
2710 if let AlterKind::RenameTable { new_table_name } = alter_kind {
2711 ensure!(
2712 NAME_PATTERN_REG.is_match(new_table_name),
2713 error::UnexpectedSnafu {
2714 violated: format!("Invalid table name: {}", new_table_name)
2715 }
2716 );
2717 ensure_table_definition_writable(&table_info.schema_name, new_table_name)?;
2720 } else if let AlterKind::AddColumns { columns } = alter_kind {
2721 let column_names: HashSet<_> = table_info
2724 .meta
2725 .schema
2726 .column_schemas()
2727 .iter()
2728 .map(|schema| &schema.name)
2729 .collect();
2730 if columns.iter().all(|column| {
2731 column_names.contains(&column.column_schema.name) && column.add_if_not_exists
2732 }) {
2733 return Ok(false);
2734 }
2735 }
2736
2737 let new_meta = table_info
2738 .meta
2739 .builder_with_alter_kind(table_name, &request.alter_kind)
2740 .context(error::TableSnafu)?
2741 .build()
2742 .context(error::BuildTableMetaSnafu { table_name })?;
2743
2744 validate_json2_primary_keys(&new_meta.schema, &new_meta.primary_key_indices)?;
2745
2746 Ok(true)
2747}
2748
2749pub(crate) fn is_retention_ttl(ttl: &str) -> bool {
2752 TimeToLive::from_humantime_or_str(ttl).is_ok_and(|ttl| !ttl.is_instant())
2753}
2754
2755pub fn create_table_info(
2756 create_table: &CreateTableExpr,
2757 partition_columns: Vec<String>,
2758) -> Result<TableInfo> {
2759 let mut column_schemas = Vec::with_capacity(create_table.column_defs.len());
2760 let mut column_name_to_index_map = HashMap::new();
2761
2762 for (idx, column) in create_table.column_defs.iter().enumerate() {
2763 let schema =
2764 column_def::try_as_column_schema(column).context(error::InvalidColumnDefSnafu {
2765 column: &column.name,
2766 })?;
2767 let schema = schema.with_time_index(column.name == create_table.time_index);
2768
2769 column_schemas.push(schema);
2770 let _ = column_name_to_index_map.insert(column.name.clone(), idx);
2771 }
2772
2773 let next_column_id = column_schemas.len() as u32;
2774 let schema = Arc::new(Schema::try_new(column_schemas).context(ConvertSchemaSnafu)?);
2775
2776 let primary_key_indices = create_table
2777 .primary_keys
2778 .iter()
2779 .map(|name| {
2780 column_name_to_index_map
2781 .get(name)
2782 .cloned()
2783 .context(ColumnNotFoundSnafu { msg: name })
2784 })
2785 .collect::<Result<Vec<_>>>()?;
2786
2787 let partition_key_indices = partition_columns
2788 .into_iter()
2789 .map(|col_name| {
2790 column_name_to_index_map
2791 .get(&col_name)
2792 .cloned()
2793 .context(ColumnNotFoundSnafu { msg: col_name })
2794 })
2795 .collect::<Result<Vec<_>>>()?;
2796
2797 let mut table_options = TableOptions::try_from_iter(&create_table.table_options)
2798 .context(UnrecognizedTableOptionSnafu)?;
2799
2800 validate_json2_primary_keys(&schema, &primary_key_indices)?;
2801
2802 validate_and_normalize_annotations(&mut table_options, &schema, &partition_key_indices)?;
2803
2804 let meta = TableMeta {
2805 schema,
2806 primary_key_indices,
2807 value_indices: vec![],
2808 engine: create_table.engine.clone(),
2809 next_column_id,
2810 options: table_options,
2811 created_on: Utc::now(),
2812 updated_on: Utc::now(),
2813 partition_key_indices,
2814 column_ids: vec![],
2815 };
2816
2817 let desc = if create_table.desc.is_empty() {
2818 create_table.table_options.get(COMMENT_KEY).cloned()
2819 } else {
2820 Some(create_table.desc.clone())
2821 };
2822
2823 let table_info = TableInfo {
2824 ident: metadata::TableIdent {
2825 table_id: 0,
2827 version: 0,
2828 },
2829 name: create_table.table_name.clone(),
2830 desc,
2831 catalog_name: create_table.catalog_name.clone(),
2832 schema_name: create_table.schema_name.clone(),
2833 meta,
2834 table_type: TableType::Base,
2835 };
2836 Ok(table_info)
2837}
2838
2839fn validate_json2_primary_keys(schema: &Schema, primary_key_indices: &[usize]) -> Result<()> {
2840 for &index in primary_key_indices {
2841 let column = &schema.column_schemas()[index];
2842 ensure!(
2843 !column.data_type.is_json2(),
2844 InvalidSqlSnafu {
2845 err_msg: format!("JSON2 column `{}` cannot be a primary key", column.name),
2846 }
2847 );
2848 }
2849 Ok(())
2850}
2851
2852fn ensure_table_writable(schema: &str, table: &str) -> Result<()> {
2854 ensure!(
2855 !is_readonly_schema(schema),
2856 SchemaReadOnlySnafu {
2857 name: schema.to_string()
2858 }
2859 );
2860 ensure!(
2861 !is_readonly_table(schema, table),
2862 TableReadOnlySnafu {
2863 name: table.to_string()
2864 }
2865 );
2866 Ok(())
2867}
2868
2869fn ensure_table_definition_writable(schema: &str, table: &str) -> Result<()> {
2875 ensure_table_writable(schema, table)?;
2876 ensure!(
2877 !is_ddl_reserved_table(schema, table),
2878 TableDdlReservedSnafu {
2879 name: table.to_string()
2880 }
2881 );
2882 Ok(())
2883}
2884
2885fn validate_and_normalize_annotations(
2889 options: &mut TableOptions,
2890 schema: &Schema,
2891 partition_key_indices: &[usize],
2892) -> Result<()> {
2893 use table::requests::AnnotationValidationError as CheckError;
2894 let handle_error = |e: CheckError| match e {
2895 CheckError::ColumnNotFound { column } => ColumnNotFoundSnafu { msg: column }.build(),
2896 e @ (CheckError::UnknownKey { .. }
2897 | CheckError::InvalidValue { .. }
2898 | CheckError::InvalidPartitionNumHint { .. }
2899 | CheckError::ColumnNotStringForm { .. }) => InvalidSqlSnafu {
2900 err_msg: e.to_string(),
2901 }
2902 .build(),
2903 e @ (CheckError::NotSingleColumn
2904 | CheckError::PartitionMetadataConflict
2905 | CheckError::TimeIndexConflict) => InvalidPartitionRuleSnafu {
2906 reason: e.to_string(),
2907 }
2908 .build(),
2909 };
2910 validate_and_normalize_annotation_options(options, schema, partition_key_indices)
2911 .map_err(handle_error)
2912}
2913
2914fn find_partition_columns(partitions: &Option<Partitions>) -> Result<Vec<String>> {
2915 let columns = if let Some(partitions) = partitions {
2916 partitions
2917 .column_list
2918 .iter()
2919 .map(|x| x.value.clone())
2920 .collect::<Vec<_>>()
2921 } else {
2922 vec![]
2923 };
2924 Ok(columns)
2925}
2926
2927fn find_partition_entries(
2931 create_table: &CreateTableExpr,
2932 partitions: &Option<Partitions>,
2933 partition_columns: &[String],
2934 query_ctx: &QueryContextRef,
2935) -> Result<Vec<PartitionExpr>> {
2936 let Some(partitions) = partitions else {
2937 return Ok(vec![]);
2938 };
2939
2940 let column_name_and_type = partition_columns
2942 .iter()
2943 .map(|pc| {
2944 let column = create_table
2945 .column_defs
2946 .iter()
2947 .find(|c| &c.name == pc)
2948 .unwrap();
2950 let column_name = &column.name;
2951 let data_type = ConcreteDataType::from(
2952 ColumnDataTypeWrapper::try_new(column.data_type, column.datatype_extension.clone())
2953 .context(ColumnDataTypeSnafu)?,
2954 );
2955 Ok((column_name, data_type))
2956 })
2957 .collect::<Result<HashMap<_, _>>>()?;
2958
2959 let mut partition_exprs = Vec::with_capacity(partitions.exprs.len());
2961 for partition in &partitions.exprs {
2962 let partition_expr =
2963 convert_one_expr(partition, &column_name_and_type, &query_ctx.timezone())?;
2964 partition_exprs.push(partition_expr);
2965 }
2966
2967 Ok(partition_exprs)
2968}
2969
2970fn column_name_and_type<'a>(
2971 partition_columns: &'a [&'a ColumnSchema],
2972) -> HashMap<&'a String, ConcreteDataType> {
2973 partition_columns
2974 .iter()
2975 .map(|column| (&column.name, column.data_type.clone()))
2976 .collect()
2977}
2978
2979fn validate_and_collect_partition_columns<'a>(
2980 column_names: &[String],
2981 column_schemas: &'a [ColumnSchema],
2982) -> Result<Vec<&'a ColumnSchema>> {
2983 let mut seen = HashSet::with_capacity(column_names.len());
2984 column_names
2985 .iter()
2986 .map(|column_name| {
2987 ensure!(
2988 seen.insert(column_name),
2989 InvalidPartitionRuleSnafu {
2990 reason: format!("duplicate partition column '{}'", column_name)
2991 }
2992 );
2993 column_schemas
2994 .iter()
2995 .find(|column| &column.name == column_name)
2996 .with_context(|| ColumnNotFoundSnafu { msg: column_name })
2997 })
2998 .collect()
2999}
3000
3001fn ensure_partition_expr_columns_in_target(
3002 partition_exprs: &[PartitionExpr],
3003 target_partition_columns: &HashSet<&String>,
3004) -> Result<()> {
3005 for expr in partition_exprs {
3006 ensure_partition_operand_columns_in_target(&expr.lhs, target_partition_columns)?;
3007 ensure_partition_operand_columns_in_target(&expr.rhs, target_partition_columns)?;
3008 }
3009
3010 Ok(())
3011}
3012
3013fn ensure_partition_operand_columns_in_target(
3014 operand: &Operand,
3015 target_partition_columns: &HashSet<&String>,
3016) -> Result<()> {
3017 match operand {
3018 Operand::Column(column) => ensure!(
3019 target_partition_columns.contains(column),
3020 InvalidPartitionRuleSnafu {
3021 reason: format!(
3022 "partition expression references column '{}' that is not in target partition columns",
3023 column
3024 )
3025 }
3026 ),
3027 Operand::Expr(expr) => {
3028 ensure_partition_operand_columns_in_target(&expr.lhs, target_partition_columns)?;
3029 ensure_partition_operand_columns_in_target(&expr.rhs, target_partition_columns)?;
3030 }
3031 Operand::IcebergHash(hash) => ensure!(
3032 target_partition_columns.contains(&hash.column().to_string()),
3033 InvalidPartitionRuleSnafu {
3034 reason: format!(
3035 "partition expression references column '{}' that is not in target partition columns",
3036 hash.column()
3037 )
3038 }
3039 ),
3040 Operand::Value(_) => {}
3041 }
3042
3043 Ok(())
3044}
3045
3046fn convert_one_expr(
3047 expr: &Expr,
3048 column_name_and_type: &HashMap<&String, ConcreteDataType>,
3049 timezone: &Timezone,
3050) -> Result<PartitionExpr> {
3051 let Expr::BinaryOp { left, op, right } = expr else {
3052 return InvalidPartitionRuleSnafu {
3053 reason: "partition rule must be a binary expression",
3054 }
3055 .fail();
3056 };
3057
3058 let op =
3059 RestrictedOp::try_from_parser(&op.clone()).with_context(|| InvalidPartitionRuleSnafu {
3060 reason: format!("unsupported operator in partition expr {op}"),
3061 })?;
3062
3063 let (lhs, op, rhs) = match (left.as_ref(), right.as_ref()) {
3065 (Expr::Identifier(ident), Expr::Value(value)) => {
3067 let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3068 let value = convert_value(&value.value, data_type, timezone, None)?;
3069 (Operand::Column(column_name), op, Operand::Value(value))
3070 }
3071 (Expr::Identifier(ident), Expr::UnaryOp { op: unary_op, expr })
3072 if let Expr::Value(v) = &**expr =>
3073 {
3074 let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3075 let value = convert_value(&v.value, data_type, timezone, Some(*unary_op))?;
3076 (Operand::Column(column_name), op, Operand::Value(value))
3077 }
3078 (Expr::Value(value), Expr::Identifier(ident)) => {
3080 let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3081 let value = convert_value(&value.value, data_type, timezone, None)?;
3082 (Operand::Value(value), op, Operand::Column(column_name))
3083 }
3084 (Expr::UnaryOp { op: unary_op, expr }, Expr::Identifier(ident))
3085 if let Expr::Value(v) = &**expr =>
3086 {
3087 let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3088 let value = convert_value(&v.value, data_type, timezone, Some(*unary_op))?;
3089 (Operand::Value(value), op, Operand::Column(column_name))
3090 }
3091 (Expr::Function(function), literal)
3093 if !matches!(op, RestrictedOp::And | RestrictedOp::Or) =>
3094 {
3095 let hash = convert_iceberg_hash(function, column_name_and_type)?;
3096 let value = convert_literal(literal, IcebergHash::output_type(), timezone)?;
3097 (Operand::IcebergHash(hash), op, Operand::Value(value))
3098 }
3099 (literal, Expr::Function(function))
3101 if !matches!(op, RestrictedOp::And | RestrictedOp::Or) =>
3102 {
3103 let hash = convert_iceberg_hash(function, column_name_and_type)?;
3104 let value = convert_literal(literal, IcebergHash::output_type(), timezone)?;
3105 (Operand::Value(value), op, Operand::IcebergHash(hash))
3106 }
3107 (Expr::BinaryOp { .. }, Expr::BinaryOp { .. }) => {
3108 let lhs = convert_one_expr(left, column_name_and_type, timezone)?;
3110 let rhs = convert_one_expr(right, column_name_and_type, timezone)?;
3111 (Operand::Expr(lhs), op, Operand::Expr(rhs))
3112 }
3113 _ => {
3114 return InvalidPartitionRuleSnafu {
3115 reason: format!("invalid partition expr {expr}"),
3116 }
3117 .fail();
3118 }
3119 };
3120
3121 Ok(PartitionExpr::new(lhs, op, rhs))
3122}
3123
3124fn convert_iceberg_hash(
3126 function: &Function,
3127 column_name_and_type: &HashMap<&String, ConcreteDataType>,
3128) -> Result<IcebergHash> {
3129 let invalid = || {
3130 InvalidPartitionRuleSnafu {
3131 reason: format!("unsupported partition function call {function}"),
3132 }
3133 .build()
3134 };
3135 let FunctionArguments::List(args) = &function.args else {
3136 return Err(invalid());
3137 };
3138 let [FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(ident)))] =
3139 args.args.as_slice()
3140 else {
3141 return Err(invalid());
3142 };
3143 if function.filter.is_some()
3144 || function.over.is_some()
3145 || function.null_treatment.is_some()
3146 || !function.within_group.is_empty()
3147 || !matches!(function.parameters, FunctionArguments::None)
3148 || args.duplicate_treatment.is_some()
3149 || !args.clauses.is_empty()
3150 {
3151 return Err(invalid());
3152 }
3153 ensure!(
3154 function
3155 .name
3156 .to_string()
3157 .eq_ignore_ascii_case(IcebergHash::NAME),
3158 InvalidPartitionRuleSnafu {
3159 reason: format!("unsupported partition function {}", function.name),
3160 }
3161 );
3162 let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3163 ensure!(
3164 IcebergHash::accepts(&data_type),
3165 InvalidPartitionRuleSnafu {
3166 reason: format!(
3167 "partition function {} does not support column type {data_type}",
3168 IcebergHash::NAME
3169 ),
3170 }
3171 );
3172 Ok(IcebergHash::new(column_name))
3173}
3174
3175fn convert_literal(expr: &Expr, data_type: ConcreteDataType, timezone: &Timezone) -> Result<Value> {
3176 match expr {
3177 Expr::Value(value) => convert_value(&value.value, data_type, timezone, None),
3178 Expr::UnaryOp { op, expr } if let Expr::Value(value) = &**expr => {
3179 convert_value(&value.value, data_type, timezone, Some(*op))
3180 }
3181 _ => InvalidPartitionRuleSnafu {
3182 reason: format!("expect a literal in partition expr, got {expr}"),
3183 }
3184 .fail(),
3185 }
3186}
3187
3188fn convert_identifier(
3189 ident: &Ident,
3190 column_name_and_type: &HashMap<&String, ConcreteDataType>,
3191) -> Result<(String, ConcreteDataType)> {
3192 let column_name = ident.value.clone();
3193 let data_type = column_name_and_type
3194 .get(&column_name)
3195 .cloned()
3196 .with_context(|| ColumnNotFoundSnafu { msg: &column_name })?;
3197 Ok((column_name, data_type))
3198}
3199
3200fn convert_value(
3201 value: &ParserValue,
3202 data_type: ConcreteDataType,
3203 timezone: &Timezone,
3204 unary_op: Option<UnaryOperator>,
3205) -> Result<Value> {
3206 sql_value_to_value(
3207 &ColumnSchema::new("<NONAME>", data_type, true),
3208 value,
3209 Some(timezone),
3210 unary_op,
3211 false,
3212 )
3213 .context(error::SqlCommonSnafu)
3214}
3215
3216#[cfg(feature = "enterprise")]
3217async fn execute_undrop_table(
3218 table_metadata_manager: &TableMetadataManagerRef,
3219 procedure_executor: &ProcedureExecutorRef,
3220 cache_invalidator: &CacheInvalidatorRef,
3221 table_name: TableName,
3222 query_context: QueryContextRef,
3223) -> Result<Output> {
3224 ensure_table_definition_writable(&table_name.schema_name, &table_name.table_name)?;
3227
3228 let dropped = table_metadata_manager
3229 .get_dropped_table(&table_name)
3230 .await
3231 .context(TableMetadataManagerSnafu)?
3232 .with_context(|| TableNotFoundSnafu {
3233 table_name: table_name.to_string(),
3234 })?;
3235
3236 let executor_context = to_executor_context(query_context, TriggerReason::Manual);
3237 let request = SubmitDdlTaskRequest::new(DdlTask::new_undrop_table(dropped.table_id));
3238 procedure_executor
3239 .submit_ddl_task(executor_context, request)
3240 .await
3241 .context(error::ExecuteDdlSnafu)?;
3242
3243 if let Err(err) = cache_invalidator
3244 .invalidate(
3245 &Context::default(),
3246 &[
3247 CacheIdent::TableId(dropped.table_id),
3248 CacheIdent::TableName(table_name),
3249 ],
3250 )
3251 .await
3252 {
3253 warn!(
3254 "Failed to invalidate cache after restoring table '{}' (id={}): {}",
3255 dropped.table_name, dropped.table_id, err
3256 );
3257 }
3258
3259 Ok(Output::new_with_affected_rows(0))
3260}
3261
3262#[cfg(test)]
3263mod test {
3264 #[cfg(feature = "enterprise")]
3265 use std::sync::{Arc, Mutex};
3266 use std::time::Duration;
3267
3268 #[cfg(feature = "enterprise")]
3269 use api::v1::meta::{ProcedureDetailResponse, ReconcileRequest, ReconcileResponse};
3270 use arrow::datatypes::{DataType as ArrowDataType, Field as ArrowField};
3271 #[cfg(feature = "enterprise")]
3272 use common_meta::cache_invalidator::{CacheInvalidator, CacheInvalidatorRef};
3273 #[cfg(feature = "enterprise")]
3274 use common_meta::instruction::CacheIdent;
3275 #[cfg(feature = "enterprise")]
3276 use common_meta::key::table_route::TableRouteValue;
3277 #[cfg(feature = "enterprise")]
3278 use common_meta::key::test_utils::new_test_table_info_with_name;
3279 #[cfg(feature = "enterprise")]
3280 use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
3281 #[cfg(feature = "enterprise")]
3282 use common_meta::kv_backend::memory::MemoryKvBackend;
3283 #[cfg(feature = "enterprise")]
3284 use common_meta::procedure_executor::{
3285 ExecutorContext, ProcedureExecutor, ProcedureExecutorRef,
3286 };
3287 #[cfg(feature = "enterprise")]
3288 use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse};
3289 #[cfg(feature = "enterprise")]
3290 use common_meta::rpc::procedure::{
3291 MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse,
3292 };
3293 use datafusion::functions_aggregate::expr_fn::count;
3294 use datafusion::logical_expr::builder::LogicalTableSource;
3295 use datafusion::logical_expr::{LogicalPlanBuilder, col};
3296 use session::context::QueryContext;
3297 use sql::dialect::GreptimeDbDialect;
3298 use sql::parser::{ParseOptions, ParserContext};
3299 use sql::statements::statement::Statement;
3300 use sqlparser::parser::Parser;
3301 use table::requests::REPARTITION_COLUMN_HINT_KEY;
3302
3303 use super::*;
3304 use crate::expr_helper;
3305
3306 #[cfg(feature = "enterprise")]
3307 #[derive(Default)]
3308 struct RecordingProcedureExecutor {
3309 requests: Mutex<Vec<SubmitDdlTaskRequest>>,
3310 contexts: Mutex<Vec<ExecutorContext>>,
3311 fail: bool,
3312 }
3313
3314 #[cfg(feature = "enterprise")]
3315 #[async_trait::async_trait]
3316 impl ProcedureExecutor for RecordingProcedureExecutor {
3317 async fn submit_ddl_task(
3318 &self,
3319 ctx: ExecutorContext,
3320 request: SubmitDdlTaskRequest,
3321 ) -> common_meta::error::Result<SubmitDdlTaskResponse> {
3322 self.contexts.lock().unwrap().push(ctx);
3323 self.requests.lock().unwrap().push(request);
3324 if self.fail {
3325 return common_meta::error::UnsupportedSnafu {
3326 operation: "test submit failure",
3327 }
3328 .fail();
3329 }
3330 Ok(SubmitDdlTaskResponse::default())
3331 }
3332
3333 async fn migrate_region(
3334 &self,
3335 _: &ExecutorContext,
3336 _: MigrateRegionRequest,
3337 ) -> common_meta::error::Result<MigrateRegionResponse> {
3338 unimplemented!()
3339 }
3340 async fn reconcile(
3341 &self,
3342 _: &ExecutorContext,
3343 _: ReconcileRequest,
3344 ) -> common_meta::error::Result<ReconcileResponse> {
3345 unimplemented!()
3346 }
3347 async fn query_procedure_state(
3348 &self,
3349 _: &ExecutorContext,
3350 _: &str,
3351 ) -> common_meta::error::Result<ProcedureStateResponse> {
3352 unimplemented!()
3353 }
3354 async fn list_procedures(
3355 &self,
3356 _: &ExecutorContext,
3357 ) -> common_meta::error::Result<ProcedureDetailResponse> {
3358 unimplemented!()
3359 }
3360 }
3361
3362 #[cfg(feature = "enterprise")]
3363 #[derive(Default)]
3364 struct RecordingCacheInvalidator {
3365 invalidations: Mutex<Vec<Vec<CacheIdent>>>,
3366 fail: bool,
3367 }
3368
3369 #[cfg(feature = "enterprise")]
3370 #[async_trait::async_trait]
3371 impl CacheInvalidator for RecordingCacheInvalidator {
3372 async fn invalidate(
3373 &self,
3374 _: &Context,
3375 caches: &[CacheIdent],
3376 ) -> common_meta::error::Result<()> {
3377 self.invalidations.lock().unwrap().push(caches.to_vec());
3378 if self.fail {
3379 return common_meta::error::UnsupportedSnafu {
3380 operation: "test cache failure",
3381 }
3382 .fail();
3383 }
3384 Ok(())
3385 }
3386
3387 fn invalidate_all(&self) -> common_meta::error::Result<()> {
3388 Ok(())
3389 }
3390 }
3391
3392 #[cfg(feature = "enterprise")]
3393 async fn dropped_table_manager(table_id: TableId, name: &TableName) -> TableMetadataManagerRef {
3394 let backend = Arc::new(MemoryKvBackend::default());
3395 let manager = Arc::new(TableMetadataManager::new(backend));
3396 let mut info = new_test_table_info_with_name(table_id, &name.table_name);
3397 info.catalog_name = name.catalog_name.clone();
3398 info.schema_name = name.schema_name.clone();
3399 let route = TableRouteValue::physical(vec![]);
3400 manager
3401 .create_table_metadata(info, route.clone(), HashMap::new())
3402 .await
3403 .unwrap();
3404 manager
3405 .delete_table_metadata(table_id, name, &route, &HashMap::new(), None)
3406 .await
3407 .unwrap();
3408 manager
3409 }
3410
3411 #[cfg(feature = "enterprise")]
3412 #[tokio::test]
3413 async fn test_undrop_table_execution_path() {
3414 let name = TableName::new("greptime", "public", "metrics");
3415 let manager = dropped_table_manager(42, &name).await;
3416 let mut live = new_test_table_info_with_name(99, "metrics");
3418 live.catalog_name = "greptime".to_string();
3419 live.schema_name = "public".to_string();
3420 manager
3421 .create_table_metadata(live, TableRouteValue::physical(vec![]), HashMap::new())
3422 .await
3423 .unwrap();
3424 let procedure = Arc::new(RecordingProcedureExecutor::default());
3425 let cache = Arc::new(RecordingCacheInvalidator::default());
3426
3427 execute_undrop_table(
3428 &manager,
3429 &(procedure.clone() as ProcedureExecutorRef),
3430 &(cache.clone() as CacheInvalidatorRef),
3431 name.clone(),
3432 QueryContext::arc(),
3433 )
3434 .await
3435 .unwrap();
3436
3437 let requests = procedure.requests.lock().unwrap();
3438 assert!(matches!(&requests[0].task, DdlTask::UndropTable(task) if task.table_id == 42));
3439 let contexts = procedure.contexts.lock().unwrap();
3440 assert_eq!(
3441 contexts[0].event_input.as_ref().map(|input| input.reason),
3442 Some(TriggerReason::Manual)
3443 );
3444 assert_eq!(
3445 cache.invalidations.lock().unwrap()[0],
3446 vec![CacheIdent::TableId(42), CacheIdent::TableName(name)]
3447 );
3448 }
3449
3450 #[cfg(feature = "enterprise")]
3451 #[tokio::test]
3452 async fn test_undrop_table_missing_tombstone_submits_nothing() {
3453 let manager = Arc::new(TableMetadataManager::new(Arc::new(
3454 MemoryKvBackend::default(),
3455 )));
3456 let procedure = Arc::new(RecordingProcedureExecutor::default());
3457 let cache = Arc::new(RecordingCacheInvalidator::default());
3458 let err = execute_undrop_table(
3459 &manager,
3460 &(procedure.clone() as ProcedureExecutorRef),
3461 &(cache as CacheInvalidatorRef),
3462 TableName::new("greptime", "public", "missing"),
3463 QueryContext::arc(),
3464 )
3465 .await
3466 .unwrap_err();
3467 assert!(matches!(err, crate::error::Error::TableNotFound { .. }));
3468 assert!(procedure.requests.lock().unwrap().is_empty());
3469 }
3470
3471 #[cfg(feature = "enterprise")]
3472 #[tokio::test]
3473 async fn test_undrop_table_submit_failure_does_not_invalidate() {
3474 let name = TableName::new("greptime", "public", "metrics");
3475 let manager = dropped_table_manager(42, &name).await;
3476 let procedure = Arc::new(RecordingProcedureExecutor {
3477 fail: true,
3478 ..Default::default()
3479 });
3480 let cache = Arc::new(RecordingCacheInvalidator::default());
3481 execute_undrop_table(
3482 &manager,
3483 &(procedure as ProcedureExecutorRef),
3484 &(cache.clone() as CacheInvalidatorRef),
3485 name,
3486 QueryContext::arc(),
3487 )
3488 .await
3489 .unwrap_err();
3490 assert!(cache.invalidations.lock().unwrap().is_empty());
3491 }
3492
3493 #[cfg(feature = "enterprise")]
3494 #[tokio::test]
3495 async fn test_undrop_table_cache_failure_returns_success_after_submit() {
3496 let name = TableName::new("greptime", "public", "metrics");
3497 let manager = dropped_table_manager(42, &name).await;
3498 let procedure = Arc::new(RecordingProcedureExecutor::default());
3499 let cache = Arc::new(RecordingCacheInvalidator {
3500 fail: true,
3501 ..Default::default()
3502 });
3503 execute_undrop_table(
3504 &manager,
3505 &(procedure.clone() as ProcedureExecutorRef),
3506 &(cache.clone() as CacheInvalidatorRef),
3507 name.clone(),
3508 QueryContext::arc(),
3509 )
3510 .await
3511 .unwrap();
3512 assert_eq!(procedure.requests.lock().unwrap().len(), 1);
3513 assert_eq!(
3514 cache.invalidations.lock().unwrap()[0],
3515 vec![CacheIdent::TableId(42), CacheIdent::TableName(name)]
3516 );
3517 }
3518
3519 #[test]
3520 fn test_parse_ddl_options() {
3521 let defaults = parse_ddl_options(&OptionMap::default()).unwrap();
3522 assert!(defaults.wait);
3523 assert_eq!(defaults.timeout, SubmitDdlTaskRequest::default_timeout());
3524
3525 let options = OptionMap::from([
3526 ("timeout".to_string(), "5m".to_string()),
3527 ("wait".to_string(), "false".to_string()),
3528 ]);
3529 let ddl_options = parse_ddl_options(&options).unwrap();
3530 assert!(!ddl_options.wait);
3531 assert_eq!(Duration::from_secs(300), ddl_options.timeout);
3532
3533 let options = OptionMap::from([("timeout".to_string(), "1500ms".to_string())]);
3534 assert_eq!(
3535 parse_ddl_options(&options).unwrap().timeout,
3536 Duration::from_millis(1500)
3537 );
3538 for (key, value) in [
3539 ("wait", "yes"),
3540 ("timeout", "not-a-duration"),
3541 ("timeout", "-1s"),
3542 ] {
3543 let options = OptionMap::from([(key.to_string(), value.to_string())]);
3544 assert!(matches!(
3545 parse_ddl_options(&options),
3546 Err(error::Error::InvalidSql { .. })
3547 ));
3548 }
3549 }
3550
3551 #[test]
3552 fn test_validate_and_normalize_annotations() {
3553 let schema = Schema::new(vec![
3554 ColumnSchema::new("service_name", ConcreteDataType::string_datatype(), true),
3555 ColumnSchema::new("host_id", ConcreteDataType::string_datatype(), true),
3556 ColumnSchema::new("value", ConcreteDataType::float64_datatype(), true),
3557 ColumnSchema::new("payload", ConcreteDataType::binary_datatype(), true),
3558 ]);
3559 let opts = |pairs: &[(&str, &str)]| TableOptions {
3560 extra_options: pairs
3561 .iter()
3562 .map(|(k, v)| (k.to_string(), v.to_string()))
3563 .collect(),
3564 ..Default::default()
3565 };
3566 let check = |pairs: &[(&str, &str)]| {
3567 let mut options = opts(pairs);
3568 validate_and_normalize_annotations(&mut options, &schema, &[]).map(|()| options)
3569 };
3570
3571 for pairs in [
3573 &[(
3574 "greptime.semantic.entity.process.id",
3575 "service_name,host_id",
3576 )][..],
3577 &[("greptime.semantic.entity.service.id", "value")][..],
3578 ] {
3579 assert!(check(pairs).is_ok());
3580 }
3581
3582 let missing = check(&[("greptime.semantic.entity.service.id", "nope")]).unwrap_err();
3584 assert!(matches!(missing, error::Error::ColumnNotFound { .. }));
3585 assert_eq!(
3586 common_error::status_code::StatusCode::InvalidArguments,
3587 common_error::ext::ErrorExt::status_code(&missing)
3588 );
3589
3590 let binary_id = check(&[("greptime.semantic.entity.service.id", "payload")]).unwrap_err();
3591 assert!(matches!(binary_id, error::Error::InvalidSql { .. }));
3592
3593 let bad_value = check(&[("greptime.semantic.signal_type", "garbage")]).unwrap_err();
3596 assert!(matches!(bad_value, error::Error::InvalidSql { .. }));
3597
3598 let unknown_key = check(&[("greptime.semantic.nonsense", "x")]).unwrap_err();
3599 assert!(matches!(unknown_key, error::Error::InvalidSql { .. }));
3600 }
3601
3602 #[test]
3603 fn test_validate_and_normalize_flow_options_empty() {
3604 assert!(
3605 validate_and_normalize_flow_options(HashMap::new(), None)
3606 .unwrap()
3607 .is_empty()
3608 );
3609 }
3610
3611 #[test]
3612 fn test_validate_and_normalize_flow_options_valid() {
3613 let options = HashMap::from([
3614 (DEFER_ON_MISSING_SOURCE_KEY.to_string(), "TRUE".to_string()),
3615 (
3616 FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
3617 "FALSE".to_string(),
3618 ),
3619 ]);
3620
3621 assert_eq!(
3622 validate_and_normalize_flow_options(options, None).unwrap(),
3623 HashMap::from([
3624 (DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string(),),
3625 (
3626 FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
3627 "false".to_string(),
3628 )
3629 ])
3630 );
3631 }
3632
3633 #[cfg(not(feature = "enterprise"))]
3634 #[test]
3635 fn test_validate_and_normalize_flow_options_unknown_option() {
3636 let err = validate_and_normalize_flow_options(
3637 HashMap::from([("foo".to_string(), "bar".to_string())]),
3638 None,
3639 )
3640 .unwrap_err();
3641
3642 assert!(
3643 err.to_string()
3644 .contains("unknown flow option 'foo', supported options:")
3645 );
3646 }
3647
3648 #[test]
3649 fn test_validate_and_normalize_flow_options_reserved_option() {
3650 let err = validate_and_normalize_flow_options(
3651 HashMap::from([(
3652 FlowType::FLOW_TYPE_KEY.to_string(),
3653 FlowType::BATCHING.to_string(),
3654 )]),
3655 None,
3656 )
3657 .unwrap_err();
3658
3659 assert!(
3660 err.to_string()
3661 .contains("flow option 'flow_type' is reserved for internal use")
3662 );
3663 }
3664
3665 #[test]
3666 fn test_validate_and_normalize_flow_options_invalid_bool() {
3667 let err = validate_and_normalize_flow_options(
3668 HashMap::from([(
3669 DEFER_ON_MISSING_SOURCE_KEY.to_string(),
3670 "not-a-bool".to_string(),
3671 )]),
3672 None,
3673 )
3674 .unwrap_err();
3675
3676 assert!(
3677 err.to_string()
3678 .contains("invalid flow option 'defer_on_missing_source': 'not-a-bool'")
3679 );
3680 }
3681
3682 #[test]
3683 fn test_parser_flow_option_access_key_id() {
3684 let sql = r"
3685CREATE FLOW task_6
3686SINK TO schema_1.table_1
3687WITH (access_key_id = [true])
3688AS
3689SELECT max(c1), min(c2) FROM schema_2.table_2;";
3690 let stmt =
3691 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
3692 .unwrap()
3693 .pop()
3694 .unwrap();
3695
3696 let Statement::CreateFlow(create_flow) = stmt else {
3697 unreachable!()
3698 };
3699 let expr =
3700 expr_helper::to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
3701
3702 #[cfg(not(feature = "enterprise"))]
3703 {
3704 let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err();
3705 assert!(
3706 err.to_string()
3707 .contains("unknown flow option 'access_key_id'")
3708 );
3709 }
3710 #[cfg(feature = "enterprise")]
3711 assert_eq!(
3712 validate_and_normalize_flow_options(expr.flow_options, None).unwrap(),
3713 HashMap::from([("access_key_id".to_string(), "['true']".to_string())])
3714 );
3715 }
3716
3717 fn stateless_test_scan(name: &str) -> LogicalPlan {
3718 let schema = arrow::datatypes::Schema::new(vec![ArrowField::new(
3719 "value",
3720 ArrowDataType::Int32,
3721 true,
3722 )]);
3723 LogicalPlanBuilder::scan(
3724 name,
3725 Arc::new(LogicalTableSource::new(Arc::new(schema))),
3726 None,
3727 )
3728 .unwrap()
3729 .build()
3730 .unwrap()
3731 }
3732
3733 #[test]
3734 fn test_is_stateless_flow_plan_accepts_scan_projection_and_filter() {
3735 let scan = stateless_test_scan("source");
3736 assert!(is_stateless_flow_plan(&scan));
3737
3738 let projection = LogicalPlanBuilder::from(scan.clone())
3739 .project(vec![col("value")])
3740 .unwrap()
3741 .build()
3742 .unwrap();
3743 assert!(is_stateless_flow_plan(&projection));
3744
3745 let filter = LogicalPlanBuilder::from(projection)
3746 .filter(col("value").gt(datafusion_expr::lit(0)))
3747 .unwrap()
3748 .build()
3749 .unwrap();
3750 assert!(is_stateless_flow_plan(&filter));
3751 }
3752
3753 #[test]
3754 fn test_is_stateless_flow_plan_rejects_aggregate_and_multiple_scans() {
3755 let scan = stateless_test_scan("source");
3756 let aggregate = LogicalPlanBuilder::from(scan.clone())
3757 .aggregate(
3758 Vec::<datafusion_expr::Expr>::new(),
3759 vec![count(col("value"))],
3760 )
3761 .unwrap()
3762 .build()
3763 .unwrap();
3764 assert!(!is_stateless_flow_plan(&aggregate));
3765
3766 let distinct = LogicalPlanBuilder::from(scan.clone())
3767 .distinct()
3768 .unwrap()
3769 .build()
3770 .unwrap();
3771 assert!(is_stateless_flow_plan(&distinct));
3772
3773 let distinct_on_aggregate = LogicalPlanBuilder::from(aggregate)
3774 .distinct()
3775 .unwrap()
3776 .build()
3777 .unwrap();
3778 assert!(!is_stateless_flow_plan(&distinct_on_aggregate));
3779
3780 let multiple_scans = LogicalPlanBuilder::from(stateless_test_scan("left"))
3781 .cross_join(stateless_test_scan("right"))
3782 .unwrap()
3783 .build()
3784 .unwrap();
3785 assert!(!is_stateless_flow_plan(&multiple_scans));
3786 }
3787
3788 #[test]
3791 fn test_eval_interval_rejected_non_positive() {
3792 let err = validate_and_normalize_flow_options(HashMap::new(), Some(0)).unwrap_err();
3794 assert!(err.to_string().contains("EVAL INTERVAL must be positive"));
3795
3796 let err = validate_and_normalize_flow_options(HashMap::new(), Some(-5)).unwrap_err();
3798 assert!(err.to_string().contains("EVAL INTERVAL must be positive"));
3799
3800 let result = validate_and_normalize_flow_options(HashMap::new(), Some(300));
3802 assert!(result.is_ok());
3803 }
3804
3805 #[test]
3806 fn test_internal_schedule_key_rejected() {
3807 let err = validate_and_normalize_flow_options(
3808 HashMap::from([(INTERNAL_EVAL_SCHEDULE_KEY.to_string(), "value".to_string())]),
3809 Some(300),
3810 )
3811 .unwrap_err();
3812 assert!(err.to_string().contains("reserved for internal use"));
3813 }
3814
3815 #[cfg(not(feature = "enterprise"))]
3816 #[test]
3817 fn test_schedule_keys_rejected_as_unknown_options() {
3818 for key in [
3819 "eval_interval_anchor",
3820 "eval_interval_start",
3821 "eval_interval_missed_tick_policy",
3822 "eval_interval_catchup_max_runs",
3823 "eval_interval_catchup_max_lag",
3824 ] {
3825 let err = validate_and_normalize_flow_options(
3826 HashMap::from([(key.to_string(), "value".to_string())]),
3827 Some(300),
3828 )
3829 .unwrap_err();
3830 assert!(
3831 err.to_string()
3832 .contains(&format!("unknown flow option '{key}'"))
3833 );
3834 }
3835 }
3836
3837 #[cfg(feature = "enterprise")]
3838 #[test]
3839 fn test_schedule_keys_preserved() {
3840 let options = HashMap::from([
3841 ("eval_interval_anchor".to_string(), "anchor".to_string()),
3842 ("eval_interval_start".to_string(), "start".to_string()),
3843 (
3844 "eval_interval_missed_tick_policy".to_string(),
3845 "missed".to_string(),
3846 ),
3847 (
3848 "eval_interval_catchup_max_runs".to_string(),
3849 "runs".to_string(),
3850 ),
3851 (
3852 "eval_interval_catchup_max_lag".to_string(),
3853 "lag".to_string(),
3854 ),
3855 ]);
3856 assert_eq!(
3857 validate_and_normalize_flow_options(options.clone(), Some(300)).unwrap(),
3858 options
3859 );
3860 }
3861
3862 #[test]
3863 fn test_internal_transport_keys_rejected_as_reserved() {
3864 for key in [
3865 "__greptime_internal_eval_schedule",
3866 "__greptime_internal_eval_offset_secs",
3867 ] {
3868 let err = validate_and_normalize_flow_options(
3869 HashMap::from([(key.to_string(), "120".to_string())]),
3870 Some(300),
3871 )
3872 .unwrap_err();
3873
3874 assert!(
3875 err.to_string()
3876 .contains(&format!("flow option '{key}' is reserved for internal use")),
3877 "unexpected error for {key}: {err}"
3878 );
3879 }
3880 }
3881
3882 #[test]
3883 fn test_determine_flow_type_for_source_state_missing_sources_require_opt_in() {
3884 let err =
3885 determine_flow_type_for_source_state("my_flow", &HashMap::new(), true, false, false)
3886 .unwrap_err();
3887
3888 assert!(err.to_string().contains(
3889 "missing source tables for flow 'my_flow'; use WITH (defer_on_missing_source = true) to create a pending flow"
3890 ));
3891 }
3892
3893 #[test]
3894 fn test_determine_flow_type_for_source_state_missing_sources_prefer_batching() {
3895 let flow_options =
3896 HashMap::from([(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string())]);
3897
3898 assert_eq!(
3899 determine_flow_type_for_source_state("my_flow", &flow_options, true, true, false)
3900 .unwrap(),
3901 Some(FlowType::Batching)
3902 );
3903 let err = determine_flow_type_for_source_state("my_flow", &flow_options, true, true, true)
3904 .unwrap_err();
3905 assert!(err.to_string().contains(
3906 "flow 'my_flow' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode"
3907 ));
3908 }
3909
3910 #[test]
3911 fn test_determine_flow_type_for_source_state_existing_sources_require_plan() {
3912 assert_eq!(
3913 determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true, false)
3914 .unwrap(),
3915 None
3916 );
3917 }
3918
3919 #[test]
3920 fn test_determine_flow_type_for_source_state_instant_ttl_with_eval_interval_is_rejected() {
3921 let err =
3926 determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true, true)
3927 .unwrap_err();
3928 assert!(
3929 err.to_string().contains(
3930 "flow 'my_flow' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode"
3931 ),
3932 "unexpected error: {err}"
3933 );
3934 }
3935
3936 #[test]
3937 fn test_name_is_match() {
3938 assert!(!NAME_PATTERN_REG.is_match("/adaf"));
3939 assert!(!NAME_PATTERN_REG.is_match("🈲"));
3940 assert!(NAME_PATTERN_REG.is_match("hello"));
3941 assert!(NAME_PATTERN_REG.is_match("test@"));
3942 assert!(!NAME_PATTERN_REG.is_match("@test"));
3943 assert!(NAME_PATTERN_REG.is_match("test#"));
3944 assert!(!NAME_PATTERN_REG.is_match("#test"));
3945 assert!(!NAME_PATTERN_REG.is_match("@"));
3946 assert!(!NAME_PATTERN_REG.is_match("#"));
3947 }
3948
3949 #[test]
3950 fn test_convert_iceberg_hash() {
3951 let names = ["host", "count", "device", "measure", "host-name"].map(String::from);
3952 let column_name_and_type = HashMap::from([
3953 (&names[0], ConcreteDataType::string_datatype()),
3954 (&names[1], ConcreteDataType::int16_datatype()),
3955 (&names[2], ConcreteDataType::uint64_datatype()),
3956 (&names[3], ConcreteDataType::float64_datatype()),
3957 (&names[4], ConcreteDataType::string_datatype()),
3958 ]);
3959 let timezone = Timezone::from_tz_string("UTC").unwrap();
3960 let dialect = GreptimeDbDialect {};
3961 let parse = |sql: &str| {
3962 Parser::new(&dialect)
3963 .try_with_sql(sql)
3964 .unwrap()
3965 .parse_expr()
3966 .unwrap()
3967 };
3968
3969 for sql in [
3970 "iceberg_hash(host) < 0",
3971 "ICEBERG_HASH(device) >= -1073741824",
3972 "0 > iceberg_hash(count)",
3973 "iceberg_hash(host) < 100 AND host < 'm'",
3974 r#"iceberg_hash("host-name") < 0 AND "host-name" < 'm'"#,
3975 ] {
3976 let expr = convert_one_expr(&parse(sql), &column_name_and_type, &timezone).unwrap();
3977 let mut partitions = Partitions {
3979 column_list: vec![],
3980 exprs: vec![expr.to_parser_expr()],
3981 };
3982 partitions.set_quote('"');
3983 let restored = parse(&partitions.exprs[0].to_string());
3984 assert_eq!(
3985 convert_one_expr(&restored, &column_name_and_type, &timezone).unwrap(),
3986 expr,
3987 "{sql}"
3988 );
3989 }
3990
3991 for sql in [
3992 "iceberg_hash(measure) < 0",
3993 "iceberg_hash(host, count) < 0",
3994 "iceberg_hash('a') < 0",
3995 "iceberg_hash(missing) < 0",
3996 "iceberg_hash(DISTINCT host) < 0",
3997 "md5(host) < 0",
3998 "iceberg_hash(host) < host",
3999 "iceberg_hash(host) < 'abc'",
4000 "iceberg_hash(host) < 2147483648",
4001 ] {
4002 assert!(
4003 convert_one_expr(&parse(sql), &column_name_and_type, &timezone).is_err(),
4004 "{sql}"
4005 );
4006 }
4007 }
4008
4009 #[test]
4010 fn test_partition_expr_equivalence_with_swapped_operands() {
4011 let column_name = "device_id".to_string();
4012 let column_name_and_type =
4013 HashMap::from([(&column_name, ConcreteDataType::int32_datatype())]);
4014 let timezone = Timezone::from_tz_string("UTC").unwrap();
4015 let dialect = GreptimeDbDialect {};
4016
4017 let mut parser = Parser::new(&dialect)
4018 .try_with_sql("device_id < 100")
4019 .unwrap();
4020 let expr_left = parser.parse_expr().unwrap();
4021
4022 let mut parser = Parser::new(&dialect)
4023 .try_with_sql("100 > device_id")
4024 .unwrap();
4025 let expr_right = parser.parse_expr().unwrap();
4026
4027 let partition_left =
4028 convert_one_expr(&expr_left, &column_name_and_type, &timezone).unwrap();
4029 let partition_right =
4030 convert_one_expr(&expr_right, &column_name_and_type, &timezone).unwrap();
4031
4032 assert_eq!(partition_left, partition_right);
4033 assert!([partition_left.clone()].contains(&partition_right));
4034
4035 let mut physical_partition_exprs = vec![partition_left];
4036 let mut logical_partition_exprs = vec![partition_right];
4037 physical_partition_exprs.sort_unstable();
4038 logical_partition_exprs.sort_unstable();
4039 assert_eq!(physical_partition_exprs, logical_partition_exprs);
4040 }
4041
4042 #[test]
4043 fn test_repartition_target_partition_columns_are_overwrite_context() {
4044 let device_id = ColumnSchema::new("device_id", ConcreteDataType::int32_datatype(), true);
4045 let area = ColumnSchema::new("area", ConcreteDataType::string_datatype(), true);
4046 let existing_partition_columns = vec![&device_id];
4047 let target_partition_columns = vec![&device_id, &area];
4048 let existing_column_name_and_type = column_name_and_type(&existing_partition_columns);
4049 let target_column_name_and_type = column_name_and_type(&target_partition_columns);
4050 let timezone = Timezone::from_tz_string("UTC").unwrap();
4051 let dialect = GreptimeDbDialect {};
4052
4053 let mut parser = Parser::new(&dialect)
4054 .try_with_sql("device_id < 100 AND area < 'South'")
4055 .unwrap();
4056 let expr = parser.parse_expr().unwrap();
4057
4058 let err = convert_one_expr(&expr, &existing_column_name_and_type, &timezone).unwrap_err();
4059 assert!(err.to_string().contains("area"));
4060
4061 let partition_expr = convert_one_expr(&expr, &target_column_name_and_type, &timezone)
4062 .expect("target columns should overwrite the conversion context");
4063 let partition_expr = partition_expr.to_string();
4064 assert!(partition_expr.contains("device_id"));
4065 assert!(partition_expr.contains("area"));
4066 assert!(partition_expr.contains("South"));
4067 }
4068
4069 #[test]
4070 fn test_repartition_rejects_remaining_expr_outside_target_columns() {
4071 let device_id = "device_id".to_string();
4072 let area = "area".to_string();
4073 let timezone = Timezone::from_tz_string("UTC").unwrap();
4074 let column_name_and_type = HashMap::from([
4075 (&device_id, ConcreteDataType::int32_datatype()),
4076 (&area, ConcreteDataType::string_datatype()),
4077 ]);
4078 let dialect = GreptimeDbDialect {};
4079 let target_partition_columns = HashSet::from([&area]);
4080 for sql in ["device_id >= 100", "iceberg_hash(device_id) >= 0"] {
4081 let mut parser = Parser::new(&dialect).try_with_sql(sql).unwrap();
4082 let remaining_old_expr = convert_one_expr(
4083 &parser.parse_expr().unwrap(),
4084 &column_name_and_type,
4085 &timezone,
4086 )
4087 .unwrap();
4088
4089 let err = ensure_partition_expr_columns_in_target(
4090 &[remaining_old_expr],
4091 &target_partition_columns,
4092 )
4093 .unwrap_err();
4094
4095 assert!(err.to_string().contains("device_id"), "{sql}");
4096 assert!(
4097 err.to_string().contains("target partition columns"),
4098 "{sql}"
4099 );
4100 }
4101 }
4102
4103 #[test]
4104 fn test_repartition_rejects_duplicate_target_partition_columns() {
4105 let device_id = ColumnSchema::new("device_id", ConcreteDataType::int32_datatype(), true);
4106 let column_schemas = vec![device_id];
4107 let target_partition_columns = vec!["device_id".to_string(), "device_id".to_string()];
4108
4109 let err =
4110 validate_and_collect_partition_columns(&target_partition_columns, &column_schemas)
4111 .unwrap_err();
4112
4113 assert!(err.to_string().contains("duplicate partition column"));
4114 assert!(err.to_string().contains("device_id"));
4115 }
4116
4117 fn create_expr_from_sql(sql: &str) -> CreateTableExpr {
4118 let result =
4119 ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
4120 .unwrap();
4121
4122 match &result[0] {
4123 Statement::CreateTable(create) => {
4124 expr_helper::create_to_expr(create, &QueryContext::arc()).unwrap()
4125 }
4126 _ => unreachable!(),
4127 }
4128 }
4129
4130 #[test]
4131 fn test_create_table_with_repartition_partition_num_hint() {
4132 let key = table::requests::REPARTITION_PARTITION_NUM_HINT_KEY;
4133 for (value, expected) in [
4134 (" 8 ", Some("8")),
4135 ("0", None),
4136 ("-1", None),
4137 ("4294967296", None),
4138 ] {
4139 let expr = create_expr_from_sql(&format!(
4140 "CREATE TABLE metrics (host STRING, ts TIMESTAMP TIME INDEX, PRIMARY KEY(host)) WITH ('{key}' = '{value}')"
4141 ));
4142 let result = create_table_info(&expr, vec![]);
4143 if let Some(expected) = expected {
4144 let info = result.unwrap();
4145 assert_eq!(
4146 info.meta.options.extra_options.get(key).map(String::as_str),
4147 Some(expected)
4148 );
4149 } else {
4150 assert!(
4151 result
4152 .unwrap_err()
4153 .to_string()
4154 .contains("expects a positive integer")
4155 );
4156 }
4157 }
4158 }
4159
4160 #[test]
4161 fn test_create_table_with_repartition_column_hint() {
4162 let expr = create_expr_from_sql(
4163 r"
4164CREATE TABLE metrics (
4165 host STRING,
4166 ts TIMESTAMP TIME INDEX,
4167 cpu DOUBLE,
4168 PRIMARY KEY(host)
4169)
4170WITH ('repartition.column.hint' = ' host ')",
4171 );
4172
4173 let table_info = create_table_info(&expr, vec![]).unwrap();
4174 assert_eq!(
4175 table_info
4176 .meta
4177 .options
4178 .extra_options
4179 .get(REPARTITION_COLUMN_HINT_KEY),
4180 Some(&"host".to_string())
4181 );
4182 }
4183
4184 #[test]
4185 fn test_create_table_info_rejects_non_timestamp_time_index() {
4186 let expr = CreateTableExpr {
4187 catalog_name: "greptime".to_string(),
4188 schema_name: "public".to_string(),
4189 table_name: "demo".to_string(),
4190 desc: String::new(),
4191 column_defs: vec![api::v1::ColumnDef {
4192 name: "host".to_string(),
4193 data_type: api::v1::ColumnDataType::String as i32,
4194 is_nullable: true,
4195 default_constraint: vec![],
4196 semantic_type: 0,
4197 comment: String::new(),
4198 datatype_extension: None,
4199 options: None,
4200 }],
4201 time_index: "host".to_string(),
4202 primary_keys: vec![],
4203 create_if_not_exists: false,
4204 table_options: HashMap::new(),
4205 table_id: None,
4206 engine: "mito".to_string(),
4207 };
4208
4209 let err = create_table_info(&expr, vec![]).unwrap_err();
4210 assert_eq!(
4211 common_error::ext::ErrorExt::status_code(&err),
4212 common_error::status_code::StatusCode::InvalidArguments
4213 );
4214 }
4215
4216 #[test]
4217 fn test_verify_alter_guards_entity_column_types() {
4218 let expr = create_expr_from_sql(
4219 "CREATE TABLE t (svc STRING, ts TIMESTAMP TIME INDEX) \
4220 WITH ('greptime.semantic.entity.service.id'='svc');",
4221 );
4222 let table_info = Arc::new(create_table_info(&expr, vec![]).unwrap());
4223 let alter = |kind| AlterTableExpr {
4224 catalog_name: "greptime".to_string(),
4225 schema_name: "public".to_string(),
4226 table_name: "t".to_string(),
4227 kind: Some(kind),
4228 };
4229
4230 let modify = alter(Kind::ModifyColumnTypes(api::v1::ModifyColumnTypes {
4231 modify_column_types: vec![api::v1::ModifyColumnType {
4232 column_name: "svc".to_string(),
4233 target_type: api::v1::ColumnDataType::Binary as i32,
4234 target_type_extension: None,
4235 }],
4236 }));
4237 let err = verify_alter(1, table_info.clone(), modify).unwrap_err();
4238 let msg = common_error::ext::ErrorExt::output_msg(&err);
4239 assert!(
4240 msg.contains("must keep a type that renders as a string"),
4241 "{msg}"
4242 );
4243
4244 let drop = alter(Kind::DropColumns(api::v1::DropColumns {
4247 drop_columns: vec![api::v1::DropColumn {
4248 name: "svc".to_string(),
4249 }],
4250 }));
4251 assert!(verify_alter(1, table_info, drop).unwrap());
4252 }
4253
4254 #[test]
4255 fn test_json2_supports_non_append_mode() -> std::result::Result<(), Box<dyn std::error::Error>>
4256 {
4257 for options in [
4258 "",
4259 "WITH (append_mode='false')",
4260 "WITH (append_mode='true')",
4261 "WITH (merge_mode='last_non_null')",
4262 ] {
4263 let expr = create_expr_from_sql(&format!(
4264 "CREATE TABLE monitor (payload JSON2, ts TIMESTAMP TIME INDEX) {options};"
4265 ));
4266 create_table_info(&expr, vec![])?;
4267 }
4268 Ok(())
4269 }
4270
4271 #[test]
4272 fn test_json2_cannot_be_primary_key() -> std::result::Result<(), Box<dyn std::error::Error>> {
4273 let expr = create_expr_from_sql(
4274 "CREATE TABLE monitor (payload JSON2, ts TIMESTAMP TIME INDEX, PRIMARY KEY(payload));",
4275 );
4276 let err = create_table_info(&expr, vec![])
4277 .err()
4278 .ok_or("expected JSON2 primary key rejection")?;
4279 assert!(
4280 err.to_string()
4281 .contains("JSON2 column `payload` cannot be a primary key"),
4282 "{err}"
4283 );
4284 Ok(())
4285 }
4286
4287 #[test]
4288 fn test_create_table_with_empty_repartition_column_hint() {
4289 let expr = create_expr_from_sql(
4290 r"
4291CREATE TABLE metrics (
4292 host STRING,
4293 ts TIMESTAMP TIME INDEX,
4294 cpu DOUBLE,
4295 PRIMARY KEY(host)
4296)
4297WITH ('repartition.column.hint' = '')",
4298 );
4299
4300 let err = create_table_info(&expr, vec![]).unwrap_err();
4301 assert!(
4302 err.to_string()
4303 .contains("repartition.column.hint expects exactly one column name")
4304 );
4305 }
4306
4307 #[test]
4308 fn test_create_table_with_multiple_repartition_column_hints() {
4309 let expr = create_expr_from_sql(
4310 r"
4311CREATE TABLE metrics (
4312 host STRING,
4313 region_id STRING,
4314 ts TIMESTAMP TIME INDEX,
4315 cpu DOUBLE,
4316 PRIMARY KEY(host)
4317)
4318WITH ('repartition.column.hint' = 'host,region_id')",
4319 );
4320
4321 let err = create_table_info(&expr, vec![]).unwrap_err();
4322 assert!(
4323 err.to_string()
4324 .contains("repartition.column.hint expects exactly one column name")
4325 );
4326 }
4327
4328 #[test]
4329 fn test_create_table_with_missing_repartition_column_hint() {
4330 let expr = create_expr_from_sql(
4331 r"
4332CREATE TABLE metrics (
4333 host STRING,
4334 ts TIMESTAMP TIME INDEX,
4335 cpu DOUBLE,
4336 PRIMARY KEY(host)
4337)
4338WITH ('repartition.column.hint' = 'region_id')",
4339 );
4340
4341 let err = create_table_info(&expr, vec![]).unwrap_err();
4342 assert!(
4343 err.to_string()
4344 .contains("Cannot find column by name: region")
4345 );
4346 }
4347
4348 #[test]
4349 fn test_create_table_with_time_index_repartition_column_hint() {
4350 let expr = create_expr_from_sql(
4351 r"
4352CREATE TABLE metrics (
4353 host STRING,
4354 ts TIMESTAMP TIME INDEX,
4355 cpu DOUBLE,
4356 PRIMARY KEY(host)
4357)
4358WITH ('repartition.column.hint' = 'ts')",
4359 );
4360
4361 let err = create_table_info(&expr, vec![]).unwrap_err();
4362 assert!(
4363 err.to_string()
4364 .contains("cannot set repartition.column.hint to the time index column")
4365 );
4366 }
4367
4368 #[test]
4369 fn test_create_partitioned_table_with_repartition_column_hint() {
4370 let expr = create_expr_from_sql(
4371 r"
4372CREATE TABLE metrics (
4373 host STRING,
4374 ts TIMESTAMP TIME INDEX,
4375 cpu DOUBLE,
4376 PRIMARY KEY(host)
4377)
4378WITH ('repartition.column.hint' = 'host')",
4379 );
4380
4381 let err = create_table_info(&expr, vec!["host".to_string()]).unwrap_err();
4382 assert!(
4383 err.to_string()
4384 .contains("cannot set repartition.column.hint on a table with partition metadata")
4385 );
4386 }
4387}