Skip to main content

operator/statement/
ddl.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{HashMap, HashSet};
16use std::sync::Arc;
17use std::time::Duration;
18
19use api::helper::ColumnDataTypeWrapper;
20use api::v1::alter_table_expr::Kind;
21use api::v1::meta::CreateFlowTask as PbCreateFlowTask;
22use api::v1::repartition::Source;
23use api::v1::{
24    AlterDatabaseExpr, AlterTableExpr, CreateFlowExpr, CreateTableExpr, CreateViewExpr,
25    PartitionedSource, Repartition, TargetPartitionColumns, UnpartitionedSource, column_def,
26};
27#[cfg(feature = "enterprise")]
28use api::v1::{
29    CreateTriggerExpr as PbCreateTriggerExpr, meta::CreateTriggerTask as PbCreateTriggerTask,
30};
31use catalog::CatalogManagerRef;
32use chrono::Utc;
33use common_base::regex_pattern::NAME_PATTERN_REG;
34use common_catalog::consts::{
35    DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, is_ddl_reserved_table, is_readonly_schema,
36    is_readonly_table,
37};
38use common_catalog::{format_full_flow_name, format_full_table_name};
39use common_error::ext::BoxedError;
40#[cfg(feature = "enterprise")]
41use common_meta::cache_invalidator::CacheInvalidatorRef;
42use common_meta::cache_invalidator::Context;
43use common_meta::ddl::create_flow::{
44    DEFER_ON_MISSING_SOURCE_KEY, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType,
45    INTERNAL_EVAL_OFFSET_KEY, INTERNAL_EVAL_SCHEDULE_KEY,
46};
47use common_meta::instruction::CacheIdent;
48#[cfg(feature = "enterprise")]
49use common_meta::key::TableMetadataManagerRef;
50use common_meta::key::schema_name::{SchemaName, SchemaNameKey};
51#[cfg(feature = "enterprise")]
52use common_meta::procedure_executor::ProcedureExecutorRef;
53#[cfg(feature = "enterprise")]
54use common_meta::rpc::ddl::trigger::CreateTriggerTask;
55#[cfg(feature = "enterprise")]
56use common_meta::rpc::ddl::trigger::DropTriggerTask;
57use common_meta::rpc::ddl::{
58    ALTER_LOGICAL_TABLE_IDS_EXTENSION_KEY, CreateFlowTask, CreatorGrantIntent, DdlTask,
59    DropFlowTask, DropViewTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse, TriggerReason,
60};
61use common_query::Output;
62use common_recordbatch::{RecordBatch, RecordBatches};
63use common_sql::convert::sql_value_to_value;
64use common_telemetry::{debug, info, tracing, warn};
65use common_time::{TimeToLive, Timestamp, Timezone};
66use datafusion_common::tree_node::TreeNodeVisitor;
67use datafusion_expr::LogicalPlan;
68use datafusion_expr::logical_plan::Distinct;
69use datatypes::prelude::ConcreteDataType;
70use datatypes::schema::{ColumnSchema, Schema};
71use datatypes::value::Value;
72use datatypes::vectors::{StringVector, VectorRef};
73use humantime::parse_duration;
74use partition::expr::{Operand, PartitionExpr, RestrictedOp};
75use partition::function::IcebergHash;
76use partition::multi_dim::MultiDimPartitionRule;
77use query::parser::QueryStatement;
78use query::plan::extract_and_rewrite_full_table_names;
79use query::promql::planner::NATIVE_PROMQL_EXTENSION_KEY;
80use query::query_engine::DefaultSerializer;
81use query::sql::create_table_stmt;
82use session::context::QueryContextRef;
83use session::table_name::table_idents_to_full_name;
84use snafu::{OptionExt, ResultExt, ensure};
85use sql::parser::{ParseOptions, ParserContext};
86use sql::parsers::utils::is_tql;
87use sql::statements::OptionMap;
88#[cfg(feature = "enterprise")]
89use sql::statements::alter::trigger::AlterTrigger;
90use sql::statements::alter::{AlterDatabase, AlterTable, AlterTableOperation};
91#[cfg(feature = "enterprise")]
92use sql::statements::create::trigger::CreateTrigger;
93use sql::statements::create::{
94    CreateExternalTable, CreateFlow, CreateTable, CreateTableLike, CreateView, Partitions,
95};
96use sql::statements::statement::Statement;
97use sqlparser::ast::{
98    Expr, Function, FunctionArg, FunctionArgExpr, FunctionArguments, Ident, UnaryOperator,
99    Value as ParserValue,
100};
101use store_api::metric_engine_consts::{
102    LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY,
103};
104use store_api::mito_engine_options::TTL_KEY;
105use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
106use table::TableRef;
107use table::dist_table::DistTable;
108use table::metadata::{self, TableId, TableInfo, TableMeta, TableType};
109use table::requests::{
110    AlterKind, AlterTableRequest, COMMENT_KEY, DDL_TIMEOUT, DDL_WAIT, INGEST_ROWS_RATE_LIMIT_KEY,
111    TableOptions, validate_and_normalize_annotation_options,
112};
113use table::table_name::TableName;
114use table::table_reference::TableReference;
115
116use crate::error::{
117    self, AlterExprToRequestSnafu, BuildDfLogicalPlanSnafu, CatalogSnafu, ColumnDataTypeSnafu,
118    ColumnNotFoundSnafu, ConvertSchemaSnafu, CreateLogicalTablesSnafu,
119    DeserializePartitionExprSnafu, EmptyDdlExprSnafu, ExternalSnafu, ExtractTableNamesSnafu,
120    FlowNotFoundSnafu, InvalidPartitionRuleSnafu, InvalidPartitionSnafu, InvalidSqlSnafu,
121    InvalidTableNameSnafu, InvalidViewNameSnafu, InvalidViewStmtSnafu, NotSupportedSnafu,
122    PartitionExprToPbSnafu, Result, RetentionTtlOnLogicalTableSnafu, SchemaInUseSnafu,
123    SchemaNotFoundSnafu, SchemaReadOnlySnafu, SerializePartitionExprSnafu, SubstraitCodecSnafu,
124    TableAlreadyExistsSnafu, TableDdlReservedSnafu, TableMetadataManagerSnafu, TableNotFoundSnafu,
125    TableReadOnlySnafu, UnrecognizedTableOptionSnafu, ViewAlreadyExistsSnafu,
126};
127use crate::expr_helper::{self, RepartitionRequest, RepartitionSource};
128use crate::statement::StatementExecutor;
129use crate::statement::show::create_partitions_stmt;
130use crate::utils::{to_executor_context, to_executor_context_with_origin_frontend};
131
132/// Options shared by SQL DDL submission paths.
133#[derive(Debug, Clone, Copy)]
134pub(crate) struct DdlSubmitOptions {
135    wait: bool,
136    timeout: Duration,
137}
138
139impl Default for DdlSubmitOptions {
140    fn default() -> Self {
141        Self {
142            wait: SubmitDdlTaskRequest::default_wait(),
143            timeout: SubmitDdlTaskRequest::default_timeout(),
144        }
145    }
146}
147
148#[cfg(not(feature = "enterprise"))]
149const ALLOWED_FLOW_OPTIONS: [&str; 2] = [
150    DEFER_ON_MISSING_SOURCE_KEY,
151    FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY,
152];
153
154fn build_procedure_id_output(procedure_id: &str) -> Result<Output> {
155    let vector: VectorRef = Arc::new(StringVector::from(vec![procedure_id]));
156    let schema = Arc::new(Schema::new(vec![ColumnSchema::new(
157        "Procedure ID",
158        vector.data_type(),
159        false,
160    )]));
161    let batch =
162        RecordBatch::new(schema.clone(), vec![vector]).context(error::BuildRecordBatchSnafu)?;
163    let batches =
164        RecordBatches::try_new(schema, vec![batch]).context(error::BuildRecordBatchSnafu)?;
165    Ok(Output::new_with_record_batches(batches))
166}
167
168/// Parses DDL submission options using the existing request defaults.
169pub(crate) fn parse_ddl_options(options: &OptionMap) -> Result<DdlSubmitOptions> {
170    let defaults = DdlSubmitOptions::default();
171    let wait = match options.get(DDL_WAIT) {
172        Some(value) => value.parse::<bool>().map_err(|_| {
173            InvalidSqlSnafu {
174                err_msg: format!("invalid DDL option '{DDL_WAIT}': '{value}'"),
175            }
176            .build()
177        })?,
178        None => defaults.wait,
179    };
180
181    let timeout = match options.get(DDL_TIMEOUT) {
182        Some(value) => parse_duration(value).map_err(|err| {
183            InvalidSqlSnafu {
184                err_msg: format!("invalid DDL option '{DDL_TIMEOUT}': '{value}': {err}"),
185            }
186            .build()
187        })?,
188        None => defaults.timeout,
189    };
190
191    Ok(DdlSubmitOptions { wait, timeout })
192}
193
194#[cfg(not(feature = "enterprise"))]
195fn supported_flow_options() -> String {
196    ALLOWED_FLOW_OPTIONS.join(", ")
197}
198
199fn normalize_flow_bool_option(key: &str, value: &str) -> Result<String> {
200    value
201        .trim()
202        .to_ascii_lowercase()
203        .parse::<bool>()
204        .map(|value| value.to_string())
205        .map_err(|_| {
206            InvalidSqlSnafu {
207                err_msg: format!("invalid flow option '{key}': '{value}'"),
208            }
209            .build()
210        })
211}
212
213fn validate_and_normalize_flow_options(
214    options: HashMap<String, String>,
215    eval_interval: Option<i64>,
216) -> Result<HashMap<String, String>> {
217    // Reject non-positive eval_interval (zero or negative).
218    if let Some(secs) = eval_interval
219        && secs <= 0
220    {
221        return InvalidSqlSnafu {
222            err_msg: format!("EVAL INTERVAL must be positive, got {secs} seconds"),
223        }
224        .fail();
225    }
226
227    options
228        .into_iter()
229        .map(|(key, value)| {
230            if matches!(
231                key.as_str(),
232                FlowType::FLOW_TYPE_KEY | INTERNAL_EVAL_OFFSET_KEY | INTERNAL_EVAL_SCHEDULE_KEY
233            ) {
234                return InvalidSqlSnafu {
235                    err_msg: format!("flow option '{key}' is reserved for internal use"),
236                }
237                .fail();
238            }
239
240            let normalized_value = match key.as_str() {
241                DEFER_ON_MISSING_SOURCE_KEY | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY => {
242                    normalize_flow_bool_option(&key, &value)?
243                }
244                #[cfg(feature = "enterprise")]
245                _ => value,
246                #[cfg(not(feature = "enterprise"))]
247                _ => {
248                    return InvalidSqlSnafu {
249                        err_msg: format!(
250                            "unknown flow option '{key}', supported options: {}",
251                            supported_flow_options()
252                        ),
253                    }
254                    .fail();
255                }
256            };
257
258            Ok((key, normalized_value))
259        })
260        .collect()
261}
262
263/// Validates `EVAL OFFSET` semantics at the operator boundary: an offset is
264/// only legal together with `EVAL INTERVAL` and must lie in
265/// `[0, eval_interval)`. Never modulo-normalized.
266fn validate_eval_offset(
267    eval_offset_secs: Option<i64>,
268    eval_interval_secs: Option<i64>,
269) -> Result<()> {
270    if let Some(offset_secs) = eval_offset_secs {
271        let Some(eval_interval_secs) = eval_interval_secs else {
272            return InvalidSqlSnafu {
273                err_msg: "EVAL OFFSET requires EVAL INTERVAL to be specified".to_string(),
274            }
275            .fail();
276        };
277        if !(0..eval_interval_secs).contains(&offset_secs) {
278            return InvalidSqlSnafu {
279                err_msg: format!(
280                    "EVAL OFFSET must be in range [0, EVAL INTERVAL), got {offset_secs} seconds with EVAL INTERVAL {eval_interval_secs} seconds"
281                ),
282            }
283            .fail();
284        }
285    }
286    Ok(())
287}
288
289fn determine_flow_type_for_source_state(
290    flow_name: &str,
291    flow_options: &HashMap<String, String>,
292    has_missing_source_table: bool,
293    has_instant_ttl_source_table: bool,
294    force_batching: bool,
295) -> Result<Option<FlowType>> {
296    if has_instant_ttl_source_table && force_batching {
297        // The batching scheduler cannot read instant-TTL source tables, so
298        // reject this combination even when another source table is missing.
299        return InvalidSqlSnafu {
300            err_msg: format!(
301                "flow '{}' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode; use a TTL longer than the flush interval",
302                flow_name
303            ),
304        }
305        .fail();
306    }
307
308    if has_missing_source_table {
309        let defer_on_missing_source = flow_options
310            .get(DEFER_ON_MISSING_SOURCE_KEY)
311            .is_some_and(|value| value == "true");
312        ensure!(
313            defer_on_missing_source,
314            InvalidSqlSnafu {
315                err_msg: format!(
316                    "missing source tables for flow '{}'; use WITH ({DEFER_ON_MISSING_SOURCE_KEY} = true) to create a pending flow",
317                    flow_name
318                )
319            }
320        );
321        info!(
322            "Flow `{}` is created as a pending batching flow because source tables are missing and defer_on_missing_source=true",
323            flow_name
324        );
325        return Ok(Some(FlowType::Batching));
326    }
327
328    Ok(None)
329}
330
331/// The stateless streaming runtime accepts only a single source scan wrapped by
332/// projections and filters. Keep this check local to the operator: the flow
333/// validator is intentionally private to the flow crate.
334fn is_stateless_flow_plan(plan: &LogicalPlan) -> bool {
335    let mut scans = 0;
336    let mut supported_nodes = true;
337    let result = plan.apply_with_subqueries(|node| {
338        match node {
339            LogicalPlan::TableScan(_) => scans += 1,
340            LogicalPlan::Projection(_) | LogicalPlan::Filter(_) => {}
341            LogicalPlan::Distinct(Distinct::All(_)) => {}
342            _ => supported_nodes = false,
343        }
344        Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)
345    });
346
347    result.is_ok() && supported_nodes && scans == 1
348}
349
350const INSTANT_TTL_FLOW_QUERY_ERROR: &str = "instant-TTL flow sources support only stateless projection/filter queries over one source; non-stateless queries require a persisted source";
351
352const STATELESS_FLOW_QUERY_ERROR: &str =
353    "flow streaming supports only stateless projection/filter queries over one source";
354
355impl StatementExecutor {
356    pub fn catalog_manager(&self) -> CatalogManagerRef {
357        self.catalog_manager.clone()
358    }
359
360    #[tracing::instrument(skip_all)]
361    pub async fn create_table(&self, stmt: CreateTable, ctx: QueryContextRef) -> Result<TableRef> {
362        let mut create_expr = self.prepare_create_table(&stmt, &ctx).await?;
363        self.create_table_inner(
364            &mut create_expr,
365            stmt.partitions,
366            ctx,
367            TriggerReason::Manual,
368        )
369        .await
370    }
371
372    async fn prepare_create_table(
373        &self,
374        stmt: &CreateTable,
375        ctx: &QueryContextRef,
376    ) -> Result<CreateTableExpr> {
377        let (catalog, schema, _table) = table_idents_to_full_name(&stmt.name, ctx)
378            .map_err(BoxedError::new)
379            .context(error::ExternalSnafu)?;
380
381        let schema_options = self
382            .table_metadata_manager
383            .schema_manager()
384            .get(SchemaNameKey {
385                catalog: &catalog,
386                schema: &schema,
387            })
388            .await
389            .context(TableMetadataManagerSnafu)?
390            .map(|v| v.into_inner());
391
392        let mut create_expr = expr_helper::create_to_expr(stmt, ctx)?;
393        // TTL and compaction options are handled separately; ingestion quota is database-only.
394        if let Some(schema_options) = schema_options {
395            for (key, value) in schema_options.extra_options.iter() {
396                if key == INGEST_ROWS_RATE_LIMIT_KEY || key.starts_with("compaction.") {
397                    continue;
398                }
399                create_expr
400                    .table_options
401                    .entry(key.clone())
402                    .or_insert(value.clone());
403            }
404        }
405
406        Ok(create_expr)
407    }
408
409    /// Prepares every explicit logical CREATE before submitting one batch procedure.
410    pub async fn batch_create_logical_tables(
411        &self,
412        statements: Vec<CreateTable>,
413        ctx: QueryContextRef,
414    ) -> Result<()> {
415        let mut expressions = Vec::with_capacity(statements.len());
416        let mut targets = std::collections::HashSet::new();
417        let mut group = None;
418        for statement in statements {
419            ensure!(
420                statement.engine == METRIC_ENGINE_NAME
421                    && statement.options.get(LOGICAL_TABLE_METADATA_KEY).is_some()
422                    && statement
423                        .options
424                        .value(PHYSICAL_TABLE_METADATA_KEY)
425                        .is_none(),
426                InvalidSqlSnafu {
427                    err_msg: "batch requires explicit Metric logical CREATE TABLE statements"
428                }
429            );
430            let expr = self.prepare_create_table(&statement, &ctx).await?;
431            let key = (
432                expr.catalog_name.clone(),
433                expr.schema_name.clone(),
434                expr.table_options.get(LOGICAL_TABLE_METADATA_KEY).cloned(),
435            );
436            ensure!(
437                group.as_ref().is_none_or(|group| group == &key),
438                InvalidSqlSnafu {
439                    err_msg: "batch must share catalog, schema and literal on_physical_table"
440                }
441            );
442            group = Some(key);
443            ensure!(
444                targets.insert(expr.table_name.clone()),
445                InvalidSqlSnafu {
446                    err_msg: "duplicate target in logical-table batch"
447                }
448            );
449            self.validate_logical_create(&expr, statement.partitions.as_ref(), &ctx)
450                .await?;
451            expressions.push(expr);
452        }
453        self.create_logical_tables(&expressions, ctx, TriggerReason::Manual)
454            .await?;
455        Ok(())
456    }
457
458    async fn validate_logical_create(
459        &self,
460        expr: &CreateTableExpr,
461        partitions: Option<&Partitions>,
462        ctx: &QueryContextRef,
463    ) -> Result<()> {
464        ensure_table_definition_writable(&expr.schema_name, &expr.table_name)?;
465        if let Some(partitions) = partitions
466            && !partitions.exprs.is_empty()
467        {
468            self.validate_logical_table_partition_rule(expr, partitions, ctx)
469                .await?;
470        }
471        Ok(())
472    }
473
474    #[tracing::instrument(skip_all)]
475    pub async fn create_table_like(
476        &self,
477        stmt: CreateTableLike,
478        ctx: QueryContextRef,
479    ) -> Result<TableRef> {
480        let (catalog, schema, table) = table_idents_to_full_name(&stmt.source_name, &ctx)
481            .map_err(BoxedError::new)
482            .context(error::ExternalSnafu)?;
483        let table_ref = self
484            .catalog_manager
485            .table(&catalog, &schema, &table, Some(&ctx))
486            .await
487            .context(CatalogSnafu)?
488            .context(TableNotFoundSnafu { table_name: &table })?;
489        let partition_info = self
490            .partition_manager
491            .find_physical_partition_info(table_ref.table_info().table_id())
492            .await
493            .context(error::FindTablePartitionRuleSnafu { table_name: table })?;
494
495        // CREATE TABLE LIKE also inherits database level options.
496        let schema_options = self
497            .table_metadata_manager
498            .schema_manager()
499            .get(SchemaNameKey {
500                catalog: &catalog,
501                schema: &schema,
502            })
503            .await
504            .context(TableMetadataManagerSnafu)?
505            .map(|v| v.into_inner());
506
507        let quote_style = ctx.quote_style();
508        let mut create_stmt =
509            create_table_stmt(&table_ref.table_info(), schema_options, quote_style)
510                .context(error::ParseQuerySnafu)?;
511        create_stmt.name = stmt.table_name;
512        create_stmt.if_not_exists = false;
513
514        let table_info = table_ref.table_info();
515        let partitions = create_partitions_stmt(&table_info, &partition_info.partitions)?.and_then(
516            |mut partitions| {
517                if !partitions.column_list.is_empty() {
518                    partitions.set_quote(quote_style);
519                    Some(partitions)
520                } else {
521                    None
522                }
523            },
524        );
525
526        let create_expr = &mut expr_helper::create_to_expr(&create_stmt, &ctx)?;
527        self.create_table_inner(create_expr, partitions, ctx, TriggerReason::Manual)
528            .await
529    }
530
531    #[tracing::instrument(skip_all)]
532    pub async fn create_external_table(
533        &self,
534        create_expr: CreateExternalTable,
535        ctx: QueryContextRef,
536    ) -> Result<TableRef> {
537        if create_expr.if_not_exists {
538            let (catalog_name, schema_name, table_name) =
539                table_idents_to_full_name(&create_expr.name, &ctx)
540                    .map_err(BoxedError::new)
541                    .context(ExternalSnafu)?;
542            if let Some(table) = self
543                .catalog_manager
544                .table(&catalog_name, &schema_name, &table_name, Some(&ctx))
545                .await
546                .context(CatalogSnafu)?
547            {
548                return Ok(table);
549            }
550        }
551
552        let create_expr =
553            &mut expr_helper::create_external_expr(create_expr, &ctx, &self.local_file_access)
554                .await?;
555        self.create_table_inner(create_expr, None, ctx, TriggerReason::Manual)
556            .await
557    }
558
559    /// Creates the declared-edge table with its canonical schema, entering
560    /// below the user-DDL guard ([`ensure_table_definition_writable`]).
561    /// `create_if_not_exists` makes concurrent first inserts race safely.
562    pub async fn create_declared_relationships_table(
563        &self,
564        catalog: &str,
565        query_ctx: QueryContextRef,
566    ) -> Result<TableRef> {
567        let mut expr = super::semantic_graph::build_declared_relationships_expr(catalog);
568        self.create_non_logic_table(&mut expr, None, query_ctx, TriggerReason::AutoCreate)
569            .await
570    }
571
572    #[tracing::instrument(skip_all)]
573    pub async fn create_table_inner(
574        &self,
575        create_table: &mut CreateTableExpr,
576        partitions: Option<Partitions>,
577        query_ctx: QueryContextRef,
578        trigger_reason: TriggerReason,
579    ) -> Result<TableRef> {
580        ensure_table_definition_writable(&create_table.schema_name, &create_table.table_name)?;
581
582        if create_table.engine == METRIC_ENGINE_NAME
583            && create_table
584                .table_options
585                .contains_key(LOGICAL_TABLE_METADATA_KEY)
586        {
587            self.validate_logical_create(create_table, partitions.as_ref(), &query_ctx)
588                .await?;
589            // Create logical tables
590            self.create_logical_tables(
591                std::slice::from_ref(create_table),
592                query_ctx,
593                trigger_reason,
594            )
595            .await?
596            .into_iter()
597            .next()
598            .context(error::UnexpectedSnafu {
599                violated: "expected to create logical tables",
600            })
601        } else {
602            // Create other normal table
603            self.create_non_logic_table(create_table, partitions, query_ctx, trigger_reason)
604                .await
605        }
606    }
607
608    #[tracing::instrument(skip_all)]
609    pub async fn create_non_logic_table(
610        &self,
611        create_table: &mut CreateTableExpr,
612        partitions: Option<Partitions>,
613        query_ctx: QueryContextRef,
614        trigger_reason: TriggerReason,
615    ) -> Result<TableRef> {
616        let _timer = crate::metrics::DIST_CREATE_TABLE.start_timer();
617
618        // Check if schema exists
619        let schema = self
620            .table_metadata_manager
621            .schema_manager()
622            .get(SchemaNameKey::new(
623                &create_table.catalog_name,
624                &create_table.schema_name,
625            ))
626            .await
627            .context(TableMetadataManagerSnafu)?;
628        ensure!(
629            schema.is_some(),
630            SchemaNotFoundSnafu {
631                schema_info: &create_table.schema_name,
632            }
633        );
634
635        // if table exists.
636        if let Some(table) = self
637            .catalog_manager
638            .table(
639                &create_table.catalog_name,
640                &create_table.schema_name,
641                &create_table.table_name,
642                Some(&query_ctx),
643            )
644            .await
645            .context(CatalogSnafu)?
646        {
647            return if create_table.create_if_not_exists {
648                Ok(table)
649            } else {
650                TableAlreadyExistsSnafu {
651                    table: format_full_table_name(
652                        &create_table.catalog_name,
653                        &create_table.schema_name,
654                        &create_table.table_name,
655                    ),
656                }
657                .fail()
658            };
659        }
660
661        ensure!(
662            NAME_PATTERN_REG.is_match(&create_table.table_name),
663            InvalidTableNameSnafu {
664                table_name: &create_table.table_name,
665            }
666        );
667
668        let table_name = TableName::new(
669            &create_table.catalog_name,
670            &create_table.schema_name,
671            &create_table.table_name,
672        );
673
674        let (partitions, partition_cols) = parse_partitions(create_table, partitions, &query_ctx)?;
675        let mut table_info = create_table_info(create_table, partition_cols)?;
676
677        let resp = self
678            .create_table_procedure(
679                create_table.clone(),
680                partitions,
681                table_info.clone(),
682                query_ctx,
683                trigger_reason,
684            )
685            .await?;
686
687        let table_id = resp
688            .table_ids
689            .into_iter()
690            .next()
691            .context(error::UnexpectedSnafu {
692                violated: "expected table_id",
693            })?;
694        info!("Successfully created table '{table_name}' with table id {table_id}");
695
696        table_info.ident.table_id = table_id;
697
698        let table_info = Arc::new(table_info);
699        create_table.table_id = Some(api::v1::TableId { id: table_id });
700
701        let table = DistTable::table(table_info);
702
703        Ok(table)
704    }
705
706    #[tracing::instrument(skip_all)]
707    pub async fn create_logical_tables(
708        &self,
709        create_table_exprs: &[CreateTableExpr],
710        query_context: QueryContextRef,
711        trigger_reason: TriggerReason,
712    ) -> Result<Vec<TableRef>> {
713        let _timer = crate::metrics::DIST_CREATE_TABLES.start_timer();
714        ensure!(
715            !create_table_exprs.is_empty(),
716            EmptyDdlExprSnafu {
717                name: "create logic tables"
718            }
719        );
720
721        // Retention is enforced on the physical table only. Drop it from logical
722        // creates (ingest hints, DDL exported by older versions), but keep
723        // `ttl=instant`: the inserter reads it from the logical table to forward
724        // rows to flows without persisting them.
725        let create_table_exprs = create_table_exprs
726            .iter()
727            .cloned()
728            .map(|mut create| {
729                if create
730                    .table_options
731                    .get(TTL_KEY)
732                    .is_some_and(|ttl| is_retention_ttl(ttl))
733                {
734                    create.table_options.remove(TTL_KEY);
735                }
736                create
737            })
738            .collect::<Vec<_>>();
739
740        // Check table names
741        for create_table in &create_table_exprs {
742            ensure!(
743                NAME_PATTERN_REG.is_match(&create_table.table_name),
744                InvalidTableNameSnafu {
745                    table_name: &create_table.table_name,
746                }
747            );
748        }
749
750        let raw_tables_info = create_table_exprs
751            .iter()
752            .map(|create| create_table_info(create, vec![]))
753            .collect::<Result<Vec<_>>>()?;
754        let tables_data = create_table_exprs
755            .iter()
756            .cloned()
757            .zip(raw_tables_info.iter().cloned())
758            .collect::<Vec<_>>();
759
760        let resp = self
761            .create_logical_tables_procedure(tables_data, query_context.clone(), trigger_reason)
762            .await?;
763
764        let table_ids = resp.table_ids;
765        ensure!(
766            table_ids.len() == raw_tables_info.len(),
767            CreateLogicalTablesSnafu {
768                reason: format!(
769                    "The number of tables is inconsistent with the expected number to be created, expected: {}, actual: {}",
770                    raw_tables_info.len(),
771                    table_ids.len()
772                )
773            }
774        );
775        info!("Successfully created logical tables: {:?}", table_ids);
776
777        // Reacquire table infos from catalog so logical tables inherit the latest partition
778        // metadata (e.g. partition_key_indices) from their physical tables.
779        // And the returned table info also included extra partition columns that are in physical table but not in logical table's create table expr
780        let mut tables_info = Vec::with_capacity(table_ids.len());
781        for (table_id, create_table) in table_ids.iter().zip(create_table_exprs.iter()) {
782            let table = self
783                .catalog_manager
784                .table(
785                    &create_table.catalog_name,
786                    &create_table.schema_name,
787                    &create_table.table_name,
788                    Some(&query_context),
789                )
790                .await
791                .context(CatalogSnafu)?
792                .with_context(|| TableNotFoundSnafu {
793                    table_name: format_full_table_name(
794                        &create_table.catalog_name,
795                        &create_table.schema_name,
796                        &create_table.table_name,
797                    ),
798                })?;
799
800            let table_info = table.table_info();
801            // Safety check: ensure we are returning the table info that matches the newly created table id.
802            ensure!(
803                table_info.table_id() == *table_id,
804                CreateLogicalTablesSnafu {
805                    reason: format!(
806                        "Table id mismatch after creation, expected {}, got {} for table {}",
807                        table_id,
808                        table_info.table_id(),
809                        format_full_table_name(
810                            &create_table.catalog_name,
811                            &create_table.schema_name,
812                            &create_table.table_name
813                        )
814                    )
815                }
816            );
817
818            tables_info.push(table_info);
819        }
820
821        Ok(tables_info.into_iter().map(DistTable::table).collect())
822    }
823
824    async fn validate_logical_table_partition_rule(
825        &self,
826        create_table: &CreateTableExpr,
827        partitions: &Partitions,
828        query_ctx: &QueryContextRef,
829    ) -> Result<()> {
830        let (_, mut logical_partition_exprs) =
831            parse_partitions_for_logical_validation(create_table, partitions, query_ctx)?;
832
833        let physical_table_name = create_table
834            .table_options
835            .get(LOGICAL_TABLE_METADATA_KEY)
836            .with_context(|| CreateLogicalTablesSnafu {
837                reason: format!(
838                    "expect `{LOGICAL_TABLE_METADATA_KEY}` option on creating logical table"
839                ),
840            })?;
841
842        let physical_table = self
843            .catalog_manager
844            .table(
845                &create_table.catalog_name,
846                &create_table.schema_name,
847                physical_table_name,
848                Some(query_ctx),
849            )
850            .await
851            .context(CatalogSnafu)?
852            .context(TableNotFoundSnafu {
853                table_name: physical_table_name.clone(),
854            })?;
855
856        let physical_table_info = physical_table.table_info();
857        let (partition_rule, _) = self
858            .partition_manager
859            .find_table_partition_rule(&physical_table_info)
860            .await
861            .context(error::FindTablePartitionRuleSnafu {
862                table_name: physical_table_name.clone(),
863            })?;
864
865        let multi_dim_rule = partition_rule
866            .as_ref()
867            .as_any()
868            .downcast_ref::<MultiDimPartitionRule>()
869            .context(InvalidPartitionRuleSnafu {
870                reason: "physical table partition rule is not range-based",
871            })?;
872
873        // TODO(ruihang): project physical partition exprs to logical partition column
874        let mut physical_partition_exprs = multi_dim_rule.exprs().to_vec();
875        logical_partition_exprs.sort_unstable();
876        physical_partition_exprs.sort_unstable();
877
878        ensure!(
879            physical_partition_exprs == logical_partition_exprs,
880            InvalidPartitionRuleSnafu {
881                reason: format!(
882                    "logical table partition rule must match the corresponding physical table's\n logical table partition exprs:\t\t {:?}\n physical table partition exprs:\t {:?}",
883                    logical_partition_exprs, physical_partition_exprs
884                ),
885            }
886        );
887
888        Ok(())
889    }
890
891    #[cfg(feature = "enterprise")]
892    #[tracing::instrument(skip_all)]
893    pub async fn create_trigger(
894        &self,
895        stmt: CreateTrigger,
896        query_context: QueryContextRef,
897    ) -> Result<Output> {
898        let expr = expr_helper::to_create_trigger_task_expr(stmt, &query_context)?;
899        self.create_trigger_inner(expr, query_context).await
900    }
901
902    #[cfg(feature = "enterprise")]
903    pub async fn create_trigger_inner(
904        &self,
905        expr: PbCreateTriggerExpr,
906        query_context: QueryContextRef,
907    ) -> Result<Output> {
908        self.create_trigger_procedure(expr, query_context).await?;
909        Ok(Output::new_with_affected_rows(0))
910    }
911
912    #[cfg(feature = "enterprise")]
913    async fn create_trigger_procedure(
914        &self,
915        expr: PbCreateTriggerExpr,
916        query_context: QueryContextRef,
917    ) -> Result<SubmitDdlTaskResponse> {
918        let task = CreateTriggerTask::try_from(PbCreateTriggerTask {
919            create_trigger: Some(expr),
920        })
921        .context(error::InvalidExprSnafu)?;
922
923        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
924        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_trigger(task));
925
926        self.procedure_executor
927            .submit_ddl_task(executor_context, request)
928            .await
929            .context(error::ExecuteDdlSnafu)
930    }
931
932    #[tracing::instrument(skip_all)]
933    pub async fn create_flow(
934        &self,
935        stmt: CreateFlow,
936        query_context: QueryContextRef,
937    ) -> Result<Output> {
938        // TODO(ruihang): do some verification
939        let eval_offset_secs = stmt.eval_offset;
940        let expr = expr_helper::to_create_flow_task_expr(stmt, &query_context)?;
941
942        // The typed `EVAL OFFSET` comes straight from the parsed SQL AST; it is
943        // the only trusted source of the offset at this boundary.
944        self.create_flow_procedure(expr, eval_offset_secs, query_context)
945            .await?;
946        Ok(Output::new_with_affected_rows(0))
947    }
948
949    /// Direct gRPC entry point for `CREATE FLOW`.
950    pub async fn create_flow_inner(
951        &self,
952        expr: CreateFlowExpr,
953        query_context: QueryContextRef,
954    ) -> Result<Output> {
955        self.create_flow_procedure(expr, None, query_context)
956            .await?;
957        Ok(Output::new_with_affected_rows(0))
958    }
959
960    async fn create_flow_procedure(
961        &self,
962        mut expr: CreateFlowExpr,
963        eval_offset_secs: Option<i64>,
964        query_context: QueryContextRef,
965    ) -> Result<SubmitDdlTaskResponse> {
966        let eval_interval_secs = expr.eval_interval.as_ref().map(|e| e.seconds);
967
968        // Reject non-positive eval_interval (zero or negative).
969        if let Some(secs) = eval_interval_secs
970            && secs <= 0
971        {
972            return InvalidSqlSnafu {
973                err_msg: format!("EVAL INTERVAL must be positive, got {secs} seconds"),
974            }
975            .fail();
976        }
977
978        // Validate EVAL OFFSET semantics at the operator boundary (defense in
979        // depth; the parser already enforces them deterministically). The
980        // offset arrives as a typed parameter, never from `flow_options`.
981        validate_eval_offset(eval_offset_secs, eval_interval_secs)?;
982
983        // Validate user options. `validate_and_normalize_flow_options` also
984        // rejects the internal transport keys, so a spoofed key smuggled into
985        // the expr can never be honored here.
986        expr.flow_options =
987            validate_and_normalize_flow_options(expr.flow_options, eval_interval_secs)?;
988
989        let flow_type = self
990            .determine_flow_type(&expr, query_context.clone())
991            .await?;
992        info!("determined flow={} type: {:#?}", expr.flow_name, flow_type);
993
994        expr.flow_options
995            .insert(FlowType::FLOW_TYPE_KEY.to_string(), flow_type.to_string());
996
997        let mut task = CreateFlowTask::try_from(PbCreateFlowTask {
998            create_flow: Some(expr),
999        })
1000        .context(error::InvalidExprSnafu)?;
1001        task.eval_offset_secs = eval_offset_secs;
1002        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1003        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_flow(task));
1004
1005        self.procedure_executor
1006            .submit_ddl_task(executor_context, request)
1007            .await
1008            .context(error::ExecuteDdlSnafu)
1009    }
1010
1011    /// Determines the flow type from source-table state, schedule requirements,
1012    /// and SQL shape.
1013    ///
1014    /// Aggregates and persisted-source DISTINCT use batching. Plain DISTINCT is request-local only
1015    /// for instant-TTL sources, where it retains the legacy streaming compatibility behavior.
1016    async fn determine_flow_type(
1017        &self,
1018        expr: &CreateFlowExpr,
1019        query_ctx: QueryContextRef,
1020    ) -> Result<FlowType> {
1021        let has_eval_interval = expr.eval_interval.is_some();
1022
1023        let mut has_missing_source_table = false;
1024        let mut has_instant_ttl_source_table = false;
1025
1026        for src_table_name in &expr.source_table_names {
1027            let table = self
1028                .catalog_manager()
1029                .table(
1030                    &src_table_name.catalog_name,
1031                    &src_table_name.schema_name,
1032                    &src_table_name.table_name,
1033                    Some(&query_ctx),
1034                )
1035                .await
1036                .map_err(BoxedError::new)
1037                .context(ExternalSnafu)?;
1038
1039            let Some(table) = table else {
1040                has_missing_source_table = true;
1041                continue;
1042            };
1043
1044            if table.table_info().meta.options.ttl == Some(common_time::TimeToLive::Instant) {
1045                warn!(
1046                    "Source table `{}` for flow `{}`'s ttl=instant, {}",
1047                    format_full_table_name(
1048                        &src_table_name.catalog_name,
1049                        &src_table_name.schema_name,
1050                        &src_table_name.table_name
1051                    ),
1052                    expr.flow_name,
1053                    if has_eval_interval {
1054                        "rejecting flow because EVAL INTERVAL requires the batching scheduler which does not support instant TTL"
1055                    } else {
1056                        "fallback to streaming mode"
1057                    }
1058                );
1059                has_instant_ttl_source_table = true;
1060            }
1061        }
1062
1063        if let Some(flow_type) = determine_flow_type_for_source_state(
1064            &expr.flow_name,
1065            &expr.flow_options,
1066            has_missing_source_table,
1067            has_instant_ttl_source_table,
1068            has_eval_interval,
1069        )? {
1070            return Ok(flow_type);
1071        }
1072
1073        // A flow with `EVAL INTERVAL` (and therefore possibly `EVAL OFFSET`)
1074        // always uses the batching scheduler: the fixed epoch-phase schedule is
1075        // only honored by the batching scheduler, regardless of the SQL shape.
1076        // Non-aggregate SQL is supported as an explicit full-query flow (the
1077        // batching engine requires a time-window expression or EVAL INTERVAL).
1078        if has_eval_interval {
1079            return Ok(FlowType::Batching);
1080        }
1081
1082        let engine = &self.query_engine;
1083        let stmts = ParserContext::create_with_dialect(
1084            &expr.sql,
1085            query_ctx.sql_dialect(),
1086            ParseOptions::default(),
1087        )
1088        .map_err(BoxedError::new)
1089        .context(ExternalSnafu)?;
1090
1091        ensure!(
1092            stmts.len() == 1,
1093            InvalidSqlSnafu {
1094                err_msg: format!("Expect only one statement, found {}", stmts.len())
1095            }
1096        );
1097        let stmt = &stmts[0];
1098
1099        let is_tql_query = is_tql(query_ctx.sql_dialect(), &expr.sql)
1100            .map_err(BoxedError::new)
1101            .context(ExternalSnafu)?;
1102        if is_tql_query {
1103            if has_instant_ttl_source_table {
1104                return InvalidSqlSnafu {
1105                    err_msg: INSTANT_TTL_FLOW_QUERY_ERROR.to_string(),
1106                }
1107                .fail();
1108            }
1109            return Ok(FlowType::Batching);
1110        }
1111
1112        // Plan before selecting a mode. In particular, instant-TTL sources
1113        // must not bypass validation of the stateless streaming subset.
1114        let plan = match stmt {
1115            Statement::Tql(_) => {
1116                if has_instant_ttl_source_table {
1117                    return InvalidSqlSnafu {
1118                        err_msg: INSTANT_TTL_FLOW_QUERY_ERROR.to_string(),
1119                    }
1120                    .fail();
1121                }
1122                return Ok(FlowType::Batching);
1123            }
1124            _ => engine
1125                .planner()
1126                .plan(&QueryStatement::Sql(stmt.clone()), query_ctx)
1127                .await
1128                .map_err(BoxedError::new)
1129                .context(ExternalSnafu)?,
1130        };
1131
1132        /// Visitor to classify aggregation and DISTINCT separately. Plain DISTINCT is request
1133        /// local only for instant-TTL sources; persisted-source DISTINCT keeps batching semantics.
1134        struct FindQueryShape {
1135            is_aggr: bool,
1136            is_distinct: bool,
1137            has_distinct_on: bool,
1138        }
1139
1140        impl TreeNodeVisitor<'_> for FindQueryShape {
1141            type Node = LogicalPlan;
1142            fn f_down(
1143                &mut self,
1144                node: &Self::Node,
1145            ) -> datafusion_common::Result<datafusion_common::tree_node::TreeNodeRecursion>
1146            {
1147                match node {
1148                    LogicalPlan::Aggregate(_) => self.is_aggr = true,
1149                    LogicalPlan::Distinct(Distinct::All(_)) => self.is_distinct = true,
1150                    LogicalPlan::Distinct(Distinct::On(_)) => self.has_distinct_on = true,
1151                    _ => (),
1152                }
1153                Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue)
1154            }
1155        }
1156
1157        let mut query_shape = FindQueryShape {
1158            is_aggr: false,
1159            is_distinct: false,
1160            has_distinct_on: false,
1161        };
1162
1163        plan.visit_with_subqueries(&mut query_shape)
1164            .context(BuildDfLogicalPlanSnafu)?;
1165        if query_shape.has_distinct_on {
1166            return InvalidSqlSnafu {
1167                err_msg: if has_instant_ttl_source_table {
1168                    INSTANT_TTL_FLOW_QUERY_ERROR.to_string()
1169                } else {
1170                    STATELESS_FLOW_QUERY_ERROR.to_string()
1171                },
1172            }
1173            .fail();
1174        }
1175        if query_shape.is_aggr {
1176            return if has_instant_ttl_source_table {
1177                InvalidSqlSnafu {
1178                    err_msg: INSTANT_TTL_FLOW_QUERY_ERROR.to_string(),
1179                }
1180                .fail()
1181            } else {
1182                Ok(FlowType::Batching)
1183            };
1184        }
1185        if query_shape.is_distinct && !has_instant_ttl_source_table {
1186            return Ok(FlowType::Batching);
1187        }
1188
1189        ensure!(
1190            is_stateless_flow_plan(&plan),
1191            InvalidSqlSnafu {
1192                err_msg: if has_instant_ttl_source_table {
1193                    INSTANT_TTL_FLOW_QUERY_ERROR.to_string()
1194                } else {
1195                    STATELESS_FLOW_QUERY_ERROR.to_string()
1196                },
1197            }
1198        );
1199        Ok(FlowType::Streaming)
1200    }
1201
1202    #[tracing::instrument(skip_all)]
1203    pub async fn create_view(
1204        &self,
1205        create_view: CreateView,
1206        ctx: QueryContextRef,
1207    ) -> Result<TableRef> {
1208        // convert input into logical plan
1209        let logical_plan = match &*create_view.query {
1210            Statement::Query(query) => {
1211                self.plan(
1212                    &QueryStatement::Sql(Statement::Query(query.clone())),
1213                    ctx.clone(),
1214                )
1215                .await?
1216            }
1217            Statement::Tql(query) => {
1218                // The serialized view plan must not depend on the native execution extension.
1219                let mut view_ctx = ctx.as_ref().clone();
1220                view_ctx.set_extension(NATIVE_PROMQL_EXTENSION_KEY, "false");
1221                self.plan_tql(query.clone(), &Arc::new(view_ctx)).await?
1222            }
1223            _ => {
1224                return InvalidViewStmtSnafu {}.fail();
1225            }
1226        };
1227        // Save the definition for `show create view`.
1228        let definition = create_view.to_string();
1229
1230        // Save the columns in plan, it may changed when the schemas of tables in plan
1231        // are altered.
1232        let schema: Schema = logical_plan
1233            .schema()
1234            .clone()
1235            .try_into()
1236            .context(ConvertSchemaSnafu)?;
1237        let plan_columns: Vec<_> = schema
1238            .column_schemas()
1239            .iter()
1240            .map(|c| c.name.clone())
1241            .collect();
1242
1243        let columns: Vec<_> = create_view
1244            .columns
1245            .iter()
1246            .map(|ident| ident.to_string())
1247            .collect();
1248
1249        // Validate columns
1250        if !columns.is_empty() {
1251            ensure!(
1252                columns.len() == plan_columns.len(),
1253                error::ViewColumnsMismatchSnafu {
1254                    view_name: create_view.name.to_string(),
1255                    expected: plan_columns.len(),
1256                    actual: columns.len(),
1257                }
1258            );
1259        }
1260
1261        // Extract the table names from the original plan
1262        // and rewrite them as fully qualified names.
1263        let (table_names, plan) = extract_and_rewrite_full_table_names(logical_plan, ctx.clone())
1264            .context(ExtractTableNamesSnafu)?;
1265
1266        let table_names = table_names.into_iter().map(|t| t.into()).collect();
1267
1268        // TODO(dennis): we don't save the optimized plan yet,
1269        // because there are some serialization issue with our own defined plan node (such as `MergeScanLogicalPlan`).
1270        // When the issues are fixed, we can use the `optimized_plan` instead.
1271        // let optimized_plan = self.optimize_logical_plan(logical_plan)?.unwrap_df_plan();
1272
1273        // encode logical plan
1274        let encoded_plan = DFLogicalSubstraitConvertor
1275            .encode(&plan, DefaultSerializer)
1276            .context(SubstraitCodecSnafu)?;
1277
1278        let expr = expr_helper::to_create_view_expr(
1279            create_view,
1280            encoded_plan.to_vec(),
1281            table_names,
1282            columns,
1283            plan_columns,
1284            definition,
1285            ctx.clone(),
1286        )?;
1287
1288        // TODO(dennis): validate the logical plan
1289        self.create_view_by_expr(expr, ctx).await
1290    }
1291
1292    pub async fn create_view_by_expr(
1293        &self,
1294        expr: CreateViewExpr,
1295        ctx: QueryContextRef,
1296    ) -> Result<TableRef> {
1297        // A view could otherwise squat a reserved name and block the canonical
1298        // table's first-write creation.
1299        ensure_table_definition_writable(&expr.schema_name, &expr.view_name)?;
1300        ensure! {
1301            !(expr.create_if_not_exists & expr.or_replace),
1302            InvalidSqlSnafu {
1303                err_msg: "syntax error Create Or Replace and If Not Exist cannot be used together",
1304            }
1305        };
1306        let _timer = crate::metrics::DIST_CREATE_VIEW.start_timer();
1307
1308        let schema_exists = self
1309            .table_metadata_manager
1310            .schema_manager()
1311            .exists(SchemaNameKey::new(&expr.catalog_name, &expr.schema_name))
1312            .await
1313            .context(TableMetadataManagerSnafu)?;
1314
1315        ensure!(
1316            schema_exists,
1317            SchemaNotFoundSnafu {
1318                schema_info: &expr.schema_name,
1319            }
1320        );
1321
1322        // if view or table exists.
1323        if let Some(table) = self
1324            .catalog_manager
1325            .table(
1326                &expr.catalog_name,
1327                &expr.schema_name,
1328                &expr.view_name,
1329                Some(&ctx),
1330            )
1331            .await
1332            .context(CatalogSnafu)?
1333        {
1334            let table_type = table.table_info().table_type;
1335
1336            match (table_type, expr.create_if_not_exists, expr.or_replace) {
1337                (TableType::View, true, false) => {
1338                    return Ok(table);
1339                }
1340                (TableType::View, false, false) => {
1341                    return ViewAlreadyExistsSnafu {
1342                        name: format_full_table_name(
1343                            &expr.catalog_name,
1344                            &expr.schema_name,
1345                            &expr.view_name,
1346                        ),
1347                    }
1348                    .fail();
1349                }
1350                (TableType::View, _, true) => {
1351                    // Try to replace an exists view
1352                }
1353                _ => {
1354                    return TableAlreadyExistsSnafu {
1355                        table: format_full_table_name(
1356                            &expr.catalog_name,
1357                            &expr.schema_name,
1358                            &expr.view_name,
1359                        ),
1360                    }
1361                    .fail();
1362                }
1363            }
1364        }
1365
1366        ensure!(
1367            NAME_PATTERN_REG.is_match(&expr.view_name),
1368            InvalidViewNameSnafu {
1369                name: expr.view_name.clone(),
1370            }
1371        );
1372
1373        let view_name = TableName::new(&expr.catalog_name, &expr.schema_name, &expr.view_name);
1374
1375        let mut view_info = TableInfo {
1376            ident: metadata::TableIdent {
1377                // The view id of distributed table is assigned by Meta, set "0" here as a placeholder.
1378                table_id: 0,
1379                version: 0,
1380            },
1381            name: expr.view_name.clone(),
1382            desc: None,
1383            catalog_name: expr.catalog_name.clone(),
1384            schema_name: expr.schema_name.clone(),
1385            // The meta doesn't make sense for views, so using a default one.
1386            meta: TableMeta::empty(),
1387            table_type: TableType::View,
1388        };
1389
1390        let executor_context = to_executor_context(ctx, TriggerReason::Manual);
1391        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_view(expr, view_info.clone()));
1392
1393        let resp = self
1394            .procedure_executor
1395            .submit_ddl_task(executor_context, request)
1396            .await
1397            .context(error::ExecuteDdlSnafu)?;
1398
1399        debug!(
1400            "Submit creating view '{view_name}' task response: {:?}",
1401            resp
1402        );
1403
1404        let view_id = resp
1405            .table_ids
1406            .into_iter()
1407            .next()
1408            .context(error::UnexpectedSnafu {
1409                violated: "expected table_id",
1410            })?;
1411        info!("Successfully created view '{view_name}' with view id {view_id}");
1412
1413        view_info.ident.table_id = view_id;
1414
1415        let view_info = Arc::new(view_info);
1416
1417        let table = DistTable::table(view_info);
1418
1419        // Invalidates local cache ASAP.
1420        self.cache_invalidator
1421            .invalidate(
1422                &Context::default(),
1423                &[
1424                    CacheIdent::TableId(view_id),
1425                    CacheIdent::TableName(view_name.clone()),
1426                ],
1427            )
1428            .await
1429            .context(error::InvalidateTableCacheSnafu)?;
1430
1431        Ok(table)
1432    }
1433
1434    #[tracing::instrument(skip_all)]
1435    pub async fn drop_flow(
1436        &self,
1437        catalog_name: String,
1438        flow_name: String,
1439        drop_if_exists: bool,
1440        query_context: QueryContextRef,
1441    ) -> Result<Output> {
1442        if let Some(flow) = self
1443            .flow_metadata_manager
1444            .flow_name_manager()
1445            .get(&catalog_name, &flow_name)
1446            .await
1447            .context(error::TableMetadataManagerSnafu)?
1448        {
1449            let flow_id = flow.flow_id();
1450            let task = DropFlowTask {
1451                catalog_name,
1452                flow_name,
1453                flow_id,
1454                drop_if_exists,
1455            };
1456            self.drop_flow_procedure(task, query_context).await?;
1457
1458            Ok(Output::new_with_affected_rows(0))
1459        } else if drop_if_exists {
1460            Ok(Output::new_with_affected_rows(0))
1461        } else {
1462            FlowNotFoundSnafu {
1463                flow_name: format_full_flow_name(&catalog_name, &flow_name),
1464            }
1465            .fail()
1466        }
1467    }
1468
1469    async fn drop_flow_procedure(
1470        &self,
1471        expr: DropFlowTask,
1472        query_context: QueryContextRef,
1473    ) -> Result<SubmitDdlTaskResponse> {
1474        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1475        let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_flow(expr));
1476
1477        self.procedure_executor
1478            .submit_ddl_task(executor_context, request)
1479            .await
1480            .context(error::ExecuteDdlSnafu)
1481    }
1482
1483    #[cfg(feature = "enterprise")]
1484    #[tracing::instrument(skip_all)]
1485    pub(super) async fn drop_trigger(
1486        &self,
1487        catalog_name: String,
1488        trigger_name: String,
1489        drop_if_exists: bool,
1490        query_context: QueryContextRef,
1491    ) -> Result<Output> {
1492        let task = DropTriggerTask {
1493            catalog_name,
1494            trigger_name,
1495            drop_if_exists,
1496        };
1497        self.drop_trigger_procedure(task, query_context).await?;
1498        Ok(Output::new_with_affected_rows(0))
1499    }
1500
1501    #[cfg(feature = "enterprise")]
1502    async fn drop_trigger_procedure(
1503        &self,
1504        expr: DropTriggerTask,
1505        query_context: QueryContextRef,
1506    ) -> Result<SubmitDdlTaskResponse> {
1507        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1508        let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_trigger(expr));
1509
1510        self.procedure_executor
1511            .submit_ddl_task(executor_context, request)
1512            .await
1513            .context(error::ExecuteDdlSnafu)
1514    }
1515
1516    /// Drop a view
1517    #[tracing::instrument(skip_all)]
1518    pub async fn drop_view(
1519        &self,
1520        catalog: String,
1521        schema: String,
1522        view: String,
1523        drop_if_exists: bool,
1524        query_context: QueryContextRef,
1525    ) -> Result<Output> {
1526        let view_info = if let Some(view) = self
1527            .catalog_manager
1528            .table(&catalog, &schema, &view, None)
1529            .await
1530            .context(CatalogSnafu)?
1531        {
1532            view.table_info()
1533        } else if drop_if_exists {
1534            // DROP VIEW IF EXISTS meets view not found - ignored
1535            return Ok(Output::new_with_affected_rows(0));
1536        } else {
1537            return TableNotFoundSnafu {
1538                table_name: format_full_table_name(&catalog, &schema, &view),
1539            }
1540            .fail();
1541        };
1542
1543        // Ensure the exists one is view, we can't drop other table types
1544        ensure!(
1545            view_info.table_type == TableType::View,
1546            error::InvalidViewSnafu {
1547                msg: "not a view",
1548                view_name: format_full_table_name(&catalog, &schema, &view),
1549            }
1550        );
1551
1552        let view_id = view_info.table_id();
1553        let view_name = TableName::new(&catalog, &schema, &view);
1554
1555        let task = DropViewTask {
1556            catalog,
1557            schema,
1558            view,
1559            view_id,
1560            drop_if_exists,
1561        };
1562
1563        self.drop_view_procedure(task, query_context).await?;
1564
1565        // Invalidates local cache ASAP.
1566        self.cache_invalidator
1567            .invalidate(
1568                &Context::default(),
1569                &[
1570                    CacheIdent::TableId(view_id),
1571                    CacheIdent::TableName(view_name),
1572                ],
1573            )
1574            .await
1575            .context(error::InvalidateTableCacheSnafu)?;
1576
1577        Ok(Output::new_with_affected_rows(0))
1578    }
1579
1580    /// Submit [DropViewTask] to procedure executor.
1581    async fn drop_view_procedure(
1582        &self,
1583        expr: DropViewTask,
1584        query_context: QueryContextRef,
1585    ) -> Result<SubmitDdlTaskResponse> {
1586        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
1587        let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_view(expr));
1588
1589        self.procedure_executor
1590            .submit_ddl_task(executor_context, request)
1591            .await
1592            .context(error::ExecuteDdlSnafu)
1593    }
1594
1595    #[tracing::instrument(skip_all)]
1596    pub async fn alter_logical_tables(
1597        &self,
1598        alter_table_exprs: Vec<AlterTableExpr>,
1599        query_context: QueryContextRef,
1600        trigger_reason: TriggerReason,
1601    ) -> Result<Output> {
1602        let mut tables = Vec::with_capacity(alter_table_exprs.len());
1603        for expr in alter_table_exprs {
1604            // Get table_id from catalog_manager
1605            let catalog = if expr.catalog_name.is_empty() {
1606                query_context.current_catalog()
1607            } else {
1608                &expr.catalog_name
1609            };
1610            let schema = if expr.schema_name.is_empty() {
1611                query_context.current_schema()
1612            } else {
1613                expr.schema_name.clone()
1614            };
1615            let table_name = &expr.table_name;
1616            let table = self
1617                .catalog_manager
1618                .table(catalog, &schema, table_name, Some(&query_context))
1619                .await
1620                .context(CatalogSnafu)?
1621                .with_context(|| TableNotFoundSnafu {
1622                    table_name: format_full_table_name(catalog, &schema, table_name),
1623                })?;
1624            let table_id = table.table_info().ident.table_id;
1625            tables.push((table_id, expr));
1626        }
1627        self.alter_logical_tables_with_ids(tables, query_context, trigger_reason)
1628            .await
1629    }
1630
1631    /// Alters the captured table generations, rejecting concurrent drop/recreate under DDL locks.
1632    pub(crate) async fn alter_logical_tables_with_ids(
1633        &self,
1634        tables: Vec<(TableId, AlterTableExpr)>,
1635        query_context: QueryContextRef,
1636        trigger_reason: TriggerReason,
1637    ) -> Result<Output> {
1638        let _timer = crate::metrics::DIST_ALTER_TABLES.start_timer();
1639        ensure!(
1640            !tables.is_empty(),
1641            EmptyDdlExprSnafu {
1642                name: "alter logical tables"
1643            }
1644        );
1645        let mut groups: HashMap<TableId, Vec<(TableId, AlterTableExpr)>> = HashMap::new();
1646        for (table_id, expr) in tables {
1647            let physical_table_id = self
1648                .table_metadata_manager
1649                .table_route_manager()
1650                .get_physical_table_id(table_id)
1651                .await
1652                .context(TableMetadataManagerSnafu)?;
1653            groups
1654                .entry(physical_table_id)
1655                .or_default()
1656                .push((table_id, expr));
1657        }
1658
1659        // Submit procedure for each physical table
1660        let mut handles = Vec::with_capacity(groups.len());
1661        for (physical_table_id, exprs) in groups {
1662            let fut = self.alter_logical_tables_procedure(
1663                physical_table_id,
1664                exprs,
1665                query_context.clone(),
1666                trigger_reason,
1667            );
1668            handles.push(fut);
1669        }
1670        let _results = futures::future::try_join_all(handles).await?;
1671
1672        Ok(Output::new_with_affected_rows(0))
1673    }
1674
1675    #[tracing::instrument(skip_all)]
1676    pub async fn drop_table(
1677        &self,
1678        table_name: TableName,
1679        drop_if_exists: bool,
1680        query_context: QueryContextRef,
1681    ) -> Result<Output> {
1682        // Reserved for grpc call
1683        self.drop_tables(&[table_name], drop_if_exists, query_context)
1684            .await
1685    }
1686
1687    #[tracing::instrument(skip_all)]
1688    pub async fn drop_tables(
1689        &self,
1690        table_names: &[TableName],
1691        drop_if_exists: bool,
1692        query_context: QueryContextRef,
1693    ) -> Result<Output> {
1694        let mut tables = Vec::with_capacity(table_names.len());
1695        for table_name in table_names {
1696            ensure_table_writable(&table_name.schema_name, &table_name.table_name)?;
1697
1698            if let Some(table) = self
1699                .catalog_manager
1700                .table(
1701                    &table_name.catalog_name,
1702                    &table_name.schema_name,
1703                    &table_name.table_name,
1704                    Some(&query_context),
1705                )
1706                .await
1707                .context(CatalogSnafu)?
1708            {
1709                tables.push(table.table_info().table_id());
1710            } else if drop_if_exists {
1711                // DROP TABLE IF EXISTS meets table not found - ignored
1712                continue;
1713            } else {
1714                return TableNotFoundSnafu {
1715                    table_name: table_name.to_string(),
1716                }
1717                .fail();
1718            }
1719        }
1720
1721        for (table_name, table_id) in table_names.iter().zip(tables.into_iter()) {
1722            self.drop_table_procedure(table_name, table_id, drop_if_exists, query_context.clone())
1723                .await?;
1724
1725            // Invalidates local cache ASAP.
1726            self.cache_invalidator
1727                .invalidate(
1728                    &Context::default(),
1729                    &[
1730                        CacheIdent::TableId(table_id),
1731                        CacheIdent::TableName(table_name.clone()),
1732                    ],
1733                )
1734                .await
1735                .context(error::InvalidateTableCacheSnafu)?;
1736        }
1737        Ok(Output::new_with_affected_rows(0))
1738    }
1739
1740    #[cfg(feature = "enterprise")]
1741    #[tracing::instrument(skip_all)]
1742    pub async fn undrop_table(
1743        &self,
1744        table_name: TableName,
1745        query_context: QueryContextRef,
1746    ) -> Result<Output> {
1747        execute_undrop_table(
1748            &self.table_metadata_manager,
1749            &self.procedure_executor,
1750            &self.cache_invalidator,
1751            table_name,
1752            query_context,
1753        )
1754        .await
1755    }
1756
1757    #[tracing::instrument(skip_all)]
1758    pub async fn drop_database(
1759        &self,
1760        catalog: String,
1761        schema: String,
1762        drop_if_exists: bool,
1763        query_context: QueryContextRef,
1764    ) -> Result<Output> {
1765        ensure!(
1766            !is_readonly_schema(&schema),
1767            SchemaReadOnlySnafu { name: schema }
1768        );
1769
1770        if self
1771            .catalog_manager
1772            .schema_exists(&catalog, &schema, None)
1773            .await
1774            .context(CatalogSnafu)?
1775        {
1776            if schema == query_context.current_schema() {
1777                SchemaInUseSnafu { name: schema }.fail()
1778            } else {
1779                self.drop_database_procedure(catalog, schema, drop_if_exists, query_context)
1780                    .await?;
1781
1782                Ok(Output::new_with_affected_rows(0))
1783            }
1784        } else if drop_if_exists {
1785            // DROP TABLE IF EXISTS meets table not found - ignored
1786            Ok(Output::new_with_affected_rows(0))
1787        } else {
1788            SchemaNotFoundSnafu {
1789                schema_info: schema,
1790            }
1791            .fail()
1792        }
1793    }
1794
1795    pub async fn truncate_table(
1796        &self,
1797        table_name: TableName,
1798        time_ranges: Vec<(Timestamp, Timestamp)>,
1799        query_context: QueryContextRef,
1800    ) -> Result<Output> {
1801        self.truncate_table_with_options(
1802            table_name,
1803            time_ranges,
1804            DdlSubmitOptions::default(),
1805            query_context,
1806        )
1807        .await
1808    }
1809
1810    /// Truncates a table, optionally waiting for the submitted procedure to finish.
1811    #[tracing::instrument(name = "truncate_table", skip_all)]
1812    pub(crate) async fn truncate_table_with_options(
1813        &self,
1814        table_name: TableName,
1815        time_ranges: Vec<(Timestamp, Timestamp)>,
1816        ddl_options: DdlSubmitOptions,
1817        query_context: QueryContextRef,
1818    ) -> Result<Output> {
1819        ensure_table_writable(&table_name.schema_name, &table_name.table_name)?;
1820
1821        let table = self
1822            .catalog_manager
1823            .table(
1824                &table_name.catalog_name,
1825                &table_name.schema_name,
1826                &table_name.table_name,
1827                Some(&query_context),
1828            )
1829            .await
1830            .context(CatalogSnafu)?
1831            .with_context(|| TableNotFoundSnafu {
1832                table_name: table_name.to_string(),
1833            })?;
1834        let table_id = table.table_info().table_id();
1835        let response = self
1836            .truncate_table_procedure(
1837                &table_name,
1838                table_id,
1839                time_ranges,
1840                ddl_options,
1841                query_context.clone(),
1842            )
1843            .await?;
1844
1845        if !ddl_options.wait {
1846            let procedure_id = String::from_utf8_lossy(&response.key).into_owned();
1847            let output = build_procedure_id_output(&procedure_id)?;
1848            query_context.set_last_procedure_id(procedure_id);
1849            return Ok(output);
1850        }
1851
1852        Ok(Output::new_with_affected_rows(0))
1853    }
1854
1855    #[tracing::instrument(skip_all)]
1856    pub async fn alter_table(
1857        &self,
1858        alter_table: AlterTable,
1859        query_context: QueryContextRef,
1860    ) -> Result<Output> {
1861        if matches!(
1862            alter_table.alter_operation(),
1863            AlterTableOperation::Repartition { .. } | AlterTableOperation::Partition { .. }
1864        ) {
1865            let request = expr_helper::to_repartition_request(alter_table, &query_context)?;
1866            return self.repartition_table(request, &query_context).await;
1867        }
1868
1869        let expr = expr_helper::to_alter_table_expr(alter_table, &query_context)?;
1870        self.alter_table_inner(expr, query_context, TriggerReason::Manual)
1871            .await
1872    }
1873
1874    #[tracing::instrument(skip_all)]
1875    pub async fn repartition_table(
1876        &self,
1877        request: RepartitionRequest,
1878        query_context: &QueryContextRef,
1879    ) -> Result<Output> {
1880        // Check if the schema is read-only.
1881        ensure_table_definition_writable(&request.schema_name, &request.table_name)?;
1882
1883        let table_ref = TableReference::full(
1884            &request.catalog_name,
1885            &request.schema_name,
1886            &request.table_name,
1887        );
1888        // Get the table from the catalog.
1889        let table = self
1890            .catalog_manager
1891            .table(
1892                &request.catalog_name,
1893                &request.schema_name,
1894                &request.table_name,
1895                Some(query_context),
1896            )
1897            .await
1898            .context(CatalogSnafu)?
1899            .with_context(|| TableNotFoundSnafu {
1900                table_name: table_ref.to_string(),
1901            })?;
1902        let table_id = table.table_info().ident.table_id;
1903        // Get existing partition expressions from the table route.
1904        let (physical_table_id, physical_table_route) = self
1905            .table_metadata_manager
1906            .table_route_manager()
1907            .get_physical_table_route(table_id)
1908            .await
1909            .context(TableMetadataManagerSnafu)?;
1910
1911        ensure!(
1912            physical_table_id == table_id,
1913            NotSupportedSnafu {
1914                feat: "REPARTITION on logical tables"
1915            }
1916        );
1917
1918        let table_info = table.table_info();
1919        let existing_partition_columns = table_info.meta.partition_columns().collect::<Vec<_>>();
1920        let column_schemas = table_info.meta.schema.column_schemas();
1921        // `REPARTITION ... ON COLUMNS` uses overwrite semantics: the provided
1922        // columns are the full target partition columns, not an extension of the
1923        // current ones. Therefore source expressions are converted with the
1924        // existing partition columns, while target expressions and the final
1925        // partition rule are validated against this effective target column set.
1926        let target_partition_columns = match &request.source {
1927            RepartitionSource::Partitions {
1928                target_partition_columns,
1929                ..
1930            } => {
1931                ensure!(
1932                    !existing_partition_columns.is_empty(),
1933                    InvalidPartitionRuleSnafu {
1934                        reason: format!(
1935                            "table {} does not have partition columns, cannot repartition",
1936                            table_ref
1937                        )
1938                    }
1939                );
1940
1941                if let Some(target_partition_columns) = target_partition_columns {
1942                    ensure!(
1943                        !target_partition_columns.is_empty(),
1944                        InvalidPartitionRuleSnafu {
1945                            reason: "ON COLUMNS requires at least one partition column"
1946                        }
1947                    );
1948                    validate_and_collect_partition_columns(
1949                        target_partition_columns,
1950                        column_schemas,
1951                    )?
1952                } else {
1953                    existing_partition_columns.clone()
1954                }
1955            }
1956            RepartitionSource::Unpartitioned { partition_columns } => {
1957                ensure!(
1958                    !partition_columns.is_empty(),
1959                    InvalidPartitionRuleSnafu {
1960                        reason: "PARTITION ON COLUMNS requires at least one partition column"
1961                    }
1962                );
1963                ensure!(
1964                    existing_partition_columns.is_empty(),
1965                    InvalidPartitionRuleSnafu {
1966                        reason: format!("table {} already has partition columns", table_ref)
1967                    }
1968                );
1969                partition_columns
1970                    .iter()
1971                    .map(|column_name| {
1972                        column_schemas
1973                            .iter()
1974                            .find(|column| &column.name == column_name)
1975                            .with_context(|| ColumnNotFoundSnafu { msg: column_name })
1976                    })
1977                    .collect::<Result<Vec<_>>>()?
1978            }
1979        };
1980
1981        let from_column_name_and_type = column_name_and_type(&existing_partition_columns);
1982        let target_column_name_and_type = column_name_and_type(&target_partition_columns);
1983        let target_partition_column_names = target_partition_columns
1984            .iter()
1985            .map(|column| column.name.clone())
1986            .collect::<Vec<_>>();
1987        let timezone = query_context.timezone();
1988        // Convert SQL Exprs to PartitionExprs.
1989        let from_partition_exprs = match &request.source {
1990            RepartitionSource::Partitions { from_exprs, .. } => from_exprs
1991                .iter()
1992                .map(|expr| convert_one_expr(expr, &from_column_name_and_type, &timezone))
1993                .collect::<Result<Vec<_>>>()?,
1994            RepartitionSource::Unpartitioned { .. } => vec![],
1995        };
1996
1997        let mut into_partition_exprs = request
1998            .into_exprs
1999            .iter()
2000            .map(|expr| convert_one_expr(expr, &target_column_name_and_type, &timezone))
2001            .collect::<Result<Vec<_>>>()?;
2002
2003        // `MERGE PARTITION` (and some `REPARTITION`) generates a single `OR` expression from
2004        // multiple source partitions; try to simplify it for better readability and stability.
2005        if matches!(&request.source, RepartitionSource::Partitions { .. })
2006            && from_partition_exprs.len() > 1
2007            && into_partition_exprs.len() == 1
2008            && let Some(expr) = into_partition_exprs.pop()
2009        {
2010            into_partition_exprs.push(partition::simplify::simplify_merged_partition_expr(expr));
2011        }
2012
2013        // Parse existing partition expressions from region routes.
2014        let mut existing_partition_exprs =
2015            Vec::with_capacity(physical_table_route.region_routes.len());
2016        for route in &physical_table_route.region_routes {
2017            let expr_json = route.region.partition_expr();
2018            if !expr_json.is_empty() {
2019                match PartitionExpr::from_json_str(&expr_json) {
2020                    Ok(Some(expr)) => existing_partition_exprs.push(expr),
2021                    Ok(None) => {
2022                        // Empty
2023                    }
2024                    Err(e) => {
2025                        return Err(e).context(DeserializePartitionExprSnafu);
2026                    }
2027                }
2028            }
2029        }
2030
2031        // Validate that from_partition_exprs are a subset of existing partition exprs.
2032        // We compare PartitionExpr directly since it implements Eq.
2033        if matches!(&request.source, RepartitionSource::Partitions { .. }) {
2034            for from_expr in &from_partition_exprs {
2035                ensure!(
2036                    existing_partition_exprs.contains(from_expr),
2037                    InvalidPartitionRuleSnafu {
2038                        reason: format!(
2039                            "partition expression '{}' does not exist in table {}",
2040                            from_expr, table_ref
2041                        )
2042                    }
2043                );
2044            }
2045        }
2046
2047        // Build the new partition expressions:
2048        // new_exprs = existing_exprs - from_exprs + into_exprs
2049        let new_partition_exprs: Vec<PartitionExpr> = match &request.source {
2050            RepartitionSource::Partitions { .. } => existing_partition_exprs
2051                .into_iter()
2052                .filter(|expr| !from_partition_exprs.contains(expr))
2053                .chain(into_partition_exprs.clone().into_iter())
2054                .collect(),
2055            RepartitionSource::Unpartitioned { .. } => into_partition_exprs.clone(),
2056        };
2057        ensure_partition_expr_columns_in_target(
2058            &new_partition_exprs,
2059            &target_partition_column_names.iter().collect(),
2060        )?;
2061        let new_partition_exprs_len = new_partition_exprs.len();
2062        let from_partition_exprs_len = from_partition_exprs.len();
2063
2064        // Validate the new partition expressions using MultiDimPartitionRule and PartitionChecker.
2065        let _ = MultiDimPartitionRule::try_new(
2066            target_partition_column_names,
2067            vec![],
2068            new_partition_exprs,
2069            true,
2070        )
2071        .context(InvalidPartitionSnafu)?;
2072
2073        let ddl_options = parse_ddl_options(&request.options)?;
2074        let serialize_exprs = |exprs: Vec<PartitionExpr>| -> Result<Vec<String>> {
2075            let mut json_exprs = Vec::with_capacity(exprs.len());
2076            for expr in exprs {
2077                json_exprs.push(expr.as_json_str().context(SerializePartitionExprSnafu)?);
2078            }
2079            Ok(json_exprs)
2080        };
2081        let from_partition_exprs_json = serialize_exprs(from_partition_exprs)?;
2082        let into_partition_exprs_json = serialize_exprs(into_partition_exprs)?;
2083        let source = match &request.source {
2084            RepartitionSource::Partitions {
2085                target_partition_columns,
2086                ..
2087            } => Source::PartitionExprs(PartitionedSource {
2088                exprs: from_partition_exprs_json,
2089                target_partition_columns: target_partition_columns
2090                    .clone()
2091                    .map(|columns| TargetPartitionColumns { columns }),
2092            }),
2093            RepartitionSource::Unpartitioned { partition_columns } => {
2094                Source::Unpartitioned(UnpartitionedSource {
2095                    partition_columns: partition_columns.clone(),
2096                })
2097            }
2098        };
2099        let repartition = Repartition {
2100            into_partition_exprs: into_partition_exprs_json,
2101            source: Some(source),
2102            ..Default::default()
2103        };
2104        let executor_context = to_executor_context(query_context.clone(), TriggerReason::Manual);
2105        let mut req = SubmitDdlTaskRequest::new(DdlTask::new_alter_table(AlterTableExpr {
2106            catalog_name: request.catalog_name.clone(),
2107            schema_name: request.schema_name.clone(),
2108            table_name: request.table_name.clone(),
2109            kind: Some(Kind::Repartition(repartition)),
2110        }));
2111        req.wait = ddl_options.wait;
2112        req.timeout = ddl_options.timeout;
2113
2114        info!(
2115            "Submitting repartition task for table {} (table_id={}), from {} to {} partitions, timeout: {:?}, wait: {}",
2116            table_ref,
2117            table_id,
2118            from_partition_exprs_len,
2119            new_partition_exprs_len,
2120            ddl_options.timeout,
2121            ddl_options.wait
2122        );
2123
2124        let response = self
2125            .procedure_executor
2126            .submit_ddl_task(executor_context, req)
2127            .await
2128            .context(error::ExecuteDdlSnafu)?;
2129
2130        if !ddl_options.wait {
2131            let procedure_id = String::from_utf8_lossy(&response.key).into_owned();
2132            let output = build_procedure_id_output(&procedure_id)?;
2133            query_context.set_last_procedure_id(procedure_id);
2134            return Ok(output);
2135        }
2136
2137        // Only invalidate cache if wait is true.
2138        let invalidate_keys = vec![
2139            CacheIdent::TableId(table_id),
2140            CacheIdent::TableName(TableName::new(
2141                request.catalog_name,
2142                request.schema_name,
2143                request.table_name,
2144            )),
2145        ];
2146
2147        // Invalidates local cache ASAP.
2148        self.cache_invalidator
2149            .invalidate(&Context::default(), &invalidate_keys)
2150            .await
2151            .context(error::InvalidateTableCacheSnafu)?;
2152
2153        Ok(Output::new_with_affected_rows(0))
2154    }
2155
2156    #[tracing::instrument(skip_all)]
2157    pub async fn alter_table_inner(
2158        &self,
2159        expr: AlterTableExpr,
2160        query_context: QueryContextRef,
2161        trigger_reason: TriggerReason,
2162    ) -> Result<Output> {
2163        ensure_table_definition_writable(&expr.schema_name, &expr.table_name)?;
2164
2165        let catalog_name = if expr.catalog_name.is_empty() {
2166            DEFAULT_CATALOG_NAME.to_string()
2167        } else {
2168            expr.catalog_name.clone()
2169        };
2170
2171        let schema_name = if expr.schema_name.is_empty() {
2172            DEFAULT_SCHEMA_NAME.to_string()
2173        } else {
2174            expr.schema_name.clone()
2175        };
2176
2177        let table_name = expr.table_name.clone();
2178
2179        let table = self
2180            .catalog_manager
2181            .table(
2182                &catalog_name,
2183                &schema_name,
2184                &table_name,
2185                Some(&query_context),
2186            )
2187            .await
2188            .context(CatalogSnafu)?
2189            .with_context(|| TableNotFoundSnafu {
2190                table_name: format_full_table_name(&catalog_name, &schema_name, &table_name),
2191            })?;
2192
2193        let table_id = table.table_info().ident.table_id;
2194        let need_alter = verify_alter(table_id, table.table_info(), expr.clone())?;
2195        if !need_alter {
2196            return Ok(Output::new_with_affected_rows(0));
2197        }
2198        info!(
2199            "Table info before alter is {:?}, expr: {:?}",
2200            table.table_info(),
2201            expr
2202        );
2203
2204        let physical_table_id = self
2205            .table_metadata_manager
2206            .table_route_manager()
2207            .get_physical_table_id(table_id)
2208            .await
2209            .context(TableMetadataManagerSnafu)?;
2210
2211        let executor_context = to_executor_context(query_context, trigger_reason);
2212
2213        let (req, invalidate_keys) = if physical_table_id == table_id {
2214            // This is physical table
2215            let req = SubmitDdlTaskRequest::new(DdlTask::new_alter_table(expr));
2216
2217            let invalidate_keys = vec![
2218                CacheIdent::TableId(table_id),
2219                CacheIdent::TableName(TableName::new(catalog_name, schema_name, table_name)),
2220            ];
2221
2222            (req, invalidate_keys)
2223        } else {
2224            // This is logical table. Retention lives on the physical table;
2225            // reject it here with an error that says so, instead of the
2226            // procedure's generic "only add columns" error.
2227            if let Some(Kind::SetTableOptions(options)) = &expr.kind {
2228                ensure!(
2229                    !options
2230                        .table_options
2231                        .iter()
2232                        .any(|option| option.key == TTL_KEY && is_retention_ttl(&option.value)),
2233                    RetentionTtlOnLogicalTableSnafu
2234                );
2235            }
2236            // Annotation alters only rewrite the logical table's own metadata;
2237            // `AlterLogicalTablesProcedure` only handles column adds.
2238            let annotation_alter = match expr.kind.as_ref() {
2239                Some(kind) => common_grpc_expr::annotation_alter_family(kind)
2240                    .context(AlterExprToRequestSnafu)?
2241                    .is_some_and(|family| family.allows_logical_tables()),
2242                None => false,
2243            };
2244            let task = if annotation_alter {
2245                DdlTask::new_alter_table(expr)
2246            } else {
2247                DdlTask::new_alter_logical_tables(vec![expr])
2248            };
2249            let req = SubmitDdlTaskRequest::new(task);
2250
2251            let mut invalidate_keys = vec![
2252                CacheIdent::TableId(physical_table_id),
2253                CacheIdent::TableId(table_id),
2254                CacheIdent::TableName(TableName::new(catalog_name, schema_name, table_name)),
2255            ];
2256
2257            let physical_table = self
2258                .table_metadata_manager
2259                .table_info_manager()
2260                .get(physical_table_id)
2261                .await
2262                .context(TableMetadataManagerSnafu)?
2263                .map(|x| x.into_inner());
2264            if let Some(physical_table) = physical_table {
2265                let physical_table_name = TableName::new(
2266                    physical_table.table_info.catalog_name,
2267                    physical_table.table_info.schema_name,
2268                    physical_table.table_info.name,
2269                );
2270                invalidate_keys.push(CacheIdent::TableName(physical_table_name));
2271            }
2272
2273            (req, invalidate_keys)
2274        };
2275
2276        self.procedure_executor
2277            .submit_ddl_task(executor_context, req)
2278            .await
2279            .context(error::ExecuteDdlSnafu)?;
2280
2281        // Invalidates local cache ASAP.
2282        self.cache_invalidator
2283            .invalidate(&Context::default(), &invalidate_keys)
2284            .await
2285            .context(error::InvalidateTableCacheSnafu)?;
2286
2287        Ok(Output::new_with_affected_rows(0))
2288    }
2289
2290    #[cfg(feature = "enterprise")]
2291    #[tracing::instrument(skip_all)]
2292    pub async fn alter_trigger(
2293        &self,
2294        _alter_expr: AlterTrigger,
2295        _query_context: QueryContextRef,
2296    ) -> Result<Output> {
2297        crate::error::NotSupportedSnafu {
2298            feat: "alter trigger",
2299        }
2300        .fail()
2301    }
2302
2303    #[tracing::instrument(skip_all)]
2304    pub async fn alter_database(
2305        &self,
2306        alter_expr: AlterDatabase,
2307        query_context: QueryContextRef,
2308    ) -> Result<Output> {
2309        let alter_expr = expr_helper::to_alter_database_expr(alter_expr, &query_context)?;
2310        self.alter_database_inner(alter_expr, query_context).await
2311    }
2312
2313    #[tracing::instrument(skip_all)]
2314    pub async fn alter_database_inner(
2315        &self,
2316        alter_expr: AlterDatabaseExpr,
2317        query_context: QueryContextRef,
2318    ) -> Result<Output> {
2319        ensure!(
2320            !is_readonly_schema(&alter_expr.schema_name),
2321            SchemaReadOnlySnafu {
2322                name: query_context.current_schema().clone()
2323            }
2324        );
2325
2326        let exists = self
2327            .catalog_manager
2328            .schema_exists(&alter_expr.catalog_name, &alter_expr.schema_name, None)
2329            .await
2330            .context(CatalogSnafu)?;
2331        ensure!(
2332            exists,
2333            SchemaNotFoundSnafu {
2334                schema_info: alter_expr.schema_name,
2335            }
2336        );
2337
2338        let cache_ident = [CacheIdent::SchemaName(SchemaName {
2339            catalog_name: alter_expr.catalog_name.clone(),
2340            schema_name: alter_expr.schema_name.clone(),
2341        })];
2342
2343        self.alter_database_procedure(alter_expr, query_context)
2344            .await?;
2345
2346        // Invalidates local cache ASAP.
2347        self.cache_invalidator
2348            .invalidate(&Context::default(), &cache_ident)
2349            .await
2350            .context(error::InvalidateTableCacheSnafu)?;
2351
2352        Ok(Output::new_with_affected_rows(0))
2353    }
2354
2355    async fn create_table_procedure(
2356        &self,
2357        create_table: CreateTableExpr,
2358        partitions: Vec<PartitionExpr>,
2359        table_info: TableInfo,
2360        query_context: QueryContextRef,
2361        trigger_reason: TriggerReason,
2362    ) -> Result<SubmitDdlTaskResponse> {
2363        let partitions = partitions
2364            .into_iter()
2365            .map(|expr| expr.as_pb_partition().context(PartitionExprToPbSnafu))
2366            .collect::<Result<Vec<_>>>()?;
2367
2368        let executor_context = to_executor_context_with_origin_frontend(
2369            query_context,
2370            &self.origin_frontend_addr,
2371            trigger_reason,
2372        );
2373        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_table(
2374            create_table,
2375            partitions,
2376            table_info,
2377        ));
2378
2379        self.procedure_executor
2380            .submit_ddl_task(executor_context, request)
2381            .await
2382            .context(error::ExecuteDdlSnafu)
2383    }
2384
2385    async fn create_logical_tables_procedure(
2386        &self,
2387        tables_data: Vec<(CreateTableExpr, TableInfo)>,
2388        query_context: QueryContextRef,
2389        trigger_reason: TriggerReason,
2390    ) -> Result<SubmitDdlTaskResponse> {
2391        let executor_context = to_executor_context_with_origin_frontend(
2392            query_context,
2393            &self.origin_frontend_addr,
2394            trigger_reason,
2395        );
2396        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_logical_tables(tables_data));
2397
2398        self.procedure_executor
2399            .submit_ddl_task(executor_context, request)
2400            .await
2401            .context(error::ExecuteDdlSnafu)
2402    }
2403
2404    async fn alter_logical_tables_procedure(
2405        &self,
2406        physical_table_id: TableId,
2407        tables_data: Vec<(TableId, AlterTableExpr)>,
2408        query_context: QueryContextRef,
2409        trigger_reason: TriggerReason,
2410    ) -> Result<SubmitDdlTaskResponse> {
2411        let mut invalidate_keys = vec![CacheIdent::TableId(physical_table_id)];
2412        if let Some(physical) = self
2413            .table_metadata_manager
2414            .table_info_manager()
2415            .get(physical_table_id)
2416            .await
2417            .context(TableMetadataManagerSnafu)?
2418        {
2419            invalidate_keys.push(CacheIdent::TableName(physical.table_name()));
2420        }
2421        for (id, expr) in &tables_data {
2422            invalidate_keys.push(CacheIdent::TableId(*id));
2423            invalidate_keys.push(CacheIdent::TableName(TableName::new(
2424                &expr.catalog_name,
2425                &expr.schema_name,
2426                &expr.table_name,
2427            )));
2428        }
2429        let mut executor_context = to_executor_context(query_context, trigger_reason);
2430        let (table_ids, exprs): (Vec<_>, Vec<_>) = tables_data.into_iter().unzip();
2431        if let Some(context) = &mut executor_context.query_context {
2432            context.extensions.insert(
2433                ALTER_LOGICAL_TABLE_IDS_EXTENSION_KEY.to_string(),
2434                table_ids
2435                    .iter()
2436                    .map(ToString::to_string)
2437                    .collect::<Vec<_>>()
2438                    .join(","),
2439            );
2440        }
2441        let request = SubmitDdlTaskRequest::new(DdlTask::new_alter_logical_tables(exprs));
2442
2443        let response = self
2444            .procedure_executor
2445            .submit_ddl_task(executor_context, request)
2446            .await
2447            .context(error::ExecuteDdlSnafu)?;
2448        // Metasrv notifications are asynchronous; callers need the updated local schema now.
2449        self.cache_invalidator
2450            .invalidate(&Context::default(), &invalidate_keys)
2451            .await
2452            .context(error::InvalidateTableCacheSnafu)?;
2453        Ok(response)
2454    }
2455
2456    async fn drop_table_procedure(
2457        &self,
2458        table_name: &TableName,
2459        table_id: TableId,
2460        drop_if_exists: bool,
2461        query_context: QueryContextRef,
2462    ) -> Result<SubmitDdlTaskResponse> {
2463        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2464        let request = SubmitDdlTaskRequest::new(DdlTask::new_drop_table(
2465            table_name.catalog_name.clone(),
2466            table_name.schema_name.clone(),
2467            table_name.table_name.clone(),
2468            table_id,
2469            drop_if_exists,
2470        ));
2471
2472        self.procedure_executor
2473            .submit_ddl_task(executor_context, request)
2474            .await
2475            .context(error::ExecuteDdlSnafu)
2476    }
2477
2478    async fn drop_database_procedure(
2479        &self,
2480        catalog: String,
2481        schema: String,
2482        drop_if_exists: bool,
2483        query_context: QueryContextRef,
2484    ) -> Result<SubmitDdlTaskResponse> {
2485        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2486        let request =
2487            SubmitDdlTaskRequest::new(DdlTask::new_drop_database(catalog, schema, drop_if_exists));
2488
2489        self.procedure_executor
2490            .submit_ddl_task(executor_context, request)
2491            .await
2492            .context(error::ExecuteDdlSnafu)
2493    }
2494
2495    async fn alter_database_procedure(
2496        &self,
2497        alter_expr: AlterDatabaseExpr,
2498        query_context: QueryContextRef,
2499    ) -> Result<SubmitDdlTaskResponse> {
2500        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2501        let request = SubmitDdlTaskRequest::new(DdlTask::new_alter_database(alter_expr));
2502
2503        self.procedure_executor
2504            .submit_ddl_task(executor_context, request)
2505            .await
2506            .context(error::ExecuteDdlSnafu)
2507    }
2508
2509    async fn truncate_table_procedure(
2510        &self,
2511        table_name: &TableName,
2512        table_id: TableId,
2513        time_ranges: Vec<(Timestamp, Timestamp)>,
2514        ddl_options: DdlSubmitOptions,
2515        query_context: QueryContextRef,
2516    ) -> Result<SubmitDdlTaskResponse> {
2517        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2518        let mut request = SubmitDdlTaskRequest::new(DdlTask::new_truncate_table(
2519            table_name.catalog_name.clone(),
2520            table_name.schema_name.clone(),
2521            table_name.table_name.clone(),
2522            table_id,
2523            time_ranges,
2524        ));
2525        request.wait = ddl_options.wait;
2526        request.timeout = ddl_options.timeout;
2527
2528        self.procedure_executor
2529            .submit_ddl_task(executor_context, request)
2530            .await
2531            .context(error::ExecuteDdlSnafu)
2532    }
2533
2534    #[tracing::instrument(skip_all)]
2535    pub async fn create_database(
2536        &self,
2537        database: &str,
2538        create_if_not_exists: bool,
2539        options: HashMap<String, String>,
2540        query_context: QueryContextRef,
2541    ) -> Result<Output> {
2542        let catalog = query_context.current_catalog();
2543        ensure!(
2544            NAME_PATTERN_REG.is_match(catalog),
2545            error::UnexpectedSnafu {
2546                violated: format!("Invalid catalog name: {}", catalog)
2547            }
2548        );
2549
2550        ensure!(
2551            NAME_PATTERN_REG.is_match(database),
2552            error::UnexpectedSnafu {
2553                violated: format!("Invalid database name: {}", database)
2554            }
2555        );
2556
2557        #[cfg(feature = "enterprise")]
2558        let creator = match &self.create_database_handler {
2559            Some(handler) => handler.creator(&query_context).context(ExternalSnafu)?,
2560            None => None,
2561        };
2562        #[cfg(not(feature = "enterprise"))]
2563        let creator = None;
2564
2565        let output = if !self
2566            .catalog_manager
2567            .schema_exists(catalog, database, None)
2568            .await
2569            .context(CatalogSnafu)?
2570            && !self.catalog_manager.is_reserved_schema_name(database)
2571        {
2572            self.create_database_procedure(
2573                catalog.to_string(),
2574                database.to_string(),
2575                create_if_not_exists,
2576                options,
2577                query_context.clone(),
2578                creator,
2579            )
2580            .await?;
2581
2582            Output::new_with_affected_rows(1)
2583        } else if create_if_not_exists {
2584            Output::new_with_affected_rows(1)
2585        } else {
2586            return error::SchemaExistsSnafu { name: database }.fail();
2587        };
2588
2589        #[cfg(feature = "enterprise")]
2590        if let Some(handler) = &self.create_database_handler {
2591            handler
2592                .refresh_current_user(&query_context)
2593                .await
2594                .context(ExternalSnafu)?;
2595        }
2596
2597        Ok(output)
2598    }
2599
2600    async fn create_database_procedure(
2601        &self,
2602        catalog: String,
2603        database: String,
2604        create_if_not_exists: bool,
2605        options: HashMap<String, String>,
2606        query_context: QueryContextRef,
2607        creator: Option<CreatorGrantIntent>,
2608    ) -> Result<SubmitDdlTaskResponse> {
2609        let executor_context = to_executor_context(query_context, TriggerReason::Manual);
2610        let request = SubmitDdlTaskRequest::new(DdlTask::new_create_database(
2611            catalog,
2612            database,
2613            create_if_not_exists,
2614            options,
2615            creator,
2616        ));
2617
2618        self.procedure_executor
2619            .submit_ddl_task(executor_context, request)
2620            .await
2621            .context(error::ExecuteDdlSnafu)
2622    }
2623}
2624
2625/// Parse partition statement [Partitions] into [PartitionExpr] and partition columns.
2626pub fn parse_partitions(
2627    create_table: &CreateTableExpr,
2628    partitions: Option<Partitions>,
2629    query_ctx: &QueryContextRef,
2630) -> Result<(Vec<PartitionExpr>, Vec<String>)> {
2631    // If partitions are not defined by user, use the timestamp column (which has to be existed) as
2632    // the partition column, and create only one partition.
2633    let partition_columns = find_partition_columns(&partitions)?;
2634    let partition_exprs =
2635        find_partition_entries(create_table, &partitions, &partition_columns, query_ctx)?;
2636
2637    // Validates partition
2638    let exprs = partition_exprs.clone();
2639    MultiDimPartitionRule::try_new(partition_columns.clone(), vec![], exprs, true)
2640        .context(InvalidPartitionSnafu)?;
2641
2642    Ok((partition_exprs, partition_columns))
2643}
2644
2645fn parse_partitions_for_logical_validation(
2646    create_table: &CreateTableExpr,
2647    partitions: &Partitions,
2648    query_ctx: &QueryContextRef,
2649) -> Result<(Vec<String>, Vec<PartitionExpr>)> {
2650    let partition_columns = partitions
2651        .column_list
2652        .iter()
2653        .map(|ident| ident.value.clone())
2654        .collect::<Vec<_>>();
2655
2656    let column_name_and_type = partition_columns
2657        .iter()
2658        .map(|pc| {
2659            let column = create_table
2660                .column_defs
2661                .iter()
2662                .find(|c| &c.name == pc)
2663                .context(ColumnNotFoundSnafu { msg: pc.clone() })?;
2664            let column_name = &column.name;
2665            let data_type = ConcreteDataType::from(
2666                ColumnDataTypeWrapper::try_new(column.data_type, column.datatype_extension.clone())
2667                    .context(ColumnDataTypeSnafu)?,
2668            );
2669            Ok((column_name, data_type))
2670        })
2671        .collect::<Result<HashMap<_, _>>>()?;
2672
2673    let mut partition_exprs = Vec::with_capacity(partitions.exprs.len());
2674    for expr in &partitions.exprs {
2675        let partition_expr = convert_one_expr(expr, &column_name_and_type, &query_ctx.timezone())?;
2676        partition_exprs.push(partition_expr);
2677    }
2678
2679    MultiDimPartitionRule::try_new(
2680        partition_columns.clone(),
2681        vec![],
2682        partition_exprs.clone(),
2683        true,
2684    )
2685    .context(InvalidPartitionSnafu)?;
2686
2687    Ok((partition_columns, partition_exprs))
2688}
2689
2690/// Verifies an alter and returns whether it is necessary to perform the alter.
2691///
2692/// # Returns
2693///
2694/// Returns true if the alter need to be porformed; otherwise, it returns false.
2695pub fn verify_alter(
2696    table_id: TableId,
2697    table_info: Arc<TableInfo>,
2698    expr: AlterTableExpr,
2699) -> Result<bool> {
2700    let request: AlterTableRequest =
2701        common_grpc_expr::alter_expr_to_request(table_id, expr, Some(&table_info.meta))
2702            .context(AlterExprToRequestSnafu)?;
2703
2704    let AlterTableRequest {
2705        table_name,
2706        alter_kind,
2707        ..
2708    } = &request;
2709
2710    if let AlterKind::RenameTable { new_table_name } = alter_kind {
2711        ensure!(
2712            NAME_PATTERN_REG.is_match(new_table_name),
2713            error::UnexpectedSnafu {
2714                violated: format!("Invalid table name: {}", new_table_name)
2715            }
2716        );
2717        // Renaming INTO a computed graph table's name would let the overlay
2718        // shadow the renamed physical table, orphaning its data.
2719        ensure_table_definition_writable(&table_info.schema_name, new_table_name)?;
2720    } else if let AlterKind::AddColumns { columns } = alter_kind {
2721        // If all the columns are marked as add_if_not_exists and they already exist in the table,
2722        // there is no need to perform the alter.
2723        let column_names: HashSet<_> = table_info
2724            .meta
2725            .schema
2726            .column_schemas()
2727            .iter()
2728            .map(|schema| &schema.name)
2729            .collect();
2730        if columns.iter().all(|column| {
2731            column_names.contains(&column.column_schema.name) && column.add_if_not_exists
2732        }) {
2733            return Ok(false);
2734        }
2735    }
2736
2737    let new_meta = table_info
2738        .meta
2739        .builder_with_alter_kind(table_name, &request.alter_kind)
2740        .context(error::TableSnafu)?
2741        .build()
2742        .context(error::BuildTableMetaSnafu { table_name })?;
2743
2744    validate_json2_primary_keys(&new_meta.schema, &new_meta.primary_key_indices)?;
2745
2746    Ok(true)
2747}
2748
2749/// Returns whether `ttl` is a retention period, i.e. a valid TTL other than
2750/// `instant`. Invalid values return false so they fail normal option validation.
2751pub(crate) fn is_retention_ttl(ttl: &str) -> bool {
2752    TimeToLive::from_humantime_or_str(ttl).is_ok_and(|ttl| !ttl.is_instant())
2753}
2754
2755pub fn create_table_info(
2756    create_table: &CreateTableExpr,
2757    partition_columns: Vec<String>,
2758) -> Result<TableInfo> {
2759    let mut column_schemas = Vec::with_capacity(create_table.column_defs.len());
2760    let mut column_name_to_index_map = HashMap::new();
2761
2762    for (idx, column) in create_table.column_defs.iter().enumerate() {
2763        let schema =
2764            column_def::try_as_column_schema(column).context(error::InvalidColumnDefSnafu {
2765                column: &column.name,
2766            })?;
2767        let schema = schema.with_time_index(column.name == create_table.time_index);
2768
2769        column_schemas.push(schema);
2770        let _ = column_name_to_index_map.insert(column.name.clone(), idx);
2771    }
2772
2773    let next_column_id = column_schemas.len() as u32;
2774    let schema = Arc::new(Schema::try_new(column_schemas).context(ConvertSchemaSnafu)?);
2775
2776    let primary_key_indices = create_table
2777        .primary_keys
2778        .iter()
2779        .map(|name| {
2780            column_name_to_index_map
2781                .get(name)
2782                .cloned()
2783                .context(ColumnNotFoundSnafu { msg: name })
2784        })
2785        .collect::<Result<Vec<_>>>()?;
2786
2787    let partition_key_indices = partition_columns
2788        .into_iter()
2789        .map(|col_name| {
2790            column_name_to_index_map
2791                .get(&col_name)
2792                .cloned()
2793                .context(ColumnNotFoundSnafu { msg: col_name })
2794        })
2795        .collect::<Result<Vec<_>>>()?;
2796
2797    let mut table_options = TableOptions::try_from_iter(&create_table.table_options)
2798        .context(UnrecognizedTableOptionSnafu)?;
2799
2800    validate_json2_primary_keys(&schema, &primary_key_indices)?;
2801
2802    validate_and_normalize_annotations(&mut table_options, &schema, &partition_key_indices)?;
2803
2804    let meta = TableMeta {
2805        schema,
2806        primary_key_indices,
2807        value_indices: vec![],
2808        engine: create_table.engine.clone(),
2809        next_column_id,
2810        options: table_options,
2811        created_on: Utc::now(),
2812        updated_on: Utc::now(),
2813        partition_key_indices,
2814        column_ids: vec![],
2815    };
2816
2817    let desc = if create_table.desc.is_empty() {
2818        create_table.table_options.get(COMMENT_KEY).cloned()
2819    } else {
2820        Some(create_table.desc.clone())
2821    };
2822
2823    let table_info = TableInfo {
2824        ident: metadata::TableIdent {
2825            // The table id of distributed table is assigned by Meta, set "0" here as a placeholder.
2826            table_id: 0,
2827            version: 0,
2828        },
2829        name: create_table.table_name.clone(),
2830        desc,
2831        catalog_name: create_table.catalog_name.clone(),
2832        schema_name: create_table.schema_name.clone(),
2833        meta,
2834        table_type: TableType::Base,
2835    };
2836    Ok(table_info)
2837}
2838
2839fn validate_json2_primary_keys(schema: &Schema, primary_key_indices: &[usize]) -> Result<()> {
2840    for &index in primary_key_indices {
2841        let column = &schema.column_schemas()[index];
2842        ensure!(
2843            !column.data_type.is_json2(),
2844            InvalidSqlSnafu {
2845                err_msg: format!("JSON2 column `{}` cannot be a primary key", column.name),
2846            }
2847        );
2848    }
2849    Ok(())
2850}
2851
2852/// Rejects DDL against read-only schemas and computed entity-graph tables.
2853fn ensure_table_writable(schema: &str, table: &str) -> Result<()> {
2854    ensure!(
2855        !is_readonly_schema(schema),
2856        SchemaReadOnlySnafu {
2857            name: schema.to_string()
2858        }
2859    );
2860    ensure!(
2861        !is_readonly_table(schema, table),
2862        TableReadOnlySnafu {
2863            name: table.to_string()
2864        }
2865    );
2866    Ok(())
2867}
2868
2869/// [`ensure_table_writable`] plus the definition guard for system-defined
2870/// tables: user CREATE, ALTER and RENAME-into are rejected so the canonical
2871/// schema cannot be squatted or mutated. DROP and TRUNCATE stay allowed — the
2872/// next INSERT recreates the table canonically, which is also the recovery
2873/// path if the canonical definition changes across an upgrade.
2874fn ensure_table_definition_writable(schema: &str, table: &str) -> Result<()> {
2875    ensure_table_writable(schema, table)?;
2876    ensure!(
2877        !is_ddl_reserved_table(schema, table),
2878        TableDdlReservedSnafu {
2879            name: table.to_string()
2880        }
2881    );
2882    Ok(())
2883}
2884
2885/// CREATE-side annotation validation: one rule source in the table crate,
2886/// mapped onto this crate's existing error variants so client-visible codes
2887/// and messages stay put.
2888fn validate_and_normalize_annotations(
2889    options: &mut TableOptions,
2890    schema: &Schema,
2891    partition_key_indices: &[usize],
2892) -> Result<()> {
2893    use table::requests::AnnotationValidationError as CheckError;
2894    let handle_error = |e: CheckError| match e {
2895        CheckError::ColumnNotFound { column } => ColumnNotFoundSnafu { msg: column }.build(),
2896        e @ (CheckError::UnknownKey { .. }
2897        | CheckError::InvalidValue { .. }
2898        | CheckError::InvalidPartitionNumHint { .. }
2899        | CheckError::ColumnNotStringForm { .. }) => InvalidSqlSnafu {
2900            err_msg: e.to_string(),
2901        }
2902        .build(),
2903        e @ (CheckError::NotSingleColumn
2904        | CheckError::PartitionMetadataConflict
2905        | CheckError::TimeIndexConflict) => InvalidPartitionRuleSnafu {
2906            reason: e.to_string(),
2907        }
2908        .build(),
2909    };
2910    validate_and_normalize_annotation_options(options, schema, partition_key_indices)
2911        .map_err(handle_error)
2912}
2913
2914fn find_partition_columns(partitions: &Option<Partitions>) -> Result<Vec<String>> {
2915    let columns = if let Some(partitions) = partitions {
2916        partitions
2917            .column_list
2918            .iter()
2919            .map(|x| x.value.clone())
2920            .collect::<Vec<_>>()
2921    } else {
2922        vec![]
2923    };
2924    Ok(columns)
2925}
2926
2927/// Parse [Partitions] into a group of partition entries.
2928///
2929/// Returns a list of [PartitionExpr], each of which defines a partition.
2930fn find_partition_entries(
2931    create_table: &CreateTableExpr,
2932    partitions: &Option<Partitions>,
2933    partition_columns: &[String],
2934    query_ctx: &QueryContextRef,
2935) -> Result<Vec<PartitionExpr>> {
2936    let Some(partitions) = partitions else {
2937        return Ok(vec![]);
2938    };
2939
2940    // extract concrete data type of partition columns
2941    let column_name_and_type = partition_columns
2942        .iter()
2943        .map(|pc| {
2944            let column = create_table
2945                .column_defs
2946                .iter()
2947                .find(|c| &c.name == pc)
2948                // unwrap is safe here because we have checked that partition columns are defined
2949                .unwrap();
2950            let column_name = &column.name;
2951            let data_type = ConcreteDataType::from(
2952                ColumnDataTypeWrapper::try_new(column.data_type, column.datatype_extension.clone())
2953                    .context(ColumnDataTypeSnafu)?,
2954            );
2955            Ok((column_name, data_type))
2956        })
2957        .collect::<Result<HashMap<_, _>>>()?;
2958
2959    // Transform parser expr to partition expr
2960    let mut partition_exprs = Vec::with_capacity(partitions.exprs.len());
2961    for partition in &partitions.exprs {
2962        let partition_expr =
2963            convert_one_expr(partition, &column_name_and_type, &query_ctx.timezone())?;
2964        partition_exprs.push(partition_expr);
2965    }
2966
2967    Ok(partition_exprs)
2968}
2969
2970fn column_name_and_type<'a>(
2971    partition_columns: &'a [&'a ColumnSchema],
2972) -> HashMap<&'a String, ConcreteDataType> {
2973    partition_columns
2974        .iter()
2975        .map(|column| (&column.name, column.data_type.clone()))
2976        .collect()
2977}
2978
2979fn validate_and_collect_partition_columns<'a>(
2980    column_names: &[String],
2981    column_schemas: &'a [ColumnSchema],
2982) -> Result<Vec<&'a ColumnSchema>> {
2983    let mut seen = HashSet::with_capacity(column_names.len());
2984    column_names
2985        .iter()
2986        .map(|column_name| {
2987            ensure!(
2988                seen.insert(column_name),
2989                InvalidPartitionRuleSnafu {
2990                    reason: format!("duplicate partition column '{}'", column_name)
2991                }
2992            );
2993            column_schemas
2994                .iter()
2995                .find(|column| &column.name == column_name)
2996                .with_context(|| ColumnNotFoundSnafu { msg: column_name })
2997        })
2998        .collect()
2999}
3000
3001fn ensure_partition_expr_columns_in_target(
3002    partition_exprs: &[PartitionExpr],
3003    target_partition_columns: &HashSet<&String>,
3004) -> Result<()> {
3005    for expr in partition_exprs {
3006        ensure_partition_operand_columns_in_target(&expr.lhs, target_partition_columns)?;
3007        ensure_partition_operand_columns_in_target(&expr.rhs, target_partition_columns)?;
3008    }
3009
3010    Ok(())
3011}
3012
3013fn ensure_partition_operand_columns_in_target(
3014    operand: &Operand,
3015    target_partition_columns: &HashSet<&String>,
3016) -> Result<()> {
3017    match operand {
3018        Operand::Column(column) => ensure!(
3019            target_partition_columns.contains(column),
3020            InvalidPartitionRuleSnafu {
3021                reason: format!(
3022                    "partition expression references column '{}' that is not in target partition columns",
3023                    column
3024                )
3025            }
3026        ),
3027        Operand::Expr(expr) => {
3028            ensure_partition_operand_columns_in_target(&expr.lhs, target_partition_columns)?;
3029            ensure_partition_operand_columns_in_target(&expr.rhs, target_partition_columns)?;
3030        }
3031        Operand::IcebergHash(hash) => ensure!(
3032            target_partition_columns.contains(&hash.column().to_string()),
3033            InvalidPartitionRuleSnafu {
3034                reason: format!(
3035                    "partition expression references column '{}' that is not in target partition columns",
3036                    hash.column()
3037                )
3038            }
3039        ),
3040        Operand::Value(_) => {}
3041    }
3042
3043    Ok(())
3044}
3045
3046fn convert_one_expr(
3047    expr: &Expr,
3048    column_name_and_type: &HashMap<&String, ConcreteDataType>,
3049    timezone: &Timezone,
3050) -> Result<PartitionExpr> {
3051    let Expr::BinaryOp { left, op, right } = expr else {
3052        return InvalidPartitionRuleSnafu {
3053            reason: "partition rule must be a binary expression",
3054        }
3055        .fail();
3056    };
3057
3058    let op =
3059        RestrictedOp::try_from_parser(&op.clone()).with_context(|| InvalidPartitionRuleSnafu {
3060            reason: format!("unsupported operator in partition expr {op}"),
3061        })?;
3062
3063    // convert leaf node.
3064    let (lhs, op, rhs) = match (left.as_ref(), right.as_ref()) {
3065        // col, val
3066        (Expr::Identifier(ident), Expr::Value(value)) => {
3067            let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3068            let value = convert_value(&value.value, data_type, timezone, None)?;
3069            (Operand::Column(column_name), op, Operand::Value(value))
3070        }
3071        (Expr::Identifier(ident), Expr::UnaryOp { op: unary_op, expr })
3072            if let Expr::Value(v) = &**expr =>
3073        {
3074            let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3075            let value = convert_value(&v.value, data_type, timezone, Some(*unary_op))?;
3076            (Operand::Column(column_name), op, Operand::Value(value))
3077        }
3078        // val, col
3079        (Expr::Value(value), Expr::Identifier(ident)) => {
3080            let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3081            let value = convert_value(&value.value, data_type, timezone, None)?;
3082            (Operand::Value(value), op, Operand::Column(column_name))
3083        }
3084        (Expr::UnaryOp { op: unary_op, expr }, Expr::Identifier(ident))
3085            if let Expr::Value(v) = &**expr =>
3086        {
3087            let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3088            let value = convert_value(&v.value, data_type, timezone, Some(*unary_op))?;
3089            (Operand::Value(value), op, Operand::Column(column_name))
3090        }
3091        // func, val
3092        (Expr::Function(function), literal)
3093            if !matches!(op, RestrictedOp::And | RestrictedOp::Or) =>
3094        {
3095            let hash = convert_iceberg_hash(function, column_name_and_type)?;
3096            let value = convert_literal(literal, IcebergHash::output_type(), timezone)?;
3097            (Operand::IcebergHash(hash), op, Operand::Value(value))
3098        }
3099        // val, func
3100        (literal, Expr::Function(function))
3101            if !matches!(op, RestrictedOp::And | RestrictedOp::Or) =>
3102        {
3103            let hash = convert_iceberg_hash(function, column_name_and_type)?;
3104            let value = convert_literal(literal, IcebergHash::output_type(), timezone)?;
3105            (Operand::Value(value), op, Operand::IcebergHash(hash))
3106        }
3107        (Expr::BinaryOp { .. }, Expr::BinaryOp { .. }) => {
3108            // sub-expr must against another sub-expr
3109            let lhs = convert_one_expr(left, column_name_and_type, timezone)?;
3110            let rhs = convert_one_expr(right, column_name_and_type, timezone)?;
3111            (Operand::Expr(lhs), op, Operand::Expr(rhs))
3112        }
3113        _ => {
3114            return InvalidPartitionRuleSnafu {
3115                reason: format!("invalid partition expr {expr}"),
3116            }
3117            .fail();
3118        }
3119    };
3120
3121    Ok(PartitionExpr::new(lhs, op, rhs))
3122}
3123
3124/// Converts an `iceberg_hash` call, which takes exactly one partition column.
3125fn convert_iceberg_hash(
3126    function: &Function,
3127    column_name_and_type: &HashMap<&String, ConcreteDataType>,
3128) -> Result<IcebergHash> {
3129    let invalid = || {
3130        InvalidPartitionRuleSnafu {
3131            reason: format!("unsupported partition function call {function}"),
3132        }
3133        .build()
3134    };
3135    let FunctionArguments::List(args) = &function.args else {
3136        return Err(invalid());
3137    };
3138    let [FunctionArg::Unnamed(FunctionArgExpr::Expr(Expr::Identifier(ident)))] =
3139        args.args.as_slice()
3140    else {
3141        return Err(invalid());
3142    };
3143    if function.filter.is_some()
3144        || function.over.is_some()
3145        || function.null_treatment.is_some()
3146        || !function.within_group.is_empty()
3147        || !matches!(function.parameters, FunctionArguments::None)
3148        || args.duplicate_treatment.is_some()
3149        || !args.clauses.is_empty()
3150    {
3151        return Err(invalid());
3152    }
3153    ensure!(
3154        function
3155            .name
3156            .to_string()
3157            .eq_ignore_ascii_case(IcebergHash::NAME),
3158        InvalidPartitionRuleSnafu {
3159            reason: format!("unsupported partition function {}", function.name),
3160        }
3161    );
3162    let (column_name, data_type) = convert_identifier(ident, column_name_and_type)?;
3163    ensure!(
3164        IcebergHash::accepts(&data_type),
3165        InvalidPartitionRuleSnafu {
3166            reason: format!(
3167                "partition function {} does not support column type {data_type}",
3168                IcebergHash::NAME
3169            ),
3170        }
3171    );
3172    Ok(IcebergHash::new(column_name))
3173}
3174
3175fn convert_literal(expr: &Expr, data_type: ConcreteDataType, timezone: &Timezone) -> Result<Value> {
3176    match expr {
3177        Expr::Value(value) => convert_value(&value.value, data_type, timezone, None),
3178        Expr::UnaryOp { op, expr } if let Expr::Value(value) = &**expr => {
3179            convert_value(&value.value, data_type, timezone, Some(*op))
3180        }
3181        _ => InvalidPartitionRuleSnafu {
3182            reason: format!("expect a literal in partition expr, got {expr}"),
3183        }
3184        .fail(),
3185    }
3186}
3187
3188fn convert_identifier(
3189    ident: &Ident,
3190    column_name_and_type: &HashMap<&String, ConcreteDataType>,
3191) -> Result<(String, ConcreteDataType)> {
3192    let column_name = ident.value.clone();
3193    let data_type = column_name_and_type
3194        .get(&column_name)
3195        .cloned()
3196        .with_context(|| ColumnNotFoundSnafu { msg: &column_name })?;
3197    Ok((column_name, data_type))
3198}
3199
3200fn convert_value(
3201    value: &ParserValue,
3202    data_type: ConcreteDataType,
3203    timezone: &Timezone,
3204    unary_op: Option<UnaryOperator>,
3205) -> Result<Value> {
3206    sql_value_to_value(
3207        &ColumnSchema::new("<NONAME>", data_type, true),
3208        value,
3209        Some(timezone),
3210        unary_op,
3211        false,
3212    )
3213    .context(error::SqlCommonSnafu)
3214}
3215
3216#[cfg(feature = "enterprise")]
3217async fn execute_undrop_table(
3218    table_metadata_manager: &TableMetadataManagerRef,
3219    procedure_executor: &ProcedureExecutorRef,
3220    cache_invalidator: &CacheInvalidatorRef,
3221    table_name: TableName,
3222    query_context: QueryContextRef,
3223) -> Result<Output> {
3224    // Undropping restores a table definition and could resurrect a
3225    // pre-canonical shape of a DDL-reserved table; rejected like CREATE.
3226    ensure_table_definition_writable(&table_name.schema_name, &table_name.table_name)?;
3227
3228    let dropped = table_metadata_manager
3229        .get_dropped_table(&table_name)
3230        .await
3231        .context(TableMetadataManagerSnafu)?
3232        .with_context(|| TableNotFoundSnafu {
3233            table_name: table_name.to_string(),
3234        })?;
3235
3236    let executor_context = to_executor_context(query_context, TriggerReason::Manual);
3237    let request = SubmitDdlTaskRequest::new(DdlTask::new_undrop_table(dropped.table_id));
3238    procedure_executor
3239        .submit_ddl_task(executor_context, request)
3240        .await
3241        .context(error::ExecuteDdlSnafu)?;
3242
3243    if let Err(err) = cache_invalidator
3244        .invalidate(
3245            &Context::default(),
3246            &[
3247                CacheIdent::TableId(dropped.table_id),
3248                CacheIdent::TableName(table_name),
3249            ],
3250        )
3251        .await
3252    {
3253        warn!(
3254            "Failed to invalidate cache after restoring table '{}' (id={}): {}",
3255            dropped.table_name, dropped.table_id, err
3256        );
3257    }
3258
3259    Ok(Output::new_with_affected_rows(0))
3260}
3261
3262#[cfg(test)]
3263mod test {
3264    #[cfg(feature = "enterprise")]
3265    use std::sync::{Arc, Mutex};
3266    use std::time::Duration;
3267
3268    #[cfg(feature = "enterprise")]
3269    use api::v1::meta::{ProcedureDetailResponse, ReconcileRequest, ReconcileResponse};
3270    use arrow::datatypes::{DataType as ArrowDataType, Field as ArrowField};
3271    #[cfg(feature = "enterprise")]
3272    use common_meta::cache_invalidator::{CacheInvalidator, CacheInvalidatorRef};
3273    #[cfg(feature = "enterprise")]
3274    use common_meta::instruction::CacheIdent;
3275    #[cfg(feature = "enterprise")]
3276    use common_meta::key::table_route::TableRouteValue;
3277    #[cfg(feature = "enterprise")]
3278    use common_meta::key::test_utils::new_test_table_info_with_name;
3279    #[cfg(feature = "enterprise")]
3280    use common_meta::key::{TableMetadataManager, TableMetadataManagerRef};
3281    #[cfg(feature = "enterprise")]
3282    use common_meta::kv_backend::memory::MemoryKvBackend;
3283    #[cfg(feature = "enterprise")]
3284    use common_meta::procedure_executor::{
3285        ExecutorContext, ProcedureExecutor, ProcedureExecutorRef,
3286    };
3287    #[cfg(feature = "enterprise")]
3288    use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse};
3289    #[cfg(feature = "enterprise")]
3290    use common_meta::rpc::procedure::{
3291        MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse,
3292    };
3293    use datafusion::functions_aggregate::expr_fn::count;
3294    use datafusion::logical_expr::builder::LogicalTableSource;
3295    use datafusion::logical_expr::{LogicalPlanBuilder, col};
3296    use session::context::QueryContext;
3297    use sql::dialect::GreptimeDbDialect;
3298    use sql::parser::{ParseOptions, ParserContext};
3299    use sql::statements::statement::Statement;
3300    use sqlparser::parser::Parser;
3301    use table::requests::REPARTITION_COLUMN_HINT_KEY;
3302
3303    use super::*;
3304    use crate::expr_helper;
3305
3306    #[cfg(feature = "enterprise")]
3307    #[derive(Default)]
3308    struct RecordingProcedureExecutor {
3309        requests: Mutex<Vec<SubmitDdlTaskRequest>>,
3310        contexts: Mutex<Vec<ExecutorContext>>,
3311        fail: bool,
3312    }
3313
3314    #[cfg(feature = "enterprise")]
3315    #[async_trait::async_trait]
3316    impl ProcedureExecutor for RecordingProcedureExecutor {
3317        async fn submit_ddl_task(
3318            &self,
3319            ctx: ExecutorContext,
3320            request: SubmitDdlTaskRequest,
3321        ) -> common_meta::error::Result<SubmitDdlTaskResponse> {
3322            self.contexts.lock().unwrap().push(ctx);
3323            self.requests.lock().unwrap().push(request);
3324            if self.fail {
3325                return common_meta::error::UnsupportedSnafu {
3326                    operation: "test submit failure",
3327                }
3328                .fail();
3329            }
3330            Ok(SubmitDdlTaskResponse::default())
3331        }
3332
3333        async fn migrate_region(
3334            &self,
3335            _: &ExecutorContext,
3336            _: MigrateRegionRequest,
3337        ) -> common_meta::error::Result<MigrateRegionResponse> {
3338            unimplemented!()
3339        }
3340        async fn reconcile(
3341            &self,
3342            _: &ExecutorContext,
3343            _: ReconcileRequest,
3344        ) -> common_meta::error::Result<ReconcileResponse> {
3345            unimplemented!()
3346        }
3347        async fn query_procedure_state(
3348            &self,
3349            _: &ExecutorContext,
3350            _: &str,
3351        ) -> common_meta::error::Result<ProcedureStateResponse> {
3352            unimplemented!()
3353        }
3354        async fn list_procedures(
3355            &self,
3356            _: &ExecutorContext,
3357        ) -> common_meta::error::Result<ProcedureDetailResponse> {
3358            unimplemented!()
3359        }
3360    }
3361
3362    #[cfg(feature = "enterprise")]
3363    #[derive(Default)]
3364    struct RecordingCacheInvalidator {
3365        invalidations: Mutex<Vec<Vec<CacheIdent>>>,
3366        fail: bool,
3367    }
3368
3369    #[cfg(feature = "enterprise")]
3370    #[async_trait::async_trait]
3371    impl CacheInvalidator for RecordingCacheInvalidator {
3372        async fn invalidate(
3373            &self,
3374            _: &Context,
3375            caches: &[CacheIdent],
3376        ) -> common_meta::error::Result<()> {
3377            self.invalidations.lock().unwrap().push(caches.to_vec());
3378            if self.fail {
3379                return common_meta::error::UnsupportedSnafu {
3380                    operation: "test cache failure",
3381                }
3382                .fail();
3383            }
3384            Ok(())
3385        }
3386
3387        fn invalidate_all(&self) -> common_meta::error::Result<()> {
3388            Ok(())
3389        }
3390    }
3391
3392    #[cfg(feature = "enterprise")]
3393    async fn dropped_table_manager(table_id: TableId, name: &TableName) -> TableMetadataManagerRef {
3394        let backend = Arc::new(MemoryKvBackend::default());
3395        let manager = Arc::new(TableMetadataManager::new(backend));
3396        let mut info = new_test_table_info_with_name(table_id, &name.table_name);
3397        info.catalog_name = name.catalog_name.clone();
3398        info.schema_name = name.schema_name.clone();
3399        let route = TableRouteValue::physical(vec![]);
3400        manager
3401            .create_table_metadata(info, route.clone(), HashMap::new())
3402            .await
3403            .unwrap();
3404        manager
3405            .delete_table_metadata(table_id, name, &route, &HashMap::new(), None)
3406            .await
3407            .unwrap();
3408        manager
3409    }
3410
3411    #[cfg(feature = "enterprise")]
3412    #[tokio::test]
3413    async fn test_undrop_table_execution_path() {
3414        let name = TableName::new("greptime", "public", "metrics");
3415        let manager = dropped_table_manager(42, &name).await;
3416        // A same-name live table must not redirect restoration to its ID.
3417        let mut live = new_test_table_info_with_name(99, "metrics");
3418        live.catalog_name = "greptime".to_string();
3419        live.schema_name = "public".to_string();
3420        manager
3421            .create_table_metadata(live, TableRouteValue::physical(vec![]), HashMap::new())
3422            .await
3423            .unwrap();
3424        let procedure = Arc::new(RecordingProcedureExecutor::default());
3425        let cache = Arc::new(RecordingCacheInvalidator::default());
3426
3427        execute_undrop_table(
3428            &manager,
3429            &(procedure.clone() as ProcedureExecutorRef),
3430            &(cache.clone() as CacheInvalidatorRef),
3431            name.clone(),
3432            QueryContext::arc(),
3433        )
3434        .await
3435        .unwrap();
3436
3437        let requests = procedure.requests.lock().unwrap();
3438        assert!(matches!(&requests[0].task, DdlTask::UndropTable(task) if task.table_id == 42));
3439        let contexts = procedure.contexts.lock().unwrap();
3440        assert_eq!(
3441            contexts[0].event_input.as_ref().map(|input| input.reason),
3442            Some(TriggerReason::Manual)
3443        );
3444        assert_eq!(
3445            cache.invalidations.lock().unwrap()[0],
3446            vec![CacheIdent::TableId(42), CacheIdent::TableName(name)]
3447        );
3448    }
3449
3450    #[cfg(feature = "enterprise")]
3451    #[tokio::test]
3452    async fn test_undrop_table_missing_tombstone_submits_nothing() {
3453        let manager = Arc::new(TableMetadataManager::new(Arc::new(
3454            MemoryKvBackend::default(),
3455        )));
3456        let procedure = Arc::new(RecordingProcedureExecutor::default());
3457        let cache = Arc::new(RecordingCacheInvalidator::default());
3458        let err = execute_undrop_table(
3459            &manager,
3460            &(procedure.clone() as ProcedureExecutorRef),
3461            &(cache as CacheInvalidatorRef),
3462            TableName::new("greptime", "public", "missing"),
3463            QueryContext::arc(),
3464        )
3465        .await
3466        .unwrap_err();
3467        assert!(matches!(err, crate::error::Error::TableNotFound { .. }));
3468        assert!(procedure.requests.lock().unwrap().is_empty());
3469    }
3470
3471    #[cfg(feature = "enterprise")]
3472    #[tokio::test]
3473    async fn test_undrop_table_submit_failure_does_not_invalidate() {
3474        let name = TableName::new("greptime", "public", "metrics");
3475        let manager = dropped_table_manager(42, &name).await;
3476        let procedure = Arc::new(RecordingProcedureExecutor {
3477            fail: true,
3478            ..Default::default()
3479        });
3480        let cache = Arc::new(RecordingCacheInvalidator::default());
3481        execute_undrop_table(
3482            &manager,
3483            &(procedure as ProcedureExecutorRef),
3484            &(cache.clone() as CacheInvalidatorRef),
3485            name,
3486            QueryContext::arc(),
3487        )
3488        .await
3489        .unwrap_err();
3490        assert!(cache.invalidations.lock().unwrap().is_empty());
3491    }
3492
3493    #[cfg(feature = "enterprise")]
3494    #[tokio::test]
3495    async fn test_undrop_table_cache_failure_returns_success_after_submit() {
3496        let name = TableName::new("greptime", "public", "metrics");
3497        let manager = dropped_table_manager(42, &name).await;
3498        let procedure = Arc::new(RecordingProcedureExecutor::default());
3499        let cache = Arc::new(RecordingCacheInvalidator {
3500            fail: true,
3501            ..Default::default()
3502        });
3503        execute_undrop_table(
3504            &manager,
3505            &(procedure.clone() as ProcedureExecutorRef),
3506            &(cache.clone() as CacheInvalidatorRef),
3507            name.clone(),
3508            QueryContext::arc(),
3509        )
3510        .await
3511        .unwrap();
3512        assert_eq!(procedure.requests.lock().unwrap().len(), 1);
3513        assert_eq!(
3514            cache.invalidations.lock().unwrap()[0],
3515            vec![CacheIdent::TableId(42), CacheIdent::TableName(name)]
3516        );
3517    }
3518
3519    #[test]
3520    fn test_parse_ddl_options() {
3521        let defaults = parse_ddl_options(&OptionMap::default()).unwrap();
3522        assert!(defaults.wait);
3523        assert_eq!(defaults.timeout, SubmitDdlTaskRequest::default_timeout());
3524
3525        let options = OptionMap::from([
3526            ("timeout".to_string(), "5m".to_string()),
3527            ("wait".to_string(), "false".to_string()),
3528        ]);
3529        let ddl_options = parse_ddl_options(&options).unwrap();
3530        assert!(!ddl_options.wait);
3531        assert_eq!(Duration::from_secs(300), ddl_options.timeout);
3532
3533        let options = OptionMap::from([("timeout".to_string(), "1500ms".to_string())]);
3534        assert_eq!(
3535            parse_ddl_options(&options).unwrap().timeout,
3536            Duration::from_millis(1500)
3537        );
3538        for (key, value) in [
3539            ("wait", "yes"),
3540            ("timeout", "not-a-duration"),
3541            ("timeout", "-1s"),
3542        ] {
3543            let options = OptionMap::from([(key.to_string(), value.to_string())]);
3544            assert!(matches!(
3545                parse_ddl_options(&options),
3546                Err(error::Error::InvalidSql { .. })
3547            ));
3548        }
3549    }
3550
3551    #[test]
3552    fn test_validate_and_normalize_annotations() {
3553        let schema = Schema::new(vec![
3554            ColumnSchema::new("service_name", ConcreteDataType::string_datatype(), true),
3555            ColumnSchema::new("host_id", ConcreteDataType::string_datatype(), true),
3556            ColumnSchema::new("value", ConcreteDataType::float64_datatype(), true),
3557            ColumnSchema::new("payload", ConcreteDataType::binary_datatype(), true),
3558        ]);
3559        let opts = |pairs: &[(&str, &str)]| TableOptions {
3560            extra_options: pairs
3561                .iter()
3562                .map(|(k, v)| (k.to_string(), v.to_string()))
3563                .collect(),
3564            ..Default::default()
3565        };
3566        let check = |pairs: &[(&str, &str)]| {
3567            let mut options = opts(pairs);
3568            validate_and_normalize_annotations(&mut options, &schema, &[]).map(|()| options)
3569        };
3570
3571        // Any existing column with a string form may be an id, tag or field.
3572        for pairs in [
3573            &[(
3574                "greptime.semantic.entity.process.id",
3575                "service_name,host_id",
3576            )][..],
3577            &[("greptime.semantic.entity.service.id", "value")][..],
3578        ] {
3579            assert!(check(pairs).is_ok());
3580        }
3581
3582        // Missing columns keep this crate's error variant and status code.
3583        let missing = check(&[("greptime.semantic.entity.service.id", "nope")]).unwrap_err();
3584        assert!(matches!(missing, error::Error::ColumnNotFound { .. }));
3585        assert_eq!(
3586            common_error::status_code::StatusCode::InvalidArguments,
3587            common_error::ext::ErrorExt::status_code(&missing)
3588        );
3589
3590        let binary_id = check(&[("greptime.semantic.entity.service.id", "payload")]).unwrap_err();
3591        assert!(matches!(binary_id, error::Error::InvalidSql { .. }));
3592
3593        // The SQL parser checks keys and value domains, but gRPC expressions
3594        // bypass it.
3595        let bad_value = check(&[("greptime.semantic.signal_type", "garbage")]).unwrap_err();
3596        assert!(matches!(bad_value, error::Error::InvalidSql { .. }));
3597
3598        let unknown_key = check(&[("greptime.semantic.nonsense", "x")]).unwrap_err();
3599        assert!(matches!(unknown_key, error::Error::InvalidSql { .. }));
3600    }
3601
3602    #[test]
3603    fn test_validate_and_normalize_flow_options_empty() {
3604        assert!(
3605            validate_and_normalize_flow_options(HashMap::new(), None)
3606                .unwrap()
3607                .is_empty()
3608        );
3609    }
3610
3611    #[test]
3612    fn test_validate_and_normalize_flow_options_valid() {
3613        let options = HashMap::from([
3614            (DEFER_ON_MISSING_SOURCE_KEY.to_string(), "TRUE".to_string()),
3615            (
3616                FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
3617                "FALSE".to_string(),
3618            ),
3619        ]);
3620
3621        assert_eq!(
3622            validate_and_normalize_flow_options(options, None).unwrap(),
3623            HashMap::from([
3624                (DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string(),),
3625                (
3626                    FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(),
3627                    "false".to_string(),
3628                )
3629            ])
3630        );
3631    }
3632
3633    #[cfg(not(feature = "enterprise"))]
3634    #[test]
3635    fn test_validate_and_normalize_flow_options_unknown_option() {
3636        let err = validate_and_normalize_flow_options(
3637            HashMap::from([("foo".to_string(), "bar".to_string())]),
3638            None,
3639        )
3640        .unwrap_err();
3641
3642        assert!(
3643            err.to_string()
3644                .contains("unknown flow option 'foo', supported options:")
3645        );
3646    }
3647
3648    #[test]
3649    fn test_validate_and_normalize_flow_options_reserved_option() {
3650        let err = validate_and_normalize_flow_options(
3651            HashMap::from([(
3652                FlowType::FLOW_TYPE_KEY.to_string(),
3653                FlowType::BATCHING.to_string(),
3654            )]),
3655            None,
3656        )
3657        .unwrap_err();
3658
3659        assert!(
3660            err.to_string()
3661                .contains("flow option 'flow_type' is reserved for internal use")
3662        );
3663    }
3664
3665    #[test]
3666    fn test_validate_and_normalize_flow_options_invalid_bool() {
3667        let err = validate_and_normalize_flow_options(
3668            HashMap::from([(
3669                DEFER_ON_MISSING_SOURCE_KEY.to_string(),
3670                "not-a-bool".to_string(),
3671            )]),
3672            None,
3673        )
3674        .unwrap_err();
3675
3676        assert!(
3677            err.to_string()
3678                .contains("invalid flow option 'defer_on_missing_source': 'not-a-bool'")
3679        );
3680    }
3681
3682    #[test]
3683    fn test_parser_flow_option_access_key_id() {
3684        let sql = r"
3685CREATE FLOW task_6
3686SINK TO schema_1.table_1
3687WITH (access_key_id = [true])
3688AS
3689SELECT max(c1), min(c2) FROM schema_2.table_2;";
3690        let stmt =
3691            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
3692                .unwrap()
3693                .pop()
3694                .unwrap();
3695
3696        let Statement::CreateFlow(create_flow) = stmt else {
3697            unreachable!()
3698        };
3699        let expr =
3700            expr_helper::to_create_flow_task_expr(create_flow, &QueryContext::arc()).unwrap();
3701
3702        #[cfg(not(feature = "enterprise"))]
3703        {
3704            let err = validate_and_normalize_flow_options(expr.flow_options, None).unwrap_err();
3705            assert!(
3706                err.to_string()
3707                    .contains("unknown flow option 'access_key_id'")
3708            );
3709        }
3710        #[cfg(feature = "enterprise")]
3711        assert_eq!(
3712            validate_and_normalize_flow_options(expr.flow_options, None).unwrap(),
3713            HashMap::from([("access_key_id".to_string(), "['true']".to_string())])
3714        );
3715    }
3716
3717    fn stateless_test_scan(name: &str) -> LogicalPlan {
3718        let schema = arrow::datatypes::Schema::new(vec![ArrowField::new(
3719            "value",
3720            ArrowDataType::Int32,
3721            true,
3722        )]);
3723        LogicalPlanBuilder::scan(
3724            name,
3725            Arc::new(LogicalTableSource::new(Arc::new(schema))),
3726            None,
3727        )
3728        .unwrap()
3729        .build()
3730        .unwrap()
3731    }
3732
3733    #[test]
3734    fn test_is_stateless_flow_plan_accepts_scan_projection_and_filter() {
3735        let scan = stateless_test_scan("source");
3736        assert!(is_stateless_flow_plan(&scan));
3737
3738        let projection = LogicalPlanBuilder::from(scan.clone())
3739            .project(vec![col("value")])
3740            .unwrap()
3741            .build()
3742            .unwrap();
3743        assert!(is_stateless_flow_plan(&projection));
3744
3745        let filter = LogicalPlanBuilder::from(projection)
3746            .filter(col("value").gt(datafusion_expr::lit(0)))
3747            .unwrap()
3748            .build()
3749            .unwrap();
3750        assert!(is_stateless_flow_plan(&filter));
3751    }
3752
3753    #[test]
3754    fn test_is_stateless_flow_plan_rejects_aggregate_and_multiple_scans() {
3755        let scan = stateless_test_scan("source");
3756        let aggregate = LogicalPlanBuilder::from(scan.clone())
3757            .aggregate(
3758                Vec::<datafusion_expr::Expr>::new(),
3759                vec![count(col("value"))],
3760            )
3761            .unwrap()
3762            .build()
3763            .unwrap();
3764        assert!(!is_stateless_flow_plan(&aggregate));
3765
3766        let distinct = LogicalPlanBuilder::from(scan.clone())
3767            .distinct()
3768            .unwrap()
3769            .build()
3770            .unwrap();
3771        assert!(is_stateless_flow_plan(&distinct));
3772
3773        let distinct_on_aggregate = LogicalPlanBuilder::from(aggregate)
3774            .distinct()
3775            .unwrap()
3776            .build()
3777            .unwrap();
3778        assert!(!is_stateless_flow_plan(&distinct_on_aggregate));
3779
3780        let multiple_scans = LogicalPlanBuilder::from(stateless_test_scan("left"))
3781            .cross_join(stateless_test_scan("right"))
3782            .unwrap()
3783            .build()
3784            .unwrap();
3785        assert!(!is_stateless_flow_plan(&multiple_scans));
3786    }
3787
3788    // --- Schedule option tests ---
3789
3790    #[test]
3791    fn test_eval_interval_rejected_non_positive() {
3792        // Zero eval_interval should be rejected.
3793        let err = validate_and_normalize_flow_options(HashMap::new(), Some(0)).unwrap_err();
3794        assert!(err.to_string().contains("EVAL INTERVAL must be positive"));
3795
3796        // Negative eval_interval should be rejected.
3797        let err = validate_and_normalize_flow_options(HashMap::new(), Some(-5)).unwrap_err();
3798        assert!(err.to_string().contains("EVAL INTERVAL must be positive"));
3799
3800        // Positive eval_interval should be accepted.
3801        let result = validate_and_normalize_flow_options(HashMap::new(), Some(300));
3802        assert!(result.is_ok());
3803    }
3804
3805    #[test]
3806    fn test_internal_schedule_key_rejected() {
3807        let err = validate_and_normalize_flow_options(
3808            HashMap::from([(INTERNAL_EVAL_SCHEDULE_KEY.to_string(), "value".to_string())]),
3809            Some(300),
3810        )
3811        .unwrap_err();
3812        assert!(err.to_string().contains("reserved for internal use"));
3813    }
3814
3815    #[cfg(not(feature = "enterprise"))]
3816    #[test]
3817    fn test_schedule_keys_rejected_as_unknown_options() {
3818        for key in [
3819            "eval_interval_anchor",
3820            "eval_interval_start",
3821            "eval_interval_missed_tick_policy",
3822            "eval_interval_catchup_max_runs",
3823            "eval_interval_catchup_max_lag",
3824        ] {
3825            let err = validate_and_normalize_flow_options(
3826                HashMap::from([(key.to_string(), "value".to_string())]),
3827                Some(300),
3828            )
3829            .unwrap_err();
3830            assert!(
3831                err.to_string()
3832                    .contains(&format!("unknown flow option '{key}'"))
3833            );
3834        }
3835    }
3836
3837    #[cfg(feature = "enterprise")]
3838    #[test]
3839    fn test_schedule_keys_preserved() {
3840        let options = HashMap::from([
3841            ("eval_interval_anchor".to_string(), "anchor".to_string()),
3842            ("eval_interval_start".to_string(), "start".to_string()),
3843            (
3844                "eval_interval_missed_tick_policy".to_string(),
3845                "missed".to_string(),
3846            ),
3847            (
3848                "eval_interval_catchup_max_runs".to_string(),
3849                "runs".to_string(),
3850            ),
3851            (
3852                "eval_interval_catchup_max_lag".to_string(),
3853                "lag".to_string(),
3854            ),
3855        ]);
3856        assert_eq!(
3857            validate_and_normalize_flow_options(options.clone(), Some(300)).unwrap(),
3858            options
3859        );
3860    }
3861
3862    #[test]
3863    fn test_internal_transport_keys_rejected_as_reserved() {
3864        for key in [
3865            "__greptime_internal_eval_schedule",
3866            "__greptime_internal_eval_offset_secs",
3867        ] {
3868            let err = validate_and_normalize_flow_options(
3869                HashMap::from([(key.to_string(), "120".to_string())]),
3870                Some(300),
3871            )
3872            .unwrap_err();
3873
3874            assert!(
3875                err.to_string()
3876                    .contains(&format!("flow option '{key}' is reserved for internal use")),
3877                "unexpected error for {key}: {err}"
3878            );
3879        }
3880    }
3881
3882    #[test]
3883    fn test_determine_flow_type_for_source_state_missing_sources_require_opt_in() {
3884        let err =
3885            determine_flow_type_for_source_state("my_flow", &HashMap::new(), true, false, false)
3886                .unwrap_err();
3887
3888        assert!(err.to_string().contains(
3889            "missing source tables for flow 'my_flow'; use WITH (defer_on_missing_source = true) to create a pending flow"
3890        ));
3891    }
3892
3893    #[test]
3894    fn test_determine_flow_type_for_source_state_missing_sources_prefer_batching() {
3895        let flow_options =
3896            HashMap::from([(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string())]);
3897
3898        assert_eq!(
3899            determine_flow_type_for_source_state("my_flow", &flow_options, true, true, false)
3900                .unwrap(),
3901            Some(FlowType::Batching)
3902        );
3903        let err = determine_flow_type_for_source_state("my_flow", &flow_options, true, true, true)
3904            .unwrap_err();
3905        assert!(err.to_string().contains(
3906            "flow 'my_flow' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode"
3907        ));
3908    }
3909
3910    #[test]
3911    fn test_determine_flow_type_for_source_state_existing_sources_require_plan() {
3912        assert_eq!(
3913            determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true, false)
3914                .unwrap(),
3915            None
3916        );
3917    }
3918
3919    #[test]
3920    fn test_determine_flow_type_for_source_state_instant_ttl_with_eval_interval_is_rejected() {
3921        // A flow with `EVAL INTERVAL` is forced onto the batching scheduler,
3922        // which cannot read instant-TTL source tables: reject instead of
3923        // silently falling back to streaming (where the schedule/offset would
3924        // be ignored).
3925        let err =
3926            determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true, true)
3927                .unwrap_err();
3928        assert!(
3929            err.to_string().contains(
3930                "flow 'my_flow' with EVAL INTERVAL requires the batching scheduler, but source tables with ttl=instant are not supported under batching mode"
3931            ),
3932            "unexpected error: {err}"
3933        );
3934    }
3935
3936    #[test]
3937    fn test_name_is_match() {
3938        assert!(!NAME_PATTERN_REG.is_match("/adaf"));
3939        assert!(!NAME_PATTERN_REG.is_match("🈲"));
3940        assert!(NAME_PATTERN_REG.is_match("hello"));
3941        assert!(NAME_PATTERN_REG.is_match("test@"));
3942        assert!(!NAME_PATTERN_REG.is_match("@test"));
3943        assert!(NAME_PATTERN_REG.is_match("test#"));
3944        assert!(!NAME_PATTERN_REG.is_match("#test"));
3945        assert!(!NAME_PATTERN_REG.is_match("@"));
3946        assert!(!NAME_PATTERN_REG.is_match("#"));
3947    }
3948
3949    #[test]
3950    fn test_convert_iceberg_hash() {
3951        let names = ["host", "count", "device", "measure", "host-name"].map(String::from);
3952        let column_name_and_type = HashMap::from([
3953            (&names[0], ConcreteDataType::string_datatype()),
3954            (&names[1], ConcreteDataType::int16_datatype()),
3955            (&names[2], ConcreteDataType::uint64_datatype()),
3956            (&names[3], ConcreteDataType::float64_datatype()),
3957            (&names[4], ConcreteDataType::string_datatype()),
3958        ]);
3959        let timezone = Timezone::from_tz_string("UTC").unwrap();
3960        let dialect = GreptimeDbDialect {};
3961        let parse = |sql: &str| {
3962            Parser::new(&dialect)
3963                .try_with_sql(sql)
3964                .unwrap()
3965                .parse_expr()
3966                .unwrap()
3967        };
3968
3969        for sql in [
3970            "iceberg_hash(host) < 0",
3971            "ICEBERG_HASH(device) >= -1073741824",
3972            "0 > iceberg_hash(count)",
3973            "iceberg_hash(host) < 100 AND host < 'm'",
3974            r#"iceberg_hash("host-name") < 0 AND "host-name" < 'm'"#,
3975        ] {
3976            let expr = convert_one_expr(&parse(sql), &column_name_and_type, &timezone).unwrap();
3977            // SHOW CREATE TABLE prints the rule back as SQL, quoting identifiers.
3978            let mut partitions = Partitions {
3979                column_list: vec![],
3980                exprs: vec![expr.to_parser_expr()],
3981            };
3982            partitions.set_quote('"');
3983            let restored = parse(&partitions.exprs[0].to_string());
3984            assert_eq!(
3985                convert_one_expr(&restored, &column_name_and_type, &timezone).unwrap(),
3986                expr,
3987                "{sql}"
3988            );
3989        }
3990
3991        for sql in [
3992            "iceberg_hash(measure) < 0",
3993            "iceberg_hash(host, count) < 0",
3994            "iceberg_hash('a') < 0",
3995            "iceberg_hash(missing) < 0",
3996            "iceberg_hash(DISTINCT host) < 0",
3997            "md5(host) < 0",
3998            "iceberg_hash(host) < host",
3999            "iceberg_hash(host) < 'abc'",
4000            "iceberg_hash(host) < 2147483648",
4001        ] {
4002            assert!(
4003                convert_one_expr(&parse(sql), &column_name_and_type, &timezone).is_err(),
4004                "{sql}"
4005            );
4006        }
4007    }
4008
4009    #[test]
4010    fn test_partition_expr_equivalence_with_swapped_operands() {
4011        let column_name = "device_id".to_string();
4012        let column_name_and_type =
4013            HashMap::from([(&column_name, ConcreteDataType::int32_datatype())]);
4014        let timezone = Timezone::from_tz_string("UTC").unwrap();
4015        let dialect = GreptimeDbDialect {};
4016
4017        let mut parser = Parser::new(&dialect)
4018            .try_with_sql("device_id < 100")
4019            .unwrap();
4020        let expr_left = parser.parse_expr().unwrap();
4021
4022        let mut parser = Parser::new(&dialect)
4023            .try_with_sql("100 > device_id")
4024            .unwrap();
4025        let expr_right = parser.parse_expr().unwrap();
4026
4027        let partition_left =
4028            convert_one_expr(&expr_left, &column_name_and_type, &timezone).unwrap();
4029        let partition_right =
4030            convert_one_expr(&expr_right, &column_name_and_type, &timezone).unwrap();
4031
4032        assert_eq!(partition_left, partition_right);
4033        assert!([partition_left.clone()].contains(&partition_right));
4034
4035        let mut physical_partition_exprs = vec![partition_left];
4036        let mut logical_partition_exprs = vec![partition_right];
4037        physical_partition_exprs.sort_unstable();
4038        logical_partition_exprs.sort_unstable();
4039        assert_eq!(physical_partition_exprs, logical_partition_exprs);
4040    }
4041
4042    #[test]
4043    fn test_repartition_target_partition_columns_are_overwrite_context() {
4044        let device_id = ColumnSchema::new("device_id", ConcreteDataType::int32_datatype(), true);
4045        let area = ColumnSchema::new("area", ConcreteDataType::string_datatype(), true);
4046        let existing_partition_columns = vec![&device_id];
4047        let target_partition_columns = vec![&device_id, &area];
4048        let existing_column_name_and_type = column_name_and_type(&existing_partition_columns);
4049        let target_column_name_and_type = column_name_and_type(&target_partition_columns);
4050        let timezone = Timezone::from_tz_string("UTC").unwrap();
4051        let dialect = GreptimeDbDialect {};
4052
4053        let mut parser = Parser::new(&dialect)
4054            .try_with_sql("device_id < 100 AND area < 'South'")
4055            .unwrap();
4056        let expr = parser.parse_expr().unwrap();
4057
4058        let err = convert_one_expr(&expr, &existing_column_name_and_type, &timezone).unwrap_err();
4059        assert!(err.to_string().contains("area"));
4060
4061        let partition_expr = convert_one_expr(&expr, &target_column_name_and_type, &timezone)
4062            .expect("target columns should overwrite the conversion context");
4063        let partition_expr = partition_expr.to_string();
4064        assert!(partition_expr.contains("device_id"));
4065        assert!(partition_expr.contains("area"));
4066        assert!(partition_expr.contains("South"));
4067    }
4068
4069    #[test]
4070    fn test_repartition_rejects_remaining_expr_outside_target_columns() {
4071        let device_id = "device_id".to_string();
4072        let area = "area".to_string();
4073        let timezone = Timezone::from_tz_string("UTC").unwrap();
4074        let column_name_and_type = HashMap::from([
4075            (&device_id, ConcreteDataType::int32_datatype()),
4076            (&area, ConcreteDataType::string_datatype()),
4077        ]);
4078        let dialect = GreptimeDbDialect {};
4079        let target_partition_columns = HashSet::from([&area]);
4080        for sql in ["device_id >= 100", "iceberg_hash(device_id) >= 0"] {
4081            let mut parser = Parser::new(&dialect).try_with_sql(sql).unwrap();
4082            let remaining_old_expr = convert_one_expr(
4083                &parser.parse_expr().unwrap(),
4084                &column_name_and_type,
4085                &timezone,
4086            )
4087            .unwrap();
4088
4089            let err = ensure_partition_expr_columns_in_target(
4090                &[remaining_old_expr],
4091                &target_partition_columns,
4092            )
4093            .unwrap_err();
4094
4095            assert!(err.to_string().contains("device_id"), "{sql}");
4096            assert!(
4097                err.to_string().contains("target partition columns"),
4098                "{sql}"
4099            );
4100        }
4101    }
4102
4103    #[test]
4104    fn test_repartition_rejects_duplicate_target_partition_columns() {
4105        let device_id = ColumnSchema::new("device_id", ConcreteDataType::int32_datatype(), true);
4106        let column_schemas = vec![device_id];
4107        let target_partition_columns = vec!["device_id".to_string(), "device_id".to_string()];
4108
4109        let err =
4110            validate_and_collect_partition_columns(&target_partition_columns, &column_schemas)
4111                .unwrap_err();
4112
4113        assert!(err.to_string().contains("duplicate partition column"));
4114        assert!(err.to_string().contains("device_id"));
4115    }
4116
4117    fn create_expr_from_sql(sql: &str) -> CreateTableExpr {
4118        let result =
4119            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
4120                .unwrap();
4121
4122        match &result[0] {
4123            Statement::CreateTable(create) => {
4124                expr_helper::create_to_expr(create, &QueryContext::arc()).unwrap()
4125            }
4126            _ => unreachable!(),
4127        }
4128    }
4129
4130    #[test]
4131    fn test_create_table_with_repartition_partition_num_hint() {
4132        let key = table::requests::REPARTITION_PARTITION_NUM_HINT_KEY;
4133        for (value, expected) in [
4134            (" 8 ", Some("8")),
4135            ("0", None),
4136            ("-1", None),
4137            ("4294967296", None),
4138        ] {
4139            let expr = create_expr_from_sql(&format!(
4140                "CREATE TABLE metrics (host STRING, ts TIMESTAMP TIME INDEX, PRIMARY KEY(host)) WITH ('{key}' = '{value}')"
4141            ));
4142            let result = create_table_info(&expr, vec![]);
4143            if let Some(expected) = expected {
4144                let info = result.unwrap();
4145                assert_eq!(
4146                    info.meta.options.extra_options.get(key).map(String::as_str),
4147                    Some(expected)
4148                );
4149            } else {
4150                assert!(
4151                    result
4152                        .unwrap_err()
4153                        .to_string()
4154                        .contains("expects a positive integer")
4155                );
4156            }
4157        }
4158    }
4159
4160    #[test]
4161    fn test_create_table_with_repartition_column_hint() {
4162        let expr = create_expr_from_sql(
4163            r"
4164CREATE TABLE metrics (
4165  host STRING,
4166  ts TIMESTAMP TIME INDEX,
4167  cpu DOUBLE,
4168  PRIMARY KEY(host)
4169)
4170WITH ('repartition.column.hint' = ' host ')",
4171        );
4172
4173        let table_info = create_table_info(&expr, vec![]).unwrap();
4174        assert_eq!(
4175            table_info
4176                .meta
4177                .options
4178                .extra_options
4179                .get(REPARTITION_COLUMN_HINT_KEY),
4180            Some(&"host".to_string())
4181        );
4182    }
4183
4184    #[test]
4185    fn test_create_table_info_rejects_non_timestamp_time_index() {
4186        let expr = CreateTableExpr {
4187            catalog_name: "greptime".to_string(),
4188            schema_name: "public".to_string(),
4189            table_name: "demo".to_string(),
4190            desc: String::new(),
4191            column_defs: vec![api::v1::ColumnDef {
4192                name: "host".to_string(),
4193                data_type: api::v1::ColumnDataType::String as i32,
4194                is_nullable: true,
4195                default_constraint: vec![],
4196                semantic_type: 0,
4197                comment: String::new(),
4198                datatype_extension: None,
4199                options: None,
4200            }],
4201            time_index: "host".to_string(),
4202            primary_keys: vec![],
4203            create_if_not_exists: false,
4204            table_options: HashMap::new(),
4205            table_id: None,
4206            engine: "mito".to_string(),
4207        };
4208
4209        let err = create_table_info(&expr, vec![]).unwrap_err();
4210        assert_eq!(
4211            common_error::ext::ErrorExt::status_code(&err),
4212            common_error::status_code::StatusCode::InvalidArguments
4213        );
4214    }
4215
4216    #[test]
4217    fn test_verify_alter_guards_entity_column_types() {
4218        let expr = create_expr_from_sql(
4219            "CREATE TABLE t (svc STRING, ts TIMESTAMP TIME INDEX) \
4220             WITH ('greptime.semantic.entity.service.id'='svc');",
4221        );
4222        let table_info = Arc::new(create_table_info(&expr, vec![]).unwrap());
4223        let alter = |kind| AlterTableExpr {
4224            catalog_name: "greptime".to_string(),
4225            schema_name: "public".to_string(),
4226            table_name: "t".to_string(),
4227            kind: Some(kind),
4228        };
4229
4230        let modify = alter(Kind::ModifyColumnTypes(api::v1::ModifyColumnTypes {
4231            modify_column_types: vec![api::v1::ModifyColumnType {
4232                column_name: "svc".to_string(),
4233                target_type: api::v1::ColumnDataType::Binary as i32,
4234                target_type_extension: None,
4235            }],
4236        }));
4237        let err = verify_alter(1, table_info.clone(), modify).unwrap_err();
4238        let msg = common_error::ext::ErrorExt::output_msg(&err);
4239        assert!(
4240            msg.contains("must keep a type that renders as a string"),
4241            "{msg}"
4242        );
4243
4244        // Dropping a declared column stays allowed: the derivation skips the
4245        // stale declaration.
4246        let drop = alter(Kind::DropColumns(api::v1::DropColumns {
4247            drop_columns: vec![api::v1::DropColumn {
4248                name: "svc".to_string(),
4249            }],
4250        }));
4251        assert!(verify_alter(1, table_info, drop).unwrap());
4252    }
4253
4254    #[test]
4255    fn test_json2_supports_non_append_mode() -> std::result::Result<(), Box<dyn std::error::Error>>
4256    {
4257        for options in [
4258            "",
4259            "WITH (append_mode='false')",
4260            "WITH (append_mode='true')",
4261            "WITH (merge_mode='last_non_null')",
4262        ] {
4263            let expr = create_expr_from_sql(&format!(
4264                "CREATE TABLE monitor (payload JSON2, ts TIMESTAMP TIME INDEX) {options};"
4265            ));
4266            create_table_info(&expr, vec![])?;
4267        }
4268        Ok(())
4269    }
4270
4271    #[test]
4272    fn test_json2_cannot_be_primary_key() -> std::result::Result<(), Box<dyn std::error::Error>> {
4273        let expr = create_expr_from_sql(
4274            "CREATE TABLE monitor (payload JSON2, ts TIMESTAMP TIME INDEX, PRIMARY KEY(payload));",
4275        );
4276        let err = create_table_info(&expr, vec![])
4277            .err()
4278            .ok_or("expected JSON2 primary key rejection")?;
4279        assert!(
4280            err.to_string()
4281                .contains("JSON2 column `payload` cannot be a primary key"),
4282            "{err}"
4283        );
4284        Ok(())
4285    }
4286
4287    #[test]
4288    fn test_create_table_with_empty_repartition_column_hint() {
4289        let expr = create_expr_from_sql(
4290            r"
4291CREATE TABLE metrics (
4292  host STRING,
4293  ts TIMESTAMP TIME INDEX,
4294  cpu DOUBLE,
4295  PRIMARY KEY(host)
4296)
4297WITH ('repartition.column.hint' = '')",
4298        );
4299
4300        let err = create_table_info(&expr, vec![]).unwrap_err();
4301        assert!(
4302            err.to_string()
4303                .contains("repartition.column.hint expects exactly one column name")
4304        );
4305    }
4306
4307    #[test]
4308    fn test_create_table_with_multiple_repartition_column_hints() {
4309        let expr = create_expr_from_sql(
4310            r"
4311CREATE TABLE metrics (
4312  host STRING,
4313  region_id STRING,
4314  ts TIMESTAMP TIME INDEX,
4315  cpu DOUBLE,
4316  PRIMARY KEY(host)
4317)
4318WITH ('repartition.column.hint' = 'host,region_id')",
4319        );
4320
4321        let err = create_table_info(&expr, vec![]).unwrap_err();
4322        assert!(
4323            err.to_string()
4324                .contains("repartition.column.hint expects exactly one column name")
4325        );
4326    }
4327
4328    #[test]
4329    fn test_create_table_with_missing_repartition_column_hint() {
4330        let expr = create_expr_from_sql(
4331            r"
4332CREATE TABLE metrics (
4333  host STRING,
4334  ts TIMESTAMP TIME INDEX,
4335  cpu DOUBLE,
4336  PRIMARY KEY(host)
4337)
4338WITH ('repartition.column.hint' = 'region_id')",
4339        );
4340
4341        let err = create_table_info(&expr, vec![]).unwrap_err();
4342        assert!(
4343            err.to_string()
4344                .contains("Cannot find column by name: region")
4345        );
4346    }
4347
4348    #[test]
4349    fn test_create_table_with_time_index_repartition_column_hint() {
4350        let expr = create_expr_from_sql(
4351            r"
4352CREATE TABLE metrics (
4353  host STRING,
4354  ts TIMESTAMP TIME INDEX,
4355  cpu DOUBLE,
4356  PRIMARY KEY(host)
4357)
4358WITH ('repartition.column.hint' = 'ts')",
4359        );
4360
4361        let err = create_table_info(&expr, vec![]).unwrap_err();
4362        assert!(
4363            err.to_string()
4364                .contains("cannot set repartition.column.hint to the time index column")
4365        );
4366    }
4367
4368    #[test]
4369    fn test_create_partitioned_table_with_repartition_column_hint() {
4370        let expr = create_expr_from_sql(
4371            r"
4372CREATE TABLE metrics (
4373  host STRING,
4374  ts TIMESTAMP TIME INDEX,
4375  cpu DOUBLE,
4376  PRIMARY KEY(host)
4377)
4378WITH ('repartition.column.hint' = 'host')",
4379        );
4380
4381        let err = create_table_info(&expr, vec!["host".to_string()]).unwrap_err();
4382        assert!(
4383            err.to_string()
4384                .contains("cannot set repartition.column.hint on a table with partition metadata")
4385        );
4386    }
4387}