Skip to main content

operator/
insert.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod otlp;
16
17pub use otlp::MetricInsertPreflight;
18
19use std::collections::{BTreeMap, BTreeSet};
20use std::sync::atomic::{AtomicU64, Ordering};
21use std::sync::{Arc, LazyLock};
22use std::time::{Duration, Instant};
23
24use ahash::{HashMap, HashMapExt, HashSet, HashSetExt};
25use api::v1::alter_table_expr::Kind;
26use api::v1::column_def::{options_from_compression, options_from_skipping, try_as_column_def};
27use api::v1::region::{
28    InsertRequest as RegionInsertRequest, InsertRequests as RegionInsertRequests,
29    RegionRequestHeader,
30};
31use api::v1::value::ValueData;
32use api::v1::{
33    AlterTableExpr, ColumnDataType, ColumnSchema, CreateTableExpr, InsertRequests,
34    RowInsertRequest, RowInsertRequests, Rows, SemanticType,
35};
36use arrow::datatypes::{DataType as ArrowDataType, Schema as ArrowSchema};
37use arrow_schema::extension::ExtensionType;
38use catalog::CatalogManagerRef;
39use client::{OutputData, OutputMeta};
40use common_base::readable_size::ReadableSize;
41use common_catalog::consts::{
42    DEFAULT_PRIVATE_SCHEMA_NAME, METRICS_EXEMPLARS_TABLE, PARENT_SPAN_ID_COLUMN,
43    SERVICE_NAME_COLUMN, SPAN_ID_COLUMN, TIMESTAMP_END_COLUMN, TRACE_ID_COLUMN, TRACE_TABLE_NAME,
44    TRACE_TABLE_NAME_SESSION_KEY, default_engine, is_ddl_reserved_table,
45    trace_operations_table_name, trace_services_table_name,
46};
47use common_event_recorder::DEFAULT_EVENTS_TABLE_NAME;
48use common_frontend::slow_query_event::SLOW_QUERY_TABLE_NAME;
49use common_grpc_expr::util::ColumnExpr;
50use common_meta::cache::TableFlownodeSetCacheRef;
51use common_meta::datanode::REGION_STATS_HISTORY_TABLE_NAME;
52use common_meta::node_manager::NodeManagerRef;
53use common_meta::peer::Peer;
54use common_meta::rpc::ddl::TriggerReason;
55use common_query::Output;
56use common_query::native_histogram::{is_native_histogram_value_type, native_histogram_value_type};
57use common_query::prelude::{
58    EXEMPLAR_METRIC_COLUMN, greptime_exemplar_labels, greptime_timestamp, greptime_value,
59};
60use common_telemetry::tracing_context::TracingContext;
61use common_telemetry::{debug, error, warn};
62use common_time::Timestamp;
63use common_time::timestamp::TimeUnit;
64use datatypes::schema::{
65    ColumnDefaultConstraint, ColumnEncoding, CompressionOptions, SkippingIndexOptions,
66};
67use futures_util::{StreamExt, TryStreamExt, future, stream};
68use meter_core::data::MeterRecord;
69use meter_macros::write_meter;
70use partition::manager::PartitionRuleManagerRef;
71use session::context::QueryContextRef;
72use snafu::ResultExt;
73use snafu::prelude::*;
74use sql::partition::partition_rule_for_hexstring;
75use sql::statements::create::Partitions;
76use sql::statements::insert::Insert;
77use store_api::metric_engine_consts::{
78    DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME, LOGICAL_TABLE_METADATA_KEY,
79    METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY,
80};
81use store_api::mito_engine_options::{
82    APPEND_MODE_KEY, COMPACTION_OPTION_KEYS, COMPACTION_TWCS_PREFIX, COMPACTION_TYPE,
83    COMPACTION_TYPE_TWCS, MERGE_MODE_KEY, TTL_KEY,
84};
85use store_api::storage::consts::ReservedColumnId;
86use store_api::storage::{RegionId, TableId};
87use table::TableRef;
88use table::metadata::{TableInfo, TableInfoRef};
89use table::requests::{
90    AUTO_CREATE_TABLE_KEY, InsertRequest as TableInsertRequest, SEMANTIC_PER_TABLE_INDEX_KEY,
91    SEMANTIC_PIPELINE, TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1, TABLE_DATA_MODEL_TRACE_V2,
92    TRACE_TABLE_PARTITIONS_HINT_KEY, VALID_TABLE_OPTION_KEYS, is_semantic_option_key,
93    validate_semantic_option,
94};
95use table::table_reference::TableReference;
96use tokio_util::sync::CancellationToken;
97
98use crate::batcher::PendingRowsBatcher;
99use crate::error::{
100    CatalogSnafu, ColumnOptionsSnafu, CreatePartitionRulesSnafu, InvalidInsertRequestSnafu,
101    JoinTaskSnafu, RequestInsertsSnafu, Result, TableNotFoundSnafu, WriteRejectedSnafu,
102};
103use crate::expr_helper;
104use crate::region_req_factory::RegionRequestFactory;
105use crate::req_convert::common::preprocess_row_insert_requests;
106use crate::req_convert::insert::{
107    ColumnToRow, ImpureDefaultFiller, RowToRegion, StatementToRegion, TableToRegion,
108    fill_reqs_with_impure_default, rows_to_record_batch, value_matches_type,
109};
110use crate::statement::StatementExecutor;
111use crate::statement::ddl::is_retention_ttl;
112use crate::write_dispatcher::{WriteDispatcher, WriteDispatcherRef};
113
114pub struct Inserter {
115    catalog_manager: CatalogManagerRef,
116    pub(crate) partition_manager: PartitionRuleManagerRef,
117    pub(crate) node_manager: NodeManagerRef,
118    pub(crate) write_dispatcher: WriteDispatcherRef,
119    pub(crate) table_flownode_set_cache: TableFlownodeSetCacheRef,
120    /// Server-side upper bound for auto table creation on write.
121    /// When `false`, missing tables are never auto-created regardless of the
122    /// per-request `auto_create_table` hint. When `true`, the hint still applies.
123    auto_create_table: bool,
124    /// Time index unit for physical metric tables created by auto table creation.
125    /// New logical tables created under such a physical table inherit its unit,
126    /// so all metric-engine ingestion paths write timestamps in this unit.
127    /// Defaults to milliseconds for compatibility.
128    auto_create_physical_table_timestamp_unit: TimeUnit,
129    pending_rows_batcher: Option<Arc<dyn PendingRowsBatcher>>,
130    /// Rows handed to detached flow mirror tasks that have not finished yet.
131    /// Bounds how much row data in-flight mirror writes may keep alive, see
132    /// [`MAX_MIRROR_PENDING_ROWS`].
133    mirror_pending_rows: Arc<AtomicU64>,
134}
135
136pub type InserterRef = Arc<Inserter>;
137
138/// Hint for the table type to create automatically.
139#[derive(Clone)]
140pub enum AutoCreateTableType {
141    /// A logical table with the physical table name.
142    Logical(String),
143    /// A physical table.
144    Physical,
145    /// The ordinary sparse append table for metric exemplars.
146    Exemplar,
147    /// A log table which is append-only.
148    Log,
149    /// A table that merges rows by `last_non_null` strategy.
150    LastNonNull,
151    /// Create table that build index and default partition rules on trace_id
152    Trace { alter_existing: bool },
153}
154
155impl AutoCreateTableType {
156    pub fn as_str(&self) -> &'static str {
157        match self {
158            AutoCreateTableType::Logical(_) => "logical",
159            AutoCreateTableType::Physical => "physical",
160            AutoCreateTableType::Exemplar => "exemplar",
161            AutoCreateTableType::Log => "log",
162            AutoCreateTableType::LastNonNull => "last_non_null",
163            AutoCreateTableType::Trace { .. } => "trace",
164        }
165    }
166
167    fn alter_existing(&self) -> bool {
168        !matches!(
169            self,
170            Self::Trace {
171                alter_existing: false
172            }
173        )
174    }
175}
176
177/// Split insert requests into normal and instant requests.
178///
179/// Where instant requests are requests with ttl=instant,
180/// and normal requests are requests with ttl set to other values.
181///
182/// This is used to split requests for different processing.
183#[derive(Clone, Default)]
184pub struct InstantAndNormalInsertRequests {
185    /// Requests with normal ttl.
186    pub normal_requests: RegionInsertRequests,
187    /// Requests with ttl=instant.
188    /// Will be discarded immediately at frontend, wouldn't even insert into memtable, and only sent to flow node if needed.
189    pub instant_requests: RegionInsertRequests,
190}
191
192/// Metric writes bound to the same resolved tables used by their exemplars.
193#[derive(Default)]
194pub struct PreparedMetricInserts {
195    requests: InstantAndNormalInsertRequests,
196    table_infos: HashMap<TableId, TableInfoRef>,
197}
198
199impl PreparedMetricInserts {
200    /// Returns the source tables captured during routing, including auto-created tables.
201    pub fn source_tables(&self) -> impl Iterator<Item = &TableInfoRef> {
202        self.table_infos.values()
203    }
204}
205
206impl Inserter {
207    /// Checks the assumptions of the logical bulk path without changing tables.
208    /// Unsupported requests retain ordinary insertion, including schema policy,
209    /// defaults, instant TTL and row-based Flow delivery.
210    pub async fn can_batch_metric_rows(
211        &self,
212        requests: &RowInsertRequests,
213        ctx: &QueryContextRef,
214        physical_table: &str,
215    ) -> Result<bool> {
216        if self.auto_create_disabled_reason(ctx)?.is_some() || ctx.extension(TTL_KEY).is_some() {
217            return Ok(false);
218        }
219        for request in &requests.inserts {
220            // The logical bulk encoder only supports scalar metric schemas;
221            // any time index unit is accepted — requests are converted to the
222            // destination table's unit during batch alignment.
223            // Check new tables too, before catalog lookup or schema changes.
224            if request.rows.as_ref().is_some_and(|rows| {
225                rows.schema.iter().any(|column| {
226                    column.datatype_extension.is_some()
227                        || !matches!(
228                            ColumnDataType::try_from(column.datatype),
229                            Ok(ColumnDataType::TimestampSecond
230                                | ColumnDataType::TimestampMillisecond
231                                | ColumnDataType::TimestampMicrosecond
232                                | ColumnDataType::TimestampNanosecond
233                                | ColumnDataType::Float64
234                                | ColumnDataType::String)
235                        )
236                })
237            }) {
238                return Ok(false);
239            }
240
241            let Some(table) = self
242                .get_table(
243                    ctx.current_catalog(),
244                    &ctx.current_schema(),
245                    &request.table_name,
246                )
247                .await?
248            else {
249                continue;
250            };
251            let info = table.table_info();
252            if info.meta.engine != METRIC_ENGINE_NAME
253                || info.is_ttl_instant_table()
254                || info
255                    .meta
256                    .options
257                    .extra_options
258                    .get(LOGICAL_TABLE_METADATA_KEY)
259                    .map(String::as_str)
260                    != Some(physical_table)
261                || info
262                    .meta
263                    .schema
264                    .column_schemas()
265                    .iter()
266                    .any(|column| column.default_constraint().is_some())
267            {
268                return Ok(false);
269            }
270            // Physical metric tags are nullable even when their logical schema
271            // is not. Arrow alignment is stricter than ordinary metric insertion.
272            if info
273                .meta
274                .primary_key_indices
275                .iter()
276                .any(|&index| !info.meta.schema.column_schemas()[index].is_nullable())
277            {
278                return Ok(false);
279            }
280            // The current Flow cache does not distinguish streaming and batch
281            // flows. Keep all Flow sources on the row-based delivery path.
282            match self.table_flownode_set_cache.get(info.table_id()).await {
283                Ok(None) => {}
284                Ok(Some(flows)) if flows.is_empty() => {}
285                _ => return Ok(false),
286            }
287        }
288        Ok(true)
289    }
290
291    /// Meters an original logical-table request before bulk routing, without
292    /// cloning its rows or changing the request boundary used for accounting.
293    pub async fn meter_row_inserts(
294        requests: &mut RowInsertRequests,
295        ctx: &QueryContextRef,
296    ) -> Result<u64> {
297        let metered = InstantAndNormalInsertRequests {
298            normal_requests: RegionInsertRequests {
299                requests: requests
300                    .inserts
301                    .iter_mut()
302                    .map(|request| RegionInsertRequest {
303                        rows: request.rows.take(),
304                        ..Default::default()
305                    })
306                    .collect(),
307            },
308            instant_requests: RegionInsertRequests::default(),
309        };
310        let cost = write_meter!(
311            ctx.current_catalog(),
312            ctx.current_schema(),
313            metered,
314            ctx.write_rows_to_admit(
315                ctx.current_catalog(),
316                &ctx.current_schema(),
317                count_insert_rows(&metered)?
318            ),
319            ctx.channel() as u8
320        )
321        .await
322        .context(WriteRejectedSnafu);
323        for (request, region) in requests
324            .inserts
325            .iter_mut()
326            .zip(metered.normal_requests.requests)
327        {
328            request.rows = region.rows;
329        }
330        cost
331    }
332
333    /// Creates a row inserter with the default write dispatcher.
334    pub fn new(
335        catalog_manager: CatalogManagerRef,
336        partition_manager: PartitionRuleManagerRef,
337        node_manager: NodeManagerRef,
338        table_flownode_set_cache: TableFlownodeSetCacheRef,
339        auto_create_table: bool,
340    ) -> Self {
341        Self {
342            write_dispatcher: Arc::new(WriteDispatcher::new(
343                node_manager.clone(),
344                partition_manager.clone(),
345            )),
346            catalog_manager,
347            partition_manager,
348            node_manager,
349            table_flownode_set_cache,
350            auto_create_table,
351            auto_create_physical_table_timestamp_unit: TimeUnit::Millisecond,
352            pending_rows_batcher: None,
353            mirror_pending_rows: Arc::new(AtomicU64::new(0)),
354        }
355    }
356
357    /// Sets the time index unit for auto-created physical metric tables.
358    pub fn with_auto_create_physical_table_timestamp_unit(mut self, unit: TimeUnit) -> Self {
359        self.auto_create_physical_table_timestamp_unit = unit;
360        self
361    }
362
363    /// Installs the shared dispatcher used by direct writes and batch flushes.
364    pub fn with_write_dispatcher(mut self, dispatcher: WriteDispatcherRef) -> Self {
365        self.write_dispatcher = dispatcher;
366        self
367    }
368
369    /// Installs the shared batcher; callers explicitly select its ingestion entry point.
370    pub fn with_pending_rows_batcher(
371        mut self,
372        batcher: Option<Arc<dyn PendingRowsBatcher>>,
373    ) -> Self {
374        self.pending_rows_batcher = batcher;
375        self
376    }
377
378    pub async fn handle_column_inserts(
379        &self,
380        requests: InsertRequests,
381        ctx: QueryContextRef,
382        statement_executor: &StatementExecutor,
383    ) -> Result<Output> {
384        let row_inserts = ColumnToRow::convert(requests)?;
385        self.handle_row_inserts(row_inserts, ctx, statement_executor, false, false)
386            .await
387    }
388
389    /// Handles row inserts request and creates a physical table on demand.
390    pub async fn handle_row_inserts(
391        &self,
392        mut requests: RowInsertRequests,
393        ctx: QueryContextRef,
394        statement_executor: &StatementExecutor,
395        accommodate_existing_schema: bool,
396        is_single_value: bool,
397    ) -> Result<Output> {
398        preprocess_row_insert_requests(&mut requests.inserts)?;
399        self.handle_row_inserts_with_create_type(
400            requests,
401            ctx,
402            statement_executor,
403            AutoCreateTableType::Physical,
404            accommodate_existing_schema,
405            is_single_value,
406        )
407        .await
408    }
409
410    /// Inserts exemplars using ordinary Mito storage and nanosecond auto-create defaults.
411    pub async fn handle_exemplar_inserts(
412        &self,
413        mut requests: RowInsertRequests,
414        ctx: QueryContextRef,
415        statement_executor: &StatementExecutor,
416        sources: &[TableInfoRef],
417    ) -> Result<Output> {
418        ensure!(
419            requests
420                .inserts
421                .iter()
422                .all(|r| r.table_name == METRICS_EXEMPLARS_TABLE),
423            InvalidInsertRequestSnafu {
424                reason: "invalid exemplar destination"
425            }
426        );
427        validate_column_count_match(&requests)?;
428        preprocess_row_insert_requests(&mut requests.inserts)?;
429        self.bind_exemplar_table_ids(&mut requests, &ctx, sources, statement_executor)
430            .await?;
431        self.handle_row_inserts_with_create_type(
432            requests,
433            ctx,
434            statement_executor,
435            AutoCreateTableType::Exemplar,
436            false,
437            false,
438        )
439        .await
440    }
441
442    async fn bind_exemplar_table_ids(
443        &self,
444        requests: &mut RowInsertRequests,
445        ctx: &QueryContextRef,
446        sources: &[TableInfoRef],
447        statement_executor: &StatementExecutor,
448    ) -> Result<()> {
449        let mut tables = HashMap::new();
450        let mut required_tags = BTreeMap::<String, BTreeSet<String>>::new();
451        for info in sources.iter().filter(|info| {
452            info.catalog_name == ctx.current_catalog() && info.schema_name == ctx.current_schema()
453        }) {
454            if let Some(previous) = tables.insert(info.name.clone(), info.clone()) {
455                ensure!(
456                    previous.table_id() == info.table_id(),
457                    InvalidInsertRequestSnafu {
458                        reason: format!(
459                            "exemplar source {} changed during write preparation",
460                            info.name
461                        ),
462                    }
463                );
464            }
465        }
466        for request in &mut requests.inserts {
467            let Some(rows) = request.rows.as_mut() else {
468                continue;
469            };
470            validate_exemplar_rows(rows)?;
471            ensure!(
472                !rows
473                    .schema
474                    .iter()
475                    .any(|c| c.column_name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME),
476                InvalidInsertRequestSnafu {
477                    reason: "exemplar table IDs must be resolved from their source tables",
478                }
479            );
480            let metric_index = rows
481                .schema
482                .iter()
483                .position(|c| c.column_name == EXEMPLAR_METRIC_COLUMN)
484                .context(InvalidInsertRequestSnafu {
485                    reason: "missing exemplar source metric",
486                })?;
487            for row in &mut rows.rows {
488                let Some(ValueData::StringValue(name)) = &row.values[metric_index].value_data
489                else {
490                    return InvalidInsertRequestSnafu {
491                        reason: "invalid exemplar source metric",
492                    }
493                    .fail();
494                };
495                if !tables.contains_key(name) {
496                    let table = self.get_table(ctx.current_catalog(), &ctx.current_schema(), name).await?
497                        .with_context(|| InvalidInsertRequestSnafu {
498                            reason: format!("exemplar source {name} must be an existing logical metric-engine table"),
499                        })?;
500                    tables.insert(name.clone(), table.table_info());
501                }
502                let info = &tables[name];
503                ensure!(
504                    info.meta.engine == METRIC_ENGINE_NAME
505                        && info
506                            .meta
507                            .options
508                            .extra_options
509                            .contains_key(LOGICAL_TABLE_METADATA_KEY),
510                    InvalidInsertRequestSnafu {
511                        reason: format!(
512                            "exemplar source {name} must be an existing logical metric-engine table"
513                        ),
514                    }
515                );
516                let tags = required_tags.entry(name.clone()).or_default();
517                for (column, value) in rows.schema.iter().zip(&row.values) {
518                    if column.semantic_type == SemanticType::Tag as i32
519                        && column.column_name != EXEMPLAR_METRIC_COLUMN
520                        && column.column_name != DATA_SCHEMA_TSID_COLUMN_NAME
521                        && value.value_data.is_some()
522                    {
523                        tags.insert(column.column_name.clone());
524                    }
525                }
526                row.values.push(ValueData::U32Value(info.table_id()).into());
527            }
528            rows.schema.push(api::v1::helper::tag_column_schema(
529                DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
530                ColumnDataType::Uint32,
531            ));
532        }
533
534        let mut alters = Vec::new();
535        let mut refresh = Vec::new();
536        for (name, tags) in required_tags {
537            let info = &tables[&name];
538            let tags = tags
539                .into_iter()
540                .map(|name| api::v1::helper::tag_column_schema(&name, ColumnDataType::String))
541                .collect::<Vec<_>>();
542            if let Some(add_columns) = exemplar_source_add_columns(info, &tags)? {
543                ensure!(
544                    self.auto_create_disabled_reason(ctx)?.is_none(),
545                    InvalidInsertRequestSnafu {
546                        reason: format!(
547                            "exemplar source {name} requires new tags, but automatic DDL is disabled"
548                        ),
549                    }
550                );
551                alters.push((
552                    info.table_id(),
553                    AlterTableExpr {
554                        catalog_name: info.catalog_name.clone(),
555                        schema_name: info.schema_name.clone(),
556                        table_name: name,
557                        kind: Some(Kind::AddColumns(add_columns)),
558                    },
559                ));
560                refresh.push((info, tags));
561            }
562        }
563        if !alters.is_empty() {
564            if let Some(table) = self
565                .get_table(
566                    ctx.current_catalog(),
567                    &ctx.current_schema(),
568                    METRICS_EXEMPLARS_TABLE,
569                )
570                .await?
571            {
572                // Reject an incompatible destination before changing any source schema.
573                validate_exemplar_table(&table.table_info())?;
574            }
575            statement_executor
576                .alter_logical_tables_with_ids(alters, ctx.clone(), TriggerReason::AutoAlter)
577                .await?;
578            for (pinned, tags) in refresh {
579                let table = self
580                    .get_table(&pinned.catalog_name, &pinned.schema_name, &pinned.name)
581                    .await?
582                    .with_context(|| InvalidInsertRequestSnafu {
583                        reason: format!(
584                            "exemplar source {} disappeared during reconciliation",
585                            pinned.name
586                        ),
587                    })?;
588                let info = table.table_info();
589                ensure!(
590                    info.table_id() == pinned.table_id(),
591                    InvalidInsertRequestSnafu {
592                        reason: format!(
593                            "exemplar source {} changed during reconciliation",
594                            pinned.name
595                        ),
596                    }
597                );
598                // Concurrent ADD IF NOT EXISTS can succeed on names alone; verify their definitions.
599                ensure!(
600                    exemplar_source_add_columns(&info, &tags)?.is_none(),
601                    InvalidInsertRequestSnafu {
602                        reason: format!(
603                            "exemplar source {} is still missing inherited tags",
604                            pinned.name
605                        ),
606                    }
607                );
608            }
609        }
610        Ok(())
611    }
612
613    /// Handles row inserts request and creates a log table on demand.
614    pub async fn handle_log_inserts(
615        &self,
616        requests: RowInsertRequests,
617        ctx: QueryContextRef,
618        statement_executor: &StatementExecutor,
619    ) -> Result<Output> {
620        self.handle_row_inserts_with_create_type(
621            requests,
622            ctx,
623            statement_executor,
624            AutoCreateTableType::Log,
625            false,
626            false,
627        )
628        .await
629    }
630
631    pub async fn handle_trace_inserts(
632        &self,
633        requests: RowInsertRequests,
634        ctx: QueryContextRef,
635        statement_executor: &StatementExecutor,
636    ) -> Result<Output> {
637        self.handle_row_inserts_with_create_type(
638            requests,
639            ctx,
640            statement_executor,
641            AutoCreateTableType::Trace {
642                alter_existing: true,
643            },
644            false,
645            false,
646        )
647        .await
648    }
649
650    /// Handles row inserts request and creates a table with `last_non_null` merge mode on demand.
651    pub async fn handle_last_non_null_inserts(
652        &self,
653        requests: RowInsertRequests,
654        ctx: QueryContextRef,
655        statement_executor: &StatementExecutor,
656        accommodate_existing_schema: bool,
657        is_single_value: bool,
658    ) -> Result<Output> {
659        self.handle_row_inserts_with_create_type(
660            requests,
661            ctx,
662            statement_executor,
663            AutoCreateTableType::LastNonNull,
664            accommodate_existing_schema,
665            is_single_value,
666        )
667        .await
668    }
669
670    /// Handles row inserts request with specified [AutoCreateTableType].
671    async fn handle_row_inserts_with_create_type(
672        &self,
673        mut requests: RowInsertRequests,
674        ctx: QueryContextRef,
675        statement_executor: &StatementExecutor,
676        create_type: AutoCreateTableType,
677        accommodate_existing_schema: bool,
678        is_single_value: bool,
679    ) -> Result<Output> {
680        let skip_wal = ctx.skip_wal();
681        let align_time_index_unit =
682            matches!(create_type, AutoCreateTableType::Exemplar).then_some(TimeUnit::Nanosecond);
683
684        let batcher = self
685            .pending_rows_batcher
686            .as_ref()
687            .filter(|_| ctx.batching_enabled());
688
689        // remove empty requests
690        requests.inserts.retain(|req| {
691            req.rows
692                .as_ref()
693                .map(|r| !r.rows.is_empty())
694                .unwrap_or_default()
695        });
696        validate_column_count_match(&requests)?;
697
698        let CreateAlterTableResult {
699            instant_table_ids,
700            table_infos,
701        } = self
702            .create_or_alter_tables_on_demand(
703                &mut requests,
704                &ctx,
705                create_type,
706                statement_executor,
707                accommodate_existing_schema,
708                is_single_value,
709                align_time_index_unit,
710            )
711            .await?;
712
713        // Instant tables have no persisted data for dirty-window Flow to read.
714        // Metric tables keep their existing dedicated ingestion path.
715        if let Some(batcher) = batcher
716            && instant_table_ids.is_empty()
717            && table_infos
718                .values()
719                .all(|info| info.meta.engine == default_engine())
720        {
721            return self
722                .submit_pending_rows(requests, table_infos, ctx, batcher)
723                .await;
724        }
725
726        let name_to_info = table_infos
727            .values()
728            .map(|info| (info.name.clone(), info.clone()))
729            .collect::<HashMap<_, _>>();
730        let inserts = RowToRegion::new(
731            name_to_info,
732            instant_table_ids,
733            self.partition_manager.as_ref(),
734        )
735        .convert(requests, skip_wal)
736        .await?;
737
738        self.do_request(inserts, &table_infos, &ctx).await
739    }
740
741    async fn submit_pending_rows(
742        &self,
743        mut requests: RowInsertRequests,
744        table_infos: HashMap<TableId, Arc<TableInfo>>,
745        ctx: QueryContextRef,
746        batcher: &Arc<dyn PendingRowsBatcher>,
747    ) -> Result<Output> {
748        // All entry points, including single-table and SQL writes, skip empty input
749        // before evaluating defaults or converting prepared rows.
750        requests.inserts.retain(|request| {
751            request
752                .rows
753                .as_ref()
754                .is_some_and(|rows| !rows.rows.is_empty())
755        });
756        let by_name = table_infos
757            .values()
758            .map(|info| (info.name.as_str(), info))
759            .collect::<HashMap<_, _>>();
760        let mut estimated_bytes = Some(0usize);
761        for request in &mut requests.inserts {
762            let table_info =
763                by_name
764                    .get(request.table_name.as_str())
765                    .context(TableNotFoundSnafu {
766                        table_name: &request.table_name,
767                    })?;
768            let Some(rows) = &mut request.rows else {
769                continue;
770            };
771            ImpureDefaultFiller::new((*table_info).clone())?.fill_rows(rows);
772            if batcher.memory_limit_bytes() != 0 {
773                estimated_bytes = estimated_bytes.and_then(|bytes| {
774                    bytes.checked_add(crate::req_convert::insert::estimated_record_batch_bytes(
775                        rows, table_info,
776                    )?)
777                });
778            }
779        }
780        // Reserve once for the whole request before constructing any Arrow arrays.
781        // PB rows and the converted batches coexist, so this is additional memory.
782        let permit = if requests.inserts.is_empty() {
783            None
784        } else {
785            Some(
786                batcher
787                    .acquire(
788                        estimated_bytes.unwrap_or(usize::MAX),
789                        ctx.held_memory_bytes(),
790                    )
791                    .await?,
792            )
793        };
794        let mut prepared = Vec::with_capacity(requests.inserts.len());
795        for request in &requests.inserts {
796            let table_info = by_name[request.table_name.as_str()];
797            if let Some(rows) = &request.rows {
798                prepared.push((table_info.clone(), rows_to_record_batch(rows, table_info)?));
799            }
800        }
801        // Preserve the existing meter input and original request boundary. These
802        // envelopes are only for accounting; routing happens after batching.
803        let metered = InstantAndNormalInsertRequests {
804            normal_requests: RegionInsertRequests {
805                requests: requests
806                    .inserts
807                    .into_iter()
808                    .map(|request| RegionInsertRequest {
809                        rows: request.rows,
810                        ..Default::default()
811                    })
812                    .collect(),
813            },
814            instant_requests: RegionInsertRequests::default(),
815        };
816        let table_info = table_infos.values().next();
817        let catalog = table_info.map_or(ctx.current_catalog(), |info| info.catalog_name.as_str());
818        let schema =
819            table_info.map_or_else(|| ctx.current_schema(), |info| info.schema_name.clone());
820        let write_cost = write_meter!(
821            catalog,
822            &schema,
823            metered,
824            ctx.write_rows_to_admit(catalog, &schema, count_insert_rows(&metered)?),
825            ctx.channel() as u8
826        )
827        .await
828        .context(WriteRejectedSnafu)?;
829        prepared.retain(|(_, batch)| batch.num_rows() != 0);
830        let results = if let Some(permit) = permit {
831            // One original request shares admission across all table submissions.
832            let submissions = prepared.into_iter().map(|(info, batch)| {
833                // Route to the same target database used for admission above.
834                let mut target_ctx = ctx.fork();
835                target_ctx.set_current_catalog(&info.catalog_name);
836                target_ctx.set_current_schema(&info.schema_name);
837                batcher.submit(info, batch, Arc::new(target_ctx), permit.clone())
838            });
839            // Observe every table completion even when another table fails.
840            future::join_all(submissions).await
841        } else {
842            Vec::new()
843        };
844        let mut affected_rows = 0;
845        let mut meta = OutputMeta::new_with_cost(write_cost as _);
846        let mut first_error = None;
847        for result in results {
848            match result {
849                Ok(output) => {
850                    affected_rows += output.extract_rows_and_cost().0;
851                    meta.write_completions.extend(output.meta.write_completions);
852                }
853                Err(error) if ctx.confirmed_writes() => {
854                    meta.write_completions
855                        .push(common_query::WriteCompletion::with_progress(async move {
856                            common_query::WriteProgress::failed(common_error::ext::BoxedError::new(
857                                error,
858                            ))
859                        }));
860                }
861                Err(error) => {
862                    first_error.get_or_insert(error);
863                }
864            }
865        }
866        if let Some(error) = first_error {
867            for completion in meta.write_completions {
868                let _ = completion.wait().await;
869            }
870            return Err(error);
871        }
872        Ok(Output::new(OutputData::AffectedRows(affected_rows), meta))
873    }
874
875    /// Handles row inserts request with metric engine.
876    ///
877    /// Resolves or creates `physical_table` only when a logical target is missing.
878    /// Existing targets retain their backing tables and time-index units.
879    pub async fn handle_metric_row_inserts(
880        &self,
881        requests: RowInsertRequests,
882        ctx: QueryContextRef,
883        statement_executor: &StatementExecutor,
884        physical_table: String,
885    ) -> Result<Output> {
886        let prepared = self
887            .prepare_metric_row_inserts(requests, ctx.clone(), statement_executor, physical_table)
888            .await?;
889        self.handle_prepared_metric_inserts(prepared, ctx).await
890    }
891
892    /// Resolves schemas and routes metric rows before dependent exemplar writes start.
893    pub async fn prepare_metric_row_inserts(
894        &self,
895        mut requests: RowInsertRequests,
896        ctx: QueryContextRef,
897        statement_executor: &StatementExecutor,
898        physical_table: String,
899    ) -> Result<PreparedMetricInserts> {
900        let skip_wal = ctx.skip_wal();
901
902        // remove empty requests
903        requests.inserts.retain(|req| {
904            req.rows
905                .as_ref()
906                .map(|r| !r.rows.is_empty())
907                .unwrap_or_default()
908        });
909        validate_column_count_match(&requests)?;
910
911        // The selected backing table is needed only by new logical targets.
912        let mut has_new_tables = false;
913        for request in &requests.inserts {
914            if self
915                .get_table(
916                    ctx.current_catalog(),
917                    &ctx.current_schema(),
918                    &request.table_name,
919                )
920                .await?
921                .is_none()
922            {
923                has_new_tables = true;
924                break;
925            }
926        }
927        let new_table_unit = if has_new_tables {
928            let physical = self
929                .create_physical_table_on_demand(&ctx, physical_table.clone(), statement_executor)
930                .await?;
931            table_time_index_unit(&physical)
932        } else {
933            None
934        };
935        let CreateAlterTableResult {
936            instant_table_ids,
937            table_infos,
938        } = self
939            .create_or_alter_tables_on_demand(
940                &mut requests,
941                &ctx,
942                AutoCreateTableType::Logical(physical_table.clone()),
943                statement_executor,
944                true,
945                true,
946                new_table_unit,
947            )
948            .await?;
949        let name_to_info = table_infos
950            .values()
951            .map(|info| (info.name.clone(), info.clone()))
952            .collect::<HashMap<_, _>>();
953        let inserts = RowToRegion::new(name_to_info, instant_table_ids, &self.partition_manager)
954            .convert(requests, skip_wal)
955            .await?;
956
957        Ok(PreparedMetricInserts {
958            requests: inserts,
959            table_infos,
960        })
961    }
962
963    /// Dispatches metric rows using the source table definitions captured during preparation.
964    pub async fn handle_prepared_metric_inserts(
965        &self,
966        prepared: PreparedMetricInserts,
967        ctx: QueryContextRef,
968    ) -> Result<Output> {
969        self.do_request(prepared.requests, &prepared.table_infos, &ctx)
970            .await
971    }
972
973    fn table_batcher(
974        &self,
975        table_info: &TableInfoRef,
976        ctx: &QueryContextRef,
977    ) -> Option<&Arc<dyn PendingRowsBatcher>> {
978        self.pending_rows_batcher.as_ref().filter(|_| {
979            ctx.batching_enabled()
980                && !table_info.is_ttl_instant_table()
981                && table_info.meta.engine == default_engine()
982        })
983    }
984
985    async fn submit_table_rows(
986        &self,
987        rows: Rows,
988        table_info: TableInfoRef,
989        ctx: QueryContextRef,
990        batcher: &Arc<dyn PendingRowsBatcher>,
991    ) -> Result<Output> {
992        let requests = RowInsertRequests {
993            inserts: vec![RowInsertRequest {
994                table_name: table_info.name.clone(),
995                rows: Some(rows),
996            }],
997        };
998        let table_infos = HashMap::from_iter([(table_info.table_id(), table_info)]);
999        self.submit_pending_rows(requests, table_infos, ctx, batcher)
1000            .await
1001    }
1002
1003    pub async fn handle_table_insert(
1004        &self,
1005        request: TableInsertRequest,
1006        ctx: QueryContextRef,
1007    ) -> Result<Output> {
1008        let catalog = request.catalog_name.as_str();
1009        let schema = request.schema_name.as_str();
1010        let table_name = request.table_name.as_str();
1011        let table = self.get_table(catalog, schema, table_name).await?;
1012        let table = table.with_context(|| TableNotFoundSnafu {
1013            table_name: common_catalog::format_full_table_name(catalog, schema, table_name),
1014        })?;
1015        let table_info = table.table_info();
1016
1017        let converter = TableToRegion::new(&table_info, &self.partition_manager);
1018        let skip_wal = request.skip_wal;
1019        let rows = converter.prepare(request)?;
1020        if let Some(batcher) = self.table_batcher(&table_info, &ctx) {
1021            return self.submit_table_rows(rows, table_info, ctx, batcher).await;
1022        }
1023        let inserts = converter.partition(rows, skip_wal).await?;
1024
1025        let table_infos = HashMap::from_iter([(table_info.table_id(), table_info.clone())]);
1026
1027        self.do_request(inserts, &table_infos, &ctx).await
1028    }
1029
1030    pub async fn handle_statement_insert(
1031        &self,
1032        insert: &Insert,
1033        ctx: &QueryContextRef,
1034    ) -> Result<Output> {
1035        let converter =
1036            StatementToRegion::new(self.catalog_manager.as_ref(), &self.partition_manager, ctx);
1037        let (rows, table_info) = converter.prepare(insert, ctx).await?;
1038        if let Some(batcher) = self.table_batcher(&table_info, ctx) {
1039            return self
1040                .submit_table_rows(rows, table_info, ctx.clone(), batcher)
1041                .await;
1042        }
1043        let inserts = converter.partition(rows, table_info.clone(), ctx).await?;
1044
1045        let table_infos = HashMap::from_iter([(table_info.table_id(), table_info.clone())]);
1046
1047        self.do_request(inserts, &table_infos, ctx).await
1048    }
1049}
1050
1051/// Admits a finite request before it is split into internal writes.
1052/// The returned context preserves accounting while preventing a second row debit.
1053pub async fn admit_write(rows: u64, ctx: &QueryContextRef) -> Result<QueryContextRef> {
1054    // The zero value is WCU: this record only admits rows. Actual inserts retain
1055    // their existing WCU accounting, so charging here would count it twice.
1056    write_meter!(MeterRecord::new(
1057        ctx.current_catalog().to_string(),
1058        ctx.current_schema(),
1059        0,
1060        ctx.write_rows_to_admit(ctx.current_catalog(), &ctx.current_schema(), rows),
1061        ctx.channel() as u8,
1062    ))
1063    .await
1064    .context(WriteRejectedSnafu)?;
1065    Ok(Arc::new(ctx.with_write_admission()))
1066}
1067
1068/// Admits all database totals before dispatching any batch of a finite request.
1069/// Each batch keeps its own protocol options and target database.
1070pub async fn admit_row_insert_batches(
1071    batches: &mut [(QueryContextRef, RowInsertRequests)],
1072) -> Result<()> {
1073    let mut totals = BTreeMap::<_, (QueryContextRef, u64)>::new();
1074    for (ctx, requests) in batches.iter() {
1075        let catalog = ctx.current_catalog();
1076        let schema = ctx.current_schema();
1077
1078        if ctx.write_rows_to_admit(catalog, &schema, 1) == 0 {
1079            continue;
1080        }
1081        let (_, total) = totals
1082            .entry((catalog.to_string(), schema.clone()))
1083            .or_insert_with(|| (ctx.clone(), 0));
1084        for rows in requests.inserts.iter().filter_map(|r| r.rows.as_ref()) {
1085            *total =
1086                total
1087                    .checked_add(rows.rows.len() as u64)
1088                    .context(InvalidInsertRequestSnafu {
1089                        reason: "Insert row count exceeds u64::MAX",
1090                    })?;
1091        }
1092    }
1093    for (ctx, rows) in totals.values() {
1094        admit_write(*rows, ctx).await?;
1095    }
1096    for (ctx, _) in batches {
1097        *ctx = Arc::new(ctx.with_write_admission());
1098    }
1099    Ok(())
1100}
1101
1102fn count_insert_rows(requests: &InstantAndNormalInsertRequests) -> Result<u64> {
1103    requests
1104        .normal_requests
1105        .requests
1106        .iter()
1107        .chain(&requests.instant_requests.requests)
1108        .filter_map(|request| request.rows.as_ref())
1109        .try_fold(0u64, |total, rows| {
1110            total
1111                .checked_add(rows.rows.len() as u64)
1112                .context(InvalidInsertRequestSnafu {
1113                    reason: "Insert row count exceeds u64::MAX",
1114                })
1115        })
1116}
1117
1118impl Inserter {
1119    async fn do_request(
1120        &self,
1121        requests: InstantAndNormalInsertRequests,
1122        table_infos: &HashMap<TableId, Arc<TableInfo>>,
1123        ctx: &QueryContextRef,
1124    ) -> Result<Output> {
1125        // Fill impure default values in the request
1126        let requests = fill_reqs_with_impure_default(table_infos, requests)?;
1127
1128        // All tables in a batch resolve to the same database. Qualified SQL
1129        // inserts may target a different database than the session's current one.
1130        let table_info = table_infos.values().next();
1131        let catalog = table_info.map_or(ctx.current_catalog(), |info| info.catalog_name.as_str());
1132        let schema =
1133            table_info.map_or_else(|| ctx.current_schema(), |info| info.schema_name.clone());
1134        let write_cost = write_meter!(
1135            catalog,
1136            schema.clone(),
1137            requests,
1138            ctx.write_rows_to_admit(catalog, &schema, count_insert_rows(&requests)?),
1139            ctx.channel() as u8
1140        )
1141        .await
1142        .context(WriteRejectedSnafu)?;
1143        let request_factory = RegionRequestFactory::new(RegionRequestHeader {
1144            tracing_context: TracingContext::from_current_span().to_w3c(),
1145            dbname: ctx.get_db_string(),
1146            ..Default::default()
1147        });
1148
1149        let InstantAndNormalInsertRequests {
1150            normal_requests,
1151            instant_requests,
1152        } = requests;
1153
1154        // Mirror requests for source table to flownode asynchronously
1155        let flow_mirror_task = FlowMirrorTask::new(
1156            &self.table_flownode_set_cache,
1157            normal_requests
1158                .requests
1159                .iter()
1160                .chain(instant_requests.requests.iter()),
1161        )
1162        .await?;
1163        let has_instant_rows = instant_requests.requests.iter().any(|request| {
1164            request
1165                .rows
1166                .as_ref()
1167                .is_some_and(|rows| !rows.rows.is_empty())
1168        });
1169        flow_mirror_task.detach(
1170            self.node_manager.clone(),
1171            self.mirror_pending_rows.clone(),
1172            has_instant_rows,
1173        )?;
1174
1175        let dispatcher = self.write_dispatcher.clone();
1176        if ctx.confirmed_writes() {
1177            let writes = normal_requests
1178                .requests
1179                .into_iter()
1180                .map(|request| {
1181                    let rows = request.rows.as_ref().map_or(0, |rows| rows.rows.len());
1182                    crate::write_dispatcher::TrackedWrite {
1183                        request: request_factory.build_insert(RegionInsertRequests {
1184                            requests: vec![request],
1185                        }),
1186                        owners: vec![0; rows],
1187                    }
1188                })
1189                .collect();
1190            let (writes, progress) = dispatcher.prepare_tracked(writes, 1).await;
1191            let cancellation = CancellationToken::new();
1192            let _guard = cancellation.clone().drop_guard();
1193            let counter = ctx.write_progress().cloned();
1194            let progress = common_runtime::spawn_global(async move {
1195                let progress = dispatcher
1196                    .execute_tracked(writes, progress, None, Some(&cancellation))
1197                    .await;
1198                if let Some(counter) = counter {
1199                    counter.add_written(progress.iter().map(|part| part.affected_rows).sum());
1200                }
1201                progress
1202            })
1203            .await
1204            .context(JoinTaskSnafu)?
1205            .into_iter()
1206            .fold(
1207                common_query::WriteProgress::default(),
1208                |mut result, part| {
1209                    result.merge(part);
1210                    result
1211                },
1212            );
1213            crate::metrics::DIST_INGEST_ROW_COUNT
1214                .with_label_values(&[ctx.get_db_string().as_str()])
1215                .inc_by(progress.affected_rows as u64);
1216            let mut meta = OutputMeta::new_with_cost(write_cost as _);
1217            let rows = progress.affected_rows;
1218            meta.write_completions
1219                .push(common_query::WriteCompletion::with_progress(async move {
1220                    progress
1221                }));
1222            return Ok(Output::new(OutputData::AffectedRows(rows), meta));
1223        }
1224        let request = request_factory.build_insert(normal_requests);
1225        let writes = dispatcher.prepare(vec![request]).await?;
1226        // Dropping the caller stops retries without abandoning in-flight writes.
1227        let cancellation = CancellationToken::new();
1228        let _guard = cancellation.clone().drop_guard();
1229        let progress = ctx.write_progress().cloned();
1230        let affected_rows = common_runtime::spawn_global(async move {
1231            dispatcher
1232                .execute_with_progress(writes, None, Some(&cancellation), progress)
1233                .await
1234        })
1235        .await
1236        .context(JoinTaskSnafu)?
1237        .context(RequestInsertsSnafu)?
1238        .affected_rows;
1239        crate::metrics::DIST_INGEST_ROW_COUNT
1240            .with_label_values(&[ctx.get_db_string().as_str()])
1241            .inc_by(affected_rows as u64);
1242        Ok(Output::new(
1243            OutputData::AffectedRows(affected_rows),
1244            OutputMeta::new_with_cost(write_cost as _),
1245        ))
1246    }
1247
1248    /// Returns `Some(reason)` if the config or request hint disables automatic
1249    /// table creation. Exempt private system tables are handled by
1250    /// [`Self::is_auto_create_exempt_private_table`].
1251    fn auto_create_disabled_reason(&self, ctx: &QueryContextRef) -> Result<Option<&'static str>> {
1252        let auto_create_table_hint = ctx
1253            .extension(AUTO_CREATE_TABLE_KEY)
1254            .map(|v| v.parse::<bool>())
1255            .transpose()
1256            .map_err(|_| {
1257                InvalidInsertRequestSnafu {
1258                    reason: "`auto_create_table` hint must be a boolean",
1259                }
1260                .build()
1261            })?
1262            .unwrap_or(true);
1263        Ok(if !self.auto_create_table {
1264            Some("auto-create table is disabled by frontend config")
1265        } else if !auto_create_table_hint {
1266            Some("`auto_create_table` hint is disabled")
1267        } else {
1268            None
1269        })
1270    }
1271
1272    /// Returns whether a private system table may infer and reconcile its schema
1273    /// even when automatic table creation is disabled.
1274    fn is_auto_create_exempt_private_table(schema: &str, table: &str) -> bool {
1275        schema == DEFAULT_PRIVATE_SCHEMA_NAME
1276            && matches!(
1277                table,
1278                DEFAULT_EVENTS_TABLE_NAME | SLOW_QUERY_TABLE_NAME | REGION_STATS_HISTORY_TABLE_NAME
1279            )
1280    }
1281
1282    /// Adds missing columns from a bulk stream's schema and returns the refreshed table.
1283    /// Call once when initializing the stream, before writing its first batch.
1284    /// Does not infer new nested or dictionary columns.
1285    pub async fn ensure_bulk_insert_schema(
1286        &self,
1287        table: TableRef,
1288        request_schema: &ArrowSchema,
1289        ctx: &QueryContextRef,
1290        statement_executor: &StatementExecutor,
1291    ) -> Result<TableRef> {
1292        let table_info = table.table_info();
1293        if self.auto_create_disabled_reason(ctx)?.is_some()
1294            && !Self::is_auto_create_exempt_private_table(&table_info.schema_name, &table_info.name)
1295        {
1296            return Ok(table);
1297        }
1298
1299        let table_schema = table.schema();
1300        let schema = request_schema
1301            .fields()
1302            .iter()
1303            .filter(|field| table_schema.column_schema_by_name(field.name()).is_none())
1304            .map(|field| {
1305                let data_type = field.data_type();
1306                // Dictionary values can reach the same infallible child-type conversion
1307                // as nested types, even when Arrow's is_nested() returns false.
1308                ensure!(
1309                    !data_type.is_nested() && !matches!(data_type, ArrowDataType::Dictionary(..)),
1310                    crate::error::NotSupportedSnafu {
1311                        feat: format!(
1312                            "automatically adding bulk insert column '{}' with type {:?}",
1313                            field.name(),
1314                            data_type
1315                        ),
1316                    }
1317                );
1318                let column = datatypes::schema::ColumnSchema::try_from(field.as_ref())
1319                    .context(crate::error::ConvertSchemaSnafu)?;
1320                // Arrow fields do not carry primary-key semantics. New columns are
1321                // fields, unless explicitly marked as a time index.
1322                let column_def =
1323                    try_as_column_def(&column, false).context(crate::error::ColumnDataTypeSnafu)?;
1324                Ok(ColumnSchema {
1325                    column_name: column_def.name,
1326                    datatype: column_def.data_type,
1327                    semantic_type: column_def.semantic_type,
1328                    datatype_extension: column_def.datatype_extension,
1329                    options: column_def.options,
1330                })
1331            })
1332            .collect::<Result<Vec<_>>>()?;
1333        let mut request = RowInsertRequest {
1334            table_name: table_info.name.clone(),
1335            rows: Some(Rows {
1336                schema,
1337                rows: Vec::new(),
1338            }),
1339        };
1340        let Some(alter_expr) =
1341            self.get_alter_table_expr_on_demand(&mut request, &table, ctx, false, false, true)?
1342        else {
1343            return Ok(table);
1344        };
1345
1346        statement_executor
1347            .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
1348            .await?;
1349        self.get_table(
1350            &table_info.catalog_name,
1351            &table_info.schema_name,
1352            &table_info.name,
1353        )
1354        .await?
1355        .with_context(|| TableNotFoundSnafu {
1356            table_name: table_info.full_table_name(),
1357        })
1358    }
1359
1360    /// Ensures a trace table has the request-global schema without requiring a
1361    /// padded data row to drive on-demand creation or alteration. When
1362    /// `alter_existing` is false, a table created after planning is left for the
1363    /// caller to re-plan.
1364    pub async fn ensure_trace_table_on_demand(
1365        &self,
1366        table_name: &str,
1367        request_schema: Vec<ColumnSchema>,
1368        alter_existing: bool,
1369        ctx: &QueryContextRef,
1370        statement_executor: &StatementExecutor,
1371    ) -> Result<()> {
1372        let mut requests = RowInsertRequests {
1373            inserts: vec![RowInsertRequest {
1374                table_name: table_name.to_string(),
1375                rows: Some(api::v1::Rows {
1376                    schema: request_schema,
1377                    rows: Vec::new(),
1378                }),
1379            }],
1380        };
1381        self.create_or_alter_tables_on_demand(
1382            &mut requests,
1383            ctx,
1384            AutoCreateTableType::Trace { alter_existing },
1385            statement_executor,
1386            false,
1387            false,
1388            None,
1389        )
1390        .await?;
1391        Ok(())
1392    }
1393
1394    /// Creates or alter tables on demand:
1395    /// - if table does not exist, create table by inferred CreateExpr
1396    /// - if table exist, check if schema matches. If any new column found, alter table by inferred `AlterExpr`
1397    ///
1398    /// Returns a mapping from table name to table id, where table name is the table name involved in the requests.
1399    /// This mapping is used in the conversion of RowToRegion.
1400    ///
1401    /// `accommodate_existing_schema` is used to determine if the existing schema should override the new schema.
1402    /// It only works for TIME_INDEX and single VALUE columns. This is for the case where the user creates a table with
1403    /// custom schema, and then inserts data with endpoints that have default schema setting, like prometheus
1404    /// remote write. This will modify the `RowInsertRequests` in place.
1405    /// `is_single_value` indicates whether the default schema only contains single value column so we can accommodate it.
1406    ///
1407    /// `align_time_index_unit` supplies the selected physical table's time-index
1408    /// unit for new logical tables. Existing logical targets always use their
1409    /// own unit, even when this argument is `None` because no new target needs
1410    /// the selected backing table.
1411    #[allow(clippy::too_many_arguments)]
1412    async fn create_or_alter_tables_on_demand(
1413        &self,
1414        requests: &mut RowInsertRequests,
1415        ctx: &QueryContextRef,
1416        auto_create_table_type: AutoCreateTableType,
1417        statement_executor: &StatementExecutor,
1418        accommodate_existing_schema: bool,
1419        is_single_value: bool,
1420        align_time_index_unit: Option<TimeUnit>,
1421    ) -> Result<CreateAlterTableResult> {
1422        let is_exemplar = matches!(&auto_create_table_type, AutoCreateTableType::Exemplar);
1423        let _timer = crate::metrics::CREATE_ALTER_ON_DEMAND
1424            .with_label_values(&[auto_create_table_type.as_str()])
1425            .start_timer();
1426        let catalog = ctx.current_catalog();
1427        let schema = ctx.current_schema();
1428
1429        let auto_create_disabled_reason = self.auto_create_disabled_reason(ctx)?;
1430        // Enabled batches permit every table, so only disabled batches need a whitelist scan.
1431        let has_auto_create_exempt_table = auto_create_disabled_reason.is_some()
1432            && requests
1433                .inserts
1434                .iter()
1435                .any(|req| Self::is_auto_create_exempt_private_table(&schema, &req.table_name));
1436        let mut table_infos = HashMap::new();
1437        // Without exempt tables, verify existing tables and reject missing ones without inferring schemas.
1438        if let Some(disabled_reason) = auto_create_disabled_reason
1439            && !has_auto_create_exempt_table
1440        {
1441            let mut instant_table_ids = HashSet::new();
1442            for req in &mut requests.inserts {
1443                let table = match self.get_table(catalog, &schema, &req.table_name).await? {
1444                    Some(table) => table,
1445                    // System-defined table: created canonically by the system,
1446                    // so the auto-create config/hint does not apply.
1447                    None if is_ddl_reserved_table(&schema, &req.table_name) => {
1448                        statement_executor
1449                            .create_declared_relationships_table(catalog, ctx.clone())
1450                            .await?
1451                    }
1452                    None => {
1453                        return InvalidInsertRequestSnafu {
1454                            reason: format!(
1455                                "Table `{}` does not exist, and {}",
1456                                req.table_name, disabled_reason
1457                            ),
1458                        }
1459                        .fail();
1460                    }
1461                };
1462                // Existing metric and exemplar destinations keep their time index unit.
1463                if matches!(
1464                    auto_create_table_type,
1465                    AutoCreateTableType::Logical(_) | AutoCreateTableType::Exemplar
1466                ) && let Some(rows) = req.rows.as_mut()
1467                    && let Some(target_unit) = table_time_index_unit(&table)
1468                {
1469                    convert_rows_time_unit(rows, target_unit)?;
1470                }
1471                let table_info = table.table_info();
1472                if matches!(auto_create_table_type, AutoCreateTableType::Exemplar) {
1473                    validate_exemplar_table(&table_info)?;
1474                }
1475                if matches!(auto_create_table_type, AutoCreateTableType::Trace { .. }) {
1476                    validate_trace_table_model(&table_info, ctx)?;
1477                }
1478                if table_info.is_ttl_instant_table() {
1479                    instant_table_ids.insert(table_info.table_id());
1480                }
1481                table_infos.insert(table_info.table_id(), table.table_info());
1482            }
1483            let ret = CreateAlterTableResult {
1484                instant_table_ids,
1485                table_infos,
1486            };
1487            return Ok(ret);
1488        }
1489
1490        let mut create_tables = vec![];
1491        let mut alter_tables = vec![];
1492        let mut need_refresh_table_infos = HashSet::new();
1493        let mut instant_table_ids = HashSet::new();
1494        let mut per_table_semantics: Option<Option<PerTableSemanticIndex>> = None;
1495
1496        for req in &mut requests.inserts {
1497            // Mixed batches need a per-table decision so an exempt table cannot authorize others.
1498            let auto_create_allowed = auto_create_disabled_reason.is_none()
1499                || Self::is_auto_create_exempt_private_table(&schema, &req.table_name);
1500            match self.get_table(catalog, &schema, &req.table_name).await? {
1501                Some(table) => {
1502                    let table_info = table.table_info();
1503                    if matches!(auto_create_table_type, AutoCreateTableType::Exemplar) {
1504                        validate_exemplar_table(&table_info)?;
1505                    }
1506                    if matches!(auto_create_table_type, AutoCreateTableType::Trace { .. }) {
1507                        validate_trace_table_model(&table_info, ctx)?;
1508                    }
1509                    if table_info.is_ttl_instant_table() {
1510                        instant_table_ids.insert(table_info.table_id());
1511                    }
1512                    // Metric path: an existing destination table keeps its
1513                    // own time index unit (it may be bound to another
1514                    // physical table than the one selected by this request).
1515                    if matches!(auto_create_table_type, AutoCreateTableType::Logical(_))
1516                        && let Some(rows) = req.rows.as_mut()
1517                        && let Some(target_unit) = table_time_index_unit(&table)
1518                    {
1519                        convert_rows_time_unit(rows, target_unit)?;
1520                    }
1521                    if auto_create_allowed
1522                        && let Some(alter_expr) = self.get_alter_table_expr_on_demand(
1523                            req,
1524                            &table,
1525                            ctx,
1526                            accommodate_existing_schema,
1527                            is_single_value,
1528                            auto_create_table_type.alter_existing(),
1529                        )?
1530                    {
1531                        alter_tables.push(alter_expr);
1532                        need_refresh_table_infos.insert((
1533                            catalog.to_string(),
1534                            schema.clone(),
1535                            req.table_name.clone(),
1536                        ));
1537                    } else {
1538                        table_infos.insert(table_info.table_id(), table.table_info());
1539                    }
1540                }
1541                // A DDL-reserved table's definition never derives from the
1542                // write request; the system creates it canonically, below the
1543                // user-DDL guard that rejects the generic create path.
1544                None if is_ddl_reserved_table(&schema, &req.table_name) => {
1545                    let table = statement_executor
1546                        .create_declared_relationships_table(catalog, ctx.clone())
1547                        .await?;
1548                    let table_info = table.table_info();
1549                    if table_info.is_ttl_instant_table() {
1550                        instant_table_ids.insert(table_info.table_id());
1551                    }
1552                    table_infos.insert(table_info.table_id(), table_info);
1553                }
1554                None if !auto_create_allowed
1555                    && let Some(disabled_reason) = auto_create_disabled_reason =>
1556                {
1557                    return InvalidInsertRequestSnafu {
1558                        reason: format!(
1559                            "Table `{}` does not exist, and {}",
1560                            req.table_name, disabled_reason,
1561                        ),
1562                    }
1563                    .fail();
1564                }
1565                None => {
1566                    // Metric path: a new table uses the selected physical
1567                    // table's unit; convert before the create expression is
1568                    // derived from the request schema.
1569                    if let Some(physical_unit) = align_time_index_unit
1570                        && let Some(rows) = req.rows.as_mut()
1571                    {
1572                        convert_rows_time_unit(rows, physical_unit)?;
1573                    }
1574                    let semantic_index = per_table_semantics
1575                        .get_or_insert_with(|| parse_per_table_semantic_index(ctx))
1576                        .as_ref();
1577                    let create_expr = self.get_create_table_expr_on_demand(
1578                        req,
1579                        &auto_create_table_type,
1580                        ctx,
1581                        semantic_index,
1582                    )?;
1583                    create_tables.push(create_expr);
1584                }
1585            }
1586        }
1587
1588        match auto_create_table_type {
1589            AutoCreateTableType::Logical(_) => {
1590                if !create_tables.is_empty() {
1591                    // Creates logical tables in batch.
1592                    let tables = self
1593                        .create_logical_tables(create_tables, ctx, statement_executor)
1594                        .await?;
1595
1596                    for table in tables {
1597                        let table_info = table.table_info();
1598                        if table_info.is_ttl_instant_table() {
1599                            instant_table_ids.insert(table_info.table_id());
1600                        }
1601                        table_infos.insert(table_info.table_id(), table.table_info());
1602                    }
1603                }
1604                if !alter_tables.is_empty() {
1605                    // Alter logical tables in batch.
1606                    statement_executor
1607                        .alter_logical_tables(alter_tables, ctx.clone(), TriggerReason::AutoAlter)
1608                        .await?;
1609                }
1610            }
1611            AutoCreateTableType::Physical
1612            | AutoCreateTableType::Exemplar
1613            | AutoCreateTableType::Log
1614            | AutoCreateTableType::LastNonNull => {
1615                // note that auto create table shouldn't be ttl instant table
1616                // for it's a very unexpected behavior and should be set by user explicitly
1617                for create_table in create_tables {
1618                    let table = self
1619                        .create_physical_table(create_table, None, ctx, statement_executor)
1620                        .await?;
1621                    // A concurrent CREATE can return our proposed schema with the winner's ID.
1622                    let table = if is_exemplar {
1623                        let name = table.table_info().name.clone();
1624                        self.get_table(catalog, &schema, &name)
1625                            .await?
1626                            .context(TableNotFoundSnafu { table_name: name })?
1627                    } else {
1628                        table
1629                    };
1630                    if matches!(auto_create_table_type, AutoCreateTableType::Exemplar) {
1631                        validate_exemplar_table(&table.table_info())?;
1632                        if let Some(req) = requests
1633                            .inserts
1634                            .iter_mut()
1635                            .find(|r| r.table_name == table.table_info().name)
1636                            && let Some(alter) = self.get_alter_table_expr_on_demand(
1637                                req, &table, ctx, false, false, true,
1638                            )?
1639                        {
1640                            alter_tables.push(alter);
1641                            need_refresh_table_infos.insert((
1642                                catalog.to_string(),
1643                                schema.clone(),
1644                                req.table_name.clone(),
1645                            ));
1646                        }
1647                    }
1648                    let table_info = table.table_info();
1649                    if table_info.is_ttl_instant_table() {
1650                        instant_table_ids.insert(table_info.table_id());
1651                    }
1652                    table_infos.insert(table_info.table_id(), table.table_info());
1653                }
1654                for alter_expr in alter_tables.into_iter() {
1655                    statement_executor
1656                        .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
1657                        .await?;
1658                }
1659            }
1660
1661            AutoCreateTableType::Trace { .. } => {
1662                let trace_table_name = ctx
1663                    .extension(TRACE_TABLE_NAME_SESSION_KEY)
1664                    .unwrap_or(TRACE_TABLE_NAME);
1665
1666                let trace_table_partitions = if let Some(trace_table_partitions) =
1667                    ctx.extension(TRACE_TABLE_PARTITIONS_HINT_KEY)
1668                {
1669                    let p = trace_table_partitions.parse::<u32>().map_err(|_| {
1670                        InvalidInsertRequestSnafu {
1671                            reason: format!(
1672                                "Failed to parse trace_table_partitions: {}",
1673                                trace_table_partitions
1674                            ),
1675                        }
1676                        .build()
1677                    })?;
1678                    Some(p)
1679                } else {
1680                    None
1681                };
1682
1683                // note that auto create table shouldn't be ttl instant table
1684                // for it's a very unexpected behavior and should be set by user explicitly
1685                for mut create_table in create_tables {
1686                    if create_table.table_name == trace_services_table_name(trace_table_name)
1687                        || create_table.table_name == trace_operations_table_name(trace_table_name)
1688                    {
1689                        // Disable append mode for auxiliary tables (services/operations) since they require upsert behavior.
1690                        create_table
1691                            .table_options
1692                            .insert(APPEND_MODE_KEY.to_string(), "false".to_string());
1693                        // Remove `ttl` key from table options if it exists
1694                        create_table.table_options.remove(TTL_KEY);
1695
1696                        let table = self
1697                            .create_physical_table(create_table, None, ctx, statement_executor)
1698                            .await?;
1699                        let table_info = table.table_info();
1700                        if table_info.is_ttl_instant_table() {
1701                            instant_table_ids.insert(table_info.table_id());
1702                        }
1703                        table_infos.insert(table_info.table_id(), table.table_info());
1704                    } else {
1705                        // prebuilt partition rules for uuid data: see the function
1706                        // for more information
1707                        let partitions = if matches!(trace_table_partitions, Some(0) | Some(1)) {
1708                            // disable partitions
1709                            None
1710                        } else {
1711                            let p = partition_rule_for_hexstring(
1712                                TRACE_ID_COLUMN,
1713                                trace_table_partitions,
1714                            )
1715                            .context(CreatePartitionRulesSnafu)?;
1716                            Some(p)
1717                        };
1718
1719                        // add skip index to
1720                        // - trace_id: when searching by trace id
1721                        // - parent_span_id: when searching root span
1722                        // - span_name: when searching certain types of span
1723                        let index_columns =
1724                            [TRACE_ID_COLUMN, PARENT_SPAN_ID_COLUMN, SERVICE_NAME_COLUMN];
1725                        for index_column in index_columns {
1726                            if let Some(col) = create_table
1727                                .column_defs
1728                                .iter_mut()
1729                                .find(|c| c.name == index_column)
1730                            {
1731                                col.options =
1732                                    options_from_skipping(&SkippingIndexOptions::default())
1733                                        .context(ColumnOptionsSnafu)?;
1734                            } else {
1735                                warn!(
1736                                    "Column {} not found when creating index for trace table: {}.",
1737                                    index_column, create_table.table_name
1738                                );
1739                            }
1740                        }
1741
1742                        // Span ids and end timestamps rarely repeat within an SST, so a
1743                        // dictionary costs more than it saves. Small span id pages let
1744                        // lookups by trace id decode only the pages holding matched rows.
1745                        let id_compression = CompressionOptions {
1746                            encoding: Some(ColumnEncoding::Plain),
1747                            page_size: Some(ReadableSize::kb(64)),
1748                            ..Default::default()
1749                        };
1750                        let compressed_columns = [
1751                            (SPAN_ID_COLUMN, id_compression),
1752                            (PARENT_SPAN_ID_COLUMN, id_compression),
1753                            (
1754                                TIMESTAMP_END_COLUMN,
1755                                CompressionOptions {
1756                                    encoding: Some(ColumnEncoding::DeltaBinaryPacked),
1757                                    ..Default::default()
1758                                },
1759                            ),
1760                        ];
1761                        for (name, compression) in compressed_columns {
1762                            if let Some(col) =
1763                                create_table.column_defs.iter_mut().find(|c| c.name == name)
1764                            {
1765                                let compression = options_from_compression(&compression)
1766                                    .context(ColumnOptionsSnafu)?;
1767                                col.options
1768                                    .get_or_insert_with(Default::default)
1769                                    .options
1770                                    .extend(compression.options);
1771                            }
1772                        }
1773
1774                        // use table_options to mark table model version
1775                        create_table.table_options.insert(
1776                            TABLE_DATA_MODEL.to_string(),
1777                            ctx.extension(SEMANTIC_PIPELINE)
1778                                .unwrap_or(TABLE_DATA_MODEL_TRACE_V1)
1779                                .to_string(),
1780                        );
1781
1782                        let table = self
1783                            .create_physical_table(
1784                                create_table,
1785                                partitions,
1786                                ctx,
1787                                statement_executor,
1788                            )
1789                            .await?;
1790                        let table_info = table.table_info();
1791                        if table_info.is_ttl_instant_table() {
1792                            instant_table_ids.insert(table_info.table_id());
1793                        }
1794                        table_infos.insert(table_info.table_id(), table.table_info());
1795                    }
1796                }
1797                for alter_expr in alter_tables.into_iter() {
1798                    statement_executor
1799                        .alter_table_inner(alter_expr, ctx.clone(), TriggerReason::AutoAlter)
1800                        .await?;
1801                }
1802            }
1803        }
1804
1805        // refresh table infos for altered tables
1806        for (catalog, schema, table_name) in need_refresh_table_infos {
1807            let table = self
1808                .get_table(&catalog, &schema, &table_name)
1809                .await?
1810                .context(TableNotFoundSnafu {
1811                    table_name: common_catalog::format_full_table_name(
1812                        &catalog,
1813                        &schema,
1814                        &table_name,
1815                    ),
1816                })?;
1817            let table_info = table.table_info();
1818            if is_exemplar {
1819                validate_exemplar_table(&table_info)?;
1820            }
1821            table_infos.insert(table_info.table_id(), table.table_info());
1822        }
1823
1824        if is_exemplar {
1825            for info in table_infos.values() {
1826                if let Some(unit) = info
1827                    .meta
1828                    .schema
1829                    .timestamp_column()
1830                    .and_then(|column| column.data_type.as_timestamp())
1831                    .map(|timestamp| timestamp.unit())
1832                {
1833                    for request in &mut requests.inserts {
1834                        if request.table_name == info.name
1835                            && let Some(rows) = request.rows.as_mut()
1836                        {
1837                            convert_rows_time_unit(rows, unit)?;
1838                        }
1839                    }
1840                }
1841            }
1842        }
1843
1844        Ok(CreateAlterTableResult {
1845            instant_table_ids,
1846            table_infos,
1847        })
1848    }
1849
1850    async fn create_physical_table_on_demand(
1851        &self,
1852        ctx: &QueryContextRef,
1853        physical_table: String,
1854        statement_executor: &StatementExecutor,
1855    ) -> Result<TableRef> {
1856        let catalog_name = ctx.current_catalog();
1857        let schema_name = ctx.current_schema();
1858
1859        // check if exist
1860        if let Some(table) = self
1861            .get_table(catalog_name, &schema_name, &physical_table)
1862            .await?
1863        {
1864            return Ok(table);
1865        }
1866
1867        // Gate here too, otherwise a disabled switch would still leak the physical table.
1868        if let Some(disabled_reason) = self.auto_create_disabled_reason(ctx)? {
1869            return InvalidInsertRequestSnafu {
1870                reason: format!(
1871                    "Physical table `{physical_table}` does not exist, and {disabled_reason}"
1872                ),
1873            }
1874            .fail();
1875        }
1876
1877        let table_reference = TableReference::full(catalog_name, &schema_name, &physical_table);
1878        debug!("Ensuring physical metric table `{table_reference}` exists for insert");
1879
1880        // schema with timestamp and field column
1881        let default_schema = vec![
1882            ColumnSchema {
1883                column_name: greptime_timestamp().to_string(),
1884                datatype: api::helper::timestamp_datatype(
1885                    self.auto_create_physical_table_timestamp_unit,
1886                ) as _,
1887                semantic_type: SemanticType::Timestamp as _,
1888                datatype_extension: None,
1889                options: None,
1890            },
1891            ColumnSchema {
1892                column_name: greptime_value().to_string(),
1893                datatype: ColumnDataType::Float64 as _,
1894                semantic_type: SemanticType::Field as _,
1895                datatype_extension: None,
1896                options: None,
1897            },
1898        ];
1899        let create_table_expr =
1900            &mut build_create_table_expr(&table_reference, &default_schema, default_engine())?;
1901
1902        create_table_expr.engine = METRIC_ENGINE_NAME.to_string();
1903        create_table_expr
1904            .table_options
1905            .insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), "true".to_string());
1906        apply_retention_ttl_hint(&mut create_table_expr.table_options, ctx);
1907
1908        // create physical table
1909        let res = statement_executor
1910            .create_table_inner(
1911                create_table_expr,
1912                None,
1913                ctx.clone(),
1914                TriggerReason::AutoCreate,
1915            )
1916            .await;
1917
1918        match res {
1919            Ok(table) => Ok(table),
1920            Err(err) => {
1921                error!(err; "Failed to create table {table_reference}");
1922                Err(err)
1923            }
1924        }
1925    }
1926
1927    async fn get_table(
1928        &self,
1929        catalog: &str,
1930        schema: &str,
1931        table: &str,
1932    ) -> Result<Option<TableRef>> {
1933        self.catalog_manager
1934            .table(catalog, schema, table, None)
1935            .await
1936            .context(CatalogSnafu)
1937    }
1938
1939    fn get_create_table_expr_on_demand(
1940        &self,
1941        req: &RowInsertRequest,
1942        create_type: &AutoCreateTableType,
1943        ctx: &QueryContextRef,
1944        semantic_index: Option<&PerTableSemanticIndex>,
1945    ) -> Result<CreateTableExpr> {
1946        let schema = ctx.current_schema();
1947        let mut table_options = std::collections::HashMap::with_capacity(4);
1948        fill_table_options_for_create(&mut table_options, create_type, ctx);
1949        if !matches!(create_type, AutoCreateTableType::Exemplar) {
1950            apply_per_table_semantic_options(
1951                &mut table_options,
1952                semantic_index,
1953                ctx.current_schema().as_str(),
1954                &req.table_name,
1955            );
1956        }
1957
1958        let engine_name = if let AutoCreateTableType::Logical(_) = create_type {
1959            // engine should be metric engine when creating logical tables.
1960            METRIC_ENGINE_NAME
1961        } else {
1962            default_engine()
1963        };
1964
1965        let table_ref = TableReference::full(ctx.current_catalog(), &schema, &req.table_name);
1966        // SAFETY: `req.rows` is guaranteed to be `Some` by `handle_row_inserts_with_create_type()`.
1967        let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
1968        let mut create_table_expr =
1969            build_create_table_expr(&table_ref, request_schema, engine_name)?;
1970
1971        if matches!(create_type, AutoCreateTableType::Exemplar) {
1972            // Preserve the stored schema order without shifting every incoming row.
1973            create_table_expr
1974                .column_defs
1975                .sort_by_key(|column| column.name != DATA_SCHEMA_TABLE_ID_COLUMN_NAME);
1976            create_table_expr
1977                .primary_keys
1978                .sort_by_key(|name| name != DATA_SCHEMA_TABLE_ID_COLUMN_NAME);
1979            for column in &mut create_table_expr.column_defs {
1980                if [
1981                    DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
1982                    DATA_SCHEMA_TSID_COLUMN_NAME,
1983                ]
1984                .contains(&column.name.as_str())
1985                {
1986                    column.is_nullable = false;
1987                }
1988            }
1989        }
1990
1991        // extension set by the Splunk HEC handler for identity path
1992        if ctx.extension(SPLUNK_PK_METADATA_ORDER_KEY).is_some() {
1993            reorder_splunk_primary_keys(&mut create_table_expr.primary_keys);
1994        }
1995
1996        debug!("Ensuring table `{table_ref}` exists for insert");
1997        create_table_expr.table_options.extend(table_options);
1998        Ok(create_table_expr)
1999    }
2000
2001    /// Returns an alter table expression if it finds new columns in the request.
2002    /// When `accommodate_existing_schema` is false, it always adds columns if not exist.
2003    /// When `accommodate_existing_schema` is true, it may modify the input `req` to
2004    /// accommodate it with existing schema. See [`create_or_alter_tables_on_demand`](Self::create_or_alter_tables_on_demand)
2005    /// for more details.
2006    /// When `is_single_value` is true, it also rejects native-histogram/float kind
2007    /// changes even when no columns need adding.
2008    /// When both options are true, it considers fields when modifying the input `req`.
2009    fn get_alter_table_expr_on_demand(
2010        &self,
2011        req: &mut RowInsertRequest,
2012        table: &TableRef,
2013        ctx: &QueryContextRef,
2014        accommodate_existing_schema: bool,
2015        is_single_value: bool,
2016        alter_existing: bool,
2017    ) -> Result<Option<AlterTableExpr>> {
2018        if !alter_existing {
2019            return Ok(None);
2020        }
2021
2022        let catalog_name = ctx.current_catalog();
2023        let schema_name = ctx.current_schema();
2024        let table_name = table.table_info().name.clone();
2025
2026        // Never auto-alter a system-defined table to fit a write; a request
2027        // with unknown columns fails instead.
2028        if is_ddl_reserved_table(&schema_name, &table_name) {
2029            return Ok(None);
2030        }
2031
2032        let request_schema = req.rows.as_ref().unwrap().schema.as_slice();
2033        let request_field_count = request_schema
2034            .iter()
2035            .filter(|col| col.semantic_type == SemanticType::Field as i32)
2036            .count();
2037        if is_single_value {
2038            let request_is_native_histogram = request_is_native_histogram(request_schema);
2039            let table_is_native_histogram = table_is_native_histogram(table);
2040            ensure!(
2041                request_is_native_histogram == table_is_native_histogram,
2042                InvalidInsertRequestSnafu {
2043                    reason: format!(
2044                        "Table `{table_name}` cannot mix native histogram and float sample fields"
2045                    ),
2046                }
2047            );
2048        }
2049
2050        let column_exprs = ColumnExpr::from_column_schemas(request_schema);
2051        let add_columns = expr_helper::extract_add_columns_expr(&table.schema(), column_exprs)?;
2052        let Some(mut add_columns) = add_columns else {
2053            return Ok(None);
2054        };
2055
2056        // If accommodate_existing_schema is true, update request schema for Timestamp/Field columns
2057        if accommodate_existing_schema {
2058            let table_schema = table.schema();
2059            // Find timestamp column name
2060            let ts_col_name = table_schema.timestamp_column().map(|c| c.name.clone());
2061            // Find field column name if there is only one and `is_single_value` is true.
2062            let mut field_col_name = None;
2063            if is_single_value && request_field_count <= 1 {
2064                let mut multiple_field_cols = false;
2065                table.field_columns().for_each(|col| {
2066                    if field_col_name.is_none() {
2067                        field_col_name = Some(col.name.clone());
2068                    } else {
2069                        multiple_field_cols = true;
2070                    }
2071                });
2072                if multiple_field_cols {
2073                    field_col_name = None;
2074                }
2075            }
2076
2077            // Update column name in request schema for Timestamp/Field columns
2078            if let Some(rows) = req.rows.as_mut() {
2079                for col in &mut rows.schema {
2080                    match col.semantic_type {
2081                        x if x == SemanticType::Timestamp as i32 => {
2082                            if let Some(ref ts_name) = ts_col_name
2083                                && col.column_name != *ts_name
2084                            {
2085                                col.column_name = ts_name.clone();
2086                            }
2087                        }
2088                        x if x == SemanticType::Field as i32 => {
2089                            if let Some(ref field_name) = field_col_name
2090                                && col.column_name != *field_name
2091                            {
2092                                col.column_name = field_name.clone();
2093                            }
2094                        }
2095                        _ => {}
2096                    }
2097                }
2098            }
2099
2100            // Only keep columns that are tags or non-single field.
2101            add_columns.add_columns.retain(|col| {
2102                let def = col.column_def.as_ref().unwrap();
2103                def.semantic_type == SemanticType::Tag as i32
2104                    || (def.semantic_type == SemanticType::Field as i32 && field_col_name.is_none())
2105            });
2106
2107            if add_columns.add_columns.is_empty() {
2108                return Ok(None);
2109            }
2110        }
2111
2112        Ok(Some(AlterTableExpr {
2113            catalog_name: catalog_name.to_string(),
2114            schema_name: schema_name.clone(),
2115            table_name: table_name.clone(),
2116            kind: Some(Kind::AddColumns(add_columns)),
2117        }))
2118    }
2119
2120    /// Creates a table with options.
2121    async fn create_physical_table(
2122        &self,
2123        mut create_table_expr: CreateTableExpr,
2124        partitions: Option<Partitions>,
2125        ctx: &QueryContextRef,
2126        statement_executor: &StatementExecutor,
2127    ) -> Result<TableRef> {
2128        let res = statement_executor
2129            .create_table_inner(
2130                &mut create_table_expr,
2131                partitions,
2132                ctx.clone(),
2133                TriggerReason::AutoCreate,
2134            )
2135            .await;
2136
2137        let table_ref = TableReference::full(
2138            &create_table_expr.catalog_name,
2139            &create_table_expr.schema_name,
2140            &create_table_expr.table_name,
2141        );
2142
2143        match res {
2144            Ok(table) => {
2145                validate_trace_table_model(&table.table_info(), ctx)?;
2146                Ok(table)
2147            }
2148            Err(err) => {
2149                error!(err; "Failed to create table {}", table_ref);
2150                Err(err)
2151            }
2152        }
2153    }
2154
2155    async fn create_logical_tables(
2156        &self,
2157        create_table_exprs: Vec<CreateTableExpr>,
2158        ctx: &QueryContextRef,
2159        statement_executor: &StatementExecutor,
2160    ) -> Result<Vec<TableRef>> {
2161        let res = statement_executor
2162            .create_logical_tables(&create_table_exprs, ctx.clone(), TriggerReason::AutoCreate)
2163            .await;
2164
2165        match res {
2166            Ok(res) => Ok(res),
2167            Err(err) => {
2168                let failed_tables = create_table_exprs
2169                    .into_iter()
2170                    .map(|expr| {
2171                        format!(
2172                            "{}.{}.{}",
2173                            expr.catalog_name, expr.schema_name, expr.table_name
2174                        )
2175                    })
2176                    .collect::<Vec<_>>();
2177                error!(
2178                    err;
2179                    "Failed to create logical tables {:?}",
2180                    failed_tables
2181                );
2182                Err(err)
2183            }
2184        }
2185    }
2186
2187    /// Returns the dispatcher shared with batching execution paths.
2188    pub fn write_dispatcher(&self) -> &WriteDispatcherRef {
2189        &self.write_dispatcher
2190    }
2191
2192    pub fn node_manager(&self) -> &NodeManagerRef {
2193        &self.node_manager
2194    }
2195
2196    pub fn partition_manager(&self) -> &PartitionRuleManagerRef {
2197        &self.partition_manager
2198    }
2199
2200    pub fn table_flownode_set_cache(&self) -> &TableFlownodeSetCacheRef {
2201        &self.table_flownode_set_cache
2202    }
2203}
2204
2205fn request_is_native_histogram(request_schema: &[ColumnSchema]) -> bool {
2206    let mut fields = request_schema
2207        .iter()
2208        .filter(|col| col.semantic_type == SemanticType::Field as i32);
2209    let Some(col) = fields.next() else {
2210        return false;
2211    };
2212
2213    fields.next().is_none()
2214        && api::helper::is_column_type_value_eq(
2215            col.datatype,
2216            col.datatype_extension.clone(),
2217            native_histogram_value_type(),
2218        )
2219}
2220
2221/// Returns the table's time index unit, if any. A metric table without a
2222/// timestamp column is left alone by the unit alignment: the metric engine
2223/// rejects it anyway.
2224fn table_time_index_unit(table: &TableRef) -> Option<TimeUnit> {
2225    table
2226        .table_info()
2227        .meta
2228        .schema
2229        .timestamp_column()
2230        .and_then(|col| col.data_type.as_timestamp().map(|ts| ts.unit()))
2231}
2232
2233fn convert_rows_time_unit(rows: &mut Rows, target_unit: TimeUnit) -> Result<()> {
2234    let Some(ts_index) = rows
2235        .schema
2236        .iter()
2237        .position(|col| col.semantic_type == SemanticType::Timestamp as i32)
2238    else {
2239        return Ok(());
2240    };
2241    let Some(source_unit) = ColumnDataType::try_from(rows.schema[ts_index].datatype)
2242        .ok()
2243        .and_then(api::helper::timestamp_unit)
2244    else {
2245        return Ok(());
2246    };
2247    if source_unit == target_unit {
2248        return Ok(());
2249    }
2250
2251    rows.schema[ts_index].datatype = api::helper::timestamp_datatype(target_unit) as i32;
2252    // Timestamp columns never carry a datatype extension.
2253    rows.schema[ts_index].datatype_extension = None;
2254
2255    // Note: the schema is rewritten before the rows are converted, so an
2256    // overflow error mid-batch leaves this request half-converted. That is
2257    // harmless: the error aborts the whole insert request.
2258    //
2259    // `validate_column_count_match` guarantees every row carries exactly one
2260    // value per schema column, so the time index position is directly in
2261    // bounds; no per-value search is needed.
2262    for row in &mut rows.rows {
2263        debug_assert_eq!(row.values.len(), rows.schema.len());
2264        let value = &mut row.values[ts_index];
2265        let Some(value_data) = value.value_data.take() else {
2266            continue;
2267        };
2268        value.value_data =
2269            convert_timestamp_value_data(value_data, source_unit, target_unit, ts_index)?;
2270    }
2271    Ok(())
2272}
2273
2274fn convert_timestamp_value_data(
2275    value_data: ValueData,
2276    source_unit: TimeUnit,
2277    target_unit: TimeUnit,
2278    column_index: usize,
2279) -> Result<Option<ValueData>> {
2280    let timestamp = match value_data {
2281        ValueData::TimestampSecondValue(v) => Timestamp::new_second(v),
2282        ValueData::TimestampMillisecondValue(v) => Timestamp::new_millisecond(v),
2283        ValueData::TimestampMicrosecondValue(v) => Timestamp::new_microsecond(v),
2284        ValueData::TimestampNanosecondValue(v) => Timestamp::new_nanosecond(v),
2285        // Null or non-timestamp value; nothing to convert.
2286        other => return Ok(Some(other)),
2287    };
2288    let converted = timestamp
2289        .convert_to(target_unit)
2290        .with_context(|| InvalidInsertRequestSnafu {
2291            reason: format!(
2292                "timestamp column {column_index} value {} in unit {source_unit:?} overflows when converting to unit {target_unit:?}",
2293                timestamp.value()
2294            ),
2295        })?;
2296    Ok(api::helper::to_grpc_value(datatypes::value::Value::Timestamp(converted)).value_data)
2297}
2298
2299fn table_is_native_histogram(table: &TableRef) -> bool {
2300    let mut fields = table.field_columns();
2301    let Some(col) = fields.next() else {
2302        return false;
2303    };
2304
2305    fields.next().is_none() && is_native_histogram_value_type(&col.data_type)
2306}
2307
2308fn validate_column_count_match(requests: &RowInsertRequests) -> Result<()> {
2309    for request in &requests.inserts {
2310        let rows = request.rows.as_ref().unwrap();
2311        let column_count = rows.schema.len();
2312        rows.rows.iter().try_for_each(|r| {
2313            ensure!(
2314                r.values.len() == column_count,
2315                InvalidInsertRequestSnafu {
2316                    reason: format!(
2317                        "column count mismatch, columns: {}, values: {}",
2318                        column_count,
2319                        r.values.len()
2320                    )
2321                }
2322            );
2323            Ok(())
2324        })?;
2325    }
2326    Ok(())
2327}
2328
2329fn validate_exemplar_rows(rows: &Rows) -> Result<()> {
2330    let mut names = HashSet::new();
2331    for (index, column) in rows.schema.iter().enumerate() {
2332        let data_type: datatypes::prelude::ConcreteDataType =
2333            api::helper::ColumnDataTypeWrapper::try_new(
2334                column.datatype,
2335                column.datatype_extension.clone(),
2336            )
2337            .context(crate::error::ColumnDataTypeSnafu)?
2338            .into();
2339        let valid = match column.column_name.as_str() {
2340            DATA_SCHEMA_TSID_COLUMN_NAME => {
2341                data_type == datatypes::prelude::ConcreteDataType::uint64_datatype()
2342                    && column.semantic_type == SemanticType::Tag as i32
2343            }
2344            EXEMPLAR_METRIC_COLUMN => {
2345                data_type.is_string() && column.semantic_type == SemanticType::Tag as i32
2346            }
2347            name if name == greptime_timestamp() => {
2348                data_type.is_timestamp() && column.semantic_type == SemanticType::Timestamp as i32
2349            }
2350            name if name == greptime_value() => {
2351                data_type == datatypes::prelude::ConcreteDataType::float64_datatype()
2352                    && column.semantic_type == SemanticType::Field as i32
2353            }
2354            name if name == greptime_exemplar_labels() => {
2355                data_type.is_json2() && column.semantic_type == SemanticType::Field as i32
2356            }
2357            _ => data_type.is_string() && column.semantic_type == SemanticType::Tag as i32,
2358        };
2359        ensure!(
2360            valid && names.insert(column.column_name.as_str()),
2361            InvalidInsertRequestSnafu {
2362                reason: format!(
2363                    "invalid or duplicate exemplar column {}",
2364                    column.column_name
2365                ),
2366            }
2367        );
2368        ensure!(
2369            rows.rows.iter().all(|row| value_matches_type(
2370                &row.values[index],
2371                column.datatype,
2372                column.datatype_extension.as_ref(),
2373            )),
2374            InvalidInsertRequestSnafu {
2375                reason: format!("invalid value for exemplar column {}", column.column_name),
2376            }
2377        );
2378    }
2379    ensure!(
2380        [
2381            DATA_SCHEMA_TSID_COLUMN_NAME,
2382            EXEMPLAR_METRIC_COLUMN,
2383            greptime_timestamp(),
2384            greptime_value(),
2385            greptime_exemplar_labels(),
2386        ]
2387        .iter()
2388        .all(|name| names.contains(name)),
2389        InvalidInsertRequestSnafu {
2390            reason: "missing required exemplar columns",
2391        }
2392    );
2393    Ok(())
2394}
2395
2396fn exemplar_source_add_columns(
2397    info: &TableInfo,
2398    tags: &[ColumnSchema],
2399) -> Result<Option<api::v1::AddColumns>> {
2400    for tag in tags {
2401        if let Some(index) = info.meta.schema.column_index_by_name(&tag.column_name) {
2402            let column = &info.meta.schema.column_schemas()[index];
2403            // Existing source defaults may already be part of the normalized series identity.
2404            ensure!(
2405                info.meta.primary_key_indices.contains(&index) && column.data_type.is_string(),
2406                InvalidInsertRequestSnafu {
2407                    reason: format!(
2408                        "exemplar source {} column {} must be a string tag",
2409                        info.name, tag.column_name
2410                    ),
2411                }
2412            );
2413        }
2414    }
2415    expr_helper::extract_add_columns_expr(&info.meta.schema, ColumnExpr::from_column_schemas(tags))
2416}
2417
2418/// Validates the fixed exemplar contract before writing or scanning a shared table.
2419pub fn validate_exemplar_table(table: &TableInfo) -> Result<()> {
2420    let meta = &table.meta;
2421    let options = &meta.options.extra_options;
2422    ensure!(
2423        meta.engine == default_engine()
2424            && options
2425                .get(APPEND_MODE_KEY)
2426                .is_some_and(|v| v.eq_ignore_ascii_case("true"))
2427            && options
2428                .get(store_api::metric_engine_consts::PRIMARY_KEY_ENCODING)
2429                .is_some_and(|v| v.eq_ignore_ascii_case("sparse")),
2430        InvalidInsertRequestSnafu {
2431            reason: format!(
2432                "{METRICS_EXEMPLARS_TABLE} requires Mito, append_mode=true and primary_key_encoding=sparse"
2433            )
2434        }
2435    );
2436    let columns = meta.schema.column_schemas();
2437    let lookup = |name: &str| {
2438        columns
2439            .iter()
2440            .enumerate()
2441            .find(|(_, c)| c.name == name)
2442            .with_context(|| InvalidInsertRequestSnafu {
2443                reason: format!("missing exemplar column {name}"),
2444            })
2445    };
2446    let (_, timestamp) = lookup(greptime_timestamp())?;
2447    for (position, name, id, data_type) in [
2448        (
2449            0,
2450            DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
2451            ReservedColumnId::table_id(),
2452            datatypes::prelude::ConcreteDataType::uint32_datatype(),
2453        ),
2454        (
2455            1,
2456            DATA_SCHEMA_TSID_COLUMN_NAME,
2457            ReservedColumnId::tsid(),
2458            datatypes::prelude::ConcreteDataType::uint64_datatype(),
2459        ),
2460    ] {
2461        let (index, column) = lookup(name)?;
2462        ensure!(
2463            meta.primary_key_indices.get(position) == Some(&index)
2464                && meta.column_ids.get(index) == Some(&id)
2465                && column.data_type == data_type
2466                && !column.is_nullable()
2467                && column.default_constraint().is_none(),
2468            InvalidInsertRequestSnafu {
2469                reason: format!("invalid exemplar source identity column {name}"),
2470            }
2471        );
2472    }
2473    ensure!(
2474        timestamp.is_time_index() && timestamp.data_type.is_timestamp(),
2475        InvalidInsertRequestSnafu {
2476            reason: "invalid exemplar time index"
2477        }
2478    );
2479    let (metric_index, metric) = lookup(EXEMPLAR_METRIC_COLUMN)?;
2480    ensure!(
2481        metric.data_type.is_string() && meta.primary_key_indices.contains(&metric_index),
2482        InvalidInsertRequestSnafu {
2483            reason: "exemplar metric name must be a string tag"
2484        }
2485    );
2486    let (value_index, value) = lookup(greptime_value())?;
2487    ensure!(
2488        value.data_type == datatypes::prelude::ConcreteDataType::float64_datatype()
2489            && !meta.primary_key_indices.contains(&value_index),
2490        InvalidInsertRequestSnafu {
2491            reason: "exemplar value must be a DOUBLE field"
2492        }
2493    );
2494    let (labels_index, labels) = lookup(greptime_exemplar_labels())?;
2495    ensure!(
2496        labels.data_type.is_json2() && !meta.primary_key_indices.contains(&labels_index),
2497        InvalidInsertRequestSnafu {
2498            reason: format!("{} must be a JSON2 field", greptime_exemplar_labels())
2499        }
2500    );
2501    if let Some(extension) = labels
2502        .extension_type::<datatypes::extension::json::Json2ExtensionType>()
2503        .map_err(|error| {
2504            InvalidInsertRequestSnafu {
2505                reason: error.to_string(),
2506            }
2507            .build()
2508        })?
2509    {
2510        ensure!(
2511            extension
2512                .metadata()
2513                .json_settings()
2514                .type_hints()
2515                .iter()
2516                .all(|hint| hint.path.len() == 1 && hint.data_type.is_string()),
2517            InvalidInsertRequestSnafu {
2518                reason: "exemplar JSON2 hints must preserve string labels"
2519            }
2520        );
2521    }
2522    for (index, column) in columns.iter().enumerate() {
2523        if [
2524            greptime_timestamp(),
2525            greptime_value(),
2526            EXEMPLAR_METRIC_COLUMN,
2527            greptime_exemplar_labels(),
2528            DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
2529            DATA_SCHEMA_TSID_COLUMN_NAME,
2530        ]
2531        .contains(&column.name.as_str())
2532        {
2533            continue;
2534        }
2535        ensure!(
2536            meta.primary_key_indices.contains(&index)
2537                && column.data_type.is_string()
2538                && column.is_nullable(),
2539            InvalidInsertRequestSnafu {
2540                reason: "inherited exemplar labels must be nullable string tags"
2541            }
2542        );
2543        // An absent inherited label must remain absent from the source series identity.
2544        ensure!(
2545            matches!(
2546                column.default_constraint(),
2547                None | Some(ColumnDefaultConstraint::Value(
2548                    datatypes::value::Value::Null
2549                ))
2550            ),
2551            InvalidInsertRequestSnafu {
2552                reason: "inherited exemplar labels must not have non-NULL defaults"
2553            }
2554        );
2555    }
2556    Ok(())
2557}
2558
2559/// Rejects writes from a different built-in trace model before schema mutation.
2560/// Unstamped, explicitly created tables remain subject to normal schema validation.
2561pub fn validate_trace_table_model(table_info: &TableInfo, ctx: &QueryContextRef) -> Result<()> {
2562    let Some(expected @ (TABLE_DATA_MODEL_TRACE_V1 | TABLE_DATA_MODEL_TRACE_V2)) =
2563        ctx.extension(SEMANTIC_PIPELINE)
2564    else {
2565        return Ok(());
2566    };
2567    if let Some(actual) = table_info.meta.options.data_model() {
2568        ensure!(
2569            actual == expected,
2570            InvalidInsertRequestSnafu {
2571                reason: format!(
2572                    "Trace table `{}` uses {actual}, but the request uses {expected}",
2573                    table_info.name,
2574                ),
2575            }
2576        );
2577    }
2578    Ok(())
2579}
2580
2581/// Copies a retention `ttl` hint onto a physical metric table being created.
2582/// Only `ttl` is copied: other ingest hints must not land on the shared physical
2583/// table, and `ttl=instant` stays on the logical table for flow forwarding.
2584/// Once the physical table exists, the hint has no effect.
2585pub fn apply_retention_ttl_hint(
2586    table_options: &mut std::collections::HashMap<String, String>,
2587    ctx: &QueryContextRef,
2588) {
2589    if let Some(ttl) = ctx.extension(TTL_KEY).filter(|ttl| is_retention_ttl(ttl)) {
2590        table_options.insert(TTL_KEY.to_string(), ttl.to_string());
2591    }
2592}
2593
2594/// Fill table options for a new table by create type.
2595pub fn fill_table_options_for_create(
2596    table_options: &mut std::collections::HashMap<String, String>,
2597    create_type: &AutoCreateTableType,
2598    ctx: &QueryContextRef,
2599) {
2600    for key in VALID_TABLE_OPTION_KEYS {
2601        if let Some(value) = ctx.extension(key) {
2602            table_options.insert(key.to_string(), value.to_string());
2603        }
2604    }
2605
2606    if !matches!(create_type, AutoCreateTableType::Logical(_)) {
2607        let mut has_twcs_option = false;
2608        for key in COMPACTION_OPTION_KEYS {
2609            if let Some(value) = ctx.extension(key) {
2610                has_twcs_option |= key.starts_with(COMPACTION_TWCS_PREFIX);
2611                table_options.insert(key.to_string(), value.to_string());
2612            }
2613        }
2614        if has_twcs_option {
2615            table_options
2616                .entry(COMPACTION_TYPE.to_string())
2617                .or_insert_with(|| COMPACTION_TYPE_TWCS.to_string());
2618        }
2619    }
2620
2621    // Semantic keys use their own vocabulary instead of the fixed option list.
2622    for (key, value) in ctx.extensions() {
2623        if is_semantic_option_key(&key) && validate_semantic_option(&key, &value) {
2624            table_options.insert(key, value);
2625        }
2626    }
2627
2628    match create_type {
2629        AutoCreateTableType::Logical(physical_table) => {
2630            table_options.insert(
2631                LOGICAL_TABLE_METADATA_KEY.to_string(),
2632                physical_table.clone(),
2633            );
2634        }
2635        AutoCreateTableType::Exemplar => {
2636            table_options.retain(|key, _| {
2637                !is_semantic_option_key(key)
2638                    && key != SEMANTIC_PER_TABLE_INDEX_KEY
2639                    && key != LOGICAL_TABLE_METADATA_KEY
2640                    && key != PHYSICAL_TABLE_METADATA_KEY
2641                    && key != MERGE_MODE_KEY
2642                    && key != "otlp_metric_compat"
2643            });
2644            table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
2645            table_options.insert(
2646                store_api::metric_engine_consts::PRIMARY_KEY_ENCODING.to_string(),
2647                "sparse".to_string(),
2648            );
2649        }
2650        AutoCreateTableType::Physical => {
2651            if let Some(append_mode) = ctx.extension(APPEND_MODE_KEY) {
2652                table_options.insert(APPEND_MODE_KEY.to_string(), append_mode.to_string());
2653            }
2654            if let Some(merge_mode) = ctx.extension(MERGE_MODE_KEY) {
2655                table_options.insert(MERGE_MODE_KEY.to_string(), merge_mode.to_string());
2656            }
2657        }
2658        // Set append_mode to true for log table.
2659        // because log tables should keep rows with the same ts and tags.
2660        AutoCreateTableType::Log => {
2661            table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
2662        }
2663        AutoCreateTableType::LastNonNull => {
2664            if ctx
2665                .extension(APPEND_MODE_KEY)
2666                .is_some_and(|value| value.eq_ignore_ascii_case("true"))
2667            {
2668                table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
2669                table_options.insert(MERGE_MODE_KEY.to_string(), "last_row".to_string());
2670            } else if let Some(merge_mode) = ctx.extension(MERGE_MODE_KEY) {
2671                table_options.insert(MERGE_MODE_KEY.to_string(), merge_mode.to_string());
2672            } else {
2673                table_options.insert(MERGE_MODE_KEY.to_string(), "last_non_null".to_string());
2674            }
2675        }
2676        AutoCreateTableType::Trace { .. } => {
2677            table_options.insert(APPEND_MODE_KEY.to_string(), "true".to_string());
2678        }
2679    }
2680}
2681
2682/// The parsed per-table semantic index: `{schema -> {table -> {key -> value}}}`,
2683/// produced by the OTLP metrics encode path (where one metric can fan out into
2684/// several tables with distinct keys) and the Prometheus remote write v2 path
2685/// (where per-series metadata declares type/unit, and a series may override its
2686/// target schema).
2687pub type PerTableSemanticIndex = BTreeMap<String, BTreeMap<String, BTreeMap<String, String>>>;
2688
2689/// Parses the per-table semantic index off the context extension. Call once per
2690/// create-planning round: a first write creating N tables would otherwise
2691/// re-parse the whole index N times. `None` when the request carries no index
2692/// (logs, traces, Prom RW v1) or it fails to parse.
2693pub fn parse_per_table_semantic_index(ctx: &QueryContextRef) -> Option<PerTableSemanticIndex> {
2694    let raw = ctx.extension(SEMANTIC_PER_TABLE_INDEX_KEY)?;
2695    match serde_json::from_str(raw) {
2696        Ok(index) => Some(index),
2697        Err(_) => {
2698            warn!("failed to parse semantic per-table index, skipping per-table options");
2699            None
2700        }
2701    }
2702}
2703
2704/// Folds the semantic keys of the table being created into `table_options`.
2705///
2706/// Common keys shared by every table in a request travel as plain semantic
2707/// extensions and are handled by [`fill_table_options_for_create`]; this
2708/// carries only the per-table tail and is applied after it, so a per-table
2709/// value (e.g. `declared` quality) wins. Keys are re-checked against the
2710/// vocabulary defensively.
2711pub fn apply_per_table_semantic_options(
2712    table_options: &mut std::collections::HashMap<String, String>,
2713    index: Option<&PerTableSemanticIndex>,
2714    schema: &str,
2715    table_name: &str,
2716) {
2717    let Some(entry) = index
2718        .and_then(|index| index.get(schema))
2719        .and_then(|tables| tables.get(table_name))
2720    else {
2721        return;
2722    };
2723    for (key, value) in entry {
2724        if is_semantic_option_key(key) && validate_semantic_option(key, value) {
2725            table_options.insert(key.clone(), value.clone());
2726        }
2727    }
2728}
2729
2730pub fn build_create_table_expr(
2731    table: &TableReference,
2732    request_schema: &[ColumnSchema],
2733    engine: &str,
2734) -> Result<CreateTableExpr> {
2735    expr_helper::create_table_expr_by_column_schemas(table, request_schema, engine, None)
2736}
2737
2738/// `QueryContext` extension key the Splunk HEC handler sets (to `"true"`) on its identity
2739/// path to request metadata-first primary-key ordering at table creation. It is absent for
2740/// user-supplied pipelines, so their primary-key order is left untouched.
2741pub const SPLUNK_PK_METADATA_ORDER_KEY: &str = "splunk_pk_metadata_order";
2742
2743/// Moves Splunk's metadata tags (`host`, `source`, `sourcetype`) to the front of the
2744/// primary key, keeping the relative order of the remaining tags.
2745fn reorder_splunk_primary_keys(primary_keys: &mut [String]) {
2746    const LEAD: [&str; 3] = ["host", "source", "sourcetype"];
2747    // Stable sort: `LEAD` columns move to the front in `host`/`source`/`sourcetype` order;
2748    // every other column keeps its existing relative position.
2749    primary_keys.sort_by_key(|name| {
2750        LEAD.iter()
2751            .position(|&lead| lead == name.as_str())
2752            .unwrap_or(LEAD.len())
2753    });
2754}
2755
2756/// Result of `create_or_alter_tables_on_demand`.
2757struct CreateAlterTableResult {
2758    /// table ids of ttl=instant tables.
2759    instant_table_ids: HashSet<TableId>,
2760    /// Table Info of the created tables.
2761    table_infos: HashMap<TableId, Arc<TableInfo>>,
2762}
2763
2764/// Upper bound of rows buffered by detached flow mirror tasks on this frontend.
2765///
2766/// Rows retained by in-flight mirror RPCs count against this bound. If ordinary
2767/// persisted source-table mirroring exceeds it, that mirror batch is best-effort
2768/// dropped while its datanode write continues. Instant-TTL batches fail retryably
2769/// on temporary saturation; oversized batches fail non-retryably so the client
2770/// can reduce the batch size.
2771const MAX_MIRROR_PENDING_ROWS: usize = 1_000_000;
2772
2773/// How often at most each mirror event is reported.
2774const MIRROR_LOG_INTERVAL: Duration = Duration::from_secs(10);
2775const MIRROR_LOG_NEVER_REPORTED: u64 = u64::MAX;
2776
2777struct MirrorLog {
2778    last_log_millis: AtomicU64,
2779    events: AtomicU64,
2780}
2781
2782impl MirrorLog {
2783    const fn new() -> Self {
2784        Self {
2785            last_log_millis: AtomicU64::new(MIRROR_LOG_NEVER_REPORTED),
2786            events: AtomicU64::new(0),
2787        }
2788    }
2789
2790    /// Returns the count since the previous report, including this event, to
2791    /// the single caller that claims the interval.
2792    fn claim_report(&self, now_millis: u64) -> Option<u64> {
2793        self.events.fetch_add(1, Ordering::Relaxed);
2794        let last = self.last_log_millis.load(Ordering::Relaxed);
2795        if last != MIRROR_LOG_NEVER_REPORTED
2796            && now_millis.saturating_sub(last) < MIRROR_LOG_INTERVAL.as_millis() as u64
2797        {
2798            return None;
2799        }
2800        if self
2801            .last_log_millis
2802            .compare_exchange(last, now_millis, Ordering::Relaxed, Ordering::Relaxed)
2803            .is_err()
2804        {
2805            return None;
2806        }
2807        Some(self.events.swap(0, Ordering::Relaxed))
2808    }
2809}
2810
2811static MIRROR_DROP_LOG: MirrorLog = MirrorLog::new();
2812static MIRROR_FAILURE_LOG: MirrorLog = MirrorLog::new();
2813
2814/// Milliseconds since the first call, so the drop log is rate limited on a
2815/// monotonic clock instead of the wall clock, which can jump.
2816fn mirror_drop_log_millis() -> u64 {
2817    static START: LazyLock<Instant> = LazyLock::new(Instant::now);
2818    START.elapsed().as_millis() as u64
2819}
2820
2821struct FlowMirrorTask {
2822    requests: HashMap<Peer, RegionInsertRequests>,
2823}
2824
2825impl FlowMirrorTask {
2826    async fn new(
2827        cache: &TableFlownodeSetCacheRef,
2828        requests: impl Iterator<Item = &RegionInsertRequest>,
2829    ) -> Result<Self> {
2830        // A cold table needs a prefix scan to discover its flow partitions.
2831        const MAX_CONCURRENT_FLOW_LOOKUPS: usize = 8;
2832
2833        let mut by_table: HashMap<TableId, Vec<&RegionInsertRequest>> = HashMap::new();
2834        for req in requests {
2835            let table_id = RegionId::from_u64(req.region_id).table_id();
2836            by_table.entry(table_id).or_default().push(req);
2837        }
2838
2839        let table_ids: Vec<_> = by_table.keys().copied().collect();
2840        let mut lookups = stream::iter(table_ids)
2841            .map(|id| async move { cache.get(id).await.map(|flows| (id, flows)) })
2842            .buffer_unordered(MAX_CONCURRENT_FLOW_LOOKUPS);
2843        let mut inserts: HashMap<Peer, RegionInsertRequests> = HashMap::new();
2844        while let Some((id, flows)) = lookups.try_next().await.context(RequestInsertsSnafu)? {
2845            if let Some(flows) = flows {
2846                for peer in flows.values().collect::<HashSet<_>>() {
2847                    inserts
2848                        .entry(peer.clone())
2849                        .or_default()
2850                        .requests
2851                        .extend(by_table[&id].iter().copied().cloned());
2852                }
2853            }
2854        }
2855
2856        Ok(Self { requests: inserts })
2857    }
2858
2859    /// Rows of row data this task keeps alive once detached: the payload it
2860    /// spawns per peer, not the source requests it was built from.
2861    ///
2862    /// A source table mapped to several flownodes clones its requests per peer
2863    /// and every clone owns its own copy of the rows, so reserving from
2864    /// `self.requests` is what the per-peer shares released by [`Self::detach`]
2865    /// sum to. Counting the source requests instead under-reserves whenever a
2866    /// batch holds several requests for the same source table.
2867    fn pending_rows(&self) -> u64 {
2868        self.requests.values().map(region_inserts_rows).sum()
2869    }
2870
2871    fn detach(
2872        self,
2873        node_manager: NodeManagerRef,
2874        mirror_pending_rows: Arc<AtomicU64>,
2875        has_instant_rows: bool,
2876    ) -> Result<()> {
2877        // Reserve the cloned per-peer payload before spawning. A full budget
2878        // drops normal mirrors or rejects instant writes before datanode dispatch.
2879        let num_rows = self.pending_rows();
2880        if num_rows == 0 {
2881            return Ok(());
2882        }
2883        if has_instant_rows && num_rows > MAX_MIRROR_PENDING_ROWS as u64 {
2884            return InvalidInsertRequestSnafu {
2885                reason: format!(
2886                    "flow mirror batch has {num_rows} peer-fan-out rows, exceeding the limit of {}; reduce the batch size",
2887                    MAX_MIRROR_PENDING_ROWS
2888                ),
2889            }
2890            .fail();
2891        }
2892
2893        let pending = reserve_mirror_pending_rows(&mirror_pending_rows, num_rows);
2894        if pending > MAX_MIRROR_PENDING_ROWS as u64 {
2895            release_mirror_pending_rows(&mirror_pending_rows, num_rows);
2896            if has_instant_rows {
2897                return Err(meter_core::collect::WriteRejected::new(
2898                    "flow mirror pending rows limit exceeded for instant-TTL insert",
2899                ))
2900                .context(WriteRejectedSnafu);
2901            }
2902            crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.inc_by(num_rows);
2903            if let Some(events) = MIRROR_DROP_LOG.claim_report(mirror_drop_log_millis()) {
2904                warn!(
2905                    "Flow mirror write dropped: {} pending rows exceeds limit {}, {} drop events since last report",
2906                    pending, MAX_MIRROR_PENDING_ROWS, events
2907                );
2908            }
2909            return Ok(());
2910        }
2911
2912        for (peer, inserts) in self.requests {
2913            // Each spawned task releases exactly its own share of the
2914            // reservation. The shares are computed over the same per-peer
2915            // payload as `pending_rows`, so they sum to the reservation.
2916            let peer_rows = region_inserts_rows(&inserts);
2917            let node_manager = node_manager.clone();
2918            let mirror_pending_rows = mirror_pending_rows.clone();
2919            common_runtime::spawn_global(async move {
2920                let result = node_manager
2921                    .flownode(&peer)
2922                    .await
2923                    .handle_inserts(inserts)
2924                    .await
2925                    .context(RequestInsertsSnafu);
2926
2927                match result {
2928                    Ok(resp) => {
2929                        let affected_rows = resp.affected_rows;
2930                        crate::metrics::DIST_MIRROR_ROW_COUNT.inc_by(affected_rows);
2931                    }
2932                    Err(err) => {
2933                        if let Some(events) =
2934                            MIRROR_FAILURE_LOG.claim_report(mirror_drop_log_millis())
2935                        {
2936                            error!(err; flownode_id = peer.id, flownode_addr = %peer.addr, "Failed to insert data into flownode ({} total mirror failures across peers since last report)", events);
2937                        }
2938                    }
2939                }
2940                // Release the reservation on both the success and the failure
2941                // path, otherwise a broken flownode would fill the budget forever.
2942                release_mirror_pending_rows(&mirror_pending_rows, peer_rows);
2943            });
2944        }
2945
2946        Ok(())
2947    }
2948}
2949
2950/// Rows of row data carried by `inserts`. A request without a `rows` payload
2951/// carries nothing to keep buffered and contributes zero.
2952fn region_inserts_rows(inserts: &RegionInsertRequests) -> u64 {
2953    inserts
2954        .requests
2955        .iter()
2956        .filter_map(|req| req.rows.as_ref())
2957        .map(|rows| rows.rows.len() as u64)
2958        .sum()
2959}
2960
2961/// Reserves `rows` of the pending mirror budget and returns the resulting
2962/// pending row count. The caller must [`release_mirror_pending_rows`] exactly
2963/// these rows once they are no longer buffered, whether it spawns the batch or
2964/// drops it.
2965fn reserve_mirror_pending_rows(mirror_pending_rows: &AtomicU64, rows: u64) -> u64 {
2966    let pending = mirror_pending_rows.fetch_add(rows, Ordering::Relaxed) + rows;
2967    // Keep the gauge in step by applying the same increment, never by setting an
2968    // absolute value: another task may move the budget in between.
2969    crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.add(rows as i64);
2970    pending
2971}
2972
2973/// Releases `rows` of the pending mirror budget.
2974///
2975/// Every release mirrors the reservation it hands back, so a well-behaved
2976/// release never subtracts more than is pending; the saturating update only
2977/// keeps a duplicated release from wrapping the shared budget around.
2978fn release_mirror_pending_rows(mirror_pending_rows: &AtomicU64, rows: u64) {
2979    let prev = mirror_pending_rows
2980        .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |pending| {
2981            Some(pending.saturating_sub(rows))
2982        })
2983        .unwrap_or_else(|prev| prev);
2984    // Subtract from the gauge with the opposite sign of
2985    // [`reserve_mirror_pending_rows`]; reading the budget here and `set`-ting it
2986    // would publish a stale value whenever another task releases concurrently.
2987    let released = rows.min(prev);
2988    if released > 0 {
2989        crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.sub(released as i64);
2990    }
2991}
2992
2993#[cfg(test)]
2994mod tests {
2995    use std::future::Future;
2996    use std::sync::Arc;
2997    use std::sync::atomic::{AtomicU64, Ordering};
2998    use std::task::Poll;
2999
3000    use api::helper::ColumnDataTypeWrapper;
3001    use api::region::RegionResponse;
3002    use api::v1::flow::FlowResponse;
3003    use api::v1::helper::{field_column_schema, time_index_column_schema};
3004    use api::v1::{Row, RowInsertRequest, Rows, Value};
3005    use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
3006    use common_error::ext::{ErrorExt, RetryHint};
3007    use common_error::mock::MockError;
3008    use common_error::status_code::StatusCode;
3009    use common_meta::cache::new_table_flownode_set_cache;
3010    use common_meta::cache_invalidator::DummyCacheInvalidator;
3011    use common_meta::ddl::test_util::datanode_handler::NaiveDatanodeHandler;
3012    use common_meta::instruction::{CacheIdent, CreateFlow};
3013    use common_meta::node_manager::{
3014        Datanode, DatanodeManager, DatanodeRef, FlownodeManager, FlownodeRef,
3015    };
3016    use common_meta::test_util::{MockDatanodeManager, MockFlownodeHandler, MockFlownodeManager};
3017    use common_query::native_histogram::NATIVE_HISTOGRAM_FIELD;
3018    use common_query::prelude::{greptime_native_histogram, set_default_prefix};
3019    use datatypes::data_type::ConcreteDataType;
3020    use datatypes::schema::ColumnSchema;
3021    use moka::future::Cache;
3022    use session::context::QueryContext;
3023    use table::TableRef;
3024    use table::dist_table::DummyDataSource;
3025    use table::metadata::{TableInfoBuilder, TableMetaBuilder, TableType};
3026    use tokio::sync::{Semaphore, mpsc};
3027
3028    use crate::insert::*;
3029    use crate::test_util::{
3030        create_partition_rule_manager, new_test_table_info, prepare_mocked_backend,
3031    };
3032    use crate::write_dispatcher::{TrackedWrite, WriteRetryOptions};
3033
3034    async fn cache_test_flow(cache: &TableFlownodeSetCacheRef, flow: CreateFlow) {
3035        cache.get_many(&flow.source_table_ids).await.unwrap();
3036        cache
3037            .invalidate(&[CacheIdent::CreateFlow(flow)])
3038            .await
3039            .unwrap();
3040    }
3041
3042    fn make_table_ref_with_schema(
3043        ts_name: &str,
3044        field_name: &str,
3045        field_type: ConcreteDataType,
3046    ) -> TableRef {
3047        let schema = datatypes::schema::SchemaBuilder::try_from_columns(vec![
3048            ColumnSchema::new(
3049                ts_name,
3050                ConcreteDataType::timestamp_millisecond_datatype(),
3051                false,
3052            )
3053            .with_time_index(true),
3054            ColumnSchema::new(field_name, field_type, true),
3055        ])
3056        .unwrap()
3057        .build()
3058        .unwrap();
3059        let meta = TableMetaBuilder::empty()
3060            .schema(Arc::new(schema))
3061            .primary_key_indices(vec![])
3062            .value_indices(vec![1])
3063            .engine("mito")
3064            .next_column_id(0)
3065            .options(Default::default())
3066            .created_on(Default::default())
3067            .build()
3068            .unwrap();
3069        let info = Arc::new(
3070            TableInfoBuilder::default()
3071                .table_id(1)
3072                .table_version(0)
3073                .name("test_table")
3074                .schema_name(DEFAULT_SCHEMA_NAME)
3075                .catalog_name(DEFAULT_CATALOG_NAME)
3076                .desc(None)
3077                .table_type(TableType::Base)
3078                .meta(meta)
3079                .build()
3080                .unwrap(),
3081        );
3082        Arc::new(table::Table::new(
3083            info,
3084            table::metadata::FilterPushDownType::Unsupported,
3085            Arc::new(DummyDataSource),
3086        ))
3087    }
3088
3089    fn make_metric_physical_table_ref_with_time_unit(unit: TimeUnit) -> TableRef {
3090        let schema = datatypes::schema::SchemaBuilder::try_from_columns(vec![
3091            ColumnSchema::new(
3092                greptime_timestamp(),
3093                ConcreteDataType::timestamp_datatype(unit),
3094                false,
3095            )
3096            .with_time_index(true),
3097            ColumnSchema::new(greptime_value(), ConcreteDataType::float64_datatype(), true),
3098        ])
3099        .unwrap()
3100        .build()
3101        .unwrap();
3102        let meta = TableMetaBuilder::empty()
3103            .schema(Arc::new(schema))
3104            .primary_key_indices(vec![])
3105            .value_indices(vec![1])
3106            .engine("metric")
3107            .next_column_id(0)
3108            .options(Default::default())
3109            .created_on(Default::default())
3110            .build()
3111            .unwrap();
3112        let info = Arc::new(
3113            TableInfoBuilder::default()
3114                .table_id(1)
3115                .table_version(0)
3116                .name("greptime_physical_table")
3117                .schema_name(DEFAULT_SCHEMA_NAME)
3118                .catalog_name(DEFAULT_CATALOG_NAME)
3119                .desc(None)
3120                .table_type(TableType::Base)
3121                .meta(meta)
3122                .build()
3123                .unwrap(),
3124        );
3125        Arc::new(table::Table::new(
3126            info,
3127            table::metadata::FilterPushDownType::Unsupported,
3128            Arc::new(DummyDataSource),
3129        ))
3130    }
3131
3132    fn ms_row_insert_request(timestamp_ms: i64) -> RowInsertRequest {
3133        ms_row_insert_request_named("my_metric", timestamp_ms)
3134    }
3135
3136    fn ms_row_insert_request_named(table: &str, timestamp_ms: i64) -> RowInsertRequest {
3137        RowInsertRequest {
3138            table_name: table.to_string(),
3139            rows: Some(Rows {
3140                schema: vec![
3141                    time_index_column_schema(
3142                        greptime_timestamp(),
3143                        ColumnDataType::TimestampMillisecond,
3144                    ),
3145                    field_column_schema(greptime_value(), ColumnDataType::Float64),
3146                ],
3147                rows: vec![Row {
3148                    values: vec![
3149                        Value {
3150                            value_data: Some(ValueData::TimestampMillisecondValue(timestamp_ms)),
3151                        },
3152                        Value {
3153                            value_data: Some(ValueData::F64Value(1.0)),
3154                        },
3155                    ],
3156                }],
3157            }),
3158        }
3159    }
3160
3161    /// Converts the time index column of each request to `target_unit`.
3162    fn convert_row_insert_requests_time_unit(
3163        requests: &mut RowInsertRequests,
3164        target_unit: TimeUnit,
3165    ) -> Result<()> {
3166        for request in &mut requests.inserts {
3167            let Some(rows) = request.rows.as_mut() else {
3168                continue;
3169            };
3170            convert_rows_time_unit(rows, target_unit)?;
3171        }
3172        Ok(())
3173    }
3174
3175    #[test]
3176    fn test_convert_row_insert_requests_time_unit_noop_when_matching() {
3177        let mut requests = RowInsertRequests {
3178            inserts: vec![ms_row_insert_request(123)],
3179        };
3180        convert_row_insert_requests_time_unit(&mut requests, TimeUnit::Millisecond).unwrap();
3181        let rows = requests.inserts[0].rows.as_ref().unwrap();
3182        assert_eq!(
3183            rows.schema[0].datatype,
3184            ColumnDataType::TimestampMillisecond as i32
3185        );
3186        assert!(matches!(
3187            rows.rows[0].values[0].value_data,
3188            Some(ValueData::TimestampMillisecondValue(123))
3189        ));
3190    }
3191
3192    #[test]
3193    fn test_convert_row_insert_requests_time_unit_widens_losslessly() {
3194        let mut requests = RowInsertRequests {
3195            inserts: vec![ms_row_insert_request(123)],
3196        };
3197        convert_row_insert_requests_time_unit(&mut requests, TimeUnit::Microsecond).unwrap();
3198        let rows = requests.inserts[0].rows.as_ref().unwrap();
3199        assert_eq!(
3200            rows.schema[0].datatype,
3201            ColumnDataType::TimestampMicrosecond as i32
3202        );
3203        assert!(matches!(
3204            rows.rows[0].values[0].value_data,
3205            Some(ValueData::TimestampMicrosecondValue(123_000))
3206        ));
3207    }
3208
3209    #[test]
3210    fn test_convert_row_insert_requests_time_unit_truncates_on_narrowing() {
3211        // 123_456_789 ns floors to 123_456 us and 123 ms; negative values
3212        // floor towards negative infinity, matching `Timestamp::convert_to`.
3213        let requests = |value_ns: i64| RowInsertRequests {
3214            inserts: vec![RowInsertRequest {
3215                table_name: "my_metric".to_string(),
3216                rows: Some(Rows {
3217                    schema: vec![
3218                        time_index_column_schema(
3219                            greptime_timestamp(),
3220                            ColumnDataType::TimestampNanosecond,
3221                        ),
3222                        field_column_schema(greptime_value(), ColumnDataType::Float64),
3223                    ],
3224                    rows: vec![Row {
3225                        values: vec![
3226                            Value {
3227                                value_data: Some(ValueData::TimestampNanosecondValue(value_ns)),
3228                            },
3229                            Value {
3230                                value_data: Some(ValueData::F64Value(1.0)),
3231                            },
3232                        ],
3233                    }],
3234                }),
3235            }],
3236        };
3237
3238        let mut reqs = requests(123_456_789);
3239        convert_row_insert_requests_time_unit(&mut reqs, TimeUnit::Microsecond).unwrap();
3240        let rows = reqs.inserts[0].rows.as_ref().unwrap();
3241        assert!(matches!(
3242            rows.rows[0].values[0].value_data,
3243            Some(ValueData::TimestampMicrosecondValue(123_456))
3244        ));
3245
3246        let mut reqs = requests(123_456_789);
3247        convert_row_insert_requests_time_unit(&mut reqs, TimeUnit::Millisecond).unwrap();
3248        let rows = reqs.inserts[0].rows.as_ref().unwrap();
3249        assert!(matches!(
3250            rows.rows[0].values[0].value_data,
3251            Some(ValueData::TimestampMillisecondValue(123))
3252        ));
3253
3254        let mut reqs = requests(-123_456_789);
3255        convert_row_insert_requests_time_unit(&mut reqs, TimeUnit::Microsecond).unwrap();
3256        let rows = reqs.inserts[0].rows.as_ref().unwrap();
3257        assert!(matches!(
3258            rows.rows[0].values[0].value_data,
3259            Some(ValueData::TimestampMicrosecondValue(-123_457))
3260        ));
3261    }
3262
3263    #[test]
3264    fn test_convert_row_insert_requests_time_unit_overflow() {
3265        let mut requests = RowInsertRequests {
3266            inserts: vec![ms_row_insert_request(i64::MAX)],
3267        };
3268        let err =
3269            convert_row_insert_requests_time_unit(&mut requests, TimeUnit::Nanosecond).unwrap_err();
3270        assert!(err.to_string().contains("overflows"), "{err}");
3271    }
3272
3273    #[test]
3274    fn test_table_time_index_unit() {
3275        assert_eq!(
3276            table_time_index_unit(&make_metric_physical_table_ref_with_time_unit(
3277                TimeUnit::Microsecond
3278            )),
3279            Some(TimeUnit::Microsecond)
3280        );
3281        assert_eq!(
3282            table_time_index_unit(&make_metric_physical_table_ref_with_time_unit(
3283                TimeUnit::Millisecond
3284            )),
3285            Some(TimeUnit::Millisecond)
3286        );
3287    }
3288
3289    #[tokio::test]
3290    async fn test_accommodate_existing_schema_and_reject_kind_changes() {
3291        let ts_name = "my_ts";
3292        let field_name = "my_field";
3293        let table =
3294            make_table_ref_with_schema(ts_name, field_name, ConcreteDataType::float64_datatype());
3295
3296        // The request uses different names for timestamp and field columns
3297        let mut req = RowInsertRequest {
3298            table_name: "test_table".to_string(),
3299            rows: Some(Rows {
3300                schema: vec![
3301                    time_index_column_schema("ts_wrong", ColumnDataType::TimestampMillisecond),
3302                    field_column_schema("field_wrong", ColumnDataType::Float64),
3303                ],
3304                rows: vec![api::v1::Row {
3305                    values: vec![Value::default(), Value::default()],
3306                }],
3307            }),
3308        };
3309        let ctx = Arc::new(QueryContext::with(
3310            DEFAULT_CATALOG_NAME,
3311            DEFAULT_SCHEMA_NAME,
3312        ));
3313
3314        let kv_backend = prepare_mocked_backend().await;
3315        let inserter = Inserter::new(
3316            catalog::memory::MemoryCatalogManager::new(),
3317            create_partition_rule_manager(kv_backend.clone()).await,
3318            Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler)),
3319            Arc::new(new_table_flownode_set_cache(
3320                String::new(),
3321                Cache::new(100),
3322                kv_backend.clone(),
3323            )),
3324            true,
3325        );
3326        // Do not apply an absent-table plan to a table that appeared concurrently.
3327        assert!(
3328            inserter
3329                .get_alter_table_expr_on_demand(&mut req, &table, &ctx, false, false, false)
3330                .unwrap()
3331                .is_none()
3332        );
3333        let alter_expr = inserter
3334            .get_alter_table_expr_on_demand(&mut req, &table, &ctx, true, true, true)
3335            .unwrap();
3336        assert!(alter_expr.is_none());
3337
3338        // The request's schema should have updated names for timestamp and field columns
3339        let req_schema = req.rows.as_ref().unwrap().schema.clone();
3340        assert_eq!(req_schema[0].column_name, ts_name);
3341        assert_eq!(req_schema[1].column_name, field_name);
3342
3343        let (datatype, datatype_extension) =
3344            ColumnDataTypeWrapper::try_from(native_histogram_value_type().clone())
3345                .unwrap()
3346                .into_parts();
3347        let mut histogram_req = RowInsertRequest {
3348            table_name: "test_table".to_string(),
3349            rows: Some(Rows {
3350                schema: vec![
3351                    time_index_column_schema("ts", ColumnDataType::TimestampMillisecond),
3352                    api::v1::ColumnSchema {
3353                        column_name: greptime_native_histogram().to_string(),
3354                        datatype: datatype as i32,
3355                        semantic_type: SemanticType::Field as i32,
3356                        datatype_extension,
3357                        options: None,
3358                    },
3359                ],
3360                rows: vec![],
3361            }),
3362        };
3363        let error = inserter
3364            .get_alter_table_expr_on_demand(&mut histogram_req, &table, &ctx, false, true, true)
3365            .unwrap_err();
3366        assert!(
3367            error
3368                .to_string()
3369                .contains("cannot mix native histogram and float sample fields")
3370        );
3371
3372        let histogram_table = make_table_ref_with_schema(
3373            "ts",
3374            greptime_native_histogram(),
3375            native_histogram_value_type().clone(),
3376        );
3377        let mut sample_req = RowInsertRequest {
3378            table_name: "test_table".to_string(),
3379            rows: Some(Rows {
3380                schema: vec![
3381                    time_index_column_schema("ts", ColumnDataType::TimestampMillisecond),
3382                    field_column_schema(greptime_value(), ColumnDataType::Float64),
3383                ],
3384                rows: vec![],
3385            }),
3386        };
3387        let error = inserter
3388            .get_alter_table_expr_on_demand(
3389                &mut sample_req,
3390                &histogram_table,
3391                &ctx,
3392                false,
3393                true,
3394                true,
3395            )
3396            .unwrap_err();
3397        assert!(
3398            error
3399                .to_string()
3400                .contains("cannot mix native histogram and float sample fields")
3401        );
3402    }
3403
3404    #[test]
3405    fn test_native_histogram_detection_survives_prefix_change() {
3406        set_default_prefix(Some("custom")).unwrap();
3407        let table = make_table_ref_with_schema(
3408            "custom_timestamp",
3409            NATIVE_HISTOGRAM_FIELD,
3410            native_histogram_value_type().clone(),
3411        );
3412        let (datatype, datatype_extension) =
3413            ColumnDataTypeWrapper::try_from(native_histogram_value_type().clone())
3414                .unwrap()
3415                .into_parts();
3416        let request_schema = [api::v1::ColumnSchema {
3417            column_name: greptime_native_histogram().to_string(),
3418            datatype: datatype as i32,
3419            semantic_type: SemanticType::Field as i32,
3420            datatype_extension,
3421            options: None,
3422        }];
3423
3424        assert!(request_is_native_histogram(&request_schema));
3425        assert!(table_is_native_histogram(&table));
3426    }
3427
3428    #[derive(Clone)]
3429    struct GatedDatanodeHandler {
3430        started: mpsc::UnboundedSender<u32>,
3431        completed: mpsc::UnboundedSender<u32>,
3432        release: Arc<Semaphore>,
3433    }
3434
3435    #[async_trait::async_trait]
3436    impl Datanode for GatedDatanodeHandler {
3437        async fn handle_request(
3438            &self,
3439            request: Arc<api::v1::region::RegionRequest>,
3440        ) -> common_meta::error::Result<RegionResponse> {
3441            let Some(api::v1::region::region_request::Body::Inserts(requests)) = &request.body
3442            else {
3443                panic!("expected inserts");
3444            };
3445            let region = RegionId::from(requests.requests[0].region_id).region_number();
3446            self.started.send(region).unwrap();
3447            self.release.acquire().await.unwrap().forget();
3448            self.completed.send(region).unwrap();
3449            if region == 2 {
3450                Err(common_meta::error::Error::retry_later(MockError::new(
3451                    StatusCode::RegionBusy,
3452                )))
3453            } else {
3454                Ok(RegionResponse::new(1))
3455            }
3456        }
3457
3458        async fn handle_query(
3459            &self,
3460            _request: common_query::request::QueryRequest,
3461        ) -> common_meta::error::Result<common_recordbatch::SendableRecordBatchStream> {
3462            unreachable!()
3463        }
3464    }
3465
3466    #[async_trait::async_trait]
3467    impl DatanodeManager for GatedDatanodeHandler {
3468        async fn datanode(&self, _peer: &Peer) -> DatanodeRef {
3469            Arc::new(self.clone())
3470        }
3471    }
3472
3473    #[async_trait::async_trait]
3474    impl FlownodeManager for GatedDatanodeHandler {
3475        async fn flownode(&self, _peer: &Peer) -> FlownodeRef {
3476            unreachable!()
3477        }
3478    }
3479
3480    #[tokio::test]
3481    async fn test_insert_cancellation_drains_inflight_writes() {
3482        for confirmed in [false, true] {
3483            for max_retries in [0, 2] {
3484                let mut inserter = batcher_test_inserter().await;
3485                let (started, mut starts) = mpsc::unbounded_channel();
3486                let (completed, mut completions) = mpsc::unbounded_channel();
3487                let release = Arc::new(Semaphore::new(0));
3488                let node_manager = Arc::new(GatedDatanodeHandler {
3489                    started,
3490                    completed,
3491                    release: release.clone(),
3492                });
3493                inserter.node_manager = node_manager.clone();
3494                let dispatcher = Arc::new(
3495                    WriteDispatcher::new(node_manager, inserter.partition_manager.clone())
3496                        .with_retry(
3497                            WriteRetryOptions {
3498                                max_retries,
3499                                initial_backoff: Duration::ZERO,
3500                                max_backoff: Duration::ZERO,
3501                                ..Default::default()
3502                            },
3503                            inserter.catalog_manager.clone(),
3504                            Arc::new(DummyCacheInvalidator),
3505                        ),
3506                );
3507                let inserter = inserter.with_write_dispatcher(dispatcher);
3508                let table_infos = HashMap::from_iter([(
3509                    1,
3510                    Arc::new(new_test_table_info(1, "table_1", [1].into_iter())),
3511                )]);
3512                let counter = Arc::new(session::write_progress::WriteProgress::default());
3513                let mut ctx = QueryContext::arc().fork();
3514                ctx.set_write_progress(counter.clone());
3515                let ctx = Arc::new(ctx);
3516                let ctx = if confirmed {
3517                    Arc::new(ctx.with_confirmed_writes())
3518                } else {
3519                    ctx
3520                };
3521                let requests = InstantAndNormalInsertRequests {
3522                    normal_requests: RegionInsertRequests {
3523                        requests: [1, 2]
3524                            .map(|region| RegionInsertRequest {
3525                                region_id: RegionId::new(1, region).as_u64(),
3526                                rows: Some(Rows {
3527                                    schema: vec![],
3528                                    rows: vec![Row { values: vec![] }],
3529                                }),
3530                                ..Default::default()
3531                            })
3532                            .to_vec(),
3533                    },
3534                    instant_requests: RegionInsertRequests::default(),
3535                };
3536                let caller =
3537                    tokio::spawn(
3538                        async move { inserter.do_request(requests, &table_infos, &ctx).await },
3539                    );
3540                for _ in 0..2 {
3541                    tokio::time::timeout(Duration::from_secs(5), starts.recv())
3542                        .await
3543                        .unwrap()
3544                        .unwrap();
3545                }
3546                caller.abort();
3547                assert!(caller.await.unwrap_err().is_cancelled());
3548                release.add_permits(2);
3549                // Closing the channel proves the detached dispatcher has finished.
3550                let mut completed = tokio::time::timeout(Duration::from_secs(5), async {
3551                    let mut regions = Vec::new();
3552                    while let Some(region) = completions.recv().await {
3553                        regions.push(region);
3554                    }
3555                    regions
3556                })
3557                .await
3558                .unwrap();
3559                completed.sort_unstable();
3560                assert_eq!(
3561                    completed,
3562                    vec![1, 2],
3563                    "confirmed={confirmed}, retries={max_retries}"
3564                );
3565                assert!(
3566                    starts.try_recv().is_err(),
3567                    "cancelled caller started another retry"
3568                );
3569                assert_eq!(counter.written(), 1);
3570            }
3571        }
3572    }
3573
3574    #[tokio::test]
3575    async fn test_tracked_cancellation_stops_backoff() {
3576        let inserter = batcher_test_inserter().await;
3577        let (started, mut starts) = mpsc::unbounded_channel();
3578        let (completed, mut completions) = mpsc::unbounded_channel();
3579        let node_manager = Arc::new(GatedDatanodeHandler {
3580            started,
3581            completed,
3582            release: Arc::new(Semaphore::new(2)),
3583        });
3584        let dispatcher = WriteDispatcher::new(node_manager, inserter.partition_manager.clone())
3585            .with_retry(
3586                WriteRetryOptions {
3587                    max_retries: 2,
3588                    initial_backoff: Duration::from_secs(60),
3589                    max_backoff: Duration::from_secs(60),
3590                    ..Default::default()
3591                },
3592                inserter.catalog_manager.clone(),
3593                Arc::new(DummyCacheInvalidator),
3594            );
3595        let factory = RegionRequestFactory::new(RegionRequestHeader::default());
3596        let requests = [1, 2]
3597            .into_iter()
3598            .map(|region| TrackedWrite {
3599                request: factory.build_insert(RegionInsertRequests {
3600                    requests: vec![RegionInsertRequest {
3601                        region_id: RegionId::new(1, region).as_u64(),
3602                        ..Default::default()
3603                    }],
3604                }),
3605                owners: vec![(region - 1) as usize],
3606            })
3607            .collect();
3608        let (writes, progress) = dispatcher.prepare_tracked(requests, 2).await;
3609        let cancellation = CancellationToken::new();
3610        let execute = dispatcher.execute_tracked(writes, progress, None, Some(&cancellation));
3611        tokio::pin!(execute);
3612        // Wait for both sends to complete before cancelling the backoff.
3613        tokio::time::timeout(
3614            Duration::from_secs(1),
3615            future::poll_fn(|cx| {
3616                assert!(execute.as_mut().poll(cx).is_pending());
3617                if completions.len() == 2 {
3618                    Poll::Ready(())
3619                } else {
3620                    Poll::Pending
3621                }
3622            }),
3623        )
3624        .await
3625        .unwrap();
3626        for region in [1, 2] {
3627            assert_eq!(starts.try_recv().unwrap(), region);
3628            assert_eq!(completions.try_recv().unwrap(), region);
3629        }
3630        cancellation.cancel();
3631        let progress = tokio::time::timeout(Duration::from_secs(1), execute)
3632            .await
3633            .unwrap();
3634        assert_eq!(progress[0].affected_rows, 1);
3635        assert!(progress[0].error.is_none());
3636        assert_eq!(progress[1].affected_rows, 0);
3637        assert_eq!(
3638            progress[1].error.as_ref().unwrap().retry_hint(),
3639            RetryHint::Retryable
3640        );
3641        assert!(starts.try_recv().is_err());
3642    }
3643
3644    // Keep global meter registration in one test, isolated by nextest's per-test process.
3645    #[tokio::test]
3646    async fn test_write_meter_admission() {
3647        use std::cell::Cell;
3648        use std::sync::Mutex;
3649        use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
3650
3651        use api::region::RegionResponse;
3652        use api::v1::region::region_request::Body;
3653        use arrow::array::{Int32Array, TimestampMillisecondArray};
3654        use arrow::record_batch::RecordBatch;
3655        use bytes::Bytes;
3656        use common_error::ext::{ErrorExt, RetryHint};
3657        use common_error::status_code::StatusCode;
3658        use common_grpc::flight::{FlightEncoder, FlightMessage};
3659        use common_meta::ddl::test_util::datanode_handler::DatanodeWatcher;
3660        use futures::future::BoxFuture;
3661        use meter_core::ItemCalculator;
3662        use meter_core::collect::{Collect, WriteRejected};
3663        use meter_core::data::MeterRecord;
3664        use meter_core::global::global_registry;
3665        use session::context::Channel;
3666
3667        const CATALOG: &str = "write_meter_test";
3668
3669        #[derive(Default)]
3670        struct Meter {
3671            reject: AtomicBool,
3672            attempts: Mutex<Vec<MeterRecord>>,
3673            accepted_value: AtomicU64,
3674        }
3675
3676        impl Collect for Meter {
3677            fn on_write(
3678                &self,
3679                record: MeterRecord,
3680            ) -> BoxFuture<'_, std::result::Result<(), WriteRejected>> {
3681                Box::pin(async move {
3682                    if record.catalog != CATALOG {
3683                        return Ok(());
3684                    }
3685                    let value = record.value;
3686                    self.attempts.lock().unwrap().push(record);
3687                    if self.reject.load(Ordering::Relaxed) {
3688                        return Err(WriteRejected::new("database row quota exhausted"));
3689                    }
3690                    self.accepted_value.fetch_add(value, Ordering::Relaxed);
3691                    Ok(())
3692                })
3693            }
3694
3695            fn on_read(&self, _: MeterRecord) {}
3696        }
3697
3698        impl ItemCalculator<InstantAndNormalInsertRequests> for Meter {
3699            fn calc(&self, _: &InstantAndNormalInsertRequests) -> u64 {
3700                17
3701            }
3702        }
3703
3704        let kv_backend = prepare_mocked_backend().await;
3705        let partition_manager = create_partition_rule_manager(kv_backend.clone()).await;
3706        let (sender, mut dispatched) = tokio::sync::mpsc::channel(16);
3707        let watcher = DatanodeWatcher::new(sender).with_handler(|_, request| {
3708            let rows = match request.body.unwrap() {
3709                Body::Inserts(requests) => requests
3710                    .requests
3711                    .iter()
3712                    .filter_map(|request| request.rows.as_ref())
3713                    .map(|rows| rows.rows.len())
3714                    .sum(),
3715                // The bulk batches below each contain two rows.
3716                Body::BulkInsert(_) => 2,
3717                body => panic!("unexpected request: {body:?}"),
3718            };
3719            Ok(RegionResponse::new(rows))
3720        });
3721        let inserter = Inserter::new(
3722            catalog::memory::MemoryCatalogManager::new(),
3723            partition_manager,
3724            Arc::new(MockDatanodeManager::new(watcher)),
3725            Arc::new(new_table_flownode_set_cache(
3726                String::new(),
3727                Cache::new(100),
3728                kv_backend,
3729            )),
3730            true,
3731        );
3732        let mut table_info = new_test_table_info(1, "table_1", [1].into_iter());
3733        table_info.catalog_name = CATALOG.to_string();
3734        table_info.schema_name = "target_db".to_string();
3735        let table_info = Arc::new(table_info);
3736        let table_infos = HashMap::from_iter([(1, table_info.clone())]);
3737        let ctx = Arc::new(QueryContext::with_channel(
3738            DEFAULT_CATALOG_NAME,
3739            DEFAULT_SCHEMA_NAME,
3740            Channel::Postgres,
3741        ));
3742        let rows_request = || {
3743            let build = |num_rows| RegionInsertRequests {
3744                requests: vec![RegionInsertRequest {
3745                    region_id: RegionId::new(1, 1).as_u64(),
3746                    rows: Some(Rows {
3747                        schema: vec![],
3748                        rows: vec![api::v1::Row { values: vec![] }; num_rows],
3749                    }),
3750                    ..Default::default()
3751                }],
3752            };
3753            InstantAndNormalInsertRequests {
3754                normal_requests: build(3),
3755                instant_requests: build(2),
3756            }
3757        };
3758
3759        // No collector or calculator: ordinary OSS insertion still succeeds.
3760        let output = inserter
3761            .do_request(rows_request(), &table_infos, &ctx)
3762            .await
3763            .unwrap();
3764        assert_eq!(output.meta.cost, 0);
3765        assert!(matches!(output.data, OutputData::AffectedRows(3)));
3766        dispatched.try_recv().unwrap();
3767
3768        let meter = Arc::new(Meter::default());
3769        global_registry().set_collector(meter.clone());
3770        global_registry().register_calculator(meter.clone());
3771        // The dependency controls noop mode; exercise both builds with this test.
3772        let enabled = Cell::new(false);
3773        write_meter!({
3774            enabled.set(true);
3775            MeterRecord::new("probe".into(), "probe".into(), 0, 0, 0)
3776        })
3777        .await
3778        .unwrap();
3779        let enabled = enabled.get();
3780
3781        let output = inserter
3782            .do_request(rows_request(), &table_infos, &ctx)
3783            .await
3784            .unwrap();
3785        assert_eq!(output.meta.cost, if enabled { 17 } else { 0 });
3786        assert!(matches!(output.data, OutputData::AffectedRows(3)));
3787        dispatched.try_recv().unwrap();
3788        let flow_cache = inserter.table_flownode_set_cache();
3789        assert!(flow_cache.contains_key(&1));
3790        common_meta::cache_invalidator::CacheInvalidator::invalidate_all(flow_cache.as_ref())
3791            .unwrap();
3792        meter.reject.store(true, Ordering::Relaxed);
3793        let result = inserter
3794            .do_request(rows_request(), &table_infos, &ctx)
3795            .await;
3796        if enabled {
3797            let error = result.unwrap_err();
3798            assert_eq!(error.status_code(), StatusCode::RateLimited);
3799            assert_eq!(error.retry_hint(), RetryHint::Retryable);
3800            assert!(error.to_string().contains("database row quota exhausted"));
3801            assert!(dispatched.try_recv().is_err());
3802            assert!(
3803                !flow_cache.contains_key(&1),
3804                "rejected write reached flow mirroring"
3805            );
3806            assert_eq!(meter.accepted_value.load(Ordering::Relaxed), 17);
3807            let attempts = meter.attempts.lock().unwrap();
3808            assert_eq!(attempts.len(), 2);
3809            for record in attempts.iter() {
3810                assert_eq!(record.catalog, CATALOG);
3811                assert_eq!(record.schema, "target_db");
3812                assert_eq!(
3813                    (record.rows, record.value, record.source),
3814                    (5, 17, Channel::Postgres as u8)
3815                );
3816            }
3817        } else {
3818            assert_eq!(result.unwrap().meta.cost, 0);
3819            dispatched.try_recv().unwrap();
3820            assert!(meter.attempts.lock().unwrap().is_empty());
3821        }
3822        meter.attempts.lock().unwrap().clear();
3823
3824        // Rejection must precede queue submission, but conversion memory is reserved first.
3825        if enabled {
3826            let batcher: Arc<dyn PendingRowsBatcher> = Arc::new(UnexpectedBatcher {
3827                allow_acquire: true,
3828            });
3829            let rows = Rows {
3830                schema: vec![
3831                    api::v1::helper::tag_column_schema("a", ColumnDataType::Int32),
3832                    time_index_column_schema("ts", ColumnDataType::TimestampMillisecond),
3833                    field_column_schema("b", ColumnDataType::Int32),
3834                ],
3835                rows: vec![api::v1::Row {
3836                    values: vec![
3837                        api::v1::value::ValueData::I32Value(60).into(),
3838                        Value {
3839                            value_data: Some(api::v1::value::ValueData::TimestampMillisecondValue(
3840                                0,
3841                            )),
3842                        },
3843                        api::v1::value::ValueData::I32Value(0).into(),
3844                    ],
3845                }],
3846            };
3847            let error = inserter
3848                .submit_table_rows(rows, table_info.clone(), ctx.clone(), &batcher)
3849                .await
3850                .unwrap_err();
3851            assert_eq!(error.status_code(), StatusCode::RateLimited);
3852            let mut attempts = meter.attempts.lock().unwrap();
3853            assert_eq!(attempts.len(), 1);
3854            assert_eq!(attempts[0].catalog, CATALOG);
3855            assert_eq!(attempts[0].schema, "target_db");
3856            assert_eq!(attempts[0].rows, 1);
3857            attempts.clear();
3858        }
3859
3860        let table = Arc::new(table::Table::new(
3861            table_info.clone(),
3862            table::metadata::FilterPushDownType::Unsupported,
3863            Arc::new(DummyDataSource),
3864        ));
3865        let batch = RecordBatch::try_new(
3866            table_info.meta.schema.arrow_schema().clone(),
3867            vec![
3868                Arc::new(Int32Array::from(vec![60, 70])),
3869                Arc::new(TimestampMillisecondArray::from(vec![0, 1])),
3870                Arc::new(Int32Array::from(vec![0, 0])),
3871            ],
3872        )
3873        .unwrap();
3874        let bulk_insert = |batch: RecordBatch| {
3875            let flight_data = FlightEncoder::default()
3876                .encode(FlightMessage::RecordBatch(batch.clone()))
3877                .into_iter()
3878                .next()
3879                .unwrap();
3880            inserter.handle_bulk_insert(
3881                table.clone(),
3882                flight_data,
3883                batch,
3884                Bytes::new(),
3885                false,
3886                Channel::Grpc,
3887            )
3888        };
3889        // Empty batches bypass admission even while the collector rejects.
3890        assert_eq!(bulk_insert(batch.slice(0, 0)).await.unwrap(), 0);
3891        assert!(meter.attempts.lock().unwrap().is_empty());
3892        assert!(dispatched.try_recv().is_err());
3893
3894        // A collector change between batches takes effect on the very next batch.
3895        for reject in [false, true, false] {
3896            meter.reject.store(reject, Ordering::Relaxed);
3897            let result = bulk_insert(batch.clone()).await;
3898            if enabled && reject {
3899                let error = result.unwrap_err();
3900                assert_eq!(error.status_code(), StatusCode::RateLimited);
3901                assert_eq!(error.retry_hint(), RetryHint::Retryable);
3902                assert!(dispatched.try_recv().is_err());
3903            } else {
3904                assert_eq!(result.unwrap(), 2);
3905                dispatched.try_recv().unwrap();
3906            }
3907        }
3908        {
3909            let attempts = meter.attempts.lock().unwrap();
3910            assert_eq!(attempts.len(), if enabled { 3 } else { 0 });
3911            for record in attempts.iter() {
3912                assert_eq!(record.catalog, CATALOG);
3913                assert_eq!(record.schema, "target_db");
3914                assert_eq!(
3915                    (record.rows, record.value, record.source),
3916                    (2, 0, Channel::Grpc as u8)
3917                );
3918            }
3919        }
3920        assert_eq!(
3921            meter.accepted_value.load(Ordering::Relaxed),
3922            if enabled { 17 } else { 0 }
3923        );
3924
3925        // Aggregate repeated database targets and retain admission across nested
3926        // batching without changing the caller's reusable context.
3927        meter.attempts.lock().unwrap().clear();
3928        let original = Arc::new(QueryContext::with_channel(
3929            CATALOG,
3930            "a",
3931            Channel::Prometheus,
3932        ));
3933        let mut batches = ["a", "b", "a"].map(|schema| {
3934            let ctx = if schema == "a" {
3935                original.clone()
3936            } else {
3937                Arc::new(QueryContext::with_channel(
3938                    CATALOG,
3939                    schema,
3940                    Channel::Prometheus,
3941                ))
3942            };
3943            (
3944                ctx,
3945                RowInsertRequests {
3946                    inserts: vec![RowInsertRequest {
3947                        table_name: "data".into(),
3948                        rows: Some(Rows {
3949                            schema: vec![],
3950                            rows: vec![api::v1::Row::default(); 2],
3951                        }),
3952                    }],
3953                },
3954            )
3955        });
3956        admit_row_insert_batches(&mut batches).await.unwrap();
3957        admit_row_insert_batches(&mut batches).await.unwrap();
3958        assert_eq!(original.write_rows_to_admit(CATALOG, "a", 4), 4);
3959        for (ctx, _) in &batches {
3960            assert_eq!(
3961                ctx.write_rows_to_admit(CATALOG, &ctx.current_schema(), 2),
3962                0
3963            );
3964            assert_eq!(ctx.channel(), Channel::Prometheus);
3965        }
3966        {
3967            let attempts = meter.attempts.lock().unwrap();
3968            let totals = attempts
3969                .iter()
3970                .map(|r| (r.schema.as_str(), r.rows, r.value))
3971                .collect::<Vec<_>>();
3972            assert_eq!(
3973                totals,
3974                if enabled {
3975                    vec![("a", 4, 0), ("b", 2, 0)]
3976                } else {
3977                    vec![]
3978                }
3979            );
3980        }
3981        meter.attempts.lock().unwrap().clear();
3982        meter.reject.store(false, Ordering::Relaxed);
3983        let ctx = Arc::new(QueryContext::with_channel(
3984            CATALOG,
3985            "logical",
3986            Channel::Otlp,
3987        ));
3988        let admitted = admit_write(2, &ctx).await.unwrap();
3989        let mut requests = RowInsertRequests {
3990            inserts: vec![RowInsertRequest {
3991                table_name: "metric".to_string(),
3992                rows: Some(Rows {
3993                    schema: vec![],
3994                    rows: vec![api::v1::Row::default(); 2],
3995                }),
3996            }],
3997        };
3998        let original = requests.clone();
3999        let cost = Inserter::meter_row_inserts(&mut requests, &admitted)
4000            .await
4001            .unwrap();
4002        assert_eq!(cost, if enabled { 17 } else { 0 });
4003        assert_eq!(requests, original);
4004        let records = meter
4005            .attempts
4006            .lock()
4007            .unwrap()
4008            .iter()
4009            .map(|record| (record.rows, record.value))
4010            .collect::<Vec<_>>();
4011        assert_eq!(
4012            records,
4013            if enabled {
4014                vec![(2, 0), (0, 17)]
4015            } else {
4016                vec![]
4017            }
4018        );
4019        meter.reject.store(true, Ordering::Relaxed);
4020        let result = Inserter::meter_row_inserts(&mut requests, &admitted).await;
4021        if enabled {
4022            assert_eq!(result.unwrap_err().status_code(), StatusCode::RateLimited);
4023        } else {
4024            assert_eq!(result.unwrap(), 0);
4025        }
4026        assert_eq!(requests, original);
4027    }
4028
4029    #[test]
4030    fn test_trace_create_options_copy_compaction_hints() {
4031        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4032        ctx.set_extension("compaction.twcs.time_window", "1h");
4033        ctx.set_extension("compaction.twcs.max_output_file_size", "256MB");
4034        ctx.set_extension("compaction.twcs.remote_compaction", "true");
4035        ctx.set_extension("compaction.unknown", "ignored");
4036        let ctx = Arc::new(ctx);
4037        let mut table_options = Default::default();
4038
4039        fill_table_options_for_create(
4040            &mut table_options,
4041            &AutoCreateTableType::Trace {
4042                alter_existing: true,
4043            },
4044            &ctx,
4045        );
4046
4047        assert_eq!(
4048            Some("twcs"),
4049            table_options.get(COMPACTION_TYPE).map(String::as_str)
4050        );
4051        assert_eq!(
4052            Some("1h"),
4053            table_options
4054                .get("compaction.twcs.time_window")
4055                .map(String::as_str)
4056        );
4057        assert_eq!(
4058            Some("256MB"),
4059            table_options
4060                .get("compaction.twcs.max_output_file_size")
4061                .map(String::as_str)
4062        );
4063        assert_eq!(
4064            Some("true"),
4065            table_options
4066                .get("compaction.twcs.remote_compaction")
4067                .map(String::as_str)
4068        );
4069        assert!(!table_options.contains_key("compaction.unknown"));
4070    }
4071
4072    #[test]
4073    fn test_exemplar_create_options_isolate_metric_context() {
4074        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4075        for (key, value) in [
4076            (APPEND_MODE_KEY, "false"),
4077            (MERGE_MODE_KEY, "last_non_null"),
4078            (PHYSICAL_TABLE_METADATA_KEY, "true"),
4079            (LOGICAL_TABLE_METADATA_KEY, "physical_metrics"),
4080            ("otlp_metric_compat", "prometheus"),
4081            (table::requests::SEMANTIC_SIGNAL_TYPE, "metric"),
4082            (SEMANTIC_PER_TABLE_INDEX_KEY, "{}"),
4083            ("ttl", "7d"),
4084            ("compaction.twcs.time_window", "1h"),
4085        ] {
4086            ctx.set_extension(key, value);
4087        }
4088        let mut options = Default::default();
4089        fill_table_options_for_create(&mut options, &AutoCreateTableType::Exemplar, &Arc::new(ctx));
4090        assert_eq!(
4091            options,
4092            std::collections::HashMap::from([
4093                (APPEND_MODE_KEY.into(), "true".into()),
4094                (
4095                    store_api::metric_engine_consts::PRIMARY_KEY_ENCODING.into(),
4096                    "sparse".into()
4097                ),
4098                ("ttl".into(), "7d".into()),
4099                (COMPACTION_TYPE.into(), "twcs".into()),
4100                ("compaction.twcs.time_window".into(), "1h".into()),
4101            ])
4102        );
4103    }
4104
4105    #[test]
4106    fn test_skip_wal_does_not_change_table_options() {
4107        check_skip_wal_does_not_change_table_options(false);
4108        check_skip_wal_does_not_change_table_options(true);
4109    }
4110
4111    fn check_skip_wal_does_not_change_table_options(skip_wal: bool) {
4112        let ctx = Arc::new(QueryContext::with(
4113            DEFAULT_CATALOG_NAME,
4114            DEFAULT_SCHEMA_NAME,
4115        ));
4116        ctx.set_skip_wal(skip_wal);
4117        let mut options = Default::default();
4118        fill_table_options_for_create(&mut options, &AutoCreateTableType::Physical, &ctx);
4119        assert!(!options.contains_key(session::hints::INSERT_SKIP_WAL_HINT));
4120        assert!(!options.contains_key("skip_wal"));
4121    }
4122
4123    #[test]
4124    fn test_last_non_null_create_options_preserve_default_without_append_mode() {
4125        let ctx = Arc::new(QueryContext::with(
4126            DEFAULT_CATALOG_NAME,
4127            DEFAULT_SCHEMA_NAME,
4128        ));
4129        let mut table_options = Default::default();
4130
4131        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
4132
4133        assert_eq!(
4134            Some("last_non_null"),
4135            table_options.get(MERGE_MODE_KEY).map(String::as_str)
4136        );
4137        assert!(!table_options.contains_key(APPEND_MODE_KEY));
4138    }
4139
4140    #[test]
4141    fn test_fill_table_options_copies_semantic_extensions() {
4142        use table::requests::{
4143            SEMANTIC_METRIC_TYPE, SEMANTIC_PER_TABLE_INDEX_KEY, SEMANTIC_SIGNAL_TYPE,
4144            SEMANTIC_SOURCE, SEMANTIC_SOURCE_VERSION, SIGNAL_TYPE_METRIC, SOURCE_OPENTELEMETRY,
4145        };
4146
4147        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4148        ctx.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC);
4149        ctx.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
4150        ctx.set_extension(SEMANTIC_SOURCE_VERSION, "2.0");
4151        ctx.set_extension(SEMANTIC_METRIC_TYPE, "bogus");
4152        // The internal transport key must NOT be copied into table options.
4153        ctx.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, "{}");
4154        let ctx = Arc::new(ctx);
4155        let mut table_options = Default::default();
4156
4157        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::Physical, &ctx);
4158
4159        assert_eq!(
4160            Some(SIGNAL_TYPE_METRIC),
4161            table_options.get(SEMANTIC_SIGNAL_TYPE).map(String::as_str)
4162        );
4163        assert_eq!(
4164            Some(SOURCE_OPENTELEMETRY),
4165            table_options.get(SEMANTIC_SOURCE).map(String::as_str)
4166        );
4167        assert_eq!(
4168            Some("2.0"),
4169            table_options
4170                .get(SEMANTIC_SOURCE_VERSION)
4171                .map(String::as_str)
4172        );
4173        assert!(!table_options.contains_key(SEMANTIC_METRIC_TYPE));
4174        assert!(!table_options.contains_key(SEMANTIC_PER_TABLE_INDEX_KEY));
4175    }
4176
4177    #[test]
4178    fn test_apply_retention_ttl_hint_copies_only_retention_ttl() {
4179        let mut instant_ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4180        instant_ctx.set_extension(TTL_KEY, "instant");
4181        instant_ctx.set_extension(APPEND_MODE_KEY, "true");
4182        let instant_ctx = Arc::new(instant_ctx);
4183        let mut instant_options = std::collections::HashMap::new();
4184        apply_retention_ttl_hint(&mut instant_options, &instant_ctx);
4185        assert!(instant_options.is_empty());
4186
4187        let mut retention_ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4188        retention_ctx.set_extension(TTL_KEY, "7d");
4189        retention_ctx.set_extension(APPEND_MODE_KEY, "true");
4190        let retention_ctx = Arc::new(retention_ctx);
4191        let mut retention_options = std::collections::HashMap::new();
4192        apply_retention_ttl_hint(&mut retention_options, &retention_ctx);
4193        assert_eq!(
4194            retention_options.get(TTL_KEY).map(String::as_str),
4195            Some("7d")
4196        );
4197        assert!(!retention_options.contains_key(APPEND_MODE_KEY));
4198    }
4199
4200    #[test]
4201    fn test_apply_per_table_semantic_options() {
4202        use table::requests::{
4203            SEMANTIC_METRIC_TYPE, SEMANTIC_METRIC_UNIT, SEMANTIC_PER_TABLE_INDEX_KEY,
4204        };
4205
4206        let index = format!(
4207            r#"{{
4208            "{DEFAULT_SCHEMA_NAME}": {{
4209                "http_requests_total": {{
4210                    "greptime.semantic.metric.type": "counter",
4211                    "greptime.semantic.metric.unit": "By",
4212                    "greptime.semantic.metric.type_BOGUS": "x"
4213                }},
4214                "other_table": {{
4215                    "greptime.semantic.metric.type": "gauge"
4216                }}
4217            }},
4218            "other_schema": {{
4219                "http_requests_total": {{
4220                    "greptime.semantic.metric.type": "gauge"
4221                }}
4222            }}
4223        }}"#
4224        );
4225        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4226        ctx.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, index);
4227        let ctx = Arc::new(ctx);
4228
4229        let index = parse_per_table_semantic_index(&ctx);
4230        assert!(index.is_some());
4231        let index = index.as_ref();
4232
4233        let mut table_options = std::collections::HashMap::new();
4234        apply_per_table_semantic_options(
4235            &mut table_options,
4236            index,
4237            DEFAULT_SCHEMA_NAME,
4238            "http_requests_total",
4239        );
4240        // The write schema's entry applies — not other_schema's `gauge`.
4241        assert_eq!(
4242            table_options.get(SEMANTIC_METRIC_TYPE).map(String::as_str),
4243            Some("counter")
4244        );
4245        assert_eq!(
4246            table_options.get(SEMANTIC_METRIC_UNIT).map(String::as_str),
4247            Some("By")
4248        );
4249        // The unknown key is rejected by the vocabulary check; other tables' keys
4250        // never appear.
4251        assert!(!table_options.contains_key("greptime.semantic.metric.type_BOGUS"));
4252        assert_eq!(table_options.len(), 2);
4253
4254        let mut empty = std::collections::HashMap::new();
4255        apply_per_table_semantic_options(&mut empty, index, DEFAULT_SCHEMA_NAME, "not_in_index");
4256        assert!(empty.is_empty());
4257
4258        // A schema with no entry is a no-op even when the table name matches
4259        // elsewhere.
4260        let mut opts = std::collections::HashMap::new();
4261        apply_per_table_semantic_options(
4262            &mut opts,
4263            index,
4264            "schema_without_entry",
4265            "http_requests_total",
4266        );
4267        assert!(opts.is_empty());
4268
4269        // No extension at all parses to no index (e.g. logs / Prom RW v1).
4270        let bare = Arc::new(QueryContext::with(
4271            DEFAULT_CATALOG_NAME,
4272            DEFAULT_SCHEMA_NAME,
4273        ));
4274        assert!(parse_per_table_semantic_index(&bare).is_none());
4275        let mut opts = std::collections::HashMap::new();
4276        apply_per_table_semantic_options(
4277            &mut opts,
4278            None,
4279            DEFAULT_SCHEMA_NAME,
4280            "http_requests_total",
4281        );
4282        assert!(opts.is_empty());
4283    }
4284
4285    #[test]
4286    fn test_last_non_null_create_options_preserve_default_with_append_mode_false() {
4287        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4288        ctx.set_extension(APPEND_MODE_KEY, "false");
4289        let ctx = Arc::new(ctx);
4290        let mut table_options = Default::default();
4291
4292        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
4293
4294        assert!(!table_options.contains_key(APPEND_MODE_KEY));
4295        assert_eq!(
4296            Some("last_non_null"),
4297            table_options.get(MERGE_MODE_KEY).map(String::as_str)
4298        );
4299    }
4300
4301    #[test]
4302    fn test_last_non_null_create_options_use_configured_merge_mode() {
4303        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4304        ctx.set_extension(MERGE_MODE_KEY, "last_row");
4305        let ctx = Arc::new(ctx);
4306        let mut table_options = Default::default();
4307
4308        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
4309
4310        assert_eq!(
4311            Some("last_row"),
4312            table_options.get(MERGE_MODE_KEY).map(String::as_str)
4313        );
4314        assert!(!table_options.contains_key(APPEND_MODE_KEY));
4315    }
4316
4317    #[test]
4318    fn test_last_non_null_create_options_use_last_row_with_append_mode_true() {
4319        let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME);
4320        ctx.set_extension(APPEND_MODE_KEY, "true");
4321        let ctx = Arc::new(ctx);
4322        let mut table_options = Default::default();
4323
4324        fill_table_options_for_create(&mut table_options, &AutoCreateTableType::LastNonNull, &ctx);
4325
4326        assert_eq!(
4327            Some("true"),
4328            table_options.get(APPEND_MODE_KEY).map(String::as_str)
4329        );
4330        assert_eq!(
4331            Some("last_row"),
4332            table_options.get(MERGE_MODE_KEY).map(String::as_str)
4333        );
4334    }
4335
4336    #[derive(Default)]
4337    struct UnexpectedBatcher {
4338        allow_acquire: bool,
4339    }
4340
4341    #[async_trait::async_trait]
4342    impl PendingRowsBatcher for UnexpectedBatcher {
4343        async fn acquire(&self, _: usize, _: u64) -> Result<crate::batcher::PendingRowsPermit> {
4344            assert!(
4345                self.allow_acquire,
4346                "empty writes must not acquire batch admission"
4347            );
4348            Ok(Arc::new(()))
4349        }
4350
4351        async fn submit(
4352            &self,
4353            _table_info: TableInfoRef,
4354            _batch: arrow::record_batch::RecordBatch,
4355            _ctx: QueryContextRef,
4356            _permit: crate::batcher::PendingRowsPermit,
4357        ) -> Result<Output> {
4358            panic!("writes must not submit a batch")
4359        }
4360    }
4361
4362    async fn batcher_test_inserter() -> Inserter {
4363        let kv_backend = prepare_mocked_backend().await;
4364        Inserter::new(
4365            catalog::memory::MemoryCatalogManager::new(),
4366            create_partition_rule_manager(kv_backend.clone()).await,
4367            Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler)),
4368            Arc::new(new_table_flownode_set_cache(
4369                String::new(),
4370                Cache::new(100),
4371                kv_backend,
4372            )),
4373            true,
4374        )
4375    }
4376
4377    #[tokio::test]
4378    async fn test_logical_batcher_eligibility() {
4379        use catalog::RegisterTableRequest;
4380        use catalog::memory::MemoryCatalogManager;
4381        use common_meta::kv_backend::KvBackendRef;
4382        use common_meta::kv_backend::memory::MemoryKvBackend;
4383        use datatypes::schema::{ColumnDefaultConstraint, SchemaBuilder};
4384        let requests = RowInsertRequests {
4385            inserts: vec![RowInsertRequest {
4386                table_name: "test_table".to_string(),
4387                rows: None,
4388            }],
4389        };
4390        let original =
4391            make_table_ref_with_schema("ts", "value", ConcreteDataType::float64_datatype())
4392                .table_info();
4393        for case in [
4394            "eligible",
4395            "physical",
4396            "ordinary",
4397            "instant",
4398            "disabled",
4399            "hint",
4400            "flow",
4401            "required_tag",
4402            "default",
4403        ] {
4404            let mut info = (*original).clone();
4405            info.meta.engine = METRIC_ENGINE_NAME.to_string();
4406            info.meta.options.extra_options.insert(
4407                LOGICAL_TABLE_METADATA_KEY.to_string(),
4408                "physical".to_string(),
4409            );
4410            let mut ctx = QueryContext::arc().fork();
4411            match case {
4412                "physical" => {
4413                    info.meta
4414                        .options
4415                        .extra_options
4416                        .insert(LOGICAL_TABLE_METADATA_KEY.to_string(), "other".to_string());
4417                }
4418                "ordinary" => info.meta.engine = "mito".to_string(),
4419                "instant" => info.meta.options.ttl = Some(common_time::ttl::TimeToLive::Instant),
4420                "hint" => ctx.set_extension(AUTO_CREATE_TABLE_KEY, "false"),
4421                "required_tag" | "default" => {
4422                    let mut columns = info.meta.schema.column_schemas().to_vec();
4423                    let mut tag = ColumnSchema::new(
4424                        "tag",
4425                        ConcreteDataType::string_datatype(),
4426                        case != "required_tag",
4427                    );
4428                    if case == "default" {
4429                        tag = tag
4430                            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
4431                            .unwrap();
4432                    }
4433                    columns.push(tag);
4434                    info.meta.schema = Arc::new(
4435                        SchemaBuilder::try_from_columns(columns)
4436                            .unwrap()
4437                            .build()
4438                            .unwrap(),
4439                    );
4440                    info.meta.primary_key_indices = vec![2];
4441                }
4442                _ => {}
4443            }
4444            let catalog = MemoryCatalogManager::with_default_setup();
4445            let table = Arc::new(table::Table::new(
4446                Arc::new(info),
4447                table::metadata::FilterPushDownType::Unsupported,
4448                Arc::new(DummyDataSource),
4449            ));
4450            catalog
4451                .register_table_sync(RegisterTableRequest {
4452                    catalog: DEFAULT_CATALOG_NAME.to_string(),
4453                    schema: DEFAULT_SCHEMA_NAME.to_string(),
4454                    table_name: "test_table".to_string(),
4455                    table_id: 1,
4456                    table,
4457                })
4458                .unwrap();
4459            let mut inserter = batcher_test_inserter().await;
4460            inserter.catalog_manager = catalog;
4461            inserter.auto_create_table = case != "disabled";
4462            let kv_backend: KvBackendRef = Arc::new(MemoryKvBackend::default());
4463            inserter.table_flownode_set_cache = Arc::new(new_table_flownode_set_cache(
4464                String::new(),
4465                Cache::new(10),
4466                kv_backend,
4467            ));
4468            if case == "flow" {
4469                cache_test_flow(
4470                    &inserter.table_flownode_set_cache,
4471                    CreateFlow {
4472                        flow_id: 1,
4473                        source_table_ids: vec![1],
4474                        partition_to_peer_mapping: vec![(0, Peer::empty(1))],
4475                    },
4476                )
4477                .await;
4478            }
4479            assert_eq!(
4480                inserter
4481                    .can_batch_metric_rows(&requests, &Arc::new(ctx), "physical")
4482                    .await
4483                    .unwrap(),
4484                case == "eligible",
4485                "{case}"
4486            );
4487        }
4488    }
4489
4490    #[tokio::test]
4491    async fn test_batcher_meter_preserves_request() {
4492        let mut requests = RowInsertRequests {
4493            inserts: vec![RowInsertRequest {
4494                table_name: "sample".to_string(),
4495                rows: Some(Rows {
4496                    schema: vec![],
4497                    rows: vec![api::v1::Row {
4498                        values: vec![Value {
4499                            value_data: Some(api::v1::value::ValueData::F64Value(1.5)),
4500                        }],
4501                    }],
4502                }),
4503            }],
4504        };
4505        let expected = requests.clone();
4506        Inserter::meter_row_inserts(&mut requests, &QueryContext::arc())
4507            .await
4508            .unwrap();
4509        assert_eq!(requests, expected);
4510    }
4511
4512    #[tokio::test]
4513    async fn test_instant_table_bypasses_batcher() {
4514        let batcher: Arc<dyn PendingRowsBatcher> = Arc::new(UnexpectedBatcher::default());
4515        let inserter = batcher_test_inserter()
4516            .await
4517            .with_pending_rows_batcher(Some(batcher));
4518        let mut ctx = session::context::QueryContextBuilder::default().build();
4519        ctx.set_batching_enabled(true);
4520        let ctx = Arc::new(ctx);
4521        let table = make_table_ref_with_schema("ts", "value", ConcreteDataType::float64_datatype())
4522            .table_info();
4523        assert!(inserter.table_batcher(&table, &ctx).is_some());
4524        let mut instant = (*table).clone();
4525        instant.meta.options.ttl = Some(common_time::ttl::TimeToLive::Instant);
4526        assert!(inserter.table_batcher(&Arc::new(instant), &ctx).is_none());
4527    }
4528
4529    #[tokio::test]
4530    async fn test_empty_prepared_rows_skip_batcher() {
4531        let inserter = batcher_test_inserter().await;
4532        let table = make_table_ref_with_schema("ts", "value", ConcreteDataType::float64_datatype())
4533            .table_info();
4534        let batcher: Arc<dyn PendingRowsBatcher> = Arc::new(UnexpectedBatcher::default());
4535        let ctx = QueryContext::arc();
4536        let output = inserter
4537            .submit_table_rows(
4538                Rows {
4539                    schema: vec![],
4540                    rows: vec![],
4541                },
4542                table.clone(),
4543                ctx.clone(),
4544                &batcher,
4545            )
4546            .await
4547            .unwrap();
4548        assert!(matches!(output.data, OutputData::AffectedRows(0)));
4549        let output = inserter
4550            .submit_pending_rows(
4551                RowInsertRequests {
4552                    inserts: vec![RowInsertRequest {
4553                        table_name: table.name.clone(),
4554                        rows: None,
4555                    }],
4556                },
4557                HashMap::from_iter([(table.table_id(), table)]),
4558                ctx,
4559                &batcher,
4560            )
4561            .await
4562            .unwrap();
4563        assert!(matches!(output.data, OutputData::AffectedRows(0)));
4564    }
4565
4566    /// Flownode handler that keeps mirror inserts in flight until the test
4567    /// releases the gate, so the pending mirror budget can be observed
4568    /// deterministically.
4569    #[derive(Clone)]
4570    struct GatedFlownodeHandler {
4571        gate: Arc<tokio::sync::Semaphore>,
4572        affected_rows: u64,
4573    }
4574
4575    #[async_trait::async_trait]
4576    impl MockFlownodeHandler for GatedFlownodeHandler {
4577        async fn handle_inserts(
4578            &self,
4579            _peer: &Peer,
4580            _requests: api::v1::region::InsertRequests,
4581        ) -> common_meta::error::Result<FlowResponse> {
4582            let _permit = self.gate.acquire().await.unwrap();
4583            Ok(FlowResponse {
4584                affected_rows: self.affected_rows,
4585                ..Default::default()
4586            })
4587        }
4588    }
4589
4590    fn flownode_peer() -> Peer {
4591        Peer {
4592            id: 1,
4593            addr: "127.0.0.1:4001".to_string(),
4594        }
4595    }
4596
4597    #[derive(Clone)]
4598    struct RecordingNodeManager {
4599        datanode_dispatch: tokio::sync::mpsc::UnboundedSender<usize>,
4600        flownode_dispatch: tokio::sync::mpsc::UnboundedSender<Peer>,
4601        gates: Arc<std::sync::Mutex<HashMap<u64, Arc<tokio::sync::Semaphore>>>>,
4602        fail_flownode: bool,
4603    }
4604
4605    #[async_trait::async_trait]
4606    impl common_meta::node_manager::DatanodeManager for RecordingNodeManager {
4607        async fn datanode(&self, peer: &Peer) -> common_meta::node_manager::DatanodeRef {
4608            Arc::new(RecordingNode {
4609                peer: peer.clone(),
4610                manager: self.clone(),
4611            })
4612        }
4613    }
4614
4615    #[async_trait::async_trait]
4616    impl common_meta::node_manager::FlownodeManager for RecordingNodeManager {
4617        async fn flownode(&self, peer: &Peer) -> common_meta::node_manager::FlownodeRef {
4618            Arc::new(RecordingNode {
4619                peer: peer.clone(),
4620                manager: self.clone(),
4621            })
4622        }
4623    }
4624
4625    struct RecordingNode {
4626        peer: Peer,
4627        manager: RecordingNodeManager,
4628    }
4629
4630    #[async_trait::async_trait]
4631    impl common_meta::node_manager::Datanode for RecordingNode {
4632        async fn handle_request(
4633            &self,
4634            _request: Arc<api::v1::region::RegionRequest>,
4635        ) -> common_meta::error::Result<api::region::RegionResponse> {
4636            let rows = match _request.body.as_ref() {
4637                Some(api::v1::region::region_request::Body::Inserts(requests)) => requests
4638                    .requests
4639                    .iter()
4640                    .filter_map(|request| request.rows.as_ref())
4641                    .map(|rows| rows.rows.len())
4642                    .sum(),
4643                _ => 0,
4644            };
4645            let _ = self.manager.datanode_dispatch.send(rows);
4646            Ok(api::region::RegionResponse::new(rows))
4647        }
4648
4649        async fn handle_query(
4650            &self,
4651            _request: common_query::request::QueryRequest,
4652        ) -> common_meta::error::Result<common_recordbatch::SendableRecordBatchStream> {
4653            unreachable!()
4654        }
4655    }
4656
4657    #[async_trait::async_trait]
4658    impl common_meta::node_manager::Flownode for RecordingNode {
4659        async fn handle(
4660            &self,
4661            _request: api::v1::flow::FlowRequest,
4662        ) -> common_meta::error::Result<FlowResponse> {
4663            unreachable!()
4664        }
4665
4666        async fn handle_inserts(
4667            &self,
4668            _request: api::v1::region::InsertRequests,
4669        ) -> common_meta::error::Result<FlowResponse> {
4670            let _ = self.manager.flownode_dispatch.send(self.peer.clone());
4671            if self.manager.fail_flownode {
4672                return Err(common_meta::error::UnexpectedSnafu {
4673                    err_msg: "test flownode failure".to_string(),
4674                }
4675                .build());
4676            }
4677            let gate = self
4678                .manager
4679                .gates
4680                .lock()
4681                .unwrap()
4682                .get(&self.peer.id)
4683                .cloned();
4684            if let Some(gate) = gate {
4685                let _permit = gate.acquire().await.unwrap();
4686            }
4687            Ok(FlowResponse::default())
4688        }
4689
4690        async fn handle_mark_window_dirty(
4691            &self,
4692            _request: api::v1::flow::DirtyWindowRequests,
4693        ) -> common_meta::error::Result<FlowResponse> {
4694            unreachable!()
4695        }
4696    }
4697
4698    fn mirror_requests(peer: &Peer, num_rows: usize) -> HashMap<Peer, RegionInsertRequests> {
4699        HashMap::from_iter([(
4700            peer.clone(),
4701            RegionInsertRequests {
4702                requests: vec![RegionInsertRequest {
4703                    region_id: RegionId::new(1, 1).as_u64(),
4704                    rows: Some(Rows {
4705                        schema: vec![],
4706                        rows: vec![api::v1::Row { values: vec![] }; num_rows],
4707                    }),
4708                    ..Default::default()
4709                }],
4710            },
4711        )])
4712    }
4713
4714    /// Serializes the mirror metric tests below: the metrics are process-wide
4715    /// singletons, so these tests would otherwise observe each other's updates
4716    /// when the test binary runs tests in parallel (they are isolated per
4717    /// process only under nextest).
4718    static MIRROR_METRIC_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
4719
4720    async fn lock_mirror_metrics() -> tokio::sync::MutexGuard<'static, ()> {
4721        MIRROR_METRIC_TEST_LOCK.lock().await
4722    }
4723
4724    // The tests below share the process-wide mirror metrics, so they take
4725    // `MIRROR_METRIC_TEST_LOCK` and assert on the difference their own budget
4726    // publishes instead of on absolute values.
4727    #[tokio::test]
4728    async fn flow_mirror_saturation_preserves_instant_and_normal_contracts() {
4729        use common_error::ext::{ErrorExt, RetryHint};
4730        use common_error::status_code::StatusCode;
4731
4732        let _guard = lock_mirror_metrics().await;
4733        let kv_backend = prepare_mocked_backend().await;
4734        let partition_manager = create_partition_rule_manager(kv_backend.clone()).await;
4735        let (datanode_tx, mut datanode_rx) = tokio::sync::mpsc::unbounded_channel();
4736        let (flownode_tx, mut flownode_rx) = tokio::sync::mpsc::unbounded_channel();
4737        let node_manager = Arc::new(RecordingNodeManager {
4738            datanode_dispatch: datanode_tx,
4739            flownode_dispatch: flownode_tx,
4740            gates: Arc::new(std::sync::Mutex::new(HashMap::new())),
4741            fail_flownode: false,
4742        });
4743        let flow_cache = Cache::new(10);
4744        let flow_cache_backend = prepare_mocked_backend().await;
4745        let inserter = Inserter::new(
4746            catalog::memory::MemoryCatalogManager::new(),
4747            partition_manager,
4748            node_manager,
4749            Arc::new(new_table_flownode_set_cache(
4750                String::new(),
4751                flow_cache.clone(),
4752                flow_cache_backend,
4753            )),
4754            true,
4755        );
4756        let peer = flownode_peer();
4757        cache_test_flow(
4758            &inserter.table_flownode_set_cache,
4759            CreateFlow {
4760                flow_id: 1,
4761                source_table_ids: vec![1, 2],
4762                partition_to_peer_mapping: vec![(0, peer)],
4763            },
4764        )
4765        .await;
4766        let mut normal_info = new_test_table_info(1, "normal_table", [1].into_iter());
4767        normal_info.catalog_name = DEFAULT_CATALOG_NAME.to_string();
4768        normal_info.schema_name = DEFAULT_SCHEMA_NAME.to_string();
4769        let mut instant_info = new_test_table_info(2, "instant_table", [1].into_iter());
4770        instant_info.meta.options.ttl = Some(common_time::ttl::TimeToLive::Instant);
4771        let table_infos =
4772            HashMap::from_iter([(1, Arc::new(normal_info)), (2, Arc::new(instant_info))]);
4773        let ctx = Arc::new(QueryContext::with(
4774            DEFAULT_CATALOG_NAME,
4775            DEFAULT_SCHEMA_NAME,
4776        ));
4777        let insert = |table_id, num_rows| RegionInsertRequests {
4778            requests: vec![RegionInsertRequest {
4779                region_id: RegionId::new(table_id, 1).as_u64(),
4780                rows: Some(Rows {
4781                    schema: vec![],
4782                    rows: vec![api::v1::Row::default(); num_rows],
4783                }),
4784                ..Default::default()
4785            }],
4786        };
4787        let make_request = |normal, instant| InstantAndNormalInsertRequests {
4788            normal_requests: if normal == 0 {
4789                RegionInsertRequests::default()
4790            } else {
4791                insert(1, normal)
4792            },
4793            instant_requests: if instant == 0 {
4794                RegionInsertRequests::default()
4795            } else {
4796                insert(2, instant)
4797            },
4798        };
4799
4800        let pending = inserter.mirror_pending_rows.clone();
4801        let gauge_before = crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get();
4802        crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.add(MAX_MIRROR_PENDING_ROWS as i64);
4803        pending.store(MAX_MIRROR_PENDING_ROWS as u64, Ordering::Relaxed);
4804        let dropped_before = crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get();
4805        for request in [make_request(0, 1), make_request(1, 1)] {
4806            let error = inserter
4807                .do_request(request, &table_infos, &ctx)
4808                .await
4809                .unwrap_err();
4810            assert_eq!(error.status_code(), StatusCode::RateLimited);
4811            assert_eq!(error.retry_hint(), RetryHint::Retryable);
4812            assert!(datanode_rx.try_recv().is_err());
4813            assert!(flownode_rx.try_recv().is_err());
4814            assert_eq!(
4815                pending.load(Ordering::Relaxed),
4816                MAX_MIRROR_PENDING_ROWS as u64
4817            );
4818        }
4819        assert_eq!(
4820            crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get(),
4821            dropped_before
4822        );
4823
4824        for pending_before in [0, MAX_MIRROR_PENDING_ROWS as u64] {
4825            if pending_before == 0 {
4826                pending.store(0, Ordering::Relaxed);
4827                crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.sub(MAX_MIRROR_PENDING_ROWS as i64);
4828            }
4829            let gauge_before_rejection = crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get();
4830            for (normal, instant) in [
4831                (0, MAX_MIRROR_PENDING_ROWS + 1),
4832                (MAX_MIRROR_PENDING_ROWS / 2, MAX_MIRROR_PENDING_ROWS / 2 + 1),
4833            ] {
4834                let error = inserter
4835                    .do_request(make_request(normal, instant), &table_infos, &ctx)
4836                    .await
4837                    .unwrap_err();
4838                assert_eq!(error.status_code(), StatusCode::InvalidArguments);
4839                assert_eq!(error.retry_hint(), RetryHint::NonRetryable);
4840                let message = error.to_string();
4841                assert!(message.contains("1000001"), "{message}");
4842                assert!(message.contains("1000000"), "{message}");
4843                assert!(message.contains("reduce the batch size"), "{message}");
4844                assert!(datanode_rx.try_recv().is_err());
4845                assert!(flownode_rx.try_recv().is_err());
4846                assert_eq!(pending.load(Ordering::Relaxed), pending_before);
4847                assert_eq!(
4848                    crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get(),
4849                    gauge_before_rejection
4850                );
4851                assert_eq!(
4852                    crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get(),
4853                    dropped_before
4854                );
4855            }
4856            if pending_before == 0 {
4857                crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.add(MAX_MIRROR_PENDING_ROWS as i64);
4858                pending.store(MAX_MIRROR_PENDING_ROWS as u64, Ordering::Relaxed);
4859            }
4860        }
4861
4862        let result = inserter
4863            .do_request(make_request(1, 0), &table_infos, &ctx)
4864            .await
4865            .unwrap();
4866        assert!(matches!(result.data, OutputData::AffectedRows(1)));
4867        assert_eq!(1, datanode_rx.try_recv().unwrap());
4868        assert!(flownode_rx.try_recv().is_err());
4869        assert_eq!(
4870            crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get(),
4871            dropped_before + 1
4872        );
4873        pending.store(0, Ordering::Relaxed);
4874        crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.sub(MAX_MIRROR_PENDING_ROWS as i64);
4875        assert_eq!(
4876            crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get(),
4877            gauge_before
4878        );
4879    }
4880
4881    #[tokio::test]
4882    async fn flow_mirror_new_counts_regions_and_peer_clones() {
4883        let peer1 = flownode_peer();
4884        let peer2 = Peer {
4885            id: 2,
4886            addr: "127.0.0.1:4002".to_string(),
4887        };
4888        let cache_backend = prepare_mocked_backend().await;
4889        let cache = Arc::new(new_table_flownode_set_cache(
4890            String::new(),
4891            Cache::new(10),
4892            cache_backend,
4893        ));
4894        cache_test_flow(
4895            &cache,
4896            CreateFlow {
4897                flow_id: 1,
4898                source_table_ids: vec![1],
4899                partition_to_peer_mapping: vec![(0, peer1), (1, peer2)],
4900            },
4901        )
4902        .await;
4903        let request = |region_id| RegionInsertRequest {
4904            region_id,
4905            rows: Some(Rows {
4906                schema: vec![],
4907                rows: vec![api::v1::Row { values: vec![] }; 3],
4908            }),
4909            ..Default::default()
4910        };
4911        let requests = [
4912            request(RegionId::new(1, 0).as_u64()),
4913            request(RegionId::new(1, 1).as_u64()),
4914        ];
4915        let task = FlowMirrorTask::new(&cache, requests.iter()).await.unwrap();
4916        assert_eq!(12, task.pending_rows());
4917        assert_eq!(2, task.requests.len());
4918        for mirrored in task.requests.values() {
4919            assert_eq!(mirrored.requests.as_slice(), requests.as_slice());
4920        }
4921    }
4922
4923    #[tokio::test]
4924    async fn flow_mirror_bounds_and_deduplicates_cold_lookups() {
4925        let (started, mut loads) = tokio::sync::mpsc::unbounded_channel();
4926        let cache = Arc::new(common_meta::cache::CacheContainer::new(
4927            "test".into(),
4928            Cache::new(32),
4929            Arc::new(move |&id| {
4930                let (resume, gate) = tokio::sync::oneshot::channel();
4931                started.send((id, resume)).unwrap();
4932                Box::pin(async move {
4933                    gate.await.unwrap();
4934                    Ok(Some(Default::default()))
4935                })
4936            }),
4937            common_meta::instruction::CacheIdent::table_id_key,
4938        ));
4939        let requests: Vec<_> = (1..=12)
4940            .flat_map(|id| {
4941                (0..2).map(move |region| RegionInsertRequest {
4942                    region_id: RegionId::new(id, region).as_u64(),
4943                    ..Default::default()
4944                })
4945            })
4946            .collect();
4947        let mut task = Box::pin(FlowMirrorTask::new(&cache, requests.iter()));
4948        let mut seen = BTreeSet::new();
4949        // Every table occurs twice. Only eight distinct misses may start before one completes.
4950        for expected in [8, 4] {
4951            assert!(futures_util::poll!(&mut task).is_pending());
4952            let mut gates = Vec::new();
4953            while let Ok((id, resume)) = loads.try_recv() {
4954                assert!(seen.insert(id));
4955                gates.push(resume);
4956            }
4957            assert_eq!(gates.len(), expected);
4958            for gate in gates {
4959                gate.send(()).unwrap();
4960            }
4961        }
4962        assert!(task.await.unwrap().requests.is_empty());
4963        assert_eq!(seen.len(), 12);
4964    }
4965
4966    #[tokio::test]
4967    async fn flow_mirror_dropped_when_pending_exceeds_limit() {
4968        let _guard = lock_mirror_metrics().await;
4969        let pending = Arc::new(AtomicU64::new(MAX_MIRROR_PENDING_ROWS as u64));
4970        let dropped_before = crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get();
4971        let gauge_before = crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get();
4972        let num_rows = 100;
4973
4974        let task = FlowMirrorTask {
4975            requests: mirror_requests(&flownode_peer(), num_rows),
4976        };
4977        // The budget is already full, so nothing may be spawned for this batch.
4978        task.detach(
4979            Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler)),
4980            pending.clone(),
4981            false,
4982        )
4983        .unwrap();
4984
4985        assert_eq!(
4986            num_rows as u64,
4987            crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get() - dropped_before
4988        );
4989        // The dropped batch must not leak its reservation.
4990        assert_eq!(
4991            MAX_MIRROR_PENDING_ROWS as u64,
4992            pending.load(Ordering::Relaxed)
4993        );
4994        // The reservation taken before the limit check is released again, so the
4995        // gauge nets out and keeps matching the pending budget.
4996        assert_eq!(
4997            gauge_before,
4998            crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get(),
4999            "a dropped batch must not change the pending gauge"
5000        );
5001
5002        let task = FlowMirrorTask {
5003            requests: mirror_requests(&flownode_peer(), num_rows),
5004        };
5005        let error = task
5006            .detach(
5007                Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler)),
5008                pending.clone(),
5009                true,
5010            )
5011            .err()
5012            .unwrap();
5013        use common_error::ext::{ErrorExt, RetryHint};
5014        use common_error::status_code::StatusCode;
5015        assert_eq!(error.status_code(), StatusCode::RateLimited);
5016        assert_eq!(error.retry_hint(), RetryHint::Retryable);
5017        assert_eq!(
5018            crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get(),
5019            dropped_before + num_rows as u64,
5020            "rejected instant rows are not successful best-effort drops"
5021        );
5022        assert_eq!(
5023            MAX_MIRROR_PENDING_ROWS as u64,
5024            pending.load(Ordering::Relaxed)
5025        );
5026        assert_eq!(
5027            gauge_before,
5028            crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get()
5029        );
5030    }
5031
5032    /// A source table mapped to several flownodes clones its requests per peer,
5033    /// so the reservation must cover every clone and each spawned task must hand
5034    /// back exactly its own share; otherwise the shared budget drifts.
5035    #[tokio::test]
5036    async fn flow_mirror_reserves_and_releases_per_peer_shares() {
5037        let _guard = lock_mirror_metrics().await;
5038        let per_peer_rows = 10;
5039        let total_rows = (2 * per_peer_rows) as u64;
5040        // Exactly enough room for the whole batch, cloned rows included.
5041        let start = MAX_MIRROR_PENDING_ROWS as u64 - total_rows;
5042        let pending = Arc::new(AtomicU64::new(start));
5043        let gate = Arc::new(tokio::sync::Semaphore::new(0));
5044        let node_manager = Arc::new(MockFlownodeManager::new(GatedFlownodeHandler {
5045            gate: gate.clone(),
5046            affected_rows: per_peer_rows as u64,
5047        }));
5048
5049        let mut requests = mirror_requests(&flownode_peer(), per_peer_rows);
5050        requests.extend(mirror_requests(
5051            &Peer {
5052                id: 2,
5053                addr: "127.0.0.1:4002".to_string(),
5054            },
5055            per_peer_rows,
5056        ));
5057
5058        let gauge_before = crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get();
5059        FlowMirrorTask { requests }
5060            .detach(node_manager, pending.clone(), false)
5061            .unwrap();
5062
5063        // Both per-peer payloads are counted, so the budget is exactly full
5064        // instead of over-reserved by the duplicated rows.
5065        assert_eq!(
5066            MAX_MIRROR_PENDING_ROWS as u64,
5067            pending.load(Ordering::Relaxed)
5068        );
5069
5070        // Let both tasks finish and check that the budget lands back on its
5071        // starting value: over-releasing a shared reservation would saturate it.
5072        // The gauge is checked too because `release_mirror_pending_rows` moves
5073        // the budget before the gauge; exiting on the budget alone could leak a
5074        // pending `gauge.sub` into the next test holding the metrics lock.
5075        gate.add_permits(2);
5076        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
5077        while pending.load(Ordering::Relaxed) != start
5078            || crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get() != gauge_before
5079        {
5080            assert!(
5081                std::time::Instant::now() < deadline,
5082                "the mirror tasks must release exactly their own shares"
5083            );
5084            std::thread::sleep(std::time::Duration::from_millis(1));
5085        }
5086    }
5087
5088    #[tokio::test]
5089    async fn flow_mirror_fast_peer_releases_only_its_share_before_drop() {
5090        let _guard = lock_mirror_metrics().await;
5091        let peer1 = flownode_peer();
5092        let peer2 = Peer {
5093            id: 2,
5094            addr: "127.0.0.1:4002".to_string(),
5095        };
5096        let gate1 = Arc::new(tokio::sync::Semaphore::new(0));
5097        let gate2 = Arc::new(tokio::sync::Semaphore::new(0));
5098        let gates = Arc::new(std::sync::Mutex::new(HashMap::from_iter([
5099            (peer1.id, gate1.clone()),
5100            (peer2.id, gate2.clone()),
5101        ])));
5102        let (datanode_tx, _datanode_rx) = tokio::sync::mpsc::unbounded_channel();
5103        let (flownode_tx, mut flownode_rx) = tokio::sync::mpsc::unbounded_channel();
5104        let node_manager = Arc::new(RecordingNodeManager {
5105            datanode_dispatch: datanode_tx,
5106            flownode_dispatch: flownode_tx,
5107            gates,
5108            fail_flownode: false,
5109        });
5110        let pending = Arc::new(AtomicU64::new(MAX_MIRROR_PENDING_ROWS as u64 - 20));
5111        let gauge_before = crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get();
5112        let mut requests = mirror_requests(&peer1, 10);
5113        requests.extend(mirror_requests(&peer2, 10));
5114        let task = FlowMirrorTask { requests };
5115        task.detach(node_manager.clone(), pending.clone(), false)
5116            .unwrap();
5117        let first = tokio::time::timeout(Duration::from_secs(10), flownode_rx.recv())
5118            .await
5119            .unwrap()
5120            .unwrap();
5121        let second = tokio::time::timeout(Duration::from_secs(10), flownode_rx.recv())
5122            .await
5123            .unwrap()
5124            .unwrap();
5125        assert_ne!(first.id, second.id);
5126        let fast_gate = if first.id == peer1.id {
5127            gate1.clone()
5128        } else {
5129            gate2.clone()
5130        };
5131        let slow_gate = if first.id == peer1.id {
5132            gate2.clone()
5133        } else {
5134            gate1.clone()
5135        };
5136        fast_gate.add_permits(1);
5137        let deadline = std::time::Instant::now() + Duration::from_secs(10);
5138        while pending.load(Ordering::Relaxed) != MAX_MIRROR_PENDING_ROWS as u64 - 10 {
5139            assert!(
5140                std::time::Instant::now() < deadline,
5141                "fast peer did not release"
5142            );
5143            std::thread::sleep(Duration::from_millis(1));
5144        }
5145        let dropped_before = crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get();
5146        FlowMirrorTask {
5147            requests: mirror_requests(&peer1, 11),
5148        }
5149        .detach(node_manager, pending.clone(), false)
5150        .unwrap();
5151        assert_eq!(
5152            crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get(),
5153            dropped_before + 11
5154        );
5155        assert!(flownode_rx.try_recv().is_err());
5156        slow_gate.add_permits(1);
5157        while pending.load(Ordering::Relaxed) != MAX_MIRROR_PENDING_ROWS as u64 - 20
5158            || crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get() != gauge_before
5159        {
5160            assert!(
5161                std::time::Instant::now() < deadline,
5162                "slow peer did not release"
5163            );
5164            std::thread::sleep(Duration::from_millis(1));
5165        }
5166    }
5167
5168    #[tokio::test]
5169    async fn flow_mirror_failure_releases_budget_and_gauge() {
5170        let _guard = lock_mirror_metrics().await;
5171        let (datanode_tx, _datanode_rx) = tokio::sync::mpsc::unbounded_channel();
5172        let (flownode_tx, mut flownode_rx) = tokio::sync::mpsc::unbounded_channel();
5173        let node_manager = Arc::new(RecordingNodeManager {
5174            datanode_dispatch: datanode_tx,
5175            flownode_dispatch: flownode_tx,
5176            gates: Arc::new(std::sync::Mutex::new(HashMap::new())),
5177            fail_flownode: true,
5178        });
5179        let pending = Arc::new(AtomicU64::new(0));
5180        let gauge_before = crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get();
5181        FlowMirrorTask {
5182            requests: mirror_requests(&flownode_peer(), 5),
5183        }
5184        .detach(node_manager, pending.clone(), false)
5185        .unwrap();
5186        assert_eq!(
5187            tokio::time::timeout(Duration::from_secs(10), flownode_rx.recv())
5188                .await
5189                .unwrap()
5190                .unwrap()
5191                .id,
5192            flownode_peer().id
5193        );
5194        let deadline = std::time::Instant::now() + Duration::from_secs(10);
5195        while pending.load(Ordering::Relaxed) != 0
5196            || crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get() != gauge_before
5197        {
5198            assert!(
5199                std::time::Instant::now() < deadline,
5200                "failed task leaked reservation"
5201            );
5202            std::thread::sleep(Duration::from_millis(1));
5203        }
5204    }
5205
5206    #[tokio::test]
5207    async fn flow_mirror_pending_counter_consistent() {
5208        let _guard = lock_mirror_metrics().await;
5209        let pending = Arc::new(AtomicU64::new(0));
5210        let gate = Arc::new(tokio::sync::Semaphore::new(0));
5211        let node_manager = Arc::new(MockFlownodeManager::new(GatedFlownodeHandler {
5212            gate: gate.clone(),
5213            affected_rows: 7,
5214        }));
5215
5216        // A batch without rows short-circuits and leaves the budget untouched.
5217        let gauge_before = crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get();
5218        FlowMirrorTask {
5219            requests: HashMap::new(),
5220        }
5221        .detach(node_manager.clone(), pending.clone(), false)
5222        .unwrap();
5223        assert_eq!(0, pending.load(Ordering::Relaxed));
5224        assert_eq!(
5225            gauge_before,
5226            crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get(),
5227            "an empty batch must not touch the pending gauge"
5228        );
5229
5230        let dropped_before = crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get();
5231        let num_rows = 7;
5232
5233        FlowMirrorTask {
5234            requests: mirror_requests(&flownode_peer(), num_rows),
5235        }
5236        .detach(node_manager, pending.clone(), false)
5237        .unwrap();
5238
5239        // The spawned task is still waiting on the gated flownode, so the
5240        // reservation made by `detach` is the value observable here.
5241        assert_eq!(num_rows as u64, pending.load(Ordering::Relaxed));
5242        assert_eq!(
5243            num_rows as i64,
5244            crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get() - gauge_before,
5245            "the pending gauge must track the pending rows"
5246        );
5247        assert_eq!(
5248            dropped_before,
5249            crate::metrics::DIST_MIRROR_DROPPED_ROW_COUNT.get(),
5250            "a batch within the limit must not be counted as dropped"
5251        );
5252
5253        // Release the in-flight task and wait for it to hand its reservation
5254        // back, so no task is left holding the shared metrics after this test.
5255        gate.add_permits(1);
5256        let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10);
5257        while pending.load(Ordering::Relaxed) != 0
5258            || crate::metrics::DIST_MIRROR_PENDING_ROW_COUNT.get() != gauge_before
5259        {
5260            assert!(
5261                std::time::Instant::now() < deadline,
5262                "the mirror task did not release its pending reservation"
5263            );
5264            std::thread::sleep(std::time::Duration::from_millis(1));
5265        }
5266    }
5267
5268    #[test]
5269    fn mirror_logs_are_rate_limited() {
5270        let log = Arc::new(MirrorLog::new());
5271        assert_eq!(Some(1), log.claim_report(1_000));
5272        assert_eq!(None, log.claim_report(1_500));
5273        assert_eq!(None, log.claim_report(10_999));
5274        assert_eq!(Some(3), log.claim_report(11_000));
5275        assert_eq!(None, log.claim_report(11_001));
5276        assert_eq!(Some(2), log.claim_report(21_000));
5277
5278        let concurrent = Arc::new(MirrorLog::new());
5279        assert_eq!(Some(1), concurrent.claim_report(50_000));
5280        let barrier = Arc::new(std::sync::Barrier::new(8));
5281        let threads = (0..8)
5282            .map(|_| {
5283                let log = concurrent.clone();
5284                let barrier = barrier.clone();
5285                std::thread::spawn(move || {
5286                    barrier.wait();
5287                    log.claim_report(60_000)
5288                })
5289            })
5290            .collect::<Vec<_>>();
5291        let winners = threads
5292            .into_iter()
5293            .filter_map(|thread| thread.join().unwrap())
5294            .collect::<Vec<_>>();
5295        assert_eq!(winners.len(), 1);
5296        let residual = concurrent.events.swap(0, Ordering::Relaxed);
5297        assert_eq!(winners[0] + residual, 8);
5298    }
5299}