Skip to main content

frontend/
instance.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod builder;
16mod copy_in;
17mod dashboard;
18mod entity_graph;
19mod exemplar;
20mod export_database;
21mod grpc;
22mod import_packed;
23mod influxdb;
24mod jaeger;
25mod log_handler;
26mod logical_batcher;
27mod logs;
28mod opentsdb;
29mod otlp;
30pub mod prom_store;
31mod promql;
32mod region_query;
33mod schema_selection;
34
35use std::collections::HashSet;
36use std::pin::Pin;
37use std::sync::atomic::AtomicBool;
38use std::sync::{Arc, OnceLock, atomic};
39use std::time::{Duration, SystemTime};
40
41use async_stream::stream;
42use async_trait::async_trait;
43use auth::{
44    COPY_TO, PROMQL_QUERY, PermissionChecker, PermissionCheckerRef, PermissionReq,
45    PermissionTableTarget, PermissionTableTargets, TableTargetPrecheck,
46};
47use catalog::CatalogManagerRef;
48use catalog::process_manager::{
49    ProcessManagerRef, QueryStatement as CatalogQueryStatement, SlowQueryRecorder, SlowQueryTimer,
50};
51use client::OutputData;
52use common_base::Plugins;
53use common_base::cancellation::CancellableFuture;
54use common_batcher::request_limiter::RequestLimiter;
55use common_error::ext::{BoxedError, ErrorExt};
56use common_event_recorder::EventRecorderRef;
57use common_meta::cache::TableFlownodeSetCacheRef;
58use common_meta::cache_invalidator::CacheInvalidatorRef;
59use common_meta::key::TableMetadataManagerRef;
60use common_meta::node_manager::NodeManagerRef;
61use common_meta::procedure_executor::ProcedureExecutorRef;
62use common_query::Output;
63use common_recordbatch::RecordBatchStreamWrapper;
64use common_recordbatch::error::StreamTimeoutSnafu;
65use common_telemetry::logging::SlowQueryOptions;
66use common_telemetry::{debug, error, tracing};
67use common_time::timestamp::TimeUnit;
68use datafusion::dataframe::DataFrame;
69use datafusion::physical_plan::ExecutionPlan;
70use datafusion_expr::LogicalPlan;
71use futures::TryStreamExt;
72use futures::{Stream, StreamExt, future};
73use operator::delete::DeleterRef;
74use operator::insert::InserterRef;
75use operator::statement::{StatementExecutor, StatementExecutorRef};
76use partition::manager::PartitionRuleManagerRef;
77use pipeline::pipeline_operator::PipelineOperator;
78use prometheus::HistogramTimer;
79use promql_parser::label::{METRIC_NAME, MatchOp, Matcher};
80use promql_parser::util::{ExprVisitor, walk_expr};
81use query::QueryEngineRef;
82use query::metrics::OnDone;
83use query::parser::{PromQuery, QueryStatement};
84use query::query_engine::DescribeResult;
85use query::query_engine::options::{QueryOptions, validate_catalog_and_schema};
86use servers::batcher::logical_table::LogicalTablePendingRowsBatcher;
87use servers::error::{self as server_error, AuthSnafu, ExecuteQuerySnafu, UnexpectedResultSnafu};
88use servers::interceptor::{
89    PromQueryInterceptor, PromQueryInterceptorRef, SqlQueryInterceptor, SqlQueryInterceptorRef,
90};
91use servers::prometheus_handler::{
92    ParsedPromQuery, PrometheusHandler, resolve_schema_from_matchers,
93};
94use servers::query_handler::sql::SqlQueryHandler;
95use servers::request_memory_limiter::ServerMemoryLimiter;
96use session::context::{Channel, MetricDiscoveryScope, QueryContextRef};
97use session::table_name::table_idents_to_full_name;
98use snafu::prelude::*;
99use sql::ast::ObjectNamePartExt;
100use sql::dialect::Dialect;
101use sql::parser::{ParseOptions, ParserContext};
102use sql::statements::comment::CommentObject;
103use sql::statements::copy::{Copy, CopyDatabase, CopyTable};
104use sql::statements::statement::Statement;
105use sql::statements::tql::Tql;
106use sql::util::{
107    extract_tables_from_prom_expr_checked, extract_tables_from_prom_expr_partial,
108    extract_tables_from_statement_checked,
109};
110use sqlparser::ast::{AnalyzeFormat, ObjectName};
111use tracing::Span;
112
113use crate::error::{
114    self, CatalogSnafu, CollectRecordbatchSnafu, Error, ExecLogicalPlanSnafu, ExecutePromqlSnafu,
115    ExternalSnafu, InvalidSqlSnafu, ParseSqlSnafu, PermissionSnafu, PlanStatementSnafu, Result,
116    SqlExecInterceptedSnafu, StatementTimeoutSnafu, TableOperationSnafu,
117};
118use crate::service_config::InfluxdbMergeMode;
119use crate::stream_wrapper::CancellableStreamWrapper;
120
121fn deduplicate_permission_targets(targets: PermissionTableTargets) -> PermissionTableTargets {
122    let PermissionTableTargets::Resolved(mut targets) = targets else {
123        return PermissionTableTargets::Unresolved;
124    };
125    if targets.len() > 1 {
126        let mut seen = HashSet::with_capacity(targets.len());
127        targets.retain(|target| seen.insert(target.clone()));
128    }
129    PermissionTableTargets::resolved(targets)
130}
131
132/// The frontend instance contains necessary components, and implements many
133/// traits, like [`servers::query_handler::grpc::GrpcQueryHandler`],
134/// [`servers::query_handler::sql::SqlQueryHandler`], etc.
135#[derive(Clone)]
136pub struct Instance {
137    server_memory_limiter: ServerMemoryLimiter,
138    frontend_peer_addr: String,
139    catalog_manager: CatalogManagerRef,
140    pipeline_operator: Arc<PipelineOperator>,
141    statement_executor: Arc<StatementExecutor>,
142    query_engine: QueryEngineRef,
143    plugins: Plugins,
144    inserter: InserterRef,
145    logical_batcher: Arc<OnceLock<Option<Arc<LogicalTablePendingRowsBatcher>>>>,
146    deleter: DeleterRef,
147    table_metadata_manager: TableMetadataManagerRef,
148    event_recorder: EventRecorderRef,
149    slow_query_recorder: EventRecorderRef,
150    process_manager: ProcessManagerRef,
151    slow_query_options: SlowQueryOptions,
152    influxdb_default_merge_mode: InfluxdbMergeMode,
153    /// Time index unit for physical metric tables created by auto table creation
154    /// through the frontend instance (e.g., batched logical-table flushes).
155    auto_create_physical_table_timestamp_unit: TimeUnit,
156    trace_ingest_chunk_size: usize,
157    trace_aux_cache: otlp::TraceAuxCache,
158    trace_aux_limiter: Option<RequestLimiter>,
159    otlp_resource_info: bool,
160    suspend: Arc<AtomicBool>,
161}
162
163impl Instance {
164    /// Returns the memory pool shared by protocol admission and table-batch conversion.
165    pub fn server_memory_limiter(&self) -> &ServerMemoryLimiter {
166        &self.server_memory_limiter
167    }
168
169    pub fn frontend_peer_addr(&self) -> &str {
170        &self.frontend_peer_addr
171    }
172
173    pub fn catalog_manager(&self) -> &CatalogManagerRef {
174        &self.catalog_manager
175    }
176
177    pub fn query_engine(&self) -> &QueryEngineRef {
178        &self.query_engine
179    }
180
181    pub fn plugins(&self) -> &Plugins {
182        &self.plugins
183    }
184
185    fn check_permission(
186        &self,
187        ctx: &QueryContextRef,
188        req: PermissionReq<'_>,
189    ) -> server_error::Result<()> {
190        self.plugins
191            .get::<PermissionCheckerRef>()
192            .as_ref()
193            .check_permission(ctx.current_user(), req)
194            .context(AuthSnafu)?;
195        Ok(())
196    }
197
198    pub fn statement_executor(&self) -> &StatementExecutorRef {
199        &self.statement_executor
200    }
201
202    pub fn table_metadata_manager(&self) -> &TableMetadataManagerRef {
203        &self.table_metadata_manager
204    }
205
206    pub fn inserter(&self) -> &InserterRef {
207        &self.inserter
208    }
209
210    pub fn process_manager(&self) -> &ProcessManagerRef {
211        &self.process_manager
212    }
213
214    /// Returns the event recorder configured for this frontend instance.
215    pub fn event_recorder(&self) -> EventRecorderRef {
216        self.event_recorder.clone()
217    }
218
219    pub fn node_manager(&self) -> &NodeManagerRef {
220        self.inserter.node_manager()
221    }
222
223    pub fn partition_manager(&self) -> &PartitionRuleManagerRef {
224        self.inserter.partition_manager()
225    }
226
227    pub fn table_flownode_set_cache(&self) -> &TableFlownodeSetCacheRef {
228        self.inserter.table_flownode_set_cache()
229    }
230
231    pub fn cache_invalidator(&self) -> &CacheInvalidatorRef {
232        self.statement_executor.cache_invalidator()
233    }
234
235    pub fn procedure_executor(&self) -> &ProcedureExecutorRef {
236        self.statement_executor.procedure_executor()
237    }
238
239    pub fn suspend_state(&self) -> Arc<AtomicBool> {
240        self.suspend.clone()
241    }
242
243    pub(crate) fn is_suspended(&self) -> bool {
244        self.suspend.load(atomic::Ordering::Relaxed)
245    }
246}
247
248fn parse_stmt(sql: &str, dialect: &(dyn Dialect + Send + Sync)) -> Result<Vec<Statement>> {
249    ParserContext::create_with_dialect(sql, dialect, ParseOptions::default()).context(ParseSqlSnafu)
250}
251
252fn is_explain_analyze_verbose(stmt: &Statement) -> bool {
253    matches!(stmt, Statement::Explain(explain) if explain.analyze && explain.verbose)
254        || matches!(stmt, Statement::Tql(Tql::Analyze(analyze)) if analyze.is_verbose)
255}
256
257fn validate_analyze_stream_statement(stmt: &mut Statement) -> Result<()> {
258    let (is_verbose, format) = match stmt {
259        Statement::Explain(explain) => (explain.analyze && explain.verbose, &mut explain.format),
260        Statement::Tql(Tql::Analyze(analyze)) => (analyze.is_verbose, &mut analyze.format),
261        _ => {
262            return InvalidSqlSnafu {
263                err_msg: "only EXPLAIN ANALYZE VERBOSE or TQL ANALYZE VERBOSE statement is supported",
264            }
265            .fail();
266        }
267    };
268
269    ensure!(
270        is_verbose,
271        InvalidSqlSnafu {
272            err_msg: "statement must be EXPLAIN ANALYZE VERBOSE or TQL ANALYZE VERBOSE"
273        }
274    );
275    match format {
276        None | Some(AnalyzeFormat::JSON) => {
277            // Keep explicit FORMAT JSON accepted, but pass JSON through
278            // QueryContext.explain_format instead of the statement to avoid
279            // the planner's current `EXPLAIN VERBOSE with FORMAT` limitation.
280            *format = None;
281            Ok(())
282        }
283        Some(_) => InvalidSqlSnafu {
284            err_msg: "only FORMAT JSON is supported for EXPLAIN ANALYZE VERBOSE or TQL ANALYZE VERBOSE",
285        }
286        .fail(),
287    }
288}
289
290impl Instance {
291    fn statement_slow_query_timer(
292        &self,
293        stmt: &Statement,
294        schema_name: String,
295    ) -> Option<SlowQueryTimer> {
296        if !should_track_statement_process(stmt) || !self.slow_query_options.enable {
297            return None;
298        }
299
300        Some(SlowQueryTimer::new(
301            CatalogQueryStatement::Sql(stmt.clone()),
302            schema_name,
303            self.slow_query_options.threshold,
304            self.slow_query_options.sample_ratio,
305            self.slow_query_options.record_type,
306            self.slow_query_recorder.clone(),
307        ))
308    }
309
310    async fn query_statement(&self, stmt: Statement, query_ctx: QueryContextRef) -> Result<Output> {
311        check_permission(self.plugins.clone(), &stmt, &query_ctx)?;
312        let query_ctx = if matches!(&stmt, Statement::SetVariables(_)) {
313            query_ctx
314        } else {
315            Arc::new(
316                query_ctx
317                    .query_option_snapshot()
318                    .map_err(|error| {
319                        query::error::InvalidQueryContextExtensionSnafu {
320                            reason: error.to_string(),
321                        }
322                        .build()
323                    })
324                    .context(PlanStatementSnafu)?,
325            )
326        };
327
328        let query_interceptor = self.plugins.get::<SqlQueryInterceptorRef<Error>>();
329        let query_interceptor = query_interceptor.as_ref();
330
331        if should_track_statement_process(&stmt) {
332            let catalog_name = query_ctx.current_catalog().to_string();
333            let schema_name = query_ctx.current_schema();
334            let slow_query_timer = self.statement_slow_query_timer(&stmt, schema_name.clone());
335            let timeout_recorder = is_explain_analyze_verbose(&stmt)
336                .then(|| slow_query_timer.as_ref().map(SlowQueryTimer::recorder))
337                .flatten();
338
339            let ticket = self.process_manager.register_query(
340                catalog_name,
341                vec![schema_name],
342                stmt.to_string(),
343                query_ctx.conn_info().to_string(),
344                Some(query_ctx.process_id()),
345                slow_query_timer,
346            );
347
348            let query_fut = self.exec_statement_with_timeout(
349                stmt,
350                query_ctx.clone(),
351                query_interceptor,
352                timeout_recorder,
353            );
354
355            let result = CancellableFuture::new(query_fut, ticket.cancellation_handle.clone())
356                .await
357                .map_err(|_| error::CancelledSnafu.build())?;
358            result.map(|output| {
359                let Output { meta, data } = output;
360                let data = match data {
361                    OutputData::Stream(stream) => {
362                        OutputData::Stream(Box::pin(CancellableStreamWrapper::new(stream, ticket)))
363                    }
364                    other => other,
365                };
366                Output { data, meta }
367            })
368        } else {
369            self.exec_statement_with_timeout(stmt, query_ctx, query_interceptor, None)
370                .await
371        }
372    }
373
374    async fn exec_statement_with_timeout(
375        &self,
376        stmt: Statement,
377        query_ctx: QueryContextRef,
378        query_interceptor: Option<&SqlQueryInterceptorRef<Error>>,
379        timeout_recorder: Option<SlowQueryRecorder>,
380    ) -> Result<Output> {
381        let timeout = derive_timeout(&stmt, &query_ctx);
382        match timeout {
383            Some(timeout) => {
384                let start = tokio::time::Instant::now();
385                let output = tokio::time::timeout(
386                    timeout,
387                    self.exec_statement(stmt, query_ctx, query_interceptor),
388                )
389                .await
390                .map_err(|_| StatementTimeoutSnafu.build())??;
391                let output = map_query_output(output)?;
392                // compute remaining timeout
393                let remaining_timeout = timeout.checked_sub(start.elapsed()).unwrap_or_default();
394                attach_timeout(output, remaining_timeout, timeout_recorder)
395            }
396            None => self
397                .exec_statement(stmt, query_ctx, query_interceptor)
398                .await
399                .and_then(map_query_output),
400        }
401    }
402
403    async fn exec_statement(
404        &self,
405        stmt: Statement,
406        query_ctx: QueryContextRef,
407        query_interceptor: Option<&SqlQueryInterceptorRef<Error>>,
408    ) -> Result<Output> {
409        match stmt {
410            Statement::Query(_) | Statement::Explain(_) | Statement::Delete(_) => {
411                // TODO: remove this when format is supported in datafusion
412                if let Statement::Explain(explain) = &stmt
413                    && let Some(format) = explain.format()
414                {
415                    query_ctx.set_explain_format(format.to_string());
416                }
417
418                self.plan_and_exec_sql(stmt, &query_ctx, query_interceptor)
419                    .await
420            }
421            Statement::Tql(tql) => {
422                self.plan_and_exec_tql(&query_ctx, query_interceptor, tql)
423                    .await
424            }
425            _ => {
426                query_interceptor.pre_execute(Some(&stmt), None, query_ctx.clone())?;
427                if let Statement::SetVariables(set) = &stmt
428                    && query_ctx.channel() == Channel::Postgres
429                    && let Some(schemas) = set.search_path()
430                {
431                    return self.set_search_path(schemas, query_ctx).await;
432                }
433                if let Statement::Use(schema) = &stmt {
434                    self.authorize_database(query_ctx.current_catalog(), schema, &query_ctx)
435                        .await?;
436                }
437                if let Statement::Copy(sql::statements::copy::Copy::CopyDatabase(
438                    CopyDatabase::From(arg),
439                )) = &stmt
440                    && arg.with.get("metric_data_layout").is_some()
441                {
442                    return self
443                        .copy_packed_database(arg.clone(), &stmt, query_ctx)
444                        .await;
445                }
446                if let Statement::Copy(sql::statements::copy::Copy::CopyDatabase(CopyDatabase::To(
447                    arg,
448                ))) = &stmt
449                    && export_database::parse_metric_export_requested(&arg.with)?
450                {
451                    return self.copy_metric_database(arg.clone(), query_ctx).await;
452                }
453                if let Statement::Copy(sql::statements::copy::Copy::CopyDatabase(copy)) = &stmt {
454                    return self.copy_database(copy.clone(), &stmt, query_ctx).await;
455                }
456                self.statement_executor
457                    .execute_sql(stmt, query_ctx)
458                    .await
459                    .context(TableOperationSnafu)
460            }
461        }
462    }
463
464    async fn plan_and_exec_sql(
465        &self,
466        stmt: Statement,
467        query_ctx: &QueryContextRef,
468        query_interceptor: Option<&SqlQueryInterceptorRef<Error>>,
469    ) -> Result<Output> {
470        let stmt = QueryStatement::Sql(stmt);
471        let plan = self
472            .statement_executor
473            .plan(&stmt, query_ctx.clone())
474            .await?;
475        let QueryStatement::Sql(stmt) = stmt else {
476            unreachable!()
477        };
478        query_interceptor.pre_execute(Some(&stmt), Some(&plan), query_ctx.clone())?;
479
480        self.statement_executor
481            .exec_plan(plan, query_ctx.clone())
482            .await
483            .context(TableOperationSnafu)
484    }
485
486    async fn plan_and_exec_tql(
487        &self,
488        query_ctx: &QueryContextRef,
489        query_interceptor: Option<&SqlQueryInterceptorRef<Error>>,
490        tql: Tql,
491    ) -> Result<Output> {
492        let plan = self
493            .statement_executor
494            .plan_tql(tql.clone(), query_ctx)
495            .await?;
496        query_interceptor.pre_execute(
497            Some(&Statement::Tql(tql)),
498            Some(&plan),
499            query_ctx.clone(),
500        )?;
501        self.statement_executor
502            .exec_plan(plan, query_ctx.clone())
503            .await
504            .context(TableOperationSnafu)
505    }
506}
507
508/// If the relevant variables are set, the timeout is enforced for all PostgreSQL statements.
509/// For MySQL, it applies only to read-only statements.
510fn derive_timeout(stmt: &Statement, query_ctx: &QueryContextRef) -> Option<Duration> {
511    let query_timeout = query_ctx.query_timeout()?;
512    if query_timeout.is_zero() {
513        return None;
514    }
515    match query_ctx.channel() {
516        Channel::Mysql if stmt.is_readonly() => Some(query_timeout),
517        Channel::Postgres => Some(query_timeout),
518        _ => None,
519    }
520}
521
522/// Derives timeout for plan execution.
523fn derive_timeout_for_plan(plan: &LogicalPlan, query_ctx: &QueryContextRef) -> Option<Duration> {
524    let query_timeout = query_ctx.query_timeout()?;
525    if query_timeout.is_zero() {
526        return None;
527    }
528    match query_ctx.channel() {
529        Channel::Mysql if is_readonly_plan(plan) => Some(query_timeout),
530        Channel::Postgres => Some(query_timeout),
531        _ => None,
532    }
533}
534
535fn record_explain_analyze_timeout(
536    recorder: Option<&SlowQueryRecorder>,
537    plan: Option<&Arc<dyn ExecutionPlan>>,
538) {
539    let Some(recorder) = recorder else {
540        return;
541    };
542    let metrics = plan
543        .and_then(|plan| query::analyze_plan_metrics_to_json_value(plan, true).ok())
544        .unwrap_or_else(|| serde_json::json!([]));
545    recorder.force_record_with_payload(serde_json::json!({
546        "timed_out": true,
547        "metrics": metrics,
548    }));
549}
550
551fn attach_timeout(
552    output: Output,
553    mut timeout: Duration,
554    timeout_recorder: Option<SlowQueryRecorder>,
555) -> Result<Output> {
556    if timeout.is_zero() {
557        return StatementTimeoutSnafu.fail();
558    }
559
560    let plan = timeout_recorder
561        .as_ref()
562        .and_then(|_| output.meta.plan.clone());
563    let output = match output.data {
564        OutputData::AffectedRows(_) | OutputData::RecordBatches(_) => output,
565        OutputData::Stream(mut stream) => {
566            let schema = stream.schema();
567            let s = Box::pin(stream! {
568                let mut start = tokio::time::Instant::now();
569                while let Some(item) = tokio::time::timeout(timeout, stream.next()).await.map_err(|_| {
570                    record_explain_analyze_timeout(timeout_recorder.as_ref(), plan.as_ref());
571                    StreamTimeoutSnafu.build()
572                })? {
573                    yield item;
574
575                    let now = tokio::time::Instant::now();
576                    timeout = timeout.checked_sub(now - start).unwrap_or(Duration::ZERO);
577                    start = now;
578                    // tokio::time::timeout may not return an error immediately when timeout is 0.
579                    if timeout.is_zero() {
580                        record_explain_analyze_timeout(timeout_recorder.as_ref(), plan.as_ref());
581                        StreamTimeoutSnafu.fail()?;
582                    }
583                }
584            }) as Pin<Box<dyn Stream<Item = _> + Send>>;
585            let stream = RecordBatchStreamWrapper {
586                schema,
587                stream: s,
588                output_ordering: None,
589                metrics: Default::default(),
590                span: Span::current(),
591            };
592            Output::new(OutputData::Stream(Box::pin(stream)), output.meta)
593        }
594    };
595
596    Ok(output)
597}
598
599impl Instance {
600    async fn copy_database(
601        &self,
602        copy: CopyDatabase,
603        stmt: &Statement,
604        ctx: QueryContextRef,
605    ) -> Result<Output> {
606        match copy {
607            CopyDatabase::To(arg) => {
608                let req = operator::statement::to_copy_database_request(arg, &ctx)?;
609                let tables = self
610                    .statement_executor
611                    .capture_database_export_tables(&req, None, &ctx)
612                    .await?;
613                let targets = PermissionTableTargets::resolved(
614                    tables
615                        .iter()
616                        .map(|table| {
617                            let info = table.table_info();
618                            PermissionTableTarget::new(
619                                &info.catalog_name,
620                                &info.schema_name,
621                                &info.name,
622                            )
623                        })
624                        .collect(),
625                );
626                self.check_table_permission(
627                    &ctx,
628                    PermissionReq::SqlStatement(stmt),
629                    targets.clone(),
630                )
631                .context(PermissionSnafu)?;
632                self.check_table_permission(&ctx, PermissionReq::Action(COPY_TO), targets)
633                    .context(PermissionSnafu)?;
634                self.statement_executor
635                    .copy_database_to_tables(req, tables, ctx)
636                    .await
637                    .context(TableOperationSnafu)
638            }
639            CopyDatabase::From(arg) => {
640                let req = operator::statement::to_copy_database_request(arg, &ctx)?;
641                let requests = self
642                    .statement_executor
643                    .prepare_database_import(&req)
644                    .await?;
645                let targets = PermissionTableTargets::resolved(
646                    requests
647                        .iter()
648                        .map(|request| {
649                            PermissionTableTarget::new(
650                                &request.catalog_name,
651                                &request.schema_name,
652                                &request.table_name,
653                            )
654                        })
655                        .collect(),
656                );
657                self.check_resolved_table_permission(
658                    &ctx,
659                    PermissionReq::SqlStatement(stmt),
660                    targets,
661                )
662                .await?;
663                self.statement_executor
664                    .copy_database_from_requests(req, requests, ctx)
665                    .await
666                    .context(TableOperationSnafu)
667            }
668        }
669    }
670
671    async fn check_sql_permission(
672        &self,
673        stmt: &Statement,
674        query_ctx: &QueryContextRef,
675    ) -> Result<()> {
676        self.check_unselected_mysql_database(stmt, query_ctx)
677            .await?;
678        self.plugins
679            .get::<PermissionCheckerRef>()
680            .as_ref()
681            .check_permission_with_context(
682                query_ctx.current_user(),
683                PermissionReq::SqlStatement(stmt),
684                Some(&query_ctx.current_schema()),
685            )
686            .context(PermissionSnafu)?;
687
688        // Database COPY needs the full table set before table-target authorization.
689        if matches!(stmt, Statement::Copy(Copy::CopyDatabase(_))) {
690            return Ok(());
691        }
692
693        let targets = match extract_tables_from_statement_checked(stmt) {
694            Some(tables) => PermissionTableTargets::resolved(
695                tables
696                    .map(|name| {
697                        table_idents_to_full_name(&name, query_ctx).map(
698                            |(catalog, schema, table)| {
699                                PermissionTableTarget::new(catalog, schema, table)
700                            },
701                        )
702                    })
703                    .collect::<std::result::Result<Vec<_>, _>>()
704                    .map_err(BoxedError::new)
705                    .context(ExternalSnafu)?,
706            ),
707            None => PermissionTableTargets::Unresolved,
708        };
709        let req = PermissionReq::SqlStatement(stmt);
710        if matches!(
711            stmt,
712            Statement::Copy(Copy::CopyTable(CopyTable::To(_)) | Copy::CopyQueryTo(_))
713        ) {
714            // COPY retains its write permission requirement and also reads source data.
715            self.check_resolved_table_permissions(
716                query_ctx,
717                [PermissionReq::Action(COPY_TO), req],
718                targets,
719            )
720            .await
721        } else {
722            self.check_resolved_table_permission(query_ctx, req, targets)
723                .await
724        }
725    }
726
727    #[tracing::instrument(skip_all, name = "SqlQueryHandler::do_analyze_stream_query")]
728    async fn do_analyze_stream_query_inner(
729        &self,
730        query: &str,
731        query_ctx: QueryContextRef,
732    ) -> Result<Output> {
733        ensure!(!self.is_suspended(), error::SuspendedSnafu);
734
735        let query_interceptor_opt = self.plugins.get::<SqlQueryInterceptorRef<Error>>();
736        let query_interceptor = query_interceptor_opt.as_ref();
737        let query = query_interceptor.pre_parsing(query, query_ctx.clone())?;
738        let mut stmts = parse_stmt(query.as_ref(), query_ctx.sql_dialect())
739            .and_then(|stmts| query_interceptor.post_parsing(stmts, query_ctx.clone()))?;
740
741        ensure!(
742            stmts.len() == 1,
743            InvalidSqlSnafu {
744                err_msg: "only a single EXPLAIN ANALYZE VERBOSE or TQL ANALYZE VERBOSE statement is supported"
745            }
746        );
747        let mut stmt = stmts.remove(0);
748        validate_analyze_stream_statement(&mut stmt)?;
749        query_ctx.set_explain_format(AnalyzeFormat::JSON.to_string());
750
751        self.check_sql_permission(&stmt, &query_ctx).await?;
752        check_permission(self.plugins.clone(), &stmt, &query_ctx)?;
753        let catalog_name = query_ctx.current_catalog().to_string();
754        let schema_name = query_ctx.current_schema();
755        let slow_query_timer = self.statement_slow_query_timer(&stmt, schema_name.clone());
756        let ticket = self.process_manager.register_query(
757            catalog_name,
758            vec![schema_name],
759            stmt.to_string(),
760            query_ctx.conn_info().to_string(),
761            Some(query_ctx.process_id()),
762            slow_query_timer,
763        );
764        let query_fut =
765            self.exec_statement_with_timeout(stmt, query_ctx.clone(), query_interceptor, None);
766        let output = CancellableFuture::new(query_fut, ticket.cancellation_handle.clone())
767            .await
768            .map_err(|_| error::CancelledSnafu.build())??;
769        let Output { meta, data } = output;
770        let data = match data {
771            OutputData::Stream(stream) => OutputData::Stream(Box::pin(
772                CancellableStreamWrapper::new_cancel_on_drop(stream, ticket),
773            )),
774            other => other,
775        };
776        query_interceptor.post_execute(Output { data, meta }, query_ctx)
777    }
778
779    async fn batch_create_logical_tables(
780        &self,
781        query: &str,
782        query_ctx: QueryContextRef,
783    ) -> Result<Vec<Output>> {
784        use servers::query_handler::sql::{
785            MAX_LOGICAL_TABLE_DDL_BYTES, MAX_LOGICAL_TABLE_DDL_STATEMENTS,
786        };
787        ensure!(!self.is_suspended(), error::SuspendedSnafu);
788        ensure!(
789            query.len() <= MAX_LOGICAL_TABLE_DDL_BYTES,
790            InvalidSqlSnafu {
791                err_msg: "logical-table DDL exceeds 1 MiB"
792            }
793        );
794        let interceptor = self.plugins.get::<SqlQueryInterceptorRef<Error>>();
795        let interceptor = interceptor.as_ref();
796        let query = interceptor.pre_parsing(query, query_ctx.clone())?;
797        ensure!(
798            query.len() <= MAX_LOGICAL_TABLE_DDL_BYTES,
799            InvalidSqlSnafu {
800                err_msg: "logical-table DDL exceeds 1 MiB"
801            }
802        );
803        let statements = parse_stmt(query.as_ref(), query_ctx.sql_dialect()).map_err(|_| {
804            InvalidSqlSnafu {
805                err_msg: "invalid logical-table DDL",
806            }
807            .build()
808        })?;
809        ensure!(
810            !statements.is_empty() && statements.len() <= MAX_LOGICAL_TABLE_DDL_STATEMENTS,
811            InvalidSqlSnafu {
812                err_msg: "logical-table batch requires 1 to 128 statements"
813            }
814        );
815        let original_statements = interceptor.map(|_| statements.clone());
816        let statements = interceptor.post_parsing(statements, query_ctx.clone())?;
817        ensure!(
818            !statements.is_empty() && statements.len() <= MAX_LOGICAL_TABLE_DDL_STATEMENTS,
819            InvalidSqlSnafu {
820                err_msg: "logical-table batch requires 1 to 128 statements"
821            }
822        );
823        let measure_rewritten_sql = || {
824            let mut remaining = MAX_LOGICAL_TABLE_DDL_BYTES - (statements.len() - 1);
825            statements.iter().all(|statement| {
826                matches!(statement, Statement::CreateTable(create) if create.consume_sql_size_budget(&mut remaining))
827            })
828        };
829        ensure!(
830            original_statements
831                .as_ref()
832                .is_none_or(|original| original == &statements)
833                || measure_rewritten_sql(),
834            InvalidSqlSnafu {
835                err_msg: "invalid or oversized rewritten logical-table DDL"
836            }
837        );
838        for statement in &statements {
839            ensure!(
840                matches!(statement, Statement::CreateTable(_)),
841                InvalidSqlSnafu {
842                    err_msg: "batch requires explicit Metric logical CREATE TABLE statements"
843                }
844            );
845            interceptor.pre_execute(Some(statement), None, query_ctx.clone())?;
846        }
847        for statement in &statements {
848            self.check_sql_permission(statement, &query_ctx).await?;
849            check_permission(self.plugins.clone(), statement, &query_ctx)?;
850        }
851        let count = statements.len();
852        let creates = statements
853            .into_iter()
854            .filter_map(|statement| {
855                if let Statement::CreateTable(create) = statement {
856                    Some(create)
857                } else {
858                    None
859                }
860            })
861            .collect();
862        self.statement_executor
863            .batch_create_logical_tables(creates, query_ctx.clone())
864            .await
865            .context(TableOperationSnafu)?;
866        (0..count)
867            .map(|_| interceptor.post_execute(Output::new_with_affected_rows(0), query_ctx.clone()))
868            .collect()
869    }
870
871    #[tracing::instrument(skip_all, name = "SqlQueryHandler::do_query")]
872    async fn do_query_inner(&self, query: &str, query_ctx: QueryContextRef) -> Vec<Result<Output>> {
873        if self.is_suspended() {
874            return vec![error::SuspendedSnafu {}.fail()];
875        }
876
877        let query_interceptor_opt = self.plugins.get::<SqlQueryInterceptorRef<Error>>();
878        let query_interceptor = query_interceptor_opt.as_ref();
879        let query = match query_interceptor.pre_parsing(query, query_ctx.clone()) {
880            Ok(q) => q,
881            Err(e) => return vec![Err(e)],
882        };
883
884        match parse_stmt(query.as_ref(), query_ctx.sql_dialect())
885            .and_then(|stmts| query_interceptor.post_parsing(stmts, query_ctx.clone()))
886        {
887            Ok(stmts) => {
888                if stmts.is_empty() {
889                    return vec![
890                        InvalidSqlSnafu {
891                            err_msg: "empty statements",
892                        }
893                        .fail(),
894                    ];
895                }
896
897                let mut results = Vec::with_capacity(stmts.len());
898                for stmt in stmts {
899                    if let Err(e) = self.check_sql_permission(&stmt, &query_ctx).await {
900                        results.push(Err(e));
901                        break;
902                    }
903
904                    match self.query_statement(stmt.clone(), query_ctx.clone()).await {
905                        Ok(output) => {
906                            let output_result =
907                                query_interceptor.post_execute(output, query_ctx.clone());
908                            results.push(output_result);
909                        }
910                        Err(e) => {
911                            if e.status_code().should_log_error() {
912                                error!(e; user = query_ctx.current_user().username(), protocol = %query_ctx.channel(),
913                                    catalog = query_ctx.current_catalog(), schema = %query_ctx.current_schema(),
914                                    process_id = query_ctx.process_id(), "Failed to execute query: {stmt}");
915                            } else {
916                                debug!("Failed to execute query: {stmt}, {e}");
917                            }
918                            results.push(Err(e));
919                            break;
920                        }
921                    }
922                }
923                results
924            }
925            Err(e) => {
926                vec![Err(e)]
927            }
928        }
929    }
930
931    async fn exec_plan(&self, plan: LogicalPlan, query_ctx: QueryContextRef) -> Result<Output> {
932        self.query_engine
933            .execute(plan, query_ctx)
934            .await
935            .context(ExecLogicalPlanSnafu)
936    }
937
938    async fn exec_plan_with_timeout(
939        &self,
940        plan: LogicalPlan,
941        query_ctx: QueryContextRef,
942        timeout_recorder: Option<SlowQueryRecorder>,
943    ) -> Result<Output> {
944        let timeout = derive_timeout_for_plan(&plan, &query_ctx);
945        match timeout {
946            Some(timeout) => {
947                let start = tokio::time::Instant::now();
948                let output = tokio::time::timeout(timeout, self.exec_plan(plan, query_ctx))
949                    .await
950                    .map_err(|_| StatementTimeoutSnafu.build())??;
951                let output = map_query_output(output)?;
952                let remaining_timeout = timeout.checked_sub(start.elapsed()).unwrap_or_default();
953                attach_timeout(output, remaining_timeout, timeout_recorder)
954            }
955            None => self
956                .exec_plan(plan, query_ctx)
957                .await
958                .and_then(map_query_output),
959        }
960    }
961
962    async fn do_exec_plan_inner(
963        &self,
964        plan: LogicalPlan,
965        stmt: Option<Statement>,
966        query_ctx: QueryContextRef,
967    ) -> Result<Output> {
968        ensure!(!self.is_suspended(), error::SuspendedSnafu);
969        if let Some(stmt) = &stmt {
970            self.check_unselected_mysql_database(stmt, &query_ctx)
971                .await?;
972        } else if query_ctx.channel() == Channel::Mysql && query_ctx.current_schema().is_empty() {
973            return Err(BoxedError::new(
974                session::SessionError::NoDatabaseSelected {},
975            ))
976            .context(ExternalSnafu);
977        }
978
979        let query_interceptor_opt = self.plugins.get::<SqlQueryInterceptorRef<Error>>();
980        let query_interceptor = query_interceptor_opt.as_ref();
981
982        query_interceptor.pre_execute(stmt.as_ref(), Some(&plan), query_ctx.clone())?;
983
984        // TQL EXPLAIN/ANALYZE formats are consumed from the query context at
985        // execution time (see `optimize_physical_plan`); re-apply the side
986        // effect of `plan_tql` that was lost when the plan was built during
987        // Describe. `explain_format` is per-query state, so this never
988        // overwrites anything.
989        if let Some(Statement::Tql(tql)) = &stmt {
990            let format = match tql {
991                Tql::Explain(explain) => explain.format.as_ref(),
992                Tql::Analyze(analyze) => analyze.format.as_ref(),
993                Tql::Eval(_) => None,
994            };
995            if let Some(format) = format {
996                query_ctx.set_explain_format(format.to_string());
997            }
998        }
999
1000        let query = stmt
1001            .as_ref()
1002            .map(|s| s.to_string())
1003            .unwrap_or_else(|| plan.display_indent().to_string());
1004
1005        let result = if should_track_plan_process(stmt.as_ref(), &plan) {
1006            let catalog_name = query_ctx.current_catalog().to_string();
1007            let schema_name = query_ctx.current_schema();
1008            let slow_query_timer = self.slow_query_options.enable.then(|| {
1009                SlowQueryTimer::new(
1010                    CatalogQueryStatement::Plan(query.clone()),
1011                    schema_name.clone(),
1012                    self.slow_query_options.threshold,
1013                    self.slow_query_options.sample_ratio,
1014                    self.slow_query_options.record_type,
1015                    self.slow_query_recorder.clone(),
1016                )
1017            });
1018
1019            let timeout_recorder = stmt
1020                .as_ref()
1021                .is_some_and(is_explain_analyze_verbose)
1022                .then(|| slow_query_timer.as_ref().map(SlowQueryTimer::recorder))
1023                .flatten();
1024            let ticket = self.process_manager.register_query(
1025                catalog_name,
1026                vec![schema_name],
1027                query,
1028                query_ctx.conn_info().to_string(),
1029                Some(query_ctx.process_id()),
1030                slow_query_timer,
1031            );
1032
1033            let query_fut = self.exec_plan_with_timeout(plan, query_ctx.clone(), timeout_recorder);
1034
1035            let result = CancellableFuture::new(query_fut, ticket.cancellation_handle.clone())
1036                .await
1037                .map_err(|_| error::CancelledSnafu.build())?;
1038            result.map(|output| {
1039                let Output { meta, data } = output;
1040                let data = match data {
1041                    OutputData::Stream(stream) => {
1042                        OutputData::Stream(Box::pin(CancellableStreamWrapper::new(stream, ticket)))
1043                    }
1044                    other => other,
1045                };
1046                Output { data, meta }
1047            })
1048        } else {
1049            self.exec_plan_with_timeout(plan, query_ctx.clone(), None)
1050                .await
1051        };
1052
1053        result.and_then(|output| query_interceptor.post_execute(output, query_ctx))
1054    }
1055
1056    #[tracing::instrument(skip_all, name = "SqlQueryHandler::do_promql_query")]
1057    async fn do_promql_query_inner(
1058        &self,
1059        query: &PromQuery,
1060        query_ctx: QueryContextRef,
1061    ) -> Vec<Result<Output>> {
1062        if self.is_suspended() {
1063            return vec![error::SuspendedSnafu {}.fail()];
1064        }
1065
1066        // check will be done in prometheus handler's do_query
1067        let result = PrometheusHandler::do_query(self, query, query_ctx)
1068            .await
1069            .with_context(|_| ExecutePromqlSnafu {
1070                query: format!("{query:?}"),
1071            });
1072        vec![result]
1073    }
1074
1075    /// Builds the [`DataFrame`] for an information-schema-backed `SHOW`
1076    /// statement; `None` for other statements. The future is boxed to keep
1077    /// `do_describe_inner`'s state machine small.
1078    fn show_statement_dataframe<'a>(
1079        &'a self,
1080        stmt: &'a Statement,
1081        query_ctx: &'a QueryContextRef,
1082    ) -> Pin<Box<dyn Future<Output = Option<query::error::Result<DataFrame>>> + Send + 'a>> {
1083        Box::pin(async move {
1084            let engine = &self.query_engine;
1085            let catalog_manager = self.catalog_manager();
1086            let ctx = query_ctx.clone();
1087            let dataframe = match stmt {
1088                Statement::ShowDatabases(show) => {
1089                    query::sql::show_databases_dataframe(show, engine, catalog_manager, ctx).await
1090                }
1091                Statement::ShowTables(show) => {
1092                    query::sql::show_tables_dataframe(show, engine, catalog_manager, ctx).await
1093                }
1094                Statement::ShowViews(show) => {
1095                    query::sql::show_views_dataframe(show, engine, catalog_manager, ctx).await
1096                }
1097                Statement::ShowFlows(show) => {
1098                    query::sql::show_flows_dataframe(show, engine, catalog_manager, ctx).await
1099                }
1100                Statement::ShowColumns(show) => {
1101                    query::sql::show_columns_dataframe(show, engine, catalog_manager, ctx).await
1102                }
1103                Statement::ShowTableStatus(show) => {
1104                    query::sql::show_table_status_dataframe(show, engine, catalog_manager, ctx)
1105                        .await
1106                }
1107                Statement::ShowCharset(kind) => {
1108                    query::sql::show_charsets_dataframe(kind, engine, catalog_manager, ctx).await
1109                }
1110                Statement::ShowCollation(kind) => {
1111                    query::sql::show_collations_dataframe(kind, engine, catalog_manager, ctx).await
1112                }
1113                Statement::ShowIndex(show) => {
1114                    query::sql::show_index_dataframe(show, engine, catalog_manager, ctx).await
1115                }
1116                Statement::ShowRegion(show) => {
1117                    query::sql::show_region_dataframe(show, engine, catalog_manager, ctx).await
1118                }
1119                Statement::ShowProcesslist(show) => {
1120                    query::sql::show_processlist_dataframe(show, engine, catalog_manager, ctx).await
1121                }
1122                _ => return None,
1123            };
1124            Some(dataframe)
1125        })
1126    }
1127
1128    async fn do_describe_inner(
1129        &self,
1130        stmt: Statement,
1131        query_ctx: QueryContextRef,
1132    ) -> Result<Option<DescribeResult>> {
1133        ensure!(!self.is_suspended(), error::SuspendedSnafu);
1134        self.check_unselected_mysql_database(&stmt, &query_ctx)
1135            .await?;
1136
1137        // EXPLAIN / EXPLAIN ANALYZE wrap an inner statement; describe them when the
1138        // wrapped statement is something we already plan (so that bind parameters
1139        // in the inner query get their types inferred). See #8029.
1140        let is_inner_plannable = |s: &Statement| {
1141            matches!(
1142                s,
1143                Statement::Insert(_) | Statement::Query(_) | Statement::Delete(_)
1144            )
1145        };
1146        let plannable = is_inner_plannable(&stmt)
1147            || matches!(&stmt, Statement::Explain(explain) if is_inner_plannable(explain.statement.as_ref()));
1148
1149        if let Statement::Tql(tql) = stmt {
1150            // TQL produces a logical plan; describe it from the plan so the
1151            // extended-protocol RowDescription matches the executed DataRows.
1152            self.check_sql_permission(&Statement::Tql(tql.clone()), &query_ctx)
1153                .await?;
1154            let plan = self.statement_executor.plan_tql(tql, &query_ctx).await?;
1155            return self
1156                .query_engine
1157                .describe(plan, query_ctx)
1158                .await
1159                .map(Some)
1160                .context(error::DescribeStatementSnafu);
1161        }
1162
1163        // Describe SHOW statements from the same projection the executor builds.
1164        if let Some(dataframe) = self
1165            .show_statement_dataframe(&stmt, &query_ctx)
1166            .await
1167            .transpose()
1168            .context(PlanStatementSnafu)?
1169        {
1170            self.check_sql_permission(&stmt, &query_ctx).await?;
1171            let plan = dataframe.into_unoptimized_plan();
1172            return self
1173                .query_engine
1174                .describe(plan, query_ctx)
1175                .await
1176                .map(Some)
1177                .context(error::DescribeStatementSnafu);
1178        }
1179
1180        if plannable {
1181            self.check_sql_permission(&stmt, &query_ctx).await?;
1182
1183            let plan = self
1184                .query_engine
1185                .planner()
1186                .plan(&QueryStatement::Sql(stmt), query_ctx.clone())
1187                .await
1188                .context(PlanStatementSnafu)?;
1189            self.query_engine
1190                .describe(plan, query_ctx)
1191                .await
1192                .map(Some)
1193                .context(error::DescribeStatementSnafu)
1194        } else {
1195            Ok(None)
1196        }
1197    }
1198
1199    async fn is_valid_schema_inner(&self, catalog: &str, schema: &str) -> Result<bool> {
1200        self.catalog_manager
1201            .schema_exists(catalog, schema, None)
1202            .await
1203            .context(error::CatalogSnafu)
1204    }
1205}
1206
1207#[async_trait]
1208impl SqlQueryHandler for Instance {
1209    fn supports_metric_batch_ddl(&self) -> bool {
1210        true
1211    }
1212
1213    async fn create_logical_tables(
1214        &self,
1215        query: &str,
1216        query_ctx: QueryContextRef,
1217    ) -> server_error::Result<Vec<Output>> {
1218        self.batch_create_logical_tables(query, query_ctx)
1219            .await
1220            .map_err(BoxedError::new)
1221            .context(ExecuteQuerySnafu)
1222    }
1223
1224    async fn do_query(
1225        &self,
1226        query: &str,
1227        query_ctx: QueryContextRef,
1228    ) -> Vec<server_error::Result<Output>> {
1229        self.do_query_inner(query, query_ctx)
1230            .await
1231            .into_iter()
1232            .map(|result| result.map_err(BoxedError::new).context(ExecuteQuerySnafu))
1233            .collect()
1234    }
1235
1236    async fn do_analyze_stream_query(
1237        &self,
1238        query: &str,
1239        query_ctx: QueryContextRef,
1240    ) -> server_error::Result<Output> {
1241        self.do_analyze_stream_query_inner(query, query_ctx)
1242            .await
1243            .map_err(BoxedError::new)
1244            .context(ExecuteQuerySnafu)
1245    }
1246
1247    async fn do_exec_plan(
1248        &self,
1249        plan: LogicalPlan,
1250        stmt: Option<Statement>,
1251        query_ctx: QueryContextRef,
1252    ) -> server_error::Result<Output> {
1253        // Prepared statements reach here with a plan built before their current execution.
1254        let plan = query::plan::refresh_partition_columns(plan, &self.catalog_manager, &query_ctx)
1255            .await
1256            .map_err(BoxedError::new)
1257            .context(server_error::ExecutePlanSnafu)?;
1258        self.do_exec_plan_inner(plan, stmt, query_ctx)
1259            .await
1260            .map_err(BoxedError::new)
1261            .context(server_error::ExecutePlanSnafu)
1262    }
1263
1264    async fn do_promql_query(
1265        &self,
1266        query: &PromQuery,
1267        query_ctx: QueryContextRef,
1268    ) -> Vec<server_error::Result<Output>> {
1269        self.do_promql_query_inner(query, query_ctx)
1270            .await
1271            .into_iter()
1272            .map(|result| result.map_err(BoxedError::new).context(ExecuteQuerySnafu))
1273            .collect()
1274    }
1275
1276    async fn do_describe(
1277        &self,
1278        stmt: Statement,
1279        query_ctx: QueryContextRef,
1280    ) -> server_error::Result<Option<DescribeResult>> {
1281        self.do_describe_inner(stmt, query_ctx)
1282            .await
1283            .map_err(BoxedError::new)
1284            .context(server_error::DescribeStatementSnafu)
1285    }
1286
1287    async fn is_valid_schema(&self, catalog: &str, schema: &str) -> server_error::Result<bool> {
1288        self.is_valid_schema_inner(catalog, schema)
1289            .await
1290            .map_err(BoxedError::new)
1291            .context(server_error::CheckDatabaseValiditySnafu)
1292    }
1293}
1294
1295/// Expands scan-time dictionaries only when a query result leaves the frontend.
1296pub(crate) fn map_query_output(output: Output) -> Result<Output> {
1297    output
1298        .map_dictionary_to_values()
1299        .context(CollectRecordbatchSnafu)
1300}
1301
1302/// Attaches a timer to the output and observes it once the output is exhausted.
1303pub fn attach_timer(output: Output, timer: HistogramTimer) -> Output {
1304    match output.data {
1305        OutputData::AffectedRows(_) | OutputData::RecordBatches(_) => output,
1306        OutputData::Stream(stream) => {
1307            let stream = OnDone::new(stream, move || {
1308                timer.observe_duration();
1309            });
1310            Output::new(OutputData::Stream(Box::pin(stream)), output.meta)
1311        }
1312    }
1313}
1314
1315impl Instance {
1316    fn check_prom_query_privilege(&self, query_ctx: &QueryContextRef) -> server_error::Result<()> {
1317        self.plugins
1318            .get::<PermissionCheckerRef>()
1319            .as_ref()
1320            .check_permission(
1321                query_ctx.current_user(),
1322                PermissionReq::Action(PROMQL_QUERY),
1323            )
1324            .context(AuthSnafu)?;
1325        Ok(())
1326    }
1327
1328    fn prom_expr_permission_targets(
1329        &self,
1330        expr: &promql_parser::parser::Expr,
1331        query_ctx: &QueryContextRef,
1332    ) -> server_error::Result<Option<Vec<PermissionTableTarget>>> {
1333        extract_tables_from_prom_expr_checked(expr)
1334            .map(|tables| {
1335                tables
1336                    .map(|name| {
1337                        table_idents_to_full_name(&name, query_ctx).map(
1338                            |(catalog, schema, table)| {
1339                                PermissionTableTarget::new(catalog, schema, table)
1340                            },
1341                        )
1342                    })
1343                    .collect::<std::result::Result<Vec<_>, _>>()
1344                    .map_err(BoxedError::new)
1345                    .context(ExecuteQuerySnafu)
1346            })
1347            .transpose()
1348    }
1349
1350    async fn is_physical_query_permission_target(
1351        &self,
1352        target: &PermissionTableTarget,
1353        query_ctx: &QueryContextRef,
1354    ) -> Result<bool> {
1355        self.catalog_manager
1356            .table(
1357                &target.catalog,
1358                &target.schema,
1359                &target.table,
1360                Some(query_ctx),
1361            )
1362            .await
1363            .map(|table| table.is_some_and(|table| table.table_info().is_physical_table()))
1364            .context(CatalogSnafu)
1365    }
1366
1367    pub(crate) async fn check_resolved_table_permission(
1368        &self,
1369        ctx: &QueryContextRef,
1370        req: PermissionReq<'_>,
1371        targets: PermissionTableTargets,
1372    ) -> Result<()> {
1373        self.check_resolved_table_permissions(ctx, [req], targets)
1374            .await
1375    }
1376
1377    async fn check_resolved_table_permissions<const N: usize>(
1378        &self,
1379        ctx: &QueryContextRef,
1380        requests: [PermissionReq<'_>; N],
1381        targets: PermissionTableTargets,
1382    ) -> Result<()> {
1383        let checker = self.plugins.get::<PermissionCheckerRef>();
1384        if checker.is_none() {
1385            return Ok(());
1386        }
1387        let checker = checker.as_ref();
1388        let mut targets = if checker.uses_table_targets() {
1389            deduplicate_permission_targets(targets)
1390        } else {
1391            targets
1392        };
1393
1394        let user = ctx.current_user();
1395        let check = |req, targets| {
1396            checker
1397                .check_permission_with_table_targets(user.clone(), req, targets)
1398                .map(|_| ())
1399                .context(PermissionSnafu)
1400        };
1401        let mut looked_up = HashSet::new();
1402        for (i, req) in requests.into_iter().enumerate() {
1403            match checker
1404                .precheck_table_targets(user.clone(), req.clone(), &targets)
1405                .context(PermissionSnafu)?
1406            {
1407                TableTargetPrecheck::Allow => continue,
1408                TableTargetPrecheck::Reject => {
1409                    unreachable!("Option<&PermissionCheckerRef> maps Reject to PermissionDenied")
1410                }
1411                TableTargetPrecheck::NeedsPhysicalResolution(indices) => {
1412                    self.resolve_physical_permission_targets(
1413                        ctx,
1414                        &mut targets,
1415                        indices,
1416                        &mut looked_up,
1417                    )
1418                    .await?;
1419                }
1420            }
1421
1422            if i + 1 == N {
1423                return check(req, targets);
1424            }
1425            check(req, targets.clone())?;
1426        }
1427        Ok(())
1428    }
1429
1430    async fn resolve_physical_permission_targets(
1431        &self,
1432        ctx: &QueryContextRef,
1433        targets: &mut PermissionTableTargets,
1434        mut indices: Vec<usize>,
1435        looked_up: &mut HashSet<usize>,
1436    ) -> Result<()> {
1437        const CONCURRENCY: usize = 8;
1438
1439        if let PermissionTableTargets::Resolved(resolved) = targets {
1440            if let Some(index) = indices.iter().find(|&&index| index >= resolved.len()) {
1441                return auth::error::InternalStateSnafu {
1442                    msg: format!(
1443                        "permission target precheck returned out-of-bounds index {index} for {} targets",
1444                        resolved.len()
1445                    ),
1446                }
1447                .fail()
1448                .context(PermissionSnafu);
1449            }
1450            indices.retain(|index| looked_up.insert(*index));
1451            // Bound catalog load and inspect results in target order to preserve serial semantics.
1452            for chunk in indices.chunks(CONCURRENCY) {
1453                let results =
1454                    future::join_all(chunk.iter().map(|&index| {
1455                        self.is_physical_query_permission_target(&resolved[index], ctx)
1456                    }))
1457                    .await;
1458                for (&index, result) in chunk.iter().zip(results) {
1459                    if result? {
1460                        resolved[index].is_physical_metric_table = true;
1461                    }
1462                }
1463            }
1464        } else if !indices.is_empty() {
1465            return auth::error::InternalStateSnafu {
1466                msg: "permission target precheck returned indices for unresolved targets"
1467                    .to_string(),
1468            }
1469            .fail()
1470            .context(PermissionSnafu);
1471        }
1472
1473        Ok(())
1474    }
1475
1476    pub(crate) async fn check_server_table_permission(
1477        &self,
1478        ctx: &QueryContextRef,
1479        req: PermissionReq<'_>,
1480        targets: PermissionTableTargets,
1481    ) -> server_error::Result<()> {
1482        match self
1483            .check_resolved_table_permission(ctx, req, targets)
1484            .await
1485        {
1486            Ok(()) => Ok(()),
1487            Err(Error::Permission { source, .. }) => Err(source).context(AuthSnafu),
1488            Err(err) => Err(BoxedError::new(err)).context(ExecuteQuerySnafu),
1489        }
1490    }
1491
1492    fn prom_queries_permission_targets(
1493        &self,
1494        queries: &[ParsedPromQuery],
1495        query_ctx: &QueryContextRef,
1496    ) -> server_error::Result<PermissionTableTargets> {
1497        let mut targets = Vec::new();
1498        let mut resolved = true;
1499
1500        for query in queries {
1501            let QueryStatement::Promql(eval_stmt, _) = query.statement() else {
1502                unreachable!("query is parsed from promql");
1503            };
1504
1505            if let Some(query_targets) =
1506                self.prom_expr_permission_targets(&eval_stmt.expr, query_ctx)?
1507            {
1508                targets.extend(query_targets);
1509            } else {
1510                resolved = false;
1511            }
1512        }
1513
1514        Ok(if resolved {
1515            PermissionTableTargets::resolved(targets)
1516        } else {
1517            PermissionTableTargets::Unresolved
1518        })
1519    }
1520}
1521
1522/// What one PromQL expression needs in order to be answered by discovering metrics.
1523#[derive(Default)]
1524struct MetricDiscoveryVisitor {
1525    needs_discovery: bool,
1526    /// True only while every selector that names no exact table is one discovery can answer. If
1527    /// any selector is not discoverable, the check must stay unresolved rather than be replaced by
1528    /// a set that silently stops naming a table the query reads.
1529    all_unresolved_are_discoverable: bool,
1530    schemas: HashSet<String>,
1531    current_schema: String,
1532    /// Addresses of the label selectors of `info` calls, which the walk reaches after their call.
1533    /// Compared only, never read through; plain integers keep the visitor `Send` across awaits.
1534    info_label_selectors: Vec<usize>,
1535}
1536
1537impl MetricDiscoveryVisitor {
1538    fn new(current_schema: String) -> Self {
1539        Self {
1540            all_unresolved_are_discoverable: true,
1541            current_schema,
1542            ..Default::default()
1543        }
1544    }
1545
1546    /// The planner reads the info table this selector stands for and never discovers with it, so
1547    /// it must not authorize a schema here. A shape the table extractor leaves unresolved keeps
1548    /// the whole expression unresolved.
1549    fn inspect_info_labels(&mut self, labels: &promql_parser::parser::VectorSelector) {
1550        self.info_label_selectors
1551            .push(std::ptr::from_ref(labels) as usize);
1552        let names = labels.matchers.find_matchers(METRIC_NAME);
1553        let exact_name =
1554            labels.name.is_some() || (names.len() == 1 && names[0].op == MatchOp::Equal);
1555        self.all_unresolved_are_discoverable &=
1556            exact_name || sql::util::reads_default_info_metric(labels);
1557    }
1558
1559    /// Whether this selector is answered by enumerating the schema's metric tables.
1560    ///
1561    /// The planner decides the same thing before it scans anything, and the two answers have to
1562    /// match: this walk authorizes what the planner will scan, so a selector the planner declines
1563    /// must not authorize a whole schema here. It is one function in `query`, called from both.
1564    fn is_discovery_selector(selector: &promql_parser::parser::VectorSelector) -> bool {
1565        query::promql::planner::discovers_metric_name(selector)
1566    }
1567
1568    fn inspect_selector(&mut self, selector: &promql_parser::parser::VectorSelector) {
1569        let names = selector.matchers.find_matchers(METRIC_NAME);
1570        let exact_name =
1571            selector.name.is_some() || (names.len() == 1 && names[0].op == MatchOp::Equal);
1572        let discover = Self::is_discovery_selector(selector);
1573        if !exact_name {
1574            self.all_unresolved_are_discoverable &= discover;
1575        }
1576        if !discover {
1577            return;
1578        }
1579        self.needs_discovery = true;
1580        let effective_schema = selector
1581            .matchers
1582            .matchers
1583            .iter()
1584            .rev()
1585            .find(|m| {
1586                (m.name == "__schema__" || m.name == "__database__") && m.op == MatchOp::Equal
1587            })
1588            .map(|m| m.value.clone())
1589            .unwrap_or_else(|| self.current_schema.clone());
1590        self.schemas.insert(effective_schema);
1591    }
1592}
1593
1594impl ExprVisitor for MetricDiscoveryVisitor {
1595    type Error = std::convert::Infallible;
1596
1597    fn pre_visit(
1598        &mut self,
1599        expr: &promql_parser::parser::Expr,
1600    ) -> std::result::Result<bool, Self::Error> {
1601        match expr {
1602            promql_parser::parser::Expr::Call(call) if call.func.name == "info" => {
1603                if let Some(promql_parser::parser::Expr::VectorSelector(labels)) =
1604                    call.args.args.get(1).map(|labels| labels.as_ref())
1605                {
1606                    self.inspect_info_labels(labels);
1607                }
1608            }
1609            promql_parser::parser::Expr::VectorSelector(selector)
1610                if !self
1611                    .info_label_selectors
1612                    .contains(&(std::ptr::from_ref(selector) as usize)) =>
1613            {
1614                self.inspect_selector(selector);
1615            }
1616            promql_parser::parser::Expr::MatrixSelector(selector) => {
1617                self.inspect_selector(&selector.vs);
1618            }
1619            _ => {}
1620        }
1621        Ok(true)
1622    }
1623}
1624
1625impl Instance {
1626    /// Bounds the metric tables this request may discover by the caller's grants, before anything
1627    /// reads the catalog. `Unavailable` would make the planner decline discovery entirely, so this
1628    /// always resolves to something explicit.
1629    async fn prepare_metric_discovery_scope(
1630        &self,
1631        expr: &promql_parser::parser::Expr,
1632        query_ctx: &mut QueryContextRef,
1633    ) -> server_error::Result<()> {
1634        // Discovery scope is read by the native PromQL route only, so a query that does not ask
1635        // for it pays nothing here beyond resolving the switch: the value is read through the
1636        // query engine's own resolver, which takes what the query states (a request hint, or the
1637        // session `SET` it inherits) over the engine default - the same two sources planning
1638        // reads, so the two cannot disagree. A route that is off leaves discovery unrestricted,
1639        // which is what it was before the switch existed; a route that is on is bounded below
1640        // whether or not the fallback option later keeps the plan on the existing route.
1641        let native_requested = query::promql::planner::native_promql_resolved(
1642            query_ctx,
1643            self.query_engine
1644                .engine_state()
1645                .experimental_native_promql(),
1646        )
1647        .map_err(BoxedError::new)
1648        .context(ExecuteQuerySnafu)?;
1649        if !native_requested {
1650            Arc::make_mut(query_ctx).set_metric_discovery_scope(MetricDiscoveryScope::Unrestricted);
1651            return Ok(());
1652        }
1653        let checker = self.plugins.get::<PermissionCheckerRef>();
1654        if !checker.as_ref().uses_table_targets() {
1655            Arc::make_mut(query_ctx).set_metric_discovery_scope(MetricDiscoveryScope::Unrestricted);
1656            return Ok(());
1657        }
1658        let mut visitor = MetricDiscoveryVisitor::new(query_ctx.current_schema());
1659        let _ = walk_expr(&mut visitor, expr);
1660        if !visitor.needs_discovery {
1661            return Ok(());
1662        }
1663        let schemas = visitor.schemas;
1664        let mut allowed = HashSet::new();
1665        for schema in schemas {
1666            let mut tables = self
1667                .catalog_manager
1668                .tables(query_ctx.current_catalog(), &schema, Some(query_ctx))
1669                .try_collect::<Vec<_>>()
1670                .await
1671                .map_err(BoxedError::new)
1672                .context(ExecuteQuerySnafu)?;
1673            tables.sort_by_key(|table| table.table_info().name.clone());
1674            for table in tables {
1675                let info = table.table_info();
1676                if info.meta.engine != store_api::metric_engine_consts::METRIC_ENGINE_NAME
1677                    || info.is_physical_table()
1678                {
1679                    continue;
1680                }
1681                if info.catalog_name != query_ctx.current_catalog() || info.schema_name != schema {
1682                    continue;
1683                }
1684                let target = PermissionTableTarget::new(
1685                    query_ctx.current_catalog(),
1686                    &schema,
1687                    info.name.as_str(),
1688                );
1689                match checker
1690                    .as_ref()
1691                    .check_permission_with_table_targets(
1692                        query_ctx.current_user(),
1693                        PermissionReq::Action(PROMQL_QUERY),
1694                        PermissionTableTargets::resolved(vec![target.clone()]),
1695                    )
1696                    .context(AuthSnafu)
1697                {
1698                    Ok(_) => {
1699                        allowed.insert(target);
1700                    }
1701                    // A rejected metric is skipped rather than failing the query, which is what
1702                    // makes discovery mean "every metric you can see": a table the caller cannot
1703                    // read is one the expression never mentions, the same way a query naming it
1704                    // would be rejected only where it names it. The split route instead rejects a
1705                    // query whose name matchers reach a denied table, so the same regex query
1706                    // answers with the readable metrics here and with `PermissionDenied` there.
1707                    Err(error)
1708                        if error.status_code()
1709                            == common_error::status_code::StatusCode::PermissionDenied => {}
1710                    Err(error) => return Err(error),
1711                }
1712            }
1713        }
1714        Arc::make_mut(query_ctx).set_metric_discovery_scope(MetricDiscoveryScope::Allowed(allowed));
1715        Ok(())
1716    }
1717
1718    /// Fails closed: a path that must have set the scope and did not is an internal error, not a
1719    /// silent return to an unfiltered discovery.
1720    fn validate_metric_discovery_scope(
1721        expr: &promql_parser::parser::Expr,
1722        scope: &MetricDiscoveryScope,
1723    ) -> server_error::Result<()> {
1724        let mut visitor = MetricDiscoveryVisitor::new(String::new());
1725        let _ = walk_expr(&mut visitor, expr);
1726        if visitor.needs_discovery && matches!(scope, MetricDiscoveryScope::Unavailable) {
1727            return Err(server_error::Error::Internal {
1728                err_msg: "Metric discovery reached without an authorization scope".to_string(),
1729            });
1730        }
1731        Ok(())
1732    }
1733
1734    /// The statically named tables of an expression, kept even when another selector cannot be
1735    /// resolved. The complete extractor would answer "no tables at all" for that expression and
1736    /// drop these, which is how a mixed query lost the tables it names explicitly.
1737    fn prom_expr_permission_targets_partial(
1738        &self,
1739        expr: &promql_parser::parser::Expr,
1740        query_ctx: &QueryContextRef,
1741    ) -> server_error::Result<Vec<PermissionTableTarget>> {
1742        extract_tables_from_prom_expr_partial(expr)
1743            .into_iter()
1744            .map(|name| {
1745                table_idents_to_full_name(&name, query_ctx).map(|(catalog, schema, table)| {
1746                    PermissionTableTarget::new(catalog, schema, table)
1747                })
1748            })
1749            .collect::<std::result::Result<Vec<_>, _>>()
1750            .map_err(BoxedError::new)
1751            .context(ExecuteQuerySnafu)
1752    }
1753}
1754
1755#[async_trait]
1756impl PrometheusHandler for Instance {
1757    async fn query_exemplars(
1758        &self,
1759        query: servers::exemplar::query::Query,
1760        ctx: QueryContextRef,
1761    ) -> server_error::Result<Vec<servers::exemplar::query::Series>> {
1762        self.query_exemplars_inner(query, ctx).await
1763    }
1764
1765    #[tracing::instrument(skip_all)]
1766    async fn do_query(
1767        &self,
1768        query: &PromQuery,
1769        query_ctx: QueryContextRef,
1770    ) -> server_error::Result<Output> {
1771        let query = ParsedPromQuery::parse(query.clone(), &query_ctx)?;
1772        self.do_query_parsed(query, query_ctx).await
1773    }
1774
1775    #[tracing::instrument(skip_all)]
1776    async fn do_query_parsed(
1777        &self,
1778        query: ParsedPromQuery,
1779        query_ctx: QueryContextRef,
1780    ) -> server_error::Result<Output> {
1781        let interceptor = self
1782            .plugins
1783            .get::<PromQueryInterceptorRef<server_error::Error>>();
1784
1785        self.check_prom_query_privilege(&query_ctx)?;
1786
1787        let parsed_expr = match query.statement() {
1788            QueryStatement::Promql(eval_stmt, _) => &eval_stmt.expr,
1789            _ => unreachable!("query is parsed from promql"),
1790        };
1791        // Authorization precedes any catalog read for discovery, so a denied name never reaches
1792        // the plan, an error message, or an output column.
1793        let mut scoped_ctx = query_ctx.clone();
1794        self.prepare_metric_discovery_scope(parsed_expr, &mut scoped_ctx)
1795            .await?;
1796        let query_ctx = scoped_ctx;
1797        Self::validate_metric_discovery_scope(parsed_expr, query_ctx.metric_discovery_scope())?;
1798        let mut targets =
1799            self.prom_queries_permission_targets(std::slice::from_ref(&query), &query_ctx)?;
1800        if matches!(targets, PermissionTableTargets::Unresolved) {
1801            let mut visitor = MetricDiscoveryVisitor::new(query_ctx.current_schema());
1802            let _ = walk_expr(&mut visitor, parsed_expr);
1803            if visitor.needs_discovery && visitor.all_unresolved_are_discoverable {
1804                match query_ctx.metric_discovery_scope() {
1805                    MetricDiscoveryScope::Allowed(allowed) => {
1806                        let mut resolved =
1807                            self.prom_expr_permission_targets_partial(parsed_expr, &query_ctx)?;
1808                        resolved.extend(allowed.iter().cloned());
1809                        targets = PermissionTableTargets::resolved(resolved);
1810                    }
1811                    // The partial answer, because a selector that cannot be resolved statically
1812                    // is what brought the query here: the complete one would answer "no tables",
1813                    // dropping the named ones that the same query also reads.
1814                    MetricDiscoveryScope::Unrestricted => {
1815                        targets = PermissionTableTargets::resolved(
1816                            self.prom_expr_permission_targets_partial(parsed_expr, &query_ctx)?,
1817                        );
1818                    }
1819                    MetricDiscoveryScope::Unavailable => {}
1820                }
1821            }
1822        }
1823        self.check_query_target_permission(targets, &query_ctx)
1824            .await?;
1825
1826        let requires_output_ordering = query.requires_output_ordering();
1827        let (query, stmt) = query.into_parts();
1828
1829        let QueryStatement::Promql(eval_stmt, _) = &stmt else {
1830            unreachable!("query is parsed from promql");
1831        };
1832
1833        let plan = self
1834            .statement_executor
1835            .plan(&stmt, query_ctx.clone())
1836            .await
1837            .map_err(BoxedError::new)
1838            .context(ExecuteQuerySnafu)?;
1839
1840        let plan = if requires_output_ordering {
1841            plan
1842        } else {
1843            promql::remove_output_sort(plan)
1844        };
1845
1846        interceptor.pre_execute(&query, &eval_stmt.expr, Some(&plan), query_ctx.clone())?;
1847
1848        // Take the EvalStmt from the original QueryStatement and use it to create the CatalogQueryStatement.
1849        let query_statement = if let QueryStatement::Promql(eval_stmt, alias) = stmt {
1850            CatalogQueryStatement::Promql(eval_stmt, alias)
1851        } else {
1852            // It should not happen since the query is already parsed successfully.
1853            return UnexpectedResultSnafu {
1854                reason: "The query should always be promql.".to_string(),
1855            }
1856            .fail();
1857        };
1858        let raw_query = query_statement.to_string();
1859
1860        let slow_query_timer = self.slow_query_options.enable.then(|| {
1861            SlowQueryTimer::new(
1862                query_statement,
1863                query_ctx.current_schema(),
1864                self.slow_query_options.threshold,
1865                self.slow_query_options.sample_ratio,
1866                self.slow_query_options.record_type,
1867                self.slow_query_recorder.clone(),
1868            )
1869        });
1870
1871        let ticket = self.process_manager.register_query(
1872            query_ctx.current_catalog().to_string(),
1873            vec![query_ctx.current_schema()],
1874            raw_query,
1875            query_ctx.conn_info().to_string(),
1876            Some(query_ctx.process_id()),
1877            slow_query_timer,
1878        );
1879
1880        let query_fut = self.statement_executor.exec_plan(plan, query_ctx.clone());
1881
1882        let output = CancellableFuture::new(query_fut, ticket.cancellation_handle.clone())
1883            .await
1884            .map_err(|_| servers::error::CancelledSnafu.build())?
1885            .map_err(BoxedError::new)
1886            .context(ExecuteQuerySnafu)?;
1887        let output = map_query_output(output)
1888            .map_err(BoxedError::new)
1889            .context(ExecuteQuerySnafu)?;
1890        let Output { meta, data } = output;
1891        let data = match data {
1892            OutputData::Stream(stream) => {
1893                OutputData::Stream(Box::pin(CancellableStreamWrapper::new(stream, ticket)))
1894            }
1895            other => other,
1896        };
1897        let output = Output { data, meta };
1898        Ok(interceptor.post_execute(output, query_ctx)?)
1899    }
1900
1901    async fn check_query_permission(
1902        &self,
1903        queries: &[PromQuery],
1904        query_ctx: &QueryContextRef,
1905    ) -> server_error::Result<()> {
1906        let queries = queries
1907            .iter()
1908            .cloned()
1909            .map(|query| ParsedPromQuery::parse(query, query_ctx))
1910            .collect::<server_error::Result<Vec<_>>>()?;
1911        self.check_query_permission_parsed(&queries, query_ctx)
1912            .await
1913    }
1914
1915    async fn check_query_permission_parsed(
1916        &self,
1917        queries: &[ParsedPromQuery],
1918        query_ctx: &QueryContextRef,
1919    ) -> server_error::Result<()> {
1920        self.check_prom_query_privilege(query_ctx)?;
1921        let targets = self.prom_queries_permission_targets(queries, query_ctx)?;
1922        self.check_query_target_permission(targets, query_ctx).await
1923    }
1924
1925    async fn check_query_target_permission(
1926        &self,
1927        targets: PermissionTableTargets,
1928        query_ctx: &QueryContextRef,
1929    ) -> server_error::Result<()> {
1930        self.check_server_table_permission(query_ctx, PermissionReq::Action(PROMQL_QUERY), targets)
1931            .await
1932    }
1933
1934    async fn filter_metadata_metric_names(
1935        &self,
1936        metric_names: Vec<String>,
1937        schema: &str,
1938        query_ctx: &QueryContextRef,
1939    ) -> server_error::Result<Vec<String>> {
1940        let checker = self.plugins.get::<PermissionCheckerRef>();
1941        if !checker.as_ref().uses_table_targets() {
1942            let Some(metric) = metric_names.first() else {
1943                return Ok(metric_names);
1944            };
1945            let target =
1946                PermissionTableTarget::new(query_ctx.current_catalog(), schema, metric.as_str());
1947            let result = checker
1948                .as_ref()
1949                .check_permission_with_table_targets(
1950                    query_ctx.current_user(),
1951                    PermissionReq::Action(PROMQL_QUERY),
1952                    PermissionTableTargets::resolved(vec![target]),
1953                )
1954                .context(AuthSnafu);
1955            return match result {
1956                Ok(_) => Ok(metric_names),
1957                Err(error)
1958                    if error.status_code()
1959                        == common_error::status_code::StatusCode::PermissionDenied =>
1960                {
1961                    Ok(Vec::new())
1962                }
1963                Err(error) => Err(error),
1964            };
1965        }
1966
1967        let mut allowed = Vec::with_capacity(metric_names.len());
1968        for metric in metric_names {
1969            let target =
1970                PermissionTableTarget::new(query_ctx.current_catalog(), schema, metric.as_str());
1971            match checker
1972                .as_ref()
1973                .check_permission_with_table_targets(
1974                    query_ctx.current_user(),
1975                    PermissionReq::Action(PROMQL_QUERY),
1976                    PermissionTableTargets::resolved(vec![target]),
1977                )
1978                .context(AuthSnafu)
1979            {
1980                Ok(_) => allowed.push(metric),
1981                Err(error)
1982                    if error.status_code()
1983                        == common_error::status_code::StatusCode::PermissionDenied => {}
1984                Err(error) => return Err(error),
1985            }
1986        }
1987        Ok(allowed)
1988    }
1989
1990    async fn filter_metadata_schema_names(
1991        &self,
1992        schema_names: Vec<String>,
1993        query_ctx: &QueryContextRef,
1994    ) -> server_error::Result<Vec<String>> {
1995        let checker = self.plugins.get::<PermissionCheckerRef>();
1996        let mut allowed = Vec::with_capacity(schema_names.len());
1997        for schema in schema_names {
1998            match checker
1999                .as_ref()
2000                .check_permission(
2001                    query_ctx.current_user(),
2002                    PermissionReq::SchemaRead {
2003                        catalog: query_ctx.current_catalog(),
2004                        schema: &schema,
2005                    },
2006                )
2007                .context(AuthSnafu)
2008            {
2009                Ok(_) => allowed.push(schema),
2010                Err(error)
2011                    if error.status_code()
2012                        == common_error::status_code::StatusCode::PermissionDenied => {}
2013                Err(error) => return Err(error),
2014            }
2015        }
2016        Ok(allowed)
2017    }
2018
2019    async fn query_metric_names(
2020        &self,
2021        matchers: Vec<Matcher>,
2022        schema: &str,
2023        ctx: &QueryContextRef,
2024    ) -> server_error::Result<Vec<String>> {
2025        self.handle_query_metric_names(matchers, schema, ctx)
2026            .await
2027            .map_err(BoxedError::new)
2028            .context(ExecuteQuerySnafu)
2029    }
2030
2031    async fn query_metric_names_by_labels(
2032        &self,
2033        matchers: Vec<Matcher>,
2034        schema: &str,
2035        start: SystemTime,
2036        end: SystemTime,
2037        ctx: &QueryContextRef,
2038    ) -> server_error::Result<Vec<String>> {
2039        self.handle_query_metric_names_by_labels(matchers, schema, start, end, ctx)
2040            .await
2041            .map_err(BoxedError::new)
2042            .context(ExecuteQuerySnafu)
2043    }
2044
2045    async fn query_label_values(
2046        &self,
2047        metric: String,
2048        label_name: String,
2049        matchers: Vec<Matcher>,
2050        start: SystemTime,
2051        end: SystemTime,
2052        ctx: &QueryContextRef,
2053    ) -> server_error::Result<Vec<String>> {
2054        let schema =
2055            resolve_schema_from_matchers(&matchers)?.unwrap_or_else(|| ctx.current_schema());
2056        let target = PermissionTableTarget::new(ctx.current_catalog(), schema.as_str(), &metric);
2057        self.check_query_target_permission(
2058            PermissionTableTargets::resolved(vec![target.clone()]),
2059            ctx,
2060        )
2061        .await?;
2062
2063        self.handle_query_label_values(target, label_name, matchers, start, end, ctx)
2064            .await
2065            .map_err(BoxedError::new)
2066            .context(ExecuteQuerySnafu)
2067    }
2068
2069    fn catalog_manager(&self) -> CatalogManagerRef {
2070        self.catalog_manager.clone()
2071    }
2072}
2073
2074/// Validate `stmt.database` permission if it's presented.
2075macro_rules! validate_db_permission {
2076    ($stmt: expr, $query_ctx: expr) => {
2077        if let Some(database) = &$stmt.database {
2078            validate_catalog_and_schema($query_ctx.current_catalog(), database, $query_ctx)
2079                .map_err(BoxedError::new)
2080                .context(SqlExecInterceptedSnafu)?;
2081        }
2082    };
2083}
2084
2085pub fn check_permission(
2086    plugins: Plugins,
2087    stmt: &Statement,
2088    query_ctx: &QueryContextRef,
2089) -> Result<()> {
2090    let need_validate = plugins
2091        .get::<QueryOptions>()
2092        .map(|opts| opts.disallow_cross_catalog_query)
2093        .unwrap_or_default();
2094
2095    if !need_validate {
2096        return Ok(());
2097    }
2098
2099    match stmt {
2100        // Will be checked in execution.
2101        // TODO(dennis): add a hook for admin commands.
2102        Statement::Admin(_) => {}
2103        // These are executed by query engine, and will be checked there.
2104        Statement::Query(_)
2105        | Statement::Explain(_)
2106        | Statement::Tql(_)
2107        | Statement::Delete(_)
2108        | Statement::DeclareCursor(_)
2109        | Statement::Copy(sql::statements::copy::Copy::CopyQueryTo(_)) => {}
2110        // database ops won't be checked
2111        Statement::CreateDatabase(_)
2112        | Statement::ShowDatabases(_)
2113        | Statement::DropDatabase(_)
2114        | Statement::AlterDatabase(_)
2115        | Statement::DropFlow(_)
2116        | Statement::Use(_) => {}
2117        #[cfg(feature = "enterprise")]
2118        Statement::DropTrigger(_) => {}
2119        Statement::ShowCreateDatabase(stmt) => {
2120            validate_database(&stmt.database_name, query_ctx)?;
2121        }
2122        Statement::ShowCreateTable(stmt) => {
2123            validate_param(&stmt.table_name, query_ctx)?;
2124        }
2125        Statement::ShowCreateFlow(stmt) => {
2126            validate_flow(&stmt.flow_name, query_ctx)?;
2127        }
2128        #[cfg(feature = "enterprise")]
2129        Statement::ShowCreateTrigger(stmt) => {
2130            validate_param(&stmt.trigger_name, query_ctx)?;
2131        }
2132        Statement::ShowCreateView(stmt) => {
2133            validate_param(&stmt.view_name, query_ctx)?;
2134        }
2135        Statement::CreateExternalTable(stmt) => {
2136            validate_param(&stmt.name, query_ctx)?;
2137        }
2138        Statement::CreateFlow(stmt) => {
2139            // TODO: should also validate source table name here?
2140            validate_param(&stmt.sink_table_name, query_ctx)?;
2141        }
2142        #[cfg(feature = "enterprise")]
2143        Statement::CreateTrigger(stmt) => {
2144            validate_param(&stmt.trigger_name, query_ctx)?;
2145        }
2146        Statement::CreateView(stmt) => {
2147            validate_param(&stmt.name, query_ctx)?;
2148        }
2149        Statement::AlterTable(stmt) => {
2150            validate_param(stmt.table_name(), query_ctx)?;
2151        }
2152        #[cfg(feature = "enterprise")]
2153        Statement::AlterTrigger(_) => {}
2154        // set/show variable now only alter/show variable in session
2155        Statement::SetVariables(_) | Statement::ShowVariables(_) => {}
2156        // show charset and show collation won't be checked
2157        Statement::ShowCharset(_) | Statement::ShowCollation(_) => {}
2158
2159        Statement::Comment(comment) => match &comment.object {
2160            CommentObject::Table(table) => validate_param(table, query_ctx)?,
2161            CommentObject::Column { table, .. } => validate_param(table, query_ctx)?,
2162            CommentObject::Flow(flow) => validate_flow(flow, query_ctx)?,
2163        },
2164
2165        Statement::Insert(insert) => {
2166            let name = insert.table_name().context(ParseSqlSnafu)?;
2167            validate_param(name, query_ctx)?;
2168        }
2169        Statement::CreateTable(stmt) => {
2170            validate_param(&stmt.name, query_ctx)?;
2171        }
2172        Statement::CreateTableLike(stmt) => {
2173            validate_param(&stmt.table_name, query_ctx)?;
2174            validate_param(&stmt.source_name, query_ctx)?;
2175        }
2176        Statement::DropTable(drop_stmt) => {
2177            for table_name in drop_stmt.table_names() {
2178                validate_param(table_name, query_ctx)?;
2179            }
2180        }
2181        #[cfg(feature = "enterprise")]
2182        Statement::UndropTable(stmt) => {
2183            validate_param(stmt.table_name(), query_ctx)?;
2184        }
2185        Statement::DropView(stmt) => {
2186            validate_param(&stmt.view_name, query_ctx)?;
2187        }
2188        Statement::ShowTables(stmt) => {
2189            validate_db_permission!(stmt, query_ctx);
2190        }
2191        Statement::ShowTableStatus(stmt) => {
2192            validate_db_permission!(stmt, query_ctx);
2193        }
2194        Statement::ShowColumns(stmt) => {
2195            validate_db_permission!(stmt, query_ctx);
2196        }
2197        Statement::ShowIndex(stmt) => {
2198            validate_db_permission!(stmt, query_ctx);
2199        }
2200        Statement::ShowRegion(stmt) => {
2201            validate_db_permission!(stmt, query_ctx);
2202        }
2203        Statement::ShowViews(stmt) => {
2204            validate_db_permission!(stmt, query_ctx);
2205        }
2206        Statement::ShowFlows(stmt) => {
2207            validate_db_permission!(stmt, query_ctx);
2208        }
2209        Statement::ShowFlowStatus(_stmt) => {
2210            // Flow statistics are organized based on the catalog dimension and
2211            // filtered by the current catalog, so there is no need to check the
2212            // permission of the database(schema).
2213        }
2214        #[cfg(feature = "enterprise")]
2215        Statement::ShowTriggers(_stmt) => {
2216            // The trigger is organized based on the catalog dimension, so there
2217            // is no need to check the permission of the database(schema).
2218        }
2219        Statement::ShowStatus(_stmt) => {}
2220        Statement::ShowSearchPath(_stmt) => {}
2221        Statement::DescribeTable(stmt) => {
2222            validate_param(stmt.name(), query_ctx)?;
2223        }
2224        Statement::Copy(sql::statements::copy::Copy::CopyTable(stmt)) => match stmt {
2225            CopyTable::To(copy_table_to) => validate_param(&copy_table_to.table_name, query_ctx)?,
2226            CopyTable::From(copy_table_from) => {
2227                validate_param(&copy_table_from.table_name, query_ctx)?
2228            }
2229        },
2230        Statement::Copy(sql::statements::copy::Copy::CopyDatabase(copy_database)) => {
2231            match copy_database {
2232                CopyDatabase::To(stmt) => validate_database(&stmt.database_name, query_ctx)?,
2233                CopyDatabase::From(stmt) => validate_database(&stmt.database_name, query_ctx)?,
2234            }
2235        }
2236        Statement::TruncateTable(stmt) => {
2237            validate_param(stmt.table_name(), query_ctx)?;
2238        }
2239        // cursor operations are always allowed once it's created
2240        Statement::FetchCursor(_) | Statement::CloseCursor(_) => {}
2241        // User can only kill process in their own catalog.
2242        Statement::Kill(_) => {}
2243        // SHOW PROCESSLIST
2244        Statement::ShowProcesslist(_) => {}
2245    }
2246    Ok(())
2247}
2248
2249fn validate_param(name: &ObjectName, query_ctx: &QueryContextRef) -> Result<()> {
2250    let (catalog, schema, _) = table_idents_to_full_name(name, query_ctx)
2251        .map_err(BoxedError::new)
2252        .context(ExternalSnafu)?;
2253
2254    validate_catalog_and_schema(&catalog, &schema, query_ctx)
2255        .map_err(BoxedError::new)
2256        .context(SqlExecInterceptedSnafu)
2257}
2258
2259fn validate_flow(name: &ObjectName, query_ctx: &QueryContextRef) -> Result<()> {
2260    let catalog = match &name.0[..] {
2261        [_flow] => query_ctx.current_catalog().to_string(),
2262        [catalog, _flow] => catalog.to_string_unquoted(),
2263        _ => {
2264            return InvalidSqlSnafu {
2265                err_msg: format!(
2266                    "expect flow name to be <catalog>.<flow_name> or <flow_name>, actual: {name}",
2267                ),
2268            }
2269            .fail();
2270        }
2271    };
2272
2273    let schema = query_ctx.current_schema();
2274
2275    validate_catalog_and_schema(&catalog, &schema, query_ctx)
2276        .map_err(BoxedError::new)
2277        .context(SqlExecInterceptedSnafu)
2278}
2279
2280fn validate_database(name: &ObjectName, query_ctx: &QueryContextRef) -> Result<()> {
2281    let (catalog, schema) = match &name.0[..] {
2282        [schema] => (
2283            query_ctx.current_catalog().to_string(),
2284            schema.to_string_unquoted(),
2285        ),
2286        [catalog, schema] => (catalog.to_string_unquoted(), schema.to_string_unquoted()),
2287        _ => InvalidSqlSnafu {
2288            err_msg: format!(
2289                "expect database name to be <catalog>.<schema> or <schema>, actual: {name}",
2290            ),
2291        }
2292        .fail()?,
2293    };
2294
2295    validate_catalog_and_schema(&catalog, &schema, query_ctx)
2296        .map_err(BoxedError::new)
2297        .context(SqlExecInterceptedSnafu)
2298}
2299
2300fn is_readonly_plan(plan: &LogicalPlan) -> bool {
2301    !matches!(plan, LogicalPlan::Dml(_) | LogicalPlan::Ddl(_))
2302}
2303
2304fn should_track_statement_process(stmt: &Statement) -> bool {
2305    stmt.is_readonly()
2306        || matches!(stmt, Statement::Insert(insert) if insert.has_non_values_query_source())
2307}
2308
2309fn should_track_plan_process(stmt: Option<&Statement>, plan: &LogicalPlan) -> bool {
2310    is_readonly_plan(plan)
2311        || matches!(stmt, Some(Statement::Insert(insert)) if insert.has_non_values_query_source())
2312}
2313
2314#[cfg(test)]
2315mod tests {
2316    use std::any::Any;
2317    use std::collections::HashMap;
2318    use std::future::Future;
2319    use std::pin::Pin;
2320    use std::sync::Arc;
2321    use std::task::{Context, Poll};
2322    use std::time::Duration;
2323
2324    use api::prom_store::remote::label_matcher::Type as PromMatcherType;
2325    use api::prom_store::remote::{
2326        Label, LabelMatcher, Query as RemoteQuery, ReadRequest, ReadResponse, Sample,
2327    };
2328    use api::v1::greptime_request::Request;
2329    use api::v1::meta::{ProcedureDetailResponse, ReconcileRequest, ReconcileResponse};
2330    use api::v1::query_request::Query;
2331    use auth::{
2332        DASHBOARD_DELETE, DASHBOARD_QUERY, DASHBOARD_SAVE, JAEGER_QUERY, PIPELINE_DELETE,
2333        PIPELINE_INSERT, PIPELINE_QUERY, PermissionAction, PermissionResp, UserInfo, UserInfoRef,
2334    };
2335    use catalog::process_manager::{ProcessManager, QueryStatement, SlowQueryTimer};
2336    use common_base::Plugins;
2337    use common_catalog::consts::DEFAULT_PRIVATE_SCHEMA_NAME;
2338    use common_error::ext::{BoxedError, ErrorExt, PlainError};
2339    use common_error::status_code::StatusCode;
2340    use common_event_recorder::{Event, EventRecorder, EventTypeFilter, EventTypeFilterRef};
2341    use common_frontend::metrics::{CACHE_HIT, CACHE_MISS};
2342    use common_frontend::slow_query_event::SlowQueryEvent;
2343    use common_meta::cache::LayeredCacheRegistryBuilder;
2344    use common_meta::kv_backend::memory::MemoryKvBackend;
2345    use common_meta::procedure_executor::{ExecutorContext, ProcedureExecutor};
2346    use common_meta::rpc::ddl::{DdlTask, SubmitDdlTaskRequest, SubmitDdlTaskResponse};
2347    use common_meta::rpc::procedure::{
2348        MigrateRegionRequest, MigrateRegionResponse, ProcedureStateResponse,
2349    };
2350    use common_query::prelude::greptime_value;
2351    use common_query::{Output, OutputMeta};
2352    use common_recordbatch::{
2353        OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream,
2354    };
2355    use common_telemetry::logging::SlowQueriesRecordType;
2356    use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
2357    use datafusion::physical_plan::empty::EmptyExec;
2358    use datafusion_expr::dml::InsertOp;
2359    use datafusion_expr::{LogicalPlanBuilder, LogicalTableSource};
2360    use datatypes::prelude::ConcreteDataType;
2361    use datatypes::schema::{ColumnSchema, Schema as GtSchema, SchemaRef as GtSchemaRef};
2362    use datatypes::vectors::{
2363        Float64Vector, StringVector, TimestampMillisecondVector, TimestampNanosecondVector,
2364        VectorRef,
2365    };
2366    use log_query::LogQuery;
2367    use prost::Message;
2368    use query::query_engine::options::QueryOptions;
2369    use servers::query_handler::{
2370        DashboardHandler, JaegerQueryHandler, LogQueryHandler, PipelineHandler, PipelineHandlerRef,
2371        PromStoreProtocolHandler,
2372    };
2373    use session::context::{Channel, ConnInfo, QueryContext, QueryContextBuilder};
2374    use snafu::{Location, Snafu};
2375    use sql::dialect::GreptimeDbDialect;
2376    use store_api::data_source::DataSource;
2377    use store_api::metric_engine_consts::{
2378        LOGICAL_TABLE_METADATA_KEY, METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY,
2379    };
2380    use store_api::storage::ScanRequest;
2381    use strfmt::Format;
2382    use table::metadata::{
2383        FilterPushDownType, TableInfo, TableInfoBuilder, TableMetaBuilder, TableType,
2384    };
2385    use table::table_name::TableName;
2386    use table::test_util::{EmptyTable, MemTable};
2387    use table::{Table, TableRef};
2388    use tokio::sync::{mpsc, oneshot};
2389    use tower::ServiceExt;
2390
2391    use crate::frontend::FrontendOptions;
2392    use crate::instance::builder::FrontendBuilder;
2393    use crate::instance::*;
2394
2395    fn parse_test_sql(sql: &str) -> Vec<Statement> {
2396        parse_stmt(sql, &GreptimeDbDialect {}).unwrap()
2397    }
2398
2399    #[derive(Debug, Default)]
2400    struct RecordingSlowQueryEventRecorder {
2401        queries: std::sync::Mutex<Vec<String>>,
2402        payloads: std::sync::Mutex<Vec<serde_json::Value>>,
2403    }
2404
2405    impl EventRecorder for RecordingSlowQueryEventRecorder {
2406        fn record(&self, event: Box<dyn Event>) {
2407            let event = event
2408                .as_any()
2409                .downcast_ref::<SlowQueryEvent>()
2410                .expect("expected a slow query event");
2411            self.queries.lock().unwrap().push(event.query.clone());
2412            self.payloads.lock().unwrap().push(event.payload.clone());
2413        }
2414
2415        fn event_type_filter(&self) -> EventTypeFilterRef {
2416            Arc::new(EventTypeFilter::All)
2417        }
2418
2419        fn close(&self) {}
2420    }
2421
2422    #[test]
2423    fn test_validate_analyze_stream_statement_strictness() {
2424        for sql in [
2425            "select 1",
2426            "explain analyze select 1",
2427            "explain analyze verbose format text select 1",
2428            "explain analyze verbose format graphviz select 1",
2429            "TQL ANALYZE (0, 10, '5s') physical_metric",
2430            "TQL EXPLAIN VERBOSE (0, 10, '5s') physical_metric",
2431            "TQL ANALYZE VERBOSE FORMAT TEXT (0, 10, '5s') physical_metric",
2432        ] {
2433            let mut stmts = parse_test_sql(sql);
2434            assert!(
2435                validate_analyze_stream_statement(&mut stmts[0]).is_err(),
2436                "{sql}"
2437            );
2438        }
2439
2440        for sql in [
2441            "explain analyze verbose select 1",
2442            "explain analyze verbose format json select 1",
2443            "TQL ANALYZE VERBOSE (0, 10, '5s') physical_metric",
2444            "TQL ANALYZE VERBOSE FORMAT JSON (0, 10, '5s') physical_metric",
2445        ] {
2446            let mut stmts = parse_test_sql(sql);
2447            assert!(
2448                validate_analyze_stream_statement(&mut stmts[0]).is_ok(),
2449                "{sql}"
2450            );
2451            match &stmts[0] {
2452                Statement::Explain(explain) => assert!(explain.format.is_none()),
2453                Statement::Tql(Tql::Analyze(analyze)) => assert!(analyze.format.is_none()),
2454                _ => unreachable!(),
2455            }
2456        }
2457
2458        assert_eq!(
2459            parse_test_sql("explain analyze verbose select 1; select 2").len(),
2460            2
2461        );
2462
2463        assert!(is_explain_analyze_verbose(
2464            &parse_test_sql("explain analyze verbose select 1")[0]
2465        ));
2466        assert!(is_explain_analyze_verbose(
2467            &parse_test_sql("TQL ANALYZE VERBOSE (0, 10, '5s') physical_metric")[0]
2468        ));
2469        for sql in [
2470            "select 1",
2471            "explain select 1",
2472            "explain analyze select 1",
2473            "explain verbose select 1",
2474            "TQL ANALYZE (0, 10, '5s') physical_metric",
2475            "TQL EXPLAIN VERBOSE (0, 10, '5s') physical_metric",
2476        ] {
2477            assert!(
2478                !is_explain_analyze_verbose(&parse_test_sql(sql)[0]),
2479                "{sql}"
2480            );
2481        }
2482    }
2483
2484    #[derive(Debug, Snafu)]
2485    enum TestError {
2486        #[snafu(display("Failed to build test cache registry"))]
2487        BuildCacheRegistry {
2488            source: cache::error::Error,
2489            #[snafu(implicit)]
2490            location: Location,
2491        },
2492
2493        #[snafu(display("Failed to build test table meta for table: {table_name}"))]
2494        BuildTableMeta {
2495            table_name: String,
2496            source: table::metadata::TableMetaBuilderError,
2497            #[snafu(implicit)]
2498            location: Location,
2499        },
2500
2501        #[snafu(display("Failed to build test table info for table: {table_name}"))]
2502        BuildTableInfo {
2503            table_name: String,
2504            source: table::metadata::TableInfoBuilderError,
2505            #[snafu(implicit)]
2506            location: Location,
2507        },
2508
2509        #[snafu(display("Failed to register test table: {table_name}"))]
2510        RegisterTable {
2511            table_name: String,
2512            source: catalog::error::Error,
2513            #[snafu(implicit)]
2514            location: Location,
2515        },
2516
2517        #[snafu(display("Failed to build test frontend instance"))]
2518        BuildFrontend {
2519            source: crate::error::Error,
2520            #[snafu(implicit)]
2521            location: Location,
2522        },
2523
2524        #[snafu(display("Expected exactly one output for SQL `{sql}`, got {actual}"))]
2525        UnexpectedOutputCount {
2526            sql: String,
2527            actual: usize,
2528            #[snafu(implicit)]
2529            location: Location,
2530        },
2531
2532        #[snafu(display("Failed to execute SQL `{sql}`"))]
2533        ExecuteSql {
2534            sql: String,
2535            source: crate::error::Error,
2536            #[snafu(implicit)]
2537            location: Location,
2538        },
2539
2540        #[snafu(display("Timed out waiting for insert-select start notification"))]
2541        InsertStartTimeout {
2542            source: tokio::time::error::Elapsed,
2543            #[snafu(implicit)]
2544            location: Location,
2545        },
2546
2547        #[snafu(display("Insert-select start notification channel closed"))]
2548        InsertStartChannelClosed {
2549            #[snafu(implicit)]
2550            location: Location,
2551        },
2552
2553        #[snafu(display("Failed to release blocking insert-select interceptor"))]
2554        ReleaseBlockedInsert {
2555            #[snafu(implicit)]
2556            location: Location,
2557        },
2558
2559        #[snafu(display("Timed out waiting for insert-select source to be polled"))]
2560        SourcePollTimeout {
2561            source: tokio::time::error::Elapsed,
2562            #[snafu(implicit)]
2563            location: Location,
2564        },
2565
2566        #[snafu(display("Insert-select source poll notification channel closed"))]
2567        SourcePollChannelClosed {
2568            source: oneshot::error::RecvError,
2569            #[snafu(implicit)]
2570            location: Location,
2571        },
2572
2573        #[snafu(display("Timed out waiting for insert task to finish"))]
2574        InsertTaskTimeout {
2575            source: tokio::time::error::Elapsed,
2576            #[snafu(implicit)]
2577            location: Location,
2578        },
2579
2580        #[snafu(display("Insert task panicked"))]
2581        InsertTaskPanic {
2582            source: tokio::task::JoinError,
2583            #[snafu(implicit)]
2584            location: Location,
2585        },
2586
2587        #[snafu(display("Expected insert-select to be cancelled"))]
2588        InsertSelectNotCancelled {
2589            #[snafu(implicit)]
2590            location: Location,
2591        },
2592    }
2593
2594    type TestResult<T> = std::result::Result<T, TestError>;
2595
2596    fn parse_one_sql(sql: &str) -> Statement {
2597        parse_stmt(sql, &GreptimeDbDialect {}).unwrap().remove(0)
2598    }
2599
2600    fn test_query_ctx(process_id: u32) -> QueryContextRef {
2601        Arc::new(
2602            QueryContextBuilder::default()
2603                .channel(Channel::Mysql)
2604                .conn_info(ConnInfo::new(None, Channel::Mysql))
2605                .process_id(process_id)
2606                .build(),
2607        )
2608    }
2609
2610    #[derive(Debug)]
2611    struct AdminUserInfo;
2612
2613    impl UserInfo for AdminUserInfo {
2614        fn as_any(&self) -> &dyn Any {
2615            self
2616        }
2617
2618        fn username(&self) -> &str {
2619            "admin"
2620        }
2621
2622        fn is_admin(&self) -> bool {
2623            true
2624        }
2625    }
2626
2627    /// A checker that authorizes per table and counts the grants it was asked for, so a test can
2628    /// state that a path did (or did not) ask.
2629    #[derive(Default)]
2630    struct CountingPermissionChecker {
2631        table_target_checks: atomic::AtomicUsize,
2632    }
2633
2634    impl CountingPermissionChecker {
2635        fn table_target_checks(&self) -> usize {
2636            self.table_target_checks.load(atomic::Ordering::Relaxed)
2637        }
2638    }
2639
2640    impl PermissionChecker for CountingPermissionChecker {
2641        fn check_permission(
2642            &self,
2643            _user_info: UserInfoRef,
2644            _req: PermissionReq,
2645        ) -> auth::error::Result<PermissionResp> {
2646            Ok(PermissionResp::Allow)
2647        }
2648
2649        fn check_permission_with_table_targets(
2650            &self,
2651            _user_info: UserInfoRef,
2652            _req: PermissionReq,
2653            _targets: PermissionTableTargets,
2654        ) -> auth::error::Result<PermissionResp> {
2655            self.table_target_checks
2656                .fetch_add(1, atomic::Ordering::Relaxed);
2657            Ok(PermissionResp::Allow)
2658        }
2659
2660        fn uses_table_targets(&self) -> bool {
2661            true
2662        }
2663    }
2664
2665    struct RejectUnresolvedPermissionChecker;
2666
2667    impl PermissionChecker for RejectUnresolvedPermissionChecker {
2668        fn check_permission(
2669            &self,
2670            _user_info: UserInfoRef,
2671            _req: PermissionReq,
2672        ) -> auth::error::Result<PermissionResp> {
2673            Ok(PermissionResp::Allow)
2674        }
2675
2676        fn check_permission_with_table_targets(
2677            &self,
2678            _user_info: UserInfoRef,
2679            _req: PermissionReq,
2680            targets: PermissionTableTargets,
2681        ) -> auth::error::Result<PermissionResp> {
2682            let reject = match targets {
2683                PermissionTableTargets::Unresolved => true,
2684                PermissionTableTargets::Resolved(targets) => targets
2685                    .iter()
2686                    .any(|target| target.table == "denied" || target.is_physical_metric_table),
2687            };
2688            Ok(if reject {
2689                PermissionResp::Reject
2690            } else {
2691                PermissionResp::Allow
2692            })
2693        }
2694    }
2695
2696    #[derive(Debug, PartialEq, Eq)]
2697    struct CheckedAction {
2698        action: PermissionAction,
2699        targets: Option<PermissionTableTargets>,
2700    }
2701
2702    #[derive(Default)]
2703    struct RejectEndpointPermissionChecker {
2704        checks: std::sync::Mutex<Vec<CheckedAction>>,
2705    }
2706
2707    impl RejectEndpointPermissionChecker {
2708        fn reject(
2709            &self,
2710            action: PermissionAction,
2711            targets: Option<PermissionTableTargets>,
2712        ) -> PermissionResp {
2713            self.checks
2714                .lock()
2715                .unwrap()
2716                .push(CheckedAction { action, targets });
2717            PermissionResp::Reject
2718        }
2719
2720        fn take_check(&self) -> CheckedAction {
2721            let mut checks = self.checks.lock().unwrap();
2722            assert_eq!(1, checks.len());
2723            checks.pop().unwrap()
2724        }
2725    }
2726
2727    impl PermissionChecker for RejectEndpointPermissionChecker {
2728        fn check_permission(
2729            &self,
2730            _user_info: UserInfoRef,
2731            req: PermissionReq,
2732        ) -> auth::error::Result<PermissionResp> {
2733            Ok(match req {
2734                PermissionReq::Action(action) => self.reject(action, None),
2735                _ => PermissionResp::Allow,
2736            })
2737        }
2738
2739        fn check_permission_with_table_targets(
2740            &self,
2741            _user_info: UserInfoRef,
2742            req: PermissionReq,
2743            targets: PermissionTableTargets,
2744        ) -> auth::error::Result<PermissionResp> {
2745            Ok(match req {
2746                PermissionReq::Action(action) => self.reject(action, Some(targets)),
2747                _ => PermissionResp::Allow,
2748            })
2749        }
2750    }
2751
2752    struct WriteOnlyPermissionChecker;
2753
2754    impl PermissionChecker for WriteOnlyPermissionChecker {
2755        fn check_permission(
2756            &self,
2757            _user_info: UserInfoRef,
2758            req: PermissionReq,
2759        ) -> auth::error::Result<PermissionResp> {
2760            Ok(if req.is_readonly() {
2761                PermissionResp::Reject
2762            } else {
2763                PermissionResp::Allow
2764            })
2765        }
2766
2767        fn check_permission_with_table_targets(
2768            &self,
2769            user_info: UserInfoRef,
2770            req: PermissionReq,
2771            _targets: PermissionTableTargets,
2772        ) -> auth::error::Result<PermissionResp> {
2773            self.check_permission(user_info, req)
2774        }
2775    }
2776
2777    #[derive(Default)]
2778    struct TargetIndependentPermissionChecker {
2779        checks: atomic::AtomicUsize,
2780    }
2781
2782    impl PermissionChecker for TargetIndependentPermissionChecker {
2783        fn check_permission(
2784            &self,
2785            _user_info: UserInfoRef,
2786            _req: PermissionReq,
2787        ) -> auth::error::Result<PermissionResp> {
2788            self.checks.fetch_add(1, atomic::Ordering::Relaxed);
2789            Ok(PermissionResp::Allow)
2790        }
2791
2792        fn uses_table_targets(&self) -> bool {
2793            false
2794        }
2795
2796        fn check_permission_with_table_targets(
2797            &self,
2798            user_info: UserInfoRef,
2799            req: PermissionReq,
2800            _targets: PermissionTableTargets,
2801        ) -> auth::error::Result<PermissionResp> {
2802            self.check_permission(user_info, req)
2803        }
2804    }
2805
2806    struct BlockingInsertSelectInterceptor {
2807        started_tx: mpsc::UnboundedSender<()>,
2808        finish_rx: std::sync::Mutex<Option<oneshot::Receiver<()>>>,
2809    }
2810
2811    impl BlockingInsertSelectInterceptor {
2812        fn new(started_tx: mpsc::UnboundedSender<()>, finish_rx: oneshot::Receiver<()>) -> Self {
2813            Self {
2814                started_tx,
2815                finish_rx: std::sync::Mutex::new(Some(finish_rx)),
2816            }
2817        }
2818    }
2819
2820    impl SqlQueryInterceptor for BlockingInsertSelectInterceptor {
2821        type Error = Error;
2822
2823        fn pre_execute(
2824            &self,
2825            statement: Option<&Statement>,
2826            _plan: Option<&LogicalPlan>,
2827            _query_ctx: QueryContextRef,
2828        ) -> Result<()> {
2829            let Some(Statement::Insert(insert)) = statement else {
2830                return Ok(());
2831            };
2832            if !insert.has_non_values_query_source() {
2833                return Ok(());
2834            }
2835
2836            let finish_rx = self.finish_rx.lock().unwrap().take().unwrap();
2837            let _ = self.started_tx.send(());
2838            tokio::task::block_in_place(|| {
2839                tokio::runtime::Handle::current()
2840                    .block_on(finish_rx)
2841                    .unwrap();
2842            });
2843            Ok(())
2844        }
2845    }
2846
2847    struct PendingRecordBatchStream {
2848        schema: GtSchemaRef,
2849        polled_tx: Option<oneshot::Sender<()>>,
2850        _finish_tx: oneshot::Sender<()>,
2851        finish_rx: Pin<Box<oneshot::Receiver<()>>>,
2852    }
2853
2854    impl RecordBatchStream for PendingRecordBatchStream {
2855        fn schema(&self) -> GtSchemaRef {
2856            self.schema.clone()
2857        }
2858
2859        fn output_ordering(&self) -> Option<&[OrderOption]> {
2860            None
2861        }
2862
2863        fn metrics(&self) -> Option<common_recordbatch::adapter::RecordBatchMetrics> {
2864            None
2865        }
2866    }
2867
2868    impl Stream for PendingRecordBatchStream {
2869        type Item = common_recordbatch::error::Result<RecordBatch>;
2870
2871        fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
2872            if let Some(polled_tx) = self.polled_tx.take() {
2873                let _ = polled_tx.send(());
2874            }
2875
2876            match self.finish_rx.as_mut().poll(cx) {
2877                Poll::Ready(_) => Poll::Ready(None),
2878                Poll::Pending => Poll::Pending,
2879            }
2880        }
2881    }
2882
2883    impl Unpin for PendingRecordBatchStream {}
2884
2885    #[test]
2886    fn test_record_explain_analyze_timeout_uses_empty_metrics_without_plan() {
2887        let event_recorder = Arc::new(RecordingSlowQueryEventRecorder::default());
2888        let timer = SlowQueryTimer::new(
2889            QueryStatement::Plan("EXPLAIN ANALYZE VERBOSE SELECT 1".to_string()),
2890            "public".to_string(),
2891            Duration::from_secs(3600),
2892            0.0,
2893            SlowQueriesRecordType::SystemTable,
2894            event_recorder.clone(),
2895        );
2896        let timeout_recorder = timer.recorder();
2897
2898        record_explain_analyze_timeout(Some(&timeout_recorder), None);
2899        drop(timer);
2900
2901        let payloads = event_recorder.payloads.lock().unwrap();
2902        assert_eq!(payloads.len(), 1);
2903        assert_eq!(payloads[0]["timed_out"], true);
2904        assert_eq!(payloads[0]["metrics"], serde_json::json!([]));
2905    }
2906
2907    #[tokio::test]
2908    async fn test_attach_timeout_records_explain_analyze_metrics() {
2909        let event_recorder = Arc::new(RecordingSlowQueryEventRecorder::default());
2910        let timer = SlowQueryTimer::new(
2911            QueryStatement::Plan("EXPLAIN ANALYZE VERBOSE SELECT 1".to_string()),
2912            "public".to_string(),
2913            Duration::from_secs(3600),
2914            0.0,
2915            SlowQueriesRecordType::SystemTable,
2916            event_recorder.clone(),
2917        );
2918        let timeout_recorder = timer.recorder();
2919        let plan: Arc<dyn ExecutionPlan> = Arc::new(EmptyExec::new(Arc::new(Schema::empty())));
2920        let (finish_tx, finish_rx) = oneshot::channel();
2921        let stream = PendingRecordBatchStream {
2922            schema: Arc::new(GtSchema::new(vec![])),
2923            polled_tx: None,
2924            _finish_tx: finish_tx,
2925            finish_rx: Box::pin(finish_rx),
2926        };
2927        let output = Output::new(
2928            OutputData::Stream(Box::pin(stream)),
2929            OutputMeta::new_with_plan(plan),
2930        );
2931        let output =
2932            attach_timeout(output, Duration::from_millis(10), Some(timeout_recorder)).unwrap();
2933        let OutputData::Stream(mut stream) = output.data else {
2934            unreachable!();
2935        };
2936
2937        let err = stream.next().await.unwrap().unwrap_err();
2938        assert_eq!(err.to_string(), "Stream timeout");
2939        drop(stream);
2940        drop(timer);
2941
2942        let payloads = event_recorder.payloads.lock().unwrap();
2943        assert_eq!(payloads.len(), 1);
2944        assert_eq!(payloads[0]["timed_out"], true);
2945        assert!(
2946            payloads[0]["metrics"]
2947                .as_array()
2948                .is_some_and(|metrics| !metrics.is_empty())
2949        );
2950    }
2951
2952    struct PendingDataSource {
2953        schema: GtSchemaRef,
2954        polled_tx: std::sync::Mutex<Option<oneshot::Sender<()>>>,
2955    }
2956
2957    impl DataSource for PendingDataSource {
2958        fn get_stream(
2959            &self,
2960            _request: ScanRequest,
2961        ) -> std::result::Result<SendableRecordBatchStream, BoxedError> {
2962            let (finish_tx, finish_rx) = oneshot::channel();
2963            let mut polled_tx = self.polled_tx.lock().map_err(|_| {
2964                BoxedError::new(PlainError::new(
2965                    "pending data source lock poisoned".to_string(),
2966                    StatusCode::Unexpected,
2967                ))
2968            })?;
2969            Ok(Box::pin(PendingRecordBatchStream {
2970                schema: self.schema.clone(),
2971                polled_tx: polled_tx.take(),
2972                _finish_tx: finish_tx,
2973                finish_rx: Box::pin(finish_rx),
2974            }))
2975        }
2976    }
2977
2978    struct NoopProcedureExecutor;
2979
2980    #[async_trait::async_trait]
2981    impl ProcedureExecutor for NoopProcedureExecutor {
2982        async fn submit_ddl_task(
2983            &self,
2984            _ctx: ExecutorContext,
2985            _request: SubmitDdlTaskRequest,
2986        ) -> common_meta::error::Result<SubmitDdlTaskResponse> {
2987            common_meta::error::UnsupportedSnafu {
2988                operation: "submit_ddl_task",
2989            }
2990            .fail()
2991        }
2992
2993        async fn migrate_region(
2994            &self,
2995            _ctx: &ExecutorContext,
2996            _request: MigrateRegionRequest,
2997        ) -> common_meta::error::Result<MigrateRegionResponse> {
2998            common_meta::error::UnsupportedSnafu {
2999                operation: "migrate_region",
3000            }
3001            .fail()
3002        }
3003
3004        async fn reconcile(
3005            &self,
3006            _ctx: &ExecutorContext,
3007            _request: ReconcileRequest,
3008        ) -> common_meta::error::Result<ReconcileResponse> {
3009            common_meta::error::UnsupportedSnafu {
3010                operation: "reconcile",
3011            }
3012            .fail()
3013        }
3014
3015        async fn query_procedure_state(
3016            &self,
3017            _ctx: &ExecutorContext,
3018            _pid: &str,
3019        ) -> common_meta::error::Result<ProcedureStateResponse> {
3020            common_meta::error::UnsupportedSnafu {
3021                operation: "query_procedure_state",
3022            }
3023            .fail()
3024        }
3025
3026        async fn list_procedures(
3027            &self,
3028            _ctx: &ExecutorContext,
3029        ) -> common_meta::error::Result<ProcedureDetailResponse> {
3030            common_meta::error::UnsupportedSnafu {
3031                operation: "list_procedures",
3032            }
3033            .fail()
3034        }
3035    }
3036
3037    /// A test [`ProcedureExecutor`] that completes create/drop DDL tasks against the
3038    /// in-memory catalog, mimicking what the meta DDL procedures do in production.
3039    /// This allows happy-path DDL requests (create/drop table/view) to be exercised
3040    /// end to end through the gRPC ingress.
3041    struct MockProcedureExecutor {
3042        catalog_manager: Arc<catalog::memory::MemoryCatalogManager>,
3043        next_table_id: std::sync::atomic::AtomicU32,
3044        submitted: std::sync::Mutex<Vec<DdlTask>>,
3045    }
3046
3047    impl MockProcedureExecutor {
3048        fn new(catalog_manager: Arc<catalog::memory::MemoryCatalogManager>) -> Self {
3049            Self {
3050                catalog_manager,
3051                next_table_id: std::sync::atomic::AtomicU32::new(1026),
3052                submitted: std::sync::Mutex::new(Vec::new()),
3053            }
3054        }
3055    }
3056
3057    #[async_trait::async_trait]
3058    impl ProcedureExecutor for MockProcedureExecutor {
3059        async fn submit_ddl_task(
3060            &self,
3061            _ctx: ExecutorContext,
3062            request: SubmitDdlTaskRequest,
3063        ) -> common_meta::error::Result<SubmitDdlTaskResponse> {
3064            self.submitted.lock().unwrap().push(request.task.clone());
3065            match request.task {
3066                DdlTask::CreateTable(task) => {
3067                    let table_id = self
3068                        .next_table_id
3069                        .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3070                    let mut table_info = task.table_info;
3071                    table_info.ident.table_id = table_id;
3072                    self.catalog_manager
3073                        .register_table_sync(catalog::RegisterTableRequest {
3074                            catalog: table_info.catalog_name.clone(),
3075                            schema: table_info.schema_name.clone(),
3076                            table_name: table_info.name.clone(),
3077                            table_id,
3078                            table: table::dist_table::DistTable::table(Arc::new(table_info)),
3079                        })
3080                        .map_err(BoxedError::new)
3081                        .context(common_meta::error::ExternalSnafu)?;
3082                    Ok(SubmitDdlTaskResponse {
3083                        key: Vec::new(),
3084                        table_ids: vec![table_id],
3085                    })
3086                }
3087                DdlTask::CreateView(task) => {
3088                    let view_id = self
3089                        .next_table_id
3090                        .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3091                    let mut view_info = task.view_info;
3092                    view_info.ident.table_id = view_id;
3093                    self.catalog_manager
3094                        .register_table_sync(catalog::RegisterTableRequest {
3095                            catalog: task.create_view.catalog_name.clone(),
3096                            schema: task.create_view.schema_name.clone(),
3097                            table_name: task.create_view.view_name.clone(),
3098                            table_id: view_id,
3099                            table: table::dist_table::DistTable::table(Arc::new(view_info)),
3100                        })
3101                        .map_err(BoxedError::new)
3102                        .context(common_meta::error::ExternalSnafu)?;
3103                    Ok(SubmitDdlTaskResponse {
3104                        key: Vec::new(),
3105                        table_ids: vec![view_id],
3106                    })
3107                }
3108                DdlTask::DropView(task) => {
3109                    self.catalog_manager
3110                        .deregister_table_sync(catalog::DeregisterTableRequest {
3111                            catalog: task.catalog.clone(),
3112                            schema: task.schema.clone(),
3113                            table_name: task.view.clone(),
3114                        })
3115                        .map_err(BoxedError::new)
3116                        .context(common_meta::error::ExternalSnafu)?;
3117                    Ok(SubmitDdlTaskResponse::default())
3118                }
3119                other => common_meta::error::UnsupportedSnafu {
3120                    operation: format!("mock submit_ddl_task: {other:?}"),
3121                }
3122                .fail(),
3123            }
3124        }
3125
3126        async fn migrate_region(
3127            &self,
3128            _ctx: &ExecutorContext,
3129            _request: MigrateRegionRequest,
3130        ) -> common_meta::error::Result<MigrateRegionResponse> {
3131            common_meta::error::UnsupportedSnafu {
3132                operation: "migrate_region",
3133            }
3134            .fail()
3135        }
3136
3137        async fn reconcile(
3138            &self,
3139            _ctx: &ExecutorContext,
3140            _request: ReconcileRequest,
3141        ) -> common_meta::error::Result<ReconcileResponse> {
3142            common_meta::error::UnsupportedSnafu {
3143                operation: "reconcile",
3144            }
3145            .fail()
3146        }
3147
3148        async fn query_procedure_state(
3149            &self,
3150            _ctx: &ExecutorContext,
3151            _pid: &str,
3152        ) -> common_meta::error::Result<ProcedureStateResponse> {
3153            common_meta::error::UnsupportedSnafu {
3154                operation: "query_procedure_state",
3155            }
3156            .fail()
3157        }
3158
3159        async fn list_procedures(
3160            &self,
3161            _ctx: &ExecutorContext,
3162        ) -> common_meta::error::Result<ProcedureDetailResponse> {
3163            common_meta::error::UnsupportedSnafu {
3164                operation: "list_procedures",
3165            }
3166            .fail()
3167        }
3168    }
3169
3170    fn test_cache_registry(
3171        kv_backend: common_meta::kv_backend::KvBackendRef,
3172    ) -> TestResult<common_meta::cache::LayeredCacheRegistryRef> {
3173        Ok(Arc::new(
3174            cache::with_default_composite_cache_registry(
3175                LayeredCacheRegistryBuilder::default()
3176                    .add_cache_registry(cache::build_fundamental_cache_registry(kv_backend)),
3177            )
3178            .context(BuildCacheRegistrySnafu)?
3179            .build(),
3180        ))
3181    }
3182
3183    fn test_table_info(table_id: u32, table_name: &str) -> TestResult<TableInfo> {
3184        let schema = Arc::new(GtSchema::new(vec![
3185            ColumnSchema::new("id", ConcreteDataType::int32_datatype(), false),
3186            ColumnSchema::new(
3187                "ts",
3188                ConcreteDataType::timestamp_millisecond_datatype(),
3189                false,
3190            )
3191            .with_time_index(true),
3192        ]));
3193        let table_meta = TableMetaBuilder::empty()
3194            .schema(schema)
3195            .primary_key_indices(vec![0])
3196            .value_indices(vec![1])
3197            .next_column_id(1024)
3198            .build()
3199            .with_context(|_| BuildTableMetaSnafu {
3200                table_name: table_name.to_string(),
3201            })?;
3202
3203        TableInfoBuilder::new(table_name, table_meta)
3204            .table_id(table_id)
3205            .build()
3206            .with_context(|_| BuildTableInfoSnafu {
3207                table_name: table_name.to_string(),
3208            })
3209    }
3210
3211    fn test_table(table_id: u32, table_name: &str) -> TestResult<table::TableRef> {
3212        let table_info = test_table_info(table_id, table_name)?;
3213        Ok(EmptyTable::from_table_info(&table_info))
3214    }
3215
3216    fn test_physical_table(table_id: u32, table_name: &str) -> TestResult<table::TableRef> {
3217        let mut table_info = test_table_info(table_id, table_name)?;
3218        table_info
3219            .meta
3220            .options
3221            .extra_options
3222            .insert(PHYSICAL_TABLE_METADATA_KEY.to_string(), String::new());
3223        Ok(EmptyTable::from_table_info(&table_info))
3224    }
3225
3226    fn test_logical_table(table_id: u32, table_name: &str) -> TestResult<table::TableRef> {
3227        let mut table_info = test_table_info(table_id, table_name)?;
3228        table_info.meta.engine = METRIC_ENGINE_NAME.to_string();
3229        table_info.meta.options.extra_options.insert(
3230            LOGICAL_TABLE_METADATA_KEY.to_string(),
3231            "physical_metric".to_string(),
3232        );
3233        Ok(EmptyTable::from_table_info(&table_info))
3234    }
3235
3236    fn test_metric_names_table() -> TableRef {
3237        let schema = Arc::new(GtSchema::new(vec![
3238            ColumnSchema::new("table_catalog", ConcreteDataType::string_datatype(), false),
3239            ColumnSchema::new("table_schema", ConcreteDataType::string_datatype(), false),
3240            ColumnSchema::new("table_name", ConcreteDataType::string_datatype(), false),
3241            ColumnSchema::new("engine", ConcreteDataType::string_datatype(), false),
3242            ColumnSchema::new("create_options", ConcreteDataType::string_datatype(), false),
3243        ]));
3244        let columns: Vec<VectorRef> = vec![
3245            Arc::new(StringVector::from(vec!["greptime", "greptime"])),
3246            Arc::new(StringVector::from(vec!["public", "public"])),
3247            Arc::new(StringVector::from(vec!["denied", "target"])),
3248            Arc::new(StringVector::from(vec!["metric", "metric"])),
3249            Arc::new(StringVector::from(vec![
3250                "on_physical_table=physical_metric",
3251                "on_physical_table=physical_metric",
3252            ])),
3253        ];
3254        let record_batch = RecordBatch::new(schema, columns).unwrap();
3255        MemTable::new_with_catalog(
3256            "tables",
3257            record_batch,
3258            2048,
3259            "greptime".to_string(),
3260            "information_schema".to_string(),
3261        )
3262    }
3263
3264    fn test_pipeline_table() -> TableRef {
3265        let schema = Arc::new(GtSchema::new(vec![
3266            ColumnSchema::new("name", ConcreteDataType::string_datatype(), false),
3267            ColumnSchema::new("schema", ConcreteDataType::string_datatype(), false),
3268            ColumnSchema::new("content_type", ConcreteDataType::string_datatype(), false),
3269            ColumnSchema::new("pipeline", ConcreteDataType::string_datatype(), false),
3270            ColumnSchema::new(
3271                "created_at",
3272                ConcreteDataType::timestamp_nanosecond_datatype(),
3273                false,
3274            )
3275            .with_time_index(true),
3276        ]));
3277        let columns: Vec<VectorRef> = vec![
3278            Arc::new(StringVector::from(vec!["pipeline"])),
3279            Arc::new(StringVector::from(vec!["public"])),
3280            Arc::new(StringVector::from(vec!["application/yaml"])),
3281            Arc::new(StringVector::from(vec![
3282                "transform:\n- field: ts\n  type: timestamp, ns\n  index: time\n",
3283            ])),
3284            Arc::new(TimestampNanosecondVector::from_values([1])),
3285        ];
3286        let record_batch = RecordBatch::new(schema, columns).unwrap();
3287        MemTable::new_with_catalog(
3288            "pipelines",
3289            record_batch,
3290            2049,
3291            "greptime".to_string(),
3292            DEFAULT_PRIVATE_SCHEMA_NAME.to_string(),
3293        )
3294    }
3295
3296    struct CancellableExportSource {
3297        schema: GtSchemaRef,
3298        channels: std::sync::Mutex<Option<(oneshot::Sender<()>, oneshot::Receiver<()>)>>,
3299    }
3300
3301    impl DataSource for CancellableExportSource {
3302        fn get_stream(
3303            &self,
3304            _request: ScanRequest,
3305        ) -> std::result::Result<SendableRecordBatchStream, BoxedError> {
3306            let (started, release) = self.channels.lock().unwrap().take().unwrap();
3307            let schema = self.schema.clone();
3308            let stream = futures::stream::once(async move {
3309                started.send(()).unwrap();
3310                release.await.unwrap();
3311                Ok(RecordBatch::new_empty(schema))
3312            });
3313            Ok(Box::pin(RecordBatchStreamWrapper::new(
3314                self.schema.clone(),
3315                Box::pin(stream),
3316            )))
3317        }
3318    }
3319
3320    #[tokio::test]
3321    async fn test_copy_database_enforces_read_and_write_targets() -> TestResult<()> {
3322        struct DatabaseCopyAcl {
3323            reject_read: bool,
3324        }
3325        impl PermissionChecker for DatabaseCopyAcl {
3326            fn check_permission(
3327                &self,
3328                _: UserInfoRef,
3329                _: PermissionReq,
3330            ) -> auth::error::Result<PermissionResp> {
3331                Ok(PermissionResp::Allow)
3332            }
3333            fn check_permission_with_table_targets(
3334                &self,
3335                _: UserInfoRef,
3336                req: PermissionReq,
3337                targets: PermissionTableTargets,
3338            ) -> auth::error::Result<PermissionResp> {
3339                let applies = match req {
3340                    PermissionReq::Action(action) => action == COPY_TO && self.reject_read,
3341                    PermissionReq::SqlStatement(Statement::Copy(Copy::CopyDatabase(_))) => {
3342                        !self.reject_read
3343                    }
3344                    _ => false,
3345                };
3346                let PermissionTableTargets::Resolved(targets) = targets else {
3347                    return Ok(PermissionResp::Reject);
3348                };
3349                Ok(
3350                    if applies
3351                        && targets.iter().any(|target| {
3352                            target.table == "denied" || target.is_physical_metric_table
3353                        })
3354                    {
3355                        PermissionResp::Reject
3356                    } else {
3357                        PermissionResp::Allow
3358                    },
3359                )
3360            }
3361        }
3362        for (table, direction, reject_read) in [
3363            (test_table(1024, "denied")?, "TO", true),
3364            (test_table(1024, "denied")?, "TO", false),
3365            (test_table(1024, "denied")?, "FROM", false),
3366            (test_physical_table(1024, "physical")?, "FROM", false),
3367            (test_table(1024, "source")?, "TO", true),
3368            (test_table(1024, "source")?, "TO", false),
3369            (test_table(1024, "source")?, "FROM", false),
3370        ] {
3371            let dir = common_test_util::temp_dir::create_temp_dir("copy_database_permissions");
3372            let name = table.table_info().name.clone();
3373            let catalog = catalog::memory::MemoryCatalogManager::new_with_table(table);
3374            catalog
3375                .register_table_sync(catalog::RegisterTableRequest {
3376                    catalog: "greptime".into(),
3377                    schema: "public".into(),
3378                    table_name: "allowed".into(),
3379                    table_id: 1025,
3380                    table: test_table(1025, "allowed")?,
3381                })
3382                .unwrap();
3383            if direction == "FROM" {
3384                std::fs::write(dir.path().join("allowed.parquet"), b"not parquet").unwrap();
3385                std::fs::write(dir.path().join(format!("{name}.parquet")), b"not parquet").unwrap();
3386            }
3387            let kv = Arc::new(MemoryKvBackend::new());
3388            let plugins = Plugins::new();
3389            plugins.insert::<PermissionCheckerRef>(Arc::new(DatabaseCopyAcl { reject_read }));
3390            let instance = FrontendBuilder::new(
3391                FrontendOptions::default(),
3392                kv.clone(),
3393                test_cache_registry(kv)?,
3394                catalog,
3395                Arc::new(client::client_manager::NodeClients::default()),
3396                Arc::new(NoopProcedureExecutor),
3397                Arc::new(ProcessManager::new("copy-permissions".into(), None)),
3398            )
3399            .with_plugin(plugins)
3400            .with_local_file_access(
3401                common_datasource::object_store::LocalFileAccess::sandboxed(dir.path()).unwrap(),
3402            )
3403            .try_build()
3404            .await
3405            .context(BuildFrontendSnafu)?;
3406            let location = reqwest::Url::from_directory_path(dir.path()).unwrap();
3407            let sql = format!(
3408                "COPY DATABASE public {direction} '{location}' WITH (continue_on_error='true')"
3409            );
3410            let result = instance
3411                .do_query_inner(&sql, QueryContext::arc())
3412                .await
3413                .remove(0);
3414            if name == "source" {
3415                assert!(result.is_ok(), "{result:?}");
3416            } else {
3417                assert_eq!(
3418                    StatusCode::PermissionDenied,
3419                    result.unwrap_err().status_code()
3420                );
3421                if direction == "TO" {
3422                    assert_eq!(0, std::fs::read_dir(dir.path()).unwrap().count());
3423                }
3424            }
3425        }
3426        Ok(())
3427    }
3428
3429    #[tokio::test]
3430    async fn test_metric_export_http_timeout_cancels_ordinary_source() {
3431        let destination = common_test_util::temp_dir::create_temp_dir("metric_export_timeout");
3432        let (started_tx, started_rx) = oneshot::channel();
3433        let (mut release_tx, release_rx) = oneshot::channel();
3434        let info = test_table_info(1024, "source").unwrap();
3435        let source = Arc::new(Table::new(
3436            Arc::new(info.clone()),
3437            FilterPushDownType::Unsupported,
3438            Arc::new(CancellableExportSource {
3439                schema: info.meta.schema.clone(),
3440                channels: std::sync::Mutex::new(Some((started_tx, release_rx))),
3441            }),
3442        ));
3443        let catalog = catalog::memory::MemoryCatalogManager::new_with_table(source);
3444        let kv = Arc::new(MemoryKvBackend::new());
3445        let instance = FrontendBuilder::new(
3446            FrontendOptions::default(),
3447            kv.clone(),
3448            test_cache_registry(kv).unwrap(),
3449            catalog,
3450            Arc::new(client::client_manager::NodeClients::default()),
3451            Arc::new(NoopProcedureExecutor),
3452            Arc::new(ProcessManager::new("export-timeout".into(), None)),
3453        )
3454        .with_local_file_access(
3455            common_datasource::object_store::LocalFileAccess::sandboxed(destination.path())
3456                .unwrap(),
3457        )
3458        .try_build()
3459        .await
3460        .unwrap();
3461        let server = servers::http::HttpServerBuilder::new(servers::http::HttpOptions {
3462            timeout: Duration::from_secs(2),
3463            ..Default::default()
3464        })
3465        .with_sql_handler(Arc::new(instance))
3466        .build();
3467        let app = server.build(server.make_app()).unwrap();
3468        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
3469        let addr = listener.local_addr().unwrap();
3470        let server_task = tokio::spawn(async move {
3471            axum::serve(listener, app).await.unwrap();
3472        });
3473        let uri = reqwest::Url::from_directory_path(destination.path()).unwrap();
3474        let sql = format!(
3475            "COPY DATABASE greptime.public TO '{uri}' WITH (experimental_metric_export='true')"
3476        );
3477        let request = tokio::spawn(async move {
3478            reqwest::Client::builder()
3479                .no_proxy()
3480                .build()
3481                .unwrap()
3482                .post(format!("http://{addr}/v1/sql"))
3483                .form(&[("sql", sql)])
3484                .send()
3485                .await
3486                .unwrap()
3487        });
3488        tokio::time::timeout(Duration::from_secs(5), started_rx)
3489            .await
3490            .unwrap()
3491            .unwrap();
3492        let response = request.await.unwrap();
3493        assert_eq!(response.status(), reqwest::StatusCode::REQUEST_TIMEOUT);
3494        tokio::time::timeout(Duration::from_secs(5), release_tx.closed())
3495            .await
3496            .unwrap();
3497        assert!(!destination.path().join("source.parquet").exists());
3498        server_task.abort();
3499    }
3500
3501    fn pending_table(
3502        table_id: u32,
3503        table_name: &str,
3504        polled_tx: oneshot::Sender<()>,
3505    ) -> TestResult<table::TableRef> {
3506        let table_info = test_table_info(table_id, table_name)?;
3507        let data_source = Arc::new(PendingDataSource {
3508            schema: table_info.meta.schema.clone(),
3509            polled_tx: std::sync::Mutex::new(Some(polled_tx)),
3510        });
3511
3512        Ok(Arc::new(Table::new(
3513            Arc::new(table_info),
3514            FilterPushDownType::Unsupported,
3515            data_source,
3516        )))
3517    }
3518
3519    async fn test_instance_with_tables(
3520        source_table: TableRef,
3521        target_table: TableRef,
3522    ) -> TestResult<Instance> {
3523        test_instance_with_plugins(source_table, target_table, Plugins::new()).await
3524    }
3525
3526    async fn test_instance_with_insert_select_interceptor(
3527        interceptor: SqlQueryInterceptorRef<Error>,
3528    ) -> TestResult<Instance> {
3529        let plugins = Plugins::new();
3530        plugins.insert::<SqlQueryInterceptorRef<Error>>(interceptor);
3531
3532        test_instance_with_plugins(
3533            test_table(1024, "source")?,
3534            test_table(1025, "target")?,
3535            plugins,
3536        )
3537        .await
3538    }
3539
3540    async fn test_instance_with_plugins(
3541        source_table: TableRef,
3542        target_table: TableRef,
3543        plugins: Plugins,
3544    ) -> TestResult<Instance> {
3545        test_instance_with_plugins_and_metric_names(source_table, target_table, plugins, None).await
3546    }
3547
3548    async fn test_instance_with_plugins_and_metric_names(
3549        source_table: TableRef,
3550        target_table: TableRef,
3551        plugins: Plugins,
3552        metric_names_table: Option<TableRef>,
3553    ) -> TestResult<Instance> {
3554        let catalog_manager = catalog::memory::MemoryCatalogManager::new_with_table(source_table);
3555        test_instance_with_catalog_manager(
3556            catalog_manager,
3557            target_table,
3558            plugins,
3559            metric_names_table,
3560            Arc::new(NoopProcedureExecutor),
3561        )
3562        .await
3563    }
3564
3565    /// Builds a test frontend `Instance` over the given (already source-registered)
3566    /// catalog manager, completing DDL tasks through `procedure_executor`.
3567    async fn test_instance_with_catalog_manager(
3568        catalog_manager: Arc<catalog::memory::MemoryCatalogManager>,
3569        target_table: TableRef,
3570        plugins: Plugins,
3571        metric_names_table: Option<TableRef>,
3572        procedure_executor: ProcedureExecutorRef,
3573    ) -> TestResult<Instance> {
3574        let kv_backend = Arc::new(MemoryKvBackend::new());
3575        let process_manager = Arc::new(ProcessManager::new("test-frontend".to_string(), None));
3576        let target_table_name = "target";
3577        catalog_manager
3578            .register_table_sync(catalog::RegisterTableRequest {
3579                catalog: "greptime".to_string(),
3580                schema: "public".to_string(),
3581                table_name: target_table_name.to_string(),
3582                table_id: 1025,
3583                table: target_table,
3584            })
3585            .with_context(|_| RegisterTableSnafu {
3586                table_name: target_table_name.to_string(),
3587            })?;
3588        if let Some(table) = metric_names_table {
3589            catalog_manager
3590                .deregister_table_sync(catalog::DeregisterTableRequest {
3591                    catalog: "greptime".to_string(),
3592                    schema: "information_schema".to_string(),
3593                    table_name: "tables".to_string(),
3594                })
3595                .unwrap();
3596            catalog_manager
3597                .register_table_sync(catalog::RegisterTableRequest {
3598                    catalog: "greptime".to_string(),
3599                    schema: "information_schema".to_string(),
3600                    table_name: "tables".to_string(),
3601                    table_id: 2048,
3602                    table,
3603                })
3604                .unwrap();
3605        }
3606        catalog_manager.register_process_list_table(process_manager.clone());
3607
3608        let cache_registry = test_cache_registry(kv_backend.clone())?;
3609
3610        FrontendBuilder::new(
3611            FrontendOptions::default(),
3612            kv_backend,
3613            cache_registry,
3614            catalog_manager,
3615            Arc::new(client::client_manager::NodeClients::default()),
3616            procedure_executor,
3617            process_manager,
3618        )
3619        .with_plugin(plugins)
3620        .try_build()
3621        .await
3622        .context(BuildFrontendSnafu)
3623    }
3624
3625    async fn execute_one_sql(
3626        instance: &Instance,
3627        sql: &str,
3628        query_ctx: QueryContextRef,
3629    ) -> TestResult<Output> {
3630        let mut results = instance.do_query_inner(sql, query_ctx).await;
3631        ensure!(
3632            results.len() == 1,
3633            UnexpectedOutputCountSnafu {
3634                sql: sql.to_string(),
3635                actual: results.len(),
3636            }
3637        );
3638        results.remove(0).with_context(|_| ExecuteSqlSnafu {
3639            sql: sql.to_string(),
3640        })
3641    }
3642
3643    fn assert_permission_denied<T>(result: servers::error::Result<T>) {
3644        let err = match result {
3645            Ok(_) => panic!("request should be rejected"),
3646            Err(err) => err,
3647        };
3648        assert_eq!(StatusCode::PermissionDenied, err.status_code());
3649    }
3650
3651    fn assert_action_checked(
3652        checker: &RejectEndpointPermissionChecker,
3653        action: PermissionAction,
3654        targets: Option<PermissionTableTargets>,
3655    ) {
3656        assert_eq!(CheckedAction { action, targets }, checker.take_check());
3657    }
3658
3659    #[tokio::test]
3660    async fn database_export_authorizes_all_tables_before_preparation() -> TestResult<()> {
3661        struct ExportAcl(std::sync::Mutex<Vec<PermissionTableTargets>>);
3662        impl PermissionChecker for ExportAcl {
3663            fn check_permission(
3664                &self,
3665                _: UserInfoRef,
3666                req: PermissionReq,
3667            ) -> auth::error::Result<PermissionResp> {
3668                assert!(matches!(
3669                    req,
3670                    PermissionReq::SqlStatement(Statement::Copy(
3671                        sql::statements::copy::Copy::CopyDatabase(
3672                            sql::statements::copy::CopyDatabase::To(_)
3673                        )
3674                    ))
3675                ));
3676                Ok(PermissionResp::Allow)
3677            }
3678            fn check_permission_with_table_targets(
3679                &self,
3680                _: UserInfoRef,
3681                req: PermissionReq,
3682                targets: PermissionTableTargets,
3683            ) -> auth::error::Result<PermissionResp> {
3684                self.0.lock().unwrap().push(targets.clone());
3685                let PermissionTableTargets::Resolved(tables) = targets else {
3686                    panic!("unresolved export")
3687                };
3688                if req.is_readonly() {
3689                    assert!(matches!(req, PermissionReq::Action(action) if action == COPY_TO));
3690                    // The first table is readable; the later table has only write access.
3691                    Ok(if tables.iter().any(|t| t.table == "target") {
3692                        PermissionResp::Reject
3693                    } else {
3694                        PermissionResp::Allow
3695                    })
3696                } else {
3697                    self.check_permission(QueryContext::arc().current_user(), req)
3698                }
3699            }
3700        }
3701        let checker = Arc::new(ExportAcl(Default::default()));
3702        let plugins = Plugins::new();
3703        plugins.insert::<PermissionCheckerRef>(checker.clone());
3704        let instance = test_instance_with_plugins(
3705            test_logical_table(1024, "source")?,
3706            test_table(1025, "target")?,
3707            plugins,
3708        )
3709        .await?;
3710        let req = table::requests::CopyDatabaseRequest {
3711            catalog_name: "greptime".into(),
3712            schema_name: "public".into(),
3713            location: "invalid-destination".into(),
3714            with: Default::default(),
3715            connection: Default::default(),
3716            time_range: None,
3717        };
3718        let result = instance
3719            .export_database_for_test(
3720                req.clone(),
3721                None,
3722                &tokio_util::sync::CancellationToken::new(),
3723                QueryContext::arc(),
3724            )
3725            .await;
3726        assert!(matches!(result, Err(Error::Permission { .. })));
3727        let expected = PermissionTableTargets::resolved(vec![
3728            PermissionTableTarget::new("greptime", "public", "source"),
3729            PermissionTableTarget::new("greptime", "public", "target"),
3730        ]);
3731        assert_eq!(*checker.0.lock().unwrap(), vec![expected.clone(), expected]);
3732        // An empty selection must still check the operation privilege.
3733        instance
3734            .plugins
3735            .map_mut::<PermissionCheckerRef, _, _>(|checker| {
3736                *checker.unwrap() = Arc::new(WriteOnlyPermissionChecker)
3737            });
3738        let result = instance
3739            .export_database_for_test(
3740                req,
3741                Some(&[]),
3742                &tokio_util::sync::CancellationToken::new(),
3743                QueryContext::arc(),
3744            )
3745            .await;
3746        assert!(matches!(result, Err(Error::Permission { .. })));
3747        Ok(())
3748    }
3749
3750    #[tokio::test]
3751    async fn test_prom_remote_read_with_custom_timestamp_and_value_columns() -> TestResult<()> {
3752        let schema = Arc::new(GtSchema::new(vec![
3753            ColumnSchema::new(
3754                "custom_ts",
3755                ConcreteDataType::timestamp_millisecond_datatype(),
3756                false,
3757            )
3758            .with_time_index(true),
3759            ColumnSchema::new("custom_value", ConcreteDataType::float64_datatype(), false),
3760        ]));
3761        let recordbatch = RecordBatch::new(
3762            schema,
3763            vec![
3764                Arc::new(TimestampMillisecondVector::from_vec(vec![1000, 2000, 3000])) as VectorRef,
3765                Arc::new(Float64Vector::from_vec(vec![1.0, 2.0, 3.0])) as VectorRef,
3766            ],
3767        )
3768        .unwrap();
3769        let instance = test_instance_with_tables(
3770            MemTable::table("custom_metric", recordbatch),
3771            test_table(1025, "target")?,
3772        )
3773        .await?;
3774
3775        let response = PromStoreProtocolHandler::read(
3776            &instance,
3777            ReadRequest {
3778                queries: vec![RemoteQuery {
3779                    start_timestamp_ms: 1500,
3780                    end_timestamp_ms: 2500,
3781                    matchers: vec![LabelMatcher {
3782                        r#type: PromMatcherType::Eq as i32,
3783                        name: servers::prom_store::METRIC_NAME_LABEL.to_string(),
3784                        value: "custom_metric".to_string(),
3785                    }],
3786                    ..Default::default()
3787                }],
3788                ..Default::default()
3789            },
3790            test_query_ctx(1),
3791        )
3792        .await
3793        .unwrap();
3794        let body = servers::prom_store::snappy_decompress(&response.body).unwrap();
3795        let response = ReadResponse::decode(body.as_slice()).unwrap();
3796
3797        assert_eq!(1, response.results.len());
3798        assert_eq!(1, response.results[0].timeseries.len());
3799        let timeseries = &response.results[0].timeseries[0];
3800        assert_eq!(
3801            vec![Label {
3802                name: servers::prom_store::METRIC_NAME_LABEL.to_string(),
3803                value: "custom_metric".to_string(),
3804            }],
3805            timeseries.labels
3806        );
3807        assert_eq!(
3808            vec![Sample {
3809                value: 2.0,
3810                timestamp: 2000,
3811            }],
3812            timeseries.samples
3813        );
3814
3815        Ok(())
3816    }
3817
3818    #[tokio::test]
3819    async fn test_prom_remote_read_prefers_default_value_column() -> TestResult<()> {
3820        let schema = Arc::new(GtSchema::new(vec![
3821            ColumnSchema::new(
3822                "custom_ts",
3823                ConcreteDataType::timestamp_millisecond_datatype(),
3824                false,
3825            )
3826            .with_time_index(true),
3827            ColumnSchema::new("extra_field", ConcreteDataType::string_datatype(), false),
3828            ColumnSchema::new(
3829                greptime_value(),
3830                ConcreteDataType::float64_datatype(),
3831                false,
3832            ),
3833        ]));
3834        let recordbatch = RecordBatch::new(
3835            schema,
3836            vec![
3837                Arc::new(TimestampMillisecondVector::from_vec(vec![1000, 2000, 3000])) as VectorRef,
3838                Arc::new(StringVector::from(vec!["99", "99", "99"])) as VectorRef,
3839                Arc::new(Float64Vector::from_vec(vec![1.0, 2.0, 3.0])) as VectorRef,
3840            ],
3841        )
3842        .unwrap();
3843        let instance = test_instance_with_tables(
3844            MemTable::table("multi_field_metric", recordbatch),
3845            test_table(1025, "target")?,
3846        )
3847        .await?;
3848
3849        let response = PromStoreProtocolHandler::read(
3850            &instance,
3851            ReadRequest {
3852                queries: vec![RemoteQuery {
3853                    start_timestamp_ms: 1500,
3854                    end_timestamp_ms: 2500,
3855                    matchers: vec![LabelMatcher {
3856                        r#type: PromMatcherType::Eq as i32,
3857                        name: servers::prom_store::METRIC_NAME_LABEL.to_string(),
3858                        value: "multi_field_metric".to_string(),
3859                    }],
3860                    ..Default::default()
3861                }],
3862                ..Default::default()
3863            },
3864            test_query_ctx(1),
3865        )
3866        .await
3867        .unwrap();
3868        let body = servers::prom_store::snappy_decompress(&response.body).unwrap();
3869        let response = ReadResponse::decode(body.as_slice()).unwrap();
3870
3871        assert_eq!(1, response.results.len());
3872        assert_eq!(1, response.results[0].timeseries.len());
3873        let timeseries = &response.results[0].timeseries[0];
3874        assert_eq!(
3875            vec![
3876                Label {
3877                    name: servers::prom_store::METRIC_NAME_LABEL.to_string(),
3878                    value: "multi_field_metric".to_string(),
3879                },
3880                Label {
3881                    name: "extra_field".to_string(),
3882                    value: "99".to_string(),
3883                },
3884            ],
3885            timeseries.labels
3886        );
3887        assert_eq!(
3888            vec![Sample {
3889                value: 2.0,
3890                timestamp: 2000,
3891            }],
3892            timeseries.samples
3893        );
3894
3895        Ok(())
3896    }
3897
3898    #[tokio::test]
3899    async fn test_prom_remote_read_rejects_ambiguous_value_columns() -> TestResult<()> {
3900        let schema = Arc::new(GtSchema::new(vec![
3901            ColumnSchema::new(
3902                "custom_ts",
3903                ConcreteDataType::timestamp_millisecond_datatype(),
3904                false,
3905            )
3906            .with_time_index(true),
3907            ColumnSchema::new("field_a", ConcreteDataType::float64_datatype(), false),
3908            ColumnSchema::new("field_b", ConcreteDataType::float64_datatype(), false),
3909        ]));
3910        let recordbatch = RecordBatch::new(
3911            schema,
3912            vec![
3913                Arc::new(TimestampMillisecondVector::from_vec(vec![1000])) as VectorRef,
3914                Arc::new(Float64Vector::from_vec(vec![1.0])) as VectorRef,
3915                Arc::new(Float64Vector::from_vec(vec![2.0])) as VectorRef,
3916            ],
3917        )
3918        .unwrap();
3919        let instance = test_instance_with_tables(
3920            MemTable::table("ambiguous_metric", recordbatch),
3921            test_table(1025, "target")?,
3922        )
3923        .await?;
3924
3925        let err = PromStoreProtocolHandler::read(
3926            &instance,
3927            ReadRequest {
3928                queries: vec![RemoteQuery {
3929                    matchers: vec![LabelMatcher {
3930                        r#type: PromMatcherType::Eq as i32,
3931                        name: servers::prom_store::METRIC_NAME_LABEL.to_string(),
3932                        value: "ambiguous_metric".to_string(),
3933                    }],
3934                    ..Default::default()
3935                }],
3936                ..Default::default()
3937            },
3938            test_query_ctx(1),
3939        )
3940        .await
3941        .err()
3942        .expect("ambiguous value columns should fail remote read");
3943
3944        assert_eq!(StatusCode::InvalidArguments, err.status_code());
3945        assert!(format!("{err:?}").contains("Ambiguous value column"));
3946
3947        Ok(())
3948    }
3949
3950    #[tokio::test]
3951    async fn test_restricted_endpoint_handlers_check_permissions() -> TestResult<()> {
3952        let checker = Arc::new(RejectEndpointPermissionChecker::default());
3953        let plugins = Plugins::new();
3954        plugins.insert::<PermissionCheckerRef>(checker.clone());
3955        let instance = test_instance_with_plugins(
3956            test_table(1024, "denied")?,
3957            test_table(1025, "target")?,
3958            plugins,
3959        )
3960        .await?;
3961        let mut ctx = test_query_ctx(1);
3962        Arc::get_mut(&mut ctx).unwrap().set_extension(
3963            servers::http::jaeger::JAEGER_QUERY_TABLE_NAME_KEY,
3964            "denied".to_string(),
3965        );
3966        let jaeger_targets = Some(PermissionTableTargets::resolved(vec![
3967            PermissionTableTarget::new("greptime", "public", "denied"),
3968        ]));
3969
3970        assert_permission_denied(JaegerQueryHandler::get_services(&instance, ctx.clone()).await);
3971        assert_action_checked(&checker, JAEGER_QUERY, jaeger_targets.clone());
3972        assert_permission_denied(
3973            JaegerQueryHandler::get_operations(&instance, ctx.clone(), "service", None).await,
3974        );
3975        assert_action_checked(&checker, JAEGER_QUERY, jaeger_targets.clone());
3976        assert_permission_denied(
3977            JaegerQueryHandler::get_trace(&instance, ctx.clone(), "trace", None, None, None).await,
3978        );
3979        assert_action_checked(&checker, JAEGER_QUERY, jaeger_targets.clone());
3980        assert_permission_denied(
3981            JaegerQueryHandler::find_traces(
3982                &instance,
3983                ctx.clone(),
3984                servers::http::jaeger::QueryTraceParams {
3985                    service_name: Some("service".to_string()),
3986                    ..Default::default()
3987                },
3988            )
3989            .await,
3990        );
3991        assert_action_checked(&checker, JAEGER_QUERY, jaeger_targets);
3992
3993        assert_permission_denied(
3994            PipelineHandler::get_pipeline_str(&instance, "pipeline", None, ctx.clone()).await,
3995        );
3996        assert_action_checked(&checker, PIPELINE_QUERY, None);
3997        assert_permission_denied(
3998            PipelineHandler::insert_pipeline(
3999                &instance,
4000                "pipeline",
4001                "application/yaml",
4002                "",
4003                ctx.clone(),
4004            )
4005            .await,
4006        );
4007        assert_action_checked(&checker, PIPELINE_INSERT, None);
4008        assert_permission_denied(
4009            PipelineHandler::delete_pipeline(&instance, "pipeline", None, ctx.clone()).await,
4010        );
4011        assert_action_checked(&checker, PIPELINE_DELETE, None);
4012        let app = axum::Router::new()
4013            .route(
4014                "/pipelines/_dryrun",
4015                axum::routing::post(servers::http::event::pipeline_dryrun),
4016            )
4017            .with_state(servers::http::event::LogState {
4018                log_handler: Arc::new(instance.clone()),
4019                log_validator: None,
4020                ingest_interceptor: None,
4021            })
4022            .layer(axum::Extension((*ctx).clone()));
4023        let response = app
4024            .oneshot(
4025                axum::http::Request::post("/pipelines/_dryrun")
4026                    .header("content-type", "application/json")
4027                    .body(axum::body::Body::from("{}"))
4028                    .unwrap(),
4029            )
4030            .await
4031            .unwrap();
4032        assert_eq!(axum::http::StatusCode::FORBIDDEN, response.status());
4033        assert_action_checked(&checker, PIPELINE_QUERY, None);
4034
4035        assert_permission_denied(
4036            DashboardHandler::save(&instance, "dashboard", "{}", ctx.clone()).await,
4037        );
4038        assert_action_checked(&checker, DASHBOARD_SAVE, None);
4039        assert_permission_denied(DashboardHandler::list(&instance, ctx.clone()).await);
4040        assert_action_checked(&checker, DASHBOARD_QUERY, None);
4041        assert_permission_denied(
4042            DashboardHandler::delete(&instance, "dashboard", ctx.clone()).await,
4043        );
4044        assert_action_checked(&checker, DASHBOARD_DELETE, None);
4045
4046        Ok(())
4047    }
4048
4049    #[tokio::test]
4050    async fn test_write_only_ingestion_loads_named_pipeline() -> TestResult<()> {
4051        let plugins = Plugins::new();
4052        plugins.insert::<PermissionCheckerRef>(Arc::new(WriteOnlyPermissionChecker));
4053        let instance = test_instance_with_plugins(
4054            test_table(1024, "source")?,
4055            test_table(1025, "target")?,
4056            plugins,
4057        )
4058        .await?;
4059        instance
4060            .catalog_manager()
4061            .as_any()
4062            .downcast_ref::<catalog::memory::MemoryCatalogManager>()
4063            .unwrap()
4064            .register_table_sync(catalog::RegisterTableRequest {
4065                catalog: "greptime".to_string(),
4066                schema: DEFAULT_PRIVATE_SCHEMA_NAME.to_string(),
4067                table_name: "pipelines".to_string(),
4068                table_id: 2049,
4069                table: test_pipeline_table(),
4070            })
4071            .with_context(|_| RegisterTableSnafu {
4072                table_name: "pipelines".to_string(),
4073            })?;
4074        let ctx = test_query_ctx(1);
4075        let handler: PipelineHandlerRef = Arc::new(instance.clone());
4076        let hits = CACHE_HIT.with_label_values(&["pipeline_table"]);
4077        let misses = CACHE_MISS.with_label_values(&["pipeline_table"]);
4078        let before = (hits.get(), misses.get());
4079
4080        handler
4081            .get_pipeline("pipeline", None, ctx.clone())
4082            .await
4083            .unwrap();
4084        assert_eq!((hits.get(), misses.get()), (before.0 + 1, before.1 + 1));
4085        handler
4086            .get_pipeline("pipeline", None, ctx.clone())
4087            .await
4088            .unwrap();
4089        assert_eq!((hits.get(), misses.get()), (before.0 + 3, before.1 + 1));
4090        assert_permission_denied(
4091            PipelineHandler::get_pipeline_str(&instance, "pipeline", None, ctx.clone()).await,
4092        );
4093
4094        let app = axum::Router::new()
4095            .route(
4096                "/pipelines/_dryrun",
4097                axum::routing::post(servers::http::event::pipeline_dryrun),
4098            )
4099            .with_state(servers::http::event::LogState {
4100                log_handler: handler,
4101                log_validator: None,
4102                ingest_interceptor: None,
4103            })
4104            .layer(axum::Extension((*ctx).clone()));
4105        let response = app
4106            .oneshot(
4107                axum::http::Request::post("/pipelines/_dryrun")
4108                    .header("content-type", "application/json")
4109                    .body(axum::body::Body::from("{}"))
4110                    .unwrap(),
4111            )
4112            .await
4113            .unwrap();
4114        assert_eq!(axum::http::StatusCode::FORBIDDEN, response.status());
4115
4116        Ok(())
4117    }
4118
4119    #[tokio::test]
4120    async fn test_copy_export_read_action_has_source_targets() -> TestResult<()> {
4121        let checker = Arc::new(RejectEndpointPermissionChecker::default());
4122        let plugins = Plugins::new();
4123        plugins.insert::<PermissionCheckerRef>(checker.clone());
4124        let instance = test_instance_with_plugins(
4125            test_table(1024, "source")?,
4126            test_physical_table(1025, "target")?,
4127            plugins,
4128        )
4129        .await?;
4130        for (sql, name, physical) in [
4131            ("COPY source TO '/unused.parquet'", "source", false),
4132            (
4133                "COPY (SELECT * FROM source) TO '/unused.parquet'",
4134                "source",
4135                false,
4136            ),
4137            ("COPY target TO '/unused.parquet'", "target", true),
4138        ] {
4139            assert_eq!(
4140                StatusCode::PermissionDenied,
4141                instance
4142                    .check_sql_permission(&parse_one_sql(sql), &QueryContext::arc())
4143                    .await
4144                    .unwrap_err()
4145                    .status_code(),
4146            );
4147            assert_action_checked(
4148                &checker,
4149                COPY_TO,
4150                Some(PermissionTableTargets::resolved(vec![
4151                    PermissionTableTarget::new("greptime", "public", name)
4152                        .with_physical_metric_table(physical),
4153                ])),
4154            );
4155        }
4156        Ok(())
4157    }
4158
4159    #[tokio::test]
4160    async fn test_copy_export_requires_read_and_write_permission() -> TestResult<()> {
4161        use auth::UserProvider;
4162
4163        let plugins = Plugins::new();
4164        plugins.insert::<PermissionCheckerRef>(auth::DefaultPermissionChecker::arc());
4165        let instance = test_instance_with_plugins(
4166            test_table(1024, "source")?,
4167            test_table(1025, "target")?,
4168            plugins,
4169        )
4170        .await?;
4171        let provider = auth::static_user_provider_from_option(
4172            "static_user_provider:cmd:reader:ro=pwd,writer:wo=pwd,owner:rw=pwd",
4173        )
4174        .unwrap();
4175        for (username, can_export, can_import) in [
4176            ("reader", false, false),
4177            ("writer", false, true),
4178            ("owner", true, true),
4179        ] {
4180            let ctx = QueryContext::arc();
4181            ctx.set_current_user(
4182                provider
4183                    .authenticate(
4184                        auth::Identity::UserId(username, None),
4185                        auth::Password::PlainText("pwd".to_string().into()),
4186                    )
4187                    .await
4188                    .unwrap(),
4189            );
4190            for sql in [
4191                "COPY source TO '/unused.parquet'",
4192                "COPY (SELECT * FROM source) TO '/unused.parquet'",
4193            ] {
4194                let result = instance
4195                    .check_sql_permission(&parse_one_sql(sql), &ctx)
4196                    .await;
4197                if can_export {
4198                    result.unwrap();
4199                } else {
4200                    assert_eq!(
4201                        StatusCode::PermissionDenied,
4202                        result.unwrap_err().status_code()
4203                    );
4204                }
4205            }
4206            let result = instance
4207                .check_sql_permission(&parse_one_sql("COPY target FROM '/unused.parquet'"), &ctx)
4208                .await;
4209            if can_import {
4210                result.unwrap();
4211            } else {
4212                assert_eq!(
4213                    StatusCode::PermissionDenied,
4214                    result.unwrap_err().status_code()
4215                );
4216            }
4217        }
4218        Ok(())
4219    }
4220
4221    #[tokio::test]
4222    async fn test_sql_schema_selection_authorizes_before_mutation() -> TestResult<()> {
4223        struct SchemaProvider;
4224        #[async_trait::async_trait]
4225        impl auth::UserProvider for SchemaProvider {
4226            fn name(&self) -> &str {
4227                "schema-test"
4228            }
4229            async fn authenticate(
4230                &self,
4231                _: auth::Identity<'_>,
4232                _: auth::Password<'_>,
4233            ) -> auth::error::Result<UserInfoRef> {
4234                unreachable!("the request already has an authenticated user")
4235            }
4236            async fn authorize(
4237                &self,
4238                _: &str,
4239                schema: &str,
4240                _: &UserInfoRef,
4241            ) -> auth::error::Result<()> {
4242                if schema == "backend_error" {
4243                    return auth::error::InternalStateSnafu {
4244                        msg: "backend unavailable",
4245                    }
4246                    .fail();
4247                }
4248                if matches!(schema, "public" | "allowed" | "allowed_missing" | "admin") {
4249                    Ok(())
4250                } else {
4251                    auth::error::PermissionDeniedSnafu.fail()
4252                }
4253            }
4254        }
4255        let plugins = Plugins::new();
4256        plugins.insert::<auth::UserProviderRef>(Arc::new(SchemaProvider));
4257        let catalog =
4258            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "source")?);
4259        for schema in ["denied", "allowed", "admin"] {
4260            catalog
4261                .register_schema_sync(catalog::RegisterSchemaRequest {
4262                    catalog: "greptime".to_string(),
4263                    schema: schema.to_string(),
4264                })
4265                .unwrap();
4266        }
4267        let instance = test_instance_with_catalog_manager(
4268            catalog,
4269            test_table(1025, "target")?,
4270            plugins,
4271            None,
4272            Arc::new(NoopProcedureExecutor),
4273        )
4274        .await?;
4275        for sql in [
4276            "Use denied",
4277            " USE denied",
4278            "/* switch */ USE denied",
4279            "USE public; Use denied",
4280            "USE missing",
4281        ] {
4282            let ctx = Arc::new(QueryContext::with_channel(
4283                "greptime",
4284                "public",
4285                Channel::Postgres,
4286            ));
4287            let results = instance.do_query_inner(sql, ctx.clone()).await;
4288            assert_eq!(
4289                StatusCode::PermissionDenied,
4290                results.last().unwrap().as_ref().unwrap_err().status_code(),
4291                "{sql}"
4292            );
4293            assert_eq!("public", ctx.current_schema());
4294        }
4295        for sql in ["USE allowed", "SET search_path TO allowed"] {
4296            let ctx = Arc::new(QueryContext::with_channel(
4297                "greptime",
4298                "public",
4299                Channel::Postgres,
4300            ));
4301            assert!(
4302                instance
4303                    .do_query_inner(sql, ctx.clone())
4304                    .await
4305                    .remove(0)
4306                    .is_ok()
4307            );
4308            assert_eq!("allowed", ctx.current_schema());
4309        }
4310        let ctx = Arc::new(QueryContext::with_channel(
4311            "greptime",
4312            "public",
4313            Channel::Postgres,
4314        ));
4315        let error = instance
4316            .do_query_inner("USE allowed_missing", ctx.clone())
4317            .await
4318            .remove(0)
4319            .unwrap_err();
4320        assert_eq!(StatusCode::InvalidArguments, error.status_code());
4321        assert_eq!("public", ctx.current_schema());
4322        for (sql, expected) in [
4323            ("SET search_path TO denied, public", "public"),
4324            ("SET search_path TO DEFAULT", "admin"),
4325            ("SET search_path TO allowed_missing, allowed", "allowed"),
4326            (
4327                "SET search_path TO denied, allowed_missing, public",
4328                "public",
4329            ),
4330            (r#"SET search_path TO "$user", public"#, "admin"),
4331            (r#"SET "search_path" TO denied, allowed"#, "allowed"),
4332            ("SET SEARCH_PATH TO 'denied'", ""),
4333            ("SET search_path TO allowed_missing", ""),
4334            ("SET search_path TO ''", ""),
4335            ("SET search_path TO allowed; SET search_path TO denied", ""),
4336        ] {
4337            let ctx = Arc::new(QueryContext::with_channel(
4338                "greptime",
4339                "public",
4340                Channel::Postgres,
4341            ));
4342            ctx.set_current_user(Arc::new(AdminUserInfo));
4343            let results = instance.do_query_inner(sql, ctx.clone()).await;
4344            assert!(
4345                results.iter().all(|result| matches!(
4346                    result,
4347                    Ok(Output {
4348                        data: OutputData::AffectedRows(0),
4349                        ..
4350                    })
4351                )),
4352                "{sql}: {results:?}"
4353            );
4354            assert_eq!(expected, ctx.current_schema(), "{sql}");
4355        }
4356
4357        let ctx = Arc::new(QueryContext::with_channel(
4358            "greptime",
4359            "public",
4360            Channel::Postgres,
4361        ));
4362        // The default user's schema is absent or unauthorized, so public remains usable.
4363        assert!(
4364            instance
4365                .do_query_inner(r#"SET search_path TO "$user", public"#, ctx.clone())
4366                .await
4367                .remove(0)
4368                .is_ok()
4369        );
4370        assert_eq!("public", ctx.current_schema());
4371        assert!(
4372            instance
4373                .do_query_inner("SET search_path TO DEFAULT", ctx.clone())
4374                .await
4375                .remove(0)
4376                .is_ok()
4377        );
4378        assert_eq!("public", ctx.current_schema());
4379        let error = instance
4380            .do_query_inner("SET search_path TO backend_error, allowed", ctx.clone())
4381            .await
4382            .remove(0)
4383            .unwrap_err();
4384        assert_eq!(StatusCode::Unexpected, error.status_code());
4385        assert_eq!("public", ctx.current_schema());
4386        Ok(())
4387    }
4388
4389    #[tokio::test]
4390    async fn test_mysql_without_selected_database() -> TestResult<()> {
4391        let plugins = Plugins::new();
4392        plugins.insert::<auth::UserProviderRef>(Arc::new(auth::tests::MockUserProvider::default()));
4393        let instance = test_instance_with_plugins(
4394            test_table(1024, "source")?,
4395            test_table(1025, "target")?,
4396            plugins,
4397        )
4398        .await?;
4399        let ctx = Arc::new(QueryContext::with_channel("greptime", "", Channel::Mysql));
4400        for sql in [
4401            "SELECT * FROM source",
4402            "INSERT INTO source VALUES (1, 0)",
4403            "CREATE TABLE t (ts TIMESTAMP TIME INDEX)",
4404            "SHOW TABLES",
4405            "TRUNCATE TABLE source",
4406            "SHOW CREATE TABLE source",
4407            "COPY source TO '/tmp/out'",
4408            "SELECT flush_table('public.source')",
4409        ] {
4410            let result = instance.do_query_inner(sql, ctx.clone()).await.remove(0);
4411            let error = match result {
4412                Err(error) => Err::<(), _>(BoxedError::new(error))
4413                    .context(servers::error::ExecuteQuerySnafu)
4414                    .unwrap_err(),
4415                Ok(output) => {
4416                    let OutputData::Stream(stream) = output.data else {
4417                        panic!("Expected a query stream: {sql}");
4418                    };
4419                    common_recordbatch::util::collect(stream)
4420                        .await
4421                        .context(servers::error::CollectRecordbatchSnafu)
4422                        .unwrap_err()
4423                }
4424            };
4425            let (kind, message) = servers::mysql::writer::handle_err(error, ctx.clone());
4426            assert_eq!(1046, kind as u16, "{sql}: {message}");
4427            assert!(message.contains("No database selected"), "{message}");
4428        }
4429        let output = instance
4430            .do_query_inner(
4431                "SELECT 1, DATABASE() IS NULL AS no_database, SCHEMA() IS NULL AS no_schema",
4432                ctx.clone(),
4433            )
4434            .await
4435            .remove(0)
4436            .unwrap();
4437        assert_eq!(2, output.data.pretty_print().await.matches("true").count());
4438        for sql in [
4439            "SELECT * FROM denied.source",
4440            "SHOW TABLES FROM denied",
4441            "USE denied",
4442        ] {
4443            let error = instance
4444                .do_query_inner(sql, ctx.clone())
4445                .await
4446                .remove(0)
4447                .unwrap_err();
4448            assert_eq!(
4449                StatusCode::AccessDenied,
4450                error.status_code(),
4451                "{sql}: {error}"
4452            );
4453            assert!(ctx.current_schema().is_empty());
4454        }
4455        for sql in [
4456            "SELECT * FROM public.source",
4457            "SHOW TABLES FROM public",
4458            "SET time_zone = '+00:00'",
4459        ] {
4460            assert!(
4461                instance
4462                    .do_query_inner(sql, ctx.clone())
4463                    .await
4464                    .remove(0)
4465                    .is_ok(),
4466                "{sql}"
4467            );
4468        }
4469        let plan = instance
4470            .do_describe_inner(parse_one_sql("SELECT * FROM public.source"), ctx.clone())
4471            .await
4472            .unwrap()
4473            .unwrap()
4474            .logical_plan;
4475        instance
4476            .do_exec_plan_inner(
4477                plan.clone(),
4478                Some(parse_one_sql("SELECT * FROM public.source")),
4479                ctx.clone(),
4480            )
4481            .await
4482            .unwrap();
4483        let error = instance
4484            .do_describe_inner(parse_one_sql("SELECT * FROM denied.source"), ctx.clone())
4485            .await
4486            .unwrap_err();
4487        assert_eq!(StatusCode::AccessDenied, error.status_code());
4488        assert!(ctx.current_schema().is_empty());
4489        let error = instance
4490            .do_exec_plan_inner(plan.clone(), None, ctx.clone())
4491            .await
4492            .unwrap_err();
4493        assert_eq!(StatusCode::DatabaseNotFound, error.status_code());
4494        let current_user = ctx.current_user();
4495        ctx.set_current_user(Arc::new(AdminUserInfo));
4496        let error = instance
4497            .do_exec_plan_inner(
4498                plan,
4499                Some(parse_one_sql("SELECT * FROM public.source")),
4500                ctx.clone(),
4501            )
4502            .await
4503            .unwrap_err();
4504        assert_eq!(StatusCode::AccessDenied, error.status_code());
4505        ctx.set_current_user(current_user);
4506        instance
4507            .do_query_inner("USE public", ctx.clone())
4508            .await
4509            .remove(0)
4510            .unwrap();
4511        assert_eq!("public", ctx.current_schema());
4512        instance
4513            .do_query_inner("SELECT * FROM source", ctx)
4514            .await
4515            .remove(0)
4516            .unwrap();
4517        Ok(())
4518    }
4519
4520    #[tokio::test]
4521    async fn test_empty_search_path_statements() -> TestResult<()> {
4522        let instance = test_instance_with_plugins(
4523            test_table(1024, "source")?,
4524            test_table(1025, "target")?,
4525            Plugins::new(),
4526        )
4527        .await?;
4528        let ctx = Arc::new(QueryContext::with_channel(
4529            "greptime",
4530            "public",
4531            Channel::Postgres,
4532        ));
4533        instance
4534            .do_query_inner("SET search_path TO ''", ctx.clone())
4535            .await
4536            .remove(0)
4537            .unwrap();
4538        let tables = instance
4539            .do_query_inner("SHOW TABLES", ctx.clone())
4540            .await
4541            .remove(0)
4542            .unwrap()
4543            .data
4544            .pretty_print()
4545            .await;
4546        assert!(
4547            !tables.contains("source") && !tables.contains("target"),
4548            "{tables}"
4549        );
4550        let error = instance
4551            .do_query_inner("INSERT INTO source VALUES (1, 0)", ctx.clone())
4552            .await
4553            .remove(0)
4554            .unwrap_err();
4555        assert!(
4556            matches!(error, Error::TableOperation { source: operator::error::Error::TableNotFound { ref table_name, .. }, .. } if table_name == "greptime..source"),
4557            "{error:?}"
4558        );
4559        for sql in [
4560            "CREATE TABLE no_schema (ts TIMESTAMP TIME INDEX)",
4561            "ALTER TABLE source ADD COLUMN no_schema_column INT",
4562            "CREATE VIEW no_schema_view AS SELECT 1 AS n",
4563            "CREATE FLOW no_schema_flow SINK TO sink AS SELECT * FROM public.source",
4564            "CREATE FLOW no_schema_flow SINK TO public.sink AS SELECT * FROM source",
4565        ] {
4566            let error = instance
4567                .do_query_inner(sql, ctx.clone())
4568                .await
4569                .remove(0)
4570                .unwrap_err();
4571            assert!(
4572                matches!(error, Error::TableOperation { source: operator::error::Error::SchemaNotFound { ref schema_info, .. }, .. } if schema_info.is_empty()),
4573                "{sql}: {error:?}"
4574            );
4575        }
4576
4577        for sql in ["SELECT * FROM source", "DELETE FROM source"] {
4578            assert!(
4579                instance
4580                    .do_query_inner(sql, ctx.clone())
4581                    .await
4582                    .remove(0)
4583                    .is_err(),
4584                "{sql}"
4585            );
4586        }
4587        let output = instance
4588            .do_query_inner(
4589                "SELECT current_schema() IS NULL AS empty, current_schemas(false) AS schemas",
4590                ctx.clone(),
4591            )
4592            .await
4593            .remove(0)
4594            .unwrap();
4595        let printed = output.data.pretty_print().await;
4596        assert!(
4597            printed.contains("true") && printed.contains("[]"),
4598            "{printed}"
4599        );
4600        let output = instance
4601            .do_query_inner("DROP TABLE IF EXISTS source", ctx.clone())
4602            .await
4603            .remove(0)
4604            .unwrap();
4605        assert!(matches!(output.data, OutputData::AffectedRows(0)));
4606        instance
4607            .do_query_inner("USE public", ctx.clone())
4608            .await
4609            .remove(0)
4610            .unwrap();
4611        assert!(
4612            instance
4613                .do_query_inner("SELECT * FROM source", ctx)
4614                .await
4615                .remove(0)
4616                .is_ok()
4617        );
4618        Ok(())
4619    }
4620
4621    #[tokio::test]
4622    async fn test_operator_rejects_search_path_without_frontend() -> TestResult<()> {
4623        let instance = test_instance_with_plugins(
4624            test_table(1024, "source")?,
4625            test_table(1025, "target")?,
4626            Plugins::new(),
4627        )
4628        .await?;
4629        let ctx = Arc::new(QueryContext::with_channel(
4630            "greptime",
4631            "public",
4632            Channel::Postgres,
4633        ));
4634        let error = instance
4635            .statement_executor
4636            .execute_sql(parse_one_sql("SET search_path TO denied"), ctx.clone())
4637            .await
4638            .unwrap_err();
4639        assert!(matches!(error, operator::error::Error::NotSupported { .. }));
4640        assert_eq!("public", ctx.current_schema());
4641        Ok(())
4642    }
4643
4644    #[tokio::test]
4645    async fn test_write_only_grpc_sql_is_checked_after_parsing() -> TestResult<()> {
4646        let plugins = Plugins::new();
4647        plugins.insert::<PermissionCheckerRef>(Arc::new(WriteOnlyPermissionChecker));
4648        let instance = test_instance_with_plugins(
4649            test_table(1024, "source")?,
4650            test_table(1025, "target")?,
4651            plugins,
4652        )
4653        .await?;
4654
4655        let insert = Request::Query(api::v1::QueryRequest {
4656            query: Some(Query::Sql(
4657                "INSERT INTO target SELECT * FROM source".to_string(),
4658            )),
4659        });
4660        servers::query_handler::grpc::GrpcQueryHandler::do_query(
4661            &instance,
4662            insert,
4663            QueryContext::arc(),
4664        )
4665        .await
4666        .unwrap();
4667
4668        let select = Request::Query(api::v1::QueryRequest {
4669            query: Some(Query::Sql("SELECT * FROM source".to_string())),
4670        });
4671        assert_permission_denied(
4672            servers::query_handler::grpc::GrpcQueryHandler::do_query(
4673                &instance,
4674                select,
4675                QueryContext::arc(),
4676            )
4677            .await,
4678        );
4679
4680        Ok(())
4681    }
4682
4683    #[tokio::test]
4684    async fn test_target_independent_checker_skips_target_resolution() -> TestResult<()> {
4685        let physical_table = "physical_metric";
4686        let checker = Arc::new(TargetIndependentPermissionChecker::default());
4687        let plugins = Plugins::new();
4688        plugins.insert::<PermissionCheckerRef>(checker.clone());
4689        let catalog = catalog::memory::MemoryCatalogManager::new_with_table(test_physical_table(
4690            1024,
4691            physical_table,
4692        )?);
4693        catalog
4694            .register_table_sync(catalog::RegisterTableRequest {
4695                catalog: "greptime".to_string(),
4696                schema: "public".to_string(),
4697                table_name: "target".to_string(),
4698                table_id: 1025,
4699                table: test_table(1025, "target")?,
4700            })
4701            .with_context(|_| RegisterTableSnafu {
4702                table_name: "target".to_string(),
4703            })?;
4704        let counting_catalog = Arc::new(CountingCatalogManager::new(catalog));
4705        let instance =
4706            test_instance_with_counting_catalog(counting_catalog.clone(), plugins).await?;
4707        counting_catalog.reset_counts();
4708
4709        let ctx = test_query_ctx(1);
4710        let physical_target = PermissionTableTarget::new("greptime", "public", physical_table);
4711        instance
4712            .check_resolved_table_permission(
4713                &ctx,
4714                PermissionReq::Action(PROMQL_QUERY),
4715                PermissionTableTargets::resolved(vec![physical_target]),
4716            )
4717            .await
4718            .unwrap();
4719        assert_eq!(0, counting_catalog.total_table_lookups());
4720        assert_eq!(1, checker.checks.load(atomic::Ordering::Relaxed));
4721        assert_eq!(
4722            vec![physical_table.to_string(), "target".to_string()],
4723            PrometheusHandler::filter_metadata_metric_names(
4724                &instance,
4725                vec![physical_table.to_string(), "target".to_string()],
4726                "public",
4727                &ctx,
4728            )
4729            .await
4730            .unwrap()
4731        );
4732        assert_eq!(0, counting_catalog.total_table_lookups());
4733        assert_eq!(2, checker.checks.load(atomic::Ordering::Relaxed));
4734
4735        Ok(())
4736    }
4737
4738    #[tokio::test]
4739    async fn test_query_permission_targets_are_deduplicated() -> TestResult<()> {
4740        struct RecordingTargetChecker(std::sync::Mutex<Vec<PermissionTableTargets>>);
4741        impl PermissionChecker for RecordingTargetChecker {
4742            fn check_permission(
4743                &self,
4744                _: UserInfoRef,
4745                _: PermissionReq,
4746            ) -> auth::error::Result<PermissionResp> {
4747                Ok(PermissionResp::Allow)
4748            }
4749
4750            fn check_permission_with_table_targets(
4751                &self,
4752                _: UserInfoRef,
4753                _: PermissionReq,
4754                targets: PermissionTableTargets,
4755            ) -> auth::error::Result<PermissionResp> {
4756                self.0.lock().unwrap().push(targets);
4757                Ok(PermissionResp::Allow)
4758            }
4759        }
4760
4761        let checker = Arc::new(RecordingTargetChecker(Default::default()));
4762        let plugins = Plugins::new();
4763        plugins.insert::<PermissionCheckerRef>(checker.clone());
4764        let catalog =
4765            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "source")?);
4766        catalog
4767            .register_table_sync(catalog::RegisterTableRequest {
4768                catalog: "greptime".to_string(),
4769                schema: "public".to_string(),
4770                table_name: "target".to_string(),
4771                table_id: 1025,
4772                table: test_table(1025, "target")?,
4773            })
4774            .with_context(|_| RegisterTableSnafu {
4775                table_name: "target".to_string(),
4776            })?;
4777        let counting_catalog = Arc::new(CountingCatalogManager::new(catalog));
4778        let instance =
4779            test_instance_with_counting_catalog(counting_catalog.clone(), plugins).await?;
4780        counting_catalog.reset_counts();
4781        let ctx = test_query_ctx(1);
4782        let target = PermissionTableTarget::new("greptime", "public", "target");
4783
4784        instance
4785            .check_resolved_table_permission(
4786                &ctx,
4787                PermissionReq::Action(PROMQL_QUERY),
4788                PermissionTableTargets::resolved(vec![target.clone(), target.clone()]),
4789            )
4790            .await
4791            .unwrap();
4792        assert_eq!(1, counting_catalog.table_calls());
4793        assert_eq!(0, counting_catalog.table_id_calls());
4794        assert_eq!(0, counting_catalog.table_info_by_id_calls());
4795        assert_eq!(
4796            *checker.0.lock().unwrap(),
4797            vec![PermissionTableTargets::Resolved(vec![target])]
4798        );
4799
4800        counting_catalog.reset_counts();
4801        instance
4802            .check_sql_permission(&parse_one_sql("COPY target TO '/unused.parquet'"), &ctx)
4803            .await
4804            .unwrap();
4805        assert_eq!(1, counting_catalog.total_table_lookups());
4806        assert_eq!(3, checker.0.lock().unwrap().len());
4807
4808        Ok(())
4809    }
4810
4811    #[tokio::test]
4812    async fn test_physical_target_permission_reaches_checker() -> TestResult<()> {
4813        struct PhysicalChecker {
4814            allow: bool,
4815        }
4816        impl PermissionChecker for PhysicalChecker {
4817            fn check_permission(
4818                &self,
4819                _: UserInfoRef,
4820                _: PermissionReq,
4821            ) -> auth::error::Result<PermissionResp> {
4822                Ok(PermissionResp::Allow)
4823            }
4824
4825            fn grpc_table_targets(
4826                &self,
4827                request: &api::v1::greptime_request::Request,
4828                catalog: &str,
4829                schema: &str,
4830            ) -> PermissionTableTargets {
4831                let api::v1::greptime_request::Request::Ddl(ddl) = request else {
4832                    panic!("expected DDL")
4833                };
4834                let Some(api::v1::ddl_request::Expr::CommentOn(comment)) = &ddl.expr else {
4835                    panic!("expected comment")
4836                };
4837                PermissionTableTargets::resolved(vec![PermissionTableTarget::new(
4838                    catalog,
4839                    schema,
4840                    &comment.object_name,
4841                )])
4842            }
4843
4844            fn check_permission_with_table_targets(
4845                &self,
4846                _: UserInfoRef,
4847                _: PermissionReq,
4848                targets: PermissionTableTargets,
4849            ) -> auth::error::Result<PermissionResp> {
4850                let PermissionTableTargets::Resolved(targets) = targets else {
4851                    panic!("physical target must retain its identity")
4852                };
4853                assert_eq!(targets.len(), 1);
4854                assert_eq!(targets[0].catalog, "greptime");
4855                assert_eq!(targets[0].schema, "public");
4856                assert_eq!(targets[0].table, "physical_metric");
4857                assert!(targets[0].is_physical_metric_table);
4858                Ok(if self.allow {
4859                    PermissionResp::Allow
4860                } else {
4861                    PermissionResp::Reject
4862                })
4863            }
4864        }
4865        for allow in [true, false] {
4866            let plugins = Plugins::new();
4867            plugins.insert::<PermissionCheckerRef>(Arc::new(PhysicalChecker { allow }));
4868            let instance = test_instance_with_plugins(
4869                test_physical_table(1024, "physical_metric")?,
4870                test_table(1025, "target")?,
4871                plugins,
4872            )
4873            .await?;
4874            let ctx = test_query_ctx(1);
4875            for sql in [
4876                "SHOW CREATE TABLE physical_metric",
4877                "SELECT * FROM physical_metric",
4878            ] {
4879                let result = instance
4880                    .check_sql_permission(&parse_one_sql(sql), &ctx)
4881                    .await;
4882                assert_eq!(allow, result.is_ok());
4883                if let Err(error) = result {
4884                    assert_eq!(StatusCode::PermissionDenied, error.status_code());
4885                }
4886            }
4887            if !allow {
4888                let request = api::v1::greptime_request::Request::Ddl(api::v1::DdlRequest {
4889                    expr: Some(api::v1::ddl_request::Expr::CommentOn(
4890                        api::v1::CommentOnExpr {
4891                            object_name: "physical_metric".to_string(),
4892                            object_type: api::v1::CommentObjectType::Table as i32,
4893                            ..Default::default()
4894                        },
4895                    )),
4896                });
4897                let err = servers::query_handler::grpc::GrpcQueryHandler::do_query(
4898                    &instance, request, ctx,
4899                )
4900                .await
4901                .unwrap_err();
4902                assert_eq!(StatusCode::PermissionDenied, err.status_code());
4903            }
4904        }
4905        Ok(())
4906    }
4907
4908    #[tokio::test]
4909    async fn test_physical_query_targets_fail_closed() -> TestResult<()> {
4910        let physical_table = "physical_metric";
4911        let plugins = Plugins::new();
4912        plugins.insert::<PermissionCheckerRef>(Arc::new(RejectUnresolvedPermissionChecker));
4913        let instance = test_instance_with_plugins(
4914            test_physical_table(1024, physical_table)?,
4915            test_table(1025, "target")?,
4916            plugins,
4917        )
4918        .await?;
4919
4920        let ctx = test_query_ctx(1);
4921        let logical_target = PermissionTableTarget::new("greptime", "public", "target");
4922        instance
4923            .check_resolved_table_permission(
4924                &ctx,
4925                PermissionReq::Action(PROMQL_QUERY),
4926                PermissionTableTargets::resolved(vec![logical_target.clone()]),
4927            )
4928            .await
4929            .unwrap();
4930        let physical_target = PermissionTableTarget::new("greptime", "public", physical_table)
4931            .with_physical_metric_table(true);
4932        let err = instance
4933            .check_resolved_table_permission(
4934                &ctx,
4935                PermissionReq::Action(PROMQL_QUERY),
4936                PermissionTableTargets::resolved(vec![
4937                    logical_target,
4938                    PermissionTableTarget::new("greptime", "public", physical_table),
4939                ]),
4940            )
4941            .await
4942            .unwrap_err();
4943        assert_eq!(StatusCode::PermissionDenied, err.status_code());
4944        assert_eq!(
4945            vec!["target".to_string()],
4946            PrometheusHandler::filter_metadata_metric_names(
4947                &instance,
4948                vec!["target".to_string(), "denied".to_string()],
4949                "public",
4950                &ctx,
4951            )
4952            .await
4953            .unwrap()
4954        );
4955
4956        let query = PromQuery {
4957            query: physical_table.to_string(),
4958            ..Default::default()
4959        };
4960        let err = PrometheusHandler::check_query_target_permission(
4961            &instance,
4962            PermissionTableTargets::resolved(vec![physical_target]),
4963            &ctx,
4964        )
4965        .await
4966        .unwrap_err();
4967        assert_eq!(StatusCode::PermissionDenied, err.status_code());
4968        let err = PrometheusHandler::check_query_permission(
4969            &instance,
4970            std::slice::from_ref(&query),
4971            &ctx,
4972        )
4973        .await
4974        .unwrap_err();
4975        assert_eq!(StatusCode::PermissionDenied, err.status_code());
4976        let err = PrometheusHandler::do_query(&instance, &query, ctx.clone())
4977            .await
4978            .unwrap_err();
4979        assert_eq!(StatusCode::PermissionDenied, err.status_code());
4980
4981        for sql in [
4982            "SELECT * FROM physical_metric",
4983            "TQL EVAL (0, 10, '5s') physical_metric",
4984            "INSERT INTO target SELECT * FROM physical_metric",
4985        ] {
4986            let mut results = instance.do_query_inner(sql, ctx.clone()).await;
4987            assert_eq!(1, results.len(), "{sql}");
4988            let err = results.remove(0).unwrap_err();
4989            assert_eq!(StatusCode::PermissionDenied, err.status_code(), "{sql}");
4990        }
4991        let err = LogQueryHandler::query(
4992            &instance,
4993            LogQuery {
4994                table: TableName::new("greptime", "public", physical_table),
4995                ..Default::default()
4996            },
4997            ctx.clone(),
4998        )
4999        .await
5000        .unwrap_err();
5001        assert_eq!(StatusCode::PermissionDenied, err.status_code());
5002        let err = instance
5003            .do_describe_inner(parse_one_sql("SELECT * FROM physical_metric"), ctx.clone())
5004            .await
5005            .unwrap_err();
5006        assert_eq!(StatusCode::PermissionDenied, err.status_code());
5007
5008        let request = ReadRequest {
5009            queries: vec![RemoteQuery {
5010                matchers: vec![LabelMatcher {
5011                    r#type: PromMatcherType::Eq as i32,
5012                    name: servers::prom_store::METRIC_NAME_LABEL.to_string(),
5013                    value: physical_table.to_string(),
5014                }],
5015                ..Default::default()
5016            }],
5017            ..Default::default()
5018        };
5019        let Err(err) = PromStoreProtocolHandler::read(&instance, request, ctx.clone()).await else {
5020            panic!("physical remote-read target must be rejected");
5021        };
5022        assert_eq!(StatusCode::PermissionDenied, err.status_code());
5023
5024        let err = PrometheusHandler::query_label_values(
5025            &instance,
5026            physical_table.to_string(),
5027            "host".to_string(),
5028            vec![],
5029            SystemTime::UNIX_EPOCH,
5030            SystemTime::UNIX_EPOCH,
5031            &ctx,
5032        )
5033        .await
5034        .unwrap_err();
5035        assert_eq!(StatusCode::PermissionDenied, err.status_code());
5036
5037        Ok(())
5038    }
5039
5040    #[tokio::test]
5041    async fn test_non_exact_query_discovery_keeps_denied_targets_for_batch_check() -> TestResult<()>
5042    {
5043        let plugins = Plugins::new();
5044        plugins.insert::<PermissionCheckerRef>(Arc::new(RejectUnresolvedPermissionChecker));
5045        let instance = test_instance_with_plugins_and_metric_names(
5046            test_logical_table(1024, "denied")?,
5047            test_logical_table(1025, "target")?,
5048            plugins,
5049            Some(test_metric_names_table()),
5050        )
5051        .await?;
5052        let ctx = test_query_ctx(1);
5053
5054        let mut metric_names = PrometheusHandler::query_metric_names(
5055            &instance,
5056            vec![Matcher::new(
5057                promql_parser::label::MatchOp::NotEqual,
5058                "__name__",
5059                "",
5060            )],
5061            "public",
5062            &ctx,
5063        )
5064        .await
5065        .unwrap();
5066        metric_names.sort_unstable();
5067        assert_eq!(
5068            vec!["denied".to_string(), "target".to_string()],
5069            metric_names
5070        );
5071
5072        let queries = metric_names
5073            .into_iter()
5074            .map(|query| PromQuery {
5075                query,
5076                ..Default::default()
5077            })
5078            .collect::<Vec<_>>();
5079        let err = PrometheusHandler::check_query_permission(&instance, &queries, &ctx)
5080            .await
5081            .unwrap_err();
5082        assert_eq!(StatusCode::PermissionDenied, err.status_code());
5083
5084        Ok(())
5085    }
5086
5087    /// `info` reads `target_info` without naming it. With a label selector the query used to be
5088    /// answered as a discovery over the tables the caller may read, which never named it either.
5089    #[tokio::test]
5090    async fn test_info_query_checks_the_info_table() -> TestResult<()> {
5091        struct DenyInfoTable;
5092
5093        impl PermissionChecker for DenyInfoTable {
5094            fn check_permission(
5095                &self,
5096                _user_info: UserInfoRef,
5097                _req: PermissionReq,
5098            ) -> auth::error::Result<PermissionResp> {
5099                Ok(PermissionResp::Allow)
5100            }
5101
5102            fn check_permission_with_table_targets(
5103                &self,
5104                _user_info: UserInfoRef,
5105                _req: PermissionReq,
5106                targets: PermissionTableTargets,
5107            ) -> auth::error::Result<PermissionResp> {
5108                let reject = match targets {
5109                    PermissionTableTargets::Unresolved => true,
5110                    PermissionTableTargets::Resolved(targets) => {
5111                        targets.iter().any(|target| target.table == "target_info")
5112                    }
5113                };
5114                Ok(if reject {
5115                    PermissionResp::Reject
5116                } else {
5117                    PermissionResp::Allow
5118                })
5119            }
5120        }
5121
5122        let plugins = Plugins::new();
5123        plugins.insert::<PermissionCheckerRef>(Arc::new(DenyInfoTable));
5124        let instance = test_instance_with_plugins_and_metric_names(
5125            test_logical_table(1024, "target_info")?,
5126            test_logical_table(1025, "target")?,
5127            plugins,
5128            None,
5129        )
5130        .await?;
5131        let ctx = Arc::new(
5132            QueryContextBuilder::default()
5133                .channel(Channel::Mysql)
5134                .conn_info(ConnInfo::new(None, Channel::Mysql))
5135                .extensions(HashMap::from([(
5136                    query::promql::planner::NATIVE_PROMQL_EXTENSION_KEY.to_string(),
5137                    "true".to_string(),
5138                )]))
5139                .build(),
5140        );
5141
5142        for promql in [
5143            "info(target)",
5144            r#"info(target, {version="v1"})"#,
5145            // Not a shape that reads `target_info`, and not a discovery either: unresolved.
5146            r#"info(target, {__schema__="public", version="v1"})"#,
5147        ] {
5148            let query = PromQuery {
5149                query: promql.to_string(),
5150                ..Default::default()
5151            };
5152            let err = PrometheusHandler::do_query(&instance, &query, ctx.clone())
5153                .await
5154                .unwrap_err();
5155            assert_eq!(StatusCode::PermissionDenied, err.status_code(), "{promql}");
5156        }
5157
5158        Ok(())
5159    }
5160
5161    #[test]
5162    fn test_should_track_statement_process() {
5163        assert!(should_track_statement_process(&parse_one_sql(
5164            "SELECT * FROM demo"
5165        )));
5166        assert!(should_track_statement_process(&parse_one_sql(
5167            "INSERT INTO demo SELECT * FROM source"
5168        )));
5169        assert!(!should_track_statement_process(&parse_one_sql(
5170            "INSERT INTO demo VALUES (1)"
5171        )));
5172        assert!(!should_track_statement_process(&parse_one_sql(
5173            "INSERT INTO demo VALUES (now())"
5174        )));
5175    }
5176
5177    #[test]
5178    fn test_should_track_plan_process() {
5179        let select_stmt = parse_one_sql("SELECT * FROM demo");
5180        let insert_select_stmt = parse_one_sql("INSERT INTO demo SELECT * FROM source");
5181        let insert_values_stmt = parse_one_sql("INSERT INTO demo VALUES (now())");
5182
5183        let empty_plan = LogicalPlanBuilder::empty(false).build().unwrap();
5184        assert!(should_track_plan_process(Some(&select_stmt), &empty_plan));
5185        assert!(should_track_plan_process(
5186            Some(&insert_select_stmt),
5187            &insert_dml_plan()
5188        ));
5189        assert!(!should_track_plan_process(
5190            Some(&insert_values_stmt),
5191            &insert_dml_plan()
5192        ));
5193        assert!(!should_track_plan_process(None, &insert_dml_plan()));
5194    }
5195
5196    #[tokio::test]
5197    async fn test_exemplar_slow_query() -> TestResult<()> {
5198        use common_catalog::consts::METRICS_EXEMPLARS_TABLE;
5199        use servers::exemplar::query::Query as ExemplarQuery;
5200
5201        for invalid_destination in [false, true] {
5202            let table_name = if invalid_destination {
5203                METRICS_EXEMPLARS_TABLE
5204            } else {
5205                "source"
5206            };
5207            let mut instance = test_instance_with_tables(
5208                test_table(1024, table_name)?,
5209                test_table(1025, "target")?,
5210            )
5211            .await?;
5212            let recorder = Arc::new(RecordingSlowQueryEventRecorder::default());
5213            instance.slow_query_recorder = recorder.clone();
5214            instance.slow_query_options.sample_ratio = 1.0;
5215            instance.slow_query_options.record_type = SlowQueriesRecordType::SystemTable;
5216
5217            for (enabled, threshold, should_record) in [
5218                (true, Duration::ZERO, true),
5219                (false, Duration::ZERO, false),
5220                (true, Duration::from_secs(3600), false),
5221            ] {
5222                instance.slow_query_options.enable = enabled;
5223                instance.slow_query_options.threshold = threshold;
5224                let query =
5225                    ExemplarQuery::parse("source".into(), "-0.001", "0.001", "public").unwrap();
5226                let result = instance.query_exemplars(query, QueryContext::arc()).await;
5227                if invalid_destination {
5228                    assert!(result.is_err());
5229                } else {
5230                    assert!(result.unwrap().is_empty());
5231                }
5232
5233                let queries = std::mem::take(&mut *recorder.queries.lock().unwrap());
5234                let expected = if should_record {
5235                    vec!["query_exemplars(source, start_ns=-1000000, end_ns=1999999)"]
5236                } else {
5237                    vec![]
5238                };
5239                assert_eq!(
5240                    queries, expected,
5241                    "invalid_destination={invalid_destination}, enabled={enabled}, threshold={threshold:?}"
5242                );
5243            }
5244        }
5245        Ok(())
5246    }
5247
5248    #[tokio::test]
5249    async fn test_insert_select_slow_query() -> TestResult<()> {
5250        let mut instance =
5251            test_instance_with_tables(test_table(1024, "source")?, test_table(1025, "target")?)
5252                .await?;
5253        let recorder = Arc::new(RecordingSlowQueryEventRecorder::default());
5254        instance.slow_query_recorder = recorder.clone();
5255        let sql = "INSERT INTO target SELECT * FROM source";
5256
5257        for prepared in [false, true] {
5258            for (enabled, threshold, should_record) in [
5259                (true, Duration::ZERO, true),
5260                (false, Duration::ZERO, false),
5261                (true, Duration::from_secs(3600), false),
5262            ] {
5263                instance.slow_query_options.enable = enabled;
5264                instance.slow_query_options.threshold = threshold;
5265                let query_ctx = test_query_ctx(42);
5266                let output = if prepared {
5267                    let stmt = parse_one_sql(sql);
5268                    let described = instance
5269                        .do_describe_inner(stmt.clone(), query_ctx.clone())
5270                        .await
5271                        .unwrap()
5272                        .unwrap();
5273                    instance
5274                        .do_exec_plan_inner(described.logical_plan, Some(stmt), query_ctx)
5275                        .await
5276                        .unwrap()
5277                } else {
5278                    execute_one_sql(&instance, sql, query_ctx).await?
5279                };
5280                assert!(matches!(output.data, OutputData::AffectedRows(0)));
5281
5282                let queries = std::mem::take(&mut *recorder.queries.lock().unwrap());
5283                let expected = if should_record { vec![sql] } else { vec![] };
5284                assert_eq!(
5285                    queries, expected,
5286                    "prepared={prepared}, enabled={enabled}, threshold={threshold:?}"
5287                );
5288            }
5289        }
5290
5291        Ok(())
5292    }
5293
5294    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5295    async fn test_insert_select_is_visible_in_show_processlist() -> TestResult<()> {
5296        let insert_sql = "INSERT INTO target SELECT * FROM source";
5297        let (started_tx, mut started_rx) = mpsc::unbounded_channel();
5298        let (finish_tx, finish_rx) = oneshot::channel();
5299        let interceptor = Arc::new(BlockingInsertSelectInterceptor::new(started_tx, finish_rx));
5300        let instance = Arc::new(test_instance_with_insert_select_interceptor(interceptor).await?);
5301
5302        let insert_task = tokio::spawn({
5303            let instance = instance.clone();
5304            async move { execute_one_sql(&instance, insert_sql, test_query_ctx(4242)).await }
5305        });
5306
5307        tokio::time::timeout(Duration::from_secs(5), started_rx.recv())
5308            .await
5309            .context(InsertStartTimeoutSnafu)?
5310            .context(InsertStartChannelClosedSnafu)?;
5311
5312        let output = execute_one_sql(&instance, "SHOW PROCESSLIST", test_query_ctx(43)).await?;
5313        let process_list = output.data.pretty_print().await;
5314        assert!(
5315            process_list.contains(insert_sql),
5316            "process list did not contain running insert:\n{process_list}"
5317        );
5318
5319        finish_tx
5320            .send(())
5321            .map_err(|_| ReleaseBlockedInsertSnafu.build())?;
5322        insert_task.await.context(InsertTaskPanicSnafu)??;
5323
5324        Ok(())
5325    }
5326
5327    #[tokio::test]
5328    async fn test_show_processlist_catalog_scope() -> TestResult<()> {
5329        let instance =
5330            test_instance_with_tables(test_table(1024, "source")?, test_table(1025, "target")?)
5331                .await?;
5332        let _current_catalog = instance.process_manager().register_query(
5333            "greptime".to_string(),
5334            vec!["public".to_string()],
5335            "current_catalog_query".to_string(),
5336            String::new(),
5337            None,
5338            None,
5339        );
5340        let _other_catalog = instance.process_manager().register_query(
5341            "other".to_string(),
5342            vec!["public".to_string()],
5343            "other_catalog_query".to_string(),
5344            String::new(),
5345            None,
5346            None,
5347        );
5348
5349        for sql in ["SHOW PROCESSLIST", "SHOW FULL PROCESSLIST"] {
5350            let output = execute_one_sql(&instance, sql, test_query_ctx(43)).await?;
5351            let process_list = output.data.pretty_print().await;
5352            assert!(
5353                process_list.contains("current_catalog_query"),
5354                "{process_list}"
5355            );
5356            assert!(
5357                !process_list.contains("other_catalog_query"),
5358                "{process_list}"
5359            );
5360
5361            let admin_ctx = test_query_ctx(44);
5362            admin_ctx.set_current_user(Arc::new(AdminUserInfo));
5363            let output = execute_one_sql(&instance, sql, admin_ctx).await?;
5364            let process_list = output.data.pretty_print().await;
5365            assert!(
5366                process_list.contains("current_catalog_query"),
5367                "{process_list}"
5368            );
5369            assert!(
5370                process_list.contains("other_catalog_query"),
5371                "{process_list}"
5372            );
5373        }
5374
5375        Ok(())
5376    }
5377
5378    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5379    async fn test_kill_query_cancels_insert_select() -> TestResult<()> {
5380        assert_kill_cancels_insert_select("KILL QUERY 4242").await
5381    }
5382
5383    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
5384    async fn test_kill_process_id_cancels_insert_select() -> TestResult<()> {
5385        assert_kill_cancels_insert_select("KILL 'test-frontend/4242'").await
5386    }
5387
5388    async fn assert_kill_cancels_insert_select(kill_sql: &str) -> TestResult<()> {
5389        let insert_sql = "INSERT INTO target SELECT * FROM source";
5390        let (source_polled_tx, source_polled_rx) = oneshot::channel();
5391        let instance = Arc::new(
5392            test_instance_with_tables(
5393                pending_table(1024, "source", source_polled_tx)?,
5394                test_table(1025, "target")?,
5395            )
5396            .await?,
5397        );
5398
5399        let insert_task = tokio::spawn({
5400            let instance = instance.clone();
5401            async move { execute_one_sql(&instance, insert_sql, test_query_ctx(4242)).await }
5402        });
5403
5404        tokio::time::timeout(Duration::from_secs(5), source_polled_rx)
5405            .await
5406            .context(SourcePollTimeoutSnafu)?
5407            .context(SourcePollChannelClosedSnafu)?;
5408
5409        let output = execute_one_sql(&instance, kill_sql, test_query_ctx(43)).await?;
5410        assert!(matches!(output.data, OutputData::AffectedRows(1)));
5411
5412        let insert_result = tokio::time::timeout(Duration::from_secs(5), insert_task)
5413            .await
5414            .context(InsertTaskTimeoutSnafu)?
5415            .context(InsertTaskPanicSnafu)?;
5416        let err = match insert_result {
5417            Ok(_) => return InsertSelectNotCancelledSnafu.fail(),
5418            Err(TestError::ExecuteSql { source, .. }) => source,
5419            Err(err) => return Err(err),
5420        };
5421        assert_eq!(StatusCode::Cancelled, err.status_code());
5422
5423        let output = execute_one_sql(&instance, "SHOW PROCESSLIST", test_query_ctx(43)).await?;
5424        let process_list = output.data.pretty_print().await;
5425        assert!(
5426            !process_list.contains(insert_sql),
5427            "process list still contains killed insert:\n{process_list}"
5428        );
5429
5430        Ok(())
5431    }
5432
5433    fn insert_dml_plan() -> LogicalPlan {
5434        let schema = SchemaRef::new(Schema::new(vec![Field::new(
5435            "value",
5436            DataType::Int64,
5437            true,
5438        )]));
5439        let target = Arc::new(LogicalTableSource::new(schema));
5440        let input = LogicalPlanBuilder::empty(false).build().unwrap();
5441
5442        LogicalPlanBuilder::insert_into(input, "demo", target, InsertOp::Append)
5443            .unwrap()
5444            .build()
5445            .unwrap()
5446    }
5447
5448    #[test]
5449    fn test_exec_validation() {
5450        let query_ctx = QueryContext::arc();
5451        let plugins: Plugins = Plugins::new();
5452        plugins.insert(QueryOptions {
5453            disallow_cross_catalog_query: true,
5454        });
5455
5456        let sql = r#"
5457        SELECT * FROM demo;
5458        EXPLAIN SELECT * FROM demo;
5459        CREATE DATABASE test_database;
5460        SHOW DATABASES;
5461        "#;
5462        let stmts = parse_stmt(sql, &GreptimeDbDialect {}).unwrap();
5463        assert_eq!(stmts.len(), 4);
5464        for stmt in stmts {
5465            let re = check_permission(plugins.clone(), &stmt, &query_ctx);
5466            re.unwrap();
5467        }
5468
5469        let sql = r#"
5470        SHOW CREATE TABLE demo;
5471        ALTER TABLE demo ADD COLUMN new_col INT;
5472        "#;
5473        let stmts = parse_stmt(sql, &GreptimeDbDialect {}).unwrap();
5474        assert_eq!(stmts.len(), 2);
5475        for stmt in stmts {
5476            let re = check_permission(plugins.clone(), &stmt, &query_ctx);
5477            re.unwrap();
5478        }
5479
5480        fn replace_test(template_sql: &str, plugins: Plugins, query_ctx: &QueryContextRef) {
5481            // test right
5482            let right = vec![("", ""), ("", "public."), ("greptime.", "public.")];
5483            for (catalog, schema) in right {
5484                let sql = do_fmt(template_sql, catalog, schema);
5485                do_test(&sql, plugins.clone(), query_ctx, true);
5486            }
5487
5488            let wrong = vec![
5489                ("wrongcatalog.", "public."),
5490                ("wrongcatalog.", "wrongschema."),
5491            ];
5492            for (catalog, schema) in wrong {
5493                let sql = do_fmt(template_sql, catalog, schema);
5494                do_test(&sql, plugins.clone(), query_ctx, false);
5495            }
5496        }
5497
5498        fn do_fmt(template: &str, catalog: &str, schema: &str) -> String {
5499            let vars = HashMap::from([
5500                ("catalog".to_string(), catalog),
5501                ("schema".to_string(), schema),
5502            ]);
5503            template.format(&vars).unwrap()
5504        }
5505
5506        fn do_test(sql: &str, plugins: Plugins, query_ctx: &QueryContextRef, is_ok: bool) {
5507            let stmt = &parse_stmt(sql, &GreptimeDbDialect {}).unwrap()[0];
5508            let re = check_permission(plugins, stmt, query_ctx);
5509            if is_ok {
5510                re.unwrap();
5511            } else {
5512                assert!(re.is_err());
5513            }
5514        }
5515
5516        // test insert
5517        let sql = "INSERT INTO {catalog}{schema}monitor(host) VALUES ('host1');";
5518        replace_test(sql, plugins.clone(), &query_ctx);
5519
5520        // test create table
5521        let sql = r#"CREATE TABLE {catalog}{schema}demo(
5522                            host STRING,
5523                            ts TIMESTAMP,
5524                            TIME INDEX (ts),
5525                            PRIMARY KEY(host)
5526                        ) engine=mito;"#;
5527        replace_test(sql, plugins.clone(), &query_ctx);
5528
5529        // test drop table
5530        let sql = "DROP TABLE {catalog}{schema}demo;";
5531        replace_test(sql, plugins.clone(), &query_ctx);
5532
5533        // test undrop table
5534        #[cfg(feature = "enterprise")]
5535        {
5536            let sql = "UNDROP TABLE {catalog}{schema}demo;";
5537            replace_test(sql, plugins.clone(), &query_ctx);
5538        }
5539
5540        // test show tables
5541        let sql = "SHOW TABLES FROM public";
5542        let stmt = parse_stmt(sql, &GreptimeDbDialect {}).unwrap();
5543        check_permission(plugins.clone(), &stmt[0], &query_ctx).unwrap();
5544
5545        let sql = "SHOW TABLES FROM private";
5546        let stmt = parse_stmt(sql, &GreptimeDbDialect {}).unwrap();
5547        let re = check_permission(plugins.clone(), &stmt[0], &query_ctx);
5548        assert!(re.is_ok());
5549
5550        // test describe table
5551        let sql = "DESC TABLE {catalog}{schema}demo;";
5552        replace_test(sql, plugins.clone(), &query_ctx);
5553
5554        let comment_flow_cases = [
5555            ("COMMENT ON FLOW my_flow IS 'comment';", true),
5556            ("COMMENT ON FLOW greptime.my_flow IS 'comment';", true),
5557            ("COMMENT ON FLOW wrongcatalog.my_flow IS 'comment';", false),
5558        ];
5559        for (sql, is_ok) in comment_flow_cases {
5560            let stmt = &parse_stmt(sql, &GreptimeDbDialect {}).unwrap()[0];
5561            let result = check_permission(plugins.clone(), stmt, &query_ctx);
5562            assert_eq!(result.is_ok(), is_ok);
5563        }
5564
5565        let show_flow_cases = [
5566            ("SHOW CREATE FLOW my_flow;", true),
5567            ("SHOW CREATE FLOW greptime.my_flow;", true),
5568            ("SHOW CREATE FLOW wrongcatalog.my_flow;", false),
5569        ];
5570        for (sql, is_ok) in show_flow_cases {
5571            let stmt = &parse_stmt(sql, &GreptimeDbDialect {}).unwrap()[0];
5572            let result = check_permission(plugins.clone(), stmt, &query_ctx);
5573            assert_eq!(result.is_ok(), is_ok);
5574        }
5575    }
5576
5577    /// A `DropView` DDL sent through the direct gRPC ingress must return an error
5578    /// (e.g. table not found) instead of panicking on `todo!()`.
5579    #[tokio::test]
5580    async fn qx_152_drop_view_via_grpc_ddl_returns_error_not_panic() -> TestResult<()> {
5581        let instance =
5582            test_instance_with_tables(test_table(1024, "source")?, test_table(1025, "target")?)
5583                .await?;
5584
5585        let request = api::v1::greptime_request::Request::Ddl(api::v1::DdlRequest {
5586            expr: Some(api::v1::ddl_request::Expr::DropView(
5587                api::v1::DropViewExpr {
5588                    catalog_name: String::new(),
5589                    schema_name: String::new(),
5590                    view_name: "non_existent_view".to_string(),
5591                    view_id: None,
5592                    drop_if_exists: false,
5593                },
5594            )),
5595        });
5596
5597        let result = servers::query_handler::grpc::GrpcQueryHandler::do_query(
5598            &instance,
5599            request,
5600            QueryContext::arc(),
5601        )
5602        .await;
5603
5604        let err = match result {
5605            Ok(_) => panic!("DropView DDL request must return an error instead of panicking"),
5606            Err(err) => err,
5607        };
5608        assert_eq!(
5609            err.status_code(),
5610            StatusCode::TableNotFound,
5611            "dropping a non-existent view without IF EXISTS must report TableNotFound, got {err}"
5612        );
5613        Ok(())
5614    }
5615
5616    #[tokio::test]
5617    async fn logical_ddl_validates_and_authorizes_rewritten_batch_before_submission()
5618    -> TestResult<()> {
5619        struct Rewrite(&'static str);
5620        impl SqlQueryInterceptor for Rewrite {
5621            type Error = Error;
5622            fn pre_parsing<'a>(
5623                &self,
5624                query: &'a str,
5625                _: QueryContextRef,
5626            ) -> Result<std::borrow::Cow<'a, str>> {
5627                Ok(if self.0 == "sql_bytes" {
5628                    std::borrow::Cow::Owned(format!("{query}{}", " ".repeat(1024 * 1024 + 1)))
5629                } else {
5630                    std::borrow::Cow::Borrowed(query)
5631                })
5632            }
5633            fn pre_execute(
5634                &self,
5635                statement: Option<&Statement>,
5636                _: Option<&LogicalPlan>,
5637                _: QueryContextRef,
5638            ) -> Result<()> {
5639                ensure!(
5640                    !(self.0 == "pre_execute"
5641                        && matches!(statement,
5642                    Some(Statement::CreateTable(create)) if create.name.to_string() == "allowed2")),
5643                    InvalidSqlSnafu {
5644                        err_msg: "second member rejected by interceptor"
5645                    }
5646                );
5647                Ok(())
5648            }
5649            fn post_parsing(
5650                &self,
5651                mut statements: Vec<Statement>,
5652                _: QueryContextRef,
5653            ) -> Result<Vec<Statement>> {
5654                match self.0 {
5655                    "target" => {
5656                        statements[1] = parse_one_sql(
5657                            "CREATE TABLE denied (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH (on_physical_table='phy')",
5658                        )
5659                    }
5660                    "count" => statements = (0..129).map(|i| parse_one_sql(&format!(
5661                        "CREATE TABLE allowed_{i} (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH (on_physical_table='phy')"
5662                    ))).collect(),
5663                    "type" => statements[1] = parse_one_sql("SELECT 1"),
5664                    "secret_bytes" | "array_bytes" | "small_option" => {
5665                        if let Statement::CreateTable(create) = &mut statements[1] {
5666                            let value = "x".repeat(if self.0 == "small_option" { 10 } else { 2 * 1024 * 1024 });
5667                            if self.0 == "array_bytes" {
5668                                create.options.insert_options("non_scalar", vec![value.as_str()].into());
5669                            } else {
5670                                create.options.insert("secret_access_key".to_string(), value);
5671                            }
5672                        }
5673                    }
5674                    "partition_shape" => {
5675                        if let Statement::CreateTable(create) = &mut statements[1] {
5676                            create.partitions = Some(sql::statements::create::Partitions {
5677                                column_list: vec![],
5678                                exprs: vec![sqlparser::ast::Expr::Identifier(sqlparser::ast::Ident::new("hidden"))],
5679                            });
5680                        }
5681                    }
5682                    "vector_hidden" => {
5683                        statements[1] = parse_one_sql("CREATE TABLE allowed2 (ts TIMESTAMP TIME INDEX, v VECTOR(3)) ENGINE=metric WITH (on_physical_table='phy')");
5684                        if let Statement::CreateTable(create) = &mut statements[1] {
5685                            create.columns[1].column_def.options.push(sqlparser::ast::ColumnOptionDef {
5686                                name: None,
5687                                option: sqlparser::ast::ColumnOption::Comment("private-value".repeat(200_000)),
5688                            });
5689                        }
5690                    }
5691                    mode if mode.starts_with("fulltext_") || mode.starts_with("skipping_")
5692                        || mode.starts_with("inverted_") || mode.starts_with("vector_") => {
5693                        if let Statement::CreateTable(create) = &mut statements[1] {
5694                            let (kind, value_type) = mode.split_once('_').unwrap();
5695                            let value = "x".repeat(if value_type == "small" { 10 } else { 2 * 1024 * 1024 });
5696                            let extensions = &mut create.columns[1].extensions;
5697                            let options = match kind {
5698                                "fulltext" => &mut extensions.fulltext_index_options,
5699                                "skipping" => &mut extensions.skipping_index_options,
5700                                "inverted" => &mut extensions.inverted_index_options,
5701                                _ => &mut extensions.vector_options,
5702                            }.get_or_insert_default();
5703                            if kind == "vector" {
5704                                options.insert("dim".to_string(), "3".to_string());
5705                            }
5706                            if value_type == "array" {
5707                                options.insert_options("non_scalar", vec![value.as_str()].into());
5708                            } else {
5709                                options.insert("secret_access_key".to_string(), value);
5710                            }
5711                        }
5712                    }
5713                    "physical_null" | "physical_array" => {
5714                        let value = if self.0 == "physical_null" { "NULL" } else { "['x']" };
5715                        statements[1] = parse_one_sql(&format!(
5716                            "CREATE TABLE allowed2 (ts TIMESTAMP TIME INDEX) ENGINE=metric WITH (on_physical_table='phy', physical_metric_table={value})"
5717                        ));
5718                    }
5719                    "ast_bytes" => {
5720                        if let Statement::CreateTable(create) = &mut statements[1] {
5721                            create.name =
5722                                ObjectName(vec![sqlparser::ast::ObjectNamePart::Identifier(
5723                                    sqlparser::ast::Ident::new("t".repeat(1024 * 1024)),
5724                                )]);
5725                        }
5726                    }
5727                    _ => {}
5728                }
5729                Ok(statements)
5730            }
5731        }
5732        for mode in [
5733            "none",
5734            "target",
5735            "count",
5736            "type",
5737            "sql_bytes",
5738            "ast_bytes",
5739            "secret_bytes",
5740            "array_bytes",
5741            "small_option",
5742            "physical_null",
5743            "physical_array",
5744            "fulltext_secret",
5745            "fulltext_array",
5746            "fulltext_small",
5747            "skipping_secret",
5748            "skipping_array",
5749            "skipping_small",
5750            "inverted_secret",
5751            "inverted_array",
5752            "inverted_small",
5753            "vector_secret",
5754            "vector_array",
5755            "vector_small",
5756            "vector_hidden",
5757            "partition_shape",
5758            "pre_execute",
5759            "cross_catalog",
5760            "valid",
5761        ] {
5762            let catalog =
5763                catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "source")?);
5764            let executor = Arc::new(MockProcedureExecutor::new(catalog.clone()));
5765            let plugins = Plugins::new();
5766            plugins.insert::<PermissionCheckerRef>(Arc::new(RejectUnresolvedPermissionChecker));
5767            plugins.insert::<SqlQueryInterceptorRef<Error>>(Arc::new(Rewrite(mode)));
5768            plugins.insert(QueryOptions {
5769                disallow_cross_catalog_query: mode == "cross_catalog",
5770            });
5771            let instance = test_instance_with_catalog_manager(
5772                catalog,
5773                test_table(1025, "target")?,
5774                plugins,
5775                None,
5776                executor.clone(),
5777            )
5778            .await?;
5779            let create = |name| {
5780                format!(
5781                    "CREATE TABLE {name} (ts TIMESTAMP TIME INDEX, host STRING) ENGINE=metric WITH (on_physical_table='phy')"
5782                )
5783            };
5784            let sql = format!(
5785                "{};{}",
5786                create("allowed"),
5787                create(match mode {
5788                    "none" => "denied",
5789                    "cross_catalog" => "other.public.allowed2",
5790                    _ => "allowed2",
5791                })
5792            );
5793            let result =
5794                SqlQueryHandler::create_logical_tables(&instance, &sql, QueryContext::arc()).await;
5795            let submits = matches!(mode, "valid" | "small_option")
5796                || (mode.ends_with("_small") && !mode.starts_with("vector_"));
5797            if matches!(mode, "none" | "target") {
5798                let err = result.unwrap_err();
5799                assert_eq!(
5800                    err.status_code(),
5801                    StatusCode::PermissionDenied,
5802                    "{mode}: {err}"
5803                );
5804            } else {
5805                assert_eq!(
5806                    result.unwrap_err().status_code(),
5807                    match mode {
5808                        _ if submits => StatusCode::Unsupported,
5809                        "cross_catalog" => StatusCode::AccessDenied,
5810                        _ => StatusCode::InvalidArguments,
5811                    },
5812                    "{mode}"
5813                );
5814            }
5815            assert_eq!(
5816                executor.submitted.lock().unwrap().len(),
5817                usize::from(submits),
5818                "{mode}"
5819            );
5820        }
5821        Ok(())
5822    }
5823
5824    /// `DROP VIEW IF EXISTS` on a missing view through the direct gRPC ingress must
5825    /// succeed with 0 affected rows (no error, no DDL task submitted), instead of
5826    /// returning `TableNotFound`.
5827    #[tokio::test]
5828    async fn qx_152_drop_view_if_exists_missing_view_via_grpc_ddl_succeeds() -> TestResult<()> {
5829        let catalog_manager =
5830            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "source")?);
5831        let procedure_executor = Arc::new(MockProcedureExecutor::new(catalog_manager.clone()));
5832        let instance = test_instance_with_catalog_manager(
5833            catalog_manager,
5834            test_table(1025, "target")?,
5835            Plugins::new(),
5836            None,
5837            procedure_executor.clone() as ProcedureExecutorRef,
5838        )
5839        .await?;
5840
5841        let request = api::v1::greptime_request::Request::Ddl(api::v1::DdlRequest {
5842            expr: Some(api::v1::ddl_request::Expr::DropView(
5843                api::v1::DropViewExpr {
5844                    catalog_name: String::new(),
5845                    schema_name: String::new(),
5846                    view_name: "non_existent_view".to_string(),
5847                    view_id: None,
5848                    drop_if_exists: true,
5849                },
5850            )),
5851        });
5852
5853        let result = servers::query_handler::grpc::GrpcQueryHandler::do_query(
5854            &instance,
5855            request,
5856            QueryContext::arc(),
5857        )
5858        .await;
5859
5860        let output = match result {
5861            Ok(output) => output,
5862            Err(err) => {
5863                panic!("DROP VIEW IF EXISTS on a missing view must succeed, got error: {err}")
5864            }
5865        };
5866        assert!(
5867            matches!(output.data, OutputData::AffectedRows(0)),
5868            "DROP VIEW IF EXISTS on a missing view must report 0 affected rows"
5869        );
5870        assert!(
5871            procedure_executor.submitted.lock().unwrap().is_empty(),
5872            "DROP VIEW IF EXISTS on a missing view must not submit a DDL task"
5873        );
5874        Ok(())
5875    }
5876
5877    /// A `CREATE VIEW` followed by `DROP VIEW` through the direct gRPC ingress must
5878    /// succeed end to end: the view is registered in the catalog and then removed.
5879    #[tokio::test]
5880    async fn qx_152_drop_existing_view_via_grpc_ddl_succeeds() -> TestResult<()> {
5881        let catalog_manager =
5882            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "source")?);
5883        let procedure_executor = Arc::new(MockProcedureExecutor::new(catalog_manager.clone()));
5884        let instance = test_instance_with_catalog_manager(
5885            catalog_manager,
5886            test_table(1025, "target")?,
5887            Plugins::new(),
5888            None,
5889            procedure_executor.clone() as ProcedureExecutorRef,
5890        )
5891        .await?;
5892
5893        // The default "greptime.public" schema must be visible to the kv-backed table
5894        // metadata manager for `CREATE VIEW`/`CREATE TABLE` to pass validation.
5895        instance
5896            .table_metadata_manager()
5897            .schema_manager()
5898            .create(
5899                common_meta::key::schema_name::SchemaNameKey::new("greptime", "public"),
5900                None,
5901                true,
5902            )
5903            .await
5904            .unwrap();
5905
5906        let create_view_request = api::v1::greptime_request::Request::Ddl(api::v1::DdlRequest {
5907            expr: Some(api::v1::ddl_request::Expr::CreateView(
5908                api::v1::CreateViewExpr {
5909                    catalog_name: String::new(),
5910                    schema_name: String::new(),
5911                    view_name: "my_view".to_string(),
5912                    logical_plan: vec![1, 2, 3],
5913                    create_if_not_exists: false,
5914                    or_replace: false,
5915                    table_names: vec![],
5916                    columns: vec![],
5917                    plan_columns: vec![],
5918                    definition: "CREATE VIEW my_view AS SELECT * FROM source".to_string(),
5919                },
5920            )),
5921        });
5922
5923        let output = match servers::query_handler::grpc::GrpcQueryHandler::do_query(
5924            &instance,
5925            create_view_request,
5926            QueryContext::arc(),
5927        )
5928        .await
5929        {
5930            Ok(output) => output,
5931            Err(err) => panic!("CREATE VIEW via gRPC DDL must succeed, got error: {err}"),
5932        };
5933        assert!(
5934            matches!(output.data, OutputData::AffectedRows(0)),
5935            "CREATE VIEW via gRPC DDL must report 0 affected rows"
5936        );
5937
5938        // The view is registered in the catalog as a view.
5939        let view = instance
5940            .catalog_manager()
5941            .table("greptime", "public", "my_view", None)
5942            .await
5943            .unwrap()
5944            .expect("view should exist after CREATE VIEW");
5945        assert_eq!(view.table_info().table_type, TableType::View);
5946
5947        let drop_view_request = api::v1::greptime_request::Request::Ddl(api::v1::DdlRequest {
5948            expr: Some(api::v1::ddl_request::Expr::DropView(
5949                api::v1::DropViewExpr {
5950                    catalog_name: String::new(),
5951                    schema_name: String::new(),
5952                    view_name: "my_view".to_string(),
5953                    view_id: None,
5954                    drop_if_exists: false,
5955                },
5956            )),
5957        });
5958
5959        let output = match servers::query_handler::grpc::GrpcQueryHandler::do_query(
5960            &instance,
5961            drop_view_request,
5962            QueryContext::arc(),
5963        )
5964        .await
5965        {
5966            Ok(output) => output,
5967            Err(err) => panic!("DROP VIEW via gRPC DDL must succeed, got error: {err}"),
5968        };
5969        assert!(
5970            matches!(output.data, OutputData::AffectedRows(0)),
5971            "DROP VIEW via gRPC DDL must report 0 affected rows"
5972        );
5973
5974        // The view is gone after the drop.
5975        assert!(
5976            instance
5977                .catalog_manager()
5978                .table("greptime", "public", "my_view", None)
5979                .await
5980                .unwrap()
5981                .is_none(),
5982            "view should be removed after DROP VIEW"
5983        );
5984
5985        let submitted = procedure_executor.submitted.lock().unwrap();
5986        assert_eq!(
5987            submitted.len(),
5988            2,
5989            "expected create and drop view tasks, got {submitted:?}"
5990        );
5991        assert!(matches!(&submitted[0], DdlTask::CreateView(_)));
5992        assert!(matches!(&submitted[1], DdlTask::DropView(_)));
5993        Ok(())
5994    }
5995
5996    /// A direct gRPC `CreateTable` whose time index column is not a timestamp must
5997    /// be rejected with `InvalidArguments` instead of panicking while building the schema.
5998    #[tokio::test]
5999    async fn qx_153_create_table_with_non_timestamp_time_index_via_grpc_returns_error()
6000    -> TestResult<()> {
6001        let instance =
6002            test_instance_with_tables(test_table(1024, "source")?, test_table(1025, "target")?)
6003                .await?;
6004
6005        let request = api::v1::greptime_request::Request::Ddl(api::v1::DdlRequest {
6006            expr: Some(api::v1::ddl_request::Expr::CreateTable(
6007                api::v1::CreateTableExpr {
6008                    catalog_name: String::new(),
6009                    schema_name: String::new(),
6010                    table_name: "demo".to_string(),
6011                    desc: String::new(),
6012                    column_defs: vec![api::v1::ColumnDef {
6013                        name: "host".to_string(),
6014                        data_type: api::v1::ColumnDataType::String as i32,
6015                        is_nullable: true,
6016                        default_constraint: vec![],
6017                        semantic_type: 0,
6018                        comment: String::new(),
6019                        datatype_extension: None,
6020                        options: None,
6021                    }],
6022                    time_index: "host".to_string(),
6023                    primary_keys: vec![],
6024                    create_if_not_exists: false,
6025                    table_options: HashMap::new(),
6026                    table_id: None,
6027                    engine: "mito".to_string(),
6028                },
6029            )),
6030        });
6031
6032        let result = servers::query_handler::grpc::GrpcQueryHandler::do_query(
6033            &instance,
6034            request,
6035            QueryContext::arc(),
6036        )
6037        .await;
6038
6039        let err = match result {
6040            Ok(_) => panic!("CreateTable with a non-timestamp time index must be rejected"),
6041            Err(err) => err,
6042        };
6043        assert_eq!(err.status_code(), StatusCode::InvalidArguments, "{err}");
6044        Ok(())
6045    }
6046
6047    /// A valid `CREATE TABLE` (timestamp time index) through the direct gRPC ingress
6048    /// must succeed, guarding that the `validate_create_expr` ingress check doesn't
6049    /// accidentally reject good requests.
6050    #[tokio::test]
6051    async fn qx_153_create_table_with_timestamp_time_index_via_grpc_succeeds() -> TestResult<()> {
6052        let catalog_manager =
6053            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "source")?);
6054        let procedure_executor = Arc::new(MockProcedureExecutor::new(catalog_manager.clone()));
6055        let instance = test_instance_with_catalog_manager(
6056            catalog_manager,
6057            test_table(1025, "target")?,
6058            Plugins::new(),
6059            None,
6060            procedure_executor.clone() as ProcedureExecutorRef,
6061        )
6062        .await?;
6063
6064        // The default "greptime.public" schema must be visible to the kv-backed table
6065        // metadata manager for `CREATE TABLE` to pass validation.
6066        instance
6067            .table_metadata_manager()
6068            .schema_manager()
6069            .create(
6070                common_meta::key::schema_name::SchemaNameKey::new("greptime", "public"),
6071                None,
6072                true,
6073            )
6074            .await
6075            .unwrap();
6076
6077        let request = api::v1::greptime_request::Request::Ddl(api::v1::DdlRequest {
6078            expr: Some(api::v1::ddl_request::Expr::CreateTable(
6079                api::v1::CreateTableExpr {
6080                    catalog_name: String::new(),
6081                    schema_name: String::new(),
6082                    table_name: "demo".to_string(),
6083                    desc: String::new(),
6084                    column_defs: vec![
6085                        api::v1::ColumnDef {
6086                            name: "host".to_string(),
6087                            data_type: api::v1::ColumnDataType::String as i32,
6088                            is_nullable: true,
6089                            default_constraint: vec![],
6090                            semantic_type: 0,
6091                            comment: String::new(),
6092                            datatype_extension: None,
6093                            options: None,
6094                        },
6095                        api::v1::ColumnDef {
6096                            name: "ts".to_string(),
6097                            data_type: api::v1::ColumnDataType::TimestampMillisecond as i32,
6098                            is_nullable: true,
6099                            default_constraint: vec![],
6100                            semantic_type: 0,
6101                            comment: String::new(),
6102                            datatype_extension: None,
6103                            options: None,
6104                        },
6105                    ],
6106                    time_index: "ts".to_string(),
6107                    primary_keys: vec![],
6108                    create_if_not_exists: false,
6109                    table_options: HashMap::new(),
6110                    table_id: None,
6111                    engine: "mito".to_string(),
6112                },
6113            )),
6114        });
6115
6116        let output = match servers::query_handler::grpc::GrpcQueryHandler::do_query(
6117            &instance,
6118            request,
6119            QueryContext::arc(),
6120        )
6121        .await
6122        {
6123            Ok(output) => output,
6124            Err(err) => panic!("CREATE TABLE via gRPC DDL must succeed, got error: {err}"),
6125        };
6126        assert!(
6127            matches!(output.data, OutputData::AffectedRows(0)),
6128            "CREATE TABLE via gRPC DDL must report 0 affected rows"
6129        );
6130
6131        // The table is registered in the catalog.
6132        let table = instance
6133            .catalog_manager()
6134            .table("greptime", "public", "demo", None)
6135            .await
6136            .unwrap()
6137            .expect("table should exist after CREATE TABLE");
6138        assert_eq!(table.table_info().table_type, TableType::Base);
6139
6140        let submitted = procedure_executor.submitted.lock().unwrap();
6141        assert_eq!(
6142            submitted.len(),
6143            1,
6144            "expected one create table task, got {submitted:?}"
6145        );
6146        assert!(matches!(&submitted[0], DdlTask::CreateTable(_)));
6147        Ok(())
6148    }
6149
6150    #[tokio::test]
6151    async fn test_copy_in_checks_table_permissions() -> TestResult<()> {
6152        use servers::query_handler::CopyInHandler;
6153
6154        for source in [
6155            test_table(1024, "denied")?,
6156            test_physical_table(1024, "physical")?,
6157        ] {
6158            let name = source.table_info().name.clone();
6159            let plugins = Plugins::new();
6160            plugins.insert::<PermissionCheckerRef>(Arc::new(RejectUnresolvedPermissionChecker));
6161            let instance =
6162                test_instance_with_plugins(source, test_table(1025, "target")?, plugins).await?;
6163            let ctx = QueryContext::arc();
6164            assert_permission_denied(
6165                instance
6166                    .copy_in_table("greptime", "public", &name, ctx.clone())
6167                    .await,
6168            );
6169            assert!(
6170                instance
6171                    .copy_in_table("greptime", "public", "target", ctx.clone())
6172                    .await
6173                    .unwrap()
6174                    .is_some()
6175            );
6176            assert!(
6177                instance
6178                    .copy_in_table("greptime", "public", "missing", ctx)
6179                    .await
6180                    .unwrap()
6181                    .is_none()
6182            );
6183        }
6184        Ok(())
6185    }
6186
6187    /// COPY FROM STDIN must enforce the same cross-catalog restriction as
6188    /// SQL INSERT when `disallow_cross_catalog_query` is configured.
6189    #[tokio::test]
6190    async fn test_copy_in_cross_catalog_restriction() -> TestResult<()> {
6191        use catalog::memory::MemoryCatalogManager;
6192        use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
6193        use servers::query_handler::CopyInHandler;
6194
6195        let catalog = MemoryCatalogManager::new_with_table(test_table(1024, "source")?);
6196        // A table in another catalog.
6197        catalog.register_catalog_sync("other").unwrap();
6198        catalog
6199            .register_schema_sync(catalog::RegisterSchemaRequest {
6200                catalog: "other".to_string(),
6201                schema: DEFAULT_SCHEMA_NAME.to_string(),
6202            })
6203            .unwrap();
6204        catalog
6205            .register_table_sync(catalog::RegisterTableRequest {
6206                catalog: "other".to_string(),
6207                schema: DEFAULT_SCHEMA_NAME.to_string(),
6208                table_name: "copy_guard".to_string(),
6209                table_id: 2048,
6210                table: test_table(2048, "copy_guard")?,
6211            })
6212            .unwrap();
6213        let executor = Arc::new(MockProcedureExecutor::new(catalog.clone()));
6214        let plugins = Plugins::new();
6215        plugins.insert(QueryOptions {
6216            disallow_cross_catalog_query: true,
6217        });
6218        let instance = test_instance_with_catalog_manager(
6219            catalog,
6220            test_table(1025, "target")?,
6221            plugins,
6222            None,
6223            executor,
6224        )
6225        .await?;
6226
6227        let ctx = Arc::new(QueryContext::with(
6228            DEFAULT_CATALOG_NAME,
6229            DEFAULT_SCHEMA_NAME,
6230        ));
6231
6232        // Cross-catalog COPY is rejected before any data is streamed.
6233        let err = instance
6234            .copy_in_table("other", DEFAULT_SCHEMA_NAME, "copy_guard", ctx.clone())
6235            .await
6236            .unwrap_err();
6237        assert!(
6238            format!("{err:?}").contains("Illegal access to catalog"),
6239            "{err}"
6240        );
6241
6242        // Same-catalog COPY resolves normally.
6243        let found = instance
6244            .copy_in_table(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "target", ctx)
6245            .await
6246            .unwrap();
6247        assert!(found.is_some());
6248
6249        Ok(())
6250    }
6251
6252    /// Scope-targeted COPY formats (OTLP logs/traces) run the same
6253    /// cross-catalog restriction through `copy_in_validate_scope` before
6254    /// any data is streamed.
6255    #[tokio::test]
6256    async fn test_copy_in_validate_scope_cross_catalog_restriction() -> TestResult<()> {
6257        use catalog::memory::MemoryCatalogManager;
6258        use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
6259        use servers::query_handler::CopyInHandler;
6260
6261        let catalog = MemoryCatalogManager::new_with_table(test_table(1024, "source")?);
6262        let executor = Arc::new(MockProcedureExecutor::new(catalog.clone()));
6263        let plugins = Plugins::new();
6264        plugins.insert(QueryOptions {
6265            disallow_cross_catalog_query: true,
6266        });
6267        let instance = test_instance_with_catalog_manager(
6268            catalog,
6269            test_table(1025, "target")?,
6270            plugins,
6271            None,
6272            executor,
6273        )
6274        .await?;
6275
6276        let ctx = Arc::new(QueryContext::with(
6277            DEFAULT_CATALOG_NAME,
6278            DEFAULT_SCHEMA_NAME,
6279        ));
6280
6281        // A target outside the session's catalog is rejected before any
6282        // data is streamed; a same-catalog schema is not.
6283        let err = instance
6284            .copy_in_validate_scope("other", DEFAULT_SCHEMA_NAME, ctx.clone())
6285            .await
6286            .unwrap_err();
6287        assert!(
6288            format!("{err:?}").contains("Illegal access to catalog"),
6289            "{err}"
6290        );
6291        instance
6292            .copy_in_validate_scope(DEFAULT_CATALOG_NAME, "other_schema", ctx.clone())
6293            .await
6294            .unwrap();
6295
6296        // Without the restriction every scope validates.
6297        let plugins = Plugins::new();
6298        plugins.insert(QueryOptions {
6299            disallow_cross_catalog_query: false,
6300        });
6301        let catalog = MemoryCatalogManager::new_with_table(test_table(1026, "source")?);
6302        let instance = test_instance_with_catalog_manager(
6303            catalog.clone(),
6304            test_table(1027, "target")?,
6305            plugins,
6306            None,
6307            Arc::new(MockProcedureExecutor::new(catalog)),
6308        )
6309        .await?;
6310        instance
6311            .copy_in_validate_scope("other", DEFAULT_SCHEMA_NAME, ctx)
6312            .await
6313            .unwrap();
6314
6315        Ok(())
6316    }
6317
6318    const NATIVE_PROMQL_OPTION: &str = "query.experimental_native_promql";
6319
6320    /// A frontend whose `public` schema holds two metric-engine logical tables, `cpu_usage` and
6321    /// `target`, with `checker` as its permission checker.
6322    async fn test_metric_discovery_instance(checker: PermissionCheckerRef) -> TestResult<Instance> {
6323        let plugins = Plugins::new();
6324        plugins.insert::<PermissionCheckerRef>(checker);
6325        test_instance_with_plugins_and_metric_names(
6326            test_logical_table(1024, "cpu_usage")?,
6327            test_logical_table(1025, "target")?,
6328            plugins,
6329            None,
6330        )
6331        .await
6332    }
6333
6334    /// An expression that has to be answered by discovering the schema's metric tables: it names
6335    /// no exact metric, only a name matcher.
6336    fn discovery_metric_expr() -> promql_parser::parser::Expr {
6337        promql_parser::parser::parse(r#"{__name__=~"cpu.*"}"#).unwrap()
6338    }
6339
6340    /// The tables a bounded scope allows, sorted so a test can state them.
6341    fn allowed_tables(scope: &MetricDiscoveryScope) -> Vec<String> {
6342        let MetricDiscoveryScope::Allowed(allowed) = scope else {
6343            panic!("expected a table-bounded discovery scope, got {scope:?}");
6344        };
6345        let mut tables = allowed
6346            .iter()
6347            .map(|target| target.table.clone())
6348            .collect::<Vec<_>>();
6349        tables.sort_unstable();
6350        tables
6351    }
6352
6353    /// A session `SET` turns the native route on without any request hint, so the scope has to be
6354    /// bounded by the caller's grants in exactly that case too.
6355    #[tokio::test]
6356    async fn test_discovery_scope_bounds_a_session_enabled_native_route() -> TestResult<()> {
6357        let instance =
6358            test_metric_discovery_instance(Arc::new(RejectUnresolvedPermissionChecker)).await?;
6359        let mut ctx = test_query_ctx(1);
6360        ctx.configuration_parameter()
6361            .set_query_option(NATIVE_PROMQL_OPTION, "true")
6362            .unwrap();
6363        assert!(
6364            ctx.extension(NATIVE_PROMQL_OPTION).is_none(),
6365            "a session SET is not a request hint"
6366        );
6367
6368        instance
6369            .prepare_metric_discovery_scope(&discovery_metric_expr(), &mut ctx)
6370            .await
6371            .unwrap();
6372
6373        assert_eq!(
6374            vec!["cpu_usage".to_string(), "target".to_string()],
6375            allowed_tables(ctx.metric_discovery_scope())
6376        );
6377
6378        Ok(())
6379    }
6380
6381    /// With the switch off and nothing stated, the route never reads discovery, so the scope stays
6382    /// unrestricted and no per-table grant is asked for: a query that does not use the route pays
6383    /// nothing here.
6384    #[tokio::test]
6385    async fn test_discovery_scope_is_not_computed_when_the_route_is_off() -> TestResult<()> {
6386        let counter = Arc::new(CountingPermissionChecker::default());
6387        let instance = test_metric_discovery_instance(counter.clone()).await?;
6388        let mut ctx = test_query_ctx(1);
6389        assert!(ctx.extension(NATIVE_PROMQL_OPTION).is_none());
6390        assert_eq!(
6391            None,
6392            ctx.effective_query_options()
6393                .unwrap()
6394                .get(NATIVE_PROMQL_OPTION),
6395            "nothing states the option here"
6396        );
6397
6398        instance
6399            .prepare_metric_discovery_scope(&discovery_metric_expr(), &mut ctx)
6400            .await
6401            .unwrap();
6402
6403        assert!(
6404            matches!(
6405                ctx.metric_discovery_scope(),
6406                MetricDiscoveryScope::Unrestricted
6407            ),
6408            "a route that is off reads no scope: {:?}",
6409            ctx.metric_discovery_scope()
6410        );
6411        assert_eq!(
6412            0,
6413            counter.table_target_checks(),
6414            "an off route asked for per-table grants"
6415        );
6416        assert!(
6417            ctx.extension(NATIVE_PROMQL_OPTION).is_none(),
6418            "resolving the switch must not state the option"
6419        );
6420
6421        Ok(())
6422    }
6423
6424    /// A checker that decides without table targets needs no bound: there is nothing to narrow.
6425    #[tokio::test]
6426    async fn test_discovery_scope_stays_unrestricted_without_table_targets() -> TestResult<()> {
6427        let instance =
6428            test_metric_discovery_instance(Arc::new(TargetIndependentPermissionChecker::default()))
6429                .await?;
6430        let mut ctx = test_query_ctx(1);
6431
6432        instance
6433            .prepare_metric_discovery_scope(&discovery_metric_expr(), &mut ctx)
6434            .await
6435            .unwrap();
6436
6437        assert!(
6438            matches!(
6439                ctx.metric_discovery_scope(),
6440                MetricDiscoveryScope::Unrestricted
6441            ),
6442            "got {:?}",
6443            ctx.metric_discovery_scope()
6444        );
6445
6446        Ok(())
6447    }
6448
6449    /// Resolving the switch must not state it: the option a query inherits stays inherited, so
6450    /// nothing this function does can change what planning later resolves. With nothing stated and
6451    /// the engine default off the route is off, which leaves discovery unrestricted - the scope is
6452    /// the only thing this function changes, and the route reads the option, not the scope.
6453    ///
6454    /// The routing itself is asserted where the plan is built
6455    /// (`query::planner`'s `test_engine_option_is_the_default_a_request_or_session_overrides`
6456    /// covers the "nothing stated, engine default" case): a catalog of plain in-memory tables
6457    /// cannot plan a discovery expression natively, so a frontend-level plan assertion would only
6458    /// restate the legacy planner's own rejections.
6459    #[tokio::test]
6460    async fn test_discovery_scope_does_not_state_the_native_option() -> TestResult<()> {
6461        let instance =
6462            test_metric_discovery_instance(Arc::new(RejectUnresolvedPermissionChecker)).await?;
6463        let mut ctx = test_query_ctx(1);
6464        assert!(ctx.extension(NATIVE_PROMQL_OPTION).is_none());
6465
6466        instance
6467            .prepare_metric_discovery_scope(&discovery_metric_expr(), &mut ctx)
6468            .await
6469            .unwrap();
6470
6471        assert!(
6472            matches!(
6473                ctx.metric_discovery_scope(),
6474                MetricDiscoveryScope::Unrestricted
6475            ),
6476            "an off route reads no scope: {:?}",
6477            ctx.metric_discovery_scope()
6478        );
6479        assert!(
6480            ctx.extension(NATIVE_PROMQL_OPTION).is_none(),
6481            "computing the scope must not state the option"
6482        );
6483        assert_eq!(
6484            None,
6485            ctx.effective_query_options()
6486                .unwrap()
6487                .get(NATIVE_PROMQL_OPTION),
6488            "the route stays on the engine default"
6489        );
6490
6491        Ok(())
6492    }
6493
6494    struct CountingCatalogManager {
6495        inner: Arc<catalog::memory::MemoryCatalogManager>,
6496        table_calls: atomic::AtomicUsize,
6497        table_id_calls: atomic::AtomicUsize,
6498        table_info_by_id_calls: atomic::AtomicUsize,
6499    }
6500
6501    impl CountingCatalogManager {
6502        fn new(inner: Arc<catalog::memory::MemoryCatalogManager>) -> Self {
6503            Self {
6504                inner,
6505                table_calls: atomic::AtomicUsize::new(0),
6506                table_id_calls: atomic::AtomicUsize::new(0),
6507                table_info_by_id_calls: atomic::AtomicUsize::new(0),
6508            }
6509        }
6510
6511        fn reset_counts(&self) {
6512            self.table_calls.store(0, atomic::Ordering::Relaxed);
6513            self.table_id_calls.store(0, atomic::Ordering::Relaxed);
6514            self.table_info_by_id_calls
6515                .store(0, atomic::Ordering::Relaxed);
6516        }
6517
6518        fn table_calls(&self) -> usize {
6519            self.table_calls.load(atomic::Ordering::Relaxed)
6520        }
6521
6522        fn table_id_calls(&self) -> usize {
6523            self.table_id_calls.load(atomic::Ordering::Relaxed)
6524        }
6525
6526        fn table_info_by_id_calls(&self) -> usize {
6527            self.table_info_by_id_calls.load(atomic::Ordering::Relaxed)
6528        }
6529
6530        fn total_table_lookups(&self) -> usize {
6531            self.table_calls() + self.table_id_calls() + self.table_info_by_id_calls()
6532        }
6533    }
6534
6535    #[async_trait::async_trait]
6536    impl catalog::CatalogManager for CountingCatalogManager {
6537        fn as_any(&self) -> &dyn std::any::Any {
6538            self
6539        }
6540
6541        async fn catalog_names(&self) -> catalog::error::Result<Vec<String>> {
6542            self.inner.catalog_names().await
6543        }
6544
6545        async fn schema_names(
6546            &self,
6547            catalog: &str,
6548            query_ctx: Option<&QueryContext>,
6549        ) -> catalog::error::Result<Vec<String>> {
6550            self.inner.schema_names(catalog, query_ctx).await
6551        }
6552
6553        async fn table_names(
6554            &self,
6555            catalog: &str,
6556            schema: &str,
6557            query_ctx: Option<&QueryContext>,
6558        ) -> catalog::error::Result<Vec<String>> {
6559            self.inner.table_names(catalog, schema, query_ctx).await
6560        }
6561
6562        async fn catalog_exists(&self, catalog: &str) -> catalog::error::Result<bool> {
6563            self.inner.catalog_exists(catalog).await
6564        }
6565
6566        async fn schema_exists(
6567            &self,
6568            catalog: &str,
6569            schema: &str,
6570            query_ctx: Option<&QueryContext>,
6571        ) -> catalog::error::Result<bool> {
6572            self.inner.schema_exists(catalog, schema, query_ctx).await
6573        }
6574
6575        async fn table_exists(
6576            &self,
6577            catalog: &str,
6578            schema: &str,
6579            table: &str,
6580            query_ctx: Option<&QueryContext>,
6581        ) -> catalog::error::Result<bool> {
6582            self.inner
6583                .table_exists(catalog, schema, table, query_ctx)
6584                .await
6585        }
6586
6587        async fn table(
6588            &self,
6589            catalog: &str,
6590            schema: &str,
6591            table_name: &str,
6592            query_ctx: Option<&QueryContext>,
6593        ) -> catalog::error::Result<Option<TableRef>> {
6594            self.table_calls.fetch_add(1, atomic::Ordering::Relaxed);
6595            self.inner
6596                .table(catalog, schema, table_name, query_ctx)
6597                .await
6598        }
6599
6600        async fn table_id(
6601            &self,
6602            catalog: &str,
6603            schema: &str,
6604            table_name: &str,
6605            query_ctx: Option<&QueryContext>,
6606        ) -> catalog::error::Result<Option<table::metadata::TableId>> {
6607            self.table_id_calls.fetch_add(1, atomic::Ordering::Relaxed);
6608            self.inner
6609                .table_id(catalog, schema, table_name, query_ctx)
6610                .await
6611        }
6612
6613        async fn table_info_by_id(
6614            &self,
6615            table_id: table::metadata::TableId,
6616        ) -> catalog::error::Result<Option<table::metadata::TableInfoRef>> {
6617            self.table_info_by_id_calls
6618                .fetch_add(1, atomic::Ordering::Relaxed);
6619            self.inner.table_info_by_id(table_id).await
6620        }
6621
6622        async fn tables_by_ids(
6623            &self,
6624            catalog: &str,
6625            schema: &str,
6626            table_ids: &[table::metadata::TableId],
6627        ) -> catalog::error::Result<Vec<TableRef>> {
6628            self.inner.tables_by_ids(catalog, schema, table_ids).await
6629        }
6630
6631        fn tables<'a>(
6632            &'a self,
6633            catalog: &'a str,
6634            schema: &'a str,
6635            query_ctx: Option<&'a QueryContext>,
6636        ) -> futures::stream::BoxStream<'a, catalog::error::Result<TableRef>> {
6637            self.inner.tables(catalog, schema, query_ctx)
6638        }
6639    }
6640
6641    async fn test_instance_with_counting_catalog(
6642        catalog_manager: Arc<CountingCatalogManager>,
6643        plugins: Plugins,
6644    ) -> TestResult<Instance> {
6645        let kv_backend = Arc::new(MemoryKvBackend::new());
6646        let process_manager = Arc::new(ProcessManager::new("test-frontend".to_string(), None));
6647        catalog_manager
6648            .inner
6649            .register_process_list_table(process_manager.clone());
6650        let cache_registry = test_cache_registry(kv_backend.clone())?;
6651
6652        FrontendBuilder::new(
6653            FrontendOptions::default(),
6654            kv_backend,
6655            cache_registry,
6656            catalog_manager,
6657            Arc::new(client::client_manager::NodeClients::default()),
6658            Arc::new(NoopProcedureExecutor),
6659            process_manager,
6660        )
6661        .with_plugin(plugins)
6662        .try_build()
6663        .await
6664        .context(BuildFrontendSnafu)
6665    }
6666
6667    #[tokio::test]
6668    async fn test_precheck_allow_skips_catalog_lookups() -> TestResult<()> {
6669        struct PrecheckAllowChecker;
6670        impl PermissionChecker for PrecheckAllowChecker {
6671            fn check_permission(
6672                &self,
6673                _: UserInfoRef,
6674                _: PermissionReq,
6675            ) -> auth::error::Result<PermissionResp> {
6676                Ok(PermissionResp::Allow)
6677            }
6678
6679            fn precheck_table_targets(
6680                &self,
6681                _: UserInfoRef,
6682                _: PermissionReq,
6683                _: &PermissionTableTargets,
6684            ) -> auth::error::Result<TableTargetPrecheck> {
6685                Ok(TableTargetPrecheck::Allow)
6686            }
6687
6688            fn check_permission_with_table_targets(
6689                &self,
6690                _: UserInfoRef,
6691                _: PermissionReq,
6692                _: PermissionTableTargets,
6693            ) -> auth::error::Result<PermissionResp> {
6694                panic!("check_permission_with_table_targets should be skipped on precheck Allow")
6695            }
6696        }
6697
6698        let plugins = Plugins::new();
6699        plugins.insert::<PermissionCheckerRef>(Arc::new(PrecheckAllowChecker));
6700        let catalog =
6701            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "target")?);
6702        let counting_catalog = Arc::new(CountingCatalogManager::new(catalog));
6703        let instance =
6704            test_instance_with_counting_catalog(counting_catalog.clone(), plugins).await?;
6705        counting_catalog.reset_counts();
6706
6707        instance
6708            .check_resolved_table_permission(
6709                &test_query_ctx(1),
6710                PermissionReq::Action(PROMQL_QUERY),
6711                PermissionTableTargets::resolved(vec![PermissionTableTarget::new(
6712                    "greptime", "public", "target",
6713                )]),
6714            )
6715            .await
6716            .unwrap();
6717        assert_eq!(0, counting_catalog.total_table_lookups());
6718        Ok(())
6719    }
6720
6721    #[tokio::test]
6722    async fn test_precheck_reject_skips_catalog_lookups() -> TestResult<()> {
6723        struct PrecheckRejectChecker;
6724        impl PermissionChecker for PrecheckRejectChecker {
6725            fn check_permission(
6726                &self,
6727                _: UserInfoRef,
6728                _: PermissionReq,
6729            ) -> auth::error::Result<PermissionResp> {
6730                Ok(PermissionResp::Allow)
6731            }
6732
6733            fn precheck_table_targets(
6734                &self,
6735                _: UserInfoRef,
6736                req: PermissionReq,
6737                targets: &PermissionTableTargets,
6738            ) -> auth::error::Result<TableTargetPrecheck> {
6739                Ok(match (req, targets) {
6740                    (PermissionReq::Action(COPY_TO), PermissionTableTargets::Resolved(_)) => {
6741                        TableTargetPrecheck::Allow
6742                    }
6743                    _ => TableTargetPrecheck::Reject,
6744                })
6745            }
6746
6747            fn check_permission_with_table_targets(
6748                &self,
6749                _: UserInfoRef,
6750                _: PermissionReq,
6751                _: PermissionTableTargets,
6752            ) -> auth::error::Result<PermissionResp> {
6753                panic!("check_permission_with_table_targets should be skipped on precheck Reject")
6754            }
6755        }
6756
6757        let plugins = Plugins::new();
6758        plugins.insert::<PermissionCheckerRef>(Arc::new(PrecheckRejectChecker));
6759        let catalog =
6760            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "target")?);
6761        let counting_catalog = Arc::new(CountingCatalogManager::new(catalog));
6762        let instance =
6763            test_instance_with_counting_catalog(counting_catalog.clone(), plugins).await?;
6764        counting_catalog.reset_counts();
6765
6766        let err = instance
6767            .check_resolved_table_permission(
6768                &test_query_ctx(1),
6769                PermissionReq::Action(PROMQL_QUERY),
6770                PermissionTableTargets::resolved(vec![PermissionTableTarget::new(
6771                    "greptime", "public", "target",
6772                )]),
6773            )
6774            .await
6775            .unwrap_err();
6776        assert!(matches!(err, Error::Permission { .. }));
6777        assert_eq!(StatusCode::PermissionDenied, err.status_code());
6778        // A plugin can construct the public variant with an incomplete target.
6779        let err = instance
6780            .check_resolved_table_permission(
6781                &test_query_ctx(1),
6782                PermissionReq::Action(COPY_TO),
6783                PermissionTableTargets::Resolved(vec![PermissionTableTarget::new(
6784                    "greptime", "public", "",
6785                )]),
6786            )
6787            .await
6788            .unwrap_err();
6789        assert_eq!(StatusCode::PermissionDenied, err.status_code());
6790        let err = instance
6791            .check_sql_permission(
6792                &parse_one_sql("COPY target TO '/unused.parquet'"),
6793                &test_query_ctx(1),
6794            )
6795            .await
6796            .unwrap_err();
6797        assert_eq!(StatusCode::PermissionDenied, err.status_code());
6798        assert_eq!(0, counting_catalog.total_table_lookups());
6799        Ok(())
6800    }
6801
6802    #[tokio::test]
6803    async fn test_precheck_partial_indices_only_resolves_listed_targets() -> TestResult<()> {
6804        struct PartialPrecheckChecker(std::sync::Mutex<Vec<PermissionTableTargets>>);
6805        impl PermissionChecker for PartialPrecheckChecker {
6806            fn check_permission(
6807                &self,
6808                _: UserInfoRef,
6809                _: PermissionReq,
6810            ) -> auth::error::Result<PermissionResp> {
6811                Ok(PermissionResp::Allow)
6812            }
6813
6814            fn precheck_table_targets(
6815                &self,
6816                _: UserInfoRef,
6817                _: PermissionReq,
6818                targets: &PermissionTableTargets,
6819            ) -> auth::error::Result<TableTargetPrecheck> {
6820                let PermissionTableTargets::Resolved(targets) = targets else {
6821                    return Ok(TableTargetPrecheck::Reject);
6822                };
6823                if targets.len() == 2 {
6824                    Ok(TableTargetPrecheck::NeedsPhysicalResolution(vec![1]))
6825                } else {
6826                    Ok(TableTargetPrecheck::Allow)
6827                }
6828            }
6829
6830            fn check_permission_with_table_targets(
6831                &self,
6832                _: UserInfoRef,
6833                _: PermissionReq,
6834                targets: PermissionTableTargets,
6835            ) -> auth::error::Result<PermissionResp> {
6836                self.0.lock().unwrap().push(targets);
6837                Ok(PermissionResp::Allow)
6838            }
6839        }
6840
6841        let checker = Arc::new(PartialPrecheckChecker(Default::default()));
6842        let plugins = Plugins::new();
6843        plugins.insert::<PermissionCheckerRef>(checker.clone());
6844        let catalog =
6845            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "wide")?);
6846        catalog
6847            .register_table_sync(catalog::RegisterTableRequest {
6848                catalog: "greptime".to_string(),
6849                schema: "public".to_string(),
6850                table_name: "narrow".to_string(),
6851                table_id: 1025,
6852                table: test_physical_table(1025, "narrow")?,
6853            })
6854            .with_context(|_| RegisterTableSnafu {
6855                table_name: "narrow".to_string(),
6856            })?;
6857        let counting_catalog = Arc::new(CountingCatalogManager::new(catalog));
6858        let instance =
6859            test_instance_with_counting_catalog(counting_catalog.clone(), plugins).await?;
6860        counting_catalog.reset_counts();
6861
6862        instance
6863            .check_resolved_table_permission(
6864                &test_query_ctx(1),
6865                PermissionReq::Action(PROMQL_QUERY),
6866                PermissionTableTargets::resolved(vec![
6867                    PermissionTableTarget::new("greptime", "public", "wide"),
6868                    PermissionTableTarget::new("greptime", "public", "narrow"),
6869                ]),
6870            )
6871            .await
6872            .unwrap();
6873        assert_eq!(1, counting_catalog.table_calls());
6874        assert_eq!(0, counting_catalog.table_id_calls());
6875        assert_eq!(0, counting_catalog.table_info_by_id_calls());
6876        assert_eq!(
6877            *checker.0.lock().unwrap(),
6878            vec![PermissionTableTargets::Resolved(vec![
6879                PermissionTableTarget::new("greptime", "public", "wide"),
6880                PermissionTableTarget::new("greptime", "public", "narrow")
6881                    .with_physical_metric_table(true),
6882            ])]
6883        );
6884        Ok(())
6885    }
6886
6887    #[tokio::test]
6888    async fn test_precheck_runs_once_for_multi_chunk_resolution() -> TestResult<()> {
6889        struct MultiChunkPrecheckChecker {
6890            precheck_calls: std::sync::Mutex<Vec<PermissionTableTargets>>,
6891            final_calls: std::sync::Mutex<Vec<PermissionTableTargets>>,
6892        }
6893        impl PermissionChecker for MultiChunkPrecheckChecker {
6894            fn check_permission(
6895                &self,
6896                _: UserInfoRef,
6897                _: PermissionReq,
6898            ) -> auth::error::Result<PermissionResp> {
6899                Ok(PermissionResp::Allow)
6900            }
6901
6902            fn precheck_table_targets(
6903                &self,
6904                _: UserInfoRef,
6905                _: PermissionReq,
6906                targets: &PermissionTableTargets,
6907            ) -> auth::error::Result<TableTargetPrecheck> {
6908                self.precheck_calls.lock().unwrap().push(targets.clone());
6909                let PermissionTableTargets::Resolved(targets) = targets else {
6910                    return Ok(TableTargetPrecheck::Reject);
6911                };
6912                Ok(TableTargetPrecheck::NeedsPhysicalResolution(
6913                    (0..targets.len()).collect(),
6914                ))
6915            }
6916
6917            fn check_permission_with_table_targets(
6918                &self,
6919                _: UserInfoRef,
6920                _: PermissionReq,
6921                targets: PermissionTableTargets,
6922            ) -> auth::error::Result<PermissionResp> {
6923                self.final_calls.lock().unwrap().push(targets);
6924                Ok(PermissionResp::Allow)
6925            }
6926        }
6927
6928        let checker = Arc::new(MultiChunkPrecheckChecker {
6929            precheck_calls: Default::default(),
6930            final_calls: Default::default(),
6931        });
6932        let plugins = Plugins::new();
6933        plugins.insert::<PermissionCheckerRef>(checker.clone());
6934        let catalog =
6935            catalog::memory::MemoryCatalogManager::new_with_table(test_physical_table(1024, "t0")?);
6936        let mut targets = vec![PermissionTableTarget::new("greptime", "public", "t0")];
6937        for i in 1..10u32 {
6938            let name = format!("t{i}");
6939            catalog
6940                .register_table_sync(catalog::RegisterTableRequest {
6941                    catalog: "greptime".to_string(),
6942                    schema: "public".to_string(),
6943                    table_name: name.clone(),
6944                    table_id: 1024 + i,
6945                    table: test_table(1024 + i, &name)?,
6946                })
6947                .with_context(|_| RegisterTableSnafu { table_name: name })?;
6948            targets.push(PermissionTableTarget::new(
6949                "greptime",
6950                "public",
6951                format!("t{i}"),
6952            ));
6953        }
6954        let counting_catalog = Arc::new(CountingCatalogManager::new(catalog));
6955        let instance =
6956            test_instance_with_counting_catalog(counting_catalog.clone(), plugins).await?;
6957        counting_catalog.reset_counts();
6958
6959        let mut request_targets = targets.clone();
6960        request_targets.push(targets[0].clone());
6961        instance
6962            .check_resolved_table_permission(
6963                &test_query_ctx(1),
6964                PermissionReq::Action(PROMQL_QUERY),
6965                PermissionTableTargets::resolved(request_targets),
6966            )
6967            .await
6968            .unwrap();
6969        assert_eq!(10, counting_catalog.table_calls());
6970        assert_eq!(0, counting_catalog.table_id_calls());
6971        assert_eq!(0, counting_catalog.table_info_by_id_calls());
6972
6973        let precheck_calls = checker.precheck_calls.lock().unwrap();
6974        assert_eq!(1, precheck_calls.len());
6975        assert_eq!(
6976            PermissionTableTargets::Resolved(targets.clone()),
6977            precheck_calls[0]
6978        );
6979
6980        let mut expected_final = targets;
6981        expected_final[0].is_physical_metric_table = true;
6982        let final_calls = checker.final_calls.lock().unwrap();
6983        assert_eq!(1, final_calls.len());
6984        assert_eq!(
6985            PermissionTableTargets::Resolved(expected_final),
6986            final_calls[0]
6987        );
6988        Ok(())
6989    }
6990
6991    #[tokio::test]
6992    async fn test_target_independent_checker_preserves_target_hook_without_lookups()
6993    -> TestResult<()> {
6994        struct EndpointHookChecker {
6995            target_hook_calls: atomic::AtomicUsize,
6996        }
6997        impl PermissionChecker for EndpointHookChecker {
6998            fn check_permission(
6999                &self,
7000                _: UserInfoRef,
7001                _: PermissionReq,
7002            ) -> auth::error::Result<PermissionResp> {
7003                Ok(PermissionResp::Allow)
7004            }
7005
7006            fn uses_table_targets(&self) -> bool {
7007                false
7008            }
7009
7010            fn check_permission_with_table_targets(
7011                &self,
7012                _: UserInfoRef,
7013                _: PermissionReq,
7014                _: PermissionTableTargets,
7015            ) -> auth::error::Result<PermissionResp> {
7016                self.target_hook_calls
7017                    .fetch_add(1, atomic::Ordering::Relaxed);
7018                Ok(PermissionResp::Reject)
7019            }
7020        }
7021
7022        let checker = Arc::new(EndpointHookChecker {
7023            target_hook_calls: atomic::AtomicUsize::new(0),
7024        });
7025        let plugins = Plugins::new();
7026        plugins.insert::<PermissionCheckerRef>(checker.clone());
7027        let catalog =
7028            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "target")?);
7029        let counting_catalog = Arc::new(CountingCatalogManager::new(catalog));
7030        let instance =
7031            test_instance_with_counting_catalog(counting_catalog.clone(), plugins).await?;
7032        counting_catalog.reset_counts();
7033
7034        let err = instance
7035            .check_resolved_table_permission(
7036                &test_query_ctx(1),
7037                PermissionReq::Action(PROMQL_QUERY),
7038                PermissionTableTargets::resolved(vec![PermissionTableTarget::new(
7039                    "greptime", "public", "target",
7040                )]),
7041            )
7042            .await
7043            .unwrap_err();
7044        assert_eq!(StatusCode::PermissionDenied, err.status_code());
7045        assert_eq!(0, counting_catalog.total_table_lookups());
7046        assert_eq!(1, checker.target_hook_calls.load(atomic::Ordering::Relaxed));
7047        Ok(())
7048    }
7049
7050    #[tokio::test]
7051    async fn test_precheck_out_of_bounds_index_returns_internal_error() -> TestResult<()> {
7052        struct BadIndexChecker;
7053        impl PermissionChecker for BadIndexChecker {
7054            fn check_permission(
7055                &self,
7056                _: UserInfoRef,
7057                _: PermissionReq,
7058            ) -> auth::error::Result<PermissionResp> {
7059                Ok(PermissionResp::Allow)
7060            }
7061
7062            fn precheck_table_targets(
7063                &self,
7064                _: UserInfoRef,
7065                _: PermissionReq,
7066                _: &PermissionTableTargets,
7067            ) -> auth::error::Result<TableTargetPrecheck> {
7068                Ok(TableTargetPrecheck::NeedsPhysicalResolution(vec![99]))
7069            }
7070
7071            fn check_permission_with_table_targets(
7072                &self,
7073                _: UserInfoRef,
7074                _: PermissionReq,
7075                _: PermissionTableTargets,
7076            ) -> auth::error::Result<PermissionResp> {
7077                unreachable!("must fail before check_permission_with_table_targets")
7078            }
7079        }
7080
7081        let plugins = Plugins::new();
7082        plugins.insert::<PermissionCheckerRef>(Arc::new(BadIndexChecker));
7083        let catalog =
7084            catalog::memory::MemoryCatalogManager::new_with_table(test_table(1024, "target")?);
7085        let counting_catalog = Arc::new(CountingCatalogManager::new(catalog));
7086        let instance =
7087            test_instance_with_counting_catalog(counting_catalog.clone(), plugins).await?;
7088        counting_catalog.reset_counts();
7089        let ctx = test_query_ctx(1);
7090
7091        for targets in [
7092            PermissionTableTargets::resolved(vec![PermissionTableTarget::new(
7093                "greptime", "public", "target",
7094            )]),
7095            PermissionTableTargets::Unresolved,
7096        ] {
7097            let err = instance
7098                .check_resolved_table_permission(&ctx, PermissionReq::Action(PROMQL_QUERY), targets)
7099                .await
7100                .unwrap_err();
7101            assert!(matches!(err, Error::Permission { .. }));
7102            assert_eq!(StatusCode::Unexpected, err.status_code());
7103            assert_eq!(0, counting_catalog.total_table_lookups());
7104        }
7105        Ok(())
7106    }
7107}