1use std::collections::HashMap;
16use std::convert::Infallible;
17use std::fmt::Display;
18use std::net::SocketAddr;
19use std::sync::{Arc, Mutex as StdMutex};
20use std::time::Duration;
21
22use async_trait::async_trait;
23use auth::UserProviderRef;
24use axum::extract::{DefaultBodyLimit, Request, State};
25use axum::http::StatusCode as HttpStatusCode;
26use axum::middleware::Next;
27use axum::response::{IntoResponse, Response};
28use axum::routing::Route;
29use axum::serve::ListenerExt;
30use axum::{Extension, Router, middleware, routing};
31use common_base::readable_size::ReadableSize;
32use common_recordbatch::RecordBatch;
33use common_telemetry::{error, info};
34use common_time::Timestamp;
35use common_time::timestamp::TimeUnit;
36use datatypes::data_type::DataType;
37use datatypes::schema::SchemaRef;
38use event::{LogState, LogValidatorRef};
39use futures::FutureExt;
40use http::{HeaderValue, Method};
41use serde::{Deserialize, Serialize};
42use serde_json::Value;
43use session::context::QueryContext;
44use snafu::{ResultExt, ensure};
45use tokio::sync::Mutex;
46use tokio::sync::oneshot::{self, Sender};
47use tonic::codegen::Service;
48use tower::{Layer, ServiceBuilder};
49use tower_http::compression::CompressionLayer;
50use tower_http::cors::{AllowOrigin, Any, CorsLayer};
51use tower_http::decompression::RequestDecompressionLayer;
52use tower_http::trace::TraceLayer;
53
54use self::authorize::AuthState;
55use self::result::table_result::TableResponse;
56use crate::batcher::BatchingProtocol;
57use crate::batcher::logical_table::LogicalTablePendingRowsBatcher;
58use crate::elasticsearch;
59use crate::error::{
60 AddressBindSnafu, AlreadyStartedSnafu, Error, InternalIoSnafu, InvalidHeaderValueSnafu, Result,
61};
62use crate::http::influxdb::{influxdb_health, influxdb_ping, influxdb_write_v1, influxdb_write_v2};
63use crate::http::otlp::OtlpState;
64use crate::http::prom_store::PromStoreState;
65use crate::http::prometheus::{
66 build_info_query, format_query, instant_query, label_values_query, labels_query,
67 metadata_query, parse_query, query_exemplars, range_query, series_query,
68};
69use crate::http::result::arrow_result::ArrowResponse;
70use crate::http::result::csv_result::CsvResponse;
71use crate::http::result::error_result::ErrorResponse;
72use crate::http::result::greptime_result_v1::GreptimedbV1Response;
73use crate::http::result::influxdb_result_v1::InfluxdbV1Response;
74use crate::http::result::json_result::JsonResponse;
75use crate::http::result::null_result::NullResponse;
76use crate::interceptor::LogIngestInterceptorRef;
77use crate::metrics::http_metrics_layer;
78use crate::metrics_handler::MetricsHandler;
79use crate::prometheus_handler::PrometheusHandlerRef;
80use crate::query_handler::sql::ServerSqlQueryHandlerRef;
81use crate::query_handler::{
82 DashboardHandlerRef, InfluxdbLineProtocolHandlerRef, JaegerQueryHandlerRef, LogQueryHandlerRef,
83 OpenTelemetryProtocolHandlerRef, OpentsdbProtocolHandlerRef, PipelineHandlerRef,
84 PromStoreProtocolHandlerRef,
85};
86use crate::request_memory_limiter::ServerMemoryLimiter;
87use crate::server::Server;
88
89pub mod authorize;
90#[cfg(feature = "dashboard")]
91mod dashboard;
92pub mod dyn_log;
93pub mod dyn_trace;
94pub mod event;
95pub mod extractor;
96pub mod handler;
97pub mod header;
98pub mod influxdb;
99pub mod jaeger;
100pub mod logs;
101pub mod loki;
102pub mod mem_prof;
103mod memory_limit;
104pub mod opentsdb;
105pub mod otlp;
106pub mod pprof;
107pub mod prom_store;
108pub mod prometheus;
109pub mod result;
110pub mod splunk;
111mod timeout;
112pub mod utils;
113mod workload_scheduler;
114
115pub use memory_limit::decoded_body_accounting_middleware;
116use result::HttpOutputWriter;
117pub(crate) use timeout::DynamicTimeoutLayer;
118
119mod client_ip;
120use crate::prom_remote_write::validation::PromValidationMode;
121mod hints;
122mod read_preference;
123mod skip_wal;
124#[cfg(any(test, feature = "testing"))]
125pub mod test_helpers;
126
127pub const HTTP_API_VERSION: &str = "v1";
128pub const HTTP_API_PREFIX: &str = "/v1/";
129pub const HTTP_API_PREFIX_WITHOUT_TRAILING_SLASH: &str = "/v1";
130
131pub trait ExtraHttpRouterProvider: Send + Sync {
133 fn router(&self) -> Router;
135}
136
137pub type ExtraHttpRouterProviderRef = Arc<dyn ExtraHttpRouterProvider>;
138
139#[derive(Clone, Default)]
141pub struct ExtraHttpRouterProviders {
142 providers: Vec<ExtraHttpRouterProviderRef>,
143}
144
145impl ExtraHttpRouterProviders {
146 pub fn new() -> Self {
148 Self::default()
149 }
150
151 pub fn add(&mut self, provider: ExtraHttpRouterProviderRef) {
153 self.providers.push(provider);
154 }
155
156 pub fn iter(&self) -> impl Iterator<Item = &dyn ExtraHttpRouterProvider> {
158 self.providers.iter().map(|x| x.as_ref())
159 }
160}
161
162const DEFAULT_BODY_LIMIT: ReadableSize = ReadableSize::mb(64);
164const DEFAULT_HTTP_API_ADDR_PORT: u16 = 4006;
166
167pub const AUTHORIZATION_HEADER: &str = "x-greptime-auth";
169
170pub static PUBLIC_API_PREFIX: [&str; 5] = [
173 "/v1/auth/config",
174 "/v1/influxdb/ping",
175 "/v1/influxdb/health",
176 "/v1/health",
177 "/v1/splunk/services/collector/health",
178];
179
180#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
186pub(crate) enum HttpServerKind {
187 #[default]
190 Full,
191 Api,
193}
194
195#[derive(Default)]
196pub struct HttpServer {
197 router: StdMutex<Router>,
198 shutdown_tx: Mutex<Option<Sender<()>>>,
199 user_provider: Option<UserProviderRef>,
200 memory_limiter: ServerMemoryLimiter,
201
202 options: HttpOptions,
204 batching_protocols: Vec<BatchingProtocol>,
205 logical_batching_protocols: Vec<BatchingProtocol>,
206 bind_addr: Option<SocketAddr>,
207 kind: HttpServerKind,
209}
210
211pub(crate) fn is_namespace(path: &str, root: &str) -> bool {
215 path == root
216 || path
217 .strip_prefix(root)
218 .is_some_and(|suffix| suffix.starts_with('/'))
219}
220
221pub fn is_api_listener_path(path: &str) -> bool {
223 is_namespace(path, HTTP_API_PREFIX_WITHOUT_TRAILING_SLASH) || is_namespace(path, "/dashboard")
224}
225
226#[derive(Clone)]
227struct LogicalBatchingProtocols(Vec<BatchingProtocol>);
228
229pub async fn set_http_write_batching(
231 State(protocol): State<BatchingProtocol>,
232 mut req: Request,
233 next: Next,
234) -> Response {
235 let enabled = req
236 .extensions()
237 .get::<Arc<Vec<BatchingProtocol>>>()
238 .is_some_and(|protocols| protocols.contains(&protocol));
239 let logical_enabled = req
240 .extensions()
241 .get::<LogicalBatchingProtocols>()
242 .is_some_and(|protocols| protocols.0.contains(&protocol));
243 let body_memory_bytes = req
244 .extensions()
245 .get::<memory_limit::BodyMemoryAccounting>()
246 .map(memory_limit::BodyMemoryAccounting::reserved_bytes_counter);
247 if let Some(ctx) = req.extensions_mut().get_mut::<QueryContext>() {
248 ctx.set_batching_enabled(enabled);
249 ctx.set_logical_batching_enabled(logical_enabled);
250 if let Some(bytes) = body_memory_bytes {
251 ctx.set_body_memory_bytes(bytes);
252 }
253 }
254 next.run(req).await
255}
256
257async fn enforce_api_surface(req: Request, next: Next) -> Response {
261 if !is_api_listener_path(req.uri().path()) {
262 return HttpStatusCode::NOT_FOUND.into_response();
263 }
264 next.run(req).await
265}
266
267impl HttpServer {
268 fn kind(&self) -> &'static str {
270 match self.kind {
271 HttpServerKind::Api => "HTTP API",
272 HttpServerKind::Full => "HTTP",
273 }
274 }
275}
276
277#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
278#[serde(default)]
279pub struct HttpOptions {
280 pub addr: String,
281
282 #[serde(with = "humantime_serde")]
285 pub timeout: Duration,
286
287 #[serde(skip)]
288 pub disable_dashboard: bool,
289
290 pub body_limit: ReadableSize,
291
292 pub cors_allowed_origins: Vec<String>,
293
294 pub enable_cors: bool,
295
296 pub enable_api_server: bool,
301 pub api_server_addr: String,
304}
305
306impl Default for HttpOptions {
307 fn default() -> Self {
308 Self {
309 addr: "127.0.0.1:4000".to_string(),
310 timeout: Duration::from_secs(0),
311 disable_dashboard: false,
312 body_limit: DEFAULT_BODY_LIMIT,
313 cors_allowed_origins: Vec::new(),
314 enable_cors: true,
315 enable_api_server: false,
316 api_server_addr: format!("127.0.0.1:{}", DEFAULT_HTTP_API_ADDR_PORT),
317 }
318 }
319}
320
321#[derive(Debug, Serialize, Deserialize, Eq, PartialEq)]
322pub struct ColumnSchema {
323 name: String,
324 data_type: String,
325}
326
327impl ColumnSchema {
328 pub fn new(name: String, data_type: String) -> ColumnSchema {
329 ColumnSchema { name, data_type }
330 }
331}
332
333#[derive(Debug, Serialize, Deserialize, Eq, PartialEq)]
334pub struct OutputSchema {
335 column_schemas: Vec<ColumnSchema>,
336}
337
338impl OutputSchema {
339 pub fn new(columns: Vec<ColumnSchema>) -> OutputSchema {
340 OutputSchema {
341 column_schemas: columns,
342 }
343 }
344}
345
346impl From<SchemaRef> for OutputSchema {
347 fn from(schema: SchemaRef) -> OutputSchema {
348 OutputSchema {
349 column_schemas: schema
350 .column_schemas()
351 .iter()
352 .map(|cs| ColumnSchema {
353 name: cs.name.clone(),
354 data_type: cs.data_type.name(),
355 })
356 .collect(),
357 }
358 }
359}
360
361#[derive(Debug, Serialize, Deserialize, Eq, PartialEq)]
362pub struct HttpRecordsOutput {
363 schema: OutputSchema,
364 rows: Vec<Vec<Value>>,
365 #[serde(default)]
368 total_rows: usize,
369
370 #[serde(skip_serializing_if = "HashMap::is_empty")]
372 #[serde(default)]
373 metrics: HashMap<String, Value>,
374}
375
376impl HttpRecordsOutput {
377 pub fn num_rows(&self) -> usize {
378 self.rows.len()
379 }
380
381 pub fn num_cols(&self) -> usize {
382 self.schema.column_schemas.len()
383 }
384
385 pub fn schema(&self) -> &OutputSchema {
386 &self.schema
387 }
388
389 pub fn rows(&self) -> &Vec<Vec<Value>> {
390 &self.rows
391 }
392}
393
394impl HttpRecordsOutput {
395 pub fn try_new(
396 schema: SchemaRef,
397 recordbatches: Vec<RecordBatch>,
398 ) -> std::result::Result<HttpRecordsOutput, Error> {
399 if recordbatches.is_empty() {
400 Ok(HttpRecordsOutput {
401 schema: OutputSchema::from(schema),
402 rows: vec![],
403 total_rows: 0,
404 metrics: Default::default(),
405 })
406 } else {
407 let num_rows = recordbatches.iter().map(|r| r.num_rows()).sum::<usize>();
408 let mut rows = Vec::with_capacity(num_rows);
409
410 for recordbatch in recordbatches {
411 let mut writer = HttpOutputWriter::new(schema.num_columns(), None);
412 writer.write(recordbatch, &mut rows)?;
413 }
414
415 Ok(HttpRecordsOutput {
416 schema: OutputSchema::from(schema),
417 total_rows: rows.len(),
418 rows,
419 metrics: Default::default(),
420 })
421 }
422 }
423}
424
425#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
426#[serde(rename_all = "lowercase")]
427pub enum GreptimeQueryOutput {
428 AffectedRows(usize),
429 Records(HttpRecordsOutput),
430}
431
432#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
434pub enum ResponseFormat {
435 Arrow,
436 Csv(bool, bool),
438 Table,
439 #[default]
440 GreptimedbV1,
441 InfluxdbV1,
442 Json,
443 Null,
444}
445
446impl ResponseFormat {
447 pub fn parse(s: &str) -> Option<Self> {
448 match s {
449 "arrow" => Some(ResponseFormat::Arrow),
450 "csv" => Some(ResponseFormat::Csv(false, false)),
451 "csvwithnames" => Some(ResponseFormat::Csv(true, false)),
452 "csvwithnamesandtypes" => Some(ResponseFormat::Csv(true, true)),
453 "table" => Some(ResponseFormat::Table),
454 "greptimedb_v1" => Some(ResponseFormat::GreptimedbV1),
455 "influxdb_v1" => Some(ResponseFormat::InfluxdbV1),
456 "json" => Some(ResponseFormat::Json),
457 "null" => Some(ResponseFormat::Null),
458 _ => None,
459 }
460 }
461
462 pub fn as_str(&self) -> &'static str {
463 match self {
464 ResponseFormat::Arrow => "arrow",
465 ResponseFormat::Csv(_, _) => "csv",
466 ResponseFormat::Table => "table",
467 ResponseFormat::GreptimedbV1 => "greptimedb_v1",
468 ResponseFormat::InfluxdbV1 => "influxdb_v1",
469 ResponseFormat::Json => "json",
470 ResponseFormat::Null => "null",
471 }
472 }
473}
474
475#[derive(Debug, Clone, Copy, PartialEq, Eq)]
476pub enum Epoch {
477 Nanosecond,
478 Microsecond,
479 Millisecond,
480 Second,
481}
482
483impl Epoch {
484 pub fn parse(s: &str) -> Option<Epoch> {
485 match s {
490 "ns" => Some(Epoch::Nanosecond),
491 "u" | "µ" => Some(Epoch::Microsecond),
492 "ms" => Some(Epoch::Millisecond),
493 "s" => Some(Epoch::Second),
494 _ => None, }
496 }
497
498 pub fn convert_timestamp(&self, ts: Timestamp) -> Option<Timestamp> {
499 match self {
500 Epoch::Nanosecond => ts.convert_to(TimeUnit::Nanosecond),
501 Epoch::Microsecond => ts.convert_to(TimeUnit::Microsecond),
502 Epoch::Millisecond => ts.convert_to(TimeUnit::Millisecond),
503 Epoch::Second => ts.convert_to(TimeUnit::Second),
504 }
505 }
506}
507
508impl Display for Epoch {
509 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
510 match self {
511 Epoch::Nanosecond => write!(f, "Epoch::Nanosecond"),
512 Epoch::Microsecond => write!(f, "Epoch::Microsecond"),
513 Epoch::Millisecond => write!(f, "Epoch::Millisecond"),
514 Epoch::Second => write!(f, "Epoch::Second"),
515 }
516 }
517}
518
519#[derive(Serialize, Deserialize, Debug)]
520pub enum HttpResponse {
521 Arrow(ArrowResponse),
522 Csv(CsvResponse),
523 Table(TableResponse),
524 Error(ErrorResponse),
525 GreptimedbV1(GreptimedbV1Response),
526 InfluxdbV1(InfluxdbV1Response),
527 Json(JsonResponse),
528 Null(NullResponse),
529}
530
531impl HttpResponse {
532 pub fn with_execution_time(self, execution_time: u64) -> Self {
533 match self {
534 HttpResponse::Arrow(resp) => resp.with_execution_time(execution_time).into(),
535 HttpResponse::Csv(resp) => resp.with_execution_time(execution_time).into(),
536 HttpResponse::Table(resp) => resp.with_execution_time(execution_time).into(),
537 HttpResponse::GreptimedbV1(resp) => resp.with_execution_time(execution_time).into(),
538 HttpResponse::InfluxdbV1(resp) => resp.with_execution_time(execution_time).into(),
539 HttpResponse::Json(resp) => resp.with_execution_time(execution_time).into(),
540 HttpResponse::Null(resp) => resp.with_execution_time(execution_time).into(),
541 HttpResponse::Error(resp) => resp.with_execution_time(execution_time).into(),
542 }
543 }
544
545 pub fn with_limit(self, limit: usize) -> Self {
546 match self {
547 HttpResponse::Csv(resp) => resp.with_limit(limit).into(),
548 HttpResponse::Table(resp) => resp.with_limit(limit).into(),
549 HttpResponse::GreptimedbV1(resp) => resp.with_limit(limit).into(),
550 HttpResponse::Json(resp) => resp.with_limit(limit).into(),
551 _ => self,
552 }
553 }
554}
555
556pub fn process_with_limit(
557 mut outputs: Vec<GreptimeQueryOutput>,
558 limit: usize,
559) -> Vec<GreptimeQueryOutput> {
560 outputs
561 .drain(..)
562 .map(|data| match data {
563 GreptimeQueryOutput::Records(mut records) => {
564 if records.rows.len() > limit {
565 records.rows.truncate(limit);
566 records.total_rows = limit;
567 }
568 GreptimeQueryOutput::Records(records)
569 }
570 _ => data,
571 })
572 .collect()
573}
574
575impl IntoResponse for HttpResponse {
576 fn into_response(self) -> Response {
577 match self {
578 HttpResponse::Arrow(resp) => resp.into_response(),
579 HttpResponse::Csv(resp) => resp.into_response(),
580 HttpResponse::Table(resp) => resp.into_response(),
581 HttpResponse::GreptimedbV1(resp) => resp.into_response(),
582 HttpResponse::InfluxdbV1(resp) => resp.into_response(),
583 HttpResponse::Json(resp) => resp.into_response(),
584 HttpResponse::Null(resp) => resp.into_response(),
585 HttpResponse::Error(resp) => resp.into_response(),
586 }
587 }
588}
589
590impl From<ArrowResponse> for HttpResponse {
591 fn from(value: ArrowResponse) -> Self {
592 HttpResponse::Arrow(value)
593 }
594}
595
596impl From<CsvResponse> for HttpResponse {
597 fn from(value: CsvResponse) -> Self {
598 HttpResponse::Csv(value)
599 }
600}
601
602impl From<TableResponse> for HttpResponse {
603 fn from(value: TableResponse) -> Self {
604 HttpResponse::Table(value)
605 }
606}
607
608impl From<ErrorResponse> for HttpResponse {
609 fn from(value: ErrorResponse) -> Self {
610 HttpResponse::Error(value)
611 }
612}
613
614impl From<GreptimedbV1Response> for HttpResponse {
615 fn from(value: GreptimedbV1Response) -> Self {
616 HttpResponse::GreptimedbV1(value)
617 }
618}
619
620impl From<InfluxdbV1Response> for HttpResponse {
621 fn from(value: InfluxdbV1Response) -> Self {
622 HttpResponse::InfluxdbV1(value)
623 }
624}
625
626impl From<JsonResponse> for HttpResponse {
627 fn from(value: JsonResponse) -> Self {
628 HttpResponse::Json(value)
629 }
630}
631
632impl From<NullResponse> for HttpResponse {
633 fn from(value: NullResponse) -> Self {
634 HttpResponse::Null(value)
635 }
636}
637
638#[derive(Clone)]
639pub struct ApiState {
640 pub sql_handler: ServerSqlQueryHandlerRef,
641}
642
643#[derive(Clone)]
644pub struct GreptimeOptionsConfigState {
645 pub greptime_config_options: String,
646}
647
648#[derive(Clone)]
649pub struct DashboardState {
650 pub handler: DashboardHandlerRef,
651}
652
653pub struct HttpServerBuilder {
654 options: HttpOptions,
655 batching_protocols: Vec<BatchingProtocol>,
656 logical_batching_protocols: Vec<BatchingProtocol>,
657 user_provider: Option<UserProviderRef>,
658 router: Router,
659 memory_limiter: ServerMemoryLimiter,
660}
661
662impl HttpServerBuilder {
663 pub fn new(options: HttpOptions) -> Self {
664 Self {
665 options,
666 batching_protocols: Vec::new(),
667 logical_batching_protocols: Vec::new(),
668 user_provider: None,
669 router: Router::new(),
670 memory_limiter: ServerMemoryLimiter::default(),
671 }
672 }
673
674 pub fn with_logical_batching_protocols(mut self, protocols: Vec<BatchingProtocol>) -> Self {
676 self.logical_batching_protocols = protocols;
677 self
678 }
679
680 pub fn with_batching_protocols(mut self, protocols: Vec<BatchingProtocol>) -> Self {
682 self.batching_protocols = protocols;
683 self
684 }
685
686 pub fn with_memory_limiter(mut self, limiter: ServerMemoryLimiter) -> Self {
688 self.memory_limiter = limiter;
689 self
690 }
691
692 pub fn with_sql_handler(self, sql_handler: ServerSqlQueryHandlerRef) -> Self {
693 let batch_body_limit = if self.options.body_limit.0 == 0 {
694 4 * 1024 * 1024
695 } else {
696 (self.options.body_limit.0 as usize).min(4 * 1024 * 1024)
697 };
698 let sql_router = HttpServer::route_sql(ApiState { sql_handler }, batch_body_limit);
699
700 Self {
701 router: self
702 .router
703 .nest(&format!("/{HTTP_API_VERSION}"), sql_router),
704 ..self
705 }
706 }
707
708 pub fn with_logs_handler(self, logs_handler: LogQueryHandlerRef) -> Self {
709 let logs_router = HttpServer::route_logs(logs_handler);
710
711 Self {
712 router: self
713 .router
714 .nest(&format!("/{HTTP_API_VERSION}"), logs_router),
715 ..self
716 }
717 }
718
719 pub fn with_opentsdb_handler(self, handler: OpentsdbProtocolHandlerRef) -> Self {
720 Self {
721 router: self.router.nest(
722 &format!("/{HTTP_API_VERSION}/opentsdb"),
723 HttpServer::route_opentsdb(handler),
724 ),
725 ..self
726 }
727 }
728
729 pub fn with_influxdb_handler(self, handler: InfluxdbLineProtocolHandlerRef) -> Self {
730 Self {
731 router: self.router.nest(
732 &format!("/{HTTP_API_VERSION}/influxdb"),
733 HttpServer::route_influxdb(handler, self.memory_limiter.clone()),
734 ),
735 ..self
736 }
737 }
738
739 pub fn with_prom_handler(
740 self,
741 handler: PromStoreProtocolHandlerRef,
742 pipeline_handler: Option<PipelineHandlerRef>,
743 prom_store_with_metric_engine: bool,
744 prom_validation_mode: PromValidationMode,
745 pending_rows_batcher: Option<Arc<LogicalTablePendingRowsBatcher>>,
746 ) -> Self {
747 let state = PromStoreState {
748 prom_store_handler: handler,
749 pipeline_handler,
750 prom_store_with_metric_engine,
751 prom_validation_mode,
752 pending_rows_batcher,
753 memory_limiter: self.memory_limiter.clone(),
754 };
755
756 Self {
757 router: self.router.nest(
758 &format!("/{HTTP_API_VERSION}/prometheus"),
759 HttpServer::route_prom(state),
760 ),
761 ..self
762 }
763 }
764
765 pub fn with_prometheus_handler(self, handler: PrometheusHandlerRef) -> Self {
766 Self {
767 router: self.router.nest(
768 &format!("/{HTTP_API_VERSION}/prometheus/api/v1"),
769 HttpServer::route_prometheus(handler),
770 ),
771 ..self
772 }
773 }
774
775 pub fn with_otlp_handler(
776 self,
777 handler: OpenTelemetryProtocolHandlerRef,
778 with_metric_engine: bool,
779 ) -> Self {
780 Self {
781 router: self.router.nest(
782 &format!("/{HTTP_API_VERSION}/otlp"),
783 HttpServer::route_otlp(handler, with_metric_engine, self.memory_limiter.clone()),
784 ),
785 ..self
786 }
787 }
788
789 pub fn with_user_provider(self, user_provider: UserProviderRef) -> Self {
790 Self {
791 user_provider: Some(user_provider),
792 ..self
793 }
794 }
795
796 pub fn with_metrics_handler(self, handler: MetricsHandler) -> Self {
797 Self {
798 router: self.router.merge(HttpServer::route_metrics(handler)),
799 ..self
800 }
801 }
802
803 pub fn with_log_ingest_handler(
804 self,
805 handler: PipelineHandlerRef,
806 validator: Option<LogValidatorRef>,
807 ingest_interceptor: Option<LogIngestInterceptorRef<Error>>,
808 ) -> Self {
809 let log_state = LogState {
810 log_handler: handler,
811 log_validator: validator,
812 ingest_interceptor,
813 };
814
815 let router = self.router.nest(
816 &format!("/{HTTP_API_VERSION}"),
817 HttpServer::route_pipelines(log_state.clone(), self.memory_limiter.clone()),
818 );
819 let router = router.nest(
821 &format!("/{HTTP_API_VERSION}/events"),
822 #[allow(deprecated)]
823 HttpServer::route_log_deprecated(log_state.clone(), self.memory_limiter.clone()),
824 );
825
826 let router = router.nest(
827 &format!("/{HTTP_API_VERSION}/loki"),
828 HttpServer::route_loki(log_state.clone(), self.memory_limiter.clone()),
829 );
830
831 let router = router.nest(
832 &format!("/{HTTP_API_VERSION}/elasticsearch"),
833 HttpServer::route_elasticsearch(log_state.clone(), self.memory_limiter.clone()),
834 );
835
836 let router = router.nest(
837 &format!("/{HTTP_API_VERSION}/elasticsearch/"),
838 Router::new()
839 .route("/", routing::get(elasticsearch::handle_get_version))
840 .with_state(log_state.clone()),
841 );
842
843 let router = router.nest(
844 &format!("/{HTTP_API_VERSION}/splunk"),
845 HttpServer::route_splunk(log_state, self.memory_limiter.clone()),
846 );
847
848 Self { router, ..self }
849 }
850
851 pub fn with_greptime_config_options(self, opts: String) -> Self {
852 let config_router = HttpServer::route_config(GreptimeOptionsConfigState {
853 greptime_config_options: opts,
854 });
855
856 Self {
857 router: self.router.merge(config_router),
858 ..self
859 }
860 }
861
862 pub fn with_jaeger_handler(self, handler: JaegerQueryHandlerRef) -> Self {
863 Self {
864 router: self.router.nest(
865 &format!("/{HTTP_API_VERSION}/jaeger"),
866 HttpServer::route_jaeger(handler),
867 ),
868 ..self
869 }
870 }
871
872 pub fn with_dashboard_handler(self, handler: DashboardHandlerRef) -> Self {
873 Self {
874 router: self.router.nest(
875 &format!("/{HTTP_API_VERSION}/dashboards"),
876 HttpServer::route_dashboard(handler, self.memory_limiter.clone()),
877 ),
878 ..self
879 }
880 }
881
882 pub fn with_extra_router(self, router: Router) -> Self {
883 Self {
884 router: self.router.merge(router),
885 ..self
886 }
887 }
888
889 pub fn add_layer<L>(self, layer: L) -> Self
890 where
891 L: Layer<Route> + Clone + Send + Sync + 'static,
892 L::Service: Service<Request> + Clone + Send + Sync + 'static,
893 <L::Service as Service<Request>>::Response: IntoResponse + 'static,
894 <L::Service as Service<Request>>::Error: Into<Infallible> + 'static,
895 <L::Service as Service<Request>>::Future: Send + 'static,
896 {
897 Self {
898 router: self.router.layer(layer),
899 ..self
900 }
901 }
902
903 pub fn build(self) -> HttpServer {
904 HttpServer {
905 options: self.options,
906 batching_protocols: self.batching_protocols.clone(),
907 logical_batching_protocols: self.logical_batching_protocols.clone(),
908 user_provider: self.user_provider,
909 shutdown_tx: Mutex::new(None),
910 router: StdMutex::new(self.router),
911 bind_addr: None,
912 memory_limiter: self.memory_limiter,
913 kind: HttpServerKind::Full,
914 }
915 }
916
917 pub fn build_servers(self) -> (HttpServer, Option<HttpServer>) {
936 let api_enabled = self.options.enable_api_server;
937 let api_addr = self.options.api_server_addr.clone();
938
939 let internal = HttpServer {
940 options: self.options,
941 batching_protocols: self.batching_protocols.clone(),
942 logical_batching_protocols: self.logical_batching_protocols.clone(),
943 user_provider: self.user_provider.clone(),
944 shutdown_tx: Mutex::new(None),
945 router: StdMutex::new(self.router.clone()),
946 bind_addr: None,
947 memory_limiter: self.memory_limiter.clone(),
948 kind: HttpServerKind::Full,
949 };
950
951 let api = if api_enabled {
952 let api_options = HttpOptions {
955 addr: api_addr,
956 ..internal.options.clone()
957 };
958 Some(HttpServer {
959 options: api_options,
960 batching_protocols: self.batching_protocols,
961 logical_batching_protocols: self.logical_batching_protocols,
962 user_provider: self.user_provider.clone(),
963 shutdown_tx: Mutex::new(None),
964 router: StdMutex::new(self.router),
965 bind_addr: None,
966 memory_limiter: self.memory_limiter,
967 kind: HttpServerKind::Api,
968 })
969 } else {
970 None
971 };
972
973 (internal, api)
974 }
975}
976
977impl HttpServer {
978 pub fn make_app(&self) -> Router {
988 let mut router = self.router.lock().unwrap().clone();
989 let auth_router = Router::new()
990 .route("/v1/auth/config", routing::get(authorize::auth_config))
991 .with_state(AuthState::new(self.user_provider.clone()));
992
993 router = router
994 .merge(auth_router)
995 .route(
996 &format!("/{HTTP_API_VERSION}/health"),
997 routing::get(handler::health).post(handler::health),
998 )
999 .route("/", routing::get(handler::index))
1000 .route(
1001 "/health",
1002 routing::get(handler::health).post(handler::health),
1003 )
1004 .route(
1005 "/ready",
1006 routing::get(handler::health).post(handler::health),
1007 )
1008 .route("/status", routing::get(handler::status));
1009
1010 #[cfg(feature = "dashboard")]
1013 {
1014 if !self.options.disable_dashboard {
1015 info!("Enable dashboard service at '/dashboard'");
1016 router = router.route(
1018 "/dashboard",
1019 routing::get(|uri: axum::http::uri::Uri| async move {
1020 let path = uri.path();
1021 let query = uri.query().map(|q| format!("?{}", q)).unwrap_or_default();
1022
1023 let new_uri = format!("{}/{}", path, query);
1024 axum::response::Redirect::permanent(&new_uri)
1025 }),
1026 );
1027
1028 router = router
1032 .route(
1033 "/dashboard/",
1034 routing::get(dashboard::static_handler).post(dashboard::static_handler),
1035 )
1036 .route(
1037 "/dashboard/{*x}",
1038 routing::get(dashboard::static_handler).post(dashboard::static_handler),
1039 );
1040 }
1041 }
1042
1043 router = router.route_layer(middleware::from_fn(http_metrics_layer));
1045
1046 router
1047 }
1048
1049 pub fn build(&self, router: Router) -> Result<Router> {
1052 let timeout_layer = ServiceBuilder::new().layer(
1053 DynamicTimeoutLayer::new(self.options.timeout)
1054 .with_status_code_fn(Self::request_timeout_status_code),
1055 );
1056 let body_limit_layer = if self.options.body_limit != ReadableSize(0) {
1057 Some(
1058 ServiceBuilder::new()
1059 .layer(DefaultBodyLimit::max(self.options.body_limit.0 as usize)),
1060 )
1061 } else {
1062 info!("HTTP server body limit is disabled");
1063 None
1064 };
1065 let cors_layer = if self.options.enable_cors {
1066 Some(
1067 CorsLayer::new()
1068 .allow_methods([
1069 Method::GET,
1070 Method::POST,
1071 Method::PUT,
1072 Method::DELETE,
1073 Method::HEAD,
1074 ])
1075 .allow_origin(cors_allow_origin(&self.options.cors_allowed_origins)?)
1076 .allow_headers(Any),
1077 )
1078 } else {
1079 info!("HTTP server cross-origin is disabled");
1080 None
1081 };
1082
1083 let router = router
1084 .layer(
1086 ServiceBuilder::new()
1087 .layer(TraceLayer::new_for_http().on_failure(()))
1090 .option_layer(cors_layer)
1091 .layer(timeout_layer)
1092 .option_layer(body_limit_layer)
1093 .layer(middleware::from_fn_with_state(
1095 self.memory_limiter.clone(),
1096 memory_limit::memory_limit_middleware,
1097 ))
1098 .layer(middleware::from_fn_with_state(
1100 AuthState::new(self.user_provider.clone()),
1101 authorize::check_http_auth,
1102 ))
1103 .layer(Extension(Arc::new(self.batching_protocols.clone())))
1104 .layer(Extension(LogicalBatchingProtocols(
1105 self.logical_batching_protocols.clone(),
1106 )))
1107 .layer(middleware::from_fn(hints::extract_hints))
1108 .layer(middleware::from_fn(client_ip::log_error_with_client_ip))
1109 .layer(middleware::from_fn(
1110 read_preference::extract_read_preference,
1111 ))
1112 .layer(middleware::from_fn(skip_wal::extract_skip_wal)),
1113 );
1114
1115 let router = router.nest(
1118 "/debug",
1119 Router::new()
1120 .route("/log_level", routing::post(dyn_log::dyn_log_handler))
1122 .route(
1123 "/workload_scheduler",
1124 routing::get(workload_scheduler::get_status_handler),
1125 )
1126 .route(
1127 "/workload_scheduler/enabled",
1128 routing::post(workload_scheduler::set_enabled_handler),
1129 )
1130 .route(
1131 "/workload_scheduler/weights",
1132 routing::post(workload_scheduler::set_weights_handler),
1133 )
1134 .route("/enable_trace", routing::post(dyn_trace::dyn_trace_handler))
1135 .nest(
1136 "/prof",
1137 Router::new()
1138 .route("/cpu", routing::post(pprof::pprof_handler))
1139 .route("/mem", routing::post(mem_prof::mem_prof_handler))
1140 .route("/mem/symbol", routing::post(mem_prof::symbolicate_handler))
1141 .route(
1142 "/mem/activate",
1143 routing::post(mem_prof::activate_heap_prof_handler),
1144 )
1145 .route(
1146 "/mem/deactivate",
1147 routing::post(mem_prof::deactivate_heap_prof_handler),
1148 )
1149 .route(
1150 "/mem/status",
1151 routing::get(mem_prof::heap_prof_status_handler),
1152 ) .route(
1154 "/mem/gdump",
1155 routing::get(mem_prof::gdump_status_handler)
1156 .post(mem_prof::gdump_toggle_handler),
1157 ),
1158 ),
1159 );
1160
1161 if self.kind == HttpServerKind::Api {
1166 Ok(router.layer(middleware::from_fn(enforce_api_surface)))
1167 } else {
1168 Ok(router)
1169 }
1170 }
1171
1172 fn request_timeout_status_code(request: &Request) -> HttpStatusCode {
1173 if request.uri().path() == "/v1/prometheus/write" {
1174 HttpStatusCode::GATEWAY_TIMEOUT
1175 } else {
1176 HttpStatusCode::REQUEST_TIMEOUT
1177 }
1178 }
1179
1180 fn route_metrics<S>(metrics_handler: MetricsHandler) -> Router<S> {
1181 Router::new()
1182 .route("/metrics", routing::get(handler::metrics))
1183 .with_state(metrics_handler)
1184 }
1185
1186 fn route_loki<S>(log_state: LogState, memory_limiter: ServerMemoryLimiter) -> Router<S> {
1187 Router::new()
1188 .route("/api/v1/push", routing::post(loki::loki_ingest))
1189 .layer(middleware::from_fn_with_state(
1190 memory_limiter,
1191 memory_limit::decoded_body_accounting_middleware,
1192 ))
1193 .layer(
1194 ServiceBuilder::new()
1195 .layer(RequestDecompressionLayer::new().pass_through_unaccepted(true)),
1196 )
1197 .layer(middleware::from_fn_with_state(
1198 BatchingProtocol::Loki,
1199 set_http_write_batching,
1200 ))
1201 .with_state(log_state)
1202 }
1203
1204 fn route_splunk<S>(log_state: LogState, memory_limiter: ServerMemoryLimiter) -> Router<S> {
1205 Router::new()
1206 .route(
1207 "/services/collector/health",
1208 routing::get(splunk::handle_health),
1209 )
1210 .route(
1211 "/services/collector/health/1.0",
1212 routing::get(splunk::handle_health),
1213 )
1214 .route(
1217 "/services/collector/event",
1218 routing::post(splunk::handle_event),
1219 )
1220 .route("/services/collector", routing::post(splunk::handle_event))
1221 .route(
1222 "/services/collector/event/1.0",
1223 routing::post(splunk::handle_event),
1224 )
1225 .route("/services/collector/raw", routing::post(splunk::handle_raw))
1228 .route(
1229 "/services/collector/raw/1.0",
1230 routing::post(splunk::handle_raw),
1231 )
1232 .layer(middleware::from_fn_with_state(
1233 memory_limiter,
1234 memory_limit::decoded_body_accounting_middleware,
1235 ))
1236 .layer(
1237 ServiceBuilder::new()
1238 .layer(RequestDecompressionLayer::new().pass_through_unaccepted(true)),
1239 )
1240 .layer(middleware::from_fn_with_state(
1241 BatchingProtocol::Splunk,
1242 set_http_write_batching,
1243 ))
1244 .with_state(log_state)
1245 }
1246
1247 fn route_elasticsearch<S>(
1248 log_state: LogState,
1249 memory_limiter: ServerMemoryLimiter,
1250 ) -> Router<S> {
1251 Router::new()
1252 .route(
1254 "/",
1255 routing::head((HttpStatusCode::OK, elasticsearch::elasticsearch_headers())),
1256 )
1257 .route("/", routing::get(elasticsearch::handle_get_version))
1259 .route("/_license", routing::get(elasticsearch::handle_get_license))
1261 .route("/_bulk", routing::post(elasticsearch::handle_bulk_api))
1262 .route(
1263 "/{index}/_bulk",
1264 routing::post(elasticsearch::handle_bulk_api_with_index),
1265 )
1266 .route(
1268 "/_ilm/policy/{*path}",
1269 routing::any((
1270 HttpStatusCode::OK,
1271 elasticsearch::elasticsearch_headers(),
1272 axum::Json(serde_json::json!({})),
1273 )),
1274 )
1275 .route(
1277 "/_index_template/{*path}",
1278 routing::any((
1279 HttpStatusCode::OK,
1280 elasticsearch::elasticsearch_headers(),
1281 axum::Json(serde_json::json!({})),
1282 )),
1283 )
1284 .route(
1287 "/_ingest/{*path}",
1288 routing::any((
1289 HttpStatusCode::OK,
1290 elasticsearch::elasticsearch_headers(),
1291 axum::Json(serde_json::json!({})),
1292 )),
1293 )
1294 .route(
1297 "/_nodes/{*path}",
1298 routing::any((
1299 HttpStatusCode::OK,
1300 elasticsearch::elasticsearch_headers(),
1301 axum::Json(serde_json::json!({})),
1302 )),
1303 )
1304 .route(
1307 "/logstash/{*path}",
1308 routing::any((
1309 HttpStatusCode::OK,
1310 elasticsearch::elasticsearch_headers(),
1311 axum::Json(serde_json::json!({})),
1312 )),
1313 )
1314 .route(
1315 "/_logstash/{*path}",
1316 routing::any((
1317 HttpStatusCode::OK,
1318 elasticsearch::elasticsearch_headers(),
1319 axum::Json(serde_json::json!({})),
1320 )),
1321 )
1322 .layer(middleware::from_fn_with_state(
1323 memory_limiter,
1324 memory_limit::decoded_body_accounting_middleware,
1325 ))
1326 .layer(ServiceBuilder::new().layer(RequestDecompressionLayer::new()))
1327 .layer(middleware::from_fn_with_state(
1328 BatchingProtocol::Elasticsearch,
1329 set_http_write_batching,
1330 ))
1331 .with_state(log_state)
1332 }
1333
1334 #[deprecated(since = "0.11.0", note = "Use `route_pipelines()` instead.")]
1335 fn route_log_deprecated<S>(
1336 log_state: LogState,
1337 memory_limiter: ServerMemoryLimiter,
1338 ) -> Router<S> {
1339 Router::new()
1340 .route("/logs", routing::post(event::log_ingester))
1341 .route(
1342 "/pipelines/{pipeline_name}",
1343 routing::get(event::query_pipeline),
1344 )
1345 .route(
1346 "/pipelines/{pipeline_name}",
1347 routing::post(event::add_pipeline),
1348 )
1349 .route(
1350 "/pipelines/{pipeline_name}",
1351 routing::delete(event::delete_pipeline),
1352 )
1353 .route("/pipelines/dryrun", routing::post(event::pipeline_dryrun))
1354 .layer(middleware::from_fn_with_state(
1355 memory_limiter,
1356 memory_limit::decoded_body_accounting_middleware,
1357 ))
1358 .layer(
1359 ServiceBuilder::new()
1360 .layer(RequestDecompressionLayer::new().pass_through_unaccepted(true)),
1361 )
1362 .layer(middleware::from_fn_with_state(
1363 BatchingProtocol::Logs,
1364 set_http_write_batching,
1365 ))
1366 .with_state(log_state)
1367 }
1368
1369 fn route_pipelines<S>(log_state: LogState, memory_limiter: ServerMemoryLimiter) -> Router<S> {
1370 Router::new()
1371 .route("/ingest", routing::post(event::log_ingester))
1372 .route(
1373 "/pipelines/{pipeline_name}",
1374 routing::get(event::query_pipeline),
1375 )
1376 .route(
1377 "/pipelines/{pipeline_name}/ddl",
1378 routing::get(event::query_pipeline_ddl),
1379 )
1380 .route(
1381 "/pipelines/{pipeline_name}",
1382 routing::post(event::add_pipeline),
1383 )
1384 .route(
1385 "/pipelines/{pipeline_name}",
1386 routing::delete(event::delete_pipeline),
1387 )
1388 .route("/pipelines/_dryrun", routing::post(event::pipeline_dryrun))
1389 .layer(middleware::from_fn_with_state(
1390 memory_limiter,
1391 memory_limit::decoded_body_accounting_middleware,
1392 ))
1393 .layer(
1394 ServiceBuilder::new()
1395 .layer(RequestDecompressionLayer::new().pass_through_unaccepted(true)),
1396 )
1397 .layer(middleware::from_fn_with_state(
1398 BatchingProtocol::Logs,
1399 set_http_write_batching,
1400 ))
1401 .with_state(log_state)
1402 }
1403
1404 fn route_sql<S>(api_state: ApiState, batch_body_limit: usize) -> Router<S> {
1405 Router::new()
1406 .route(
1407 "/capabilities",
1408 routing::get(|State(state): State<ApiState>| async move {
1409 let mut capabilities =
1410 serde_json::json!({"metric_packed_import": 1, "metric_packed_export": 1});
1411 if state.sql_handler.supports_metric_batch_ddl() {
1412 capabilities["metric_batch_ddl"] = serde_json::json!(1);
1413 }
1414 axum::Json(capabilities)
1415 }),
1416 )
1417 .route(
1418 "/ddl/logical-tables",
1419 routing::post(handler::create_logical_tables)
1420 .layer(DefaultBodyLimit::max(batch_body_limit)),
1421 )
1422 .route(
1423 "/sql",
1424 routing::get(handler::sql).post(handler::sql).layer(
1425 middleware::from_fn_with_state(
1426 BatchingProtocol::HttpSql,
1427 set_http_write_batching,
1428 ),
1429 ),
1430 )
1431 .route(
1432 "/sql/parse",
1433 routing::get(handler::sql_parse).post(handler::sql_parse),
1434 )
1435 .route(
1436 "/sql/format",
1437 routing::get(handler::sql_format).post(handler::sql_format),
1438 )
1439 .route(
1440 "/promql",
1441 routing::get(handler::promql).post(handler::promql),
1442 )
1443 .route(
1444 "/sql/analyze/stream",
1445 routing::post(handler::sql_analyze_stream),
1446 )
1447 .with_state(api_state)
1448 }
1449
1450 fn route_logs<S>(log_handler: LogQueryHandlerRef) -> Router<S> {
1451 Router::new()
1452 .route("/logs", routing::get(logs::logs).post(logs::logs))
1453 .with_state(log_handler)
1454 }
1455
1456 pub fn route_prometheus<S>(prometheus_handler: PrometheusHandlerRef) -> Router<S> {
1460 Router::new()
1461 .route(
1462 "/format_query",
1463 routing::post(format_query).get(format_query),
1464 )
1465 .route("/status/buildinfo", routing::get(build_info_query))
1466 .route("/query", routing::post(instant_query).get(instant_query))
1467 .route("/query_range", routing::post(range_query).get(range_query))
1468 .route(
1469 "/query_exemplars",
1470 routing::post(query_exemplars).get(query_exemplars),
1471 )
1472 .route("/labels", routing::post(labels_query).get(labels_query))
1473 .route("/metadata", routing::get(metadata_query))
1474 .route("/series", routing::post(series_query).get(series_query))
1475 .route("/parse_query", routing::post(parse_query).get(parse_query))
1476 .route(
1477 "/label/{label_name}/values",
1478 routing::get(label_values_query),
1479 )
1480 .layer(ServiceBuilder::new().layer(CompressionLayer::new()))
1481 .with_state(prometheus_handler)
1482 }
1483
1484 fn route_prom<S>(state: PromStoreState) -> Router<S> {
1490 let write = routing::post(prom_store::remote_write);
1491 let write = if state.prom_store_with_metric_engine {
1492 write
1493 } else {
1494 write.layer(middleware::from_fn_with_state(
1495 BatchingProtocol::Prom,
1496 set_http_write_batching,
1497 ))
1498 };
1499 Router::new()
1500 .route("/read", routing::post(prom_store::remote_read))
1501 .route("/write", write)
1502 .with_state(state)
1503 }
1504
1505 fn route_influxdb<S>(
1506 influxdb_handler: InfluxdbLineProtocolHandlerRef,
1507 memory_limiter: ServerMemoryLimiter,
1508 ) -> Router<S> {
1509 Router::new()
1510 .route("/write", routing::post(influxdb_write_v1))
1511 .route("/api/v2/write", routing::post(influxdb_write_v2))
1512 .layer(middleware::from_fn_with_state(
1513 memory_limiter,
1514 memory_limit::decoded_body_accounting_middleware,
1515 ))
1516 .layer(
1517 ServiceBuilder::new()
1518 .layer(RequestDecompressionLayer::new().pass_through_unaccepted(true)),
1519 )
1520 .route("/ping", routing::get(influxdb_ping))
1521 .route("/health", routing::get(influxdb_health))
1522 .layer(middleware::from_fn_with_state(
1523 BatchingProtocol::Influxdb,
1524 set_http_write_batching,
1525 ))
1526 .with_state(influxdb_handler)
1527 }
1528
1529 fn route_opentsdb<S>(opentsdb_handler: OpentsdbProtocolHandlerRef) -> Router<S> {
1530 Router::new()
1531 .route("/api/put", routing::post(opentsdb::put))
1532 .layer(middleware::from_fn_with_state(
1533 BatchingProtocol::Opentsdb,
1534 set_http_write_batching,
1535 ))
1536 .with_state(opentsdb_handler)
1537 }
1538
1539 fn route_otlp<S>(
1540 otlp_handler: OpenTelemetryProtocolHandlerRef,
1541 with_metric_engine: bool,
1542 memory_limiter: ServerMemoryLimiter,
1543 ) -> Router<S> {
1544 Router::new()
1545 .route("/v1/metrics", routing::post(otlp::metrics))
1546 .route("/v1/traces", routing::post(otlp::traces))
1547 .route("/v1/logs", routing::post(otlp::logs))
1548 .layer(middleware::from_fn_with_state(
1549 memory_limiter,
1550 memory_limit::decoded_body_accounting_middleware,
1551 ))
1552 .layer(
1553 ServiceBuilder::new()
1554 .layer(RequestDecompressionLayer::new().pass_through_unaccepted(true)),
1555 )
1556 .layer(middleware::from_fn_with_state(
1557 BatchingProtocol::Otlp,
1558 set_http_write_batching,
1559 ))
1560 .with_state(OtlpState {
1561 with_metric_engine,
1562 handler: otlp_handler,
1563 })
1564 }
1565
1566 fn route_config<S>(state: GreptimeOptionsConfigState) -> Router<S> {
1567 Router::new()
1568 .route("/config", routing::get(handler::config))
1569 .with_state(state)
1570 }
1571
1572 fn route_jaeger<S>(handler: JaegerQueryHandlerRef) -> Router<S> {
1573 Router::new()
1574 .route("/api/services", routing::get(jaeger::handle_get_services))
1575 .route(
1576 "/api/services/{service_name}/operations",
1577 routing::get(jaeger::handle_get_operations_by_service),
1578 )
1579 .route(
1580 "/api/operations",
1581 routing::get(jaeger::handle_get_operations),
1582 )
1583 .route("/api/traces", routing::get(jaeger::handle_find_traces))
1584 .route(
1585 "/api/traces/{trace_id}",
1586 routing::get(jaeger::handle_get_trace),
1587 )
1588 .route(
1589 "/api/v3/services",
1590 routing::get(jaeger::v3::handle_get_services),
1591 )
1592 .route(
1593 "/api/v3/operations",
1594 routing::get(jaeger::v3::handle_get_operations),
1595 )
1596 .route(
1597 "/api/v3/traces",
1598 routing::get(jaeger::v3::handle_find_traces),
1599 )
1600 .route(
1601 "/api/v3/trace-summaries",
1602 routing::get(jaeger::v3::handle_find_trace_summaries),
1603 )
1604 .route(
1605 "/api/v3/traces/{trace_id}",
1606 routing::get(jaeger::v3::handle_get_trace),
1607 )
1608 .with_state(handler)
1609 }
1610
1611 #[cfg(feature = "dashboard")]
1612 fn route_dashboard<S>(
1613 handler: DashboardHandlerRef,
1614 memory_limiter: ServerMemoryLimiter,
1615 ) -> Router<S> {
1616 use crate::http::dashboard::{add_dashboard, delete_dashboard, list_dashboards};
1617
1618 Router::new()
1619 .route("/", routing::get(list_dashboards))
1620 .route("/{dashboard_name}", routing::post(add_dashboard))
1621 .route("/{dashboard_name}", routing::delete(delete_dashboard))
1622 .layer(middleware::from_fn_with_state(
1623 memory_limiter,
1624 memory_limit::decoded_body_accounting_middleware,
1625 ))
1626 .layer(
1627 ServiceBuilder::new()
1628 .layer(RequestDecompressionLayer::new().pass_through_unaccepted(true)),
1629 )
1630 .with_state(DashboardState { handler })
1631 }
1632
1633 #[cfg(not(feature = "dashboard"))]
1634 fn route_dashboard<S>(
1635 handler: DashboardHandlerRef,
1636 _memory_limiter: ServerMemoryLimiter,
1637 ) -> Router<S> {
1638 Router::new().with_state(DashboardState { handler })
1639 }
1640}
1641
1642pub const HTTP_SERVER: &str = "HTTP_SERVER";
1643pub const HTTP_API_SERVER: &str = "HTTP_API_SERVER";
1644
1645pub(crate) fn cors_allow_origin(origins: &[String]) -> Result<AllowOrigin> {
1647 if origins.is_empty() {
1648 return Ok(AllowOrigin::any());
1649 }
1650 let origins = origins
1651 .iter()
1652 .map(|origin| HeaderValue::from_str(origin).context(InvalidHeaderValueSnafu))
1653 .collect::<Result<Vec<_>>>()?;
1654 Ok(AllowOrigin::list(origins))
1655}
1656
1657#[async_trait]
1658impl Server for HttpServer {
1659 async fn shutdown(&self) -> Result<()> {
1660 let mut shutdown_tx = self.shutdown_tx.lock().await;
1661 if let Some(tx) = shutdown_tx.take()
1662 && tx.send(()).is_err()
1663 {
1664 info!("Receiver dropped, the HTTP server has already exited");
1665 }
1666 info!("Shutdown {}", self.kind());
1667
1668 Ok(())
1669 }
1670
1671 async fn start(&mut self, listening: SocketAddr) -> Result<()> {
1672 let (tx, rx) = oneshot::channel();
1673 let serve = {
1674 let mut shutdown_tx = self.shutdown_tx.lock().await;
1675 ensure!(
1676 shutdown_tx.is_none(),
1677 AlreadyStartedSnafu {
1678 server: self.kind()
1679 }
1680 );
1681
1682 let app = self.build(self.make_app())?;
1683 let listener = tokio::net::TcpListener::bind(listening)
1684 .await
1685 .context(AddressBindSnafu { addr: listening })?
1686 .tap_io(|tcp_stream| {
1687 if let Err(e) = tcp_stream.set_nodelay(true) {
1688 error!(e; "Failed to set TCP_NODELAY on incoming connection");
1689 }
1690 });
1691 let serve = axum::serve(
1692 listener,
1693 app.into_make_service_with_connect_info::<SocketAddr>(),
1694 );
1695
1696 *shutdown_tx = Some(tx);
1713
1714 serve
1715 };
1716 let listening = serve.local_addr().context(InternalIoSnafu)?;
1717 info!("{} server is bound to {}", self.kind(), listening);
1718
1719 common_runtime::spawn_global(async move {
1720 if let Err(e) = serve
1721 .with_graceful_shutdown(rx.map(drop))
1722 .await
1723 .context(InternalIoSnafu)
1724 {
1725 error!(e; "Failed to shutdown http server");
1726 }
1727 });
1728
1729 self.bind_addr = Some(listening);
1730 Ok(())
1731 }
1732
1733 fn name(&self) -> &str {
1734 match self.kind {
1735 HttpServerKind::Api => HTTP_API_SERVER,
1736 HttpServerKind::Full => HTTP_SERVER,
1737 }
1738 }
1739
1740 fn bind_addr(&self) -> Option<SocketAddr> {
1741 self.bind_addr
1742 }
1743
1744 fn as_any(&self) -> &dyn std::any::Any {
1745 self
1746 }
1747}
1748
1749#[cfg(test)]
1750mod test {
1751 use std::future::pending;
1752 use std::io::Cursor;
1753 use std::sync::Arc;
1754
1755 use arrow_ipc::reader::StreamReader;
1756 use arrow_schema::DataType;
1757 use axum::http::StatusCode;
1758 use axum::routing::{get, post};
1759 use common_query::{Output, OutputData};
1760 use common_recordbatch::RecordBatches;
1761 use datafusion_expr::LogicalPlan;
1762 use datatypes::prelude::*;
1763 use datatypes::schema::{ColumnSchema, Schema};
1764 use datatypes::vectors::{StringVector, UInt32Vector};
1765 use header::constants::GREPTIME_DB_HEADER_TIMEOUT;
1766 use query::parser::PromQuery;
1767 use query::query_engine::DescribeResult;
1768 use session::context::QueryContextRef;
1769 use sql::statements::statement::Statement;
1770 use tokio::sync::mpsc;
1771 use tokio::time::Instant;
1772
1773 use crate::http::test_helpers::TestClient;
1774 use crate::http::*;
1775 use crate::prom_remote_write::validation::validate_label_name;
1776 use crate::query_handler::sql::SqlQueryHandler;
1777
1778 struct DummyInstance {
1779 _tx: mpsc::Sender<(String, Vec<u8>)>,
1780 }
1781
1782 #[async_trait]
1783 impl SqlQueryHandler for DummyInstance {
1784 async fn do_query(&self, _: &str, _: QueryContextRef) -> Vec<Result<Output>> {
1785 unimplemented!()
1786 }
1787
1788 async fn do_analyze_stream_query(&self, _: &str, _: QueryContextRef) -> Result<Output> {
1789 let stream = common_recordbatch::RecordBatches::empty().as_stream();
1790 Ok(Output::new(OutputData::Stream(stream), Default::default()))
1791 }
1792
1793 async fn do_promql_query(&self, _: &PromQuery, _: QueryContextRef) -> Vec<Result<Output>> {
1794 unimplemented!()
1795 }
1796
1797 async fn do_exec_plan(
1798 &self,
1799 _plan: LogicalPlan,
1800 _stmt: Option<Statement>,
1801 _query_ctx: QueryContextRef,
1802 ) -> Result<Output> {
1803 unimplemented!()
1804 }
1805
1806 async fn do_describe(
1807 &self,
1808 _stmt: sql::statements::statement::Statement,
1809 _query_ctx: QueryContextRef,
1810 ) -> Result<Option<DescribeResult>> {
1811 unimplemented!()
1812 }
1813
1814 async fn is_valid_schema(&self, _catalog: &str, _schema: &str) -> Result<bool> {
1815 Ok(true)
1816 }
1817 }
1818
1819 async fn forever() {
1820 pending().await
1821 }
1822
1823 fn make_test_app(tx: mpsc::Sender<(String, Vec<u8>)>) -> Router {
1824 make_test_app_custom(tx, HttpOptions::default())
1825 }
1826
1827 fn make_test_app_custom(tx: mpsc::Sender<(String, Vec<u8>)>, options: HttpOptions) -> Router {
1828 let instance = Arc::new(DummyInstance { _tx: tx });
1829 let server = HttpServerBuilder::new(options)
1830 .with_sql_handler(instance.clone())
1831 .build();
1832 let app = server
1833 .make_app()
1834 .route("/test/timeout", get(forever))
1835 .route("/v1/prometheus/write", post(forever));
1836 server.build(app).unwrap()
1837 }
1838
1839 fn make_split_builder() -> HttpServerBuilder {
1840 let (tx, _rx) = mpsc::channel(100);
1841 let instance = Arc::new(DummyInstance { _tx: tx });
1842 let options = HttpOptions {
1844 enable_api_server: true,
1845 ..HttpOptions::default()
1846 };
1847 HttpServerBuilder::new(options)
1848 .with_sql_handler(instance)
1849 .with_metrics_handler(MetricsHandler)
1850 .with_greptime_config_options("dummy = \"value\"".to_string())
1851 }
1852
1853 #[tokio::test]
1854 pub async fn test_http_api_options_defaults() {
1855 let opts = HttpOptions::default();
1856 assert!(!opts.enable_api_server);
1858 assert_eq!(opts.api_server_addr, "127.0.0.1:4006");
1859 }
1860
1861 #[tokio::test]
1862 async fn test_auth_config_without_provider_disables_auth() {
1863 let server = HttpServerBuilder::new(HttpOptions::default()).build();
1864 let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
1865 let response = client.get("/v1/auth/config").send().await;
1866
1867 assert_eq!(response.status(), StatusCode::OK);
1868 assert_eq!(
1869 response.json::<serde_json::Value>().await,
1870 serde_json::json!({"auth_required": false, "methods": []})
1871 );
1872 }
1873
1874 #[tokio::test]
1875 async fn test_auth_config_with_provider_is_public() {
1876 let provider =
1877 auth::static_user_provider_from_option("static_user_provider:cmd:user=password")
1878 .unwrap();
1879 let server = HttpServerBuilder::new(HttpOptions::default())
1880 .with_user_provider(Arc::new(provider))
1881 .build();
1882 let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
1883 let response = client.get("/v1/auth/config").send().await;
1884
1885 assert_eq!(response.status(), StatusCode::OK);
1886 assert_eq!(
1887 response.json::<serde_json::Value>().await,
1888 serde_json::json!({
1889 "auth_required": true,
1890 "methods": [{"type": "password"}]
1891 })
1892 );
1893 }
1894
1895 #[tokio::test]
1896 pub async fn test_build_serves_all_routes() {
1897 let server = make_split_builder().build();
1901 assert_eq!(server.name(), HTTP_SERVER);
1902
1903 let app = server.build(server.make_app()).unwrap();
1904 let client = TestClient::new(app).await;
1905
1906 for path in ["/v1/health", "/health", "/status", "/metrics", "/config"] {
1907 let res = client.get(path).send().await;
1908 assert_eq!(
1909 res.status(),
1910 StatusCode::OK,
1911 "internal/full server should serve {path}"
1912 );
1913 }
1914 }
1915
1916 #[tokio::test]
1917 pub async fn test_build_servers_separates_api_and_internal_routes() {
1918 let (internal, api) = make_split_builder().build_servers();
1921 let api = api.expect("API server is explicitly enabled in make_split_builder");
1922 assert_eq!(internal.name(), HTTP_SERVER);
1923 assert_eq!(api.name(), HTTP_API_SERVER);
1924
1925 let internal_app = internal.build(internal.make_app()).unwrap();
1926 let api_app = api.build(api.make_app()).unwrap();
1927
1928 let internal_client = TestClient::new(internal_app).await;
1929 let api_client = TestClient::new(api_app).await;
1930
1931 assert_eq!(
1933 internal_client.get("/v1/health").send().await.status(),
1934 StatusCode::OK
1935 );
1936 assert_eq!(
1937 api_client.get("/v1/health").send().await.status(),
1938 StatusCode::OK
1939 );
1940
1941 for path in ["/health", "/status", "/metrics", "/config"] {
1943 assert_eq!(
1944 internal_client.get(path).send().await.status(),
1945 StatusCode::OK,
1946 "internal/full server should serve {path}"
1947 );
1948 assert_eq!(
1950 api_client.get(path).send().await.status(),
1951 StatusCode::NOT_FOUND,
1952 "API server should NOT serve {path}"
1953 );
1954 }
1955 }
1956
1957 #[test]
1958 fn test_build_servers_api_inherits_http_options() {
1959 let http_opts = HttpOptions {
1961 timeout: Duration::from_secs(42),
1962 body_limit: ReadableSize::mb(128),
1963 cors_allowed_origins: vec!["https://example.com".to_string()],
1964 enable_api_server: true,
1965 ..HttpOptions::default()
1966 };
1967 let (internal, api) = HttpServerBuilder::new(http_opts.clone()).build_servers();
1968 let api = api.expect("API server is explicitly enabled");
1969
1970 assert_eq!(internal.options.addr, http_opts.addr);
1972 assert_eq!(api.options.addr, "127.0.0.1:4006");
1973 assert_eq!(api.options.timeout, http_opts.timeout);
1975 assert_eq!(api.options.body_limit, http_opts.body_limit);
1976 assert_eq!(
1977 api.options.cors_allowed_origins,
1978 http_opts.cors_allowed_origins
1979 );
1980 }
1981
1982 #[test]
1983 fn test_is_api_listener_path() {
1984 assert!(is_api_listener_path("/v1"));
1986 assert!(is_api_listener_path("/v1/"));
1987 assert!(is_api_listener_path("/v1/sql"));
1988 assert!(!is_api_listener_path("/v10/sql"));
1989 assert!(!is_api_listener_path("/v1-internal"));
1990 assert!(is_api_listener_path("/dashboard"));
1992 assert!(is_api_listener_path("/dashboard/app.js"));
1993 assert!(!is_api_listener_path("/dashboard-admin"));
1994 assert!(!is_api_listener_path("/metrics"));
1996 assert!(!is_api_listener_path("/status/plugin"));
1997 assert!(!is_api_listener_path("/health"));
1998 }
1999
2000 #[tokio::test]
2001 async fn packed_capability_requires_auth_on_full_and_api_listeners() {
2002 let (tx, _rx) = mpsc::channel(1);
2003 let provider =
2004 auth::static_user_provider_from_option("static_user_provider:cmd:user=password")
2005 .unwrap();
2006 let (full, api) = HttpServerBuilder::new(HttpOptions {
2007 enable_api_server: true,
2008 ..Default::default()
2009 })
2010 .with_sql_handler(Arc::new(DummyInstance { _tx: tx }))
2011 .with_user_provider(Arc::new(provider))
2012 .build_servers();
2013 for server in [full, api.unwrap()] {
2014 let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
2015 assert_eq!(
2016 client.get("/v1/capabilities").send().await.status(),
2017 StatusCode::UNAUTHORIZED
2018 );
2019 assert_eq!(
2020 client
2021 .post("/v1/ddl/logical-tables")
2022 .form(&[("sql", "SELECT 1")])
2023 .send()
2024 .await
2025 .status(),
2026 StatusCode::UNAUTHORIZED
2027 );
2028 let response = client
2029 .get("/v1/capabilities")
2030 .header("Authorization", "Basic dXNlcjpwYXNzd29yZA==")
2031 .send()
2032 .await;
2033 assert_eq!(response.status(), StatusCode::OK);
2034 assert_eq!(
2035 response.json::<serde_json::Value>().await,
2036 serde_json::json!({"metric_packed_import": 1, "metric_packed_export": 1})
2037 );
2038 }
2039 }
2040
2041 #[tokio::test]
2042 async fn logical_ddl_form_limits_and_unsupported_handler() {
2043 use axum::body::{Body, to_bytes};
2044 use tower::ServiceExt;
2045
2046 async fn post_form(app: Router, body: String) -> (StatusCode, serde_json::Value) {
2047 let response = app
2048 .oneshot(
2049 Request::builder()
2050 .method(Method::POST)
2051 .uri("/v1/ddl/logical-tables")
2052 .header("Content-Type", "application/x-www-form-urlencoded")
2053 .body(Body::from(body))
2054 .unwrap(),
2055 )
2056 .await
2057 .unwrap();
2058 let status = response.status();
2059 let body = to_bytes(response.into_body(), 4096).await.unwrap();
2060 (status, serde_json::from_slice(&body).unwrap())
2061 }
2062
2063 for limit in [0, 64] {
2064 let (tx, _rx) = mpsc::channel(1);
2065 let server = HttpServerBuilder::new(HttpOptions {
2066 body_limit: ReadableSize(limit),
2067 ..Default::default()
2068 })
2069 .with_sql_handler(Arc::new(DummyInstance { _tx: tx }))
2070 .build();
2071 let app = server.build(server.make_app()).unwrap();
2072 let client = TestClient::new(app.clone()).await;
2073 assert_eq!(
2074 client.get("/v1/ddl/logical-tables").send().await.status(),
2075 StatusCode::METHOD_NOT_ALLOWED
2076 );
2077 let small = client
2078 .post("/v1/ddl/logical-tables")
2079 .form(&[("sql", "CREATE TABLE t")])
2080 .send()
2081 .await;
2082 assert_eq!(
2083 small.json::<serde_json::Value>().await["code"],
2084 common_error::status_code::StatusCode::Unsupported as u32
2085 );
2086 let (_, body) = post_form(
2087 app.clone(),
2088 if limit == 0 {
2089 format!("sql={}", "%20".repeat(1024 * 1024))
2090 } else {
2091 "sql=".to_string() + &" ".repeat(65)
2092 },
2093 )
2094 .await;
2095 let code = body["code"].clone();
2096 assert_eq!(
2097 code,
2098 if limit == 0 {
2099 common_error::status_code::StatusCode::Unsupported as u32
2100 } else {
2101 common_error::status_code::StatusCode::InvalidArguments as u32
2102 }
2103 );
2104 for body in [
2105 format!("sql={}", "x".repeat(1024 * 1024 + 1)),
2106 format!("sql={}", "%C3%A9".repeat(512 * 1024 + 1)),
2107 format!("sql=CREATE+TABLE+t&padding={}", "x".repeat(4 * 1024 * 1024)),
2108 ] {
2109 let (status, body) = post_form(app.clone(), body).await;
2110 assert_eq!(status, StatusCode::BAD_REQUEST);
2111 assert_eq!(
2112 body["code"],
2113 common_error::status_code::StatusCode::InvalidArguments as u32
2114 );
2115 }
2116 }
2117 }
2118
2119 #[tokio::test]
2120 pub async fn test_extra_router_respects_api_surface() {
2121 let (tx, _rx) = mpsc::channel(100);
2127 let instance = Arc::new(DummyInstance { _tx: tx });
2128 let options = HttpOptions {
2129 enable_api_server: true,
2130 ..HttpOptions::default()
2131 };
2132 let builder = HttpServerBuilder::new(options)
2133 .with_sql_handler(instance)
2134 .with_extra_router(
2135 Router::new()
2136 .route("/status/plugin", routing::get(|| async { "ok" }))
2137 .route("/v1/plugin", routing::get(|| async { "ok" })),
2138 );
2139 let (full, api) = builder.build_servers();
2140 let api = api.expect("API server is explicitly enabled");
2141
2142 let full_client = TestClient::new(full.build(full.make_app()).unwrap()).await;
2143 let api_client = TestClient::new(api.build(api.make_app()).unwrap()).await;
2144
2145 assert_eq!(
2147 full_client.get("/status/plugin").send().await.status(),
2148 StatusCode::OK
2149 );
2150 assert_eq!(
2152 api_client.get("/status/plugin").send().await.status(),
2153 StatusCode::NOT_FOUND
2154 );
2155
2156 assert_eq!(
2158 full_client.get("/v1/plugin").send().await.status(),
2159 StatusCode::OK
2160 );
2161 assert_eq!(
2162 api_client.get("/v1/plugin").send().await.status(),
2163 StatusCode::OK
2164 );
2165
2166 assert_eq!(
2168 api_client.get("/v10/plugin").send().await.status(),
2169 StatusCode::NOT_FOUND
2170 );
2171 }
2172
2173 #[test]
2174 fn test_build_servers_api_disabled_by_default() {
2175 let (_internal, api) = HttpServerBuilder::new(HttpOptions::default()).build_servers();
2178 assert!(api.is_none());
2179 }
2180
2181 #[tokio::test]
2182 pub async fn test_cors() {
2183 let (tx, _rx) = mpsc::channel(100);
2185 let app = make_test_app(tx);
2186 let client = TestClient::new(app).await;
2187
2188 let res = client.get("/health").send().await;
2189
2190 assert_eq!(res.status(), StatusCode::OK);
2191 assert_eq!(
2192 res.headers()
2193 .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
2194 .expect("expect cors header origin"),
2195 "*"
2196 );
2197
2198 let res = client.get("/v1/health").send().await;
2199
2200 assert_eq!(res.status(), StatusCode::OK);
2201 assert_eq!(
2202 res.headers()
2203 .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
2204 .expect("expect cors header origin"),
2205 "*"
2206 );
2207
2208 let res = client
2209 .options("/health")
2210 .header("Access-Control-Request-Headers", "x-greptime-auth")
2211 .header("Access-Control-Request-Method", "DELETE")
2212 .header("Origin", "https://example.com")
2213 .send()
2214 .await;
2215 assert_eq!(res.status(), StatusCode::OK);
2216 assert_eq!(
2217 res.headers()
2218 .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
2219 .expect("expect cors header origin"),
2220 "*"
2221 );
2222 assert_eq!(
2223 res.headers()
2224 .get(http::header::ACCESS_CONTROL_ALLOW_HEADERS)
2225 .expect("expect cors header headers"),
2226 "*"
2227 );
2228 assert_eq!(
2229 res.headers()
2230 .get(http::header::ACCESS_CONTROL_ALLOW_METHODS)
2231 .expect("expect cors header methods"),
2232 "GET,POST,PUT,DELETE,HEAD"
2233 );
2234 }
2235
2236 #[tokio::test]
2237 pub async fn test_cors_custom_origins() {
2238 let (tx, _rx) = mpsc::channel(100);
2240 let origin = "https://example.com";
2241
2242 let options = HttpOptions {
2243 cors_allowed_origins: vec![origin.to_string()],
2244 ..Default::default()
2245 };
2246
2247 let app = make_test_app_custom(tx, options);
2248 let client = TestClient::new(app).await;
2249
2250 let res = client.get("/health").header("Origin", origin).send().await;
2251
2252 assert_eq!(res.status(), StatusCode::OK);
2253 assert_eq!(
2254 res.headers()
2255 .get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
2256 .expect("expect cors header origin"),
2257 origin
2258 );
2259
2260 let res = client
2261 .get("/health")
2262 .header("Origin", "https://notallowed.com")
2263 .send()
2264 .await;
2265
2266 assert_eq!(res.status(), StatusCode::OK);
2267 assert!(
2268 !res.headers()
2269 .contains_key(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
2270 );
2271 }
2272
2273 #[tokio::test]
2274 pub async fn test_cors_disabled() {
2275 let (tx, _rx) = mpsc::channel(100);
2277
2278 let options = HttpOptions {
2279 enable_cors: false,
2280 ..Default::default()
2281 };
2282
2283 let app = make_test_app_custom(tx, options);
2284 let client = TestClient::new(app).await;
2285
2286 let res = client.get("/health").send().await;
2287
2288 assert_eq!(res.status(), StatusCode::OK);
2289 assert!(
2290 !res.headers()
2291 .contains_key(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
2292 );
2293 }
2294
2295 #[test]
2296 fn test_http_options_default() {
2297 let default = HttpOptions::default();
2298 assert_eq!("127.0.0.1:4000".to_string(), default.addr);
2299 assert_eq!(Duration::from_secs(0), default.timeout)
2300 }
2301
2302 #[tokio::test]
2303 async fn test_http_options_legacy_analyze_stream_config_is_ignored() {
2304 let options: HttpOptions = serde_json::from_value(serde_json::json!({
2305 "addr": "127.0.0.1:4000",
2306 "timeout": "0s",
2307 "body_limit": "64MiB",
2308 "cors_allowed_origins": [],
2309 "enable_cors": true,
2310 "experimental_enable_explain_analyze_stream": false,
2311 "enable_api_server": false,
2312 "api_server_addr": "127.0.0.1:4006"
2313 }))
2314 .unwrap();
2315 let serialized = serde_json::to_string(&options).unwrap();
2316 assert!(!serialized.contains("experimental_enable_explain_analyze_stream"));
2317
2318 let (tx, _rx) = mpsc::channel(100);
2319 let app = make_test_app_custom(tx, options);
2320 let client = TestClient::new(app).await;
2321 let response = client
2322 .post("/v1/sql/analyze/stream")
2323 .form(&handler::SqlQuery {
2324 sql: Some("EXPLAIN ANALYZE VERBOSE SELECT 1".to_string()),
2325 ..Default::default()
2326 })
2327 .send()
2328 .await;
2329 assert_ne!(response.status(), StatusCode::NOT_FOUND);
2330 }
2331
2332 #[tokio::test]
2333 async fn test_http_server_request_timeout() {
2334 common_telemetry::init_default_ut_logging();
2335
2336 let (tx, _rx) = mpsc::channel(100);
2337 let options = HttpOptions {
2338 timeout: Duration::from_millis(10),
2339 ..Default::default()
2340 };
2341 let app = make_test_app_custom(tx, options);
2342 let client = TestClient::new(app).await;
2343 let res = client.get("/test/timeout").send().await;
2344 assert_eq!(res.status(), StatusCode::REQUEST_TIMEOUT);
2345
2346 let res = client.post("/v1/prometheus/write").send().await;
2347 assert_eq!(res.status(), StatusCode::GATEWAY_TIMEOUT);
2348
2349 let now = Instant::now();
2350 let res = client
2351 .get("/test/timeout")
2352 .header(GREPTIME_DB_HEADER_TIMEOUT, "20ms")
2353 .send()
2354 .await;
2355 assert_eq!(res.status(), StatusCode::REQUEST_TIMEOUT);
2356 let elapsed = now.elapsed();
2357 assert!(elapsed > Duration::from_millis(15));
2358
2359 tokio::time::timeout(
2360 Duration::from_millis(15),
2361 client
2362 .get("/test/timeout")
2363 .header(GREPTIME_DB_HEADER_TIMEOUT, "0s")
2364 .send(),
2365 )
2366 .await
2367 .unwrap_err();
2368
2369 tokio::time::timeout(
2370 Duration::from_millis(15),
2371 client
2372 .get("/test/timeout")
2373 .header(
2374 GREPTIME_DB_HEADER_TIMEOUT,
2375 humantime::format_duration(Duration::default()).to_string(),
2376 )
2377 .send(),
2378 )
2379 .await
2380 .unwrap_err();
2381 }
2382
2383 #[tokio::test]
2384 async fn test_schema_for_empty_response() {
2385 let column_schemas = vec![
2386 ColumnSchema::new("numbers", ConcreteDataType::uint32_datatype(), false),
2387 ColumnSchema::new("strings", ConcreteDataType::string_datatype(), true),
2388 ];
2389 let schema = Arc::new(Schema::new(column_schemas));
2390
2391 let recordbatches = RecordBatches::try_new(schema.clone(), vec![]).unwrap();
2392 let outputs = vec![Ok(Output::new_with_record_batches(recordbatches))];
2393
2394 let json_resp = GreptimedbV1Response::from_output(outputs).await;
2395 if let HttpResponse::GreptimedbV1(json_resp) = json_resp {
2396 let json_output = &json_resp.output[0];
2397 if let GreptimeQueryOutput::Records(r) = json_output {
2398 assert_eq!(r.num_rows(), 0);
2399 assert_eq!(r.num_cols(), 2);
2400 assert_eq!(r.schema.column_schemas[0].name, "numbers");
2401 assert_eq!(r.schema.column_schemas[0].data_type, "UInt32");
2402 } else {
2403 panic!("invalid output type");
2404 }
2405 } else {
2406 panic!("invalid format")
2407 }
2408 }
2409
2410 #[tokio::test]
2411 async fn test_recordbatches_conversion() {
2412 let column_schemas = vec![
2413 ColumnSchema::new("numbers", ConcreteDataType::uint32_datatype(), false),
2414 ColumnSchema::new("strings", ConcreteDataType::string_datatype(), true),
2415 ];
2416 let schema = Arc::new(Schema::new(column_schemas));
2417 let columns: Vec<VectorRef> = vec![
2418 Arc::new(UInt32Vector::from_slice(vec![1, 2, 3, 4])),
2419 Arc::new(StringVector::from(vec![
2420 None,
2421 Some("hello"),
2422 Some("greptime"),
2423 None,
2424 ])),
2425 ];
2426 let recordbatch = RecordBatch::new(schema.clone(), columns).unwrap();
2427
2428 for format in [
2429 ResponseFormat::GreptimedbV1,
2430 ResponseFormat::InfluxdbV1,
2431 ResponseFormat::Csv(true, true),
2432 ResponseFormat::Table,
2433 ResponseFormat::Arrow,
2434 ResponseFormat::Json,
2435 ResponseFormat::Null,
2436 ] {
2437 let recordbatches =
2438 RecordBatches::try_new(schema.clone(), vec![recordbatch.clone()]).unwrap();
2439 let outputs = vec![Ok(Output::new_with_record_batches(recordbatches))];
2440 let json_resp = match format {
2441 ResponseFormat::Arrow => ArrowResponse::from_output(outputs, None).await,
2442 ResponseFormat::Csv(with_names, with_types) => {
2443 CsvResponse::from_output(outputs, with_names, with_types).await
2444 }
2445 ResponseFormat::Table => TableResponse::from_output(outputs).await,
2446 ResponseFormat::GreptimedbV1 => GreptimedbV1Response::from_output(outputs).await,
2447 ResponseFormat::InfluxdbV1 => InfluxdbV1Response::from_output(outputs, None).await,
2448 ResponseFormat::Json => JsonResponse::from_output(outputs).await,
2449 ResponseFormat::Null => NullResponse::from_output(outputs).await,
2450 };
2451
2452 match json_resp {
2453 HttpResponse::GreptimedbV1(resp) => {
2454 let json_output = &resp.output[0];
2455 if let GreptimeQueryOutput::Records(r) = json_output {
2456 assert_eq!(r.num_rows(), 4);
2457 assert_eq!(r.num_cols(), 2);
2458 assert_eq!(r.schema.column_schemas[0].name, "numbers");
2459 assert_eq!(r.schema.column_schemas[0].data_type, "UInt32");
2460 assert_eq!(r.rows[0][0], serde_json::Value::from(1));
2461 assert_eq!(r.rows[0][1], serde_json::Value::Null);
2462 } else {
2463 panic!("invalid output type");
2464 }
2465 }
2466 HttpResponse::InfluxdbV1(resp) => {
2467 let json_output = &resp.results()[0];
2468 assert_eq!(json_output.num_rows(), 4);
2469 assert_eq!(json_output.num_cols(), 2);
2470 assert_eq!(json_output.series[0].columns.clone()[0], "numbers");
2471 assert_eq!(
2472 json_output.series[0].values[0][0],
2473 serde_json::Value::from(1)
2474 );
2475 assert_eq!(json_output.series[0].values[0][1], serde_json::Value::Null);
2476 }
2477 HttpResponse::Csv(resp) => {
2478 let output = &resp.output()[0];
2479 if let GreptimeQueryOutput::Records(r) = output {
2480 assert_eq!(r.num_rows(), 4);
2481 assert_eq!(r.num_cols(), 2);
2482 assert_eq!(r.schema.column_schemas[0].name, "numbers");
2483 assert_eq!(r.schema.column_schemas[0].data_type, "UInt32");
2484 assert_eq!(r.rows[0][0], serde_json::Value::from(1));
2485 assert_eq!(r.rows[0][1], serde_json::Value::Null);
2486 } else {
2487 panic!("invalid output type");
2488 }
2489 }
2490
2491 HttpResponse::Table(resp) => {
2492 let output = &resp.output()[0];
2493 if let GreptimeQueryOutput::Records(r) = output {
2494 assert_eq!(r.num_rows(), 4);
2495 assert_eq!(r.num_cols(), 2);
2496 assert_eq!(r.schema.column_schemas[0].name, "numbers");
2497 assert_eq!(r.schema.column_schemas[0].data_type, "UInt32");
2498 assert_eq!(r.rows[0][0], serde_json::Value::from(1));
2499 assert_eq!(r.rows[0][1], serde_json::Value::Null);
2500 } else {
2501 panic!("invalid output type");
2502 }
2503 }
2504
2505 HttpResponse::Arrow(resp) => {
2506 let output = resp.data;
2507 let mut reader = StreamReader::try_new(Cursor::new(output), None)
2508 .expect("Arrow reader error");
2509 let schema = reader.schema();
2510 assert_eq!(schema.fields[0].name(), "numbers");
2511 assert_eq!(schema.fields[0].data_type(), &DataType::UInt32);
2512 assert_eq!(schema.fields[1].name(), "strings");
2513 assert_eq!(schema.fields[1].data_type(), &DataType::Utf8);
2514
2515 let rb = reader.next().unwrap().expect("read record batch failed");
2516 assert_eq!(rb.num_columns(), 2);
2517 assert_eq!(rb.num_rows(), 4);
2518 }
2519
2520 HttpResponse::Json(resp) => {
2521 let output = &resp.output()[0];
2522 if let GreptimeQueryOutput::Records(r) = output {
2523 assert_eq!(r.num_rows(), 4);
2524 assert_eq!(r.num_cols(), 2);
2525 assert_eq!(r.schema.column_schemas[0].name, "numbers");
2526 assert_eq!(r.schema.column_schemas[0].data_type, "UInt32");
2527 assert_eq!(r.rows[0][0], serde_json::Value::from(1));
2528 assert_eq!(r.rows[0][1], serde_json::Value::Null);
2529 } else {
2530 panic!("invalid output type");
2531 }
2532 }
2533
2534 HttpResponse::Null(resp) => {
2535 assert_eq!(resp.rows(), 4);
2536 }
2537
2538 HttpResponse::Error(err) => unreachable!("{err:?}"),
2539 }
2540 }
2541 }
2542
2543 #[test]
2544 fn test_response_format_misc() {
2545 assert_eq!(ResponseFormat::default(), ResponseFormat::GreptimedbV1);
2546 assert_eq!(ResponseFormat::parse("arrow"), Some(ResponseFormat::Arrow));
2547 assert_eq!(
2548 ResponseFormat::parse("csv"),
2549 Some(ResponseFormat::Csv(false, false))
2550 );
2551 assert_eq!(
2552 ResponseFormat::parse("csvwithnames"),
2553 Some(ResponseFormat::Csv(true, false))
2554 );
2555 assert_eq!(
2556 ResponseFormat::parse("csvwithnamesandtypes"),
2557 Some(ResponseFormat::Csv(true, true))
2558 );
2559 assert_eq!(ResponseFormat::parse("table"), Some(ResponseFormat::Table));
2560 assert_eq!(
2561 ResponseFormat::parse("greptimedb_v1"),
2562 Some(ResponseFormat::GreptimedbV1)
2563 );
2564 assert_eq!(
2565 ResponseFormat::parse("influxdb_v1"),
2566 Some(ResponseFormat::InfluxdbV1)
2567 );
2568 assert_eq!(ResponseFormat::parse("json"), Some(ResponseFormat::Json));
2569 assert_eq!(ResponseFormat::parse("null"), Some(ResponseFormat::Null));
2570
2571 assert_eq!(ResponseFormat::parse("invalid"), None);
2573 assert_eq!(ResponseFormat::parse(""), None);
2574 assert_eq!(ResponseFormat::parse("CSV"), None); assert_eq!(ResponseFormat::Arrow.as_str(), "arrow");
2578 assert_eq!(ResponseFormat::Csv(false, false).as_str(), "csv");
2579 assert_eq!(ResponseFormat::Csv(true, true).as_str(), "csv");
2580 assert_eq!(ResponseFormat::Table.as_str(), "table");
2581 assert_eq!(ResponseFormat::GreptimedbV1.as_str(), "greptimedb_v1");
2582 assert_eq!(ResponseFormat::InfluxdbV1.as_str(), "influxdb_v1");
2583 assert_eq!(ResponseFormat::Json.as_str(), "json");
2584 assert_eq!(ResponseFormat::Null.as_str(), "null");
2585 assert_eq!(ResponseFormat::default().as_str(), "greptimedb_v1");
2586 }
2587
2588 #[test]
2589 fn test_decode_label_name_strict() {
2590 let strict = PromValidationMode::Strict;
2591
2592 assert!(strict.decode_label_name(b"__name__").is_ok());
2594 assert!(strict.decode_label_name(b"job").is_ok());
2595 assert!(strict.decode_label_name(b"instance").is_ok());
2596 assert!(strict.decode_label_name(b"_private").is_ok());
2597 assert!(strict.decode_label_name(b"label_with_underscores").is_ok());
2598 assert!(strict.decode_label_name(b"abc123").is_ok());
2599 assert!(strict.decode_label_name(b"A").is_ok());
2600 assert!(strict.decode_label_name(b"_").is_ok());
2601
2602 assert!(strict.decode_label_name(b"0abc").is_err());
2604 assert!(strict.decode_label_name(b"123").is_err());
2605
2606 assert!(strict.decode_label_name(b"label-name").is_err());
2608 assert!(strict.decode_label_name(b"label.name").is_err());
2609 assert!(strict.decode_label_name(b"label name").is_err());
2610 assert!(strict.decode_label_name(b"label/name").is_err());
2611
2612 assert!(strict.decode_label_name(b"").is_err());
2614
2615 assert!(strict.decode_label_name("ラベル".as_bytes()).is_err());
2617
2618 assert!(strict.decode_label_name(&[0xff, 0xfe]).is_err());
2620 }
2621
2622 #[test]
2623 fn test_decode_label_name_lossy() {
2624 let lossy = PromValidationMode::Lossy;
2625
2626 assert!(lossy.decode_label_name(b"__name__").is_ok());
2628 assert!(lossy.decode_label_name(b"label-name").is_err());
2629 assert!(lossy.decode_label_name(b"0abc").is_err());
2630
2631 assert!(lossy.decode_label_name(&[0xff, 0xfe]).is_err());
2633 }
2634
2635 #[test]
2636 fn test_decode_label_name_unchecked() {
2637 let unchecked = PromValidationMode::Unchecked;
2638
2639 assert!(unchecked.decode_label_name(b"__name__").is_ok());
2641 assert!(unchecked.decode_label_name(b"label-name").is_err());
2642 assert!(unchecked.decode_label_name(b"0abc").is_err());
2643 }
2644
2645 #[test]
2646 fn test_is_valid_prom_label_name_bytes() {
2647 assert!(validate_label_name(b"__name__"));
2648 assert!(validate_label_name(b"job"));
2649 assert!(validate_label_name(b"_"));
2650 assert!(validate_label_name(b"A"));
2651 assert!(validate_label_name(b"abc123"));
2652 assert!(validate_label_name(b"_leading_underscore"));
2653
2654 assert!(!validate_label_name(b""));
2655 assert!(!validate_label_name(b"0starts_with_digit"));
2656 assert!(!validate_label_name(b"has-dash"));
2657 assert!(!validate_label_name(b"has.dot"));
2658 assert!(!validate_label_name(b"has space"));
2659 assert!(!validate_label_name(&[0xff, 0xfe]));
2660 }
2661}
2662
2663#[cfg(test)]
2664mod batching_tests {
2665 use std::sync::{Arc, Mutex};
2666
2667 use axum::http::StatusCode;
2668 use common_query::Output;
2669 use session::context::QueryContextRef;
2670
2671 use crate::batcher::BatchingProtocol;
2672 use crate::error::Result as ServerResult;
2673 use crate::http::test_helpers::TestClient;
2674 use crate::http::{HttpOptions, HttpServerBuilder};
2675 use crate::influxdb::InfluxdbRequest;
2676 use crate::opentsdb::codec::DataPoint;
2677 use crate::query_handler::{InfluxdbLineProtocolHandler, OpentsdbProtocolHandler};
2678
2679 #[tokio::test]
2680 async fn test_batching_selectors_are_independent() {
2681 use axum::routing::post;
2682 use axum::{Extension, Json, Router, middleware};
2683 use session::context::{QueryContext, QueryContextBuilder};
2684
2685 use crate::http::{LogicalBatchingProtocols, set_http_write_batching};
2686 for table in [false, true] {
2687 for logical in [false, true] {
2688 let app = Router::new()
2689 .route(
2690 "/",
2691 post(|Extension(ctx): Extension<QueryContext>| async move {
2692 Json([ctx.batching_enabled(), ctx.logical_batching_enabled()])
2693 }),
2694 )
2695 .route_layer(middleware::from_fn_with_state(
2696 BatchingProtocol::Otlp,
2697 set_http_write_batching,
2698 ))
2699 .layer(Extension(QueryContextBuilder::default().build()))
2700 .layer(Extension(Arc::new(if table {
2701 vec![BatchingProtocol::Otlp]
2702 } else {
2703 vec![]
2704 })))
2705 .layer(Extension(LogicalBatchingProtocols(if logical {
2706 vec![BatchingProtocol::Otlp]
2707 } else {
2708 vec![]
2709 })));
2710 let client = TestClient::new(app).await;
2711 let actual: [bool; 2] = client.post("/").send().await.json().await;
2712 assert_eq!(actual, [table, logical]);
2713 }
2714 }
2715 }
2716
2717 #[derive(Default)]
2718 struct RecordingWriteHandler {
2719 selections: Mutex<Vec<bool>>,
2720 }
2721
2722 #[async_trait::async_trait]
2723 impl OpentsdbProtocolHandler for RecordingWriteHandler {
2724 async fn preflight(&self, _: &[DataPoint], _: QueryContextRef) -> ServerResult<()> {
2725 Ok(())
2726 }
2727
2728 async fn exec(&self, points: Vec<DataPoint>, ctx: QueryContextRef) -> ServerResult<usize> {
2729 self.selections.lock().unwrap().push(ctx.batching_enabled());
2730 Ok(points.len())
2731 }
2732 }
2733
2734 #[async_trait::async_trait]
2735 impl InfluxdbLineProtocolHandler for RecordingWriteHandler {
2736 async fn exec(&self, _: InfluxdbRequest, ctx: QueryContextRef) -> ServerResult<Output> {
2737 self.selections.lock().unwrap().push(ctx.batching_enabled());
2738 Ok(Output::new_with_affected_rows(1))
2739 }
2740 }
2741
2742 #[tokio::test]
2743 async fn test_real_influx_and_opentsdb_routes_keep_selection() {
2744 for protocols in [
2745 vec![],
2746 vec![BatchingProtocol::Influxdb],
2747 vec![BatchingProtocol::Opentsdb],
2748 vec![BatchingProtocol::Influxdb, BatchingProtocol::Opentsdb],
2749 ] {
2750 let handler = Arc::new(RecordingWriteHandler::default());
2751 let influx_enabled = protocols.contains(&BatchingProtocol::Influxdb);
2752 let opentsdb_enabled = protocols.contains(&BatchingProtocol::Opentsdb);
2753 let server = HttpServerBuilder::new(HttpOptions::default())
2754 .with_batching_protocols(protocols)
2755 .with_influxdb_handler(handler.clone())
2756 .with_opentsdb_handler(handler.clone())
2757 .build();
2758 let client = TestClient::new(server.build(server.make_app()).unwrap()).await;
2759 for path in [
2760 "/v1/influxdb/write",
2761 "/v1/influxdb/api/v2/write?bucket=public",
2762 ] {
2763 assert_eq!(
2764 client
2765 .post(path)
2766 .body("cpu value=1 42")
2767 .send()
2768 .await
2769 .status(),
2770 StatusCode::NO_CONTENT
2771 );
2772 }
2773 let point =
2774 serde_json::json!({"metric":"cpu", "timestamp":42, "value":1, "tags":{"host":"a"}});
2775 for query in ["", "?summary", "?details"] {
2776 let response = client
2777 .post(&format!("/v1/opentsdb/api/put{query}"))
2778 .json(&point)
2779 .send()
2780 .await;
2781 assert_eq!(
2782 response.status(),
2783 if query.is_empty() {
2784 StatusCode::NO_CONTENT
2785 } else {
2786 StatusCode::OK
2787 }
2788 );
2789 }
2790 assert_eq!(
2791 *handler.selections.lock().unwrap(),
2792 vec![
2793 influx_enabled,
2794 influx_enabled,
2795 opentsdb_enabled,
2796 opentsdb_enabled,
2797 opentsdb_enabled
2798 ]
2799 );
2800 }
2801 }
2802}