servers/grpc/
builder.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::convert::Infallible;
16
17use api::v1::frontend::frontend_server::FrontendServer;
18use api::v1::greptime_database_server::GreptimeDatabaseServer;
19use api::v1::prometheus_gateway_server::PrometheusGatewayServer;
20use api::v1::region::region_server::RegionServer;
21use arrow_flight::flight_service_server::FlightServiceServer;
22use auth::UserProviderRef;
23use axum::extract::Request;
24use axum::response::IntoResponse;
25use axum::routing::Route;
26use common_grpc::error::{InvalidConfigFilePathSnafu, Result};
27use common_runtime::Runtime;
28use common_telemetry::warn;
29use otel_arrow_rust::proto::opentelemetry::arrow::v1::arrow_metrics_service_server::ArrowMetricsServiceServer;
30use snafu::ResultExt;
31use tokio::sync::Mutex;
32use tonic::codec::CompressionEncoding;
33use tonic::codegen::Service;
34use tonic::service::RoutesBuilder;
35use tonic::service::interceptor::InterceptedService;
36use tonic::transport::{Identity, ServerTlsConfig};
37use tower::{Layer, ServiceBuilder};
38
39use crate::grpc::database::DatabaseService;
40use crate::grpc::flight::{FlightCraftRef, FlightCraftWrapper};
41use crate::grpc::frontend_grpc_handler::FrontendGrpcHandler;
42use crate::grpc::greptime_handler::GreptimeRequestHandler;
43use crate::grpc::memory_limit::{MemoryLimiterExtensionLayer, MemoryLimiterExtensionService};
44use crate::grpc::prom_query_gateway::PrometheusGatewayService;
45use crate::grpc::region_server::{RegionServerHandlerRef, RegionServerRequestHandler};
46use crate::grpc::{GrpcServer, GrpcServerConfig};
47use crate::otel_arrow::{HeaderInterceptor, OtelArrowServiceHandler};
48use crate::prometheus_handler::PrometheusHandlerRef;
49use crate::query_handler::OpenTelemetryProtocolHandlerRef;
50use crate::request_memory_limiter::ServerMemoryLimiter;
51use crate::tls::TlsOption;
52
53/// Add a gRPC service (`service`) to a `builder`([RoutesBuilder]).
54/// This macro will automatically add some gRPC properties to the service.
55#[macro_export]
56macro_rules! add_service {
57    ($builder: ident, $service: expr) => {
58        let max_recv_message_size = $builder.config().max_recv_message_size;
59        let max_send_message_size = $builder.config().max_send_message_size;
60
61        use tonic::codec::CompressionEncoding;
62        let service_builder = $service
63            .max_decoding_message_size(max_recv_message_size)
64            .max_encoding_message_size(max_send_message_size)
65            .accept_compressed(CompressionEncoding::Gzip)
66            .accept_compressed(CompressionEncoding::Zstd)
67            .send_compressed(CompressionEncoding::Gzip)
68            .send_compressed(CompressionEncoding::Zstd);
69
70        // Apply memory limiter layer
71        use $crate::grpc::memory_limit::MemoryLimiterExtensionLayer;
72        let service_with_limiter = $crate::tower::ServiceBuilder::new()
73            .layer(MemoryLimiterExtensionLayer::new(
74                $builder.memory_limiter().clone(),
75                max_recv_message_size,
76            ))
77            .service(service_builder);
78
79        $builder
80            .routes_builder_mut()
81            .add_service(service_with_limiter);
82    };
83}
84
85pub struct GrpcServerBuilder {
86    name: Option<String>,
87    config: GrpcServerConfig,
88    runtime: Runtime,
89    routes_builder: RoutesBuilder,
90    tls_config: Option<ServerTlsConfig>,
91    otel_arrow_service: Option<
92        MemoryLimiterExtensionService<
93            InterceptedService<
94                ArrowMetricsServiceServer<OtelArrowServiceHandler<OpenTelemetryProtocolHandlerRef>>,
95                HeaderInterceptor,
96            >,
97        >,
98    >,
99    memory_limiter: ServerMemoryLimiter,
100}
101
102impl GrpcServerBuilder {
103    pub fn new(config: GrpcServerConfig, runtime: Runtime) -> Self {
104        // Create a default unlimited limiter (can be overridden with with_memory_limiter)
105        let memory_limiter = ServerMemoryLimiter::default();
106
107        Self {
108            name: None,
109            config,
110            runtime,
111            routes_builder: RoutesBuilder::default(),
112            tls_config: None,
113            otel_arrow_service: None,
114            memory_limiter,
115        }
116    }
117
118    /// Set a global memory limiter for all server protocols.
119    pub fn with_memory_limiter(mut self, limiter: ServerMemoryLimiter) -> Self {
120        self.memory_limiter = limiter;
121        self
122    }
123
124    pub fn config(&self) -> &GrpcServerConfig {
125        &self.config
126    }
127
128    pub fn runtime(&self) -> &Runtime {
129        &self.runtime
130    }
131
132    pub fn memory_limiter(&self) -> &ServerMemoryLimiter {
133        &self.memory_limiter
134    }
135
136    pub fn name(self, name: Option<String>) -> Self {
137        Self { name, ..self }
138    }
139
140    /// Add handler for [DatabaseService] service.
141    pub fn database_handler(mut self, database_handler: GreptimeRequestHandler) -> Self {
142        add_service!(
143            self,
144            GreptimeDatabaseServer::new(DatabaseService::new(database_handler))
145        );
146        self
147    }
148
149    /// Add handler for Prometheus-compatible PromQL queries ([PrometheusGateway]).
150    pub fn prometheus_handler(
151        mut self,
152        prometheus_handler: PrometheusHandlerRef,
153        user_provider: Option<UserProviderRef>,
154    ) -> Self {
155        add_service!(
156            self,
157            PrometheusGatewayServer::new(PrometheusGatewayService::new(
158                prometheus_handler,
159                user_provider,
160            ))
161        );
162        self
163    }
164
165    /// Add handler for [FlightService](arrow_flight::flight_service_server::FlightService).
166    pub fn flight_handler(mut self, flight_handler: FlightCraftRef) -> Self {
167        add_service!(
168            self,
169            FlightServiceServer::new(FlightCraftWrapper(flight_handler.clone()))
170        );
171        self
172    }
173
174    /// Add handler for Frontend gRPC service.
175    pub fn frontend_grpc_handler(mut self, handler: FrontendGrpcHandler) -> Self {
176        add_service!(self, FrontendServer::new(handler));
177        self
178    }
179
180    /// Add handler for [OtelArrowService].
181    pub fn otel_arrow_handler(
182        mut self,
183        handler: OtelArrowServiceHandler<OpenTelemetryProtocolHandlerRef>,
184    ) -> Self {
185        let mut server = ArrowMetricsServiceServer::new(handler);
186        server = server
187            .max_decoding_message_size(self.config.max_recv_message_size)
188            .max_encoding_message_size(self.config.max_send_message_size)
189            .accept_compressed(CompressionEncoding::Zstd)
190            .send_compressed(CompressionEncoding::Zstd);
191        let svc = InterceptedService::new(server, HeaderInterceptor {});
192        // Same pre-decode memory admission as `add_service!`.
193        let svc = ServiceBuilder::new()
194            .layer(MemoryLimiterExtensionLayer::new(
195                self.memory_limiter.clone(),
196                self.config.max_recv_message_size,
197            ))
198            .service(svc);
199        self.otel_arrow_service = Some(svc);
200        self
201    }
202
203    /// Add handler for [RegionServer].
204    pub fn region_server_handler(mut self, region_server_handler: RegionServerHandlerRef) -> Self {
205        let handler = RegionServerRequestHandler::new(region_server_handler, self.runtime.clone());
206        add_service!(self, RegionServer::new(handler));
207        self
208    }
209
210    pub fn routes_builder_mut(&mut self) -> &mut RoutesBuilder {
211        &mut self.routes_builder
212    }
213
214    pub fn with_tls_config(mut self, tls_option: TlsOption) -> Result<Self> {
215        // tonic does not support watching for tls config changes
216        // so we don't support it either for now
217        if tls_option.watch {
218            warn!("Certificates watch and reloading for gRPC is NOT supported at the moment");
219        }
220        self.tls_config = if tls_option.should_force_tls() {
221            let cert = std::fs::read_to_string(tls_option.cert_path)
222                .context(InvalidConfigFilePathSnafu)?;
223            let key =
224                std::fs::read_to_string(tls_option.key_path).context(InvalidConfigFilePathSnafu)?;
225            let identity = Identity::from_pem(cert, key);
226            Some(ServerTlsConfig::new().identity(identity))
227        } else {
228            None
229        };
230        Ok(self)
231    }
232
233    pub fn add_layer<L>(self, layer: L) -> Self
234    where
235        L: Layer<Route> + Clone + Send + Sync + 'static,
236        L::Service: Service<Request> + Clone + Send + Sync + 'static,
237        <L::Service as Service<Request>>::Response: IntoResponse + 'static,
238        <L::Service as Service<Request>>::Error: Into<Infallible> + 'static,
239        <L::Service as Service<Request>>::Future: Send + 'static,
240    {
241        let routes = self.routes_builder.routes();
242        let router = routes.into_axum_router();
243        let router = router.layer(layer);
244        Self {
245            routes_builder: RoutesBuilder::from(router),
246            ..self
247        }
248    }
249
250    pub fn build(self) -> GrpcServer {
251        GrpcServer {
252            routes: Mutex::new(Some(self.routes_builder.routes())),
253            shutdown_tx: Mutex::new(None),
254            serve_state: Mutex::new(None),
255            tls_config: self.tls_config,
256            otel_arrow_service: Mutex::new(self.otel_arrow_service),
257            bind_addr: None,
258            name: self.name,
259            config: self.config,
260        }
261    }
262}
servers/grpc/builder.rs

servers/grpc/
builder.rs
