Skip to main content

servers/grpc/
region_server.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16
17use api::v1::region::region_server::Region as RegionServer;
18use api::v1::region::{RegionRequest, RegionResponse, region_request};
19use async_trait::async_trait;
20use common_error::ext::ErrorExt;
21use common_runtime::Runtime;
22use common_runtime::runtime::RuntimeTrait;
23use common_telemetry::tracing::info_span;
24use common_telemetry::tracing_context::{FutureExt, TracingContext};
25use common_telemetry::{debug, error, warn};
26use snafu::{OptionExt, ResultExt};
27use tonic::{Request, Response, Status};
28
29use crate::error::{InvalidQuerySnafu, JoinTaskSnafu, Result};
30use crate::grpc::{TonicResult, cancellation};
31
32#[async_trait]
33pub trait RegionServerHandler: Send + Sync {
34    async fn handle(&self, request: region_request::Body) -> Result<RegionResponse>;
35
36    async fn handle_region_request(&self, request: RegionRequest) -> Result<RegionResponse> {
37        let body = request.body.context(InvalidQuerySnafu {
38            reason: "Expecting non-empty region request body.",
39        })?;
40        self.handle(body).await
41    }
42}
43
44pub type RegionServerHandlerRef = Arc<dyn RegionServerHandler>;
45
46#[derive(Clone)]
47pub struct RegionServerRequestHandler {
48    handler: Arc<dyn RegionServerHandler>,
49    runtime: Runtime,
50}
51
52impl RegionServerRequestHandler {
53    pub fn new(handler: Arc<dyn RegionServerHandler>, runtime: Runtime) -> Self {
54        Self { handler, runtime }
55    }
56
57    async fn handle(&self, request: RegionRequest) -> Result<RegionResponse> {
58        let tracing_context = TracingContext::from_w3c(
59            &request
60                .header
61                .as_ref()
62                .context(InvalidQuerySnafu {
63                    reason: "Expecting non-empty region request header.",
64                })?
65                .tracing_context,
66        );
67        let handler = self.handler.clone();
68
69        // Executes requests in another runtime to
70        // 1. prevent the execution from being cancelled unexpected by Tonic runtime;
71        //   - Refer to our blog for the rational behind it:
72        //     https://www.greptime.com/blogs/2023-01-12-hidden-control-flow.html
73        //   - Obtaining a `JoinHandle` to get the panic message (if there's any).
74        //     From its docs, `JoinHandle` is cancel safe. The task keeps running even it's handle been dropped.
75        // 2. avoid the handler blocks the gRPC runtime incidentally.
76        let handle = self.runtime.spawn(async move {
77            handler
78                .handle_region_request(request)
79                .trace(tracing_context.attach(info_span!("RegionServerRequestHandler::handle")))
80                .await
81                .map_err(|e| {
82                    if e.status_code().should_log_error() {
83                        error!(e; "Failed to handle request");
84                    } else {
85                        // Currently, we still print a debug log.
86                        debug!("Failed to handle request, err: {}", e);
87                    }
88                    e
89                })
90        });
91
92        handle.await.context(JoinTaskSnafu)?
93    }
94}
95
96#[async_trait]
97impl RegionServer for RegionServerRequestHandler {
98    async fn handle(
99        &self,
100        request: Request<RegionRequest>,
101    ) -> TonicResult<Response<RegionResponse>> {
102        let remote_addr = request.remote_addr();
103        let self_cloned = self.clone();
104        let request_future = async move {
105            let request = request.into_inner();
106            let response = self_cloned.handle(request).await?;
107
108            Ok(Response::new(response))
109        };
110
111        let cancellation_future = async move {
112            warn!("Region request from {:?} cancelled by client", remote_addr);
113            // If this future is executed it means the request future was dropped,
114            // so it doesn't actually matter what is returned here
115            Err(Status::cancelled("Region request cancelled by client"))
116        };
117        cancellation::with_cancellation_handler(request_future, cancellation_future).await
118    }
119}
120
121#[cfg(test)]
122mod tests {
123    use std::sync::Mutex;
124    use std::time::Duration;
125
126    use api::v1::region::region_client::RegionClient;
127    use api::v1::region::region_server::RegionServer as RegionService;
128    use api::v1::region::{CancelCompactionRequest, RegionRequestHeader};
129    use api::v1::{ResponseHeader, Status as ResponseStatus};
130    use common_error::status_code::StatusCode;
131    use tokio::net::TcpListener;
132    use tokio_stream::wrappers::TcpListenerStream;
133
134    use super::*;
135
136    #[derive(Default)]
137    struct RecordingHandler {
138        requests: Mutex<Vec<region_request::Body>>,
139    }
140
141    #[async_trait]
142    impl RegionServerHandler for RecordingHandler {
143        async fn handle(&self, request: region_request::Body) -> Result<RegionResponse> {
144            self.requests.lock().unwrap().push(request);
145            Ok(RegionResponse {
146                header: Some(ResponseHeader {
147                    status: Some(ResponseStatus {
148                        status_code: StatusCode::Success as u32,
149                        ..Default::default()
150                    }),
151                }),
152                affected_rows: 1,
153                ..Default::default()
154            })
155        }
156    }
157
158    #[tokio::test]
159    async fn test_cancel_compaction_forwards_through_region_handler() {
160        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
161        let addr = listener.local_addr().unwrap();
162        let handler = Arc::new(RecordingHandler::default());
163        let bridge =
164            RegionServerRequestHandler::new(handler.clone(), common_runtime::global_runtime());
165        let (shutdown, stopped) = tokio::sync::oneshot::channel();
166        let server = tokio::spawn(async move {
167            tonic::transport::Server::builder()
168                .add_service(RegionService::new(bridge))
169                .serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
170                    let _ = stopped.await;
171                })
172                .await
173                .unwrap();
174        });
175        tokio::time::timeout(Duration::from_secs(10), async {
176            let mut client = RegionClient::connect(format!("http://{addr}"))
177                .await
178                .unwrap();
179            let error = client
180                .handle(RegionRequest {
181                    header: None,
182                    body: Some(region_request::Body::CancelCompaction(
183                        CancelCompactionRequest {
184                            region_id: 42,
185                            task_id: "cycle-id".into(),
186                        },
187                    )),
188                })
189                .await
190                .unwrap_err();
191            assert_eq!(error.code(), tonic::Code::InvalidArgument);
192            assert!(handler.requests.lock().unwrap().is_empty());
193
194            let request = RegionRequest {
195                header: Some(RegionRequestHeader {
196                    dbname: "greptime-public".into(),
197                    tracing_context: TracingContext::from_current_span().to_w3c(),
198                    ..Default::default()
199                }),
200                body: Some(region_request::Body::CancelCompaction(
201                    CancelCompactionRequest {
202                        region_id: 42,
203                        task_id: "cycle-id".into(),
204                    },
205                )),
206            };
207            let response = client.handle(request.clone()).await.unwrap().into_inner();
208            assert_eq!(response.affected_rows, 1);
209            assert_eq!(
210                response.header.unwrap().status.unwrap().status_code,
211                StatusCode::Success as u32
212            );
213            assert_eq!(
214                *handler.requests.lock().unwrap(),
215                vec![request.body.unwrap()]
216            );
217        })
218        .await
219        .expect("cancellation RPC did not complete");
220        shutdown.send(()).unwrap();
221        tokio::time::timeout(Duration::from_secs(10), server)
222            .await
223            .unwrap()
224            .unwrap();
225    }
226}