1mod codec;
16
17use std::borrow::Borrow;
18use std::sync::Arc;
19use std::time::Duration;
20
21use api::region::{BatchRegionResponse, RegionResponse};
22use api::v1::ResponseHeader;
23use api::v1::region::RegionResponse as PbRegionResponse;
24use api::v1::region::{
25 RegionRequest, RegionRequestHeader, RemoteDynFilterRequest, RemoteDynFilterUnregister,
26 RemoteDynFilterUpdate, region_request, remote_dyn_filter_request,
27};
28use arc_swap::ArcSwapOption;
29use arrow_flight::Ticket;
30use async_stream::stream;
31use async_trait::async_trait;
32use common_error::ext::{BoxedError, ErrorExt};
33use common_error::status_code::StatusCode;
34use common_grpc::flight::{FlightDecoder, FlightMessage};
35use common_meta::error::{self as meta_error, Result as MetaResult};
36use common_meta::node_manager::Datanode;
37use common_query::request::QueryRequest;
38use common_recordbatch::error::ExternalSnafu;
39use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper, SendableRecordBatchStream};
40use common_telemetry::error;
41use common_telemetry::tracing::Span;
42use common_telemetry::tracing_context::TracingContext;
43use futures_util::Stream;
44use prost::Message;
45use query::query_engine::DefaultSerializer;
46use snafu::{OptionExt, ResultExt, location};
47use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
48use tokio_stream::StreamExt;
49use tonic::codec::CompressionEncoding;
50
51use crate::error::{
52 self, FlightGetSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
53 MissingFieldSnafu, Result,
54};
55use crate::flight::{FlightMessageReader, decode_flight_data};
56use crate::region::codec::RegionCodec;
57use crate::{Client, metrics};
58
59const FLIGHT_DO_GET_TIMEOUT: Duration = Duration::from_secs(10);
60
61#[derive(Debug)]
63pub struct RegionRequester {
64 client: Client,
65 send_compression: bool,
66 accept_compression: bool,
67}
68
69#[async_trait]
70impl Datanode for RegionRequester {
71 async fn handle_request(&self, request: Arc<RegionRequest>) -> MetaResult<RegionResponse> {
72 self.handle_inner(request).await.map_err(Into::into)
73 }
74
75 async fn handle_query(&self, request: QueryRequest) -> MetaResult<SendableRecordBatchStream> {
76 let plan = DFLogicalSubstraitConvertor
77 .encode(&request.plan, DefaultSerializer)
78 .map_err(BoxedError::new)
79 .context(meta_error::ExternalSnafu)?
80 .to_vec();
81 let request = api::v1::region::QueryRequest {
82 header: request.header,
83 region_id: request.region_id.as_u64(),
84 plan,
85 };
86
87 let ticket = Ticket {
88 ticket: request.encode_to_vec().into(),
89 };
90 self.do_get_inner(ticket)
91 .await
92 .map_err(BoxedError::new)
93 .context(meta_error::ExternalSnafu)
94 }
95}
96
97impl RegionRequester {
98 pub fn new(client: Client, send_compression: bool, accept_compression: bool) -> Self {
99 Self {
100 client,
101 send_compression,
102 accept_compression,
103 }
104 }
105
106 pub async fn do_get_inner(&self, ticket: Ticket) -> Result<SendableRecordBatchStream> {
107 let mut flight_client = self
108 .client
109 .make_flight_client(self.send_compression, self.accept_compression)?;
110 let addr = flight_client.addr().to_string();
112 let mut request = tonic::Request::new(ticket);
113 request.set_timeout(FLIGHT_DO_GET_TIMEOUT);
114 let response = flight_client
115 .mut_inner()
116 .do_get(request)
117 .await
118 .or_else(|e| {
119 let tonic_code = e.code();
120 let e: error::Error = e.into();
121 error!(
122 e; "Failed to do Flight get, addr: {}, code: {}",
123 addr,
124 tonic_code
125 );
126 Err(BoxedError::new(e)).with_context(|_| FlightGetSnafu {
127 addr: addr.clone(),
128 tonic_code,
129 })
130 })?;
131
132 let flight_data_stream = response.into_inner();
133 let mut decoder = FlightDecoder::default();
134
135 let flight_message_stream = flight_data_stream
136 .filter_map(move |flight_data| decode_flight_data(&mut decoder, flight_data));
137
138 recordbatches_from_flight_message_stream(addr, flight_message_stream).await
139 }
140
141 pub async fn handle<R>(&self, request: R) -> Result<RegionResponse>
148 where
149 R: Borrow<RegionRequest> + Send + Sync + 'static,
150 {
151 self.handle_inner(request).await
152 }
153
154 pub async fn handle_remote_dyn_filter_update(
156 &self,
157 query_id: impl Into<String>,
158 update: RemoteDynFilterUpdate,
159 ) -> Result<RegionResponse> {
160 self.handle_inner(build_remote_dyn_filter_update_request(query_id, update))
161 .await
162 }
163
164 pub async fn handle_remote_dyn_filter_unregister(
166 &self,
167 query_id: impl Into<String>,
168 unregister: RemoteDynFilterUnregister,
169 ) -> Result<RegionResponse> {
170 self.handle_inner(build_remote_dyn_filter_unregister_request(
171 query_id, unregister,
172 ))
173 .await
174 }
175
176 async fn handle_inner<R>(&self, request: R) -> Result<RegionResponse>
177 where
178 R: Borrow<RegionRequest> + Send + Sync + 'static,
179 {
180 let request_body = request
181 .borrow()
182 .body
183 .as_ref()
184 .context(MissingFieldSnafu { field: "body" })?;
185 let request_count = match request_body {
186 region_request::Body::Inserts(requests) => Some(requests.requests.len()),
187 region_request::Body::Deletes(requests) => Some(requests.requests.len()),
188 _ => None,
189 };
190 let is_insert = matches!(
191 request_body,
192 region_request::Body::Inserts(_) | region_request::Body::BulkInsert(_)
193 );
194 let request_type = request_body.as_ref().to_string();
195 let _timer = metrics::METRIC_REGION_REQUEST_GRPC
196 .with_label_values(&[request_type.as_str()])
197 .start_timer();
198
199 let (addr, mut client) = self.client.raw_region_client()?;
200 if is_insert {
201 if self.send_compression {
202 client = client.send_compressed(CompressionEncoding::Zstd);
203 }
204 if self.accept_compression {
205 client = client.accept_compressed(CompressionEncoding::Zstd);
206 }
207 }
208
209 let accept_partial_result = request
210 .borrow()
211 .header
212 .as_ref()
213 .and_then(|header| header.query_context.as_ref())
214 .and_then(|context| context.extensions.get(api::region::ACCEPT_PARTIAL_RESULT))
215 .is_some_and(|value| value == "true");
216 let response = async {
217 client
218 .ready()
219 .await
220 .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {e}")))?;
221 let mut request = tonic::Request::new(request);
222 request.extensions_mut().insert(tonic::GrpcMethod::new(
223 "greptime.v1.region.Region",
224 "Handle",
225 ));
226 client
227 .unary(
228 request,
229 tonic::codegen::http::uri::PathAndQuery::from_static(
230 "/greptime.v1.region.Region/Handle",
231 ),
232 RegionCodec::default(),
233 )
234 .await
235 }
236 .await
237 .map_err(|status| map_region_error(status, addr))?
238 .into_inner();
239
240 match request_count {
241 Some(count) if accept_partial_result => decode_write_response(response, count),
242 _ => decode_region_response(response),
243 }
244 }
245}
246
247fn map_region_error(status: tonic::Status, addr: String) -> error::Error {
248 error::Error::RegionServer {
249 addr,
250 code: status.code(),
251 source: BoxedError::new(error::Error::from(status)),
252 location: location!(),
253 }
254}
255
256async fn recordbatches_from_flight_message_stream<S>(
257 addr: String,
258 flight_message_stream: S,
259) -> Result<SendableRecordBatchStream>
260where
261 S: Stream<Item = Result<FlightMessage>> + Send + Unpin + 'static,
262{
263 let mut reader = FlightMessageReader::new(addr.clone(), flight_message_stream);
264 let FlightMessage::Schema(schema) = reader
265 .read_first()
266 .await
267 .map_err(|error| flight_stream_error(reader.remote_addr(), error))?
268 else {
269 return IllegalFlightMessagesSnafu {
270 reason: "Expect schema to be the first flight message",
271 }
272 .fail()
273 .map_err(|error| flight_stream_error(reader.remote_addr(), error));
274 };
275
276 let metrics = Arc::new(ArcSwapOption::from(None));
277 let metrics_ref = metrics.clone();
278
279 let tracing_context = TracingContext::from_current_span();
280
281 let schema =
282 Arc::new(datatypes::schema::Schema::try_from(schema).context(error::ConvertSchemaSnafu)?);
283 let schema_cloned = schema.clone();
284 let stream_addr = addr;
285 let stream = Box::pin(stream!({
286 let _span = tracing_context.attach(common_telemetry::tracing::info_span!(
287 "poll_flight_data_stream"
288 ));
289
290 loop {
291 let flight_message = match reader.read_next().await {
292 Ok(Some(message)) => message,
293 Ok(None) => break,
294 Err(error) => {
295 yield Err(BoxedError::new(flight_stream_error(&stream_addr, error)))
296 .context(ExternalSnafu);
297 break;
298 }
299 };
300
301 match flight_message {
302 FlightMessage::RecordBatch(record_batch) => {
303 yield Ok(RecordBatch::from_df_record_batch(
307 schema_cloned.clone(),
308 record_batch,
309 ));
310 }
311 FlightMessage::Metrics(s) => {
312 match serde_json::from_str(&s) {
314 Ok(metrics) => {
315 metrics_ref.swap(Some(Arc::new(metrics)));
316 }
317 Err(error) => {
318 common_telemetry::warn!(
319 "Failed to decode region Flight metrics: {}",
320 error
321 );
322 }
323 }
324 continue;
325 }
326 _ => {
327 yield IllegalFlightMessagesSnafu {
328 reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"
329 }
330 .fail()
331 .map_err(BoxedError::new)
332 .context(ExternalSnafu);
333 break;
334 }
335 }
336 }
337 }));
338 let record_batch_stream = RecordBatchStreamWrapper {
339 schema,
340 stream,
341 output_ordering: None,
342 metrics,
343 span: Span::current(),
344 };
345 Ok(Box::pin(record_batch_stream))
346}
347
348fn flight_stream_error(addr: &str, error: error::Error) -> error::Error {
349 let tonic_code = error.tonic_code().unwrap_or(tonic::Code::Unknown);
350 if error.status_code().should_log_error() {
351 error!(
352 error; "Failed to receive Flight data, addr: {}, code: {}",
353 addr,
354 tonic_code
355 );
356 }
357
358 let message = error.to_string();
361 error::Error::FlightStream {
362 addr: addr.to_string(),
363 tonic_code,
364 message,
365 source: BoxedError::new(error),
366 location: snafu::location!(),
367 }
368}
369
370pub fn build_remote_dyn_filter_update_request(
371 query_id: impl Into<String>,
372 update: RemoteDynFilterUpdate,
373) -> RegionRequest {
374 build_remote_dyn_filter_request(
375 query_id.into(),
376 remote_dyn_filter_request::Action::Update(update),
377 )
378}
379
380pub fn build_remote_dyn_filter_unregister_request(
381 query_id: impl Into<String>,
382 unregister: RemoteDynFilterUnregister,
383) -> RegionRequest {
384 build_remote_dyn_filter_request(
385 query_id.into(),
386 remote_dyn_filter_request::Action::Unregister(unregister),
387 )
388}
389
390fn build_remote_dyn_filter_request(
391 query_id: String,
392 action: remote_dyn_filter_request::Action,
393) -> RegionRequest {
394 RegionRequest {
395 header: Some(RegionRequestHeader {
396 tracing_context: TracingContext::from_current_span().to_w3c(),
397 ..Default::default()
398 }),
399 body: Some(region_request::Body::RemoteDynFilter(
400 RemoteDynFilterRequest {
401 query_id,
402 action: Some(action),
403 },
404 )),
405 }
406}
407
408pub fn check_response_header(header: &Option<ResponseHeader>) -> Result<()> {
410 let status = header
411 .as_ref()
412 .and_then(|header| header.status.as_ref())
413 .context(IllegalDatabaseResponseSnafu {
414 err_msg: "either response header or status is missing",
415 })?;
416
417 if StatusCode::is_success(status.status_code) {
418 Ok(())
419 } else {
420 Err(error::Error::try_from_response_status(status)?)
421 }
422}
423
424pub fn decode_region_response(response: PbRegionResponse) -> Result<RegionResponse> {
426 if !response.failures.is_empty() {
427 return IllegalDatabaseResponseSnafu {
428 err_msg: "Failure details are only valid for inserts/deletes",
429 }
430 .fail();
431 }
432 check_response_header(&response.header)?;
433 Ok(RegionResponse::from_region_response(response))
434}
435
436fn decode_write_response(response: PbRegionResponse, entry_count: usize) -> Result<RegionResponse> {
439 if response.failures.is_empty() {
440 if response
441 .header
442 .as_ref()
443 .and_then(|header| header.status.as_ref())
444 .is_some_and(|status| status.status_code == StatusCode::PartialFailure as u32)
445 {
446 return IllegalDatabaseResponseSnafu {
447 err_msg: "Partial failure has no failed entries",
448 }
449 .fail();
450 }
451 return decode_region_response(response);
452 }
453 let status = response
454 .header
455 .as_ref()
456 .and_then(|header| header.status.as_ref())
457 .context(IllegalDatabaseResponseSnafu {
458 err_msg: "Missing region response status",
459 })?;
460 if status.status_code != StatusCode::PartialFailure as u32 {
461 return IllegalDatabaseResponseSnafu {
462 err_msg: "Failure details require a partial-failure status",
463 }
464 .fail();
465 }
466 let mut seen = std::collections::HashSet::with_capacity(response.failures.len());
467 let mut failures = Vec::with_capacity(response.failures.len());
468 for error in response.failures {
469 let index = error.request_index as usize;
470 if index >= entry_count || !seen.insert(index) {
471 return IllegalDatabaseResponseSnafu {
472 err_msg: "Invalid or duplicate batch error index",
473 }
474 .fail();
475 }
476 let status = error.status.context(IllegalDatabaseResponseSnafu {
477 err_msg: "Missing batch error status",
478 })?;
479 if StatusCode::is_success(status.status_code) {
480 return IllegalDatabaseResponseSnafu {
481 err_msg: "Batch error contains a success status",
482 }
483 .fail();
484 }
485 failures.push((index, error::Error::try_from_response_status(&status)?));
486 }
487 if entry_count == 1
489 && let Some((_, error)) = failures.pop()
490 {
491 return Err(error);
492 }
493 Err(error::Error::PartialRegion {
494 response: Box::new(BatchRegionResponse {
495 affected_rows: response.affected_rows as _,
496 extensions: response.extensions,
497 failures,
498 }),
499 })
500}
501
502#[cfg(test)]
503#[allow(deprecated)]
504mod tests {
505 use std::collections::VecDeque;
506 use std::sync::Mutex;
507
508 use api::v1::ResponseHeader;
509 use api::v1::region::region_server::{Region, RegionServer};
510 use api::v1::region::{
511 BulkInsertRequest, InsertRequest, InsertRequests, RegionRequestFailure,
512 RegionResponse as PbRegionResponse, RemoteDynFilterUnregister, RemoteDynFilterUpdate,
513 region_request, remote_dyn_filter_request,
514 };
515 use api::v1::value::ValueData;
516 use api::v1::{
517 ColumnDataType, ColumnSchema as PbColumnSchema, Row, Rows, Status as PbStatus, Value,
518 };
519 use common_error::ext::RetryHint;
520 use common_error::status_code::StatusCode;
521 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
522 use common_recordbatch::adapter::RecordBatchMetrics;
523 use datatypes::arrow::array::Int32Array;
524 use datatypes::prelude::{ConcreteDataType, VectorRef};
525 use datatypes::schema::{ColumnSchema, Schema};
526 use datatypes::vectors::Int32Vector;
527 use futures_util::stream;
528 use tokio::net::TcpListener;
529 use tokio::task::JoinHandle;
530 use tokio_stream::wrappers::TcpListenerStream;
531 use tonic::codec::CompressionEncoding;
532 use tonic::{Request, Response, Status};
533
534 use super::*;
535 use crate::Error;
536
537 #[derive(Clone, Default)]
538 struct RecordingRegionService {
539 requests: Arc<Mutex<Vec<(RegionRequest, bool, bool)>>>,
540 responses: Arc<Mutex<VecDeque<std::result::Result<PbRegionResponse, Status>>>>,
541 }
542
543 #[tonic::async_trait]
544 impl Region for RecordingRegionService {
545 async fn handle(
546 &self,
547 request: Request<RegionRequest>,
548 ) -> std::result::Result<Response<PbRegionResponse>, Status> {
549 let metadata = request.metadata();
550 let sends_zstd = metadata.get("grpc-encoding").is_some_and(|v| v == "zstd");
551 let accepts_zstd = metadata
552 .get("grpc-accept-encoding")
553 .and_then(|v| v.to_str().ok())
554 .is_some_and(|v| v.split(',').any(|encoding| encoding == "zstd"));
555 self.requests
556 .lock()
557 .unwrap()
558 .push((request.into_inner(), sends_zstd, accepts_zstd));
559 self.responses
560 .lock()
561 .unwrap()
562 .pop_front()
563 .unwrap_or_else(|| Ok(build_response(StatusCode::Success)))
564 .map(Response::new)
565 }
566 }
567
568 struct RegionTestServer {
569 addr: String,
570 service: RecordingRegionService,
571 task: JoinHandle<()>,
572 }
573
574 impl RegionTestServer {
575 async fn start() -> Self {
576 let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
577 let addr = listener.local_addr().unwrap().to_string();
578 let service = RecordingRegionService::default();
579 let server_service = service.clone();
580 let task = tokio::spawn(async move {
581 tonic::transport::Server::builder()
582 .add_service(
583 RegionServer::new(server_service)
584 .accept_compressed(CompressionEncoding::Zstd)
585 .send_compressed(CompressionEncoding::Zstd),
586 )
587 .serve_with_incoming(TcpListenerStream::new(listener))
588 .await
589 .unwrap();
590 });
591 Self {
592 addr,
593 service,
594 task,
595 }
596 }
597
598 fn create_requester(
599 &self,
600 config: ChannelConfig,
601 send: bool,
602 accept: bool,
603 ) -> RegionRequester {
604 RegionRequester::new(
605 Client::with_manager_and_urls(
606 ChannelManager::with_config(config, None),
607 [self.addr.clone()],
608 ),
609 send,
610 accept,
611 )
612 }
613 }
614
615 impl Drop for RegionTestServer {
616 fn drop(&mut self) {
617 self.task.abort();
618 }
619 }
620
621 fn build_response(code: StatusCode) -> PbRegionResponse {
622 PbRegionResponse {
623 header: Some(ResponseHeader {
624 status: Some(PbStatus {
625 status_code: code as u32,
626 ..Default::default()
627 }),
628 }),
629 affected_rows: 1,
630 ..Default::default()
631 }
632 }
633
634 #[tokio::test]
635 async fn test_cancel_compaction_through_handle() {
636 let server = RegionTestServer::start().await;
637 let requester = server.create_requester(ChannelConfig::default(), false, false);
638 let request = RegionRequest {
639 header: Some(RegionRequestHeader::default()),
640 body: Some(region_request::Body::CancelCompaction(
641 api::v1::region::CancelCompactionRequest {
642 region_id: 42,
643 task_id: "cycle-id".into(),
644 },
645 )),
646 };
647 for affected_rows in [0, 1] {
648 let mut response = build_response(StatusCode::Success);
649 response.affected_rows = affected_rows;
650 server
651 .service
652 .responses
653 .lock()
654 .unwrap()
655 .push_back(Ok(response));
656 assert_eq!(
657 requester
658 .handle(request.clone())
659 .await
660 .unwrap()
661 .affected_rows,
662 affected_rows as usize
663 );
664 }
665 let requests = server.service.requests.lock().unwrap();
666 assert_eq!(requests.len(), 2);
667 for (actual, _, _) in requests.iter() {
668 assert_eq!(actual.body, request.body);
669 }
670 }
671
672 fn build_insert_request() -> RegionRequest {
673 RegionRequest {
674 header: Some(RegionRequestHeader {
675 dbname: "shared-request-test".to_string(),
676 ..Default::default()
677 }),
678 body: Some(region_request::Body::Inserts(InsertRequests {
679 requests: vec![InsertRequest {
680 region_id: 42,
681 rows: Some(Rows {
682 schema: vec![
683 PbColumnSchema {
684 column_name: "message".to_string(),
685 datatype: ColumnDataType::String as i32,
686 ..Default::default()
687 },
688 PbColumnSchema {
689 column_name: "payload".to_string(),
690 datatype: ColumnDataType::Binary as i32,
691 ..Default::default()
692 },
693 ],
694 rows: vec![Row {
695 values: vec![
696 Value {
697 value_data: Some(ValueData::StringValue(
698 "hello 世界".to_string(),
699 )),
700 },
701 Value {
702 value_data: Some(ValueData::BinaryValue(vec![0, 1, 127, 255])),
703 },
704 ],
705 }],
706 }),
707 ..Default::default()
708 }],
709 })),
710 }
711 }
712
713 fn with_partial_result(mut request: RegionRequest) -> RegionRequest {
714 request
715 .header
716 .get_or_insert_with(Default::default)
717 .query_context
718 .get_or_insert_with(Default::default)
719 .extensions
720 .insert(
721 api::region::ACCEPT_PARTIAL_RESULT.to_string(),
722 "true".to_string(),
723 );
724 request
725 }
726
727 #[tokio::test]
728 async fn test_owned_and_shared_requests_preserve_payload_and_compression() {
729 let server = RegionTestServer::start().await;
730 let shared = Arc::new(build_insert_request());
731 for (send, accept) in [(false, false), (true, false), (false, true), (true, true)] {
732 let requester = server.create_requester(ChannelConfig::default(), send, accept);
733 let channel =
734 tonic::transport::Endpoint::from_shared(format!("http://{}", server.addr))
735 .unwrap()
736 .connect()
737 .await
738 .unwrap();
739 let mut baseline = api::v1::region::region_client::RegionClient::new(channel);
740 if send {
741 baseline = baseline.send_compressed(CompressionEncoding::Zstd);
742 }
743 if accept {
744 baseline = baseline.accept_compressed(CompressionEncoding::Zstd);
745 }
746 assert_eq!(
747 baseline
748 .handle(build_insert_request())
749 .await
750 .unwrap()
751 .into_inner()
752 .affected_rows,
753 1
754 );
755 assert_eq!(
756 requester
757 .handle(build_insert_request())
758 .await
759 .unwrap()
760 .affected_rows,
761 1
762 );
763 assert_eq!(
764 requester
765 .handle(Arc::clone(&shared))
766 .await
767 .unwrap()
768 .affected_rows,
769 1
770 );
771 }
772 let requests = server.service.requests.lock().unwrap();
773 assert_eq!(requests.len(), 12);
774 for ((request, sends, accepts), (send, accept)) in requests.iter().zip(
775 [(false, false), (true, false), (false, true), (true, true)]
776 .into_iter()
777 .flat_map(|settings| [settings, settings, settings]),
778 ) {
779 assert_eq!(request, shared.as_ref());
780 assert_eq!((*sends, *accepts), (send, accept));
781 }
782 }
783
784 #[tokio::test]
785 async fn test_partial_result_requires_explicit_acceptance() {
786 let server = RegionTestServer::start().await;
787 let requester = server.create_requester(ChannelConfig::default(), false, false);
788 for accepted in [false, true] {
789 let request = if accepted {
790 with_partial_result(build_insert_request())
791 } else {
792 build_insert_request()
793 };
794 let shared = Arc::new(request);
795 server
796 .service
797 .responses
798 .lock()
799 .unwrap()
800 .push_back(Ok(PbRegionResponse {
801 header: Some(ResponseHeader {
802 status: Some(PbStatus {
803 status_code: StatusCode::PartialFailure as u32,
804 ..Default::default()
805 }),
806 }),
807 failures: vec![RegionRequestFailure {
808 request_index: 0,
809 status: Some(PbStatus {
810 status_code: StatusCode::RegionNotFound as u32,
811 retry_hint: RetryHint::RetryAfterRouteRefresh.as_str().to_string(),
812 ..Default::default()
813 }),
814 }],
815 ..Default::default()
816 }));
817 let error = requester.handle(Arc::clone(&shared)).await.unwrap_err();
818 if accepted {
819 assert_eq!(error.status_code(), StatusCode::RegionNotFound);
820 assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
821 } else {
822 assert!(matches!(error, Error::IllegalDatabaseResponse { .. }));
823 }
824 assert_eq!(
825 &server.service.requests.lock().unwrap().last().unwrap().0,
826 shared.as_ref()
827 );
828 assert_eq!(Arc::strong_count(&shared), 1);
829 }
830 }
831
832 #[tokio::test]
833 async fn test_shared_request_can_be_resent_after_transport_error() {
834 let server = RegionTestServer::start().await;
835 let mut status = Status::unavailable("migrating");
836 status.metadata_mut().insert(
837 common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
838 RetryHint::Retryable.as_str().parse().unwrap(),
839 );
840 server
841 .service
842 .responses
843 .lock()
844 .unwrap()
845 .push_back(Err(status));
846 let requester = server.create_requester(ChannelConfig::default(), false, false);
847 let shared = Arc::new(build_insert_request());
848 let error = requester.handle(Arc::clone(&shared)).await.unwrap_err();
849 assert_eq!(error.retry_hint(), RetryHint::Retryable);
850 assert!(
851 matches!(error, Error::RegionServer { addr, code: tonic::Code::Unavailable, .. } if addr == server.addr)
852 );
853 requester.handle(Arc::clone(&shared)).await.unwrap();
854 let requests = server.service.requests.lock().unwrap();
855 assert_eq!(requests.len(), 2);
856 assert_eq!(&requests[0].0, shared.as_ref());
857 assert_eq!(&requests[1].0, shared.as_ref());
858 }
859
860 #[test]
861 fn test_retry_hint_wire_compatibility() {
862 for (name, wire_hint, expected_hint) in [
863 ("legacy_missing", None, RetryHint::NonRetryable),
864 ("legacy_retryable", Some("retryable"), RetryHint::Retryable),
865 (
866 "legacy_non_retryable",
867 Some("non_retryable"),
868 RetryHint::NonRetryable,
869 ),
870 (
871 "route_refresh",
872 Some("retry_after_route_refresh"),
873 RetryHint::RetryAfterRouteRefresh,
874 ),
875 (
876 "unknown_future_hint",
877 Some("future_retry_hint"),
878 RetryHint::NonRetryable,
879 ),
880 ] {
881 let mut status = Status::failed_precondition("region unavailable");
882 status.metadata_mut().insert(
883 common_error::GREPTIME_DB_HEADER_ERROR_CODE,
884 (StatusCode::RegionNotReady as u32)
885 .to_string()
886 .parse()
887 .unwrap(),
888 );
889 if let Some(hint) = wire_hint {
890 status.metadata_mut().insert(
891 common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
892 hint.parse().unwrap(),
893 );
894 }
895 let error: Error = status.into();
896 assert_eq!(error.retry_hint(), expected_hint, "{name}");
897 assert_eq!(error.status_code(), StatusCode::RegionNotReady, "{name}");
898 assert_eq!(
899 error.tonic_code(),
900 Some(tonic::Code::FailedPrecondition),
901 "{name}"
902 );
903 }
904 }
905
906 #[tokio::test]
907 async fn test_transport_error_parity_and_datanode_mapping() {
908 let server = RegionTestServer::start().await;
909 let requester = server.create_requester(ChannelConfig::default(), false, false);
910 for hint in [
911 RetryHint::Retryable,
912 RetryHint::RetryAfterRouteRefresh,
913 RetryHint::NonRetryable,
914 ] {
915 let mut status = Status::failed_precondition("region unavailable");
916 status.metadata_mut().insert(
917 common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
918 hint.as_str().parse().unwrap(),
919 );
920 status.metadata_mut().insert(
921 common_error::GREPTIME_DB_HEADER_ERROR_CODE,
922 (StatusCode::RegionNotFound as u32)
923 .to_string()
924 .parse()
925 .unwrap(),
926 );
927 server
928 .service
929 .responses
930 .lock()
931 .unwrap()
932 .extend((0..4).map(|_| Err(status.clone())));
933 let mut baseline = api::v1::region::region_client::RegionClient::connect(format!(
934 "http://{}",
935 server.addr
936 ))
937 .await
938 .unwrap();
939 let baseline: Error = baseline
940 .handle(build_insert_request())
941 .await
942 .unwrap_err()
943 .into();
944 assert_eq!(baseline.status_code(), StatusCode::RegionNotFound);
945 assert_eq!(baseline.retry_hint(), hint);
946 assert_eq!(baseline.tonic_code(), Some(tonic::Code::FailedPrecondition));
947 let owned = requester.handle(build_insert_request()).await.unwrap_err();
948 let shared = requester
949 .handle(Arc::new(build_insert_request()))
950 .await
951 .unwrap_err();
952 for error in [owned, shared] {
953 assert_eq!(error.status_code(), baseline.status_code());
954 assert_eq!(error.retry_hint(), baseline.retry_hint());
955 assert_eq!(error.tonic_code(), baseline.tonic_code());
956 }
957 let datanode: &dyn Datanode = &requester;
958 let error = datanode
959 .handle_request(Arc::new(build_insert_request()))
960 .await
961 .unwrap_err();
962 assert_eq!(error.status_code(), StatusCode::RegionNotFound);
963 assert_eq!(error.retry_hint(), hint);
964 assert!(matches!(error, meta_error::Error::External { .. }));
965 }
966 }
967
968 #[tokio::test]
969 async fn test_datanode_transport_retry_hints() {
970 let server = RegionTestServer::start().await;
971 let requester = server.create_requester(ChannelConfig::default(), false, false);
972 let datanode: &dyn Datanode = &requester;
973 let statuses = [
974 tonic::Code::Unavailable,
975 tonic::Code::Cancelled,
976 tonic::Code::DeadlineExceeded,
977 tonic::Code::FailedPrecondition,
978 ]
979 .into_iter()
980 .flat_map(|code| {
981 [
982 None,
983 Some(RetryHint::NonRetryable),
984 Some(RetryHint::Retryable),
985 Some(RetryHint::RetryAfterRouteRefresh),
986 ]
987 .into_iter()
988 .map(move |hint| (code, hint))
989 });
990 for (code, hint) in statuses {
991 for body in [
992 region_request::Body::Inserts(Default::default()),
993 region_request::Body::Deletes(Default::default()),
994 region_request::Body::BulkInsert(Default::default()),
995 region_request::Body::Create(Default::default()),
996 region_request::Body::RemoteDynFilter(Default::default()),
997 ] {
998 let request_type = body.as_ref().to_string();
999 let mut status = Status::new(code, "request failed");
1000 status.metadata_mut().insert(
1002 common_error::GREPTIME_DB_HEADER_ERROR_CODE,
1003 (StatusCode::RegionNotReady as u32)
1004 .to_string()
1005 .parse()
1006 .unwrap(),
1007 );
1008 if let Some(hint) = hint {
1009 status.metadata_mut().insert(
1010 common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
1011 hint.as_str().parse().unwrap(),
1012 );
1013 }
1014 let original = Error::from(status.clone());
1015 let expected = if original.should_retry() && !original.retry_hint().is_retryable() {
1016 RetryHint::Retryable
1017 } else {
1018 original.retry_hint()
1019 };
1020 server
1021 .service
1022 .responses
1023 .lock()
1024 .unwrap()
1025 .extend([Err(status.clone()), Err(status)]);
1026 let request = RegionRequest {
1027 header: None,
1028 body: Some(body),
1029 };
1030 let shared = Arc::new(request);
1031 let shared_error = datanode
1032 .handle_request(Arc::clone(&shared))
1033 .await
1034 .unwrap_err();
1035 assert_eq!(Arc::strong_count(&shared), 1);
1036 let request = Arc::try_unwrap(shared).unwrap();
1037 let owned_error = datanode
1038 .handle_request(Arc::new(request))
1039 .await
1040 .unwrap_err();
1041 for error in [owned_error, shared_error] {
1042 assert_eq!(
1043 error.retry_hint(),
1044 expected,
1045 "{code:?}, {hint:?}, {request_type}"
1046 );
1047 assert_eq!(error.retry_hint().is_retryable(), original.should_retry());
1048 assert_eq!(error.status_code(), StatusCode::RegionNotReady);
1049 let meta_error::Error::External { source, .. } = error else {
1050 panic!("expected an external error");
1051 };
1052 assert_eq!(source.retry_hint(), expected);
1053 let Error::RegionServer {
1054 source,
1055 code: actual_code,
1056 ..
1057 } = source.as_any().downcast_ref::<Error>().unwrap()
1058 else {
1059 panic!("expected a region server error");
1060 };
1061 assert_eq!(*actual_code, code);
1062 assert_eq!(source.retry_hint(), original.retry_hint());
1063 assert_eq!(source.to_string(), original.to_string());
1064 }
1065 }
1066 }
1067 }
1068
1069 #[tokio::test]
1070 async fn test_shared_request_preserves_response_header_errors() {
1071 let server = RegionTestServer::start().await;
1072 server.service.responses.lock().unwrap().extend([
1073 Ok(PbRegionResponse::default()),
1074 Ok(build_response(StatusCode::RegionNotFound)),
1075 ]);
1076 let requester = server.create_requester(ChannelConfig::default(), false, false);
1077 let shared = Arc::new(build_insert_request());
1078 assert!(matches!(
1079 requester.handle(Arc::clone(&shared)).await.unwrap_err(),
1080 Error::IllegalDatabaseResponse { .. }
1081 ));
1082 assert!(matches!(
1083 requester.handle(shared).await.unwrap_err(),
1084 Error::Server {
1085 code: StatusCode::RegionNotFound,
1086 ..
1087 }
1088 ));
1089 }
1090
1091 #[tokio::test]
1092 async fn test_shared_request_respects_configured_message_limits() {
1093 let server = RegionTestServer::start().await;
1094 let shared = Arc::new(build_insert_request());
1095 for send_limit in [true, false] {
1096 let mut config = ChannelConfig::default();
1097 if send_limit {
1098 config.max_send_message_size = "1B".parse().unwrap();
1099 } else {
1100 config.max_recv_message_size = "1B".parse().unwrap();
1101 }
1102 let mut baseline = api::v1::region::region_client::RegionClient::connect(format!(
1103 "http://{}",
1104 server.addr
1105 ))
1106 .await
1107 .unwrap()
1108 .max_encoding_message_size(config.max_send_message_size.as_bytes() as usize)
1109 .max_decoding_message_size(config.max_recv_message_size.as_bytes() as usize);
1110 let baseline: Error = baseline
1111 .handle(build_insert_request())
1112 .await
1113 .unwrap_err()
1114 .into();
1115 if !send_limit {
1116 assert_eq!(
1117 baseline.tonic_code(),
1118 Some(tonic::Code::OutOfRange),
1119 "{baseline:?}"
1120 );
1121 }
1122 let requester = server.create_requester(config, false, false);
1123 let error = requester.handle(Arc::clone(&shared)).await.unwrap_err();
1124 assert_eq!(
1125 error.tonic_code(),
1126 baseline.tonic_code(),
1127 "shared: {error:?}; generated: {baseline:?}"
1128 );
1129 assert_eq!(
1130 error.status_code(),
1131 baseline.status_code(),
1132 "shared: {error:?}; generated: {baseline:?}"
1133 );
1134 let Error::RegionServer { source, .. } = &error else {
1135 panic!("expected a region server error: {error:?}");
1136 };
1137 assert_eq!(
1138 source.retry_hint(),
1139 baseline.retry_hint(),
1140 "shared: {error:?}; generated: {baseline:?}"
1141 );
1142 assert_eq!(
1143 server.service.requests.lock().unwrap().len(),
1144 if send_limit { 0 } else { 2 }
1145 );
1146 }
1147 let requester = server.create_requester(ChannelConfig::default(), false, false);
1148 requester.handle(shared).await.unwrap();
1149 }
1150
1151 #[tokio::test]
1152 async fn test_inserts_and_bulk_insert_use_transport_compression() {
1153 let server = RegionTestServer::start().await;
1154 let requester = server.create_requester(ChannelConfig::default(), true, true);
1155 let send_only_requester = server.create_requester(ChannelConfig::default(), true, false);
1156 let accept_only_requester = server.create_requester(ChannelConfig::default(), false, true);
1157 let disabled_requester = server.create_requester(ChannelConfig::default(), false, false);
1158 let inserts = || RegionRequest {
1159 body: Some(region_request::Body::Inserts(Default::default())),
1160 ..Default::default()
1161 };
1162 let bulk_insert = || RegionRequest {
1163 body: Some(region_request::Body::BulkInsert(
1164 BulkInsertRequest::default(),
1165 )),
1166 ..Default::default()
1167 };
1168
1169 requester.handle(inserts()).await.unwrap();
1170 send_only_requester.handle(inserts()).await.unwrap();
1171 accept_only_requester.handle(inserts()).await.unwrap();
1172 requester.handle(bulk_insert()).await.unwrap();
1173 disabled_requester.handle(bulk_insert()).await.unwrap();
1174 requester
1175 .handle(build_remote_dyn_filter_unregister_request(
1176 "query-1",
1177 RemoteDynFilterUnregister {
1178 filter_id: "filter-1".to_string(),
1179 },
1180 ))
1181 .await
1182 .unwrap();
1183
1184 assert_eq!(
1185 vec![
1186 (true, true),
1187 (true, false),
1188 (false, true),
1189 (true, true),
1190 (false, false),
1191 (false, false)
1192 ],
1193 server
1194 .service
1195 .requests
1196 .lock()
1197 .unwrap()
1198 .iter()
1199 .map(|(_, send, accept)| (*send, *accept))
1200 .collect::<Vec<_>>()
1201 );
1202 }
1203
1204 #[test]
1205 fn test_flight_stream_error_preserves_peer_address() {
1206 let error = flight_stream_error(
1207 "127.0.0.1:4001",
1208 tonic::Status::unavailable("datanode unavailable").into(),
1209 );
1210
1211 assert!(matches!(
1212 error,
1213 error::Error::FlightStream {
1214 addr,
1215 tonic_code: tonic::Code::Unavailable,
1216 ..
1217 } if addr == "127.0.0.1:4001"
1218 ));
1219 }
1220
1221 #[tokio::test]
1222 async fn test_empty_flight_stream_preserves_peer_address() {
1223 let Err(error) = recordbatches_from_flight_message_stream(
1224 "127.0.0.1:4001".to_string(),
1225 stream::empty::<Result<FlightMessage>>(),
1226 )
1227 .await
1228 else {
1229 panic!("expected empty Flight stream to fail");
1230 };
1231
1232 assert!(matches!(
1233 error,
1234 error::Error::FlightStream {
1235 addr,
1236 tonic_code: tonic::Code::Unknown,
1237 ..
1238 } if addr == "127.0.0.1:4001"
1239 ));
1240 }
1241
1242 fn test_schema() -> Arc<Schema> {
1243 Arc::new(Schema::new(vec![ColumnSchema::new(
1244 "v",
1245 ConcreteDataType::int32_datatype(),
1246 false,
1247 )]))
1248 }
1249
1250 fn test_metrics_json() -> String {
1251 serde_json::to_string(&RecordBatchMetrics {
1252 elapsed_compute: 7,
1253 ..Default::default()
1254 })
1255 .unwrap()
1256 }
1257
1258 #[tokio::test]
1259 async fn test_partial_response_validation_and_compression() {
1260 let server = RegionTestServer::start().await;
1261 let requester = server.create_requester(ChannelConfig::default(), true, true);
1262 let mut request = build_insert_request();
1263 if let Some(region_request::Body::Inserts(inserts)) = &mut request.body {
1264 inserts.requests.push(inserts.requests[0].clone());
1265 }
1266 let request = Arc::new(with_partial_result(request));
1267 let failure = RegionRequestFailure {
1268 request_index: 0,
1269 status: Some(PbStatus {
1270 status_code: StatusCode::RegionNotFound as u32,
1271 err_msg: "moved".to_string(),
1272 retry_hint: "retry_after_route_refresh".to_string(),
1273 }),
1274 };
1275 for (failures, valid) in [
1276 (vec![], true),
1277 (vec![failure.clone()], true),
1278 (vec![failure.clone(), failure.clone()], false),
1279 (
1280 vec![RegionRequestFailure {
1281 request_index: u32::MAX,
1282 ..failure.clone()
1283 }],
1284 false,
1285 ),
1286 (
1287 vec![RegionRequestFailure {
1288 request_index: 0,
1289 status: None,
1290 }],
1291 false,
1292 ),
1293 (
1294 vec![RegionRequestFailure {
1295 request_index: 0,
1296 status: Some(PbStatus::default()),
1297 }],
1298 false,
1299 ),
1300 (
1301 vec![RegionRequestFailure {
1302 request_index: 0,
1303 status: Some(PbStatus {
1304 status_code: u32::MAX,
1305 ..Default::default()
1306 }),
1307 }],
1308 false,
1309 ),
1310 ] {
1311 let has_error = !failures.is_empty();
1312 server
1313 .service
1314 .responses
1315 .lock()
1316 .unwrap()
1317 .push_back(Ok(PbRegionResponse {
1318 header: Some(ResponseHeader {
1319 status: Some(PbStatus {
1320 status_code: if has_error {
1321 StatusCode::PartialFailure
1322 } else {
1323 StatusCode::Success
1324 } as u32,
1325 ..Default::default()
1326 }),
1327 }),
1328 affected_rows: 3,
1329 extensions: std::collections::HashMap::from([("test".to_string(), vec![1, 2])]),
1330 failures,
1331 ..Default::default()
1332 }));
1333 let response = requester.handle_inner(Arc::clone(&request)).await;
1334 if valid {
1335 if has_error {
1336 let error = response.unwrap_err();
1337 assert!(error.output_msg().contains("entry 0"));
1338 assert!(error.output_msg().contains("moved"));
1339 let error: common_meta::error::Error = error.into();
1340 assert!(error.output_msg().contains("moved"));
1341 let common_meta::error::Error::PartialRegion { response } = error else {
1342 panic!("Expected partial region failure");
1343 };
1344 assert_eq!(response.affected_rows, 3);
1345 assert_eq!(response.extensions["test"], vec![1, 2]);
1346 assert_eq!(
1347 response.failures[0].1.retry_hint(),
1348 RetryHint::RetryAfterRouteRefresh
1349 );
1350 } else {
1351 let response = response.unwrap();
1352 assert_eq!(response.affected_rows, 3);
1353 assert_eq!(response.extensions["test"], vec![1, 2]);
1354 }
1355 } else {
1356 assert!(matches!(
1357 response.unwrap_err(),
1358 Error::IllegalDatabaseResponse { .. }
1359 ));
1360 }
1361 }
1362 for (received, send, accept) in server.service.requests.lock().unwrap().iter() {
1363 assert_eq!(received, request.as_ref());
1364 assert!(*send && *accept);
1365 }
1366 assert_eq!(Arc::strong_count(&request), 1);
1367 }
1368
1369 #[test]
1370 fn test_partial_response_envelope() {
1371 let failure = RegionRequestFailure {
1372 request_index: 0,
1373 status: Some(PbStatus {
1374 status_code: StatusCode::RegionNotFound as u32,
1375 retry_hint: RetryHint::RetryAfterRouteRefresh.as_str().to_string(),
1376 ..Default::default()
1377 }),
1378 };
1379 for (status, failures, count) in [
1380 (Some(StatusCode::Success), vec![failure.clone()], Some(2)),
1381 (Some(StatusCode::PartialFailure), vec![], Some(2)),
1382 (
1383 Some(StatusCode::PartialFailure),
1384 vec![failure.clone()],
1385 None,
1386 ),
1387 (None, vec![failure.clone()], Some(2)),
1388 ] {
1389 let response = PbRegionResponse {
1390 header: status.map(|code| ResponseHeader {
1391 status: Some(PbStatus {
1392 status_code: code as u32,
1393 ..Default::default()
1394 }),
1395 }),
1396 failures,
1397 ..Default::default()
1398 };
1399 assert!(matches!(
1400 match count {
1401 Some(count) => decode_write_response(response, count),
1402 None => decode_region_response(response),
1403 },
1404 Err(Error::IllegalDatabaseResponse { .. })
1405 ));
1406 }
1407 let response = PbRegionResponse {
1408 header: Some(ResponseHeader {
1409 status: Some(PbStatus {
1410 status_code: StatusCode::PartialFailure as u32,
1411 ..Default::default()
1412 }),
1413 }),
1414 failures: vec![failure],
1415 ..Default::default()
1416 };
1417 assert!(check_response_header(&response.header).is_err());
1419 let error = decode_write_response(response, 1).unwrap_err();
1420 assert_eq!(error.status_code(), StatusCode::RegionNotFound);
1421 assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
1422 }
1423
1424 #[test]
1425 fn test_partial_error_preserves_status_and_retry_hint() {
1426 let expected_message = "Region request has 2 failed entries; entry 0: first input failed";
1427 for code in [StatusCode::StorageUnavailable, StatusCode::InvalidArguments] {
1428 let response = PbRegionResponse {
1429 header: Some(ResponseHeader {
1430 status: Some(PbStatus {
1431 status_code: StatusCode::PartialFailure as u32,
1432 ..Default::default()
1433 }),
1434 }),
1435 failures: vec![
1436 RegionRequestFailure {
1437 request_index: 1,
1438 status: Some(PbStatus {
1439 status_code: StatusCode::Internal as u32,
1440 err_msg: "later input failed".to_string(),
1441 ..Default::default()
1442 }),
1443 },
1444 RegionRequestFailure {
1445 request_index: 0,
1446 status: Some(PbStatus {
1447 status_code: code as u32,
1448 err_msg: "first input failed".to_string(),
1449 retry_hint: RetryHint::NonRetryable.as_str().to_string(),
1450 }),
1451 },
1452 ],
1453 ..Default::default()
1454 };
1455 let error = decode_write_response(response, 2).unwrap_err();
1456 assert_eq!(code, error.status_code());
1457 assert_eq!(expected_message, error.to_string());
1458 assert_eq!(RetryHint::Retryable, error.retry_hint());
1459 let error: common_meta::error::Error = error.into();
1460 assert_eq!(code, error.status_code());
1461 assert_eq!(expected_message, error.to_string());
1462 assert_eq!(RetryHint::Retryable, error.retry_hint());
1463 let common_meta::error::Error::PartialRegion { response } = error else {
1464 panic!("expected partial result")
1465 };
1466 assert!(
1467 response
1468 .failures
1469 .iter()
1470 .all(|(_, error)| error.retry_hint() == RetryHint::NonRetryable)
1471 );
1472 }
1473 }
1474
1475 #[tokio::test]
1476 async fn test_region_rpc_errors() {
1477 let server = RegionTestServer::start().await;
1478 let requester = server.create_requester(ChannelConfig::default(), false, false);
1479 let request = Arc::new(build_insert_request());
1480 let mut business_error = Status::unimplemented("unsupported write");
1481 business_error.metadata_mut().insert(
1482 common_error::GREPTIME_DB_HEADER_ERROR_CODE,
1483 (StatusCode::Unsupported as u32)
1484 .to_string()
1485 .parse()
1486 .unwrap(),
1487 );
1488 for status in [
1489 Status::unimplemented("unknown method"),
1490 business_error,
1491 Status::unavailable("lost response"),
1492 Status::deadline_exceeded("timeout"),
1493 ] {
1494 let code = status.code();
1495 server
1496 .service
1497 .responses
1498 .lock()
1499 .unwrap()
1500 .push_back(Err(status));
1501 let error = requester
1502 .handle_inner(Arc::clone(&request))
1503 .await
1504 .unwrap_err();
1505 assert!(matches!(error, Error::RegionServer { code: actual, .. } if actual == code));
1506 }
1507 assert_eq!(server.service.requests.lock().unwrap().len(), 4);
1508 }
1509
1510 #[test]
1511 fn test_check_response_header() {
1512 for (hint, expected) in [
1513 ("retryable", RetryHint::Retryable),
1514 (
1515 "retry_after_route_refresh",
1516 RetryHint::RetryAfterRouteRefresh,
1517 ),
1518 ("non_retryable", RetryHint::NonRetryable),
1519 ("", RetryHint::NonRetryable),
1520 ("future_hint", RetryHint::NonRetryable),
1521 ] {
1522 let error = check_response_header(&Some(ResponseHeader {
1523 status: Some(PbStatus {
1524 status_code: StatusCode::RegionNotFound as u32,
1525 err_msg: "region moved".to_string(),
1526 retry_hint: hint.to_string(),
1527 }),
1528 }))
1529 .unwrap_err();
1530 assert_eq!(error.retry_hint(), expected, "{hint}");
1531 }
1532
1533 let result = check_response_header(&None);
1534 assert!(matches!(
1535 result.unwrap_err(),
1536 Error::IllegalDatabaseResponse { .. }
1537 ));
1538
1539 let result = check_response_header(&Some(ResponseHeader { status: None }));
1540 assert!(matches!(
1541 result.unwrap_err(),
1542 Error::IllegalDatabaseResponse { .. }
1543 ));
1544
1545 let result = check_response_header(&Some(ResponseHeader {
1546 status: Some(PbStatus {
1547 status_code: StatusCode::Success as u32,
1548 err_msg: String::default(),
1549 ..Default::default()
1550 }),
1551 }));
1552 assert!(result.is_ok());
1553
1554 let result = check_response_header(&Some(ResponseHeader {
1555 status: Some(PbStatus {
1556 status_code: u32::MAX,
1557 err_msg: String::default(),
1558 ..Default::default()
1559 }),
1560 }));
1561 assert!(matches!(
1562 result.unwrap_err(),
1563 Error::IllegalDatabaseResponse { .. }
1564 ));
1565
1566 let result = check_response_header(&Some(ResponseHeader {
1567 status: Some(PbStatus {
1568 status_code: StatusCode::Internal as u32,
1569 err_msg: "blabla".to_string(),
1570 ..Default::default()
1571 }),
1572 }));
1573 let Error::Server { code, msg, .. } = result.unwrap_err() else {
1574 unreachable!()
1575 };
1576 assert_eq!(code, StatusCode::Internal);
1577 assert_eq!(msg, "blabla");
1578 }
1579
1580 #[test]
1581 fn test_build_remote_dyn_filter_request_sets_header_and_body() {
1582 let request = build_remote_dyn_filter_update_request(
1583 "query-1",
1584 RemoteDynFilterUpdate {
1585 filter_id: "filter-1".to_string(),
1586 payload: vec![1, 2, 3],
1587 generation: 7,
1588 is_complete: false,
1589 },
1590 );
1591
1592 request.header.expect("remote dyn filter header must exist");
1593
1594 let body = request.body.expect("remote dyn filter body must exist");
1595 let region_request::Body::RemoteDynFilter(remote_request) = body else {
1596 panic!("expected remote dyn filter request body");
1597 };
1598
1599 assert_eq!(remote_request.query_id, "query-1");
1600 assert!(matches!(
1601 remote_request.action,
1602 Some(remote_dyn_filter_request::Action::Update(_))
1603 ));
1604 }
1605
1606 #[test]
1607 fn test_build_remote_dyn_filter_unregister_request_sets_header_and_body() {
1608 let request = build_remote_dyn_filter_unregister_request(
1609 "query-1",
1610 RemoteDynFilterUnregister {
1611 filter_id: "filter-9".to_string(),
1612 },
1613 );
1614
1615 request.header.expect("remote dyn filter header must exist");
1616
1617 let body = request.body.expect("remote dyn filter body must exist");
1618 let region_request::Body::RemoteDynFilter(remote_request) = body else {
1619 panic!("expected remote dyn filter request body");
1620 };
1621
1622 assert_eq!(remote_request.query_id, "query-1");
1623 assert!(matches!(
1624 remote_request.action,
1625 Some(remote_dyn_filter_request::Action::Unregister(_))
1626 ));
1627 }
1628
1629 #[tokio::test]
1630 async fn test_record_batch_stream_continues_after_pre_batch_metrics() {
1631 let schema = test_schema();
1632 let batch = RecordBatch::new(
1633 schema.clone(),
1634 vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1635 )
1636 .unwrap();
1637
1638 let mut recordbatches = recordbatches_from_flight_message_stream(
1639 "test-peer".to_string(),
1640 stream::iter(vec![
1641 Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1642 Ok(FlightMessage::Metrics(test_metrics_json())),
1643 Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())),
1644 ]),
1645 )
1646 .await
1647 .unwrap();
1648
1649 let batch = recordbatches.next().await.unwrap().unwrap();
1650 assert_eq!(batch.num_rows(), 1);
1651 assert!(recordbatches.next().await.is_none());
1652
1653 let metrics = recordbatches.metrics().unwrap();
1654 assert_eq!(metrics.elapsed_compute, 7);
1655 }
1656
1657 #[tokio::test]
1658 async fn test_record_batch_is_yielded_without_waiting_for_next_message() {
1659 let schema = test_schema();
1660 let batch = RecordBatch::new(
1661 schema.clone(),
1662 vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1663 )
1664 .unwrap();
1665
1666 let messages = stream::iter(vec![
1667 Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1668 Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())),
1669 ])
1670 .chain(stream::pending::<Result<FlightMessage>>());
1671 let mut recordbatches =
1672 recordbatches_from_flight_message_stream("test-peer".to_string(), messages)
1673 .await
1674 .unwrap();
1675
1676 let batch = tokio::time::timeout(Duration::from_secs(1), recordbatches.next())
1677 .await
1678 .expect("the first batch must not wait for lookahead")
1679 .unwrap()
1680 .unwrap();
1681 assert_eq!(batch.num_rows(), 1);
1682 }
1683
1684 #[tokio::test]
1685 async fn test_malformed_region_metrics_are_non_fatal() {
1686 let schema = test_schema();
1687 let batch = RecordBatch::new(
1688 schema.clone(),
1689 vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1690 )
1691 .unwrap();
1692 let mut recordbatches = recordbatches_from_flight_message_stream(
1693 "test-peer".to_string(),
1694 stream::iter(vec![
1695 Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1696 Ok(FlightMessage::Metrics("{not-json}".to_string())),
1697 Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())),
1698 ]),
1699 )
1700 .await
1701 .unwrap();
1702
1703 assert_eq!(recordbatches.next().await.unwrap().unwrap().num_rows(), 1);
1704 assert!(recordbatches.next().await.is_none());
1705 }
1706
1707 #[tokio::test]
1708 async fn test_record_batch_stream_preserves_following_record_batch() {
1709 let schema = test_schema();
1710 let first_batch = RecordBatch::new(
1711 schema.clone(),
1712 vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1713 )
1714 .unwrap();
1715 let second_batch = RecordBatch::new(
1716 schema.clone(),
1717 vec![Arc::new(Int32Vector::from_slice([2])) as VectorRef],
1718 )
1719 .unwrap();
1720
1721 let mut recordbatches = recordbatches_from_flight_message_stream(
1722 "test-peer".to_string(),
1723 stream::iter(vec![
1724 Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1725 Ok(FlightMessage::RecordBatch(
1726 first_batch.into_df_record_batch(),
1727 )),
1728 Ok(FlightMessage::RecordBatch(
1729 second_batch.into_df_record_batch(),
1730 )),
1731 ]),
1732 )
1733 .await
1734 .unwrap();
1735
1736 let first_batch = recordbatches.next().await.unwrap().unwrap();
1737 let second_batch = recordbatches.next().await.unwrap().unwrap();
1738 assert_eq!(
1739 first_batch
1740 .column(0)
1741 .as_any()
1742 .downcast_ref::<Int32Array>()
1743 .unwrap()
1744 .value(0),
1745 1
1746 );
1747 assert_eq!(
1748 second_batch
1749 .column(0)
1750 .as_any()
1751 .downcast_ref::<Int32Array>()
1752 .unwrap()
1753 .value(0),
1754 2
1755 );
1756 assert!(recordbatches.next().await.is_none());
1757 }
1758
1759 #[tokio::test]
1760 async fn test_record_batch_stream_captures_final_metrics_after_record_batch() {
1761 let schema = test_schema();
1762 let batch = RecordBatch::new(
1763 schema.clone(),
1764 vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1765 )
1766 .unwrap();
1767 let final_metrics = serde_json::to_string(&RecordBatchMetrics {
1768 elapsed_compute: 99,
1769 ..Default::default()
1770 })
1771 .unwrap();
1772 let mut recordbatches = recordbatches_from_flight_message_stream(
1773 "test-peer".to_string(),
1774 stream::iter(vec![
1775 Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1776 Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())),
1777 Ok(FlightMessage::Metrics(final_metrics)),
1778 ]),
1779 )
1780 .await
1781 .unwrap();
1782
1783 assert_eq!(recordbatches.next().await.unwrap().unwrap().num_rows(), 1);
1784 assert!(recordbatches.next().await.is_none());
1785 assert_eq!(recordbatches.metrics().unwrap().elapsed_compute, 99);
1786 }
1787
1788 #[tokio::test]
1789 async fn test_record_batch_stream_exposes_error_after_pre_batch_metrics() {
1790 let schema = test_schema();
1791 let mut recordbatches = recordbatches_from_flight_message_stream(
1792 "test-peer".to_string(),
1793 stream::iter(vec![
1794 Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1795 Ok(FlightMessage::Metrics(test_metrics_json())),
1796 Err(Error::from(Status::internal("boom after metrics"))),
1797 ]),
1798 )
1799 .await
1800 .unwrap();
1801
1802 let err = recordbatches.next().await.unwrap().unwrap_err();
1803 assert!(
1806 err.to_string().starts_with("External error: "),
1807 "unexpected error: {err}"
1808 );
1809 assert!(
1810 err.to_string().contains("boom after metrics"),
1811 "unexpected error: {err}"
1812 );
1813 assert!(recordbatches.next().await.is_none());
1814 }
1815}