Skip to main content

client/
region.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod codec;
16
17use std::borrow::Borrow;
18use std::sync::Arc;
19use std::time::Duration;
20
21use api::region::{BatchRegionResponse, RegionResponse};
22use api::v1::ResponseHeader;
23use api::v1::region::RegionResponse as PbRegionResponse;
24use api::v1::region::{
25    RegionRequest, RegionRequestHeader, RemoteDynFilterRequest, RemoteDynFilterUnregister,
26    RemoteDynFilterUpdate, region_request, remote_dyn_filter_request,
27};
28use arc_swap::ArcSwapOption;
29use arrow_flight::Ticket;
30use async_stream::stream;
31use async_trait::async_trait;
32use common_error::ext::{BoxedError, ErrorExt};
33use common_error::status_code::StatusCode;
34use common_grpc::flight::{FlightDecoder, FlightMessage};
35use common_meta::error::{self as meta_error, Result as MetaResult};
36use common_meta::node_manager::Datanode;
37use common_query::request::QueryRequest;
38use common_recordbatch::error::ExternalSnafu;
39use common_recordbatch::{RecordBatch, RecordBatchStreamWrapper, SendableRecordBatchStream};
40use common_telemetry::error;
41use common_telemetry::tracing::Span;
42use common_telemetry::tracing_context::TracingContext;
43use futures_util::Stream;
44use prost::Message;
45use query::query_engine::DefaultSerializer;
46use snafu::{OptionExt, ResultExt, location};
47use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan};
48use tokio_stream::StreamExt;
49use tonic::codec::CompressionEncoding;
50
51use crate::error::{
52    self, FlightGetSnafu, IllegalDatabaseResponseSnafu, IllegalFlightMessagesSnafu,
53    MissingFieldSnafu, Result,
54};
55use crate::flight::{FlightMessageReader, decode_flight_data};
56use crate::region::codec::RegionCodec;
57use crate::{Client, metrics};
58
59const FLIGHT_DO_GET_TIMEOUT: Duration = Duration::from_secs(10);
60
61/// Sends region operations through a configured gRPC client.
62#[derive(Debug)]
63pub struct RegionRequester {
64    client: Client,
65    send_compression: bool,
66    accept_compression: bool,
67}
68
69#[async_trait]
70impl Datanode for RegionRequester {
71    async fn handle_request(&self, request: Arc<RegionRequest>) -> MetaResult<RegionResponse> {
72        self.handle_inner(request).await.map_err(Into::into)
73    }
74
75    async fn handle_query(&self, request: QueryRequest) -> MetaResult<SendableRecordBatchStream> {
76        let plan = DFLogicalSubstraitConvertor
77            .encode(&request.plan, DefaultSerializer)
78            .map_err(BoxedError::new)
79            .context(meta_error::ExternalSnafu)?
80            .to_vec();
81        let request = api::v1::region::QueryRequest {
82            header: request.header,
83            region_id: request.region_id.as_u64(),
84            plan,
85        };
86
87        let ticket = Ticket {
88            ticket: request.encode_to_vec().into(),
89        };
90        self.do_get_inner(ticket)
91            .await
92            .map_err(BoxedError::new)
93            .context(meta_error::ExternalSnafu)
94    }
95}
96
97impl RegionRequester {
98    pub fn new(client: Client, send_compression: bool, accept_compression: bool) -> Self {
99        Self {
100            client,
101            send_compression,
102            accept_compression,
103        }
104    }
105
106    pub async fn do_get_inner(&self, ticket: Ticket) -> Result<SendableRecordBatchStream> {
107        let mut flight_client = self
108            .client
109            .make_flight_client(self.send_compression, self.accept_compression)?;
110        // Limit Flight DoGet response time without limiting query stream execution.
111        let addr = flight_client.addr().to_string();
112        let mut request = tonic::Request::new(ticket);
113        request.set_timeout(FLIGHT_DO_GET_TIMEOUT);
114        let response = flight_client
115            .mut_inner()
116            .do_get(request)
117            .await
118            .or_else(|e| {
119                let tonic_code = e.code();
120                let e: error::Error = e.into();
121                error!(
122                    e; "Failed to do Flight get, addr: {}, code: {}",
123                    addr,
124                    tonic_code
125                );
126                Err(BoxedError::new(e)).with_context(|_| FlightGetSnafu {
127                    addr: addr.clone(),
128                    tonic_code,
129                })
130            })?;
131
132        let flight_data_stream = response.into_inner();
133        let mut decoder = FlightDecoder::default();
134
135        let flight_message_stream = flight_data_stream
136            .filter_map(move |flight_data| decode_flight_data(&mut decoder, flight_data));
137
138        recordbatches_from_flight_message_stream(addr, flight_message_stream).await
139    }
140
141    /// Sends an owned or shared request without deep-cloning its protobuf payload.
142    ///
143    /// Accepts both [`RegionRequest`] and [`Arc<RegionRequest>`]. Callers can retain a
144    /// shared request for another attempt; this method does not retry or determine
145    /// whether replaying the request is safe. Callers accepting partial results in the
146    /// request header must inspect failed entries before retrying.
147    pub async fn handle<R>(&self, request: R) -> Result<RegionResponse>
148    where
149        R: Borrow<RegionRequest> + Send + Sync + 'static,
150    {
151        self.handle_inner(request).await
152    }
153
154    /// Sends a remote dynamic filter update to the region service.
155    pub async fn handle_remote_dyn_filter_update(
156        &self,
157        query_id: impl Into<String>,
158        update: RemoteDynFilterUpdate,
159    ) -> Result<RegionResponse> {
160        self.handle_inner(build_remote_dyn_filter_update_request(query_id, update))
161            .await
162    }
163
164    /// Unregisters a remote dynamic filter in the region service.
165    pub async fn handle_remote_dyn_filter_unregister(
166        &self,
167        query_id: impl Into<String>,
168        unregister: RemoteDynFilterUnregister,
169    ) -> Result<RegionResponse> {
170        self.handle_inner(build_remote_dyn_filter_unregister_request(
171            query_id, unregister,
172        ))
173        .await
174    }
175
176    async fn handle_inner<R>(&self, request: R) -> Result<RegionResponse>
177    where
178        R: Borrow<RegionRequest> + Send + Sync + 'static,
179    {
180        let request_body = request
181            .borrow()
182            .body
183            .as_ref()
184            .context(MissingFieldSnafu { field: "body" })?;
185        let request_count = match request_body {
186            region_request::Body::Inserts(requests) => Some(requests.requests.len()),
187            region_request::Body::Deletes(requests) => Some(requests.requests.len()),
188            _ => None,
189        };
190        let is_insert = matches!(
191            request_body,
192            region_request::Body::Inserts(_) | region_request::Body::BulkInsert(_)
193        );
194        let request_type = request_body.as_ref().to_string();
195        let _timer = metrics::METRIC_REGION_REQUEST_GRPC
196            .with_label_values(&[request_type.as_str()])
197            .start_timer();
198
199        let (addr, mut client) = self.client.raw_region_client()?;
200        if is_insert {
201            if self.send_compression {
202                client = client.send_compressed(CompressionEncoding::Zstd);
203            }
204            if self.accept_compression {
205                client = client.accept_compressed(CompressionEncoding::Zstd);
206            }
207        }
208
209        let accept_partial_result = request
210            .borrow()
211            .header
212            .as_ref()
213            .and_then(|header| header.query_context.as_ref())
214            .and_then(|context| context.extensions.get(api::region::ACCEPT_PARTIAL_RESULT))
215            .is_some_and(|value| value == "true");
216        let response = async {
217            client
218                .ready()
219                .await
220                .map_err(|e| tonic::Status::unknown(format!("Service was not ready: {e}")))?;
221            let mut request = tonic::Request::new(request);
222            request.extensions_mut().insert(tonic::GrpcMethod::new(
223                "greptime.v1.region.Region",
224                "Handle",
225            ));
226            client
227                .unary(
228                    request,
229                    tonic::codegen::http::uri::PathAndQuery::from_static(
230                        "/greptime.v1.region.Region/Handle",
231                    ),
232                    RegionCodec::default(),
233                )
234                .await
235        }
236        .await
237        .map_err(|status| map_region_error(status, addr))?
238        .into_inner();
239
240        match request_count {
241            Some(count) if accept_partial_result => decode_write_response(response, count),
242            _ => decode_region_response(response),
243        }
244    }
245}
246
247fn map_region_error(status: tonic::Status, addr: String) -> error::Error {
248    error::Error::RegionServer {
249        addr,
250        code: status.code(),
251        source: BoxedError::new(error::Error::from(status)),
252        location: location!(),
253    }
254}
255
256async fn recordbatches_from_flight_message_stream<S>(
257    addr: String,
258    flight_message_stream: S,
259) -> Result<SendableRecordBatchStream>
260where
261    S: Stream<Item = Result<FlightMessage>> + Send + Unpin + 'static,
262{
263    let mut reader = FlightMessageReader::new(addr.clone(), flight_message_stream);
264    let FlightMessage::Schema(schema) = reader
265        .read_first()
266        .await
267        .map_err(|error| flight_stream_error(reader.remote_addr(), error))?
268    else {
269        return IllegalFlightMessagesSnafu {
270            reason: "Expect schema to be the first flight message",
271        }
272        .fail()
273        .map_err(|error| flight_stream_error(reader.remote_addr(), error));
274    };
275
276    let metrics = Arc::new(ArcSwapOption::from(None));
277    let metrics_ref = metrics.clone();
278
279    let tracing_context = TracingContext::from_current_span();
280
281    let schema =
282        Arc::new(datatypes::schema::Schema::try_from(schema).context(error::ConvertSchemaSnafu)?);
283    let schema_cloned = schema.clone();
284    let stream_addr = addr;
285    let stream = Box::pin(stream!({
286        let _span = tracing_context.attach(common_telemetry::tracing::info_span!(
287            "poll_flight_data_stream"
288        ));
289
290        loop {
291            let flight_message = match reader.read_next().await {
292                Ok(Some(message)) => message,
293                Ok(None) => break,
294                Err(error) => {
295                    yield Err(BoxedError::new(flight_stream_error(&stream_addr, error)))
296                        .context(ExternalSnafu);
297                    break;
298                }
299            };
300
301            match flight_message {
302                FlightMessage::RecordBatch(record_batch) => {
303                    // Deliver each batch immediately. In particular, do not
304                    // wait for a possible following Metrics message; it is
305                    // consumed on the next poll of this stream.
306                    yield Ok(RecordBatch::from_df_record_batch(
307                        schema_cloned.clone(),
308                        record_batch,
309                    ));
310                }
311                FlightMessage::Metrics(s) => {
312                    // Metrics may arrive before the next RecordBatch.
313                    match serde_json::from_str(&s) {
314                        Ok(metrics) => {
315                            metrics_ref.swap(Some(Arc::new(metrics)));
316                        }
317                        Err(error) => {
318                            common_telemetry::warn!(
319                                "Failed to decode region Flight metrics: {}",
320                                error
321                            );
322                        }
323                    }
324                    continue;
325                }
326                _ => {
327                    yield IllegalFlightMessagesSnafu {
328                        reason: "A Schema message must be succeeded exclusively by a set of RecordBatch messages"
329                    }
330                    .fail()
331                    .map_err(BoxedError::new)
332                    .context(ExternalSnafu);
333                    break;
334                }
335            }
336        }
337    }));
338    let record_batch_stream = RecordBatchStreamWrapper {
339        schema,
340        stream,
341        output_ordering: None,
342        metrics,
343        span: Span::current(),
344    };
345    Ok(Box::pin(record_batch_stream))
346}
347
348fn flight_stream_error(addr: &str, error: error::Error) -> error::Error {
349    let tonic_code = error.tonic_code().unwrap_or(tonic::Code::Unknown);
350    if error.status_code().should_log_error() {
351        error!(
352            error; "Failed to receive Flight data, addr: {}, code: {}",
353            addr,
354            tonic_code
355        );
356    }
357
358    // The failure's own sentence travels with it: a region read crosses a process boundary, and
359    // the message is the only thing that says what the region refused.
360    let message = error.to_string();
361    error::Error::FlightStream {
362        addr: addr.to_string(),
363        tonic_code,
364        message,
365        source: BoxedError::new(error),
366        location: snafu::location!(),
367    }
368}
369
370pub fn build_remote_dyn_filter_update_request(
371    query_id: impl Into<String>,
372    update: RemoteDynFilterUpdate,
373) -> RegionRequest {
374    build_remote_dyn_filter_request(
375        query_id.into(),
376        remote_dyn_filter_request::Action::Update(update),
377    )
378}
379
380pub fn build_remote_dyn_filter_unregister_request(
381    query_id: impl Into<String>,
382    unregister: RemoteDynFilterUnregister,
383) -> RegionRequest {
384    build_remote_dyn_filter_request(
385        query_id.into(),
386        remote_dyn_filter_request::Action::Unregister(unregister),
387    )
388}
389
390fn build_remote_dyn_filter_request(
391    query_id: String,
392    action: remote_dyn_filter_request::Action,
393) -> RegionRequest {
394    RegionRequest {
395        header: Some(RegionRequestHeader {
396            tracing_context: TracingContext::from_current_span().to_w3c(),
397            ..Default::default()
398        }),
399        body: Some(region_request::Body::RemoteDynFilter(
400            RemoteDynFilterRequest {
401                query_id,
402                action: Some(action),
403            },
404        )),
405    }
406}
407
408/// Checks that the response contains a successful application status.
409pub fn check_response_header(header: &Option<ResponseHeader>) -> Result<()> {
410    let status = header
411        .as_ref()
412        .and_then(|header| header.status.as_ref())
413        .context(IllegalDatabaseResponseSnafu {
414            err_msg: "either response header or status is missing",
415        })?;
416
417    if StatusCode::is_success(status.status_code) {
418        Ok(())
419    } else {
420        Err(error::Error::try_from_response_status(status)?)
421    }
422}
423
424/// Decodes an ordinary region response without per-entry write failures.
425pub fn decode_region_response(response: PbRegionResponse) -> Result<RegionResponse> {
426    if !response.failures.is_empty() {
427        return IllegalDatabaseResponseSnafu {
428            err_msg: "Failure details are only valid for inserts/deletes",
429        }
430        .fail();
431    }
432    check_response_header(&response.header)?;
433    Ok(RegionResponse::from_region_response(response))
434}
435
436/// Decodes an inserts/deletes response, validating failures against the original entry count.
437/// A failed single-entry request returns its underlying error.
438fn decode_write_response(response: PbRegionResponse, entry_count: usize) -> Result<RegionResponse> {
439    if response.failures.is_empty() {
440        if response
441            .header
442            .as_ref()
443            .and_then(|header| header.status.as_ref())
444            .is_some_and(|status| status.status_code == StatusCode::PartialFailure as u32)
445        {
446            return IllegalDatabaseResponseSnafu {
447                err_msg: "Partial failure has no failed entries",
448            }
449            .fail();
450        }
451        return decode_region_response(response);
452    }
453    let status = response
454        .header
455        .as_ref()
456        .and_then(|header| header.status.as_ref())
457        .context(IllegalDatabaseResponseSnafu {
458            err_msg: "Missing region response status",
459        })?;
460    if status.status_code != StatusCode::PartialFailure as u32 {
461        return IllegalDatabaseResponseSnafu {
462            err_msg: "Failure details require a partial-failure status",
463        }
464        .fail();
465    }
466    let mut seen = std::collections::HashSet::with_capacity(response.failures.len());
467    let mut failures = Vec::with_capacity(response.failures.len());
468    for error in response.failures {
469        let index = error.request_index as usize;
470        if index >= entry_count || !seen.insert(index) {
471            return IllegalDatabaseResponseSnafu {
472                err_msg: "Invalid or duplicate batch error index",
473            }
474            .fail();
475        }
476        let status = error.status.context(IllegalDatabaseResponseSnafu {
477            err_msg: "Missing batch error status",
478        })?;
479        if StatusCode::is_success(status.status_code) {
480            return IllegalDatabaseResponseSnafu {
481                err_msg: "Batch error contains a success status",
482            }
483            .fail();
484        }
485        failures.push((index, error::Error::try_from_response_status(&status)?));
486    }
487    // A single failed entry has no successful peers to preserve.
488    if entry_count == 1
489        && let Some((_, error)) = failures.pop()
490    {
491        return Err(error);
492    }
493    Err(error::Error::PartialRegion {
494        response: Box::new(BatchRegionResponse {
495            affected_rows: response.affected_rows as _,
496            extensions: response.extensions,
497            failures,
498        }),
499    })
500}
501
502#[cfg(test)]
503#[allow(deprecated)]
504mod tests {
505    use std::collections::VecDeque;
506    use std::sync::Mutex;
507
508    use api::v1::ResponseHeader;
509    use api::v1::region::region_server::{Region, RegionServer};
510    use api::v1::region::{
511        BulkInsertRequest, InsertRequest, InsertRequests, RegionRequestFailure,
512        RegionResponse as PbRegionResponse, RemoteDynFilterUnregister, RemoteDynFilterUpdate,
513        region_request, remote_dyn_filter_request,
514    };
515    use api::v1::value::ValueData;
516    use api::v1::{
517        ColumnDataType, ColumnSchema as PbColumnSchema, Row, Rows, Status as PbStatus, Value,
518    };
519    use common_error::ext::RetryHint;
520    use common_error::status_code::StatusCode;
521    use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
522    use common_recordbatch::adapter::RecordBatchMetrics;
523    use datatypes::arrow::array::Int32Array;
524    use datatypes::prelude::{ConcreteDataType, VectorRef};
525    use datatypes::schema::{ColumnSchema, Schema};
526    use datatypes::vectors::Int32Vector;
527    use futures_util::stream;
528    use tokio::net::TcpListener;
529    use tokio::task::JoinHandle;
530    use tokio_stream::wrappers::TcpListenerStream;
531    use tonic::codec::CompressionEncoding;
532    use tonic::{Request, Response, Status};
533
534    use super::*;
535    use crate::Error;
536
537    #[derive(Clone, Default)]
538    struct RecordingRegionService {
539        requests: Arc<Mutex<Vec<(RegionRequest, bool, bool)>>>,
540        responses: Arc<Mutex<VecDeque<std::result::Result<PbRegionResponse, Status>>>>,
541    }
542
543    #[tonic::async_trait]
544    impl Region for RecordingRegionService {
545        async fn handle(
546            &self,
547            request: Request<RegionRequest>,
548        ) -> std::result::Result<Response<PbRegionResponse>, Status> {
549            let metadata = request.metadata();
550            let sends_zstd = metadata.get("grpc-encoding").is_some_and(|v| v == "zstd");
551            let accepts_zstd = metadata
552                .get("grpc-accept-encoding")
553                .and_then(|v| v.to_str().ok())
554                .is_some_and(|v| v.split(',').any(|encoding| encoding == "zstd"));
555            self.requests
556                .lock()
557                .unwrap()
558                .push((request.into_inner(), sends_zstd, accepts_zstd));
559            self.responses
560                .lock()
561                .unwrap()
562                .pop_front()
563                .unwrap_or_else(|| Ok(build_response(StatusCode::Success)))
564                .map(Response::new)
565        }
566    }
567
568    struct RegionTestServer {
569        addr: String,
570        service: RecordingRegionService,
571        task: JoinHandle<()>,
572    }
573
574    impl RegionTestServer {
575        async fn start() -> Self {
576            let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
577            let addr = listener.local_addr().unwrap().to_string();
578            let service = RecordingRegionService::default();
579            let server_service = service.clone();
580            let task = tokio::spawn(async move {
581                tonic::transport::Server::builder()
582                    .add_service(
583                        RegionServer::new(server_service)
584                            .accept_compressed(CompressionEncoding::Zstd)
585                            .send_compressed(CompressionEncoding::Zstd),
586                    )
587                    .serve_with_incoming(TcpListenerStream::new(listener))
588                    .await
589                    .unwrap();
590            });
591            Self {
592                addr,
593                service,
594                task,
595            }
596        }
597
598        fn create_requester(
599            &self,
600            config: ChannelConfig,
601            send: bool,
602            accept: bool,
603        ) -> RegionRequester {
604            RegionRequester::new(
605                Client::with_manager_and_urls(
606                    ChannelManager::with_config(config, None),
607                    [self.addr.clone()],
608                ),
609                send,
610                accept,
611            )
612        }
613    }
614
615    impl Drop for RegionTestServer {
616        fn drop(&mut self) {
617            self.task.abort();
618        }
619    }
620
621    fn build_response(code: StatusCode) -> PbRegionResponse {
622        PbRegionResponse {
623            header: Some(ResponseHeader {
624                status: Some(PbStatus {
625                    status_code: code as u32,
626                    ..Default::default()
627                }),
628            }),
629            affected_rows: 1,
630            ..Default::default()
631        }
632    }
633
634    #[tokio::test]
635    async fn test_cancel_compaction_through_handle() {
636        let server = RegionTestServer::start().await;
637        let requester = server.create_requester(ChannelConfig::default(), false, false);
638        let request = RegionRequest {
639            header: Some(RegionRequestHeader::default()),
640            body: Some(region_request::Body::CancelCompaction(
641                api::v1::region::CancelCompactionRequest {
642                    region_id: 42,
643                    task_id: "cycle-id".into(),
644                },
645            )),
646        };
647        for affected_rows in [0, 1] {
648            let mut response = build_response(StatusCode::Success);
649            response.affected_rows = affected_rows;
650            server
651                .service
652                .responses
653                .lock()
654                .unwrap()
655                .push_back(Ok(response));
656            assert_eq!(
657                requester
658                    .handle(request.clone())
659                    .await
660                    .unwrap()
661                    .affected_rows,
662                affected_rows as usize
663            );
664        }
665        let requests = server.service.requests.lock().unwrap();
666        assert_eq!(requests.len(), 2);
667        for (actual, _, _) in requests.iter() {
668            assert_eq!(actual.body, request.body);
669        }
670    }
671
672    fn build_insert_request() -> RegionRequest {
673        RegionRequest {
674            header: Some(RegionRequestHeader {
675                dbname: "shared-request-test".to_string(),
676                ..Default::default()
677            }),
678            body: Some(region_request::Body::Inserts(InsertRequests {
679                requests: vec![InsertRequest {
680                    region_id: 42,
681                    rows: Some(Rows {
682                        schema: vec![
683                            PbColumnSchema {
684                                column_name: "message".to_string(),
685                                datatype: ColumnDataType::String as i32,
686                                ..Default::default()
687                            },
688                            PbColumnSchema {
689                                column_name: "payload".to_string(),
690                                datatype: ColumnDataType::Binary as i32,
691                                ..Default::default()
692                            },
693                        ],
694                        rows: vec![Row {
695                            values: vec![
696                                Value {
697                                    value_data: Some(ValueData::StringValue(
698                                        "hello 世界".to_string(),
699                                    )),
700                                },
701                                Value {
702                                    value_data: Some(ValueData::BinaryValue(vec![0, 1, 127, 255])),
703                                },
704                            ],
705                        }],
706                    }),
707                    ..Default::default()
708                }],
709            })),
710        }
711    }
712
713    fn with_partial_result(mut request: RegionRequest) -> RegionRequest {
714        request
715            .header
716            .get_or_insert_with(Default::default)
717            .query_context
718            .get_or_insert_with(Default::default)
719            .extensions
720            .insert(
721                api::region::ACCEPT_PARTIAL_RESULT.to_string(),
722                "true".to_string(),
723            );
724        request
725    }
726
727    #[tokio::test]
728    async fn test_owned_and_shared_requests_preserve_payload_and_compression() {
729        let server = RegionTestServer::start().await;
730        let shared = Arc::new(build_insert_request());
731        for (send, accept) in [(false, false), (true, false), (false, true), (true, true)] {
732            let requester = server.create_requester(ChannelConfig::default(), send, accept);
733            let channel =
734                tonic::transport::Endpoint::from_shared(format!("http://{}", server.addr))
735                    .unwrap()
736                    .connect()
737                    .await
738                    .unwrap();
739            let mut baseline = api::v1::region::region_client::RegionClient::new(channel);
740            if send {
741                baseline = baseline.send_compressed(CompressionEncoding::Zstd);
742            }
743            if accept {
744                baseline = baseline.accept_compressed(CompressionEncoding::Zstd);
745            }
746            assert_eq!(
747                baseline
748                    .handle(build_insert_request())
749                    .await
750                    .unwrap()
751                    .into_inner()
752                    .affected_rows,
753                1
754            );
755            assert_eq!(
756                requester
757                    .handle(build_insert_request())
758                    .await
759                    .unwrap()
760                    .affected_rows,
761                1
762            );
763            assert_eq!(
764                requester
765                    .handle(Arc::clone(&shared))
766                    .await
767                    .unwrap()
768                    .affected_rows,
769                1
770            );
771        }
772        let requests = server.service.requests.lock().unwrap();
773        assert_eq!(requests.len(), 12);
774        for ((request, sends, accepts), (send, accept)) in requests.iter().zip(
775            [(false, false), (true, false), (false, true), (true, true)]
776                .into_iter()
777                .flat_map(|settings| [settings, settings, settings]),
778        ) {
779            assert_eq!(request, shared.as_ref());
780            assert_eq!((*sends, *accepts), (send, accept));
781        }
782    }
783
784    #[tokio::test]
785    async fn test_partial_result_requires_explicit_acceptance() {
786        let server = RegionTestServer::start().await;
787        let requester = server.create_requester(ChannelConfig::default(), false, false);
788        for accepted in [false, true] {
789            let request = if accepted {
790                with_partial_result(build_insert_request())
791            } else {
792                build_insert_request()
793            };
794            let shared = Arc::new(request);
795            server
796                .service
797                .responses
798                .lock()
799                .unwrap()
800                .push_back(Ok(PbRegionResponse {
801                    header: Some(ResponseHeader {
802                        status: Some(PbStatus {
803                            status_code: StatusCode::PartialFailure as u32,
804                            ..Default::default()
805                        }),
806                    }),
807                    failures: vec![RegionRequestFailure {
808                        request_index: 0,
809                        status: Some(PbStatus {
810                            status_code: StatusCode::RegionNotFound as u32,
811                            retry_hint: RetryHint::RetryAfterRouteRefresh.as_str().to_string(),
812                            ..Default::default()
813                        }),
814                    }],
815                    ..Default::default()
816                }));
817            let error = requester.handle(Arc::clone(&shared)).await.unwrap_err();
818            if accepted {
819                assert_eq!(error.status_code(), StatusCode::RegionNotFound);
820                assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
821            } else {
822                assert!(matches!(error, Error::IllegalDatabaseResponse { .. }));
823            }
824            assert_eq!(
825                &server.service.requests.lock().unwrap().last().unwrap().0,
826                shared.as_ref()
827            );
828            assert_eq!(Arc::strong_count(&shared), 1);
829        }
830    }
831
832    #[tokio::test]
833    async fn test_shared_request_can_be_resent_after_transport_error() {
834        let server = RegionTestServer::start().await;
835        let mut status = Status::unavailable("migrating");
836        status.metadata_mut().insert(
837            common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
838            RetryHint::Retryable.as_str().parse().unwrap(),
839        );
840        server
841            .service
842            .responses
843            .lock()
844            .unwrap()
845            .push_back(Err(status));
846        let requester = server.create_requester(ChannelConfig::default(), false, false);
847        let shared = Arc::new(build_insert_request());
848        let error = requester.handle(Arc::clone(&shared)).await.unwrap_err();
849        assert_eq!(error.retry_hint(), RetryHint::Retryable);
850        assert!(
851            matches!(error, Error::RegionServer { addr, code: tonic::Code::Unavailable, .. } if addr == server.addr)
852        );
853        requester.handle(Arc::clone(&shared)).await.unwrap();
854        let requests = server.service.requests.lock().unwrap();
855        assert_eq!(requests.len(), 2);
856        assert_eq!(&requests[0].0, shared.as_ref());
857        assert_eq!(&requests[1].0, shared.as_ref());
858    }
859
860    #[test]
861    fn test_retry_hint_wire_compatibility() {
862        for (name, wire_hint, expected_hint) in [
863            ("legacy_missing", None, RetryHint::NonRetryable),
864            ("legacy_retryable", Some("retryable"), RetryHint::Retryable),
865            (
866                "legacy_non_retryable",
867                Some("non_retryable"),
868                RetryHint::NonRetryable,
869            ),
870            (
871                "route_refresh",
872                Some("retry_after_route_refresh"),
873                RetryHint::RetryAfterRouteRefresh,
874            ),
875            (
876                "unknown_future_hint",
877                Some("future_retry_hint"),
878                RetryHint::NonRetryable,
879            ),
880        ] {
881            let mut status = Status::failed_precondition("region unavailable");
882            status.metadata_mut().insert(
883                common_error::GREPTIME_DB_HEADER_ERROR_CODE,
884                (StatusCode::RegionNotReady as u32)
885                    .to_string()
886                    .parse()
887                    .unwrap(),
888            );
889            if let Some(hint) = wire_hint {
890                status.metadata_mut().insert(
891                    common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
892                    hint.parse().unwrap(),
893                );
894            }
895            let error: Error = status.into();
896            assert_eq!(error.retry_hint(), expected_hint, "{name}");
897            assert_eq!(error.status_code(), StatusCode::RegionNotReady, "{name}");
898            assert_eq!(
899                error.tonic_code(),
900                Some(tonic::Code::FailedPrecondition),
901                "{name}"
902            );
903        }
904    }
905
906    #[tokio::test]
907    async fn test_transport_error_parity_and_datanode_mapping() {
908        let server = RegionTestServer::start().await;
909        let requester = server.create_requester(ChannelConfig::default(), false, false);
910        for hint in [
911            RetryHint::Retryable,
912            RetryHint::RetryAfterRouteRefresh,
913            RetryHint::NonRetryable,
914        ] {
915            let mut status = Status::failed_precondition("region unavailable");
916            status.metadata_mut().insert(
917                common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
918                hint.as_str().parse().unwrap(),
919            );
920            status.metadata_mut().insert(
921                common_error::GREPTIME_DB_HEADER_ERROR_CODE,
922                (StatusCode::RegionNotFound as u32)
923                    .to_string()
924                    .parse()
925                    .unwrap(),
926            );
927            server
928                .service
929                .responses
930                .lock()
931                .unwrap()
932                .extend((0..4).map(|_| Err(status.clone())));
933            let mut baseline = api::v1::region::region_client::RegionClient::connect(format!(
934                "http://{}",
935                server.addr
936            ))
937            .await
938            .unwrap();
939            let baseline: Error = baseline
940                .handle(build_insert_request())
941                .await
942                .unwrap_err()
943                .into();
944            assert_eq!(baseline.status_code(), StatusCode::RegionNotFound);
945            assert_eq!(baseline.retry_hint(), hint);
946            assert_eq!(baseline.tonic_code(), Some(tonic::Code::FailedPrecondition));
947            let owned = requester.handle(build_insert_request()).await.unwrap_err();
948            let shared = requester
949                .handle(Arc::new(build_insert_request()))
950                .await
951                .unwrap_err();
952            for error in [owned, shared] {
953                assert_eq!(error.status_code(), baseline.status_code());
954                assert_eq!(error.retry_hint(), baseline.retry_hint());
955                assert_eq!(error.tonic_code(), baseline.tonic_code());
956            }
957            let datanode: &dyn Datanode = &requester;
958            let error = datanode
959                .handle_request(Arc::new(build_insert_request()))
960                .await
961                .unwrap_err();
962            assert_eq!(error.status_code(), StatusCode::RegionNotFound);
963            assert_eq!(error.retry_hint(), hint);
964            assert!(matches!(error, meta_error::Error::External { .. }));
965        }
966    }
967
968    #[tokio::test]
969    async fn test_datanode_transport_retry_hints() {
970        let server = RegionTestServer::start().await;
971        let requester = server.create_requester(ChannelConfig::default(), false, false);
972        let datanode: &dyn Datanode = &requester;
973        let statuses = [
974            tonic::Code::Unavailable,
975            tonic::Code::Cancelled,
976            tonic::Code::DeadlineExceeded,
977            tonic::Code::FailedPrecondition,
978        ]
979        .into_iter()
980        .flat_map(|code| {
981            [
982                None,
983                Some(RetryHint::NonRetryable),
984                Some(RetryHint::Retryable),
985                Some(RetryHint::RetryAfterRouteRefresh),
986            ]
987            .into_iter()
988            .map(move |hint| (code, hint))
989        });
990        for (code, hint) in statuses {
991            for body in [
992                region_request::Body::Inserts(Default::default()),
993                region_request::Body::Deletes(Default::default()),
994                region_request::Body::BulkInsert(Default::default()),
995                region_request::Body::Create(Default::default()),
996                region_request::Body::RemoteDynFilter(Default::default()),
997            ] {
998                let request_type = body.as_ref().to_string();
999                let mut status = Status::new(code, "request failed");
1000                // Metadata status is deliberately different from the transport status.
1001                status.metadata_mut().insert(
1002                    common_error::GREPTIME_DB_HEADER_ERROR_CODE,
1003                    (StatusCode::RegionNotReady as u32)
1004                        .to_string()
1005                        .parse()
1006                        .unwrap(),
1007                );
1008                if let Some(hint) = hint {
1009                    status.metadata_mut().insert(
1010                        common_error::GREPTIME_DB_HEADER_ERROR_RETRY_HINT,
1011                        hint.as_str().parse().unwrap(),
1012                    );
1013                }
1014                let original = Error::from(status.clone());
1015                let expected = if original.should_retry() && !original.retry_hint().is_retryable() {
1016                    RetryHint::Retryable
1017                } else {
1018                    original.retry_hint()
1019                };
1020                server
1021                    .service
1022                    .responses
1023                    .lock()
1024                    .unwrap()
1025                    .extend([Err(status.clone()), Err(status)]);
1026                let request = RegionRequest {
1027                    header: None,
1028                    body: Some(body),
1029                };
1030                let shared = Arc::new(request);
1031                let shared_error = datanode
1032                    .handle_request(Arc::clone(&shared))
1033                    .await
1034                    .unwrap_err();
1035                assert_eq!(Arc::strong_count(&shared), 1);
1036                let request = Arc::try_unwrap(shared).unwrap();
1037                let owned_error = datanode
1038                    .handle_request(Arc::new(request))
1039                    .await
1040                    .unwrap_err();
1041                for error in [owned_error, shared_error] {
1042                    assert_eq!(
1043                        error.retry_hint(),
1044                        expected,
1045                        "{code:?}, {hint:?}, {request_type}"
1046                    );
1047                    assert_eq!(error.retry_hint().is_retryable(), original.should_retry());
1048                    assert_eq!(error.status_code(), StatusCode::RegionNotReady);
1049                    let meta_error::Error::External { source, .. } = error else {
1050                        panic!("expected an external error");
1051                    };
1052                    assert_eq!(source.retry_hint(), expected);
1053                    let Error::RegionServer {
1054                        source,
1055                        code: actual_code,
1056                        ..
1057                    } = source.as_any().downcast_ref::<Error>().unwrap()
1058                    else {
1059                        panic!("expected a region server error");
1060                    };
1061                    assert_eq!(*actual_code, code);
1062                    assert_eq!(source.retry_hint(), original.retry_hint());
1063                    assert_eq!(source.to_string(), original.to_string());
1064                }
1065            }
1066        }
1067    }
1068
1069    #[tokio::test]
1070    async fn test_shared_request_preserves_response_header_errors() {
1071        let server = RegionTestServer::start().await;
1072        server.service.responses.lock().unwrap().extend([
1073            Ok(PbRegionResponse::default()),
1074            Ok(build_response(StatusCode::RegionNotFound)),
1075        ]);
1076        let requester = server.create_requester(ChannelConfig::default(), false, false);
1077        let shared = Arc::new(build_insert_request());
1078        assert!(matches!(
1079            requester.handle(Arc::clone(&shared)).await.unwrap_err(),
1080            Error::IllegalDatabaseResponse { .. }
1081        ));
1082        assert!(matches!(
1083            requester.handle(shared).await.unwrap_err(),
1084            Error::Server {
1085                code: StatusCode::RegionNotFound,
1086                ..
1087            }
1088        ));
1089    }
1090
1091    #[tokio::test]
1092    async fn test_shared_request_respects_configured_message_limits() {
1093        let server = RegionTestServer::start().await;
1094        let shared = Arc::new(build_insert_request());
1095        for send_limit in [true, false] {
1096            let mut config = ChannelConfig::default();
1097            if send_limit {
1098                config.max_send_message_size = "1B".parse().unwrap();
1099            } else {
1100                config.max_recv_message_size = "1B".parse().unwrap();
1101            }
1102            let mut baseline = api::v1::region::region_client::RegionClient::connect(format!(
1103                "http://{}",
1104                server.addr
1105            ))
1106            .await
1107            .unwrap()
1108            .max_encoding_message_size(config.max_send_message_size.as_bytes() as usize)
1109            .max_decoding_message_size(config.max_recv_message_size.as_bytes() as usize);
1110            let baseline: Error = baseline
1111                .handle(build_insert_request())
1112                .await
1113                .unwrap_err()
1114                .into();
1115            if !send_limit {
1116                assert_eq!(
1117                    baseline.tonic_code(),
1118                    Some(tonic::Code::OutOfRange),
1119                    "{baseline:?}"
1120                );
1121            }
1122            let requester = server.create_requester(config, false, false);
1123            let error = requester.handle(Arc::clone(&shared)).await.unwrap_err();
1124            assert_eq!(
1125                error.tonic_code(),
1126                baseline.tonic_code(),
1127                "shared: {error:?}; generated: {baseline:?}"
1128            );
1129            assert_eq!(
1130                error.status_code(),
1131                baseline.status_code(),
1132                "shared: {error:?}; generated: {baseline:?}"
1133            );
1134            let Error::RegionServer { source, .. } = &error else {
1135                panic!("expected a region server error: {error:?}");
1136            };
1137            assert_eq!(
1138                source.retry_hint(),
1139                baseline.retry_hint(),
1140                "shared: {error:?}; generated: {baseline:?}"
1141            );
1142            assert_eq!(
1143                server.service.requests.lock().unwrap().len(),
1144                if send_limit { 0 } else { 2 }
1145            );
1146        }
1147        let requester = server.create_requester(ChannelConfig::default(), false, false);
1148        requester.handle(shared).await.unwrap();
1149    }
1150
1151    #[tokio::test]
1152    async fn test_inserts_and_bulk_insert_use_transport_compression() {
1153        let server = RegionTestServer::start().await;
1154        let requester = server.create_requester(ChannelConfig::default(), true, true);
1155        let send_only_requester = server.create_requester(ChannelConfig::default(), true, false);
1156        let accept_only_requester = server.create_requester(ChannelConfig::default(), false, true);
1157        let disabled_requester = server.create_requester(ChannelConfig::default(), false, false);
1158        let inserts = || RegionRequest {
1159            body: Some(region_request::Body::Inserts(Default::default())),
1160            ..Default::default()
1161        };
1162        let bulk_insert = || RegionRequest {
1163            body: Some(region_request::Body::BulkInsert(
1164                BulkInsertRequest::default(),
1165            )),
1166            ..Default::default()
1167        };
1168
1169        requester.handle(inserts()).await.unwrap();
1170        send_only_requester.handle(inserts()).await.unwrap();
1171        accept_only_requester.handle(inserts()).await.unwrap();
1172        requester.handle(bulk_insert()).await.unwrap();
1173        disabled_requester.handle(bulk_insert()).await.unwrap();
1174        requester
1175            .handle(build_remote_dyn_filter_unregister_request(
1176                "query-1",
1177                RemoteDynFilterUnregister {
1178                    filter_id: "filter-1".to_string(),
1179                },
1180            ))
1181            .await
1182            .unwrap();
1183
1184        assert_eq!(
1185            vec![
1186                (true, true),
1187                (true, false),
1188                (false, true),
1189                (true, true),
1190                (false, false),
1191                (false, false)
1192            ],
1193            server
1194                .service
1195                .requests
1196                .lock()
1197                .unwrap()
1198                .iter()
1199                .map(|(_, send, accept)| (*send, *accept))
1200                .collect::<Vec<_>>()
1201        );
1202    }
1203
1204    #[test]
1205    fn test_flight_stream_error_preserves_peer_address() {
1206        let error = flight_stream_error(
1207            "127.0.0.1:4001",
1208            tonic::Status::unavailable("datanode unavailable").into(),
1209        );
1210
1211        assert!(matches!(
1212            error,
1213            error::Error::FlightStream {
1214                addr,
1215                tonic_code: tonic::Code::Unavailable,
1216                ..
1217            } if addr == "127.0.0.1:4001"
1218        ));
1219    }
1220
1221    #[tokio::test]
1222    async fn test_empty_flight_stream_preserves_peer_address() {
1223        let Err(error) = recordbatches_from_flight_message_stream(
1224            "127.0.0.1:4001".to_string(),
1225            stream::empty::<Result<FlightMessage>>(),
1226        )
1227        .await
1228        else {
1229            panic!("expected empty Flight stream to fail");
1230        };
1231
1232        assert!(matches!(
1233            error,
1234            error::Error::FlightStream {
1235                addr,
1236                tonic_code: tonic::Code::Unknown,
1237                ..
1238            } if addr == "127.0.0.1:4001"
1239        ));
1240    }
1241
1242    fn test_schema() -> Arc<Schema> {
1243        Arc::new(Schema::new(vec![ColumnSchema::new(
1244            "v",
1245            ConcreteDataType::int32_datatype(),
1246            false,
1247        )]))
1248    }
1249
1250    fn test_metrics_json() -> String {
1251        serde_json::to_string(&RecordBatchMetrics {
1252            elapsed_compute: 7,
1253            ..Default::default()
1254        })
1255        .unwrap()
1256    }
1257
1258    #[tokio::test]
1259    async fn test_partial_response_validation_and_compression() {
1260        let server = RegionTestServer::start().await;
1261        let requester = server.create_requester(ChannelConfig::default(), true, true);
1262        let mut request = build_insert_request();
1263        if let Some(region_request::Body::Inserts(inserts)) = &mut request.body {
1264            inserts.requests.push(inserts.requests[0].clone());
1265        }
1266        let request = Arc::new(with_partial_result(request));
1267        let failure = RegionRequestFailure {
1268            request_index: 0,
1269            status: Some(PbStatus {
1270                status_code: StatusCode::RegionNotFound as u32,
1271                err_msg: "moved".to_string(),
1272                retry_hint: "retry_after_route_refresh".to_string(),
1273            }),
1274        };
1275        for (failures, valid) in [
1276            (vec![], true),
1277            (vec![failure.clone()], true),
1278            (vec![failure.clone(), failure.clone()], false),
1279            (
1280                vec![RegionRequestFailure {
1281                    request_index: u32::MAX,
1282                    ..failure.clone()
1283                }],
1284                false,
1285            ),
1286            (
1287                vec![RegionRequestFailure {
1288                    request_index: 0,
1289                    status: None,
1290                }],
1291                false,
1292            ),
1293            (
1294                vec![RegionRequestFailure {
1295                    request_index: 0,
1296                    status: Some(PbStatus::default()),
1297                }],
1298                false,
1299            ),
1300            (
1301                vec![RegionRequestFailure {
1302                    request_index: 0,
1303                    status: Some(PbStatus {
1304                        status_code: u32::MAX,
1305                        ..Default::default()
1306                    }),
1307                }],
1308                false,
1309            ),
1310        ] {
1311            let has_error = !failures.is_empty();
1312            server
1313                .service
1314                .responses
1315                .lock()
1316                .unwrap()
1317                .push_back(Ok(PbRegionResponse {
1318                    header: Some(ResponseHeader {
1319                        status: Some(PbStatus {
1320                            status_code: if has_error {
1321                                StatusCode::PartialFailure
1322                            } else {
1323                                StatusCode::Success
1324                            } as u32,
1325                            ..Default::default()
1326                        }),
1327                    }),
1328                    affected_rows: 3,
1329                    extensions: std::collections::HashMap::from([("test".to_string(), vec![1, 2])]),
1330                    failures,
1331                    ..Default::default()
1332                }));
1333            let response = requester.handle_inner(Arc::clone(&request)).await;
1334            if valid {
1335                if has_error {
1336                    let error = response.unwrap_err();
1337                    assert!(error.output_msg().contains("entry 0"));
1338                    assert!(error.output_msg().contains("moved"));
1339                    let error: common_meta::error::Error = error.into();
1340                    assert!(error.output_msg().contains("moved"));
1341                    let common_meta::error::Error::PartialRegion { response } = error else {
1342                        panic!("Expected partial region failure");
1343                    };
1344                    assert_eq!(response.affected_rows, 3);
1345                    assert_eq!(response.extensions["test"], vec![1, 2]);
1346                    assert_eq!(
1347                        response.failures[0].1.retry_hint(),
1348                        RetryHint::RetryAfterRouteRefresh
1349                    );
1350                } else {
1351                    let response = response.unwrap();
1352                    assert_eq!(response.affected_rows, 3);
1353                    assert_eq!(response.extensions["test"], vec![1, 2]);
1354                }
1355            } else {
1356                assert!(matches!(
1357                    response.unwrap_err(),
1358                    Error::IllegalDatabaseResponse { .. }
1359                ));
1360            }
1361        }
1362        for (received, send, accept) in server.service.requests.lock().unwrap().iter() {
1363            assert_eq!(received, request.as_ref());
1364            assert!(*send && *accept);
1365        }
1366        assert_eq!(Arc::strong_count(&request), 1);
1367    }
1368
1369    #[test]
1370    fn test_partial_response_envelope() {
1371        let failure = RegionRequestFailure {
1372            request_index: 0,
1373            status: Some(PbStatus {
1374                status_code: StatusCode::RegionNotFound as u32,
1375                retry_hint: RetryHint::RetryAfterRouteRefresh.as_str().to_string(),
1376                ..Default::default()
1377            }),
1378        };
1379        for (status, failures, count) in [
1380            (Some(StatusCode::Success), vec![failure.clone()], Some(2)),
1381            (Some(StatusCode::PartialFailure), vec![], Some(2)),
1382            (
1383                Some(StatusCode::PartialFailure),
1384                vec![failure.clone()],
1385                None,
1386            ),
1387            (None, vec![failure.clone()], Some(2)),
1388        ] {
1389            let response = PbRegionResponse {
1390                header: status.map(|code| ResponseHeader {
1391                    status: Some(PbStatus {
1392                        status_code: code as u32,
1393                        ..Default::default()
1394                    }),
1395                }),
1396                failures,
1397                ..Default::default()
1398            };
1399            assert!(matches!(
1400                match count {
1401                    Some(count) => decode_write_response(response, count),
1402                    None => decode_region_response(response),
1403                },
1404                Err(Error::IllegalDatabaseResponse { .. })
1405            ));
1406        }
1407        let response = PbRegionResponse {
1408            header: Some(ResponseHeader {
1409                status: Some(PbStatus {
1410                    status_code: StatusCode::PartialFailure as u32,
1411                    ..Default::default()
1412                }),
1413            }),
1414            failures: vec![failure],
1415            ..Default::default()
1416        };
1417        // A partial envelope must not be interpreted as success by the header-only decoder.
1418        assert!(check_response_header(&response.header).is_err());
1419        let error = decode_write_response(response, 1).unwrap_err();
1420        assert_eq!(error.status_code(), StatusCode::RegionNotFound);
1421        assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
1422    }
1423
1424    #[test]
1425    fn test_partial_error_preserves_status_and_retry_hint() {
1426        let expected_message = "Region request has 2 failed entries; entry 0: first input failed";
1427        for code in [StatusCode::StorageUnavailable, StatusCode::InvalidArguments] {
1428            let response = PbRegionResponse {
1429                header: Some(ResponseHeader {
1430                    status: Some(PbStatus {
1431                        status_code: StatusCode::PartialFailure as u32,
1432                        ..Default::default()
1433                    }),
1434                }),
1435                failures: vec![
1436                    RegionRequestFailure {
1437                        request_index: 1,
1438                        status: Some(PbStatus {
1439                            status_code: StatusCode::Internal as u32,
1440                            err_msg: "later input failed".to_string(),
1441                            ..Default::default()
1442                        }),
1443                    },
1444                    RegionRequestFailure {
1445                        request_index: 0,
1446                        status: Some(PbStatus {
1447                            status_code: code as u32,
1448                            err_msg: "first input failed".to_string(),
1449                            retry_hint: RetryHint::NonRetryable.as_str().to_string(),
1450                        }),
1451                    },
1452                ],
1453                ..Default::default()
1454            };
1455            let error = decode_write_response(response, 2).unwrap_err();
1456            assert_eq!(code, error.status_code());
1457            assert_eq!(expected_message, error.to_string());
1458            assert_eq!(RetryHint::Retryable, error.retry_hint());
1459            let error: common_meta::error::Error = error.into();
1460            assert_eq!(code, error.status_code());
1461            assert_eq!(expected_message, error.to_string());
1462            assert_eq!(RetryHint::Retryable, error.retry_hint());
1463            let common_meta::error::Error::PartialRegion { response } = error else {
1464                panic!("expected partial result")
1465            };
1466            assert!(
1467                response
1468                    .failures
1469                    .iter()
1470                    .all(|(_, error)| error.retry_hint() == RetryHint::NonRetryable)
1471            );
1472        }
1473    }
1474
1475    #[tokio::test]
1476    async fn test_region_rpc_errors() {
1477        let server = RegionTestServer::start().await;
1478        let requester = server.create_requester(ChannelConfig::default(), false, false);
1479        let request = Arc::new(build_insert_request());
1480        let mut business_error = Status::unimplemented("unsupported write");
1481        business_error.metadata_mut().insert(
1482            common_error::GREPTIME_DB_HEADER_ERROR_CODE,
1483            (StatusCode::Unsupported as u32)
1484                .to_string()
1485                .parse()
1486                .unwrap(),
1487        );
1488        for status in [
1489            Status::unimplemented("unknown method"),
1490            business_error,
1491            Status::unavailable("lost response"),
1492            Status::deadline_exceeded("timeout"),
1493        ] {
1494            let code = status.code();
1495            server
1496                .service
1497                .responses
1498                .lock()
1499                .unwrap()
1500                .push_back(Err(status));
1501            let error = requester
1502                .handle_inner(Arc::clone(&request))
1503                .await
1504                .unwrap_err();
1505            assert!(matches!(error, Error::RegionServer { code: actual, .. } if actual == code));
1506        }
1507        assert_eq!(server.service.requests.lock().unwrap().len(), 4);
1508    }
1509
1510    #[test]
1511    fn test_check_response_header() {
1512        for (hint, expected) in [
1513            ("retryable", RetryHint::Retryable),
1514            (
1515                "retry_after_route_refresh",
1516                RetryHint::RetryAfterRouteRefresh,
1517            ),
1518            ("non_retryable", RetryHint::NonRetryable),
1519            ("", RetryHint::NonRetryable),
1520            ("future_hint", RetryHint::NonRetryable),
1521        ] {
1522            let error = check_response_header(&Some(ResponseHeader {
1523                status: Some(PbStatus {
1524                    status_code: StatusCode::RegionNotFound as u32,
1525                    err_msg: "region moved".to_string(),
1526                    retry_hint: hint.to_string(),
1527                }),
1528            }))
1529            .unwrap_err();
1530            assert_eq!(error.retry_hint(), expected, "{hint}");
1531        }
1532
1533        let result = check_response_header(&None);
1534        assert!(matches!(
1535            result.unwrap_err(),
1536            Error::IllegalDatabaseResponse { .. }
1537        ));
1538
1539        let result = check_response_header(&Some(ResponseHeader { status: None }));
1540        assert!(matches!(
1541            result.unwrap_err(),
1542            Error::IllegalDatabaseResponse { .. }
1543        ));
1544
1545        let result = check_response_header(&Some(ResponseHeader {
1546            status: Some(PbStatus {
1547                status_code: StatusCode::Success as u32,
1548                err_msg: String::default(),
1549                ..Default::default()
1550            }),
1551        }));
1552        assert!(result.is_ok());
1553
1554        let result = check_response_header(&Some(ResponseHeader {
1555            status: Some(PbStatus {
1556                status_code: u32::MAX,
1557                err_msg: String::default(),
1558                ..Default::default()
1559            }),
1560        }));
1561        assert!(matches!(
1562            result.unwrap_err(),
1563            Error::IllegalDatabaseResponse { .. }
1564        ));
1565
1566        let result = check_response_header(&Some(ResponseHeader {
1567            status: Some(PbStatus {
1568                status_code: StatusCode::Internal as u32,
1569                err_msg: "blabla".to_string(),
1570                ..Default::default()
1571            }),
1572        }));
1573        let Error::Server { code, msg, .. } = result.unwrap_err() else {
1574            unreachable!()
1575        };
1576        assert_eq!(code, StatusCode::Internal);
1577        assert_eq!(msg, "blabla");
1578    }
1579
1580    #[test]
1581    fn test_build_remote_dyn_filter_request_sets_header_and_body() {
1582        let request = build_remote_dyn_filter_update_request(
1583            "query-1",
1584            RemoteDynFilterUpdate {
1585                filter_id: "filter-1".to_string(),
1586                payload: vec![1, 2, 3],
1587                generation: 7,
1588                is_complete: false,
1589            },
1590        );
1591
1592        request.header.expect("remote dyn filter header must exist");
1593
1594        let body = request.body.expect("remote dyn filter body must exist");
1595        let region_request::Body::RemoteDynFilter(remote_request) = body else {
1596            panic!("expected remote dyn filter request body");
1597        };
1598
1599        assert_eq!(remote_request.query_id, "query-1");
1600        assert!(matches!(
1601            remote_request.action,
1602            Some(remote_dyn_filter_request::Action::Update(_))
1603        ));
1604    }
1605
1606    #[test]
1607    fn test_build_remote_dyn_filter_unregister_request_sets_header_and_body() {
1608        let request = build_remote_dyn_filter_unregister_request(
1609            "query-1",
1610            RemoteDynFilterUnregister {
1611                filter_id: "filter-9".to_string(),
1612            },
1613        );
1614
1615        request.header.expect("remote dyn filter header must exist");
1616
1617        let body = request.body.expect("remote dyn filter body must exist");
1618        let region_request::Body::RemoteDynFilter(remote_request) = body else {
1619            panic!("expected remote dyn filter request body");
1620        };
1621
1622        assert_eq!(remote_request.query_id, "query-1");
1623        assert!(matches!(
1624            remote_request.action,
1625            Some(remote_dyn_filter_request::Action::Unregister(_))
1626        ));
1627    }
1628
1629    #[tokio::test]
1630    async fn test_record_batch_stream_continues_after_pre_batch_metrics() {
1631        let schema = test_schema();
1632        let batch = RecordBatch::new(
1633            schema.clone(),
1634            vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1635        )
1636        .unwrap();
1637
1638        let mut recordbatches = recordbatches_from_flight_message_stream(
1639            "test-peer".to_string(),
1640            stream::iter(vec![
1641                Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1642                Ok(FlightMessage::Metrics(test_metrics_json())),
1643                Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())),
1644            ]),
1645        )
1646        .await
1647        .unwrap();
1648
1649        let batch = recordbatches.next().await.unwrap().unwrap();
1650        assert_eq!(batch.num_rows(), 1);
1651        assert!(recordbatches.next().await.is_none());
1652
1653        let metrics = recordbatches.metrics().unwrap();
1654        assert_eq!(metrics.elapsed_compute, 7);
1655    }
1656
1657    #[tokio::test]
1658    async fn test_record_batch_is_yielded_without_waiting_for_next_message() {
1659        let schema = test_schema();
1660        let batch = RecordBatch::new(
1661            schema.clone(),
1662            vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1663        )
1664        .unwrap();
1665
1666        let messages = stream::iter(vec![
1667            Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1668            Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())),
1669        ])
1670        .chain(stream::pending::<Result<FlightMessage>>());
1671        let mut recordbatches =
1672            recordbatches_from_flight_message_stream("test-peer".to_string(), messages)
1673                .await
1674                .unwrap();
1675
1676        let batch = tokio::time::timeout(Duration::from_secs(1), recordbatches.next())
1677            .await
1678            .expect("the first batch must not wait for lookahead")
1679            .unwrap()
1680            .unwrap();
1681        assert_eq!(batch.num_rows(), 1);
1682    }
1683
1684    #[tokio::test]
1685    async fn test_malformed_region_metrics_are_non_fatal() {
1686        let schema = test_schema();
1687        let batch = RecordBatch::new(
1688            schema.clone(),
1689            vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1690        )
1691        .unwrap();
1692        let mut recordbatches = recordbatches_from_flight_message_stream(
1693            "test-peer".to_string(),
1694            stream::iter(vec![
1695                Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1696                Ok(FlightMessage::Metrics("{not-json}".to_string())),
1697                Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())),
1698            ]),
1699        )
1700        .await
1701        .unwrap();
1702
1703        assert_eq!(recordbatches.next().await.unwrap().unwrap().num_rows(), 1);
1704        assert!(recordbatches.next().await.is_none());
1705    }
1706
1707    #[tokio::test]
1708    async fn test_record_batch_stream_preserves_following_record_batch() {
1709        let schema = test_schema();
1710        let first_batch = RecordBatch::new(
1711            schema.clone(),
1712            vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1713        )
1714        .unwrap();
1715        let second_batch = RecordBatch::new(
1716            schema.clone(),
1717            vec![Arc::new(Int32Vector::from_slice([2])) as VectorRef],
1718        )
1719        .unwrap();
1720
1721        let mut recordbatches = recordbatches_from_flight_message_stream(
1722            "test-peer".to_string(),
1723            stream::iter(vec![
1724                Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1725                Ok(FlightMessage::RecordBatch(
1726                    first_batch.into_df_record_batch(),
1727                )),
1728                Ok(FlightMessage::RecordBatch(
1729                    second_batch.into_df_record_batch(),
1730                )),
1731            ]),
1732        )
1733        .await
1734        .unwrap();
1735
1736        let first_batch = recordbatches.next().await.unwrap().unwrap();
1737        let second_batch = recordbatches.next().await.unwrap().unwrap();
1738        assert_eq!(
1739            first_batch
1740                .column(0)
1741                .as_any()
1742                .downcast_ref::<Int32Array>()
1743                .unwrap()
1744                .value(0),
1745            1
1746        );
1747        assert_eq!(
1748            second_batch
1749                .column(0)
1750                .as_any()
1751                .downcast_ref::<Int32Array>()
1752                .unwrap()
1753                .value(0),
1754            2
1755        );
1756        assert!(recordbatches.next().await.is_none());
1757    }
1758
1759    #[tokio::test]
1760    async fn test_record_batch_stream_captures_final_metrics_after_record_batch() {
1761        let schema = test_schema();
1762        let batch = RecordBatch::new(
1763            schema.clone(),
1764            vec![Arc::new(Int32Vector::from_slice([1])) as VectorRef],
1765        )
1766        .unwrap();
1767        let final_metrics = serde_json::to_string(&RecordBatchMetrics {
1768            elapsed_compute: 99,
1769            ..Default::default()
1770        })
1771        .unwrap();
1772        let mut recordbatches = recordbatches_from_flight_message_stream(
1773            "test-peer".to_string(),
1774            stream::iter(vec![
1775                Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1776                Ok(FlightMessage::RecordBatch(batch.into_df_record_batch())),
1777                Ok(FlightMessage::Metrics(final_metrics)),
1778            ]),
1779        )
1780        .await
1781        .unwrap();
1782
1783        assert_eq!(recordbatches.next().await.unwrap().unwrap().num_rows(), 1);
1784        assert!(recordbatches.next().await.is_none());
1785        assert_eq!(recordbatches.metrics().unwrap().elapsed_compute, 99);
1786    }
1787
1788    #[tokio::test]
1789    async fn test_record_batch_stream_exposes_error_after_pre_batch_metrics() {
1790        let schema = test_schema();
1791        let mut recordbatches = recordbatches_from_flight_message_stream(
1792            "test-peer".to_string(),
1793            stream::iter(vec![
1794                Ok(FlightMessage::Schema(schema.arrow_schema().clone())),
1795                Ok(FlightMessage::Metrics(test_metrics_json())),
1796                Err(Error::from(Status::internal("boom after metrics"))),
1797            ]),
1798        )
1799        .await
1800        .unwrap();
1801
1802        let err = recordbatches.next().await.unwrap().unwrap_err();
1803        // The failure keeps its message: a client reading the error learns what went wrong, not
1804        // just that a stream failed.
1805        assert!(
1806            err.to_string().starts_with("External error: "),
1807            "unexpected error: {err}"
1808        );
1809        assert!(
1810            err.to_string().contains("boom after metrics"),
1811            "unexpected error: {err}"
1812        );
1813        assert!(recordbatches.next().await.is_none());
1814    }
1815}