Skip to main content

meta_srv/procedure/region_migration/
downgrade_leader_region.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::time::Duration;
17
18use api::v1::meta::MailboxMessage;
19use common_error::ext::BoxedError;
20use common_meta::distributed_time_constants::default_distributed_time_constants;
21use common_meta::instruction::{
22    DowngradeRegion, DowngradeRegionReply, DowngradeRegionsReply, Instruction, InstructionReply,
23};
24use common_procedure::{Context as ProcedureContext, Status};
25use common_telemetry::tracing_context::TracingContext;
26use common_telemetry::{debug, error, info, warn};
27use common_time::util::current_time_millis;
28use serde::{Deserialize, Serialize};
29use snafu::{OptionExt, ResultExt};
30use tokio::time::{Instant, sleep};
31
32use crate::discovery::utils::find_datanode_lease_value;
33use crate::error::{self, Result};
34use crate::handler::HeartbeatMailbox;
35use crate::procedure::region_migration::update_metadata::UpdateMetadata;
36use crate::procedure::region_migration::upgrade_candidate_region::UpgradeCandidateRegion;
37use crate::procedure::region_migration::{Context, State};
38use crate::procedure::utils::instruction_error_result;
39use crate::service::mailbox::Channel;
40
41#[derive(Debug, Serialize, Deserialize)]
42pub struct DowngradeLeaderRegion {
43    // The optimistic retry times.
44    optimistic_retry: usize,
45    // The retry initial interval.
46    retry_initial_interval: Duration,
47}
48
49impl Default for DowngradeLeaderRegion {
50    fn default() -> Self {
51        Self {
52            optimistic_retry: 3,
53            retry_initial_interval: Duration::from_millis(500),
54        }
55    }
56}
57
58#[async_trait::async_trait]
59#[typetag::serde]
60impl State for DowngradeLeaderRegion {
61    async fn next(
62        &mut self,
63        ctx: &mut Context,
64        _procedure_ctx: &ProcedureContext,
65    ) -> Result<(Box<dyn State>, Status)> {
66        let now = Instant::now();
67        // Ensures the `leader_region_lease_deadline` must exist after recovering.
68        ctx.volatile_ctx
69            .set_leader_region_lease_deadline(default_distributed_time_constants().region_lease);
70
71        match self.downgrade_region_with_retry(ctx).await {
72            Ok(_) => {
73                // Do nothing
74                info!(
75                    "Downgraded region leader success, region: {:?}",
76                    ctx.persistent_ctx.region_ids
77                );
78            }
79            Err(error::Error::ExceededDeadline { .. }) => {
80                info!(
81                    "Downgrade region leader exceeded deadline, region: {:?}",
82                    ctx.persistent_ctx.region_ids
83                );
84                // Rollbacks the metadata if procedure is timeout
85                return Ok((Box::new(UpdateMetadata::Rollback), Status::executing(false)));
86            }
87            Err(err) => {
88                error!(err; "Occurs non-retryable error, region: {:?}", ctx.persistent_ctx.region_ids);
89                if let Some(deadline) = ctx.volatile_ctx.leader_region_lease_deadline.as_ref() {
90                    info!(
91                        "Running into the downgrade region leader slow path, region: {:?}, sleep until {:?}",
92                        ctx.persistent_ctx.region_ids, deadline
93                    );
94                    tokio::time::sleep_until(*deadline).await;
95                } else {
96                    warn!(
97                        "Leader region lease deadline is not set, region: {:?}",
98                        ctx.persistent_ctx.region_ids
99                    );
100                }
101            }
102        }
103        ctx.update_downgrade_leader_region_elapsed(now);
104
105        Ok((
106            Box::new(UpgradeCandidateRegion::default()),
107            Status::executing(false),
108        ))
109    }
110
111    fn as_any(&self) -> &dyn Any {
112        self
113    }
114}
115
116impl DowngradeLeaderRegion {
117    /// Builds downgrade region instruction.
118    fn build_downgrade_region_instruction(
119        &self,
120        ctx: &Context,
121        flush_timeout: Duration,
122    ) -> Instruction {
123        let region_ids = &ctx.persistent_ctx.region_ids;
124        let mut downgrade_regions = Vec::with_capacity(region_ids.len());
125        for region_id in region_ids {
126            downgrade_regions.push(DowngradeRegion {
127                region_id: *region_id,
128                flush_timeout: Some(flush_timeout),
129            });
130        }
131
132        Instruction::DowngradeRegions(downgrade_regions)
133    }
134
135    fn handle_downgrade_region_reply(
136        &self,
137        ctx: &mut Context,
138        reply: &DowngradeRegionReply,
139        now: &Instant,
140    ) -> Result<()> {
141        let leader = &ctx.persistent_ctx.from_peer;
142        let DowngradeRegionReply {
143            region_id,
144            last_entry_id,
145            metadata_last_entry_id,
146            exists,
147            error,
148        } = reply;
149
150        if let Some(error) = error {
151            return instruction_error_result(
152                error,
153                format!(
154                    "Failed to downgrade the region {} on datanode {:?}, error: {:?}, elapsed: {:?}",
155                    region_id,
156                    leader,
157                    error,
158                    now.elapsed()
159                ),
160            );
161        }
162
163        // A closing region is no longer found on the datanode while its in-flight
164        // manifest updates may still be written, so a missing region does not prove
165        // that the leader has stopped.
166        if !exists {
167            return error::RetryLaterSnafu {
168                reason: format!(
169                    "Region {} doesn't exist on datanode {:?}, elapsed: {:?}",
170                    region_id,
171                    leader,
172                    now.elapsed()
173                ),
174            }
175            .fail();
176        }
177
178        info!(
179            "Region {} leader is downgraded on datanode {:?}, last_entry_id: {:?}, metadata_last_entry_id: {:?}, elapsed: {:?}",
180            region_id,
181            leader,
182            last_entry_id,
183            metadata_last_entry_id,
184            now.elapsed()
185        );
186
187        if let Some(last_entry_id) = last_entry_id {
188            debug!(
189                "set last_entry_id: {:?}, region_id: {:?}",
190                last_entry_id, region_id
191            );
192            ctx.volatile_ctx
193                .set_last_entry_id(*region_id, *last_entry_id);
194        }
195
196        if let Some(metadata_last_entry_id) = metadata_last_entry_id {
197            ctx.volatile_ctx
198                .set_metadata_last_entry_id(*region_id, *metadata_last_entry_id);
199        }
200
201        Ok(())
202    }
203
204    /// Tries to downgrade a leader region.
205    ///
206    /// Retry:
207    /// - [MailboxTimeout](error::Error::MailboxTimeout), Timeout.
208    /// - The Datanode replies with a retryable error.
209    /// - The Datanode replies, without an error, that the region doesn't exist. A closing
210    ///   region leaves the region map while its manifest write may still be in flight.
211    ///
212    /// Abort:
213    /// - The Datanode replies with a non-retryable error.
214    /// - [PusherNotFound](error::Error::PusherNotFound), The datanode is unreachable.
215    /// - [PushMessage](error::Error::PushMessage), The receiver is dropped.
216    /// - [MailboxReceiver](error::Error::MailboxReceiver), The sender is dropped without sending (impossible).
217    /// - [UnexpectedInstructionReply](error::Error::UnexpectedInstructionReply).
218    /// - [ExceededDeadline](error::Error::ExceededDeadline)
219    /// - Invalid JSON.
220    async fn downgrade_region(&self, ctx: &mut Context) -> Result<()> {
221        let region_ids = &ctx.persistent_ctx.region_ids;
222        let operation_timeout =
223            ctx.next_operation_timeout()
224                .context(error::ExceededDeadlineSnafu {
225                    operation: "Downgrade region",
226                })?;
227        let downgrade_instruction = self.build_downgrade_region_instruction(ctx, operation_timeout);
228
229        let leader = &ctx.persistent_ctx.from_peer;
230        let tracing_ctx = TracingContext::from_current_span();
231        let msg = MailboxMessage::json_message(
232            &format!("Downgrade leader regions: {:?}", region_ids),
233            &format!("Metasrv@{}", ctx.server_addr()),
234            &format!("Datanode-{}@{}", leader.id, leader.addr),
235            common_time::util::current_time_millis(),
236            &downgrade_instruction,
237            Some(tracing_ctx.to_w3c()),
238        )
239        .with_context(|_| error::SerializeToJsonSnafu {
240            input: downgrade_instruction.to_string(),
241        })?;
242
243        let ch = Channel::Datanode(leader.id);
244        let now = Instant::now();
245        let receiver = ctx.mailbox.send(&ch, msg, operation_timeout).await?;
246
247        match receiver.await {
248            Ok(msg) => {
249                let reply = HeartbeatMailbox::json_reply(&msg)?;
250                info!(
251                    "Received downgrade region reply: {:?}, region: {:?}, elapsed: {:?}",
252                    reply,
253                    region_ids,
254                    now.elapsed()
255                );
256                let InstructionReply::DowngradeRegions(DowngradeRegionsReply { replies }) = reply
257                else {
258                    return error::UnexpectedInstructionReplySnafu {
259                        mailbox_message: msg.to_string(),
260                        reason: "expect downgrade region reply",
261                    }
262                    .fail();
263                };
264
265                let mut result = Ok(());
266                for reply in replies {
267                    let handled = self.handle_downgrade_region_reply(ctx, &reply, &now);
268                    // The first failure in reply order decides the retry; later replies
269                    // are still handled so that their last entry ids are recorded.
270                    if result.is_ok() {
271                        result = handled;
272                    }
273                }
274                result
275            }
276            Err(error::Error::MailboxTimeout { .. }) => {
277                let reason = format!(
278                    "Mailbox received timeout for downgrade leader region {region_ids:?} on datanode {:?}, elapsed: {:?}",
279                    leader,
280                    now.elapsed()
281                );
282                error::RetryLaterSnafu { reason }.fail()
283            }
284            Err(err) => Err(err),
285        }
286    }
287
288    async fn update_leader_region_lease_deadline(&self, ctx: &mut Context) {
289        let leader = &ctx.persistent_ctx.from_peer;
290
291        let last_connection_at = match find_datanode_lease_value(&ctx.in_memory, leader.id).await {
292            Ok(lease_value) => lease_value.map(|lease_value| lease_value.timestamp_millis),
293            Err(err) => {
294                error!(err; "Failed to find datanode lease value for datanode: {}, during region migration, region: {:?}", leader, ctx.persistent_ctx.region_ids);
295                return;
296            }
297        };
298
299        if let Some(last_connection_at) = last_connection_at {
300            let now = current_time_millis();
301            let elapsed = now - last_connection_at;
302            let region_lease = default_distributed_time_constants().region_lease;
303
304            // It's safe to update the region leader lease deadline here because:
305            // 1. The old region leader has already been marked as downgraded in metadata,
306            //    which means any attempts to renew its lease will be rejected.
307            // 2. The pusher disconnect time record only gets removed when the datanode (from_peer)
308            //    establishes a new heartbeat connection stream.
309            if elapsed >= (region_lease.as_secs() * 1000) as i64 {
310                ctx.volatile_ctx.reset_leader_region_lease_deadline();
311                info!(
312                    "Datanode {}({}) has been disconnected for longer than the region lease period ({:?}), reset leader region lease deadline to None, region: {:?}",
313                    leader, last_connection_at, region_lease, ctx.persistent_ctx.region_ids
314                );
315            } else if elapsed > 0 {
316                // `now - last_connection_at` < REGION_LEASE_SECS * 1000
317                let lease_timeout =
318                    region_lease - Duration::from_millis((now - last_connection_at) as u64);
319                ctx.volatile_ctx.reset_leader_region_lease_deadline();
320                ctx.volatile_ctx
321                    .set_leader_region_lease_deadline(lease_timeout);
322                info!(
323                    "Datanode {}({}) last connected {:?} ago, updated leader region lease deadline to {:?}, region: {:?}",
324                    leader,
325                    last_connection_at,
326                    elapsed,
327                    ctx.volatile_ctx.leader_region_lease_deadline,
328                    ctx.persistent_ctx.region_ids
329                );
330            } else {
331                warn!(
332                    "Datanode {} has invalid last connection timestamp: {} (which is after current time: {}), region: {:?}",
333                    leader, last_connection_at, now, ctx.persistent_ctx.region_ids
334                )
335            }
336        } else {
337            warn!(
338                "Failed to find last connection time for datanode {}, unable to update region lease deadline, region: {:?}",
339                leader, ctx.persistent_ctx.region_ids
340            )
341        }
342    }
343
344    /// Downgrades a leader region.
345    ///
346    /// Fast path:
347    /// - Waits for the reply of downgrade instruction.
348    ///
349    /// Slow path:
350    /// - Waits for the lease of the leader region expired.
351    ///
352    /// Abort:
353    /// - ExceededDeadline
354    async fn downgrade_region_with_retry(&self, ctx: &mut Context) -> Result<()> {
355        let mut retry = 0;
356
357        loop {
358            let timer = Instant::now();
359            if let Err(err) = self.downgrade_region(ctx).await {
360                ctx.update_operations_elapsed(timer);
361                retry += 1;
362                // Throws the error immediately if the procedure exceeded the deadline.
363                if matches!(err, error::Error::ExceededDeadline { .. }) {
364                    error!(err; "Failed to downgrade region leader, regions: {:?}, exceeded deadline", ctx.persistent_ctx.region_ids);
365                    return Err(err);
366                } else if matches!(err, error::Error::PusherNotFound { .. }) {
367                    // Throws the error immediately if the datanode is unreachable.
368                    error!(err; "Failed to downgrade region leader, regions: {:?}, datanode({}) is unreachable(PusherNotFound)", ctx.persistent_ctx.region_ids, ctx.persistent_ctx.from_peer.id);
369                    self.update_leader_region_lease_deadline(ctx).await;
370                    return Err(err);
371                } else if err.is_retryable() && retry < self.optimistic_retry {
372                    error!(err; "Failed to downgrade region leader, regions: {:?}, retry later", ctx.persistent_ctx.region_ids);
373                    sleep(self.retry_initial_interval).await;
374                } else {
375                    return Err(BoxedError::new(err)).context(error::DowngradeLeaderSnafu {
376                        // TODO(weny): handle multiple regions.
377                        region_id: ctx.persistent_ctx.region_ids[0],
378                    })?;
379                }
380            } else {
381                ctx.update_operations_elapsed(timer);
382                // Resets the deadline.
383                ctx.volatile_ctx.reset_leader_region_lease_deadline();
384                break;
385            }
386        }
387
388        Ok(())
389    }
390}
391
392#[cfg(test)]
393mod tests {
394    use std::assert_matches;
395    use std::collections::HashMap;
396    use std::sync::Arc;
397    use std::sync::atomic::{AtomicUsize, Ordering};
398
399    use common_error::ext::RetryHint;
400    use common_error::status_code::StatusCode;
401    use common_meta::instruction::InstructionError;
402    use common_meta::key::table_route::TableRouteValue;
403    use common_meta::key::test_utils::new_test_table_info;
404    use common_meta::peer::Peer;
405    use common_meta::rpc::router::{Region, RegionRoute};
406    use common_meta::wal_provider::RegionWalOptions;
407    use store_api::storage::RegionId;
408    use tokio::time::Instant;
409
410    use super::*;
411    use crate::error::Error;
412    use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context};
413    use crate::procedure::region_migration::{
414        ContextFactory, PersistentContext, RegionMigrationTriggerReason,
415    };
416    use crate::procedure::test_util::{
417        new_close_region_reply, new_downgrade_region_reply, new_downgrade_regions_reply,
418        send_mock_reply,
419    };
420
421    fn new_persistent_context() -> PersistentContext {
422        PersistentContext::new(
423            vec![("greptime".into(), "public".into())],
424            Peer::empty(1),
425            Peer::empty(2),
426            vec![RegionId::new(1024, 1)],
427            Duration::from_millis(1000),
428            RegionMigrationTriggerReason::Unknown,
429        )
430    }
431
432    async fn prepare_table_metadata(ctx: &Context, wal_options: RegionWalOptions) {
433        let region_id = ctx.persistent_ctx.region_ids[0];
434        let table_info = new_test_table_info(region_id.table_id());
435        let region_routes = vec![RegionRoute {
436            region: Region::new_test(region_id),
437            leader_peer: Some(ctx.persistent_ctx.from_peer.clone()),
438            follower_peers: vec![ctx.persistent_ctx.to_peer.clone()],
439            ..Default::default()
440        }];
441        ctx.table_metadata_manager
442            .create_table_metadata(
443                table_info,
444                TableRouteValue::physical(region_routes),
445                wal_options,
446            )
447            .await
448            .unwrap();
449    }
450
451    #[tokio::test]
452    async fn test_datanode_is_unreachable() {
453        let state = DowngradeLeaderRegion::default();
454        let persistent_context = new_persistent_context();
455        let env = TestingEnv::new();
456        let mut ctx = env.context_factory().new_context(persistent_context);
457        prepare_table_metadata(&ctx, HashMap::default()).await;
458        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
459
460        assert_matches!(err, Error::PusherNotFound { .. });
461        assert!(!err.is_retryable());
462    }
463
464    #[tokio::test]
465    async fn test_pusher_dropped() {
466        let state = DowngradeLeaderRegion::default();
467        let persistent_context = new_persistent_context();
468        let from_peer_id = persistent_context.from_peer.id;
469
470        let mut env = TestingEnv::new();
471        let mut ctx = env.context_factory().new_context(persistent_context);
472        prepare_table_metadata(&ctx, HashMap::default()).await;
473        let mailbox_ctx = env.mailbox_context();
474
475        let (tx, rx) = tokio::sync::mpsc::channel(1);
476
477        mailbox_ctx
478            .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
479            .await;
480
481        drop(rx);
482
483        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
484
485        assert_matches!(err, Error::PushMessage { .. });
486        assert!(!err.is_retryable());
487    }
488
489    #[tokio::test]
490    async fn test_procedure_exceeded_deadline() {
491        let state = DowngradeLeaderRegion::default();
492        let persistent_context = new_persistent_context();
493        let env = TestingEnv::new();
494        let mut ctx = env.context_factory().new_context(persistent_context);
495        prepare_table_metadata(&ctx, HashMap::default()).await;
496        ctx.volatile_ctx.metrics.operations_elapsed =
497            ctx.persistent_ctx.timeout + Duration::from_secs(1);
498
499        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
500
501        assert_matches!(err, Error::ExceededDeadline { .. });
502        assert!(!err.is_retryable());
503
504        let err = state
505            .downgrade_region_with_retry(&mut ctx)
506            .await
507            .unwrap_err();
508        assert_matches!(err, Error::ExceededDeadline { .. });
509        assert!(!err.is_retryable());
510    }
511
512    #[tokio::test]
513    async fn test_unexpected_instruction_reply() {
514        let state = DowngradeLeaderRegion::default();
515        let persistent_context = new_persistent_context();
516        let from_peer_id = persistent_context.from_peer.id;
517
518        let mut env = TestingEnv::new();
519        let mut ctx = env.context_factory().new_context(persistent_context);
520        prepare_table_metadata(&ctx, HashMap::default()).await;
521        let mailbox_ctx = env.mailbox_context();
522        let mailbox = mailbox_ctx.mailbox().clone();
523
524        let (tx, rx) = tokio::sync::mpsc::channel(1);
525
526        mailbox_ctx
527            .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
528            .await;
529
530        // Sends an incorrect reply.
531        send_mock_reply(mailbox, rx, |id| Ok(new_close_region_reply(id)));
532
533        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
534
535        assert_matches!(err, Error::UnexpectedInstructionReply { .. });
536        assert!(!err.is_retryable());
537    }
538
539    #[tokio::test]
540    async fn test_instruction_exceeded_deadline() {
541        let state = DowngradeLeaderRegion::default();
542        let persistent_context = new_persistent_context();
543        let from_peer_id = persistent_context.from_peer.id;
544
545        let mut env = TestingEnv::new();
546        let mut ctx = env.context_factory().new_context(persistent_context);
547        prepare_table_metadata(&ctx, HashMap::default()).await;
548        let mailbox_ctx = env.mailbox_context();
549        let mailbox = mailbox_ctx.mailbox().clone();
550
551        let (tx, rx) = tokio::sync::mpsc::channel(1);
552
553        mailbox_ctx
554            .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
555            .await;
556
557        send_mock_reply(mailbox, rx, |id| {
558            Err(error::MailboxTimeoutSnafu { id }.build())
559        });
560
561        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
562
563        assert_matches!(err, Error::RetryLater { .. });
564        assert!(err.is_retryable());
565    }
566
567    #[tokio::test]
568    async fn test_downgrade_region_failed() {
569        let state = DowngradeLeaderRegion::default();
570        let persistent_context = new_persistent_context();
571        let from_peer_id = persistent_context.from_peer.id;
572
573        let mut env = TestingEnv::new();
574        let mut ctx = env.context_factory().new_context(persistent_context);
575        prepare_table_metadata(&ctx, HashMap::default()).await;
576        let mailbox_ctx = env.mailbox_context();
577        let mailbox = mailbox_ctx.mailbox().clone();
578
579        let (tx, rx) = tokio::sync::mpsc::channel(1);
580
581        mailbox_ctx
582            .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
583            .await;
584
585        send_mock_reply(mailbox, rx, |id| {
586            Ok(new_downgrade_region_reply(
587                id,
588                None,
589                false,
590                Some("test mocked".to_string()),
591            ))
592        });
593
594        let err = state.downgrade_region(&mut ctx).await.unwrap_err();
595
596        assert_matches!(err, Error::RetryLater { .. });
597        assert!(err.is_retryable());
598        assert!(format!("{err:?}").contains("test mocked"), "err: {err:?}",);
599    }
600
601    #[tokio::test]
602    async fn test_downgrade_region_returns_first_failure_of_batch() {
603        let missing = DowngradeRegionReply {
604            region_id: RegionId::new(1024, 1),
605            last_entry_id: None,
606            metadata_last_entry_id: None,
607            exists: false,
608            error: None,
609        };
610        let failed = DowngradeRegionReply {
611            region_id: RegionId::new(1024, 2),
612            last_entry_id: None,
613            metadata_last_entry_id: None,
614            exists: true,
615            error: Some(InstructionError::new(
616                StatusCode::Internal,
617                "non retryable mocked",
618                RetryHint::NonRetryable,
619            )),
620        };
621        let downgraded = DowngradeRegionReply {
622            region_id: RegionId::new(1024, 3),
623            last_entry_id: Some(1),
624            metadata_last_entry_id: None,
625            exists: true,
626            error: None,
627        };
628
629        for (replies, retryable) in [
630            (
631                vec![missing.clone(), failed.clone(), downgraded.clone()],
632                true,
633            ),
634            (vec![failed, missing, downgraded.clone()], false),
635        ] {
636            let state = DowngradeLeaderRegion::default();
637            let persistent_context = new_persistent_context();
638            let from_peer_id = persistent_context.from_peer.id;
639
640            let mut env = TestingEnv::new();
641            let mut ctx = env.context_factory().new_context(persistent_context);
642            prepare_table_metadata(&ctx, HashMap::default()).await;
643            let mailbox_ctx = env.mailbox_context();
644            let mailbox = mailbox_ctx.mailbox().clone();
645
646            let (tx, rx) = tokio::sync::mpsc::channel(1);
647
648            mailbox_ctx
649                .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
650                .await;
651
652            send_mock_reply(mailbox, rx, move |id| {
653                Ok(new_downgrade_regions_reply(id, replies.clone()))
654            });
655
656            let err = state.downgrade_region(&mut ctx).await.unwrap_err();
657
658            assert_eq!(err.is_retryable(), retryable, "err: {err:?}");
659            assert_eq!(
660                ctx.volatile_ctx.leader_region_last_entry_ids,
661                HashMap::from([(downgraded.region_id, 1)])
662            );
663        }
664    }
665
666    #[tokio::test]
667    async fn test_downgrade_region_with_retry_fast_path() {
668        let state = DowngradeLeaderRegion::default();
669        let persistent_context = new_persistent_context();
670        let from_peer_id = persistent_context.from_peer.id;
671
672        let mut env = TestingEnv::new();
673        let mut ctx = env.context_factory().new_context(persistent_context);
674        prepare_table_metadata(&ctx, HashMap::default()).await;
675        let mailbox_ctx = env.mailbox_context();
676        let mailbox = mailbox_ctx.mailbox().clone();
677
678        let (tx, mut rx) = tokio::sync::mpsc::channel(1);
679
680        mailbox_ctx
681            .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
682            .await;
683
684        common_runtime::spawn_global(async move {
685            // retry: 0.
686            let resp = rx.recv().await.unwrap().unwrap();
687            let reply_id = resp.mailbox_message.unwrap().id;
688            mailbox
689                .on_recv(
690                    reply_id,
691                    Err(error::MailboxTimeoutSnafu { id: reply_id }.build()),
692                )
693                .await
694                .unwrap();
695
696            // retry: 1.
697            let resp = rx.recv().await.unwrap().unwrap();
698            let reply_id = resp.mailbox_message.unwrap().id;
699            mailbox
700                .on_recv(
701                    reply_id,
702                    Ok(new_downgrade_region_reply(reply_id, Some(1), true, None)),
703                )
704                .await
705                .unwrap();
706        });
707
708        state.downgrade_region_with_retry(&mut ctx).await.unwrap();
709        assert_eq!(
710            ctx.volatile_ctx
711                .leader_region_last_entry_ids
712                .get(&RegionId::new(0, 0))
713                .cloned(),
714            Some(1)
715        );
716        assert!(ctx.volatile_ctx.leader_region_lease_deadline.is_none());
717    }
718
719    #[tokio::test]
720    async fn test_downgrade_region_with_retry_slow_path() {
721        let state = DowngradeLeaderRegion {
722            optimistic_retry: 3,
723            retry_initial_interval: Duration::from_millis(100),
724        };
725        let persistent_context = new_persistent_context();
726        let from_peer_id = persistent_context.from_peer.id;
727
728        let mut env = TestingEnv::new();
729        let mut ctx = env.context_factory().new_context(persistent_context);
730        let mailbox_ctx = env.mailbox_context();
731        let mailbox = mailbox_ctx.mailbox().clone();
732
733        let (tx, mut rx) = tokio::sync::mpsc::channel(1);
734
735        mailbox_ctx
736            .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
737            .await;
738
739        common_runtime::spawn_global(async move {
740            for _ in 0..3 {
741                let resp = rx.recv().await.unwrap().unwrap();
742                let reply_id = resp.mailbox_message.unwrap().id;
743                mailbox
744                    .on_recv(
745                        reply_id,
746                        Err(error::MailboxTimeoutSnafu { id: reply_id }.build()),
747                    )
748                    .await
749                    .unwrap();
750            }
751        });
752
753        ctx.volatile_ctx
754            .set_leader_region_lease_deadline(Duration::from_secs(5));
755        let expected_deadline = ctx.volatile_ctx.leader_region_lease_deadline.unwrap();
756        let err = state
757            .downgrade_region_with_retry(&mut ctx)
758            .await
759            .unwrap_err();
760        assert_matches!(err, error::Error::DowngradeLeader { .. });
761        // assert_eq!(ctx.volatile_ctx.leader_region_last_entry_id, None);
762        // Should remain no change.
763        assert_eq!(
764            ctx.volatile_ctx.leader_region_lease_deadline.unwrap(),
765            expected_deadline
766        )
767    }
768
769    #[tokio::test]
770    async fn test_next_upgrade_candidate_state() {
771        let mut state = Box::<DowngradeLeaderRegion>::default();
772        let persistent_context = new_persistent_context();
773        let from_peer_id = persistent_context.from_peer.id;
774
775        let mut env = TestingEnv::new();
776        let mut ctx = env.context_factory().new_context(persistent_context);
777        prepare_table_metadata(&ctx, HashMap::default()).await;
778        let mailbox_ctx = env.mailbox_context();
779        let mailbox = mailbox_ctx.mailbox().clone();
780
781        let (tx, rx) = tokio::sync::mpsc::channel(1);
782
783        mailbox_ctx
784            .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
785            .await;
786
787        send_mock_reply(mailbox, rx, |id| {
788            Ok(new_downgrade_region_reply(id, Some(1), true, None))
789        });
790
791        let timer = Instant::now();
792        let procedure_ctx = new_procedure_context();
793        let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
794        let elapsed = timer.elapsed().as_secs();
795        let region_lease = default_distributed_time_constants().region_lease.as_secs();
796        assert!(elapsed < region_lease / 2);
797        assert_eq!(
798            ctx.volatile_ctx
799                .leader_region_last_entry_ids
800                .get(&RegionId::new(0, 0))
801                .cloned(),
802            Some(1)
803        );
804        assert!(ctx.volatile_ctx.leader_region_lease_deadline.is_none());
805
806        let _ = next
807            .as_any()
808            .downcast_ref::<UpgradeCandidateRegion>()
809            .unwrap();
810    }
811
812    #[tokio::test]
813    async fn test_next_waits_for_lease_deadline_if_region_not_found() {
814        let mut state = Box::new(DowngradeLeaderRegion {
815            optimistic_retry: 3,
816            retry_initial_interval: Duration::from_millis(100),
817        });
818        let persistent_context = new_persistent_context();
819        let from_peer_id = persistent_context.from_peer.id;
820
821        let mut env = TestingEnv::new();
822        let mut ctx = env.context_factory().new_context(persistent_context);
823        prepare_table_metadata(&ctx, HashMap::default()).await;
824        let mailbox_ctx = env.mailbox_context();
825        let mailbox = mailbox_ctx.mailbox().clone();
826
827        let (tx, rx) = tokio::sync::mpsc::channel(1);
828
829        mailbox_ctx
830            .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
831            .await;
832
833        let attempts = Arc::new(AtomicUsize::new(0));
834        let attempts_moved = attempts.clone();
835        let missing_region_id = RegionId::new(1024, 1);
836        let downgraded_region_id = RegionId::new(1024, 2);
837        send_mock_reply(mailbox, rx, move |id| {
838            attempts_moved.fetch_add(1, Ordering::Relaxed);
839            let replies = vec![
840                DowngradeRegionReply {
841                    region_id: missing_region_id,
842                    last_entry_id: None,
843                    metadata_last_entry_id: None,
844                    exists: false,
845                    error: None,
846                },
847                DowngradeRegionReply {
848                    region_id: downgraded_region_id,
849                    last_entry_id: Some(1),
850                    metadata_last_entry_id: Some(2),
851                    exists: true,
852                    error: None,
853                },
854            ];
855            Ok(new_downgrade_regions_reply(id, replies))
856        });
857
858        ctx.volatile_ctx
859            .set_leader_region_lease_deadline(Duration::from_secs(1));
860        let expected_deadline = ctx.volatile_ctx.leader_region_lease_deadline.unwrap();
861        let procedure_ctx = new_procedure_context();
862        let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
863
864        assert!(Instant::now() >= expected_deadline);
865        assert_eq!(attempts.load(Ordering::Relaxed), state.optimistic_retry);
866        assert_eq!(
867            ctx.volatile_ctx.leader_region_lease_deadline,
868            Some(expected_deadline)
869        );
870        assert_eq!(
871            ctx.volatile_ctx.leader_region_last_entry_ids,
872            HashMap::from([(downgraded_region_id, 1)])
873        );
874        assert_eq!(
875            ctx.volatile_ctx.leader_region_metadata_last_entry_ids,
876            HashMap::from([(downgraded_region_id, 2)])
877        );
878        let _ = next
879            .as_any()
880            .downcast_ref::<UpgradeCandidateRegion>()
881            .unwrap();
882    }
883
884    #[tokio::test]
885    async fn test_downgrade_region_procedure_exceeded_deadline() {
886        let mut state = Box::<UpgradeCandidateRegion>::default();
887        state.retry_initial_interval = Duration::from_millis(100);
888        let persistent_context = new_persistent_context();
889        let to_peer_id = persistent_context.to_peer.id;
890
891        let mut env = TestingEnv::new();
892        let mut ctx = env.context_factory().new_context(persistent_context);
893        let mailbox_ctx = env.mailbox_context();
894        let mailbox = mailbox_ctx.mailbox().clone();
895        ctx.volatile_ctx.metrics.operations_elapsed =
896            ctx.persistent_ctx.timeout + Duration::from_secs(1);
897
898        let (tx, rx) = tokio::sync::mpsc::channel(1);
899        mailbox_ctx
900            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
901            .await;
902
903        send_mock_reply(mailbox, rx, |id| {
904            Ok(new_downgrade_region_reply(id, None, true, None))
905        });
906        let procedure_ctx = new_procedure_context();
907        let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
908        let update_metadata = next.as_any().downcast_ref::<UpdateMetadata>().unwrap();
909        assert_matches!(update_metadata, UpdateMetadata::Rollback);
910    }
911}