1use std::any::Any;
16use std::time::Duration;
17
18use api::v1::meta::MailboxMessage;
19use common_error::ext::BoxedError;
20use common_meta::distributed_time_constants::default_distributed_time_constants;
21use common_meta::instruction::{
22 DowngradeRegion, DowngradeRegionReply, DowngradeRegionsReply, Instruction, InstructionReply,
23};
24use common_procedure::{Context as ProcedureContext, Status};
25use common_telemetry::tracing_context::TracingContext;
26use common_telemetry::{debug, error, info, warn};
27use common_time::util::current_time_millis;
28use serde::{Deserialize, Serialize};
29use snafu::{OptionExt, ResultExt};
30use tokio::time::{Instant, sleep};
31
32use crate::discovery::utils::find_datanode_lease_value;
33use crate::error::{self, Result};
34use crate::handler::HeartbeatMailbox;
35use crate::procedure::region_migration::update_metadata::UpdateMetadata;
36use crate::procedure::region_migration::upgrade_candidate_region::UpgradeCandidateRegion;
37use crate::procedure::region_migration::{Context, State};
38use crate::procedure::utils::instruction_error_result;
39use crate::service::mailbox::Channel;
40
41#[derive(Debug, Serialize, Deserialize)]
42pub struct DowngradeLeaderRegion {
43 optimistic_retry: usize,
45 retry_initial_interval: Duration,
47}
48
49impl Default for DowngradeLeaderRegion {
50 fn default() -> Self {
51 Self {
52 optimistic_retry: 3,
53 retry_initial_interval: Duration::from_millis(500),
54 }
55 }
56}
57
58#[async_trait::async_trait]
59#[typetag::serde]
60impl State for DowngradeLeaderRegion {
61 async fn next(
62 &mut self,
63 ctx: &mut Context,
64 _procedure_ctx: &ProcedureContext,
65 ) -> Result<(Box<dyn State>, Status)> {
66 let now = Instant::now();
67 ctx.volatile_ctx
69 .set_leader_region_lease_deadline(default_distributed_time_constants().region_lease);
70
71 match self.downgrade_region_with_retry(ctx).await {
72 Ok(_) => {
73 info!(
75 "Downgraded region leader success, region: {:?}",
76 ctx.persistent_ctx.region_ids
77 );
78 }
79 Err(error::Error::ExceededDeadline { .. }) => {
80 info!(
81 "Downgrade region leader exceeded deadline, region: {:?}",
82 ctx.persistent_ctx.region_ids
83 );
84 return Ok((Box::new(UpdateMetadata::Rollback), Status::executing(false)));
86 }
87 Err(err) => {
88 error!(err; "Occurs non-retryable error, region: {:?}", ctx.persistent_ctx.region_ids);
89 if let Some(deadline) = ctx.volatile_ctx.leader_region_lease_deadline.as_ref() {
90 info!(
91 "Running into the downgrade region leader slow path, region: {:?}, sleep until {:?}",
92 ctx.persistent_ctx.region_ids, deadline
93 );
94 tokio::time::sleep_until(*deadline).await;
95 } else {
96 warn!(
97 "Leader region lease deadline is not set, region: {:?}",
98 ctx.persistent_ctx.region_ids
99 );
100 }
101 }
102 }
103 ctx.update_downgrade_leader_region_elapsed(now);
104
105 Ok((
106 Box::new(UpgradeCandidateRegion::default()),
107 Status::executing(false),
108 ))
109 }
110
111 fn as_any(&self) -> &dyn Any {
112 self
113 }
114}
115
116impl DowngradeLeaderRegion {
117 fn build_downgrade_region_instruction(
119 &self,
120 ctx: &Context,
121 flush_timeout: Duration,
122 ) -> Instruction {
123 let region_ids = &ctx.persistent_ctx.region_ids;
124 let mut downgrade_regions = Vec::with_capacity(region_ids.len());
125 for region_id in region_ids {
126 downgrade_regions.push(DowngradeRegion {
127 region_id: *region_id,
128 flush_timeout: Some(flush_timeout),
129 });
130 }
131
132 Instruction::DowngradeRegions(downgrade_regions)
133 }
134
135 fn handle_downgrade_region_reply(
136 &self,
137 ctx: &mut Context,
138 reply: &DowngradeRegionReply,
139 now: &Instant,
140 ) -> Result<()> {
141 let leader = &ctx.persistent_ctx.from_peer;
142 let DowngradeRegionReply {
143 region_id,
144 last_entry_id,
145 metadata_last_entry_id,
146 exists,
147 error,
148 } = reply;
149
150 if let Some(error) = error {
151 return instruction_error_result(
152 error,
153 format!(
154 "Failed to downgrade the region {} on datanode {:?}, error: {:?}, elapsed: {:?}",
155 region_id,
156 leader,
157 error,
158 now.elapsed()
159 ),
160 );
161 }
162
163 if !exists {
167 return error::RetryLaterSnafu {
168 reason: format!(
169 "Region {} doesn't exist on datanode {:?}, elapsed: {:?}",
170 region_id,
171 leader,
172 now.elapsed()
173 ),
174 }
175 .fail();
176 }
177
178 info!(
179 "Region {} leader is downgraded on datanode {:?}, last_entry_id: {:?}, metadata_last_entry_id: {:?}, elapsed: {:?}",
180 region_id,
181 leader,
182 last_entry_id,
183 metadata_last_entry_id,
184 now.elapsed()
185 );
186
187 if let Some(last_entry_id) = last_entry_id {
188 debug!(
189 "set last_entry_id: {:?}, region_id: {:?}",
190 last_entry_id, region_id
191 );
192 ctx.volatile_ctx
193 .set_last_entry_id(*region_id, *last_entry_id);
194 }
195
196 if let Some(metadata_last_entry_id) = metadata_last_entry_id {
197 ctx.volatile_ctx
198 .set_metadata_last_entry_id(*region_id, *metadata_last_entry_id);
199 }
200
201 Ok(())
202 }
203
204 async fn downgrade_region(&self, ctx: &mut Context) -> Result<()> {
221 let region_ids = &ctx.persistent_ctx.region_ids;
222 let operation_timeout =
223 ctx.next_operation_timeout()
224 .context(error::ExceededDeadlineSnafu {
225 operation: "Downgrade region",
226 })?;
227 let downgrade_instruction = self.build_downgrade_region_instruction(ctx, operation_timeout);
228
229 let leader = &ctx.persistent_ctx.from_peer;
230 let tracing_ctx = TracingContext::from_current_span();
231 let msg = MailboxMessage::json_message(
232 &format!("Downgrade leader regions: {:?}", region_ids),
233 &format!("Metasrv@{}", ctx.server_addr()),
234 &format!("Datanode-{}@{}", leader.id, leader.addr),
235 common_time::util::current_time_millis(),
236 &downgrade_instruction,
237 Some(tracing_ctx.to_w3c()),
238 )
239 .with_context(|_| error::SerializeToJsonSnafu {
240 input: downgrade_instruction.to_string(),
241 })?;
242
243 let ch = Channel::Datanode(leader.id);
244 let now = Instant::now();
245 let receiver = ctx.mailbox.send(&ch, msg, operation_timeout).await?;
246
247 match receiver.await {
248 Ok(msg) => {
249 let reply = HeartbeatMailbox::json_reply(&msg)?;
250 info!(
251 "Received downgrade region reply: {:?}, region: {:?}, elapsed: {:?}",
252 reply,
253 region_ids,
254 now.elapsed()
255 );
256 let InstructionReply::DowngradeRegions(DowngradeRegionsReply { replies }) = reply
257 else {
258 return error::UnexpectedInstructionReplySnafu {
259 mailbox_message: msg.to_string(),
260 reason: "expect downgrade region reply",
261 }
262 .fail();
263 };
264
265 let mut result = Ok(());
266 for reply in replies {
267 let handled = self.handle_downgrade_region_reply(ctx, &reply, &now);
268 if result.is_ok() {
271 result = handled;
272 }
273 }
274 result
275 }
276 Err(error::Error::MailboxTimeout { .. }) => {
277 let reason = format!(
278 "Mailbox received timeout for downgrade leader region {region_ids:?} on datanode {:?}, elapsed: {:?}",
279 leader,
280 now.elapsed()
281 );
282 error::RetryLaterSnafu { reason }.fail()
283 }
284 Err(err) => Err(err),
285 }
286 }
287
288 async fn update_leader_region_lease_deadline(&self, ctx: &mut Context) {
289 let leader = &ctx.persistent_ctx.from_peer;
290
291 let last_connection_at = match find_datanode_lease_value(&ctx.in_memory, leader.id).await {
292 Ok(lease_value) => lease_value.map(|lease_value| lease_value.timestamp_millis),
293 Err(err) => {
294 error!(err; "Failed to find datanode lease value for datanode: {}, during region migration, region: {:?}", leader, ctx.persistent_ctx.region_ids);
295 return;
296 }
297 };
298
299 if let Some(last_connection_at) = last_connection_at {
300 let now = current_time_millis();
301 let elapsed = now - last_connection_at;
302 let region_lease = default_distributed_time_constants().region_lease;
303
304 if elapsed >= (region_lease.as_secs() * 1000) as i64 {
310 ctx.volatile_ctx.reset_leader_region_lease_deadline();
311 info!(
312 "Datanode {}({}) has been disconnected for longer than the region lease period ({:?}), reset leader region lease deadline to None, region: {:?}",
313 leader, last_connection_at, region_lease, ctx.persistent_ctx.region_ids
314 );
315 } else if elapsed > 0 {
316 let lease_timeout =
318 region_lease - Duration::from_millis((now - last_connection_at) as u64);
319 ctx.volatile_ctx.reset_leader_region_lease_deadline();
320 ctx.volatile_ctx
321 .set_leader_region_lease_deadline(lease_timeout);
322 info!(
323 "Datanode {}({}) last connected {:?} ago, updated leader region lease deadline to {:?}, region: {:?}",
324 leader,
325 last_connection_at,
326 elapsed,
327 ctx.volatile_ctx.leader_region_lease_deadline,
328 ctx.persistent_ctx.region_ids
329 );
330 } else {
331 warn!(
332 "Datanode {} has invalid last connection timestamp: {} (which is after current time: {}), region: {:?}",
333 leader, last_connection_at, now, ctx.persistent_ctx.region_ids
334 )
335 }
336 } else {
337 warn!(
338 "Failed to find last connection time for datanode {}, unable to update region lease deadline, region: {:?}",
339 leader, ctx.persistent_ctx.region_ids
340 )
341 }
342 }
343
344 async fn downgrade_region_with_retry(&self, ctx: &mut Context) -> Result<()> {
355 let mut retry = 0;
356
357 loop {
358 let timer = Instant::now();
359 if let Err(err) = self.downgrade_region(ctx).await {
360 ctx.update_operations_elapsed(timer);
361 retry += 1;
362 if matches!(err, error::Error::ExceededDeadline { .. }) {
364 error!(err; "Failed to downgrade region leader, regions: {:?}, exceeded deadline", ctx.persistent_ctx.region_ids);
365 return Err(err);
366 } else if matches!(err, error::Error::PusherNotFound { .. }) {
367 error!(err; "Failed to downgrade region leader, regions: {:?}, datanode({}) is unreachable(PusherNotFound)", ctx.persistent_ctx.region_ids, ctx.persistent_ctx.from_peer.id);
369 self.update_leader_region_lease_deadline(ctx).await;
370 return Err(err);
371 } else if err.is_retryable() && retry < self.optimistic_retry {
372 error!(err; "Failed to downgrade region leader, regions: {:?}, retry later", ctx.persistent_ctx.region_ids);
373 sleep(self.retry_initial_interval).await;
374 } else {
375 return Err(BoxedError::new(err)).context(error::DowngradeLeaderSnafu {
376 region_id: ctx.persistent_ctx.region_ids[0],
378 })?;
379 }
380 } else {
381 ctx.update_operations_elapsed(timer);
382 ctx.volatile_ctx.reset_leader_region_lease_deadline();
384 break;
385 }
386 }
387
388 Ok(())
389 }
390}
391
392#[cfg(test)]
393mod tests {
394 use std::assert_matches;
395 use std::collections::HashMap;
396 use std::sync::Arc;
397 use std::sync::atomic::{AtomicUsize, Ordering};
398
399 use common_error::ext::RetryHint;
400 use common_error::status_code::StatusCode;
401 use common_meta::instruction::InstructionError;
402 use common_meta::key::table_route::TableRouteValue;
403 use common_meta::key::test_utils::new_test_table_info;
404 use common_meta::peer::Peer;
405 use common_meta::rpc::router::{Region, RegionRoute};
406 use common_meta::wal_provider::RegionWalOptions;
407 use store_api::storage::RegionId;
408 use tokio::time::Instant;
409
410 use super::*;
411 use crate::error::Error;
412 use crate::procedure::region_migration::test_util::{TestingEnv, new_procedure_context};
413 use crate::procedure::region_migration::{
414 ContextFactory, PersistentContext, RegionMigrationTriggerReason,
415 };
416 use crate::procedure::test_util::{
417 new_close_region_reply, new_downgrade_region_reply, new_downgrade_regions_reply,
418 send_mock_reply,
419 };
420
421 fn new_persistent_context() -> PersistentContext {
422 PersistentContext::new(
423 vec![("greptime".into(), "public".into())],
424 Peer::empty(1),
425 Peer::empty(2),
426 vec![RegionId::new(1024, 1)],
427 Duration::from_millis(1000),
428 RegionMigrationTriggerReason::Unknown,
429 )
430 }
431
432 async fn prepare_table_metadata(ctx: &Context, wal_options: RegionWalOptions) {
433 let region_id = ctx.persistent_ctx.region_ids[0];
434 let table_info = new_test_table_info(region_id.table_id());
435 let region_routes = vec![RegionRoute {
436 region: Region::new_test(region_id),
437 leader_peer: Some(ctx.persistent_ctx.from_peer.clone()),
438 follower_peers: vec![ctx.persistent_ctx.to_peer.clone()],
439 ..Default::default()
440 }];
441 ctx.table_metadata_manager
442 .create_table_metadata(
443 table_info,
444 TableRouteValue::physical(region_routes),
445 wal_options,
446 )
447 .await
448 .unwrap();
449 }
450
451 #[tokio::test]
452 async fn test_datanode_is_unreachable() {
453 let state = DowngradeLeaderRegion::default();
454 let persistent_context = new_persistent_context();
455 let env = TestingEnv::new();
456 let mut ctx = env.context_factory().new_context(persistent_context);
457 prepare_table_metadata(&ctx, HashMap::default()).await;
458 let err = state.downgrade_region(&mut ctx).await.unwrap_err();
459
460 assert_matches!(err, Error::PusherNotFound { .. });
461 assert!(!err.is_retryable());
462 }
463
464 #[tokio::test]
465 async fn test_pusher_dropped() {
466 let state = DowngradeLeaderRegion::default();
467 let persistent_context = new_persistent_context();
468 let from_peer_id = persistent_context.from_peer.id;
469
470 let mut env = TestingEnv::new();
471 let mut ctx = env.context_factory().new_context(persistent_context);
472 prepare_table_metadata(&ctx, HashMap::default()).await;
473 let mailbox_ctx = env.mailbox_context();
474
475 let (tx, rx) = tokio::sync::mpsc::channel(1);
476
477 mailbox_ctx
478 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
479 .await;
480
481 drop(rx);
482
483 let err = state.downgrade_region(&mut ctx).await.unwrap_err();
484
485 assert_matches!(err, Error::PushMessage { .. });
486 assert!(!err.is_retryable());
487 }
488
489 #[tokio::test]
490 async fn test_procedure_exceeded_deadline() {
491 let state = DowngradeLeaderRegion::default();
492 let persistent_context = new_persistent_context();
493 let env = TestingEnv::new();
494 let mut ctx = env.context_factory().new_context(persistent_context);
495 prepare_table_metadata(&ctx, HashMap::default()).await;
496 ctx.volatile_ctx.metrics.operations_elapsed =
497 ctx.persistent_ctx.timeout + Duration::from_secs(1);
498
499 let err = state.downgrade_region(&mut ctx).await.unwrap_err();
500
501 assert_matches!(err, Error::ExceededDeadline { .. });
502 assert!(!err.is_retryable());
503
504 let err = state
505 .downgrade_region_with_retry(&mut ctx)
506 .await
507 .unwrap_err();
508 assert_matches!(err, Error::ExceededDeadline { .. });
509 assert!(!err.is_retryable());
510 }
511
512 #[tokio::test]
513 async fn test_unexpected_instruction_reply() {
514 let state = DowngradeLeaderRegion::default();
515 let persistent_context = new_persistent_context();
516 let from_peer_id = persistent_context.from_peer.id;
517
518 let mut env = TestingEnv::new();
519 let mut ctx = env.context_factory().new_context(persistent_context);
520 prepare_table_metadata(&ctx, HashMap::default()).await;
521 let mailbox_ctx = env.mailbox_context();
522 let mailbox = mailbox_ctx.mailbox().clone();
523
524 let (tx, rx) = tokio::sync::mpsc::channel(1);
525
526 mailbox_ctx
527 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
528 .await;
529
530 send_mock_reply(mailbox, rx, |id| Ok(new_close_region_reply(id)));
532
533 let err = state.downgrade_region(&mut ctx).await.unwrap_err();
534
535 assert_matches!(err, Error::UnexpectedInstructionReply { .. });
536 assert!(!err.is_retryable());
537 }
538
539 #[tokio::test]
540 async fn test_instruction_exceeded_deadline() {
541 let state = DowngradeLeaderRegion::default();
542 let persistent_context = new_persistent_context();
543 let from_peer_id = persistent_context.from_peer.id;
544
545 let mut env = TestingEnv::new();
546 let mut ctx = env.context_factory().new_context(persistent_context);
547 prepare_table_metadata(&ctx, HashMap::default()).await;
548 let mailbox_ctx = env.mailbox_context();
549 let mailbox = mailbox_ctx.mailbox().clone();
550
551 let (tx, rx) = tokio::sync::mpsc::channel(1);
552
553 mailbox_ctx
554 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
555 .await;
556
557 send_mock_reply(mailbox, rx, |id| {
558 Err(error::MailboxTimeoutSnafu { id }.build())
559 });
560
561 let err = state.downgrade_region(&mut ctx).await.unwrap_err();
562
563 assert_matches!(err, Error::RetryLater { .. });
564 assert!(err.is_retryable());
565 }
566
567 #[tokio::test]
568 async fn test_downgrade_region_failed() {
569 let state = DowngradeLeaderRegion::default();
570 let persistent_context = new_persistent_context();
571 let from_peer_id = persistent_context.from_peer.id;
572
573 let mut env = TestingEnv::new();
574 let mut ctx = env.context_factory().new_context(persistent_context);
575 prepare_table_metadata(&ctx, HashMap::default()).await;
576 let mailbox_ctx = env.mailbox_context();
577 let mailbox = mailbox_ctx.mailbox().clone();
578
579 let (tx, rx) = tokio::sync::mpsc::channel(1);
580
581 mailbox_ctx
582 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
583 .await;
584
585 send_mock_reply(mailbox, rx, |id| {
586 Ok(new_downgrade_region_reply(
587 id,
588 None,
589 false,
590 Some("test mocked".to_string()),
591 ))
592 });
593
594 let err = state.downgrade_region(&mut ctx).await.unwrap_err();
595
596 assert_matches!(err, Error::RetryLater { .. });
597 assert!(err.is_retryable());
598 assert!(format!("{err:?}").contains("test mocked"), "err: {err:?}",);
599 }
600
601 #[tokio::test]
602 async fn test_downgrade_region_returns_first_failure_of_batch() {
603 let missing = DowngradeRegionReply {
604 region_id: RegionId::new(1024, 1),
605 last_entry_id: None,
606 metadata_last_entry_id: None,
607 exists: false,
608 error: None,
609 };
610 let failed = DowngradeRegionReply {
611 region_id: RegionId::new(1024, 2),
612 last_entry_id: None,
613 metadata_last_entry_id: None,
614 exists: true,
615 error: Some(InstructionError::new(
616 StatusCode::Internal,
617 "non retryable mocked",
618 RetryHint::NonRetryable,
619 )),
620 };
621 let downgraded = DowngradeRegionReply {
622 region_id: RegionId::new(1024, 3),
623 last_entry_id: Some(1),
624 metadata_last_entry_id: None,
625 exists: true,
626 error: None,
627 };
628
629 for (replies, retryable) in [
630 (
631 vec![missing.clone(), failed.clone(), downgraded.clone()],
632 true,
633 ),
634 (vec![failed, missing, downgraded.clone()], false),
635 ] {
636 let state = DowngradeLeaderRegion::default();
637 let persistent_context = new_persistent_context();
638 let from_peer_id = persistent_context.from_peer.id;
639
640 let mut env = TestingEnv::new();
641 let mut ctx = env.context_factory().new_context(persistent_context);
642 prepare_table_metadata(&ctx, HashMap::default()).await;
643 let mailbox_ctx = env.mailbox_context();
644 let mailbox = mailbox_ctx.mailbox().clone();
645
646 let (tx, rx) = tokio::sync::mpsc::channel(1);
647
648 mailbox_ctx
649 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
650 .await;
651
652 send_mock_reply(mailbox, rx, move |id| {
653 Ok(new_downgrade_regions_reply(id, replies.clone()))
654 });
655
656 let err = state.downgrade_region(&mut ctx).await.unwrap_err();
657
658 assert_eq!(err.is_retryable(), retryable, "err: {err:?}");
659 assert_eq!(
660 ctx.volatile_ctx.leader_region_last_entry_ids,
661 HashMap::from([(downgraded.region_id, 1)])
662 );
663 }
664 }
665
666 #[tokio::test]
667 async fn test_downgrade_region_with_retry_fast_path() {
668 let state = DowngradeLeaderRegion::default();
669 let persistent_context = new_persistent_context();
670 let from_peer_id = persistent_context.from_peer.id;
671
672 let mut env = TestingEnv::new();
673 let mut ctx = env.context_factory().new_context(persistent_context);
674 prepare_table_metadata(&ctx, HashMap::default()).await;
675 let mailbox_ctx = env.mailbox_context();
676 let mailbox = mailbox_ctx.mailbox().clone();
677
678 let (tx, mut rx) = tokio::sync::mpsc::channel(1);
679
680 mailbox_ctx
681 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
682 .await;
683
684 common_runtime::spawn_global(async move {
685 let resp = rx.recv().await.unwrap().unwrap();
687 let reply_id = resp.mailbox_message.unwrap().id;
688 mailbox
689 .on_recv(
690 reply_id,
691 Err(error::MailboxTimeoutSnafu { id: reply_id }.build()),
692 )
693 .await
694 .unwrap();
695
696 let resp = rx.recv().await.unwrap().unwrap();
698 let reply_id = resp.mailbox_message.unwrap().id;
699 mailbox
700 .on_recv(
701 reply_id,
702 Ok(new_downgrade_region_reply(reply_id, Some(1), true, None)),
703 )
704 .await
705 .unwrap();
706 });
707
708 state.downgrade_region_with_retry(&mut ctx).await.unwrap();
709 assert_eq!(
710 ctx.volatile_ctx
711 .leader_region_last_entry_ids
712 .get(&RegionId::new(0, 0))
713 .cloned(),
714 Some(1)
715 );
716 assert!(ctx.volatile_ctx.leader_region_lease_deadline.is_none());
717 }
718
719 #[tokio::test]
720 async fn test_downgrade_region_with_retry_slow_path() {
721 let state = DowngradeLeaderRegion {
722 optimistic_retry: 3,
723 retry_initial_interval: Duration::from_millis(100),
724 };
725 let persistent_context = new_persistent_context();
726 let from_peer_id = persistent_context.from_peer.id;
727
728 let mut env = TestingEnv::new();
729 let mut ctx = env.context_factory().new_context(persistent_context);
730 let mailbox_ctx = env.mailbox_context();
731 let mailbox = mailbox_ctx.mailbox().clone();
732
733 let (tx, mut rx) = tokio::sync::mpsc::channel(1);
734
735 mailbox_ctx
736 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
737 .await;
738
739 common_runtime::spawn_global(async move {
740 for _ in 0..3 {
741 let resp = rx.recv().await.unwrap().unwrap();
742 let reply_id = resp.mailbox_message.unwrap().id;
743 mailbox
744 .on_recv(
745 reply_id,
746 Err(error::MailboxTimeoutSnafu { id: reply_id }.build()),
747 )
748 .await
749 .unwrap();
750 }
751 });
752
753 ctx.volatile_ctx
754 .set_leader_region_lease_deadline(Duration::from_secs(5));
755 let expected_deadline = ctx.volatile_ctx.leader_region_lease_deadline.unwrap();
756 let err = state
757 .downgrade_region_with_retry(&mut ctx)
758 .await
759 .unwrap_err();
760 assert_matches!(err, error::Error::DowngradeLeader { .. });
761 assert_eq!(
764 ctx.volatile_ctx.leader_region_lease_deadline.unwrap(),
765 expected_deadline
766 )
767 }
768
769 #[tokio::test]
770 async fn test_next_upgrade_candidate_state() {
771 let mut state = Box::<DowngradeLeaderRegion>::default();
772 let persistent_context = new_persistent_context();
773 let from_peer_id = persistent_context.from_peer.id;
774
775 let mut env = TestingEnv::new();
776 let mut ctx = env.context_factory().new_context(persistent_context);
777 prepare_table_metadata(&ctx, HashMap::default()).await;
778 let mailbox_ctx = env.mailbox_context();
779 let mailbox = mailbox_ctx.mailbox().clone();
780
781 let (tx, rx) = tokio::sync::mpsc::channel(1);
782
783 mailbox_ctx
784 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
785 .await;
786
787 send_mock_reply(mailbox, rx, |id| {
788 Ok(new_downgrade_region_reply(id, Some(1), true, None))
789 });
790
791 let timer = Instant::now();
792 let procedure_ctx = new_procedure_context();
793 let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
794 let elapsed = timer.elapsed().as_secs();
795 let region_lease = default_distributed_time_constants().region_lease.as_secs();
796 assert!(elapsed < region_lease / 2);
797 assert_eq!(
798 ctx.volatile_ctx
799 .leader_region_last_entry_ids
800 .get(&RegionId::new(0, 0))
801 .cloned(),
802 Some(1)
803 );
804 assert!(ctx.volatile_ctx.leader_region_lease_deadline.is_none());
805
806 let _ = next
807 .as_any()
808 .downcast_ref::<UpgradeCandidateRegion>()
809 .unwrap();
810 }
811
812 #[tokio::test]
813 async fn test_next_waits_for_lease_deadline_if_region_not_found() {
814 let mut state = Box::new(DowngradeLeaderRegion {
815 optimistic_retry: 3,
816 retry_initial_interval: Duration::from_millis(100),
817 });
818 let persistent_context = new_persistent_context();
819 let from_peer_id = persistent_context.from_peer.id;
820
821 let mut env = TestingEnv::new();
822 let mut ctx = env.context_factory().new_context(persistent_context);
823 prepare_table_metadata(&ctx, HashMap::default()).await;
824 let mailbox_ctx = env.mailbox_context();
825 let mailbox = mailbox_ctx.mailbox().clone();
826
827 let (tx, rx) = tokio::sync::mpsc::channel(1);
828
829 mailbox_ctx
830 .insert_heartbeat_response_receiver(Channel::Datanode(from_peer_id), tx)
831 .await;
832
833 let attempts = Arc::new(AtomicUsize::new(0));
834 let attempts_moved = attempts.clone();
835 let missing_region_id = RegionId::new(1024, 1);
836 let downgraded_region_id = RegionId::new(1024, 2);
837 send_mock_reply(mailbox, rx, move |id| {
838 attempts_moved.fetch_add(1, Ordering::Relaxed);
839 let replies = vec![
840 DowngradeRegionReply {
841 region_id: missing_region_id,
842 last_entry_id: None,
843 metadata_last_entry_id: None,
844 exists: false,
845 error: None,
846 },
847 DowngradeRegionReply {
848 region_id: downgraded_region_id,
849 last_entry_id: Some(1),
850 metadata_last_entry_id: Some(2),
851 exists: true,
852 error: None,
853 },
854 ];
855 Ok(new_downgrade_regions_reply(id, replies))
856 });
857
858 ctx.volatile_ctx
859 .set_leader_region_lease_deadline(Duration::from_secs(1));
860 let expected_deadline = ctx.volatile_ctx.leader_region_lease_deadline.unwrap();
861 let procedure_ctx = new_procedure_context();
862 let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
863
864 assert!(Instant::now() >= expected_deadline);
865 assert_eq!(attempts.load(Ordering::Relaxed), state.optimistic_retry);
866 assert_eq!(
867 ctx.volatile_ctx.leader_region_lease_deadline,
868 Some(expected_deadline)
869 );
870 assert_eq!(
871 ctx.volatile_ctx.leader_region_last_entry_ids,
872 HashMap::from([(downgraded_region_id, 1)])
873 );
874 assert_eq!(
875 ctx.volatile_ctx.leader_region_metadata_last_entry_ids,
876 HashMap::from([(downgraded_region_id, 2)])
877 );
878 let _ = next
879 .as_any()
880 .downcast_ref::<UpgradeCandidateRegion>()
881 .unwrap();
882 }
883
884 #[tokio::test]
885 async fn test_downgrade_region_procedure_exceeded_deadline() {
886 let mut state = Box::<UpgradeCandidateRegion>::default();
887 state.retry_initial_interval = Duration::from_millis(100);
888 let persistent_context = new_persistent_context();
889 let to_peer_id = persistent_context.to_peer.id;
890
891 let mut env = TestingEnv::new();
892 let mut ctx = env.context_factory().new_context(persistent_context);
893 let mailbox_ctx = env.mailbox_context();
894 let mailbox = mailbox_ctx.mailbox().clone();
895 ctx.volatile_ctx.metrics.operations_elapsed =
896 ctx.persistent_ctx.timeout + Duration::from_secs(1);
897
898 let (tx, rx) = tokio::sync::mpsc::channel(1);
899 mailbox_ctx
900 .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
901 .await;
902
903 send_mock_reply(mailbox, rx, |id| {
904 Ok(new_downgrade_region_reply(id, None, true, None))
905 });
906 let procedure_ctx = new_procedure_context();
907 let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
908 let update_metadata = next.as_any().downcast_ref::<UpdateMetadata>().unwrap();
909 assert_matches!(update_metadata, UpdateMetadata::Rollback);
910 }
911}