1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
16use std::fmt::Debug;
17use std::sync::{Arc, Mutex};
18use std::time::{Duration, Instant};
19
20use async_trait::async_trait;
21use common_meta::DatanodeId;
22use common_meta::affinity::{AffinityFallback, AffinityReference, AffinityResolverRef};
23use common_meta::datanode::Stat;
24use common_meta::ddl::{DetectingRegion, RegionFailureDetectorController};
25use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
26use common_meta::key::table_route::{TableRouteKey, TableRouteStorage, TableRouteValue};
27use common_meta::key::{MetadataKey, MetadataValue};
28use common_meta::kv_backend::KvBackendRef;
29use common_meta::leadership_notifier::LeadershipChangeListener;
30use common_meta::peer::{Peer, PeerResolverRef};
31use common_meta::range_stream::{DEFAULT_PAGE_SIZE, PaginationStream};
32use common_meta::rpc::store::RangeRequest;
33use common_runtime::JoinHandle;
34use common_telemetry::{debug, error, info, warn};
35use common_time::util::current_time_millis;
36use futures::{StreamExt, TryStreamExt};
37use itertools::Itertools;
38use snafu::{ResultExt, ensure};
39use store_api::storage::RegionId;
40use tokio::sync::mpsc::error::TrySendError;
41use tokio::sync::mpsc::{Receiver, Sender};
42use tokio::sync::oneshot;
43use tokio::time::{MissedTickBehavior, interval, interval_at};
44
45use crate::discovery::utils::accept_ingest_workload;
46use crate::error::{self, Result};
47use crate::failure_detector::PhiAccrualFailureDetectorOptions;
48use crate::metasrv::{RegionStatAwareSelectorRef, SelectorContext, SelectorRef};
49use crate::metrics::METRIC_META_HEARTBEAT_DROPPED;
50use crate::procedure::region_migration::manager::{
51 RegionMigrationManagerRef, RegionMigrationTriggerReason, SubmitRegionMigrationTaskResult,
52};
53use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
54use crate::procedure::region_migration::{
55 DEFAULT_REGION_MIGRATION_TIMEOUT, RegionMigrationProcedureTask,
56};
57use crate::region::failure_detector::RegionFailureDetector;
58use crate::selector::SelectorOptions;
59use crate::state::StateRef;
60
61#[derive(Debug)]
65pub(crate) struct DatanodeHeartbeat {
66 datanode_id: DatanodeId,
67 regions: Vec<RegionId>,
69 pub(crate) opening_regions: Vec<RegionId>,
70 timestamp: i64,
71}
72
73impl From<&Stat> for DatanodeHeartbeat {
74 fn from(value: &Stat) -> Self {
75 DatanodeHeartbeat {
76 datanode_id: value.id,
77 regions: value.region_stats.iter().map(|x| x.id).collect(),
78 opening_regions: Vec::new(),
79 timestamp: value.timestamp_millis,
80 }
81 }
82}
83
84pub(crate) enum Event {
101 Tick,
102 InitializeAllRegions(tokio::sync::oneshot::Sender<()>),
103 RegisterFailureDetectors(Vec<DetectingRegion>),
104 DeregisterFailureDetectors(Vec<DetectingRegion>),
105 ResetFailureDetectors(Vec<DetectingRegion>),
106 HeartbeatArrived(DatanodeHeartbeat),
107 Clear,
108 #[cfg(test)]
109 Dump(tokio::sync::oneshot::Sender<RegionFailureDetector>),
110}
111
112impl Debug for Event {
113 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
114 match self {
115 Self::Tick => write!(f, "Tick"),
116 Self::HeartbeatArrived(arg0) => f.debug_tuple("HeartbeatArrived").field(arg0).finish(),
117 Self::Clear => write!(f, "Clear"),
118 Self::InitializeAllRegions(_) => write!(f, "InspectAndRegisterRegions"),
119 Self::RegisterFailureDetectors(arg0) => f
120 .debug_tuple("RegisterFailureDetectors")
121 .field(arg0)
122 .finish(),
123 Self::ResetFailureDetectors(arg0) => {
124 f.debug_tuple("ResetFailureDetectors").field(arg0).finish()
125 }
126 Self::DeregisterFailureDetectors(arg0) => f
127 .debug_tuple("DeregisterFailureDetectors")
128 .field(arg0)
129 .finish(),
130 #[cfg(test)]
131 Self::Dump(_) => f.debug_struct("Dump").finish(),
132 }
133 }
134}
135
136pub type RegionSupervisorTickerRef = Arc<RegionSupervisorTicker>;
137
138#[derive(Debug)]
140pub struct RegionSupervisorTicker {
141 tick_handle: Mutex<Option<JoinHandle<()>>>,
143
144 initialization_handle: Mutex<Option<JoinHandle<()>>>,
146
147 tick_interval: Duration,
149
150 initialization_delay: Duration,
152
153 initialization_retry_period: Duration,
155
156 sender: Sender<Event>,
158}
159
160#[async_trait]
161impl LeadershipChangeListener for RegionSupervisorTicker {
162 fn name(&self) -> &'static str {
163 "RegionSupervisorTicker"
164 }
165
166 async fn on_leader_start(&self) -> common_meta::error::Result<()> {
167 self.start();
168 Ok(())
169 }
170
171 async fn on_leader_stop(&self) -> common_meta::error::Result<()> {
172 self.stop();
173 Ok(())
174 }
175}
176
177impl RegionSupervisorTicker {
178 pub(crate) fn new(
179 tick_interval: Duration,
180 initialization_delay: Duration,
181 initialization_retry_period: Duration,
182 sender: Sender<Event>,
183 ) -> Self {
184 info!(
185 "RegionSupervisorTicker is created, tick_interval: {:?}, initialization_delay: {:?}, initialization_retry_period: {:?}",
186 tick_interval, initialization_delay, initialization_retry_period
187 );
188 Self {
189 tick_handle: Mutex::new(None),
190 initialization_handle: Mutex::new(None),
191 tick_interval,
192 initialization_delay,
193 initialization_retry_period,
194 sender,
195 }
196 }
197
198 pub fn start(&self) {
200 let mut handle = self.tick_handle.lock().unwrap();
201 if handle.is_none() {
202 let sender = self.sender.clone();
203 let tick_interval = self.tick_interval;
204 let initialization_delay = self.initialization_delay;
205
206 let mut initialization_interval = interval_at(
207 tokio::time::Instant::now() + initialization_delay,
208 self.initialization_retry_period,
209 );
210 initialization_interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
211 let initialization_handler = common_runtime::spawn_global(async move {
212 loop {
213 initialization_interval.tick().await;
214 let (tx, rx) = oneshot::channel();
215 if sender.send(Event::InitializeAllRegions(tx)).await.is_err() {
216 info!(
217 "EventReceiver is dropped, region failure detectors initialization loop is stopped"
218 );
219 break;
220 }
221 if rx.await.is_ok() {
222 info!("All region failure detectors are initialized.");
223 break;
224 }
225 }
226 });
227 *self.initialization_handle.lock().unwrap() = Some(initialization_handler);
228
229 let sender = self.sender.clone();
230 let ticker_loop = tokio::spawn(async move {
231 let mut tick_interval = interval(tick_interval);
232 tick_interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
233
234 if let Err(err) = sender.send(Event::Clear).await {
235 warn!(err; "EventReceiver is dropped, failed to send Event::Clear");
236 return;
237 }
238 loop {
239 tick_interval.tick().await;
240 if sender.send(Event::Tick).await.is_err() {
241 info!("EventReceiver is dropped, tick loop is stopped");
242 break;
243 }
244 }
245 });
246 *handle = Some(ticker_loop);
247 }
248 }
249
250 pub fn stop(&self) {
252 let handle = self.tick_handle.lock().unwrap().take();
253 if let Some(handle) = handle {
254 handle.abort();
255 info!("The tick loop is stopped.");
256 }
257 let initialization_handler = self.initialization_handle.lock().unwrap().take();
258 if let Some(initialization_handler) = initialization_handler {
259 initialization_handler.abort();
260 info!("The initialization loop is stopped.");
261 }
262 }
263}
264
265impl Drop for RegionSupervisorTicker {
266 fn drop(&mut self) {
267 self.stop();
268 }
269}
270
271pub type RegionSupervisorRef = Arc<RegionSupervisor>;
272
273pub const DEFAULT_TICK_INTERVAL: Duration = Duration::from_secs(1);
275pub const DEFAULT_INITIALIZATION_RETRY_PERIOD: Duration = Duration::from_secs(60);
277
278pub enum RegionSupervisorSelector {
280 NaiveSelector(SelectorRef),
281 RegionStatAwareSelector(RegionStatAwareSelectorRef),
282}
283
284pub struct RegionSupervisor {
287 failure_detector: RegionFailureDetector,
289 failover_counts: HashMap<DetectingRegion, u32>,
291 receiver: Receiver<Event>,
293 selector_context: SelectorContext,
295 selector: RegionSupervisorSelector,
297 affinity_resolver: Option<AffinityResolverRef>,
299 region_migration_manager: RegionMigrationManagerRef,
301 runtime_switch_manager: RuntimeSwitchManagerRef,
303 peer_resolver: PeerResolverRef,
305 kv_backend: KvBackendRef,
307 state: Option<StateRef>,
309}
310
311#[derive(Debug, Clone)]
313pub struct RegionFailureDetectorControl {
314 sender: Sender<Event>,
315}
316
317impl RegionFailureDetectorControl {
318 pub(crate) fn new(sender: Sender<Event>) -> Self {
319 Self { sender }
320 }
321}
322
323#[async_trait::async_trait]
324impl RegionFailureDetectorController for RegionFailureDetectorControl {
325 async fn register_failure_detectors(&self, detecting_regions: Vec<DetectingRegion>) {
326 if let Err(err) = self
327 .sender
328 .send(Event::RegisterFailureDetectors(detecting_regions))
329 .await
330 {
331 error!(err; "RegionSupervisor has stop receiving heartbeat.");
332 }
333 }
334
335 async fn reset_failure_detectors(&self, detecting_regions: Vec<DetectingRegion>) {
336 if let Err(err) = self
337 .sender
338 .send(Event::ResetFailureDetectors(detecting_regions))
339 .await
340 {
341 error!(err; "RegionSupervisor has stop receiving heartbeat.");
342 }
343 }
344
345 async fn deregister_failure_detectors(&self, detecting_regions: Vec<DetectingRegion>) {
346 if let Err(err) = self
347 .sender
348 .send(Event::DeregisterFailureDetectors(detecting_regions))
349 .await
350 {
351 error!(err; "RegionSupervisor has stop receiving heartbeat.");
352 }
353 }
354}
355
356#[derive(Clone)]
358pub(crate) struct HeartbeatAcceptor {
359 sender: Sender<Event>,
360}
361
362impl HeartbeatAcceptor {
363 pub(crate) fn new(sender: Sender<Event>) -> Self {
364 Self { sender }
365 }
366
367 pub(crate) fn accept(&self, heartbeat: DatanodeHeartbeat) {
371 let datanode_id = heartbeat.datanode_id;
372 match self.sender.try_send(Event::HeartbeatArrived(heartbeat)) {
373 Ok(()) => {}
374 Err(TrySendError::Full(_)) => {
375 METRIC_META_HEARTBEAT_DROPPED
376 .with_label_values(&["full"])
377 .inc();
378 warn!(
379 "Dropping heartbeat because the region supervisor event queue is full, datanode_id: {}, queue_capacity: {}",
380 datanode_id,
381 self.sender.max_capacity()
382 );
383 }
384 Err(TrySendError::Closed(err)) => {
385 METRIC_META_HEARTBEAT_DROPPED
386 .with_label_values(&["closed"])
387 .inc();
388 error!(err; "RegionSupervisor has stop receiving heartbeat.");
389 }
390 }
391 }
392}
393
394impl RegionSupervisor {
395 pub(crate) fn channel() -> (Sender<Event>, Receiver<Event>) {
397 tokio::sync::mpsc::channel(1024)
398 }
399
400 #[allow(clippy::too_many_arguments)]
401 pub(crate) fn new(
402 event_receiver: Receiver<Event>,
403 options: PhiAccrualFailureDetectorOptions,
404 selector_context: SelectorContext,
405 selector: RegionSupervisorSelector,
406 region_migration_manager: RegionMigrationManagerRef,
407 runtime_switch_manager: RuntimeSwitchManagerRef,
408 peer_resolver: PeerResolverRef,
409 kv_backend: KvBackendRef,
410 ) -> Self {
411 Self {
412 failure_detector: RegionFailureDetector::new(options),
413 failover_counts: HashMap::new(),
414 receiver: event_receiver,
415 selector_context: SelectorContext {
417 affinity_fallback: AffinityFallback::Unconstrained,
418 ..selector_context
419 },
420 selector,
421 affinity_resolver: None,
422 region_migration_manager,
423 runtime_switch_manager,
424 peer_resolver,
425 kv_backend,
426 state: None,
427 }
428 }
429
430 pub(crate) fn with_affinity_resolver(mut self, resolver: AffinityResolverRef) -> Self {
432 self.affinity_resolver = Some(resolver);
433 self
434 }
435
436 pub(crate) fn with_state(mut self, state: StateRef) -> Self {
438 self.state = Some(state);
439 self
440 }
441
442 pub(crate) async fn run(&mut self) {
444 while let Some(event) = self.receiver.recv().await {
445 if let Some(state) = self.state.as_ref()
446 && !state.read().unwrap().is_leader()
447 {
448 warn!(
449 "The current metasrv is not the leader, ignore {:?} event",
450 event
451 );
452 continue;
453 }
454
455 match event {
456 Event::InitializeAllRegions(sender) => {
457 match self.is_maintenance_mode_enabled().await {
458 Ok(false) => {}
459 Ok(true) => {
460 warn!(
461 "Skipping initialize all regions since maintenance mode is enabled."
462 );
463 continue;
464 }
465 Err(err) => {
466 error!(err; "Failed to check maintenance mode during initialize all regions.");
467 continue;
468 }
469 }
470
471 if let Err(err) = self.initialize_all().await {
472 error!(err; "Failed to initialize all regions.");
473 } else {
474 let _ = sender.send(());
476 }
477 }
478 Event::Tick => {
479 let regions = self.detect_region_failure();
480 self.handle_region_failures(regions).await;
481 }
482 Event::RegisterFailureDetectors(detecting_regions) => {
483 self.register_failure_detectors(detecting_regions).await
484 }
485 Event::ResetFailureDetectors(detecting_regions) => {
486 self.reset_failure_detectors(detecting_regions).await
487 }
488 Event::DeregisterFailureDetectors(detecting_regions) => {
489 self.deregister_failure_detectors(detecting_regions).await
490 }
491 Event::HeartbeatArrived(heartbeat) => self.on_heartbeat_arrived(heartbeat),
492 Event::Clear => {
493 self.clear();
494 info!("Region supervisor is initialized.");
495 }
496 #[cfg(test)]
497 Event::Dump(sender) => {
498 let _ = sender.send(self.failure_detector.dump());
499 }
500 }
501 }
502 info!("RegionSupervisor is stopped!");
503 }
504
505 async fn initialize_all(&self) -> Result<()> {
506 let now = Instant::now();
507 let regions = self.regions();
508 let req = RangeRequest::new().with_prefix(TableRouteKey::range_prefix());
509 let stream = PaginationStream::new(self.kv_backend.clone(), req, DEFAULT_PAGE_SIZE, |kv| {
510 TableRouteKey::from_bytes(&kv.key).map(|v| (v.table_id, kv.value))
511 })
512 .into_stream();
513
514 let mut stream = stream
515 .map_ok(|(_, value)| {
516 TableRouteValue::try_from_raw_value(&value)
517 .context(error::TableMetadataManagerSnafu)
518 })
519 .boxed();
520 let mut detecting_regions = Vec::new();
521 while let Some(route) = stream
522 .try_next()
523 .await
524 .context(error::TableMetadataManagerSnafu)?
525 {
526 let route = route?;
527 if !route.is_physical() {
528 continue;
529 }
530
531 let physical_table_route = route.into_physical_table_route();
532 physical_table_route
533 .region_routes
534 .iter()
535 .for_each(|region_route| {
536 if !regions.contains(®ion_route.region.id)
537 && let Some(leader_peer) = ®ion_route.leader_peer
538 {
539 detecting_regions.push((leader_peer.id, region_route.region.id));
540 }
541 });
542 }
543
544 let num_detecting_regions = detecting_regions.len();
545 if !detecting_regions.is_empty() {
546 self.register_failure_detectors(detecting_regions).await;
547 }
548
549 info!(
550 "Initialize {} region failure detectors, elapsed: {:?}",
551 num_detecting_regions,
552 now.elapsed()
553 );
554
555 Ok(())
556 }
557
558 async fn register_failure_detectors(&self, detecting_regions: Vec<DetectingRegion>) {
559 let ts_millis = current_time_millis();
560 for region in detecting_regions {
561 self.failure_detector
563 .maybe_init_region_failure_detector(region, ts_millis);
564 }
565 }
566
567 async fn reset_failure_detectors(&mut self, detecting_regions: Vec<DetectingRegion>) {
568 let ts_millis = current_time_millis();
569 for region in detecting_regions {
570 self.failure_detector
571 .reset_region_failure_detector(region, ts_millis);
572 }
573 }
574
575 async fn deregister_failure_detectors(&mut self, detecting_regions: Vec<DetectingRegion>) {
576 for region in detecting_regions {
577 self.failure_detector.remove(®ion);
578 self.failover_counts.remove(®ion);
579 }
580 }
581
582 async fn handle_region_failures(&mut self, mut regions: Vec<(DatanodeId, RegionId)>) {
583 if regions.is_empty() {
584 return;
585 }
586 match self.is_maintenance_mode_enabled().await {
587 Ok(false) => {}
588 Ok(true) => {
589 warn!(
590 "Skipping failover since maintenance mode is enabled. Detected region failures: {:?}",
591 regions
592 );
593 return;
594 }
595 Err(err) => {
596 error!(err; "Failed to check maintenance mode");
597 return;
598 }
599 }
600
601 let migrating_regions = regions
603 .extract_if(.., |(_, region_id)| {
604 self.region_migration_manager.tracker().contains(*region_id)
605 })
606 .collect::<Vec<_>>();
607
608 for (datanode_id, region_id) in migrating_regions {
609 debug!(
610 "Removed region failover for region: {region_id}, datanode: {datanode_id} because it's migrating"
611 );
612 }
613
614 if regions.is_empty() {
615 return;
617 }
618
619 let mut grouped_regions: HashMap<u64, Vec<RegionId>> =
620 HashMap::with_capacity(regions.len());
621 for (datanode_id, region_id) in regions {
622 grouped_regions
623 .entry(datanode_id)
624 .or_default()
625 .push(region_id);
626 }
627
628 for (datanode_id, regions) in grouped_regions {
629 warn!(
630 "Detects region failures on datanode: {}, regions: {:?}",
631 datanode_id, regions
632 );
633 let failed_datanodes = [datanode_id];
637 match self
638 .generate_failover_tasks(datanode_id, ®ions, &failed_datanodes)
639 .await
640 {
641 Ok(tasks) => {
642 let mut grouped_tasks: HashMap<(u64, u64), Vec<_>> = HashMap::new();
643 for (task, count) in tasks {
644 grouped_tasks
645 .entry((task.from_peer.id, task.to_peer.id))
646 .or_default()
647 .push((task, count));
648 }
649
650 for ((from_peer_id, to_peer_id), tasks) in grouped_tasks {
651 if tasks.is_empty() {
652 continue;
653 }
654 let task = RegionMigrationTaskBatch::from_tasks(tasks);
655 let region_ids = task.region_ids.clone();
656 if let Err(err) = self.do_failover_tasks(task).await {
657 error!(err; "Failed to execute region failover for regions: {:?}, from_peer: {}, to_peer: {}", region_ids, from_peer_id, to_peer_id);
658 }
659 }
660 }
661 Err(err) => error!(err; "Failed to generate failover tasks"),
662 }
663 }
664 }
665
666 pub(crate) async fn is_maintenance_mode_enabled(&self) -> Result<bool> {
667 self.runtime_switch_manager
668 .maintenance_mode()
669 .await
670 .context(error::RuntimeSwitchManagerSnafu)
671 }
672
673 async fn select_peers(
677 &self,
678 from_peer_id: DatanodeId,
679 regions: &[RegionId],
680 failure_datanodes: &[DatanodeId],
681 exclude_peer_ids_by_region: &HashMap<RegionId, BTreeSet<DatanodeId>>,
682 ) -> Result<Vec<(RegionId, Peer)>> {
683 let affinity = if let Some(resolver) = &self.affinity_resolver {
684 match resolver
685 .resolve(&[AffinityReference::Datanode(from_peer_id)])
686 .await
687 {
688 Ok(mut resolved) => {
689 ensure!(
690 resolved.len() == 1,
691 error::UnexpectedSnafu {
692 violated: "affinity resolution must preserve the number and order of references",
693 }
694 );
695 let affinity = resolved.pop().flatten();
696 if affinity.is_none() {
697 warn!(
698 "Source affinity unavailable for datanode {}; selecting failover peers without affinity",
699 from_peer_id
700 );
701 }
702 affinity
703 }
704 Err(error) => {
705 warn!(error; "Failed to resolve failover affinity for datanode {}; selecting without affinity", from_peer_id);
706 None
707 }
708 }
709 } else {
710 None
711 };
712 let mut exclude_peer_ids: HashSet<u64> =
713 HashSet::from_iter(failure_datanodes.iter().copied());
714 let unschedulable_datanodes = self
715 .selector_context
716 .peer_discovery
717 .active_datanodes(None)
718 .await
719 .map_err(common_error::ext::BoxedError::new)
720 .context(error::OtherSnafu)?
721 .into_iter()
722 .filter(|datanode| !datanode.is_schedulable())
723 .map(|datanode| datanode.peer.id);
724 exclude_peer_ids.extend(unschedulable_datanodes);
725
726 let region_peers = match &self.selector {
727 RegionSupervisorSelector::NaiveSelector(selector) => {
728 let empty = BTreeSet::new();
731 let mut groups = BTreeMap::<_, Vec<_>>::new();
732 for region_id in regions {
733 let excluded = exclude_peer_ids_by_region.get(region_id).unwrap_or(&empty);
734 groups.entry(excluded).or_default().push(*region_id);
735 }
736 let mut region_peers = Vec::with_capacity(regions.len());
737 for (excluded, regions) in groups {
738 let mut exclude_peer_ids = exclude_peer_ids.clone();
739 exclude_peer_ids.extend(excluded);
740 let opts = SelectorOptions {
741 affinity: affinity.as_ref(),
742 min_required_items: regions.len(),
743 allow_duplication: true,
744 exclude_peer_ids,
745 workload_filter: Some(accept_ingest_workload),
746 };
747 match selector.select(&self.selector_context, opts).await {
748 Ok(peers) => region_peers.extend(regions.into_iter().zip(peers)),
749 Err(error::Error::NoEnoughAvailableNode { .. }) => {}
751 Err(err) => return Err(err),
752 }
753 }
754 region_peers
755 }
756 RegionSupervisorSelector::RegionStatAwareSelector(selector) => {
757 let selected = selector
758 .select(
759 &self.selector_context,
760 from_peer_id,
761 regions,
762 exclude_peer_ids,
763 exclude_peer_ids_by_region,
764 affinity.as_ref(),
765 )
766 .await;
767 match selected {
768 Err(error::Error::NoEnoughAvailableNode { .. }) => vec![],
769 selected => selected?,
770 }
771 }
772 };
773 let placed = region_peers
774 .iter()
775 .map(|(region_id, _)| *region_id)
776 .collect::<HashSet<_>>();
777 let unplaced = regions
778 .iter()
779 .filter(|region_id| !placed.contains(region_id))
780 .collect::<Vec<_>>();
781 if !unplaced.is_empty() {
782 error!("No failover target for regions: {:?}", unplaced);
783 }
784 Ok(region_peers)
785 }
786
787 async fn generate_failover_tasks(
788 &mut self,
789 from_peer_id: DatanodeId,
790 regions: &[RegionId],
791 failed_datanodes: &[DatanodeId],
792 ) -> Result<Vec<(RegionMigrationProcedureTask, u32)>> {
793 let mut tasks = Vec::with_capacity(regions.len());
794 let from_peer = self
795 .peer_resolver
796 .datanode(from_peer_id)
797 .await
798 .ok()
799 .flatten()
800 .unwrap_or_else(|| Peer::empty(from_peer_id));
801
802 let table_ids = regions
804 .iter()
805 .map(|region_id| region_id.table_id())
806 .unique()
807 .collect::<Vec<_>>();
808 let table_routes = TableRouteStorage::new(self.kv_backend.clone())
809 .batch_get(&table_ids)
810 .await
811 .unwrap_or_else(|err| {
812 warn!(err; "Failed to read the routes of failover regions: {:?}", regions);
814 vec![]
815 });
816 let exclude_peer_ids_by_region = table_routes
817 .iter()
818 .flatten()
819 .filter_map(|table_route| table_route.region_routes().ok())
820 .flatten()
821 .map(|route| {
822 let peers = route.follower_peers.iter().map(|peer| peer.id);
823 (route.region.id, peers.collect())
824 })
825 .collect();
826 let region_peers = self
827 .select_peers(
828 from_peer_id,
829 regions,
830 failed_datanodes,
831 &exclude_peer_ids_by_region,
832 )
833 .await?;
834
835 for (region_id, peer) in region_peers {
836 let count = *self
837 .failover_counts
838 .entry((from_peer_id, region_id))
839 .and_modify(|count| *count += 1)
840 .or_insert(1);
841 let task = RegionMigrationProcedureTask {
842 region_id,
843 from_peer: from_peer.clone(),
844 to_peer: peer,
845 timeout: DEFAULT_REGION_MIGRATION_TIMEOUT * count,
846 trigger_reason: RegionMigrationTriggerReason::Failover,
847 };
848 tasks.push((task, count));
849 }
850
851 Ok(tasks)
852 }
853
854 async fn do_failover_tasks(&mut self, task: RegionMigrationTaskBatch) -> Result<()> {
855 let from_peer_id = task.from_peer.id;
856 let to_peer_id = task.to_peer.id;
857 let timeout = task.timeout;
858 let trigger_reason = task.trigger_reason;
859 let result = self
860 .region_migration_manager
861 .submit_region_migration_task(task)
862 .await?;
863 self.handle_submit_region_migration_task_result(
864 from_peer_id,
865 to_peer_id,
866 timeout,
867 trigger_reason,
868 result,
869 )
870 .await
871 }
872
873 async fn handle_submit_region_migration_task_result(
874 &mut self,
875 from_peer_id: DatanodeId,
876 to_peer_id: DatanodeId,
877 timeout: Duration,
878 trigger_reason: RegionMigrationTriggerReason,
879 result: SubmitRegionMigrationTaskResult,
880 ) -> Result<()> {
881 if !result.migrated.is_empty() {
882 let detecting_regions = result
883 .migrated
884 .iter()
885 .map(|region_id| (from_peer_id, *region_id))
886 .collect::<Vec<_>>();
887 self.deregister_failure_detectors(detecting_regions).await;
888 info!(
889 "Region has been migrated to target peer: {}, removed failover detectors for regions: {:?}",
890 to_peer_id, result.migrated,
891 )
892 }
893 if !result.migrating.is_empty() {
894 info!(
895 "Region is still migrating, skipping failover for regions: {:?}",
896 result.migrating
897 );
898 }
899 if !result.region_not_found.is_empty() {
900 let detecting_regions = result
901 .region_not_found
902 .iter()
903 .map(|region_id| (from_peer_id, *region_id))
904 .collect::<Vec<_>>();
905 self.deregister_failure_detectors(detecting_regions).await;
906 info!(
907 "Region route not found, removed failover detectors for regions: {:?}",
908 result.region_not_found
909 );
910 }
911 if !result.table_not_found.is_empty() {
912 let detecting_regions = result
913 .table_not_found
914 .iter()
915 .map(|region_id| (from_peer_id, *region_id))
916 .collect::<Vec<_>>();
917 self.deregister_failure_detectors(detecting_regions).await;
918 info!(
919 "Table is not found, removed failover detectors for regions: {:?}",
920 result.table_not_found
921 );
922 }
923 if !result.leader_changed.is_empty() {
924 let detecting_regions = result
925 .leader_changed
926 .iter()
927 .map(|region_id| (from_peer_id, *region_id))
928 .collect::<Vec<_>>();
929 self.deregister_failure_detectors(detecting_regions).await;
930 info!(
931 "Region's leader peer changed, removed failover detectors for regions: {:?}",
932 result.leader_changed
933 );
934 }
935 if !result.peer_conflict.is_empty() {
936 info!(
937 "Region has peer conflict, ignore failover for regions: {:?}",
938 result.peer_conflict
939 );
940 }
941 if !result.submitted.is_empty() {
942 info!(
943 "Failover for regions: {:?}, from_peer: {}, to_peer: {}, procedure_id: {:?}, timeout: {:?}, trigger_reason: {:?}",
944 result.submitted,
945 from_peer_id,
946 to_peer_id,
947 result.procedure_id,
948 timeout,
949 trigger_reason,
950 );
951 }
952
953 Ok(())
954 }
955
956 fn detect_region_failure(&self) -> Vec<(DatanodeId, RegionId)> {
958 self.failure_detector
959 .iter()
960 .filter_map(|e| {
961 if !e.failure_detector().is_available(current_time_millis()) {
968 Some(*e.region_ident())
969 } else {
970 None
971 }
972 })
973 .collect::<Vec<_>>()
974 }
975
976 fn regions(&self) -> HashSet<RegionId> {
978 self.failure_detector
979 .iter()
980 .map(|e| e.region_ident().1)
981 .collect::<HashSet<_>>()
982 }
983
984 fn on_heartbeat_arrived(&self, heartbeat: DatanodeHeartbeat) {
986 let mut regions = heartbeat.regions;
987 if !heartbeat.opening_regions.is_empty() {
988 let mut reported_regions: HashSet<_> = regions.iter().copied().collect();
989 regions.extend(
991 heartbeat
992 .opening_regions
993 .into_iter()
994 .filter(|region_id| reported_regions.insert(*region_id)),
995 );
996 }
997 for region_id in regions {
998 let detecting_region = (heartbeat.datanode_id, region_id);
999 let mut detector = self
1000 .failure_detector
1001 .region_failure_detector(detecting_region);
1002 detector.heartbeat(heartbeat.timestamp);
1003 }
1004 }
1005
1006 fn clear(&self) {
1007 self.failure_detector.clear();
1008 }
1009}
1010
1011#[cfg(test)]
1012pub(crate) mod tests {
1013 use std::assert_matches;
1014 use std::collections::HashMap;
1015 use std::sync::{Arc, Mutex};
1016 use std::time::Duration;
1017
1018 use common_meta::affinity::NoopAffinity;
1019 use common_meta::ddl::RegionFailureDetectorController;
1020 use common_meta::ddl::test_util::{
1021 test_create_logical_table_task, test_create_physical_table_task,
1022 };
1023 use common_meta::key::table_route::{
1024 LogicalTableRouteValue, PhysicalTableRouteValue, TableRouteValue,
1025 };
1026 use common_meta::key::{TableMetadataManager, runtime_switch};
1027 use common_meta::peer::Peer;
1028 use common_meta::rpc::router::{Region, RegionRoute};
1029 use common_meta::test_util::NoopPeerResolver;
1030 use common_telemetry::info;
1031 use common_time::util::current_time_millis;
1032 use rand::Rng;
1033 use store_api::storage::RegionId;
1034 use tokio::sync::mpsc::Sender;
1035 use tokio::sync::oneshot;
1036 use tokio::time::sleep;
1037
1038 use super::RegionSupervisorSelector;
1039 use crate::metasrv::SelectorContext;
1040 use crate::metrics::METRIC_META_HEARTBEAT_DROPPED;
1041 use crate::procedure::region_migration::RegionMigrationTriggerReason;
1042 use crate::procedure::region_migration::manager::{
1043 RegionMigrationManager, SubmitRegionMigrationTaskResult,
1044 };
1045 use crate::procedure::region_migration::test_util::TestingEnv;
1046 use crate::region::supervisor::{
1047 DatanodeHeartbeat, Event, HeartbeatAcceptor, RegionFailureDetectorControl,
1048 RegionSupervisor, RegionSupervisorTicker,
1049 };
1050 use crate::selector::test_utils::{RandomNodeSelector, new_test_selector_context};
1051
1052 pub(crate) fn new_test_supervisor() -> (RegionSupervisor, Sender<Event>) {
1053 new_test_supervisor_with(
1054 new_test_selector_context(),
1055 RegionSupervisorSelector::NaiveSelector(Arc::new(RandomNodeSelector::new(vec![
1056 Peer::empty(1),
1057 ]))),
1058 )
1059 }
1060
1061 pub(crate) fn new_test_supervisor_with(
1062 selector_context: SelectorContext,
1063 selector: RegionSupervisorSelector,
1064 ) -> (RegionSupervisor, Sender<Event>) {
1065 let env = TestingEnv::new();
1066 let context_factory = env.context_factory();
1067 let region_migration_manager = Arc::new(RegionMigrationManager::new(
1068 env.procedure_manager().clone(),
1069 context_factory,
1070 ));
1071 let runtime_switch_manager =
1072 Arc::new(runtime_switch::RuntimeSwitchManager::new(env.kv_backend()));
1073 let peer_resolver = Arc::new(NoopPeerResolver);
1074 let (tx, rx) = RegionSupervisor::channel();
1075 let kv_backend = env.kv_backend();
1076
1077 (
1078 RegionSupervisor::new(
1079 rx,
1080 Default::default(),
1081 selector_context,
1082 selector,
1083 region_migration_manager,
1084 runtime_switch_manager,
1085 peer_resolver,
1086 kv_backend,
1087 ),
1088 tx,
1089 )
1090 }
1091
1092 #[tokio::test]
1093 async fn test_failover_affinity_fallback_and_selector_errors() {
1094 use std::collections::{BTreeSet, HashSet};
1095 use std::sync::atomic::{AtomicUsize, Ordering};
1096
1097 use common_meta::affinity::{
1098 AffinityFallback, AffinityReference, AffinityRequirements, AffinityResolver,
1099 };
1100 use common_meta::error::{self as meta_error, Result as MetaResult};
1101
1102 use crate::error::{self, Result};
1103 use crate::metasrv::{SelectTarget, SelectorContext};
1104 use crate::selector::{RegionStatAwareSelector, Selector, SelectorOptions};
1105
1106 #[derive(Clone, Copy, PartialEq)]
1107 enum Resolution {
1108 Disabled,
1109 Available,
1110 Missing,
1111 Error,
1112 }
1113
1114 struct Resolver {
1115 result: Resolution,
1116 calls: AtomicUsize,
1117 }
1118
1119 #[async_trait::async_trait]
1120 impl AffinityResolver for Resolver {
1121 async fn resolve(
1122 &self,
1123 references: &[AffinityReference],
1124 ) -> MetaResult<Vec<Option<AffinityRequirements>>> {
1125 assert_eq!(references, &[AffinityReference::Datanode(7)]);
1126 self.calls.fetch_add(1, Ordering::Relaxed);
1127 match self.result {
1128 Resolution::Available => {
1129 Ok(vec![AffinityRequirements::new(HashMap::from([(
1130 "pool".into(),
1131 "a".into(),
1132 )]))])
1133 }
1134 Resolution::Disabled => unreachable!(),
1135 Resolution::Missing => Ok(vec![None]),
1136 Resolution::Error => meta_error::UnexpectedSnafu {
1137 err_msg: "source lookup failed",
1138 }
1139 .fail(),
1140 }
1141 }
1142 }
1143
1144 #[derive(Clone, Copy)]
1145 enum Outcome {
1146 Success,
1147 Short,
1148 Exhausted,
1149 Error,
1150 }
1151 struct FailoverSelector {
1152 outcome: Outcome,
1153 expected_fallback: AffinityFallback,
1154 calls: Mutex<Vec<Option<AffinityRequirements>>>,
1155 }
1156 impl FailoverSelector {
1157 fn select_peers(&self, affinity: Option<&AffinityRequirements>) -> Result<Vec<Peer>> {
1158 self.calls.lock().unwrap().push(affinity.cloned());
1159 if affinity.is_some() {
1160 match self.outcome {
1161 Outcome::Success => {}
1162 Outcome::Short => return Ok(vec![Peer::empty(9)]),
1163 Outcome::Exhausted => {
1164 return error::NoEnoughAvailableNodeSnafu {
1165 required: 2usize,
1166 available: 0usize,
1167 select_target: SelectTarget::Datanode,
1168 }
1169 .fail();
1170 }
1171 Outcome::Error => {
1172 return error::UnexpectedSnafu {
1173 violated: "selection backend failed",
1174 }
1175 .fail();
1176 }
1177 }
1178 }
1179 Ok(vec![Peer::empty(9); 2])
1180 }
1181 }
1182 #[async_trait::async_trait]
1183 impl Selector for FailoverSelector {
1184 type Context = SelectorContext;
1185 type Output = Vec<Peer>;
1186 async fn select(
1187 &self,
1188 ctx: &SelectorContext,
1189 opts: SelectorOptions<'_>,
1190 ) -> Result<Self::Output> {
1191 assert_eq!(ctx.affinity_fallback, self.expected_fallback);
1192 assert_eq!(opts.exclude_peer_ids, HashSet::from([8]));
1193 assert!(opts.workload_filter.is_some());
1194 assert!(opts.allow_duplication);
1195 assert_eq!(opts.min_required_items, 2);
1196 self.select_peers(opts.affinity)
1197 }
1198 }
1199 #[async_trait::async_trait]
1200 impl RegionStatAwareSelector for FailoverSelector {
1201 type Context = SelectorContext;
1202 type Output = Vec<(RegionId, Peer)>;
1203 async fn select(
1204 &self,
1205 ctx: &SelectorContext,
1206 source: u64,
1207 regions: &[RegionId],
1208 excluded: HashSet<u64>,
1209 exclude_peer_ids_by_region: &HashMap<RegionId, BTreeSet<u64>>,
1210 affinity: Option<&AffinityRequirements>,
1211 ) -> Result<Self::Output> {
1212 assert_eq!(ctx.affinity_fallback, self.expected_fallback);
1213 assert_eq!(source, 7);
1214 assert_eq!(excluded, HashSet::from([8]));
1215 assert!(exclude_peer_ids_by_region.is_empty());
1216 Ok(regions
1217 .iter()
1218 .copied()
1219 .zip(self.select_peers(affinity)?)
1220 .collect())
1221 }
1222 }
1223 for stat_aware in [false, true] {
1224 for (resolution, outcome, succeeds) in [
1225 (Resolution::Available, Outcome::Success, true),
1226 (Resolution::Available, Outcome::Short, false),
1227 (Resolution::Available, Outcome::Exhausted, false),
1228 (Resolution::Available, Outcome::Error, false),
1229 (Resolution::Missing, Outcome::Success, true),
1230 (Resolution::Error, Outcome::Success, true),
1231 (Resolution::Disabled, Outcome::Success, true),
1232 ] {
1233 let resolver = Arc::new(Resolver {
1234 result: resolution,
1235 calls: AtomicUsize::new(0),
1236 });
1237 let selector = Arc::new(FailoverSelector {
1238 outcome,
1239 expected_fallback: AffinityFallback::Unconstrained,
1240 calls: Mutex::new(vec![]),
1241 });
1242 let (mut supervisor, _sender) = new_test_supervisor();
1243 if resolution != Resolution::Disabled {
1244 supervisor = supervisor.with_affinity_resolver(resolver.clone());
1245 }
1246 supervisor.selector = if stat_aware {
1247 RegionSupervisorSelector::RegionStatAwareSelector(selector.clone())
1248 } else {
1249 RegionSupervisorSelector::NaiveSelector(selector.clone())
1250 };
1251 let regions = [RegionId::new(1, 1), RegionId::new(1, 2)];
1252 let result = supervisor
1254 .select_peers(7, ®ions, &[8], &HashMap::new())
1255 .await;
1256 if succeeds {
1257 assert_eq!(
1258 result.unwrap(),
1259 vec![(regions[0], Peer::empty(9)), (regions[1], Peer::empty(9))]
1260 );
1261 } else {
1262 match outcome {
1263 Outcome::Short => {
1265 assert_eq!(result.unwrap(), vec![(regions[0], Peer::empty(9))])
1266 }
1267 Outcome::Exhausted => assert!(result.unwrap().is_empty()),
1268 Outcome::Error => assert!(matches!(
1269 result.unwrap_err(),
1270 error::Error::Unexpected { .. }
1271 )),
1272 Outcome::Success => unreachable!(),
1273 }
1274 }
1275 assert_eq!(
1276 resolver.calls.load(Ordering::Relaxed),
1277 usize::from(resolution != Resolution::Disabled)
1278 );
1279 let calls = selector.calls.lock().unwrap();
1280 assert_eq!(calls.len(), 1);
1281 if let Some(affinity) = calls.first() {
1282 assert_eq!(affinity.is_some(), resolution == Resolution::Available);
1283 }
1284 }
1285 }
1286 }
1287
1288 fn filler_heartbeat() -> DatanodeHeartbeat {
1289 DatanodeHeartbeat {
1290 opening_regions: Vec::new(),
1291 datanode_id: 1,
1292 regions: vec![],
1293 timestamp: 0,
1294 }
1295 }
1296
1297 #[tokio::test]
1298 async fn test_select_peers_excludes_unschedulable_datanodes() {
1299 use std::collections::HashSet as StdHashSet;
1300
1301 use crate::key::NodeSchedulingRole;
1302 use crate::selector::{Selector, SelectorOptions};
1303 use crate::test_util::{
1304 create_meta_peer_client, new_node_scheduling_discovery, put_datanodes,
1305 };
1306
1307 struct ExcludeAssertSelector;
1309
1310 #[async_trait::async_trait]
1311 impl Selector for ExcludeAssertSelector {
1312 type Context = SelectorContext;
1313 type Output = Vec<Peer>;
1314
1315 async fn select(
1316 &self,
1317 _ctx: &Self::Context,
1318 opts: SelectorOptions<'_>,
1319 ) -> crate::error::Result<Self::Output> {
1320 assert_eq!(StdHashSet::from([7, 8]), opts.exclude_peer_ids);
1321 Ok(vec![Peer::empty(9)])
1322 }
1323 }
1324
1325 let client = create_meta_peer_client();
1327 put_datanodes(&client, vec![Peer::empty(7)]).await;
1328 let (manager, discovery) = new_node_scheduling_discovery(&client);
1329 manager
1330 .mark_unschedulable(NodeSchedulingRole::Datanode, 7)
1331 .await
1332 .unwrap();
1333
1334 let selector_context = SelectorContext {
1335 affinity_fallback: Default::default(),
1336 affinity: Arc::new(NoopAffinity),
1337 peer_discovery: Arc::new(discovery),
1338 };
1339 let (supervisor, _tx) = new_test_supervisor_with(
1340 selector_context,
1341 RegionSupervisorSelector::NaiveSelector(Arc::new(ExcludeAssertSelector)),
1342 );
1343
1344 let region_peers = supervisor
1345 .select_peers(7, &[RegionId::new(1, 1)], &[8], &HashMap::new())
1346 .await
1347 .unwrap();
1348 assert_eq!(1, region_peers.len());
1349 assert_eq!(Peer::empty(9), region_peers[0].1);
1350 }
1351
1352 #[tokio::test]
1353 async fn test_failover_excludes_route_followers() {
1354 use std::collections::{BTreeSet, HashSet as StdHashSet};
1355
1356 use common_meta::affinity::AffinityRequirements;
1357 use common_meta::key::MetadataKey;
1358 use common_meta::key::table_route::TableRouteKey;
1359 use common_meta::rpc::store::PutRequest;
1360
1361 use crate::error::{self, Result};
1362 use crate::metasrv::SelectTarget;
1363 use crate::selector::{RegionStatAwareSelector, Selector, SelectorOptions};
1364
1365 struct RecordingSelector(Mutex<Vec<(StdHashSet<u64>, usize)>>);
1367
1368 #[async_trait::async_trait]
1369 impl Selector for RecordingSelector {
1370 type Context = SelectorContext;
1371 type Output = Vec<Peer>;
1372
1373 async fn select(
1374 &self,
1375 _ctx: &Self::Context,
1376 opts: SelectorOptions<'_>,
1377 ) -> Result<Self::Output> {
1378 let required = opts.min_required_items;
1379 let no_candidate = opts.exclude_peer_ids.contains(&3);
1380 self.0
1381 .lock()
1382 .unwrap()
1383 .push((opts.exclude_peer_ids, required));
1384 if no_candidate {
1385 return error::NoEnoughAvailableNodeSnafu {
1386 required,
1387 available: 0usize,
1388 select_target: SelectTarget::Datanode,
1389 }
1390 .fail();
1391 }
1392 Ok(vec![Peer::empty(3); required])
1393 }
1394 }
1395
1396 type Batch = (Vec<RegionId>, HashMap<RegionId, BTreeSet<u64>>);
1398
1399 struct StatAwareRecorder(Mutex<Vec<Batch>>);
1401
1402 #[async_trait::async_trait]
1403 impl RegionStatAwareSelector for StatAwareRecorder {
1404 type Context = SelectorContext;
1405 type Output = Vec<(RegionId, Peer)>;
1406
1407 async fn select(
1408 &self,
1409 _ctx: &Self::Context,
1410 _from_peer_id: u64,
1411 region_ids: &[RegionId],
1412 exclude_peer_ids: StdHashSet<u64>,
1413 exclude_peer_ids_by_region: &HashMap<RegionId, BTreeSet<u64>>,
1414 _affinity: Option<&AffinityRequirements>,
1415 ) -> Result<Self::Output> {
1416 assert_eq!(exclude_peer_ids, StdHashSet::from([8]));
1417 self.0
1418 .lock()
1419 .unwrap()
1420 .push((region_ids.to_vec(), exclude_peer_ids_by_region.clone()));
1421 Ok(region_ids
1422 .iter()
1423 .map(|region_id| (*region_id, Peer::empty(9)))
1424 .collect())
1425 }
1426 }
1427
1428 let selector = Arc::new(RecordingSelector(Mutex::new(vec![])));
1429 let (mut supervisor, _tx) = new_test_supervisor_with(
1430 new_test_selector_context(),
1431 RegionSupervisorSelector::NaiveSelector(selector.clone()),
1432 );
1433 let table_id = 1024;
1434 let region_route = |region_number, followers: &[u64]| RegionRoute {
1435 region: Region {
1436 id: RegionId::new(table_id, region_number),
1437 ..Default::default()
1438 },
1439 leader_peer: Some(Peer::empty(8)),
1440 follower_peers: followers.iter().map(|id| Peer::empty(*id)).collect(),
1441 ..Default::default()
1442 };
1443 let table_route = PhysicalTableRouteValue::new(vec![
1444 region_route(0, &[]),
1445 region_route(1, &[3]),
1446 region_route(2, &[4]),
1447 region_route(3, &[4]),
1448 ]);
1449 let mut task = test_create_physical_table_task("my_physical_table");
1450 task.set_table_id(table_id);
1451 TableMetadataManager::new(supervisor.kv_backend.clone())
1452 .create_table_metadata(
1453 task.table_info,
1454 TableRouteValue::Physical(table_route),
1455 HashMap::new(),
1456 )
1457 .await
1458 .unwrap();
1459
1460 let mut regions = (0..4)
1461 .map(|number| RegionId::new(table_id, number))
1462 .collect::<Vec<_>>();
1463 regions.push(RegionId::new(table_id + 1, 0));
1465 let tasks = supervisor
1466 .generate_failover_tasks(8, ®ions, &[8])
1467 .await
1468 .unwrap();
1469
1470 assert_eq!(
1472 *selector.0.lock().unwrap(),
1473 vec![
1474 (StdHashSet::from([8]), 2),
1475 (StdHashSet::from([8, 3]), 1),
1476 (StdHashSet::from([8, 4]), 2),
1477 ]
1478 );
1479 let failed_over = tasks
1481 .iter()
1482 .map(|(task, _)| task.region_id)
1483 .collect::<Vec<_>>();
1484 assert_eq!(
1485 failed_over,
1486 [regions[0], regions[4], regions[2], regions[3]]
1487 );
1488
1489 let stat_aware = Arc::new(StatAwareRecorder(Mutex::new(vec![])));
1491 supervisor.selector = RegionSupervisorSelector::RegionStatAwareSelector(stat_aware.clone());
1492 let tasks = supervisor
1493 .generate_failover_tasks(8, ®ions, &[8])
1494 .await
1495 .unwrap();
1496 let followers = HashMap::from([
1497 (regions[0], BTreeSet::new()),
1498 (regions[1], BTreeSet::from([3])),
1499 (regions[2], BTreeSet::from([4])),
1500 (regions[3], BTreeSet::from([4])),
1501 ]);
1502 assert_eq!(
1503 *stat_aware.0.lock().unwrap(),
1504 vec![(regions.clone(), followers)]
1505 );
1506 let failed_over = tasks
1507 .iter()
1508 .map(|(task, _)| task.region_id)
1509 .collect::<Vec<_>>();
1510 assert_eq!(failed_over, regions);
1511 supervisor.selector = RegionSupervisorSelector::NaiveSelector(selector.clone());
1512
1513 supervisor
1515 .kv_backend
1516 .put(
1517 PutRequest::new()
1518 .with_key(TableRouteKey::new(table_id + 1).to_bytes())
1519 .with_value(b"corrupted".to_vec()),
1520 )
1521 .await
1522 .unwrap();
1523 selector.0.lock().unwrap().clear();
1524 let tasks = supervisor
1525 .generate_failover_tasks(8, ®ions, &[8])
1526 .await
1527 .unwrap();
1528 assert_eq!(
1529 *selector.0.lock().unwrap(),
1530 vec![(StdHashSet::from([8]), 5)]
1531 );
1532 assert_eq!(tasks.len(), 5);
1533 }
1534
1535 #[tokio::test]
1536 async fn test_opening_regions_refresh_detectors_until_serving() {
1537 let (supervisor, _sender) = new_test_supervisor();
1538 let region_id = RegionId::new(1, 1);
1539 let other_region = RegionId::new(1, 2);
1540 let new_region = RegionId::new(1, 3);
1541 for region in [(1, region_id), (1, other_region), (2, region_id)] {
1542 let mut detector = supervisor.failure_detector.region_failure_detector(region);
1543 detector.heartbeat(1_000);
1544 detector.heartbeat(2_000);
1545 }
1546
1547 let mut expected = supervisor
1548 .failure_detector
1549 .region_failure_detector((1, region_id))
1550 .clone();
1551 for timestamp in [60_000, 120_000] {
1552 expected.heartbeat(timestamp);
1553 assert!(
1554 !supervisor
1555 .failure_detector
1556 .region_failure_detector((1, region_id))
1557 .is_available(timestamp)
1558 );
1559 supervisor.on_heartbeat_arrived(DatanodeHeartbeat {
1560 datanode_id: 1,
1561 regions: vec![],
1562 opening_regions: vec![region_id, new_region],
1563 timestamp,
1564 });
1565 assert_eq!(
1566 expected,
1567 *supervisor
1568 .failure_detector
1569 .region_failure_detector((1, region_id))
1570 );
1571 for region in [region_id, new_region] {
1572 let detector = supervisor
1573 .failure_detector
1574 .region_failure_detector((1, region));
1575 assert_eq!(Some(timestamp), detector.last_heartbeat_millis());
1576 assert!(detector.is_available(timestamp));
1577 assert!(!detector.is_available(timestamp + 60_000));
1579 }
1580 for region in [(1, other_region), (2, region_id)] {
1581 let detector = supervisor.failure_detector.region_failure_detector(region);
1582 assert_eq!(Some(2_000), detector.last_heartbeat_millis());
1583 }
1584 }
1585
1586 for (timestamp, opening_regions) in [(121_000, vec![]), (122_000, vec![region_id])] {
1588 expected.heartbeat(timestamp);
1589 supervisor.on_heartbeat_arrived(DatanodeHeartbeat {
1590 datanode_id: 1,
1591 regions: vec![region_id],
1592 opening_regions,
1593 timestamp,
1594 });
1595 assert_eq!(
1596 expected,
1597 *supervisor
1598 .failure_detector
1599 .region_failure_detector((1, region_id))
1600 );
1601 }
1602 }
1603
1604 #[tokio::test]
1605 async fn test_heartbeat() {
1606 let (mut supervisor, sender) = new_test_supervisor();
1607 tokio::spawn(async move { supervisor.run().await });
1608
1609 sender
1610 .send(Event::HeartbeatArrived(DatanodeHeartbeat {
1611 opening_regions: Vec::new(),
1612 datanode_id: 0,
1613 regions: vec![RegionId::new(1, 1)],
1614 timestamp: 100,
1615 }))
1616 .await
1617 .unwrap();
1618 let (tx, rx) = oneshot::channel();
1619 sender.send(Event::Dump(tx)).await.unwrap();
1620 let detector = rx.await.unwrap();
1621 assert!(detector.contains(&(0, RegionId::new(1, 1))));
1622
1623 sender.send(Event::Clear).await.unwrap();
1625 let (tx, rx) = oneshot::channel();
1626 sender.send(Event::Dump(tx)).await.unwrap();
1627 assert!(rx.await.unwrap().is_empty());
1628
1629 fn generate_heartbeats(datanode_id: u64, region_ids: Vec<u32>) -> Vec<DatanodeHeartbeat> {
1630 let mut rng = rand::rng();
1631 let start = current_time_millis();
1632 (0..2000)
1633 .map(|i| DatanodeHeartbeat {
1634 opening_regions: Vec::new(),
1635 timestamp: start + i * 1000 + rng.random_range(0..100),
1636 datanode_id,
1637 regions: region_ids
1638 .iter()
1639 .map(|number| RegionId::new(0, *number))
1640 .collect(),
1641 })
1642 .collect::<Vec<_>>()
1643 }
1644
1645 let heartbeats = generate_heartbeats(100, vec![1, 2, 3]);
1646 let last_heartbeat_time = heartbeats.last().unwrap().timestamp;
1647 for heartbeat in heartbeats {
1648 sender
1649 .send(Event::HeartbeatArrived(heartbeat))
1650 .await
1651 .unwrap();
1652 }
1653
1654 let (tx, rx) = oneshot::channel();
1655 sender.send(Event::Dump(tx)).await.unwrap();
1656 let detector = rx.await.unwrap();
1657 assert_eq!(detector.len(), 3);
1658
1659 for e in detector.iter() {
1660 let fd = e.failure_detector();
1661 let acceptable_heartbeat_pause_millis = fd.acceptable_heartbeat_pause_millis() as i64;
1662 let start = last_heartbeat_time;
1663
1664 for i in 1..=acceptable_heartbeat_pause_millis / 1000 {
1666 let now = start + i * 1000;
1667 assert_eq!(fd.phi(now), 0.0);
1668 }
1669
1670 let now = start + acceptable_heartbeat_pause_millis + 1000;
1673 assert!(fd.phi(now) < fd.threshold() as _);
1674 let now = start + acceptable_heartbeat_pause_millis + 2000;
1675 assert!(fd.phi(now) > fd.threshold() as _);
1676 }
1677 }
1678
1679 #[tokio::test]
1680 async fn test_supervisor_ticker() {
1681 let (tx, mut rx) = tokio::sync::mpsc::channel(128);
1682 let ticker = RegionSupervisorTicker {
1683 tick_handle: Mutex::new(None),
1684 initialization_handle: Mutex::new(None),
1685 tick_interval: Duration::from_millis(10),
1686 initialization_delay: Duration::from_millis(100),
1687 initialization_retry_period: Duration::from_millis(100),
1688 sender: tx,
1689 };
1690 for _ in 0..2 {
1692 ticker.start();
1693 sleep(Duration::from_millis(100)).await;
1694 ticker.stop();
1695 assert!(!rx.is_empty());
1696 while let Ok(event) = rx.try_recv() {
1697 assert_matches!(
1698 event,
1699 Event::Tick | Event::Clear | Event::InitializeAllRegions(_)
1700 );
1701 }
1702 assert!(ticker.initialization_handle.lock().unwrap().is_none());
1703 assert!(ticker.tick_handle.lock().unwrap().is_none());
1704 }
1705 }
1706
1707 #[tokio::test]
1708 async fn test_initialize_all_regions_event_handling() {
1709 common_telemetry::init_default_ut_logging();
1710 let (tx, mut rx) = tokio::sync::mpsc::channel(128);
1711 let ticker = RegionSupervisorTicker {
1712 tick_handle: Mutex::new(None),
1713 initialization_handle: Mutex::new(None),
1714 tick_interval: Duration::from_millis(1000),
1715 initialization_delay: Duration::from_millis(50),
1716 initialization_retry_period: Duration::from_millis(50),
1717 sender: tx,
1718 };
1719 ticker.start();
1720 sleep(Duration::from_millis(60)).await;
1721 let handle = tokio::spawn(async move {
1722 let mut counter = 0;
1723 while let Some(event) = rx.recv().await {
1724 if let Event::InitializeAllRegions(tx) = event {
1725 if counter == 0 {
1726 counter += 1;
1728 continue;
1729 }
1730 tx.send(()).unwrap();
1731 info!("Responded initialize all regions event");
1732 break;
1733 }
1734 }
1735 rx
1736 });
1737
1738 let rx = handle.await.unwrap();
1739 for _ in 0..3 {
1740 sleep(Duration::from_millis(100)).await;
1741 assert!(rx.is_empty());
1742 }
1743 }
1744
1745 #[tokio::test]
1746 async fn test_initialize_all_regions() {
1747 common_telemetry::init_default_ut_logging();
1748 let (mut supervisor, sender) = new_test_supervisor();
1749 let table_metadata_manager = TableMetadataManager::new(supervisor.kv_backend.clone());
1750
1751 let table_id = 1024;
1753 let mut create_physical_table_task = test_create_physical_table_task("my_physical_table");
1754 create_physical_table_task.set_table_id(table_id);
1755 let table_info = create_physical_table_task.table_info;
1756 let table_route = PhysicalTableRouteValue::new(vec![RegionRoute {
1757 region: Region {
1758 id: RegionId::new(table_id, 0),
1759 ..Default::default()
1760 },
1761 leader_peer: Some(Peer::empty(1)),
1762 ..Default::default()
1763 }]);
1764 let table_route_value = TableRouteValue::Physical(table_route);
1765 table_metadata_manager
1766 .create_table_metadata(table_info, table_route_value, HashMap::new())
1767 .await
1768 .unwrap();
1769
1770 let logical_table_id = 1025;
1772 let mut test_create_logical_table_task = test_create_logical_table_task("my_logical_table");
1773 test_create_logical_table_task.set_table_id(logical_table_id);
1774 let table_info = test_create_logical_table_task.table_info;
1775 let table_route = LogicalTableRouteValue::new(1024);
1776 let table_route_value = TableRouteValue::Logical(table_route);
1777 table_metadata_manager
1778 .create_table_metadata(table_info, table_route_value, HashMap::new())
1779 .await
1780 .unwrap();
1781 tokio::spawn(async move { supervisor.run().await });
1782 let (tx, rx) = oneshot::channel();
1783 sender.send(Event::InitializeAllRegions(tx)).await.unwrap();
1784 assert!(rx.await.is_ok());
1785
1786 let (tx, rx) = oneshot::channel();
1787 sender.send(Event::Dump(tx)).await.unwrap();
1788 let detector = rx.await.unwrap();
1789 assert_eq!(detector.len(), 1);
1790 assert!(detector.contains(&(1, RegionId::new(1024, 0))));
1791 }
1792
1793 #[tokio::test]
1794 async fn test_initialize_all_regions_with_maintenance_mode() {
1795 common_telemetry::init_default_ut_logging();
1796 let (mut supervisor, sender) = new_test_supervisor();
1797
1798 supervisor
1799 .runtime_switch_manager
1800 .set_maintenance_mode()
1801 .await
1802 .unwrap();
1803 tokio::spawn(async move { supervisor.run().await });
1804 let (tx, rx) = oneshot::channel();
1805 sender.send(Event::InitializeAllRegions(tx)).await.unwrap();
1806 assert!(rx.await.is_err());
1808 }
1809
1810 #[tokio::test]
1811 async fn test_region_failure_detector_controller() {
1812 let (mut supervisor, sender) = new_test_supervisor();
1813 let controller = RegionFailureDetectorControl::new(sender.clone());
1814 tokio::spawn(async move { supervisor.run().await });
1815 let detecting_region = (1, RegionId::new(1, 1));
1816 controller
1817 .register_failure_detectors(vec![detecting_region])
1818 .await;
1819
1820 let (tx, rx) = oneshot::channel();
1821 sender.send(Event::Dump(tx)).await.unwrap();
1822 let detector = rx.await.unwrap();
1823 let region_detector = detector.region_failure_detector(detecting_region).clone();
1824
1825 controller
1827 .register_failure_detectors(vec![detecting_region])
1828 .await;
1829 let (tx, rx) = oneshot::channel();
1830 sender.send(Event::Dump(tx)).await.unwrap();
1831 let detector = rx.await.unwrap();
1832 let got = detector.region_failure_detector(detecting_region).clone();
1833 assert_eq!(region_detector, got);
1834
1835 controller
1836 .deregister_failure_detectors(vec![detecting_region])
1837 .await;
1838 let (tx, rx) = oneshot::channel();
1839 sender.send(Event::Dump(tx)).await.unwrap();
1840 assert!(rx.await.unwrap().is_empty());
1841 }
1842
1843 #[tokio::test]
1844 async fn test_handle_submit_region_migration_task_result_migrated() {
1845 common_telemetry::init_default_ut_logging();
1846 let (mut supervisor, _) = new_test_supervisor();
1847 let region_id = RegionId::new(1, 1);
1848 let detecting_region = (1, region_id);
1849 supervisor
1850 .register_failure_detectors(vec![detecting_region])
1851 .await;
1852 supervisor.failover_counts.insert(detecting_region, 1);
1853 let result = SubmitRegionMigrationTaskResult {
1854 migrated: vec![region_id],
1855 ..Default::default()
1856 };
1857 supervisor
1858 .handle_submit_region_migration_task_result(
1859 1,
1860 2,
1861 Duration::from_millis(1000),
1862 RegionMigrationTriggerReason::Manual,
1863 result,
1864 )
1865 .await
1866 .unwrap();
1867 assert!(!supervisor.failure_detector.contains(&detecting_region));
1868 assert!(supervisor.failover_counts.is_empty());
1869 }
1870
1871 #[tokio::test]
1872 async fn test_handle_submit_region_migration_task_result_migrating() {
1873 common_telemetry::init_default_ut_logging();
1874 let (mut supervisor, _) = new_test_supervisor();
1875 let region_id = RegionId::new(1, 1);
1876 let detecting_region = (1, region_id);
1877 supervisor
1878 .register_failure_detectors(vec![detecting_region])
1879 .await;
1880 supervisor.failover_counts.insert(detecting_region, 1);
1881 let result = SubmitRegionMigrationTaskResult {
1882 migrating: vec![region_id],
1883 ..Default::default()
1884 };
1885 supervisor
1886 .handle_submit_region_migration_task_result(
1887 1,
1888 2,
1889 Duration::from_millis(1000),
1890 RegionMigrationTriggerReason::Manual,
1891 result,
1892 )
1893 .await
1894 .unwrap();
1895 assert!(supervisor.failure_detector.contains(&detecting_region));
1896 assert!(supervisor.failover_counts.contains_key(&detecting_region));
1897 }
1898
1899 #[tokio::test]
1900 async fn test_handle_submit_region_migration_task_result_table_not_found() {
1901 common_telemetry::init_default_ut_logging();
1902 let (mut supervisor, _) = new_test_supervisor();
1903 let region_id = RegionId::new(1, 1);
1904 let detecting_region = (1, region_id);
1905 supervisor
1906 .register_failure_detectors(vec![detecting_region])
1907 .await;
1908 supervisor.failover_counts.insert(detecting_region, 1);
1909 let result = SubmitRegionMigrationTaskResult {
1910 table_not_found: vec![region_id],
1911 ..Default::default()
1912 };
1913 supervisor
1914 .handle_submit_region_migration_task_result(
1915 1,
1916 2,
1917 Duration::from_millis(1000),
1918 RegionMigrationTriggerReason::Manual,
1919 result,
1920 )
1921 .await
1922 .unwrap();
1923 assert!(!supervisor.failure_detector.contains(&detecting_region));
1924 assert!(supervisor.failover_counts.is_empty());
1925 }
1926
1927 #[tokio::test]
1928 async fn test_handle_submit_region_migration_task_result_region_not_found() {
1929 common_telemetry::init_default_ut_logging();
1930 let (mut supervisor, _) = new_test_supervisor();
1931 let region_id = RegionId::new(1, 1);
1932 let detecting_region = (1, region_id);
1933 supervisor
1934 .register_failure_detectors(vec![detecting_region])
1935 .await;
1936 supervisor.failover_counts.insert(detecting_region, 1);
1937 let result = SubmitRegionMigrationTaskResult {
1938 region_not_found: vec![region_id],
1939 ..Default::default()
1940 };
1941 supervisor
1942 .handle_submit_region_migration_task_result(
1943 1,
1944 2,
1945 Duration::from_millis(1000),
1946 RegionMigrationTriggerReason::Manual,
1947 result,
1948 )
1949 .await
1950 .unwrap();
1951 assert!(!supervisor.failure_detector.contains(&detecting_region));
1952 assert!(supervisor.failover_counts.is_empty());
1953 }
1954
1955 #[tokio::test]
1956 async fn test_handle_submit_region_migration_task_result_leader_changed() {
1957 common_telemetry::init_default_ut_logging();
1958 let (mut supervisor, _) = new_test_supervisor();
1959 let region_id = RegionId::new(1, 1);
1960 let detecting_region = (1, region_id);
1961 supervisor
1962 .register_failure_detectors(vec![detecting_region])
1963 .await;
1964 supervisor.failover_counts.insert(detecting_region, 1);
1965 let result = SubmitRegionMigrationTaskResult {
1966 leader_changed: vec![region_id],
1967 ..Default::default()
1968 };
1969 supervisor
1970 .handle_submit_region_migration_task_result(
1971 1,
1972 2,
1973 Duration::from_millis(1000),
1974 RegionMigrationTriggerReason::Manual,
1975 result,
1976 )
1977 .await
1978 .unwrap();
1979 assert!(!supervisor.failure_detector.contains(&detecting_region));
1980 assert!(supervisor.failover_counts.is_empty());
1981 }
1982
1983 #[tokio::test]
1984 async fn test_handle_submit_region_migration_task_result_peer_conflict() {
1985 common_telemetry::init_default_ut_logging();
1986 let (mut supervisor, _) = new_test_supervisor();
1987 let region_id = RegionId::new(1, 1);
1988 let detecting_region = (1, region_id);
1989 supervisor
1990 .register_failure_detectors(vec![detecting_region])
1991 .await;
1992 supervisor.failover_counts.insert(detecting_region, 1);
1993 let result = SubmitRegionMigrationTaskResult {
1994 peer_conflict: vec![region_id],
1995 ..Default::default()
1996 };
1997 supervisor
1998 .handle_submit_region_migration_task_result(
1999 1,
2000 2,
2001 Duration::from_millis(1000),
2002 RegionMigrationTriggerReason::Manual,
2003 result,
2004 )
2005 .await
2006 .unwrap();
2007 assert!(supervisor.failure_detector.contains(&detecting_region));
2008 assert!(supervisor.failover_counts.contains_key(&detecting_region));
2009 }
2010
2011 #[tokio::test]
2012 async fn test_handle_submit_region_migration_task_result_submitted() {
2013 common_telemetry::init_default_ut_logging();
2014 let (mut supervisor, _) = new_test_supervisor();
2015 let region_id = RegionId::new(1, 1);
2016 let detecting_region = (1, region_id);
2017 supervisor
2018 .register_failure_detectors(vec![detecting_region])
2019 .await;
2020 supervisor.failover_counts.insert(detecting_region, 1);
2021 let result = SubmitRegionMigrationTaskResult {
2022 submitted: vec![region_id],
2023 ..Default::default()
2024 };
2025 supervisor
2026 .handle_submit_region_migration_task_result(
2027 1,
2028 2,
2029 Duration::from_millis(1000),
2030 RegionMigrationTriggerReason::Manual,
2031 result,
2032 )
2033 .await
2034 .unwrap();
2035 assert!(supervisor.failure_detector.contains(&detecting_region));
2036 assert!(supervisor.failover_counts.contains_key(&detecting_region));
2037 }
2038
2039 #[tokio::test]
2040 async fn test_heartbeat_accept_does_not_block_on_full_queue() {
2041 let (mut supervisor, sender) = new_test_supervisor();
2044
2045 let mut queued = 0;
2046 while sender
2047 .try_send(Event::HeartbeatArrived(filler_heartbeat()))
2048 .is_ok()
2049 {
2050 queued += 1;
2051 }
2052 assert_eq!(queued, sender.max_capacity());
2053 assert_eq!(sender.capacity(), 0);
2054
2055 let acceptor = HeartbeatAcceptor::new(sender.clone());
2056 let dropped = || {
2057 METRIC_META_HEARTBEAT_DROPPED
2058 .with_label_values(&["full"])
2059 .get()
2060 };
2061 let dropped_before = dropped();
2062 acceptor.accept(filler_heartbeat());
2064 assert_eq!(dropped(), dropped_before + 1);
2065 assert_eq!(sender.capacity(), 0);
2066
2067 supervisor.receiver.try_recv().unwrap();
2069 acceptor.accept(filler_heartbeat());
2070 assert_eq!(dropped(), dropped_before + 1);
2071 assert_eq!(sender.capacity(), 0);
2072 }
2073}