Skip to main content

meta_srv/region/
supervisor.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
16use std::fmt::Debug;
17use std::sync::{Arc, Mutex};
18use std::time::{Duration, Instant};
19
20use async_trait::async_trait;
21use common_meta::DatanodeId;
22use common_meta::affinity::{AffinityFallback, AffinityReference, AffinityResolverRef};
23use common_meta::datanode::Stat;
24use common_meta::ddl::{DetectingRegion, RegionFailureDetectorController};
25use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
26use common_meta::key::table_route::{TableRouteKey, TableRouteStorage, TableRouteValue};
27use common_meta::key::{MetadataKey, MetadataValue};
28use common_meta::kv_backend::KvBackendRef;
29use common_meta::leadership_notifier::LeadershipChangeListener;
30use common_meta::peer::{Peer, PeerResolverRef};
31use common_meta::range_stream::{DEFAULT_PAGE_SIZE, PaginationStream};
32use common_meta::rpc::store::RangeRequest;
33use common_runtime::JoinHandle;
34use common_telemetry::{debug, error, info, warn};
35use common_time::util::current_time_millis;
36use futures::{StreamExt, TryStreamExt};
37use itertools::Itertools;
38use snafu::{ResultExt, ensure};
39use store_api::storage::RegionId;
40use tokio::sync::mpsc::error::TrySendError;
41use tokio::sync::mpsc::{Receiver, Sender};
42use tokio::sync::oneshot;
43use tokio::time::{MissedTickBehavior, interval, interval_at};
44
45use crate::discovery::utils::accept_ingest_workload;
46use crate::error::{self, Result};
47use crate::failure_detector::PhiAccrualFailureDetectorOptions;
48use crate::metasrv::{RegionStatAwareSelectorRef, SelectorContext, SelectorRef};
49use crate::metrics::METRIC_META_HEARTBEAT_DROPPED;
50use crate::procedure::region_migration::manager::{
51    RegionMigrationManagerRef, RegionMigrationTriggerReason, SubmitRegionMigrationTaskResult,
52};
53use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
54use crate::procedure::region_migration::{
55    DEFAULT_REGION_MIGRATION_TIMEOUT, RegionMigrationProcedureTask,
56};
57use crate::region::failure_detector::RegionFailureDetector;
58use crate::selector::SelectorOptions;
59use crate::state::StateRef;
60
61/// `DatanodeHeartbeat` represents the heartbeat signal sent from a datanode.
62/// It includes identifiers for the cluster and datanode, a list of regions being monitored,
63/// and a timestamp indicating when the heartbeat was sent.
64#[derive(Debug)]
65pub(crate) struct DatanodeHeartbeat {
66    datanode_id: DatanodeId,
67    // TODO(weny): Considers collecting the memtable size in regions.
68    regions: Vec<RegionId>,
69    pub(crate) opening_regions: Vec<RegionId>,
70    timestamp: i64,
71}
72
73impl From<&Stat> for DatanodeHeartbeat {
74    fn from(value: &Stat) -> Self {
75        DatanodeHeartbeat {
76            datanode_id: value.id,
77            regions: value.region_stats.iter().map(|x| x.id).collect(),
78            opening_regions: Vec::new(),
79            timestamp: value.timestamp_millis,
80        }
81    }
82}
83
84/// `Event` represents various types of events that can be processed by the region supervisor.
85/// These events are crucial for managing state transitions and handling specific scenarios
86/// in the region lifecycle.
87///
88/// Variants:
89/// - `Tick`: This event is used to trigger region failure detection periodically.
90/// - `InitializeAllRegions`: This event is used to initialize all region failure detectors.
91/// - `RegisterFailureDetectors`: This event is used to register failure detectors for regions.
92/// - `ResetFailureDetectors`: This event is used to reset failure detectors for regions.
93/// - `DeregisterFailureDetectors`: This event is used to deregister failure detectors for regions.
94/// - `HeartbeatArrived`: This event presents the metasrv received [`DatanodeHeartbeat`] from the datanodes.
95/// - `Clear`: This event is used to reset the state of the supervisor, typically used
96///   when a system-wide reset or reinitialization is needed.
97/// - `Dump`: (Available only in test) This event triggers a dump of the
98///   current state for debugging purposes. It allows developers to inspect the internal state
99///   of the supervisor during tests.
100pub(crate) enum Event {
101    Tick,
102    InitializeAllRegions(tokio::sync::oneshot::Sender<()>),
103    RegisterFailureDetectors(Vec<DetectingRegion>),
104    DeregisterFailureDetectors(Vec<DetectingRegion>),
105    ResetFailureDetectors(Vec<DetectingRegion>),
106    HeartbeatArrived(DatanodeHeartbeat),
107    Clear,
108    #[cfg(test)]
109    Dump(tokio::sync::oneshot::Sender<RegionFailureDetector>),
110}
111
112impl Debug for Event {
113    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
114        match self {
115            Self::Tick => write!(f, "Tick"),
116            Self::HeartbeatArrived(arg0) => f.debug_tuple("HeartbeatArrived").field(arg0).finish(),
117            Self::Clear => write!(f, "Clear"),
118            Self::InitializeAllRegions(_) => write!(f, "InspectAndRegisterRegions"),
119            Self::RegisterFailureDetectors(arg0) => f
120                .debug_tuple("RegisterFailureDetectors")
121                .field(arg0)
122                .finish(),
123            Self::ResetFailureDetectors(arg0) => {
124                f.debug_tuple("ResetFailureDetectors").field(arg0).finish()
125            }
126            Self::DeregisterFailureDetectors(arg0) => f
127                .debug_tuple("DeregisterFailureDetectors")
128                .field(arg0)
129                .finish(),
130            #[cfg(test)]
131            Self::Dump(_) => f.debug_struct("Dump").finish(),
132        }
133    }
134}
135
136pub type RegionSupervisorTickerRef = Arc<RegionSupervisorTicker>;
137
138/// A background job to generate [`Event::Tick`] type events.
139#[derive(Debug)]
140pub struct RegionSupervisorTicker {
141    /// The [`Option`] wrapper allows us to abort the job while dropping the [`RegionSupervisor`].
142    tick_handle: Mutex<Option<JoinHandle<()>>>,
143
144    /// The [`Option`] wrapper allows us to abort the job while dropping the [`RegionSupervisor`].
145    initialization_handle: Mutex<Option<JoinHandle<()>>>,
146
147    /// The interval of tick.
148    tick_interval: Duration,
149
150    /// The delay before initializing all region failure detectors.
151    initialization_delay: Duration,
152
153    /// The retry period for initializing all region failure detectors.
154    initialization_retry_period: Duration,
155
156    /// Sends [Event]s.
157    sender: Sender<Event>,
158}
159
160#[async_trait]
161impl LeadershipChangeListener for RegionSupervisorTicker {
162    fn name(&self) -> &'static str {
163        "RegionSupervisorTicker"
164    }
165
166    async fn on_leader_start(&self) -> common_meta::error::Result<()> {
167        self.start();
168        Ok(())
169    }
170
171    async fn on_leader_stop(&self) -> common_meta::error::Result<()> {
172        self.stop();
173        Ok(())
174    }
175}
176
177impl RegionSupervisorTicker {
178    pub(crate) fn new(
179        tick_interval: Duration,
180        initialization_delay: Duration,
181        initialization_retry_period: Duration,
182        sender: Sender<Event>,
183    ) -> Self {
184        info!(
185            "RegionSupervisorTicker is created, tick_interval: {:?}, initialization_delay: {:?}, initialization_retry_period: {:?}",
186            tick_interval, initialization_delay, initialization_retry_period
187        );
188        Self {
189            tick_handle: Mutex::new(None),
190            initialization_handle: Mutex::new(None),
191            tick_interval,
192            initialization_delay,
193            initialization_retry_period,
194            sender,
195        }
196    }
197
198    /// Starts the ticker.
199    pub fn start(&self) {
200        let mut handle = self.tick_handle.lock().unwrap();
201        if handle.is_none() {
202            let sender = self.sender.clone();
203            let tick_interval = self.tick_interval;
204            let initialization_delay = self.initialization_delay;
205
206            let mut initialization_interval = interval_at(
207                tokio::time::Instant::now() + initialization_delay,
208                self.initialization_retry_period,
209            );
210            initialization_interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
211            let initialization_handler = common_runtime::spawn_global(async move {
212                loop {
213                    initialization_interval.tick().await;
214                    let (tx, rx) = oneshot::channel();
215                    if sender.send(Event::InitializeAllRegions(tx)).await.is_err() {
216                        info!(
217                            "EventReceiver is dropped, region failure detectors initialization loop is stopped"
218                        );
219                        break;
220                    }
221                    if rx.await.is_ok() {
222                        info!("All region failure detectors are initialized.");
223                        break;
224                    }
225                }
226            });
227            *self.initialization_handle.lock().unwrap() = Some(initialization_handler);
228
229            let sender = self.sender.clone();
230            let ticker_loop = tokio::spawn(async move {
231                let mut tick_interval = interval(tick_interval);
232                tick_interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
233
234                if let Err(err) = sender.send(Event::Clear).await {
235                    warn!(err; "EventReceiver is dropped, failed to send Event::Clear");
236                    return;
237                }
238                loop {
239                    tick_interval.tick().await;
240                    if sender.send(Event::Tick).await.is_err() {
241                        info!("EventReceiver is dropped, tick loop is stopped");
242                        break;
243                    }
244                }
245            });
246            *handle = Some(ticker_loop);
247        }
248    }
249
250    /// Stops the ticker.
251    pub fn stop(&self) {
252        let handle = self.tick_handle.lock().unwrap().take();
253        if let Some(handle) = handle {
254            handle.abort();
255            info!("The tick loop is stopped.");
256        }
257        let initialization_handler = self.initialization_handle.lock().unwrap().take();
258        if let Some(initialization_handler) = initialization_handler {
259            initialization_handler.abort();
260            info!("The initialization loop is stopped.");
261        }
262    }
263}
264
265impl Drop for RegionSupervisorTicker {
266    fn drop(&mut self) {
267        self.stop();
268    }
269}
270
271pub type RegionSupervisorRef = Arc<RegionSupervisor>;
272
273/// The default tick interval.
274pub const DEFAULT_TICK_INTERVAL: Duration = Duration::from_secs(1);
275/// The default initialization retry period.
276pub const DEFAULT_INITIALIZATION_RETRY_PERIOD: Duration = Duration::from_secs(60);
277
278/// Selector for region supervisor.
279pub enum RegionSupervisorSelector {
280    NaiveSelector(SelectorRef),
281    RegionStatAwareSelector(RegionStatAwareSelectorRef),
282}
283
284/// The [`RegionSupervisor`] is used to detect Region failures
285/// and initiate Region failover upon detection, ensuring uninterrupted region service.
286pub struct RegionSupervisor {
287    /// Used to detect the failure of regions.
288    failure_detector: RegionFailureDetector,
289    /// Tracks the number of failovers for each region.
290    failover_counts: HashMap<DetectingRegion, u32>,
291    /// Receives [Event]s.
292    receiver: Receiver<Event>,
293    /// The context of [`SelectorRef`]
294    selector_context: SelectorContext,
295    /// Candidate node selector.
296    selector: RegionSupervisorSelector,
297    /// Resolves the failed node attributes before selection.
298    affinity_resolver: Option<AffinityResolverRef>,
299    /// Region migration manager.
300    region_migration_manager: RegionMigrationManagerRef,
301    /// The maintenance mode manager.
302    runtime_switch_manager: RuntimeSwitchManagerRef,
303    /// Peer resolver
304    peer_resolver: PeerResolverRef,
305    /// The kv backend.
306    kv_backend: KvBackendRef,
307    /// The meta state, used to check if the current metasrv is the leader.
308    state: Option<StateRef>,
309}
310
311/// Controller for managing failure detectors for regions.
312#[derive(Debug, Clone)]
313pub struct RegionFailureDetectorControl {
314    sender: Sender<Event>,
315}
316
317impl RegionFailureDetectorControl {
318    pub(crate) fn new(sender: Sender<Event>) -> Self {
319        Self { sender }
320    }
321}
322
323#[async_trait::async_trait]
324impl RegionFailureDetectorController for RegionFailureDetectorControl {
325    async fn register_failure_detectors(&self, detecting_regions: Vec<DetectingRegion>) {
326        if let Err(err) = self
327            .sender
328            .send(Event::RegisterFailureDetectors(detecting_regions))
329            .await
330        {
331            error!(err; "RegionSupervisor has stop receiving heartbeat.");
332        }
333    }
334
335    async fn reset_failure_detectors(&self, detecting_regions: Vec<DetectingRegion>) {
336        if let Err(err) = self
337            .sender
338            .send(Event::ResetFailureDetectors(detecting_regions))
339            .await
340        {
341            error!(err; "RegionSupervisor has stop receiving heartbeat.");
342        }
343    }
344
345    async fn deregister_failure_detectors(&self, detecting_regions: Vec<DetectingRegion>) {
346        if let Err(err) = self
347            .sender
348            .send(Event::DeregisterFailureDetectors(detecting_regions))
349            .await
350        {
351            error!(err; "RegionSupervisor has stop receiving heartbeat.");
352        }
353    }
354}
355
356/// [`HeartbeatAcceptor`] forwards heartbeats to [`RegionSupervisor`].
357#[derive(Clone)]
358pub(crate) struct HeartbeatAcceptor {
359    sender: Sender<Event>,
360}
361
362impl HeartbeatAcceptor {
363    pub(crate) fn new(sender: Sender<Event>) -> Self {
364        Self { sender }
365    }
366
367    /// Accepts heartbeats from datanodes without waiting for queue capacity: a full queue
368    /// drops the heartbeat, counted in [`METRIC_META_HEARTBEAT_DROPPED`]. A dropped
369    /// heartbeat costs the failure detector one sample.
370    pub(crate) fn accept(&self, heartbeat: DatanodeHeartbeat) {
371        let datanode_id = heartbeat.datanode_id;
372        match self.sender.try_send(Event::HeartbeatArrived(heartbeat)) {
373            Ok(()) => {}
374            Err(TrySendError::Full(_)) => {
375                METRIC_META_HEARTBEAT_DROPPED
376                    .with_label_values(&["full"])
377                    .inc();
378                warn!(
379                    "Dropping heartbeat because the region supervisor event queue is full, datanode_id: {}, queue_capacity: {}",
380                    datanode_id,
381                    self.sender.max_capacity()
382                );
383            }
384            Err(TrySendError::Closed(err)) => {
385                METRIC_META_HEARTBEAT_DROPPED
386                    .with_label_values(&["closed"])
387                    .inc();
388                error!(err; "RegionSupervisor has stop receiving heartbeat.");
389            }
390        }
391    }
392}
393
394impl RegionSupervisor {
395    /// Returns a mpsc channel with a buffer capacity of 1024 for sending and receiving `Event` messages.
396    pub(crate) fn channel() -> (Sender<Event>, Receiver<Event>) {
397        tokio::sync::mpsc::channel(1024)
398    }
399
400    #[allow(clippy::too_many_arguments)]
401    pub(crate) fn new(
402        event_receiver: Receiver<Event>,
403        options: PhiAccrualFailureDetectorOptions,
404        selector_context: SelectorContext,
405        selector: RegionSupervisorSelector,
406        region_migration_manager: RegionMigrationManagerRef,
407        runtime_switch_manager: RuntimeSwitchManagerRef,
408        peer_resolver: PeerResolverRef,
409        kv_backend: KvBackendRef,
410    ) -> Self {
411        Self {
412            failure_detector: RegionFailureDetector::new(options),
413            failover_counts: HashMap::new(),
414            receiver: event_receiver,
415            // Failover prioritizes recovery over affinity, independently of DDL policy.
416            selector_context: SelectorContext {
417                affinity_fallback: AffinityFallback::Unconstrained,
418                ..selector_context
419            },
420            selector,
421            affinity_resolver: None,
422            region_migration_manager,
423            runtime_switch_manager,
424            peer_resolver,
425            kv_backend,
426            state: None,
427        }
428    }
429
430    /// Sets the resolver used to prepare failover affinity requirements.
431    pub(crate) fn with_affinity_resolver(mut self, resolver: AffinityResolverRef) -> Self {
432        self.affinity_resolver = Some(resolver);
433        self
434    }
435
436    /// Sets the meta state.
437    pub(crate) fn with_state(mut self, state: StateRef) -> Self {
438        self.state = Some(state);
439        self
440    }
441
442    /// Runs the main loop.
443    pub(crate) async fn run(&mut self) {
444        while let Some(event) = self.receiver.recv().await {
445            if let Some(state) = self.state.as_ref()
446                && !state.read().unwrap().is_leader()
447            {
448                warn!(
449                    "The current metasrv is not the leader, ignore {:?} event",
450                    event
451                );
452                continue;
453            }
454
455            match event {
456                Event::InitializeAllRegions(sender) => {
457                    match self.is_maintenance_mode_enabled().await {
458                        Ok(false) => {}
459                        Ok(true) => {
460                            warn!(
461                                "Skipping initialize all regions since maintenance mode is enabled."
462                            );
463                            continue;
464                        }
465                        Err(err) => {
466                            error!(err; "Failed to check maintenance mode during initialize all regions.");
467                            continue;
468                        }
469                    }
470
471                    if let Err(err) = self.initialize_all().await {
472                        error!(err; "Failed to initialize all regions.");
473                    } else {
474                        // Ignore the error.
475                        let _ = sender.send(());
476                    }
477                }
478                Event::Tick => {
479                    let regions = self.detect_region_failure();
480                    self.handle_region_failures(regions).await;
481                }
482                Event::RegisterFailureDetectors(detecting_regions) => {
483                    self.register_failure_detectors(detecting_regions).await
484                }
485                Event::ResetFailureDetectors(detecting_regions) => {
486                    self.reset_failure_detectors(detecting_regions).await
487                }
488                Event::DeregisterFailureDetectors(detecting_regions) => {
489                    self.deregister_failure_detectors(detecting_regions).await
490                }
491                Event::HeartbeatArrived(heartbeat) => self.on_heartbeat_arrived(heartbeat),
492                Event::Clear => {
493                    self.clear();
494                    info!("Region supervisor is initialized.");
495                }
496                #[cfg(test)]
497                Event::Dump(sender) => {
498                    let _ = sender.send(self.failure_detector.dump());
499                }
500            }
501        }
502        info!("RegionSupervisor is stopped!");
503    }
504
505    async fn initialize_all(&self) -> Result<()> {
506        let now = Instant::now();
507        let regions = self.regions();
508        let req = RangeRequest::new().with_prefix(TableRouteKey::range_prefix());
509        let stream = PaginationStream::new(self.kv_backend.clone(), req, DEFAULT_PAGE_SIZE, |kv| {
510            TableRouteKey::from_bytes(&kv.key).map(|v| (v.table_id, kv.value))
511        })
512        .into_stream();
513
514        let mut stream = stream
515            .map_ok(|(_, value)| {
516                TableRouteValue::try_from_raw_value(&value)
517                    .context(error::TableMetadataManagerSnafu)
518            })
519            .boxed();
520        let mut detecting_regions = Vec::new();
521        while let Some(route) = stream
522            .try_next()
523            .await
524            .context(error::TableMetadataManagerSnafu)?
525        {
526            let route = route?;
527            if !route.is_physical() {
528                continue;
529            }
530
531            let physical_table_route = route.into_physical_table_route();
532            physical_table_route
533                .region_routes
534                .iter()
535                .for_each(|region_route| {
536                    if !regions.contains(&region_route.region.id)
537                        && let Some(leader_peer) = &region_route.leader_peer
538                    {
539                        detecting_regions.push((leader_peer.id, region_route.region.id));
540                    }
541                });
542        }
543
544        let num_detecting_regions = detecting_regions.len();
545        if !detecting_regions.is_empty() {
546            self.register_failure_detectors(detecting_regions).await;
547        }
548
549        info!(
550            "Initialize {} region failure detectors, elapsed: {:?}",
551            num_detecting_regions,
552            now.elapsed()
553        );
554
555        Ok(())
556    }
557
558    async fn register_failure_detectors(&self, detecting_regions: Vec<DetectingRegion>) {
559        let ts_millis = current_time_millis();
560        for region in detecting_regions {
561            // The corresponding region has `acceptable_heartbeat_pause_millis` to send heartbeat from datanode.
562            self.failure_detector
563                .maybe_init_region_failure_detector(region, ts_millis);
564        }
565    }
566
567    async fn reset_failure_detectors(&mut self, detecting_regions: Vec<DetectingRegion>) {
568        let ts_millis = current_time_millis();
569        for region in detecting_regions {
570            self.failure_detector
571                .reset_region_failure_detector(region, ts_millis);
572        }
573    }
574
575    async fn deregister_failure_detectors(&mut self, detecting_regions: Vec<DetectingRegion>) {
576        for region in detecting_regions {
577            self.failure_detector.remove(&region);
578            self.failover_counts.remove(&region);
579        }
580    }
581
582    async fn handle_region_failures(&mut self, mut regions: Vec<(DatanodeId, RegionId)>) {
583        if regions.is_empty() {
584            return;
585        }
586        match self.is_maintenance_mode_enabled().await {
587            Ok(false) => {}
588            Ok(true) => {
589                warn!(
590                    "Skipping failover since maintenance mode is enabled. Detected region failures: {:?}",
591                    regions
592                );
593                return;
594            }
595            Err(err) => {
596                error!(err; "Failed to check maintenance mode");
597                return;
598            }
599        }
600
601        // Extracts regions that are migrating(failover), which means they are already being triggered failover.
602        let migrating_regions = regions
603            .extract_if(.., |(_, region_id)| {
604                self.region_migration_manager.tracker().contains(*region_id)
605            })
606            .collect::<Vec<_>>();
607
608        for (datanode_id, region_id) in migrating_regions {
609            debug!(
610                "Removed region failover for region: {region_id}, datanode: {datanode_id} because it's migrating"
611            );
612        }
613
614        if regions.is_empty() {
615            // If all detected regions are failover or migrating, just return.
616            return;
617        }
618
619        let mut grouped_regions: HashMap<u64, Vec<RegionId>> =
620            HashMap::with_capacity(regions.len());
621        for (datanode_id, region_id) in regions {
622            grouped_regions
623                .entry(datanode_id)
624                .or_default()
625                .push(region_id);
626        }
627
628        for (datanode_id, regions) in grouped_regions {
629            warn!(
630                "Detects region failures on datanode: {}, regions: {:?}",
631                datanode_id, regions
632            );
633            // We can't use `grouped_regions.keys().cloned().collect::<Vec<_>>()` here
634            // because there may be false positives in failure detection on the datanode.
635            // So we only consider the datanode that reports the failure.
636            let failed_datanodes = [datanode_id];
637            match self
638                .generate_failover_tasks(datanode_id, &regions, &failed_datanodes)
639                .await
640            {
641                Ok(tasks) => {
642                    let mut grouped_tasks: HashMap<(u64, u64), Vec<_>> = HashMap::new();
643                    for (task, count) in tasks {
644                        grouped_tasks
645                            .entry((task.from_peer.id, task.to_peer.id))
646                            .or_default()
647                            .push((task, count));
648                    }
649
650                    for ((from_peer_id, to_peer_id), tasks) in grouped_tasks {
651                        if tasks.is_empty() {
652                            continue;
653                        }
654                        let task = RegionMigrationTaskBatch::from_tasks(tasks);
655                        let region_ids = task.region_ids.clone();
656                        if let Err(err) = self.do_failover_tasks(task).await {
657                            error!(err; "Failed to execute region failover for regions: {:?}, from_peer: {}, to_peer: {}", region_ids, from_peer_id, to_peer_id);
658                        }
659                    }
660                }
661                Err(err) => error!(err; "Failed to generate failover tasks"),
662            }
663        }
664    }
665
666    pub(crate) async fn is_maintenance_mode_enabled(&self) -> Result<bool> {
667        self.runtime_switch_manager
668            .maintenance_mode()
669            .await
670            .context(error::RuntimeSwitchManagerSnafu)
671    }
672
673    /// Selects a target for each of `regions`, excluding `failure_datanodes`, the unschedulable
674    /// datanodes and the region's datanodes in `exclude_peer_ids_by_region`. Regions without a
675    /// candidate are left out.
676    async fn select_peers(
677        &self,
678        from_peer_id: DatanodeId,
679        regions: &[RegionId],
680        failure_datanodes: &[DatanodeId],
681        exclude_peer_ids_by_region: &HashMap<RegionId, BTreeSet<DatanodeId>>,
682    ) -> Result<Vec<(RegionId, Peer)>> {
683        let affinity = if let Some(resolver) = &self.affinity_resolver {
684            match resolver
685                .resolve(&[AffinityReference::Datanode(from_peer_id)])
686                .await
687            {
688                Ok(mut resolved) => {
689                    ensure!(
690                        resolved.len() == 1,
691                        error::UnexpectedSnafu {
692                            violated: "affinity resolution must preserve the number and order of references",
693                        }
694                    );
695                    let affinity = resolved.pop().flatten();
696                    if affinity.is_none() {
697                        warn!(
698                            "Source affinity unavailable for datanode {}; selecting failover peers without affinity",
699                            from_peer_id
700                        );
701                    }
702                    affinity
703                }
704                Err(error) => {
705                    warn!(error; "Failed to resolve failover affinity for datanode {}; selecting without affinity", from_peer_id);
706                    None
707                }
708            }
709        } else {
710            None
711        };
712        let mut exclude_peer_ids: HashSet<u64> =
713            HashSet::from_iter(failure_datanodes.iter().copied());
714        let unschedulable_datanodes = self
715            .selector_context
716            .peer_discovery
717            .active_datanodes(None)
718            .await
719            .map_err(common_error::ext::BoxedError::new)
720            .context(error::OtherSnafu)?
721            .into_iter()
722            .filter(|datanode| !datanode.is_schedulable())
723            .map(|datanode| datanode.peer.id);
724        exclude_peer_ids.extend(unschedulable_datanodes);
725
726        let region_peers = match &self.selector {
727            RegionSupervisorSelector::NaiveSelector(selector) => {
728                // `Selector` takes one exclusion set, so select once per distinct set of excluded
729                // datanodes.
730                let empty = BTreeSet::new();
731                let mut groups = BTreeMap::<_, Vec<_>>::new();
732                for region_id in regions {
733                    let excluded = exclude_peer_ids_by_region.get(region_id).unwrap_or(&empty);
734                    groups.entry(excluded).or_default().push(*region_id);
735                }
736                let mut region_peers = Vec::with_capacity(regions.len());
737                for (excluded, regions) in groups {
738                    let mut exclude_peer_ids = exclude_peer_ids.clone();
739                    exclude_peer_ids.extend(excluded);
740                    let opts = SelectorOptions {
741                        affinity: affinity.as_ref(),
742                        min_required_items: regions.len(),
743                        allow_duplication: true,
744                        exclude_peer_ids,
745                        workload_filter: Some(accept_ingest_workload),
746                    };
747                    match selector.select(&self.selector_context, opts).await {
748                        Ok(peers) => region_peers.extend(regions.into_iter().zip(peers)),
749                        // Other groups may still have candidates.
750                        Err(error::Error::NoEnoughAvailableNode { .. }) => {}
751                        Err(err) => return Err(err),
752                    }
753                }
754                region_peers
755            }
756            RegionSupervisorSelector::RegionStatAwareSelector(selector) => {
757                let selected = selector
758                    .select(
759                        &self.selector_context,
760                        from_peer_id,
761                        regions,
762                        exclude_peer_ids,
763                        exclude_peer_ids_by_region,
764                        affinity.as_ref(),
765                    )
766                    .await;
767                match selected {
768                    Err(error::Error::NoEnoughAvailableNode { .. }) => vec![],
769                    selected => selected?,
770                }
771            }
772        };
773        let placed = region_peers
774            .iter()
775            .map(|(region_id, _)| *region_id)
776            .collect::<HashSet<_>>();
777        let unplaced = regions
778            .iter()
779            .filter(|region_id| !placed.contains(region_id))
780            .collect::<Vec<_>>();
781        if !unplaced.is_empty() {
782            error!("No failover target for regions: {:?}", unplaced);
783        }
784        Ok(region_peers)
785    }
786
787    async fn generate_failover_tasks(
788        &mut self,
789        from_peer_id: DatanodeId,
790        regions: &[RegionId],
791        failed_datanodes: &[DatanodeId],
792    ) -> Result<Vec<(RegionMigrationProcedureTask, u32)>> {
793        let mut tasks = Vec::with_capacity(regions.len());
794        let from_peer = self
795            .peer_resolver
796            .datanode(from_peer_id)
797            .await
798            .ok()
799            .flatten()
800            .unwrap_or_else(|| Peer::empty(from_peer_id));
801
802        // A region's route followers can't take it over.
803        let table_ids = regions
804            .iter()
805            .map(|region_id| region_id.table_id())
806            .unique()
807            .collect::<Vec<_>>();
808        let table_routes = TableRouteStorage::new(self.kv_backend.clone())
809            .batch_get(&table_ids)
810            .await
811            .unwrap_or_else(|err| {
812                // Exclude no followers; the submit still rejects a follower as the target.
813                warn!(err; "Failed to read the routes of failover regions: {:?}", regions);
814                vec![]
815            });
816        let exclude_peer_ids_by_region = table_routes
817            .iter()
818            .flatten()
819            .filter_map(|table_route| table_route.region_routes().ok())
820            .flatten()
821            .map(|route| {
822                let peers = route.follower_peers.iter().map(|peer| peer.id);
823                (route.region.id, peers.collect())
824            })
825            .collect();
826        let region_peers = self
827            .select_peers(
828                from_peer_id,
829                regions,
830                failed_datanodes,
831                &exclude_peer_ids_by_region,
832            )
833            .await?;
834
835        for (region_id, peer) in region_peers {
836            let count = *self
837                .failover_counts
838                .entry((from_peer_id, region_id))
839                .and_modify(|count| *count += 1)
840                .or_insert(1);
841            let task = RegionMigrationProcedureTask {
842                region_id,
843                from_peer: from_peer.clone(),
844                to_peer: peer,
845                timeout: DEFAULT_REGION_MIGRATION_TIMEOUT * count,
846                trigger_reason: RegionMigrationTriggerReason::Failover,
847            };
848            tasks.push((task, count));
849        }
850
851        Ok(tasks)
852    }
853
854    async fn do_failover_tasks(&mut self, task: RegionMigrationTaskBatch) -> Result<()> {
855        let from_peer_id = task.from_peer.id;
856        let to_peer_id = task.to_peer.id;
857        let timeout = task.timeout;
858        let trigger_reason = task.trigger_reason;
859        let result = self
860            .region_migration_manager
861            .submit_region_migration_task(task)
862            .await?;
863        self.handle_submit_region_migration_task_result(
864            from_peer_id,
865            to_peer_id,
866            timeout,
867            trigger_reason,
868            result,
869        )
870        .await
871    }
872
873    async fn handle_submit_region_migration_task_result(
874        &mut self,
875        from_peer_id: DatanodeId,
876        to_peer_id: DatanodeId,
877        timeout: Duration,
878        trigger_reason: RegionMigrationTriggerReason,
879        result: SubmitRegionMigrationTaskResult,
880    ) -> Result<()> {
881        if !result.migrated.is_empty() {
882            let detecting_regions = result
883                .migrated
884                .iter()
885                .map(|region_id| (from_peer_id, *region_id))
886                .collect::<Vec<_>>();
887            self.deregister_failure_detectors(detecting_regions).await;
888            info!(
889                "Region has been migrated to target peer: {}, removed failover detectors for regions: {:?}",
890                to_peer_id, result.migrated,
891            )
892        }
893        if !result.migrating.is_empty() {
894            info!(
895                "Region is still migrating, skipping failover for regions: {:?}",
896                result.migrating
897            );
898        }
899        if !result.region_not_found.is_empty() {
900            let detecting_regions = result
901                .region_not_found
902                .iter()
903                .map(|region_id| (from_peer_id, *region_id))
904                .collect::<Vec<_>>();
905            self.deregister_failure_detectors(detecting_regions).await;
906            info!(
907                "Region route not found, removed failover detectors for regions: {:?}",
908                result.region_not_found
909            );
910        }
911        if !result.table_not_found.is_empty() {
912            let detecting_regions = result
913                .table_not_found
914                .iter()
915                .map(|region_id| (from_peer_id, *region_id))
916                .collect::<Vec<_>>();
917            self.deregister_failure_detectors(detecting_regions).await;
918            info!(
919                "Table is not found, removed failover detectors for regions: {:?}",
920                result.table_not_found
921            );
922        }
923        if !result.leader_changed.is_empty() {
924            let detecting_regions = result
925                .leader_changed
926                .iter()
927                .map(|region_id| (from_peer_id, *region_id))
928                .collect::<Vec<_>>();
929            self.deregister_failure_detectors(detecting_regions).await;
930            info!(
931                "Region's leader peer changed, removed failover detectors for regions: {:?}",
932                result.leader_changed
933            );
934        }
935        if !result.peer_conflict.is_empty() {
936            info!(
937                "Region has peer conflict, ignore failover for regions: {:?}",
938                result.peer_conflict
939            );
940        }
941        if !result.submitted.is_empty() {
942            info!(
943                "Failover for regions: {:?}, from_peer: {}, to_peer: {}, procedure_id: {:?}, timeout: {:?}, trigger_reason: {:?}",
944                result.submitted,
945                from_peer_id,
946                to_peer_id,
947                result.procedure_id,
948                timeout,
949                trigger_reason,
950            );
951        }
952
953        Ok(())
954    }
955
956    /// Detects the failure of regions.
957    fn detect_region_failure(&self) -> Vec<(DatanodeId, RegionId)> {
958        self.failure_detector
959            .iter()
960            .filter_map(|e| {
961                // Intentionally not place `current_time_millis()` out of the iteration.
962                // The failure detection determination should be happened "just in time",
963                // i.e., failed or not has to be compared with the most recent "now".
964                // Besides, it might reduce the false positive of failure detection,
965                // because during the iteration, heartbeats are coming in as usual,
966                // and the `phi`s are still updating.
967                if !e.failure_detector().is_available(current_time_millis()) {
968                    Some(*e.region_ident())
969                } else {
970                    None
971                }
972            })
973            .collect::<Vec<_>>()
974    }
975
976    /// Returns all regions that registered in the failure detector.
977    fn regions(&self) -> HashSet<RegionId> {
978        self.failure_detector
979            .iter()
980            .map(|e| e.region_ident().1)
981            .collect::<HashSet<_>>()
982    }
983
984    /// Updates the state of corresponding failure detectors.
985    fn on_heartbeat_arrived(&self, heartbeat: DatanodeHeartbeat) {
986        let mut regions = heartbeat.regions;
987        if !heartbeat.opening_regions.is_empty() {
988            let mut reported_regions: HashSet<_> = regions.iter().copied().collect();
989            // Update each detector once, even if both snapshots contain the region.
990            regions.extend(
991                heartbeat
992                    .opening_regions
993                    .into_iter()
994                    .filter(|region_id| reported_regions.insert(*region_id)),
995            );
996        }
997        for region_id in regions {
998            let detecting_region = (heartbeat.datanode_id, region_id);
999            let mut detector = self
1000                .failure_detector
1001                .region_failure_detector(detecting_region);
1002            detector.heartbeat(heartbeat.timestamp);
1003        }
1004    }
1005
1006    fn clear(&self) {
1007        self.failure_detector.clear();
1008    }
1009}
1010
1011#[cfg(test)]
1012pub(crate) mod tests {
1013    use std::assert_matches;
1014    use std::collections::HashMap;
1015    use std::sync::{Arc, Mutex};
1016    use std::time::Duration;
1017
1018    use common_meta::affinity::NoopAffinity;
1019    use common_meta::ddl::RegionFailureDetectorController;
1020    use common_meta::ddl::test_util::{
1021        test_create_logical_table_task, test_create_physical_table_task,
1022    };
1023    use common_meta::key::table_route::{
1024        LogicalTableRouteValue, PhysicalTableRouteValue, TableRouteValue,
1025    };
1026    use common_meta::key::{TableMetadataManager, runtime_switch};
1027    use common_meta::peer::Peer;
1028    use common_meta::rpc::router::{Region, RegionRoute};
1029    use common_meta::test_util::NoopPeerResolver;
1030    use common_telemetry::info;
1031    use common_time::util::current_time_millis;
1032    use rand::Rng;
1033    use store_api::storage::RegionId;
1034    use tokio::sync::mpsc::Sender;
1035    use tokio::sync::oneshot;
1036    use tokio::time::sleep;
1037
1038    use super::RegionSupervisorSelector;
1039    use crate::metasrv::SelectorContext;
1040    use crate::metrics::METRIC_META_HEARTBEAT_DROPPED;
1041    use crate::procedure::region_migration::RegionMigrationTriggerReason;
1042    use crate::procedure::region_migration::manager::{
1043        RegionMigrationManager, SubmitRegionMigrationTaskResult,
1044    };
1045    use crate::procedure::region_migration::test_util::TestingEnv;
1046    use crate::region::supervisor::{
1047        DatanodeHeartbeat, Event, HeartbeatAcceptor, RegionFailureDetectorControl,
1048        RegionSupervisor, RegionSupervisorTicker,
1049    };
1050    use crate::selector::test_utils::{RandomNodeSelector, new_test_selector_context};
1051
1052    pub(crate) fn new_test_supervisor() -> (RegionSupervisor, Sender<Event>) {
1053        new_test_supervisor_with(
1054            new_test_selector_context(),
1055            RegionSupervisorSelector::NaiveSelector(Arc::new(RandomNodeSelector::new(vec![
1056                Peer::empty(1),
1057            ]))),
1058        )
1059    }
1060
1061    pub(crate) fn new_test_supervisor_with(
1062        selector_context: SelectorContext,
1063        selector: RegionSupervisorSelector,
1064    ) -> (RegionSupervisor, Sender<Event>) {
1065        let env = TestingEnv::new();
1066        let context_factory = env.context_factory();
1067        let region_migration_manager = Arc::new(RegionMigrationManager::new(
1068            env.procedure_manager().clone(),
1069            context_factory,
1070        ));
1071        let runtime_switch_manager =
1072            Arc::new(runtime_switch::RuntimeSwitchManager::new(env.kv_backend()));
1073        let peer_resolver = Arc::new(NoopPeerResolver);
1074        let (tx, rx) = RegionSupervisor::channel();
1075        let kv_backend = env.kv_backend();
1076
1077        (
1078            RegionSupervisor::new(
1079                rx,
1080                Default::default(),
1081                selector_context,
1082                selector,
1083                region_migration_manager,
1084                runtime_switch_manager,
1085                peer_resolver,
1086                kv_backend,
1087            ),
1088            tx,
1089        )
1090    }
1091
1092    #[tokio::test]
1093    async fn test_failover_affinity_fallback_and_selector_errors() {
1094        use std::collections::{BTreeSet, HashSet};
1095        use std::sync::atomic::{AtomicUsize, Ordering};
1096
1097        use common_meta::affinity::{
1098            AffinityFallback, AffinityReference, AffinityRequirements, AffinityResolver,
1099        };
1100        use common_meta::error::{self as meta_error, Result as MetaResult};
1101
1102        use crate::error::{self, Result};
1103        use crate::metasrv::{SelectTarget, SelectorContext};
1104        use crate::selector::{RegionStatAwareSelector, Selector, SelectorOptions};
1105
1106        #[derive(Clone, Copy, PartialEq)]
1107        enum Resolution {
1108            Disabled,
1109            Available,
1110            Missing,
1111            Error,
1112        }
1113
1114        struct Resolver {
1115            result: Resolution,
1116            calls: AtomicUsize,
1117        }
1118
1119        #[async_trait::async_trait]
1120        impl AffinityResolver for Resolver {
1121            async fn resolve(
1122                &self,
1123                references: &[AffinityReference],
1124            ) -> MetaResult<Vec<Option<AffinityRequirements>>> {
1125                assert_eq!(references, &[AffinityReference::Datanode(7)]);
1126                self.calls.fetch_add(1, Ordering::Relaxed);
1127                match self.result {
1128                    Resolution::Available => {
1129                        Ok(vec![AffinityRequirements::new(HashMap::from([(
1130                            "pool".into(),
1131                            "a".into(),
1132                        )]))])
1133                    }
1134                    Resolution::Disabled => unreachable!(),
1135                    Resolution::Missing => Ok(vec![None]),
1136                    Resolution::Error => meta_error::UnexpectedSnafu {
1137                        err_msg: "source lookup failed",
1138                    }
1139                    .fail(),
1140                }
1141            }
1142        }
1143
1144        #[derive(Clone, Copy)]
1145        enum Outcome {
1146            Success,
1147            Short,
1148            Exhausted,
1149            Error,
1150        }
1151        struct FailoverSelector {
1152            outcome: Outcome,
1153            expected_fallback: AffinityFallback,
1154            calls: Mutex<Vec<Option<AffinityRequirements>>>,
1155        }
1156        impl FailoverSelector {
1157            fn select_peers(&self, affinity: Option<&AffinityRequirements>) -> Result<Vec<Peer>> {
1158                self.calls.lock().unwrap().push(affinity.cloned());
1159                if affinity.is_some() {
1160                    match self.outcome {
1161                        Outcome::Success => {}
1162                        Outcome::Short => return Ok(vec![Peer::empty(9)]),
1163                        Outcome::Exhausted => {
1164                            return error::NoEnoughAvailableNodeSnafu {
1165                                required: 2usize,
1166                                available: 0usize,
1167                                select_target: SelectTarget::Datanode,
1168                            }
1169                            .fail();
1170                        }
1171                        Outcome::Error => {
1172                            return error::UnexpectedSnafu {
1173                                violated: "selection backend failed",
1174                            }
1175                            .fail();
1176                        }
1177                    }
1178                }
1179                Ok(vec![Peer::empty(9); 2])
1180            }
1181        }
1182        #[async_trait::async_trait]
1183        impl Selector for FailoverSelector {
1184            type Context = SelectorContext;
1185            type Output = Vec<Peer>;
1186            async fn select(
1187                &self,
1188                ctx: &SelectorContext,
1189                opts: SelectorOptions<'_>,
1190            ) -> Result<Self::Output> {
1191                assert_eq!(ctx.affinity_fallback, self.expected_fallback);
1192                assert_eq!(opts.exclude_peer_ids, HashSet::from([8]));
1193                assert!(opts.workload_filter.is_some());
1194                assert!(opts.allow_duplication);
1195                assert_eq!(opts.min_required_items, 2);
1196                self.select_peers(opts.affinity)
1197            }
1198        }
1199        #[async_trait::async_trait]
1200        impl RegionStatAwareSelector for FailoverSelector {
1201            type Context = SelectorContext;
1202            type Output = Vec<(RegionId, Peer)>;
1203            async fn select(
1204                &self,
1205                ctx: &SelectorContext,
1206                source: u64,
1207                regions: &[RegionId],
1208                excluded: HashSet<u64>,
1209                exclude_peer_ids_by_region: &HashMap<RegionId, BTreeSet<u64>>,
1210                affinity: Option<&AffinityRequirements>,
1211            ) -> Result<Self::Output> {
1212                assert_eq!(ctx.affinity_fallback, self.expected_fallback);
1213                assert_eq!(source, 7);
1214                assert_eq!(excluded, HashSet::from([8]));
1215                assert!(exclude_peer_ids_by_region.is_empty());
1216                Ok(regions
1217                    .iter()
1218                    .copied()
1219                    .zip(self.select_peers(affinity)?)
1220                    .collect())
1221            }
1222        }
1223        for stat_aware in [false, true] {
1224            for (resolution, outcome, succeeds) in [
1225                (Resolution::Available, Outcome::Success, true),
1226                (Resolution::Available, Outcome::Short, false),
1227                (Resolution::Available, Outcome::Exhausted, false),
1228                (Resolution::Available, Outcome::Error, false),
1229                (Resolution::Missing, Outcome::Success, true),
1230                (Resolution::Error, Outcome::Success, true),
1231                (Resolution::Disabled, Outcome::Success, true),
1232            ] {
1233                let resolver = Arc::new(Resolver {
1234                    result: resolution,
1235                    calls: AtomicUsize::new(0),
1236                });
1237                let selector = Arc::new(FailoverSelector {
1238                    outcome,
1239                    expected_fallback: AffinityFallback::Unconstrained,
1240                    calls: Mutex::new(vec![]),
1241                });
1242                let (mut supervisor, _sender) = new_test_supervisor();
1243                if resolution != Resolution::Disabled {
1244                    supervisor = supervisor.with_affinity_resolver(resolver.clone());
1245                }
1246                supervisor.selector = if stat_aware {
1247                    RegionSupervisorSelector::RegionStatAwareSelector(selector.clone())
1248                } else {
1249                    RegionSupervisorSelector::NaiveSelector(selector.clone())
1250                };
1251                let regions = [RegionId::new(1, 1), RegionId::new(1, 2)];
1252                // Preserve the caller's exclusion set without adding the source.
1253                let result = supervisor
1254                    .select_peers(7, &regions, &[8], &HashMap::new())
1255                    .await;
1256                if succeeds {
1257                    assert_eq!(
1258                        result.unwrap(),
1259                        vec![(regions[0], Peer::empty(9)), (regions[1], Peer::empty(9))]
1260                    );
1261                } else {
1262                    match outcome {
1263                        // Only the regions with a candidate are placed.
1264                        Outcome::Short => {
1265                            assert_eq!(result.unwrap(), vec![(regions[0], Peer::empty(9))])
1266                        }
1267                        Outcome::Exhausted => assert!(result.unwrap().is_empty()),
1268                        Outcome::Error => assert!(matches!(
1269                            result.unwrap_err(),
1270                            error::Error::Unexpected { .. }
1271                        )),
1272                        Outcome::Success => unreachable!(),
1273                    }
1274                }
1275                assert_eq!(
1276                    resolver.calls.load(Ordering::Relaxed),
1277                    usize::from(resolution != Resolution::Disabled)
1278                );
1279                let calls = selector.calls.lock().unwrap();
1280                assert_eq!(calls.len(), 1);
1281                if let Some(affinity) = calls.first() {
1282                    assert_eq!(affinity.is_some(), resolution == Resolution::Available);
1283                }
1284            }
1285        }
1286    }
1287
1288    fn filler_heartbeat() -> DatanodeHeartbeat {
1289        DatanodeHeartbeat {
1290            opening_regions: Vec::new(),
1291            datanode_id: 1,
1292            regions: vec![],
1293            timestamp: 0,
1294        }
1295    }
1296
1297    #[tokio::test]
1298    async fn test_select_peers_excludes_unschedulable_datanodes() {
1299        use std::collections::HashSet as StdHashSet;
1300
1301        use crate::key::NodeSchedulingRole;
1302        use crate::selector::{Selector, SelectorOptions};
1303        use crate::test_util::{
1304            create_meta_peer_client, new_node_scheduling_discovery, put_datanodes,
1305        };
1306
1307        /// Selector that records the excluded peers and returns a fixed peer.
1308        struct ExcludeAssertSelector;
1309
1310        #[async_trait::async_trait]
1311        impl Selector for ExcludeAssertSelector {
1312            type Context = SelectorContext;
1313            type Output = Vec<Peer>;
1314
1315            async fn select(
1316                &self,
1317                _ctx: &Self::Context,
1318                opts: SelectorOptions<'_>,
1319            ) -> crate::error::Result<Self::Output> {
1320                assert_eq!(StdHashSet::from([7, 8]), opts.exclude_peer_ids);
1321                Ok(vec![Peer::empty(9)])
1322            }
1323        }
1324
1325        // Datanode 7 is marked unschedulable; datanode 8 failed.
1326        let client = create_meta_peer_client();
1327        put_datanodes(&client, vec![Peer::empty(7)]).await;
1328        let (manager, discovery) = new_node_scheduling_discovery(&client);
1329        manager
1330            .mark_unschedulable(NodeSchedulingRole::Datanode, 7)
1331            .await
1332            .unwrap();
1333
1334        let selector_context = SelectorContext {
1335            affinity_fallback: Default::default(),
1336            affinity: Arc::new(NoopAffinity),
1337            peer_discovery: Arc::new(discovery),
1338        };
1339        let (supervisor, _tx) = new_test_supervisor_with(
1340            selector_context,
1341            RegionSupervisorSelector::NaiveSelector(Arc::new(ExcludeAssertSelector)),
1342        );
1343
1344        let region_peers = supervisor
1345            .select_peers(7, &[RegionId::new(1, 1)], &[8], &HashMap::new())
1346            .await
1347            .unwrap();
1348        assert_eq!(1, region_peers.len());
1349        assert_eq!(Peer::empty(9), region_peers[0].1);
1350    }
1351
1352    #[tokio::test]
1353    async fn test_failover_excludes_route_followers() {
1354        use std::collections::{BTreeSet, HashSet as StdHashSet};
1355
1356        use common_meta::affinity::AffinityRequirements;
1357        use common_meta::key::MetadataKey;
1358        use common_meta::key::table_route::TableRouteKey;
1359        use common_meta::rpc::store::PutRequest;
1360
1361        use crate::error::{self, Result};
1362        use crate::metasrv::SelectTarget;
1363        use crate::selector::{RegionStatAwareSelector, Selector, SelectorOptions};
1364
1365        /// Records each selection, and has no candidate once datanode 3 is excluded.
1366        struct RecordingSelector(Mutex<Vec<(StdHashSet<u64>, usize)>>);
1367
1368        #[async_trait::async_trait]
1369        impl Selector for RecordingSelector {
1370            type Context = SelectorContext;
1371            type Output = Vec<Peer>;
1372
1373            async fn select(
1374                &self,
1375                _ctx: &Self::Context,
1376                opts: SelectorOptions<'_>,
1377            ) -> Result<Self::Output> {
1378                let required = opts.min_required_items;
1379                let no_candidate = opts.exclude_peer_ids.contains(&3);
1380                self.0
1381                    .lock()
1382                    .unwrap()
1383                    .push((opts.exclude_peer_ids, required));
1384                if no_candidate {
1385                    return error::NoEnoughAvailableNodeSnafu {
1386                        required,
1387                        available: 0usize,
1388                        select_target: SelectTarget::Datanode,
1389                    }
1390                    .fail();
1391                }
1392                Ok(vec![Peer::empty(3); required])
1393            }
1394        }
1395
1396        /// A batch of regions with their excluded peers.
1397        type Batch = (Vec<RegionId>, HashMap<RegionId, BTreeSet<u64>>);
1398
1399        /// Records each batch, and places every region on datanode 9, which no region excludes.
1400        struct StatAwareRecorder(Mutex<Vec<Batch>>);
1401
1402        #[async_trait::async_trait]
1403        impl RegionStatAwareSelector for StatAwareRecorder {
1404            type Context = SelectorContext;
1405            type Output = Vec<(RegionId, Peer)>;
1406
1407            async fn select(
1408                &self,
1409                _ctx: &Self::Context,
1410                _from_peer_id: u64,
1411                region_ids: &[RegionId],
1412                exclude_peer_ids: StdHashSet<u64>,
1413                exclude_peer_ids_by_region: &HashMap<RegionId, BTreeSet<u64>>,
1414                _affinity: Option<&AffinityRequirements>,
1415            ) -> Result<Self::Output> {
1416                assert_eq!(exclude_peer_ids, StdHashSet::from([8]));
1417                self.0
1418                    .lock()
1419                    .unwrap()
1420                    .push((region_ids.to_vec(), exclude_peer_ids_by_region.clone()));
1421                Ok(region_ids
1422                    .iter()
1423                    .map(|region_id| (*region_id, Peer::empty(9)))
1424                    .collect())
1425            }
1426        }
1427
1428        let selector = Arc::new(RecordingSelector(Mutex::new(vec![])));
1429        let (mut supervisor, _tx) = new_test_supervisor_with(
1430            new_test_selector_context(),
1431            RegionSupervisorSelector::NaiveSelector(selector.clone()),
1432        );
1433        let table_id = 1024;
1434        let region_route = |region_number, followers: &[u64]| RegionRoute {
1435            region: Region {
1436                id: RegionId::new(table_id, region_number),
1437                ..Default::default()
1438            },
1439            leader_peer: Some(Peer::empty(8)),
1440            follower_peers: followers.iter().map(|id| Peer::empty(*id)).collect(),
1441            ..Default::default()
1442        };
1443        let table_route = PhysicalTableRouteValue::new(vec![
1444            region_route(0, &[]),
1445            region_route(1, &[3]),
1446            region_route(2, &[4]),
1447            region_route(3, &[4]),
1448        ]);
1449        let mut task = test_create_physical_table_task("my_physical_table");
1450        task.set_table_id(table_id);
1451        TableMetadataManager::new(supervisor.kv_backend.clone())
1452            .create_table_metadata(
1453                task.table_info,
1454                TableRouteValue::Physical(table_route),
1455                HashMap::new(),
1456            )
1457            .await
1458            .unwrap();
1459
1460        let mut regions = (0..4)
1461            .map(|number| RegionId::new(table_id, number))
1462            .collect::<Vec<_>>();
1463        // A region of a table without a route has no followers.
1464        regions.push(RegionId::new(table_id + 1, 0));
1465        let tasks = supervisor
1466            .generate_failover_tasks(8, &regions, &[8])
1467            .await
1468            .unwrap();
1469
1470        // Regions with the same followers are selected together, excluding those followers.
1471        assert_eq!(
1472            *selector.0.lock().unwrap(),
1473            vec![
1474                (StdHashSet::from([8]), 2),
1475                (StdHashSet::from([8, 3]), 1),
1476                (StdHashSet::from([8, 4]), 2),
1477            ]
1478        );
1479        // Only the region whose follower is the last candidate is skipped.
1480        let failed_over = tasks
1481            .iter()
1482            .map(|(task, _)| task.region_id)
1483            .collect::<Vec<_>>();
1484        assert_eq!(
1485            failed_over,
1486            [regions[0], regions[4], regions[2], regions[3]]
1487        );
1488
1489        // The stat-aware selector gets the whole batch with each region's followers.
1490        let stat_aware = Arc::new(StatAwareRecorder(Mutex::new(vec![])));
1491        supervisor.selector = RegionSupervisorSelector::RegionStatAwareSelector(stat_aware.clone());
1492        let tasks = supervisor
1493            .generate_failover_tasks(8, &regions, &[8])
1494            .await
1495            .unwrap();
1496        let followers = HashMap::from([
1497            (regions[0], BTreeSet::new()),
1498            (regions[1], BTreeSet::from([3])),
1499            (regions[2], BTreeSet::from([4])),
1500            (regions[3], BTreeSet::from([4])),
1501        ]);
1502        assert_eq!(
1503            *stat_aware.0.lock().unwrap(),
1504            vec![(regions.clone(), followers)]
1505        );
1506        let failed_over = tasks
1507            .iter()
1508            .map(|(task, _)| task.region_id)
1509            .collect::<Vec<_>>();
1510        assert_eq!(failed_over, regions);
1511        supervisor.selector = RegionSupervisorSelector::NaiveSelector(selector.clone());
1512
1513        // A route that can't be read excludes no followers.
1514        supervisor
1515            .kv_backend
1516            .put(
1517                PutRequest::new()
1518                    .with_key(TableRouteKey::new(table_id + 1).to_bytes())
1519                    .with_value(b"corrupted".to_vec()),
1520            )
1521            .await
1522            .unwrap();
1523        selector.0.lock().unwrap().clear();
1524        let tasks = supervisor
1525            .generate_failover_tasks(8, &regions, &[8])
1526            .await
1527            .unwrap();
1528        assert_eq!(
1529            *selector.0.lock().unwrap(),
1530            vec![(StdHashSet::from([8]), 5)]
1531        );
1532        assert_eq!(tasks.len(), 5);
1533    }
1534
1535    #[tokio::test]
1536    async fn test_opening_regions_refresh_detectors_until_serving() {
1537        let (supervisor, _sender) = new_test_supervisor();
1538        let region_id = RegionId::new(1, 1);
1539        let other_region = RegionId::new(1, 2);
1540        let new_region = RegionId::new(1, 3);
1541        for region in [(1, region_id), (1, other_region), (2, region_id)] {
1542            let mut detector = supervisor.failure_detector.region_failure_detector(region);
1543            detector.heartbeat(1_000);
1544            detector.heartbeat(2_000);
1545        }
1546
1547        let mut expected = supervisor
1548            .failure_detector
1549            .region_failure_detector((1, region_id))
1550            .clone();
1551        for timestamp in [60_000, 120_000] {
1552            expected.heartbeat(timestamp);
1553            assert!(
1554                !supervisor
1555                    .failure_detector
1556                    .region_failure_detector((1, region_id))
1557                    .is_available(timestamp)
1558            );
1559            supervisor.on_heartbeat_arrived(DatanodeHeartbeat {
1560                datanode_id: 1,
1561                regions: vec![],
1562                opening_regions: vec![region_id, new_region],
1563                timestamp,
1564            });
1565            assert_eq!(
1566                expected,
1567                *supervisor
1568                    .failure_detector
1569                    .region_failure_detector((1, region_id))
1570            );
1571            for region in [region_id, new_region] {
1572                let detector = supervisor
1573                    .failure_detector
1574                    .region_failure_detector((1, region));
1575                assert_eq!(Some(timestamp), detector.last_heartbeat_millis());
1576                assert!(detector.is_available(timestamp));
1577                // Failure detection resumes when opening heartbeats stop.
1578                assert!(!detector.is_available(timestamp + 60_000));
1579            }
1580            for region in [(1, other_region), (2, region_id)] {
1581                let detector = supervisor.failure_detector.region_failure_detector(region);
1582                assert_eq!(Some(2_000), detector.last_heartbeat_millis());
1583            }
1584        }
1585
1586        // Serving heartbeats retain history, even with a stale opening entry.
1587        for (timestamp, opening_regions) in [(121_000, vec![]), (122_000, vec![region_id])] {
1588            expected.heartbeat(timestamp);
1589            supervisor.on_heartbeat_arrived(DatanodeHeartbeat {
1590                datanode_id: 1,
1591                regions: vec![region_id],
1592                opening_regions,
1593                timestamp,
1594            });
1595            assert_eq!(
1596                expected,
1597                *supervisor
1598                    .failure_detector
1599                    .region_failure_detector((1, region_id))
1600            );
1601        }
1602    }
1603
1604    #[tokio::test]
1605    async fn test_heartbeat() {
1606        let (mut supervisor, sender) = new_test_supervisor();
1607        tokio::spawn(async move { supervisor.run().await });
1608
1609        sender
1610            .send(Event::HeartbeatArrived(DatanodeHeartbeat {
1611                opening_regions: Vec::new(),
1612                datanode_id: 0,
1613                regions: vec![RegionId::new(1, 1)],
1614                timestamp: 100,
1615            }))
1616            .await
1617            .unwrap();
1618        let (tx, rx) = oneshot::channel();
1619        sender.send(Event::Dump(tx)).await.unwrap();
1620        let detector = rx.await.unwrap();
1621        assert!(detector.contains(&(0, RegionId::new(1, 1))));
1622
1623        // Clear up
1624        sender.send(Event::Clear).await.unwrap();
1625        let (tx, rx) = oneshot::channel();
1626        sender.send(Event::Dump(tx)).await.unwrap();
1627        assert!(rx.await.unwrap().is_empty());
1628
1629        fn generate_heartbeats(datanode_id: u64, region_ids: Vec<u32>) -> Vec<DatanodeHeartbeat> {
1630            let mut rng = rand::rng();
1631            let start = current_time_millis();
1632            (0..2000)
1633                .map(|i| DatanodeHeartbeat {
1634                    opening_regions: Vec::new(),
1635                    timestamp: start + i * 1000 + rng.random_range(0..100),
1636                    datanode_id,
1637                    regions: region_ids
1638                        .iter()
1639                        .map(|number| RegionId::new(0, *number))
1640                        .collect(),
1641                })
1642                .collect::<Vec<_>>()
1643        }
1644
1645        let heartbeats = generate_heartbeats(100, vec![1, 2, 3]);
1646        let last_heartbeat_time = heartbeats.last().unwrap().timestamp;
1647        for heartbeat in heartbeats {
1648            sender
1649                .send(Event::HeartbeatArrived(heartbeat))
1650                .await
1651                .unwrap();
1652        }
1653
1654        let (tx, rx) = oneshot::channel();
1655        sender.send(Event::Dump(tx)).await.unwrap();
1656        let detector = rx.await.unwrap();
1657        assert_eq!(detector.len(), 3);
1658
1659        for e in detector.iter() {
1660            let fd = e.failure_detector();
1661            let acceptable_heartbeat_pause_millis = fd.acceptable_heartbeat_pause_millis() as i64;
1662            let start = last_heartbeat_time;
1663
1664            // Within the "acceptable_heartbeat_pause_millis" period, phi is zero ...
1665            for i in 1..=acceptable_heartbeat_pause_millis / 1000 {
1666                let now = start + i * 1000;
1667                assert_eq!(fd.phi(now), 0.0);
1668            }
1669
1670            // ... then in less than two seconds, phi is above the threshold.
1671            // The same effect can be seen in the diagrams in Akka's document.
1672            let now = start + acceptable_heartbeat_pause_millis + 1000;
1673            assert!(fd.phi(now) < fd.threshold() as _);
1674            let now = start + acceptable_heartbeat_pause_millis + 2000;
1675            assert!(fd.phi(now) > fd.threshold() as _);
1676        }
1677    }
1678
1679    #[tokio::test]
1680    async fn test_supervisor_ticker() {
1681        let (tx, mut rx) = tokio::sync::mpsc::channel(128);
1682        let ticker = RegionSupervisorTicker {
1683            tick_handle: Mutex::new(None),
1684            initialization_handle: Mutex::new(None),
1685            tick_interval: Duration::from_millis(10),
1686            initialization_delay: Duration::from_millis(100),
1687            initialization_retry_period: Duration::from_millis(100),
1688            sender: tx,
1689        };
1690        // It's ok if we start the ticker again.
1691        for _ in 0..2 {
1692            ticker.start();
1693            sleep(Duration::from_millis(100)).await;
1694            ticker.stop();
1695            assert!(!rx.is_empty());
1696            while let Ok(event) = rx.try_recv() {
1697                assert_matches!(
1698                    event,
1699                    Event::Tick | Event::Clear | Event::InitializeAllRegions(_)
1700                );
1701            }
1702            assert!(ticker.initialization_handle.lock().unwrap().is_none());
1703            assert!(ticker.tick_handle.lock().unwrap().is_none());
1704        }
1705    }
1706
1707    #[tokio::test]
1708    async fn test_initialize_all_regions_event_handling() {
1709        common_telemetry::init_default_ut_logging();
1710        let (tx, mut rx) = tokio::sync::mpsc::channel(128);
1711        let ticker = RegionSupervisorTicker {
1712            tick_handle: Mutex::new(None),
1713            initialization_handle: Mutex::new(None),
1714            tick_interval: Duration::from_millis(1000),
1715            initialization_delay: Duration::from_millis(50),
1716            initialization_retry_period: Duration::from_millis(50),
1717            sender: tx,
1718        };
1719        ticker.start();
1720        sleep(Duration::from_millis(60)).await;
1721        let handle = tokio::spawn(async move {
1722            let mut counter = 0;
1723            while let Some(event) = rx.recv().await {
1724                if let Event::InitializeAllRegions(tx) = event {
1725                    if counter == 0 {
1726                        // Ignore the first event
1727                        counter += 1;
1728                        continue;
1729                    }
1730                    tx.send(()).unwrap();
1731                    info!("Responded initialize all regions event");
1732                    break;
1733                }
1734            }
1735            rx
1736        });
1737
1738        let rx = handle.await.unwrap();
1739        for _ in 0..3 {
1740            sleep(Duration::from_millis(100)).await;
1741            assert!(rx.is_empty());
1742        }
1743    }
1744
1745    #[tokio::test]
1746    async fn test_initialize_all_regions() {
1747        common_telemetry::init_default_ut_logging();
1748        let (mut supervisor, sender) = new_test_supervisor();
1749        let table_metadata_manager = TableMetadataManager::new(supervisor.kv_backend.clone());
1750
1751        // Create a physical table metadata
1752        let table_id = 1024;
1753        let mut create_physical_table_task = test_create_physical_table_task("my_physical_table");
1754        create_physical_table_task.set_table_id(table_id);
1755        let table_info = create_physical_table_task.table_info;
1756        let table_route = PhysicalTableRouteValue::new(vec![RegionRoute {
1757            region: Region {
1758                id: RegionId::new(table_id, 0),
1759                ..Default::default()
1760            },
1761            leader_peer: Some(Peer::empty(1)),
1762            ..Default::default()
1763        }]);
1764        let table_route_value = TableRouteValue::Physical(table_route);
1765        table_metadata_manager
1766            .create_table_metadata(table_info, table_route_value, HashMap::new())
1767            .await
1768            .unwrap();
1769
1770        // Create a logical table metadata
1771        let logical_table_id = 1025;
1772        let mut test_create_logical_table_task = test_create_logical_table_task("my_logical_table");
1773        test_create_logical_table_task.set_table_id(logical_table_id);
1774        let table_info = test_create_logical_table_task.table_info;
1775        let table_route = LogicalTableRouteValue::new(1024);
1776        let table_route_value = TableRouteValue::Logical(table_route);
1777        table_metadata_manager
1778            .create_table_metadata(table_info, table_route_value, HashMap::new())
1779            .await
1780            .unwrap();
1781        tokio::spawn(async move { supervisor.run().await });
1782        let (tx, rx) = oneshot::channel();
1783        sender.send(Event::InitializeAllRegions(tx)).await.unwrap();
1784        assert!(rx.await.is_ok());
1785
1786        let (tx, rx) = oneshot::channel();
1787        sender.send(Event::Dump(tx)).await.unwrap();
1788        let detector = rx.await.unwrap();
1789        assert_eq!(detector.len(), 1);
1790        assert!(detector.contains(&(1, RegionId::new(1024, 0))));
1791    }
1792
1793    #[tokio::test]
1794    async fn test_initialize_all_regions_with_maintenance_mode() {
1795        common_telemetry::init_default_ut_logging();
1796        let (mut supervisor, sender) = new_test_supervisor();
1797
1798        supervisor
1799            .runtime_switch_manager
1800            .set_maintenance_mode()
1801            .await
1802            .unwrap();
1803        tokio::spawn(async move { supervisor.run().await });
1804        let (tx, rx) = oneshot::channel();
1805        sender.send(Event::InitializeAllRegions(tx)).await.unwrap();
1806        // The sender is dropped, so the receiver will receive an error.
1807        assert!(rx.await.is_err());
1808    }
1809
1810    #[tokio::test]
1811    async fn test_region_failure_detector_controller() {
1812        let (mut supervisor, sender) = new_test_supervisor();
1813        let controller = RegionFailureDetectorControl::new(sender.clone());
1814        tokio::spawn(async move { supervisor.run().await });
1815        let detecting_region = (1, RegionId::new(1, 1));
1816        controller
1817            .register_failure_detectors(vec![detecting_region])
1818            .await;
1819
1820        let (tx, rx) = oneshot::channel();
1821        sender.send(Event::Dump(tx)).await.unwrap();
1822        let detector = rx.await.unwrap();
1823        let region_detector = detector.region_failure_detector(detecting_region).clone();
1824
1825        // Registers failure detector again
1826        controller
1827            .register_failure_detectors(vec![detecting_region])
1828            .await;
1829        let (tx, rx) = oneshot::channel();
1830        sender.send(Event::Dump(tx)).await.unwrap();
1831        let detector = rx.await.unwrap();
1832        let got = detector.region_failure_detector(detecting_region).clone();
1833        assert_eq!(region_detector, got);
1834
1835        controller
1836            .deregister_failure_detectors(vec![detecting_region])
1837            .await;
1838        let (tx, rx) = oneshot::channel();
1839        sender.send(Event::Dump(tx)).await.unwrap();
1840        assert!(rx.await.unwrap().is_empty());
1841    }
1842
1843    #[tokio::test]
1844    async fn test_handle_submit_region_migration_task_result_migrated() {
1845        common_telemetry::init_default_ut_logging();
1846        let (mut supervisor, _) = new_test_supervisor();
1847        let region_id = RegionId::new(1, 1);
1848        let detecting_region = (1, region_id);
1849        supervisor
1850            .register_failure_detectors(vec![detecting_region])
1851            .await;
1852        supervisor.failover_counts.insert(detecting_region, 1);
1853        let result = SubmitRegionMigrationTaskResult {
1854            migrated: vec![region_id],
1855            ..Default::default()
1856        };
1857        supervisor
1858            .handle_submit_region_migration_task_result(
1859                1,
1860                2,
1861                Duration::from_millis(1000),
1862                RegionMigrationTriggerReason::Manual,
1863                result,
1864            )
1865            .await
1866            .unwrap();
1867        assert!(!supervisor.failure_detector.contains(&detecting_region));
1868        assert!(supervisor.failover_counts.is_empty());
1869    }
1870
1871    #[tokio::test]
1872    async fn test_handle_submit_region_migration_task_result_migrating() {
1873        common_telemetry::init_default_ut_logging();
1874        let (mut supervisor, _) = new_test_supervisor();
1875        let region_id = RegionId::new(1, 1);
1876        let detecting_region = (1, region_id);
1877        supervisor
1878            .register_failure_detectors(vec![detecting_region])
1879            .await;
1880        supervisor.failover_counts.insert(detecting_region, 1);
1881        let result = SubmitRegionMigrationTaskResult {
1882            migrating: vec![region_id],
1883            ..Default::default()
1884        };
1885        supervisor
1886            .handle_submit_region_migration_task_result(
1887                1,
1888                2,
1889                Duration::from_millis(1000),
1890                RegionMigrationTriggerReason::Manual,
1891                result,
1892            )
1893            .await
1894            .unwrap();
1895        assert!(supervisor.failure_detector.contains(&detecting_region));
1896        assert!(supervisor.failover_counts.contains_key(&detecting_region));
1897    }
1898
1899    #[tokio::test]
1900    async fn test_handle_submit_region_migration_task_result_table_not_found() {
1901        common_telemetry::init_default_ut_logging();
1902        let (mut supervisor, _) = new_test_supervisor();
1903        let region_id = RegionId::new(1, 1);
1904        let detecting_region = (1, region_id);
1905        supervisor
1906            .register_failure_detectors(vec![detecting_region])
1907            .await;
1908        supervisor.failover_counts.insert(detecting_region, 1);
1909        let result = SubmitRegionMigrationTaskResult {
1910            table_not_found: vec![region_id],
1911            ..Default::default()
1912        };
1913        supervisor
1914            .handle_submit_region_migration_task_result(
1915                1,
1916                2,
1917                Duration::from_millis(1000),
1918                RegionMigrationTriggerReason::Manual,
1919                result,
1920            )
1921            .await
1922            .unwrap();
1923        assert!(!supervisor.failure_detector.contains(&detecting_region));
1924        assert!(supervisor.failover_counts.is_empty());
1925    }
1926
1927    #[tokio::test]
1928    async fn test_handle_submit_region_migration_task_result_region_not_found() {
1929        common_telemetry::init_default_ut_logging();
1930        let (mut supervisor, _) = new_test_supervisor();
1931        let region_id = RegionId::new(1, 1);
1932        let detecting_region = (1, region_id);
1933        supervisor
1934            .register_failure_detectors(vec![detecting_region])
1935            .await;
1936        supervisor.failover_counts.insert(detecting_region, 1);
1937        let result = SubmitRegionMigrationTaskResult {
1938            region_not_found: vec![region_id],
1939            ..Default::default()
1940        };
1941        supervisor
1942            .handle_submit_region_migration_task_result(
1943                1,
1944                2,
1945                Duration::from_millis(1000),
1946                RegionMigrationTriggerReason::Manual,
1947                result,
1948            )
1949            .await
1950            .unwrap();
1951        assert!(!supervisor.failure_detector.contains(&detecting_region));
1952        assert!(supervisor.failover_counts.is_empty());
1953    }
1954
1955    #[tokio::test]
1956    async fn test_handle_submit_region_migration_task_result_leader_changed() {
1957        common_telemetry::init_default_ut_logging();
1958        let (mut supervisor, _) = new_test_supervisor();
1959        let region_id = RegionId::new(1, 1);
1960        let detecting_region = (1, region_id);
1961        supervisor
1962            .register_failure_detectors(vec![detecting_region])
1963            .await;
1964        supervisor.failover_counts.insert(detecting_region, 1);
1965        let result = SubmitRegionMigrationTaskResult {
1966            leader_changed: vec![region_id],
1967            ..Default::default()
1968        };
1969        supervisor
1970            .handle_submit_region_migration_task_result(
1971                1,
1972                2,
1973                Duration::from_millis(1000),
1974                RegionMigrationTriggerReason::Manual,
1975                result,
1976            )
1977            .await
1978            .unwrap();
1979        assert!(!supervisor.failure_detector.contains(&detecting_region));
1980        assert!(supervisor.failover_counts.is_empty());
1981    }
1982
1983    #[tokio::test]
1984    async fn test_handle_submit_region_migration_task_result_peer_conflict() {
1985        common_telemetry::init_default_ut_logging();
1986        let (mut supervisor, _) = new_test_supervisor();
1987        let region_id = RegionId::new(1, 1);
1988        let detecting_region = (1, region_id);
1989        supervisor
1990            .register_failure_detectors(vec![detecting_region])
1991            .await;
1992        supervisor.failover_counts.insert(detecting_region, 1);
1993        let result = SubmitRegionMigrationTaskResult {
1994            peer_conflict: vec![region_id],
1995            ..Default::default()
1996        };
1997        supervisor
1998            .handle_submit_region_migration_task_result(
1999                1,
2000                2,
2001                Duration::from_millis(1000),
2002                RegionMigrationTriggerReason::Manual,
2003                result,
2004            )
2005            .await
2006            .unwrap();
2007        assert!(supervisor.failure_detector.contains(&detecting_region));
2008        assert!(supervisor.failover_counts.contains_key(&detecting_region));
2009    }
2010
2011    #[tokio::test]
2012    async fn test_handle_submit_region_migration_task_result_submitted() {
2013        common_telemetry::init_default_ut_logging();
2014        let (mut supervisor, _) = new_test_supervisor();
2015        let region_id = RegionId::new(1, 1);
2016        let detecting_region = (1, region_id);
2017        supervisor
2018            .register_failure_detectors(vec![detecting_region])
2019            .await;
2020        supervisor.failover_counts.insert(detecting_region, 1);
2021        let result = SubmitRegionMigrationTaskResult {
2022            submitted: vec![region_id],
2023            ..Default::default()
2024        };
2025        supervisor
2026            .handle_submit_region_migration_task_result(
2027                1,
2028                2,
2029                Duration::from_millis(1000),
2030                RegionMigrationTriggerReason::Manual,
2031                result,
2032            )
2033            .await
2034            .unwrap();
2035        assert!(supervisor.failure_detector.contains(&detecting_region));
2036        assert!(supervisor.failover_counts.contains_key(&detecting_region));
2037    }
2038
2039    #[tokio::test]
2040    async fn test_heartbeat_accept_does_not_block_on_full_queue() {
2041        // Regression test for #9419: a full queue used to park the caller.
2042        // The supervisor is never run, so the queue stays full.
2043        let (mut supervisor, sender) = new_test_supervisor();
2044
2045        let mut queued = 0;
2046        while sender
2047            .try_send(Event::HeartbeatArrived(filler_heartbeat()))
2048            .is_ok()
2049        {
2050            queued += 1;
2051        }
2052        assert_eq!(queued, sender.max_capacity());
2053        assert_eq!(sender.capacity(), 0);
2054
2055        let acceptor = HeartbeatAcceptor::new(sender.clone());
2056        let dropped = || {
2057            METRIC_META_HEARTBEAT_DROPPED
2058                .with_label_values(&["full"])
2059                .get()
2060        };
2061        let dropped_before = dropped();
2062        // The full queue drops the heartbeat and counts it.
2063        acceptor.accept(filler_heartbeat());
2064        assert_eq!(dropped(), dropped_before + 1);
2065        assert_eq!(sender.capacity(), 0);
2066
2067        // Free one slot: the next heartbeat is accepted.
2068        supervisor.receiver.try_recv().unwrap();
2069        acceptor.accept(filler_heartbeat());
2070        assert_eq!(dropped(), dropped_before + 1);
2071        assert_eq!(sender.capacity(), 0);
2072    }
2073}