meta_srv/handler/
failure_handler.rs1use api::v1::meta::{HeartbeatRequest, Role};
16use async_trait::async_trait;
17use common_telemetry::info;
18use store_api::storage::RegionId;
19
20use crate::error::Result;
21use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
22use crate::metasrv::Context;
23use crate::region::supervisor::{DatanodeHeartbeat, HeartbeatAcceptor, RegionSupervisor};
24
25pub struct RegionFailureHandler {
26 heartbeat_acceptor: HeartbeatAcceptor,
27}
28
29impl RegionFailureHandler {
30 pub(crate) fn new(
31 mut region_supervisor: RegionSupervisor,
32 heartbeat_acceptor: HeartbeatAcceptor,
33 ) -> Self {
34 info!("Starting region supervisor");
35 common_runtime::spawn_global(async move { region_supervisor.run().await });
36 Self { heartbeat_acceptor }
37 }
38}
39
40#[async_trait]
41impl HeartbeatHandler for RegionFailureHandler {
42 fn is_acceptable(&self, role: Role) -> bool {
43 role == Role::Datanode
44 }
45
46 async fn handle(
47 &self,
48 req: &HeartbeatRequest,
49 _ctx: &mut Context,
50 acc: &mut HeartbeatAccumulator,
51 ) -> Result<HandleControl> {
52 let Some(stat) = acc.stat.as_ref() else {
53 return Ok(HandleControl::Continue);
54 };
55
56 let mut heartbeat = DatanodeHeartbeat::from(stat);
59 heartbeat.opening_regions = req
60 .opening_regions
61 .iter()
62 .map(|region| RegionId::from_u64(region.region_id))
63 .collect();
64 self.heartbeat_acceptor.accept(heartbeat);
65
66 Ok(HandleControl::Continue)
67 }
68}
69
70#[cfg(test)]
71mod tests {
72 use std::time::Duration;
73
74 use api::v1::meta::HeartbeatRequest;
75 use common_catalog::consts::default_engine;
76 use common_meta::datanode::{RegionManifestInfo, RegionStat, Stat};
77 use store_api::region_engine::RegionRole;
78 use store_api::storage::RegionId;
79 use tokio::sync::oneshot;
80
81 use crate::handler::failure_handler::RegionFailureHandler;
82 use crate::handler::{HeartbeatAccumulator, HeartbeatHandler};
83 use crate::metasrv::builder::MetasrvBuilder;
84 use crate::region::supervisor::tests::new_test_supervisor;
85 use crate::region::supervisor::{DatanodeHeartbeat, Event, HeartbeatAcceptor};
86
87 fn new_region_stat(region_id: u64) -> RegionStat {
88 RegionStat {
89 id: RegionId::from_u64(region_id),
90 rcus: 0,
91 wcus: 0,
92 approximate_bytes: 0,
93 engine: default_engine().to_string(),
94 role: RegionRole::Follower,
95 num_rows: 0,
96 memtable_size: 0,
97 manifest_size: 0,
98 sst_size: 0,
99 sst_num: 0,
100 index_size: 0,
101 region_manifest: RegionManifestInfo::Mito {
102 manifest_version: 0,
103 flushed_entry_id: 0,
104 file_removed_cnt: 0,
105 },
106 data_topic_latest_entry_id: 0,
107 metadata_topic_latest_entry_id: 0,
108 written_bytes: 0,
109 ingested_parquet_bytes: 0,
110 query_cpu_time: 0,
111 query_scanned_bytes: 0,
112 min_timestamp: None,
113 max_timestamp: None,
114 has_pending_tails: false,
115 }
116 }
117
118 fn new_stat() -> Stat {
119 Stat {
120 id: 42,
121 region_stats: vec![new_region_stat(1), new_region_stat(2), new_region_stat(3)],
122 timestamp_millis: 1000,
123 ..Default::default()
124 }
125 }
126
127 #[tokio::test]
128 async fn test_handle_heartbeat() {
129 let (supervisor, sender) = new_test_supervisor();
130 let heartbeat_acceptor = HeartbeatAcceptor::new(sender.clone());
131 let handler = RegionFailureHandler::new(supervisor, heartbeat_acceptor);
132 let req = &HeartbeatRequest::default();
133 let builder = MetasrvBuilder::new();
134 let metasrv = builder.build().await.unwrap();
135 let mut ctx = metasrv.new_ctx();
136 let acc = &mut HeartbeatAccumulator::default();
137 acc.stat = Some(new_stat());
138
139 handler.handle(req, &mut ctx, acc).await.unwrap();
140 let (tx, rx) = oneshot::channel();
141 sender.send(Event::Dump(tx)).await.unwrap();
142 let detector = rx.await.unwrap();
143 assert_eq!(detector.iter().collect::<Vec<_>>().len(), 3);
144 }
145
146 #[tokio::test]
147 async fn test_handle_heartbeat_does_not_block_on_full_supervisor_queue() {
148 let (_supervisor, sender) = new_test_supervisor();
151 let handler = RegionFailureHandler {
153 heartbeat_acceptor: HeartbeatAcceptor::new(sender.clone()),
154 };
155
156 let stat = new_stat();
157 let mut queued = 0;
158 while sender
159 .try_send(Event::HeartbeatArrived(DatanodeHeartbeat::from(&stat)))
160 .is_ok()
161 {
162 queued += 1;
163 }
164 assert_eq!(queued, sender.max_capacity());
165
166 let builder = MetasrvBuilder::new();
167 let metasrv = builder.build().await.unwrap();
168 let mut ctx = metasrv.new_ctx();
169 let acc = &mut HeartbeatAccumulator::default();
170 acc.stat = Some(stat);
172
173 let result = tokio::time::timeout(
174 Duration::from_secs(5),
175 handler.handle(&HeartbeatRequest::default(), &mut ctx, acc),
176 )
177 .await;
178 assert!(result.is_ok());
179 }
180}