1use std::collections::HashMap;
16use std::sync::Arc;
17
18use api::v1::meta::{HeartbeatRequest, RegionLease, Role};
19use async_trait::async_trait;
20use common_meta::key::TableMetadataManagerRef;
21use common_meta::region_keeper::MemoryRegionKeeperRef;
22use common_telemetry::error;
23use store_api::region_engine::GrantedRegion;
24use store_api::storage::RegionId;
25
26use crate::error::Result;
27use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
28use crate::metasrv::Context;
29use crate::region::RegionLeaseKeeper;
30use crate::region::lease_keeper::{
31 RegionLeaseInfo, RegionLeaseKeeperRef, RenewRegionLeasesResponse,
32};
33
34pub struct RegionLeaseHandler {
35 region_lease_seconds: u64,
36 region_lease_keeper: RegionLeaseKeeperRef,
37 customized_region_lease_renewer: Option<CustomizedRegionLeaseRenewerRef>,
38}
39
40pub type CustomizedRegionLeaseRenewerRef = Arc<dyn CustomizedRegionLeaseRenewer>;
41
42pub trait CustomizedRegionLeaseRenewer: Send + Sync {
43 fn renew(
49 &self,
50 ctx: &mut Context,
51 regions: HashMap<RegionId, RegionLeaseInfo>,
52 ) -> Vec<GrantedRegion>;
53}
54
55impl RegionLeaseHandler {
56 pub fn new(
57 region_lease_seconds: u64,
58 table_metadata_manager: TableMetadataManagerRef,
59 memory_region_keeper: MemoryRegionKeeperRef,
60 customized_region_lease_renewer: Option<CustomizedRegionLeaseRenewerRef>,
61 ) -> Self {
62 let region_lease_keeper =
63 RegionLeaseKeeper::new(table_metadata_manager, memory_region_keeper.clone());
64
65 Self {
66 region_lease_seconds,
67 region_lease_keeper: Arc::new(region_lease_keeper),
68 customized_region_lease_renewer,
69 }
70 }
71}
72
73#[async_trait]
74impl HeartbeatHandler for RegionLeaseHandler {
75 fn is_acceptable(&self, role: Role) -> bool {
76 role == Role::Datanode
77 }
78
79 async fn handle(
80 &self,
81 req: &HeartbeatRequest,
82 ctx: &mut Context,
83 acc: &mut HeartbeatAccumulator,
84 ) -> Result<HandleControl> {
85 let Some(stat) = acc.stat.as_ref() else {
86 return Ok(HandleControl::Continue);
87 };
88
89 let regions = stat.regions();
90 let datanode_id = stat.id;
91
92 match self
93 .region_lease_keeper
94 .renew_region_leases(datanode_id, ®ions)
95 .await
96 {
97 Ok(RenewRegionLeasesResponse {
98 non_exists,
99 renewed,
100 }) => {
101 let renewed = if let Some(renewer) = &self.customized_region_lease_renewer {
102 let candidate_lease_ids = renewed
103 .iter()
104 .filter_map(|(region_id, region_lease_info)| {
105 region_lease_info
106 .candidate_lease_id
107 .map(|candidate_lease_id| (*region_id, candidate_lease_id))
108 })
109 .collect::<HashMap<_, _>>();
110 renewer
111 .renew(ctx, renewed)
112 .into_iter()
113 .map(|region| {
114 GrantedRegion {
115 candidate_lease_id: candidate_lease_ids
116 .get(®ion.region_id)
117 .copied(),
118 ..region
119 }
120 .into()
121 })
122 .collect()
123 } else {
124 renewed
125 .into_values()
126 .map(|region_lease_info| GrantedRegion::from(region_lease_info).into())
127 .collect::<Vec<_>>()
128 };
129
130 acc.region_lease = Some(RegionLease {
131 regions: renewed,
132 duration_since_epoch: req.duration_since_epoch,
133 lease_seconds: self.region_lease_seconds,
134 closeable_region_ids: non_exists.iter().map(|region| region.as_u64()).collect(),
135 });
136 acc.inactive_region_ids = non_exists;
137 }
138 Err(e) => {
139 error!(e; "Failed to renew region leases for datanode: {datanode_id:?}, regions: {:?}", regions);
140 }
143 }
144
145 Ok(HandleControl::Continue)
146 }
147}
148
149#[cfg(test)]
150mod test {
151
152 use std::collections::{HashMap, HashSet};
153 use std::sync::Arc;
154
155 use common_meta::datanode::{RegionManifestInfo, RegionStat, Stat};
156 use common_meta::distributed_time_constants::default_distributed_time_constants;
157 use common_meta::key::TableMetadataManager;
158 use common_meta::key::table_route::TableRouteValue;
159 use common_meta::key::test_utils::new_test_table_info;
160 use common_meta::kv_backend::memory::MemoryKvBackend;
161 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
162 use common_meta::peer::Peer;
163 use common_meta::region_keeper::MemoryRegionKeeper;
164 use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
165 use store_api::region_engine::RegionRole;
166 use store_api::storage::RegionId;
167
168 use super::*;
169 use crate::metasrv::builder::MetasrvBuilder;
170
171 fn new_test_keeper() -> RegionLeaseKeeper {
172 let store = Arc::new(MemoryKvBackend::new());
173
174 let table_metadata_manager = Arc::new(TableMetadataManager::new(store));
175
176 let memory_region_keeper = Arc::new(MemoryRegionKeeper::default());
177 RegionLeaseKeeper::new(table_metadata_manager, memory_region_keeper)
178 }
179
180 fn new_empty_region_stat(region_id: RegionId, role: RegionRole) -> RegionStat {
181 RegionStat {
182 id: region_id,
183 role,
184 rcus: 0,
185 wcus: 0,
186 approximate_bytes: 0,
187 engine: String::new(),
188 num_rows: 0,
189 memtable_size: 0,
190 manifest_size: 0,
191 sst_size: 0,
192 sst_num: 0,
193 index_size: 0,
194 region_manifest: RegionManifestInfo::Mito {
195 manifest_version: 0,
196 flushed_entry_id: 0,
197 file_removed_cnt: 0,
198 },
199 data_topic_latest_entry_id: 0,
200 metadata_topic_latest_entry_id: 0,
201 written_bytes: 0,
202 ingested_parquet_bytes: 0,
203 query_cpu_time: 0,
204 query_scanned_bytes: 0,
205 min_timestamp: None,
206 max_timestamp: None,
207 has_pending_tails: false,
208 }
209 }
210
211 #[tokio::test]
212 async fn test_handle_upgradable_follower() {
213 let datanode_id = 1;
214 let region_number = 1u32;
215 let table_id = 10;
216 let region_id = RegionId::new(table_id, region_number);
217 let another_region_id = RegionId::new(table_id, region_number + 1);
218 let peer = Peer::empty(datanode_id);
219 let follower_peer = Peer::empty(datanode_id + 1);
220 let table_info = new_test_table_info(table_id);
221
222 let region_routes = vec![RegionRoute {
223 region: Region::new_test(region_id),
224 leader_peer: Some(peer.clone()),
225 follower_peers: vec![follower_peer.clone()],
226 ..Default::default()
227 }];
228
229 let keeper = new_test_keeper();
230 let table_metadata_manager = keeper.table_metadata_manager();
231
232 table_metadata_manager
233 .create_table_metadata(
234 table_info,
235 TableRouteValue::physical(region_routes),
236 HashMap::default(),
237 )
238 .await
239 .unwrap();
240
241 let builder = MetasrvBuilder::new();
242 let metasrv = builder.build().await.unwrap();
243 let ctx = &mut metasrv.new_ctx();
244
245 let acc = &mut HeartbeatAccumulator::default();
246
247 acc.stat = Some(Stat {
248 id: peer.id,
249 region_stats: vec![
250 new_empty_region_stat(region_id, RegionRole::Follower),
251 new_empty_region_stat(another_region_id, RegionRole::Follower),
252 ],
253 ..Default::default()
254 });
255
256 let req = HeartbeatRequest {
257 duration_since_epoch: 1234,
258 ..Default::default()
259 };
260
261 let opening_region_keeper = Arc::new(MemoryRegionKeeper::default());
262
263 let handler = RegionLeaseHandler::new(
264 default_distributed_time_constants().region_lease.as_secs(),
265 table_metadata_manager.clone(),
266 opening_region_keeper.clone(),
267 None,
268 );
269
270 handler.handle(&req, ctx, acc).await.unwrap();
271
272 assert_region_lease(acc, vec![GrantedRegion::new(region_id, RegionRole::Leader)]);
273 assert_eq!(acc.inactive_region_ids, HashSet::from([another_region_id]));
274 assert_eq!(
275 acc.region_lease.as_ref().unwrap().closeable_region_ids,
276 vec![another_region_id]
277 );
278
279 let acc = &mut HeartbeatAccumulator::default();
280
281 acc.stat = Some(Stat {
282 id: follower_peer.id,
283 region_stats: vec![
284 new_empty_region_stat(region_id, RegionRole::Follower),
285 new_empty_region_stat(another_region_id, RegionRole::Follower),
286 ],
287 ..Default::default()
288 });
289
290 handler.handle(&req, ctx, acc).await.unwrap();
291
292 assert_eq!(
293 acc.region_lease.as_ref().unwrap().lease_seconds,
294 default_distributed_time_constants().region_lease.as_secs()
295 );
296
297 assert_region_lease(
298 acc,
299 vec![GrantedRegion::new(region_id, RegionRole::Follower)],
300 );
301 assert_eq!(acc.inactive_region_ids, HashSet::from([another_region_id]));
302 assert_eq!(
303 acc.region_lease.as_ref().unwrap().closeable_region_ids,
304 vec![another_region_id]
305 );
306
307 let opening_region_id = RegionId::new(table_id, region_number + 2);
308 let _guard = opening_region_keeper
309 .register_with_role(follower_peer.id, opening_region_id, RegionRole::Follower)
310 .unwrap();
311 let candidate_region_id = RegionId::new(table_id, region_number + 3);
312 let candidate_guard = opening_region_keeper
313 .register_candidate_region(follower_peer.id, candidate_region_id)
314 .unwrap();
315
316 let acc = &mut HeartbeatAccumulator::default();
317
318 acc.stat = Some(Stat {
319 id: follower_peer.id,
320 region_stats: vec![
321 new_empty_region_stat(region_id, RegionRole::Follower),
322 new_empty_region_stat(another_region_id, RegionRole::Follower),
323 new_empty_region_stat(opening_region_id, RegionRole::Follower),
324 new_empty_region_stat(candidate_region_id, RegionRole::Follower),
325 ],
326 ..Default::default()
327 });
328
329 handler.handle(&req, ctx, acc).await.unwrap();
330
331 assert_eq!(
332 acc.region_lease.as_ref().unwrap().lease_seconds,
333 default_distributed_time_constants().region_lease.as_secs()
334 );
335
336 assert_region_lease(
337 acc,
338 vec![
339 GrantedRegion::new(region_id, RegionRole::Follower),
340 GrantedRegion::new(opening_region_id, RegionRole::Follower),
341 GrantedRegion {
342 candidate_lease_id: Some(candidate_guard.candidate_lease_id().unwrap()),
343 ..GrantedRegion::new(candidate_region_id, RegionRole::Follower)
344 },
345 ],
346 );
347 assert_eq!(acc.inactive_region_ids, HashSet::from([another_region_id]));
348 assert_eq!(
349 acc.region_lease.as_ref().unwrap().closeable_region_ids,
350 vec![another_region_id]
351 );
352 }
353
354 struct StubRegionLeaseRenewer {
357 skipped: RegionId,
358 extra: RegionId,
359 }
360
361 impl CustomizedRegionLeaseRenewer for StubRegionLeaseRenewer {
362 fn renew(
363 &self,
364 _ctx: &mut Context,
365 regions: HashMap<RegionId, RegionLeaseInfo>,
366 ) -> Vec<GrantedRegion> {
367 regions
368 .into_values()
369 .filter(|info| info.region_id != self.skipped)
370 .map(|info| (info.region_id, info.role))
371 .chain([(self.extra, RegionRole::Follower)])
372 .map(|(region_id, role)| GrantedRegion {
373 candidate_lease_id: Some(u64::MAX),
374 ..GrantedRegion::new(region_id, role)
375 })
376 .collect()
377 }
378 }
379
380 #[tokio::test]
381 async fn test_customized_renewer_leases_carry_candidate_lease_id() {
382 let table_id = 10;
383 let region_id = RegionId::new(table_id, 1);
384 let candidate_region_id = RegionId::new(table_id, 2);
385 let skipped_region_id = RegionId::new(table_id, 3);
386 let extra_region_id = RegionId::new(table_id, 4);
387 let leader_peer = Peer::empty(1);
388 let follower_peer = Peer::empty(2);
389
390 let keeper = new_test_keeper();
391 let table_metadata_manager = keeper.table_metadata_manager();
392 table_metadata_manager
393 .create_table_metadata(
394 new_test_table_info(table_id),
395 TableRouteValue::physical(vec![RegionRoute {
396 region: Region::new_test(region_id),
397 leader_peer: Some(leader_peer),
398 follower_peers: vec![follower_peer.clone()],
399 ..Default::default()
400 }]),
401 HashMap::default(),
402 )
403 .await
404 .unwrap();
405
406 let opening_region_keeper = Arc::new(MemoryRegionKeeper::default());
407 let candidate_guard = opening_region_keeper
408 .register_candidate_region(follower_peer.id, candidate_region_id)
409 .unwrap();
410 let _skipped_guard = opening_region_keeper
411 .register_candidate_region(follower_peer.id, skipped_region_id)
412 .unwrap();
413
414 let handler = RegionLeaseHandler::new(
415 default_distributed_time_constants().region_lease.as_secs(),
416 table_metadata_manager.clone(),
417 opening_region_keeper,
418 Some(Arc::new(StubRegionLeaseRenewer {
419 skipped: skipped_region_id,
420 extra: extra_region_id,
421 })),
422 );
423 let metasrv = MetasrvBuilder::new().build().await.unwrap();
424 let ctx = &mut metasrv.new_ctx();
425 let acc = &mut HeartbeatAccumulator::default();
426 acc.stat = Some(Stat {
427 id: follower_peer.id,
428 region_stats: vec![
429 new_empty_region_stat(region_id, RegionRole::Follower),
430 new_empty_region_stat(candidate_region_id, RegionRole::Follower),
431 new_empty_region_stat(skipped_region_id, RegionRole::Follower),
432 ],
433 ..Default::default()
434 });
435
436 handler
437 .handle(&HeartbeatRequest::default(), ctx, acc)
438 .await
439 .unwrap();
440
441 assert_region_lease(
442 acc,
443 vec![
444 GrantedRegion::new(region_id, RegionRole::Follower),
445 GrantedRegion {
446 candidate_lease_id: Some(candidate_guard.candidate_lease_id().unwrap()),
447 ..GrantedRegion::new(candidate_region_id, RegionRole::Follower)
448 },
449 GrantedRegion::new(extra_region_id, RegionRole::Follower),
450 ],
451 );
452 }
453
454 #[tokio::test]
455
456 async fn test_handle_downgradable_leader() {
457 let datanode_id = 1;
458 let region_number = 1u32;
459 let table_id = 10;
460 let region_id = RegionId::new(table_id, region_number);
461 let another_region_id = RegionId::new(table_id, region_number + 1);
462 let no_exist_region_id = RegionId::new(table_id, region_number + 2);
463 let peer = Peer::empty(datanode_id);
464 let follower_peer = Peer::empty(datanode_id + 1);
465 let table_info = new_test_table_info(table_id);
466
467 let region_routes = vec![
468 RegionRoute {
469 region: Region::new_test(region_id),
470 leader_peer: Some(peer.clone()),
471 follower_peers: vec![follower_peer.clone()],
472 leader_state: Some(LeaderState::Downgrading),
473 leader_down_since: Some(1),
474 write_route_policy: None,
475 },
476 RegionRoute {
477 region: Region::new_test(another_region_id),
478 leader_peer: Some(peer.clone()),
479 ..Default::default()
480 },
481 ];
482
483 let keeper = new_test_keeper();
484 let table_metadata_manager = keeper.table_metadata_manager();
485
486 table_metadata_manager
487 .create_table_metadata(
488 table_info,
489 TableRouteValue::physical(region_routes),
490 HashMap::default(),
491 )
492 .await
493 .unwrap();
494
495 let builder = MetasrvBuilder::new();
496 let metasrv = builder.build().await.unwrap();
497 let ctx = &mut metasrv.new_ctx();
498
499 let req = HeartbeatRequest {
500 duration_since_epoch: 1234,
501 ..Default::default()
502 };
503
504 let acc = &mut HeartbeatAccumulator::default();
505
506 acc.stat = Some(Stat {
507 id: peer.id,
508 region_stats: vec![
509 new_empty_region_stat(region_id, RegionRole::Leader),
510 new_empty_region_stat(another_region_id, RegionRole::Leader),
511 new_empty_region_stat(no_exist_region_id, RegionRole::Leader),
512 ],
513 ..Default::default()
514 });
515
516 let handler = RegionLeaseHandler::new(
517 default_distributed_time_constants().region_lease.as_secs(),
518 table_metadata_manager.clone(),
519 Default::default(),
520 None,
521 );
522
523 handler.handle(&req, ctx, acc).await.unwrap();
524
525 assert_region_lease(
526 acc,
527 vec![
528 GrantedRegion::new(region_id, RegionRole::DowngradingLeader),
529 GrantedRegion::new(another_region_id, RegionRole::Leader),
530 ],
531 );
532 assert_eq!(acc.inactive_region_ids, HashSet::from([no_exist_region_id]));
533 }
534
535 #[tokio::test]
536 async fn test_handle_staging_leader() {
537 let datanode_id = 1;
538 let region_number = 1u32;
539 let table_id = 10;
540 let region_id = RegionId::new(table_id, region_number);
541 let peer = Peer::empty(datanode_id);
542 let table_info = new_test_table_info(table_id);
543
544 let region_routes = vec![RegionRoute {
545 region: Region::new_test(region_id),
546 leader_peer: Some(peer.clone()),
547 leader_state: Some(LeaderState::Staging),
548 ..Default::default()
549 }];
550
551 let keeper = new_test_keeper();
552 let table_metadata_manager = keeper.table_metadata_manager();
553
554 table_metadata_manager
555 .create_table_metadata(
556 table_info,
557 TableRouteValue::physical(region_routes),
558 HashMap::default(),
559 )
560 .await
561 .unwrap();
562
563 let builder = MetasrvBuilder::new();
564 let metasrv = builder.build().await.unwrap();
565 let ctx = &mut metasrv.new_ctx();
566
567 let req = HeartbeatRequest {
568 duration_since_epoch: 1234,
569 ..Default::default()
570 };
571
572 let acc = &mut HeartbeatAccumulator::default();
573 acc.stat = Some(Stat {
574 id: peer.id,
575 region_stats: vec![new_empty_region_stat(region_id, RegionRole::StagingLeader)],
576 ..Default::default()
577 });
578
579 let handler = RegionLeaseHandler::new(
580 default_distributed_time_constants().region_lease.as_secs(),
581 table_metadata_manager.clone(),
582 Default::default(),
583 None,
584 );
585
586 handler.handle(&req, ctx, acc).await.unwrap();
587
588 assert_region_lease(
589 acc,
590 vec![GrantedRegion::new(region_id, RegionRole::StagingLeader)],
591 );
592 }
593
594 fn assert_region_lease(acc: &HeartbeatAccumulator, expected: Vec<GrantedRegion>) {
595 let region_lease = acc.region_lease.as_ref().unwrap().clone();
596 let granted: Vec<GrantedRegion> = region_lease
597 .regions
598 .into_iter()
599 .map(Into::into)
600 .collect::<Vec<_>>();
601
602 let granted = granted
603 .into_iter()
604 .map(|region| (region.region_id, region))
605 .collect::<HashMap<_, _>>();
606
607 let expected = expected
608 .into_iter()
609 .map(|region| (region.region_id, region))
610 .collect::<HashMap<_, _>>();
611
612 assert_eq!(granted, expected);
613 }
614
615 #[tokio::test]
616 async fn test_handle_renew_region_lease_failure() {
617 common_telemetry::init_default_ut_logging();
618 let kv = MockKvBackendBuilder::default()
619 .batch_get_fn(Arc::new(|_| {
620 common_meta::error::UnexpectedSnafu {
621 err_msg: "mock err",
622 }
623 .fail()
624 }) as _)
625 .build()
626 .unwrap();
627 let kvbackend = Arc::new(kv);
628 let table_metadata_manager = Arc::new(TableMetadataManager::new(kvbackend));
629
630 let datanode_id = 1;
631 let region_number = 1u32;
632 let table_id = 10;
633 let region_id = RegionId::new(table_id, region_number);
634 let another_region_id = RegionId::new(table_id, region_number + 1);
635 let no_exist_region_id = RegionId::new(table_id, region_number + 2);
636 let peer = Peer::empty(datanode_id);
637
638 let builder = MetasrvBuilder::new();
639 let metasrv = builder.build().await.unwrap();
640 let ctx = &mut metasrv.new_ctx();
641
642 let req = HeartbeatRequest {
643 duration_since_epoch: 1234,
644 ..Default::default()
645 };
646
647 let acc = &mut HeartbeatAccumulator::default();
648 acc.stat = Some(Stat {
649 id: peer.id,
650 region_stats: vec![
651 new_empty_region_stat(region_id, RegionRole::Leader),
652 new_empty_region_stat(another_region_id, RegionRole::Leader),
653 new_empty_region_stat(no_exist_region_id, RegionRole::Leader),
654 ],
655 ..Default::default()
656 });
657 let handler = RegionLeaseHandler::new(
658 default_distributed_time_constants().region_lease.as_secs(),
659 table_metadata_manager.clone(),
660 Default::default(),
661 None,
662 );
663 handler.handle(&req, ctx, acc).await.unwrap();
664
665 assert!(acc.region_lease.is_none());
666 assert!(acc.inactive_region_ids.is_empty());
667 }
668}