Skip to main content

meta_srv/handler/
region_lease_handler.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::sync::Arc;
17
18use api::v1::meta::{HeartbeatRequest, RegionLease, Role};
19use async_trait::async_trait;
20use common_meta::key::TableMetadataManagerRef;
21use common_meta::region_keeper::MemoryRegionKeeperRef;
22use common_telemetry::error;
23use store_api::region_engine::GrantedRegion;
24use store_api::storage::RegionId;
25
26use crate::error::Result;
27use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
28use crate::metasrv::Context;
29use crate::region::RegionLeaseKeeper;
30use crate::region::lease_keeper::{
31    RegionLeaseInfo, RegionLeaseKeeperRef, RenewRegionLeasesResponse,
32};
33
34pub struct RegionLeaseHandler {
35    region_lease_seconds: u64,
36    region_lease_keeper: RegionLeaseKeeperRef,
37    customized_region_lease_renewer: Option<CustomizedRegionLeaseRenewerRef>,
38}
39
40pub type CustomizedRegionLeaseRenewerRef = Arc<dyn CustomizedRegionLeaseRenewer>;
41
42pub trait CustomizedRegionLeaseRenewer: Send + Sync {
43    /// Returns the leases to grant for the renewed `regions`.
44    ///
45    /// The handler sets the `candidate_lease_id` of each returned lease from the entry of
46    /// `regions` with the same region id, or to `None` if there is no such entry. The value set
47    /// by the renewer is ignored.
48    fn renew(
49        &self,
50        ctx: &mut Context,
51        regions: HashMap<RegionId, RegionLeaseInfo>,
52    ) -> Vec<GrantedRegion>;
53}
54
55impl RegionLeaseHandler {
56    pub fn new(
57        region_lease_seconds: u64,
58        table_metadata_manager: TableMetadataManagerRef,
59        memory_region_keeper: MemoryRegionKeeperRef,
60        customized_region_lease_renewer: Option<CustomizedRegionLeaseRenewerRef>,
61    ) -> Self {
62        let region_lease_keeper =
63            RegionLeaseKeeper::new(table_metadata_manager, memory_region_keeper.clone());
64
65        Self {
66            region_lease_seconds,
67            region_lease_keeper: Arc::new(region_lease_keeper),
68            customized_region_lease_renewer,
69        }
70    }
71}
72
73#[async_trait]
74impl HeartbeatHandler for RegionLeaseHandler {
75    fn is_acceptable(&self, role: Role) -> bool {
76        role == Role::Datanode
77    }
78
79    async fn handle(
80        &self,
81        req: &HeartbeatRequest,
82        ctx: &mut Context,
83        acc: &mut HeartbeatAccumulator,
84    ) -> Result<HandleControl> {
85        let Some(stat) = acc.stat.as_ref() else {
86            return Ok(HandleControl::Continue);
87        };
88
89        let regions = stat.regions();
90        let datanode_id = stat.id;
91
92        match self
93            .region_lease_keeper
94            .renew_region_leases(datanode_id, &regions)
95            .await
96        {
97            Ok(RenewRegionLeasesResponse {
98                non_exists,
99                renewed,
100            }) => {
101                let renewed = if let Some(renewer) = &self.customized_region_lease_renewer {
102                    let candidate_lease_ids = renewed
103                        .iter()
104                        .filter_map(|(region_id, region_lease_info)| {
105                            region_lease_info
106                                .candidate_lease_id
107                                .map(|candidate_lease_id| (*region_id, candidate_lease_id))
108                        })
109                        .collect::<HashMap<_, _>>();
110                    renewer
111                        .renew(ctx, renewed)
112                        .into_iter()
113                        .map(|region| {
114                            GrantedRegion {
115                                candidate_lease_id: candidate_lease_ids
116                                    .get(&region.region_id)
117                                    .copied(),
118                                ..region
119                            }
120                            .into()
121                        })
122                        .collect()
123                } else {
124                    renewed
125                        .into_values()
126                        .map(|region_lease_info| GrantedRegion::from(region_lease_info).into())
127                        .collect::<Vec<_>>()
128                };
129
130                acc.region_lease = Some(RegionLease {
131                    regions: renewed,
132                    duration_since_epoch: req.duration_since_epoch,
133                    lease_seconds: self.region_lease_seconds,
134                    closeable_region_ids: non_exists.iter().map(|region| region.as_u64()).collect(),
135                });
136                acc.inactive_region_ids = non_exists;
137            }
138            Err(e) => {
139                error!(e; "Failed to renew region leases for datanode: {datanode_id:?}, regions: {:?}", regions);
140                // If we throw error here, the datanode will be marked as failure by region failure handler.
141                // So we only log the error and continue.
142            }
143        }
144
145        Ok(HandleControl::Continue)
146    }
147}
148
149#[cfg(test)]
150mod test {
151
152    use std::collections::{HashMap, HashSet};
153    use std::sync::Arc;
154
155    use common_meta::datanode::{RegionManifestInfo, RegionStat, Stat};
156    use common_meta::distributed_time_constants::default_distributed_time_constants;
157    use common_meta::key::TableMetadataManager;
158    use common_meta::key::table_route::TableRouteValue;
159    use common_meta::key::test_utils::new_test_table_info;
160    use common_meta::kv_backend::memory::MemoryKvBackend;
161    use common_meta::kv_backend::test_util::MockKvBackendBuilder;
162    use common_meta::peer::Peer;
163    use common_meta::region_keeper::MemoryRegionKeeper;
164    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
165    use store_api::region_engine::RegionRole;
166    use store_api::storage::RegionId;
167
168    use super::*;
169    use crate::metasrv::builder::MetasrvBuilder;
170
171    fn new_test_keeper() -> RegionLeaseKeeper {
172        let store = Arc::new(MemoryKvBackend::new());
173
174        let table_metadata_manager = Arc::new(TableMetadataManager::new(store));
175
176        let memory_region_keeper = Arc::new(MemoryRegionKeeper::default());
177        RegionLeaseKeeper::new(table_metadata_manager, memory_region_keeper)
178    }
179
180    fn new_empty_region_stat(region_id: RegionId, role: RegionRole) -> RegionStat {
181        RegionStat {
182            id: region_id,
183            role,
184            rcus: 0,
185            wcus: 0,
186            approximate_bytes: 0,
187            engine: String::new(),
188            num_rows: 0,
189            memtable_size: 0,
190            manifest_size: 0,
191            sst_size: 0,
192            sst_num: 0,
193            index_size: 0,
194            region_manifest: RegionManifestInfo::Mito {
195                manifest_version: 0,
196                flushed_entry_id: 0,
197                file_removed_cnt: 0,
198            },
199            data_topic_latest_entry_id: 0,
200            metadata_topic_latest_entry_id: 0,
201            written_bytes: 0,
202            ingested_parquet_bytes: 0,
203            query_cpu_time: 0,
204            query_scanned_bytes: 0,
205            min_timestamp: None,
206            max_timestamp: None,
207            has_pending_tails: false,
208        }
209    }
210
211    #[tokio::test]
212    async fn test_handle_upgradable_follower() {
213        let datanode_id = 1;
214        let region_number = 1u32;
215        let table_id = 10;
216        let region_id = RegionId::new(table_id, region_number);
217        let another_region_id = RegionId::new(table_id, region_number + 1);
218        let peer = Peer::empty(datanode_id);
219        let follower_peer = Peer::empty(datanode_id + 1);
220        let table_info = new_test_table_info(table_id);
221
222        let region_routes = vec![RegionRoute {
223            region: Region::new_test(region_id),
224            leader_peer: Some(peer.clone()),
225            follower_peers: vec![follower_peer.clone()],
226            ..Default::default()
227        }];
228
229        let keeper = new_test_keeper();
230        let table_metadata_manager = keeper.table_metadata_manager();
231
232        table_metadata_manager
233            .create_table_metadata(
234                table_info,
235                TableRouteValue::physical(region_routes),
236                HashMap::default(),
237            )
238            .await
239            .unwrap();
240
241        let builder = MetasrvBuilder::new();
242        let metasrv = builder.build().await.unwrap();
243        let ctx = &mut metasrv.new_ctx();
244
245        let acc = &mut HeartbeatAccumulator::default();
246
247        acc.stat = Some(Stat {
248            id: peer.id,
249            region_stats: vec![
250                new_empty_region_stat(region_id, RegionRole::Follower),
251                new_empty_region_stat(another_region_id, RegionRole::Follower),
252            ],
253            ..Default::default()
254        });
255
256        let req = HeartbeatRequest {
257            duration_since_epoch: 1234,
258            ..Default::default()
259        };
260
261        let opening_region_keeper = Arc::new(MemoryRegionKeeper::default());
262
263        let handler = RegionLeaseHandler::new(
264            default_distributed_time_constants().region_lease.as_secs(),
265            table_metadata_manager.clone(),
266            opening_region_keeper.clone(),
267            None,
268        );
269
270        handler.handle(&req, ctx, acc).await.unwrap();
271
272        assert_region_lease(acc, vec![GrantedRegion::new(region_id, RegionRole::Leader)]);
273        assert_eq!(acc.inactive_region_ids, HashSet::from([another_region_id]));
274        assert_eq!(
275            acc.region_lease.as_ref().unwrap().closeable_region_ids,
276            vec![another_region_id]
277        );
278
279        let acc = &mut HeartbeatAccumulator::default();
280
281        acc.stat = Some(Stat {
282            id: follower_peer.id,
283            region_stats: vec![
284                new_empty_region_stat(region_id, RegionRole::Follower),
285                new_empty_region_stat(another_region_id, RegionRole::Follower),
286            ],
287            ..Default::default()
288        });
289
290        handler.handle(&req, ctx, acc).await.unwrap();
291
292        assert_eq!(
293            acc.region_lease.as_ref().unwrap().lease_seconds,
294            default_distributed_time_constants().region_lease.as_secs()
295        );
296
297        assert_region_lease(
298            acc,
299            vec![GrantedRegion::new(region_id, RegionRole::Follower)],
300        );
301        assert_eq!(acc.inactive_region_ids, HashSet::from([another_region_id]));
302        assert_eq!(
303            acc.region_lease.as_ref().unwrap().closeable_region_ids,
304            vec![another_region_id]
305        );
306
307        let opening_region_id = RegionId::new(table_id, region_number + 2);
308        let _guard = opening_region_keeper
309            .register_with_role(follower_peer.id, opening_region_id, RegionRole::Follower)
310            .unwrap();
311        let candidate_region_id = RegionId::new(table_id, region_number + 3);
312        let candidate_guard = opening_region_keeper
313            .register_candidate_region(follower_peer.id, candidate_region_id)
314            .unwrap();
315
316        let acc = &mut HeartbeatAccumulator::default();
317
318        acc.stat = Some(Stat {
319            id: follower_peer.id,
320            region_stats: vec![
321                new_empty_region_stat(region_id, RegionRole::Follower),
322                new_empty_region_stat(another_region_id, RegionRole::Follower),
323                new_empty_region_stat(opening_region_id, RegionRole::Follower),
324                new_empty_region_stat(candidate_region_id, RegionRole::Follower),
325            ],
326            ..Default::default()
327        });
328
329        handler.handle(&req, ctx, acc).await.unwrap();
330
331        assert_eq!(
332            acc.region_lease.as_ref().unwrap().lease_seconds,
333            default_distributed_time_constants().region_lease.as_secs()
334        );
335
336        assert_region_lease(
337            acc,
338            vec![
339                GrantedRegion::new(region_id, RegionRole::Follower),
340                GrantedRegion::new(opening_region_id, RegionRole::Follower),
341                GrantedRegion {
342                    candidate_lease_id: Some(candidate_guard.candidate_lease_id().unwrap()),
343                    ..GrantedRegion::new(candidate_region_id, RegionRole::Follower)
344                },
345            ],
346        );
347        assert_eq!(acc.inactive_region_ids, HashSet::from([another_region_id]));
348        assert_eq!(
349            acc.region_lease.as_ref().unwrap().closeable_region_ids,
350            vec![another_region_id]
351        );
352    }
353
354    /// Grants every renewed region except `skipped`, plus `extra`, which is not in the heartbeat.
355    /// Each lease has a candidate lease id the handler must not keep.
356    struct StubRegionLeaseRenewer {
357        skipped: RegionId,
358        extra: RegionId,
359    }
360
361    impl CustomizedRegionLeaseRenewer for StubRegionLeaseRenewer {
362        fn renew(
363            &self,
364            _ctx: &mut Context,
365            regions: HashMap<RegionId, RegionLeaseInfo>,
366        ) -> Vec<GrantedRegion> {
367            regions
368                .into_values()
369                .filter(|info| info.region_id != self.skipped)
370                .map(|info| (info.region_id, info.role))
371                .chain([(self.extra, RegionRole::Follower)])
372                .map(|(region_id, role)| GrantedRegion {
373                    candidate_lease_id: Some(u64::MAX),
374                    ..GrantedRegion::new(region_id, role)
375                })
376                .collect()
377        }
378    }
379
380    #[tokio::test]
381    async fn test_customized_renewer_leases_carry_candidate_lease_id() {
382        let table_id = 10;
383        let region_id = RegionId::new(table_id, 1);
384        let candidate_region_id = RegionId::new(table_id, 2);
385        let skipped_region_id = RegionId::new(table_id, 3);
386        let extra_region_id = RegionId::new(table_id, 4);
387        let leader_peer = Peer::empty(1);
388        let follower_peer = Peer::empty(2);
389
390        let keeper = new_test_keeper();
391        let table_metadata_manager = keeper.table_metadata_manager();
392        table_metadata_manager
393            .create_table_metadata(
394                new_test_table_info(table_id),
395                TableRouteValue::physical(vec![RegionRoute {
396                    region: Region::new_test(region_id),
397                    leader_peer: Some(leader_peer),
398                    follower_peers: vec![follower_peer.clone()],
399                    ..Default::default()
400                }]),
401                HashMap::default(),
402            )
403            .await
404            .unwrap();
405
406        let opening_region_keeper = Arc::new(MemoryRegionKeeper::default());
407        let candidate_guard = opening_region_keeper
408            .register_candidate_region(follower_peer.id, candidate_region_id)
409            .unwrap();
410        let _skipped_guard = opening_region_keeper
411            .register_candidate_region(follower_peer.id, skipped_region_id)
412            .unwrap();
413
414        let handler = RegionLeaseHandler::new(
415            default_distributed_time_constants().region_lease.as_secs(),
416            table_metadata_manager.clone(),
417            opening_region_keeper,
418            Some(Arc::new(StubRegionLeaseRenewer {
419                skipped: skipped_region_id,
420                extra: extra_region_id,
421            })),
422        );
423        let metasrv = MetasrvBuilder::new().build().await.unwrap();
424        let ctx = &mut metasrv.new_ctx();
425        let acc = &mut HeartbeatAccumulator::default();
426        acc.stat = Some(Stat {
427            id: follower_peer.id,
428            region_stats: vec![
429                new_empty_region_stat(region_id, RegionRole::Follower),
430                new_empty_region_stat(candidate_region_id, RegionRole::Follower),
431                new_empty_region_stat(skipped_region_id, RegionRole::Follower),
432            ],
433            ..Default::default()
434        });
435
436        handler
437            .handle(&HeartbeatRequest::default(), ctx, acc)
438            .await
439            .unwrap();
440
441        assert_region_lease(
442            acc,
443            vec![
444                GrantedRegion::new(region_id, RegionRole::Follower),
445                GrantedRegion {
446                    candidate_lease_id: Some(candidate_guard.candidate_lease_id().unwrap()),
447                    ..GrantedRegion::new(candidate_region_id, RegionRole::Follower)
448                },
449                GrantedRegion::new(extra_region_id, RegionRole::Follower),
450            ],
451        );
452    }
453
454    #[tokio::test]
455
456    async fn test_handle_downgradable_leader() {
457        let datanode_id = 1;
458        let region_number = 1u32;
459        let table_id = 10;
460        let region_id = RegionId::new(table_id, region_number);
461        let another_region_id = RegionId::new(table_id, region_number + 1);
462        let no_exist_region_id = RegionId::new(table_id, region_number + 2);
463        let peer = Peer::empty(datanode_id);
464        let follower_peer = Peer::empty(datanode_id + 1);
465        let table_info = new_test_table_info(table_id);
466
467        let region_routes = vec![
468            RegionRoute {
469                region: Region::new_test(region_id),
470                leader_peer: Some(peer.clone()),
471                follower_peers: vec![follower_peer.clone()],
472                leader_state: Some(LeaderState::Downgrading),
473                leader_down_since: Some(1),
474                write_route_policy: None,
475            },
476            RegionRoute {
477                region: Region::new_test(another_region_id),
478                leader_peer: Some(peer.clone()),
479                ..Default::default()
480            },
481        ];
482
483        let keeper = new_test_keeper();
484        let table_metadata_manager = keeper.table_metadata_manager();
485
486        table_metadata_manager
487            .create_table_metadata(
488                table_info,
489                TableRouteValue::physical(region_routes),
490                HashMap::default(),
491            )
492            .await
493            .unwrap();
494
495        let builder = MetasrvBuilder::new();
496        let metasrv = builder.build().await.unwrap();
497        let ctx = &mut metasrv.new_ctx();
498
499        let req = HeartbeatRequest {
500            duration_since_epoch: 1234,
501            ..Default::default()
502        };
503
504        let acc = &mut HeartbeatAccumulator::default();
505
506        acc.stat = Some(Stat {
507            id: peer.id,
508            region_stats: vec![
509                new_empty_region_stat(region_id, RegionRole::Leader),
510                new_empty_region_stat(another_region_id, RegionRole::Leader),
511                new_empty_region_stat(no_exist_region_id, RegionRole::Leader),
512            ],
513            ..Default::default()
514        });
515
516        let handler = RegionLeaseHandler::new(
517            default_distributed_time_constants().region_lease.as_secs(),
518            table_metadata_manager.clone(),
519            Default::default(),
520            None,
521        );
522
523        handler.handle(&req, ctx, acc).await.unwrap();
524
525        assert_region_lease(
526            acc,
527            vec![
528                GrantedRegion::new(region_id, RegionRole::DowngradingLeader),
529                GrantedRegion::new(another_region_id, RegionRole::Leader),
530            ],
531        );
532        assert_eq!(acc.inactive_region_ids, HashSet::from([no_exist_region_id]));
533    }
534
535    #[tokio::test]
536    async fn test_handle_staging_leader() {
537        let datanode_id = 1;
538        let region_number = 1u32;
539        let table_id = 10;
540        let region_id = RegionId::new(table_id, region_number);
541        let peer = Peer::empty(datanode_id);
542        let table_info = new_test_table_info(table_id);
543
544        let region_routes = vec![RegionRoute {
545            region: Region::new_test(region_id),
546            leader_peer: Some(peer.clone()),
547            leader_state: Some(LeaderState::Staging),
548            ..Default::default()
549        }];
550
551        let keeper = new_test_keeper();
552        let table_metadata_manager = keeper.table_metadata_manager();
553
554        table_metadata_manager
555            .create_table_metadata(
556                table_info,
557                TableRouteValue::physical(region_routes),
558                HashMap::default(),
559            )
560            .await
561            .unwrap();
562
563        let builder = MetasrvBuilder::new();
564        let metasrv = builder.build().await.unwrap();
565        let ctx = &mut metasrv.new_ctx();
566
567        let req = HeartbeatRequest {
568            duration_since_epoch: 1234,
569            ..Default::default()
570        };
571
572        let acc = &mut HeartbeatAccumulator::default();
573        acc.stat = Some(Stat {
574            id: peer.id,
575            region_stats: vec![new_empty_region_stat(region_id, RegionRole::StagingLeader)],
576            ..Default::default()
577        });
578
579        let handler = RegionLeaseHandler::new(
580            default_distributed_time_constants().region_lease.as_secs(),
581            table_metadata_manager.clone(),
582            Default::default(),
583            None,
584        );
585
586        handler.handle(&req, ctx, acc).await.unwrap();
587
588        assert_region_lease(
589            acc,
590            vec![GrantedRegion::new(region_id, RegionRole::StagingLeader)],
591        );
592    }
593
594    fn assert_region_lease(acc: &HeartbeatAccumulator, expected: Vec<GrantedRegion>) {
595        let region_lease = acc.region_lease.as_ref().unwrap().clone();
596        let granted: Vec<GrantedRegion> = region_lease
597            .regions
598            .into_iter()
599            .map(Into::into)
600            .collect::<Vec<_>>();
601
602        let granted = granted
603            .into_iter()
604            .map(|region| (region.region_id, region))
605            .collect::<HashMap<_, _>>();
606
607        let expected = expected
608            .into_iter()
609            .map(|region| (region.region_id, region))
610            .collect::<HashMap<_, _>>();
611
612        assert_eq!(granted, expected);
613    }
614
615    #[tokio::test]
616    async fn test_handle_renew_region_lease_failure() {
617        common_telemetry::init_default_ut_logging();
618        let kv = MockKvBackendBuilder::default()
619            .batch_get_fn(Arc::new(|_| {
620                common_meta::error::UnexpectedSnafu {
621                    err_msg: "mock err",
622                }
623                .fail()
624            }) as _)
625            .build()
626            .unwrap();
627        let kvbackend = Arc::new(kv);
628        let table_metadata_manager = Arc::new(TableMetadataManager::new(kvbackend));
629
630        let datanode_id = 1;
631        let region_number = 1u32;
632        let table_id = 10;
633        let region_id = RegionId::new(table_id, region_number);
634        let another_region_id = RegionId::new(table_id, region_number + 1);
635        let no_exist_region_id = RegionId::new(table_id, region_number + 2);
636        let peer = Peer::empty(datanode_id);
637
638        let builder = MetasrvBuilder::new();
639        let metasrv = builder.build().await.unwrap();
640        let ctx = &mut metasrv.new_ctx();
641
642        let req = HeartbeatRequest {
643            duration_since_epoch: 1234,
644            ..Default::default()
645        };
646
647        let acc = &mut HeartbeatAccumulator::default();
648        acc.stat = Some(Stat {
649            id: peer.id,
650            region_stats: vec![
651                new_empty_region_stat(region_id, RegionRole::Leader),
652                new_empty_region_stat(another_region_id, RegionRole::Leader),
653                new_empty_region_stat(no_exist_region_id, RegionRole::Leader),
654            ],
655            ..Default::default()
656        });
657        let handler = RegionLeaseHandler::new(
658            default_distributed_time_constants().region_lease.as_secs(),
659            table_metadata_manager.clone(),
660            Default::default(),
661            None,
662        );
663        handler.handle(&req, ctx, acc).await.unwrap();
664
665        assert!(acc.region_lease.is_none());
666        assert!(acc.inactive_region_ids.is_empty());
667    }
668}