Skip to main content

meta_srv/region/
lease_keeper.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{HashMap, HashSet};
16use std::sync::Arc;
17
18use common_meta::DatanodeId;
19use common_meta::key::TableMetadataManagerRef;
20use common_meta::key::table_route::TableRouteValue;
21use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegion};
22use common_meta::rpc::router::RegionRoute;
23use common_telemetry::{info, warn};
24use snafu::ResultExt;
25use store_api::region_engine::{GrantedRegion, RegionRole};
26use store_api::storage::{RegionId, TableId};
27
28use crate::error::{self, Result};
29
30pub type RegionLeaseKeeperRef = Arc<RegionLeaseKeeper>;
31
32/// Renews lease of regions.
33pub struct RegionLeaseKeeper {
34    table_metadata_manager: TableMetadataManagerRef,
35    memory_region_keeper: MemoryRegionKeeperRef,
36}
37
38/// The result of region lease renewal,
39/// contains the renewed region leases and [RegionId] of non-existing regions.
40pub struct RenewRegionLeasesResponse {
41    pub renewed: HashMap<RegionId, RegionLeaseInfo>,
42    pub non_exists: HashSet<RegionId>,
43}
44
45impl RegionLeaseKeeper {
46    pub fn new(
47        table_metadata_manager: TableMetadataManagerRef,
48        memory_region_keeper: MemoryRegionKeeperRef,
49    ) -> Self {
50        Self {
51            table_metadata_manager,
52            memory_region_keeper,
53        }
54    }
55}
56
57fn renew_region_lease_via_region_route(
58    region_route: &RegionRoute,
59    datanode_id: DatanodeId,
60    region_id: RegionId,
61) -> Option<(RegionId, RegionRole)> {
62    // If it's a leader region on this datanode.
63    if let Some(leader) = &region_route.leader_peer
64        && leader.id == datanode_id
65    {
66        return region_route
67            .leader_region_role()
68            .map(|region_role| (region_id, region_role));
69    }
70
71    // If it's a follower region on this datanode.
72    if region_route
73        .follower_peers
74        .iter()
75        .any(|peer| peer.id == datanode_id)
76    {
77        return Some((region_id, RegionRole::Follower));
78    }
79
80    None
81}
82
83fn renew_region_lease_via_operating_regions(
84    operating_regions: &HashMap<RegionId, OperatingRegion>,
85    datanode_id: DatanodeId,
86    region_id: RegionId,
87    reported_role: RegionRole,
88    table_route_exists: bool,
89) -> Option<RegionLeaseInfo> {
90    // `operating_regions` is filtered by the current datanode in `collect_metadata`,
91    // so looking up by `region_id` is sufficient here.
92    if let Some(operating_region) = operating_regions.get(&region_id) {
93        let role = operating_region.role;
94        let mut region_lease_info = RegionLeaseInfo::operating(region_id, role);
95        // Without the table route, the registration alone must not vouch for a migration.
96        if table_route_exists {
97            region_lease_info.candidate_lease_id = operating_region.candidate_lease_id;
98        }
99        if role != reported_role {
100            info!(
101                "The region {} on datanode {} is operating with role {:?}, but reported as {:?}",
102                region_id, datanode_id, role, reported_role
103            );
104        }
105        return Some(region_lease_info);
106    }
107
108    None
109}
110
111/// The information of region lease.
112#[derive(Debug, PartialEq, Eq)]
113pub struct RegionLeaseInfo {
114    pub region_id: RegionId,
115    /// Whether the region is operating.
116    ///
117    /// The region is under dropping or opening / migration operation.
118    pub is_operating: bool,
119    /// The role of region.
120    pub role: RegionRole,
121    /// The candidate lease id of the region migration that registered this region as its
122    /// target, if the lease is granted because of that registration and the table route exists.
123    pub candidate_lease_id: Option<u64>,
124}
125
126impl RegionLeaseInfo {
127    /// Creates a new [RegionLeaseInfo] with the given region id and role with operating status.
128    pub fn operating(region_id: RegionId, role: RegionRole) -> Self {
129        Self {
130            region_id,
131            is_operating: true,
132            role,
133            candidate_lease_id: None,
134        }
135    }
136}
137
138impl From<(RegionId, RegionRole)> for RegionLeaseInfo {
139    fn from((region_id, role): (RegionId, RegionRole)) -> Self {
140        Self {
141            region_id,
142            is_operating: false,
143            role,
144            candidate_lease_id: None,
145        }
146    }
147}
148
149impl From<RegionLeaseInfo> for GrantedRegion {
150    fn from(value: RegionLeaseInfo) -> Self {
151        GrantedRegion {
152            candidate_lease_id: value.candidate_lease_id,
153            ..GrantedRegion::new(value.region_id, value.role)
154        }
155    }
156}
157
158impl RegionLeaseKeeper {
159    async fn collect_table_metadata(
160        &self,
161        table_ids: &[TableId],
162    ) -> Result<HashMap<TableId, TableRouteValue>> {
163        let table_route_manager = self.table_metadata_manager.table_route_manager();
164
165        // The subset of all table metadata.
166        // TODO: considers storing all active regions in meta's memory.
167        let table_routes = table_route_manager
168            .table_route_storage()
169            .batch_get(table_ids)
170            .await
171            .context(error::TableMetadataManagerSnafu)?;
172
173        let metadata_subset = table_routes
174            .into_iter()
175            .zip(table_ids)
176            .filter_map(|(route, id)| route.map(|route| (*id, route)))
177            .collect::<HashMap<_, _>>();
178
179        Ok(metadata_subset)
180    }
181
182    /// Returns [None] if:
183    /// - The region doesn't belong to the datanode in metadata or operating regions.
184    /// - The region belongs to a logical table.
185    fn renew_region_lease(
186        &self,
187        table_metadata: &HashMap<TableId, TableRouteValue>,
188        operating_regions: &HashMap<RegionId, OperatingRegion>,
189        datanode_id: DatanodeId,
190        region_id: RegionId,
191        reported_role: RegionRole,
192    ) -> Option<RegionLeaseInfo> {
193        let table_route = table_metadata.get(&region_id.table_id());
194        // First try to renew via region route
195        if let Some(table_route) = table_route
196            && let Ok(Some(region_route)) = table_route.region_route(region_id)
197            && let Some(region_lease) =
198                renew_region_lease_via_region_route(&region_route, datanode_id, region_id)
199        {
200            return Some(RegionLeaseInfo::from(region_lease));
201        }
202        // Then try to renew via operating regions, which covers the opening region without region route in metadata.
203        if let Some(region_lease_info) = renew_region_lease_via_operating_regions(
204            operating_regions,
205            datanode_id,
206            region_id,
207            reported_role,
208            table_route.is_some(),
209        ) {
210            return Some(region_lease_info);
211        }
212
213        warn!(
214            "Denied to renew region lease for datanode: {datanode_id}, region_id: {region_id}, no matching metadata or operating region found",
215        );
216        None
217    }
218
219    async fn collect_metadata(
220        &self,
221        datanode_id: DatanodeId,
222        region_ids: HashSet<RegionId>,
223    ) -> Result<(
224        HashMap<TableId, TableRouteValue>,
225        HashMap<RegionId, OperatingRegion>,
226    )> {
227        let operating_regions = self
228            .memory_region_keeper
229            .extract_operating_regions(datanode_id, &region_ids);
230        let table_ids = region_ids
231            .into_iter()
232            .map(|region_id| region_id.table_id())
233            .collect::<HashSet<_>>()
234            .into_iter()
235            .collect::<Vec<_>>();
236        let table_metadata = self.collect_table_metadata(&table_ids).await?;
237        Ok((table_metadata, operating_regions))
238    }
239
240    /// Renews the lease of regions for specific datanode.
241    ///
242    /// The lease of regions will be renewed if:
243    /// -  The region of the specific datanode exists in [TableRouteValue].
244    /// -  The region of the specific datanode is opening.
245    ///
246    /// Otherwise the lease of regions will not be renewed,
247    /// and corresponding regions will be added to `non_exists` of [RenewRegionLeasesResponse].
248    pub async fn renew_region_leases(
249        &self,
250        datanode_id: DatanodeId,
251        regions: &[(RegionId, RegionRole)],
252    ) -> Result<RenewRegionLeasesResponse> {
253        let region_ids = regions
254            .iter()
255            .map(|(region_id, _)| *region_id)
256            .collect::<HashSet<_>>();
257        let (table_metadata, operating_regions) =
258            self.collect_metadata(datanode_id, region_ids).await?;
259        let mut renewed = HashMap::new();
260        let mut non_exists = HashSet::new();
261
262        for &(region, reported_role) in regions {
263            match self.renew_region_lease(
264                &table_metadata,
265                &operating_regions,
266                datanode_id,
267                region,
268                reported_role,
269            ) {
270                Some(region_lease_info) => {
271                    renewed.insert(region_lease_info.region_id, region_lease_info);
272                }
273                None => {
274                    non_exists.insert(region);
275                }
276            }
277        }
278
279        Ok(RenewRegionLeasesResponse {
280            renewed,
281            non_exists,
282        })
283    }
284
285    #[cfg(test)]
286    pub fn table_metadata_manager(&self) -> &TableMetadataManagerRef {
287        &self.table_metadata_manager
288    }
289}
290
291#[cfg(test)]
292mod tests {
293    use std::collections::{HashMap, HashSet};
294    use std::sync::Arc;
295
296    use common_meta::DatanodeId;
297    use common_meta::key::TableMetadataManager;
298    use common_meta::key::table_route::{LogicalTableRouteValue, TableRouteValue};
299    use common_meta::key::test_utils::new_test_table_info;
300    use common_meta::kv_backend::memory::MemoryKvBackend;
301    use common_meta::peer::Peer;
302    use common_meta::region_keeper::{MemoryRegionKeeper, OperatingRegion, OperatingRegionGuard};
303    use common_meta::rpc::router::{LeaderState, Region, RegionRouteBuilder};
304    use store_api::region_engine::RegionRole;
305    use store_api::storage::RegionId;
306    use table::metadata::TableInfo;
307
308    use super::{RegionLeaseKeeper, renew_region_lease_via_region_route};
309    use crate::region::lease_keeper::{RegionLeaseInfo, RenewRegionLeasesResponse};
310
311    fn new_test_keeper() -> RegionLeaseKeeper {
312        let store = Arc::new(MemoryKvBackend::new());
313
314        let table_metadata_manager = Arc::new(TableMetadataManager::new(store));
315
316        let opening_region_keeper = Arc::new(MemoryRegionKeeper::default());
317        RegionLeaseKeeper::new(table_metadata_manager, opening_region_keeper)
318    }
319
320    #[test]
321    fn test_renew_region_lease_via_region_route() {
322        let region_id = RegionId::new(1024, 1);
323        let leader_peer_id = 1024;
324        let follower_peer_id = 2048;
325        let mut region_route = RegionRouteBuilder::default()
326            .region(Region::new_test(region_id))
327            .leader_peer(Peer::empty(leader_peer_id))
328            .follower_peers(vec![Peer::empty(follower_peer_id)])
329            .build()
330            .unwrap();
331
332        // The region doesn't belong to the datanode.
333        for region_id in [RegionId::new(1024, 2), region_id] {
334            assert!(renew_region_lease_via_region_route(&region_route, 1, region_id).is_none());
335        }
336
337        // The leader region on the datanode.
338        assert_eq!(
339            renew_region_lease_via_region_route(&region_route, leader_peer_id, region_id),
340            Some((region_id, RegionRole::Leader))
341        );
342        // The follower region on the datanode.
343        assert_eq!(
344            renew_region_lease_via_region_route(&region_route, follower_peer_id, region_id),
345            Some((region_id, RegionRole::Follower))
346        );
347
348        region_route.leader_state = Some(LeaderState::Downgrading);
349        // The downgraded leader region on the datanode.
350        assert_eq!(
351            renew_region_lease_via_region_route(&region_route, leader_peer_id, region_id),
352            Some((region_id, RegionRole::DowngradingLeader))
353        );
354
355        region_route.leader_state = Some(LeaderState::Staging);
356        assert_eq!(
357            renew_region_lease_via_region_route(&region_route, leader_peer_id, region_id),
358            Some((region_id, RegionRole::StagingLeader))
359        );
360    }
361
362    #[tokio::test]
363    async fn test_renew_region_leases_non_exists_regions() {
364        let keeper = new_test_keeper();
365
366        let RenewRegionLeasesResponse {
367            non_exists,
368            renewed,
369        } = keeper
370            .renew_region_leases(
371                1,
372                &[
373                    (RegionId::new(1024, 1), RegionRole::Follower),
374                    (RegionId::new(1024, 2), RegionRole::Leader),
375                ],
376            )
377            .await
378            .unwrap();
379
380        assert!(renewed.is_empty());
381        assert_eq!(
382            non_exists,
383            HashSet::from([RegionId::new(1024, 1), RegionId::new(1024, 2)])
384        );
385    }
386
387    #[tokio::test]
388    async fn test_collect_metadata() {
389        let table_id = 1024;
390        let table_info: TableInfo = new_test_table_info(table_id);
391
392        let region_id = RegionId::new(table_id, 1);
393        let leader_peer_id = 1024;
394        let follower_peer_id = 2048;
395        let region_route = RegionRouteBuilder::default()
396            .region(Region::new_test(region_id))
397            .leader_peer(Peer::empty(leader_peer_id))
398            .follower_peers(vec![Peer::empty(follower_peer_id)])
399            .build()
400            .unwrap();
401
402        let keeper = new_test_keeper();
403        let table_metadata_manager = keeper.table_metadata_manager();
404        table_metadata_manager
405            .create_table_metadata(
406                table_info,
407                TableRouteValue::physical(vec![region_route]),
408                HashMap::default(),
409            )
410            .await
411            .unwrap();
412        let opening_region_id = RegionId::new(1025, 1);
413        let _guard = keeper
414            .memory_region_keeper
415            .register_with_role(leader_peer_id, opening_region_id, RegionRole::Leader)
416            .unwrap();
417        let another_opening_region_id = RegionId::new(1025, 2);
418        let _guard2 = keeper
419            .memory_region_keeper
420            .register_with_role(
421                follower_peer_id,
422                another_opening_region_id,
423                RegionRole::Follower,
424            )
425            .unwrap();
426
427        let (metadata, regions) = keeper
428            .collect_metadata(
429                leader_peer_id,
430                HashSet::from([region_id, opening_region_id]),
431            )
432            .await
433            .unwrap();
434        assert_eq!(
435            metadata.keys().cloned().collect::<Vec<_>>(),
436            vec![region_id.table_id()]
437        );
438        assert_eq!(
439            regions,
440            HashMap::from([(
441                opening_region_id,
442                OperatingRegion {
443                    role: RegionRole::Leader,
444                    candidate_lease_id: None,
445                }
446            )])
447        );
448    }
449
450    #[tokio::test]
451    async fn test_renew_region_leases_basic() {
452        let table_id = 1024;
453        let table_info: TableInfo = new_test_table_info(table_id);
454
455        let region_id = RegionId::new(table_id, 1);
456        let leader_peer_id = 1024;
457        let follower_peer_id = 2048;
458        let region_route = RegionRouteBuilder::default()
459            .region(Region::new_test(region_id))
460            .leader_peer(Peer::empty(leader_peer_id))
461            .follower_peers(vec![Peer::empty(follower_peer_id)])
462            .build()
463            .unwrap();
464
465        let keeper = new_test_keeper();
466        let table_metadata_manager = keeper.table_metadata_manager();
467        table_metadata_manager
468            .create_table_metadata(
469                table_info,
470                TableRouteValue::physical(vec![region_route]),
471                HashMap::default(),
472            )
473            .await
474            .unwrap();
475
476        // The region doesn't belong to the datanode.
477        for region_id in [RegionId::new(1024, 2), region_id] {
478            let RenewRegionLeasesResponse {
479                non_exists,
480                renewed,
481            } = keeper
482                .renew_region_leases(1, &[(region_id, RegionRole::Follower)])
483                .await
484                .unwrap();
485            assert!(renewed.is_empty());
486            assert_eq!(non_exists, HashSet::from([region_id]));
487        }
488
489        // The leader region on the datanode.
490        for role in [RegionRole::Leader, RegionRole::Follower] {
491            let RenewRegionLeasesResponse {
492                non_exists,
493                renewed,
494            } = keeper
495                .renew_region_leases(leader_peer_id, &[(region_id, role)])
496                .await
497                .unwrap();
498
499            assert!(non_exists.is_empty());
500            assert_eq!(
501                renewed,
502                HashMap::from([(
503                    region_id,
504                    RegionLeaseInfo::from((region_id, RegionRole::Leader))
505                )])
506            );
507        }
508
509        // The follower region on the datanode.
510        for role in [RegionRole::Leader, RegionRole::Follower] {
511            let RenewRegionLeasesResponse {
512                non_exists,
513                renewed,
514            } = keeper
515                .renew_region_leases(follower_peer_id, &[(region_id, role)])
516                .await
517                .unwrap();
518
519            assert!(non_exists.is_empty());
520            assert_eq!(
521                renewed,
522                HashMap::from([(
523                    region_id,
524                    RegionLeaseInfo::from((region_id, RegionRole::Follower))
525                )])
526            );
527        }
528
529        let opening_region_id = RegionId::new(2048, 1);
530        let _guard = keeper
531            .memory_region_keeper
532            .register_with_role(leader_peer_id, opening_region_id, RegionRole::Leader)
533            .unwrap();
534
535        // The opening region on the datanode.
536        // NOTES: The procedure lock will ensure only one opening leader.
537        for reported_role in [RegionRole::Leader, RegionRole::Follower] {
538            let RenewRegionLeasesResponse {
539                non_exists,
540                renewed,
541            } = keeper
542                .renew_region_leases(leader_peer_id, &[(opening_region_id, reported_role)])
543                .await
544                .unwrap();
545
546            assert!(non_exists.is_empty());
547            assert_eq!(
548                renewed,
549                HashMap::from([(
550                    opening_region_id,
551                    RegionLeaseInfo::operating(opening_region_id, RegionRole::Leader)
552                )])
553            );
554        }
555    }
556
557    #[tokio::test]
558    async fn test_renew_unexpected_logic_table() {
559        let table_id = 1024;
560        let table_info: TableInfo = new_test_table_info(table_id);
561
562        let region_id = RegionId::new(table_id, 1);
563        let keeper = new_test_keeper();
564        let table_metadata_manager = keeper.table_metadata_manager();
565        table_metadata_manager
566            .create_table_metadata(
567                table_info,
568                TableRouteValue::Logical(LogicalTableRouteValue::new(table_id)),
569                HashMap::default(),
570            )
571            .await
572            .unwrap();
573
574        for region_id in [region_id, RegionId::new(1024, 2)] {
575            let RenewRegionLeasesResponse {
576                non_exists,
577                renewed,
578            } = keeper
579                .renew_region_leases(
580                    1,
581                    &[
582                        (region_id, RegionRole::Follower),
583                        (region_id, RegionRole::Leader),
584                    ],
585                )
586                .await
587                .unwrap();
588            assert!(renewed.is_empty());
589            assert_eq!(non_exists, HashSet::from([region_id]));
590        }
591    }
592
593    #[tokio::test]
594    async fn test_renew_region_leases_with_downgrade_leader() {
595        let table_id = 1024;
596        let table_info: TableInfo = new_test_table_info(table_id);
597
598        let region_id = RegionId::new(table_id, 1);
599        let leader_peer_id = 1024;
600        let follower_peer_id = 2048;
601        let region_route = RegionRouteBuilder::default()
602            .region(Region::new_test(region_id))
603            .leader_peer(Peer::empty(leader_peer_id))
604            .follower_peers(vec![Peer::empty(follower_peer_id)])
605            .leader_state(LeaderState::Downgrading)
606            .build()
607            .unwrap();
608
609        let keeper = new_test_keeper();
610        let table_metadata_manager = keeper.table_metadata_manager();
611        table_metadata_manager
612            .create_table_metadata(
613                table_info,
614                TableRouteValue::physical(vec![region_route]),
615                HashMap::default(),
616            )
617            .await
618            .unwrap();
619
620        // The leader region on the datanode.
621        for role in [RegionRole::Leader, RegionRole::Follower] {
622            let RenewRegionLeasesResponse {
623                non_exists,
624                renewed,
625            } = keeper
626                .renew_region_leases(follower_peer_id, &[(region_id, role)])
627                .await
628                .unwrap();
629
630            assert!(non_exists.is_empty());
631            assert_eq!(
632                renewed,
633                HashMap::from([(
634                    region_id,
635                    RegionLeaseInfo::from((region_id, RegionRole::Follower))
636                )])
637            );
638        }
639    }
640
641    #[tokio::test]
642    async fn test_renew_region_leases_reported_staging_expected_leader() {
643        let table_id = 1024;
644        let table_info: TableInfo = new_test_table_info(table_id);
645
646        let region_id = RegionId::new(table_id, 1);
647        let leader_peer_id = 1024;
648        let region_route = RegionRouteBuilder::default()
649            .region(Region::new_test(region_id))
650            .leader_peer(Peer::empty(leader_peer_id))
651            .build()
652            .unwrap();
653
654        let keeper = new_test_keeper();
655        let table_metadata_manager = keeper.table_metadata_manager();
656        table_metadata_manager
657            .create_table_metadata(
658                table_info,
659                TableRouteValue::physical(vec![region_route]),
660                HashMap::default(),
661            )
662            .await
663            .unwrap();
664
665        let RenewRegionLeasesResponse {
666            non_exists,
667            renewed,
668        } = keeper
669            .renew_region_leases(leader_peer_id, &[(region_id, RegionRole::StagingLeader)])
670            .await
671            .unwrap();
672
673        assert!(non_exists.is_empty());
674        assert_eq!(
675            renewed,
676            HashMap::from([(
677                region_id,
678                RegionLeaseInfo::from((region_id, RegionRole::Leader))
679            )])
680        );
681    }
682
683    #[tokio::test]
684    async fn test_renew_region_leases_reported_staging_expected_staging() {
685        let table_id = 1024;
686        let table_info: TableInfo = new_test_table_info(table_id);
687
688        let region_id = RegionId::new(table_id, 1);
689        let leader_peer_id = 1024;
690        let region_route = RegionRouteBuilder::default()
691            .region(Region::new_test(region_id))
692            .leader_peer(Peer::empty(leader_peer_id))
693            .leader_state(LeaderState::Staging)
694            .build()
695            .unwrap();
696
697        let keeper = new_test_keeper();
698        let table_metadata_manager = keeper.table_metadata_manager();
699        table_metadata_manager
700            .create_table_metadata(
701                table_info,
702                TableRouteValue::physical(vec![region_route]),
703                HashMap::default(),
704            )
705            .await
706            .unwrap();
707
708        let RenewRegionLeasesResponse {
709            non_exists,
710            renewed,
711        } = keeper
712            .renew_region_leases(leader_peer_id, &[(region_id, RegionRole::StagingLeader)])
713            .await
714            .unwrap();
715
716        assert!(non_exists.is_empty());
717        assert_eq!(
718            renewed,
719            HashMap::from([(
720                region_id,
721                RegionLeaseInfo::from((region_id, RegionRole::StagingLeader))
722            )])
723        );
724    }
725
726    #[tokio::test]
727    async fn test_renew_region_leases_metadata_role_beats_keeper_role() {
728        let table_id = 2048;
729        let table_info: TableInfo = new_test_table_info(table_id);
730
731        let datanode_id = 1024;
732        let region_id = RegionId::new(table_id, 1);
733        let region_route = RegionRouteBuilder::default()
734            .region(Region::new_test(region_id))
735            .leader_peer(Peer::empty(datanode_id))
736            .build()
737            .unwrap();
738
739        let keeper = new_test_keeper();
740        let table_metadata_manager = keeper.table_metadata_manager();
741        table_metadata_manager
742            .create_table_metadata(
743                table_info,
744                TableRouteValue::physical(vec![region_route]),
745                HashMap::default(),
746            )
747            .await
748            .unwrap();
749
750        let _guard = keeper
751            .memory_region_keeper
752            .register_with_role(datanode_id, region_id, RegionRole::Follower)
753            .unwrap();
754
755        let RenewRegionLeasesResponse {
756            non_exists,
757            renewed,
758        } = keeper
759            .renew_region_leases(datanode_id, &[(region_id, RegionRole::Follower)])
760            .await
761            .unwrap();
762
763        assert!(non_exists.is_empty());
764        assert_eq!(
765            renewed,
766            HashMap::from([(
767                region_id,
768                RegionLeaseInfo::from((region_id, RegionRole::Leader))
769            )])
770        );
771    }
772
773    #[tokio::test]
774    async fn test_renew_region_leases_missing_route_falls_back_to_keeper_role() {
775        let table_id = 2048;
776        let table_info: TableInfo = new_test_table_info(table_id);
777
778        let datanode_id = 1024;
779        let region_id = RegionId::new(table_id, 1);
780        let another_region_id = RegionId::new(table_id, 2);
781        let region_route = RegionRouteBuilder::default()
782            .region(Region::new_test(another_region_id))
783            .leader_peer(Peer::empty(datanode_id))
784            .build()
785            .unwrap();
786
787        let keeper = new_test_keeper();
788        let table_metadata_manager = keeper.table_metadata_manager();
789        table_metadata_manager
790            .create_table_metadata(
791                table_info,
792                TableRouteValue::physical(vec![region_route]),
793                HashMap::default(),
794            )
795            .await
796            .unwrap();
797
798        let _guard = keeper
799            .memory_region_keeper
800            .register_with_role(datanode_id, region_id, RegionRole::DowngradingLeader)
801            .unwrap();
802
803        let RenewRegionLeasesResponse {
804            non_exists,
805            renewed,
806        } = keeper
807            .renew_region_leases(datanode_id, &[(region_id, RegionRole::StagingLeader)])
808            .await
809            .unwrap();
810
811        assert!(non_exists.is_empty());
812        assert_eq!(
813            renewed,
814            HashMap::from([(
815                region_id,
816                RegionLeaseInfo::operating(region_id, RegionRole::DowngradingLeader)
817            )])
818        );
819    }
820
821    #[tokio::test]
822    async fn test_renew_region_leases_operating_region_uses_keeper_role() {
823        let keeper = new_test_keeper();
824        let datanode_id = 1024;
825        let region_id = RegionId::new(2048, 1);
826
827        let _guard = keeper
828            .memory_region_keeper
829            .register_with_role(datanode_id, region_id, RegionRole::DowngradingLeader)
830            .unwrap();
831
832        let RenewRegionLeasesResponse {
833            non_exists,
834            renewed,
835        } = keeper
836            .renew_region_leases(datanode_id, &[(region_id, RegionRole::StagingLeader)])
837            .await
838            .unwrap();
839
840        assert!(non_exists.is_empty());
841        assert_eq!(
842            renewed,
843            HashMap::from([(
844                region_id,
845                RegionLeaseInfo::operating(region_id, RegionRole::DowngradingLeader)
846            )])
847        );
848    }
849
850    /// Creates a table whose route has the region led by `leader_peer_id`, and registers the region
851    /// on `candidate_peer_id` as the target of a migration.
852    async fn register_candidate_with_route(
853        keeper: &RegionLeaseKeeper,
854        region_id: RegionId,
855        leader_peer_id: DatanodeId,
856        candidate_peer_id: DatanodeId,
857    ) -> OperatingRegionGuard {
858        let region_route = RegionRouteBuilder::default()
859            .region(Region::new_test(region_id))
860            .leader_peer(Peer::empty(leader_peer_id))
861            .build()
862            .unwrap();
863        keeper
864            .table_metadata_manager()
865            .create_table_metadata(
866                new_test_table_info(region_id.table_id()),
867                TableRouteValue::physical(vec![region_route]),
868                HashMap::default(),
869            )
870            .await
871            .unwrap();
872        keeper
873            .memory_region_keeper
874            .register_candidate_region(candidate_peer_id, region_id)
875            .unwrap()
876    }
877
878    #[tokio::test]
879    async fn test_renew_region_lease_from_registration_carries_candidate_lease_id() {
880        let keeper = new_test_keeper();
881        let region_id = RegionId::new(2048, 1);
882        let guard = register_candidate_with_route(&keeper, region_id, 1, 2).await;
883
884        let RenewRegionLeasesResponse { renewed, .. } = keeper
885            .renew_region_leases(2, &[(region_id, RegionRole::Follower)])
886            .await
887            .unwrap();
888
889        let expected = RegionLeaseInfo {
890            candidate_lease_id: guard.candidate_lease_id(),
891            ..RegionLeaseInfo::operating(region_id, RegionRole::Follower)
892        };
893        assert!(expected.candidate_lease_id.is_some());
894        assert_eq!(renewed, HashMap::from([(region_id, expected)]));
895    }
896
897    #[tokio::test]
898    async fn test_renew_region_leases_route_lease_has_no_candidate_lease_id() {
899        let keeper = new_test_keeper();
900        let region_id = RegionId::new(2048, 1);
901        // The route already names the candidate as the leader.
902        let _guard = register_candidate_with_route(&keeper, region_id, 2, 2).await;
903
904        let RenewRegionLeasesResponse { renewed, .. } = keeper
905            .renew_region_leases(2, &[(region_id, RegionRole::Follower)])
906            .await
907            .unwrap();
908
909        assert_eq!(
910            renewed,
911            HashMap::from([(
912                region_id,
913                RegionLeaseInfo::from((region_id, RegionRole::Leader))
914            )])
915        );
916    }
917
918    #[tokio::test]
919    async fn test_renew_region_leases_without_table_route_has_no_candidate_lease_id() {
920        let keeper = new_test_keeper();
921        let region_id = RegionId::new(2048, 1);
922        let _guard = keeper
923            .memory_region_keeper
924            .register_candidate_region(2, region_id)
925            .unwrap();
926
927        let RenewRegionLeasesResponse { renewed, .. } = keeper
928            .renew_region_leases(2, &[(region_id, RegionRole::Follower)])
929            .await
930            .unwrap();
931
932        assert_eq!(
933            renewed,
934            HashMap::from([(
935                region_id,
936                RegionLeaseInfo::operating(region_id, RegionRole::Follower)
937            )])
938        );
939    }
940}