Skip to main content

meta_srv/procedure/region_migration/update_metadata/
upgrade_candidate_region.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use common_error::ext::BoxedError;
16use common_meta::key::datanode_table::RegionInfo;
17use common_meta::lock_key::TableLock;
18use common_meta::rpc::router::{RegionRoute, region_distribution};
19use common_procedure::ContextProviderRef;
20use common_telemetry::{error, info, warn};
21use snafu::{OptionExt, ResultExt, ensure};
22use store_api::storage::RegionId;
23
24use crate::error::{self, Result};
25use crate::procedure::region_migration::Context;
26use crate::procedure::region_migration::update_metadata::UpdateMetadata;
27
28impl UpdateMetadata {
29    /// Returns new [Vec<RegionRoute>].
30    fn build_upgrade_candidate_region_metadata(
31        &self,
32        ctx: &mut Context,
33        region_ids: &[RegionId],
34        mut region_routes: Vec<RegionRoute>,
35    ) -> Result<Vec<RegionRoute>> {
36        let old_leader_peer = &ctx.persistent_ctx.from_peer;
37        let new_leader_peer = &ctx.persistent_ctx.to_peer;
38        for region_id in region_ids {
39            // Find the RegionRoute for this region_id.
40            let region_route = region_routes
41                .iter_mut()
42                .find(|route| route.region.id == *region_id)
43                .context(error::RegionRouteNotFoundSnafu {
44                    region_id: *region_id,
45                })?;
46
47            // Remove any "downgraded leader" state.
48            region_route.set_leader_state(None);
49
50            // Check old leader matches expectation before upgrading to new leader.
51            ensure!(
52                region_route
53                    .leader_peer
54                    .take_if(|old_leader| old_leader.id == old_leader_peer.id)
55                    .is_some(),
56                error::UnexpectedSnafu {
57                    violated: format!(
58                        "Unexpected region leader: {:?} during the candidate-to-leader upgrade; expected: {:?}",
59                        region_route.leader_peer, old_leader_peer
60                    ),
61                }
62            );
63
64            // Set new leader.
65            region_route.leader_peer = Some(new_leader_peer.clone());
66
67            // Remove new leader from followers (avoids duplicate leader/follower).
68            let removed = region_route
69                .follower_peers
70                .extract_if(.., |peer| peer.id == new_leader_peer.id)
71                .collect::<Vec<_>>();
72
73            // Warn if more than one follower with the new leader id was present.
74            if removed.len() > 1 {
75                warn!(
76                    "Removed duplicate followers: {removed:?} during candidate-to-leader upgrade for region: {region_id}"
77                );
78            }
79        }
80
81        info!(
82            "Building metadata for upgrading candidate region to new leader: {:?} for regions: {:?}",
83            new_leader_peer, region_ids,
84        );
85
86        Ok(region_routes)
87    }
88
89    /// Checks if metadata has been upgraded for a list of regions by verifying if their
90    /// leader peers have been switched to a specified peer ID (`to_peer_id`) and that
91    /// no region is in a leader downgrading state.
92    ///
93    /// Returns:
94    /// - `Ok(true)` if all regions' leader is the target peer and no downgrading occurs.
95    /// - `Ok(false)` if any region's leader is not the target peer.
96    /// - Error if region route or leader peer cannot be found, or an unexpected state is detected.
97    fn check_metadata_updated(
98        &self,
99        ctx: &mut Context,
100        region_ids: &[RegionId],
101        region_routes: &[RegionRoute],
102    ) -> Result<bool> {
103        // Iterate through each provided region ID
104        for region_id in region_ids {
105            // Find the route info for this region
106            let region_route = region_routes
107                .iter()
108                .find(|route| route.region.id == *region_id)
109                .context(error::RegionRouteNotFoundSnafu {
110                    region_id: *region_id,
111                })?;
112
113            // Get the leader peer for the region, error if not found
114            let leader_peer = region_route.leader_peer.as_ref().with_context(||error::UnexpectedSnafu {
115                violated: format!(
116                    "The leader peer of region {region_id} is not found during the metadata upgrade check"
117                ),
118            })?;
119
120            // If the leader is not the expected peer, return false (i.e., not yet upgraded)
121            if leader_peer.id != ctx.persistent_ctx.to_peer.id {
122                return Ok(false);
123            } else {
124                // If leader matches but region is in leader downgrading state, error (unexpected state)
125                ensure!(
126                    !region_route.is_leader_downgrading(),
127                    error::UnexpectedSnafu {
128                        violated: format!(
129                            "Unexpected intermediate state is found during the metadata upgrade check for region {region_id}"
130                        ),
131                    }
132                );
133            }
134        }
135
136        // All regions' leader match expected peer and are not downgrading; considered upgraded
137        Ok(true)
138    }
139
140    /// Upgrades the candidate region.
141    ///
142    /// Abort(non-retry):
143    /// - TableRoute or RegionRoute is not found.
144    ///   Typically, it's impossible, there is no other DDL procedure executed concurrently for the current table.
145    ///
146    /// Retry:
147    /// - Failed to update [TableRouteValue](common_meta::key::table_region::TableRegionValue).
148    /// - Failed to retrieve the metadata of table.
149    pub async fn upgrade_candidate_region(
150        &self,
151        ctx: &mut Context,
152        ctx_provider: &ContextProviderRef,
153    ) -> Result<()> {
154        let table_metadata_manager = ctx.table_metadata_manager.clone();
155        let table_regions = ctx.persistent_ctx.table_regions();
156        let from_peer_id = ctx.persistent_ctx.from_peer.id;
157        let to_peer_id = ctx.persistent_ctx.to_peer.id;
158
159        for (table_id, region_ids) in table_regions {
160            let table_lock = TableLock::Write(table_id).into();
161            let _guard = ctx_provider.acquire_lock(&table_lock).await;
162
163            let table_route_value = ctx.get_table_route_value(table_id).await?;
164            let region_routes = table_route_value.region_routes().with_context(|_| {
165                error::UnexpectedLogicalRouteTableSnafu {
166                    err_msg: format!("TableRoute({table_id:?}) is a non-physical TableRouteValue."),
167                }
168            })?;
169            if self.check_metadata_updated(ctx, &region_ids, region_routes)? {
170                continue;
171            }
172            let datanode_table_value = ctx.get_from_peer_datanode_table_value(table_id).await?;
173            let RegionInfo {
174                region_storage_path,
175                region_options,
176                region_wal_options,
177                engine,
178            } = datanode_table_value.region_info.clone();
179            let new_region_routes = self.build_upgrade_candidate_region_metadata(
180                ctx,
181                &region_ids,
182                region_routes.clone(),
183            )?;
184            let region_distribution = region_distribution(region_routes);
185            info!(
186                "Trying to update region routes to {:?} for table: {}",
187                region_distribution, table_id,
188            );
189
190            if let Err(err) = table_metadata_manager
191                .update_table_route(
192                    table_id,
193                    RegionInfo {
194                        engine: engine.clone(),
195                        region_storage_path: region_storage_path.clone(),
196                        region_options: region_options.clone(),
197                        region_wal_options: region_wal_options.clone(),
198                    },
199                    &table_route_value,
200                    new_region_routes,
201                    &region_options,
202                    &region_wal_options,
203                )
204                .await
205                .context(error::TableMetadataManagerSnafu)
206            {
207                error!(err; "Failed to update the table route during the upgrading candidate region: {region_ids:?}, from_peer_id: {from_peer_id}");
208                return Err(BoxedError::new(err)).context(error::RetryLaterWithSourceSnafu {
209                    reason: format!("Failed to update the table route during the upgrading candidate region: {table_id}"),
210                });
211            };
212            info!(
213                "Upgrading candidate region table route success, table_id: {table_id}, regions: {region_ids:?}, to_peer_id: {to_peer_id}"
214            );
215        }
216
217        ctx.deregister_failure_detectors().await;
218        ctx.reset_failure_detectors(&ctx.persistent_ctx.region_ids)
219            .await;
220        // Consumes the guard.
221        ctx.volatile_ctx.opening_region_guards.clear();
222
223        Ok(())
224    }
225}
226
227#[cfg(test)]
228mod tests {
229    use std::assert_matches;
230    use std::sync::Arc;
231
232    use common_meta::key::TableMetadataManager;
233    use common_meta::key::test_utils::new_test_table_info;
234    use common_meta::kv_backend::test_util::MockKvBackendBuilder;
235    use common_meta::peer::Peer;
236    use common_meta::region_keeper::MemoryRegionKeeper;
237    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
238    use common_time::util::current_time_millis;
239    use store_api::region_engine::RegionRole;
240    use store_api::storage::RegionId;
241
242    use crate::error::Error;
243    use crate::procedure::region_migration::close_downgraded_region::CloseDowngradedRegion;
244    use crate::procedure::region_migration::test_util::{
245        self, TestingEnv, assert_table_cache_invalidated, new_procedure_context,
246    };
247    use crate::procedure::region_migration::update_metadata::UpdateMetadata;
248    use crate::procedure::region_migration::{ContextFactory, PersistentContext, State};
249
250    fn new_persistent_context() -> PersistentContext {
251        test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
252    }
253
254    #[tokio::test]
255    async fn test_table_route_is_not_found_error() {
256        let env = TestingEnv::new();
257        let persistent_context = new_persistent_context();
258        let ctx = env.context_factory().new_context(persistent_context);
259
260        let err = ctx.get_table_route_value(1024).await.unwrap_err();
261
262        assert_matches!(err, Error::TableRouteNotFound { .. });
263        assert!(!err.is_retryable());
264    }
265
266    #[tokio::test]
267    async fn test_region_route_is_not_found() {
268        let state = UpdateMetadata::Upgrade;
269        let env = TestingEnv::new();
270        let persistent_context = new_persistent_context();
271        let mut ctx = env.context_factory().new_context(persistent_context);
272
273        let table_info = new_test_table_info(1024);
274        let region_routes = vec![RegionRoute {
275            region: Region::new_test(RegionId::new(1024, 2)),
276            leader_peer: Some(Peer::empty(4)),
277            ..Default::default()
278        }];
279        env.create_physical_table_metadata(table_info, region_routes)
280            .await;
281
282        let table_route_value = ctx.get_table_route_value(1024).await.unwrap();
283        let region_routes = table_route_value
284            .into_inner()
285            .into_physical_table_route()
286            .region_routes;
287        let err = state
288            .build_upgrade_candidate_region_metadata(
289                &mut ctx,
290                &[RegionId::new(1024, 1)],
291                region_routes,
292            )
293            .unwrap_err();
294
295        assert_matches!(err, Error::RegionRouteNotFound { .. });
296        assert!(!err.is_retryable());
297    }
298
299    #[tokio::test]
300    async fn test_region_route_expected_leader() {
301        let state = UpdateMetadata::Upgrade;
302        let env = TestingEnv::new();
303        let persistent_context = new_persistent_context();
304        let mut ctx = env.context_factory().new_context(persistent_context);
305
306        let table_info = new_test_table_info(1024);
307        let region_routes = vec![RegionRoute {
308            region: Region::new_test(RegionId::new(1024, 1)),
309            leader_peer: Some(Peer::empty(3)),
310            ..Default::default()
311        }];
312
313        env.create_physical_table_metadata(table_info, region_routes)
314            .await;
315
316        let table_route_value = ctx.get_table_route_value(1024).await.unwrap();
317        let region_routes = table_route_value
318            .into_inner()
319            .into_physical_table_route()
320            .region_routes;
321        let err = state
322            .build_upgrade_candidate_region_metadata(
323                &mut ctx,
324                &[RegionId::new(1024, 1)],
325                region_routes,
326            )
327            .unwrap_err();
328
329        assert_matches!(err, Error::Unexpected { .. });
330        assert!(!err.is_retryable());
331        assert!(err.to_string().contains("Unexpected region leader"));
332    }
333
334    #[tokio::test]
335    async fn test_build_upgrade_candidate_region_metadata() {
336        let state = UpdateMetadata::Upgrade;
337        let env = TestingEnv::new();
338        let persistent_context = new_persistent_context();
339        let mut ctx = env.context_factory().new_context(persistent_context);
340
341        let table_info = new_test_table_info(1024);
342        let region_routes = vec![RegionRoute {
343            region: Region::new_test(RegionId::new(1024, 1)),
344            leader_peer: Some(Peer::empty(1)),
345            follower_peers: vec![Peer::empty(2), Peer::empty(3)],
346            leader_state: Some(LeaderState::Downgrading),
347            leader_down_since: Some(current_time_millis()),
348            write_route_policy: None,
349        }];
350
351        env.create_physical_table_metadata(table_info, region_routes)
352            .await;
353
354        let table_route_value = ctx.get_table_route_value(1024).await.unwrap();
355        let region_routes = table_route_value
356            .into_inner()
357            .into_physical_table_route()
358            .region_routes;
359        let new_region_routes = state
360            .build_upgrade_candidate_region_metadata(
361                &mut ctx,
362                &[RegionId::new(1024, 1)],
363                region_routes,
364            )
365            .unwrap();
366
367        assert!(!new_region_routes[0].is_leader_downgrading());
368        assert!(new_region_routes[0].leader_down_since.is_none());
369        assert_eq!(new_region_routes[0].follower_peers, vec![Peer::empty(3)]);
370        assert_eq!(new_region_routes[0].leader_peer.as_ref().unwrap().id, 2);
371    }
372
373    #[tokio::test]
374    async fn test_failed_upgrade_broadcasts_invalidation() {
375        // The procedure iterates the tables of a batch in the order of a `HashMap` with a random
376        // hasher (`PersistentContext::table_regions`), so the attempt is repeated until the table
377        // that fails comes after the one that is switched. Each attempt orders the two tables
378        // independently, so all 64 miss with probability 2^-64; a map with a fixed order could
379        // make this test fail every time.
380        for _ in 0..64 {
381            let mut state = Box::new(UpdateMetadata::Upgrade);
382            let mut env = TestingEnv::new();
383            // The table of the second region has been dropped.
384            let mut persistent_context = new_persistent_context();
385            persistent_context.region_ids = vec![RegionId::new(1024, 1), RegionId::new(1025, 1)];
386            let mut ctx = env.context_factory().new_context(persistent_context);
387            let region_routes = vec![RegionRoute {
388                region: Region::new_test(RegionId::new(1024, 1)),
389                leader_peer: Some(Peer::empty(1)),
390                leader_state: Some(LeaderState::Downgrading),
391                ..Default::default()
392            }];
393            env.create_physical_table_metadata(new_test_table_info(1024), region_routes)
394                .await;
395            let mut rx = env.frontend_receiver().await;
396
397            let err = state
398                .next(&mut ctx, &new_procedure_context())
399                .await
400                .unwrap_err();
401
402            assert_matches!(err, Error::TableRouteNotFound { .. });
403            assert_table_cache_invalidated(&mut rx, &[1024, 1025]);
404            let table_route = ctx.get_table_route_value(1024).await.unwrap();
405            let switched = table_route.region_routes().unwrap()[0]
406                .leader_peer
407                .as_ref()
408                .is_some_and(|leader| leader.id == 2);
409            if switched {
410                return;
411            }
412        }
413        panic!("In 64 attempts the failing table never came after the switched one");
414    }
415
416    #[tokio::test]
417    async fn test_retried_upgrade_broadcasts_invalidation() {
418        let mut state = Box::new(UpdateMetadata::Upgrade);
419        let mut env = TestingEnv::new();
420        // The table route cannot be read.
421        let kv_backend = MockKvBackendBuilder::default()
422            .range_fn(Arc::new(|_| {
423                common_meta::error::UnexpectedSnafu {
424                    err_msg: "mock range failed",
425                }
426                .fail()
427            }))
428            .build()
429            .unwrap();
430        let table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(kv_backend)));
431        let mut ctx = env
432            .context_factory_with_metadata(table_metadata_manager)
433            .new_context(new_persistent_context());
434        let mut rx = env.frontend_receiver().await;
435
436        // Every failed attempt broadcasts.
437        for _ in 0..2 {
438            let err = state
439                .next(&mut ctx, &new_procedure_context())
440                .await
441                .unwrap_err();
442
443            assert_matches!(err, Error::RetryLaterWithSource { .. });
444            assert!(err.is_retryable());
445            assert_table_cache_invalidated(&mut rx, &[1024]);
446        }
447    }
448
449    #[tokio::test]
450    async fn test_check_metadata() {
451        let state = UpdateMetadata::Upgrade;
452        let env = TestingEnv::new();
453        let persistent_context = new_persistent_context();
454        let leader_peer = persistent_context.from_peer.clone();
455
456        let mut ctx = env.context_factory().new_context(persistent_context);
457        let table_info = new_test_table_info(1024);
458        let region_routes = vec![RegionRoute {
459            region: Region::new_test(RegionId::new(1024, 1)),
460            leader_peer: Some(leader_peer),
461            follower_peers: vec![Peer::empty(2), Peer::empty(3)],
462            leader_state: None,
463            leader_down_since: None,
464            write_route_policy: None,
465        }];
466
467        env.create_physical_table_metadata(table_info, region_routes)
468            .await;
469        let table_routes = ctx.get_table_route_value(1024).await.unwrap();
470        let region_routes = table_routes.region_routes().unwrap();
471        let updated = state
472            .check_metadata_updated(&mut ctx, &[RegionId::new(1024, 1)], region_routes)
473            .unwrap();
474        assert!(!updated);
475    }
476
477    #[tokio::test]
478    async fn test_check_metadata_updated() {
479        let state = UpdateMetadata::Upgrade;
480        let env = TestingEnv::new();
481        let persistent_context = new_persistent_context();
482        let candidate_peer = persistent_context.to_peer.clone();
483
484        let mut ctx = env.context_factory().new_context(persistent_context);
485        let table_info = new_test_table_info(1024);
486        let region_routes = vec![RegionRoute {
487            region: Region::new_test(RegionId::new(1024, 1)),
488            leader_peer: Some(candidate_peer),
489            follower_peers: vec![Peer::empty(2), Peer::empty(3)],
490            leader_state: None,
491            leader_down_since: None,
492            write_route_policy: None,
493        }];
494
495        env.create_physical_table_metadata(table_info, region_routes)
496            .await;
497
498        let table_routes = ctx.get_table_route_value(1024).await.unwrap();
499        let region_routes = table_routes.region_routes().unwrap();
500        let updated = state
501            .check_metadata_updated(&mut ctx, &[RegionId::new(1024, 1)], region_routes)
502            .unwrap();
503        assert!(updated);
504    }
505
506    #[tokio::test]
507    async fn test_check_metadata_intermediate_state() {
508        let state = UpdateMetadata::Upgrade;
509        let env = TestingEnv::new();
510        let persistent_context = new_persistent_context();
511        let candidate_peer = persistent_context.to_peer.clone();
512
513        let mut ctx = env.context_factory().new_context(persistent_context);
514        let table_info = new_test_table_info(1024);
515        let region_routes = vec![RegionRoute {
516            region: Region::new_test(RegionId::new(1024, 1)),
517            leader_peer: Some(candidate_peer),
518            follower_peers: vec![Peer::empty(2), Peer::empty(3)],
519            leader_state: Some(LeaderState::Downgrading),
520            leader_down_since: None,
521            write_route_policy: None,
522        }];
523
524        env.create_physical_table_metadata(table_info, region_routes)
525            .await;
526
527        let table_routes = ctx.get_table_route_value(1024).await.unwrap();
528        let region_routes = table_routes.region_routes().unwrap();
529        let err = state
530            .check_metadata_updated(&mut ctx, &[RegionId::new(1024, 1)], region_routes)
531            .unwrap_err();
532        assert_matches!(err, Error::Unexpected { .. });
533        assert!(err.to_string().contains("intermediate state"));
534    }
535
536    #[tokio::test]
537    async fn test_next_close_downgraded_region_state() {
538        let mut state = Box::new(UpdateMetadata::Upgrade);
539        let env = TestingEnv::new();
540        let persistent_context = new_persistent_context();
541        let mut ctx = env.context_factory().new_context(persistent_context);
542        let opening_keeper = MemoryRegionKeeper::default();
543
544        let table_id = 1024;
545        let table_info = new_test_table_info(table_id);
546        let region_routes = vec![RegionRoute {
547            region: Region::new_test(RegionId::new(table_id, 1)),
548            leader_peer: Some(Peer::empty(1)),
549            leader_state: Some(LeaderState::Downgrading),
550            ..Default::default()
551        }];
552
553        let guard = opening_keeper
554            .register_with_role(2, RegionId::new(table_id, 1), RegionRole::Follower)
555            .unwrap();
556        ctx.volatile_ctx.opening_region_guards.push(guard);
557
558        env.create_physical_table_metadata(table_info, region_routes)
559            .await;
560
561        let table_metadata_manager = env.table_metadata_manager();
562
563        let procedure_ctx = new_procedure_context();
564        let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
565
566        let _ = next
567            .as_any()
568            .downcast_ref::<CloseDowngradedRegion>()
569            .unwrap();
570
571        let table_route = table_metadata_manager
572            .table_route_manager()
573            .table_route_storage()
574            .get(table_id)
575            .await
576            .unwrap()
577            .unwrap();
578        let region_routes = table_route.region_routes().unwrap();
579
580        assert!(ctx.volatile_ctx.opening_region_guards.is_empty());
581        assert_eq!(region_routes.len(), 1);
582        assert!(!region_routes[0].is_leader_downgrading());
583        assert!(region_routes[0].follower_peers.is_empty());
584        assert_eq!(region_routes[0].leader_peer.as_ref().unwrap().id, 2);
585    }
586}