1use common_error::ext::BoxedError;
16use common_meta::key::datanode_table::RegionInfo;
17use common_meta::lock_key::TableLock;
18use common_meta::rpc::router::{RegionRoute, region_distribution};
19use common_procedure::ContextProviderRef;
20use common_telemetry::{error, info, warn};
21use snafu::{OptionExt, ResultExt, ensure};
22use store_api::storage::RegionId;
23
24use crate::error::{self, Result};
25use crate::procedure::region_migration::Context;
26use crate::procedure::region_migration::update_metadata::UpdateMetadata;
27
28impl UpdateMetadata {
29 fn build_upgrade_candidate_region_metadata(
31 &self,
32 ctx: &mut Context,
33 region_ids: &[RegionId],
34 mut region_routes: Vec<RegionRoute>,
35 ) -> Result<Vec<RegionRoute>> {
36 let old_leader_peer = &ctx.persistent_ctx.from_peer;
37 let new_leader_peer = &ctx.persistent_ctx.to_peer;
38 for region_id in region_ids {
39 let region_route = region_routes
41 .iter_mut()
42 .find(|route| route.region.id == *region_id)
43 .context(error::RegionRouteNotFoundSnafu {
44 region_id: *region_id,
45 })?;
46
47 region_route.set_leader_state(None);
49
50 ensure!(
52 region_route
53 .leader_peer
54 .take_if(|old_leader| old_leader.id == old_leader_peer.id)
55 .is_some(),
56 error::UnexpectedSnafu {
57 violated: format!(
58 "Unexpected region leader: {:?} during the candidate-to-leader upgrade; expected: {:?}",
59 region_route.leader_peer, old_leader_peer
60 ),
61 }
62 );
63
64 region_route.leader_peer = Some(new_leader_peer.clone());
66
67 let removed = region_route
69 .follower_peers
70 .extract_if(.., |peer| peer.id == new_leader_peer.id)
71 .collect::<Vec<_>>();
72
73 if removed.len() > 1 {
75 warn!(
76 "Removed duplicate followers: {removed:?} during candidate-to-leader upgrade for region: {region_id}"
77 );
78 }
79 }
80
81 info!(
82 "Building metadata for upgrading candidate region to new leader: {:?} for regions: {:?}",
83 new_leader_peer, region_ids,
84 );
85
86 Ok(region_routes)
87 }
88
89 fn check_metadata_updated(
98 &self,
99 ctx: &mut Context,
100 region_ids: &[RegionId],
101 region_routes: &[RegionRoute],
102 ) -> Result<bool> {
103 for region_id in region_ids {
105 let region_route = region_routes
107 .iter()
108 .find(|route| route.region.id == *region_id)
109 .context(error::RegionRouteNotFoundSnafu {
110 region_id: *region_id,
111 })?;
112
113 let leader_peer = region_route.leader_peer.as_ref().with_context(||error::UnexpectedSnafu {
115 violated: format!(
116 "The leader peer of region {region_id} is not found during the metadata upgrade check"
117 ),
118 })?;
119
120 if leader_peer.id != ctx.persistent_ctx.to_peer.id {
122 return Ok(false);
123 } else {
124 ensure!(
126 !region_route.is_leader_downgrading(),
127 error::UnexpectedSnafu {
128 violated: format!(
129 "Unexpected intermediate state is found during the metadata upgrade check for region {region_id}"
130 ),
131 }
132 );
133 }
134 }
135
136 Ok(true)
138 }
139
140 pub async fn upgrade_candidate_region(
150 &self,
151 ctx: &mut Context,
152 ctx_provider: &ContextProviderRef,
153 ) -> Result<()> {
154 let table_metadata_manager = ctx.table_metadata_manager.clone();
155 let table_regions = ctx.persistent_ctx.table_regions();
156 let from_peer_id = ctx.persistent_ctx.from_peer.id;
157 let to_peer_id = ctx.persistent_ctx.to_peer.id;
158
159 for (table_id, region_ids) in table_regions {
160 let table_lock = TableLock::Write(table_id).into();
161 let _guard = ctx_provider.acquire_lock(&table_lock).await;
162
163 let table_route_value = ctx.get_table_route_value(table_id).await?;
164 let region_routes = table_route_value.region_routes().with_context(|_| {
165 error::UnexpectedLogicalRouteTableSnafu {
166 err_msg: format!("TableRoute({table_id:?}) is a non-physical TableRouteValue."),
167 }
168 })?;
169 if self.check_metadata_updated(ctx, ®ion_ids, region_routes)? {
170 continue;
171 }
172 let datanode_table_value = ctx.get_from_peer_datanode_table_value(table_id).await?;
173 let RegionInfo {
174 region_storage_path,
175 region_options,
176 region_wal_options,
177 engine,
178 } = datanode_table_value.region_info.clone();
179 let new_region_routes = self.build_upgrade_candidate_region_metadata(
180 ctx,
181 ®ion_ids,
182 region_routes.clone(),
183 )?;
184 let region_distribution = region_distribution(region_routes);
185 info!(
186 "Trying to update region routes to {:?} for table: {}",
187 region_distribution, table_id,
188 );
189
190 if let Err(err) = table_metadata_manager
191 .update_table_route(
192 table_id,
193 RegionInfo {
194 engine: engine.clone(),
195 region_storage_path: region_storage_path.clone(),
196 region_options: region_options.clone(),
197 region_wal_options: region_wal_options.clone(),
198 },
199 &table_route_value,
200 new_region_routes,
201 ®ion_options,
202 ®ion_wal_options,
203 )
204 .await
205 .context(error::TableMetadataManagerSnafu)
206 {
207 error!(err; "Failed to update the table route during the upgrading candidate region: {region_ids:?}, from_peer_id: {from_peer_id}");
208 return Err(BoxedError::new(err)).context(error::RetryLaterWithSourceSnafu {
209 reason: format!("Failed to update the table route during the upgrading candidate region: {table_id}"),
210 });
211 };
212 info!(
213 "Upgrading candidate region table route success, table_id: {table_id}, regions: {region_ids:?}, to_peer_id: {to_peer_id}"
214 );
215 }
216
217 ctx.deregister_failure_detectors().await;
218 ctx.reset_failure_detectors(&ctx.persistent_ctx.region_ids)
219 .await;
220 ctx.volatile_ctx.opening_region_guards.clear();
222
223 Ok(())
224 }
225}
226
227#[cfg(test)]
228mod tests {
229 use std::assert_matches;
230 use std::sync::Arc;
231
232 use common_meta::key::TableMetadataManager;
233 use common_meta::key::test_utils::new_test_table_info;
234 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
235 use common_meta::peer::Peer;
236 use common_meta::region_keeper::MemoryRegionKeeper;
237 use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
238 use common_time::util::current_time_millis;
239 use store_api::region_engine::RegionRole;
240 use store_api::storage::RegionId;
241
242 use crate::error::Error;
243 use crate::procedure::region_migration::close_downgraded_region::CloseDowngradedRegion;
244 use crate::procedure::region_migration::test_util::{
245 self, TestingEnv, assert_table_cache_invalidated, new_procedure_context,
246 };
247 use crate::procedure::region_migration::update_metadata::UpdateMetadata;
248 use crate::procedure::region_migration::{ContextFactory, PersistentContext, State};
249
250 fn new_persistent_context() -> PersistentContext {
251 test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
252 }
253
254 #[tokio::test]
255 async fn test_table_route_is_not_found_error() {
256 let env = TestingEnv::new();
257 let persistent_context = new_persistent_context();
258 let ctx = env.context_factory().new_context(persistent_context);
259
260 let err = ctx.get_table_route_value(1024).await.unwrap_err();
261
262 assert_matches!(err, Error::TableRouteNotFound { .. });
263 assert!(!err.is_retryable());
264 }
265
266 #[tokio::test]
267 async fn test_region_route_is_not_found() {
268 let state = UpdateMetadata::Upgrade;
269 let env = TestingEnv::new();
270 let persistent_context = new_persistent_context();
271 let mut ctx = env.context_factory().new_context(persistent_context);
272
273 let table_info = new_test_table_info(1024);
274 let region_routes = vec![RegionRoute {
275 region: Region::new_test(RegionId::new(1024, 2)),
276 leader_peer: Some(Peer::empty(4)),
277 ..Default::default()
278 }];
279 env.create_physical_table_metadata(table_info, region_routes)
280 .await;
281
282 let table_route_value = ctx.get_table_route_value(1024).await.unwrap();
283 let region_routes = table_route_value
284 .into_inner()
285 .into_physical_table_route()
286 .region_routes;
287 let err = state
288 .build_upgrade_candidate_region_metadata(
289 &mut ctx,
290 &[RegionId::new(1024, 1)],
291 region_routes,
292 )
293 .unwrap_err();
294
295 assert_matches!(err, Error::RegionRouteNotFound { .. });
296 assert!(!err.is_retryable());
297 }
298
299 #[tokio::test]
300 async fn test_region_route_expected_leader() {
301 let state = UpdateMetadata::Upgrade;
302 let env = TestingEnv::new();
303 let persistent_context = new_persistent_context();
304 let mut ctx = env.context_factory().new_context(persistent_context);
305
306 let table_info = new_test_table_info(1024);
307 let region_routes = vec![RegionRoute {
308 region: Region::new_test(RegionId::new(1024, 1)),
309 leader_peer: Some(Peer::empty(3)),
310 ..Default::default()
311 }];
312
313 env.create_physical_table_metadata(table_info, region_routes)
314 .await;
315
316 let table_route_value = ctx.get_table_route_value(1024).await.unwrap();
317 let region_routes = table_route_value
318 .into_inner()
319 .into_physical_table_route()
320 .region_routes;
321 let err = state
322 .build_upgrade_candidate_region_metadata(
323 &mut ctx,
324 &[RegionId::new(1024, 1)],
325 region_routes,
326 )
327 .unwrap_err();
328
329 assert_matches!(err, Error::Unexpected { .. });
330 assert!(!err.is_retryable());
331 assert!(err.to_string().contains("Unexpected region leader"));
332 }
333
334 #[tokio::test]
335 async fn test_build_upgrade_candidate_region_metadata() {
336 let state = UpdateMetadata::Upgrade;
337 let env = TestingEnv::new();
338 let persistent_context = new_persistent_context();
339 let mut ctx = env.context_factory().new_context(persistent_context);
340
341 let table_info = new_test_table_info(1024);
342 let region_routes = vec![RegionRoute {
343 region: Region::new_test(RegionId::new(1024, 1)),
344 leader_peer: Some(Peer::empty(1)),
345 follower_peers: vec![Peer::empty(2), Peer::empty(3)],
346 leader_state: Some(LeaderState::Downgrading),
347 leader_down_since: Some(current_time_millis()),
348 write_route_policy: None,
349 }];
350
351 env.create_physical_table_metadata(table_info, region_routes)
352 .await;
353
354 let table_route_value = ctx.get_table_route_value(1024).await.unwrap();
355 let region_routes = table_route_value
356 .into_inner()
357 .into_physical_table_route()
358 .region_routes;
359 let new_region_routes = state
360 .build_upgrade_candidate_region_metadata(
361 &mut ctx,
362 &[RegionId::new(1024, 1)],
363 region_routes,
364 )
365 .unwrap();
366
367 assert!(!new_region_routes[0].is_leader_downgrading());
368 assert!(new_region_routes[0].leader_down_since.is_none());
369 assert_eq!(new_region_routes[0].follower_peers, vec![Peer::empty(3)]);
370 assert_eq!(new_region_routes[0].leader_peer.as_ref().unwrap().id, 2);
371 }
372
373 #[tokio::test]
374 async fn test_failed_upgrade_broadcasts_invalidation() {
375 for _ in 0..64 {
381 let mut state = Box::new(UpdateMetadata::Upgrade);
382 let mut env = TestingEnv::new();
383 let mut persistent_context = new_persistent_context();
385 persistent_context.region_ids = vec![RegionId::new(1024, 1), RegionId::new(1025, 1)];
386 let mut ctx = env.context_factory().new_context(persistent_context);
387 let region_routes = vec![RegionRoute {
388 region: Region::new_test(RegionId::new(1024, 1)),
389 leader_peer: Some(Peer::empty(1)),
390 leader_state: Some(LeaderState::Downgrading),
391 ..Default::default()
392 }];
393 env.create_physical_table_metadata(new_test_table_info(1024), region_routes)
394 .await;
395 let mut rx = env.frontend_receiver().await;
396
397 let err = state
398 .next(&mut ctx, &new_procedure_context())
399 .await
400 .unwrap_err();
401
402 assert_matches!(err, Error::TableRouteNotFound { .. });
403 assert_table_cache_invalidated(&mut rx, &[1024, 1025]);
404 let table_route = ctx.get_table_route_value(1024).await.unwrap();
405 let switched = table_route.region_routes().unwrap()[0]
406 .leader_peer
407 .as_ref()
408 .is_some_and(|leader| leader.id == 2);
409 if switched {
410 return;
411 }
412 }
413 panic!("In 64 attempts the failing table never came after the switched one");
414 }
415
416 #[tokio::test]
417 async fn test_retried_upgrade_broadcasts_invalidation() {
418 let mut state = Box::new(UpdateMetadata::Upgrade);
419 let mut env = TestingEnv::new();
420 let kv_backend = MockKvBackendBuilder::default()
422 .range_fn(Arc::new(|_| {
423 common_meta::error::UnexpectedSnafu {
424 err_msg: "mock range failed",
425 }
426 .fail()
427 }))
428 .build()
429 .unwrap();
430 let table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(kv_backend)));
431 let mut ctx = env
432 .context_factory_with_metadata(table_metadata_manager)
433 .new_context(new_persistent_context());
434 let mut rx = env.frontend_receiver().await;
435
436 for _ in 0..2 {
438 let err = state
439 .next(&mut ctx, &new_procedure_context())
440 .await
441 .unwrap_err();
442
443 assert_matches!(err, Error::RetryLaterWithSource { .. });
444 assert!(err.is_retryable());
445 assert_table_cache_invalidated(&mut rx, &[1024]);
446 }
447 }
448
449 #[tokio::test]
450 async fn test_check_metadata() {
451 let state = UpdateMetadata::Upgrade;
452 let env = TestingEnv::new();
453 let persistent_context = new_persistent_context();
454 let leader_peer = persistent_context.from_peer.clone();
455
456 let mut ctx = env.context_factory().new_context(persistent_context);
457 let table_info = new_test_table_info(1024);
458 let region_routes = vec![RegionRoute {
459 region: Region::new_test(RegionId::new(1024, 1)),
460 leader_peer: Some(leader_peer),
461 follower_peers: vec![Peer::empty(2), Peer::empty(3)],
462 leader_state: None,
463 leader_down_since: None,
464 write_route_policy: None,
465 }];
466
467 env.create_physical_table_metadata(table_info, region_routes)
468 .await;
469 let table_routes = ctx.get_table_route_value(1024).await.unwrap();
470 let region_routes = table_routes.region_routes().unwrap();
471 let updated = state
472 .check_metadata_updated(&mut ctx, &[RegionId::new(1024, 1)], region_routes)
473 .unwrap();
474 assert!(!updated);
475 }
476
477 #[tokio::test]
478 async fn test_check_metadata_updated() {
479 let state = UpdateMetadata::Upgrade;
480 let env = TestingEnv::new();
481 let persistent_context = new_persistent_context();
482 let candidate_peer = persistent_context.to_peer.clone();
483
484 let mut ctx = env.context_factory().new_context(persistent_context);
485 let table_info = new_test_table_info(1024);
486 let region_routes = vec![RegionRoute {
487 region: Region::new_test(RegionId::new(1024, 1)),
488 leader_peer: Some(candidate_peer),
489 follower_peers: vec![Peer::empty(2), Peer::empty(3)],
490 leader_state: None,
491 leader_down_since: None,
492 write_route_policy: None,
493 }];
494
495 env.create_physical_table_metadata(table_info, region_routes)
496 .await;
497
498 let table_routes = ctx.get_table_route_value(1024).await.unwrap();
499 let region_routes = table_routes.region_routes().unwrap();
500 let updated = state
501 .check_metadata_updated(&mut ctx, &[RegionId::new(1024, 1)], region_routes)
502 .unwrap();
503 assert!(updated);
504 }
505
506 #[tokio::test]
507 async fn test_check_metadata_intermediate_state() {
508 let state = UpdateMetadata::Upgrade;
509 let env = TestingEnv::new();
510 let persistent_context = new_persistent_context();
511 let candidate_peer = persistent_context.to_peer.clone();
512
513 let mut ctx = env.context_factory().new_context(persistent_context);
514 let table_info = new_test_table_info(1024);
515 let region_routes = vec![RegionRoute {
516 region: Region::new_test(RegionId::new(1024, 1)),
517 leader_peer: Some(candidate_peer),
518 follower_peers: vec![Peer::empty(2), Peer::empty(3)],
519 leader_state: Some(LeaderState::Downgrading),
520 leader_down_since: None,
521 write_route_policy: None,
522 }];
523
524 env.create_physical_table_metadata(table_info, region_routes)
525 .await;
526
527 let table_routes = ctx.get_table_route_value(1024).await.unwrap();
528 let region_routes = table_routes.region_routes().unwrap();
529 let err = state
530 .check_metadata_updated(&mut ctx, &[RegionId::new(1024, 1)], region_routes)
531 .unwrap_err();
532 assert_matches!(err, Error::Unexpected { .. });
533 assert!(err.to_string().contains("intermediate state"));
534 }
535
536 #[tokio::test]
537 async fn test_next_close_downgraded_region_state() {
538 let mut state = Box::new(UpdateMetadata::Upgrade);
539 let env = TestingEnv::new();
540 let persistent_context = new_persistent_context();
541 let mut ctx = env.context_factory().new_context(persistent_context);
542 let opening_keeper = MemoryRegionKeeper::default();
543
544 let table_id = 1024;
545 let table_info = new_test_table_info(table_id);
546 let region_routes = vec![RegionRoute {
547 region: Region::new_test(RegionId::new(table_id, 1)),
548 leader_peer: Some(Peer::empty(1)),
549 leader_state: Some(LeaderState::Downgrading),
550 ..Default::default()
551 }];
552
553 let guard = opening_keeper
554 .register_with_role(2, RegionId::new(table_id, 1), RegionRole::Follower)
555 .unwrap();
556 ctx.volatile_ctx.opening_region_guards.push(guard);
557
558 env.create_physical_table_metadata(table_info, region_routes)
559 .await;
560
561 let table_metadata_manager = env.table_metadata_manager();
562
563 let procedure_ctx = new_procedure_context();
564 let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
565
566 let _ = next
567 .as_any()
568 .downcast_ref::<CloseDowngradedRegion>()
569 .unwrap();
570
571 let table_route = table_metadata_manager
572 .table_route_manager()
573 .table_route_storage()
574 .get(table_id)
575 .await
576 .unwrap()
577 .unwrap();
578 let region_routes = table_route.region_routes().unwrap();
579
580 assert!(ctx.volatile_ctx.opening_region_guards.is_empty());
581 assert_eq!(region_routes.len(), 1);
582 assert!(!region_routes[0].is_leader_downgrading());
583 assert!(region_routes[0].follower_peers.is_empty());
584 assert_eq!(region_routes[0].leader_peer.as_ref().unwrap().id, 2);
585 }
586}