1use std::collections::hash_map::Entry;
16use std::collections::{HashMap, HashSet};
17use std::fmt::Display;
18use std::sync::{Arc, RwLock};
19use std::time::Duration;
20
21use common_event_recorder::PersistentEventContext;
22use common_meta::key::table_info::TableInfoValue;
23use common_meta::key::table_route::TableRouteValue;
24use common_meta::peer::Peer;
25use common_meta::rpc::router::RegionRoute;
26use common_procedure::{
27 ProcedureContext, ProcedureId, ProcedureManagerRef, ProcedureWithId, watcher,
28};
29use common_telemetry::{error, info, warn};
30use serde::{Deserialize, Serialize};
31use snafu::{OptionExt, ResultExt, ensure};
32use store_api::storage::RegionId;
33use table::table_name::TableName;
34
35use crate::error::{self, Result};
36use crate::metrics::{METRIC_META_REGION_MIGRATION_DATANODES, METRIC_META_REGION_MIGRATION_FAIL};
37use crate::procedure::region_migration::utils::{
38 RegionMigrationAnalysis, RegionMigrationTaskBatch, analyze_region_migration_task,
39};
40use crate::procedure::region_migration::{
41 DefaultContextFactory, PersistentContext, RegionMigrationProcedure,
42};
43
44pub type RegionMigrationManagerRef = Arc<RegionMigrationManager>;
45
46pub struct RegionMigrationManager {
48 procedure_manager: ProcedureManagerRef,
49 context_factory: DefaultContextFactory,
50 tracker: RegionMigrationProcedureTracker,
51}
52
53#[derive(Default, Clone)]
54pub struct RegionMigrationProcedureTracker {
55 running_procedures: Arc<RwLock<HashMap<RegionId, RegionMigrationProcedureTask>>>,
56}
57
58impl RegionMigrationProcedureTracker {
59 pub(crate) fn insert_running_procedure(
61 &self,
62 task: &RegionMigrationProcedureTask,
63 ) -> Option<RegionMigrationProcedureGuard> {
64 let mut procedures = self.running_procedures.write().unwrap();
65 match procedures.entry(task.region_id) {
66 Entry::Occupied(_) => None,
67 Entry::Vacant(v) => {
68 v.insert(task.clone());
69 Some(RegionMigrationProcedureGuard {
70 region_id: task.region_id,
71 running_procedures: self.running_procedures.clone(),
72 })
73 }
74 }
75 }
76
77 pub(crate) fn contains(&self, region_id: RegionId) -> bool {
79 self.running_procedures
80 .read()
81 .unwrap()
82 .contains_key(®ion_id)
83 }
84}
85
86pub(crate) struct RegionMigrationProcedureGuard {
88 region_id: RegionId,
89 running_procedures: Arc<RwLock<HashMap<RegionId, RegionMigrationProcedureTask>>>,
90}
91
92impl Drop for RegionMigrationProcedureGuard {
93 fn drop(&mut self) {
94 let exists = self
95 .running_procedures
96 .read()
97 .unwrap()
98 .contains_key(&self.region_id);
99 if exists {
100 self.running_procedures
101 .write()
102 .unwrap()
103 .remove(&self.region_id);
104 }
105 }
106}
107
108#[derive(Debug, Clone)]
110pub struct RegionMigrationProcedureTask {
111 pub(crate) region_id: RegionId,
112 pub(crate) from_peer: Peer,
113 pub(crate) to_peer: Peer,
114 pub(crate) timeout: Duration,
115 pub(crate) trigger_reason: RegionMigrationTriggerReason,
116}
117
118#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, strum::Display)]
120#[strum(serialize_all = "PascalCase")]
121pub enum RegionMigrationTriggerReason {
122 #[default]
123 Unknown,
125 Manual,
127 AutoRebalance,
129 Failover,
131}
132
133impl RegionMigrationProcedureTask {
134 pub fn new(
135 region_id: RegionId,
136 from_peer: Peer,
137 to_peer: Peer,
138 timeout: Duration,
139 trigger_reason: RegionMigrationTriggerReason,
140 ) -> Self {
141 Self {
142 region_id,
143 from_peer,
144 to_peer,
145 timeout,
146 trigger_reason,
147 }
148 }
149}
150
151impl Display for RegionMigrationProcedureTask {
152 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
153 write!(
154 f,
155 "region: {}, from_peer: {}, to_peer: {}, trigger_reason: {}",
156 self.region_id, self.from_peer, self.to_peer, self.trigger_reason
157 )
158 }
159}
160
161#[derive(Debug, Default, PartialEq, Eq)]
163pub struct SubmitRegionMigrationTaskResult {
164 pub migrated: Vec<RegionId>,
166 pub leader_changed: Vec<RegionId>,
168 pub peer_conflict: Vec<RegionId>,
170 pub table_not_found: Vec<RegionId>,
172 pub region_not_found: Vec<RegionId>,
174 pub migrating: Vec<RegionId>,
176 pub submitted: Vec<RegionId>,
178 pub procedure_id: Option<ProcedureId>,
180}
181
182pub(crate) struct BuiltMigrationBatch {
184 pub(crate) analysis: RegionMigrationAnalysis,
186 pub(crate) procedure: Option<ProcedureWithId>,
188}
189
190impl RegionMigrationManager {
191 pub(crate) fn new(
193 procedure_manager: ProcedureManagerRef,
194 context_factory: DefaultContextFactory,
195 ) -> Self {
196 Self {
197 procedure_manager,
198 context_factory,
199 tracker: RegionMigrationProcedureTracker::default(),
200 }
201 }
202
203 pub fn tracker(&self) -> &RegionMigrationProcedureTracker {
205 &self.tracker
206 }
207
208 pub(crate) fn try_start(&self) -> Result<()> {
210 let context_factory = self.context_factory.clone();
211 let tracker = self.tracker.clone();
212 self.procedure_manager
213 .register_loader(
214 RegionMigrationProcedure::TYPE_NAME,
215 Box::new(move |json| {
216 let context_factory = context_factory.clone();
217 let tracker = tracker.clone();
218 RegionMigrationProcedure::from_json(json, context_factory, tracker)
219 .map(|p| Box::new(p) as _)
220 }),
221 )
222 .context(error::RegisterProcedureLoaderSnafu {
223 type_name: RegionMigrationProcedure::TYPE_NAME,
224 })
225 }
226
227 fn insert_running_procedure(
228 &self,
229 task: &RegionMigrationProcedureTask,
230 ) -> Option<RegionMigrationProcedureGuard> {
231 self.tracker.insert_running_procedure(task)
232 }
233
234 fn verify_task(&self, task: &RegionMigrationProcedureTask) -> Result<()> {
235 if task.to_peer.id == task.from_peer.id {
236 return error::InvalidArgumentsSnafu {
237 err_msg: "The `from_peer_id` can't equal `to_peer_id`",
238 }
239 .fail();
240 }
241
242 Ok(())
243 }
244
245 async fn retrieve_table_route(&self, region_id: RegionId) -> Result<TableRouteValue> {
246 let table_route = self
247 .context_factory
248 .table_metadata_manager
249 .table_route_manager()
250 .table_route_storage()
251 .get(region_id.table_id())
252 .await
253 .context(error::TableMetadataManagerSnafu)?
254 .context(error::TableRouteNotFoundSnafu {
255 table_id: region_id.table_id(),
256 })?;
257
258 Ok(table_route)
259 }
260
261 async fn retrieve_table_info(&self, region_id: RegionId) -> Result<TableInfoValue> {
262 let table_route = self
263 .context_factory
264 .table_metadata_manager
265 .table_info_manager()
266 .get(region_id.table_id())
267 .await
268 .context(error::TableMetadataManagerSnafu)?
269 .context(error::TableInfoNotFoundSnafu {
270 table_id: region_id.table_id(),
271 })?
272 .into_inner();
273
274 Ok(table_route)
275 }
276
277 fn verify_table_route(
279 &self,
280 table_route: &TableRouteValue,
281 task: &RegionMigrationProcedureTask,
282 ) -> Result<()> {
283 if !table_route.is_physical() {
284 return error::UnexpectedSnafu {
285 violated: format!(
286 "Trying to execute region migration on the logical table, task {task}"
287 ),
288 }
289 .fail();
290 }
291
292 Ok(())
293 }
294
295 fn has_migrated(
297 &self,
298 region_route: &RegionRoute,
299 task: &RegionMigrationProcedureTask,
300 ) -> Result<bool> {
301 if region_route.is_leader_downgrading() {
302 return Ok(false);
303 }
304
305 let leader_peer = region_route
306 .leader_peer
307 .as_ref()
308 .context(error::UnexpectedSnafu {
309 violated: "Region route leader peer is not found",
310 })?;
311
312 Ok(leader_peer.id == task.to_peer.id)
313 }
314
315 fn verify_region_leader_peer(
319 &self,
320 region_route: &RegionRoute,
321 task: &mut RegionMigrationProcedureTask,
322 ) -> Result<()> {
323 let leader_peer = region_route
324 .leader_peer
325 .as_ref()
326 .context(error::UnexpectedSnafu {
327 violated: "Region route leader peer is not found",
328 })?;
329
330 ensure!(
331 leader_peer.id == task.from_peer.id,
332 error::LeaderPeerChangedSnafu {
333 msg: format!(
334 "Region's leader peer({}) is not the `from_peer`({}), region: {}",
335 leader_peer.id, task.from_peer.id, task.region_id
336 ),
337 }
338 );
339
340 if task.from_peer.addr.is_empty() {
341 warn!(
342 "The `from_peer` is unknown, use the leader peer({}) as the `from_peer`, region: {}",
343 leader_peer, task.region_id
344 );
345 task.from_peer = leader_peer.clone();
347 }
348
349 Ok(())
350 }
351
352 fn verify_region_follower_peers(
354 &self,
355 region_route: &RegionRoute,
356 task: &RegionMigrationProcedureTask,
357 ) -> Result<()> {
358 ensure!(
359 !region_route.follower_peers.contains(&task.to_peer),
360 error::InvalidArgumentsSnafu {
361 err_msg: format!(
362 "The `to_peer`({}) is already has a region follower, region: {}",
363 task.to_peer.id, task.region_id
364 ),
365 },
366 );
367
368 Ok(())
369 }
370
371 fn extract_running_regions(
376 &self,
377 task: &mut RegionMigrationTaskBatch,
378 ) -> (Vec<RegionId>, Vec<RegionMigrationProcedureGuard>) {
379 let mut migrating_region_ids = Vec::new();
380 let mut procedure_guards = Vec::with_capacity(task.region_ids.len());
381
382 for region_id in &task.region_ids {
383 let Some(guard) = self.insert_running_procedure(&RegionMigrationProcedureTask::new(
384 *region_id,
385 task.from_peer.clone(),
386 task.to_peer.clone(),
387 task.timeout,
388 task.trigger_reason,
389 )) else {
390 migrating_region_ids.push(*region_id);
391 continue;
392 };
393 procedure_guards.push(guard);
394 }
395
396 let migrating_set = migrating_region_ids.iter().cloned().collect::<HashSet<_>>();
397 task.region_ids.retain(|id| !migrating_set.contains(id));
398
399 (migrating_region_ids, procedure_guards)
400 }
401
402 pub async fn submit_region_migration_task(
403 &self,
404 task: RegionMigrationTaskBatch,
405 ) -> Result<SubmitRegionMigrationTaskResult> {
406 let procedure_context = ProcedureContext {
407 actor: None,
408 event_context: Some(PersistentEventContext::new(
409 task.trigger_reason.to_trigger_reason(),
410 )),
411 };
412 let from_peer_id = task.from_peer.id;
413 let to_peer_id = task.to_peer.id;
414 let BuiltMigrationBatch {
415 analysis,
416 procedure,
417 } = self.build_batch_procedure(procedure_context, task).await?;
418 let procedure_id = procedure.as_ref().map(|procedure| procedure.id);
419 if let Some(procedure) = procedure {
420 let procedure_id = procedure.id;
421 info!(
422 "Starting region migration procedure {procedure_id}, regions: {:?}, from_peer: {}, to_peer: {}",
423 analysis.pending, from_peer_id, to_peer_id
424 );
425 let mut watcher = self
426 .procedure_manager
427 .submit(procedure)
428 .await
429 .context(error::SubmitProcedureSnafu)?;
430 let num_regions = analysis.pending.len();
431 common_runtime::spawn_global(async move {
432 METRIC_META_REGION_MIGRATION_DATANODES
433 .with_label_values(&["src", &from_peer_id.to_string()])
434 .inc_by(num_regions as u64);
435 METRIC_META_REGION_MIGRATION_DATANODES
436 .with_label_values(&["desc", &to_peer_id.to_string()])
437 .inc_by(num_regions as u64);
438 if let Err(e) = watcher::wait(&mut watcher).await {
439 error!(e; "Failed to wait region migration procedure {procedure_id}");
440 METRIC_META_REGION_MIGRATION_FAIL.inc();
441 return;
442 }
443 info!("Region migration procedure {procedure_id} finished successfully");
444 });
445 }
446 let RegionMigrationAnalysis {
447 migrating,
448 migrated,
449 leader_changed,
450 peer_conflict,
451 table_not_found,
452 region_not_found,
453 pending,
454 } = analysis;
455 Ok(SubmitRegionMigrationTaskResult {
456 migrated,
457 leader_changed,
458 peer_conflict,
459 table_not_found,
460 region_not_found,
461 migrating,
462 submitted: pending,
463 procedure_id,
464 })
465 }
466
467 pub(crate) async fn build_batch_procedure(
470 &self,
471 procedure_context: ProcedureContext,
472 mut task: RegionMigrationTaskBatch,
473 ) -> Result<BuiltMigrationBatch> {
474 let (migrating, procedure_guards) = self.extract_running_regions(&mut task);
475 let mut analysis =
476 analyze_region_migration_task(&task, &self.context_factory.table_metadata_manager)
477 .await?;
478 analysis.migrating = migrating;
479 if analysis.pending.is_empty() {
480 return Ok(BuiltMigrationBatch {
481 analysis,
482 procedure: None,
483 });
484 }
485
486 task.region_ids = std::mem::take(&mut analysis.pending);
488 let table_regions = task.table_regions();
489 let table_ids = table_regions.keys().cloned().collect::<Vec<_>>();
490 let table_info_values = self
491 .context_factory
492 .table_metadata_manager
493 .table_info_manager()
494 .batch_get(&table_ids)
495 .await
496 .context(error::TableMetadataManagerSnafu)?;
497 let mut catalog_and_schema = Vec::with_capacity(table_info_values.len());
498 for (table_id, regions) in table_regions {
499 match table_info_values.get(&table_id) {
500 Some(table_info) => {
501 let TableName {
502 catalog_name,
503 schema_name,
504 ..
505 } = table_info.table_name();
506 catalog_and_schema.push((catalog_name, schema_name));
507 }
508 None => {
509 task.region_ids.retain(|id| id.table_id() != table_id);
510 analysis.table_not_found.extend(regions);
511 }
512 }
513 }
514 if task.region_ids.is_empty() {
515 return Ok(BuiltMigrationBatch {
516 analysis,
517 procedure: None,
518 });
519 }
520
521 analysis.pending = task.region_ids.clone();
522 let pending = task.region_ids.iter().copied().collect::<HashSet<_>>();
523 let procedure_guards = procedure_guards
524 .into_iter()
525 .filter(|guard| pending.contains(&guard.region_id))
526 .collect();
527 let procedure = RegionMigrationProcedure::new(
528 PersistentContext::new(
529 catalog_and_schema,
530 task.from_peer,
531 task.to_peer,
532 task.region_ids,
533 task.timeout,
534 task.trigger_reason,
535 ),
536 self.context_factory.clone(),
537 procedure_guards,
538 );
539 let procedure =
540 ProcedureWithId::with_random_id(Box::new(procedure)).with_context(procedure_context);
541 Ok(BuiltMigrationBatch {
542 analysis,
543 procedure: Some(procedure),
544 })
545 }
546
547 pub async fn build_subprocedure(
552 &self,
553 procedure_context: ProcedureContext,
554 mut task: RegionMigrationProcedureTask,
555 ) -> Result<ProcedureWithId> {
556 let Some(guard) = self.insert_running_procedure(&task) else {
557 return error::MigrationRunningSnafu {
558 region_id: task.region_id,
559 }
560 .fail();
561 };
562
563 self.verify_task(&task)?;
564
565 let region_id = task.region_id;
566
567 let table_route = self.retrieve_table_route(region_id).await?;
568 self.verify_table_route(&table_route, &task)?;
569
570 let region_route = table_route
572 .region_route(region_id)
573 .context(error::UnexpectedLogicalRouteTableSnafu {
574 err_msg: format!("{table_route:?} is a non-physical TableRouteValue."),
575 })?
576 .context(error::RegionRouteNotFoundSnafu { region_id })?;
577
578 if self.has_migrated(®ion_route, &task)? {
579 info!("Skipping region migration task: {task}");
580 return error::RegionMigratedSnafu {
581 region_id,
582 target_peer_id: task.to_peer.id,
583 }
584 .fail();
585 }
586
587 self.verify_region_leader_peer(®ion_route, &mut task)?;
588 self.verify_region_follower_peers(®ion_route, &task)?;
589 let table_info = self.retrieve_table_info(region_id).await?;
590 let TableName {
591 catalog_name,
592 schema_name,
593 ..
594 } = table_info.table_name();
595 let RegionMigrationProcedureTask {
596 region_id,
597 from_peer,
598 to_peer,
599 timeout,
600 trigger_reason,
601 } = task.clone();
602 let procedure = RegionMigrationProcedure::new(
603 PersistentContext::new(
604 vec![(catalog_name, schema_name)],
605 from_peer,
606 to_peer,
607 vec![region_id],
608 timeout,
609 trigger_reason,
610 ),
611 self.context_factory.clone(),
612 vec![guard],
613 );
614 let procedure_with_id =
615 ProcedureWithId::with_random_id(Box::new(procedure)).with_context(procedure_context);
616 Ok(procedure_with_id)
617 }
618
619 pub async fn submit_procedure(
621 &self,
622 procedure_context: ProcedureContext,
623 mut task: RegionMigrationProcedureTask,
624 ) -> Result<Option<ProcedureId>> {
625 if let Some(event_context) = procedure_context.event_context.as_ref() {
626 task.trigger_reason =
627 RegionMigrationTriggerReason::from_trigger_reason(event_context.reason);
628 }
629
630 let procedure_with_id = self
631 .build_subprocedure(procedure_context, task.clone())
632 .await?;
633 let procedure_id = procedure_with_id.id;
634 info!("Starting region migration procedure {procedure_id} for {task}");
635 let mut watcher = self
636 .procedure_manager
637 .submit(procedure_with_id)
638 .await
639 .context(error::SubmitProcedureSnafu)?;
640 common_runtime::spawn_global(async move {
641 METRIC_META_REGION_MIGRATION_DATANODES
642 .with_label_values(&["src", &task.from_peer.id.to_string()])
643 .inc();
644 METRIC_META_REGION_MIGRATION_DATANODES
645 .with_label_values(&["desc", &task.to_peer.id.to_string()])
646 .inc();
647
648 if let Err(e) = watcher::wait(&mut watcher).await {
649 error!(e; "Failed to wait region migration procedure {procedure_id} for {task}");
650 METRIC_META_REGION_MIGRATION_FAIL.inc();
651 return;
652 }
653
654 info!("Region migration procedure {procedure_id} for {task} is finished successfully!");
655 });
656
657 Ok(Some(procedure_id))
658 }
659}
660
661#[cfg(test)]
662mod test {
663 use std::assert_matches;
664
665 use common_meta::key::table_route::LogicalTableRouteValue;
666 use common_meta::key::test_utils::new_test_table_info;
667 use common_meta::rpc::router::Region;
668 use common_meta::state_store::KvStateStore;
669 use common_procedure::ProcedureManager;
670 use common_procedure::local::{LocalManager, ManagerConfig, PauseAware};
671 use common_procedure::test_util::InMemoryPoisonStore;
672 use tokio::sync::Notify;
673
674 use super::*;
675 use crate::procedure::region_migration::test_util::{TestingEnv, new_persistent_context};
676
677 #[derive(Default)]
678 struct SubmissionGate {
679 entered: Notify,
680 release: Notify,
681 }
682
683 #[async_trait::async_trait]
684 impl PauseAware for SubmissionGate {
685 async fn is_paused(&self) -> std::result::Result<bool, common_error::ext::BoxedError> {
686 self.entered.notify_one();
687 self.release.notified().await;
688 Ok(false)
689 }
690 }
691
692 async fn check_submission_registration(batch: bool) {
693 let env = TestingEnv::new();
694 let gate = Arc::new(SubmissionGate::default());
695 let procedure_manager = Arc::new(LocalManager::new(
696 ManagerConfig::default(),
697 Arc::new(KvStateStore::new(env.kv_backend())),
698 Arc::new(InMemoryPoisonStore::default()),
699 Some(gate.clone()),
700 None,
701 ));
702 let manager = RegionMigrationManager::new(procedure_manager.clone(), env.context_factory());
703 let region_ids = if batch {
704 vec![RegionId::new(1024, 1), RegionId::new(1024, 2)]
705 } else {
706 vec![RegionId::new(1024, 1)]
707 };
708 let region_routes = region_ids
709 .iter()
710 .map(|region_id| RegionRoute {
711 region: Region::new_test(*region_id),
712 leader_peer: Some(Peer::empty(1)),
713 ..Default::default()
714 })
715 .collect();
716 env.create_physical_table_metadata(new_test_table_info(1024), region_routes)
717 .await;
718
719 for started in [false, true] {
722 if started {
723 procedure_manager.start().await.unwrap();
724 }
725 let submission = async {
726 if batch {
727 let result = manager
728 .submit_region_migration_task(RegionMigrationTaskBatch {
729 region_ids: region_ids.clone(),
730 from_peer: Peer::empty(1),
731 to_peer: Peer::empty(2),
732 timeout: Duration::from_secs(10),
733 trigger_reason: RegionMigrationTriggerReason::Manual,
734 })
735 .await?;
736 assert_eq!(result.submitted, region_ids);
737 Ok(result.procedure_id)
738 } else {
739 manager
740 .submit_procedure(
741 ProcedureContext::default(),
742 RegionMigrationProcedureTask::new(
743 region_ids[0],
744 Peer::empty(1),
745 Peer::empty(2),
746 Duration::from_secs(10),
747 RegionMigrationTriggerReason::Manual,
748 ),
749 )
750 .await
751 }
752 };
753 tokio::pin!(submission);
754 tokio::select! {
755 biased;
756 result = &mut submission => panic!("Submission returned before registration: {result:?}"),
757 _ = gate.entered.notified() => {}
758 }
759 assert!(futures::poll!(&mut submission).is_pending());
760 assert!(
761 procedure_manager
762 .list_procedures()
763 .await
764 .unwrap()
765 .is_empty()
766 );
767 for region_id in ®ion_ids {
768 assert!(manager.tracker.contains(*region_id));
769 }
770
771 gate.release.notify_one();
772 let result = tokio::time::timeout(Duration::from_secs(5), &mut submission)
773 .await
774 .expect("Submission should return after registration");
775 if started {
776 let procedure_id = result.unwrap().unwrap();
777 assert!(
778 procedure_manager
779 .procedure_state(procedure_id)
780 .await
781 .unwrap()
782 .is_some()
783 );
784 } else {
785 assert_matches!(
786 result.unwrap_err(),
787 error::Error::SubmitProcedure {
788 source: common_procedure::Error::ManagerNotStart { .. },
789 ..
790 }
791 );
792 for region_id in ®ion_ids {
793 assert!(!manager.tracker.contains(*region_id));
794 }
795 }
796 }
797 procedure_manager.stop().await.unwrap();
798 }
799
800 #[tokio::test]
801 async fn test_build_batch_procedure_retains_only_pending_guards() {
802 let env = TestingEnv::new();
803 let manager =
804 RegionMigrationManager::new(env.procedure_manager().clone(), env.context_factory());
805 let pending = RegionId::new(1024, 1);
806 let migrated = RegionId::new(1024, 2);
807 let migrating = RegionId::new(2048, 1);
808 let running_guard = manager
809 .insert_running_procedure(&RegionMigrationProcedureTask::new(
810 migrating,
811 Peer::empty(1),
812 Peer::empty(2),
813 Duration::from_secs(10),
814 RegionMigrationTriggerReason::Manual,
815 ))
816 .unwrap();
817 env.create_physical_table_metadata(
818 new_test_table_info(1024),
819 vec![
820 RegionRoute {
821 region: Region::new_test(pending),
822 leader_peer: Some(Peer::empty(1)),
823 ..Default::default()
824 },
825 RegionRoute {
826 region: Region::new_test(migrated),
827 leader_peer: Some(Peer::empty(2)),
828 ..Default::default()
829 },
830 ],
831 )
832 .await;
833 let batch = manager
834 .build_batch_procedure(
835 ProcedureContext::default(),
836 RegionMigrationTaskBatch {
837 region_ids: vec![pending, migrated, migrating],
838 from_peer: Peer::empty(1),
839 to_peer: Peer::empty(2),
840 timeout: Duration::from_secs(10),
841 trigger_reason: RegionMigrationTriggerReason::Manual,
842 },
843 )
844 .await
845 .unwrap();
846 assert_eq!(
847 RegionMigrationAnalysis {
848 pending: vec![pending],
849 migrated: vec![migrated],
850 migrating: vec![migrating],
851 ..Default::default()
852 },
853 batch.analysis
854 );
855 let child = batch.procedure.unwrap();
856 assert!(manager.tracker.contains(pending));
857 assert!(!manager.tracker.contains(migrated));
858 assert!(
859 env.procedure_manager()
860 .list_procedures()
861 .await
862 .unwrap()
863 .is_empty()
864 );
865 drop(child);
866 assert!(!manager.tracker.contains(pending));
867 assert!(manager.tracker.contains(migrating));
868
869 let batch = manager
870 .build_batch_procedure(
871 ProcedureContext::default(),
872 RegionMigrationTaskBatch {
873 region_ids: vec![migrating],
874 from_peer: Peer::empty(1),
875 to_peer: Peer::empty(2),
876 timeout: Duration::from_secs(10),
877 trigger_reason: RegionMigrationTriggerReason::Manual,
878 },
879 )
880 .await
881 .unwrap();
882 assert_eq!(
883 RegionMigrationAnalysis {
884 migrating: vec![migrating],
885 ..Default::default()
886 },
887 batch.analysis
888 );
889 assert!(batch.procedure.is_none());
890 assert!(manager.tracker.contains(migrating));
891 drop(running_guard);
892 assert!(!manager.tracker.contains(migrating));
893 }
894
895 #[tokio::test]
896 async fn test_submit_procedure_registration() {
897 check_submission_registration(false).await;
898 }
899
900 #[tokio::test]
901 async fn test_submit_region_migration_task_registration() {
902 check_submission_registration(true).await;
903 }
904
905 #[tokio::test]
906 async fn test_insert_running_procedure() {
907 let env = TestingEnv::new();
908 let context_factory = env.context_factory();
909 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
910 let region_id = RegionId::new(1024, 1);
911 let task = RegionMigrationProcedureTask {
912 region_id,
913 from_peer: Peer::empty(2),
914 to_peer: Peer::empty(1),
915 timeout: Duration::from_millis(1000),
916 trigger_reason: RegionMigrationTriggerReason::Manual,
917 };
918 manager
920 .tracker
921 .running_procedures
922 .write()
923 .unwrap()
924 .insert(region_id, task.clone());
925
926 let err = manager
927 .submit_procedure(
928 ProcedureContext::from_event_context(PersistentEventContext::new(
929 task.trigger_reason.to_trigger_reason(),
930 )),
931 task,
932 )
933 .await
934 .unwrap_err();
935 assert_matches!(err, error::Error::MigrationRunning { .. });
936 }
937
938 #[tokio::test]
939 async fn test_submit_procedure_invalid_task() {
940 let env = TestingEnv::new();
941 let context_factory = env.context_factory();
942 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
943 let region_id = RegionId::new(1024, 1);
944 let task = RegionMigrationProcedureTask {
945 region_id,
946 from_peer: Peer::empty(1),
947 to_peer: Peer::empty(1),
948 timeout: Duration::from_millis(1000),
949 trigger_reason: RegionMigrationTriggerReason::Manual,
950 };
951
952 let err = manager
953 .submit_procedure(
954 ProcedureContext::from_event_context(PersistentEventContext::new(
955 task.trigger_reason.to_trigger_reason(),
956 )),
957 task,
958 )
959 .await
960 .unwrap_err();
961 assert_matches!(err, error::Error::InvalidArguments { .. });
962 }
963
964 #[tokio::test]
965 async fn test_submit_procedure_table_not_found() {
966 let env = TestingEnv::new();
967 let context_factory = env.context_factory();
968 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
969 let region_id = RegionId::new(1024, 1);
970 let task = RegionMigrationProcedureTask {
971 region_id,
972 from_peer: Peer::empty(1),
973 to_peer: Peer::empty(2),
974 timeout: Duration::from_millis(1000),
975 trigger_reason: RegionMigrationTriggerReason::Manual,
976 };
977
978 let err = manager
979 .submit_procedure(
980 ProcedureContext::from_event_context(PersistentEventContext::new(
981 task.trigger_reason.to_trigger_reason(),
982 )),
983 task,
984 )
985 .await
986 .unwrap_err();
987 assert_matches!(err, error::Error::TableRouteNotFound { .. });
988 }
989
990 #[tokio::test]
991 async fn test_submit_procedure_region_route_not_found() {
992 let env = TestingEnv::new();
993 let context_factory = env.context_factory();
994 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
995 let region_id = RegionId::new(1024, 1);
996 let task = RegionMigrationProcedureTask {
997 region_id,
998 from_peer: Peer::empty(1),
999 to_peer: Peer::empty(2),
1000 timeout: Duration::from_millis(1000),
1001 trigger_reason: RegionMigrationTriggerReason::Manual,
1002 };
1003
1004 let table_info = new_test_table_info(1024);
1005 let region_routes = vec![RegionRoute {
1006 region: Region::new_test(RegionId::new(1024, 2)),
1007 leader_peer: Some(Peer::empty(3)),
1008 ..Default::default()
1009 }];
1010
1011 env.create_physical_table_metadata(table_info, region_routes)
1012 .await;
1013
1014 let err = manager
1015 .submit_procedure(
1016 ProcedureContext::from_event_context(PersistentEventContext::new(
1017 task.trigger_reason.to_trigger_reason(),
1018 )),
1019 task,
1020 )
1021 .await
1022 .unwrap_err();
1023 assert_matches!(err, error::Error::RegionRouteNotFound { .. });
1024 }
1025
1026 #[tokio::test]
1027 async fn test_submit_procedure_incorrect_from_peer() {
1028 let env = TestingEnv::new();
1029 let context_factory = env.context_factory();
1030 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1031 let region_id = RegionId::new(1024, 1);
1032 let task = RegionMigrationProcedureTask {
1033 region_id,
1034 from_peer: Peer::empty(1),
1035 to_peer: Peer::empty(2),
1036 timeout: Duration::from_millis(1000),
1037 trigger_reason: RegionMigrationTriggerReason::Manual,
1038 };
1039
1040 let table_info = new_test_table_info(1024);
1041 let region_routes = vec![RegionRoute {
1042 region: Region::new_test(RegionId::new(1024, 1)),
1043 leader_peer: Some(Peer::empty(3)),
1044 ..Default::default()
1045 }];
1046
1047 env.create_physical_table_metadata(table_info, region_routes)
1048 .await;
1049
1050 let err = manager
1051 .submit_procedure(
1052 ProcedureContext::from_event_context(PersistentEventContext::new(
1053 task.trigger_reason.to_trigger_reason(),
1054 )),
1055 task,
1056 )
1057 .await
1058 .unwrap_err();
1059 assert_matches!(err, error::Error::LeaderPeerChanged { .. });
1060 assert_eq!(
1061 err.to_string(),
1062 "Region's leader peer changed: Region's leader peer(3) is not the `from_peer`(1), region: 4398046511105(1024, 1)"
1063 );
1064 }
1065
1066 #[tokio::test]
1067 async fn test_submit_procedure_region_follower_on_to_peer() {
1068 let env = TestingEnv::new();
1069 let context_factory = env.context_factory();
1070 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1071 let region_id = RegionId::new(1024, 1);
1072 let task = RegionMigrationProcedureTask {
1073 region_id,
1074 from_peer: Peer::empty(3),
1075 to_peer: Peer::empty(2),
1076 timeout: Duration::from_millis(1000),
1077 trigger_reason: RegionMigrationTriggerReason::Manual,
1078 };
1079
1080 let table_info = new_test_table_info(1024);
1081 let region_routes = vec![RegionRoute {
1082 region: Region::new_test(region_id),
1083 leader_peer: Some(Peer::empty(3)),
1084 follower_peers: vec![Peer::empty(2)],
1085 ..Default::default()
1086 }];
1087
1088 env.create_physical_table_metadata(table_info, region_routes)
1089 .await;
1090
1091 let err = manager
1092 .submit_procedure(
1093 ProcedureContext::from_event_context(PersistentEventContext::new(
1094 task.trigger_reason.to_trigger_reason(),
1095 )),
1096 task,
1097 )
1098 .await
1099 .unwrap_err();
1100 assert_matches!(err, error::Error::InvalidArguments { .. });
1101 assert_eq!(
1102 err.to_string(),
1103 "Invalid arguments: The `to_peer`(2) is already has a region follower, region: 4398046511105(1024, 1)"
1104 );
1105 }
1106
1107 #[tokio::test]
1108 async fn test_submit_procedure_has_migrated() {
1109 common_telemetry::init_default_ut_logging();
1110 let env = TestingEnv::new();
1111 let context_factory = env.context_factory();
1112 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1113 let region_id = RegionId::new(1024, 1);
1114 let task = RegionMigrationProcedureTask {
1115 region_id,
1116 from_peer: Peer::empty(1),
1117 to_peer: Peer::empty(2),
1118 timeout: Duration::from_millis(1000),
1119 trigger_reason: RegionMigrationTriggerReason::Manual,
1120 };
1121
1122 let table_info = new_test_table_info(1024);
1123 let region_routes = vec![RegionRoute {
1124 region: Region::new_test(RegionId::new(1024, 1)),
1125 leader_peer: Some(Peer::empty(2)),
1126 ..Default::default()
1127 }];
1128
1129 env.create_physical_table_metadata(table_info, region_routes)
1130 .await;
1131
1132 let err = manager
1133 .submit_procedure(
1134 ProcedureContext::from_event_context(PersistentEventContext::new(
1135 task.trigger_reason.to_trigger_reason(),
1136 )),
1137 task,
1138 )
1139 .await
1140 .unwrap_err();
1141 assert_matches!(err, error::Error::RegionMigrated { .. });
1142 }
1143
1144 #[tokio::test]
1145 async fn test_verify_table_route_error() {
1146 let env = TestingEnv::new();
1147 let context_factory = env.context_factory();
1148 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1149 let region_id = RegionId::new(1024, 1);
1150 let task = RegionMigrationProcedureTask {
1151 region_id,
1152 from_peer: Peer::empty(1),
1153 to_peer: Peer::empty(2),
1154 timeout: Duration::from_millis(1000),
1155 trigger_reason: RegionMigrationTriggerReason::Manual,
1156 };
1157
1158 let err = manager
1159 .verify_table_route(
1160 &TableRouteValue::Logical(LogicalTableRouteValue::new(0)),
1161 &task,
1162 )
1163 .unwrap_err();
1164
1165 assert_matches!(err, error::Error::Unexpected { .. });
1166 }
1167
1168 #[tokio::test]
1169 async fn test_submit_procedure_with_multiple_regions_invalid_task() {
1170 let env = TestingEnv::new();
1171 let context_factory = env.context_factory();
1172 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1173 let task = RegionMigrationTaskBatch {
1174 region_ids: vec![RegionId::new(1024, 1)],
1175 from_peer: Peer::empty(1),
1176 to_peer: Peer::empty(1),
1177 timeout: Duration::from_millis(1000),
1178 trigger_reason: RegionMigrationTriggerReason::Manual,
1179 };
1180
1181 let err = manager
1182 .submit_region_migration_task(task)
1183 .await
1184 .unwrap_err();
1185 assert_matches!(err, error::Error::InvalidArguments { .. });
1186 }
1187
1188 #[tokio::test]
1189 async fn test_submit_procedure_with_multiple_regions_no_region_to_migrate() {
1190 common_telemetry::init_default_ut_logging();
1191 let env = TestingEnv::new();
1192 let context_factory = env.context_factory();
1193 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1194 let region_id = RegionId::new(1024, 1);
1195 let task = RegionMigrationTaskBatch {
1196 region_ids: vec![region_id],
1197 from_peer: Peer::empty(1),
1198 to_peer: Peer::empty(2),
1199 timeout: Duration::from_millis(1000),
1200 trigger_reason: RegionMigrationTriggerReason::Manual,
1201 };
1202 let table_info = new_test_table_info(1024);
1203 let region_routes = vec![RegionRoute {
1204 region: Region::new_test(region_id),
1205 leader_peer: Some(Peer::empty(2)),
1206 ..Default::default()
1207 }];
1208 env.create_physical_table_metadata(table_info, region_routes)
1209 .await;
1210 let result = manager.submit_region_migration_task(task).await.unwrap();
1211
1212 assert_eq!(
1213 result,
1214 SubmitRegionMigrationTaskResult {
1215 migrated: vec![region_id],
1216 ..Default::default()
1217 }
1218 );
1219 }
1220
1221 #[tokio::test]
1222 async fn test_submit_procedure_with_multiple_regions_leader_peer_changed() {
1223 let env = TestingEnv::new();
1224 let context_factory = env.context_factory();
1225 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1226 let region_id = RegionId::new(1024, 1);
1227 let task = RegionMigrationTaskBatch {
1228 region_ids: vec![region_id],
1229 from_peer: Peer::empty(1),
1230 to_peer: Peer::empty(2),
1231 timeout: Duration::from_millis(1000),
1232 trigger_reason: RegionMigrationTriggerReason::Manual,
1233 };
1234
1235 let table_info = new_test_table_info(1024);
1236 let region_routes = vec![RegionRoute {
1237 region: Region::new_test(RegionId::new(1024, 1)),
1238 leader_peer: Some(Peer::empty(3)),
1239 ..Default::default()
1240 }];
1241
1242 env.create_physical_table_metadata(table_info, region_routes)
1243 .await;
1244 let result = manager.submit_region_migration_task(task).await.unwrap();
1245 assert_eq!(
1246 result,
1247 SubmitRegionMigrationTaskResult {
1248 leader_changed: vec![region_id],
1249 ..Default::default()
1250 }
1251 );
1252 }
1253
1254 #[tokio::test]
1255 async fn test_submit_procedure_with_multiple_regions_peer_conflict() {
1256 let env = TestingEnv::new();
1257 let context_factory = env.context_factory();
1258 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1259 let region_id = RegionId::new(1024, 1);
1260 let task = RegionMigrationTaskBatch {
1261 region_ids: vec![region_id],
1262 from_peer: Peer::empty(3),
1263 to_peer: Peer::empty(2),
1264 timeout: Duration::from_millis(1000),
1265 trigger_reason: RegionMigrationTriggerReason::Manual,
1266 };
1267
1268 let table_info = new_test_table_info(1024);
1269 let region_routes = vec![RegionRoute {
1270 region: Region::new_test(region_id),
1271 leader_peer: Some(Peer::empty(3)),
1272 follower_peers: vec![Peer::empty(2)],
1273 ..Default::default()
1274 }];
1275
1276 env.create_physical_table_metadata(table_info, region_routes)
1277 .await;
1278 let result = manager.submit_region_migration_task(task).await.unwrap();
1279 assert_eq!(
1280 result,
1281 SubmitRegionMigrationTaskResult {
1282 peer_conflict: vec![region_id],
1283 ..Default::default()
1284 }
1285 );
1286 }
1287
1288 #[tokio::test]
1289 async fn test_running_regions() {
1290 let env = TestingEnv::new();
1291 env.procedure_manager().start().await.unwrap();
1292 let context_factory = env.context_factory();
1293 let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1294 let region_id = RegionId::new(1024, 1);
1295 let task = RegionMigrationTaskBatch {
1296 region_ids: vec![region_id, RegionId::new(1024, 2)],
1297 from_peer: Peer::empty(1),
1298 to_peer: Peer::empty(2),
1299 timeout: Duration::from_millis(1000),
1300 trigger_reason: RegionMigrationTriggerReason::Manual,
1301 };
1302 manager.tracker.running_procedures.write().unwrap().insert(
1304 region_id,
1305 RegionMigrationProcedureTask::new(
1306 region_id,
1307 task.from_peer.clone(),
1308 task.to_peer.clone(),
1309 task.timeout,
1310 task.trigger_reason,
1311 ),
1312 );
1313 let table_info = new_test_table_info(1024);
1314 let region_routes = vec![RegionRoute {
1315 region: Region::new_test(RegionId::new(1024, 2)),
1316 leader_peer: Some(Peer::empty(1)),
1317 ..Default::default()
1318 }];
1319 env.create_physical_table_metadata(table_info, region_routes)
1320 .await;
1321 let result = manager.submit_region_migration_task(task).await.unwrap();
1322 assert_eq!(result.migrating, vec![region_id]);
1323 assert_eq!(result.submitted, vec![RegionId::new(1024, 2)]);
1324 assert!(result.procedure_id.is_some());
1325 env.procedure_manager().stop().await.unwrap();
1326 }
1327
1328 #[test]
1329 fn test_dropped_procedure_releases_candidate_registration_before_tracker() {
1330 let env = TestingEnv::new();
1331 let keeper = env.opening_region_keeper().clone();
1332 let tracker = RegionMigrationProcedureTracker::default();
1333 let region_id = RegionId::new(1024, 1);
1334 let task = RegionMigrationProcedureTask::new(
1335 region_id,
1336 Peer::empty(1),
1337 Peer::empty(2),
1338 Duration::from_secs(1),
1339 RegionMigrationTriggerReason::Manual,
1340 );
1341 let tracker_guard = tracker.insert_running_procedure(&task).unwrap();
1342 let mut procedure = RegionMigrationProcedure::new(
1343 new_persistent_context(1, 2, region_id),
1344 env.context_factory(),
1345 vec![tracker_guard],
1346 );
1347 procedure
1348 .context
1349 .volatile_ctx
1350 .opening_region_guards
1351 .push(keeper.register_candidate_region(2, region_id).unwrap());
1352
1353 let running_procedures = tracker.running_procedures.read().unwrap();
1356 let dropper = std::thread::spawn(move || drop(procedure));
1357 let deadline = std::time::Instant::now() + Duration::from_secs(10);
1358 while !keeper.is_empty() {
1359 assert!(
1360 std::time::Instant::now() < deadline,
1361 "the candidate registration outlives the tracker entry"
1362 );
1363 std::thread::sleep(Duration::from_millis(1));
1364 }
1365 assert!(running_procedures.contains_key(®ion_id));
1366 drop(running_procedures);
1367 dropper.join().unwrap();
1368 assert!(tracker.running_procedures.read().unwrap().is_empty());
1369 }
1370}