Skip to main content

meta_srv/procedure/region_migration/
manager.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::hash_map::Entry;
16use std::collections::{HashMap, HashSet};
17use std::fmt::Display;
18use std::sync::{Arc, RwLock};
19use std::time::Duration;
20
21use common_event_recorder::PersistentEventContext;
22use common_meta::key::table_info::TableInfoValue;
23use common_meta::key::table_route::TableRouteValue;
24use common_meta::peer::Peer;
25use common_meta::rpc::router::RegionRoute;
26use common_procedure::{
27    ProcedureContext, ProcedureId, ProcedureManagerRef, ProcedureWithId, watcher,
28};
29use common_telemetry::{error, info, warn};
30use serde::{Deserialize, Serialize};
31use snafu::{OptionExt, ResultExt, ensure};
32use store_api::storage::RegionId;
33use table::table_name::TableName;
34
35use crate::error::{self, Result};
36use crate::metrics::{METRIC_META_REGION_MIGRATION_DATANODES, METRIC_META_REGION_MIGRATION_FAIL};
37use crate::procedure::region_migration::utils::{
38    RegionMigrationAnalysis, RegionMigrationTaskBatch, analyze_region_migration_task,
39};
40use crate::procedure::region_migration::{
41    DefaultContextFactory, PersistentContext, RegionMigrationProcedure,
42};
43
44pub type RegionMigrationManagerRef = Arc<RegionMigrationManager>;
45
46/// Manager of region migration procedure.
47pub struct RegionMigrationManager {
48    procedure_manager: ProcedureManagerRef,
49    context_factory: DefaultContextFactory,
50    tracker: RegionMigrationProcedureTracker,
51}
52
53#[derive(Default, Clone)]
54pub struct RegionMigrationProcedureTracker {
55    running_procedures: Arc<RwLock<HashMap<RegionId, RegionMigrationProcedureTask>>>,
56}
57
58impl RegionMigrationProcedureTracker {
59    /// Returns the [RegionMigrationProcedureGuard] if current region isn't migrating.
60    pub(crate) fn insert_running_procedure(
61        &self,
62        task: &RegionMigrationProcedureTask,
63    ) -> Option<RegionMigrationProcedureGuard> {
64        let mut procedures = self.running_procedures.write().unwrap();
65        match procedures.entry(task.region_id) {
66            Entry::Occupied(_) => None,
67            Entry::Vacant(v) => {
68                v.insert(task.clone());
69                Some(RegionMigrationProcedureGuard {
70                    region_id: task.region_id,
71                    running_procedures: self.running_procedures.clone(),
72                })
73            }
74        }
75    }
76
77    /// Returns true if it contains the specific region(`region_id`).
78    pub(crate) fn contains(&self, region_id: RegionId) -> bool {
79        self.running_procedures
80            .read()
81            .unwrap()
82            .contains_key(&region_id)
83    }
84}
85
86/// The guard of running [RegionMigrationProcedureTask].
87pub(crate) struct RegionMigrationProcedureGuard {
88    region_id: RegionId,
89    running_procedures: Arc<RwLock<HashMap<RegionId, RegionMigrationProcedureTask>>>,
90}
91
92impl Drop for RegionMigrationProcedureGuard {
93    fn drop(&mut self) {
94        let exists = self
95            .running_procedures
96            .read()
97            .unwrap()
98            .contains_key(&self.region_id);
99        if exists {
100            self.running_procedures
101                .write()
102                .unwrap()
103                .remove(&self.region_id);
104        }
105    }
106}
107
108/// A task of region migration procedure.
109#[derive(Debug, Clone)]
110pub struct RegionMigrationProcedureTask {
111    pub(crate) region_id: RegionId,
112    pub(crate) from_peer: Peer,
113    pub(crate) to_peer: Peer,
114    pub(crate) timeout: Duration,
115    pub(crate) trigger_reason: RegionMigrationTriggerReason,
116}
117
118/// The reason why the region migration procedure is triggered.
119#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, strum::Display)]
120#[strum(serialize_all = "PascalCase")]
121pub enum RegionMigrationTriggerReason {
122    #[default]
123    /// The region migration procedure is triggered by unknown reason.
124    Unknown,
125    /// The region migration procedure is triggered by administrator.
126    Manual,
127    /// The region migration procedure is triggered by auto rebalance.
128    AutoRebalance,
129    /// The region migration procedure is triggered by failover.
130    Failover,
131}
132
133impl RegionMigrationProcedureTask {
134    pub fn new(
135        region_id: RegionId,
136        from_peer: Peer,
137        to_peer: Peer,
138        timeout: Duration,
139        trigger_reason: RegionMigrationTriggerReason,
140    ) -> Self {
141        Self {
142            region_id,
143            from_peer,
144            to_peer,
145            timeout,
146            trigger_reason,
147        }
148    }
149}
150
151impl Display for RegionMigrationProcedureTask {
152    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
153        write!(
154            f,
155            "region: {}, from_peer: {}, to_peer: {}, trigger_reason: {}",
156            self.region_id, self.from_peer, self.to_peer, self.trigger_reason
157        )
158    }
159}
160
161/// The result of submitting a region migration task.
162#[derive(Debug, Default, PartialEq, Eq)]
163pub struct SubmitRegionMigrationTaskResult {
164    /// Regions already migrated to the `to_peer`.
165    pub migrated: Vec<RegionId>,
166    /// Regions where the leader peer has changed.
167    pub leader_changed: Vec<RegionId>,
168    /// Regions where `to_peer` is already a follower (conflict).
169    pub peer_conflict: Vec<RegionId>,
170    /// Regions whose table is not found.
171    pub table_not_found: Vec<RegionId>,
172    /// Regions whose table exists but region route is not found (e.g., removed after repartition).
173    pub region_not_found: Vec<RegionId>,
174    /// Regions still pending migration.
175    pub migrating: Vec<RegionId>,
176    /// Regions that have been submitted for migration.
177    pub submitted: Vec<RegionId>,
178    /// The procedure id of the region migration procedure.
179    pub procedure_id: Option<ProcedureId>,
180}
181
182/// A constructed migration batch that has not been submitted.
183pub(crate) struct BuiltMigrationBatch {
184    /// Region classifications from tracker, route, and table-info checks.
185    pub(crate) analysis: RegionMigrationAnalysis,
186    /// The procedure owning guards for the pending regions, if any remain.
187    pub(crate) procedure: Option<ProcedureWithId>,
188}
189
190impl RegionMigrationManager {
191    /// Returns new [`RegionMigrationManager`]
192    pub(crate) fn new(
193        procedure_manager: ProcedureManagerRef,
194        context_factory: DefaultContextFactory,
195    ) -> Self {
196        Self {
197            procedure_manager,
198            context_factory,
199            tracker: RegionMigrationProcedureTracker::default(),
200        }
201    }
202
203    /// Returns the [`RegionMigrationProcedureTracker`].
204    pub fn tracker(&self) -> &RegionMigrationProcedureTracker {
205        &self.tracker
206    }
207
208    /// Registers the loader of [RegionMigrationProcedure] to the `ProcedureManager`.
209    pub(crate) fn try_start(&self) -> Result<()> {
210        let context_factory = self.context_factory.clone();
211        let tracker = self.tracker.clone();
212        self.procedure_manager
213            .register_loader(
214                RegionMigrationProcedure::TYPE_NAME,
215                Box::new(move |json| {
216                    let context_factory = context_factory.clone();
217                    let tracker = tracker.clone();
218                    RegionMigrationProcedure::from_json(json, context_factory, tracker)
219                        .map(|p| Box::new(p) as _)
220                }),
221            )
222            .context(error::RegisterProcedureLoaderSnafu {
223                type_name: RegionMigrationProcedure::TYPE_NAME,
224            })
225    }
226
227    fn insert_running_procedure(
228        &self,
229        task: &RegionMigrationProcedureTask,
230    ) -> Option<RegionMigrationProcedureGuard> {
231        self.tracker.insert_running_procedure(task)
232    }
233
234    fn verify_task(&self, task: &RegionMigrationProcedureTask) -> Result<()> {
235        if task.to_peer.id == task.from_peer.id {
236            return error::InvalidArgumentsSnafu {
237                err_msg: "The `from_peer_id` can't equal `to_peer_id`",
238            }
239            .fail();
240        }
241
242        Ok(())
243    }
244
245    async fn retrieve_table_route(&self, region_id: RegionId) -> Result<TableRouteValue> {
246        let table_route = self
247            .context_factory
248            .table_metadata_manager
249            .table_route_manager()
250            .table_route_storage()
251            .get(region_id.table_id())
252            .await
253            .context(error::TableMetadataManagerSnafu)?
254            .context(error::TableRouteNotFoundSnafu {
255                table_id: region_id.table_id(),
256            })?;
257
258        Ok(table_route)
259    }
260
261    async fn retrieve_table_info(&self, region_id: RegionId) -> Result<TableInfoValue> {
262        let table_route = self
263            .context_factory
264            .table_metadata_manager
265            .table_info_manager()
266            .get(region_id.table_id())
267            .await
268            .context(error::TableMetadataManagerSnafu)?
269            .context(error::TableInfoNotFoundSnafu {
270                table_id: region_id.table_id(),
271            })?
272            .into_inner();
273
274        Ok(table_route)
275    }
276
277    /// Verifies the type of region migration table route.
278    fn verify_table_route(
279        &self,
280        table_route: &TableRouteValue,
281        task: &RegionMigrationProcedureTask,
282    ) -> Result<()> {
283        if !table_route.is_physical() {
284            return error::UnexpectedSnafu {
285                violated: format!(
286                    "Trying to execute region migration on the logical table, task {task}"
287                ),
288            }
289            .fail();
290        }
291
292        Ok(())
293    }
294
295    /// Returns true if the region has been migrated.
296    fn has_migrated(
297        &self,
298        region_route: &RegionRoute,
299        task: &RegionMigrationProcedureTask,
300    ) -> Result<bool> {
301        if region_route.is_leader_downgrading() {
302            return Ok(false);
303        }
304
305        let leader_peer = region_route
306            .leader_peer
307            .as_ref()
308            .context(error::UnexpectedSnafu {
309                violated: "Region route leader peer is not found",
310            })?;
311
312        Ok(leader_peer.id == task.to_peer.id)
313    }
314
315    /// Throws an error if `leader_peer` is not the `from_peer`.
316    ///
317    /// If `from_peer` is unknown, use the leader peer as the `from_peer`.
318    fn verify_region_leader_peer(
319        &self,
320        region_route: &RegionRoute,
321        task: &mut RegionMigrationProcedureTask,
322    ) -> Result<()> {
323        let leader_peer = region_route
324            .leader_peer
325            .as_ref()
326            .context(error::UnexpectedSnafu {
327                violated: "Region route leader peer is not found",
328            })?;
329
330        ensure!(
331            leader_peer.id == task.from_peer.id,
332            error::LeaderPeerChangedSnafu {
333                msg: format!(
334                    "Region's leader peer({}) is not the `from_peer`({}), region: {}",
335                    leader_peer.id, task.from_peer.id, task.region_id
336                ),
337            }
338        );
339
340        if task.from_peer.addr.is_empty() {
341            warn!(
342                "The `from_peer` is unknown, use the leader peer({}) as the `from_peer`, region: {}",
343                leader_peer, task.region_id
344            );
345            // The peer id is the same as the leader peer id.
346            task.from_peer = leader_peer.clone();
347        }
348
349        Ok(())
350    }
351
352    /// Throws an error if `to_peer` is already has a region follower.
353    fn verify_region_follower_peers(
354        &self,
355        region_route: &RegionRoute,
356        task: &RegionMigrationProcedureTask,
357    ) -> Result<()> {
358        ensure!(
359            !region_route.follower_peers.contains(&task.to_peer),
360            error::InvalidArgumentsSnafu {
361                err_msg: format!(
362                    "The `to_peer`({}) is already has a region follower, region: {}",
363                    task.to_peer.id, task.region_id
364                ),
365            },
366        );
367
368        Ok(())
369    }
370
371    /// Extracts regions from the migration task that are already running migration procedures.
372    ///
373    /// Returns a tuple containing those region ids that are already running and the newly created procedure guards.
374    /// The regions that are already running will be removed from the [`RegionMigrationTask`].
375    fn extract_running_regions(
376        &self,
377        task: &mut RegionMigrationTaskBatch,
378    ) -> (Vec<RegionId>, Vec<RegionMigrationProcedureGuard>) {
379        let mut migrating_region_ids = Vec::new();
380        let mut procedure_guards = Vec::with_capacity(task.region_ids.len());
381
382        for region_id in &task.region_ids {
383            let Some(guard) = self.insert_running_procedure(&RegionMigrationProcedureTask::new(
384                *region_id,
385                task.from_peer.clone(),
386                task.to_peer.clone(),
387                task.timeout,
388                task.trigger_reason,
389            )) else {
390                migrating_region_ids.push(*region_id);
391                continue;
392            };
393            procedure_guards.push(guard);
394        }
395
396        let migrating_set = migrating_region_ids.iter().cloned().collect::<HashSet<_>>();
397        task.region_ids.retain(|id| !migrating_set.contains(id));
398
399        (migrating_region_ids, procedure_guards)
400    }
401
402    pub async fn submit_region_migration_task(
403        &self,
404        task: RegionMigrationTaskBatch,
405    ) -> Result<SubmitRegionMigrationTaskResult> {
406        let procedure_context = ProcedureContext {
407            actor: None,
408            event_context: Some(PersistentEventContext::new(
409                task.trigger_reason.to_trigger_reason(),
410            )),
411        };
412        let from_peer_id = task.from_peer.id;
413        let to_peer_id = task.to_peer.id;
414        let BuiltMigrationBatch {
415            analysis,
416            procedure,
417        } = self.build_batch_procedure(procedure_context, task).await?;
418        let procedure_id = procedure.as_ref().map(|procedure| procedure.id);
419        if let Some(procedure) = procedure {
420            let procedure_id = procedure.id;
421            info!(
422                "Starting region migration procedure {procedure_id}, regions: {:?}, from_peer: {}, to_peer: {}",
423                analysis.pending, from_peer_id, to_peer_id
424            );
425            let mut watcher = self
426                .procedure_manager
427                .submit(procedure)
428                .await
429                .context(error::SubmitProcedureSnafu)?;
430            let num_regions = analysis.pending.len();
431            common_runtime::spawn_global(async move {
432                METRIC_META_REGION_MIGRATION_DATANODES
433                    .with_label_values(&["src", &from_peer_id.to_string()])
434                    .inc_by(num_regions as u64);
435                METRIC_META_REGION_MIGRATION_DATANODES
436                    .with_label_values(&["desc", &to_peer_id.to_string()])
437                    .inc_by(num_regions as u64);
438                if let Err(e) = watcher::wait(&mut watcher).await {
439                    error!(e; "Failed to wait region migration procedure {procedure_id}");
440                    METRIC_META_REGION_MIGRATION_FAIL.inc();
441                    return;
442                }
443                info!("Region migration procedure {procedure_id} finished successfully");
444            });
445        }
446        let RegionMigrationAnalysis {
447            migrating,
448            migrated,
449            leader_changed,
450            peer_conflict,
451            table_not_found,
452            region_not_found,
453            pending,
454        } = analysis;
455        Ok(SubmitRegionMigrationTaskResult {
456            migrated,
457            leader_changed,
458            peer_conflict,
459            table_not_found,
460            region_not_found,
461            migrating,
462            submitted: pending,
463            procedure_id,
464        })
465    }
466
467    /// Builds a batch without submitting it. The returned procedure owns the guards
468    /// of its pending regions; skipped regions are reported separately.
469    pub(crate) async fn build_batch_procedure(
470        &self,
471        procedure_context: ProcedureContext,
472        mut task: RegionMigrationTaskBatch,
473    ) -> Result<BuiltMigrationBatch> {
474        let (migrating, procedure_guards) = self.extract_running_regions(&mut task);
475        let mut analysis =
476            analyze_region_migration_task(&task, &self.context_factory.table_metadata_manager)
477                .await?;
478        analysis.migrating = migrating;
479        if analysis.pending.is_empty() {
480            return Ok(BuiltMigrationBatch {
481                analysis,
482                procedure: None,
483            });
484        }
485
486        // Updates the region ids to the pending region ids.
487        task.region_ids = std::mem::take(&mut analysis.pending);
488        let table_regions = task.table_regions();
489        let table_ids = table_regions.keys().cloned().collect::<Vec<_>>();
490        let table_info_values = self
491            .context_factory
492            .table_metadata_manager
493            .table_info_manager()
494            .batch_get(&table_ids)
495            .await
496            .context(error::TableMetadataManagerSnafu)?;
497        let mut catalog_and_schema = Vec::with_capacity(table_info_values.len());
498        for (table_id, regions) in table_regions {
499            match table_info_values.get(&table_id) {
500                Some(table_info) => {
501                    let TableName {
502                        catalog_name,
503                        schema_name,
504                        ..
505                    } = table_info.table_name();
506                    catalog_and_schema.push((catalog_name, schema_name));
507                }
508                None => {
509                    task.region_ids.retain(|id| id.table_id() != table_id);
510                    analysis.table_not_found.extend(regions);
511                }
512            }
513        }
514        if task.region_ids.is_empty() {
515            return Ok(BuiltMigrationBatch {
516                analysis,
517                procedure: None,
518            });
519        }
520
521        analysis.pending = task.region_ids.clone();
522        let pending = task.region_ids.iter().copied().collect::<HashSet<_>>();
523        let procedure_guards = procedure_guards
524            .into_iter()
525            .filter(|guard| pending.contains(&guard.region_id))
526            .collect();
527        let procedure = RegionMigrationProcedure::new(
528            PersistentContext::new(
529                catalog_and_schema,
530                task.from_peer,
531                task.to_peer,
532                task.region_ids,
533                task.timeout,
534                task.trigger_reason,
535            ),
536            self.context_factory.clone(),
537            procedure_guards,
538        );
539        let procedure =
540            ProcedureWithId::with_random_id(Box::new(procedure)).with_context(procedure_context);
541        Ok(BuiltMigrationBatch {
542            analysis,
543            procedure: Some(procedure),
544        })
545    }
546
547    /// Builds a new region migration procedure for `task` without submitting it.
548    ///
549    /// Returns the built [`ProcedureWithId`] so that a parent procedure can run it as a
550    /// child procedure.
551    pub async fn build_subprocedure(
552        &self,
553        procedure_context: ProcedureContext,
554        mut task: RegionMigrationProcedureTask,
555    ) -> Result<ProcedureWithId> {
556        let Some(guard) = self.insert_running_procedure(&task) else {
557            return error::MigrationRunningSnafu {
558                region_id: task.region_id,
559            }
560            .fail();
561        };
562
563        self.verify_task(&task)?;
564
565        let region_id = task.region_id;
566
567        let table_route = self.retrieve_table_route(region_id).await?;
568        self.verify_table_route(&table_route, &task)?;
569
570        // Safety: checked before.
571        let region_route = table_route
572            .region_route(region_id)
573            .context(error::UnexpectedLogicalRouteTableSnafu {
574                err_msg: format!("{table_route:?} is a non-physical TableRouteValue."),
575            })?
576            .context(error::RegionRouteNotFoundSnafu { region_id })?;
577
578        if self.has_migrated(&region_route, &task)? {
579            info!("Skipping region migration task: {task}");
580            return error::RegionMigratedSnafu {
581                region_id,
582                target_peer_id: task.to_peer.id,
583            }
584            .fail();
585        }
586
587        self.verify_region_leader_peer(&region_route, &mut task)?;
588        self.verify_region_follower_peers(&region_route, &task)?;
589        let table_info = self.retrieve_table_info(region_id).await?;
590        let TableName {
591            catalog_name,
592            schema_name,
593            ..
594        } = table_info.table_name();
595        let RegionMigrationProcedureTask {
596            region_id,
597            from_peer,
598            to_peer,
599            timeout,
600            trigger_reason,
601        } = task.clone();
602        let procedure = RegionMigrationProcedure::new(
603            PersistentContext::new(
604                vec![(catalog_name, schema_name)],
605                from_peer,
606                to_peer,
607                vec![region_id],
608                timeout,
609                trigger_reason,
610            ),
611            self.context_factory.clone(),
612            vec![guard],
613        );
614        let procedure_with_id =
615            ProcedureWithId::with_random_id(Box::new(procedure)).with_context(procedure_context);
616        Ok(procedure_with_id)
617    }
618
619    /// Submits a new region migration procedure.
620    pub async fn submit_procedure(
621        &self,
622        procedure_context: ProcedureContext,
623        mut task: RegionMigrationProcedureTask,
624    ) -> Result<Option<ProcedureId>> {
625        if let Some(event_context) = procedure_context.event_context.as_ref() {
626            task.trigger_reason =
627                RegionMigrationTriggerReason::from_trigger_reason(event_context.reason);
628        }
629
630        let procedure_with_id = self
631            .build_subprocedure(procedure_context, task.clone())
632            .await?;
633        let procedure_id = procedure_with_id.id;
634        info!("Starting region migration procedure {procedure_id} for {task}");
635        let mut watcher = self
636            .procedure_manager
637            .submit(procedure_with_id)
638            .await
639            .context(error::SubmitProcedureSnafu)?;
640        common_runtime::spawn_global(async move {
641            METRIC_META_REGION_MIGRATION_DATANODES
642                .with_label_values(&["src", &task.from_peer.id.to_string()])
643                .inc();
644            METRIC_META_REGION_MIGRATION_DATANODES
645                .with_label_values(&["desc", &task.to_peer.id.to_string()])
646                .inc();
647
648            if let Err(e) = watcher::wait(&mut watcher).await {
649                error!(e; "Failed to wait region migration procedure {procedure_id} for {task}");
650                METRIC_META_REGION_MIGRATION_FAIL.inc();
651                return;
652            }
653
654            info!("Region migration procedure {procedure_id} for {task} is finished successfully!");
655        });
656
657        Ok(Some(procedure_id))
658    }
659}
660
661#[cfg(test)]
662mod test {
663    use std::assert_matches;
664
665    use common_meta::key::table_route::LogicalTableRouteValue;
666    use common_meta::key::test_utils::new_test_table_info;
667    use common_meta::rpc::router::Region;
668    use common_meta::state_store::KvStateStore;
669    use common_procedure::ProcedureManager;
670    use common_procedure::local::{LocalManager, ManagerConfig, PauseAware};
671    use common_procedure::test_util::InMemoryPoisonStore;
672    use tokio::sync::Notify;
673
674    use super::*;
675    use crate::procedure::region_migration::test_util::{TestingEnv, new_persistent_context};
676
677    #[derive(Default)]
678    struct SubmissionGate {
679        entered: Notify,
680        release: Notify,
681    }
682
683    #[async_trait::async_trait]
684    impl PauseAware for SubmissionGate {
685        async fn is_paused(&self) -> std::result::Result<bool, common_error::ext::BoxedError> {
686            self.entered.notify_one();
687            self.release.notified().await;
688            Ok(false)
689        }
690    }
691
692    async fn check_submission_registration(batch: bool) {
693        let env = TestingEnv::new();
694        let gate = Arc::new(SubmissionGate::default());
695        let procedure_manager = Arc::new(LocalManager::new(
696            ManagerConfig::default(),
697            Arc::new(KvStateStore::new(env.kv_backend())),
698            Arc::new(InMemoryPoisonStore::default()),
699            Some(gate.clone()),
700            None,
701        ));
702        let manager = RegionMigrationManager::new(procedure_manager.clone(), env.context_factory());
703        let region_ids = if batch {
704            vec![RegionId::new(1024, 1), RegionId::new(1024, 2)]
705        } else {
706            vec![RegionId::new(1024, 1)]
707        };
708        let region_routes = region_ids
709            .iter()
710            .map(|region_id| RegionRoute {
711                region: Region::new_test(*region_id),
712                leader_peer: Some(Peer::empty(1)),
713                ..Default::default()
714            })
715            .collect();
716        env.create_physical_table_metadata(new_test_table_info(1024), region_routes)
717            .await;
718
719        // A stopped manager must reject submission and release all migration guards.
720        // Starting it then allows the same regions to be submitted successfully.
721        for started in [false, true] {
722            if started {
723                procedure_manager.start().await.unwrap();
724            }
725            let submission = async {
726                if batch {
727                    let result = manager
728                        .submit_region_migration_task(RegionMigrationTaskBatch {
729                            region_ids: region_ids.clone(),
730                            from_peer: Peer::empty(1),
731                            to_peer: Peer::empty(2),
732                            timeout: Duration::from_secs(10),
733                            trigger_reason: RegionMigrationTriggerReason::Manual,
734                        })
735                        .await?;
736                    assert_eq!(result.submitted, region_ids);
737                    Ok(result.procedure_id)
738                } else {
739                    manager
740                        .submit_procedure(
741                            ProcedureContext::default(),
742                            RegionMigrationProcedureTask::new(
743                                region_ids[0],
744                                Peer::empty(1),
745                                Peer::empty(2),
746                                Duration::from_secs(10),
747                                RegionMigrationTriggerReason::Manual,
748                            ),
749                        )
750                        .await
751                }
752            };
753            tokio::pin!(submission);
754            tokio::select! {
755                biased;
756                result = &mut submission => panic!("Submission returned before registration: {result:?}"),
757                _ = gate.entered.notified() => {}
758            }
759            assert!(futures::poll!(&mut submission).is_pending());
760            assert!(
761                procedure_manager
762                    .list_procedures()
763                    .await
764                    .unwrap()
765                    .is_empty()
766            );
767            for region_id in &region_ids {
768                assert!(manager.tracker.contains(*region_id));
769            }
770
771            gate.release.notify_one();
772            let result = tokio::time::timeout(Duration::from_secs(5), &mut submission)
773                .await
774                .expect("Submission should return after registration");
775            if started {
776                let procedure_id = result.unwrap().unwrap();
777                assert!(
778                    procedure_manager
779                        .procedure_state(procedure_id)
780                        .await
781                        .unwrap()
782                        .is_some()
783                );
784            } else {
785                assert_matches!(
786                    result.unwrap_err(),
787                    error::Error::SubmitProcedure {
788                        source: common_procedure::Error::ManagerNotStart { .. },
789                        ..
790                    }
791                );
792                for region_id in &region_ids {
793                    assert!(!manager.tracker.contains(*region_id));
794                }
795            }
796        }
797        procedure_manager.stop().await.unwrap();
798    }
799
800    #[tokio::test]
801    async fn test_build_batch_procedure_retains_only_pending_guards() {
802        let env = TestingEnv::new();
803        let manager =
804            RegionMigrationManager::new(env.procedure_manager().clone(), env.context_factory());
805        let pending = RegionId::new(1024, 1);
806        let migrated = RegionId::new(1024, 2);
807        let migrating = RegionId::new(2048, 1);
808        let running_guard = manager
809            .insert_running_procedure(&RegionMigrationProcedureTask::new(
810                migrating,
811                Peer::empty(1),
812                Peer::empty(2),
813                Duration::from_secs(10),
814                RegionMigrationTriggerReason::Manual,
815            ))
816            .unwrap();
817        env.create_physical_table_metadata(
818            new_test_table_info(1024),
819            vec![
820                RegionRoute {
821                    region: Region::new_test(pending),
822                    leader_peer: Some(Peer::empty(1)),
823                    ..Default::default()
824                },
825                RegionRoute {
826                    region: Region::new_test(migrated),
827                    leader_peer: Some(Peer::empty(2)),
828                    ..Default::default()
829                },
830            ],
831        )
832        .await;
833        let batch = manager
834            .build_batch_procedure(
835                ProcedureContext::default(),
836                RegionMigrationTaskBatch {
837                    region_ids: vec![pending, migrated, migrating],
838                    from_peer: Peer::empty(1),
839                    to_peer: Peer::empty(2),
840                    timeout: Duration::from_secs(10),
841                    trigger_reason: RegionMigrationTriggerReason::Manual,
842                },
843            )
844            .await
845            .unwrap();
846        assert_eq!(
847            RegionMigrationAnalysis {
848                pending: vec![pending],
849                migrated: vec![migrated],
850                migrating: vec![migrating],
851                ..Default::default()
852            },
853            batch.analysis
854        );
855        let child = batch.procedure.unwrap();
856        assert!(manager.tracker.contains(pending));
857        assert!(!manager.tracker.contains(migrated));
858        assert!(
859            env.procedure_manager()
860                .list_procedures()
861                .await
862                .unwrap()
863                .is_empty()
864        );
865        drop(child);
866        assert!(!manager.tracker.contains(pending));
867        assert!(manager.tracker.contains(migrating));
868
869        let batch = manager
870            .build_batch_procedure(
871                ProcedureContext::default(),
872                RegionMigrationTaskBatch {
873                    region_ids: vec![migrating],
874                    from_peer: Peer::empty(1),
875                    to_peer: Peer::empty(2),
876                    timeout: Duration::from_secs(10),
877                    trigger_reason: RegionMigrationTriggerReason::Manual,
878                },
879            )
880            .await
881            .unwrap();
882        assert_eq!(
883            RegionMigrationAnalysis {
884                migrating: vec![migrating],
885                ..Default::default()
886            },
887            batch.analysis
888        );
889        assert!(batch.procedure.is_none());
890        assert!(manager.tracker.contains(migrating));
891        drop(running_guard);
892        assert!(!manager.tracker.contains(migrating));
893    }
894
895    #[tokio::test]
896    async fn test_submit_procedure_registration() {
897        check_submission_registration(false).await;
898    }
899
900    #[tokio::test]
901    async fn test_submit_region_migration_task_registration() {
902        check_submission_registration(true).await;
903    }
904
905    #[tokio::test]
906    async fn test_insert_running_procedure() {
907        let env = TestingEnv::new();
908        let context_factory = env.context_factory();
909        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
910        let region_id = RegionId::new(1024, 1);
911        let task = RegionMigrationProcedureTask {
912            region_id,
913            from_peer: Peer::empty(2),
914            to_peer: Peer::empty(1),
915            timeout: Duration::from_millis(1000),
916            trigger_reason: RegionMigrationTriggerReason::Manual,
917        };
918        // Inserts one
919        manager
920            .tracker
921            .running_procedures
922            .write()
923            .unwrap()
924            .insert(region_id, task.clone());
925
926        let err = manager
927            .submit_procedure(
928                ProcedureContext::from_event_context(PersistentEventContext::new(
929                    task.trigger_reason.to_trigger_reason(),
930                )),
931                task,
932            )
933            .await
934            .unwrap_err();
935        assert_matches!(err, error::Error::MigrationRunning { .. });
936    }
937
938    #[tokio::test]
939    async fn test_submit_procedure_invalid_task() {
940        let env = TestingEnv::new();
941        let context_factory = env.context_factory();
942        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
943        let region_id = RegionId::new(1024, 1);
944        let task = RegionMigrationProcedureTask {
945            region_id,
946            from_peer: Peer::empty(1),
947            to_peer: Peer::empty(1),
948            timeout: Duration::from_millis(1000),
949            trigger_reason: RegionMigrationTriggerReason::Manual,
950        };
951
952        let err = manager
953            .submit_procedure(
954                ProcedureContext::from_event_context(PersistentEventContext::new(
955                    task.trigger_reason.to_trigger_reason(),
956                )),
957                task,
958            )
959            .await
960            .unwrap_err();
961        assert_matches!(err, error::Error::InvalidArguments { .. });
962    }
963
964    #[tokio::test]
965    async fn test_submit_procedure_table_not_found() {
966        let env = TestingEnv::new();
967        let context_factory = env.context_factory();
968        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
969        let region_id = RegionId::new(1024, 1);
970        let task = RegionMigrationProcedureTask {
971            region_id,
972            from_peer: Peer::empty(1),
973            to_peer: Peer::empty(2),
974            timeout: Duration::from_millis(1000),
975            trigger_reason: RegionMigrationTriggerReason::Manual,
976        };
977
978        let err = manager
979            .submit_procedure(
980                ProcedureContext::from_event_context(PersistentEventContext::new(
981                    task.trigger_reason.to_trigger_reason(),
982                )),
983                task,
984            )
985            .await
986            .unwrap_err();
987        assert_matches!(err, error::Error::TableRouteNotFound { .. });
988    }
989
990    #[tokio::test]
991    async fn test_submit_procedure_region_route_not_found() {
992        let env = TestingEnv::new();
993        let context_factory = env.context_factory();
994        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
995        let region_id = RegionId::new(1024, 1);
996        let task = RegionMigrationProcedureTask {
997            region_id,
998            from_peer: Peer::empty(1),
999            to_peer: Peer::empty(2),
1000            timeout: Duration::from_millis(1000),
1001            trigger_reason: RegionMigrationTriggerReason::Manual,
1002        };
1003
1004        let table_info = new_test_table_info(1024);
1005        let region_routes = vec![RegionRoute {
1006            region: Region::new_test(RegionId::new(1024, 2)),
1007            leader_peer: Some(Peer::empty(3)),
1008            ..Default::default()
1009        }];
1010
1011        env.create_physical_table_metadata(table_info, region_routes)
1012            .await;
1013
1014        let err = manager
1015            .submit_procedure(
1016                ProcedureContext::from_event_context(PersistentEventContext::new(
1017                    task.trigger_reason.to_trigger_reason(),
1018                )),
1019                task,
1020            )
1021            .await
1022            .unwrap_err();
1023        assert_matches!(err, error::Error::RegionRouteNotFound { .. });
1024    }
1025
1026    #[tokio::test]
1027    async fn test_submit_procedure_incorrect_from_peer() {
1028        let env = TestingEnv::new();
1029        let context_factory = env.context_factory();
1030        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1031        let region_id = RegionId::new(1024, 1);
1032        let task = RegionMigrationProcedureTask {
1033            region_id,
1034            from_peer: Peer::empty(1),
1035            to_peer: Peer::empty(2),
1036            timeout: Duration::from_millis(1000),
1037            trigger_reason: RegionMigrationTriggerReason::Manual,
1038        };
1039
1040        let table_info = new_test_table_info(1024);
1041        let region_routes = vec![RegionRoute {
1042            region: Region::new_test(RegionId::new(1024, 1)),
1043            leader_peer: Some(Peer::empty(3)),
1044            ..Default::default()
1045        }];
1046
1047        env.create_physical_table_metadata(table_info, region_routes)
1048            .await;
1049
1050        let err = manager
1051            .submit_procedure(
1052                ProcedureContext::from_event_context(PersistentEventContext::new(
1053                    task.trigger_reason.to_trigger_reason(),
1054                )),
1055                task,
1056            )
1057            .await
1058            .unwrap_err();
1059        assert_matches!(err, error::Error::LeaderPeerChanged { .. });
1060        assert_eq!(
1061            err.to_string(),
1062            "Region's leader peer changed: Region's leader peer(3) is not the `from_peer`(1), region: 4398046511105(1024, 1)"
1063        );
1064    }
1065
1066    #[tokio::test]
1067    async fn test_submit_procedure_region_follower_on_to_peer() {
1068        let env = TestingEnv::new();
1069        let context_factory = env.context_factory();
1070        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1071        let region_id = RegionId::new(1024, 1);
1072        let task = RegionMigrationProcedureTask {
1073            region_id,
1074            from_peer: Peer::empty(3),
1075            to_peer: Peer::empty(2),
1076            timeout: Duration::from_millis(1000),
1077            trigger_reason: RegionMigrationTriggerReason::Manual,
1078        };
1079
1080        let table_info = new_test_table_info(1024);
1081        let region_routes = vec![RegionRoute {
1082            region: Region::new_test(region_id),
1083            leader_peer: Some(Peer::empty(3)),
1084            follower_peers: vec![Peer::empty(2)],
1085            ..Default::default()
1086        }];
1087
1088        env.create_physical_table_metadata(table_info, region_routes)
1089            .await;
1090
1091        let err = manager
1092            .submit_procedure(
1093                ProcedureContext::from_event_context(PersistentEventContext::new(
1094                    task.trigger_reason.to_trigger_reason(),
1095                )),
1096                task,
1097            )
1098            .await
1099            .unwrap_err();
1100        assert_matches!(err, error::Error::InvalidArguments { .. });
1101        assert_eq!(
1102            err.to_string(),
1103            "Invalid arguments: The `to_peer`(2) is already has a region follower, region: 4398046511105(1024, 1)"
1104        );
1105    }
1106
1107    #[tokio::test]
1108    async fn test_submit_procedure_has_migrated() {
1109        common_telemetry::init_default_ut_logging();
1110        let env = TestingEnv::new();
1111        let context_factory = env.context_factory();
1112        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1113        let region_id = RegionId::new(1024, 1);
1114        let task = RegionMigrationProcedureTask {
1115            region_id,
1116            from_peer: Peer::empty(1),
1117            to_peer: Peer::empty(2),
1118            timeout: Duration::from_millis(1000),
1119            trigger_reason: RegionMigrationTriggerReason::Manual,
1120        };
1121
1122        let table_info = new_test_table_info(1024);
1123        let region_routes = vec![RegionRoute {
1124            region: Region::new_test(RegionId::new(1024, 1)),
1125            leader_peer: Some(Peer::empty(2)),
1126            ..Default::default()
1127        }];
1128
1129        env.create_physical_table_metadata(table_info, region_routes)
1130            .await;
1131
1132        let err = manager
1133            .submit_procedure(
1134                ProcedureContext::from_event_context(PersistentEventContext::new(
1135                    task.trigger_reason.to_trigger_reason(),
1136                )),
1137                task,
1138            )
1139            .await
1140            .unwrap_err();
1141        assert_matches!(err, error::Error::RegionMigrated { .. });
1142    }
1143
1144    #[tokio::test]
1145    async fn test_verify_table_route_error() {
1146        let env = TestingEnv::new();
1147        let context_factory = env.context_factory();
1148        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1149        let region_id = RegionId::new(1024, 1);
1150        let task = RegionMigrationProcedureTask {
1151            region_id,
1152            from_peer: Peer::empty(1),
1153            to_peer: Peer::empty(2),
1154            timeout: Duration::from_millis(1000),
1155            trigger_reason: RegionMigrationTriggerReason::Manual,
1156        };
1157
1158        let err = manager
1159            .verify_table_route(
1160                &TableRouteValue::Logical(LogicalTableRouteValue::new(0)),
1161                &task,
1162            )
1163            .unwrap_err();
1164
1165        assert_matches!(err, error::Error::Unexpected { .. });
1166    }
1167
1168    #[tokio::test]
1169    async fn test_submit_procedure_with_multiple_regions_invalid_task() {
1170        let env = TestingEnv::new();
1171        let context_factory = env.context_factory();
1172        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1173        let task = RegionMigrationTaskBatch {
1174            region_ids: vec![RegionId::new(1024, 1)],
1175            from_peer: Peer::empty(1),
1176            to_peer: Peer::empty(1),
1177            timeout: Duration::from_millis(1000),
1178            trigger_reason: RegionMigrationTriggerReason::Manual,
1179        };
1180
1181        let err = manager
1182            .submit_region_migration_task(task)
1183            .await
1184            .unwrap_err();
1185        assert_matches!(err, error::Error::InvalidArguments { .. });
1186    }
1187
1188    #[tokio::test]
1189    async fn test_submit_procedure_with_multiple_regions_no_region_to_migrate() {
1190        common_telemetry::init_default_ut_logging();
1191        let env = TestingEnv::new();
1192        let context_factory = env.context_factory();
1193        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1194        let region_id = RegionId::new(1024, 1);
1195        let task = RegionMigrationTaskBatch {
1196            region_ids: vec![region_id],
1197            from_peer: Peer::empty(1),
1198            to_peer: Peer::empty(2),
1199            timeout: Duration::from_millis(1000),
1200            trigger_reason: RegionMigrationTriggerReason::Manual,
1201        };
1202        let table_info = new_test_table_info(1024);
1203        let region_routes = vec![RegionRoute {
1204            region: Region::new_test(region_id),
1205            leader_peer: Some(Peer::empty(2)),
1206            ..Default::default()
1207        }];
1208        env.create_physical_table_metadata(table_info, region_routes)
1209            .await;
1210        let result = manager.submit_region_migration_task(task).await.unwrap();
1211
1212        assert_eq!(
1213            result,
1214            SubmitRegionMigrationTaskResult {
1215                migrated: vec![region_id],
1216                ..Default::default()
1217            }
1218        );
1219    }
1220
1221    #[tokio::test]
1222    async fn test_submit_procedure_with_multiple_regions_leader_peer_changed() {
1223        let env = TestingEnv::new();
1224        let context_factory = env.context_factory();
1225        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1226        let region_id = RegionId::new(1024, 1);
1227        let task = RegionMigrationTaskBatch {
1228            region_ids: vec![region_id],
1229            from_peer: Peer::empty(1),
1230            to_peer: Peer::empty(2),
1231            timeout: Duration::from_millis(1000),
1232            trigger_reason: RegionMigrationTriggerReason::Manual,
1233        };
1234
1235        let table_info = new_test_table_info(1024);
1236        let region_routes = vec![RegionRoute {
1237            region: Region::new_test(RegionId::new(1024, 1)),
1238            leader_peer: Some(Peer::empty(3)),
1239            ..Default::default()
1240        }];
1241
1242        env.create_physical_table_metadata(table_info, region_routes)
1243            .await;
1244        let result = manager.submit_region_migration_task(task).await.unwrap();
1245        assert_eq!(
1246            result,
1247            SubmitRegionMigrationTaskResult {
1248                leader_changed: vec![region_id],
1249                ..Default::default()
1250            }
1251        );
1252    }
1253
1254    #[tokio::test]
1255    async fn test_submit_procedure_with_multiple_regions_peer_conflict() {
1256        let env = TestingEnv::new();
1257        let context_factory = env.context_factory();
1258        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1259        let region_id = RegionId::new(1024, 1);
1260        let task = RegionMigrationTaskBatch {
1261            region_ids: vec![region_id],
1262            from_peer: Peer::empty(3),
1263            to_peer: Peer::empty(2),
1264            timeout: Duration::from_millis(1000),
1265            trigger_reason: RegionMigrationTriggerReason::Manual,
1266        };
1267
1268        let table_info = new_test_table_info(1024);
1269        let region_routes = vec![RegionRoute {
1270            region: Region::new_test(region_id),
1271            leader_peer: Some(Peer::empty(3)),
1272            follower_peers: vec![Peer::empty(2)],
1273            ..Default::default()
1274        }];
1275
1276        env.create_physical_table_metadata(table_info, region_routes)
1277            .await;
1278        let result = manager.submit_region_migration_task(task).await.unwrap();
1279        assert_eq!(
1280            result,
1281            SubmitRegionMigrationTaskResult {
1282                peer_conflict: vec![region_id],
1283                ..Default::default()
1284            }
1285        );
1286    }
1287
1288    #[tokio::test]
1289    async fn test_running_regions() {
1290        let env = TestingEnv::new();
1291        env.procedure_manager().start().await.unwrap();
1292        let context_factory = env.context_factory();
1293        let manager = RegionMigrationManager::new(env.procedure_manager().clone(), context_factory);
1294        let region_id = RegionId::new(1024, 1);
1295        let task = RegionMigrationTaskBatch {
1296            region_ids: vec![region_id, RegionId::new(1024, 2)],
1297            from_peer: Peer::empty(1),
1298            to_peer: Peer::empty(2),
1299            timeout: Duration::from_millis(1000),
1300            trigger_reason: RegionMigrationTriggerReason::Manual,
1301        };
1302        // Inserts one
1303        manager.tracker.running_procedures.write().unwrap().insert(
1304            region_id,
1305            RegionMigrationProcedureTask::new(
1306                region_id,
1307                task.from_peer.clone(),
1308                task.to_peer.clone(),
1309                task.timeout,
1310                task.trigger_reason,
1311            ),
1312        );
1313        let table_info = new_test_table_info(1024);
1314        let region_routes = vec![RegionRoute {
1315            region: Region::new_test(RegionId::new(1024, 2)),
1316            leader_peer: Some(Peer::empty(1)),
1317            ..Default::default()
1318        }];
1319        env.create_physical_table_metadata(table_info, region_routes)
1320            .await;
1321        let result = manager.submit_region_migration_task(task).await.unwrap();
1322        assert_eq!(result.migrating, vec![region_id]);
1323        assert_eq!(result.submitted, vec![RegionId::new(1024, 2)]);
1324        assert!(result.procedure_id.is_some());
1325        env.procedure_manager().stop().await.unwrap();
1326    }
1327
1328    #[test]
1329    fn test_dropped_procedure_releases_candidate_registration_before_tracker() {
1330        let env = TestingEnv::new();
1331        let keeper = env.opening_region_keeper().clone();
1332        let tracker = RegionMigrationProcedureTracker::default();
1333        let region_id = RegionId::new(1024, 1);
1334        let task = RegionMigrationProcedureTask::new(
1335            region_id,
1336            Peer::empty(1),
1337            Peer::empty(2),
1338            Duration::from_secs(1),
1339            RegionMigrationTriggerReason::Manual,
1340        );
1341        let tracker_guard = tracker.insert_running_procedure(&task).unwrap();
1342        let mut procedure = RegionMigrationProcedure::new(
1343            new_persistent_context(1, 2, region_id),
1344            env.context_factory(),
1345            vec![tracker_guard],
1346        );
1347        procedure
1348            .context
1349            .volatile_ctx
1350            .opening_region_guards
1351            .push(keeper.register_candidate_region(2, region_id).unwrap());
1352
1353        // While this read lock is held, the tracker entry cannot be removed, so the registration
1354        // has to go first.
1355        let running_procedures = tracker.running_procedures.read().unwrap();
1356        let dropper = std::thread::spawn(move || drop(procedure));
1357        let deadline = std::time::Instant::now() + Duration::from_secs(10);
1358        while !keeper.is_empty() {
1359            assert!(
1360                std::time::Instant::now() < deadline,
1361                "the candidate registration outlives the tracker entry"
1362            );
1363            std::thread::sleep(Duration::from_millis(1));
1364        }
1365        assert!(running_procedures.contains_key(&region_id));
1366        drop(running_procedures);
1367        dropper.join().unwrap();
1368        assert!(tracker.running_procedures.read().unwrap().is_empty());
1369    }
1370}