Skip to main content

meta_srv/procedure/
region_migration.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub(crate) mod close_downgraded_region;
16pub(crate) mod downgrade_leader_region;
17pub(crate) mod flush_leader_region;
18pub(crate) mod manager;
19pub(crate) mod migration_abort;
20pub(crate) mod migration_end;
21pub(crate) mod migration_start;
22pub(crate) mod open_candidate_region;
23#[cfg(test)]
24pub mod test_util;
25pub(crate) mod update_metadata;
26pub(crate) mod upgrade_candidate_region;
27pub(crate) mod utils;
28
29use std::any::Any;
30use std::collections::{HashMap, HashSet};
31use std::fmt::{Debug, Display};
32use std::time::Duration;
33
34use common_error::ext::BoxedError;
35use common_event_recorder::{Event, PersistentEventContext};
36use common_meta::cache_invalidator::CacheInvalidatorRef;
37use common_meta::ddl::RegionFailureDetectorControllerRef;
38use common_meta::instruction::CacheIdent;
39use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue};
40use common_meta::key::table_route::TableRouteValue;
41use common_meta::key::topic_name::TopicNameKey;
42use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey};
43use common_meta::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
44use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
45use common_meta::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock};
46use common_meta::peer::Peer;
47use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard};
48use common_meta::rpc::ddl::TriggerReason;
49use common_procedure::error::{
50    Error as ProcedureError, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu,
51};
52use common_procedure::{
53    Context as ProcedureContext, EventContext, LockKey, Procedure, Status, StringKey,
54};
55use common_telemetry::{debug, error, info};
56use manager::RegionMigrationProcedureGuard;
57pub use manager::{
58    RegionMigrationManagerRef, RegionMigrationProcedureTask, RegionMigrationProcedureTracker,
59    RegionMigrationTriggerReason,
60};
61use serde::{Deserialize, Deserializer, Serialize};
62use snafu::{OptionExt, ResultExt};
63use store_api::storage::{RegionId, TableId};
64use tokio::time::Instant;
65
66use self::migration_start::RegionMigrationStart;
67use crate::error::{self, Result};
68use crate::event::region_migration::{REGION_MIGRATION_EVENT_TYPE, RegionMigrationEvent};
69use crate::metrics::{
70    METRIC_META_REGION_MIGRATION_ERROR, METRIC_META_REGION_MIGRATION_EXECUTE,
71    METRIC_META_REGION_MIGRATION_STAGE_ELAPSED,
72};
73use crate::service::mailbox::MailboxRef;
74
75/// The default timeout for region migration.
76pub const DEFAULT_REGION_MIGRATION_TIMEOUT: Duration = Duration::from_secs(120);
77
78#[derive(Debug, Deserialize)]
79#[serde(untagged)]
80enum SingleOrMultiple<T> {
81    Single(T),
82    Multiple(Vec<T>),
83}
84
85fn single_or_multiple_from<'de, D, T>(deserializer: D) -> std::result::Result<Vec<T>, D::Error>
86where
87    D: Deserializer<'de>,
88    T: Deserialize<'de>,
89{
90    let helper = SingleOrMultiple::<T>::deserialize(deserializer)?;
91    Ok(match helper {
92        SingleOrMultiple::Single(x) => vec![x],
93        SingleOrMultiple::Multiple(xs) => xs,
94    })
95}
96
97/// It's shared in each step and available even after recovering.
98///
99/// It will only be updated/stored after the Red node has succeeded.
100///
101/// **Notes: Stores with too large data in the context might incur replication overhead.**
102#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
103pub struct PersistentContext {
104    /// The table catalog.
105    #[deprecated(note = "use `catalog_and_schema` instead")]
106    #[serde(default, skip_serializing_if = "Option::is_none")]
107    pub(crate) catalog: Option<String>,
108    /// The table schema.
109    #[deprecated(note = "use `catalog_and_schema` instead")]
110    #[serde(default, skip_serializing_if = "Option::is_none")]
111    pub(crate) schema: Option<String>,
112    /// The catalog and schema of the regions.
113    #[serde(default, skip_serializing_if = "Vec::is_empty")]
114    pub(crate) catalog_and_schema: Vec<(String, String)>,
115    /// The [Peer] of migration source.
116    pub(crate) from_peer: Peer,
117    /// The [Peer] of migration destination.
118    pub(crate) to_peer: Peer,
119    /// The [RegionId] of migration region.
120    #[serde(deserialize_with = "single_or_multiple_from", alias = "region_id")]
121    pub(crate) region_ids: Vec<RegionId>,
122    /// The timeout for downgrading leader region and upgrading candidate region operations.
123    #[serde(with = "humantime_serde", default = "default_timeout")]
124    pub(crate) timeout: Duration,
125    /// The trigger reason persisted for compatibility with versions without procedure context.
126    #[serde(default)]
127    trigger_reason: RegionMigrationTriggerReason,
128}
129
130impl PersistentContext {
131    pub fn new(
132        catalog_and_schema: Vec<(String, String)>,
133        from_peer: Peer,
134        to_peer: Peer,
135        region_ids: Vec<RegionId>,
136        timeout: Duration,
137        trigger_reason: RegionMigrationTriggerReason,
138    ) -> Self {
139        #[allow(deprecated)]
140        Self {
141            catalog: None,
142            schema: None,
143            catalog_and_schema,
144            from_peer,
145            to_peer,
146            region_ids,
147            timeout,
148            trigger_reason,
149        }
150    }
151}
152
153impl RegionMigrationTriggerReason {
154    fn to_trigger_reason(self) -> TriggerReason {
155        match self {
156            Self::Manual => TriggerReason::Manual,
157            Self::AutoRebalance => TriggerReason::AutoRebalance,
158            Self::Failover => TriggerReason::RegionFailover,
159            Self::Unknown => TriggerReason::Unknown,
160        }
161    }
162
163    pub(crate) fn from_trigger_reason(reason: TriggerReason) -> Self {
164        match reason {
165            TriggerReason::Manual => Self::Manual,
166            TriggerReason::AutoRebalance => Self::AutoRebalance,
167            TriggerReason::RegionFailover => Self::Failover,
168            _ => Self::Unknown,
169        }
170    }
171}
172
173fn default_timeout() -> Duration {
174    Duration::from_secs(10)
175}
176
177impl PersistentContext {
178    pub fn lock_key(&self) -> Vec<StringKey> {
179        let mut lock_keys =
180            Vec::with_capacity(self.region_ids.len() + 2 + self.catalog_and_schema.len() * 2);
181        #[allow(deprecated)]
182        if let (Some(catalog), Some(schema)) = (&self.catalog, &self.schema) {
183            lock_keys.push(CatalogLock::Read(catalog).into());
184            lock_keys.push(SchemaLock::read(catalog, schema).into());
185        }
186        for (catalog, schema) in self.catalog_and_schema.iter() {
187            lock_keys.push(CatalogLock::Read(catalog).into());
188            lock_keys.push(SchemaLock::read(catalog, schema).into());
189        }
190
191        // Sort the region ids to ensure the same order of region ids.
192        let mut region_ids = self.region_ids.clone();
193        region_ids.sort_unstable();
194        for region_id in region_ids {
195            lock_keys.push(RegionLock::Write(region_id).into());
196        }
197        lock_keys
198    }
199
200    /// Returns the table ids of the regions.
201    ///
202    /// The return value is a set of table ids.
203    pub fn region_table_ids(&self) -> Vec<TableId> {
204        self.region_ids
205            .iter()
206            .map(|region_id| region_id.table_id())
207            .collect::<HashSet<_>>()
208            .into_iter()
209            .collect()
210    }
211
212    /// Returns the table regions map.
213    ///
214    /// The key is the table id, the value is the region ids of the table.
215    pub fn table_regions(&self) -> HashMap<TableId, Vec<RegionId>> {
216        let mut table_regions = HashMap::new();
217        for region_id in &self.region_ids {
218            table_regions
219                .entry(region_id.table_id())
220                .or_insert_with(Vec::new)
221                .push(*region_id);
222        }
223        table_regions
224    }
225}
226
227/// Metrics of region migration.
228#[derive(Debug, Clone, Default)]
229pub struct Metrics {
230    /// Elapsed time of downgrading region and upgrading region.
231    operations_elapsed: Duration,
232    /// Elapsed time of flushing leader region.
233    flush_leader_region_elapsed: Duration,
234    /// Elapsed time of downgrading leader region.
235    downgrade_leader_region_elapsed: Duration,
236    /// Elapsed time of open candidate region.
237    open_candidate_region_elapsed: Duration,
238    /// Elapsed time of upgrade candidate region.
239    upgrade_candidate_region_elapsed: Duration,
240}
241
242impl Display for Metrics {
243    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
244        let total = self.flush_leader_region_elapsed
245            + self.downgrade_leader_region_elapsed
246            + self.open_candidate_region_elapsed
247            + self.upgrade_candidate_region_elapsed;
248        write!(
249            f,
250            "total: {:?}, flush_leader_region_elapsed: {:?}, downgrade_leader_region_elapsed: {:?}, open_candidate_region_elapsed: {:?}, upgrade_candidate_region_elapsed: {:?}",
251            total,
252            self.flush_leader_region_elapsed,
253            self.downgrade_leader_region_elapsed,
254            self.open_candidate_region_elapsed,
255            self.upgrade_candidate_region_elapsed
256        )
257    }
258}
259
260impl Metrics {
261    /// Updates the elapsed time of downgrading region and upgrading region.
262    pub fn update_operations_elapsed(&mut self, elapsed: Duration) {
263        self.operations_elapsed += elapsed;
264    }
265
266    /// Updates the elapsed time of flushing leader region.
267    pub fn update_flush_leader_region_elapsed(&mut self, elapsed: Duration) {
268        self.flush_leader_region_elapsed += elapsed;
269    }
270
271    /// Updates the elapsed time of downgrading leader region.
272    pub fn update_downgrade_leader_region_elapsed(&mut self, elapsed: Duration) {
273        self.downgrade_leader_region_elapsed += elapsed;
274    }
275
276    /// Updates the elapsed time of open candidate region.
277    pub fn update_open_candidate_region_elapsed(&mut self, elapsed: Duration) {
278        self.open_candidate_region_elapsed += elapsed;
279    }
280
281    /// Updates the elapsed time of upgrade candidate region.
282    pub fn update_upgrade_candidate_region_elapsed(&mut self, elapsed: Duration) {
283        self.upgrade_candidate_region_elapsed += elapsed;
284    }
285}
286
287impl Drop for Metrics {
288    fn drop(&mut self) {
289        let total = self.flush_leader_region_elapsed
290            + self.downgrade_leader_region_elapsed
291            + self.open_candidate_region_elapsed
292            + self.upgrade_candidate_region_elapsed;
293        METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
294            .with_label_values(&["total"])
295            .observe(total.as_secs_f64());
296
297        if !self.flush_leader_region_elapsed.is_zero() {
298            METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
299                .with_label_values(&["flush_leader_region"])
300                .observe(self.flush_leader_region_elapsed.as_secs_f64());
301        }
302
303        if !self.downgrade_leader_region_elapsed.is_zero() {
304            METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
305                .with_label_values(&["downgrade_leader_region"])
306                .observe(self.downgrade_leader_region_elapsed.as_secs_f64());
307        }
308
309        if !self.open_candidate_region_elapsed.is_zero() {
310            METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
311                .with_label_values(&["open_candidate_region"])
312                .observe(self.open_candidate_region_elapsed.as_secs_f64());
313        }
314
315        if !self.upgrade_candidate_region_elapsed.is_zero() {
316            METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
317                .with_label_values(&["upgrade_candidate_region"])
318                .observe(self.upgrade_candidate_region_elapsed.as_secs_f64());
319        }
320    }
321}
322
323/// It's shared in each step and available in executing (including retrying).
324///
325/// It will be dropped if the procedure runner crashes.
326///
327/// The additional remote fetches are only required in the worst cases.
328#[derive(Debug, Clone, Default)]
329pub struct VolatileContext {
330    /// `opening_region_guard` will be set after the
331    /// [OpenCandidateRegion](crate::procedure::region_migration::open_candidate_region::OpenCandidateRegion) step.
332    ///
333    /// On success, `opening_region_guards` is consumed after the
334    /// [RegionRoute](common_meta::rpc::router::RegionRoute) of the opening region is written into
335    /// the [TableRouteValue](common_meta::key::table_route::TableRouteValue). On a rollback it is
336    /// consumed before the route is restored, so the target leaves the candidate registration
337    /// before the source can lead again.
338    opening_region_guards: Vec<OperatingRegionGuard>,
339    /// The deadline of leader region lease.
340    leader_region_lease_deadline: Option<Instant>,
341    /// The datanode table values.
342    from_peer_datanode_table_values: Option<HashMap<TableId, DatanodeTableValue>>,
343    /// The last_entry_ids of leader regions.
344    leader_region_last_entry_ids: HashMap<RegionId, u64>,
345    /// The last_entry_ids of leader metadata regions (Only used for metric engine).
346    leader_region_metadata_last_entry_ids: HashMap<RegionId, u64>,
347    /// The regions whose upgrade reply from the candidate has been accepted. A retry of the
348    /// upgrade does not send them again.
349    upgraded_regions: HashSet<RegionId>,
350    /// Metrics of region migration.
351    metrics: Metrics,
352}
353
354impl VolatileContext {
355    /// Sets the `leader_region_lease_deadline` if it does not exist.
356    pub fn set_leader_region_lease_deadline(&mut self, lease_timeout: Duration) {
357        if self.leader_region_lease_deadline.is_none() {
358            self.leader_region_lease_deadline = Some(Instant::now() + lease_timeout);
359        }
360    }
361
362    /// Resets the `leader_region_lease_deadline`.
363    pub fn reset_leader_region_lease_deadline(&mut self) {
364        self.leader_region_lease_deadline = None;
365    }
366
367    /// Sets the `leader_region_last_entry_id`.
368    pub fn set_last_entry_id(&mut self, region_id: RegionId, last_entry_id: u64) {
369        self.leader_region_last_entry_ids
370            .insert(region_id, last_entry_id);
371    }
372
373    /// Sets the `leader_region_metadata_last_entry_id`.
374    pub fn set_metadata_last_entry_id(&mut self, region_id: RegionId, last_entry_id: u64) {
375        self.leader_region_metadata_last_entry_ids
376            .insert(region_id, last_entry_id);
377    }
378}
379
380/// Used to generate new [Context].
381pub trait ContextFactory {
382    fn new_context(self, persistent_ctx: PersistentContext) -> Context;
383}
384
385/// Default implementation.
386#[derive(Clone)]
387pub struct DefaultContextFactory {
388    volatile_ctx: VolatileContext,
389    in_memory_key: ResettableKvBackendRef,
390    table_metadata_manager: TableMetadataManagerRef,
391    opening_region_keeper: MemoryRegionKeeperRef,
392    region_failure_detector_controller: RegionFailureDetectorControllerRef,
393    mailbox: MailboxRef,
394    server_addr: String,
395    cache_invalidator: CacheInvalidatorRef,
396}
397
398impl DefaultContextFactory {
399    /// Returns an [`DefaultContextFactory`].
400    pub fn new(
401        in_memory_key: ResettableKvBackendRef,
402        table_metadata_manager: TableMetadataManagerRef,
403        opening_region_keeper: MemoryRegionKeeperRef,
404        region_failure_detector_controller: RegionFailureDetectorControllerRef,
405        mailbox: MailboxRef,
406        server_addr: String,
407        cache_invalidator: CacheInvalidatorRef,
408    ) -> Self {
409        Self {
410            volatile_ctx: VolatileContext::default(),
411            in_memory_key,
412            table_metadata_manager,
413            opening_region_keeper,
414            region_failure_detector_controller,
415            mailbox,
416            server_addr,
417            cache_invalidator,
418        }
419    }
420}
421
422impl ContextFactory for DefaultContextFactory {
423    fn new_context(self, persistent_ctx: PersistentContext) -> Context {
424        Context {
425            persistent_ctx,
426            volatile_ctx: self.volatile_ctx,
427            in_memory: self.in_memory_key,
428            table_metadata_manager: self.table_metadata_manager,
429            opening_region_keeper: self.opening_region_keeper,
430            region_failure_detector_controller: self.region_failure_detector_controller,
431            mailbox: self.mailbox,
432            server_addr: self.server_addr,
433            cache_invalidator: self.cache_invalidator,
434        }
435    }
436}
437
438/// The context of procedure execution.
439pub struct Context {
440    persistent_ctx: PersistentContext,
441    volatile_ctx: VolatileContext,
442    in_memory: KvBackendRef,
443    table_metadata_manager: TableMetadataManagerRef,
444    opening_region_keeper: MemoryRegionKeeperRef,
445    region_failure_detector_controller: RegionFailureDetectorControllerRef,
446    mailbox: MailboxRef,
447    server_addr: String,
448    cache_invalidator: CacheInvalidatorRef,
449}
450
451impl Context {
452    pub(crate) fn trigger_reason(
453        &self,
454        event_context: Option<&PersistentEventContext>,
455    ) -> RegionMigrationTriggerReason {
456        event_context
457            .map(|ctx| RegionMigrationTriggerReason::from_trigger_reason(ctx.reason))
458            .unwrap_or(self.persistent_ctx.trigger_reason)
459    }
460    /// Returns the next operation's timeout.
461    pub fn next_operation_timeout(&self) -> Option<Duration> {
462        self.persistent_ctx
463            .timeout
464            .checked_sub(self.volatile_ctx.metrics.operations_elapsed)
465    }
466
467    /// Updates operations elapsed.
468    pub fn update_operations_elapsed(&mut self, instant: Instant) {
469        self.volatile_ctx
470            .metrics
471            .update_operations_elapsed(instant.elapsed());
472    }
473
474    /// Updates the elapsed time of flushing leader region.
475    pub fn update_flush_leader_region_elapsed(&mut self, instant: Instant) {
476        self.volatile_ctx
477            .metrics
478            .update_flush_leader_region_elapsed(instant.elapsed());
479    }
480
481    /// Updates the elapsed time of downgrading leader region.
482    pub fn update_downgrade_leader_region_elapsed(&mut self, instant: Instant) {
483        self.volatile_ctx
484            .metrics
485            .update_downgrade_leader_region_elapsed(instant.elapsed());
486    }
487
488    /// Updates the elapsed time of open candidate region.
489    pub fn update_open_candidate_region_elapsed(&mut self, instant: Instant) {
490        self.volatile_ctx
491            .metrics
492            .update_open_candidate_region_elapsed(instant.elapsed());
493    }
494
495    /// Updates the elapsed time of upgrade candidate region.
496    pub fn update_upgrade_candidate_region_elapsed(&mut self, instant: Instant) {
497        self.volatile_ctx
498            .metrics
499            .update_upgrade_candidate_region_elapsed(instant.elapsed());
500    }
501
502    /// Returns address of meta server.
503    pub fn server_addr(&self) -> &str {
504        &self.server_addr
505    }
506
507    /// Returns the table ids of the regions.
508    pub fn region_table_ids(&self) -> Vec<TableId> {
509        self.persistent_ctx
510            .region_ids
511            .iter()
512            .map(|region_id| region_id.table_id())
513            .collect::<HashSet<_>>()
514            .into_iter()
515            .collect()
516    }
517
518    /// Returns the `table_routes` of [VolatileContext] if any.
519    /// Otherwise, returns the value retrieved from remote.
520    ///
521    /// Retry:
522    /// - Failed to retrieve the metadata of table.
523    pub async fn get_table_route_values(
524        &self,
525    ) -> Result<HashMap<TableId, DeserializedValueWithBytes<TableRouteValue>>> {
526        let table_ids = self.persistent_ctx.region_table_ids();
527        let table_routes = self
528            .table_metadata_manager
529            .table_route_manager()
530            .table_route_storage()
531            .batch_get_with_raw_bytes(&table_ids)
532            .await
533            .context(error::TableMetadataManagerSnafu)
534            .map_err(BoxedError::new)
535            .with_context(|_| error::RetryLaterWithSourceSnafu {
536                reason: format!("Failed to get table routes: {table_ids:?}"),
537            })?;
538        let table_routes = table_ids
539            .into_iter()
540            .zip(table_routes)
541            .filter_map(|(table_id, table_route)| {
542                table_route.map(|table_route| (table_id, table_route))
543            })
544            .collect::<HashMap<_, _>>();
545        Ok(table_routes)
546    }
547
548    /// Returns the `table_route` of [VolatileContext] if any.
549    /// Otherwise, returns the value retrieved from remote.
550    ///
551    /// Retry:
552    /// - Failed to retrieve the metadata of table.
553    pub async fn get_table_route_value(
554        &self,
555        table_id: TableId,
556    ) -> Result<DeserializedValueWithBytes<TableRouteValue>> {
557        let table_route_value = self
558            .table_metadata_manager
559            .table_route_manager()
560            .table_route_storage()
561            .get_with_raw_bytes(table_id)
562            .await
563            .context(error::TableMetadataManagerSnafu)
564            .map_err(BoxedError::new)
565            .with_context(|_| error::RetryLaterWithSourceSnafu {
566                reason: format!("Failed to get table routes: {table_id:}"),
567            })?
568            .context(error::TableRouteNotFoundSnafu { table_id })?;
569        Ok(table_route_value)
570    }
571
572    /// Returns the `from_peer_datanode_table_values` of [VolatileContext] if any.
573    /// Otherwise, returns the value retrieved from remote.
574    ///
575    /// Retry:
576    /// - Failed to retrieve the metadata of datanode table.
577    pub async fn get_from_peer_datanode_table_values(
578        &mut self,
579    ) -> Result<&HashMap<TableId, DatanodeTableValue>> {
580        let from_peer_datanode_table_values =
581            &mut self.volatile_ctx.from_peer_datanode_table_values;
582        if from_peer_datanode_table_values.is_none() {
583            let table_ids = self.persistent_ctx.region_table_ids();
584            let datanode_table_keys = table_ids
585                .iter()
586                .map(|table_id| DatanodeTableKey {
587                    datanode_id: self.persistent_ctx.from_peer.id,
588                    table_id: *table_id,
589                })
590                .collect::<Vec<_>>();
591            let datanode_table_values = self
592                .table_metadata_manager
593                .datanode_table_manager()
594                .batch_get(&datanode_table_keys)
595                .await
596                .context(error::TableMetadataManagerSnafu)
597                .map_err(BoxedError::new)
598                .with_context(|_| error::RetryLaterWithSourceSnafu {
599                    reason: format!("Failed to get DatanodeTable: {table_ids:?}"),
600                })?
601                .into_iter()
602                .map(|(k, v)| (k.table_id, v))
603                .collect();
604            *from_peer_datanode_table_values = Some(datanode_table_values);
605        }
606        Ok(from_peer_datanode_table_values.as_ref().unwrap())
607    }
608
609    /// Returns the `from_peer_datanode_table_value` of [VolatileContext] if any.
610    /// Otherwise, returns the value retrieved from remote.
611    ///
612    /// Retry:
613    /// - Failed to retrieve the metadata of datanode table.
614    pub async fn get_from_peer_datanode_table_value(
615        &self,
616        table_id: TableId,
617    ) -> Result<DatanodeTableValue> {
618        let datanode_table_value = self
619            .table_metadata_manager
620            .datanode_table_manager()
621            .get(&DatanodeTableKey {
622                datanode_id: self.persistent_ctx.from_peer.id,
623                table_id,
624            })
625            .await
626            .context(error::TableMetadataManagerSnafu)
627            .map_err(BoxedError::new)
628            .with_context(|_| error::RetryLaterWithSourceSnafu {
629                reason: format!("Failed to get DatanodeTable: {table_id}"),
630            })?
631            .context(error::DatanodeTableNotFoundSnafu {
632                table_id,
633                datanode_id: self.persistent_ctx.from_peer.id,
634            })?;
635        Ok(datanode_table_value)
636    }
637
638    /// Notifies the RegionSupervisor to register failure detectors of the given regions on the
639    /// source peer.
640    ///
641    /// The original failure detector was removed once the procedure was triggered.
642    /// Now, we need to register the failure detector for the failed region again.
643    pub async fn register_failure_detectors(&self, region_ids: &[RegionId]) {
644        let datanode_id = self.persistent_ctx.from_peer.id;
645        let detecting_regions = region_ids
646            .iter()
647            .map(|region_id| (datanode_id, *region_id))
648            .collect::<Vec<_>>();
649        self.region_failure_detector_controller
650            .register_failure_detectors(detecting_regions)
651            .await;
652        info!(
653            "Registered failure detectors after migration failures for datanode {}, regions {:?}",
654            datanode_id, region_ids
655        );
656    }
657
658    /// Notifies the RegionSupervisor to reset failure detectors of the given regions on the target
659    /// peer.
660    pub async fn reset_failure_detectors(&self, region_ids: &[RegionId]) {
661        let datanode_id = self.persistent_ctx.to_peer.id;
662        let detecting_regions = region_ids
663            .iter()
664            .map(|region_id| (datanode_id, *region_id))
665            .collect::<Vec<_>>();
666        self.region_failure_detector_controller
667            .reset_failure_detectors(detecting_regions)
668            .await;
669        info!(
670            "Reset failure detectors of regions for datanode {}, regions {:?}",
671            datanode_id, region_ids
672        );
673    }
674
675    /// Notifies the RegionSupervisor to deregister failure detectors.
676    ///
677    /// The original failure detectors won't be removed once the procedure was triggered.
678    /// We need to deregister the failure detectors for the original region if the procedure is finished.
679    pub async fn deregister_failure_detectors(&self) {
680        let datanode_id = self.persistent_ctx.from_peer.id;
681        let region_ids = &self.persistent_ctx.region_ids;
682        let detecting_regions = region_ids
683            .iter()
684            .map(|region_id| (datanode_id, *region_id))
685            .collect::<Vec<_>>();
686
687        self.region_failure_detector_controller
688            .deregister_failure_detectors(detecting_regions)
689            .await;
690    }
691
692    /// Notifies the RegionSupervisor to deregister failure detectors for the candidate regions on the destination peer.
693    ///
694    /// The candidate regions may be created on the destination peer,
695    /// so we need to deregister the failure detectors for the candidate regions if the procedure is aborted.
696    pub async fn deregister_failure_detectors_for_candidate_regions(&self) {
697        let to_peer_id = self.persistent_ctx.to_peer.id;
698        let region_ids = &self.persistent_ctx.region_ids;
699        let detecting_regions = region_ids
700            .iter()
701            .map(|region_id| (to_peer_id, *region_id))
702            .collect::<Vec<_>>();
703
704        self.region_failure_detector_controller
705            .deregister_failure_detectors(detecting_regions)
706            .await;
707    }
708
709    /// Fetches replay checkpoints and merges them with topic pruned entry ids.
710    pub async fn get_replay_checkpoints_with_topic_pruned_entry_ids(
711        &self,
712        region_topics: &[(RegionId, String, bool)],
713    ) -> Result<HashMap<RegionId, ReplayCheckpoint>> {
714        let topic_region_keys = region_topics
715            .iter()
716            .map(|(region_id, topic, _)| TopicRegionKey::new(*region_id, topic))
717            .collect::<Vec<_>>();
718        let topic_region_values = self
719            .table_metadata_manager
720            .topic_region_manager()
721            .batch_get(topic_region_keys)
722            .await
723            .context(error::TableMetadataManagerSnafu)?;
724
725        let topic_name_keys = region_topics
726            .iter()
727            .map(|(_, topic, _)| topic.as_str())
728            .collect::<HashSet<_>>()
729            .into_iter()
730            .map(TopicNameKey::new)
731            .collect::<Vec<_>>();
732        let topic_name_values = self
733            .table_metadata_manager
734            .topic_name_manager()
735            .batch_get(topic_name_keys)
736            .await
737            .context(error::TableMetadataManagerSnafu)?;
738        debug!(
739            "Fetched topic region values: {:?}, topic name values: {:?}",
740            topic_region_values, topic_name_values
741        );
742
743        let replay_checkpoints = region_topics
744            .iter()
745            .filter_map(|(region_id, topic, is_metric_engine)| {
746                let checkpoint = topic_region_values
747                    .get(region_id)
748                    .and_then(|value| value.checkpoint);
749                let pruned_entry_id = topic_name_values
750                    .get(topic)
751                    .map(|value| value.pruned_entry_id);
752
753                ReplayCheckpoint::merge_with_topic_pruned_entry_id(
754                    checkpoint,
755                    pruned_entry_id,
756                    *is_metric_engine,
757                )
758                .map(|checkpoint| (*region_id, checkpoint))
759            })
760            .collect::<HashMap<_, _>>();
761
762        Ok(replay_checkpoints)
763    }
764
765    /// Broadcasts the invalidate table cache message.
766    pub async fn invalidate_table_cache(&self) -> Result<()> {
767        let table_ids = self.region_table_ids();
768        let mut cache_idents = Vec::with_capacity(table_ids.len());
769        for table_id in &table_ids {
770            cache_idents.push(CacheIdent::TableId(*table_id));
771        }
772        // ignore the result
773        let ctx = common_meta::cache_invalidator::Context::default();
774        let _ = self.cache_invalidator.invalidate(&ctx, &cache_idents).await;
775        Ok(())
776    }
777
778    /// Returns the [PersistentContext] of the procedure.
779    pub fn persistent_ctx(&self) -> PersistentContext {
780        self.persistent_ctx.clone()
781    }
782}
783
784#[async_trait::async_trait]
785#[typetag::serde(tag = "region_migration_state")]
786pub(crate) trait State: Sync + Send + Debug {
787    fn name(&self) -> &'static str {
788        let type_name = std::any::type_name::<Self>();
789        // short name
790        type_name.split("::").last().unwrap_or(type_name)
791    }
792
793    /// Yields the next [State] and [Status].
794    async fn next(
795        &mut self,
796        ctx: &mut Context,
797        procedure_ctx: &ProcedureContext,
798    ) -> Result<(Box<dyn State>, Status)>;
799
800    /// Returns as [Any](std::any::Any).
801    fn as_any(&self) -> &dyn Any;
802}
803
804/// Persistent data of a recovered [RegionMigrationProcedure]. The persisted state is not read:
805/// the procedure starts from the route check again.
806#[derive(Debug, Deserialize)]
807pub struct RegionMigrationDataOwned {
808    persistent_ctx: PersistentContext,
809}
810
811/// Persistent data of [RegionMigrationProcedure].
812#[derive(Debug, Serialize)]
813pub struct RegionMigrationData<'a> {
814    persistent_ctx: &'a PersistentContext,
815    state: &'a dyn State,
816}
817
818pub(crate) struct RegionMigrationProcedure {
819    state: Box<dyn State>,
820    /// Declared before `_guards`: a procedure that ends without a rollback (e.g. it fails to
821    /// record the rollback, or the manager stops) drops the candidate registration in `context`
822    /// before the region leaves the tracker, so another migration cannot start meanwhile.
823    context: Context,
824    _guards: Vec<RegionMigrationProcedureGuard>,
825}
826
827impl RegionMigrationProcedure {
828    const TYPE_NAME: &'static str = "metasrv-procedure::RegionMigration";
829
830    pub fn new(
831        persistent_context: PersistentContext,
832        context_factory: impl ContextFactory,
833        guards: Vec<RegionMigrationProcedureGuard>,
834    ) -> Self {
835        let state = Box::new(RegionMigrationStart {});
836        Self::new_inner(state, persistent_context, context_factory, guards)
837    }
838
839    fn new_inner(
840        state: Box<dyn State>,
841        persistent_context: PersistentContext,
842        context_factory: impl ContextFactory,
843        guards: Vec<RegionMigrationProcedureGuard>,
844    ) -> Self {
845        Self {
846            state,
847            context: context_factory.new_context(persistent_context),
848            _guards: guards,
849        }
850    }
851
852    fn from_json(
853        json: &str,
854        context_factory: impl ContextFactory,
855        tracker: RegionMigrationProcedureTracker,
856    ) -> ProcedureResult<Self> {
857        // The routes may have changed since the state was persisted, so the procedure checks
858        // them again from the start.
859        let RegionMigrationDataOwned { persistent_ctx } =
860            serde_json::from_str(json).context(FromJsonSnafu)?;
861        let guards = persistent_ctx
862            .region_ids
863            .iter()
864            .flat_map(|region_id| {
865                tracker.insert_running_procedure(&RegionMigrationProcedureTask {
866                    region_id: *region_id,
867                    from_peer: persistent_ctx.from_peer.clone(),
868                    to_peer: persistent_ctx.to_peer.clone(),
869                    timeout: persistent_ctx.timeout,
870                    trigger_reason: persistent_ctx.trigger_reason,
871                })
872            })
873            .collect::<Vec<_>>();
874
875        let context = context_factory.new_context(persistent_ctx);
876
877        Ok(Self {
878            state: Box::new(RegionMigrationStart),
879            context,
880            _guards: guards,
881        })
882    }
883
884    async fn rollback_inner(&mut self, procedure_ctx: &ProcedureContext) -> Result<()> {
885        let _timer = METRIC_META_REGION_MIGRATION_EXECUTE
886            .with_label_values(&["rollback"])
887            .start_timer();
888        // Before the source can lead again, the target leaves the candidate registration.
889        self.context.volatile_ctx.opening_region_guards.clear();
890        let ctx = &self.context;
891        let to_peer_id = ctx.persistent_ctx.to_peer.id;
892        let table_regions = ctx.persistent_ctx.table_regions();
893        // Regions whose route was switched to the target before the procedure aborted.
894        let mut switched_regions = Vec::new();
895        let mut cleared_downgrading = false;
896        let mut failure = None;
897        for (table_id, regions) in table_regions {
898            let table_lock = TableLock::Write(table_id).into();
899            let _guard = procedure_ctx.provider.acquire_lock(&table_lock).await;
900            let table_route = match ctx.get_table_route_value(table_id).await {
901                Ok(table_route) => table_route,
902                // The table has been dropped meanwhile.
903                Err(error::Error::TableRouteNotFound { .. }) => continue,
904                Err(err) => {
905                    failure = Some(err);
906                    break;
907                }
908            };
909            let region_routes = table_route.region_routes().unwrap();
910            switched_regions.extend(
911                region_routes
912                    .iter()
913                    .filter(|route| regions.contains(&route.region.id))
914                    .filter(|route| {
915                        route.leader_peer.as_ref().map(|peer| peer.id) == Some(to_peer_id)
916                    })
917                    .map(|route| route.region.id),
918            );
919            let downgraded = region_routes
920                .iter()
921                .filter(|route| regions.contains(&route.region.id))
922                .any(|route| route.is_leader_downgrading());
923            if downgraded {
924                info!(
925                    "Rollbacking downgraded region leader table route, table: {table_id}, regions: {regions:?}"
926                );
927                let table_metadata_manager = &ctx.table_metadata_manager;
928                let cleared = table_metadata_manager
929                    .update_leader_region_status(table_id, &table_route, |route| {
930                        if regions.contains(&route.region.id) {
931                            Some(None)
932                        } else {
933                            None
934                        }
935                    })
936                    .await
937                    .context(error::TableMetadataManagerSnafu)
938                    .map_err(BoxedError::new)
939                    .with_context(|_| error::RetryLaterWithSourceSnafu {
940                        reason: format!("Failed to update the table route during the rollback downgraded leader region: {regions:?}"),
941                    });
942                if let Err(err) = cleared {
943                    failure = Some(err);
944                    break;
945                }
946                cleared_downgrading = true;
947            }
948        }
949        // Clients may have cached the routes that carried the cleared marks, also when a later
950        // table failed: a retry finds those marks gone and clears nothing.
951        if cleared_downgrading {
952            ctx.invalidate_table_cache().await?;
953        }
954        if let Some(err) = failure {
955            return Err(err);
956        }
957        self.context
958            .deregister_failure_detectors_for_candidate_regions()
959            .await;
960        // The switched regions are served by the target, so they are monitored there instead.
961        let unswitched_regions = ctx
962            .persistent_ctx
963            .region_ids
964            .iter()
965            .filter(|region_id| !switched_regions.contains(region_id))
966            .copied()
967            .collect::<Vec<_>>();
968        self.context
969            .register_failure_detectors(&unswitched_regions)
970            .await;
971        // The target's next heartbeat would register these detectors again, but nothing would if
972        // the target stopped before sending one.
973        if !switched_regions.is_empty() {
974            self.context
975                .reset_failure_detectors(&switched_regions)
976                .await;
977        }
978
979        Ok(())
980    }
981}
982
983#[async_trait::async_trait]
984impl Procedure for RegionMigrationProcedure {
985    fn type_name(&self) -> &str {
986        Self::TYPE_NAME
987    }
988
989    async fn rollback(&mut self, ctx: &ProcedureContext) -> ProcedureResult<()> {
990        self.rollback_inner(ctx)
991            .await
992            .map_err(ProcedureError::external)
993    }
994
995    fn rollback_supported(&self) -> bool {
996        true
997    }
998
999    #[tracing::instrument(skip_all, fields(
1000        state = %self.state.name(),
1001        region_count = self.context.persistent_ctx.region_ids.len(),
1002        from_peer = self.context.persistent_ctx.from_peer.id,
1003        to_peer = self.context.persistent_ctx.to_peer.id,
1004    ))]
1005    async fn execute(&mut self, ctx: &ProcedureContext) -> ProcedureResult<Status> {
1006        let state = &mut self.state;
1007
1008        let name = state.name();
1009        let _timer = METRIC_META_REGION_MIGRATION_EXECUTE
1010            .with_label_values(&[name])
1011            .start_timer();
1012        match state.next(&mut self.context, ctx).await {
1013            Ok((next, status)) => {
1014                *state = next;
1015                Ok(status)
1016            }
1017            Err(e) => {
1018                if e.is_retryable() {
1019                    METRIC_META_REGION_MIGRATION_ERROR
1020                        .with_label_values(&[name, "retryable"])
1021                        .inc();
1022                    Err(ProcedureError::retry_later(e))
1023                } else {
1024                    // Consumes the opening region guard before deregistering the failure detectors.
1025                    self.context.volatile_ctx.opening_region_guards.clear();
1026                    self.context
1027                        .deregister_failure_detectors_for_candidate_regions()
1028                        .await;
1029                    error!(
1030                        e;
1031                        "Region migration procedure failed, regions: {:?}, from_peer: {}, to_peer: {}, {}",
1032                        self.context.persistent_ctx.region_ids,
1033                        self.context.persistent_ctx.from_peer,
1034                        self.context.persistent_ctx.to_peer,
1035                        self.context.volatile_ctx.metrics,
1036                    );
1037                    METRIC_META_REGION_MIGRATION_ERROR
1038                        .with_label_values(&[name, "external"])
1039                        .inc();
1040                    Err(ProcedureError::external(e))
1041                }
1042            }
1043        }
1044    }
1045
1046    fn dump(&self) -> ProcedureResult<String> {
1047        let data = RegionMigrationData {
1048            state: self.state.as_ref(),
1049            persistent_ctx: &self.context.persistent_ctx,
1050        };
1051        serde_json::to_string(&data).context(ToJsonSnafu)
1052    }
1053
1054    fn lock_key(&self) -> LockKey {
1055        LockKey::new(self.context.persistent_ctx.lock_key())
1056    }
1057
1058    fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
1059        if !ctx.event_type_filter.allows(REGION_MIGRATION_EVENT_TYPE) {
1060            return None;
1061        }
1062
1063        Some(Box::new(RegionMigrationEvent::from_persistent_ctx(
1064            &self.context.persistent_ctx,
1065            self.context
1066                .trigger_reason(ctx.event_context)
1067                .to_trigger_reason(),
1068        )))
1069    }
1070}
1071
1072#[cfg(test)]
1073mod tests {
1074    use std::assert_matches;
1075    use std::sync::Arc;
1076
1077    use common_meta::distributed_time_constants::default_distributed_time_constants;
1078    use common_meta::instruction::Instruction;
1079    use common_meta::key::MetadataKey;
1080    use common_meta::key::table_route::TableRouteKey;
1081    use common_meta::key::test_utils::{new_test_table_info, new_test_table_info_with_name};
1082    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
1083    use common_meta::rpc::store::PutRequest;
1084
1085    use super::*;
1086    use crate::handler::HeartbeatMailbox;
1087    use crate::procedure::region_migration::open_candidate_region::OpenCandidateRegion;
1088    use crate::procedure::region_migration::test_util::*;
1089    use crate::procedure::test_util::{
1090        new_downgrade_region_reply, new_flush_region_reply_for_region, new_open_region_reply,
1091        new_upgrade_region_reply,
1092    };
1093    use crate::region::supervisor::{Event, RegionFailureDetectorControl};
1094    use crate::service::mailbox::Channel;
1095
1096    fn new_persistent_context() -> PersistentContext {
1097        test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
1098    }
1099
1100    #[tokio::test]
1101    async fn test_rollback_broadcasts_cleared_marks_before_a_later_table_fails() {
1102        // The rollback visits the tables of a batch in the order of a `HashMap` with a random
1103        // hasher, so the attempt is repeated until the table that fails comes after the one
1104        // whose mark is cleared; all 64 attempts miss with probability 2^-64.
1105        let cleared = RegionId::new(1024, 1);
1106        let failing = RegionId::new(1025, 1);
1107        for _ in 0..64 {
1108            let mut env = TestingEnv::new();
1109            for region_id in [cleared, failing] {
1110                let table_id = region_id.table_id();
1111                let table_info =
1112                    new_test_table_info_with_name(table_id, &format!("table_{table_id}"));
1113                let region_routes = vec![RegionRoute {
1114                    region: Region::new_test(region_id),
1115                    leader_peer: Some(Peer::empty(1)),
1116                    leader_state: (region_id == cleared).then_some(LeaderState::Downgrading),
1117                    ..Default::default()
1118                }];
1119                env.create_physical_table_metadata(table_info, region_routes)
1120                    .await;
1121            }
1122            // The route of the second table cannot be read.
1123            let route_key = TableRouteKey::new(failing.table_id()).to_bytes();
1124            let kv_backend = env.table_metadata_manager().kv_backend().clone();
1125            kv_backend
1126                .put(
1127                    PutRequest::new()
1128                        .with_key(route_key.clone())
1129                        .with_value(b"x".to_vec()),
1130                )
1131                .await
1132                .unwrap();
1133            let mut persistent_ctx = new_persistent_context();
1134            persistent_ctx.region_ids = vec![cleared, failing];
1135            let mut procedure =
1136                RegionMigrationProcedure::new(persistent_ctx, env.context_factory(), vec![]);
1137            let mut rx = env.frontend_receiver().await;
1138
1139            procedure
1140                .rollback_inner(&new_procedure_context())
1141                .await
1142                .unwrap_err();
1143
1144            let table_route = env
1145                .table_metadata_manager()
1146                .table_route_manager()
1147                .table_route_storage()
1148                .get(cleared.table_id())
1149                .await
1150                .unwrap()
1151                .unwrap();
1152            if table_route.region_routes().unwrap()[0]
1153                .leader_state
1154                .is_some()
1155            {
1156                // The failing table came first and nothing was cleared.
1157                assert!(rx.try_recv().is_err());
1158                continue;
1159            }
1160            assert_table_cache_invalidated(&mut rx, &[1024, 1025]);
1161            // The retry clears nothing and broadcasts nothing.
1162            kv_backend.delete(&route_key, false).await.unwrap();
1163            procedure
1164                .rollback_inner(&new_procedure_context())
1165                .await
1166                .unwrap();
1167            assert!(rx.try_recv().is_err());
1168            return;
1169        }
1170        panic!("In 64 attempts the failing table never came after the cleared one");
1171    }
1172
1173    #[tokio::test]
1174    async fn test_rollback_registers_detectors_for_switched_regions() {
1175        let env = TestingEnv::new();
1176        // The route switch aborted after the table of `switched` was switched to the target and
1177        // before the table of `unswitched`, whose leader is still downgrading; the table of
1178        // `dropped` has been dropped.
1179        let switched = RegionId::new(1024, 1);
1180        let unswitched = RegionId::new(1025, 1);
1181        let dropped = RegionId::new(1026, 1);
1182        for (region_id, leader) in [(switched, 2), (unswitched, 1)] {
1183            let table_id = region_id.table_id();
1184            let table_info = new_test_table_info_with_name(table_id, &format!("table_{table_id}"));
1185            let region_routes = vec![RegionRoute {
1186                region: Region::new_test(region_id),
1187                leader_peer: Some(Peer::empty(leader)),
1188                leader_state: (region_id == unswitched).then_some(LeaderState::Downgrading),
1189                ..Default::default()
1190            }];
1191            env.create_physical_table_metadata(table_info, region_routes)
1192                .await;
1193        }
1194        let mut persistent_ctx = new_persistent_context();
1195        persistent_ctx.region_ids = vec![switched, unswitched, dropped];
1196        let (tx, mut rx) = tokio::sync::mpsc::channel(8);
1197        let mut context_factory = env.context_factory();
1198        context_factory.region_failure_detector_controller =
1199            Arc::new(RegionFailureDetectorControl::new(tx));
1200        let mut procedure = RegionMigrationProcedure::new(persistent_ctx, context_factory, vec![]);
1201
1202        procedure
1203            .rollback_inner(&new_procedure_context())
1204            .await
1205            .unwrap();
1206
1207        // The target's detectors are replaced by the source's, except for `switched`, which stays
1208        // monitored on the target.
1209        let target = [switched, unswitched, dropped].map(|region_id| (2, region_id));
1210        let source = [unswitched, dropped].map(|region_id| (1, region_id));
1211        assert_matches!(rx.try_recv(), Ok(Event::DeregisterFailureDetectors(r)) if r == target);
1212        assert_matches!(rx.try_recv(), Ok(Event::RegisterFailureDetectors(r)) if r == source);
1213        assert_matches!(rx.try_recv(), Ok(Event::ResetFailureDetectors(r)) if r == [(2, switched)]);
1214        assert!(rx.try_recv().is_err());
1215        let table_route = env
1216            .table_metadata_manager()
1217            .table_route_manager()
1218            .table_route_storage()
1219            .get(unswitched.table_id())
1220            .await
1221            .unwrap()
1222            .unwrap();
1223        assert_eq!(table_route.region_routes().unwrap()[0].leader_state, None);
1224    }
1225
1226    #[tokio::test]
1227    async fn test_rollback_drops_candidate_registration_before_route_write() {
1228        let env = TestingEnv::new();
1229        let persistent_ctx = new_persistent_context();
1230        let rollback = CandidateRollback::new(&env, &persistent_ctx).await;
1231        let mut procedure =
1232            RegionMigrationProcedure::new(persistent_ctx, rollback.context_factory, vec![]);
1233        procedure
1234            .context
1235            .volatile_ctx
1236            .opening_region_guards
1237            .push(rollback.guard);
1238
1239        procedure
1240            .rollback_inner(&new_procedure_context())
1241            .await
1242            .unwrap();
1243
1244        assert_registration_dropped_before_route_write(&rollback.keeper_empty_at_txn);
1245    }
1246
1247    #[test]
1248    fn test_lock_key() {
1249        let persistent_context = new_persistent_context();
1250        let expected_keys = persistent_context.lock_key();
1251
1252        let env = TestingEnv::new();
1253        let context = env.context_factory();
1254
1255        let procedure = RegionMigrationProcedure::new(persistent_context, context, vec![]);
1256
1257        let key = procedure.lock_key();
1258        let keys = key.keys_to_lock().cloned().collect::<Vec<_>>();
1259
1260        for key in expected_keys {
1261            assert!(keys.contains(&key));
1262        }
1263    }
1264
1265    #[test]
1266    fn test_data_serialization() {
1267        let persistent_context = new_persistent_context();
1268
1269        let env = TestingEnv::new();
1270        let context = env.context_factory();
1271
1272        let procedure = RegionMigrationProcedure::new(persistent_context, context, vec![]);
1273
1274        let serialized = procedure.dump().unwrap();
1275        let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
1276        assert_eq!(expected, serialized);
1277    }
1278
1279    #[test]
1280    fn test_event_hook_uses_current_persistent_context() {
1281        let env = TestingEnv::new();
1282        let procedure =
1283            RegionMigrationProcedure::new(new_persistent_context(), env.context_factory(), vec![]);
1284        let state = common_procedure::ProcedureState::Running;
1285        let triggers = [
1286            common_procedure::EventTrigger::Recovered,
1287            common_procedure::EventTrigger::Retrying {
1288                phase: common_procedure::RetryPhase::Execute,
1289                attempt: 1,
1290            },
1291            common_procedure::EventTrigger::RollingBack,
1292            common_procedure::EventTrigger::Succeeded,
1293            common_procedure::EventTrigger::Failed,
1294            common_procedure::EventTrigger::Poisoned,
1295        ];
1296
1297        for trigger in triggers {
1298            let event = procedure
1299                .event(&EventContext {
1300                    procedure_id: common_procedure::ProcedureId::random(),
1301                    lifecycle_state: &state,
1302                    trigger,
1303                    event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
1304                    event_context: None,
1305                })
1306                .unwrap();
1307            assert_eq!(event.event_type(), "region_migration");
1308            assert_eq!(event.extra_rows().unwrap().len(), 1);
1309        }
1310    }
1311
1312    #[test]
1313    fn test_backward_compatibility() {
1314        let persistent_ctx = PersistentContext {
1315            #[allow(deprecated)]
1316            catalog: Some("greptime".into()),
1317            #[allow(deprecated)]
1318            schema: Some("public".into()),
1319            catalog_and_schema: vec![],
1320            from_peer: Peer::empty(1),
1321            to_peer: Peer::empty(2),
1322            region_ids: vec![RegionId::new(1024, 1)],
1323            timeout: Duration::from_secs(10),
1324            trigger_reason: RegionMigrationTriggerReason::Unknown,
1325        };
1326        // NOTES: Changes it will break backward compatibility.
1327        let serialized = r#"{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105}"#;
1328        let deserialized: PersistentContext = serde_json::from_str(serialized).unwrap();
1329
1330        assert_eq!(persistent_ctx, deserialized);
1331    }
1332
1333    #[test]
1334    fn test_legacy_trigger_reason_survives_recovery_and_repersistence() {
1335        let serialized = r#"{"persistent_ctx":{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105,"trigger_reason":"Failover"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
1336        let env = TestingEnv::new();
1337        let procedure = RegionMigrationProcedure::from_json(
1338            serialized,
1339            env.context_factory(),
1340            RegionMigrationProcedureTracker::default(),
1341        )
1342        .unwrap();
1343        assert_eq!(
1344            procedure.context.persistent_ctx.trigger_reason,
1345            RegionMigrationTriggerReason::Failover
1346        );
1347        let repersisted = procedure.dump().unwrap();
1348        assert_eq!(
1349            serde_json::from_str::<serde_json::Value>(&repersisted).unwrap()["persistent_ctx"]["trigger_reason"],
1350            "Failover"
1351        );
1352
1353        let recovered = RegionMigrationProcedure::from_json(
1354            &repersisted,
1355            env.context_factory(),
1356            RegionMigrationProcedureTracker::default(),
1357        )
1358        .unwrap();
1359        assert_eq!(
1360            recovered.context.trigger_reason(None),
1361            RegionMigrationTriggerReason::Failover
1362        );
1363        let event_context = PersistentEventContext::new(TriggerReason::AutoRebalance);
1364        assert_eq!(
1365            recovered.context.trigger_reason(Some(&event_context)),
1366            RegionMigrationTriggerReason::AutoRebalance
1367        );
1368
1369        let state = common_procedure::ProcedureState::Running;
1370        let event = recovered
1371            .event(&EventContext {
1372                procedure_id: common_procedure::ProcedureId::random(),
1373                lifecycle_state: &state,
1374                trigger: common_procedure::EventTrigger::Recovered,
1375                event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
1376                event_context: None,
1377            })
1378            .unwrap();
1379        assert_eq!(
1380            event.extra_rows().unwrap()[0].values[3].value_data,
1381            Some(api::v1::value::ValueData::StringValue(
1382                "Failover".to_string()
1383            ))
1384        );
1385    }
1386
1387    #[test]
1388    fn test_migration_reason_uses_event_context() {
1389        let env = TestingEnv::new();
1390        let context = env.context_factory().new_context(new_persistent_context());
1391        let event_context = PersistentEventContext::new(TriggerReason::RegionFailover);
1392
1393        assert_eq!(
1394            context.trigger_reason(Some(&event_context)),
1395            RegionMigrationTriggerReason::Failover
1396        );
1397
1398        let procedure =
1399            RegionMigrationProcedure::new(new_persistent_context(), env.context_factory(), vec![]);
1400        let state = common_procedure::ProcedureState::Running;
1401        let event = procedure
1402            .event(&EventContext {
1403                procedure_id: common_procedure::ProcedureId::random(),
1404                lifecycle_state: &state,
1405                trigger: common_procedure::EventTrigger::Submitted,
1406                event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
1407                event_context: Some(&event_context),
1408            })
1409            .unwrap();
1410        assert_eq!(
1411            event.extra_rows().unwrap()[0].values[3].value_data,
1412            Some(api::v1::value::ValueData::StringValue(
1413                "Failover".to_string()
1414            ))
1415        );
1416    }
1417
1418    /// Returns the procedure recovered from the dump of a migration that was persisted at
1419    /// [OpenCandidateRegion].
1420    fn recover_procedure(
1421        env: &TestingEnv,
1422        persistent_ctx: PersistentContext,
1423    ) -> RegionMigrationProcedure {
1424        let persisted = RegionMigrationProcedure::new_inner(
1425            Box::new(OpenCandidateRegion),
1426            persistent_ctx,
1427            env.context_factory(),
1428            vec![],
1429        )
1430        .dump()
1431        .unwrap();
1432        RegionMigrationProcedure::from_json(&persisted, env.context_factory(), env.tracker())
1433            .unwrap()
1434    }
1435
1436    #[tokio::test]
1437    async fn test_recovered_procedure_ends_if_regions_migrated() {
1438        let mut env = TestingEnv::new();
1439        let persistent_ctx = new_persistent_context();
1440        let region_id = persistent_ctx.region_ids[0];
1441        // The procedure stopped after it switched the route to the target.
1442        let region_routes = vec![RegionRoute {
1443            region: Region::new_test(region_id),
1444            leader_peer: Some(persistent_ctx.to_peer.clone()),
1445            ..Default::default()
1446        }];
1447        env.create_physical_table_metadata(new_test_table_info(1024), region_routes)
1448            .await;
1449        let mut frontend_rx = env.frontend_receiver().await;
1450        let (datanode_tx, mut datanode_rx) = tokio::sync::mpsc::channel(1);
1451        env.mailbox_context()
1452            .insert_heartbeat_response_receiver(Channel::Datanode(2), datanode_tx)
1453            .await;
1454
1455        let mut procedure = recover_procedure(&env, persistent_ctx);
1456        assert!(env.tracker().contains(region_id));
1457        let status = procedure
1458            .execute(&TestingEnv::procedure_context())
1459            .await
1460            .unwrap();
1461
1462        assert!(status.is_done());
1463        assert_table_cache_invalidated(&mut frontend_rx, &[1024]);
1464        // The target receives the broadcast and is not instructed again.
1465        assert_table_cache_invalidated(&mut datanode_rx, &[1024]);
1466        assert!(datanode_rx.try_recv().is_err());
1467    }
1468
1469    #[tokio::test]
1470    async fn test_recovered_procedure_rolls_back_if_leader_moved() {
1471        // The procedure stopped after it downgraded the leaders, then the leader of `moved`
1472        // changed to neither the source nor the target. The other region keeps the downgrading
1473        // mark, on the source or, in a state the route switch does not produce, on the target.
1474        for downgraded_leader in [1, 2] {
1475            let mut env = TestingEnv::new();
1476            let mut persistent_ctx = new_persistent_context();
1477            persistent_ctx.region_ids = env.create_batch_with_moved_leader(downgraded_leader).await;
1478            let mut rx = env.frontend_receiver().await;
1479
1480            let mut procedure = recover_procedure(&env, persistent_ctx);
1481            let ctx = TestingEnv::procedure_context();
1482            let status = procedure.execute(&ctx).await.unwrap();
1483            assert!(!status.is_done());
1484            let err = procedure.execute(&ctx).await.unwrap_err();
1485            assert!(!err.is_retry_later());
1486            while rx.try_recv().is_ok() {}
1487            procedure.rollback(&ctx).await.unwrap();
1488            // The routes no longer carry the marks that clients may have cached.
1489            assert_table_cache_invalidated(&mut rx, &[1024]);
1490
1491            let table_route = env
1492                .table_metadata_manager()
1493                .table_route_manager()
1494                .table_route_storage()
1495                .get(1024)
1496                .await
1497                .unwrap()
1498                .unwrap();
1499            for region_route in table_route.region_routes().unwrap() {
1500                assert_eq!(region_route.leader_state, None);
1501            }
1502        }
1503    }
1504
1505    #[tokio::test]
1506    async fn test_recovered_procedure_ignores_persisted_state() {
1507        let env = TestingEnv::new();
1508        let persisted = RegionMigrationProcedure::new_inner(
1509            Box::new(OpenCandidateRegion),
1510            new_persistent_context(),
1511            env.context_factory(),
1512            vec![],
1513        )
1514        .dump()
1515        .unwrap();
1516        // A state this version cannot parse.
1517        let unknown_state = persisted.replace("OpenCandidateRegion", "UnknownState");
1518        assert_ne!(unknown_state, persisted);
1519
1520        let procedure = RegionMigrationProcedure::from_json(
1521            &unknown_state,
1522            env.context_factory(),
1523            env.tracker(),
1524        )
1525        .unwrap();
1526
1527        assert!(procedure.state.as_any().is::<RegionMigrationStart>());
1528    }
1529
1530    #[tokio::test]
1531    async fn test_broadcast_invalidate_table_cache() {
1532        let mut env = TestingEnv::new();
1533        let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1534        let ctx = env.context_factory().new_context(persistent_context);
1535        let mailbox_ctx = env.mailbox_context();
1536
1537        // No receivers.
1538        ctx.invalidate_table_cache().await.unwrap();
1539
1540        let (tx, mut rx) = tokio::sync::mpsc::channel(1);
1541
1542        mailbox_ctx
1543            .insert_heartbeat_response_receiver(Channel::Frontend(1), tx)
1544            .await;
1545
1546        ctx.invalidate_table_cache().await.unwrap();
1547
1548        let resp = rx.recv().await.unwrap().unwrap();
1549        let msg = resp.mailbox_message.unwrap();
1550
1551        let instruction = HeartbeatMailbox::json_instruction(&msg).unwrap();
1552        assert_eq!(
1553            instruction,
1554            Instruction::InvalidateCaches(vec![CacheIdent::TableId(1024)])
1555        );
1556    }
1557
1558    fn procedure_flow_steps(from_peer_id: u64, to_peer_id: u64) -> Vec<Step> {
1559        vec![
1560            // MigrationStart
1561            Step::next(
1562                "Should be the open candidate region",
1563                None,
1564                Assertion::simple(assert_open_candidate_region, assert_need_persist),
1565            ),
1566            // OpenCandidateRegion
1567            Step::next(
1568                "Should be the flush leader region",
1569                Some(mock_datanode_reply(
1570                    to_peer_id,
1571                    Arc::new(|id| Ok(new_open_region_reply(id, true, None))),
1572                )),
1573                Assertion::simple(assert_flush_leader_region, assert_no_persist),
1574            ),
1575            // Flush Leader Region
1576            Step::next(
1577                "Should be the flush leader region",
1578                Some(mock_datanode_reply(
1579                    from_peer_id,
1580                    Arc::new(move |id| {
1581                        Ok(new_flush_region_reply_for_region(
1582                            id,
1583                            RegionId::new(1024, 1),
1584                            true,
1585                            None,
1586                        ))
1587                    }),
1588                )),
1589                Assertion::simple(assert_update_metadata_downgrade, assert_no_persist),
1590            ),
1591            // UpdateMetadata::Downgrade
1592            Step::next(
1593                "Should be the downgrade leader region",
1594                None,
1595                Assertion::simple(assert_downgrade_leader_region, assert_no_persist),
1596            ),
1597            // Downgrade Candidate
1598            Step::next(
1599                "Should be the upgrade candidate region",
1600                Some(mock_datanode_reply(
1601                    from_peer_id,
1602                    Arc::new(|id| Ok(new_downgrade_region_reply(id, None, true, None))),
1603                )),
1604                Assertion::simple(assert_upgrade_candidate_region, assert_no_persist),
1605            ),
1606            // Upgrade Candidate
1607            Step::next(
1608                "Should be the update metadata for upgrading",
1609                Some(mock_datanode_reply(
1610                    to_peer_id,
1611                    Arc::new(|id| Ok(new_upgrade_region_reply(id, true, true, None))),
1612                )),
1613                Assertion::simple(assert_update_metadata_upgrade, assert_no_persist),
1614            ),
1615            // UpdateMetadata::Upgrade
1616            Step::next(
1617                "Should be the close downgraded region",
1618                None,
1619                Assertion::simple(assert_close_downgraded_region, assert_no_persist),
1620            ),
1621            // CloseDowngradedRegion
1622            Step::next(
1623                "Should be the region migration end",
1624                None,
1625                Assertion::simple(assert_region_migration_end, assert_done),
1626            ),
1627            // RegionMigrationEnd
1628            Step::next(
1629                "Should be the region migration end again",
1630                None,
1631                Assertion::simple(assert_region_migration_end, assert_done),
1632            ),
1633        ]
1634    }
1635
1636    #[tokio::test]
1637    async fn test_procedure_flow() {
1638        common_telemetry::init_default_ut_logging();
1639
1640        let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1641        let state = Box::new(RegionMigrationStart);
1642
1643        // The table metadata.
1644        let from_peer_id = persistent_context.from_peer.id;
1645        let to_peer_id = persistent_context.to_peer.id;
1646        let from_peer = persistent_context.from_peer.clone();
1647        let to_peer = persistent_context.to_peer.clone();
1648        let region_id = persistent_context.region_ids[0];
1649        let table_info = new_test_table_info(1024);
1650        let region_routes = vec![RegionRoute {
1651            region: Region::new_test(region_id),
1652            leader_peer: Some(from_peer),
1653            follower_peers: vec![to_peer],
1654            ..Default::default()
1655        }];
1656
1657        let suite = ProcedureMigrationTestSuite::new(persistent_context, state);
1658        suite.init_table_metadata(table_info, region_routes).await;
1659
1660        let steps = procedure_flow_steps(from_peer_id, to_peer_id);
1661        let timer = Instant::now();
1662
1663        // Run the table tests.
1664        let runner = ProcedureMigrationSuiteRunner::new(suite)
1665            .steps(steps)
1666            .run_once()
1667            .await;
1668
1669        let region_lease = default_distributed_time_constants().region_lease.as_secs();
1670
1671        // Ensure it didn't run into the slow path.
1672        assert!(timer.elapsed().as_secs() < region_lease / 2);
1673
1674        runner.suite.verify_table_metadata().await;
1675    }
1676
1677    #[tokio::test]
1678    async fn test_procedure_flow_open_candidate_region_retryable_error() {
1679        common_telemetry::init_default_ut_logging();
1680
1681        let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1682        let state = Box::new(RegionMigrationStart);
1683
1684        // The table metadata.
1685        let to_peer_id = persistent_context.to_peer.id;
1686        let from_peer = persistent_context.from_peer.clone();
1687        let region_id = persistent_context.region_ids[0];
1688        let table_info = new_test_table_info(1024);
1689        let region_routes = vec![RegionRoute {
1690            region: Region::new_test(region_id),
1691            leader_peer: Some(from_peer),
1692            follower_peers: vec![],
1693            ..Default::default()
1694        }];
1695
1696        let suite = ProcedureMigrationTestSuite::new(persistent_context, state);
1697        suite.init_table_metadata(table_info, region_routes).await;
1698
1699        let steps = vec![
1700            // Migration Start
1701            Step::next(
1702                "Should be the open candidate region",
1703                None,
1704                Assertion::simple(assert_open_candidate_region, assert_need_persist),
1705            ),
1706            // OpenCandidateRegion
1707            Step::next(
1708                "Should be throwing a non-retry error",
1709                Some(mock_datanode_reply(
1710                    to_peer_id,
1711                    Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1712                )),
1713                Assertion::error(|error| assert!(error.is_retryable())),
1714            ),
1715            // OpenCandidateRegion
1716            Step::next(
1717                "Should be throwing a non-retry error again",
1718                Some(mock_datanode_reply(
1719                    to_peer_id,
1720                    Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1721                )),
1722                Assertion::error(|error| assert!(error.is_retryable())),
1723            ),
1724        ];
1725
1726        let setup_to_latest_persisted_state = Step::setup(
1727            "Sets state to UpdateMetadata::Downgrade",
1728            merge_before_test_fn(vec![
1729                setup_state(Arc::new(|| Box::new(OpenCandidateRegion))),
1730                Arc::new(reset_volatile_ctx),
1731            ]),
1732        );
1733
1734        let steps = [
1735            steps.clone(),
1736            // Mocks the volatile ctx lost(i.g., Meta leader restarts).
1737            vec![setup_to_latest_persisted_state.clone()],
1738            steps.clone()[1..].to_vec(),
1739            vec![setup_to_latest_persisted_state],
1740            steps.clone()[1..].to_vec(),
1741        ]
1742        .concat();
1743
1744        // Run the table tests.
1745        let runner = ProcedureMigrationSuiteRunner::new(suite)
1746            .steps(steps.clone())
1747            .run_once()
1748            .await;
1749
1750        let table_routes_version = runner
1751            .env()
1752            .table_metadata_manager()
1753            .table_route_manager()
1754            .table_route_storage()
1755            .get(region_id.table_id())
1756            .await
1757            .unwrap()
1758            .unwrap()
1759            .version();
1760        // Should be unchanged.
1761        assert_eq!(table_routes_version.unwrap(), 0);
1762    }
1763
1764    #[tokio::test]
1765    async fn test_procedure_flow_upgrade_candidate_with_retry_and_failed() {
1766        common_telemetry::init_default_ut_logging();
1767
1768        let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1769        let state = Box::new(RegionMigrationStart);
1770
1771        // The table metadata.
1772        let from_peer_id = persistent_context.from_peer.id;
1773        let to_peer_id = persistent_context.to_peer.id;
1774        let from_peer = persistent_context.from_peer.clone();
1775        let region_id = persistent_context.region_ids[0];
1776        let table_info = new_test_table_info(1024);
1777        let region_routes = vec![RegionRoute {
1778            region: Region::new_test(region_id),
1779            leader_peer: Some(from_peer),
1780            follower_peers: vec![],
1781            ..Default::default()
1782        }];
1783
1784        let suite = ProcedureMigrationTestSuite::new(persistent_context, state);
1785        suite.init_table_metadata(table_info, region_routes).await;
1786
1787        let steps = vec![
1788            // MigrationStart
1789            Step::next(
1790                "Should be the open candidate region",
1791                None,
1792                Assertion::simple(assert_open_candidate_region, assert_need_persist),
1793            ),
1794            // OpenCandidateRegion
1795            Step::next(
1796                "Should be the flush leader region",
1797                Some(mock_datanode_reply(
1798                    to_peer_id,
1799                    Arc::new(|id| Ok(new_open_region_reply(id, true, None))),
1800                )),
1801                Assertion::simple(assert_flush_leader_region, assert_no_persist),
1802            ),
1803            // Flush Leader Region
1804            Step::next(
1805                "Should be the flush leader region",
1806                Some(mock_datanode_reply(
1807                    from_peer_id,
1808                    Arc::new(move |id| {
1809                        Ok(new_flush_region_reply_for_region(
1810                            id,
1811                            RegionId::new(1024, 1),
1812                            true,
1813                            None,
1814                        ))
1815                    }),
1816                )),
1817                Assertion::simple(assert_update_metadata_downgrade, assert_no_persist),
1818            ),
1819            // UpdateMetadata::Downgrade
1820            Step::next(
1821                "Should be the downgrade leader region",
1822                None,
1823                Assertion::simple(assert_downgrade_leader_region, assert_no_persist),
1824            ),
1825            // Downgrade Candidate
1826            Step::next(
1827                "Should be the upgrade candidate region",
1828                Some(mock_datanode_reply(
1829                    from_peer_id,
1830                    Arc::new(|id| Ok(new_downgrade_region_reply(id, None, true, None))),
1831                )),
1832                Assertion::simple(assert_upgrade_candidate_region, assert_no_persist),
1833            ),
1834            // Upgrade Candidate
1835            Step::next(
1836                "Should be the rollback metadata",
1837                Some(mock_datanode_reply(
1838                    to_peer_id,
1839                    Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1840                )),
1841                Assertion::simple(assert_update_metadata_rollback, assert_no_persist),
1842            ),
1843            // UpdateMetadata::Rollback
1844            Step::next(
1845                "Should be the region migration abort",
1846                None,
1847                Assertion::simple(assert_region_migration_abort, assert_no_persist),
1848            ),
1849            // RegionMigrationAbort
1850            Step::next(
1851                "Should throw an error",
1852                None,
1853                Assertion::error(|error| {
1854                    assert!(!error.is_retryable());
1855                    assert_matches!(error, error::Error::MigrationAbort { .. });
1856                }),
1857            ),
1858        ];
1859
1860        let setup_to_latest_persisted_state = Step::setup(
1861            "Sets state to OpenCandidateRegion",
1862            merge_before_test_fn(vec![
1863                setup_state(Arc::new(|| Box::new(OpenCandidateRegion))),
1864                Arc::new(reset_volatile_ctx),
1865            ]),
1866        );
1867
1868        let steps = [
1869            steps.clone(),
1870            vec![setup_to_latest_persisted_state.clone()],
1871            steps.clone()[1..].to_vec(),
1872            vec![setup_to_latest_persisted_state],
1873            steps.clone()[1..].to_vec(),
1874        ]
1875        .concat();
1876
1877        // Run the table tests.
1878        ProcedureMigrationSuiteRunner::new(suite)
1879            .steps(steps.clone())
1880            .run_once()
1881            .await;
1882    }
1883
1884    #[tokio::test]
1885    async fn test_procedure_flow_upgrade_candidate_with_retry() {
1886        common_telemetry::init_default_ut_logging();
1887
1888        let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1889        let state = Box::new(RegionMigrationStart);
1890
1891        // The table metadata.
1892        let to_peer_id = persistent_context.to_peer.id;
1893        let from_peer_id = persistent_context.from_peer.id;
1894        let from_peer = persistent_context.from_peer.clone();
1895        let region_id = persistent_context.region_ids[0];
1896        let table_info = new_test_table_info(1024);
1897        let region_routes = vec![RegionRoute {
1898            region: Region::new_test(region_id),
1899            leader_peer: Some(from_peer),
1900            follower_peers: vec![],
1901            ..Default::default()
1902        }];
1903
1904        let suite = ProcedureMigrationTestSuite::new(persistent_context, state);
1905        suite.init_table_metadata(table_info, region_routes).await;
1906
1907        let steps = vec![
1908            // Migration Start
1909            Step::next(
1910                "Should be the open candidate region",
1911                None,
1912                Assertion::simple(assert_open_candidate_region, assert_need_persist),
1913            ),
1914            // OpenCandidateRegion
1915            Step::next(
1916                "Should be throwing a retryable error",
1917                Some(mock_datanode_reply(
1918                    to_peer_id,
1919                    Arc::new(|id| {
1920                        Ok(new_open_region_reply(
1921                            id,
1922                            false,
1923                            Some("mock retryable open region error".to_string()),
1924                        ))
1925                    }),
1926                )),
1927                Assertion::error(|error| assert!(error.is_retryable(), "err: {error:?}")),
1928            ),
1929            // OpenCandidateRegion
1930            Step::next(
1931                "Should be the update metadata for downgrading",
1932                Some(mock_datanode_reply(
1933                    to_peer_id,
1934                    Arc::new(|id| Ok(new_open_region_reply(id, true, None))),
1935                )),
1936                Assertion::simple(assert_flush_leader_region, assert_no_persist),
1937            ),
1938            // Flush Leader Region
1939            Step::next(
1940                "Should be the flush leader region",
1941                Some(mock_datanode_reply(
1942                    from_peer_id,
1943                    Arc::new(move |id| {
1944                        Ok(new_flush_region_reply_for_region(id, region_id, true, None))
1945                    }),
1946                )),
1947                Assertion::simple(assert_update_metadata_downgrade, assert_no_persist),
1948            ),
1949            // UpdateMetadata::Downgrade
1950            Step::next(
1951                "Should be the downgrade leader region",
1952                None,
1953                Assertion::simple(assert_downgrade_leader_region, assert_no_persist),
1954            ),
1955            // Downgrade Leader
1956            Step::next(
1957                "Should be the upgrade candidate region",
1958                Some(mock_datanode_reply(
1959                    from_peer_id,
1960                    merge_mailbox_messages(vec![
1961                        Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1962                        Arc::new(|id| Ok(new_downgrade_region_reply(id, None, true, None))),
1963                    ]),
1964                )),
1965                Assertion::simple(assert_upgrade_candidate_region, assert_no_persist),
1966            ),
1967            // Upgrade Candidate
1968            Step::next(
1969                "Should be the update metadata for upgrading",
1970                Some(mock_datanode_reply(
1971                    to_peer_id,
1972                    merge_mailbox_messages(vec![
1973                        Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1974                        Arc::new(|id| Ok(new_upgrade_region_reply(id, true, true, None))),
1975                    ]),
1976                )),
1977                Assertion::simple(assert_update_metadata_upgrade, assert_no_persist),
1978            ),
1979            // UpdateMetadata::Upgrade
1980            Step::next(
1981                "Should be the close downgraded region",
1982                None,
1983                Assertion::simple(assert_close_downgraded_region, assert_no_persist),
1984            ),
1985            // CloseDowngradedRegion
1986            Step::next(
1987                "Should be the region migration end",
1988                None,
1989                Assertion::simple(assert_region_migration_end, assert_done),
1990            ),
1991            // RegionMigrationEnd
1992            Step::next(
1993                "Should be the region migration end again",
1994                None,
1995                Assertion::simple(assert_region_migration_end, assert_done),
1996            ),
1997            // RegionMigrationStart
1998            Step::setup(
1999                "Sets state to RegionMigrationStart",
2000                merge_before_test_fn(vec![
2001                    setup_state(Arc::new(|| Box::new(RegionMigrationStart))),
2002                    Arc::new(reset_volatile_ctx),
2003                ]),
2004            ),
2005            // RegionMigrationEnd
2006            // Note: We can't run this test multiple times;
2007            // the `peer_id`'s `DatanodeTable` will be removed after first-time migration success.
2008            Step::next(
2009                "Should be the region migration end(has been migrated)",
2010                None,
2011                Assertion::simple(assert_region_migration_end, assert_done),
2012            ),
2013        ];
2014
2015        let steps = [steps.clone()].concat();
2016        let timer = Instant::now();
2017
2018        // Run the table tests.
2019        let runner = ProcedureMigrationSuiteRunner::new(suite)
2020            .steps(steps.clone())
2021            .run_once()
2022            .await;
2023
2024        let region_lease = default_distributed_time_constants().region_lease.as_secs();
2025        // Ensure it didn't run into the slow path.
2026        assert!(timer.elapsed().as_secs() < region_lease);
2027        runner.suite.verify_table_metadata().await;
2028    }
2029}