1pub(crate) mod close_downgraded_region;
16pub(crate) mod downgrade_leader_region;
17pub(crate) mod flush_leader_region;
18pub(crate) mod manager;
19pub(crate) mod migration_abort;
20pub(crate) mod migration_end;
21pub(crate) mod migration_start;
22pub(crate) mod open_candidate_region;
23#[cfg(test)]
24pub mod test_util;
25pub(crate) mod update_metadata;
26pub(crate) mod upgrade_candidate_region;
27pub(crate) mod utils;
28
29use std::any::Any;
30use std::collections::{HashMap, HashSet};
31use std::fmt::{Debug, Display};
32use std::time::Duration;
33
34use common_error::ext::BoxedError;
35use common_event_recorder::{Event, PersistentEventContext};
36use common_meta::cache_invalidator::CacheInvalidatorRef;
37use common_meta::ddl::RegionFailureDetectorControllerRef;
38use common_meta::instruction::CacheIdent;
39use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue};
40use common_meta::key::table_route::TableRouteValue;
41use common_meta::key::topic_name::TopicNameKey;
42use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey};
43use common_meta::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
44use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
45use common_meta::lock_key::{CatalogLock, RegionLock, SchemaLock, TableLock};
46use common_meta::peer::Peer;
47use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard};
48use common_meta::rpc::ddl::TriggerReason;
49use common_procedure::error::{
50 Error as ProcedureError, FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu,
51};
52use common_procedure::{
53 Context as ProcedureContext, EventContext, LockKey, Procedure, Status, StringKey,
54};
55use common_telemetry::{debug, error, info};
56use manager::RegionMigrationProcedureGuard;
57pub use manager::{
58 RegionMigrationManagerRef, RegionMigrationProcedureTask, RegionMigrationProcedureTracker,
59 RegionMigrationTriggerReason,
60};
61use serde::{Deserialize, Deserializer, Serialize};
62use snafu::{OptionExt, ResultExt};
63use store_api::storage::{RegionId, TableId};
64use tokio::time::Instant;
65
66use self::migration_start::RegionMigrationStart;
67use crate::error::{self, Result};
68use crate::event::region_migration::{REGION_MIGRATION_EVENT_TYPE, RegionMigrationEvent};
69use crate::metrics::{
70 METRIC_META_REGION_MIGRATION_ERROR, METRIC_META_REGION_MIGRATION_EXECUTE,
71 METRIC_META_REGION_MIGRATION_STAGE_ELAPSED,
72};
73use crate::service::mailbox::MailboxRef;
74
75pub const DEFAULT_REGION_MIGRATION_TIMEOUT: Duration = Duration::from_secs(120);
77
78#[derive(Debug, Deserialize)]
79#[serde(untagged)]
80enum SingleOrMultiple<T> {
81 Single(T),
82 Multiple(Vec<T>),
83}
84
85fn single_or_multiple_from<'de, D, T>(deserializer: D) -> std::result::Result<Vec<T>, D::Error>
86where
87 D: Deserializer<'de>,
88 T: Deserialize<'de>,
89{
90 let helper = SingleOrMultiple::<T>::deserialize(deserializer)?;
91 Ok(match helper {
92 SingleOrMultiple::Single(x) => vec![x],
93 SingleOrMultiple::Multiple(xs) => xs,
94 })
95}
96
97#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
103pub struct PersistentContext {
104 #[deprecated(note = "use `catalog_and_schema` instead")]
106 #[serde(default, skip_serializing_if = "Option::is_none")]
107 pub(crate) catalog: Option<String>,
108 #[deprecated(note = "use `catalog_and_schema` instead")]
110 #[serde(default, skip_serializing_if = "Option::is_none")]
111 pub(crate) schema: Option<String>,
112 #[serde(default, skip_serializing_if = "Vec::is_empty")]
114 pub(crate) catalog_and_schema: Vec<(String, String)>,
115 pub(crate) from_peer: Peer,
117 pub(crate) to_peer: Peer,
119 #[serde(deserialize_with = "single_or_multiple_from", alias = "region_id")]
121 pub(crate) region_ids: Vec<RegionId>,
122 #[serde(with = "humantime_serde", default = "default_timeout")]
124 pub(crate) timeout: Duration,
125 #[serde(default)]
127 trigger_reason: RegionMigrationTriggerReason,
128}
129
130impl PersistentContext {
131 pub fn new(
132 catalog_and_schema: Vec<(String, String)>,
133 from_peer: Peer,
134 to_peer: Peer,
135 region_ids: Vec<RegionId>,
136 timeout: Duration,
137 trigger_reason: RegionMigrationTriggerReason,
138 ) -> Self {
139 #[allow(deprecated)]
140 Self {
141 catalog: None,
142 schema: None,
143 catalog_and_schema,
144 from_peer,
145 to_peer,
146 region_ids,
147 timeout,
148 trigger_reason,
149 }
150 }
151}
152
153impl RegionMigrationTriggerReason {
154 fn to_trigger_reason(self) -> TriggerReason {
155 match self {
156 Self::Manual => TriggerReason::Manual,
157 Self::AutoRebalance => TriggerReason::AutoRebalance,
158 Self::Failover => TriggerReason::RegionFailover,
159 Self::Unknown => TriggerReason::Unknown,
160 }
161 }
162
163 pub(crate) fn from_trigger_reason(reason: TriggerReason) -> Self {
164 match reason {
165 TriggerReason::Manual => Self::Manual,
166 TriggerReason::AutoRebalance => Self::AutoRebalance,
167 TriggerReason::RegionFailover => Self::Failover,
168 _ => Self::Unknown,
169 }
170 }
171}
172
173fn default_timeout() -> Duration {
174 Duration::from_secs(10)
175}
176
177impl PersistentContext {
178 pub fn lock_key(&self) -> Vec<StringKey> {
179 let mut lock_keys =
180 Vec::with_capacity(self.region_ids.len() + 2 + self.catalog_and_schema.len() * 2);
181 #[allow(deprecated)]
182 if let (Some(catalog), Some(schema)) = (&self.catalog, &self.schema) {
183 lock_keys.push(CatalogLock::Read(catalog).into());
184 lock_keys.push(SchemaLock::read(catalog, schema).into());
185 }
186 for (catalog, schema) in self.catalog_and_schema.iter() {
187 lock_keys.push(CatalogLock::Read(catalog).into());
188 lock_keys.push(SchemaLock::read(catalog, schema).into());
189 }
190
191 let mut region_ids = self.region_ids.clone();
193 region_ids.sort_unstable();
194 for region_id in region_ids {
195 lock_keys.push(RegionLock::Write(region_id).into());
196 }
197 lock_keys
198 }
199
200 pub fn region_table_ids(&self) -> Vec<TableId> {
204 self.region_ids
205 .iter()
206 .map(|region_id| region_id.table_id())
207 .collect::<HashSet<_>>()
208 .into_iter()
209 .collect()
210 }
211
212 pub fn table_regions(&self) -> HashMap<TableId, Vec<RegionId>> {
216 let mut table_regions = HashMap::new();
217 for region_id in &self.region_ids {
218 table_regions
219 .entry(region_id.table_id())
220 .or_insert_with(Vec::new)
221 .push(*region_id);
222 }
223 table_regions
224 }
225}
226
227#[derive(Debug, Clone, Default)]
229pub struct Metrics {
230 operations_elapsed: Duration,
232 flush_leader_region_elapsed: Duration,
234 downgrade_leader_region_elapsed: Duration,
236 open_candidate_region_elapsed: Duration,
238 upgrade_candidate_region_elapsed: Duration,
240}
241
242impl Display for Metrics {
243 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
244 let total = self.flush_leader_region_elapsed
245 + self.downgrade_leader_region_elapsed
246 + self.open_candidate_region_elapsed
247 + self.upgrade_candidate_region_elapsed;
248 write!(
249 f,
250 "total: {:?}, flush_leader_region_elapsed: {:?}, downgrade_leader_region_elapsed: {:?}, open_candidate_region_elapsed: {:?}, upgrade_candidate_region_elapsed: {:?}",
251 total,
252 self.flush_leader_region_elapsed,
253 self.downgrade_leader_region_elapsed,
254 self.open_candidate_region_elapsed,
255 self.upgrade_candidate_region_elapsed
256 )
257 }
258}
259
260impl Metrics {
261 pub fn update_operations_elapsed(&mut self, elapsed: Duration) {
263 self.operations_elapsed += elapsed;
264 }
265
266 pub fn update_flush_leader_region_elapsed(&mut self, elapsed: Duration) {
268 self.flush_leader_region_elapsed += elapsed;
269 }
270
271 pub fn update_downgrade_leader_region_elapsed(&mut self, elapsed: Duration) {
273 self.downgrade_leader_region_elapsed += elapsed;
274 }
275
276 pub fn update_open_candidate_region_elapsed(&mut self, elapsed: Duration) {
278 self.open_candidate_region_elapsed += elapsed;
279 }
280
281 pub fn update_upgrade_candidate_region_elapsed(&mut self, elapsed: Duration) {
283 self.upgrade_candidate_region_elapsed += elapsed;
284 }
285}
286
287impl Drop for Metrics {
288 fn drop(&mut self) {
289 let total = self.flush_leader_region_elapsed
290 + self.downgrade_leader_region_elapsed
291 + self.open_candidate_region_elapsed
292 + self.upgrade_candidate_region_elapsed;
293 METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
294 .with_label_values(&["total"])
295 .observe(total.as_secs_f64());
296
297 if !self.flush_leader_region_elapsed.is_zero() {
298 METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
299 .with_label_values(&["flush_leader_region"])
300 .observe(self.flush_leader_region_elapsed.as_secs_f64());
301 }
302
303 if !self.downgrade_leader_region_elapsed.is_zero() {
304 METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
305 .with_label_values(&["downgrade_leader_region"])
306 .observe(self.downgrade_leader_region_elapsed.as_secs_f64());
307 }
308
309 if !self.open_candidate_region_elapsed.is_zero() {
310 METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
311 .with_label_values(&["open_candidate_region"])
312 .observe(self.open_candidate_region_elapsed.as_secs_f64());
313 }
314
315 if !self.upgrade_candidate_region_elapsed.is_zero() {
316 METRIC_META_REGION_MIGRATION_STAGE_ELAPSED
317 .with_label_values(&["upgrade_candidate_region"])
318 .observe(self.upgrade_candidate_region_elapsed.as_secs_f64());
319 }
320 }
321}
322
323#[derive(Debug, Clone, Default)]
329pub struct VolatileContext {
330 opening_region_guards: Vec<OperatingRegionGuard>,
339 leader_region_lease_deadline: Option<Instant>,
341 from_peer_datanode_table_values: Option<HashMap<TableId, DatanodeTableValue>>,
343 leader_region_last_entry_ids: HashMap<RegionId, u64>,
345 leader_region_metadata_last_entry_ids: HashMap<RegionId, u64>,
347 upgraded_regions: HashSet<RegionId>,
350 metrics: Metrics,
352}
353
354impl VolatileContext {
355 pub fn set_leader_region_lease_deadline(&mut self, lease_timeout: Duration) {
357 if self.leader_region_lease_deadline.is_none() {
358 self.leader_region_lease_deadline = Some(Instant::now() + lease_timeout);
359 }
360 }
361
362 pub fn reset_leader_region_lease_deadline(&mut self) {
364 self.leader_region_lease_deadline = None;
365 }
366
367 pub fn set_last_entry_id(&mut self, region_id: RegionId, last_entry_id: u64) {
369 self.leader_region_last_entry_ids
370 .insert(region_id, last_entry_id);
371 }
372
373 pub fn set_metadata_last_entry_id(&mut self, region_id: RegionId, last_entry_id: u64) {
375 self.leader_region_metadata_last_entry_ids
376 .insert(region_id, last_entry_id);
377 }
378}
379
380pub trait ContextFactory {
382 fn new_context(self, persistent_ctx: PersistentContext) -> Context;
383}
384
385#[derive(Clone)]
387pub struct DefaultContextFactory {
388 volatile_ctx: VolatileContext,
389 in_memory_key: ResettableKvBackendRef,
390 table_metadata_manager: TableMetadataManagerRef,
391 opening_region_keeper: MemoryRegionKeeperRef,
392 region_failure_detector_controller: RegionFailureDetectorControllerRef,
393 mailbox: MailboxRef,
394 server_addr: String,
395 cache_invalidator: CacheInvalidatorRef,
396}
397
398impl DefaultContextFactory {
399 pub fn new(
401 in_memory_key: ResettableKvBackendRef,
402 table_metadata_manager: TableMetadataManagerRef,
403 opening_region_keeper: MemoryRegionKeeperRef,
404 region_failure_detector_controller: RegionFailureDetectorControllerRef,
405 mailbox: MailboxRef,
406 server_addr: String,
407 cache_invalidator: CacheInvalidatorRef,
408 ) -> Self {
409 Self {
410 volatile_ctx: VolatileContext::default(),
411 in_memory_key,
412 table_metadata_manager,
413 opening_region_keeper,
414 region_failure_detector_controller,
415 mailbox,
416 server_addr,
417 cache_invalidator,
418 }
419 }
420}
421
422impl ContextFactory for DefaultContextFactory {
423 fn new_context(self, persistent_ctx: PersistentContext) -> Context {
424 Context {
425 persistent_ctx,
426 volatile_ctx: self.volatile_ctx,
427 in_memory: self.in_memory_key,
428 table_metadata_manager: self.table_metadata_manager,
429 opening_region_keeper: self.opening_region_keeper,
430 region_failure_detector_controller: self.region_failure_detector_controller,
431 mailbox: self.mailbox,
432 server_addr: self.server_addr,
433 cache_invalidator: self.cache_invalidator,
434 }
435 }
436}
437
438pub struct Context {
440 persistent_ctx: PersistentContext,
441 volatile_ctx: VolatileContext,
442 in_memory: KvBackendRef,
443 table_metadata_manager: TableMetadataManagerRef,
444 opening_region_keeper: MemoryRegionKeeperRef,
445 region_failure_detector_controller: RegionFailureDetectorControllerRef,
446 mailbox: MailboxRef,
447 server_addr: String,
448 cache_invalidator: CacheInvalidatorRef,
449}
450
451impl Context {
452 pub(crate) fn trigger_reason(
453 &self,
454 event_context: Option<&PersistentEventContext>,
455 ) -> RegionMigrationTriggerReason {
456 event_context
457 .map(|ctx| RegionMigrationTriggerReason::from_trigger_reason(ctx.reason))
458 .unwrap_or(self.persistent_ctx.trigger_reason)
459 }
460 pub fn next_operation_timeout(&self) -> Option<Duration> {
462 self.persistent_ctx
463 .timeout
464 .checked_sub(self.volatile_ctx.metrics.operations_elapsed)
465 }
466
467 pub fn update_operations_elapsed(&mut self, instant: Instant) {
469 self.volatile_ctx
470 .metrics
471 .update_operations_elapsed(instant.elapsed());
472 }
473
474 pub fn update_flush_leader_region_elapsed(&mut self, instant: Instant) {
476 self.volatile_ctx
477 .metrics
478 .update_flush_leader_region_elapsed(instant.elapsed());
479 }
480
481 pub fn update_downgrade_leader_region_elapsed(&mut self, instant: Instant) {
483 self.volatile_ctx
484 .metrics
485 .update_downgrade_leader_region_elapsed(instant.elapsed());
486 }
487
488 pub fn update_open_candidate_region_elapsed(&mut self, instant: Instant) {
490 self.volatile_ctx
491 .metrics
492 .update_open_candidate_region_elapsed(instant.elapsed());
493 }
494
495 pub fn update_upgrade_candidate_region_elapsed(&mut self, instant: Instant) {
497 self.volatile_ctx
498 .metrics
499 .update_upgrade_candidate_region_elapsed(instant.elapsed());
500 }
501
502 pub fn server_addr(&self) -> &str {
504 &self.server_addr
505 }
506
507 pub fn region_table_ids(&self) -> Vec<TableId> {
509 self.persistent_ctx
510 .region_ids
511 .iter()
512 .map(|region_id| region_id.table_id())
513 .collect::<HashSet<_>>()
514 .into_iter()
515 .collect()
516 }
517
518 pub async fn get_table_route_values(
524 &self,
525 ) -> Result<HashMap<TableId, DeserializedValueWithBytes<TableRouteValue>>> {
526 let table_ids = self.persistent_ctx.region_table_ids();
527 let table_routes = self
528 .table_metadata_manager
529 .table_route_manager()
530 .table_route_storage()
531 .batch_get_with_raw_bytes(&table_ids)
532 .await
533 .context(error::TableMetadataManagerSnafu)
534 .map_err(BoxedError::new)
535 .with_context(|_| error::RetryLaterWithSourceSnafu {
536 reason: format!("Failed to get table routes: {table_ids:?}"),
537 })?;
538 let table_routes = table_ids
539 .into_iter()
540 .zip(table_routes)
541 .filter_map(|(table_id, table_route)| {
542 table_route.map(|table_route| (table_id, table_route))
543 })
544 .collect::<HashMap<_, _>>();
545 Ok(table_routes)
546 }
547
548 pub async fn get_table_route_value(
554 &self,
555 table_id: TableId,
556 ) -> Result<DeserializedValueWithBytes<TableRouteValue>> {
557 let table_route_value = self
558 .table_metadata_manager
559 .table_route_manager()
560 .table_route_storage()
561 .get_with_raw_bytes(table_id)
562 .await
563 .context(error::TableMetadataManagerSnafu)
564 .map_err(BoxedError::new)
565 .with_context(|_| error::RetryLaterWithSourceSnafu {
566 reason: format!("Failed to get table routes: {table_id:}"),
567 })?
568 .context(error::TableRouteNotFoundSnafu { table_id })?;
569 Ok(table_route_value)
570 }
571
572 pub async fn get_from_peer_datanode_table_values(
578 &mut self,
579 ) -> Result<&HashMap<TableId, DatanodeTableValue>> {
580 let from_peer_datanode_table_values =
581 &mut self.volatile_ctx.from_peer_datanode_table_values;
582 if from_peer_datanode_table_values.is_none() {
583 let table_ids = self.persistent_ctx.region_table_ids();
584 let datanode_table_keys = table_ids
585 .iter()
586 .map(|table_id| DatanodeTableKey {
587 datanode_id: self.persistent_ctx.from_peer.id,
588 table_id: *table_id,
589 })
590 .collect::<Vec<_>>();
591 let datanode_table_values = self
592 .table_metadata_manager
593 .datanode_table_manager()
594 .batch_get(&datanode_table_keys)
595 .await
596 .context(error::TableMetadataManagerSnafu)
597 .map_err(BoxedError::new)
598 .with_context(|_| error::RetryLaterWithSourceSnafu {
599 reason: format!("Failed to get DatanodeTable: {table_ids:?}"),
600 })?
601 .into_iter()
602 .map(|(k, v)| (k.table_id, v))
603 .collect();
604 *from_peer_datanode_table_values = Some(datanode_table_values);
605 }
606 Ok(from_peer_datanode_table_values.as_ref().unwrap())
607 }
608
609 pub async fn get_from_peer_datanode_table_value(
615 &self,
616 table_id: TableId,
617 ) -> Result<DatanodeTableValue> {
618 let datanode_table_value = self
619 .table_metadata_manager
620 .datanode_table_manager()
621 .get(&DatanodeTableKey {
622 datanode_id: self.persistent_ctx.from_peer.id,
623 table_id,
624 })
625 .await
626 .context(error::TableMetadataManagerSnafu)
627 .map_err(BoxedError::new)
628 .with_context(|_| error::RetryLaterWithSourceSnafu {
629 reason: format!("Failed to get DatanodeTable: {table_id}"),
630 })?
631 .context(error::DatanodeTableNotFoundSnafu {
632 table_id,
633 datanode_id: self.persistent_ctx.from_peer.id,
634 })?;
635 Ok(datanode_table_value)
636 }
637
638 pub async fn register_failure_detectors(&self, region_ids: &[RegionId]) {
644 let datanode_id = self.persistent_ctx.from_peer.id;
645 let detecting_regions = region_ids
646 .iter()
647 .map(|region_id| (datanode_id, *region_id))
648 .collect::<Vec<_>>();
649 self.region_failure_detector_controller
650 .register_failure_detectors(detecting_regions)
651 .await;
652 info!(
653 "Registered failure detectors after migration failures for datanode {}, regions {:?}",
654 datanode_id, region_ids
655 );
656 }
657
658 pub async fn reset_failure_detectors(&self, region_ids: &[RegionId]) {
661 let datanode_id = self.persistent_ctx.to_peer.id;
662 let detecting_regions = region_ids
663 .iter()
664 .map(|region_id| (datanode_id, *region_id))
665 .collect::<Vec<_>>();
666 self.region_failure_detector_controller
667 .reset_failure_detectors(detecting_regions)
668 .await;
669 info!(
670 "Reset failure detectors of regions for datanode {}, regions {:?}",
671 datanode_id, region_ids
672 );
673 }
674
675 pub async fn deregister_failure_detectors(&self) {
680 let datanode_id = self.persistent_ctx.from_peer.id;
681 let region_ids = &self.persistent_ctx.region_ids;
682 let detecting_regions = region_ids
683 .iter()
684 .map(|region_id| (datanode_id, *region_id))
685 .collect::<Vec<_>>();
686
687 self.region_failure_detector_controller
688 .deregister_failure_detectors(detecting_regions)
689 .await;
690 }
691
692 pub async fn deregister_failure_detectors_for_candidate_regions(&self) {
697 let to_peer_id = self.persistent_ctx.to_peer.id;
698 let region_ids = &self.persistent_ctx.region_ids;
699 let detecting_regions = region_ids
700 .iter()
701 .map(|region_id| (to_peer_id, *region_id))
702 .collect::<Vec<_>>();
703
704 self.region_failure_detector_controller
705 .deregister_failure_detectors(detecting_regions)
706 .await;
707 }
708
709 pub async fn get_replay_checkpoints_with_topic_pruned_entry_ids(
711 &self,
712 region_topics: &[(RegionId, String, bool)],
713 ) -> Result<HashMap<RegionId, ReplayCheckpoint>> {
714 let topic_region_keys = region_topics
715 .iter()
716 .map(|(region_id, topic, _)| TopicRegionKey::new(*region_id, topic))
717 .collect::<Vec<_>>();
718 let topic_region_values = self
719 .table_metadata_manager
720 .topic_region_manager()
721 .batch_get(topic_region_keys)
722 .await
723 .context(error::TableMetadataManagerSnafu)?;
724
725 let topic_name_keys = region_topics
726 .iter()
727 .map(|(_, topic, _)| topic.as_str())
728 .collect::<HashSet<_>>()
729 .into_iter()
730 .map(TopicNameKey::new)
731 .collect::<Vec<_>>();
732 let topic_name_values = self
733 .table_metadata_manager
734 .topic_name_manager()
735 .batch_get(topic_name_keys)
736 .await
737 .context(error::TableMetadataManagerSnafu)?;
738 debug!(
739 "Fetched topic region values: {:?}, topic name values: {:?}",
740 topic_region_values, topic_name_values
741 );
742
743 let replay_checkpoints = region_topics
744 .iter()
745 .filter_map(|(region_id, topic, is_metric_engine)| {
746 let checkpoint = topic_region_values
747 .get(region_id)
748 .and_then(|value| value.checkpoint);
749 let pruned_entry_id = topic_name_values
750 .get(topic)
751 .map(|value| value.pruned_entry_id);
752
753 ReplayCheckpoint::merge_with_topic_pruned_entry_id(
754 checkpoint,
755 pruned_entry_id,
756 *is_metric_engine,
757 )
758 .map(|checkpoint| (*region_id, checkpoint))
759 })
760 .collect::<HashMap<_, _>>();
761
762 Ok(replay_checkpoints)
763 }
764
765 pub async fn invalidate_table_cache(&self) -> Result<()> {
767 let table_ids = self.region_table_ids();
768 let mut cache_idents = Vec::with_capacity(table_ids.len());
769 for table_id in &table_ids {
770 cache_idents.push(CacheIdent::TableId(*table_id));
771 }
772 let ctx = common_meta::cache_invalidator::Context::default();
774 let _ = self.cache_invalidator.invalidate(&ctx, &cache_idents).await;
775 Ok(())
776 }
777
778 pub fn persistent_ctx(&self) -> PersistentContext {
780 self.persistent_ctx.clone()
781 }
782}
783
784#[async_trait::async_trait]
785#[typetag::serde(tag = "region_migration_state")]
786pub(crate) trait State: Sync + Send + Debug {
787 fn name(&self) -> &'static str {
788 let type_name = std::any::type_name::<Self>();
789 type_name.split("::").last().unwrap_or(type_name)
791 }
792
793 async fn next(
795 &mut self,
796 ctx: &mut Context,
797 procedure_ctx: &ProcedureContext,
798 ) -> Result<(Box<dyn State>, Status)>;
799
800 fn as_any(&self) -> &dyn Any;
802}
803
804#[derive(Debug, Deserialize)]
807pub struct RegionMigrationDataOwned {
808 persistent_ctx: PersistentContext,
809}
810
811#[derive(Debug, Serialize)]
813pub struct RegionMigrationData<'a> {
814 persistent_ctx: &'a PersistentContext,
815 state: &'a dyn State,
816}
817
818pub(crate) struct RegionMigrationProcedure {
819 state: Box<dyn State>,
820 context: Context,
824 _guards: Vec<RegionMigrationProcedureGuard>,
825}
826
827impl RegionMigrationProcedure {
828 const TYPE_NAME: &'static str = "metasrv-procedure::RegionMigration";
829
830 pub fn new(
831 persistent_context: PersistentContext,
832 context_factory: impl ContextFactory,
833 guards: Vec<RegionMigrationProcedureGuard>,
834 ) -> Self {
835 let state = Box::new(RegionMigrationStart {});
836 Self::new_inner(state, persistent_context, context_factory, guards)
837 }
838
839 fn new_inner(
840 state: Box<dyn State>,
841 persistent_context: PersistentContext,
842 context_factory: impl ContextFactory,
843 guards: Vec<RegionMigrationProcedureGuard>,
844 ) -> Self {
845 Self {
846 state,
847 context: context_factory.new_context(persistent_context),
848 _guards: guards,
849 }
850 }
851
852 fn from_json(
853 json: &str,
854 context_factory: impl ContextFactory,
855 tracker: RegionMigrationProcedureTracker,
856 ) -> ProcedureResult<Self> {
857 let RegionMigrationDataOwned { persistent_ctx } =
860 serde_json::from_str(json).context(FromJsonSnafu)?;
861 let guards = persistent_ctx
862 .region_ids
863 .iter()
864 .flat_map(|region_id| {
865 tracker.insert_running_procedure(&RegionMigrationProcedureTask {
866 region_id: *region_id,
867 from_peer: persistent_ctx.from_peer.clone(),
868 to_peer: persistent_ctx.to_peer.clone(),
869 timeout: persistent_ctx.timeout,
870 trigger_reason: persistent_ctx.trigger_reason,
871 })
872 })
873 .collect::<Vec<_>>();
874
875 let context = context_factory.new_context(persistent_ctx);
876
877 Ok(Self {
878 state: Box::new(RegionMigrationStart),
879 context,
880 _guards: guards,
881 })
882 }
883
884 async fn rollback_inner(&mut self, procedure_ctx: &ProcedureContext) -> Result<()> {
885 let _timer = METRIC_META_REGION_MIGRATION_EXECUTE
886 .with_label_values(&["rollback"])
887 .start_timer();
888 self.context.volatile_ctx.opening_region_guards.clear();
890 let ctx = &self.context;
891 let to_peer_id = ctx.persistent_ctx.to_peer.id;
892 let table_regions = ctx.persistent_ctx.table_regions();
893 let mut switched_regions = Vec::new();
895 let mut cleared_downgrading = false;
896 let mut failure = None;
897 for (table_id, regions) in table_regions {
898 let table_lock = TableLock::Write(table_id).into();
899 let _guard = procedure_ctx.provider.acquire_lock(&table_lock).await;
900 let table_route = match ctx.get_table_route_value(table_id).await {
901 Ok(table_route) => table_route,
902 Err(error::Error::TableRouteNotFound { .. }) => continue,
904 Err(err) => {
905 failure = Some(err);
906 break;
907 }
908 };
909 let region_routes = table_route.region_routes().unwrap();
910 switched_regions.extend(
911 region_routes
912 .iter()
913 .filter(|route| regions.contains(&route.region.id))
914 .filter(|route| {
915 route.leader_peer.as_ref().map(|peer| peer.id) == Some(to_peer_id)
916 })
917 .map(|route| route.region.id),
918 );
919 let downgraded = region_routes
920 .iter()
921 .filter(|route| regions.contains(&route.region.id))
922 .any(|route| route.is_leader_downgrading());
923 if downgraded {
924 info!(
925 "Rollbacking downgraded region leader table route, table: {table_id}, regions: {regions:?}"
926 );
927 let table_metadata_manager = &ctx.table_metadata_manager;
928 let cleared = table_metadata_manager
929 .update_leader_region_status(table_id, &table_route, |route| {
930 if regions.contains(&route.region.id) {
931 Some(None)
932 } else {
933 None
934 }
935 })
936 .await
937 .context(error::TableMetadataManagerSnafu)
938 .map_err(BoxedError::new)
939 .with_context(|_| error::RetryLaterWithSourceSnafu {
940 reason: format!("Failed to update the table route during the rollback downgraded leader region: {regions:?}"),
941 });
942 if let Err(err) = cleared {
943 failure = Some(err);
944 break;
945 }
946 cleared_downgrading = true;
947 }
948 }
949 if cleared_downgrading {
952 ctx.invalidate_table_cache().await?;
953 }
954 if let Some(err) = failure {
955 return Err(err);
956 }
957 self.context
958 .deregister_failure_detectors_for_candidate_regions()
959 .await;
960 let unswitched_regions = ctx
962 .persistent_ctx
963 .region_ids
964 .iter()
965 .filter(|region_id| !switched_regions.contains(region_id))
966 .copied()
967 .collect::<Vec<_>>();
968 self.context
969 .register_failure_detectors(&unswitched_regions)
970 .await;
971 if !switched_regions.is_empty() {
974 self.context
975 .reset_failure_detectors(&switched_regions)
976 .await;
977 }
978
979 Ok(())
980 }
981}
982
983#[async_trait::async_trait]
984impl Procedure for RegionMigrationProcedure {
985 fn type_name(&self) -> &str {
986 Self::TYPE_NAME
987 }
988
989 async fn rollback(&mut self, ctx: &ProcedureContext) -> ProcedureResult<()> {
990 self.rollback_inner(ctx)
991 .await
992 .map_err(ProcedureError::external)
993 }
994
995 fn rollback_supported(&self) -> bool {
996 true
997 }
998
999 #[tracing::instrument(skip_all, fields(
1000 state = %self.state.name(),
1001 region_count = self.context.persistent_ctx.region_ids.len(),
1002 from_peer = self.context.persistent_ctx.from_peer.id,
1003 to_peer = self.context.persistent_ctx.to_peer.id,
1004 ))]
1005 async fn execute(&mut self, ctx: &ProcedureContext) -> ProcedureResult<Status> {
1006 let state = &mut self.state;
1007
1008 let name = state.name();
1009 let _timer = METRIC_META_REGION_MIGRATION_EXECUTE
1010 .with_label_values(&[name])
1011 .start_timer();
1012 match state.next(&mut self.context, ctx).await {
1013 Ok((next, status)) => {
1014 *state = next;
1015 Ok(status)
1016 }
1017 Err(e) => {
1018 if e.is_retryable() {
1019 METRIC_META_REGION_MIGRATION_ERROR
1020 .with_label_values(&[name, "retryable"])
1021 .inc();
1022 Err(ProcedureError::retry_later(e))
1023 } else {
1024 self.context.volatile_ctx.opening_region_guards.clear();
1026 self.context
1027 .deregister_failure_detectors_for_candidate_regions()
1028 .await;
1029 error!(
1030 e;
1031 "Region migration procedure failed, regions: {:?}, from_peer: {}, to_peer: {}, {}",
1032 self.context.persistent_ctx.region_ids,
1033 self.context.persistent_ctx.from_peer,
1034 self.context.persistent_ctx.to_peer,
1035 self.context.volatile_ctx.metrics,
1036 );
1037 METRIC_META_REGION_MIGRATION_ERROR
1038 .with_label_values(&[name, "external"])
1039 .inc();
1040 Err(ProcedureError::external(e))
1041 }
1042 }
1043 }
1044 }
1045
1046 fn dump(&self) -> ProcedureResult<String> {
1047 let data = RegionMigrationData {
1048 state: self.state.as_ref(),
1049 persistent_ctx: &self.context.persistent_ctx,
1050 };
1051 serde_json::to_string(&data).context(ToJsonSnafu)
1052 }
1053
1054 fn lock_key(&self) -> LockKey {
1055 LockKey::new(self.context.persistent_ctx.lock_key())
1056 }
1057
1058 fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn Event>> {
1059 if !ctx.event_type_filter.allows(REGION_MIGRATION_EVENT_TYPE) {
1060 return None;
1061 }
1062
1063 Some(Box::new(RegionMigrationEvent::from_persistent_ctx(
1064 &self.context.persistent_ctx,
1065 self.context
1066 .trigger_reason(ctx.event_context)
1067 .to_trigger_reason(),
1068 )))
1069 }
1070}
1071
1072#[cfg(test)]
1073mod tests {
1074 use std::assert_matches;
1075 use std::sync::Arc;
1076
1077 use common_meta::distributed_time_constants::default_distributed_time_constants;
1078 use common_meta::instruction::Instruction;
1079 use common_meta::key::MetadataKey;
1080 use common_meta::key::table_route::TableRouteKey;
1081 use common_meta::key::test_utils::{new_test_table_info, new_test_table_info_with_name};
1082 use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
1083 use common_meta::rpc::store::PutRequest;
1084
1085 use super::*;
1086 use crate::handler::HeartbeatMailbox;
1087 use crate::procedure::region_migration::open_candidate_region::OpenCandidateRegion;
1088 use crate::procedure::region_migration::test_util::*;
1089 use crate::procedure::test_util::{
1090 new_downgrade_region_reply, new_flush_region_reply_for_region, new_open_region_reply,
1091 new_upgrade_region_reply,
1092 };
1093 use crate::region::supervisor::{Event, RegionFailureDetectorControl};
1094 use crate::service::mailbox::Channel;
1095
1096 fn new_persistent_context() -> PersistentContext {
1097 test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
1098 }
1099
1100 #[tokio::test]
1101 async fn test_rollback_broadcasts_cleared_marks_before_a_later_table_fails() {
1102 let cleared = RegionId::new(1024, 1);
1106 let failing = RegionId::new(1025, 1);
1107 for _ in 0..64 {
1108 let mut env = TestingEnv::new();
1109 for region_id in [cleared, failing] {
1110 let table_id = region_id.table_id();
1111 let table_info =
1112 new_test_table_info_with_name(table_id, &format!("table_{table_id}"));
1113 let region_routes = vec![RegionRoute {
1114 region: Region::new_test(region_id),
1115 leader_peer: Some(Peer::empty(1)),
1116 leader_state: (region_id == cleared).then_some(LeaderState::Downgrading),
1117 ..Default::default()
1118 }];
1119 env.create_physical_table_metadata(table_info, region_routes)
1120 .await;
1121 }
1122 let route_key = TableRouteKey::new(failing.table_id()).to_bytes();
1124 let kv_backend = env.table_metadata_manager().kv_backend().clone();
1125 kv_backend
1126 .put(
1127 PutRequest::new()
1128 .with_key(route_key.clone())
1129 .with_value(b"x".to_vec()),
1130 )
1131 .await
1132 .unwrap();
1133 let mut persistent_ctx = new_persistent_context();
1134 persistent_ctx.region_ids = vec![cleared, failing];
1135 let mut procedure =
1136 RegionMigrationProcedure::new(persistent_ctx, env.context_factory(), vec![]);
1137 let mut rx = env.frontend_receiver().await;
1138
1139 procedure
1140 .rollback_inner(&new_procedure_context())
1141 .await
1142 .unwrap_err();
1143
1144 let table_route = env
1145 .table_metadata_manager()
1146 .table_route_manager()
1147 .table_route_storage()
1148 .get(cleared.table_id())
1149 .await
1150 .unwrap()
1151 .unwrap();
1152 if table_route.region_routes().unwrap()[0]
1153 .leader_state
1154 .is_some()
1155 {
1156 assert!(rx.try_recv().is_err());
1158 continue;
1159 }
1160 assert_table_cache_invalidated(&mut rx, &[1024, 1025]);
1161 kv_backend.delete(&route_key, false).await.unwrap();
1163 procedure
1164 .rollback_inner(&new_procedure_context())
1165 .await
1166 .unwrap();
1167 assert!(rx.try_recv().is_err());
1168 return;
1169 }
1170 panic!("In 64 attempts the failing table never came after the cleared one");
1171 }
1172
1173 #[tokio::test]
1174 async fn test_rollback_registers_detectors_for_switched_regions() {
1175 let env = TestingEnv::new();
1176 let switched = RegionId::new(1024, 1);
1180 let unswitched = RegionId::new(1025, 1);
1181 let dropped = RegionId::new(1026, 1);
1182 for (region_id, leader) in [(switched, 2), (unswitched, 1)] {
1183 let table_id = region_id.table_id();
1184 let table_info = new_test_table_info_with_name(table_id, &format!("table_{table_id}"));
1185 let region_routes = vec![RegionRoute {
1186 region: Region::new_test(region_id),
1187 leader_peer: Some(Peer::empty(leader)),
1188 leader_state: (region_id == unswitched).then_some(LeaderState::Downgrading),
1189 ..Default::default()
1190 }];
1191 env.create_physical_table_metadata(table_info, region_routes)
1192 .await;
1193 }
1194 let mut persistent_ctx = new_persistent_context();
1195 persistent_ctx.region_ids = vec![switched, unswitched, dropped];
1196 let (tx, mut rx) = tokio::sync::mpsc::channel(8);
1197 let mut context_factory = env.context_factory();
1198 context_factory.region_failure_detector_controller =
1199 Arc::new(RegionFailureDetectorControl::new(tx));
1200 let mut procedure = RegionMigrationProcedure::new(persistent_ctx, context_factory, vec![]);
1201
1202 procedure
1203 .rollback_inner(&new_procedure_context())
1204 .await
1205 .unwrap();
1206
1207 let target = [switched, unswitched, dropped].map(|region_id| (2, region_id));
1210 let source = [unswitched, dropped].map(|region_id| (1, region_id));
1211 assert_matches!(rx.try_recv(), Ok(Event::DeregisterFailureDetectors(r)) if r == target);
1212 assert_matches!(rx.try_recv(), Ok(Event::RegisterFailureDetectors(r)) if r == source);
1213 assert_matches!(rx.try_recv(), Ok(Event::ResetFailureDetectors(r)) if r == [(2, switched)]);
1214 assert!(rx.try_recv().is_err());
1215 let table_route = env
1216 .table_metadata_manager()
1217 .table_route_manager()
1218 .table_route_storage()
1219 .get(unswitched.table_id())
1220 .await
1221 .unwrap()
1222 .unwrap();
1223 assert_eq!(table_route.region_routes().unwrap()[0].leader_state, None);
1224 }
1225
1226 #[tokio::test]
1227 async fn test_rollback_drops_candidate_registration_before_route_write() {
1228 let env = TestingEnv::new();
1229 let persistent_ctx = new_persistent_context();
1230 let rollback = CandidateRollback::new(&env, &persistent_ctx).await;
1231 let mut procedure =
1232 RegionMigrationProcedure::new(persistent_ctx, rollback.context_factory, vec![]);
1233 procedure
1234 .context
1235 .volatile_ctx
1236 .opening_region_guards
1237 .push(rollback.guard);
1238
1239 procedure
1240 .rollback_inner(&new_procedure_context())
1241 .await
1242 .unwrap();
1243
1244 assert_registration_dropped_before_route_write(&rollback.keeper_empty_at_txn);
1245 }
1246
1247 #[test]
1248 fn test_lock_key() {
1249 let persistent_context = new_persistent_context();
1250 let expected_keys = persistent_context.lock_key();
1251
1252 let env = TestingEnv::new();
1253 let context = env.context_factory();
1254
1255 let procedure = RegionMigrationProcedure::new(persistent_context, context, vec![]);
1256
1257 let key = procedure.lock_key();
1258 let keys = key.keys_to_lock().cloned().collect::<Vec<_>>();
1259
1260 for key in expected_keys {
1261 assert!(keys.contains(&key));
1262 }
1263 }
1264
1265 #[test]
1266 fn test_data_serialization() {
1267 let persistent_context = new_persistent_context();
1268
1269 let env = TestingEnv::new();
1270 let context = env.context_factory();
1271
1272 let procedure = RegionMigrationProcedure::new(persistent_context, context, vec![]);
1273
1274 let serialized = procedure.dump().unwrap();
1275 let expected = r#"{"persistent_ctx":{"catalog_and_schema":[["greptime","public"]],"from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_ids":[4398046511105],"timeout":"10s","trigger_reason":"Unknown"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
1276 assert_eq!(expected, serialized);
1277 }
1278
1279 #[test]
1280 fn test_event_hook_uses_current_persistent_context() {
1281 let env = TestingEnv::new();
1282 let procedure =
1283 RegionMigrationProcedure::new(new_persistent_context(), env.context_factory(), vec![]);
1284 let state = common_procedure::ProcedureState::Running;
1285 let triggers = [
1286 common_procedure::EventTrigger::Recovered,
1287 common_procedure::EventTrigger::Retrying {
1288 phase: common_procedure::RetryPhase::Execute,
1289 attempt: 1,
1290 },
1291 common_procedure::EventTrigger::RollingBack,
1292 common_procedure::EventTrigger::Succeeded,
1293 common_procedure::EventTrigger::Failed,
1294 common_procedure::EventTrigger::Poisoned,
1295 ];
1296
1297 for trigger in triggers {
1298 let event = procedure
1299 .event(&EventContext {
1300 procedure_id: common_procedure::ProcedureId::random(),
1301 lifecycle_state: &state,
1302 trigger,
1303 event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
1304 event_context: None,
1305 })
1306 .unwrap();
1307 assert_eq!(event.event_type(), "region_migration");
1308 assert_eq!(event.extra_rows().unwrap().len(), 1);
1309 }
1310 }
1311
1312 #[test]
1313 fn test_backward_compatibility() {
1314 let persistent_ctx = PersistentContext {
1315 #[allow(deprecated)]
1316 catalog: Some("greptime".into()),
1317 #[allow(deprecated)]
1318 schema: Some("public".into()),
1319 catalog_and_schema: vec![],
1320 from_peer: Peer::empty(1),
1321 to_peer: Peer::empty(2),
1322 region_ids: vec![RegionId::new(1024, 1)],
1323 timeout: Duration::from_secs(10),
1324 trigger_reason: RegionMigrationTriggerReason::Unknown,
1325 };
1326 let serialized = r#"{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105}"#;
1328 let deserialized: PersistentContext = serde_json::from_str(serialized).unwrap();
1329
1330 assert_eq!(persistent_ctx, deserialized);
1331 }
1332
1333 #[test]
1334 fn test_legacy_trigger_reason_survives_recovery_and_repersistence() {
1335 let serialized = r#"{"persistent_ctx":{"catalog":"greptime","schema":"public","from_peer":{"id":1,"addr":""},"to_peer":{"id":2,"addr":""},"region_id":4398046511105,"trigger_reason":"Failover"},"state":{"region_migration_state":"RegionMigrationStart"}}"#;
1336 let env = TestingEnv::new();
1337 let procedure = RegionMigrationProcedure::from_json(
1338 serialized,
1339 env.context_factory(),
1340 RegionMigrationProcedureTracker::default(),
1341 )
1342 .unwrap();
1343 assert_eq!(
1344 procedure.context.persistent_ctx.trigger_reason,
1345 RegionMigrationTriggerReason::Failover
1346 );
1347 let repersisted = procedure.dump().unwrap();
1348 assert_eq!(
1349 serde_json::from_str::<serde_json::Value>(&repersisted).unwrap()["persistent_ctx"]["trigger_reason"],
1350 "Failover"
1351 );
1352
1353 let recovered = RegionMigrationProcedure::from_json(
1354 &repersisted,
1355 env.context_factory(),
1356 RegionMigrationProcedureTracker::default(),
1357 )
1358 .unwrap();
1359 assert_eq!(
1360 recovered.context.trigger_reason(None),
1361 RegionMigrationTriggerReason::Failover
1362 );
1363 let event_context = PersistentEventContext::new(TriggerReason::AutoRebalance);
1364 assert_eq!(
1365 recovered.context.trigger_reason(Some(&event_context)),
1366 RegionMigrationTriggerReason::AutoRebalance
1367 );
1368
1369 let state = common_procedure::ProcedureState::Running;
1370 let event = recovered
1371 .event(&EventContext {
1372 procedure_id: common_procedure::ProcedureId::random(),
1373 lifecycle_state: &state,
1374 trigger: common_procedure::EventTrigger::Recovered,
1375 event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
1376 event_context: None,
1377 })
1378 .unwrap();
1379 assert_eq!(
1380 event.extra_rows().unwrap()[0].values[3].value_data,
1381 Some(api::v1::value::ValueData::StringValue(
1382 "Failover".to_string()
1383 ))
1384 );
1385 }
1386
1387 #[test]
1388 fn test_migration_reason_uses_event_context() {
1389 let env = TestingEnv::new();
1390 let context = env.context_factory().new_context(new_persistent_context());
1391 let event_context = PersistentEventContext::new(TriggerReason::RegionFailover);
1392
1393 assert_eq!(
1394 context.trigger_reason(Some(&event_context)),
1395 RegionMigrationTriggerReason::Failover
1396 );
1397
1398 let procedure =
1399 RegionMigrationProcedure::new(new_persistent_context(), env.context_factory(), vec![]);
1400 let state = common_procedure::ProcedureState::Running;
1401 let event = procedure
1402 .event(&EventContext {
1403 procedure_id: common_procedure::ProcedureId::random(),
1404 lifecycle_state: &state,
1405 trigger: common_procedure::EventTrigger::Submitted,
1406 event_type_filter: Arc::new(common_event_recorder::EventTypeFilter::All),
1407 event_context: Some(&event_context),
1408 })
1409 .unwrap();
1410 assert_eq!(
1411 event.extra_rows().unwrap()[0].values[3].value_data,
1412 Some(api::v1::value::ValueData::StringValue(
1413 "Failover".to_string()
1414 ))
1415 );
1416 }
1417
1418 fn recover_procedure(
1421 env: &TestingEnv,
1422 persistent_ctx: PersistentContext,
1423 ) -> RegionMigrationProcedure {
1424 let persisted = RegionMigrationProcedure::new_inner(
1425 Box::new(OpenCandidateRegion),
1426 persistent_ctx,
1427 env.context_factory(),
1428 vec![],
1429 )
1430 .dump()
1431 .unwrap();
1432 RegionMigrationProcedure::from_json(&persisted, env.context_factory(), env.tracker())
1433 .unwrap()
1434 }
1435
1436 #[tokio::test]
1437 async fn test_recovered_procedure_ends_if_regions_migrated() {
1438 let mut env = TestingEnv::new();
1439 let persistent_ctx = new_persistent_context();
1440 let region_id = persistent_ctx.region_ids[0];
1441 let region_routes = vec![RegionRoute {
1443 region: Region::new_test(region_id),
1444 leader_peer: Some(persistent_ctx.to_peer.clone()),
1445 ..Default::default()
1446 }];
1447 env.create_physical_table_metadata(new_test_table_info(1024), region_routes)
1448 .await;
1449 let mut frontend_rx = env.frontend_receiver().await;
1450 let (datanode_tx, mut datanode_rx) = tokio::sync::mpsc::channel(1);
1451 env.mailbox_context()
1452 .insert_heartbeat_response_receiver(Channel::Datanode(2), datanode_tx)
1453 .await;
1454
1455 let mut procedure = recover_procedure(&env, persistent_ctx);
1456 assert!(env.tracker().contains(region_id));
1457 let status = procedure
1458 .execute(&TestingEnv::procedure_context())
1459 .await
1460 .unwrap();
1461
1462 assert!(status.is_done());
1463 assert_table_cache_invalidated(&mut frontend_rx, &[1024]);
1464 assert_table_cache_invalidated(&mut datanode_rx, &[1024]);
1466 assert!(datanode_rx.try_recv().is_err());
1467 }
1468
1469 #[tokio::test]
1470 async fn test_recovered_procedure_rolls_back_if_leader_moved() {
1471 for downgraded_leader in [1, 2] {
1475 let mut env = TestingEnv::new();
1476 let mut persistent_ctx = new_persistent_context();
1477 persistent_ctx.region_ids = env.create_batch_with_moved_leader(downgraded_leader).await;
1478 let mut rx = env.frontend_receiver().await;
1479
1480 let mut procedure = recover_procedure(&env, persistent_ctx);
1481 let ctx = TestingEnv::procedure_context();
1482 let status = procedure.execute(&ctx).await.unwrap();
1483 assert!(!status.is_done());
1484 let err = procedure.execute(&ctx).await.unwrap_err();
1485 assert!(!err.is_retry_later());
1486 while rx.try_recv().is_ok() {}
1487 procedure.rollback(&ctx).await.unwrap();
1488 assert_table_cache_invalidated(&mut rx, &[1024]);
1490
1491 let table_route = env
1492 .table_metadata_manager()
1493 .table_route_manager()
1494 .table_route_storage()
1495 .get(1024)
1496 .await
1497 .unwrap()
1498 .unwrap();
1499 for region_route in table_route.region_routes().unwrap() {
1500 assert_eq!(region_route.leader_state, None);
1501 }
1502 }
1503 }
1504
1505 #[tokio::test]
1506 async fn test_recovered_procedure_ignores_persisted_state() {
1507 let env = TestingEnv::new();
1508 let persisted = RegionMigrationProcedure::new_inner(
1509 Box::new(OpenCandidateRegion),
1510 new_persistent_context(),
1511 env.context_factory(),
1512 vec![],
1513 )
1514 .dump()
1515 .unwrap();
1516 let unknown_state = persisted.replace("OpenCandidateRegion", "UnknownState");
1518 assert_ne!(unknown_state, persisted);
1519
1520 let procedure = RegionMigrationProcedure::from_json(
1521 &unknown_state,
1522 env.context_factory(),
1523 env.tracker(),
1524 )
1525 .unwrap();
1526
1527 assert!(procedure.state.as_any().is::<RegionMigrationStart>());
1528 }
1529
1530 #[tokio::test]
1531 async fn test_broadcast_invalidate_table_cache() {
1532 let mut env = TestingEnv::new();
1533 let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1534 let ctx = env.context_factory().new_context(persistent_context);
1535 let mailbox_ctx = env.mailbox_context();
1536
1537 ctx.invalidate_table_cache().await.unwrap();
1539
1540 let (tx, mut rx) = tokio::sync::mpsc::channel(1);
1541
1542 mailbox_ctx
1543 .insert_heartbeat_response_receiver(Channel::Frontend(1), tx)
1544 .await;
1545
1546 ctx.invalidate_table_cache().await.unwrap();
1547
1548 let resp = rx.recv().await.unwrap().unwrap();
1549 let msg = resp.mailbox_message.unwrap();
1550
1551 let instruction = HeartbeatMailbox::json_instruction(&msg).unwrap();
1552 assert_eq!(
1553 instruction,
1554 Instruction::InvalidateCaches(vec![CacheIdent::TableId(1024)])
1555 );
1556 }
1557
1558 fn procedure_flow_steps(from_peer_id: u64, to_peer_id: u64) -> Vec<Step> {
1559 vec![
1560 Step::next(
1562 "Should be the open candidate region",
1563 None,
1564 Assertion::simple(assert_open_candidate_region, assert_need_persist),
1565 ),
1566 Step::next(
1568 "Should be the flush leader region",
1569 Some(mock_datanode_reply(
1570 to_peer_id,
1571 Arc::new(|id| Ok(new_open_region_reply(id, true, None))),
1572 )),
1573 Assertion::simple(assert_flush_leader_region, assert_no_persist),
1574 ),
1575 Step::next(
1577 "Should be the flush leader region",
1578 Some(mock_datanode_reply(
1579 from_peer_id,
1580 Arc::new(move |id| {
1581 Ok(new_flush_region_reply_for_region(
1582 id,
1583 RegionId::new(1024, 1),
1584 true,
1585 None,
1586 ))
1587 }),
1588 )),
1589 Assertion::simple(assert_update_metadata_downgrade, assert_no_persist),
1590 ),
1591 Step::next(
1593 "Should be the downgrade leader region",
1594 None,
1595 Assertion::simple(assert_downgrade_leader_region, assert_no_persist),
1596 ),
1597 Step::next(
1599 "Should be the upgrade candidate region",
1600 Some(mock_datanode_reply(
1601 from_peer_id,
1602 Arc::new(|id| Ok(new_downgrade_region_reply(id, None, true, None))),
1603 )),
1604 Assertion::simple(assert_upgrade_candidate_region, assert_no_persist),
1605 ),
1606 Step::next(
1608 "Should be the update metadata for upgrading",
1609 Some(mock_datanode_reply(
1610 to_peer_id,
1611 Arc::new(|id| Ok(new_upgrade_region_reply(id, true, true, None))),
1612 )),
1613 Assertion::simple(assert_update_metadata_upgrade, assert_no_persist),
1614 ),
1615 Step::next(
1617 "Should be the close downgraded region",
1618 None,
1619 Assertion::simple(assert_close_downgraded_region, assert_no_persist),
1620 ),
1621 Step::next(
1623 "Should be the region migration end",
1624 None,
1625 Assertion::simple(assert_region_migration_end, assert_done),
1626 ),
1627 Step::next(
1629 "Should be the region migration end again",
1630 None,
1631 Assertion::simple(assert_region_migration_end, assert_done),
1632 ),
1633 ]
1634 }
1635
1636 #[tokio::test]
1637 async fn test_procedure_flow() {
1638 common_telemetry::init_default_ut_logging();
1639
1640 let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1641 let state = Box::new(RegionMigrationStart);
1642
1643 let from_peer_id = persistent_context.from_peer.id;
1645 let to_peer_id = persistent_context.to_peer.id;
1646 let from_peer = persistent_context.from_peer.clone();
1647 let to_peer = persistent_context.to_peer.clone();
1648 let region_id = persistent_context.region_ids[0];
1649 let table_info = new_test_table_info(1024);
1650 let region_routes = vec![RegionRoute {
1651 region: Region::new_test(region_id),
1652 leader_peer: Some(from_peer),
1653 follower_peers: vec![to_peer],
1654 ..Default::default()
1655 }];
1656
1657 let suite = ProcedureMigrationTestSuite::new(persistent_context, state);
1658 suite.init_table_metadata(table_info, region_routes).await;
1659
1660 let steps = procedure_flow_steps(from_peer_id, to_peer_id);
1661 let timer = Instant::now();
1662
1663 let runner = ProcedureMigrationSuiteRunner::new(suite)
1665 .steps(steps)
1666 .run_once()
1667 .await;
1668
1669 let region_lease = default_distributed_time_constants().region_lease.as_secs();
1670
1671 assert!(timer.elapsed().as_secs() < region_lease / 2);
1673
1674 runner.suite.verify_table_metadata().await;
1675 }
1676
1677 #[tokio::test]
1678 async fn test_procedure_flow_open_candidate_region_retryable_error() {
1679 common_telemetry::init_default_ut_logging();
1680
1681 let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1682 let state = Box::new(RegionMigrationStart);
1683
1684 let to_peer_id = persistent_context.to_peer.id;
1686 let from_peer = persistent_context.from_peer.clone();
1687 let region_id = persistent_context.region_ids[0];
1688 let table_info = new_test_table_info(1024);
1689 let region_routes = vec![RegionRoute {
1690 region: Region::new_test(region_id),
1691 leader_peer: Some(from_peer),
1692 follower_peers: vec![],
1693 ..Default::default()
1694 }];
1695
1696 let suite = ProcedureMigrationTestSuite::new(persistent_context, state);
1697 suite.init_table_metadata(table_info, region_routes).await;
1698
1699 let steps = vec![
1700 Step::next(
1702 "Should be the open candidate region",
1703 None,
1704 Assertion::simple(assert_open_candidate_region, assert_need_persist),
1705 ),
1706 Step::next(
1708 "Should be throwing a non-retry error",
1709 Some(mock_datanode_reply(
1710 to_peer_id,
1711 Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1712 )),
1713 Assertion::error(|error| assert!(error.is_retryable())),
1714 ),
1715 Step::next(
1717 "Should be throwing a non-retry error again",
1718 Some(mock_datanode_reply(
1719 to_peer_id,
1720 Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1721 )),
1722 Assertion::error(|error| assert!(error.is_retryable())),
1723 ),
1724 ];
1725
1726 let setup_to_latest_persisted_state = Step::setup(
1727 "Sets state to UpdateMetadata::Downgrade",
1728 merge_before_test_fn(vec![
1729 setup_state(Arc::new(|| Box::new(OpenCandidateRegion))),
1730 Arc::new(reset_volatile_ctx),
1731 ]),
1732 );
1733
1734 let steps = [
1735 steps.clone(),
1736 vec![setup_to_latest_persisted_state.clone()],
1738 steps.clone()[1..].to_vec(),
1739 vec![setup_to_latest_persisted_state],
1740 steps.clone()[1..].to_vec(),
1741 ]
1742 .concat();
1743
1744 let runner = ProcedureMigrationSuiteRunner::new(suite)
1746 .steps(steps.clone())
1747 .run_once()
1748 .await;
1749
1750 let table_routes_version = runner
1751 .env()
1752 .table_metadata_manager()
1753 .table_route_manager()
1754 .table_route_storage()
1755 .get(region_id.table_id())
1756 .await
1757 .unwrap()
1758 .unwrap()
1759 .version();
1760 assert_eq!(table_routes_version.unwrap(), 0);
1762 }
1763
1764 #[tokio::test]
1765 async fn test_procedure_flow_upgrade_candidate_with_retry_and_failed() {
1766 common_telemetry::init_default_ut_logging();
1767
1768 let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1769 let state = Box::new(RegionMigrationStart);
1770
1771 let from_peer_id = persistent_context.from_peer.id;
1773 let to_peer_id = persistent_context.to_peer.id;
1774 let from_peer = persistent_context.from_peer.clone();
1775 let region_id = persistent_context.region_ids[0];
1776 let table_info = new_test_table_info(1024);
1777 let region_routes = vec![RegionRoute {
1778 region: Region::new_test(region_id),
1779 leader_peer: Some(from_peer),
1780 follower_peers: vec![],
1781 ..Default::default()
1782 }];
1783
1784 let suite = ProcedureMigrationTestSuite::new(persistent_context, state);
1785 suite.init_table_metadata(table_info, region_routes).await;
1786
1787 let steps = vec![
1788 Step::next(
1790 "Should be the open candidate region",
1791 None,
1792 Assertion::simple(assert_open_candidate_region, assert_need_persist),
1793 ),
1794 Step::next(
1796 "Should be the flush leader region",
1797 Some(mock_datanode_reply(
1798 to_peer_id,
1799 Arc::new(|id| Ok(new_open_region_reply(id, true, None))),
1800 )),
1801 Assertion::simple(assert_flush_leader_region, assert_no_persist),
1802 ),
1803 Step::next(
1805 "Should be the flush leader region",
1806 Some(mock_datanode_reply(
1807 from_peer_id,
1808 Arc::new(move |id| {
1809 Ok(new_flush_region_reply_for_region(
1810 id,
1811 RegionId::new(1024, 1),
1812 true,
1813 None,
1814 ))
1815 }),
1816 )),
1817 Assertion::simple(assert_update_metadata_downgrade, assert_no_persist),
1818 ),
1819 Step::next(
1821 "Should be the downgrade leader region",
1822 None,
1823 Assertion::simple(assert_downgrade_leader_region, assert_no_persist),
1824 ),
1825 Step::next(
1827 "Should be the upgrade candidate region",
1828 Some(mock_datanode_reply(
1829 from_peer_id,
1830 Arc::new(|id| Ok(new_downgrade_region_reply(id, None, true, None))),
1831 )),
1832 Assertion::simple(assert_upgrade_candidate_region, assert_no_persist),
1833 ),
1834 Step::next(
1836 "Should be the rollback metadata",
1837 Some(mock_datanode_reply(
1838 to_peer_id,
1839 Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1840 )),
1841 Assertion::simple(assert_update_metadata_rollback, assert_no_persist),
1842 ),
1843 Step::next(
1845 "Should be the region migration abort",
1846 None,
1847 Assertion::simple(assert_region_migration_abort, assert_no_persist),
1848 ),
1849 Step::next(
1851 "Should throw an error",
1852 None,
1853 Assertion::error(|error| {
1854 assert!(!error.is_retryable());
1855 assert_matches!(error, error::Error::MigrationAbort { .. });
1856 }),
1857 ),
1858 ];
1859
1860 let setup_to_latest_persisted_state = Step::setup(
1861 "Sets state to OpenCandidateRegion",
1862 merge_before_test_fn(vec![
1863 setup_state(Arc::new(|| Box::new(OpenCandidateRegion))),
1864 Arc::new(reset_volatile_ctx),
1865 ]),
1866 );
1867
1868 let steps = [
1869 steps.clone(),
1870 vec![setup_to_latest_persisted_state.clone()],
1871 steps.clone()[1..].to_vec(),
1872 vec![setup_to_latest_persisted_state],
1873 steps.clone()[1..].to_vec(),
1874 ]
1875 .concat();
1876
1877 ProcedureMigrationSuiteRunner::new(suite)
1879 .steps(steps.clone())
1880 .run_once()
1881 .await;
1882 }
1883
1884 #[tokio::test]
1885 async fn test_procedure_flow_upgrade_candidate_with_retry() {
1886 common_telemetry::init_default_ut_logging();
1887
1888 let persistent_context = test_util::new_persistent_context(1, 2, RegionId::new(1024, 1));
1889 let state = Box::new(RegionMigrationStart);
1890
1891 let to_peer_id = persistent_context.to_peer.id;
1893 let from_peer_id = persistent_context.from_peer.id;
1894 let from_peer = persistent_context.from_peer.clone();
1895 let region_id = persistent_context.region_ids[0];
1896 let table_info = new_test_table_info(1024);
1897 let region_routes = vec![RegionRoute {
1898 region: Region::new_test(region_id),
1899 leader_peer: Some(from_peer),
1900 follower_peers: vec![],
1901 ..Default::default()
1902 }];
1903
1904 let suite = ProcedureMigrationTestSuite::new(persistent_context, state);
1905 suite.init_table_metadata(table_info, region_routes).await;
1906
1907 let steps = vec![
1908 Step::next(
1910 "Should be the open candidate region",
1911 None,
1912 Assertion::simple(assert_open_candidate_region, assert_need_persist),
1913 ),
1914 Step::next(
1916 "Should be throwing a retryable error",
1917 Some(mock_datanode_reply(
1918 to_peer_id,
1919 Arc::new(|id| {
1920 Ok(new_open_region_reply(
1921 id,
1922 false,
1923 Some("mock retryable open region error".to_string()),
1924 ))
1925 }),
1926 )),
1927 Assertion::error(|error| assert!(error.is_retryable(), "err: {error:?}")),
1928 ),
1929 Step::next(
1931 "Should be the update metadata for downgrading",
1932 Some(mock_datanode_reply(
1933 to_peer_id,
1934 Arc::new(|id| Ok(new_open_region_reply(id, true, None))),
1935 )),
1936 Assertion::simple(assert_flush_leader_region, assert_no_persist),
1937 ),
1938 Step::next(
1940 "Should be the flush leader region",
1941 Some(mock_datanode_reply(
1942 from_peer_id,
1943 Arc::new(move |id| {
1944 Ok(new_flush_region_reply_for_region(id, region_id, true, None))
1945 }),
1946 )),
1947 Assertion::simple(assert_update_metadata_downgrade, assert_no_persist),
1948 ),
1949 Step::next(
1951 "Should be the downgrade leader region",
1952 None,
1953 Assertion::simple(assert_downgrade_leader_region, assert_no_persist),
1954 ),
1955 Step::next(
1957 "Should be the upgrade candidate region",
1958 Some(mock_datanode_reply(
1959 from_peer_id,
1960 merge_mailbox_messages(vec![
1961 Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1962 Arc::new(|id| Ok(new_downgrade_region_reply(id, None, true, None))),
1963 ]),
1964 )),
1965 Assertion::simple(assert_upgrade_candidate_region, assert_no_persist),
1966 ),
1967 Step::next(
1969 "Should be the update metadata for upgrading",
1970 Some(mock_datanode_reply(
1971 to_peer_id,
1972 merge_mailbox_messages(vec![
1973 Arc::new(|id| error::MailboxTimeoutSnafu { id }.fail()),
1974 Arc::new(|id| Ok(new_upgrade_region_reply(id, true, true, None))),
1975 ]),
1976 )),
1977 Assertion::simple(assert_update_metadata_upgrade, assert_no_persist),
1978 ),
1979 Step::next(
1981 "Should be the close downgraded region",
1982 None,
1983 Assertion::simple(assert_close_downgraded_region, assert_no_persist),
1984 ),
1985 Step::next(
1987 "Should be the region migration end",
1988 None,
1989 Assertion::simple(assert_region_migration_end, assert_done),
1990 ),
1991 Step::next(
1993 "Should be the region migration end again",
1994 None,
1995 Assertion::simple(assert_region_migration_end, assert_done),
1996 ),
1997 Step::setup(
1999 "Sets state to RegionMigrationStart",
2000 merge_before_test_fn(vec![
2001 setup_state(Arc::new(|| Box::new(RegionMigrationStart))),
2002 Arc::new(reset_volatile_ctx),
2003 ]),
2004 ),
2005 Step::next(
2009 "Should be the region migration end(has been migrated)",
2010 None,
2011 Assertion::simple(assert_region_migration_end, assert_done),
2012 ),
2013 ];
2014
2015 let steps = [steps.clone()].concat();
2016 let timer = Instant::now();
2017
2018 let runner = ProcedureMigrationSuiteRunner::new(suite)
2020 .steps(steps.clone())
2021 .run_once()
2022 .await;
2023
2024 let region_lease = default_distributed_time_constants().region_lease.as_secs();
2025 assert!(timer.elapsed().as_secs() < region_lease);
2027 runner.suite.verify_table_metadata().await;
2028 }
2029}