meta_srv/procedure/region_migration/update_metadata/
rollback_downgraded_region.rs1use common_error::ext::BoxedError;
16use common_meta::lock_key::TableLock;
17use common_procedure::ContextProviderRef;
18use common_telemetry::{error, info};
19use snafu::ResultExt;
20
21use crate::error::{self, Result};
22use crate::procedure::region_migration::Context;
23use crate::procedure::region_migration::update_metadata::UpdateMetadata;
24
25impl UpdateMetadata {
26 pub async fn rollback_downgraded_region(
35 &self,
36 ctx: &mut Context,
37 ctx_provider: &ContextProviderRef,
38 ) -> Result<()> {
39 let table_metadata_manager = ctx.table_metadata_manager.clone();
40 let table_regions = ctx.persistent_ctx.table_regions();
41
42 for (table_id, regions) in table_regions {
43 let table_lock = TableLock::Write(table_id).into();
44 let _guard = ctx_provider.acquire_lock(&table_lock).await;
45
46 let current_table_route_value = ctx.get_table_route_value(table_id).await?;
47 if let Err(err) = table_metadata_manager
48 .update_leader_region_status(table_id, ¤t_table_route_value, |route| {
49 if regions.contains(&route.region.id) {
50 Some(None)
51 } else {
52 None
53 }
54 })
55 .await
56 .context(error::TableMetadataManagerSnafu)
57 {
58 error!(err; "Failed to update the table route during the rollback downgraded leader regions: {regions:?}");
59 return Err(BoxedError::new(err)).with_context(|_| error::RetryLaterWithSourceSnafu {
60 reason: format!("Failed to update the table route during the rollback downgraded leader regions: {regions:?}"),
61 });
62 }
63 info!(
64 "Rolling back downgraded leader region table route success, table_id: {table_id}, regions: {regions:?}"
65 );
66 }
67 ctx.register_failure_detectors(&ctx.persistent_ctx.region_ids)
68 .await;
69
70 Ok(())
71 }
72}
73
74#[cfg(test)]
75mod tests {
76 use std::assert_matches;
77 use std::collections::HashMap;
78 use std::sync::Arc;
79
80 use common_meta::key::test_utils::new_test_table_info;
81 use common_meta::peer::Peer;
82 use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
83 use common_procedure_test::MockContextProvider;
84 use store_api::storage::RegionId;
85
86 use crate::error::Error;
87 use crate::procedure::region_migration::migration_abort::RegionMigrationAbort;
88 use crate::procedure::region_migration::test_util::{
89 self, CandidateRollback, TestingEnv, new_procedure_context,
90 };
91 use crate::procedure::region_migration::update_metadata::UpdateMetadata;
92 use crate::procedure::region_migration::{ContextFactory, PersistentContext, State};
93
94 fn new_persistent_context() -> PersistentContext {
95 test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
96 }
97
98 #[tokio::test]
99 async fn test_table_route_is_not_found_error() {
100 let state = UpdateMetadata::Rollback;
101 let env = TestingEnv::new();
102 let persistent_context = new_persistent_context();
103 let mut ctx = env.context_factory().new_context(persistent_context);
104
105 let provider = Arc::new(MockContextProvider::new(HashMap::new())) as _;
106 let err = state
107 .rollback_downgraded_region(&mut ctx, &provider)
108 .await
109 .unwrap_err();
110
111 assert_matches!(err, Error::TableRouteNotFound { .. });
112
113 assert!(!err.is_retryable());
114 }
115
116 #[tokio::test]
117 async fn test_next_migration_end_state() {
118 let mut state = Box::new(UpdateMetadata::Rollback);
119 let persistent_context = new_persistent_context();
120 let from_peer = persistent_context.from_peer.clone();
121
122 let env = TestingEnv::new();
123 let mut ctx = env.context_factory().new_context(persistent_context);
124 let table_id = ctx.persistent_ctx.region_ids[0].table_id();
125
126 let table_info = new_test_table_info(1024);
127 let region_routes = vec![
128 RegionRoute {
129 region: Region::new_test(RegionId::new(1024, 1)),
130 leader_peer: Some(from_peer.clone()),
131 leader_state: Some(LeaderState::Downgrading),
132 ..Default::default()
133 },
134 RegionRoute {
135 region: Region::new_test(RegionId::new(1024, 2)),
136 leader_peer: Some(Peer::empty(4)),
137 leader_state: Some(LeaderState::Downgrading),
138 ..Default::default()
139 },
140 RegionRoute {
141 region: Region::new_test(RegionId::new(1024, 3)),
142 leader_peer: Some(Peer::empty(5)),
143 ..Default::default()
144 },
145 ];
146
147 let expected_region_routes = {
148 let mut region_routes = region_routes.clone();
149 region_routes[0].leader_state = None;
150 region_routes
151 };
152
153 env.create_physical_table_metadata(table_info, region_routes)
154 .await;
155
156 let table_metadata_manager = env.table_metadata_manager();
157
158 let procedure_ctx = new_procedure_context();
159 let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
160
161 let _ = next
162 .as_any()
163 .downcast_ref::<RegionMigrationAbort>()
164 .unwrap();
165
166 let table_route = table_metadata_manager
167 .table_route_manager()
168 .table_route_storage()
169 .get(table_id)
170 .await
171 .unwrap()
172 .unwrap();
173 assert_eq!(
174 &expected_region_routes,
175 table_route.region_routes().unwrap()
176 );
177 }
178
179 #[tokio::test]
180 async fn test_next_drops_candidate_registration_before_route_write() {
181 let env = TestingEnv::new();
182 let persistent_context = new_persistent_context();
183 let rollback = CandidateRollback::new(&env, &persistent_context).await;
184 let mut ctx = rollback.context_factory.new_context(persistent_context);
185 ctx.volatile_ctx.opening_region_guards.push(rollback.guard);
186
187 UpdateMetadata::Rollback
188 .next(&mut ctx, &new_procedure_context())
189 .await
190 .unwrap();
191
192 test_util::assert_registration_dropped_before_route_write(&rollback.keeper_empty_at_txn);
193 }
194}