meta_srv/procedure/region_migration/
migration_start.rs1use std::any::Any;
16
17use common_meta::peer::Peer;
18use common_meta::rpc::router::RegionRoute;
19use common_procedure::{Context as ProcedureContext, Status};
20use common_telemetry::info;
21use serde::{Deserialize, Serialize};
22use snafu::{OptionExt, ResultExt};
23
24use crate::error::{self, Result};
25use crate::procedure::region_migration::migration_abort::RegionMigrationAbort;
26use crate::procedure::region_migration::migration_end::RegionMigrationEnd;
27use crate::procedure::region_migration::open_candidate_region::OpenCandidateRegion;
28use crate::procedure::region_migration::{Context, State};
29
30#[derive(Debug, Serialize, Deserialize)]
38pub struct RegionMigrationStart;
39
40#[async_trait::async_trait]
41#[typetag::serde]
42impl State for RegionMigrationStart {
43 async fn next(
51 &mut self,
52 ctx: &mut Context,
53 _procedure_ctx: &ProcedureContext,
54 ) -> Result<(Box<dyn State>, Status)> {
55 let mut region_routes = self.retrieve_region_routes(ctx).await?;
56 let to_peer = &ctx.persistent_ctx.to_peer;
57 let from_peer = &ctx.persistent_ctx.from_peer;
58 let region_ids = &ctx.persistent_ctx.region_ids;
59
60 let downgraded = region_routes
63 .iter()
64 .any(|route| route.is_leader_downgrading());
65
66 self.filter_unmigrated_regions(&mut region_routes, to_peer);
67
68 if region_routes.len() != region_ids.len() {
70 ctx.invalidate_table_cache().await?;
71 }
72
73 if region_routes.is_empty() {
75 info!(
76 "All regions have been migrated, regions: {:?}, to_peer: {:?}",
77 region_ids, to_peer
78 );
79 return Ok((Box::new(RegionMigrationEnd), Status::done()));
80 }
81
82 for region_route in ®ion_routes {
85 if self.invalid_leader_peer(region_route, from_peer)? {
86 info!(
87 "Abort region migration, region:{}, unexpected leader peer: {:?}, expected: {:?}",
88 region_route.region.id, region_route.leader_peer, from_peer,
89 );
90 let status = if downgraded {
92 Status::executing(false)
93 } else {
94 Status::done()
95 };
96 return Ok((
97 Box::new(RegionMigrationAbort::new(&format!(
98 "Invalid region leader peer: {:?}, expected: {:?}",
99 region_route.leader_peer.as_ref().unwrap(),
100 from_peer,
101 ))),
102 status,
103 ));
104 }
105 }
106
107 if region_routes.len() != region_ids.len() {
109 let unmigrated_region_ids = region_routes.iter().map(|route| route.region.id).collect();
110 info!(
111 "Some of the regions have been migrated, only migrate the following regions: {:?}, to_peer: {:?}",
112 unmigrated_region_ids, to_peer
113 );
114 ctx.persistent_ctx.region_ids = unmigrated_region_ids;
115 }
116
117 Ok((Box::new(OpenCandidateRegion), Status::executing(true)))
119 }
120
121 fn as_any(&self) -> &dyn Any {
122 self
123 }
124}
125
126impl RegionMigrationStart {
127 async fn retrieve_region_routes(&self, ctx: &mut Context) -> Result<Vec<RegionRoute>> {
136 let region_ids = &ctx.persistent_ctx.region_ids;
137 let table_route_values = ctx.get_table_route_values().await?;
138 let mut region_routes = Vec::with_capacity(region_ids.len());
139 for region_id in region_ids {
140 let table_id = region_id.table_id();
141 let region_route = table_route_values
142 .get(&table_id)
143 .context(error::TableRouteNotFoundSnafu { table_id })?
144 .region_routes()
145 .with_context(|_| error::UnexpectedLogicalRouteTableSnafu {
146 err_msg: format!("TableRoute({table_id:?}) is a non-physical TableRouteValue."),
147 })?
148 .iter()
149 .find(|route| route.region.id == *region_id)
150 .cloned()
151 .with_context(|| error::UnexpectedSnafu {
152 violated: format!(
153 "RegionRoute({}) is not found in TableRoute({})",
154 region_id, table_id
155 ),
156 })?;
157 region_routes.push(region_route);
158 }
159
160 Ok(region_routes)
161 }
162
163 fn invalid_leader_peer(&self, region_route: &RegionRoute, from_peer: &Peer) -> Result<bool> {
168 let region_id = region_route.region.id;
169
170 let is_invalid_leader_peer = region_route
171 .leader_peer
172 .as_ref()
173 .with_context(|| error::UnexpectedSnafu {
174 violated: format!("Leader peer is not found in TableRoute({})", region_id),
175 })?
176 .id
177 != from_peer.id;
178 Ok(is_invalid_leader_peer)
179 }
180
181 fn filter_unmigrated_regions(&self, region_routes: &mut Vec<RegionRoute>, to_peer: &Peer) {
183 region_routes
184 .retain(|region_route| !self.has_migrated(region_route, to_peer).unwrap_or(false));
185 }
186
187 fn has_migrated(&self, region_route: &RegionRoute, to_peer: &Peer) -> Result<bool> {
193 let region_id = region_route.region.id;
194
195 let region_migrated = region_route
196 .leader_peer
197 .as_ref()
198 .with_context(|| error::UnexpectedSnafu {
199 violated: format!("Leader peer is not found in TableRoute({})", region_id),
200 })?
201 .id
202 == to_peer.id;
203 Ok(region_migrated)
204 }
205}
206
207#[cfg(test)]
208mod tests {
209
210 use std::assert_matches;
211
212 use common_meta::key::test_utils::{new_test_table_info, new_test_table_info_with_name};
213 use common_meta::peer::Peer;
214 use common_meta::rpc::router::{Region, RegionRoute};
215 use store_api::storage::RegionId;
216
217 use super::*;
218 use crate::error::Error;
219 use crate::procedure::region_migration::test_util::{
220 self, TestingEnv, assert_done, assert_need_persist, assert_table_cache_invalidated,
221 new_procedure_context,
222 };
223 use crate::procedure::region_migration::{ContextFactory, PersistentContext};
224
225 fn new_persistent_context() -> PersistentContext {
226 test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
227 }
228
229 #[tokio::test]
230 async fn test_table_route_is_not_found_error() {
231 let state = RegionMigrationStart;
232 let env = TestingEnv::new();
233 let persistent_context = new_persistent_context();
234 let mut ctx = env.context_factory().new_context(persistent_context);
235 let err = state.retrieve_region_routes(&mut ctx).await.unwrap_err();
236 assert_matches!(err, Error::TableRouteNotFound { .. });
237 assert!(!err.is_retryable());
238 }
239
240 #[tokio::test]
241 async fn test_region_route_is_not_found_error() {
242 let state = RegionMigrationStart;
243 let persistent_context = new_persistent_context();
244 let from_peer = persistent_context.from_peer.clone();
245
246 let env = TestingEnv::new();
247 let mut ctx = env.context_factory().new_context(persistent_context);
248
249 let table_info = new_test_table_info(1024);
250 let region_route = RegionRoute {
251 region: Region::new_test(RegionId::new(1024, 3)),
252 leader_peer: Some(from_peer.clone()),
253 ..Default::default()
254 };
255
256 env.create_physical_table_metadata(table_info, vec![region_route])
257 .await;
258 let err = state.retrieve_region_routes(&mut ctx).await.unwrap_err();
259 assert_matches!(err, Error::Unexpected { .. });
260 assert!(!err.is_retryable());
261 }
262
263 #[tokio::test]
264 async fn test_next_migration_end_state() {
265 let mut state = Box::new(RegionMigrationStart);
266 let persistent_context = new_persistent_context();
269 let to_peer = persistent_context.to_peer.clone();
270 let from_peer = persistent_context.from_peer.clone();
271 let region_id = persistent_context.region_ids[0];
272
273 let mut env = TestingEnv::new();
274 let mut ctx = env.context_factory().new_context(persistent_context);
275 let mut rx = env.frontend_receiver().await;
276
277 let table_info = new_test_table_info(1024);
278 let region_routes = vec![RegionRoute {
279 region: Region::new_test(region_id),
280 leader_peer: Some(to_peer),
281 follower_peers: vec![from_peer],
282 ..Default::default()
283 }];
284
285 env.create_physical_table_metadata(table_info, region_routes)
286 .await;
287 let procedure_ctx = new_procedure_context();
288 let (next, status) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
289
290 let _ = next.as_any().downcast_ref::<RegionMigrationEnd>().unwrap();
291 assert_done(status);
292 assert_table_cache_invalidated(&mut rx, &[1024]);
293 }
294
295 #[tokio::test]
296 async fn test_next_migrates_remaining_regions() {
297 let mut state = Box::new(RegionMigrationStart);
298 let mut persistent_context = new_persistent_context();
301 let migrated = RegionId::new(1024, 1);
302 let remaining = RegionId::new(1025, 1);
303 persistent_context.region_ids = vec![migrated, remaining];
304 let mut env = TestingEnv::new();
305 let mut ctx = env.context_factory().new_context(persistent_context);
306 let mut rx = env.frontend_receiver().await;
307
308 for (region_id, leader) in [(migrated, 2), (remaining, 1)] {
309 let table_id = region_id.table_id();
310 let table_info = new_test_table_info_with_name(table_id, &format!("table_{table_id}"));
311 let region_routes = vec![RegionRoute {
312 region: Region::new_test(region_id),
313 leader_peer: Some(Peer::empty(leader)),
314 ..Default::default()
315 }];
316 env.create_physical_table_metadata(table_info, region_routes)
317 .await;
318 }
319 let procedure_ctx = new_procedure_context();
320 let (next, status) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
321
322 let _ = next.as_any().downcast_ref::<OpenCandidateRegion>().unwrap();
323 assert_need_persist(status);
324 assert_eq!(ctx.persistent_ctx.region_ids, vec![remaining]);
325 assert_table_cache_invalidated(&mut rx, &[1024, 1025]);
327 }
328
329 #[tokio::test]
330 async fn test_next_open_candidate_region_state() {
331 let mut state = Box::new(RegionMigrationStart);
332 let persistent_context = new_persistent_context();
335 let from_peer_id = persistent_context.from_peer.id;
336 let region_id = persistent_context.region_ids[0];
337 let mut env = TestingEnv::new();
338 let mut ctx = env.context_factory().new_context(persistent_context);
339 let mut rx = env.frontend_receiver().await;
340
341 let table_info = new_test_table_info(1024);
342 let region_routes = vec![RegionRoute {
343 region: Region::new_test(region_id),
344 leader_peer: Some(Peer::empty(from_peer_id)),
345 ..Default::default()
346 }];
347
348 env.create_physical_table_metadata(table_info, region_routes)
349 .await;
350 let procedure_ctx = new_procedure_context();
351 let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
352
353 let _ = next.as_any().downcast_ref::<OpenCandidateRegion>().unwrap();
354 assert!(rx.try_recv().is_err());
356 }
357
358 #[tokio::test]
359 async fn test_next_migration_abort() {
360 let mut state = Box::new(RegionMigrationStart);
361 let persistent_context = new_persistent_context();
364 let region_id = persistent_context.region_ids[0];
365 let env = TestingEnv::new();
366 let mut ctx = env.context_factory().new_context(persistent_context);
367
368 let table_info = new_test_table_info(1024);
369 let region_routes: Vec<RegionRoute> = vec![RegionRoute {
370 region: Region::new_test(region_id),
371 leader_peer: Some(Peer::empty(1024)),
372 ..Default::default()
373 }];
374
375 env.create_physical_table_metadata(table_info, region_routes)
376 .await;
377 let procedure_ctx = new_procedure_context();
378 let (next, status) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
379
380 let _ = next
381 .as_any()
382 .downcast_ref::<RegionMigrationAbort>()
383 .unwrap();
384 assert_done(status);
385 }
386
387 #[tokio::test]
388 async fn test_next_migration_abort_with_downgraded_leader() {
389 let mut state = Box::new(RegionMigrationStart);
390 let env = TestingEnv::new();
393 let mut persistent_context = new_persistent_context();
394 persistent_context.region_ids = env.create_batch_with_moved_leader(1).await;
395 let mut ctx = env.context_factory().new_context(persistent_context);
396 let procedure_ctx = new_procedure_context();
397 let (next, status) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
398
399 let _ = next
400 .as_any()
401 .downcast_ref::<RegionMigrationAbort>()
402 .unwrap();
403 assert!(!status.is_done());
405 }
406}