Skip to main content

meta_srv/procedure/region_migration/
migration_start.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16
17use common_meta::peer::Peer;
18use common_meta::rpc::router::RegionRoute;
19use common_procedure::{Context as ProcedureContext, Status};
20use common_telemetry::info;
21use serde::{Deserialize, Serialize};
22use snafu::{OptionExt, ResultExt};
23
24use crate::error::{self, Result};
25use crate::procedure::region_migration::migration_abort::RegionMigrationAbort;
26use crate::procedure::region_migration::migration_end::RegionMigrationEnd;
27use crate::procedure::region_migration::open_candidate_region::OpenCandidateRegion;
28use crate::procedure::region_migration::{Context, State};
29
30/// The behaviors:
31///
32/// - If all regions have been migrated, transitions to [RegionMigrationEnd].
33/// - If any of the region leaders is not the `from_peer`, transitions to [RegionMigrationAbort].
34/// - Otherwise, continues with [OpenCandidateRegion] to initiate the candidate region.
35///
36/// A recovered procedure starts here again, whatever state it was persisted in.
37#[derive(Debug, Serialize, Deserialize)]
38pub struct RegionMigrationStart;
39
40#[async_trait::async_trait]
41#[typetag::serde]
42impl State for RegionMigrationStart {
43    /// Yields next [State].
44    ///
45    /// Determines the next [State] for region migration:
46    ///
47    /// - If all regions have been migrated, transitions to [RegionMigrationEnd].
48    /// - If any of the region leaders is not the `from_peer`, transitions to [RegionMigrationAbort].
49    /// - Otherwise, continues with [OpenCandidateRegion] to initiate the candidate region.
50    async fn next(
51        &mut self,
52        ctx: &mut Context,
53        _procedure_ctx: &ProcedureContext,
54    ) -> Result<(Box<dyn State>, Status)> {
55        let mut region_routes = self.retrieve_region_routes(ctx).await?;
56        let to_peer = &ctx.persistent_ctx.to_peer;
57        let from_peer = &ctx.persistent_ctx.from_peer;
58        let region_ids = &ctx.persistent_ctx.region_ids;
59
60        // A recovered procedure may have downgraded the leaders of the regions. All the routes of
61        // the batch are checked, including those of the regions already led by the target.
62        let downgraded = region_routes
63            .iter()
64            .any(|route| route.is_leader_downgrading());
65
66        self.filter_unmigrated_regions(&mut region_routes, to_peer);
67
68        // A procedure that stopped right after it switched the routes has not broadcast them.
69        if region_routes.len() != region_ids.len() {
70            ctx.invalidate_table_cache().await?;
71        }
72
73        // No region to migrate, skip the migration.
74        if region_routes.is_empty() {
75            info!(
76                "All regions have been migrated, regions: {:?}, to_peer: {:?}",
77                region_ids, to_peer
78            );
79            return Ok((Box::new(RegionMigrationEnd), Status::done()));
80        }
81
82        // Checks if any of the region leaders is not the `from_peer`. The region ids are still
83        // those of the whole batch, which a rollback restores.
84        for region_route in &region_routes {
85            if self.invalid_leader_peer(region_route, from_peer)? {
86                info!(
87                    "Abort region migration, region:{}, unexpected leader peer: {:?}, expected: {:?}",
88                    region_route.region.id, region_route.leader_peer, from_peer,
89                );
90                // The abort fails the procedure, and its rollback restores the downgraded leaders.
91                let status = if downgraded {
92                    Status::executing(false)
93                } else {
94                    Status::done()
95                };
96                return Ok((
97                    Box::new(RegionMigrationAbort::new(&format!(
98                        "Invalid region leader peer: {:?}, expected: {:?}",
99                        region_route.leader_peer.as_ref().unwrap(),
100                        from_peer,
101                    ))),
102                    status,
103                ));
104            }
105        }
106
107        // Updates the region ids to the unmigrated regions.
108        if region_routes.len() != region_ids.len() {
109            let unmigrated_region_ids = region_routes.iter().map(|route| route.region.id).collect();
110            info!(
111                "Some of the regions have been migrated, only migrate the following regions: {:?}, to_peer: {:?}",
112                unmigrated_region_ids, to_peer
113            );
114            ctx.persistent_ctx.region_ids = unmigrated_region_ids;
115        }
116
117        // If all checks pass, open the candidate region.
118        Ok((Box::new(OpenCandidateRegion), Status::executing(true)))
119    }
120
121    fn as_any(&self) -> &dyn Any {
122        self
123    }
124}
125
126impl RegionMigrationStart {
127    /// Retrieves region routes for multiple regions.
128    ///
129    /// Abort(non-retry):
130    /// - TableRoute is not found.
131    /// - RegionRoute is not found.
132    ///
133    /// Retry:
134    /// - Failed to retrieve the metadata of table.
135    async fn retrieve_region_routes(&self, ctx: &mut Context) -> Result<Vec<RegionRoute>> {
136        let region_ids = &ctx.persistent_ctx.region_ids;
137        let table_route_values = ctx.get_table_route_values().await?;
138        let mut region_routes = Vec::with_capacity(region_ids.len());
139        for region_id in region_ids {
140            let table_id = region_id.table_id();
141            let region_route = table_route_values
142                .get(&table_id)
143                .context(error::TableRouteNotFoundSnafu { table_id })?
144                .region_routes()
145                .with_context(|_| error::UnexpectedLogicalRouteTableSnafu {
146                    err_msg: format!("TableRoute({table_id:?}) is a non-physical TableRouteValue."),
147                })?
148                .iter()
149                .find(|route| route.region.id == *region_id)
150                .cloned()
151                .with_context(|| error::UnexpectedSnafu {
152                    violated: format!(
153                        "RegionRoute({}) is not found in TableRoute({})",
154                        region_id, table_id
155                    ),
156                })?;
157            region_routes.push(region_route);
158        }
159
160        Ok(region_routes)
161    }
162
163    /// Returns true if the region leader is not the `from_peer`.
164    ///     
165    /// Abort(non-retry):
166    /// - Leader peer of RegionRoute is not found.
167    fn invalid_leader_peer(&self, region_route: &RegionRoute, from_peer: &Peer) -> Result<bool> {
168        let region_id = region_route.region.id;
169
170        let is_invalid_leader_peer = region_route
171            .leader_peer
172            .as_ref()
173            .with_context(|| error::UnexpectedSnafu {
174                violated: format!("Leader peer is not found in TableRoute({})", region_id),
175            })?
176            .id
177            != from_peer.id;
178        Ok(is_invalid_leader_peer)
179    }
180
181    /// Filters out regions that unmigrated.
182    fn filter_unmigrated_regions(&self, region_routes: &mut Vec<RegionRoute>, to_peer: &Peer) {
183        region_routes
184            .retain(|region_route| !self.has_migrated(region_route, to_peer).unwrap_or(false));
185    }
186
187    /// Checks whether the region has been migrated.
188    /// Returns true if it's.
189    ///     
190    /// Abort(non-retry):
191    /// - Leader peer of RegionRoute is not found.
192    fn has_migrated(&self, region_route: &RegionRoute, to_peer: &Peer) -> Result<bool> {
193        let region_id = region_route.region.id;
194
195        let region_migrated = region_route
196            .leader_peer
197            .as_ref()
198            .with_context(|| error::UnexpectedSnafu {
199                violated: format!("Leader peer is not found in TableRoute({})", region_id),
200            })?
201            .id
202            == to_peer.id;
203        Ok(region_migrated)
204    }
205}
206
207#[cfg(test)]
208mod tests {
209
210    use std::assert_matches;
211
212    use common_meta::key::test_utils::{new_test_table_info, new_test_table_info_with_name};
213    use common_meta::peer::Peer;
214    use common_meta::rpc::router::{Region, RegionRoute};
215    use store_api::storage::RegionId;
216
217    use super::*;
218    use crate::error::Error;
219    use crate::procedure::region_migration::test_util::{
220        self, TestingEnv, assert_done, assert_need_persist, assert_table_cache_invalidated,
221        new_procedure_context,
222    };
223    use crate::procedure::region_migration::{ContextFactory, PersistentContext};
224
225    fn new_persistent_context() -> PersistentContext {
226        test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
227    }
228
229    #[tokio::test]
230    async fn test_table_route_is_not_found_error() {
231        let state = RegionMigrationStart;
232        let env = TestingEnv::new();
233        let persistent_context = new_persistent_context();
234        let mut ctx = env.context_factory().new_context(persistent_context);
235        let err = state.retrieve_region_routes(&mut ctx).await.unwrap_err();
236        assert_matches!(err, Error::TableRouteNotFound { .. });
237        assert!(!err.is_retryable());
238    }
239
240    #[tokio::test]
241    async fn test_region_route_is_not_found_error() {
242        let state = RegionMigrationStart;
243        let persistent_context = new_persistent_context();
244        let from_peer = persistent_context.from_peer.clone();
245
246        let env = TestingEnv::new();
247        let mut ctx = env.context_factory().new_context(persistent_context);
248
249        let table_info = new_test_table_info(1024);
250        let region_route = RegionRoute {
251            region: Region::new_test(RegionId::new(1024, 3)),
252            leader_peer: Some(from_peer.clone()),
253            ..Default::default()
254        };
255
256        env.create_physical_table_metadata(table_info, vec![region_route])
257            .await;
258        let err = state.retrieve_region_routes(&mut ctx).await.unwrap_err();
259        assert_matches!(err, Error::Unexpected { .. });
260        assert!(!err.is_retryable());
261    }
262
263    #[tokio::test]
264    async fn test_next_migration_end_state() {
265        let mut state = Box::new(RegionMigrationStart);
266        // from_peer: 1
267        // to_peer: 2
268        let persistent_context = new_persistent_context();
269        let to_peer = persistent_context.to_peer.clone();
270        let from_peer = persistent_context.from_peer.clone();
271        let region_id = persistent_context.region_ids[0];
272
273        let mut env = TestingEnv::new();
274        let mut ctx = env.context_factory().new_context(persistent_context);
275        let mut rx = env.frontend_receiver().await;
276
277        let table_info = new_test_table_info(1024);
278        let region_routes = vec![RegionRoute {
279            region: Region::new_test(region_id),
280            leader_peer: Some(to_peer),
281            follower_peers: vec![from_peer],
282            ..Default::default()
283        }];
284
285        env.create_physical_table_metadata(table_info, region_routes)
286            .await;
287        let procedure_ctx = new_procedure_context();
288        let (next, status) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
289
290        let _ = next.as_any().downcast_ref::<RegionMigrationEnd>().unwrap();
291        assert_done(status);
292        assert_table_cache_invalidated(&mut rx, &[1024]);
293    }
294
295    #[tokio::test]
296    async fn test_next_migrates_remaining_regions() {
297        let mut state = Box::new(RegionMigrationStart);
298        // from_peer: 1
299        // to_peer: 2
300        let mut persistent_context = new_persistent_context();
301        let migrated = RegionId::new(1024, 1);
302        let remaining = RegionId::new(1025, 1);
303        persistent_context.region_ids = vec![migrated, remaining];
304        let mut env = TestingEnv::new();
305        let mut ctx = env.context_factory().new_context(persistent_context);
306        let mut rx = env.frontend_receiver().await;
307
308        for (region_id, leader) in [(migrated, 2), (remaining, 1)] {
309            let table_id = region_id.table_id();
310            let table_info = new_test_table_info_with_name(table_id, &format!("table_{table_id}"));
311            let region_routes = vec![RegionRoute {
312                region: Region::new_test(region_id),
313                leader_peer: Some(Peer::empty(leader)),
314                ..Default::default()
315            }];
316            env.create_physical_table_metadata(table_info, region_routes)
317                .await;
318        }
319        let procedure_ctx = new_procedure_context();
320        let (next, status) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
321
322        let _ = next.as_any().downcast_ref::<OpenCandidateRegion>().unwrap();
323        assert_need_persist(status);
324        assert_eq!(ctx.persistent_ctx.region_ids, vec![remaining]);
325        // The table of the migrated region is invalidated too.
326        assert_table_cache_invalidated(&mut rx, &[1024, 1025]);
327    }
328
329    #[tokio::test]
330    async fn test_next_open_candidate_region_state() {
331        let mut state = Box::new(RegionMigrationStart);
332        // from_peer: 1
333        // to_peer: 2
334        let persistent_context = new_persistent_context();
335        let from_peer_id = persistent_context.from_peer.id;
336        let region_id = persistent_context.region_ids[0];
337        let mut env = TestingEnv::new();
338        let mut ctx = env.context_factory().new_context(persistent_context);
339        let mut rx = env.frontend_receiver().await;
340
341        let table_info = new_test_table_info(1024);
342        let region_routes = vec![RegionRoute {
343            region: Region::new_test(region_id),
344            leader_peer: Some(Peer::empty(from_peer_id)),
345            ..Default::default()
346        }];
347
348        env.create_physical_table_metadata(table_info, region_routes)
349            .await;
350        let procedure_ctx = new_procedure_context();
351        let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
352
353        let _ = next.as_any().downcast_ref::<OpenCandidateRegion>().unwrap();
354        // No route has been switched.
355        assert!(rx.try_recv().is_err());
356    }
357
358    #[tokio::test]
359    async fn test_next_migration_abort() {
360        let mut state = Box::new(RegionMigrationStart);
361        // from_peer: 1
362        // to_peer: 2
363        let persistent_context = new_persistent_context();
364        let region_id = persistent_context.region_ids[0];
365        let env = TestingEnv::new();
366        let mut ctx = env.context_factory().new_context(persistent_context);
367
368        let table_info = new_test_table_info(1024);
369        let region_routes: Vec<RegionRoute> = vec![RegionRoute {
370            region: Region::new_test(region_id),
371            leader_peer: Some(Peer::empty(1024)),
372            ..Default::default()
373        }];
374
375        env.create_physical_table_metadata(table_info, region_routes)
376            .await;
377        let procedure_ctx = new_procedure_context();
378        let (next, status) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
379
380        let _ = next
381            .as_any()
382            .downcast_ref::<RegionMigrationAbort>()
383            .unwrap();
384        assert_done(status);
385    }
386
387    #[tokio::test]
388    async fn test_next_migration_abort_with_downgraded_leader() {
389        let mut state = Box::new(RegionMigrationStart);
390        // from_peer: 1
391        // to_peer: 2
392        let env = TestingEnv::new();
393        let mut persistent_context = new_persistent_context();
394        persistent_context.region_ids = env.create_batch_with_moved_leader(1).await;
395        let mut ctx = env.context_factory().new_context(persistent_context);
396        let procedure_ctx = new_procedure_context();
397        let (next, status) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
398
399        let _ = next
400            .as_any()
401            .downcast_ref::<RegionMigrationAbort>()
402            .unwrap();
403        // The abort runs, so the procedure fails and rolls back.
404        assert!(!status.is_done());
405    }
406}