meta_srv/procedure/region_migration/update_metadata/
rollback_downgraded_region.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use common_error::ext::BoxedError;
16use common_meta::lock_key::TableLock;
17use common_procedure::ContextProviderRef;
18use common_telemetry::{error, info};
19use snafu::ResultExt;
20
21use crate::error::{self, Result};
22use crate::procedure::region_migration::Context;
23use crate::procedure::region_migration::update_metadata::UpdateMetadata;
24
25impl UpdateMetadata {
26    /// Rollbacks the downgraded leader region if the candidate region is unreachable.
27    ///
28    /// Abort(non-retry):
29    /// - TableRoute is not found.
30    ///
31    /// Retry:
32    /// - Failed to update [TableRouteValue](common_meta::key::table_region::TableRegionValue).
33    /// - Failed to retrieve the metadata of table.
34    pub async fn rollback_downgraded_region(
35        &self,
36        ctx: &mut Context,
37        ctx_provider: &ContextProviderRef,
38    ) -> Result<()> {
39        let table_metadata_manager = ctx.table_metadata_manager.clone();
40        let table_regions = ctx.persistent_ctx.table_regions();
41
42        for (table_id, regions) in table_regions {
43            let table_lock = TableLock::Write(table_id).into();
44            let _guard = ctx_provider.acquire_lock(&table_lock).await;
45
46            let current_table_route_value = ctx.get_table_route_value(table_id).await?;
47            if let Err(err) = table_metadata_manager
48                .update_leader_region_status(table_id, &current_table_route_value, |route| {
49                    if regions.contains(&route.region.id) {
50                        Some(None)
51                    } else {
52                        None
53                    }
54                })
55                .await
56                .context(error::TableMetadataManagerSnafu)
57            {
58                error!(err; "Failed to update the table route during the rollback downgraded leader regions: {regions:?}");
59                return Err(BoxedError::new(err)).with_context(|_| error::RetryLaterWithSourceSnafu {
60                    reason: format!("Failed to update the table route during the rollback downgraded leader regions: {regions:?}"),
61                });
62            }
63            info!(
64                "Rolling back downgraded leader region table route success, table_id: {table_id}, regions: {regions:?}"
65            );
66        }
67        ctx.register_failure_detectors(&ctx.persistent_ctx.region_ids)
68            .await;
69
70        Ok(())
71    }
72}
73
74#[cfg(test)]
75mod tests {
76    use std::assert_matches;
77    use std::collections::HashMap;
78    use std::sync::Arc;
79
80    use common_meta::key::test_utils::new_test_table_info;
81    use common_meta::peer::Peer;
82    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
83    use common_procedure_test::MockContextProvider;
84    use store_api::storage::RegionId;
85
86    use crate::error::Error;
87    use crate::procedure::region_migration::migration_abort::RegionMigrationAbort;
88    use crate::procedure::region_migration::test_util::{self, TestingEnv, new_procedure_context};
89    use crate::procedure::region_migration::update_metadata::UpdateMetadata;
90    use crate::procedure::region_migration::{ContextFactory, PersistentContext, State};
91
92    fn new_persistent_context() -> PersistentContext {
93        test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
94    }
95
96    #[tokio::test]
97    async fn test_table_route_is_not_found_error() {
98        let state = UpdateMetadata::Rollback;
99        let env = TestingEnv::new();
100        let persistent_context = new_persistent_context();
101        let mut ctx = env.context_factory().new_context(persistent_context);
102
103        let provider = Arc::new(MockContextProvider::new(HashMap::new())) as _;
104        let err = state
105            .rollback_downgraded_region(&mut ctx, &provider)
106            .await
107            .unwrap_err();
108
109        assert_matches!(err, Error::TableRouteNotFound { .. });
110
111        assert!(!err.is_retryable());
112    }
113
114    #[tokio::test]
115    async fn test_next_migration_end_state() {
116        let mut state = Box::new(UpdateMetadata::Rollback);
117        let persistent_context = new_persistent_context();
118        let from_peer = persistent_context.from_peer.clone();
119
120        let env = TestingEnv::new();
121        let mut ctx = env.context_factory().new_context(persistent_context);
122        let table_id = ctx.persistent_ctx.region_ids[0].table_id();
123
124        let table_info = new_test_table_info(1024);
125        let region_routes = vec![
126            RegionRoute {
127                region: Region::new_test(RegionId::new(1024, 1)),
128                leader_peer: Some(from_peer.clone()),
129                leader_state: Some(LeaderState::Downgrading),
130                ..Default::default()
131            },
132            RegionRoute {
133                region: Region::new_test(RegionId::new(1024, 2)),
134                leader_peer: Some(Peer::empty(4)),
135                leader_state: Some(LeaderState::Downgrading),
136                ..Default::default()
137            },
138            RegionRoute {
139                region: Region::new_test(RegionId::new(1024, 3)),
140                leader_peer: Some(Peer::empty(5)),
141                ..Default::default()
142            },
143        ];
144
145        let expected_region_routes = {
146            let mut region_routes = region_routes.clone();
147            region_routes[0].leader_state = None;
148            region_routes
149        };
150
151        env.create_physical_table_metadata(table_info, region_routes)
152            .await;
153
154        let table_metadata_manager = env.table_metadata_manager();
155
156        let procedure_ctx = new_procedure_context();
157        let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
158
159        let _ = next
160            .as_any()
161            .downcast_ref::<RegionMigrationAbort>()
162            .unwrap();
163
164        let table_route = table_metadata_manager
165            .table_route_manager()
166            .table_route_storage()
167            .get(table_id)
168            .await
169            .unwrap()
170            .unwrap();
171        assert_eq!(
172            &expected_region_routes,
173            table_route.region_routes().unwrap()
174        );
175    }
176}
meta_srv/procedure/region_migration/update_metadata/rollback_downgraded_region.rs

meta_srv/procedure/region_migration/update_metadata/
rollback_downgraded_region.rs
