Skip to main content

meta_srv/procedure/region_migration/update_metadata/
rollback_downgraded_region.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use common_error::ext::BoxedError;
16use common_meta::lock_key::TableLock;
17use common_procedure::ContextProviderRef;
18use common_telemetry::{error, info};
19use snafu::ResultExt;
20
21use crate::error::{self, Result};
22use crate::procedure::region_migration::Context;
23use crate::procedure::region_migration::update_metadata::UpdateMetadata;
24
25impl UpdateMetadata {
26    /// Rollbacks the downgraded leader region if the candidate region is unreachable.
27    ///
28    /// Abort(non-retry):
29    /// - TableRoute is not found.
30    ///
31    /// Retry:
32    /// - Failed to update [TableRouteValue](common_meta::key::table_region::TableRegionValue).
33    /// - Failed to retrieve the metadata of table.
34    pub async fn rollback_downgraded_region(
35        &self,
36        ctx: &mut Context,
37        ctx_provider: &ContextProviderRef,
38    ) -> Result<()> {
39        let table_metadata_manager = ctx.table_metadata_manager.clone();
40        let table_regions = ctx.persistent_ctx.table_regions();
41
42        for (table_id, regions) in table_regions {
43            let table_lock = TableLock::Write(table_id).into();
44            let _guard = ctx_provider.acquire_lock(&table_lock).await;
45
46            let current_table_route_value = ctx.get_table_route_value(table_id).await?;
47            if let Err(err) = table_metadata_manager
48                .update_leader_region_status(table_id, &current_table_route_value, |route| {
49                    if regions.contains(&route.region.id) {
50                        Some(None)
51                    } else {
52                        None
53                    }
54                })
55                .await
56                .context(error::TableMetadataManagerSnafu)
57            {
58                error!(err; "Failed to update the table route during the rollback downgraded leader regions: {regions:?}");
59                return Err(BoxedError::new(err)).with_context(|_| error::RetryLaterWithSourceSnafu {
60                    reason: format!("Failed to update the table route during the rollback downgraded leader regions: {regions:?}"),
61                });
62            }
63            info!(
64                "Rolling back downgraded leader region table route success, table_id: {table_id}, regions: {regions:?}"
65            );
66        }
67        ctx.register_failure_detectors(&ctx.persistent_ctx.region_ids)
68            .await;
69
70        Ok(())
71    }
72}
73
74#[cfg(test)]
75mod tests {
76    use std::assert_matches;
77    use std::collections::HashMap;
78    use std::sync::Arc;
79
80    use common_meta::key::test_utils::new_test_table_info;
81    use common_meta::peer::Peer;
82    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
83    use common_procedure_test::MockContextProvider;
84    use store_api::storage::RegionId;
85
86    use crate::error::Error;
87    use crate::procedure::region_migration::migration_abort::RegionMigrationAbort;
88    use crate::procedure::region_migration::test_util::{
89        self, CandidateRollback, TestingEnv, new_procedure_context,
90    };
91    use crate::procedure::region_migration::update_metadata::UpdateMetadata;
92    use crate::procedure::region_migration::{ContextFactory, PersistentContext, State};
93
94    fn new_persistent_context() -> PersistentContext {
95        test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
96    }
97
98    #[tokio::test]
99    async fn test_table_route_is_not_found_error() {
100        let state = UpdateMetadata::Rollback;
101        let env = TestingEnv::new();
102        let persistent_context = new_persistent_context();
103        let mut ctx = env.context_factory().new_context(persistent_context);
104
105        let provider = Arc::new(MockContextProvider::new(HashMap::new())) as _;
106        let err = state
107            .rollback_downgraded_region(&mut ctx, &provider)
108            .await
109            .unwrap_err();
110
111        assert_matches!(err, Error::TableRouteNotFound { .. });
112
113        assert!(!err.is_retryable());
114    }
115
116    #[tokio::test]
117    async fn test_next_migration_end_state() {
118        let mut state = Box::new(UpdateMetadata::Rollback);
119        let persistent_context = new_persistent_context();
120        let from_peer = persistent_context.from_peer.clone();
121
122        let env = TestingEnv::new();
123        let mut ctx = env.context_factory().new_context(persistent_context);
124        let table_id = ctx.persistent_ctx.region_ids[0].table_id();
125
126        let table_info = new_test_table_info(1024);
127        let region_routes = vec![
128            RegionRoute {
129                region: Region::new_test(RegionId::new(1024, 1)),
130                leader_peer: Some(from_peer.clone()),
131                leader_state: Some(LeaderState::Downgrading),
132                ..Default::default()
133            },
134            RegionRoute {
135                region: Region::new_test(RegionId::new(1024, 2)),
136                leader_peer: Some(Peer::empty(4)),
137                leader_state: Some(LeaderState::Downgrading),
138                ..Default::default()
139            },
140            RegionRoute {
141                region: Region::new_test(RegionId::new(1024, 3)),
142                leader_peer: Some(Peer::empty(5)),
143                ..Default::default()
144            },
145        ];
146
147        let expected_region_routes = {
148            let mut region_routes = region_routes.clone();
149            region_routes[0].leader_state = None;
150            region_routes
151        };
152
153        env.create_physical_table_metadata(table_info, region_routes)
154            .await;
155
156        let table_metadata_manager = env.table_metadata_manager();
157
158        let procedure_ctx = new_procedure_context();
159        let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
160
161        let _ = next
162            .as_any()
163            .downcast_ref::<RegionMigrationAbort>()
164            .unwrap();
165
166        let table_route = table_metadata_manager
167            .table_route_manager()
168            .table_route_storage()
169            .get(table_id)
170            .await
171            .unwrap()
172            .unwrap();
173        assert_eq!(
174            &expected_region_routes,
175            table_route.region_routes().unwrap()
176        );
177    }
178
179    #[tokio::test]
180    async fn test_next_drops_candidate_registration_before_route_write() {
181        let env = TestingEnv::new();
182        let persistent_context = new_persistent_context();
183        let rollback = CandidateRollback::new(&env, &persistent_context).await;
184        let mut ctx = rollback.context_factory.new_context(persistent_context);
185        ctx.volatile_ctx.opening_region_guards.push(rollback.guard);
186
187        UpdateMetadata::Rollback
188            .next(&mut ctx, &new_procedure_context())
189            .await
190            .unwrap();
191
192        test_util::assert_registration_dropped_before_route_write(&rollback.keeper_empty_at_txn);
193    }
194}