Skip to main content

meta_srv/procedure/drain_datanode/
utils.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub(crate) mod batch_aggregator;
16pub(crate) mod scan;
17
18use common_meta::DatanodeId;
19use common_meta::key::TableMetadataManagerRef;
20use common_meta::key::table_route::TableRouteValue;
21use common_meta::peer::Peer;
22use common_time::util::current_time_millis;
23use snafu::ResultExt;
24use store_api::storage::RegionId;
25
26use crate::error::{self, Result};
27use crate::metasrv::SelectorContext;
28use crate::procedure::drain_datanode::Context;
29use crate::selector::common;
30
31/// Returns true if `node_id` still leads `region_id`.
32///
33/// The table route is authoritative, so a region that is not listed anymore is reported as
34/// not led by the datanode.
35pub(crate) async fn leads_region(
36    table_metadata_manager: &TableMetadataManagerRef,
37    node_id: DatanodeId,
38    region_id: RegionId,
39) -> Result<bool> {
40    let Some(table_route) = table_metadata_manager
41        .table_route_manager()
42        .table_route_storage()
43        .get(region_id.table_id())
44        .await
45        .context(error::TableMetadataManagerSnafu)?
46    else {
47        return Ok(false);
48    };
49    let TableRouteValue::Physical(table_route) = table_route else {
50        return Ok(false);
51    };
52    let Some(region_route) = table_route
53        .region_routes
54        .iter()
55        .find(|region_route| region_route.region.id == region_id)
56    else {
57        return Ok(false);
58    };
59
60    Ok(region_route.leader_peer.as_ref().map(|peer| peer.id) == Some(node_id))
61}
62
63/// Returns an error if the drain exceeded its deadline.
64pub(crate) fn ensure_drain_deadline(ctx: &Context) -> Result<()> {
65    if current_time_millis() < ctx.persistent_ctx.deadline_ms {
66        return Ok(());
67    }
68
69    error::ExceededDeadlineSnafu {
70        operation: format!(
71            "Drain datanode {}, migrated: {}",
72            ctx.persistent_ctx.node_id, ctx.volatile_ctx.migrated
73        ),
74    }
75    .fail()
76}
77
78/// Selects the migration destination of `region_id` off the drained datanode.
79///
80/// Returns the pinned target when the procedure has one, otherwise the schedulable
81/// datanode with the smallest peer id. Candidates come from the
82/// peer discovery, so the returned peer carries its address and was not marked
83/// unschedulable; the drained datanode and the existing followers are excluded.
84pub(crate) async fn select_migration_target(
85    selector_ctx: &SelectorContext,
86    node_id: DatanodeId,
87    target_node_id: Option<DatanodeId>,
88    region_id: RegionId,
89    follower_peers: &[Peer],
90) -> Result<Peer> {
91    let target = common::active_schedulable_datanodes(selector_ctx, None)
92        .await?
93        .into_iter()
94        .map(|datanode| datanode.peer)
95        .filter(|peer| peer.id != node_id)
96        .filter(|peer| target_node_id.is_none_or(|target| peer.id == target))
97        .filter(|peer| !follower_peers.iter().any(|follower| follower.id == peer.id))
98        .min_by_key(|peer| peer.id);
99
100    match target {
101        Some(peer) => Ok(peer),
102        None => error::DrainDatanodeAbortSnafu {
103            node_id,
104            failed_regions: vec![region_id],
105        }
106        .fail(),
107    }
108}
109
110#[cfg(test)]
111mod tests {
112    use common_meta::peer::Peer;
113    use store_api::storage::RegionId;
114
115    use crate::error;
116    use crate::key::NodeSchedulingRole;
117    use crate::procedure::drain_datanode::test_util::{TestEnv, new_persistent_context};
118    use crate::procedure::drain_datanode::{DrainDatanodeStrategy, utils};
119
120    #[tokio::test]
121    async fn test_select_migration_target() {
122        let env = TestEnv::new().await;
123        let region_id = RegionId::new(1024, 1);
124
125        let mut ctx = env.context(new_persistent_context(
126            None,
127            DrainDatanodeStrategy::FailFast,
128        ));
129        let peer = utils::select_migration_target(
130            &ctx.selector_ctx,
131            ctx.persistent_ctx.node_id,
132            ctx.persistent_ctx.target_node_id,
133            region_id,
134            &[],
135        )
136        .await
137        .unwrap();
138        assert_eq!(2, peer.id);
139
140        // The existing followers and the drained datanode are excluded.
141        let peer = utils::select_migration_target(
142            &ctx.selector_ctx,
143            ctx.persistent_ctx.node_id,
144            ctx.persistent_ctx.target_node_id,
145            region_id,
146            &[Peer::empty(2)],
147        )
148        .await
149        .unwrap();
150        assert_eq!(3, peer.id);
151        assert_eq!("dn3:3001", peer.addr);
152
153        // A pinned target is resolved to the active peer, so it carries its address.
154        ctx.persistent_ctx.target_node_id = Some(2);
155        let peer = utils::select_migration_target(
156            &ctx.selector_ctx,
157            ctx.persistent_ctx.node_id,
158            ctx.persistent_ctx.target_node_id,
159            region_id,
160            &[],
161        )
162        .await
163        .unwrap();
164        assert_eq!(2, peer.id);
165        assert_eq!("dn2:3001", peer.addr);
166
167        // A target that is already a follower, the drained datanode itself or an
168        // unschedulable datanode is rejected.
169        ctx.persistent_ctx.target_node_id = Some(2);
170        let error = utils::select_migration_target(
171            &ctx.selector_ctx,
172            ctx.persistent_ctx.node_id,
173            ctx.persistent_ctx.target_node_id,
174            region_id,
175            &[Peer::empty(2)],
176        )
177        .await
178        .unwrap_err();
179        assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
180
181        ctx.persistent_ctx.target_node_id = Some(1);
182        let error = utils::select_migration_target(
183            &ctx.selector_ctx,
184            ctx.persistent_ctx.node_id,
185            ctx.persistent_ctx.target_node_id,
186            region_id,
187            &[],
188        )
189        .await
190        .unwrap_err();
191        assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
192
193        env.node_scheduling_manager
194            .mark_unschedulable(NodeSchedulingRole::Datanode, 3)
195            .await
196            .unwrap();
197        ctx.persistent_ctx.target_node_id = Some(3);
198        let error = utils::select_migration_target(
199            &ctx.selector_ctx,
200            ctx.persistent_ctx.node_id,
201            ctx.persistent_ctx.target_node_id,
202            region_id,
203            &[],
204        )
205        .await
206        .unwrap_err();
207        assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
208    }
209}