Skip to main content

meta_srv/procedure/
drain_datanode.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub(crate) mod cursor;
16pub(crate) mod dispatch;
17pub(crate) mod drain_datanode_end;
18pub(crate) mod drain_datanode_start;
19pub(crate) mod manager;
20pub(crate) mod utils;
21
22#[cfg(test)]
23mod test_util;
24
25use std::any::Any;
26use std::fmt::Debug;
27use std::time::Duration;
28
29use common_meta::DatanodeId;
30use common_meta::key::TableMetadataManagerRef;
31use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
32use common_procedure::{
33    Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure, ProcedureId,
34    Result as ProcedureResult, Status,
35};
36use common_telemetry::{error, info};
37use serde::{Deserialize, Serialize};
38use snafu::ResultExt;
39use store_api::storage::RegionId;
40
41use crate::error::{self, Result};
42use crate::metasrv::SelectorContext;
43use crate::metrics::METRIC_META_REGION_MIGRATION_FAIL;
44use crate::node_scheduling::NodeSchedulingManagerRef;
45use crate::procedure::drain_datanode::drain_datanode_start::DrainDatanodeStart;
46use crate::procedure::drain_datanode::manager::{
47    DrainDatanodeProcedureGuard, DrainDatanodeProcedureTracker,
48};
49use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
50
51/// The default number of leader regions migrated concurrently.
52pub const DEFAULT_PARALLELISM: usize = 4;
53/// The maximum number of leader regions migrated concurrently.
54pub const MAX_PARALLELISM: usize = 32;
55/// The default budget of the whole drain.
56pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(20 * 60);
57/// The maximum number of scan starts per run. The initial scan, each completion
58/// confirmation scan, and each restart after a read failure consume one attempt.
59pub const MAX_RESCAN_ROUNDS: u32 = 3;
60
61/// The strategy of a drain run when a region cannot be migrated.
62#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
63pub enum DrainDatanodeStrategy {
64    /// Aborts the procedure once every migration of the batch is settled and at least one
65    /// of them failed.
66    FailFast,
67    /// Drains every remaining region and records the failed ones.
68    #[default]
69    TryAll,
70}
71
72/// The persistent data of [`DrainDatanodeProcedure`].
73///
74/// Only inputs that cannot be derived from cluster metadata are persisted. After a
75/// restart, a fresh scan discovers remaining migrations and progress counters restart.
76#[derive(Debug, Clone, Serialize, Deserialize)]
77pub struct PersistentContext {
78    /// The datanode to drain.
79    pub node_id: DatanodeId,
80    /// The pinned destination of leader regions. Resolved by the selector when `None`.
81    pub target_node_id: Option<DatanodeId>,
82    /// The number of leader regions migrated concurrently.
83    pub parallelism: usize,
84    /// The error handling strategy.
85    pub strategy: DrainDatanodeStrategy,
86    /// The absolute deadline of the whole drain, in milliseconds since the Unix epoch.
87    pub deadline_ms: i64,
88}
89
90/// A dispatched migration sub-procedure that is not settled yet.
91#[derive(Debug, Clone)]
92pub struct SubprocedureMeta {
93    /// The regions migrated by the sub-procedure.
94    pub region_ids: Vec<RegionId>,
95    /// The procedure id of the sub-procedure.
96    pub procedure_id: ProcedureId,
97}
98
99/// The in-memory data of [`DrainDatanodeProcedure`].
100///
101/// It is not persisted: a restart begins a fresh scan. Scans are bounded by
102/// [`MAX_RESCAN_ROUNDS`] per run and rediscover regions that are still led by
103/// the drained datanode and have no running migration. Recovered migrations own their
104/// lifecycle and may finish after this drain procedure ends.
105#[derive(Debug, Default)]
106pub struct VolatileContext {
107    /// Paused metadata scan; dropped and rebuilt after a metadata read error.
108    pub(crate) scan: Option<utils::scan::RegionScan>,
109    /// Number of followers seen in the current scan.
110    pub follower_count: usize,
111    /// The number of regions confirmed to have left the source while collecting batch results.
112    pub migrated: usize,
113    /// The regions given up by [`DrainDatanodeStrategy::TryAll`] in this run.
114    pub failed_regions: Vec<RegionId>,
115    /// The number of full scan attempts in this run, including failed reads.
116    pub rescan_round: u32,
117}
118
119/// The task of [`DrainDatanodeProcedure`].
120#[derive(Debug, Clone)]
121pub struct DrainDatanodeProcedureTask {
122    /// The datanode to drain.
123    pub node_id: DatanodeId,
124    /// The pinned destination of leader regions. Resolved by the selector when `None`.
125    pub target_node_id: Option<DatanodeId>,
126    /// The number of leader regions migrated concurrently.
127    pub parallelism: usize,
128    /// The error handling strategy.
129    pub strategy: DrainDatanodeStrategy,
130    /// The budget of the whole drain. `None` uses [`DEFAULT_DRAIN_TIMEOUT`].
131    pub timeout: Option<Duration>,
132}
133
134/// The context of [`DrainDatanodeProcedure`].
135pub struct Context {
136    /// The persistent data of the procedure.
137    pub persistent_ctx: PersistentContext,
138    /// The in-memory data of the procedure.
139    pub volatile_ctx: VolatileContext,
140    /// The table metadata manager.
141    pub table_metadata_manager: TableMetadataManagerRef,
142    /// The node scheduling manager, used to fence the drained datanode.
143    pub node_scheduling_manager: NodeSchedulingManagerRef,
144    /// The region migration manager, used to build migration sub-procedures.
145    pub region_migration_manager: RegionMigrationManagerRef,
146    /// The selector context, used to resolve the destination of a leader region.
147    pub selector_ctx: SelectorContext,
148}
149
150impl Context {
151    /// Builds the context of a run.
152    ///
153    /// The per-run state starts empty. Scanning authoritative table routes discovers
154    /// the remaining work; progress is counted as migrations settle.
155    pub(crate) fn new(
156        table_metadata_manager: TableMetadataManagerRef,
157        node_scheduling_manager: NodeSchedulingManagerRef,
158        region_migration_manager: RegionMigrationManagerRef,
159        selector_ctx: SelectorContext,
160
161        persistent_ctx: PersistentContext,
162    ) -> Self {
163        Self {
164            persistent_ctx,
165            volatile_ctx: VolatileContext::default(),
166            table_metadata_manager,
167            node_scheduling_manager,
168            region_migration_manager,
169            selector_ctx,
170        }
171    }
172}
173
174impl Context {
175    /// Applies the failure strategy to regions that could not be prepared or dispatched.
176    pub(crate) fn handle_migration_error(
177        &mut self,
178        e: error::Error,
179        regions: &[RegionId],
180    ) -> Result<()> {
181        if self.persistent_ctx.strategy == DrainDatanodeStrategy::FailFast {
182            return Err(e);
183        }
184        error!(e; "Gave up draining regions {:?}", regions);
185        METRIC_META_REGION_MIGRATION_FAIL.inc_by(regions.len() as u64);
186        self.volatile_ctx.failed_regions.extend_from_slice(regions);
187        Ok(())
188    }
189}
190
191/// The persistent data of [`DrainDatanodeProcedure`] as read by `from_json`.
192#[derive(Debug, Deserialize)]
193pub struct DrainDatanodeDataOwned {
194    persistent_ctx: PersistentContext,
195    state: Box<dyn State>,
196}
197
198/// The persistent data of [`DrainDatanodeProcedure`] as written by `dump`.
199#[derive(Debug, Serialize)]
200pub struct DrainDatanodeData<'a> {
201    persistent_ctx: &'a PersistentContext,
202    state: &'a dyn State,
203}
204
205/// A state of [`DrainDatanodeProcedure`].
206#[async_trait::async_trait]
207#[typetag::serde(tag = "drain_datanode_state")]
208pub(crate) trait State: Sync + Send + Debug {
209    /// Returns the short name of this state.
210    fn name(&self) -> &'static str {
211        let type_name = std::any::type_name::<Self>();
212        type_name.split("::").last().unwrap_or(type_name)
213    }
214
215    /// Yields the next [`State`] and [`Status`].
216    async fn next(
217        &mut self,
218        ctx: &mut Context,
219        procedure_ctx: &ProcedureContext,
220    ) -> Result<(Box<dyn State>, Status)>;
221
222    /// Returns as [`Any`].
223    fn as_any(&self) -> &dyn Any;
224}
225
226/// The procedure draining a datanode.
227///
228/// It fences the datanode, migrates every leader region off it through
229/// [`RegionMigrationProcedure`](crate::procedure::region_migration::RegionMigrationProcedure)
230/// sub-procedures, and rescans for regions that still need a new migration. Migrations
231/// already registered in the tracker continue independently and may outlive this
232/// procedure. Follower regions are counted but not removed yet.
233pub(crate) struct DrainDatanodeProcedure {
234    state: Box<dyn State>,
235    context: Context,
236    /// Keeps the drained datanode in the running set of the manager until the procedure ends,
237    /// so a second drain of the datanode is rejected at submission instead of being queued.
238    _guards: Vec<DrainDatanodeProcedureGuard>,
239}
240
241impl DrainDatanodeProcedure {
242    const TYPE_NAME: &'static str = "metasrv-procedure::DrainDatanode";
243
244    /// Returns a new [`DrainDatanodeProcedure`] in its initial state.
245    pub fn new(context: Context, guards: Vec<DrainDatanodeProcedureGuard>) -> Self {
246        Self {
247            state: Box::new(DrainDatanodeStart),
248            context,
249            _guards: guards,
250        }
251    }
252
253    /// Returns a new [`DrainDatanodeProcedure`] from the given serialized data.
254    ///
255    /// The procedure takes its place back in `tracker`, so a reloaded drain still rejects a
256    /// second drain of the same datanode.
257    pub fn from_json<F>(
258        json: &str,
259        tracker: &DrainDatanodeProcedureTracker,
260        ctx_factory: F,
261    ) -> ProcedureResult<Self>
262    where
263        F: FnOnce(PersistentContext) -> Context,
264    {
265        let DrainDatanodeDataOwned {
266            persistent_ctx,
267            state,
268        } = serde_json::from_str(json).context(FromJsonSnafu)?;
269        let guards = tracker
270            .insert_running_procedure(&DrainDatanodeProcedureTask {
271                node_id: persistent_ctx.node_id,
272                target_node_id: persistent_ctx.target_node_id,
273                parallelism: persistent_ctx.parallelism,
274                strategy: persistent_ctx.strategy,
275                timeout: None,
276            })
277            .into_iter()
278            .collect();
279
280        let context = ctx_factory(persistent_ctx);
281        Ok(Self {
282            state,
283            context,
284            _guards: guards,
285        })
286    }
287}
288
289#[async_trait::async_trait]
290impl Procedure for DrainDatanodeProcedure {
291    fn type_name(&self) -> &str {
292        Self::TYPE_NAME
293    }
294
295    #[tracing::instrument(skip_all, fields(
296        state = %self.state.name(),
297        node_id = %self.context.persistent_ctx.node_id
298    ))]
299    async fn execute(&mut self, ctx: &ProcedureContext) -> ProcedureResult<Status> {
300        let state_name = self.state.name();
301        info!(
302            "Drain datanode procedure executing state: {}, node_id: {}",
303            state_name, self.context.persistent_ctx.node_id
304        );
305        match self.state.next(&mut self.context, ctx).await {
306            Ok((next, status)) => {
307                self.state = next;
308                Ok(status)
309            }
310            Err(e) => {
311                if e.is_retryable() {
312                    Err(ProcedureError::retry_later(e))
313                } else {
314                    error!(
315                        e;
316                        "Drain datanode procedure failed, node_id: {}",
317                        self.context.persistent_ctx.node_id
318                    );
319                    Err(ProcedureError::external(e))
320                }
321            }
322        }
323    }
324
325    fn dump(&self) -> ProcedureResult<String> {
326        let data = DrainDatanodeData {
327            persistent_ctx: &self.context.persistent_ctx,
328            state: self.state.as_ref(),
329        };
330        serde_json::to_string(&data).context(ToJsonSnafu)
331    }
332
333    fn lock_key(&self) -> LockKey {
334        LockKey::single_exclusive(format!(
335            "__drain_datanode/{}",
336            self.context.persistent_ctx.node_id
337        ))
338    }
339}
340
341#[cfg(test)]
342mod tests {
343    use std::collections::HashMap;
344    use std::sync::Arc;
345    use std::time::Duration;
346
347    use crate::procedure::drain_datanode::cursor::DrainDatanodeCursor;
348    use crate::procedure::drain_datanode::dispatch::Dispatch;
349    use crate::procedure::drain_datanode::test_util::create_kafka_table;
350    use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
351    use api::v1::meta::heartbeat_request::NodeWorkloads;
352    use common_error::ext::ErrorExt;
353    use common_error::status_code::StatusCode;
354    use common_meta::cluster::NodeInfo;
355    use common_meta::error::Result as MetaResult;
356    use common_meta::key::TableMetadataManagerRef;
357    use common_meta::peer::{PeerDiscovery, PeerDiscoveryRef};
358    use common_meta::state_store::KvStateStore;
359    use common_procedure::store::ProcedureMessage;
360    use common_procedure::store::state_store::StateStore;
361    use common_procedure::{Procedure, ProcedureId, ProcedureState, Status};
362    use futures::TryStreamExt;
363    use store_api::storage::RegionId;
364
365    use crate::error;
366    use crate::key::NodeSchedulingRole;
367    use crate::procedure::drain_datanode::drain_datanode_start::DrainDatanodeStart;
368    use crate::procedure::drain_datanode::manager::DrainDatanodeProcedureTracker;
369    use crate::procedure::drain_datanode::test_util::{
370        TestEnv, follower_region, leader_region, new_persistent_context, run_until_done,
371        spawn_mock_datanode, wait_for_terminal_state,
372    };
373    use crate::procedure::drain_datanode::{
374        DEFAULT_PARALLELISM, DrainDatanodeProcedure, DrainDatanodeProcedureTask,
375        DrainDatanodeStrategy, State,
376    };
377    use crate::procedure::region_migration::test_util::TestingEnv;
378    use crate::service::mailbox::Channel;
379
380    #[tokio::test]
381    async fn test_drain_datanode_fences_and_completes_without_leaders() {
382        let env = TestEnv::new().await;
383        env.create_table(vec![follower_region(RegionId::new(1024, 1), 2, 1)])
384            .await;
385
386        let mut ctx = env.context(new_persistent_context(
387            None,
388            DrainDatanodeStrategy::FailFast,
389        ));
390        let procedure_ctx = TestingEnv::procedure_context();
391        let mut state: Box<dyn State> = Box::new(DrainDatanodeStart);
392
393        let status = run_until_done(&mut state, &mut ctx, &procedure_ctx)
394            .await
395            .unwrap();
396
397        assert!(matches!(status, Status::Done { .. }));
398        assert_eq!(0, ctx.volatile_ctx.migrated);
399        assert_eq!(1, ctx.volatile_ctx.follower_count);
400        assert!(
401            !env.node_scheduling_manager
402                .is_schedulable(NodeSchedulingRole::Datanode, 1)
403                .await
404                .unwrap()
405        );
406    }
407
408    #[tokio::test]
409    async fn test_drain_datanode_dump_roundtrip() {
410        let env = TestEnv::new().await;
411        let persistent_ctx = new_persistent_context(Some(2), DrainDatanodeStrategy::TryAll);
412
413        let procedure = DrainDatanodeProcedure::new(env.context(persistent_ctx), vec![]);
414        let json = procedure.dump().unwrap();
415        let tracker = DrainDatanodeProcedureTracker::default();
416        let reloaded = DrainDatanodeProcedure::from_json(&json, &tracker, |persistent_ctx| {
417            env.context(persistent_ctx)
418        })
419        .unwrap();
420
421        assert_eq!("DrainDatanodeStart", reloaded.state.name());
422        let ctx = &reloaded.context.persistent_ctx;
423        assert_eq!(1, ctx.node_id);
424        assert_eq!(Some(2), ctx.target_node_id);
425        assert_eq!(DEFAULT_PARALLELISM, ctx.parallelism);
426        assert_eq!(DrainDatanodeStrategy::TryAll, ctx.strategy);
427        // Scan state and progress are volatile, so they are not dumped.
428        assert!(!json.contains("scan"));
429        assert!(!json.contains("rescan_round"));
430    }
431
432    #[tokio::test]
433    async fn test_drain_datanode_default_strategy_continues_after_batch_failures() {
434        let env = TestEnv::new().await;
435        let regions: Vec<_> = (1..=5).map(|number| RegionId::new(1024, number)).collect();
436        env.create_table(
437            regions
438                .iter()
439                .copied()
440                .map(|region_id| leader_region(region_id, 1))
441                .collect(),
442        )
443        .await;
444        // The pinned target is unschedulable, so every region is given up.
445        env.node_scheduling_manager
446            .mark_unschedulable(NodeSchedulingRole::Datanode, 3)
447            .await
448            .unwrap();
449
450        let mut persistent_ctx = new_persistent_context(Some(3), DrainDatanodeStrategy::default());
451        persistent_ctx.parallelism = 2;
452        let mut ctx = env.context(persistent_ctx);
453        let procedure_ctx = TestingEnv::procedure_context();
454        let mut state: Box<dyn State> = Box::new(DrainDatanodeStart);
455
456        let error = run_until_done(&mut state, &mut ctx, &procedure_ctx)
457            .await
458            .unwrap_err();
459
460        let error::Error::DrainDatanodeAbort { failed_regions, .. } = error else {
461            panic!("expected the drain procedure to abort");
462        };
463        assert_eq!(regions, failed_regions);
464        assert_eq!(regions, ctx.volatile_ctx.failed_regions);
465        assert_eq!(0, ctx.volatile_ctx.migrated);
466    }
467
468    #[tokio::test]
469    async fn test_drain_datanode_recovery_obeys_deadline() {
470        for expired in [true, false] {
471            let mut env = TestEnv::new().await;
472            let first_id = RegionId::new(1024, 1);
473            let second_id = RegionId::new(1024, 2);
474            env.create_table(vec![
475                leader_region(first_id, 1),
476                leader_region(second_id, 1),
477            ])
478            .await;
479            env.mock_datanodes().await;
480            let mut persistent_ctx =
481                new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
482            if expired {
483                persistent_ctx.deadline_ms = 0;
484            }
485            let mut procedure =
486                DrainDatanodeProcedure::new(env.context(persistent_ctx.clone()), vec![]);
487            assert!(
488                procedure
489                    .execute(&TestingEnv::procedure_context())
490                    .await
491                    .unwrap()
492                    .need_persist()
493            );
494            let checkpoint = procedure.dump().unwrap();
495            let json: serde_json::Value = serde_json::from_str(&checkpoint).unwrap();
496            assert_eq!(
497                serde_json::json!({
498                    "persistent_ctx": {
499                        "node_id": 1, "target_node_id": 2,
500                        "parallelism": DEFAULT_PARALLELISM, "strategy": "FailFast",
501                        "deadline_ms": persistent_ctx.deadline_ms,
502                    },
503                    "state": {"drain_datanode_state": "DrainDatanodeCursor"},
504                }),
505                json
506            );
507            drop(procedure);
508            let recovered = DrainDatanodeProcedure::from_json(
509                &checkpoint,
510                &DrainDatanodeProcedureTracker::default(),
511                |persistent_ctx| env.context(persistent_ctx),
512            )
513            .unwrap();
514            assert!(recovered.context.volatile_ctx.scan.is_none());
515            drop(recovered);
516            let id = ProcedureId::random();
517            let message = common_procedure::store::ProcedureMessage {
518                type_name: DrainDatanodeProcedure::TYPE_NAME.to_string(),
519                data: checkpoint,
520                parent_id: None,
521                subprocedure_recovery: Default::default(),
522                step: 0,
523                error: None,
524                context: Default::default(),
525            };
526            let store = common_meta::state_store::KvStateStore::new(env.env.kv_backend());
527            common_procedure::store::state_store::StateStore::put(
528                &store,
529                &format!("procedure/{id}/0000000000.step"),
530                serde_json::to_vec(&message).unwrap(),
531            )
532            .await
533            .unwrap();
534            let manager = env.drain_manager();
535            manager.try_start().unwrap();
536            env.procedure_manager().start().await.unwrap();
537            let state = wait_for_terminal_state(env.procedure_manager(), id).await;
538            if expired {
539                let ProcedureState::Failed { error } = state else {
540                    panic!("expected expired drain: {state:?}");
541                };
542                assert_eq!(StatusCode::Cancelled, error.status_code());
543            } else {
544                assert!(state.is_done(), "{state:?}");
545            }
546            let table_route = env
547                .table_metadata_manager
548                .table_route_manager()
549                .table_route_storage()
550                .get(1024)
551                .await
552                .unwrap()
553                .unwrap();
554            let routes = table_route.region_routes().unwrap();
555            assert_eq!(2, routes.len());
556            for route in routes {
557                assert_eq!(
558                    Some(if expired { 1 } else { 2 }),
559                    route.leader_peer.as_ref().map(|peer| peer.id)
560                );
561            }
562        }
563    }
564
565    #[tokio::test]
566    async fn test_drain_datanode_recovered_migration_outlives_parent() {
567        let env = TestEnv::new().await;
568        let region_id = RegionId::new(1024, 1);
569        env.create_table(vec![leader_region(region_id, 1)]).await;
570        assert_recovered_migration_outlives_parent(env, vec![region_id]).await;
571    }
572
573    #[tokio::test]
574    async fn test_drain_datanode_recovers_every_region_in_a_migration_batch() {
575        let env = TestEnv::new().await;
576        create_kafka_table(&env, 1024, 3, 1).await;
577        let regions = (0..3).map(|i| RegionId::new(1024, i)).collect();
578        assert_recovered_migration_outlives_parent(env, regions).await;
579    }
580
581    async fn assert_recovered_migration_outlives_parent(
582        mut env: TestEnv,
583        region_ids: Vec<RegionId>,
584    ) {
585        let procedure_ctx = TestingEnv::procedure_context();
586        let mut parent = DrainDatanodeProcedure::new(
587            env.context(new_persistent_context(
588                Some(2),
589                DrainDatanodeStrategy::FailFast,
590            )),
591            vec![],
592        );
593        assert!(parent.execute(&procedure_ctx).await.unwrap().need_persist());
594        assert_eq!("DrainDatanodeCursor", parent.state.name());
595        let parent_checkpoint = parent.dump().unwrap();
596        let Status::Suspended {
597            subprocedures: mut children,
598            recovery,
599            ..
600        } = run_until_done(&mut parent.state, &mut parent.context, &procedure_ctx)
601            .await
602            .unwrap()
603        else {
604            panic!("expected a migration batch");
605        };
606        assert_eq!(1, children.len());
607        let mut child = children.pop().unwrap();
608        // Persist the first migration checkpoint, before it opens the candidate region.
609        assert!(
610            child
611                .procedure
612                .execute(&procedure_ctx)
613                .await
614                .unwrap()
615                .need_persist()
616        );
617        let parent_id = ProcedureId::random();
618        let child_id = child.id;
619        let store = KvStateStore::new(env.env.kv_backend());
620        for (id, message) in [
621            (
622                parent_id,
623                ProcedureMessage {
624                    type_name: parent.type_name().to_string(),
625                    data: parent_checkpoint,
626                    parent_id: None,
627                    subprocedure_recovery: Default::default(),
628                    step: 0,
629                    error: None,
630                    context: Default::default(),
631                },
632            ),
633            (
634                child_id,
635                ProcedureMessage {
636                    type_name: child.procedure.type_name().to_string(),
637                    data: child.procedure.dump().unwrap(),
638                    parent_id: Some(parent_id),
639                    subprocedure_recovery: recovery,
640                    step: 0,
641                    error: None,
642                    context: child.context.clone(),
643                },
644            ),
645        ] {
646            store
647                .put(
648                    &format!("procedure/{id}/0000000000.step"),
649                    serde_json::to_vec(&message).unwrap(),
650                )
651                .await
652                .unwrap();
653        }
654        drop(child);
655        drop(parent);
656        assert!(
657            region_ids
658                .iter()
659                .all(|id| !env.region_migration_manager.tracker().contains(*id))
660        );
661
662        // Queue mailbox requests without replying until the recovered parent has ended.
663        let mailbox_ctx = env.env.mailbox_context();
664        let mut receivers = Vec::new();
665        for node_id in [1, 2] {
666            let (tx, rx) = tokio::sync::mpsc::channel(8);
667            mailbox_ctx
668                .insert_heartbeat_response_receiver(Channel::Datanode(node_id), tx)
669                .await;
670            receivers.push(rx);
671        }
672        let mailbox = mailbox_ctx.mailbox().clone();
673        env.region_migration_manager.try_start().unwrap();
674        let drain_manager = env.drain_manager();
675        drain_manager.try_start().unwrap();
676        env.procedure_manager().start().await.unwrap();
677        assert!(
678            wait_for_terminal_state(env.procedure_manager(), parent_id)
679                .await
680                .is_done()
681        );
682        assert!(
683            region_ids
684                .iter()
685                .all(|id| env.region_migration_manager.tracker().contains(*id))
686        );
687        assert!(matches!(
688            env.procedure_manager()
689                .procedure_state(child_id)
690                .await
691                .unwrap(),
692            Some(ProcedureState::Running)
693        ));
694        // No replacement migration was submitted by the recovering drain.
695        assert_eq!(
696            2,
697            env.procedure_manager()
698                .list_procedures()
699                .await
700                .unwrap()
701                .len()
702        );
703
704        for rx in receivers {
705            spawn_mock_datanode(mailbox.clone(), rx);
706        }
707        assert!(
708            wait_for_terminal_state(env.procedure_manager(), child_id)
709                .await
710                .is_done()
711        );
712        for &region_id in &region_ids {
713            assert!(
714                !crate::procedure::drain_datanode::utils::leads_region(
715                    &env.table_metadata_manager,
716                    1,
717                    region_id,
718                )
719                .await
720                .unwrap()
721            );
722        }
723        // Completion must release the recovered guard and clean its own checkpoints.
724        tokio::time::timeout(Duration::from_secs(5), async {
725            loop {
726                let mut records = store.walk_top_down("procedure/").await.unwrap();
727                if records.try_next().await.unwrap().is_none()
728                    && region_ids
729                        .iter()
730                        .all(|id| !env.region_migration_manager.tracker().contains(*id))
731                {
732                    break;
733                }
734                tokio::time::sleep(Duration::from_millis(10)).await;
735            }
736        })
737        .await
738        .expect("recovered migration did not clean up");
739        env.procedure_manager().stop().await.unwrap();
740    }
741
742    #[tokio::test]
743    async fn test_drain_datanode_migrates_leader_region() {
744        assert_drain_migrates_leader_regions(1).await;
745    }
746
747    #[tokio::test]
748    async fn test_drain_datanode_migrates_multiple_batches() {
749        assert_drain_migrates_leader_regions(3).await;
750    }
751
752    async fn assert_drain_migrates_leader_regions(region_count: u32) {
753        let mut env = TestEnv::new().await;
754        let regions: Vec<_> = (1..=region_count)
755            .map(|number| RegionId::new(1024, number))
756            .collect();
757        env.create_table(
758            regions
759                .iter()
760                .copied()
761                .map(|region_id| leader_region(region_id, 1))
762                .collect(),
763        )
764        .await;
765
766        env.mock_datanodes().await;
767
768        env.procedure_manager().start().await.unwrap();
769        let manager = env.drain_manager();
770        let procedure_id = manager
771            .submit_procedure(
772                common_procedure::ProcedureContext::default(),
773                DrainDatanodeProcedureTask {
774                    node_id: 1,
775                    target_node_id: Some(2),
776                    parallelism: 1,
777                    strategy: DrainDatanodeStrategy::FailFast,
778                    timeout: None,
779                },
780            )
781            .await
782            .unwrap();
783
784        let state = wait_for_terminal_state(env.procedure_manager(), procedure_id).await;
785        assert!(
786            matches!(state, ProcedureState::Done { .. }),
787            "the drain procedure did not complete successfully: {state:?}"
788        );
789
790        let table_route = env
791            .table_metadata_manager
792            .table_route_manager()
793            .table_route_storage()
794            .get(1024)
795            .await
796            .unwrap()
797            .unwrap();
798        assert_eq!(
799            region_count as usize,
800            table_route.region_routes().unwrap().len()
801        );
802        assert!(
803            !env.node_scheduling_manager
804                .is_schedulable(NodeSchedulingRole::Datanode, 1)
805                .await
806                .unwrap()
807        );
808        for region_route in table_route.region_routes().unwrap() {
809            assert_eq!(
810                Some(2),
811                region_route.leader_peer.as_ref().map(|peer| peer.id)
812            );
813        }
814    }
815
816    #[tokio::test]
817    async fn test_drain_drains_multiple_topic_windows_and_confirms_completion() {
818        use crate::procedure::drain_datanode::DrainDatanodeProcedureTask;
819        use crate::procedure::drain_datanode::test_util::{
820            create_kafka_table, wait_for_terminal_state,
821        };
822        use common_procedure::ProcedureState;
823
824        let mut env = TestEnv::new().await;
825        create_kafka_table(&env, 1024, 13, 3).await;
826        env.mock_datanodes().await;
827        env.procedure_manager().start().await.unwrap();
828        let id = env
829            .drain_manager()
830            .submit_procedure(
831                common_procedure::ProcedureContext::default(),
832                DrainDatanodeProcedureTask {
833                    node_id: 1,
834                    target_node_id: Some(2),
835                    parallelism: 2,
836                    strategy: DrainDatanodeStrategy::FailFast,
837                    timeout: None,
838                },
839            )
840            .await
841            .unwrap();
842        let state = wait_for_terminal_state(env.procedure_manager(), id).await;
843        assert!(matches!(state, ProcedureState::Done { .. }), "{state:?}");
844        let route = env
845            .table_metadata_manager
846            .table_route_manager()
847            .table_route_storage()
848            .get(1024)
849            .await
850            .unwrap()
851            .unwrap();
852        let regions = route.region_routes().unwrap();
853        assert_eq!(13, regions.len());
854        assert!(
855            regions
856                .iter()
857                .all(|region| region.leader_peer.as_ref().is_some_and(|peer| peer.id == 2))
858        );
859        env.procedure_manager().stop().await.unwrap();
860    }
861
862    #[tokio::test]
863    async fn test_drain_skips_region_moved_or_dropped_during_discovery() {
864        for strategy in [
865            DrainDatanodeStrategy::FailFast,
866            DrainDatanodeStrategy::TryAll,
867        ] {
868            // Target 2 yields RegionMigrated; leader 3 yields LeaderPeerChanged;
869            // dropping the table yields TableRouteNotFound.
870            for leader_id in [Some(2), Some(3), None] {
871                let env = TestEnv::new().await;
872                let region_id = RegionId::new(1024, 1);
873                env.create_table(vec![leader_region(region_id, 1)]).await;
874                let mut ctx = env.context(new_persistent_context(Some(2), strategy));
875                ctx.selector_ctx.peer_discovery = Arc::new(ChangeRouteDuringDiscovery {
876                    inner: ctx.selector_ctx.peer_discovery.clone(),
877                    metadata: env.table_metadata_manager.clone(),
878                    region_id,
879                    leader_id,
880                });
881
882                let (mut dispatch, _) = DrainDatanodeCursor::default()
883                    .next(&mut ctx, &TestingEnv::procedure_context())
884                    .await
885                    .unwrap();
886
887                let (next, status) = dispatch
888                    .next(&mut ctx, &TestingEnv::procedure_context())
889                    .await
890                    .unwrap();
891                assert!(next.as_any().is::<DrainDatanodeCursor>());
892                assert!(matches!(status, Status::Executing { persist: false, .. }));
893                assert!(ctx.volatile_ctx.failed_regions.is_empty());
894                assert_eq!(0, ctx.volatile_ctx.migrated);
895                assert!(!env.region_migration_manager.tracker().contains(region_id));
896            }
897        }
898    }
899
900    #[tokio::test]
901    async fn test_grouped_migrations_reduce_child_count() {
902        common_telemetry::init_default_ut_logging();
903        let env = TestEnv::new().await;
904        create_kafka_table(&env, 1024, 8, 1).await;
905        for trial in 0..3 {
906            for grouped in [false, true] {
907                let mut persistent =
908                    new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
909                persistent.parallelism = 8;
910                let mut ctx = env.context(persistent);
911                let (prepared, _) = DrainDatanodeCursor::default()
912                    .next(&mut ctx, &TestingEnv::procedure_context())
913                    .await
914                    .unwrap();
915                let batches = prepared
916                    .as_any()
917                    .downcast_ref::<Dispatch>()
918                    .unwrap()
919                    .batches
920                    .clone();
921                let batches = if grouped {
922                    batches
923                } else {
924                    batches
925                        .into_iter()
926                        .flat_map(|batch| {
927                            batch.region_ids.clone().into_iter().map(move |region_id| {
928                                RegionMigrationTaskBatch {
929                                    region_ids: vec![region_id],
930                                    ..batch.clone()
931                                }
932                            })
933                        })
934                        .collect()
935                };
936                let mut dispatch = Dispatch::new(batches);
937                let (next, status) = dispatch
938                    .next(&mut ctx, &TestingEnv::procedure_context())
939                    .await
940                    .unwrap();
941                let Status::Suspended { subprocedures, .. } = status else {
942                    panic!("expected children");
943                };
944                let count = subprocedures.len();
945                assert_eq!(if grouped { 1 } else { 8 }, count);
946                assert!(next.as_any().is::<Dispatch>());
947                let mut regions = subprocedures
948                    .iter()
949                    .flat_map(|child| {
950                        let data: serde_json::Value =
951                            serde_json::from_str(&child.procedure.dump().unwrap()).unwrap();
952                        serde_json::from_value::<Vec<RegionId>>(
953                            data["persistent_ctx"]["region_ids"].clone(),
954                        )
955                        .unwrap()
956                    })
957                    .collect::<Vec<_>>();
958                regions.sort_unstable();
959                assert_eq!(
960                    (0..8).map(|i| RegionId::new(1024, i)).collect::<Vec<_>>(),
961                    regions
962                );
963                common_telemetry::info!(
964                    "trial={trial}, grouped={grouped}, regions={}, children={count}",
965                    regions.len()
966                );
967                drop(subprocedures);
968                assert!(
969                    regions
970                        .iter()
971                        .all(|id| !env.region_migration_manager.tracker().contains(*id))
972                );
973            }
974        }
975    }
976
977    /// Changes the route after the drain's first read, during target discovery.
978    struct ChangeRouteDuringDiscovery {
979        inner: PeerDiscoveryRef,
980        metadata: TableMetadataManagerRef,
981        region_id: RegionId,
982        leader_id: Option<u64>,
983    }
984
985    #[async_trait::async_trait]
986    impl PeerDiscovery for ChangeRouteDuringDiscovery {
987        async fn active_frontends(&self) -> MetaResult<Vec<NodeInfo>> {
988            self.inner.active_frontends().await
989        }
990
991        async fn active_datanodes(
992            &self,
993            filter: Option<for<'a> fn(&'a NodeWorkloads) -> bool>,
994        ) -> MetaResult<Vec<NodeInfo>> {
995            let storage = self.metadata.table_route_manager().table_route_storage();
996            let table_id = self.region_id.table_id();
997            let current = storage.get_with_raw_bytes(table_id).await?.unwrap();
998            if let Some(leader_id) = self.leader_id {
999                let updated = current.update(vec![leader_region(self.region_id, leader_id)])?;
1000                let (txn, _) = storage.build_update_txn(table_id, &current, &updated)?;
1001                assert!(self.metadata.kv_backend().txn(txn).await?.succeeded);
1002            } else {
1003                let table_info = self
1004                    .metadata
1005                    .table_info_manager()
1006                    .get(table_id)
1007                    .await?
1008                    .unwrap();
1009                self.metadata
1010                    .delete_table_metadata(
1011                        table_id,
1012                        &table_info.table_name(),
1013                        &current,
1014                        &HashMap::new(),
1015                        None,
1016                    )
1017                    .await?;
1018            }
1019            self.inner.active_datanodes(filter).await
1020        }
1021
1022        async fn active_flownodes(
1023            &self,
1024            filter: Option<for<'a> fn(&'a NodeWorkloads) -> bool>,
1025        ) -> MetaResult<Vec<NodeInfo>> {
1026            self.inner.active_flownodes(filter).await
1027        }
1028    }
1029
1030    #[tokio::test]
1031    async fn test_drain_datanode_dispatches_migration() {
1032        let env = TestEnv::new().await;
1033        let region_id = RegionId::new(1024, 1);
1034        env.create_table(vec![leader_region(region_id, 1)]).await;
1035
1036        let mut ctx = env.context(new_persistent_context(
1037            Some(2),
1038            DrainDatanodeStrategy::FailFast,
1039        ));
1040        let procedure_ctx = TestingEnv::procedure_context();
1041
1042        let (mut next, _) = DrainDatanodeStart
1043            .next(&mut ctx, &procedure_ctx)
1044            .await
1045            .unwrap();
1046        let status = run_until_done(&mut next, &mut ctx, &procedure_ctx)
1047            .await
1048            .unwrap();
1049
1050        let Status::Suspended {
1051            subprocedures,
1052            recovery,
1053            persist,
1054        } = status
1055        else {
1056            panic!("expected the dispatch state to suspend the procedure");
1057        };
1058        // Recovery resumes children independently and restarts the parent's scan.
1059        assert!(!persist);
1060        assert_eq!(common_procedure::SubprocedureRecovery::Detach, recovery);
1061        assert_eq!(1, subprocedures.len());
1062        assert!(env.region_migration_manager.tracker().contains(region_id));
1063        let data: serde_json::Value =
1064            serde_json::from_str(&subprocedures[0].procedure.dump().unwrap()).unwrap();
1065        assert_eq!(
1066            serde_json::json!([region_id]),
1067            data["persistent_ctx"]["region_ids"]
1068        );
1069        assert!(next.as_any().downcast_ref::<Dispatch>().is_some());
1070    }
1071}