Skip to main content

meta_srv/procedure/drain_datanode/
manager.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::collections::hash_map::Entry;
17use std::sync::{Arc, RwLock};
18
19use common_meta::DatanodeId;
20use common_meta::key::TableMetadataManagerRef;
21use common_procedure::{ProcedureContext, ProcedureId, ProcedureManagerRef, ProcedureWithId};
22use common_telemetry::info;
23use common_time::util::current_time_millis;
24use snafu::{OptionExt, ResultExt, ensure};
25use tokio::sync::Mutex;
26
27use crate::error::{self, Result};
28use crate::key::NodeSchedulingRole;
29use crate::metasrv::SelectorContext;
30use crate::node_scheduling::NodeSchedulingManagerRef;
31use crate::procedure::drain_datanode::{
32    Context as DrainDatanodeContext, DEFAULT_DRAIN_TIMEOUT, DrainDatanodeProcedure,
33    DrainDatanodeProcedureTask, MAX_PARALLELISM, PersistentContext,
34};
35use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
36
37pub type DrainDatanodeManagerRef = Arc<DrainDatanodeManager>;
38
39/// Tracks the running drains, indexed by the drained datanode.
40///
41/// Reserves each datanode for one drain and rejects concurrent submissions.
42#[derive(Default, Clone)]
43pub struct DrainDatanodeProcedureTracker {
44    running_procedures: Arc<RwLock<HashMap<DatanodeId, DrainDatanodeProcedureTask>>>,
45}
46
47impl DrainDatanodeProcedureTracker {
48    fn contains(&self, node_id: DatanodeId) -> bool {
49        self.running_procedures
50            .read()
51            .unwrap()
52            .contains_key(&node_id)
53    }
54
55    /// Returns the guard of a drain of `task.node_id`, or `None` if that datanode is already
56    /// being drained.
57    pub(crate) fn insert_running_procedure(
58        &self,
59        task: &DrainDatanodeProcedureTask,
60    ) -> Option<DrainDatanodeProcedureGuard> {
61        let mut procedures = self.running_procedures.write().unwrap();
62        match procedures.entry(task.node_id) {
63            Entry::Occupied(_) => None,
64            Entry::Vacant(v) => {
65                v.insert(task.clone());
66                Some(DrainDatanodeProcedureGuard {
67                    node_id: task.node_id,
68                    running_procedures: self.running_procedures.clone(),
69                })
70            }
71        }
72    }
73}
74
75/// Removes the drained datanode from [`DrainDatanodeProcedureTracker`] when dropped.
76pub(crate) struct DrainDatanodeProcedureGuard {
77    node_id: DatanodeId,
78    running_procedures: Arc<RwLock<HashMap<DatanodeId, DrainDatanodeProcedureTask>>>,
79}
80
81impl Drop for DrainDatanodeProcedureGuard {
82    fn drop(&mut self) {
83        self.running_procedures
84            .write()
85            .unwrap()
86            .remove(&self.node_id);
87    }
88}
89
90/// Manages the [`DrainDatanodeProcedure`]s.
91///
92/// Registers the procedure loader and serializes submissions with restoring scheduling.
93pub struct DrainDatanodeManager {
94    procedure_manager: ProcedureManagerRef,
95    table_metadata_manager: TableMetadataManagerRef,
96    node_scheduling_manager: NodeSchedulingManagerRef,
97    region_migration_manager: RegionMigrationManagerRef,
98    selector_ctx: SelectorContext,
99
100    tracker: DrainDatanodeProcedureTracker,
101    /// Serializes submissions with restoring scheduling, including their KV operations.
102    admission: Mutex<()>,
103}
104
105impl DrainDatanodeManager {
106    /// Returns a new [`DrainDatanodeManager`].
107    pub fn new(
108        procedure_manager: ProcedureManagerRef,
109        table_metadata_manager: TableMetadataManagerRef,
110        node_scheduling_manager: NodeSchedulingManagerRef,
111        region_migration_manager: RegionMigrationManagerRef,
112        selector_ctx: SelectorContext,
113    ) -> Self {
114        Self {
115            procedure_manager,
116            table_metadata_manager,
117            node_scheduling_manager,
118            region_migration_manager,
119            selector_ctx,
120
121            tracker: DrainDatanodeProcedureTracker::default(),
122            admission: Mutex::new(()),
123        }
124    }
125
126    /// Returns the context of a drain run.
127    fn context(&self, persistent_ctx: PersistentContext) -> DrainDatanodeContext {
128        DrainDatanodeContext::new(
129            self.table_metadata_manager.clone(),
130            self.node_scheduling_manager.clone(),
131            self.region_migration_manager.clone(),
132            self.selector_ctx.clone(),
133            persistent_ctx,
134        )
135    }
136
137    /// Submits a drain, rejecting the task if the datanode is already being drained.
138    pub async fn submit_procedure(
139        &self,
140        procedure_context: ProcedureContext,
141        task: DrainDatanodeProcedureTask,
142    ) -> Result<ProcedureId> {
143        let _admission = self.admission.lock().await;
144        if task.target_node_id == Some(task.node_id) {
145            return error::InvalidArgumentsSnafu {
146                err_msg: "the target datanode must differ from the drained datanode",
147            }
148            .fail();
149        }
150        if task.parallelism == 0 || task.parallelism > MAX_PARALLELISM {
151            return error::InvalidArgumentsSnafu {
152                err_msg: format!("the parallelism must be in 1..={}", MAX_PARALLELISM),
153            }
154            .fail();
155        }
156        let timeout = task.timeout.unwrap_or(DEFAULT_DRAIN_TIMEOUT);
157        if timeout.is_zero() {
158            return error::InvalidArgumentsSnafu {
159                err_msg: "the timeout must be greater than zero",
160            }
161            .fail();
162        }
163
164        // The budget starts at submission and must fit the persisted absolute deadline.
165        let deadline_ms = i64::try_from(timeout.as_millis())
166            .ok()
167            .and_then(|timeout_ms| current_time_millis().checked_add(timeout_ms))
168            .context(error::InvalidArgumentsSnafu {
169                err_msg: "the timeout exceeds the supported deadline range",
170            })?;
171
172        self.ensure_ready()?;
173        let Some(guard) = self.tracker.insert_running_procedure(&task) else {
174            return error::DrainDatanodeRunningSnafu {
175                node_id: task.node_id,
176            }
177            .fail();
178        };
179
180        let persistent_ctx = PersistentContext {
181            node_id: task.node_id,
182            target_node_id: task.target_node_id,
183            parallelism: task.parallelism,
184            strategy: task.strategy,
185            // The budget starts when the task is submitted, so neither a slow start nor a
186            // restart extends it.
187            deadline_ms,
188        };
189        let procedure = DrainDatanodeProcedure::new(self.context(persistent_ctx), vec![guard]);
190        let procedure_with_id =
191            ProcedureWithId::with_random_id(Box::new(procedure)).with_context(procedure_context);
192        let procedure_id = procedure_with_id.id;
193        info!(
194            "Starting drain datanode procedure {procedure_id} for datanode {}, timeout: {:?}",
195            task.node_id, timeout
196        );
197        self.procedure_manager
198            .submit(procedure_with_id)
199            .await
200            .context(error::SubmitProcedureSnafu)?;
201
202        Ok(procedure_id)
203    }
204
205    /// Restores scheduling only when no drain owns the node.
206    pub async fn mark_schedulable(&self, node_id: DatanodeId) -> Result<()> {
207        let _admission = self.admission.lock().await;
208        self.ensure_ready()?;
209        ensure!(
210            !self.tracker.contains(node_id),
211            error::DrainDatanodeRunningSnafu { node_id }
212        );
213        self.node_scheduling_manager
214            .mark_schedulable(NodeSchedulingRole::Datanode, node_id)
215            .await?;
216        Ok(())
217    }
218
219    fn ensure_ready(&self) -> Result<()> {
220        ensure!(
221            self.procedure_manager.is_ready(),
222            error::RetryLaterSnafu {
223                reason: "procedure manager has not completed recovery",
224            }
225        );
226        Ok(())
227    }
228
229    /// Registers the loader of [`DrainDatanodeProcedure`].
230    pub(crate) fn try_start(&self) -> Result<()> {
231        let table_metadata_manager = self.table_metadata_manager.clone();
232        let node_scheduling_manager = self.node_scheduling_manager.clone();
233        let region_migration_manager = self.region_migration_manager.clone();
234        let selector_ctx = self.selector_ctx.clone();
235        let tracker = self.tracker.clone();
236        self.procedure_manager
237            .register_loader(
238                DrainDatanodeProcedure::TYPE_NAME,
239                Box::new(move |json| {
240                    let table_metadata_manager = table_metadata_manager.clone();
241                    let node_scheduling_manager = node_scheduling_manager.clone();
242                    let region_migration_manager = region_migration_manager.clone();
243                    let selector_ctx = selector_ctx.clone();
244                    let factory = move |persistent_ctx| {
245                        DrainDatanodeContext::new(
246                            table_metadata_manager,
247                            node_scheduling_manager,
248                            region_migration_manager,
249                            selector_ctx,
250                            persistent_ctx,
251                        )
252                    };
253                    DrainDatanodeProcedure::from_json(json, &tracker, factory)
254                        .map(|procedure| Box::new(procedure) as _)
255                }),
256            )
257            .context(error::RegisterProcedureLoaderSnafu {
258                type_name: DrainDatanodeProcedure::TYPE_NAME,
259            })
260    }
261}
262
263#[cfg(test)]
264mod tests {
265    use std::sync::Arc;
266    use std::time::Duration;
267
268    use common_error::ext::ErrorExt;
269    use common_error::status_code::StatusCode;
270    use common_procedure::store::state_store::StateStore;
271    use common_procedure::{Procedure, ProcedureId, ProcedureManagerRef, Status};
272    use tokio::sync::Notify;
273
274    use crate::error;
275    use crate::key::NodeSchedulingRole;
276    use crate::procedure::drain_datanode::manager::{
277        DrainDatanodeManager, DrainDatanodeProcedureTracker,
278    };
279    use crate::procedure::drain_datanode::test_util::{TestEnv, new_persistent_context};
280    use crate::procedure::drain_datanode::{
281        DEFAULT_PARALLELISM, DrainDatanodeProcedure, DrainDatanodeProcedureTask,
282        DrainDatanodeStrategy,
283    };
284
285    struct HoldDrainLock {
286        entered: Arc<Notify>,
287        release: Arc<Notify>,
288    }
289
290    #[async_trait::async_trait]
291    impl Procedure for HoldDrainLock {
292        fn type_name(&self) -> &str {
293            "test-hold-drain-lock"
294        }
295        async fn execute(
296            &mut self,
297            _: &common_procedure::Context,
298        ) -> common_procedure::Result<Status> {
299            self.entered.notify_one();
300            self.release.notified().await;
301            Ok(Status::done())
302        }
303        fn dump(&self) -> common_procedure::Result<String> {
304            Ok(String::new())
305        }
306        fn lock_key(&self) -> common_procedure::LockKey {
307            common_procedure::LockKey::single_exclusive("__drain_datanode/1")
308        }
309    }
310
311    fn task() -> DrainDatanodeProcedureTask {
312        DrainDatanodeProcedureTask {
313            node_id: 1,
314            target_node_id: Some(2),
315            parallelism: DEFAULT_PARALLELISM,
316            strategy: DrainDatanodeStrategy::FailFast,
317            timeout: None,
318        }
319    }
320
321    async fn hold_drain_lock(procedure_manager: &ProcedureManagerRef) -> Arc<Notify> {
322        let entered = Arc::new(Notify::new());
323        let release = Arc::new(Notify::new());
324        procedure_manager
325            .submit(common_procedure::ProcedureWithId::with_random_id(Box::new(
326                HoldDrainLock {
327                    entered: entered.clone(),
328                    release: release.clone(),
329                },
330            )))
331            .await
332            .unwrap();
333        tokio::time::timeout(Duration::from_secs(5), entered.notified())
334            .await
335            .unwrap();
336        release
337    }
338
339    #[tokio::test]
340    async fn test_submit_drain_rejects_concurrent_tasks_and_scheduling() {
341        let env = TestEnv::new().await;
342        env.procedure_manager().start().await.unwrap();
343        let release = hold_drain_lock(env.procedure_manager()).await;
344        let manager = env.drain_manager();
345        let (first, second) = tokio::join!(
346            manager.submit_procedure(Default::default(), task()),
347            manager.submit_procedure(Default::default(), task()),
348        );
349        assert!(matches!(
350            (first, second),
351            (Ok(_), Err(error::Error::DrainDatanodeRunning { .. }))
352                | (Err(error::Error::DrainDatanodeRunning { .. }), Ok(_))
353        ));
354        assert_eq!(
355            2,
356            env.procedure_manager()
357                .list_procedures()
358                .await
359                .unwrap()
360                .len()
361        );
362        let mut conflicting = task();
363        conflicting.target_node_id = Some(3);
364        for err in [
365            manager
366                .submit_procedure(Default::default(), conflicting)
367                .await
368                .unwrap_err(),
369            manager.mark_schedulable(1).await.unwrap_err(),
370        ] {
371            assert!(matches!(&err, error::Error::DrainDatanodeRunning { .. }));
372            assert_eq!(StatusCode::IllegalState, err.status_code());
373        }
374        release.notify_one();
375        tokio::time::timeout(Duration::from_secs(5), async {
376            loop {
377                if !manager
378                    .tracker
379                    .running_procedures
380                    .read()
381                    .unwrap()
382                    .contains_key(&1)
383                {
384                    break;
385                }
386                tokio::task::yield_now().await;
387            }
388        })
389        .await
390        .unwrap();
391        manager.mark_schedulable(1).await.unwrap();
392        manager.mark_schedulable(1).await.unwrap();
393        env.procedure_manager().stop().await.unwrap();
394    }
395
396    struct RecoveryGate {
397        store: common_meta::state_store::KvStateStore,
398        entered: tokio::sync::Notify,
399        release: tokio::sync::Notify,
400    }
401
402    #[async_trait::async_trait]
403    impl common_procedure::store::state_store::StateStore for RecoveryGate {
404        async fn put(&self, key: &str, value: Vec<u8>) -> common_procedure::Result<()> {
405            self.store.put(key, value).await
406        }
407
408        async fn walk_top_down(
409            &self,
410            path: &str,
411        ) -> common_procedure::Result<common_procedure::store::state_store::KeyValueStream>
412        {
413            if path == "procedure/" {
414                self.entered.notify_one();
415                self.release.notified().await;
416            }
417            self.store.walk_top_down(path).await
418        }
419
420        async fn batch_delete(&self, keys: &[String]) -> common_procedure::Result<()> {
421            self.store.batch_delete(keys).await
422        }
423
424        async fn delete(&self, key: &str) -> common_procedure::Result<()> {
425            self.store.delete(key).await
426        }
427    }
428
429    #[tokio::test]
430    async fn test_drain_admission_waits_for_recovery_and_rejects_running_task() {
431        use crate::procedure::region_migration::test_util::TestingEnv;
432        use common_procedure::local::{LocalManager, ManagerConfig};
433        use common_procedure::store::ProcedureMessage;
434        use common_procedure::test_util::InMemoryPoisonStore;
435
436        let env = TestEnv::new().await;
437        let mut original = DrainDatanodeProcedure::new(
438            env.context(new_persistent_context(
439                Some(2),
440                DrainDatanodeStrategy::FailFast,
441            )),
442            vec![],
443        );
444        assert!(
445            original
446                .execute(&TestingEnv::procedure_context())
447                .await
448                .unwrap()
449                .need_persist()
450        );
451        let id = ProcedureId::random();
452        let message = ProcedureMessage {
453            type_name: DrainDatanodeProcedure::TYPE_NAME.to_string(),
454            data: original.dump().unwrap(),
455            parent_id: None,
456            subprocedure_recovery: Default::default(),
457            step: 0,
458            error: None,
459            context: Default::default(),
460        };
461        drop(original);
462        let store = Arc::new(RecoveryGate {
463            store: common_meta::state_store::KvStateStore::new(env.env.kv_backend()),
464            entered: tokio::sync::Notify::new(),
465            release: tokio::sync::Notify::new(),
466        });
467        store
468            .put(
469                &format!("procedure/{id}/0000000000.step"),
470                serde_json::to_vec(&message).unwrap(),
471            )
472            .await
473            .unwrap();
474        let procedure_manager: ProcedureManagerRef = Arc::new(LocalManager::new(
475            ManagerConfig::default(),
476            store.clone(),
477            Arc::new(InMemoryPoisonStore::default()),
478            None,
479            None,
480        ));
481        let manager = DrainDatanodeManager::new(
482            procedure_manager.clone(),
483            env.table_metadata_manager.clone(),
484            env.node_scheduling_manager.clone(),
485            env.region_migration_manager.clone(),
486            env.selector_ctx.clone(),
487        );
488        manager.try_start().unwrap();
489        let start = tokio::spawn({
490            let procedure_manager = procedure_manager.clone();
491            async move { procedure_manager.start().await }
492        });
493        tokio::time::timeout(Duration::from_secs(5), store.entered.notified())
494            .await
495            .unwrap();
496        assert!(!procedure_manager.is_ready());
497        assert!(matches!(
498            manager.submit_procedure(Default::default(), task()).await,
499            Err(error::Error::RetryLater { .. })
500        ));
501        assert!(matches!(
502            manager.mark_schedulable(1).await,
503            Err(error::Error::RetryLater { .. })
504        ));
505        assert!(
506            !env.node_scheduling_manager
507                .is_schedulable(NodeSchedulingRole::Datanode, 1)
508                .await
509                .unwrap()
510        );
511
512        // Hold execution while the real loader restores the checkpoint and tracker.
513        let release = hold_drain_lock(&procedure_manager).await;
514        store.release.notify_one();
515        tokio::time::timeout(Duration::from_secs(5), start)
516            .await
517            .unwrap()
518            .unwrap()
519            .unwrap();
520        assert!(procedure_manager.is_ready());
521        for _ in 0..2 {
522            assert!(matches!(
523                manager.submit_procedure(Default::default(), task()).await,
524                Err(error::Error::DrainDatanodeRunning { .. })
525            ));
526        }
527        let mut conflicting = task();
528        conflicting.target_node_id = Some(3);
529        for err in [
530            manager
531                .submit_procedure(Default::default(), conflicting)
532                .await
533                .unwrap_err(),
534            manager.mark_schedulable(1).await.unwrap_err(),
535        ] {
536            assert!(matches!(&err, error::Error::DrainDatanodeRunning { .. }));
537            assert_eq!(StatusCode::IllegalState, err.status_code());
538        }
539        release.notify_one();
540        tokio::time::timeout(Duration::from_secs(5), async {
541            while manager.tracker.contains(1) {
542                tokio::task::yield_now().await;
543            }
544        })
545        .await
546        .unwrap();
547        assert!(
548            procedure_manager
549                .procedure_state(id)
550                .await
551                .unwrap()
552                .unwrap()
553                .is_done()
554        );
555        let second = manager
556            .submit_procedure(Default::default(), task())
557            .await
558            .unwrap();
559        assert_ne!(id, second);
560        tokio::time::timeout(Duration::from_secs(5), async {
561            while manager.tracker.contains(1) {
562                tokio::task::yield_now().await;
563            }
564        })
565        .await
566        .unwrap();
567        assert!(
568            procedure_manager
569                .procedure_state(second)
570                .await
571                .unwrap()
572                .unwrap()
573                .is_done()
574        );
575        manager.mark_schedulable(1).await.unwrap();
576        procedure_manager.stop().await.unwrap();
577        assert!(!procedure_manager.is_ready());
578        assert!(matches!(
579            manager.mark_schedulable(1).await,
580            Err(error::Error::RetryLater { .. })
581        ));
582    }
583
584    #[tokio::test]
585    async fn test_failed_drain_can_be_submitted_again() {
586        use crate::key::NodeSchedulingRole;
587        use crate::procedure::drain_datanode::test_util::leader_region;
588        let env = TestEnv::new().await;
589        env.create_table(vec![leader_region(
590            store_api::storage::RegionId::new(1024, 1),
591            1,
592        )])
593        .await;
594        env.node_scheduling_manager
595            .mark_unschedulable(NodeSchedulingRole::Datanode, 2)
596            .await
597            .unwrap();
598        env.procedure_manager().start().await.unwrap();
599        let manager = env.drain_manager();
600        let first = manager
601            .submit_procedure(Default::default(), task())
602            .await
603            .unwrap();
604        tokio::time::timeout(Duration::from_secs(5), async {
605            loop {
606                if env
607                    .procedure_manager()
608                    .procedure_state(first)
609                    .await
610                    .unwrap()
611                    .unwrap()
612                    .is_failed()
613                    && !manager
614                        .tracker
615                        .running_procedures
616                        .read()
617                        .unwrap()
618                        .contains_key(&1)
619                {
620                    break;
621                }
622                tokio::task::yield_now().await;
623            }
624        })
625        .await
626        .unwrap();
627        let second = manager
628            .submit_procedure(Default::default(), task())
629            .await
630            .unwrap();
631        assert_ne!(first, second);
632        env.procedure_manager().stop().await.unwrap();
633    }
634
635    #[tokio::test]
636    async fn test_failed_submission_releases_drain_admission() {
637        use common_procedure::local::{LocalManager, ManagerConfig};
638        use common_procedure::test_util::InMemoryPoisonStore;
639
640        let env = TestEnv::new().await;
641        let procedure_manager: ProcedureManagerRef = Arc::new(LocalManager::new(
642            ManagerConfig {
643                max_running_procedures: 0,
644                ..Default::default()
645            },
646            Arc::new(common_meta::state_store::KvStateStore::new(
647                env.env.kv_backend(),
648            )),
649            Arc::new(InMemoryPoisonStore::default()),
650            None,
651            None,
652        ));
653        procedure_manager.start().await.unwrap();
654        let manager = DrainDatanodeManager::new(
655            procedure_manager.clone(),
656            env.table_metadata_manager.clone(),
657            env.node_scheduling_manager.clone(),
658            env.region_migration_manager.clone(),
659            env.selector_ctx.clone(),
660        );
661        assert!(matches!(
662            manager.submit_procedure(Default::default(), task()).await,
663            Err(error::Error::SubmitProcedure { .. })
664        ));
665        assert!(!manager.tracker.contains(1));
666        manager.mark_schedulable(1).await.unwrap();
667        procedure_manager.stop().await.unwrap();
668    }
669
670    #[tokio::test]
671    async fn test_submit_procedure_rejects_invalid_task() {
672        let env = TestEnv::new().await;
673        let manager = env.drain_manager();
674
675        // The pinned target must differ from the drained datanode.
676        let error = manager
677            .submit_procedure(
678                common_procedure::ProcedureContext::default(),
679                DrainDatanodeProcedureTask {
680                    node_id: 1,
681                    target_node_id: Some(1),
682                    parallelism: DEFAULT_PARALLELISM,
683                    strategy: DrainDatanodeStrategy::FailFast,
684                    timeout: None,
685                },
686            )
687            .await
688            .unwrap_err();
689        assert!(matches!(error, error::Error::InvalidArguments { .. }));
690
691        // The parallelism must be in range.
692        for parallelism in [0, crate::procedure::drain_datanode::MAX_PARALLELISM + 1] {
693            let error = manager
694                .submit_procedure(
695                    common_procedure::ProcedureContext::default(),
696                    DrainDatanodeProcedureTask {
697                        node_id: 1,
698                        target_node_id: Some(2),
699                        parallelism,
700                        strategy: DrainDatanodeStrategy::FailFast,
701                        timeout: None,
702                    },
703                )
704                .await
705                .unwrap_err();
706            assert!(matches!(error, error::Error::InvalidArguments { .. }));
707        }
708
709        // Reject zero, millisecond conversion overflow, and absolute deadline overflow.
710        for timeout in [
711            Duration::ZERO,
712            Duration::from_millis(i64::MAX as u64 + 1),
713            Duration::from_millis(i64::MAX as u64),
714        ] {
715            let error = manager
716                .submit_procedure(
717                    common_procedure::ProcedureContext::default(),
718                    DrainDatanodeProcedureTask {
719                        timeout: Some(timeout),
720                        ..task()
721                    },
722                )
723                .await
724                .unwrap_err();
725            assert!(matches!(error, error::Error::InvalidArguments { .. }));
726            assert!(!manager.tracker.contains(1));
727        }
728        assert!(
729            env.procedure_manager()
730                .list_procedures()
731                .await
732                .unwrap()
733                .is_empty()
734        );
735        assert!(
736            env.node_scheduling_manager
737                .is_schedulable(NodeSchedulingRole::Datanode, 1)
738                .await
739                .unwrap()
740        );
741    }
742
743    #[tokio::test]
744    async fn test_drain_recovery_reserves_node_until_drop() {
745        let env = TestEnv::new().await;
746        let context = env.context(new_persistent_context(
747            Some(2),
748            DrainDatanodeStrategy::FailFast,
749        ));
750        let procedure = DrainDatanodeProcedure::new(context, vec![]);
751        let json = procedure.dump().unwrap();
752        let tracker = DrainDatanodeProcedureTracker::default();
753
754        // Loading reserves the datanode before any runner starts.
755        let reloaded = DrainDatanodeProcedure::from_json(&json, &tracker, |persistent_ctx| {
756            env.context(persistent_ctx)
757        })
758        .unwrap();
759        let task = DrainDatanodeProcedureTask {
760            node_id: 1,
761            target_node_id: Some(2),
762            parallelism: DEFAULT_PARALLELISM,
763            strategy: DrainDatanodeStrategy::FailFast,
764            timeout: None,
765        };
766        assert!(tracker.contains(task.node_id));
767        assert!(tracker.insert_running_procedure(&task).is_none());
768
769        // Dropping the reloaded procedure releases the place.
770        drop(reloaded);
771        assert!(tracker.insert_running_procedure(&task).is_some());
772    }
773}