1use std::collections::HashMap;
16use std::collections::hash_map::Entry;
17use std::sync::{Arc, RwLock};
18
19use common_meta::DatanodeId;
20use common_meta::key::TableMetadataManagerRef;
21use common_procedure::{ProcedureContext, ProcedureId, ProcedureManagerRef, ProcedureWithId};
22use common_telemetry::info;
23use common_time::util::current_time_millis;
24use snafu::{OptionExt, ResultExt, ensure};
25use tokio::sync::Mutex;
26
27use crate::error::{self, Result};
28use crate::key::NodeSchedulingRole;
29use crate::metasrv::SelectorContext;
30use crate::node_scheduling::NodeSchedulingManagerRef;
31use crate::procedure::drain_datanode::{
32 Context as DrainDatanodeContext, DEFAULT_DRAIN_TIMEOUT, DrainDatanodeProcedure,
33 DrainDatanodeProcedureTask, MAX_PARALLELISM, PersistentContext,
34};
35use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
36
37pub type DrainDatanodeManagerRef = Arc<DrainDatanodeManager>;
38
39#[derive(Default, Clone)]
43pub struct DrainDatanodeProcedureTracker {
44 running_procedures: Arc<RwLock<HashMap<DatanodeId, DrainDatanodeProcedureTask>>>,
45}
46
47impl DrainDatanodeProcedureTracker {
48 fn contains(&self, node_id: DatanodeId) -> bool {
49 self.running_procedures
50 .read()
51 .unwrap()
52 .contains_key(&node_id)
53 }
54
55 pub(crate) fn insert_running_procedure(
58 &self,
59 task: &DrainDatanodeProcedureTask,
60 ) -> Option<DrainDatanodeProcedureGuard> {
61 let mut procedures = self.running_procedures.write().unwrap();
62 match procedures.entry(task.node_id) {
63 Entry::Occupied(_) => None,
64 Entry::Vacant(v) => {
65 v.insert(task.clone());
66 Some(DrainDatanodeProcedureGuard {
67 node_id: task.node_id,
68 running_procedures: self.running_procedures.clone(),
69 })
70 }
71 }
72 }
73}
74
75pub(crate) struct DrainDatanodeProcedureGuard {
77 node_id: DatanodeId,
78 running_procedures: Arc<RwLock<HashMap<DatanodeId, DrainDatanodeProcedureTask>>>,
79}
80
81impl Drop for DrainDatanodeProcedureGuard {
82 fn drop(&mut self) {
83 self.running_procedures
84 .write()
85 .unwrap()
86 .remove(&self.node_id);
87 }
88}
89
90pub struct DrainDatanodeManager {
94 procedure_manager: ProcedureManagerRef,
95 table_metadata_manager: TableMetadataManagerRef,
96 node_scheduling_manager: NodeSchedulingManagerRef,
97 region_migration_manager: RegionMigrationManagerRef,
98 selector_ctx: SelectorContext,
99
100 tracker: DrainDatanodeProcedureTracker,
101 admission: Mutex<()>,
103}
104
105impl DrainDatanodeManager {
106 pub fn new(
108 procedure_manager: ProcedureManagerRef,
109 table_metadata_manager: TableMetadataManagerRef,
110 node_scheduling_manager: NodeSchedulingManagerRef,
111 region_migration_manager: RegionMigrationManagerRef,
112 selector_ctx: SelectorContext,
113 ) -> Self {
114 Self {
115 procedure_manager,
116 table_metadata_manager,
117 node_scheduling_manager,
118 region_migration_manager,
119 selector_ctx,
120
121 tracker: DrainDatanodeProcedureTracker::default(),
122 admission: Mutex::new(()),
123 }
124 }
125
126 fn context(&self, persistent_ctx: PersistentContext) -> DrainDatanodeContext {
128 DrainDatanodeContext::new(
129 self.table_metadata_manager.clone(),
130 self.node_scheduling_manager.clone(),
131 self.region_migration_manager.clone(),
132 self.selector_ctx.clone(),
133 persistent_ctx,
134 )
135 }
136
137 pub async fn submit_procedure(
139 &self,
140 procedure_context: ProcedureContext,
141 task: DrainDatanodeProcedureTask,
142 ) -> Result<ProcedureId> {
143 let _admission = self.admission.lock().await;
144 if task.target_node_id == Some(task.node_id) {
145 return error::InvalidArgumentsSnafu {
146 err_msg: "the target datanode must differ from the drained datanode",
147 }
148 .fail();
149 }
150 if task.parallelism == 0 || task.parallelism > MAX_PARALLELISM {
151 return error::InvalidArgumentsSnafu {
152 err_msg: format!("the parallelism must be in 1..={}", MAX_PARALLELISM),
153 }
154 .fail();
155 }
156 let timeout = task.timeout.unwrap_or(DEFAULT_DRAIN_TIMEOUT);
157 if timeout.is_zero() {
158 return error::InvalidArgumentsSnafu {
159 err_msg: "the timeout must be greater than zero",
160 }
161 .fail();
162 }
163
164 let deadline_ms = i64::try_from(timeout.as_millis())
166 .ok()
167 .and_then(|timeout_ms| current_time_millis().checked_add(timeout_ms))
168 .context(error::InvalidArgumentsSnafu {
169 err_msg: "the timeout exceeds the supported deadline range",
170 })?;
171
172 self.ensure_ready()?;
173 let Some(guard) = self.tracker.insert_running_procedure(&task) else {
174 return error::DrainDatanodeRunningSnafu {
175 node_id: task.node_id,
176 }
177 .fail();
178 };
179
180 let persistent_ctx = PersistentContext {
181 node_id: task.node_id,
182 target_node_id: task.target_node_id,
183 parallelism: task.parallelism,
184 strategy: task.strategy,
185 deadline_ms,
188 };
189 let procedure = DrainDatanodeProcedure::new(self.context(persistent_ctx), vec![guard]);
190 let procedure_with_id =
191 ProcedureWithId::with_random_id(Box::new(procedure)).with_context(procedure_context);
192 let procedure_id = procedure_with_id.id;
193 info!(
194 "Starting drain datanode procedure {procedure_id} for datanode {}, timeout: {:?}",
195 task.node_id, timeout
196 );
197 self.procedure_manager
198 .submit(procedure_with_id)
199 .await
200 .context(error::SubmitProcedureSnafu)?;
201
202 Ok(procedure_id)
203 }
204
205 pub async fn mark_schedulable(&self, node_id: DatanodeId) -> Result<()> {
207 let _admission = self.admission.lock().await;
208 self.ensure_ready()?;
209 ensure!(
210 !self.tracker.contains(node_id),
211 error::DrainDatanodeRunningSnafu { node_id }
212 );
213 self.node_scheduling_manager
214 .mark_schedulable(NodeSchedulingRole::Datanode, node_id)
215 .await?;
216 Ok(())
217 }
218
219 fn ensure_ready(&self) -> Result<()> {
220 ensure!(
221 self.procedure_manager.is_ready(),
222 error::RetryLaterSnafu {
223 reason: "procedure manager has not completed recovery",
224 }
225 );
226 Ok(())
227 }
228
229 pub(crate) fn try_start(&self) -> Result<()> {
231 let table_metadata_manager = self.table_metadata_manager.clone();
232 let node_scheduling_manager = self.node_scheduling_manager.clone();
233 let region_migration_manager = self.region_migration_manager.clone();
234 let selector_ctx = self.selector_ctx.clone();
235 let tracker = self.tracker.clone();
236 self.procedure_manager
237 .register_loader(
238 DrainDatanodeProcedure::TYPE_NAME,
239 Box::new(move |json| {
240 let table_metadata_manager = table_metadata_manager.clone();
241 let node_scheduling_manager = node_scheduling_manager.clone();
242 let region_migration_manager = region_migration_manager.clone();
243 let selector_ctx = selector_ctx.clone();
244 let factory = move |persistent_ctx| {
245 DrainDatanodeContext::new(
246 table_metadata_manager,
247 node_scheduling_manager,
248 region_migration_manager,
249 selector_ctx,
250 persistent_ctx,
251 )
252 };
253 DrainDatanodeProcedure::from_json(json, &tracker, factory)
254 .map(|procedure| Box::new(procedure) as _)
255 }),
256 )
257 .context(error::RegisterProcedureLoaderSnafu {
258 type_name: DrainDatanodeProcedure::TYPE_NAME,
259 })
260 }
261}
262
263#[cfg(test)]
264mod tests {
265 use std::sync::Arc;
266 use std::time::Duration;
267
268 use common_error::ext::ErrorExt;
269 use common_error::status_code::StatusCode;
270 use common_procedure::store::state_store::StateStore;
271 use common_procedure::{Procedure, ProcedureId, ProcedureManagerRef, Status};
272 use tokio::sync::Notify;
273
274 use crate::error;
275 use crate::key::NodeSchedulingRole;
276 use crate::procedure::drain_datanode::manager::{
277 DrainDatanodeManager, DrainDatanodeProcedureTracker,
278 };
279 use crate::procedure::drain_datanode::test_util::{TestEnv, new_persistent_context};
280 use crate::procedure::drain_datanode::{
281 DEFAULT_PARALLELISM, DrainDatanodeProcedure, DrainDatanodeProcedureTask,
282 DrainDatanodeStrategy,
283 };
284
285 struct HoldDrainLock {
286 entered: Arc<Notify>,
287 release: Arc<Notify>,
288 }
289
290 #[async_trait::async_trait]
291 impl Procedure for HoldDrainLock {
292 fn type_name(&self) -> &str {
293 "test-hold-drain-lock"
294 }
295 async fn execute(
296 &mut self,
297 _: &common_procedure::Context,
298 ) -> common_procedure::Result<Status> {
299 self.entered.notify_one();
300 self.release.notified().await;
301 Ok(Status::done())
302 }
303 fn dump(&self) -> common_procedure::Result<String> {
304 Ok(String::new())
305 }
306 fn lock_key(&self) -> common_procedure::LockKey {
307 common_procedure::LockKey::single_exclusive("__drain_datanode/1")
308 }
309 }
310
311 fn task() -> DrainDatanodeProcedureTask {
312 DrainDatanodeProcedureTask {
313 node_id: 1,
314 target_node_id: Some(2),
315 parallelism: DEFAULT_PARALLELISM,
316 strategy: DrainDatanodeStrategy::FailFast,
317 timeout: None,
318 }
319 }
320
321 async fn hold_drain_lock(procedure_manager: &ProcedureManagerRef) -> Arc<Notify> {
322 let entered = Arc::new(Notify::new());
323 let release = Arc::new(Notify::new());
324 procedure_manager
325 .submit(common_procedure::ProcedureWithId::with_random_id(Box::new(
326 HoldDrainLock {
327 entered: entered.clone(),
328 release: release.clone(),
329 },
330 )))
331 .await
332 .unwrap();
333 tokio::time::timeout(Duration::from_secs(5), entered.notified())
334 .await
335 .unwrap();
336 release
337 }
338
339 #[tokio::test]
340 async fn test_submit_drain_rejects_concurrent_tasks_and_scheduling() {
341 let env = TestEnv::new().await;
342 env.procedure_manager().start().await.unwrap();
343 let release = hold_drain_lock(env.procedure_manager()).await;
344 let manager = env.drain_manager();
345 let (first, second) = tokio::join!(
346 manager.submit_procedure(Default::default(), task()),
347 manager.submit_procedure(Default::default(), task()),
348 );
349 assert!(matches!(
350 (first, second),
351 (Ok(_), Err(error::Error::DrainDatanodeRunning { .. }))
352 | (Err(error::Error::DrainDatanodeRunning { .. }), Ok(_))
353 ));
354 assert_eq!(
355 2,
356 env.procedure_manager()
357 .list_procedures()
358 .await
359 .unwrap()
360 .len()
361 );
362 let mut conflicting = task();
363 conflicting.target_node_id = Some(3);
364 for err in [
365 manager
366 .submit_procedure(Default::default(), conflicting)
367 .await
368 .unwrap_err(),
369 manager.mark_schedulable(1).await.unwrap_err(),
370 ] {
371 assert!(matches!(&err, error::Error::DrainDatanodeRunning { .. }));
372 assert_eq!(StatusCode::IllegalState, err.status_code());
373 }
374 release.notify_one();
375 tokio::time::timeout(Duration::from_secs(5), async {
376 loop {
377 if !manager
378 .tracker
379 .running_procedures
380 .read()
381 .unwrap()
382 .contains_key(&1)
383 {
384 break;
385 }
386 tokio::task::yield_now().await;
387 }
388 })
389 .await
390 .unwrap();
391 manager.mark_schedulable(1).await.unwrap();
392 manager.mark_schedulable(1).await.unwrap();
393 env.procedure_manager().stop().await.unwrap();
394 }
395
396 struct RecoveryGate {
397 store: common_meta::state_store::KvStateStore,
398 entered: tokio::sync::Notify,
399 release: tokio::sync::Notify,
400 }
401
402 #[async_trait::async_trait]
403 impl common_procedure::store::state_store::StateStore for RecoveryGate {
404 async fn put(&self, key: &str, value: Vec<u8>) -> common_procedure::Result<()> {
405 self.store.put(key, value).await
406 }
407
408 async fn walk_top_down(
409 &self,
410 path: &str,
411 ) -> common_procedure::Result<common_procedure::store::state_store::KeyValueStream>
412 {
413 if path == "procedure/" {
414 self.entered.notify_one();
415 self.release.notified().await;
416 }
417 self.store.walk_top_down(path).await
418 }
419
420 async fn batch_delete(&self, keys: &[String]) -> common_procedure::Result<()> {
421 self.store.batch_delete(keys).await
422 }
423
424 async fn delete(&self, key: &str) -> common_procedure::Result<()> {
425 self.store.delete(key).await
426 }
427 }
428
429 #[tokio::test]
430 async fn test_drain_admission_waits_for_recovery_and_rejects_running_task() {
431 use crate::procedure::region_migration::test_util::TestingEnv;
432 use common_procedure::local::{LocalManager, ManagerConfig};
433 use common_procedure::store::ProcedureMessage;
434 use common_procedure::test_util::InMemoryPoisonStore;
435
436 let env = TestEnv::new().await;
437 let mut original = DrainDatanodeProcedure::new(
438 env.context(new_persistent_context(
439 Some(2),
440 DrainDatanodeStrategy::FailFast,
441 )),
442 vec![],
443 );
444 assert!(
445 original
446 .execute(&TestingEnv::procedure_context())
447 .await
448 .unwrap()
449 .need_persist()
450 );
451 let id = ProcedureId::random();
452 let message = ProcedureMessage {
453 type_name: DrainDatanodeProcedure::TYPE_NAME.to_string(),
454 data: original.dump().unwrap(),
455 parent_id: None,
456 subprocedure_recovery: Default::default(),
457 step: 0,
458 error: None,
459 context: Default::default(),
460 };
461 drop(original);
462 let store = Arc::new(RecoveryGate {
463 store: common_meta::state_store::KvStateStore::new(env.env.kv_backend()),
464 entered: tokio::sync::Notify::new(),
465 release: tokio::sync::Notify::new(),
466 });
467 store
468 .put(
469 &format!("procedure/{id}/0000000000.step"),
470 serde_json::to_vec(&message).unwrap(),
471 )
472 .await
473 .unwrap();
474 let procedure_manager: ProcedureManagerRef = Arc::new(LocalManager::new(
475 ManagerConfig::default(),
476 store.clone(),
477 Arc::new(InMemoryPoisonStore::default()),
478 None,
479 None,
480 ));
481 let manager = DrainDatanodeManager::new(
482 procedure_manager.clone(),
483 env.table_metadata_manager.clone(),
484 env.node_scheduling_manager.clone(),
485 env.region_migration_manager.clone(),
486 env.selector_ctx.clone(),
487 );
488 manager.try_start().unwrap();
489 let start = tokio::spawn({
490 let procedure_manager = procedure_manager.clone();
491 async move { procedure_manager.start().await }
492 });
493 tokio::time::timeout(Duration::from_secs(5), store.entered.notified())
494 .await
495 .unwrap();
496 assert!(!procedure_manager.is_ready());
497 assert!(matches!(
498 manager.submit_procedure(Default::default(), task()).await,
499 Err(error::Error::RetryLater { .. })
500 ));
501 assert!(matches!(
502 manager.mark_schedulable(1).await,
503 Err(error::Error::RetryLater { .. })
504 ));
505 assert!(
506 !env.node_scheduling_manager
507 .is_schedulable(NodeSchedulingRole::Datanode, 1)
508 .await
509 .unwrap()
510 );
511
512 let release = hold_drain_lock(&procedure_manager).await;
514 store.release.notify_one();
515 tokio::time::timeout(Duration::from_secs(5), start)
516 .await
517 .unwrap()
518 .unwrap()
519 .unwrap();
520 assert!(procedure_manager.is_ready());
521 for _ in 0..2 {
522 assert!(matches!(
523 manager.submit_procedure(Default::default(), task()).await,
524 Err(error::Error::DrainDatanodeRunning { .. })
525 ));
526 }
527 let mut conflicting = task();
528 conflicting.target_node_id = Some(3);
529 for err in [
530 manager
531 .submit_procedure(Default::default(), conflicting)
532 .await
533 .unwrap_err(),
534 manager.mark_schedulable(1).await.unwrap_err(),
535 ] {
536 assert!(matches!(&err, error::Error::DrainDatanodeRunning { .. }));
537 assert_eq!(StatusCode::IllegalState, err.status_code());
538 }
539 release.notify_one();
540 tokio::time::timeout(Duration::from_secs(5), async {
541 while manager.tracker.contains(1) {
542 tokio::task::yield_now().await;
543 }
544 })
545 .await
546 .unwrap();
547 assert!(
548 procedure_manager
549 .procedure_state(id)
550 .await
551 .unwrap()
552 .unwrap()
553 .is_done()
554 );
555 let second = manager
556 .submit_procedure(Default::default(), task())
557 .await
558 .unwrap();
559 assert_ne!(id, second);
560 tokio::time::timeout(Duration::from_secs(5), async {
561 while manager.tracker.contains(1) {
562 tokio::task::yield_now().await;
563 }
564 })
565 .await
566 .unwrap();
567 assert!(
568 procedure_manager
569 .procedure_state(second)
570 .await
571 .unwrap()
572 .unwrap()
573 .is_done()
574 );
575 manager.mark_schedulable(1).await.unwrap();
576 procedure_manager.stop().await.unwrap();
577 assert!(!procedure_manager.is_ready());
578 assert!(matches!(
579 manager.mark_schedulable(1).await,
580 Err(error::Error::RetryLater { .. })
581 ));
582 }
583
584 #[tokio::test]
585 async fn test_failed_drain_can_be_submitted_again() {
586 use crate::key::NodeSchedulingRole;
587 use crate::procedure::drain_datanode::test_util::leader_region;
588 let env = TestEnv::new().await;
589 env.create_table(vec![leader_region(
590 store_api::storage::RegionId::new(1024, 1),
591 1,
592 )])
593 .await;
594 env.node_scheduling_manager
595 .mark_unschedulable(NodeSchedulingRole::Datanode, 2)
596 .await
597 .unwrap();
598 env.procedure_manager().start().await.unwrap();
599 let manager = env.drain_manager();
600 let first = manager
601 .submit_procedure(Default::default(), task())
602 .await
603 .unwrap();
604 tokio::time::timeout(Duration::from_secs(5), async {
605 loop {
606 if env
607 .procedure_manager()
608 .procedure_state(first)
609 .await
610 .unwrap()
611 .unwrap()
612 .is_failed()
613 && !manager
614 .tracker
615 .running_procedures
616 .read()
617 .unwrap()
618 .contains_key(&1)
619 {
620 break;
621 }
622 tokio::task::yield_now().await;
623 }
624 })
625 .await
626 .unwrap();
627 let second = manager
628 .submit_procedure(Default::default(), task())
629 .await
630 .unwrap();
631 assert_ne!(first, second);
632 env.procedure_manager().stop().await.unwrap();
633 }
634
635 #[tokio::test]
636 async fn test_failed_submission_releases_drain_admission() {
637 use common_procedure::local::{LocalManager, ManagerConfig};
638 use common_procedure::test_util::InMemoryPoisonStore;
639
640 let env = TestEnv::new().await;
641 let procedure_manager: ProcedureManagerRef = Arc::new(LocalManager::new(
642 ManagerConfig {
643 max_running_procedures: 0,
644 ..Default::default()
645 },
646 Arc::new(common_meta::state_store::KvStateStore::new(
647 env.env.kv_backend(),
648 )),
649 Arc::new(InMemoryPoisonStore::default()),
650 None,
651 None,
652 ));
653 procedure_manager.start().await.unwrap();
654 let manager = DrainDatanodeManager::new(
655 procedure_manager.clone(),
656 env.table_metadata_manager.clone(),
657 env.node_scheduling_manager.clone(),
658 env.region_migration_manager.clone(),
659 env.selector_ctx.clone(),
660 );
661 assert!(matches!(
662 manager.submit_procedure(Default::default(), task()).await,
663 Err(error::Error::SubmitProcedure { .. })
664 ));
665 assert!(!manager.tracker.contains(1));
666 manager.mark_schedulable(1).await.unwrap();
667 procedure_manager.stop().await.unwrap();
668 }
669
670 #[tokio::test]
671 async fn test_submit_procedure_rejects_invalid_task() {
672 let env = TestEnv::new().await;
673 let manager = env.drain_manager();
674
675 let error = manager
677 .submit_procedure(
678 common_procedure::ProcedureContext::default(),
679 DrainDatanodeProcedureTask {
680 node_id: 1,
681 target_node_id: Some(1),
682 parallelism: DEFAULT_PARALLELISM,
683 strategy: DrainDatanodeStrategy::FailFast,
684 timeout: None,
685 },
686 )
687 .await
688 .unwrap_err();
689 assert!(matches!(error, error::Error::InvalidArguments { .. }));
690
691 for parallelism in [0, crate::procedure::drain_datanode::MAX_PARALLELISM + 1] {
693 let error = manager
694 .submit_procedure(
695 common_procedure::ProcedureContext::default(),
696 DrainDatanodeProcedureTask {
697 node_id: 1,
698 target_node_id: Some(2),
699 parallelism,
700 strategy: DrainDatanodeStrategy::FailFast,
701 timeout: None,
702 },
703 )
704 .await
705 .unwrap_err();
706 assert!(matches!(error, error::Error::InvalidArguments { .. }));
707 }
708
709 for timeout in [
711 Duration::ZERO,
712 Duration::from_millis(i64::MAX as u64 + 1),
713 Duration::from_millis(i64::MAX as u64),
714 ] {
715 let error = manager
716 .submit_procedure(
717 common_procedure::ProcedureContext::default(),
718 DrainDatanodeProcedureTask {
719 timeout: Some(timeout),
720 ..task()
721 },
722 )
723 .await
724 .unwrap_err();
725 assert!(matches!(error, error::Error::InvalidArguments { .. }));
726 assert!(!manager.tracker.contains(1));
727 }
728 assert!(
729 env.procedure_manager()
730 .list_procedures()
731 .await
732 .unwrap()
733 .is_empty()
734 );
735 assert!(
736 env.node_scheduling_manager
737 .is_schedulable(NodeSchedulingRole::Datanode, 1)
738 .await
739 .unwrap()
740 );
741 }
742
743 #[tokio::test]
744 async fn test_drain_recovery_reserves_node_until_drop() {
745 let env = TestEnv::new().await;
746 let context = env.context(new_persistent_context(
747 Some(2),
748 DrainDatanodeStrategy::FailFast,
749 ));
750 let procedure = DrainDatanodeProcedure::new(context, vec![]);
751 let json = procedure.dump().unwrap();
752 let tracker = DrainDatanodeProcedureTracker::default();
753
754 let reloaded = DrainDatanodeProcedure::from_json(&json, &tracker, |persistent_ctx| {
756 env.context(persistent_ctx)
757 })
758 .unwrap();
759 let task = DrainDatanodeProcedureTask {
760 node_id: 1,
761 target_node_id: Some(2),
762 parallelism: DEFAULT_PARALLELISM,
763 strategy: DrainDatanodeStrategy::FailFast,
764 timeout: None,
765 };
766 assert!(tracker.contains(task.node_id));
767 assert!(tracker.insert_running_procedure(&task).is_none());
768
769 drop(reloaded);
771 assert!(tracker.insert_running_procedure(&task).is_some());
772 }
773}