1pub(crate) mod cursor;
16pub(crate) mod dispatch;
17pub(crate) mod drain_datanode_end;
18pub(crate) mod drain_datanode_start;
19pub(crate) mod manager;
20pub(crate) mod utils;
21
22#[cfg(test)]
23mod test_util;
24
25use std::any::Any;
26use std::fmt::Debug;
27use std::time::Duration;
28
29use common_meta::DatanodeId;
30use common_meta::key::TableMetadataManagerRef;
31use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
32use common_procedure::{
33 Context as ProcedureContext, Error as ProcedureError, LockKey, Procedure, ProcedureId,
34 Result as ProcedureResult, Status,
35};
36use common_telemetry::{error, info};
37use serde::{Deserialize, Serialize};
38use snafu::ResultExt;
39use store_api::storage::RegionId;
40
41use crate::error::{self, Result};
42use crate::metasrv::SelectorContext;
43use crate::metrics::METRIC_META_REGION_MIGRATION_FAIL;
44use crate::node_scheduling::NodeSchedulingManagerRef;
45use crate::procedure::drain_datanode::drain_datanode_start::DrainDatanodeStart;
46use crate::procedure::drain_datanode::manager::{
47 DrainDatanodeProcedureGuard, DrainDatanodeProcedureTracker,
48};
49use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
50
51pub const DEFAULT_PARALLELISM: usize = 4;
53pub const MAX_PARALLELISM: usize = 32;
55pub const DEFAULT_DRAIN_TIMEOUT: Duration = Duration::from_secs(20 * 60);
57pub const MAX_RESCAN_ROUNDS: u32 = 3;
60
61#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
63pub enum DrainDatanodeStrategy {
64 FailFast,
67 #[default]
69 TryAll,
70}
71
72#[derive(Debug, Clone, Serialize, Deserialize)]
77pub struct PersistentContext {
78 pub node_id: DatanodeId,
80 pub target_node_id: Option<DatanodeId>,
82 pub parallelism: usize,
84 pub strategy: DrainDatanodeStrategy,
86 pub deadline_ms: i64,
88}
89
90#[derive(Debug, Clone)]
92pub struct SubprocedureMeta {
93 pub region_ids: Vec<RegionId>,
95 pub procedure_id: ProcedureId,
97}
98
99#[derive(Debug, Default)]
106pub struct VolatileContext {
107 pub(crate) scan: Option<utils::scan::RegionScan>,
109 pub follower_count: usize,
111 pub migrated: usize,
113 pub failed_regions: Vec<RegionId>,
115 pub rescan_round: u32,
117}
118
119#[derive(Debug, Clone)]
121pub struct DrainDatanodeProcedureTask {
122 pub node_id: DatanodeId,
124 pub target_node_id: Option<DatanodeId>,
126 pub parallelism: usize,
128 pub strategy: DrainDatanodeStrategy,
130 pub timeout: Option<Duration>,
132}
133
134pub struct Context {
136 pub persistent_ctx: PersistentContext,
138 pub volatile_ctx: VolatileContext,
140 pub table_metadata_manager: TableMetadataManagerRef,
142 pub node_scheduling_manager: NodeSchedulingManagerRef,
144 pub region_migration_manager: RegionMigrationManagerRef,
146 pub selector_ctx: SelectorContext,
148}
149
150impl Context {
151 pub(crate) fn new(
156 table_metadata_manager: TableMetadataManagerRef,
157 node_scheduling_manager: NodeSchedulingManagerRef,
158 region_migration_manager: RegionMigrationManagerRef,
159 selector_ctx: SelectorContext,
160
161 persistent_ctx: PersistentContext,
162 ) -> Self {
163 Self {
164 persistent_ctx,
165 volatile_ctx: VolatileContext::default(),
166 table_metadata_manager,
167 node_scheduling_manager,
168 region_migration_manager,
169 selector_ctx,
170 }
171 }
172}
173
174impl Context {
175 pub(crate) fn handle_migration_error(
177 &mut self,
178 e: error::Error,
179 regions: &[RegionId],
180 ) -> Result<()> {
181 if self.persistent_ctx.strategy == DrainDatanodeStrategy::FailFast {
182 return Err(e);
183 }
184 error!(e; "Gave up draining regions {:?}", regions);
185 METRIC_META_REGION_MIGRATION_FAIL.inc_by(regions.len() as u64);
186 self.volatile_ctx.failed_regions.extend_from_slice(regions);
187 Ok(())
188 }
189}
190
191#[derive(Debug, Deserialize)]
193pub struct DrainDatanodeDataOwned {
194 persistent_ctx: PersistentContext,
195 state: Box<dyn State>,
196}
197
198#[derive(Debug, Serialize)]
200pub struct DrainDatanodeData<'a> {
201 persistent_ctx: &'a PersistentContext,
202 state: &'a dyn State,
203}
204
205#[async_trait::async_trait]
207#[typetag::serde(tag = "drain_datanode_state")]
208pub(crate) trait State: Sync + Send + Debug {
209 fn name(&self) -> &'static str {
211 let type_name = std::any::type_name::<Self>();
212 type_name.split("::").last().unwrap_or(type_name)
213 }
214
215 async fn next(
217 &mut self,
218 ctx: &mut Context,
219 procedure_ctx: &ProcedureContext,
220 ) -> Result<(Box<dyn State>, Status)>;
221
222 fn as_any(&self) -> &dyn Any;
224}
225
226pub(crate) struct DrainDatanodeProcedure {
234 state: Box<dyn State>,
235 context: Context,
236 _guards: Vec<DrainDatanodeProcedureGuard>,
239}
240
241impl DrainDatanodeProcedure {
242 const TYPE_NAME: &'static str = "metasrv-procedure::DrainDatanode";
243
244 pub fn new(context: Context, guards: Vec<DrainDatanodeProcedureGuard>) -> Self {
246 Self {
247 state: Box::new(DrainDatanodeStart),
248 context,
249 _guards: guards,
250 }
251 }
252
253 pub fn from_json<F>(
258 json: &str,
259 tracker: &DrainDatanodeProcedureTracker,
260 ctx_factory: F,
261 ) -> ProcedureResult<Self>
262 where
263 F: FnOnce(PersistentContext) -> Context,
264 {
265 let DrainDatanodeDataOwned {
266 persistent_ctx,
267 state,
268 } = serde_json::from_str(json).context(FromJsonSnafu)?;
269 let guards = tracker
270 .insert_running_procedure(&DrainDatanodeProcedureTask {
271 node_id: persistent_ctx.node_id,
272 target_node_id: persistent_ctx.target_node_id,
273 parallelism: persistent_ctx.parallelism,
274 strategy: persistent_ctx.strategy,
275 timeout: None,
276 })
277 .into_iter()
278 .collect();
279
280 let context = ctx_factory(persistent_ctx);
281 Ok(Self {
282 state,
283 context,
284 _guards: guards,
285 })
286 }
287}
288
289#[async_trait::async_trait]
290impl Procedure for DrainDatanodeProcedure {
291 fn type_name(&self) -> &str {
292 Self::TYPE_NAME
293 }
294
295 #[tracing::instrument(skip_all, fields(
296 state = %self.state.name(),
297 node_id = %self.context.persistent_ctx.node_id
298 ))]
299 async fn execute(&mut self, ctx: &ProcedureContext) -> ProcedureResult<Status> {
300 let state_name = self.state.name();
301 info!(
302 "Drain datanode procedure executing state: {}, node_id: {}",
303 state_name, self.context.persistent_ctx.node_id
304 );
305 match self.state.next(&mut self.context, ctx).await {
306 Ok((next, status)) => {
307 self.state = next;
308 Ok(status)
309 }
310 Err(e) => {
311 if e.is_retryable() {
312 Err(ProcedureError::retry_later(e))
313 } else {
314 error!(
315 e;
316 "Drain datanode procedure failed, node_id: {}",
317 self.context.persistent_ctx.node_id
318 );
319 Err(ProcedureError::external(e))
320 }
321 }
322 }
323 }
324
325 fn dump(&self) -> ProcedureResult<String> {
326 let data = DrainDatanodeData {
327 persistent_ctx: &self.context.persistent_ctx,
328 state: self.state.as_ref(),
329 };
330 serde_json::to_string(&data).context(ToJsonSnafu)
331 }
332
333 fn lock_key(&self) -> LockKey {
334 LockKey::single_exclusive(format!(
335 "__drain_datanode/{}",
336 self.context.persistent_ctx.node_id
337 ))
338 }
339}
340
341#[cfg(test)]
342mod tests {
343 use std::collections::HashMap;
344 use std::sync::Arc;
345 use std::time::Duration;
346
347 use crate::procedure::drain_datanode::cursor::DrainDatanodeCursor;
348 use crate::procedure::drain_datanode::dispatch::Dispatch;
349 use crate::procedure::drain_datanode::test_util::create_kafka_table;
350 use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
351 use api::v1::meta::heartbeat_request::NodeWorkloads;
352 use common_error::ext::ErrorExt;
353 use common_error::status_code::StatusCode;
354 use common_meta::cluster::NodeInfo;
355 use common_meta::error::Result as MetaResult;
356 use common_meta::key::TableMetadataManagerRef;
357 use common_meta::peer::{PeerDiscovery, PeerDiscoveryRef};
358 use common_meta::state_store::KvStateStore;
359 use common_procedure::store::ProcedureMessage;
360 use common_procedure::store::state_store::StateStore;
361 use common_procedure::{Procedure, ProcedureId, ProcedureState, Status};
362 use futures::TryStreamExt;
363 use store_api::storage::RegionId;
364
365 use crate::error;
366 use crate::key::NodeSchedulingRole;
367 use crate::procedure::drain_datanode::drain_datanode_start::DrainDatanodeStart;
368 use crate::procedure::drain_datanode::manager::DrainDatanodeProcedureTracker;
369 use crate::procedure::drain_datanode::test_util::{
370 TestEnv, follower_region, leader_region, new_persistent_context, run_until_done,
371 spawn_mock_datanode, wait_for_terminal_state,
372 };
373 use crate::procedure::drain_datanode::{
374 DEFAULT_PARALLELISM, DrainDatanodeProcedure, DrainDatanodeProcedureTask,
375 DrainDatanodeStrategy, State,
376 };
377 use crate::procedure::region_migration::test_util::TestingEnv;
378 use crate::service::mailbox::Channel;
379
380 #[tokio::test]
381 async fn test_drain_datanode_fences_and_completes_without_leaders() {
382 let env = TestEnv::new().await;
383 env.create_table(vec![follower_region(RegionId::new(1024, 1), 2, 1)])
384 .await;
385
386 let mut ctx = env.context(new_persistent_context(
387 None,
388 DrainDatanodeStrategy::FailFast,
389 ));
390 let procedure_ctx = TestingEnv::procedure_context();
391 let mut state: Box<dyn State> = Box::new(DrainDatanodeStart);
392
393 let status = run_until_done(&mut state, &mut ctx, &procedure_ctx)
394 .await
395 .unwrap();
396
397 assert!(matches!(status, Status::Done { .. }));
398 assert_eq!(0, ctx.volatile_ctx.migrated);
399 assert_eq!(1, ctx.volatile_ctx.follower_count);
400 assert!(
401 !env.node_scheduling_manager
402 .is_schedulable(NodeSchedulingRole::Datanode, 1)
403 .await
404 .unwrap()
405 );
406 }
407
408 #[tokio::test]
409 async fn test_drain_datanode_dump_roundtrip() {
410 let env = TestEnv::new().await;
411 let persistent_ctx = new_persistent_context(Some(2), DrainDatanodeStrategy::TryAll);
412
413 let procedure = DrainDatanodeProcedure::new(env.context(persistent_ctx), vec![]);
414 let json = procedure.dump().unwrap();
415 let tracker = DrainDatanodeProcedureTracker::default();
416 let reloaded = DrainDatanodeProcedure::from_json(&json, &tracker, |persistent_ctx| {
417 env.context(persistent_ctx)
418 })
419 .unwrap();
420
421 assert_eq!("DrainDatanodeStart", reloaded.state.name());
422 let ctx = &reloaded.context.persistent_ctx;
423 assert_eq!(1, ctx.node_id);
424 assert_eq!(Some(2), ctx.target_node_id);
425 assert_eq!(DEFAULT_PARALLELISM, ctx.parallelism);
426 assert_eq!(DrainDatanodeStrategy::TryAll, ctx.strategy);
427 assert!(!json.contains("scan"));
429 assert!(!json.contains("rescan_round"));
430 }
431
432 #[tokio::test]
433 async fn test_drain_datanode_default_strategy_continues_after_batch_failures() {
434 let env = TestEnv::new().await;
435 let regions: Vec<_> = (1..=5).map(|number| RegionId::new(1024, number)).collect();
436 env.create_table(
437 regions
438 .iter()
439 .copied()
440 .map(|region_id| leader_region(region_id, 1))
441 .collect(),
442 )
443 .await;
444 env.node_scheduling_manager
446 .mark_unschedulable(NodeSchedulingRole::Datanode, 3)
447 .await
448 .unwrap();
449
450 let mut persistent_ctx = new_persistent_context(Some(3), DrainDatanodeStrategy::default());
451 persistent_ctx.parallelism = 2;
452 let mut ctx = env.context(persistent_ctx);
453 let procedure_ctx = TestingEnv::procedure_context();
454 let mut state: Box<dyn State> = Box::new(DrainDatanodeStart);
455
456 let error = run_until_done(&mut state, &mut ctx, &procedure_ctx)
457 .await
458 .unwrap_err();
459
460 let error::Error::DrainDatanodeAbort { failed_regions, .. } = error else {
461 panic!("expected the drain procedure to abort");
462 };
463 assert_eq!(regions, failed_regions);
464 assert_eq!(regions, ctx.volatile_ctx.failed_regions);
465 assert_eq!(0, ctx.volatile_ctx.migrated);
466 }
467
468 #[tokio::test]
469 async fn test_drain_datanode_recovery_obeys_deadline() {
470 for expired in [true, false] {
471 let mut env = TestEnv::new().await;
472 let first_id = RegionId::new(1024, 1);
473 let second_id = RegionId::new(1024, 2);
474 env.create_table(vec![
475 leader_region(first_id, 1),
476 leader_region(second_id, 1),
477 ])
478 .await;
479 env.mock_datanodes().await;
480 let mut persistent_ctx =
481 new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
482 if expired {
483 persistent_ctx.deadline_ms = 0;
484 }
485 let mut procedure =
486 DrainDatanodeProcedure::new(env.context(persistent_ctx.clone()), vec![]);
487 assert!(
488 procedure
489 .execute(&TestingEnv::procedure_context())
490 .await
491 .unwrap()
492 .need_persist()
493 );
494 let checkpoint = procedure.dump().unwrap();
495 let json: serde_json::Value = serde_json::from_str(&checkpoint).unwrap();
496 assert_eq!(
497 serde_json::json!({
498 "persistent_ctx": {
499 "node_id": 1, "target_node_id": 2,
500 "parallelism": DEFAULT_PARALLELISM, "strategy": "FailFast",
501 "deadline_ms": persistent_ctx.deadline_ms,
502 },
503 "state": {"drain_datanode_state": "DrainDatanodeCursor"},
504 }),
505 json
506 );
507 drop(procedure);
508 let recovered = DrainDatanodeProcedure::from_json(
509 &checkpoint,
510 &DrainDatanodeProcedureTracker::default(),
511 |persistent_ctx| env.context(persistent_ctx),
512 )
513 .unwrap();
514 assert!(recovered.context.volatile_ctx.scan.is_none());
515 drop(recovered);
516 let id = ProcedureId::random();
517 let message = common_procedure::store::ProcedureMessage {
518 type_name: DrainDatanodeProcedure::TYPE_NAME.to_string(),
519 data: checkpoint,
520 parent_id: None,
521 subprocedure_recovery: Default::default(),
522 step: 0,
523 error: None,
524 context: Default::default(),
525 };
526 let store = common_meta::state_store::KvStateStore::new(env.env.kv_backend());
527 common_procedure::store::state_store::StateStore::put(
528 &store,
529 &format!("procedure/{id}/0000000000.step"),
530 serde_json::to_vec(&message).unwrap(),
531 )
532 .await
533 .unwrap();
534 let manager = env.drain_manager();
535 manager.try_start().unwrap();
536 env.procedure_manager().start().await.unwrap();
537 let state = wait_for_terminal_state(env.procedure_manager(), id).await;
538 if expired {
539 let ProcedureState::Failed { error } = state else {
540 panic!("expected expired drain: {state:?}");
541 };
542 assert_eq!(StatusCode::Cancelled, error.status_code());
543 } else {
544 assert!(state.is_done(), "{state:?}");
545 }
546 let table_route = env
547 .table_metadata_manager
548 .table_route_manager()
549 .table_route_storage()
550 .get(1024)
551 .await
552 .unwrap()
553 .unwrap();
554 let routes = table_route.region_routes().unwrap();
555 assert_eq!(2, routes.len());
556 for route in routes {
557 assert_eq!(
558 Some(if expired { 1 } else { 2 }),
559 route.leader_peer.as_ref().map(|peer| peer.id)
560 );
561 }
562 }
563 }
564
565 #[tokio::test]
566 async fn test_drain_datanode_recovered_migration_outlives_parent() {
567 let env = TestEnv::new().await;
568 let region_id = RegionId::new(1024, 1);
569 env.create_table(vec![leader_region(region_id, 1)]).await;
570 assert_recovered_migration_outlives_parent(env, vec![region_id]).await;
571 }
572
573 #[tokio::test]
574 async fn test_drain_datanode_recovers_every_region_in_a_migration_batch() {
575 let env = TestEnv::new().await;
576 create_kafka_table(&env, 1024, 3, 1).await;
577 let regions = (0..3).map(|i| RegionId::new(1024, i)).collect();
578 assert_recovered_migration_outlives_parent(env, regions).await;
579 }
580
581 async fn assert_recovered_migration_outlives_parent(
582 mut env: TestEnv,
583 region_ids: Vec<RegionId>,
584 ) {
585 let procedure_ctx = TestingEnv::procedure_context();
586 let mut parent = DrainDatanodeProcedure::new(
587 env.context(new_persistent_context(
588 Some(2),
589 DrainDatanodeStrategy::FailFast,
590 )),
591 vec![],
592 );
593 assert!(parent.execute(&procedure_ctx).await.unwrap().need_persist());
594 assert_eq!("DrainDatanodeCursor", parent.state.name());
595 let parent_checkpoint = parent.dump().unwrap();
596 let Status::Suspended {
597 subprocedures: mut children,
598 recovery,
599 ..
600 } = run_until_done(&mut parent.state, &mut parent.context, &procedure_ctx)
601 .await
602 .unwrap()
603 else {
604 panic!("expected a migration batch");
605 };
606 assert_eq!(1, children.len());
607 let mut child = children.pop().unwrap();
608 assert!(
610 child
611 .procedure
612 .execute(&procedure_ctx)
613 .await
614 .unwrap()
615 .need_persist()
616 );
617 let parent_id = ProcedureId::random();
618 let child_id = child.id;
619 let store = KvStateStore::new(env.env.kv_backend());
620 for (id, message) in [
621 (
622 parent_id,
623 ProcedureMessage {
624 type_name: parent.type_name().to_string(),
625 data: parent_checkpoint,
626 parent_id: None,
627 subprocedure_recovery: Default::default(),
628 step: 0,
629 error: None,
630 context: Default::default(),
631 },
632 ),
633 (
634 child_id,
635 ProcedureMessage {
636 type_name: child.procedure.type_name().to_string(),
637 data: child.procedure.dump().unwrap(),
638 parent_id: Some(parent_id),
639 subprocedure_recovery: recovery,
640 step: 0,
641 error: None,
642 context: child.context.clone(),
643 },
644 ),
645 ] {
646 store
647 .put(
648 &format!("procedure/{id}/0000000000.step"),
649 serde_json::to_vec(&message).unwrap(),
650 )
651 .await
652 .unwrap();
653 }
654 drop(child);
655 drop(parent);
656 assert!(
657 region_ids
658 .iter()
659 .all(|id| !env.region_migration_manager.tracker().contains(*id))
660 );
661
662 let mailbox_ctx = env.env.mailbox_context();
664 let mut receivers = Vec::new();
665 for node_id in [1, 2] {
666 let (tx, rx) = tokio::sync::mpsc::channel(8);
667 mailbox_ctx
668 .insert_heartbeat_response_receiver(Channel::Datanode(node_id), tx)
669 .await;
670 receivers.push(rx);
671 }
672 let mailbox = mailbox_ctx.mailbox().clone();
673 env.region_migration_manager.try_start().unwrap();
674 let drain_manager = env.drain_manager();
675 drain_manager.try_start().unwrap();
676 env.procedure_manager().start().await.unwrap();
677 assert!(
678 wait_for_terminal_state(env.procedure_manager(), parent_id)
679 .await
680 .is_done()
681 );
682 assert!(
683 region_ids
684 .iter()
685 .all(|id| env.region_migration_manager.tracker().contains(*id))
686 );
687 assert!(matches!(
688 env.procedure_manager()
689 .procedure_state(child_id)
690 .await
691 .unwrap(),
692 Some(ProcedureState::Running)
693 ));
694 assert_eq!(
696 2,
697 env.procedure_manager()
698 .list_procedures()
699 .await
700 .unwrap()
701 .len()
702 );
703
704 for rx in receivers {
705 spawn_mock_datanode(mailbox.clone(), rx);
706 }
707 assert!(
708 wait_for_terminal_state(env.procedure_manager(), child_id)
709 .await
710 .is_done()
711 );
712 for ®ion_id in ®ion_ids {
713 assert!(
714 !crate::procedure::drain_datanode::utils::leads_region(
715 &env.table_metadata_manager,
716 1,
717 region_id,
718 )
719 .await
720 .unwrap()
721 );
722 }
723 tokio::time::timeout(Duration::from_secs(5), async {
725 loop {
726 let mut records = store.walk_top_down("procedure/").await.unwrap();
727 if records.try_next().await.unwrap().is_none()
728 && region_ids
729 .iter()
730 .all(|id| !env.region_migration_manager.tracker().contains(*id))
731 {
732 break;
733 }
734 tokio::time::sleep(Duration::from_millis(10)).await;
735 }
736 })
737 .await
738 .expect("recovered migration did not clean up");
739 env.procedure_manager().stop().await.unwrap();
740 }
741
742 #[tokio::test]
743 async fn test_drain_datanode_migrates_leader_region() {
744 assert_drain_migrates_leader_regions(1).await;
745 }
746
747 #[tokio::test]
748 async fn test_drain_datanode_migrates_multiple_batches() {
749 assert_drain_migrates_leader_regions(3).await;
750 }
751
752 async fn assert_drain_migrates_leader_regions(region_count: u32) {
753 let mut env = TestEnv::new().await;
754 let regions: Vec<_> = (1..=region_count)
755 .map(|number| RegionId::new(1024, number))
756 .collect();
757 env.create_table(
758 regions
759 .iter()
760 .copied()
761 .map(|region_id| leader_region(region_id, 1))
762 .collect(),
763 )
764 .await;
765
766 env.mock_datanodes().await;
767
768 env.procedure_manager().start().await.unwrap();
769 let manager = env.drain_manager();
770 let procedure_id = manager
771 .submit_procedure(
772 common_procedure::ProcedureContext::default(),
773 DrainDatanodeProcedureTask {
774 node_id: 1,
775 target_node_id: Some(2),
776 parallelism: 1,
777 strategy: DrainDatanodeStrategy::FailFast,
778 timeout: None,
779 },
780 )
781 .await
782 .unwrap();
783
784 let state = wait_for_terminal_state(env.procedure_manager(), procedure_id).await;
785 assert!(
786 matches!(state, ProcedureState::Done { .. }),
787 "the drain procedure did not complete successfully: {state:?}"
788 );
789
790 let table_route = env
791 .table_metadata_manager
792 .table_route_manager()
793 .table_route_storage()
794 .get(1024)
795 .await
796 .unwrap()
797 .unwrap();
798 assert_eq!(
799 region_count as usize,
800 table_route.region_routes().unwrap().len()
801 );
802 assert!(
803 !env.node_scheduling_manager
804 .is_schedulable(NodeSchedulingRole::Datanode, 1)
805 .await
806 .unwrap()
807 );
808 for region_route in table_route.region_routes().unwrap() {
809 assert_eq!(
810 Some(2),
811 region_route.leader_peer.as_ref().map(|peer| peer.id)
812 );
813 }
814 }
815
816 #[tokio::test]
817 async fn test_drain_drains_multiple_topic_windows_and_confirms_completion() {
818 use crate::procedure::drain_datanode::DrainDatanodeProcedureTask;
819 use crate::procedure::drain_datanode::test_util::{
820 create_kafka_table, wait_for_terminal_state,
821 };
822 use common_procedure::ProcedureState;
823
824 let mut env = TestEnv::new().await;
825 create_kafka_table(&env, 1024, 13, 3).await;
826 env.mock_datanodes().await;
827 env.procedure_manager().start().await.unwrap();
828 let id = env
829 .drain_manager()
830 .submit_procedure(
831 common_procedure::ProcedureContext::default(),
832 DrainDatanodeProcedureTask {
833 node_id: 1,
834 target_node_id: Some(2),
835 parallelism: 2,
836 strategy: DrainDatanodeStrategy::FailFast,
837 timeout: None,
838 },
839 )
840 .await
841 .unwrap();
842 let state = wait_for_terminal_state(env.procedure_manager(), id).await;
843 assert!(matches!(state, ProcedureState::Done { .. }), "{state:?}");
844 let route = env
845 .table_metadata_manager
846 .table_route_manager()
847 .table_route_storage()
848 .get(1024)
849 .await
850 .unwrap()
851 .unwrap();
852 let regions = route.region_routes().unwrap();
853 assert_eq!(13, regions.len());
854 assert!(
855 regions
856 .iter()
857 .all(|region| region.leader_peer.as_ref().is_some_and(|peer| peer.id == 2))
858 );
859 env.procedure_manager().stop().await.unwrap();
860 }
861
862 #[tokio::test]
863 async fn test_drain_skips_region_moved_or_dropped_during_discovery() {
864 for strategy in [
865 DrainDatanodeStrategy::FailFast,
866 DrainDatanodeStrategy::TryAll,
867 ] {
868 for leader_id in [Some(2), Some(3), None] {
871 let env = TestEnv::new().await;
872 let region_id = RegionId::new(1024, 1);
873 env.create_table(vec![leader_region(region_id, 1)]).await;
874 let mut ctx = env.context(new_persistent_context(Some(2), strategy));
875 ctx.selector_ctx.peer_discovery = Arc::new(ChangeRouteDuringDiscovery {
876 inner: ctx.selector_ctx.peer_discovery.clone(),
877 metadata: env.table_metadata_manager.clone(),
878 region_id,
879 leader_id,
880 });
881
882 let (mut dispatch, _) = DrainDatanodeCursor::default()
883 .next(&mut ctx, &TestingEnv::procedure_context())
884 .await
885 .unwrap();
886
887 let (next, status) = dispatch
888 .next(&mut ctx, &TestingEnv::procedure_context())
889 .await
890 .unwrap();
891 assert!(next.as_any().is::<DrainDatanodeCursor>());
892 assert!(matches!(status, Status::Executing { persist: false, .. }));
893 assert!(ctx.volatile_ctx.failed_regions.is_empty());
894 assert_eq!(0, ctx.volatile_ctx.migrated);
895 assert!(!env.region_migration_manager.tracker().contains(region_id));
896 }
897 }
898 }
899
900 #[tokio::test]
901 async fn test_grouped_migrations_reduce_child_count() {
902 common_telemetry::init_default_ut_logging();
903 let env = TestEnv::new().await;
904 create_kafka_table(&env, 1024, 8, 1).await;
905 for trial in 0..3 {
906 for grouped in [false, true] {
907 let mut persistent =
908 new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
909 persistent.parallelism = 8;
910 let mut ctx = env.context(persistent);
911 let (prepared, _) = DrainDatanodeCursor::default()
912 .next(&mut ctx, &TestingEnv::procedure_context())
913 .await
914 .unwrap();
915 let batches = prepared
916 .as_any()
917 .downcast_ref::<Dispatch>()
918 .unwrap()
919 .batches
920 .clone();
921 let batches = if grouped {
922 batches
923 } else {
924 batches
925 .into_iter()
926 .flat_map(|batch| {
927 batch.region_ids.clone().into_iter().map(move |region_id| {
928 RegionMigrationTaskBatch {
929 region_ids: vec![region_id],
930 ..batch.clone()
931 }
932 })
933 })
934 .collect()
935 };
936 let mut dispatch = Dispatch::new(batches);
937 let (next, status) = dispatch
938 .next(&mut ctx, &TestingEnv::procedure_context())
939 .await
940 .unwrap();
941 let Status::Suspended { subprocedures, .. } = status else {
942 panic!("expected children");
943 };
944 let count = subprocedures.len();
945 assert_eq!(if grouped { 1 } else { 8 }, count);
946 assert!(next.as_any().is::<Dispatch>());
947 let mut regions = subprocedures
948 .iter()
949 .flat_map(|child| {
950 let data: serde_json::Value =
951 serde_json::from_str(&child.procedure.dump().unwrap()).unwrap();
952 serde_json::from_value::<Vec<RegionId>>(
953 data["persistent_ctx"]["region_ids"].clone(),
954 )
955 .unwrap()
956 })
957 .collect::<Vec<_>>();
958 regions.sort_unstable();
959 assert_eq!(
960 (0..8).map(|i| RegionId::new(1024, i)).collect::<Vec<_>>(),
961 regions
962 );
963 common_telemetry::info!(
964 "trial={trial}, grouped={grouped}, regions={}, children={count}",
965 regions.len()
966 );
967 drop(subprocedures);
968 assert!(
969 regions
970 .iter()
971 .all(|id| !env.region_migration_manager.tracker().contains(*id))
972 );
973 }
974 }
975 }
976
977 struct ChangeRouteDuringDiscovery {
979 inner: PeerDiscoveryRef,
980 metadata: TableMetadataManagerRef,
981 region_id: RegionId,
982 leader_id: Option<u64>,
983 }
984
985 #[async_trait::async_trait]
986 impl PeerDiscovery for ChangeRouteDuringDiscovery {
987 async fn active_frontends(&self) -> MetaResult<Vec<NodeInfo>> {
988 self.inner.active_frontends().await
989 }
990
991 async fn active_datanodes(
992 &self,
993 filter: Option<for<'a> fn(&'a NodeWorkloads) -> bool>,
994 ) -> MetaResult<Vec<NodeInfo>> {
995 let storage = self.metadata.table_route_manager().table_route_storage();
996 let table_id = self.region_id.table_id();
997 let current = storage.get_with_raw_bytes(table_id).await?.unwrap();
998 if let Some(leader_id) = self.leader_id {
999 let updated = current.update(vec![leader_region(self.region_id, leader_id)])?;
1000 let (txn, _) = storage.build_update_txn(table_id, ¤t, &updated)?;
1001 assert!(self.metadata.kv_backend().txn(txn).await?.succeeded);
1002 } else {
1003 let table_info = self
1004 .metadata
1005 .table_info_manager()
1006 .get(table_id)
1007 .await?
1008 .unwrap();
1009 self.metadata
1010 .delete_table_metadata(
1011 table_id,
1012 &table_info.table_name(),
1013 ¤t,
1014 &HashMap::new(),
1015 None,
1016 )
1017 .await?;
1018 }
1019 self.inner.active_datanodes(filter).await
1020 }
1021
1022 async fn active_flownodes(
1023 &self,
1024 filter: Option<for<'a> fn(&'a NodeWorkloads) -> bool>,
1025 ) -> MetaResult<Vec<NodeInfo>> {
1026 self.inner.active_flownodes(filter).await
1027 }
1028 }
1029
1030 #[tokio::test]
1031 async fn test_drain_datanode_dispatches_migration() {
1032 let env = TestEnv::new().await;
1033 let region_id = RegionId::new(1024, 1);
1034 env.create_table(vec![leader_region(region_id, 1)]).await;
1035
1036 let mut ctx = env.context(new_persistent_context(
1037 Some(2),
1038 DrainDatanodeStrategy::FailFast,
1039 ));
1040 let procedure_ctx = TestingEnv::procedure_context();
1041
1042 let (mut next, _) = DrainDatanodeStart
1043 .next(&mut ctx, &procedure_ctx)
1044 .await
1045 .unwrap();
1046 let status = run_until_done(&mut next, &mut ctx, &procedure_ctx)
1047 .await
1048 .unwrap();
1049
1050 let Status::Suspended {
1051 subprocedures,
1052 recovery,
1053 persist,
1054 } = status
1055 else {
1056 panic!("expected the dispatch state to suspend the procedure");
1057 };
1058 assert!(!persist);
1060 assert_eq!(common_procedure::SubprocedureRecovery::Detach, recovery);
1061 assert_eq!(1, subprocedures.len());
1062 assert!(env.region_migration_manager.tracker().contains(region_id));
1063 let data: serde_json::Value =
1064 serde_json::from_str(&subprocedures[0].procedure.dump().unwrap()).unwrap();
1065 assert_eq!(
1066 serde_json::json!([region_id]),
1067 data["persistent_ctx"]["region_ids"]
1068 );
1069 assert!(next.as_any().downcast_ref::<Dispatch>().is_some());
1070 }
1071}