1use std::any::Any;
16
17use common_procedure::{
18 Context as ProcedureContext, ProcedureContext as SubmissionContext, ProcedureWithId, Status,
19 SubprocedureRecovery, watcher,
20};
21use common_telemetry::{error, info, warn};
22use serde::{Deserialize, Serialize};
23use snafu::ResultExt;
24use store_api::storage::RegionId;
25
26use crate::error::{self, Result};
27use crate::metrics::{METRIC_META_REGION_MIGRATION_DATANODES, METRIC_META_REGION_MIGRATION_FAIL};
28use crate::procedure::drain_datanode::cursor::DrainDatanodeCursor;
29use crate::procedure::drain_datanode::{
30 Context, DrainDatanodeStrategy, State, SubprocedureMeta, utils,
31};
32use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
33
34#[derive(Debug, Clone, Serialize, Deserialize)]
37pub(crate) struct Dispatch {
38 #[serde(skip)]
40 pub(crate) batches: Vec<RegionMigrationTaskBatch>,
41 #[serde(skip)]
43 inflight_subprocedures: Vec<SubprocedureMeta>,
44}
45
46impl Dispatch {
47 pub(crate) fn new(batches: Vec<RegionMigrationTaskBatch>) -> Self {
49 Self {
50 batches,
51 inflight_subprocedures: vec![],
52 }
53 }
54
55 async fn wait_for_inflight_subprocedures(
60 &mut self,
61 ctx: &mut Context,
62 procedure_ctx: &ProcedureContext,
63 strategy: DrainDatanodeStrategy,
64 ) -> Result<()> {
65 let node_id = ctx.persistent_ctx.node_id;
66 let mut migrated = Vec::new();
67 let mut failed = Vec::new();
68 for meta in &self.inflight_subprocedures {
69 let receiver = procedure_ctx
70 .provider
71 .procedure_state_receiver(meta.procedure_id)
72 .await
73 .inspect_err(|e| {
74 error!(e; "Failed to get state receiver for migration sub-procedure {}", meta.procedure_id);
75 })
76 .context(error::QueryProcedureSnafu)?;
77 let Some(mut receiver) = receiver else {
78 warn!(
79 "Migration sub-procedure {} of regions {:?} is unknown, keep the region pending",
80 meta.procedure_id, meta.region_ids
81 );
82 continue;
83 };
84 match watcher::wait(&mut receiver).await {
85 Ok(_) => migrated.extend(meta.region_ids.iter().copied()),
86 Err(e) => {
87 error!(e; "Migration sub-procedure {} of regions {:?} failed", meta.procedure_id, meta.region_ids);
88 failed.extend(meta.region_ids.iter().copied());
89 }
90 }
91 }
92
93 let failed_count = failed.len();
94 let mut failed_regions = Vec::new();
95 for region_id in failed {
96 if utils::leads_region(&ctx.table_metadata_manager, node_id, region_id).await? {
98 failed_regions.push(region_id);
99 } else {
100 migrated.push(region_id);
101 }
102 }
103
104 METRIC_META_REGION_MIGRATION_FAIL.inc_by(failed_count as u64);
105 for region_id in migrated {
106 ctx.volatile_ctx.migrated += 1;
107 info!("Region {} left datanode {}", region_id, node_id);
108 }
109 self.inflight_subprocedures.clear();
110 match strategy {
111 DrainDatanodeStrategy::FailFast if !failed_regions.is_empty() => {
112 error::DrainDatanodeAbortSnafu {
113 node_id,
114 failed_regions,
115 }
116 .fail()
117 }
118 DrainDatanodeStrategy::TryAll => {
119 ctx.volatile_ctx.failed_regions.extend(failed_regions);
120 Ok(())
121 }
122 DrainDatanodeStrategy::FailFast => Ok(()),
123 }
124 }
125
126 async fn collect_failed_regions(
128 ctx: &mut Context,
129 region_ids: &[RegionId],
130 ) -> Result<Vec<RegionId>> {
131 let mut failed = Vec::new();
132 for ®ion_id in region_ids {
133 if ctx.region_migration_manager.tracker().contains(region_id) {
134 continue;
135 }
136 if utils::leads_region(
137 &ctx.table_metadata_manager,
138 ctx.persistent_ctx.node_id,
139 region_id,
140 )
141 .await?
142 {
143 failed.push(region_id);
144 }
145 }
146 Ok(failed)
147 }
148
149 async fn dispatch_migrations(
151 &mut self,
152 ctx: &mut Context,
153 procedure_ctx: &ProcedureContext,
154 ) -> Result<Vec<ProcedureWithId>> {
155 let node_id = ctx.persistent_ctx.node_id;
156 let mut children = Vec::new();
157 let mut inflight = Vec::new();
158 let mut targets = Vec::new();
159 let mut failures = Vec::new();
160 for task in self.batches.clone() {
163 let target_node_id = task.to_peer.id;
164 let region_ids = task.region_ids.clone();
165 let submission_context = match procedure_ctx.event_context.clone() {
166 Some(event_context) => SubmissionContext::from_event_context(event_context),
167 None => SubmissionContext::default(),
168 };
169 match ctx
170 .region_migration_manager
171 .build_batch_procedure(submission_context, task)
172 .await
173 {
174 Ok(batch) => {
175 let result = batch.analysis;
176 if !result.peer_conflict.is_empty() {
177 let err = error::DrainDatanodeAbortSnafu {
178 node_id,
179 failed_regions: result.peer_conflict.clone(),
180 }
181 .build();
182 if ctx.persistent_ctx.strategy == DrainDatanodeStrategy::FailFast {
183 return Err(err);
184 }
185 failures.push((err, result.peer_conflict));
186 }
187 if !result.table_not_found.is_empty() {
188 warn!(
189 "Skipping dropped regions while draining datanode {}: {:?}",
190 node_id, result.table_not_found
191 );
192 }
193 if let Some(child) = batch.procedure {
194 targets.push((target_node_id, result.pending.len()));
195 inflight.push(SubprocedureMeta {
196 region_ids: result.pending,
197 procedure_id: child.id,
198 });
199 children.push(child);
200 }
201 }
202 Err(e) => {
203 if ctx.persistent_ctx.strategy == DrainDatanodeStrategy::FailFast {
204 return Err(e);
205 }
206 let failed = Self::collect_failed_regions(ctx, ®ion_ids).await?;
207 if !failed.is_empty() {
208 failures.push((e, failed));
209 }
210 }
211 }
212 }
213 for (error, regions) in failures {
215 ctx.handle_migration_error(error, ®ions)?;
216 }
217 self.batches.clear();
218 self.inflight_subprocedures = inflight;
219 for (target, count) in targets {
220 METRIC_META_REGION_MIGRATION_DATANODES
221 .with_label_values(&["src", &node_id.to_string()])
222 .inc_by(count as u64);
223 METRIC_META_REGION_MIGRATION_DATANODES
224 .with_label_values(&["desc", &target.to_string()])
225 .inc_by(count as u64);
226 }
227 Ok(children)
228 }
229}
230
231#[async_trait::async_trait]
232#[typetag::serde]
233impl State for Dispatch {
234 async fn next(
235 &mut self,
236 ctx: &mut Context,
237 procedure_ctx: &ProcedureContext,
238 ) -> Result<(Box<dyn State>, Status)> {
239 if !self.inflight_subprocedures.is_empty() {
240 self.wait_for_inflight_subprocedures(ctx, procedure_ctx, ctx.persistent_ctx.strategy)
241 .await?;
242 return Ok((
243 Box::new(DrainDatanodeCursor::default()),
244 Status::executing(false),
245 ));
246 }
247 utils::ensure_drain_deadline(ctx)?;
248 let children = self.dispatch_migrations(ctx, procedure_ctx).await?;
249 if children.is_empty() {
250 return Ok((
251 Box::new(DrainDatanodeCursor::default()),
252 Status::executing(false),
253 ));
254 }
255 Ok((
256 Box::new(Self {
257 batches: vec![],
258 inflight_subprocedures: std::mem::take(&mut self.inflight_subprocedures),
259 }),
260 Status::Suspended {
261 subprocedures: children,
262 recovery: SubprocedureRecovery::Detach,
263 persist: false,
264 },
265 ))
266 }
267
268 fn as_any(&self) -> &dyn Any {
269 self
270 }
271}
272
273#[cfg(test)]
274mod tests {
275 use std::collections::HashMap;
276 use std::sync::Arc;
277 use std::time::Duration;
278
279 use crate::procedure::test_util::{procedure_context_with_receivers, procedure_state_receiver};
280 use common_error::mock::MockError;
281 use common_error::status_code::StatusCode;
282 use common_meta::key::table_route::TableRouteKey;
283 use common_meta::key::{MetadataKey, TableMetadataManager};
284 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
285 use common_meta::peer::Peer;
286 use common_meta::rpc::store::{BatchGetRequest, BatchGetResponse};
287 use common_procedure::{
288 Error as ProcedureError, ProcedureId, ProcedureState, Status, SubprocedureRecovery,
289 };
290 use store_api::storage::RegionId;
291 use tokio::sync::watch;
292
293 use crate::error;
294 use crate::procedure::drain_datanode::cursor::DrainDatanodeCursor;
295 use crate::procedure::drain_datanode::dispatch::Dispatch;
296 use crate::procedure::drain_datanode::test_util::create_kafka_table;
297 use crate::procedure::drain_datanode::test_util::{
298 TestEnv, follower_region, leader_region, new_persistent_context,
299 };
300 use crate::procedure::drain_datanode::{DrainDatanodeStrategy, State, SubprocedureMeta};
301 use crate::procedure::region_migration::manager::{
302 RegionMigrationManager, RegionMigrationTriggerReason,
303 };
304 use crate::procedure::region_migration::test_util::TestingEnv;
305 use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
306
307 #[tokio::test]
308 async fn test_drain_datanode_dispatch_transitions_to_cursor() {
309 let env = TestEnv::new().await;
310 let mut dispatch = Dispatch::new(vec![]);
311 env.create_table(vec![follower_region(RegionId::new(1024, 1), 2, 1)])
313 .await;
314
315 let mut ctx = env.context(new_persistent_context(
316 Some(2),
317 DrainDatanodeStrategy::FailFast,
318 ));
319 let procedure_ctx = TestingEnv::procedure_context();
320
321 let (next, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
322
323 assert!(
324 next.as_any()
325 .downcast_ref::<DrainDatanodeCursor>()
326 .is_some()
327 );
328 assert!(matches!(status, Status::Executing { persist: false, .. }));
329 }
330
331 #[tokio::test]
332 async fn test_drop_after_route_read_fail_fast() {
333 assert_drop_after_route_read_is_skipped(DrainDatanodeStrategy::FailFast).await;
334 }
335
336 #[tokio::test]
337 async fn test_drop_after_route_read_try_all() {
338 assert_drop_after_route_read_is_skipped(DrainDatanodeStrategy::TryAll).await;
339 }
340
341 async fn assert_drop_after_route_read_is_skipped(strategy: DrainDatanodeStrategy) {
342 let env = TestEnv::new().await;
343 let region_id = RegionId::new(1024, 1);
344 env.create_table(vec![leader_region(region_id, 1)]).await;
345 let metadata = env.table_metadata_manager.clone();
346 let route_key = TableRouteKey::new(region_id.table_id()).to_bytes();
349 let snapshot = metadata
350 .kv_backend()
351 .batch_get(BatchGetRequest {
352 keys: vec![route_key.clone()],
353 })
354 .await
355 .unwrap();
356 assert_eq!(1, snapshot.kvs.len());
357 let backend = MockKvBackendBuilder::default()
358 .batch_get_fn(Arc::new(move |request| {
359 if request.keys.contains(&route_key) {
360 Ok(snapshot.clone())
361 } else {
362 Ok(BatchGetResponse { kvs: vec![] })
363 }
364 }))
365 .build()
366 .unwrap();
367 let factory = env
368 .env
369 .context_factory_with_metadata(Arc::new(TableMetadataManager::new(Arc::new(backend))));
370 let manager = Arc::new(RegionMigrationManager::new(
371 env.procedure_manager().clone(),
372 factory,
373 ));
374 let mut ctx = env.context(new_persistent_context(Some(2), strategy));
375 ctx.region_migration_manager = manager.clone();
376
377 let mut dispatch = Dispatch::new(vec![RegionMigrationTaskBatch {
378 region_ids: vec![region_id],
379 from_peer: Peer::empty(1),
380 to_peer: Peer::empty(2),
381 timeout: Duration::from_secs(10),
382 trigger_reason: RegionMigrationTriggerReason::Manual,
383 }]);
384 let route = metadata
385 .table_route_manager()
386 .table_route_storage()
387 .get_with_raw_bytes(region_id.table_id())
388 .await
389 .unwrap()
390 .unwrap();
391 let info = metadata
392 .table_info_manager()
393 .get(region_id.table_id())
394 .await
395 .unwrap()
396 .unwrap();
397 metadata
398 .delete_table_metadata(
399 region_id.table_id(),
400 &info.table_name(),
401 &route,
402 &HashMap::new(),
403 None,
404 )
405 .await
406 .unwrap();
407
408 let (next, status) = dispatch
409 .next(&mut ctx, &TestingEnv::procedure_context())
410 .await
411 .unwrap();
412
413 assert!(next.as_any().is::<DrainDatanodeCursor>());
414 assert!(matches!(status, Status::Executing { persist: false, .. }));
415 assert!(dispatch.inflight_subprocedures.is_empty());
416 assert!(ctx.volatile_ctx.failed_regions.is_empty());
417 assert_eq!(0, ctx.volatile_ctx.migrated);
418 assert!(!manager.tracker().contains(region_id));
419 assert!(
420 env.table_metadata_manager
421 .table_info_manager()
422 .get(region_id.table_id())
423 .await
424 .unwrap()
425 .is_none()
426 );
427 }
428
429 #[tokio::test]
430 async fn test_drain_datanode_waits_for_migrated_batch() {
431 let env = TestEnv::new().await;
432 let mut dispatch = Dispatch::new(vec![]);
433 let region_id = RegionId::new(1024, 1);
434 env.create_table(vec![leader_region(region_id, 1)]).await;
435
436 let mut ctx = env.context(new_persistent_context(
437 Some(2),
438 DrainDatanodeStrategy::FailFast,
439 ));
440
441 dispatch.inflight_subprocedures = vec![SubprocedureMeta {
442 region_ids: vec![region_id],
443 procedure_id: ProcedureId::random(),
444 }];
445 let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
446 dispatch.inflight_subprocedures[0].procedure_id,
447 procedure_state_receiver(ProcedureState::Done { output: None }),
448 )]));
449
450 let (next, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
451
452 assert!(matches!(status, Status::Executing { .. }));
453 assert!(next.as_any().is::<DrainDatanodeCursor>());
454 assert_eq!(1, ctx.volatile_ctx.migrated);
455 assert!(dispatch.inflight_subprocedures.is_empty());
456 }
457
458 #[tokio::test]
459 async fn test_drain_datanode_exceeded_deadline() {
460 let env = TestEnv::new().await;
461 let region_id = RegionId::new(1024, 1);
462 let mut dispatch = Dispatch::new(vec![RegionMigrationTaskBatch {
463 region_ids: vec![region_id],
464 from_peer: Peer::empty(1),
465 to_peer: Peer::empty(2),
466 timeout: Duration::from_secs(10),
467 trigger_reason: RegionMigrationTriggerReason::Manual,
468 }]);
469 env.create_table(vec![leader_region(region_id, 1)]).await;
470
471 let mut persistent_ctx = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
472 persistent_ctx.deadline_ms = 0;
473 let mut ctx = env.context(persistent_ctx);
474
475 let procedure_ctx = TestingEnv::procedure_context();
476
477 let error = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap_err();
478
479 assert!(matches!(error, error::Error::ExceededDeadline { .. }));
480 assert!(!error.is_retryable());
481 assert_eq!(1, dispatch.batches.len());
482 assert_eq!(vec![region_id], dispatch.batches[0].region_ids);
483 assert!(!env.region_migration_manager.tracker().contains(region_id));
484 }
485
486 #[tokio::test]
487 async fn test_drain_datanode_deadline_does_not_interrupt_batch() {
488 let env = TestEnv::new().await;
489 let mut dispatch = Dispatch::new(vec![]);
490 let region_id = RegionId::new(1024, 1);
491 env.create_table(vec![leader_region(region_id, 1)]).await;
492
493 let mut persistent_ctx = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
494 persistent_ctx.deadline_ms = 0;
495 let mut ctx = env.context(persistent_ctx);
496
497 dispatch.inflight_subprocedures = vec![SubprocedureMeta {
498 region_ids: vec![region_id],
499 procedure_id: ProcedureId::random(),
500 }];
501 let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
502 dispatch.inflight_subprocedures[0].procedure_id,
503 procedure_state_receiver(ProcedureState::Done { output: None }),
504 )]));
505
506 let (mut next, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
508 assert!(matches!(status, Status::Executing { .. }));
509 assert_eq!(1, ctx.volatile_ctx.migrated);
510
511 let error = next.next(&mut ctx, &procedure_ctx).await.unwrap_err();
513 assert!(matches!(error, error::Error::ExceededDeadline { .. }));
514 }
515
516 #[tokio::test]
517 async fn test_build_failure_releases_children_and_retains_prepared_batches() {
518 let env = TestEnv::new().await;
519 let first = RegionId::new(1024, 1);
520 let second = RegionId::new(1024, 2);
521 let mut second_route = leader_region(second, 1);
522 second_route.follower_peers.push(Peer::empty(3));
523 env.create_table(vec![leader_region(first, 1), second_route])
524 .await;
525 let mut ctx = env.context(new_persistent_context(
526 Some(2),
527 DrainDatanodeStrategy::FailFast,
528 ));
529
530 let batches = vec![
531 RegionMigrationTaskBatch {
532 region_ids: vec![first],
533 from_peer: Peer::empty(1),
534 to_peer: Peer::empty(2),
535 timeout: Duration::from_secs(10),
536 trigger_reason: RegionMigrationTriggerReason::Manual,
537 },
538 RegionMigrationTaskBatch {
539 region_ids: vec![second],
540 from_peer: Peer::empty(1),
541 to_peer: Peer::empty(2),
542 timeout: Duration::from_secs(10),
543 trigger_reason: RegionMigrationTriggerReason::Manual,
544 },
545 ];
546 let mut dispatch = Dispatch::new(batches);
547 assert_eq!(2, dispatch.batches.len());
548 dispatch.batches[1].to_peer = Peer::empty(3);
550 let procedure_ctx = TestingEnv::procedure_context();
551 let error = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap_err();
552 assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
553 assert_eq!(2, dispatch.batches.len());
554 assert_eq!(
555 vec![first, second],
556 dispatch
557 .batches
558 .iter()
559 .flat_map(|batch| batch.region_ids.iter().copied())
560 .collect::<Vec<_>>()
561 );
562 let tracker = env.region_migration_manager.tracker();
563 assert!(!tracker.contains(first));
564 assert!(!tracker.contains(second));
565
566 dispatch.batches[1].to_peer = Peer::empty(2);
567 let (_, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
568 let Status::Suspended { subprocedures, .. } = status else {
569 panic!("expected both retained batches to be submitted");
570 };
571 assert_eq!(2, subprocedures.len());
572 assert!(dispatch.batches.is_empty());
573 assert!(tracker.contains(first));
574 assert!(tracker.contains(second));
575 drop(subprocedures);
576 assert!(!tracker.contains(first));
577 assert!(!tracker.contains(second));
578 }
579
580 #[tokio::test]
581 async fn test_build_failure_reconciles_each_region() {
582 use crate::procedure::region_migration::manager::RegionMigrationProcedureTask;
583
584 for strategy in [
585 DrainDatanodeStrategy::FailFast,
586 DrainDatanodeStrategy::TryAll,
587 ] {
588 for include_pending in [false, true] {
589 let env = TestEnv::new().await;
590 let pending = RegionId::new(1024, 1);
591 let moved = RegionId::new(1024, 2);
592 let running = RegionId::new(1024, 3);
593 let removed = RegionId::new(2048, 1);
594 env.create_table(vec![
595 leader_region(pending, 1),
596 leader_region(moved, 2),
597 leader_region(running, 1),
598 ])
599 .await;
600 let backend = MockKvBackendBuilder::default()
601 .batch_get_fn(Arc::new(|_| {
602 common_meta::error::ElectionNoLeaderSnafu.fail()
603 }))
604 .build()
605 .unwrap();
606 let factory =
607 env.env
608 .context_factory_with_metadata(Arc::new(TableMetadataManager::new(
609 Arc::new(backend),
610 )));
611 let manager = Arc::new(RegionMigrationManager::new(
612 env.procedure_manager().clone(),
613 factory,
614 ));
615 let guard = manager
616 .tracker()
617 .insert_running_procedure(&RegionMigrationProcedureTask::new(
618 running,
619 Peer::empty(1),
620 Peer::empty(2),
621 Duration::from_secs(10),
622 RegionMigrationTriggerReason::Manual,
623 ))
624 .unwrap();
625 let mut regions = vec![moved, removed, running];
626 if include_pending {
627 regions.push(pending);
628 }
629 let mut dispatch = Dispatch::new(vec![RegionMigrationTaskBatch {
630 region_ids: regions,
631 from_peer: Peer::empty(1),
632 to_peer: Peer::empty(2),
633 timeout: Duration::from_secs(10),
634 trigger_reason: RegionMigrationTriggerReason::Manual,
635 }]);
636 let mut ctx = env.context(new_persistent_context(Some(2), strategy));
637 ctx.region_migration_manager = manager.clone();
638 let result = dispatch
639 .next(&mut ctx, &TestingEnv::procedure_context())
640 .await;
641 if strategy == DrainDatanodeStrategy::FailFast {
642 assert!(matches!(
643 result.unwrap_err(),
644 error::Error::TableMetadataManager { .. }
645 ));
646 assert_eq!(1, dispatch.batches.len());
647 assert!(ctx.volatile_ctx.failed_regions.is_empty());
648 } else {
649 let (next, status) = result.unwrap();
650 assert!(next.as_any().is::<DrainDatanodeCursor>());
651 assert!(matches!(status, Status::Executing { .. }));
652 assert!(dispatch.batches.is_empty());
653 assert_eq!(
654 if include_pending {
655 vec![pending]
656 } else {
657 vec![]
658 },
659 ctx.volatile_ctx.failed_regions
660 );
661 }
662 assert_eq!(0, ctx.volatile_ctx.migrated);
663 assert!(!manager.tracker().contains(pending));
664 assert!(!manager.tracker().contains(moved));
665 assert!(!manager.tracker().contains(removed));
666 assert!(manager.tracker().contains(running));
667 drop(guard);
668 }
669 }
670 }
671
672 #[tokio::test]
673 async fn test_build_reconciliation_retry_preserves_batches_and_failures() {
674 use common_meta::key::table_info::TableInfoKey;
675 use std::sync::atomic::{AtomicUsize, Ordering};
676
677 let env = TestEnv::new().await;
678 let ready = RegionId::new(1024, 1);
679 let failed = RegionId::new(2048, 1);
680 let checked = RegionId::new(4096, 1);
681 let unread = RegionId::new(4096, 2);
682 for regions in [vec![ready], vec![failed], vec![checked, unread]] {
683 env.env
684 .create_physical_table_metadata(
685 common_meta::key::test_utils::new_test_table_info(regions[0].table_id()),
686 regions.into_iter().map(|id| leader_region(id, 1)).collect(),
687 )
688 .await;
689 }
690 let snapshot = env
691 .table_metadata_manager
692 .kv_backend()
693 .batch_get(BatchGetRequest {
694 keys: vec![
695 TableRouteKey::new(1024).to_bytes(),
696 TableInfoKey::new(1024).to_bytes(),
697 ],
698 })
699 .await
700 .unwrap();
701 assert_eq!(2, snapshot.kvs.len());
702 let backend = MockKvBackendBuilder::default()
703 .batch_get_fn(Arc::new(move |request| {
704 if [2048, 4096]
705 .iter()
706 .any(|id| request.keys.contains(&TableRouteKey::new(*id).to_bytes()))
707 {
708 return common_meta::error::ElectionNoLeaderSnafu.fail();
709 }
710 Ok(BatchGetResponse {
711 kvs: snapshot
712 .kvs
713 .iter()
714 .filter(|kv| request.keys.contains(&kv.key))
715 .cloned()
716 .collect(),
717 })
718 }))
719 .build()
720 .unwrap();
721 let factory = env
722 .env
723 .context_factory_with_metadata(Arc::new(TableMetadataManager::new(Arc::new(backend))));
724 let manager = Arc::new(RegionMigrationManager::new(
725 env.procedure_manager().clone(),
726 factory,
727 ));
728 let mut ctx = env.context(new_persistent_context(
729 Some(2),
730 DrainDatanodeStrategy::TryAll,
731 ));
732 ctx.region_migration_manager = manager.clone();
733 let reads = Arc::new(AtomicUsize::new(0));
734 let reads_moved = reads.clone();
735 let backend = env.table_metadata_manager.kv_backend().clone();
736 let route_key = TableRouteKey::new(4096).to_bytes();
737 ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
738 MockKvBackendBuilder::default()
739 .batch_get_fn(Arc::new(|_| Ok(BatchGetResponse { kvs: vec![] })))
740 .range_async_fn(Arc::new(move |request| {
741 let fail = request.key == route_key
742 && reads_moved.fetch_add(1, Ordering::Relaxed) == 1;
743 let backend = backend.clone();
744 Box::pin(async move {
745 if fail {
746 return common_meta::error::ElectionNoLeaderSnafu.fail();
747 }
748 backend.range(request).await
749 })
750 }))
751 .build()
752 .unwrap(),
753 )));
754 let mut dispatch = Dispatch::new(
755 [vec![ready], vec![failed], vec![checked, unread]]
756 .into_iter()
757 .map(|region_ids| RegionMigrationTaskBatch {
758 region_ids,
759 from_peer: Peer::empty(1),
760 to_peer: Peer::empty(2),
761 timeout: Duration::from_secs(10),
762 trigger_reason: RegionMigrationTriggerReason::Manual,
763 })
764 .collect(),
765 );
766 let procedure_ctx = TestingEnv::procedure_context();
767 let error = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap_err();
768 assert!(error.is_retryable());
769 assert_eq!(2, reads.load(Ordering::Relaxed));
770 assert_eq!(3, dispatch.batches.len());
771 assert!(dispatch.inflight_subprocedures.is_empty());
772 assert!(ctx.volatile_ctx.failed_regions.is_empty());
773 assert_eq!(0, ctx.volatile_ctx.migrated);
774 for region in [ready, failed, checked, unread] {
775 assert!(!manager.tracker().contains(region));
776 }
777
778 ctx.table_metadata_manager = env.table_metadata_manager.clone();
779 let (_, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
780 let Status::Suspended {
781 subprocedures,
782 recovery,
783 ..
784 } = status
785 else {
786 panic!("expected the ready batch to be submitted");
787 };
788 assert_eq!(SubprocedureRecovery::Detach, recovery);
789 assert_eq!(1, subprocedures.len());
790 assert!(dispatch.batches.is_empty());
791 assert_eq!(
792 vec![failed, checked, unread],
793 ctx.volatile_ctx.failed_regions
794 );
795 assert!(manager.tracker().contains(ready));
796 drop(subprocedures);
797 assert!(!manager.tracker().contains(ready));
798 }
799
800 #[tokio::test]
801 async fn test_settled_failed_batch_reconciles_each_region() {
802 for strategy in [
803 DrainDatanodeStrategy::FailFast,
804 DrainDatanodeStrategy::TryAll,
805 ] {
806 let env = TestEnv::new().await;
807 let pending = RegionId::new(1024, 1);
808 let moved = RegionId::new(1024, 2);
809 let removed = RegionId::new(2048, 1);
810 env.create_table(vec![leader_region(pending, 1), leader_region(moved, 2)])
811 .await;
812 let mut ctx = env.context(new_persistent_context(Some(2), strategy));
813 let regions = vec![pending, moved, removed];
814
815 let procedure_id = ProcedureId::random();
816 let mut dispatch = Dispatch::new(vec![]);
817 dispatch.inflight_subprocedures = vec![SubprocedureMeta {
818 region_ids: regions,
819 procedure_id,
820 }];
821 let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
822 procedure_id,
823 procedure_state_receiver(ProcedureState::failed(Arc::new(
824 ProcedureError::external(MockError::new(StatusCode::Internal)),
825 ))),
826 )]));
827 let result = dispatch
828 .wait_for_inflight_subprocedures(&mut ctx, &procedure_ctx, strategy)
829 .await;
830 if strategy == DrainDatanodeStrategy::FailFast {
831 let error::Error::DrainDatanodeAbort { failed_regions, .. } = result.unwrap_err()
832 else {
833 panic!("unexpected error");
834 };
835 assert_eq!(vec![pending], failed_regions);
836 } else {
837 result.unwrap();
838 assert_eq!(vec![pending], ctx.volatile_ctx.failed_regions);
839 }
840 assert_eq!(2, ctx.volatile_ctx.migrated);
841 assert!(dispatch.inflight_subprocedures.is_empty());
842 }
843 }
844
845 #[tokio::test]
846 async fn test_collect_successful_batch() {
847 let env = TestEnv::new().await;
848 let region_id = RegionId::new(1024, 1);
849 env.create_table(vec![leader_region(region_id, 1)]).await;
850
851 let mut ctx = env.context(new_persistent_context(
852 Some(2),
853 DrainDatanodeStrategy::FailFast,
854 ));
855
856 let mut dispatch = Dispatch::new(vec![]);
857 dispatch.inflight_subprocedures = vec![SubprocedureMeta {
858 region_ids: vec![region_id],
859 procedure_id: ProcedureId::random(),
860 }];
861 let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
862 dispatch.inflight_subprocedures[0].procedure_id,
863 procedure_state_receiver(ProcedureState::Done { output: None }),
864 )]));
865
866 dispatch
867 .wait_for_inflight_subprocedures(
868 &mut ctx,
869 &procedure_ctx,
870 DrainDatanodeStrategy::FailFast,
871 )
872 .await
873 .unwrap();
874
875 assert_eq!(1, ctx.volatile_ctx.migrated);
876 assert!(dispatch.inflight_subprocedures.is_empty());
877 }
878
879 #[tokio::test]
880 async fn test_drain_datanode_wait_failed_child_fail_fast() {
881 let env = TestEnv::new().await;
882 let region_id = RegionId::new(1024, 1);
883 env.create_table(vec![leader_region(region_id, 1)]).await;
884
885 let mut ctx = env.context(new_persistent_context(
886 Some(2),
887 DrainDatanodeStrategy::FailFast,
888 ));
889
890 let mut dispatch = Dispatch::new(vec![]);
891 dispatch.inflight_subprocedures = vec![SubprocedureMeta {
892 region_ids: vec![region_id],
893 procedure_id: ProcedureId::random(),
894 }];
895 let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
896 dispatch.inflight_subprocedures[0].procedure_id,
897 procedure_state_receiver(ProcedureState::failed(Arc::new(ProcedureError::external(
898 MockError::new(StatusCode::Internal),
899 )))),
900 )]));
901
902 let error = dispatch
903 .wait_for_inflight_subprocedures(
904 &mut ctx,
905 &procedure_ctx,
906 DrainDatanodeStrategy::FailFast,
907 )
908 .await
909 .unwrap_err();
910
911 assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
913 assert_eq!(0, ctx.volatile_ctx.migrated);
914 let error::Error::DrainDatanodeAbort { failed_regions, .. } = error else {
915 panic!("expected the failed region in the drain error");
916 };
917 assert_eq!(vec![region_id], failed_regions);
918 assert!(dispatch.inflight_subprocedures.is_empty());
919 }
920
921 #[tokio::test]
922 async fn test_drain_datanode_wait_failed_child_try_all() {
923 let env = TestEnv::new().await;
924 let region_id = RegionId::new(1024, 1);
925 env.create_table(vec![leader_region(region_id, 1)]).await;
926
927 let mut ctx = env.context(new_persistent_context(
928 Some(2),
929 DrainDatanodeStrategy::FailFast,
930 ));
931
932 let mut dispatch = Dispatch::new(vec![]);
933 dispatch.inflight_subprocedures = vec![SubprocedureMeta {
934 region_ids: vec![region_id],
935 procedure_id: ProcedureId::random(),
936 }];
937 let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
938 dispatch.inflight_subprocedures[0].procedure_id,
939 procedure_state_receiver(ProcedureState::failed(Arc::new(ProcedureError::external(
940 MockError::new(StatusCode::Internal),
941 )))),
942 )]));
943
944 dispatch
945 .wait_for_inflight_subprocedures(
946 &mut ctx,
947 &procedure_ctx,
948 DrainDatanodeStrategy::TryAll,
949 )
950 .await
951 .unwrap();
952
953 assert_eq!(vec![region_id], ctx.volatile_ctx.failed_regions);
954 assert_eq!(0, ctx.volatile_ctx.migrated);
955 assert!(dispatch.inflight_subprocedures.is_empty());
956 }
957
958 #[tokio::test]
959 async fn test_drain_datanode_wait_failed_child_counts_migrated_when_moved() {
960 let env = TestEnv::new().await;
961 let region_id = RegionId::new(1024, 1);
962 env.create_table(vec![leader_region(region_id, 2)]).await;
965
966 let mut ctx = env.context(new_persistent_context(
967 Some(2),
968 DrainDatanodeStrategy::FailFast,
969 ));
970
971 let mut dispatch = Dispatch::new(vec![]);
972 dispatch.inflight_subprocedures = vec![SubprocedureMeta {
973 region_ids: vec![region_id],
974 procedure_id: ProcedureId::random(),
975 }];
976 let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
977 dispatch.inflight_subprocedures[0].procedure_id,
978 procedure_state_receiver(ProcedureState::failed(Arc::new(ProcedureError::external(
979 MockError::new(StatusCode::Internal),
980 )))),
981 )]));
982
983 dispatch
984 .wait_for_inflight_subprocedures(
985 &mut ctx,
986 &procedure_ctx,
987 DrainDatanodeStrategy::FailFast,
988 )
989 .await
990 .unwrap();
991
992 assert_eq!(1, ctx.volatile_ctx.migrated);
993 assert!(ctx.volatile_ctx.failed_regions.is_empty());
994 assert!(dispatch.inflight_subprocedures.is_empty());
995 }
996
997 #[tokio::test]
998 async fn test_drain_datanode_wait_unknown_child_keeps_pending() {
999 let env = TestEnv::new().await;
1000 let region_id = RegionId::new(1024, 1);
1001 env.create_table(vec![leader_region(region_id, 1)]).await;
1002
1003 let mut ctx = env.context(new_persistent_context(
1004 Some(2),
1005 DrainDatanodeStrategy::FailFast,
1006 ));
1007
1008 let mut dispatch = Dispatch::new(vec![]);
1009 dispatch.inflight_subprocedures = vec![SubprocedureMeta {
1010 region_ids: vec![region_id],
1011 procedure_id: ProcedureId::random(),
1012 }];
1013 let procedure_ctx = procedure_context_with_receivers(HashMap::new());
1014
1015 dispatch
1016 .wait_for_inflight_subprocedures(
1017 &mut ctx,
1018 &procedure_ctx,
1019 DrainDatanodeStrategy::FailFast,
1020 )
1021 .await
1022 .unwrap();
1023
1024 let (next, _) = DrainDatanodeCursor::default()
1027 .next(&mut ctx, &procedure_ctx)
1028 .await
1029 .unwrap();
1030 let batches = &next.as_any().downcast_ref::<Dispatch>().unwrap().batches;
1031 assert_eq!(1, batches.len());
1032 assert_eq!(vec![region_id], batches[0].region_ids);
1033 assert_eq!(0, ctx.volatile_ctx.migrated);
1034 assert!(ctx.volatile_ctx.failed_regions.is_empty());
1035 }
1036
1037 #[tokio::test]
1038 async fn test_drain_datanode_waits_for_entire_batch() {
1039 for strategy in [
1040 DrainDatanodeStrategy::FailFast,
1041 DrainDatanodeStrategy::TryAll,
1042 ] {
1043 let env = TestEnv::new().await;
1044 let failed_region = RegionId::new(1024, 1);
1045 let succeeded_region = RegionId::new(1024, 2);
1046 env.create_table(vec![
1047 leader_region(failed_region, 1),
1048 leader_region(succeeded_region, 1),
1049 ])
1050 .await;
1051 let mut ctx = env.context(new_persistent_context(
1052 Some(2),
1053 DrainDatanodeStrategy::FailFast,
1054 ));
1055
1056 let failed_id = ProcedureId::random();
1057 let pending_id = ProcedureId::random();
1058 let mut dispatch = Dispatch::new(vec![]);
1059 dispatch.inflight_subprocedures = vec![
1060 SubprocedureMeta {
1061 region_ids: vec![failed_region],
1062 procedure_id: failed_id,
1063 },
1064 SubprocedureMeta {
1065 region_ids: vec![succeeded_region],
1066 procedure_id: pending_id,
1067 },
1068 ];
1069 let (sender, receiver) = watch::channel(ProcedureState::Running);
1070 let procedure_ctx = procedure_context_with_receivers(HashMap::from([
1071 (
1072 failed_id,
1073 procedure_state_receiver(ProcedureState::failed(Arc::new(
1074 ProcedureError::external(MockError::new(StatusCode::Internal)),
1075 ))),
1076 ),
1077 (pending_id, receiver),
1078 ]));
1079 let mut waiting = Box::pin(dispatch.wait_for_inflight_subprocedures(
1080 &mut ctx,
1081 &procedure_ctx,
1082 strategy,
1083 ));
1084 assert!(futures::poll!(&mut waiting).is_pending());
1085 sender.send(ProcedureState::Done { output: None }).unwrap();
1086 let result = waiting.await;
1087 match strategy {
1088 DrainDatanodeStrategy::FailFast => assert!(
1089 matches!(result, Err(error::Error::DrainDatanodeAbort { failed_regions, .. }) if failed_regions == vec![failed_region])
1090 ),
1091 DrainDatanodeStrategy::TryAll => {
1092 result.unwrap();
1093 assert_eq!(vec![failed_region], ctx.volatile_ctx.failed_regions);
1094 }
1095 }
1096 assert_eq!(1, ctx.volatile_ctx.migrated);
1097 assert!(dispatch.inflight_subprocedures.is_empty());
1098 }
1099 }
1100
1101 #[tokio::test]
1102 async fn test_drain_datanode_wait_preserves_batch_on_metadata_retry() {
1103 use common_meta::key::TableMetadataManager;
1104 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
1105
1106 for strategy in [
1107 DrainDatanodeStrategy::FailFast,
1108 DrainDatanodeStrategy::TryAll,
1109 ] {
1110 let env = TestEnv::new().await;
1111 let failed_region = RegionId::new(1024, 1);
1112 let succeeded_region = RegionId::new(1024, 2);
1113 env.create_table(vec![
1114 leader_region(failed_region, 1),
1115 leader_region(succeeded_region, 1),
1116 ])
1117 .await;
1118 let mut ctx = env.context(new_persistent_context(
1119 Some(2),
1120 DrainDatanodeStrategy::FailFast,
1121 ));
1122
1123 let failed_id = ProcedureId::random();
1124 let succeeded_id = ProcedureId::random();
1125 let mut dispatch = Dispatch::new(vec![]);
1126 dispatch.inflight_subprocedures = vec![
1127 SubprocedureMeta {
1128 region_ids: vec![succeeded_region],
1129 procedure_id: succeeded_id,
1130 },
1131 SubprocedureMeta {
1132 region_ids: vec![failed_region],
1133 procedure_id: failed_id,
1134 },
1135 ];
1136 let procedure_ctx = procedure_context_with_receivers(HashMap::from([
1137 (
1138 succeeded_id,
1139 procedure_state_receiver(ProcedureState::Done { output: None }),
1140 ),
1141 (
1142 failed_id,
1143 procedure_state_receiver(ProcedureState::failed(Arc::new(
1144 ProcedureError::external(MockError::new(StatusCode::Internal)),
1145 ))),
1146 ),
1147 ]));
1148 ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
1149 MockKvBackendBuilder::default()
1150 .range_fn(Arc::new(|_| {
1151 common_meta::error::ElectionNoLeaderSnafu.fail()
1152 }))
1153 .build()
1154 .unwrap(),
1155 )));
1156 let error = dispatch
1157 .wait_for_inflight_subprocedures(&mut ctx, &procedure_ctx, strategy)
1158 .await
1159 .unwrap_err();
1160 assert!(error.is_retryable());
1161 assert_eq!(2, dispatch.inflight_subprocedures.len());
1162 assert_eq!(0, ctx.volatile_ctx.migrated);
1163 assert!(ctx.volatile_ctx.failed_regions.is_empty());
1164 assert_eq!(
1165 vec![
1166 (succeeded_id, vec![succeeded_region]),
1167 (failed_id, vec![failed_region])
1168 ],
1169 dispatch
1170 .inflight_subprocedures
1171 .iter()
1172 .map(|meta| (meta.procedure_id, meta.region_ids.clone()))
1173 .collect::<Vec<_>>()
1174 );
1175
1176 ctx.table_metadata_manager = env.table_metadata_manager().clone();
1177 let result = dispatch
1178 .wait_for_inflight_subprocedures(&mut ctx, &procedure_ctx, strategy)
1179 .await;
1180 match strategy {
1181 DrainDatanodeStrategy::FailFast => assert!(
1182 matches!(result, Err(error::Error::DrainDatanodeAbort { failed_regions, .. }) if failed_regions == vec![failed_region])
1183 ),
1184 DrainDatanodeStrategy::TryAll => {
1185 result.unwrap();
1186 assert_eq!(vec![failed_region], ctx.volatile_ctx.failed_regions);
1187 }
1188 }
1189 assert_eq!(1, ctx.volatile_ctx.migrated);
1190 assert!(dispatch.inflight_subprocedures.is_empty());
1191 }
1192 }
1193
1194 #[tokio::test]
1195 async fn test_dispatch_registers_and_releases_all_batch_regions() {
1196 let env = TestEnv::new().await;
1197 create_kafka_table(&env, 1024, 8, 2).await;
1198 let mut persistent = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
1199 persistent.parallelism = 3;
1200 let mut ctx = env.context(persistent);
1201 let regions = vec![
1202 RegionId::new(1024, 0),
1203 RegionId::new(1024, 2),
1204 RegionId::new(1024, 4),
1205 ];
1206
1207 let mut dispatch = Dispatch::new(vec![RegionMigrationTaskBatch {
1208 region_ids: regions.clone(),
1209 from_peer: Peer::empty(1),
1210 to_peer: Peer::empty(2),
1211 timeout: Duration::from_secs(10),
1212 trigger_reason: RegionMigrationTriggerReason::Manual,
1213 }]);
1214 let (next, status) = dispatch
1215 .next(&mut ctx, &TestingEnv::procedure_context())
1216 .await
1217 .unwrap();
1218 let Status::Suspended {
1219 subprocedures,
1220 recovery,
1221 persist,
1222 } = status
1223 else {
1224 panic!("expected a detached batch");
1225 };
1226 assert!(matches!(recovery, SubprocedureRecovery::Detach));
1227 assert!(!persist);
1228 assert_eq!(1, subprocedures.len());
1229 assert_eq!(
1230 regions,
1231 next.as_any()
1232 .downcast_ref::<Dispatch>()
1233 .unwrap()
1234 .inflight_subprocedures[0]
1235 .region_ids
1236 );
1237 let tracker = env.region_migration_manager.tracker();
1238 for region_id in ®ions {
1239 assert!(tracker.contains(*region_id));
1240 }
1241 drop(subprocedures);
1242 for region_id in regions {
1243 assert!(!tracker.contains(region_id));
1244 }
1245 }
1246}