1use std::any::Any;
16
17use common_meta::key::table_route::TableRouteValue;
18use common_meta::peer::Peer;
19use common_procedure::{Context as ProcedureContext, Status};
20use common_telemetry::info;
21use serde::{Deserialize, Serialize};
22use snafu::ResultExt;
23use store_api::storage::RegionId;
24
25use crate::error::{self, Result};
26use crate::procedure::drain_datanode::dispatch::Dispatch;
27use crate::procedure::drain_datanode::drain_datanode_end::DrainDatanodeEnd;
28use crate::procedure::drain_datanode::utils::batch_aggregator::BatchAggregator;
29use crate::procedure::drain_datanode::{Context, MAX_RESCAN_ROUNDS, State, utils};
30use crate::procedure::region_migration::DEFAULT_REGION_MIGRATION_TIMEOUT;
31use crate::procedure::region_migration::manager::{
32 RegionMigrationProcedureTask, RegionMigrationTriggerReason,
33};
34use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
35
36const SCAN_BUFFER_FACTOR: usize = 4;
38
39#[derive(Debug, Default, Clone, Serialize, Deserialize)]
41pub(crate) struct DrainDatanodeCursor {
42 #[serde(skip)]
44 region_group: Vec<RegionId>,
45 #[serde(skip)]
47 batches: Vec<RegionMigrationTaskBatch>,
48}
49
50impl DrainDatanodeCursor {
51 fn start_scan(&mut self, ctx: &mut Context) -> Result<()> {
56 let node_id = ctx.persistent_ctx.node_id;
57 if ctx.volatile_ctx.rescan_round >= MAX_RESCAN_ROUNDS {
58 return error::DrainDatanodeRescanExceededSnafu {
59 node_id,
60 max_rounds: MAX_RESCAN_ROUNDS,
61 }
62 .fail();
63 }
64
65 ctx.volatile_ctx.rescan_round += 1;
66 ctx.volatile_ctx.scan = Some(utils::scan::RegionScan::new(
67 &ctx.table_metadata_manager,
68 node_id,
69 ctx.persistent_ctx.parallelism,
70 ctx.persistent_ctx.parallelism * SCAN_BUFFER_FACTOR,
71 ));
72 ctx.volatile_ctx.follower_count = 0;
73
74 Ok(())
75 }
76
77 async fn read_next_group(
80 &mut self,
81 ctx: &mut Context,
82 max_regions: usize,
83 ) -> Result<Option<Vec<RegionId>>> {
84 let Some(mut scan) = ctx.volatile_ctx.scan.take() else {
85 return Ok(None);
86 };
87 let mut next = scan
88 .next_group(
89 |region_id| {
90 ctx.volatile_ctx.failed_regions.contains(®ion_id)
91 || ctx.region_migration_manager.tracker().contains(region_id)
92 },
93 max_regions,
94 )
95 .await?;
96 if let Some(group) = &mut next {
98 group.retain(|region| {
99 !self
100 .batches
101 .iter()
102 .any(|batch| batch.region_ids.contains(region))
103 });
104 }
105 ctx.volatile_ctx.follower_count = scan.follower_count();
106 ctx.volatile_ctx.scan = Some(scan);
107 Ok(next)
108 }
109
110 async fn prepare_batches(&mut self, ctx: &mut Context) -> Result<()> {
112 loop {
113 utils::ensure_drain_deadline(ctx)?;
114 if !self.region_group.is_empty() {
115 let batches = self.prepare_migrations(ctx).await?;
116 self.batches.extend(batches);
117 self.region_group.clear();
118 }
119 let prepared_regions = self
120 .batches
121 .iter()
122 .map(|batch| batch.region_ids.len())
123 .sum::<usize>();
124 if prepared_regions >= ctx.persistent_ctx.parallelism {
125 break;
126 }
127 if ctx.volatile_ctx.scan.is_none() {
128 self.start_scan(ctx)?;
129 }
130 let Some(group) = self
131 .read_next_group(ctx, ctx.persistent_ctx.parallelism - prepared_regions)
132 .await?
133 else {
134 break;
135 };
136 self.region_group = group;
137 }
138 utils::ensure_drain_deadline(ctx)
139 }
140
141 async fn prepare_migrations(
143 &mut self,
144 ctx: &mut Context,
145 ) -> Result<Vec<RegionMigrationTaskBatch>> {
146 let regions = self.region_group.clone();
147 let mut tasks = Vec::new();
148 for region_id in regions {
149 if ctx.volatile_ctx.failed_regions.contains(®ion_id)
150 || ctx.region_migration_manager.tracker().contains(region_id)
151 {
152 continue;
153 }
154 match self.prepare_migration_task(ctx, region_id).await {
155 Ok(Some(task)) => tasks.push(task),
156 Ok(None) => {}
157 Err(e) => ctx.handle_migration_error(e, &[region_id])?,
158 }
159 }
160 let groups = BatchAggregator::group(tasks.into_iter().map(|task| (task.to_peer.id, task)));
161 let batches = groups
162 .into_iter()
163 .map(|(_, tasks)| {
164 let first = &tasks[0];
165 RegionMigrationTaskBatch {
166 region_ids: tasks.iter().map(|task| task.region_id).collect(),
167 from_peer: first.from_peer.clone(),
168 to_peer: first.to_peer.clone(),
169 timeout: first.timeout,
172 trigger_reason: first.trigger_reason,
173 }
174 })
175 .collect();
176 Ok(batches)
177 }
178
179 async fn prepare_migration_task(
180 &self,
181 ctx: &mut Context,
182 region_id: RegionId,
183 ) -> Result<Option<RegionMigrationProcedureTask>> {
184 let node_id = ctx.persistent_ctx.node_id;
185 let Some(table_route) = ctx
186 .table_metadata_manager
187 .table_route_manager()
188 .table_route_storage()
189 .get(region_id.table_id())
190 .await
191 .context(error::TableMetadataManagerSnafu)?
192 else {
193 return Ok(None);
194 };
195 let TableRouteValue::Physical(table_route) = table_route else {
196 return Ok(None);
197 };
198 let Some(region_route) = table_route
199 .region_routes
200 .iter()
201 .find(|region_route| region_route.region.id == region_id)
202 else {
203 return Ok(None);
204 };
205 if region_route.leader_peer.as_ref().map(|peer| peer.id) != Some(node_id) {
206 return Ok(None);
207 }
208
209 let to_peer = utils::select_migration_target(
210 &ctx.selector_ctx,
211 node_id,
212 ctx.persistent_ctx.target_node_id,
213 region_id,
214 ®ion_route.follower_peers,
215 )
216 .await?;
217
218 Ok(Some(RegionMigrationProcedureTask::new(
219 region_id,
220 Peer::empty(node_id),
221 to_peer,
222 DEFAULT_REGION_MIGRATION_TIMEOUT,
223 RegionMigrationTriggerReason::Manual,
224 )))
225 }
226}
227
228#[async_trait::async_trait]
229#[typetag::serde]
230impl State for DrainDatanodeCursor {
231 async fn next(
232 &mut self,
233 ctx: &mut Context,
234 _procedure_ctx: &ProcedureContext,
235 ) -> Result<(Box<dyn State>, Status)> {
236 loop {
237 self.prepare_batches(ctx).await?;
238 if !self.batches.is_empty() {
239 let batches = std::mem::take(&mut self.batches);
240 return Ok((Box::new(Dispatch::new(batches)), Status::executing(false)));
241 }
242 let produced_group = ctx
243 .volatile_ctx
244 .scan
245 .take()
246 .is_some_and(|scan| scan.has_produced_group());
247 if produced_group {
248 continue;
250 }
251 info!(
252 "No more migrations to dispatch for datanode {}, migrated regions: {}, follower regions: {}",
253 ctx.persistent_ctx.node_id,
254 ctx.volatile_ctx.migrated,
255 ctx.volatile_ctx.follower_count
256 );
257 return Ok((Box::new(DrainDatanodeEnd), Status::executing(false)));
258 }
259 }
260
261 fn as_any(&self) -> &dyn Any {
262 self
263 }
264}
265
266#[cfg(test)]
267mod tests {
268 use std::collections::HashMap;
269 use std::sync::Arc;
270 use std::sync::atomic::{AtomicUsize, Ordering};
271 use std::time::Duration;
272
273 use api::v1::meta::heartbeat_request::NodeWorkloads;
274 use common_meta::cluster::NodeInfo;
275 use common_meta::error::Result as MetaResult;
276 use common_meta::key::table_route::TableRouteValue;
277 use common_meta::key::test_utils::new_test_table_info;
278 use common_meta::peer::{Peer, PeerDiscovery};
279 use common_procedure::Status;
280 use common_time::util::current_time_millis;
281 use common_wal::options::{KafkaWalOptions, WalOptions};
282 use store_api::storage::RegionId;
283
284 use crate::error;
285 use crate::key::NodeSchedulingRole;
286 use crate::procedure::drain_datanode::cursor::DrainDatanodeCursor;
287 use crate::procedure::drain_datanode::dispatch::Dispatch;
288 use crate::procedure::drain_datanode::drain_datanode_end::DrainDatanodeEnd;
289 use crate::procedure::drain_datanode::test_util::{
290 TestEnv, create_kafka_table, leader_region, new_persistent_context,
291 };
292 use crate::procedure::drain_datanode::{DrainDatanodeStrategy, MAX_RESCAN_ROUNDS, State};
293 use crate::procedure::region_migration::DEFAULT_REGION_MIGRATION_TIMEOUT;
294 use crate::procedure::region_migration::manager::{
295 RegionMigrationProcedureTask, RegionMigrationTriggerReason,
296 };
297 use crate::procedure::region_migration::test_util::TestingEnv;
298 use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
299
300 fn dispatch_regions(state: &dyn State) -> Vec<RegionId> {
301 state
302 .as_any()
303 .downcast_ref::<Dispatch>()
304 .unwrap()
305 .batches
306 .iter()
307 .flat_map(|batch| batch.region_ids.iter().copied())
308 .collect()
309 }
310
311 async fn wait_until_deadline(deadline_ms: i64) {
312 tokio::time::timeout(Duration::from_secs(5), async {
313 while current_time_millis() < deadline_ms {
314 tokio::time::sleep(Duration::from_millis(1)).await;
315 }
316 })
317 .await
318 .expect("the deadline should elapse");
319 }
320
321 struct EmptyDiscovery {
322 deadline_ms: i64,
323 calls: AtomicUsize,
324 }
325
326 #[async_trait::async_trait]
327 impl PeerDiscovery for EmptyDiscovery {
328 async fn active_frontends(&self) -> MetaResult<Vec<NodeInfo>> {
329 unreachable!()
330 }
331
332 async fn active_datanodes(
333 &self,
334 _filter: Option<for<'a> fn(&'a NodeWorkloads) -> bool>,
335 ) -> MetaResult<Vec<NodeInfo>> {
336 let call = self.calls.fetch_add(1, Ordering::Relaxed);
337 if call == 1 {
338 wait_until_deadline(self.deadline_ms).await;
339 }
340 Ok(vec![])
341 }
342
343 async fn active_flownodes(
344 &self,
345 _filter: Option<for<'a> fn(&'a NodeWorkloads) -> bool>,
346 ) -> MetaResult<Vec<NodeInfo>> {
347 unreachable!()
348 }
349 }
350
351 #[tokio::test]
352 async fn test_cursor_checks_deadline_after_preparation_failures() {
353 let env = TestEnv::new().await;
354 let regions = (0..4).map(|i| RegionId::new(1024, i)).collect::<Vec<_>>();
355 env.create_table(regions.iter().map(|id| leader_region(*id, 1)).collect())
356 .await;
357 let mut persistent = new_persistent_context(None, DrainDatanodeStrategy::TryAll);
358 persistent.parallelism = 1;
359 persistent.deadline_ms = current_time_millis() + 1000;
360 let discovery = Arc::new(EmptyDiscovery {
361 deadline_ms: persistent.deadline_ms,
362 calls: AtomicUsize::new(0),
363 });
364 let mut ctx = env.context(persistent);
365 ctx.selector_ctx.peer_discovery = discovery.clone();
366 let error = DrainDatanodeCursor::default()
367 .next(&mut ctx, &TestingEnv::procedure_context())
368 .await
369 .unwrap_err();
370 assert!(matches!(error, error::Error::ExceededDeadline { .. }));
371 assert_eq!(2, discovery.calls.load(Ordering::Relaxed));
372 assert_eq!(®ions[..2], ctx.volatile_ctx.failed_regions);
373 assert_eq!(0, ctx.volatile_ctx.migrated);
374 }
375
376 #[tokio::test]
377 async fn test_cursor_checks_deadline_after_empty_scan() {
378 use common_meta::key::TableMetadataManager;
379 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
380
381 let env = TestEnv::new().await;
382 let mut ctx = env.context(new_persistent_context(None, DrainDatanodeStrategy::TryAll));
383 let deadline_ms = current_time_millis() + 1000;
384 ctx.persistent_ctx.deadline_ms = deadline_ms;
385 ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
386 MockKvBackendBuilder::default()
387 .range_async_fn(Arc::new(move |_| {
388 Box::pin(async move {
389 wait_until_deadline(deadline_ms).await;
390 Ok(common_meta::rpc::store::RangeResponse {
391 kvs: vec![],
392 more: false,
393 })
394 })
395 }))
396 .build()
397 .unwrap(),
398 )));
399 let error = DrainDatanodeCursor::default()
400 .next(&mut ctx, &TestingEnv::procedure_context())
401 .await
402 .unwrap_err();
403 assert!(matches!(error, error::Error::ExceededDeadline { .. }));
404 assert_eq!(1, ctx.volatile_ctx.rescan_round);
405 assert!(ctx.volatile_ctx.failed_regions.is_empty());
406 }
407
408 #[tokio::test]
409 async fn test_drain_datanode_cursor_excludes_running_migrations() {
410 for has_pending_region in [false, true] {
411 let env = TestEnv::new().await;
412 let running = RegionId::new(1024, 1);
413 let pending = RegionId::new(1024, 2);
414 let failed = RegionId::new(1024, 3);
415 let mut routes = vec![leader_region(running, 1), leader_region(failed, 1)];
416 if has_pending_region {
417 routes.push(leader_region(pending, 1));
418 }
419 env.create_table(routes).await;
420 let guard = env
421 .region_migration_manager
422 .tracker()
423 .insert_running_procedure(&RegionMigrationProcedureTask::new(
424 running,
425 Peer::empty(1),
426 Peer::empty(2),
427 Duration::from_secs(10),
428 RegionMigrationTriggerReason::Manual,
429 ))
430 .unwrap();
431 let mut ctx = env.context(new_persistent_context(
432 Some(2),
433 DrainDatanodeStrategy::TryAll,
434 ));
435 ctx.volatile_ctx.failed_regions.push(failed);
436 let procedure_ctx = TestingEnv::procedure_context();
437
438 let (next, status) = DrainDatanodeCursor::default()
439 .next(&mut ctx, &procedure_ctx)
440 .await
441 .unwrap();
442 assert!(matches!(status, Status::Executing { persist: false, .. }));
443 assert_eq!(vec![failed], ctx.volatile_ctx.failed_regions);
444 assert_eq!(0, ctx.volatile_ctx.migrated);
445 if has_pending_region {
446 assert!(next.as_any().is::<Dispatch>());
447 assert_eq!(vec![pending], dispatch_regions(next.as_ref()));
448 } else {
449 assert!(next.as_any().is::<DrainDatanodeEnd>());
450 }
451
452 drop(guard);
454
455 ctx.volatile_ctx.scan = None;
456 let (next, _) = DrainDatanodeCursor::default()
457 .next(&mut ctx, &procedure_ctx)
458 .await
459 .unwrap();
460 assert!(next.as_any().is::<Dispatch>());
461 let expected = if has_pending_region {
462 vec![running, pending]
463 } else {
464 vec![running]
465 };
466 assert_eq!(expected, dispatch_regions(next.as_ref()));
467 }
468 }
469
470 #[tokio::test]
471 async fn test_drain_datanode_cursor_transitions_to_end() {
472 let env = TestEnv::new().await;
473
474 let mut ctx = env.context(new_persistent_context(
475 Some(2),
476 DrainDatanodeStrategy::FailFast,
477 ));
478 let procedure_ctx = TestingEnv::procedure_context();
479
480 let (next, status) = DrainDatanodeCursor::default()
481 .next(&mut ctx, &procedure_ctx)
482 .await
483 .unwrap();
484
485 assert!(next.as_any().downcast_ref::<DrainDatanodeEnd>().is_some());
486 assert!(matches!(status, Status::Executing { persist: false, .. }));
487 }
488
489 #[tokio::test]
490 async fn test_drain_datanode_cursor_rescan_requeues() {
491 let env = TestEnv::new().await;
492 let region_id = RegionId::new(1024, 1);
493 env.create_table(vec![leader_region(region_id, 1)]).await;
494
495 let mut ctx = env.context(new_persistent_context(
497 None,
498 DrainDatanodeStrategy::FailFast,
499 ));
500 let procedure_ctx = TestingEnv::procedure_context();
501
502 let (next, status) = DrainDatanodeCursor::default()
503 .next(&mut ctx, &procedure_ctx)
504 .await
505 .unwrap();
506
507 assert!(matches!(status, Status::Executing { .. }));
508 assert!(next.as_any().downcast_ref::<Dispatch>().is_some());
509 assert_eq!(vec![region_id], dispatch_regions(next.as_ref()));
510 assert_eq!(1, ctx.volatile_ctx.rescan_round);
511 }
512
513 #[tokio::test]
514 async fn test_drain_datanode_cursor_exceeded_rounds() {
515 let env = TestEnv::new().await;
516 let region_id = RegionId::new(1024, 1);
517 env.create_table(vec![leader_region(region_id, 1)]).await;
518
519 let mut ctx = env.context(new_persistent_context(
520 None,
521 DrainDatanodeStrategy::FailFast,
522 ));
523 ctx.volatile_ctx.rescan_round = MAX_RESCAN_ROUNDS;
524 let procedure_ctx = TestingEnv::procedure_context();
525
526 let error = DrainDatanodeCursor::default()
527 .next(&mut ctx, &procedure_ctx)
528 .await
529 .unwrap_err();
530
531 assert!(matches!(
532 error,
533 error::Error::DrainDatanodeRescanExceeded { .. }
534 ));
535 assert!(!error.is_retryable());
536 }
537
538 #[tokio::test]
539 async fn test_drain_datanode_cursor_ignores_failed_regions() {
540 let env = TestEnv::new().await;
541 let region_id = RegionId::new(1024, 1);
542 env.create_table(vec![leader_region(region_id, 1)]).await;
543
544 let mut ctx = env.context(new_persistent_context(None, DrainDatanodeStrategy::TryAll));
545 ctx.volatile_ctx.failed_regions = vec![region_id];
546 let procedure_ctx = TestingEnv::procedure_context();
547
548 let (next, status) = DrainDatanodeCursor::default()
549 .next(&mut ctx, &procedure_ctx)
550 .await
551 .unwrap();
552
553 assert!(matches!(status, Status::Executing { .. }));
554 assert!(next.as_any().downcast_ref::<DrainDatanodeEnd>().is_some());
555 }
556
557 #[tokio::test]
558 async fn test_cursor_skips_region_moved_away() {
559 let env = TestEnv::new().await;
560 let region_id = RegionId::new(1024, 1);
561 env.create_table(vec![leader_region(region_id, 2)]).await;
563
564 let mut ctx = env.context(new_persistent_context(
565 Some(3),
566 DrainDatanodeStrategy::FailFast,
567 ));
568 let mut cursor = DrainDatanodeCursor {
569 region_group: vec![region_id],
570 ..Default::default()
571 };
572 let procedure_ctx = TestingEnv::procedure_context();
573
574 let (next, status) = cursor.next(&mut ctx, &procedure_ctx).await.unwrap();
575
576 assert!(matches!(status, Status::Executing { .. }));
577 assert!(next.as_any().is::<DrainDatanodeEnd>());
578 }
579
580 #[tokio::test]
581 async fn test_drain_datanode_skips_region_whose_migration_is_running() {
582 for scan_before_migration in [false, true] {
583 let env = TestEnv::new().await;
584 let region_id = RegionId::new(1024, 1);
585 env.create_table(vec![leader_region(region_id, 1)]).await;
586
587 let mut ctx = env.context(new_persistent_context(
588 Some(2),
589 DrainDatanodeStrategy::FailFast,
590 ));
591 let mut cursor = DrainDatanodeCursor::default();
592 if scan_before_migration {
593 cursor.start_scan(&mut ctx).unwrap();
594 cursor.region_group = cursor.read_next_group(&mut ctx, 1).await.unwrap().unwrap();
595 assert_eq!(vec![region_id], cursor.region_group);
596 }
597 let procedure_ctx = TestingEnv::procedure_context();
598
599 let _child = env
602 .region_migration_manager
603 .build_subprocedure(
604 common_procedure::ProcedureContext::default(),
605 RegionMigrationProcedureTask::new(
606 region_id,
607 Peer::empty(1),
608 Peer::empty(2),
609 Duration::from_secs(10),
610 RegionMigrationTriggerReason::Manual,
611 ),
612 )
613 .await
614 .expect("the manual migration should be buildable");
615
616 let (next, status) = cursor.next(&mut ctx, &procedure_ctx).await.unwrap();
617
618 assert!(matches!(status, Status::Executing { .. }));
620 assert!(next.as_any().is::<DrainDatanodeEnd>());
621
622 assert!(ctx.volatile_ctx.failed_regions.is_empty());
623 assert_eq!(0, ctx.volatile_ctx.migrated);
624 assert!(env.region_migration_manager.tracker().contains(region_id));
625 }
626 }
627
628 #[tokio::test]
629 async fn test_drain_datanode_fail_fast_on_unschedulable_target() {
630 let env = TestEnv::new().await;
631 let region_id = RegionId::new(1024, 1);
632 env.create_table(vec![leader_region(region_id, 1)]).await;
633 env.node_scheduling_manager
634 .mark_unschedulable(NodeSchedulingRole::Datanode, 3)
635 .await
636 .unwrap();
637
638 let mut ctx = env.context(new_persistent_context(
639 Some(3),
640 DrainDatanodeStrategy::FailFast,
641 ));
642 let procedure_ctx = TestingEnv::procedure_context();
643
644 let mut cursor = DrainDatanodeCursor::default();
645 let error = cursor.next(&mut ctx, &procedure_ctx).await.unwrap_err();
646
647 assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
648 assert!(!error.is_retryable());
649 assert_eq!(vec![region_id], cursor.region_group);
650 }
651
652 #[tokio::test]
653 async fn test_same_topic_splits_by_eligible_destination() {
654 use common_meta::peer::Peer;
655 let env = TestEnv::new().await;
656 let first = RegionId::new(1024, 0);
657 let second = RegionId::new(1024, 1);
658 let mut first_route = leader_region(first, 1);
659 first_route.follower_peers = vec![Peer::empty(3)];
660 let mut second_route = leader_region(second, 1);
661 second_route.follower_peers = vec![Peer::empty(2)];
662 env.table_metadata_manager
663 .create_table_metadata(
664 new_test_table_info(1024),
665 TableRouteValue::physical(vec![first_route, second_route]),
666 HashMap::from([
667 (0, WalOptions::Kafka(KafkaWalOptions::new("topic".into()))),
668 (1, WalOptions::Kafka(KafkaWalOptions::new("topic".into()))),
669 ]),
670 )
671 .await
672 .unwrap();
673 let mut persistent = new_persistent_context(None, DrainDatanodeStrategy::FailFast);
674 persistent.parallelism = 2;
675 let mut ctx = env.context(persistent);
676 let (dispatch, status) = DrainDatanodeCursor::default()
677 .next(&mut ctx, &TestingEnv::procedure_context())
678 .await
679 .unwrap();
680 assert!(matches!(status, Status::Executing { persist: false, .. }));
681 let dispatch = dispatch.as_any().downcast_ref::<Dispatch>().unwrap();
682 let destinations = dispatch
683 .batches
684 .iter()
685 .map(|batch| (batch.to_peer.id, batch.region_ids.clone()))
686 .collect::<Vec<_>>();
687 assert_eq!(vec![(2, vec![first]), (3, vec![second])], destinations);
688 assert!(!env.region_migration_manager.tracker().contains(first));
690 assert!(!env.region_migration_manager.tracker().contains(second));
691 }
692
693 #[tokio::test]
694 async fn test_cursor_preserves_prepared_batches_on_scan_retry() {
695 use common_meta::key::TableMetadataManager;
696 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
697
698 let env = TestEnv::new().await;
699 create_kafka_table(&env, 1024, 3, 1).await;
700 let mut persistent = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
701 persistent.parallelism = 3;
702 let mut ctx = env.context(persistent);
703 let first = RegionId::new(1024, 0);
704 let mut cursor = DrainDatanodeCursor {
705 batches: vec![RegionMigrationTaskBatch {
706 region_ids: vec![first],
707 from_peer: Peer::empty(1),
708 to_peer: Peer::empty(2),
709 timeout: Duration::from_secs(10),
710 trigger_reason: RegionMigrationTriggerReason::Manual,
711 }],
712 ..Default::default()
713 };
714 ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
715 MockKvBackendBuilder::default()
716 .range_fn(Arc::new(|_| {
717 common_meta::error::ElectionNoLeaderSnafu.fail()
718 }))
719 .build()
720 .unwrap(),
721 )));
722 let procedure_ctx = TestingEnv::procedure_context();
723 let error = cursor.next(&mut ctx, &procedure_ctx).await.unwrap_err();
724 assert!(error.is_retryable());
725 assert!(ctx.volatile_ctx.scan.is_none());
726 assert_eq!(vec![first], cursor.batches[0].region_ids);
727 ctx.table_metadata_manager = env.table_metadata_manager.clone();
728 let (dispatch, _) = cursor.next(&mut ctx, &procedure_ctx).await.unwrap();
729 let dispatch = dispatch.as_any().downcast_ref::<Dispatch>().unwrap();
730 let mut regions = dispatch
731 .batches
732 .iter()
733 .flat_map(|batch| batch.region_ids.iter().copied())
734 .collect::<Vec<_>>();
735 regions.sort_unstable();
736 assert_eq!(
737 (0..3).map(|i| RegionId::new(1024, i)).collect::<Vec<_>>(),
738 regions
739 );
740 assert_eq!(2, ctx.volatile_ctx.rescan_round);
741 }
742
743 #[tokio::test]
744 async fn test_cursor_fills_region_budget_for_kafka_and_local_wal() {
745 for kafka in [false, true] {
746 let env = TestEnv::new().await;
747 if kafka {
748 create_kafka_table(&env, 1024, 8, 2).await;
749 } else {
750 env.create_table(
751 (0..8)
752 .map(|i| leader_region(RegionId::new(1024, i), 1))
753 .collect(),
754 )
755 .await;
756 }
757 let mut persistent = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
758 persistent.parallelism = 3;
759 let mut ctx = env.context(persistent);
760 let (dispatch, status) = DrainDatanodeCursor::default()
761 .next(&mut ctx, &TestingEnv::procedure_context())
762 .await
763 .unwrap();
764 assert!(matches!(status, Status::Executing { persist: false, .. }));
765 let batches = &dispatch
766 .as_any()
767 .downcast_ref::<Dispatch>()
768 .unwrap()
769 .batches;
770 assert_eq!(
771 3,
772 batches
773 .iter()
774 .map(|batch| batch.region_ids.len())
775 .sum::<usize>()
776 );
777 assert_eq!(if kafka { 1 } else { 3 }, batches.len());
778 assert!(
779 batches
780 .iter()
781 .all(|batch| batch.timeout == DEFAULT_REGION_MIGRATION_TIMEOUT)
782 );
783 for region in batches.iter().flat_map(|batch| &batch.region_ids) {
784 assert!(!env.region_migration_manager.tracker().contains(*region));
785 }
786 }
787 }
788 #[tokio::test]
789 async fn test_cursor_retains_group_and_batches_on_preparation_retry() {
790 use common_meta::key::TableMetadataManager;
791 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
792
793 let env = TestEnv::new().await;
794 create_kafka_table(&env, 1024, 2, 1).await;
795 let first = RegionId::new(1024, 0);
796 let second = RegionId::new(1024, 1);
797 let mut persistent = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
798 persistent.parallelism = 2;
799 let mut ctx = env.context(persistent);
800 let mut cursor = DrainDatanodeCursor {
801 region_group: vec![second],
802 batches: vec![RegionMigrationTaskBatch {
803 region_ids: vec![first],
804 from_peer: Peer::empty(1),
805 to_peer: Peer::empty(2),
806 timeout: Duration::from_secs(10),
807 trigger_reason: RegionMigrationTriggerReason::Manual,
808 }],
809 };
810 ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
811 MockKvBackendBuilder::default()
812 .range_fn(Arc::new(|_| {
813 common_meta::error::ElectionNoLeaderSnafu.fail()
814 }))
815 .build()
816 .unwrap(),
817 )));
818 let procedure_ctx = TestingEnv::procedure_context();
819 let error = cursor.next(&mut ctx, &procedure_ctx).await.unwrap_err();
820 assert!(error.is_retryable());
821 assert_eq!(vec![second], cursor.region_group);
822 assert_eq!(vec![first], cursor.batches[0].region_ids);
823 ctx.table_metadata_manager = env.table_metadata_manager.clone();
824 let (dispatch, _) = cursor.next(&mut ctx, &procedure_ctx).await.unwrap();
825 let batches = &dispatch
826 .as_any()
827 .downcast_ref::<Dispatch>()
828 .unwrap()
829 .batches;
830 assert_eq!(
831 vec![first, second],
832 batches
833 .iter()
834 .flat_map(|batch| batch.region_ids.iter().copied())
835 .collect::<Vec<_>>()
836 );
837 }
838}