Skip to main content

meta_srv/procedure/drain_datanode/
cursor.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16
17use common_meta::key::table_route::TableRouteValue;
18use common_meta::peer::Peer;
19use common_procedure::{Context as ProcedureContext, Status};
20use common_telemetry::info;
21use serde::{Deserialize, Serialize};
22use snafu::ResultExt;
23use store_api::storage::RegionId;
24
25use crate::error::{self, Result};
26use crate::procedure::drain_datanode::dispatch::Dispatch;
27use crate::procedure::drain_datanode::drain_datanode_end::DrainDatanodeEnd;
28use crate::procedure::drain_datanode::utils::batch_aggregator::BatchAggregator;
29use crate::procedure::drain_datanode::{Context, MAX_RESCAN_ROUNDS, State, utils};
30use crate::procedure::region_migration::DEFAULT_REGION_MIGRATION_TIMEOUT;
31use crate::procedure::region_migration::manager::{
32    RegionMigrationProcedureTask, RegionMigrationTriggerReason,
33};
34use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
35
36/// Buffers a multiple of the round budget to gather same-topic regions while bounding memory.
37const SCAN_BUFFER_FACTOR: usize = 4;
38
39/// Scans region groups, prepares migrations, and confirms completion with a fresh scan.
40#[derive(Debug, Default, Clone, Serialize, Deserialize)]
41pub(crate) struct DrainDatanodeCursor {
42    /// The group being prepared, retained when preparation needs a retry.
43    #[serde(skip)]
44    region_group: Vec<RegionId>,
45    /// Batches prepared for this round, retained when a later scan or preparation fails.
46    #[serde(skip)]
47    batches: Vec<RegionMigrationTaskBatch>,
48}
49
50impl DrainDatanodeCursor {
51    /// Starts a fresh scan of the authoritative table routes.
52    ///
53    /// Scans are bounded by [`MAX_RESCAN_ROUNDS`]: a datanode whose leader regions keep
54    /// coming back must not be rescanned forever.
55    fn start_scan(&mut self, ctx: &mut Context) -> Result<()> {
56        let node_id = ctx.persistent_ctx.node_id;
57        if ctx.volatile_ctx.rescan_round >= MAX_RESCAN_ROUNDS {
58            return error::DrainDatanodeRescanExceededSnafu {
59                node_id,
60                max_rounds: MAX_RESCAN_ROUNDS,
61            }
62            .fail();
63        }
64
65        ctx.volatile_ctx.rescan_round += 1;
66        ctx.volatile_ctx.scan = Some(utils::scan::RegionScan::new(
67            &ctx.table_metadata_manager,
68            node_id,
69            ctx.persistent_ctx.parallelism,
70            ctx.persistent_ctx.parallelism * SCAN_BUFFER_FACTOR,
71        ));
72        ctx.volatile_ctx.follower_count = 0;
73
74        Ok(())
75    }
76
77    /// Reads only the next bounded window. A read failure invalidates the scan so
78    /// retry starts from authoritative metadata instead of losing buffered entries.
79    async fn read_next_group(
80        &mut self,
81        ctx: &mut Context,
82        max_regions: usize,
83    ) -> Result<Option<Vec<RegionId>>> {
84        let Some(mut scan) = ctx.volatile_ctx.scan.take() else {
85            return Ok(None);
86        };
87        let mut next = scan
88            .next_group(
89                |region_id| {
90                    ctx.volatile_ctx.failed_regions.contains(&region_id)
91                        || ctx.region_migration_manager.tracker().contains(region_id)
92                },
93                max_regions,
94            )
95            .await?;
96        // A rebuilt scan may encounter regions already prepared before a read error.
97        if let Some(group) = &mut next {
98            group.retain(|region| {
99                !self
100                    .batches
101                    .iter()
102                    .any(|batch| batch.region_ids.contains(region))
103            });
104        }
105        ctx.volatile_ctx.follower_count = scan.follower_count();
106        ctx.volatile_ctx.scan = Some(scan);
107        Ok(next)
108    }
109
110    /// Fills this round's region budget, retaining partial work on retryable errors.
111    async fn prepare_batches(&mut self, ctx: &mut Context) -> Result<()> {
112        loop {
113            utils::ensure_drain_deadline(ctx)?;
114            if !self.region_group.is_empty() {
115                let batches = self.prepare_migrations(ctx).await?;
116                self.batches.extend(batches);
117                self.region_group.clear();
118            }
119            let prepared_regions = self
120                .batches
121                .iter()
122                .map(|batch| batch.region_ids.len())
123                .sum::<usize>();
124            if prepared_regions >= ctx.persistent_ctx.parallelism {
125                break;
126            }
127            if ctx.volatile_ctx.scan.is_none() {
128                self.start_scan(ctx)?;
129            }
130            let Some(group) = self
131                .read_next_group(ctx, ctx.persistent_ctx.parallelism - prepared_regions)
132                .await?
133            else {
134                break;
135            };
136            self.region_group = group;
137        }
138        utils::ensure_drain_deadline(ctx)
139    }
140
141    /// Selects destinations within the current group and prepares complete migration batches.
142    async fn prepare_migrations(
143        &mut self,
144        ctx: &mut Context,
145    ) -> Result<Vec<RegionMigrationTaskBatch>> {
146        let regions = self.region_group.clone();
147        let mut tasks = Vec::new();
148        for region_id in regions {
149            if ctx.volatile_ctx.failed_regions.contains(&region_id)
150                || ctx.region_migration_manager.tracker().contains(region_id)
151            {
152                continue;
153            }
154            match self.prepare_migration_task(ctx, region_id).await {
155                Ok(Some(task)) => tasks.push(task),
156                Ok(None) => {}
157                Err(e) => ctx.handle_migration_error(e, &[region_id])?,
158            }
159        }
160        let groups = BatchAggregator::group(tasks.into_iter().map(|task| (task.to_peer.id, task)));
161        let batches = groups
162            .into_iter()
163            .map(|(_, tasks)| {
164                let first = &tasks[0];
165                RegionMigrationTaskBatch {
166                    region_ids: tasks.iter().map(|task| task.region_id).collect(),
167                    from_peer: first.from_peer.clone(),
168                    to_peer: first.to_peer.clone(),
169                    // Downgrade and catchup share this batch budget; downgrade requests run
170                    // concurrently rather than receiving a separate budget per region.
171                    timeout: first.timeout,
172                    trigger_reason: first.trigger_reason,
173                }
174            })
175            .collect();
176        Ok(batches)
177    }
178
179    async fn prepare_migration_task(
180        &self,
181        ctx: &mut Context,
182        region_id: RegionId,
183    ) -> Result<Option<RegionMigrationProcedureTask>> {
184        let node_id = ctx.persistent_ctx.node_id;
185        let Some(table_route) = ctx
186            .table_metadata_manager
187            .table_route_manager()
188            .table_route_storage()
189            .get(region_id.table_id())
190            .await
191            .context(error::TableMetadataManagerSnafu)?
192        else {
193            return Ok(None);
194        };
195        let TableRouteValue::Physical(table_route) = table_route else {
196            return Ok(None);
197        };
198        let Some(region_route) = table_route
199            .region_routes
200            .iter()
201            .find(|region_route| region_route.region.id == region_id)
202        else {
203            return Ok(None);
204        };
205        if region_route.leader_peer.as_ref().map(|peer| peer.id) != Some(node_id) {
206            return Ok(None);
207        }
208
209        let to_peer = utils::select_migration_target(
210            &ctx.selector_ctx,
211            node_id,
212            ctx.persistent_ctx.target_node_id,
213            region_id,
214            &region_route.follower_peers,
215        )
216        .await?;
217
218        Ok(Some(RegionMigrationProcedureTask::new(
219            region_id,
220            Peer::empty(node_id),
221            to_peer,
222            DEFAULT_REGION_MIGRATION_TIMEOUT,
223            RegionMigrationTriggerReason::Manual,
224        )))
225    }
226}
227
228#[async_trait::async_trait]
229#[typetag::serde]
230impl State for DrainDatanodeCursor {
231    async fn next(
232        &mut self,
233        ctx: &mut Context,
234        _procedure_ctx: &ProcedureContext,
235    ) -> Result<(Box<dyn State>, Status)> {
236        loop {
237            self.prepare_batches(ctx).await?;
238            if !self.batches.is_empty() {
239                let batches = std::mem::take(&mut self.batches);
240                return Ok((Box::new(Dispatch::new(batches)), Status::executing(false)));
241            }
242            let produced_group = ctx
243                .volatile_ctx
244                .scan
245                .take()
246                .is_some_and(|scan| scan.has_produced_group());
247            if produced_group {
248                // Confirm completion after all work from this scan has settled.
249                continue;
250            }
251            info!(
252                "No more migrations to dispatch for datanode {}, migrated regions: {}, follower regions: {}",
253                ctx.persistent_ctx.node_id,
254                ctx.volatile_ctx.migrated,
255                ctx.volatile_ctx.follower_count
256            );
257            return Ok((Box::new(DrainDatanodeEnd), Status::executing(false)));
258        }
259    }
260
261    fn as_any(&self) -> &dyn Any {
262        self
263    }
264}
265
266#[cfg(test)]
267mod tests {
268    use std::collections::HashMap;
269    use std::sync::Arc;
270    use std::sync::atomic::{AtomicUsize, Ordering};
271    use std::time::Duration;
272
273    use api::v1::meta::heartbeat_request::NodeWorkloads;
274    use common_meta::cluster::NodeInfo;
275    use common_meta::error::Result as MetaResult;
276    use common_meta::key::table_route::TableRouteValue;
277    use common_meta::key::test_utils::new_test_table_info;
278    use common_meta::peer::{Peer, PeerDiscovery};
279    use common_procedure::Status;
280    use common_time::util::current_time_millis;
281    use common_wal::options::{KafkaWalOptions, WalOptions};
282    use store_api::storage::RegionId;
283
284    use crate::error;
285    use crate::key::NodeSchedulingRole;
286    use crate::procedure::drain_datanode::cursor::DrainDatanodeCursor;
287    use crate::procedure::drain_datanode::dispatch::Dispatch;
288    use crate::procedure::drain_datanode::drain_datanode_end::DrainDatanodeEnd;
289    use crate::procedure::drain_datanode::test_util::{
290        TestEnv, create_kafka_table, leader_region, new_persistent_context,
291    };
292    use crate::procedure::drain_datanode::{DrainDatanodeStrategy, MAX_RESCAN_ROUNDS, State};
293    use crate::procedure::region_migration::DEFAULT_REGION_MIGRATION_TIMEOUT;
294    use crate::procedure::region_migration::manager::{
295        RegionMigrationProcedureTask, RegionMigrationTriggerReason,
296    };
297    use crate::procedure::region_migration::test_util::TestingEnv;
298    use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
299
300    fn dispatch_regions(state: &dyn State) -> Vec<RegionId> {
301        state
302            .as_any()
303            .downcast_ref::<Dispatch>()
304            .unwrap()
305            .batches
306            .iter()
307            .flat_map(|batch| batch.region_ids.iter().copied())
308            .collect()
309    }
310
311    async fn wait_until_deadline(deadline_ms: i64) {
312        tokio::time::timeout(Duration::from_secs(5), async {
313            while current_time_millis() < deadline_ms {
314                tokio::time::sleep(Duration::from_millis(1)).await;
315            }
316        })
317        .await
318        .expect("the deadline should elapse");
319    }
320
321    struct EmptyDiscovery {
322        deadline_ms: i64,
323        calls: AtomicUsize,
324    }
325
326    #[async_trait::async_trait]
327    impl PeerDiscovery for EmptyDiscovery {
328        async fn active_frontends(&self) -> MetaResult<Vec<NodeInfo>> {
329            unreachable!()
330        }
331
332        async fn active_datanodes(
333            &self,
334            _filter: Option<for<'a> fn(&'a NodeWorkloads) -> bool>,
335        ) -> MetaResult<Vec<NodeInfo>> {
336            let call = self.calls.fetch_add(1, Ordering::Relaxed);
337            if call == 1 {
338                wait_until_deadline(self.deadline_ms).await;
339            }
340            Ok(vec![])
341        }
342
343        async fn active_flownodes(
344            &self,
345            _filter: Option<for<'a> fn(&'a NodeWorkloads) -> bool>,
346        ) -> MetaResult<Vec<NodeInfo>> {
347            unreachable!()
348        }
349    }
350
351    #[tokio::test]
352    async fn test_cursor_checks_deadline_after_preparation_failures() {
353        let env = TestEnv::new().await;
354        let regions = (0..4).map(|i| RegionId::new(1024, i)).collect::<Vec<_>>();
355        env.create_table(regions.iter().map(|id| leader_region(*id, 1)).collect())
356            .await;
357        let mut persistent = new_persistent_context(None, DrainDatanodeStrategy::TryAll);
358        persistent.parallelism = 1;
359        persistent.deadline_ms = current_time_millis() + 1000;
360        let discovery = Arc::new(EmptyDiscovery {
361            deadline_ms: persistent.deadline_ms,
362            calls: AtomicUsize::new(0),
363        });
364        let mut ctx = env.context(persistent);
365        ctx.selector_ctx.peer_discovery = discovery.clone();
366        let error = DrainDatanodeCursor::default()
367            .next(&mut ctx, &TestingEnv::procedure_context())
368            .await
369            .unwrap_err();
370        assert!(matches!(error, error::Error::ExceededDeadline { .. }));
371        assert_eq!(2, discovery.calls.load(Ordering::Relaxed));
372        assert_eq!(&regions[..2], ctx.volatile_ctx.failed_regions);
373        assert_eq!(0, ctx.volatile_ctx.migrated);
374    }
375
376    #[tokio::test]
377    async fn test_cursor_checks_deadline_after_empty_scan() {
378        use common_meta::key::TableMetadataManager;
379        use common_meta::kv_backend::test_util::MockKvBackendBuilder;
380
381        let env = TestEnv::new().await;
382        let mut ctx = env.context(new_persistent_context(None, DrainDatanodeStrategy::TryAll));
383        let deadline_ms = current_time_millis() + 1000;
384        ctx.persistent_ctx.deadline_ms = deadline_ms;
385        ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
386            MockKvBackendBuilder::default()
387                .range_async_fn(Arc::new(move |_| {
388                    Box::pin(async move {
389                        wait_until_deadline(deadline_ms).await;
390                        Ok(common_meta::rpc::store::RangeResponse {
391                            kvs: vec![],
392                            more: false,
393                        })
394                    })
395                }))
396                .build()
397                .unwrap(),
398        )));
399        let error = DrainDatanodeCursor::default()
400            .next(&mut ctx, &TestingEnv::procedure_context())
401            .await
402            .unwrap_err();
403        assert!(matches!(error, error::Error::ExceededDeadline { .. }));
404        assert_eq!(1, ctx.volatile_ctx.rescan_round);
405        assert!(ctx.volatile_ctx.failed_regions.is_empty());
406    }
407
408    #[tokio::test]
409    async fn test_drain_datanode_cursor_excludes_running_migrations() {
410        for has_pending_region in [false, true] {
411            let env = TestEnv::new().await;
412            let running = RegionId::new(1024, 1);
413            let pending = RegionId::new(1024, 2);
414            let failed = RegionId::new(1024, 3);
415            let mut routes = vec![leader_region(running, 1), leader_region(failed, 1)];
416            if has_pending_region {
417                routes.push(leader_region(pending, 1));
418            }
419            env.create_table(routes).await;
420            let guard = env
421                .region_migration_manager
422                .tracker()
423                .insert_running_procedure(&RegionMigrationProcedureTask::new(
424                    running,
425                    Peer::empty(1),
426                    Peer::empty(2),
427                    Duration::from_secs(10),
428                    RegionMigrationTriggerReason::Manual,
429                ))
430                .unwrap();
431            let mut ctx = env.context(new_persistent_context(
432                Some(2),
433                DrainDatanodeStrategy::TryAll,
434            ));
435            ctx.volatile_ctx.failed_regions.push(failed);
436            let procedure_ctx = TestingEnv::procedure_context();
437
438            let (next, status) = DrainDatanodeCursor::default()
439                .next(&mut ctx, &procedure_ctx)
440                .await
441                .unwrap();
442            assert!(matches!(status, Status::Executing { persist: false, .. }));
443            assert_eq!(vec![failed], ctx.volatile_ctx.failed_regions);
444            assert_eq!(0, ctx.volatile_ctx.migrated);
445            if has_pending_region {
446                assert!(next.as_any().is::<Dispatch>());
447                assert_eq!(vec![pending], dispatch_regions(next.as_ref()));
448            } else {
449                assert!(next.as_any().is::<DrainDatanodeEnd>());
450            }
451
452            // Once the migration releases its guard, an unchanged route needs a new child.
453            drop(guard);
454
455            ctx.volatile_ctx.scan = None;
456            let (next, _) = DrainDatanodeCursor::default()
457                .next(&mut ctx, &procedure_ctx)
458                .await
459                .unwrap();
460            assert!(next.as_any().is::<Dispatch>());
461            let expected = if has_pending_region {
462                vec![running, pending]
463            } else {
464                vec![running]
465            };
466            assert_eq!(expected, dispatch_regions(next.as_ref()));
467        }
468    }
469
470    #[tokio::test]
471    async fn test_drain_datanode_cursor_transitions_to_end() {
472        let env = TestEnv::new().await;
473
474        let mut ctx = env.context(new_persistent_context(
475            Some(2),
476            DrainDatanodeStrategy::FailFast,
477        ));
478        let procedure_ctx = TestingEnv::procedure_context();
479
480        let (next, status) = DrainDatanodeCursor::default()
481            .next(&mut ctx, &procedure_ctx)
482            .await
483            .unwrap();
484
485        assert!(next.as_any().downcast_ref::<DrainDatanodeEnd>().is_some());
486        assert!(matches!(status, Status::Executing { persist: false, .. }));
487    }
488
489    #[tokio::test]
490    async fn test_drain_datanode_cursor_rescan_requeues() {
491        let env = TestEnv::new().await;
492        let region_id = RegionId::new(1024, 1);
493        env.create_table(vec![leader_region(region_id, 1)]).await;
494
495        // A fresh scan discovers regions still led by the source.
496        let mut ctx = env.context(new_persistent_context(
497            None,
498            DrainDatanodeStrategy::FailFast,
499        ));
500        let procedure_ctx = TestingEnv::procedure_context();
501
502        let (next, status) = DrainDatanodeCursor::default()
503            .next(&mut ctx, &procedure_ctx)
504            .await
505            .unwrap();
506
507        assert!(matches!(status, Status::Executing { .. }));
508        assert!(next.as_any().downcast_ref::<Dispatch>().is_some());
509        assert_eq!(vec![region_id], dispatch_regions(next.as_ref()));
510        assert_eq!(1, ctx.volatile_ctx.rescan_round);
511    }
512
513    #[tokio::test]
514    async fn test_drain_datanode_cursor_exceeded_rounds() {
515        let env = TestEnv::new().await;
516        let region_id = RegionId::new(1024, 1);
517        env.create_table(vec![leader_region(region_id, 1)]).await;
518
519        let mut ctx = env.context(new_persistent_context(
520            None,
521            DrainDatanodeStrategy::FailFast,
522        ));
523        ctx.volatile_ctx.rescan_round = MAX_RESCAN_ROUNDS;
524        let procedure_ctx = TestingEnv::procedure_context();
525
526        let error = DrainDatanodeCursor::default()
527            .next(&mut ctx, &procedure_ctx)
528            .await
529            .unwrap_err();
530
531        assert!(matches!(
532            error,
533            error::Error::DrainDatanodeRescanExceeded { .. }
534        ));
535        assert!(!error.is_retryable());
536    }
537
538    #[tokio::test]
539    async fn test_drain_datanode_cursor_ignores_failed_regions() {
540        let env = TestEnv::new().await;
541        let region_id = RegionId::new(1024, 1);
542        env.create_table(vec![leader_region(region_id, 1)]).await;
543
544        let mut ctx = env.context(new_persistent_context(None, DrainDatanodeStrategy::TryAll));
545        ctx.volatile_ctx.failed_regions = vec![region_id];
546        let procedure_ctx = TestingEnv::procedure_context();
547
548        let (next, status) = DrainDatanodeCursor::default()
549            .next(&mut ctx, &procedure_ctx)
550            .await
551            .unwrap();
552
553        assert!(matches!(status, Status::Executing { .. }));
554        assert!(next.as_any().downcast_ref::<DrainDatanodeEnd>().is_some());
555    }
556
557    #[tokio::test]
558    async fn test_cursor_skips_region_moved_away() {
559        let env = TestEnv::new().await;
560        let region_id = RegionId::new(1024, 1);
561        // The current batch lists the region, but the table route says another datanode leads it.
562        env.create_table(vec![leader_region(region_id, 2)]).await;
563
564        let mut ctx = env.context(new_persistent_context(
565            Some(3),
566            DrainDatanodeStrategy::FailFast,
567        ));
568        let mut cursor = DrainDatanodeCursor {
569            region_group: vec![region_id],
570            ..Default::default()
571        };
572        let procedure_ctx = TestingEnv::procedure_context();
573
574        let (next, status) = cursor.next(&mut ctx, &procedure_ctx).await.unwrap();
575
576        assert!(matches!(status, Status::Executing { .. }));
577        assert!(next.as_any().is::<DrainDatanodeEnd>());
578    }
579
580    #[tokio::test]
581    async fn test_drain_datanode_skips_region_whose_migration_is_running() {
582        for scan_before_migration in [false, true] {
583            let env = TestEnv::new().await;
584            let region_id = RegionId::new(1024, 1);
585            env.create_table(vec![leader_region(region_id, 1)]).await;
586
587            let mut ctx = env.context(new_persistent_context(
588                Some(2),
589                DrainDatanodeStrategy::FailFast,
590            ));
591            let mut cursor = DrainDatanodeCursor::default();
592            if scan_before_migration {
593                cursor.start_scan(&mut ctx).unwrap();
594                cursor.region_group = cursor.read_next_group(&mut ctx, 1).await.unwrap().unwrap();
595                assert_eq!(vec![region_id], cursor.region_group);
596            }
597            let procedure_ctx = TestingEnv::procedure_context();
598
599            // A manual migration of the same region is already running. Building it keeps the
600            // region in the running set of the migration manager until it is dropped.
601            let _child = env
602                .region_migration_manager
603                .build_subprocedure(
604                    common_procedure::ProcedureContext::default(),
605                    RegionMigrationProcedureTask::new(
606                        region_id,
607                        Peer::empty(1),
608                        Peer::empty(2),
609                        Duration::from_secs(10),
610                        RegionMigrationTriggerReason::Manual,
611                    ),
612                )
613                .await
614                .expect("the manual migration should be buildable");
615
616            let (next, status) = cursor.next(&mut ctx, &procedure_ctx).await.unwrap();
617
618            // Both scanning and preparation skip regions owned by another migration.
619            assert!(matches!(status, Status::Executing { .. }));
620            assert!(next.as_any().is::<DrainDatanodeEnd>());
621
622            assert!(ctx.volatile_ctx.failed_regions.is_empty());
623            assert_eq!(0, ctx.volatile_ctx.migrated);
624            assert!(env.region_migration_manager.tracker().contains(region_id));
625        }
626    }
627
628    #[tokio::test]
629    async fn test_drain_datanode_fail_fast_on_unschedulable_target() {
630        let env = TestEnv::new().await;
631        let region_id = RegionId::new(1024, 1);
632        env.create_table(vec![leader_region(region_id, 1)]).await;
633        env.node_scheduling_manager
634            .mark_unschedulable(NodeSchedulingRole::Datanode, 3)
635            .await
636            .unwrap();
637
638        let mut ctx = env.context(new_persistent_context(
639            Some(3),
640            DrainDatanodeStrategy::FailFast,
641        ));
642        let procedure_ctx = TestingEnv::procedure_context();
643
644        let mut cursor = DrainDatanodeCursor::default();
645        let error = cursor.next(&mut ctx, &procedure_ctx).await.unwrap_err();
646
647        assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
648        assert!(!error.is_retryable());
649        assert_eq!(vec![region_id], cursor.region_group);
650    }
651
652    #[tokio::test]
653    async fn test_same_topic_splits_by_eligible_destination() {
654        use common_meta::peer::Peer;
655        let env = TestEnv::new().await;
656        let first = RegionId::new(1024, 0);
657        let second = RegionId::new(1024, 1);
658        let mut first_route = leader_region(first, 1);
659        first_route.follower_peers = vec![Peer::empty(3)];
660        let mut second_route = leader_region(second, 1);
661        second_route.follower_peers = vec![Peer::empty(2)];
662        env.table_metadata_manager
663            .create_table_metadata(
664                new_test_table_info(1024),
665                TableRouteValue::physical(vec![first_route, second_route]),
666                HashMap::from([
667                    (0, WalOptions::Kafka(KafkaWalOptions::new("topic".into()))),
668                    (1, WalOptions::Kafka(KafkaWalOptions::new("topic".into()))),
669                ]),
670            )
671            .await
672            .unwrap();
673        let mut persistent = new_persistent_context(None, DrainDatanodeStrategy::FailFast);
674        persistent.parallelism = 2;
675        let mut ctx = env.context(persistent);
676        let (dispatch, status) = DrainDatanodeCursor::default()
677            .next(&mut ctx, &TestingEnv::procedure_context())
678            .await
679            .unwrap();
680        assert!(matches!(status, Status::Executing { persist: false, .. }));
681        let dispatch = dispatch.as_any().downcast_ref::<Dispatch>().unwrap();
682        let destinations = dispatch
683            .batches
684            .iter()
685            .map(|batch| (batch.to_peer.id, batch.region_ids.clone()))
686            .collect::<Vec<_>>();
687        assert_eq!(vec![(2, vec![first]), (3, vec![second])], destinations);
688        // The cursor prepares inputs; registration happens only when Dispatch runs.
689        assert!(!env.region_migration_manager.tracker().contains(first));
690        assert!(!env.region_migration_manager.tracker().contains(second));
691    }
692
693    #[tokio::test]
694    async fn test_cursor_preserves_prepared_batches_on_scan_retry() {
695        use common_meta::key::TableMetadataManager;
696        use common_meta::kv_backend::test_util::MockKvBackendBuilder;
697
698        let env = TestEnv::new().await;
699        create_kafka_table(&env, 1024, 3, 1).await;
700        let mut persistent = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
701        persistent.parallelism = 3;
702        let mut ctx = env.context(persistent);
703        let first = RegionId::new(1024, 0);
704        let mut cursor = DrainDatanodeCursor {
705            batches: vec![RegionMigrationTaskBatch {
706                region_ids: vec![first],
707                from_peer: Peer::empty(1),
708                to_peer: Peer::empty(2),
709                timeout: Duration::from_secs(10),
710                trigger_reason: RegionMigrationTriggerReason::Manual,
711            }],
712            ..Default::default()
713        };
714        ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
715            MockKvBackendBuilder::default()
716                .range_fn(Arc::new(|_| {
717                    common_meta::error::ElectionNoLeaderSnafu.fail()
718                }))
719                .build()
720                .unwrap(),
721        )));
722        let procedure_ctx = TestingEnv::procedure_context();
723        let error = cursor.next(&mut ctx, &procedure_ctx).await.unwrap_err();
724        assert!(error.is_retryable());
725        assert!(ctx.volatile_ctx.scan.is_none());
726        assert_eq!(vec![first], cursor.batches[0].region_ids);
727        ctx.table_metadata_manager = env.table_metadata_manager.clone();
728        let (dispatch, _) = cursor.next(&mut ctx, &procedure_ctx).await.unwrap();
729        let dispatch = dispatch.as_any().downcast_ref::<Dispatch>().unwrap();
730        let mut regions = dispatch
731            .batches
732            .iter()
733            .flat_map(|batch| batch.region_ids.iter().copied())
734            .collect::<Vec<_>>();
735        regions.sort_unstable();
736        assert_eq!(
737            (0..3).map(|i| RegionId::new(1024, i)).collect::<Vec<_>>(),
738            regions
739        );
740        assert_eq!(2, ctx.volatile_ctx.rescan_round);
741    }
742
743    #[tokio::test]
744    async fn test_cursor_fills_region_budget_for_kafka_and_local_wal() {
745        for kafka in [false, true] {
746            let env = TestEnv::new().await;
747            if kafka {
748                create_kafka_table(&env, 1024, 8, 2).await;
749            } else {
750                env.create_table(
751                    (0..8)
752                        .map(|i| leader_region(RegionId::new(1024, i), 1))
753                        .collect(),
754                )
755                .await;
756            }
757            let mut persistent = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
758            persistent.parallelism = 3;
759            let mut ctx = env.context(persistent);
760            let (dispatch, status) = DrainDatanodeCursor::default()
761                .next(&mut ctx, &TestingEnv::procedure_context())
762                .await
763                .unwrap();
764            assert!(matches!(status, Status::Executing { persist: false, .. }));
765            let batches = &dispatch
766                .as_any()
767                .downcast_ref::<Dispatch>()
768                .unwrap()
769                .batches;
770            assert_eq!(
771                3,
772                batches
773                    .iter()
774                    .map(|batch| batch.region_ids.len())
775                    .sum::<usize>()
776            );
777            assert_eq!(if kafka { 1 } else { 3 }, batches.len());
778            assert!(
779                batches
780                    .iter()
781                    .all(|batch| batch.timeout == DEFAULT_REGION_MIGRATION_TIMEOUT)
782            );
783            for region in batches.iter().flat_map(|batch| &batch.region_ids) {
784                assert!(!env.region_migration_manager.tracker().contains(*region));
785            }
786        }
787    }
788    #[tokio::test]
789    async fn test_cursor_retains_group_and_batches_on_preparation_retry() {
790        use common_meta::key::TableMetadataManager;
791        use common_meta::kv_backend::test_util::MockKvBackendBuilder;
792
793        let env = TestEnv::new().await;
794        create_kafka_table(&env, 1024, 2, 1).await;
795        let first = RegionId::new(1024, 0);
796        let second = RegionId::new(1024, 1);
797        let mut persistent = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
798        persistent.parallelism = 2;
799        let mut ctx = env.context(persistent);
800        let mut cursor = DrainDatanodeCursor {
801            region_group: vec![second],
802            batches: vec![RegionMigrationTaskBatch {
803                region_ids: vec![first],
804                from_peer: Peer::empty(1),
805                to_peer: Peer::empty(2),
806                timeout: Duration::from_secs(10),
807                trigger_reason: RegionMigrationTriggerReason::Manual,
808            }],
809        };
810        ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
811            MockKvBackendBuilder::default()
812                .range_fn(Arc::new(|_| {
813                    common_meta::error::ElectionNoLeaderSnafu.fail()
814                }))
815                .build()
816                .unwrap(),
817        )));
818        let procedure_ctx = TestingEnv::procedure_context();
819        let error = cursor.next(&mut ctx, &procedure_ctx).await.unwrap_err();
820        assert!(error.is_retryable());
821        assert_eq!(vec![second], cursor.region_group);
822        assert_eq!(vec![first], cursor.batches[0].region_ids);
823        ctx.table_metadata_manager = env.table_metadata_manager.clone();
824        let (dispatch, _) = cursor.next(&mut ctx, &procedure_ctx).await.unwrap();
825        let batches = &dispatch
826            .as_any()
827            .downcast_ref::<Dispatch>()
828            .unwrap()
829            .batches;
830        assert_eq!(
831            vec![first, second],
832            batches
833                .iter()
834                .flat_map(|batch| batch.region_ids.iter().copied())
835                .collect::<Vec<_>>()
836        );
837    }
838}