Skip to main content

meta_srv/procedure/drain_datanode/
dispatch.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16
17use common_procedure::{
18    Context as ProcedureContext, ProcedureContext as SubmissionContext, ProcedureWithId, Status,
19    SubprocedureRecovery, watcher,
20};
21use common_telemetry::{error, info, warn};
22use serde::{Deserialize, Serialize};
23use snafu::ResultExt;
24use store_api::storage::RegionId;
25
26use crate::error::{self, Result};
27use crate::metrics::{METRIC_META_REGION_MIGRATION_DATANODES, METRIC_META_REGION_MIGRATION_FAIL};
28use crate::procedure::drain_datanode::cursor::DrainDatanodeCursor;
29use crate::procedure::drain_datanode::{
30    Context, DrainDatanodeStrategy, State, SubprocedureMeta, utils,
31};
32use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
33
34/// Submits the cursor's prepared batches as detached children and collects their results.
35/// Returns to the cursor after each round; recovered children retain their own lifecycle.
36#[derive(Debug, Clone, Serialize, Deserialize)]
37pub(crate) struct Dispatch {
38    /// Prepared batches retained until all children have been built successfully.
39    #[serde(skip)]
40    pub(crate) batches: Vec<RegionMigrationTaskBatch>,
41    /// Dispatched children whose results have not been collected yet.
42    #[serde(skip)]
43    inflight_subprocedures: Vec<SubprocedureMeta>,
44}
45
46impl Dispatch {
47    /// Creates a dispatch state for the batches prepared by the cursor.
48    pub(crate) fn new(batches: Vec<RegionMigrationTaskBatch>) -> Self {
49        Self {
50            batches,
51            inflight_subprocedures: vec![],
52        }
53    }
54
55    /// Waits for every migration in the batch before applying the drain strategy.
56    ///
57    /// Keep the batch and its results unchanged until all fallible route checks succeed,
58    /// so a metadata retry cannot lose failures or apply progress twice.
59    async fn wait_for_inflight_subprocedures(
60        &mut self,
61        ctx: &mut Context,
62        procedure_ctx: &ProcedureContext,
63        strategy: DrainDatanodeStrategy,
64    ) -> Result<()> {
65        let node_id = ctx.persistent_ctx.node_id;
66        let mut migrated = Vec::new();
67        let mut failed = Vec::new();
68        for meta in &self.inflight_subprocedures {
69            let receiver = procedure_ctx
70                .provider
71                .procedure_state_receiver(meta.procedure_id)
72                .await
73                .inspect_err(|e| {
74                    error!(e; "Failed to get state receiver for migration sub-procedure {}", meta.procedure_id);
75                })
76                .context(error::QueryProcedureSnafu)?;
77            let Some(mut receiver) = receiver else {
78                warn!(
79                    "Migration sub-procedure {} of regions {:?} is unknown, keep the region pending",
80                    meta.procedure_id, meta.region_ids
81                );
82                continue;
83            };
84            match watcher::wait(&mut receiver).await {
85                Ok(_) => migrated.extend(meta.region_ids.iter().copied()),
86                Err(e) => {
87                    error!(e; "Migration sub-procedure {} of regions {:?} failed", meta.procedure_id, meta.region_ids);
88                    failed.extend(meta.region_ids.iter().copied());
89                }
90            }
91        }
92
93        let failed_count = failed.len();
94        let mut failed_regions = Vec::new();
95        for region_id in failed {
96            // A concurrent failover may have achieved the goal despite the child failure.
97            if utils::leads_region(&ctx.table_metadata_manager, node_id, region_id).await? {
98                failed_regions.push(region_id);
99            } else {
100                migrated.push(region_id);
101            }
102        }
103
104        METRIC_META_REGION_MIGRATION_FAIL.inc_by(failed_count as u64);
105        for region_id in migrated {
106            ctx.volatile_ctx.migrated += 1;
107            info!("Region {} left datanode {}", region_id, node_id);
108        }
109        self.inflight_subprocedures.clear();
110        match strategy {
111            DrainDatanodeStrategy::FailFast if !failed_regions.is_empty() => {
112                error::DrainDatanodeAbortSnafu {
113                    node_id,
114                    failed_regions,
115                }
116                .fail()
117            }
118            DrainDatanodeStrategy::TryAll => {
119                ctx.volatile_ctx.failed_regions.extend(failed_regions);
120                Ok(())
121            }
122            DrainDatanodeStrategy::FailFast => Ok(()),
123        }
124    }
125
126    /// Rechecks ownership before recording a failed batch build.
127    async fn collect_failed_regions(
128        ctx: &mut Context,
129        region_ids: &[RegionId],
130    ) -> Result<Vec<RegionId>> {
131        let mut failed = Vec::new();
132        for &region_id in region_ids {
133            if ctx.region_migration_manager.tracker().contains(region_id) {
134                continue;
135            }
136            if utils::leads_region(
137                &ctx.table_metadata_manager,
138                ctx.persistent_ctx.node_id,
139                region_id,
140            )
141            .await?
142            {
143                failed.push(region_id);
144            }
145        }
146        Ok(failed)
147    }
148
149    /// Builds detached children from batches already selected by the cursor.
150    async fn dispatch_migrations(
151        &mut self,
152        ctx: &mut Context,
153        procedure_ctx: &ProcedureContext,
154    ) -> Result<Vec<ProcedureWithId>> {
155        let node_id = ctx.persistent_ctx.node_id;
156        let mut children = Vec::new();
157        let mut inflight = Vec::new();
158        let mut targets = Vec::new();
159        let mut failures = Vec::new();
160        // Keep the prepared batches until construction succeeds, so retries do not
161        // consume work from the metadata cursor twice.
162        for task in self.batches.clone() {
163            let target_node_id = task.to_peer.id;
164            let region_ids = task.region_ids.clone();
165            let submission_context = match procedure_ctx.event_context.clone() {
166                Some(event_context) => SubmissionContext::from_event_context(event_context),
167                None => SubmissionContext::default(),
168            };
169            match ctx
170                .region_migration_manager
171                .build_batch_procedure(submission_context, task)
172                .await
173            {
174                Ok(batch) => {
175                    let result = batch.analysis;
176                    if !result.peer_conflict.is_empty() {
177                        let err = error::DrainDatanodeAbortSnafu {
178                            node_id,
179                            failed_regions: result.peer_conflict.clone(),
180                        }
181                        .build();
182                        if ctx.persistent_ctx.strategy == DrainDatanodeStrategy::FailFast {
183                            return Err(err);
184                        }
185                        failures.push((err, result.peer_conflict));
186                    }
187                    if !result.table_not_found.is_empty() {
188                        warn!(
189                            "Skipping dropped regions while draining datanode {}: {:?}",
190                            node_id, result.table_not_found
191                        );
192                    }
193                    if let Some(child) = batch.procedure {
194                        targets.push((target_node_id, result.pending.len()));
195                        inflight.push(SubprocedureMeta {
196                            region_ids: result.pending,
197                            procedure_id: child.id,
198                        });
199                        children.push(child);
200                    }
201                }
202                Err(e) => {
203                    if ctx.persistent_ctx.strategy == DrainDatanodeStrategy::FailFast {
204                        return Err(e);
205                    }
206                    let failed = Self::collect_failed_regions(ctx, &region_ids).await?;
207                    if !failed.is_empty() {
208                        failures.push((e, failed));
209                    }
210                }
211            }
212        }
213        // Reconciliation can fail; publish failures only after every batch is checked.
214        for (error, regions) in failures {
215            ctx.handle_migration_error(error, &regions)?;
216        }
217        self.batches.clear();
218        self.inflight_subprocedures = inflight;
219        for (target, count) in targets {
220            METRIC_META_REGION_MIGRATION_DATANODES
221                .with_label_values(&["src", &node_id.to_string()])
222                .inc_by(count as u64);
223            METRIC_META_REGION_MIGRATION_DATANODES
224                .with_label_values(&["desc", &target.to_string()])
225                .inc_by(count as u64);
226        }
227        Ok(children)
228    }
229}
230
231#[async_trait::async_trait]
232#[typetag::serde]
233impl State for Dispatch {
234    async fn next(
235        &mut self,
236        ctx: &mut Context,
237        procedure_ctx: &ProcedureContext,
238    ) -> Result<(Box<dyn State>, Status)> {
239        if !self.inflight_subprocedures.is_empty() {
240            self.wait_for_inflight_subprocedures(ctx, procedure_ctx, ctx.persistent_ctx.strategy)
241                .await?;
242            return Ok((
243                Box::new(DrainDatanodeCursor::default()),
244                Status::executing(false),
245            ));
246        }
247        utils::ensure_drain_deadline(ctx)?;
248        let children = self.dispatch_migrations(ctx, procedure_ctx).await?;
249        if children.is_empty() {
250            return Ok((
251                Box::new(DrainDatanodeCursor::default()),
252                Status::executing(false),
253            ));
254        }
255        Ok((
256            Box::new(Self {
257                batches: vec![],
258                inflight_subprocedures: std::mem::take(&mut self.inflight_subprocedures),
259            }),
260            Status::Suspended {
261                subprocedures: children,
262                recovery: SubprocedureRecovery::Detach,
263                persist: false,
264            },
265        ))
266    }
267
268    fn as_any(&self) -> &dyn Any {
269        self
270    }
271}
272
273#[cfg(test)]
274mod tests {
275    use std::collections::HashMap;
276    use std::sync::Arc;
277    use std::time::Duration;
278
279    use crate::procedure::test_util::{procedure_context_with_receivers, procedure_state_receiver};
280    use common_error::mock::MockError;
281    use common_error::status_code::StatusCode;
282    use common_meta::key::table_route::TableRouteKey;
283    use common_meta::key::{MetadataKey, TableMetadataManager};
284    use common_meta::kv_backend::test_util::MockKvBackendBuilder;
285    use common_meta::peer::Peer;
286    use common_meta::rpc::store::{BatchGetRequest, BatchGetResponse};
287    use common_procedure::{
288        Error as ProcedureError, ProcedureId, ProcedureState, Status, SubprocedureRecovery,
289    };
290    use store_api::storage::RegionId;
291    use tokio::sync::watch;
292
293    use crate::error;
294    use crate::procedure::drain_datanode::cursor::DrainDatanodeCursor;
295    use crate::procedure::drain_datanode::dispatch::Dispatch;
296    use crate::procedure::drain_datanode::test_util::create_kafka_table;
297    use crate::procedure::drain_datanode::test_util::{
298        TestEnv, follower_region, leader_region, new_persistent_context,
299    };
300    use crate::procedure::drain_datanode::{DrainDatanodeStrategy, State, SubprocedureMeta};
301    use crate::procedure::region_migration::manager::{
302        RegionMigrationManager, RegionMigrationTriggerReason,
303    };
304    use crate::procedure::region_migration::test_util::TestingEnv;
305    use crate::procedure::region_migration::utils::RegionMigrationTaskBatch;
306
307    #[tokio::test]
308    async fn test_drain_datanode_dispatch_transitions_to_cursor() {
309        let env = TestEnv::new().await;
310        let mut dispatch = Dispatch::new(vec![]);
311        // Only a follower region: there is no leader region to migrate.
312        env.create_table(vec![follower_region(RegionId::new(1024, 1), 2, 1)])
313            .await;
314
315        let mut ctx = env.context(new_persistent_context(
316            Some(2),
317            DrainDatanodeStrategy::FailFast,
318        ));
319        let procedure_ctx = TestingEnv::procedure_context();
320
321        let (next, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
322
323        assert!(
324            next.as_any()
325                .downcast_ref::<DrainDatanodeCursor>()
326                .is_some()
327        );
328        assert!(matches!(status, Status::Executing { persist: false, .. }));
329    }
330
331    #[tokio::test]
332    async fn test_drop_after_route_read_fail_fast() {
333        assert_drop_after_route_read_is_skipped(DrainDatanodeStrategy::FailFast).await;
334    }
335
336    #[tokio::test]
337    async fn test_drop_after_route_read_try_all() {
338        assert_drop_after_route_read_is_skipped(DrainDatanodeStrategy::TryAll).await;
339    }
340
341    async fn assert_drop_after_route_read_is_skipped(strategy: DrainDatanodeStrategy) {
342        let env = TestEnv::new().await;
343        let region_id = RegionId::new(1024, 1);
344        env.create_table(vec![leader_region(region_id, 1)]).await;
345        let metadata = env.table_metadata_manager.clone();
346        // Return the route snapshot even after DROP removes the table. The following
347        // table-info batch read must see the deletion rather than abort the drain.
348        let route_key = TableRouteKey::new(region_id.table_id()).to_bytes();
349        let snapshot = metadata
350            .kv_backend()
351            .batch_get(BatchGetRequest {
352                keys: vec![route_key.clone()],
353            })
354            .await
355            .unwrap();
356        assert_eq!(1, snapshot.kvs.len());
357        let backend = MockKvBackendBuilder::default()
358            .batch_get_fn(Arc::new(move |request| {
359                if request.keys.contains(&route_key) {
360                    Ok(snapshot.clone())
361                } else {
362                    Ok(BatchGetResponse { kvs: vec![] })
363                }
364            }))
365            .build()
366            .unwrap();
367        let factory = env
368            .env
369            .context_factory_with_metadata(Arc::new(TableMetadataManager::new(Arc::new(backend))));
370        let manager = Arc::new(RegionMigrationManager::new(
371            env.procedure_manager().clone(),
372            factory,
373        ));
374        let mut ctx = env.context(new_persistent_context(Some(2), strategy));
375        ctx.region_migration_manager = manager.clone();
376
377        let mut dispatch = Dispatch::new(vec![RegionMigrationTaskBatch {
378            region_ids: vec![region_id],
379            from_peer: Peer::empty(1),
380            to_peer: Peer::empty(2),
381            timeout: Duration::from_secs(10),
382            trigger_reason: RegionMigrationTriggerReason::Manual,
383        }]);
384        let route = metadata
385            .table_route_manager()
386            .table_route_storage()
387            .get_with_raw_bytes(region_id.table_id())
388            .await
389            .unwrap()
390            .unwrap();
391        let info = metadata
392            .table_info_manager()
393            .get(region_id.table_id())
394            .await
395            .unwrap()
396            .unwrap();
397        metadata
398            .delete_table_metadata(
399                region_id.table_id(),
400                &info.table_name(),
401                &route,
402                &HashMap::new(),
403                None,
404            )
405            .await
406            .unwrap();
407
408        let (next, status) = dispatch
409            .next(&mut ctx, &TestingEnv::procedure_context())
410            .await
411            .unwrap();
412
413        assert!(next.as_any().is::<DrainDatanodeCursor>());
414        assert!(matches!(status, Status::Executing { persist: false, .. }));
415        assert!(dispatch.inflight_subprocedures.is_empty());
416        assert!(ctx.volatile_ctx.failed_regions.is_empty());
417        assert_eq!(0, ctx.volatile_ctx.migrated);
418        assert!(!manager.tracker().contains(region_id));
419        assert!(
420            env.table_metadata_manager
421                .table_info_manager()
422                .get(region_id.table_id())
423                .await
424                .unwrap()
425                .is_none()
426        );
427    }
428
429    #[tokio::test]
430    async fn test_drain_datanode_waits_for_migrated_batch() {
431        let env = TestEnv::new().await;
432        let mut dispatch = Dispatch::new(vec![]);
433        let region_id = RegionId::new(1024, 1);
434        env.create_table(vec![leader_region(region_id, 1)]).await;
435
436        let mut ctx = env.context(new_persistent_context(
437            Some(2),
438            DrainDatanodeStrategy::FailFast,
439        ));
440
441        dispatch.inflight_subprocedures = vec![SubprocedureMeta {
442            region_ids: vec![region_id],
443            procedure_id: ProcedureId::random(),
444        }];
445        let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
446            dispatch.inflight_subprocedures[0].procedure_id,
447            procedure_state_receiver(ProcedureState::Done { output: None }),
448        )]));
449
450        let (next, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
451
452        assert!(matches!(status, Status::Executing { .. }));
453        assert!(next.as_any().is::<DrainDatanodeCursor>());
454        assert_eq!(1, ctx.volatile_ctx.migrated);
455        assert!(dispatch.inflight_subprocedures.is_empty());
456    }
457
458    #[tokio::test]
459    async fn test_drain_datanode_exceeded_deadline() {
460        let env = TestEnv::new().await;
461        let region_id = RegionId::new(1024, 1);
462        let mut dispatch = Dispatch::new(vec![RegionMigrationTaskBatch {
463            region_ids: vec![region_id],
464            from_peer: Peer::empty(1),
465            to_peer: Peer::empty(2),
466            timeout: Duration::from_secs(10),
467            trigger_reason: RegionMigrationTriggerReason::Manual,
468        }]);
469        env.create_table(vec![leader_region(region_id, 1)]).await;
470
471        let mut persistent_ctx = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
472        persistent_ctx.deadline_ms = 0;
473        let mut ctx = env.context(persistent_ctx);
474
475        let procedure_ctx = TestingEnv::procedure_context();
476
477        let error = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap_err();
478
479        assert!(matches!(error, error::Error::ExceededDeadline { .. }));
480        assert!(!error.is_retryable());
481        assert_eq!(1, dispatch.batches.len());
482        assert_eq!(vec![region_id], dispatch.batches[0].region_ids);
483        assert!(!env.region_migration_manager.tracker().contains(region_id));
484    }
485
486    #[tokio::test]
487    async fn test_drain_datanode_deadline_does_not_interrupt_batch() {
488        let env = TestEnv::new().await;
489        let mut dispatch = Dispatch::new(vec![]);
490        let region_id = RegionId::new(1024, 1);
491        env.create_table(vec![leader_region(region_id, 1)]).await;
492
493        let mut persistent_ctx = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
494        persistent_ctx.deadline_ms = 0;
495        let mut ctx = env.context(persistent_ctx);
496
497        dispatch.inflight_subprocedures = vec![SubprocedureMeta {
498            region_ids: vec![region_id],
499            procedure_id: ProcedureId::random(),
500        }];
501        let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
502            dispatch.inflight_subprocedures[0].procedure_id,
503            procedure_state_receiver(ProcedureState::Done { output: None }),
504        )]));
505
506        // The in-flight batch is settled before the deadline is checked.
507        let (mut next, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
508        assert!(matches!(status, Status::Executing { .. }));
509        assert_eq!(1, ctx.volatile_ctx.migrated);
510
511        // The next step observes the exceeded deadline.
512        let error = next.next(&mut ctx, &procedure_ctx).await.unwrap_err();
513        assert!(matches!(error, error::Error::ExceededDeadline { .. }));
514    }
515
516    #[tokio::test]
517    async fn test_build_failure_releases_children_and_retains_prepared_batches() {
518        let env = TestEnv::new().await;
519        let first = RegionId::new(1024, 1);
520        let second = RegionId::new(1024, 2);
521        let mut second_route = leader_region(second, 1);
522        second_route.follower_peers.push(Peer::empty(3));
523        env.create_table(vec![leader_region(first, 1), second_route])
524            .await;
525        let mut ctx = env.context(new_persistent_context(
526            Some(2),
527            DrainDatanodeStrategy::FailFast,
528        ));
529
530        let batches = vec![
531            RegionMigrationTaskBatch {
532                region_ids: vec![first],
533                from_peer: Peer::empty(1),
534                to_peer: Peer::empty(2),
535                timeout: Duration::from_secs(10),
536                trigger_reason: RegionMigrationTriggerReason::Manual,
537            },
538            RegionMigrationTaskBatch {
539                region_ids: vec![second],
540                from_peer: Peer::empty(1),
541                to_peer: Peer::empty(2),
542                timeout: Duration::from_secs(10),
543                trigger_reason: RegionMigrationTriggerReason::Manual,
544            },
545        ];
546        let mut dispatch = Dispatch::new(batches);
547        assert_eq!(2, dispatch.batches.len());
548        // The first child is built before the second batch encounters a target conflict.
549        dispatch.batches[1].to_peer = Peer::empty(3);
550        let procedure_ctx = TestingEnv::procedure_context();
551        let error = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap_err();
552        assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
553        assert_eq!(2, dispatch.batches.len());
554        assert_eq!(
555            vec![first, second],
556            dispatch
557                .batches
558                .iter()
559                .flat_map(|batch| batch.region_ids.iter().copied())
560                .collect::<Vec<_>>()
561        );
562        let tracker = env.region_migration_manager.tracker();
563        assert!(!tracker.contains(first));
564        assert!(!tracker.contains(second));
565
566        dispatch.batches[1].to_peer = Peer::empty(2);
567        let (_, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
568        let Status::Suspended { subprocedures, .. } = status else {
569            panic!("expected both retained batches to be submitted");
570        };
571        assert_eq!(2, subprocedures.len());
572        assert!(dispatch.batches.is_empty());
573        assert!(tracker.contains(first));
574        assert!(tracker.contains(second));
575        drop(subprocedures);
576        assert!(!tracker.contains(first));
577        assert!(!tracker.contains(second));
578    }
579
580    #[tokio::test]
581    async fn test_build_failure_reconciles_each_region() {
582        use crate::procedure::region_migration::manager::RegionMigrationProcedureTask;
583
584        for strategy in [
585            DrainDatanodeStrategy::FailFast,
586            DrainDatanodeStrategy::TryAll,
587        ] {
588            for include_pending in [false, true] {
589                let env = TestEnv::new().await;
590                let pending = RegionId::new(1024, 1);
591                let moved = RegionId::new(1024, 2);
592                let running = RegionId::new(1024, 3);
593                let removed = RegionId::new(2048, 1);
594                env.create_table(vec![
595                    leader_region(pending, 1),
596                    leader_region(moved, 2),
597                    leader_region(running, 1),
598                ])
599                .await;
600                let backend = MockKvBackendBuilder::default()
601                    .batch_get_fn(Arc::new(|_| {
602                        common_meta::error::ElectionNoLeaderSnafu.fail()
603                    }))
604                    .build()
605                    .unwrap();
606                let factory =
607                    env.env
608                        .context_factory_with_metadata(Arc::new(TableMetadataManager::new(
609                            Arc::new(backend),
610                        )));
611                let manager = Arc::new(RegionMigrationManager::new(
612                    env.procedure_manager().clone(),
613                    factory,
614                ));
615                let guard = manager
616                    .tracker()
617                    .insert_running_procedure(&RegionMigrationProcedureTask::new(
618                        running,
619                        Peer::empty(1),
620                        Peer::empty(2),
621                        Duration::from_secs(10),
622                        RegionMigrationTriggerReason::Manual,
623                    ))
624                    .unwrap();
625                let mut regions = vec![moved, removed, running];
626                if include_pending {
627                    regions.push(pending);
628                }
629                let mut dispatch = Dispatch::new(vec![RegionMigrationTaskBatch {
630                    region_ids: regions,
631                    from_peer: Peer::empty(1),
632                    to_peer: Peer::empty(2),
633                    timeout: Duration::from_secs(10),
634                    trigger_reason: RegionMigrationTriggerReason::Manual,
635                }]);
636                let mut ctx = env.context(new_persistent_context(Some(2), strategy));
637                ctx.region_migration_manager = manager.clone();
638                let result = dispatch
639                    .next(&mut ctx, &TestingEnv::procedure_context())
640                    .await;
641                if strategy == DrainDatanodeStrategy::FailFast {
642                    assert!(matches!(
643                        result.unwrap_err(),
644                        error::Error::TableMetadataManager { .. }
645                    ));
646                    assert_eq!(1, dispatch.batches.len());
647                    assert!(ctx.volatile_ctx.failed_regions.is_empty());
648                } else {
649                    let (next, status) = result.unwrap();
650                    assert!(next.as_any().is::<DrainDatanodeCursor>());
651                    assert!(matches!(status, Status::Executing { .. }));
652                    assert!(dispatch.batches.is_empty());
653                    assert_eq!(
654                        if include_pending {
655                            vec![pending]
656                        } else {
657                            vec![]
658                        },
659                        ctx.volatile_ctx.failed_regions
660                    );
661                }
662                assert_eq!(0, ctx.volatile_ctx.migrated);
663                assert!(!manager.tracker().contains(pending));
664                assert!(!manager.tracker().contains(moved));
665                assert!(!manager.tracker().contains(removed));
666                assert!(manager.tracker().contains(running));
667                drop(guard);
668            }
669        }
670    }
671
672    #[tokio::test]
673    async fn test_build_reconciliation_retry_preserves_batches_and_failures() {
674        use common_meta::key::table_info::TableInfoKey;
675        use std::sync::atomic::{AtomicUsize, Ordering};
676
677        let env = TestEnv::new().await;
678        let ready = RegionId::new(1024, 1);
679        let failed = RegionId::new(2048, 1);
680        let checked = RegionId::new(4096, 1);
681        let unread = RegionId::new(4096, 2);
682        for regions in [vec![ready], vec![failed], vec![checked, unread]] {
683            env.env
684                .create_physical_table_metadata(
685                    common_meta::key::test_utils::new_test_table_info(regions[0].table_id()),
686                    regions.into_iter().map(|id| leader_region(id, 1)).collect(),
687                )
688                .await;
689        }
690        let snapshot = env
691            .table_metadata_manager
692            .kv_backend()
693            .batch_get(BatchGetRequest {
694                keys: vec![
695                    TableRouteKey::new(1024).to_bytes(),
696                    TableInfoKey::new(1024).to_bytes(),
697                ],
698            })
699            .await
700            .unwrap();
701        assert_eq!(2, snapshot.kvs.len());
702        let backend = MockKvBackendBuilder::default()
703            .batch_get_fn(Arc::new(move |request| {
704                if [2048, 4096]
705                    .iter()
706                    .any(|id| request.keys.contains(&TableRouteKey::new(*id).to_bytes()))
707                {
708                    return common_meta::error::ElectionNoLeaderSnafu.fail();
709                }
710                Ok(BatchGetResponse {
711                    kvs: snapshot
712                        .kvs
713                        .iter()
714                        .filter(|kv| request.keys.contains(&kv.key))
715                        .cloned()
716                        .collect(),
717                })
718            }))
719            .build()
720            .unwrap();
721        let factory = env
722            .env
723            .context_factory_with_metadata(Arc::new(TableMetadataManager::new(Arc::new(backend))));
724        let manager = Arc::new(RegionMigrationManager::new(
725            env.procedure_manager().clone(),
726            factory,
727        ));
728        let mut ctx = env.context(new_persistent_context(
729            Some(2),
730            DrainDatanodeStrategy::TryAll,
731        ));
732        ctx.region_migration_manager = manager.clone();
733        let reads = Arc::new(AtomicUsize::new(0));
734        let reads_moved = reads.clone();
735        let backend = env.table_metadata_manager.kv_backend().clone();
736        let route_key = TableRouteKey::new(4096).to_bytes();
737        ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
738            MockKvBackendBuilder::default()
739                .batch_get_fn(Arc::new(|_| Ok(BatchGetResponse { kvs: vec![] })))
740                .range_async_fn(Arc::new(move |request| {
741                    let fail = request.key == route_key
742                        && reads_moved.fetch_add(1, Ordering::Relaxed) == 1;
743                    let backend = backend.clone();
744                    Box::pin(async move {
745                        if fail {
746                            return common_meta::error::ElectionNoLeaderSnafu.fail();
747                        }
748                        backend.range(request).await
749                    })
750                }))
751                .build()
752                .unwrap(),
753        )));
754        let mut dispatch = Dispatch::new(
755            [vec![ready], vec![failed], vec![checked, unread]]
756                .into_iter()
757                .map(|region_ids| RegionMigrationTaskBatch {
758                    region_ids,
759                    from_peer: Peer::empty(1),
760                    to_peer: Peer::empty(2),
761                    timeout: Duration::from_secs(10),
762                    trigger_reason: RegionMigrationTriggerReason::Manual,
763                })
764                .collect(),
765        );
766        let procedure_ctx = TestingEnv::procedure_context();
767        let error = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap_err();
768        assert!(error.is_retryable());
769        assert_eq!(2, reads.load(Ordering::Relaxed));
770        assert_eq!(3, dispatch.batches.len());
771        assert!(dispatch.inflight_subprocedures.is_empty());
772        assert!(ctx.volatile_ctx.failed_regions.is_empty());
773        assert_eq!(0, ctx.volatile_ctx.migrated);
774        for region in [ready, failed, checked, unread] {
775            assert!(!manager.tracker().contains(region));
776        }
777
778        ctx.table_metadata_manager = env.table_metadata_manager.clone();
779        let (_, status) = dispatch.next(&mut ctx, &procedure_ctx).await.unwrap();
780        let Status::Suspended {
781            subprocedures,
782            recovery,
783            ..
784        } = status
785        else {
786            panic!("expected the ready batch to be submitted");
787        };
788        assert_eq!(SubprocedureRecovery::Detach, recovery);
789        assert_eq!(1, subprocedures.len());
790        assert!(dispatch.batches.is_empty());
791        assert_eq!(
792            vec![failed, checked, unread],
793            ctx.volatile_ctx.failed_regions
794        );
795        assert!(manager.tracker().contains(ready));
796        drop(subprocedures);
797        assert!(!manager.tracker().contains(ready));
798    }
799
800    #[tokio::test]
801    async fn test_settled_failed_batch_reconciles_each_region() {
802        for strategy in [
803            DrainDatanodeStrategy::FailFast,
804            DrainDatanodeStrategy::TryAll,
805        ] {
806            let env = TestEnv::new().await;
807            let pending = RegionId::new(1024, 1);
808            let moved = RegionId::new(1024, 2);
809            let removed = RegionId::new(2048, 1);
810            env.create_table(vec![leader_region(pending, 1), leader_region(moved, 2)])
811                .await;
812            let mut ctx = env.context(new_persistent_context(Some(2), strategy));
813            let regions = vec![pending, moved, removed];
814
815            let procedure_id = ProcedureId::random();
816            let mut dispatch = Dispatch::new(vec![]);
817            dispatch.inflight_subprocedures = vec![SubprocedureMeta {
818                region_ids: regions,
819                procedure_id,
820            }];
821            let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
822                procedure_id,
823                procedure_state_receiver(ProcedureState::failed(Arc::new(
824                    ProcedureError::external(MockError::new(StatusCode::Internal)),
825                ))),
826            )]));
827            let result = dispatch
828                .wait_for_inflight_subprocedures(&mut ctx, &procedure_ctx, strategy)
829                .await;
830            if strategy == DrainDatanodeStrategy::FailFast {
831                let error::Error::DrainDatanodeAbort { failed_regions, .. } = result.unwrap_err()
832                else {
833                    panic!("unexpected error");
834                };
835                assert_eq!(vec![pending], failed_regions);
836            } else {
837                result.unwrap();
838                assert_eq!(vec![pending], ctx.volatile_ctx.failed_regions);
839            }
840            assert_eq!(2, ctx.volatile_ctx.migrated);
841            assert!(dispatch.inflight_subprocedures.is_empty());
842        }
843    }
844
845    #[tokio::test]
846    async fn test_collect_successful_batch() {
847        let env = TestEnv::new().await;
848        let region_id = RegionId::new(1024, 1);
849        env.create_table(vec![leader_region(region_id, 1)]).await;
850
851        let mut ctx = env.context(new_persistent_context(
852            Some(2),
853            DrainDatanodeStrategy::FailFast,
854        ));
855
856        let mut dispatch = Dispatch::new(vec![]);
857        dispatch.inflight_subprocedures = vec![SubprocedureMeta {
858            region_ids: vec![region_id],
859            procedure_id: ProcedureId::random(),
860        }];
861        let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
862            dispatch.inflight_subprocedures[0].procedure_id,
863            procedure_state_receiver(ProcedureState::Done { output: None }),
864        )]));
865
866        dispatch
867            .wait_for_inflight_subprocedures(
868                &mut ctx,
869                &procedure_ctx,
870                DrainDatanodeStrategy::FailFast,
871            )
872            .await
873            .unwrap();
874
875        assert_eq!(1, ctx.volatile_ctx.migrated);
876        assert!(dispatch.inflight_subprocedures.is_empty());
877    }
878
879    #[tokio::test]
880    async fn test_drain_datanode_wait_failed_child_fail_fast() {
881        let env = TestEnv::new().await;
882        let region_id = RegionId::new(1024, 1);
883        env.create_table(vec![leader_region(region_id, 1)]).await;
884
885        let mut ctx = env.context(new_persistent_context(
886            Some(2),
887            DrainDatanodeStrategy::FailFast,
888        ));
889
890        let mut dispatch = Dispatch::new(vec![]);
891        dispatch.inflight_subprocedures = vec![SubprocedureMeta {
892            region_ids: vec![region_id],
893            procedure_id: ProcedureId::random(),
894        }];
895        let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
896            dispatch.inflight_subprocedures[0].procedure_id,
897            procedure_state_receiver(ProcedureState::failed(Arc::new(ProcedureError::external(
898                MockError::new(StatusCode::Internal),
899            )))),
900        )]));
901
902        let error = dispatch
903            .wait_for_inflight_subprocedures(
904                &mut ctx,
905                &procedure_ctx,
906                DrainDatanodeStrategy::FailFast,
907            )
908            .await
909            .unwrap_err();
910
911        // The whole batch is settled before the procedure aborts.
912        assert!(matches!(error, error::Error::DrainDatanodeAbort { .. }));
913        assert_eq!(0, ctx.volatile_ctx.migrated);
914        let error::Error::DrainDatanodeAbort { failed_regions, .. } = error else {
915            panic!("expected the failed region in the drain error");
916        };
917        assert_eq!(vec![region_id], failed_regions);
918        assert!(dispatch.inflight_subprocedures.is_empty());
919    }
920
921    #[tokio::test]
922    async fn test_drain_datanode_wait_failed_child_try_all() {
923        let env = TestEnv::new().await;
924        let region_id = RegionId::new(1024, 1);
925        env.create_table(vec![leader_region(region_id, 1)]).await;
926
927        let mut ctx = env.context(new_persistent_context(
928            Some(2),
929            DrainDatanodeStrategy::FailFast,
930        ));
931
932        let mut dispatch = Dispatch::new(vec![]);
933        dispatch.inflight_subprocedures = vec![SubprocedureMeta {
934            region_ids: vec![region_id],
935            procedure_id: ProcedureId::random(),
936        }];
937        let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
938            dispatch.inflight_subprocedures[0].procedure_id,
939            procedure_state_receiver(ProcedureState::failed(Arc::new(ProcedureError::external(
940                MockError::new(StatusCode::Internal),
941            )))),
942        )]));
943
944        dispatch
945            .wait_for_inflight_subprocedures(
946                &mut ctx,
947                &procedure_ctx,
948                DrainDatanodeStrategy::TryAll,
949            )
950            .await
951            .unwrap();
952
953        assert_eq!(vec![region_id], ctx.volatile_ctx.failed_regions);
954        assert_eq!(0, ctx.volatile_ctx.migrated);
955        assert!(dispatch.inflight_subprocedures.is_empty());
956    }
957
958    #[tokio::test]
959    async fn test_drain_datanode_wait_failed_child_counts_migrated_when_moved() {
960        let env = TestEnv::new().await;
961        let region_id = RegionId::new(1024, 1);
962        // The route reports another datanode as the leader, which is what a failover that
963        // raced with the migration leaves behind.
964        env.create_table(vec![leader_region(region_id, 2)]).await;
965
966        let mut ctx = env.context(new_persistent_context(
967            Some(2),
968            DrainDatanodeStrategy::FailFast,
969        ));
970
971        let mut dispatch = Dispatch::new(vec![]);
972        dispatch.inflight_subprocedures = vec![SubprocedureMeta {
973            region_ids: vec![region_id],
974            procedure_id: ProcedureId::random(),
975        }];
976        let procedure_ctx = procedure_context_with_receivers(HashMap::from([(
977            dispatch.inflight_subprocedures[0].procedure_id,
978            procedure_state_receiver(ProcedureState::failed(Arc::new(ProcedureError::external(
979                MockError::new(StatusCode::Internal),
980            )))),
981        )]));
982
983        dispatch
984            .wait_for_inflight_subprocedures(
985                &mut ctx,
986                &procedure_ctx,
987                DrainDatanodeStrategy::FailFast,
988            )
989            .await
990            .unwrap();
991
992        assert_eq!(1, ctx.volatile_ctx.migrated);
993        assert!(ctx.volatile_ctx.failed_regions.is_empty());
994        assert!(dispatch.inflight_subprocedures.is_empty());
995    }
996
997    #[tokio::test]
998    async fn test_drain_datanode_wait_unknown_child_keeps_pending() {
999        let env = TestEnv::new().await;
1000        let region_id = RegionId::new(1024, 1);
1001        env.create_table(vec![leader_region(region_id, 1)]).await;
1002
1003        let mut ctx = env.context(new_persistent_context(
1004            Some(2),
1005            DrainDatanodeStrategy::FailFast,
1006        ));
1007
1008        let mut dispatch = Dispatch::new(vec![]);
1009        dispatch.inflight_subprocedures = vec![SubprocedureMeta {
1010            region_ids: vec![region_id],
1011            procedure_id: ProcedureId::random(),
1012        }];
1013        let procedure_ctx = procedure_context_with_receivers(HashMap::new());
1014
1015        dispatch
1016            .wait_for_inflight_subprocedures(
1017                &mut ctx,
1018                &procedure_ctx,
1019                DrainDatanodeStrategy::FailFast,
1020            )
1021            .await
1022            .unwrap();
1023
1024        // The region stays pending and is dispatched again instead of being reported as
1025        // failed.
1026        let (next, _) = DrainDatanodeCursor::default()
1027            .next(&mut ctx, &procedure_ctx)
1028            .await
1029            .unwrap();
1030        let batches = &next.as_any().downcast_ref::<Dispatch>().unwrap().batches;
1031        assert_eq!(1, batches.len());
1032        assert_eq!(vec![region_id], batches[0].region_ids);
1033        assert_eq!(0, ctx.volatile_ctx.migrated);
1034        assert!(ctx.volatile_ctx.failed_regions.is_empty());
1035    }
1036
1037    #[tokio::test]
1038    async fn test_drain_datanode_waits_for_entire_batch() {
1039        for strategy in [
1040            DrainDatanodeStrategy::FailFast,
1041            DrainDatanodeStrategy::TryAll,
1042        ] {
1043            let env = TestEnv::new().await;
1044            let failed_region = RegionId::new(1024, 1);
1045            let succeeded_region = RegionId::new(1024, 2);
1046            env.create_table(vec![
1047                leader_region(failed_region, 1),
1048                leader_region(succeeded_region, 1),
1049            ])
1050            .await;
1051            let mut ctx = env.context(new_persistent_context(
1052                Some(2),
1053                DrainDatanodeStrategy::FailFast,
1054            ));
1055
1056            let failed_id = ProcedureId::random();
1057            let pending_id = ProcedureId::random();
1058            let mut dispatch = Dispatch::new(vec![]);
1059            dispatch.inflight_subprocedures = vec![
1060                SubprocedureMeta {
1061                    region_ids: vec![failed_region],
1062                    procedure_id: failed_id,
1063                },
1064                SubprocedureMeta {
1065                    region_ids: vec![succeeded_region],
1066                    procedure_id: pending_id,
1067                },
1068            ];
1069            let (sender, receiver) = watch::channel(ProcedureState::Running);
1070            let procedure_ctx = procedure_context_with_receivers(HashMap::from([
1071                (
1072                    failed_id,
1073                    procedure_state_receiver(ProcedureState::failed(Arc::new(
1074                        ProcedureError::external(MockError::new(StatusCode::Internal)),
1075                    ))),
1076                ),
1077                (pending_id, receiver),
1078            ]));
1079            let mut waiting = Box::pin(dispatch.wait_for_inflight_subprocedures(
1080                &mut ctx,
1081                &procedure_ctx,
1082                strategy,
1083            ));
1084            assert!(futures::poll!(&mut waiting).is_pending());
1085            sender.send(ProcedureState::Done { output: None }).unwrap();
1086            let result = waiting.await;
1087            match strategy {
1088                DrainDatanodeStrategy::FailFast => assert!(
1089                    matches!(result, Err(error::Error::DrainDatanodeAbort { failed_regions, .. }) if failed_regions == vec![failed_region])
1090                ),
1091                DrainDatanodeStrategy::TryAll => {
1092                    result.unwrap();
1093                    assert_eq!(vec![failed_region], ctx.volatile_ctx.failed_regions);
1094                }
1095            }
1096            assert_eq!(1, ctx.volatile_ctx.migrated);
1097            assert!(dispatch.inflight_subprocedures.is_empty());
1098        }
1099    }
1100
1101    #[tokio::test]
1102    async fn test_drain_datanode_wait_preserves_batch_on_metadata_retry() {
1103        use common_meta::key::TableMetadataManager;
1104        use common_meta::kv_backend::test_util::MockKvBackendBuilder;
1105
1106        for strategy in [
1107            DrainDatanodeStrategy::FailFast,
1108            DrainDatanodeStrategy::TryAll,
1109        ] {
1110            let env = TestEnv::new().await;
1111            let failed_region = RegionId::new(1024, 1);
1112            let succeeded_region = RegionId::new(1024, 2);
1113            env.create_table(vec![
1114                leader_region(failed_region, 1),
1115                leader_region(succeeded_region, 1),
1116            ])
1117            .await;
1118            let mut ctx = env.context(new_persistent_context(
1119                Some(2),
1120                DrainDatanodeStrategy::FailFast,
1121            ));
1122
1123            let failed_id = ProcedureId::random();
1124            let succeeded_id = ProcedureId::random();
1125            let mut dispatch = Dispatch::new(vec![]);
1126            dispatch.inflight_subprocedures = vec![
1127                SubprocedureMeta {
1128                    region_ids: vec![succeeded_region],
1129                    procedure_id: succeeded_id,
1130                },
1131                SubprocedureMeta {
1132                    region_ids: vec![failed_region],
1133                    procedure_id: failed_id,
1134                },
1135            ];
1136            let procedure_ctx = procedure_context_with_receivers(HashMap::from([
1137                (
1138                    succeeded_id,
1139                    procedure_state_receiver(ProcedureState::Done { output: None }),
1140                ),
1141                (
1142                    failed_id,
1143                    procedure_state_receiver(ProcedureState::failed(Arc::new(
1144                        ProcedureError::external(MockError::new(StatusCode::Internal)),
1145                    ))),
1146                ),
1147            ]));
1148            ctx.table_metadata_manager = Arc::new(TableMetadataManager::new(Arc::new(
1149                MockKvBackendBuilder::default()
1150                    .range_fn(Arc::new(|_| {
1151                        common_meta::error::ElectionNoLeaderSnafu.fail()
1152                    }))
1153                    .build()
1154                    .unwrap(),
1155            )));
1156            let error = dispatch
1157                .wait_for_inflight_subprocedures(&mut ctx, &procedure_ctx, strategy)
1158                .await
1159                .unwrap_err();
1160            assert!(error.is_retryable());
1161            assert_eq!(2, dispatch.inflight_subprocedures.len());
1162            assert_eq!(0, ctx.volatile_ctx.migrated);
1163            assert!(ctx.volatile_ctx.failed_regions.is_empty());
1164            assert_eq!(
1165                vec![
1166                    (succeeded_id, vec![succeeded_region]),
1167                    (failed_id, vec![failed_region])
1168                ],
1169                dispatch
1170                    .inflight_subprocedures
1171                    .iter()
1172                    .map(|meta| (meta.procedure_id, meta.region_ids.clone()))
1173                    .collect::<Vec<_>>()
1174            );
1175
1176            ctx.table_metadata_manager = env.table_metadata_manager().clone();
1177            let result = dispatch
1178                .wait_for_inflight_subprocedures(&mut ctx, &procedure_ctx, strategy)
1179                .await;
1180            match strategy {
1181                DrainDatanodeStrategy::FailFast => assert!(
1182                    matches!(result, Err(error::Error::DrainDatanodeAbort { failed_regions, .. }) if failed_regions == vec![failed_region])
1183                ),
1184                DrainDatanodeStrategy::TryAll => {
1185                    result.unwrap();
1186                    assert_eq!(vec![failed_region], ctx.volatile_ctx.failed_regions);
1187                }
1188            }
1189            assert_eq!(1, ctx.volatile_ctx.migrated);
1190            assert!(dispatch.inflight_subprocedures.is_empty());
1191        }
1192    }
1193
1194    #[tokio::test]
1195    async fn test_dispatch_registers_and_releases_all_batch_regions() {
1196        let env = TestEnv::new().await;
1197        create_kafka_table(&env, 1024, 8, 2).await;
1198        let mut persistent = new_persistent_context(Some(2), DrainDatanodeStrategy::FailFast);
1199        persistent.parallelism = 3;
1200        let mut ctx = env.context(persistent);
1201        let regions = vec![
1202            RegionId::new(1024, 0),
1203            RegionId::new(1024, 2),
1204            RegionId::new(1024, 4),
1205        ];
1206
1207        let mut dispatch = Dispatch::new(vec![RegionMigrationTaskBatch {
1208            region_ids: regions.clone(),
1209            from_peer: Peer::empty(1),
1210            to_peer: Peer::empty(2),
1211            timeout: Duration::from_secs(10),
1212            trigger_reason: RegionMigrationTriggerReason::Manual,
1213        }]);
1214        let (next, status) = dispatch
1215            .next(&mut ctx, &TestingEnv::procedure_context())
1216            .await
1217            .unwrap();
1218        let Status::Suspended {
1219            subprocedures,
1220            recovery,
1221            persist,
1222        } = status
1223        else {
1224            panic!("expected a detached batch");
1225        };
1226        assert!(matches!(recovery, SubprocedureRecovery::Detach));
1227        assert!(!persist);
1228        assert_eq!(1, subprocedures.len());
1229        assert_eq!(
1230            regions,
1231            next.as_any()
1232                .downcast_ref::<Dispatch>()
1233                .unwrap()
1234                .inflight_subprocedures[0]
1235                .region_ids
1236        );
1237        let tracker = env.region_migration_manager.tracker();
1238        for region_id in &regions {
1239            assert!(tracker.contains(*region_id));
1240        }
1241        drop(subprocedures);
1242        for region_id in regions {
1243            assert!(!tracker.contains(region_id));
1244        }
1245    }
1246}