Skip to main content

meta_srv/procedure/
repartition.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod allocate_region;
16pub mod collect;
17pub mod deallocate_region;
18pub mod dispatch;
19pub mod gc_requirement;
20pub mod group;
21pub mod plan;
22pub mod repartition_end;
23pub mod repartition_start;
24pub mod update_partition_metadata;
25pub mod utils;
26
27use std::any::Any;
28use std::collections::{HashMap, HashSet};
29use std::fmt::{Debug, Display};
30use std::time::{Duration, Instant};
31
32use common_error::ext::BoxedError;
33use common_meta::affinity::{AffinityFallback, AffinityResolverRef};
34use common_meta::cache_invalidator::CacheInvalidatorRef;
35use common_meta::ddl::DdlContext;
36use common_meta::ddl::allocator::region_routes::RegionRoutesAllocatorRef;
37use common_meta::ddl::allocator::wal_options::WalOptionsAllocatorRef;
38use common_meta::ddl::utils::get_region_wal_options;
39use common_meta::ddl_manager::{RepartitionProcedureFactory, RepartitionSource};
40use common_meta::instruction::CacheIdent;
41use common_meta::key::datanode_table::RegionInfo;
42use common_meta::key::table_info::TableInfoValue;
43use common_meta::key::table_route::TableRouteValue;
44use common_meta::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
45use common_meta::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
46use common_meta::node_manager::NodeManagerRef;
47use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard};
48use common_meta::region_registry::LeaderRegionRegistryRef;
49use common_meta::rpc::router::{RegionRoute, operating_leader_region_roles};
50use common_meta::wal_provider::RegionWalOptions;
51use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
52use common_procedure::{
53    BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventContext,
54    EventTrigger, LockKey, Procedure, ProcedureManagerRef, Result as ProcedureResult, Status,
55    StringKey,
56};
57use common_telemetry::{error, info, warn};
58use partition::expr::PartitionExpr;
59use serde::{Deserialize, Serialize};
60use snafu::{OptionExt, ResultExt};
61use store_api::storage::{RegionNumber, TableId};
62use table::table_name::TableName;
63
64use crate::error::{self, Result};
65use crate::event::repartition::{REPARTITION_EVENT_TYPE, RepartitionEvent};
66use crate::procedure::repartition::collect::ProcedureMeta;
67use crate::procedure::repartition::deallocate_region::DeallocateRegion;
68use crate::procedure::repartition::gc_requirement::RepartitionGcRequirementManagerRef;
69use crate::procedure::repartition::group::{
70    Context as RepartitionGroupContext, RepartitionGroupProcedure, region_routes,
71};
72use crate::procedure::repartition::plan::RepartitionPlanEntry;
73use crate::procedure::repartition::repartition_start::{RepartitionFrom, RepartitionStart};
74use crate::procedure::repartition::update_partition_metadata::PartitionMetadataUpdate;
75use crate::procedure::repartition::utils::{
76    get_datanode_table_value, rollback_group_metadata_routes,
77};
78use crate::service::mailbox::MailboxRef;
79
80#[cfg(test)]
81pub mod test_util;
82
83#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
84pub struct PersistentContext {
85    pub catalog_name: String,
86    pub schema_name: String,
87    pub table_name: String,
88    pub table_id: TableId,
89    pub plans: Vec<RepartitionPlanEntry>,
90    #[serde(default)]
91    /// Records failed sub-procedures for parent rollback selection.
92    ///
93    /// The parent repartition procedure uses these entries to decide which plans
94    /// require group-metadata restoration and allocated-region cleanup.
95    pub failed_procedures: Vec<ProcedureMeta>,
96    #[serde(default)]
97    /// Records unknown sub-procedures for parent rollback selection.
98    ///
99    /// Unknown procedures are treated the same as failed ones when selecting the
100    /// plan subset that must be rolled back by the parent procedure.
101    pub unknown_procedures: Vec<ProcedureMeta>,
102    /// The timeout for repartition operations.
103    #[serde(with = "humantime_serde", default = "default_timeout")]
104    pub timeout: Duration,
105    #[serde(default)]
106    /// Records table-level partition metadata updated by this repartition.
107    pub partition_metadata_update: Option<PartitionMetadataUpdate>,
108}
109
110fn default_timeout() -> Duration {
111    Duration::from_secs(2 * 60)
112}
113
114impl PersistentContext {
115    /// Creates a new [PersistentContext] with the given table name, table id and timeout.
116    ///
117    /// If the timeout is not provided, the default timeout will be used.
118    pub fn new(
119        TableName {
120            catalog_name,
121            schema_name,
122            table_name,
123        }: TableName,
124        table_id: TableId,
125        timeout: Option<Duration>,
126    ) -> Self {
127        Self {
128            catalog_name,
129            schema_name,
130            table_name,
131            table_id,
132            plans: vec![],
133            failed_procedures: vec![],
134            unknown_procedures: vec![],
135            timeout: timeout.unwrap_or_else(default_timeout),
136            partition_metadata_update: None,
137        }
138    }
139
140    pub fn lock_key(&self) -> Vec<StringKey> {
141        vec![
142            CatalogLock::Read(&self.catalog_name).into(),
143            SchemaLock::read(&self.catalog_name, &self.schema_name).into(),
144            TableLock::Write(self.table_id).into(),
145            TableNameLock::new(&self.catalog_name, &self.schema_name, &self.table_name).into(),
146        ]
147    }
148}
149
150#[derive(Clone)]
151pub struct Context {
152    pub affinity_resolver: Option<AffinityResolverRef>,
153    pub affinity_fallback: AffinityFallback,
154    pub persistent_ctx: PersistentContext,
155    pub volatile_ctx: VolatileContext,
156    pub table_metadata_manager: TableMetadataManagerRef,
157    pub memory_region_keeper: MemoryRegionKeeperRef,
158    pub node_manager: NodeManagerRef,
159    pub leader_region_registry: LeaderRegionRegistryRef,
160    pub mailbox: MailboxRef,
161    pub server_addr: String,
162    pub cache_invalidator: CacheInvalidatorRef,
163    pub region_routes_allocator: RegionRoutesAllocatorRef,
164    pub wal_options_allocator: WalOptionsAllocatorRef,
165    pub start_time: Instant,
166}
167
168#[derive(Debug, Clone, Default)]
169pub struct VolatileContext {
170    pub metrics: Metrics,
171    pub dispatch_start_time: Option<Instant>,
172}
173
174/// Metrics of repartition.
175#[derive(Debug, Clone, Default)]
176pub struct Metrics {
177    /// Elapsed time of building plan.
178    build_plan_elapsed: Duration,
179    /// Elapsed time of allocating region.
180    allocate_region_elapsed: Duration,
181    /// Elapsed time of finishing groups.
182    finish_groups_elapsed: Duration,
183    /// Elapsed time of deallocating region.
184    deallocate_region_elapsed: Duration,
185}
186
187impl Display for Metrics {
188    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
189        let total = self.build_plan_elapsed
190            + self.allocate_region_elapsed
191            + self.finish_groups_elapsed
192            + self.deallocate_region_elapsed;
193        write!(f, "total: {:?}", total)?;
194        let mut parts = Vec::with_capacity(4);
195        if self.build_plan_elapsed > Duration::ZERO {
196            parts.push(format!("build_plan_elapsed: {:?}", self.build_plan_elapsed));
197        }
198        if self.allocate_region_elapsed > Duration::ZERO {
199            parts.push(format!(
200                "allocate_region_elapsed: {:?}",
201                self.allocate_region_elapsed
202            ));
203        }
204        if self.finish_groups_elapsed > Duration::ZERO {
205            parts.push(format!(
206                "finish_groups_elapsed: {:?}",
207                self.finish_groups_elapsed
208            ));
209        }
210        if self.deallocate_region_elapsed > Duration::ZERO {
211            parts.push(format!(
212                "deallocate_region_elapsed: {:?}",
213                self.deallocate_region_elapsed
214            ));
215        }
216
217        if !parts.is_empty() {
218            write!(f, ", {}", parts.join(", "))?;
219        }
220        Ok(())
221    }
222}
223
224impl Metrics {
225    /// Updates the elapsed time of building plan.
226    pub fn update_build_plan_elapsed(&mut self, elapsed: Duration) {
227        self.build_plan_elapsed += elapsed;
228    }
229
230    /// Updates the elapsed time of allocating region.
231    pub fn update_allocate_region_elapsed(&mut self, elapsed: Duration) {
232        self.allocate_region_elapsed += elapsed;
233    }
234
235    /// Updates the elapsed time of finishing groups.
236    pub fn update_finish_groups_elapsed(&mut self, elapsed: Duration) {
237        self.finish_groups_elapsed += elapsed;
238    }
239
240    /// Updates the elapsed time of deallocating region.
241    pub fn update_deallocate_region_elapsed(&mut self, elapsed: Duration) {
242        self.deallocate_region_elapsed += elapsed;
243    }
244}
245
246impl Context {
247    pub fn new(
248        ddl_ctx: &DdlContext,
249        mailbox: MailboxRef,
250        server_addr: String,
251        persistent_ctx: PersistentContext,
252    ) -> Self {
253        Self {
254            affinity_resolver: ddl_ctx.affinity_resolver.clone(),
255            affinity_fallback: ddl_ctx.affinity_fallback,
256            persistent_ctx,
257            table_metadata_manager: ddl_ctx.table_metadata_manager.clone(),
258            memory_region_keeper: ddl_ctx.memory_region_keeper.clone(),
259            node_manager: ddl_ctx.node_manager.clone(),
260            leader_region_registry: ddl_ctx.leader_region_registry.clone(),
261            mailbox,
262            server_addr,
263            cache_invalidator: ddl_ctx.cache_invalidator.clone(),
264            region_routes_allocator: ddl_ctx.table_metadata_allocator.region_routes_allocator(),
265            wal_options_allocator: ddl_ctx.table_metadata_allocator.wal_options_allocator(),
266            start_time: Instant::now(),
267            volatile_ctx: VolatileContext::default(),
268        }
269    }
270
271    /// Returns the next operation's timeout.
272    pub fn next_operation_timeout(&self) -> Option<Duration> {
273        self.persistent_ctx
274            .timeout
275            .checked_sub(self.start_time.elapsed())
276    }
277
278    /// Updates the elapsed time of building plan.
279    pub fn update_build_plan_elapsed(&mut self, elapsed: Duration) {
280        self.volatile_ctx.metrics.update_build_plan_elapsed(elapsed);
281    }
282
283    /// Updates the elapsed time of allocating region.
284    pub fn update_allocate_region_elapsed(&mut self, elapsed: Duration) {
285        self.volatile_ctx
286            .metrics
287            .update_allocate_region_elapsed(elapsed);
288    }
289
290    /// Updates the elapsed time of finishing groups.
291    pub fn update_finish_groups_elapsed(&mut self, elapsed: Duration) {
292        self.volatile_ctx
293            .metrics
294            .update_finish_groups_elapsed(elapsed);
295    }
296
297    /// Updates the elapsed time of deallocating region.
298    pub fn update_deallocate_region_elapsed(&mut self, elapsed: Duration) {
299        self.volatile_ctx
300            .metrics
301            .update_deallocate_region_elapsed(elapsed);
302    }
303
304    /// Retrieves the table route value for the given table id.
305    ///
306    /// Retry:
307    /// - Failed to retrieve the metadata of table.
308    ///
309    /// Abort:
310    /// - Table route not found.
311    pub async fn get_table_route_value(
312        &self,
313    ) -> Result<DeserializedValueWithBytes<TableRouteValue>> {
314        let table_id = self.persistent_ctx.table_id;
315        let table_route_value = self
316            .table_metadata_manager
317            .table_route_manager()
318            .table_route_storage()
319            .get_with_raw_bytes(table_id)
320            .await
321            .map_err(BoxedError::new)
322            .with_context(|_| error::RetryLaterWithSourceSnafu {
323                reason: format!("Failed to get table route for table: {}", table_id),
324            })?
325            .context(error::TableRouteNotFoundSnafu { table_id })?;
326
327        Ok(table_route_value)
328    }
329
330    async fn reserve_region_numbers(
331        &self,
332        current: &DeserializedValueWithBytes<TableRouteValue>,
333        count: usize,
334    ) -> Result<RegionNumber> {
335        let table_id = self.persistent_ctx.table_id;
336        self.table_metadata_manager
337            .table_route_manager()
338            .try_reserve_region_numbers(table_id, current, count)
339            .await
340            .context(error::TableMetadataManagerSnafu)?
341            .with_context(|| error::RetryLaterSnafu {
342                reason: format!(
343                    "Table route changed while reserving region numbers for table {table_id}"
344                ),
345            })
346    }
347
348    /// Retrieves the table info value for the given table id.
349    ///
350    /// Retry:
351    /// - Failed to retrieve the metadata of table.
352    ///
353    /// Abort:
354    /// - Table info not found.
355    pub async fn get_raw_table_info_value(
356        &self,
357    ) -> Result<DeserializedValueWithBytes<TableInfoValue>> {
358        let table_id = self.persistent_ctx.table_id;
359        let table_info_value = self
360            .table_metadata_manager
361            .table_info_manager()
362            .get(table_id)
363            .await
364            .map_err(BoxedError::new)
365            .with_context(|_| error::RetryLaterWithSourceSnafu {
366                reason: format!("Failed to get table info for table: {}", table_id),
367            })?
368            .context(error::TableInfoNotFoundSnafu { table_id })?;
369
370        Ok(table_info_value)
371    }
372
373    pub async fn get_table_info_value(&self) -> Result<TableInfoValue> {
374        let table_info_value = self.get_raw_table_info_value().await?.into_inner();
375        Ok(table_info_value)
376    }
377
378    /// Updates the table info.
379    pub async fn update_table_info(
380        &self,
381        current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
382        new_table_info_value: TableInfoValue,
383    ) -> Result<()> {
384        let table_id = self.persistent_ctx.table_id;
385        self.table_metadata_manager
386            .update_table_info(
387                current_table_info_value,
388                None,
389                new_table_info_value.table_info,
390            )
391            .await
392            .map_err(BoxedError::new)
393            .with_context(|_| error::RetryLaterWithSourceSnafu {
394                reason: format!("Failed to update table info for table: {}", table_id),
395            })
396    }
397
398    /// Updates the table route.
399    ///
400    /// Retry:
401    /// - Failed to retrieve the metadata of datanode table.
402    ///
403    /// Abort:
404    /// - Table route not found.
405    /// - Failed to update the table route.
406    pub async fn update_table_route(
407        &self,
408        current_table_route_value: &DeserializedValueWithBytes<TableRouteValue>,
409        new_region_routes: Vec<RegionRoute>,
410        new_region_wal_options: RegionWalOptions,
411    ) -> Result<()> {
412        let table_id = self.persistent_ctx.table_id;
413        if new_region_routes.is_empty() {
414            return error::UnexpectedSnafu {
415                violated: format!("new_region_routes is empty for table: {}", table_id),
416            }
417            .fail();
418        }
419        let datanode_id = new_region_routes
420            .first()
421            .unwrap()
422            .leader_peer
423            .as_ref()
424            .context(error::NoLeaderSnafu)?
425            .id;
426        let datanode_table_value =
427            get_datanode_table_value(&self.table_metadata_manager, table_id, datanode_id).await?;
428
429        let RegionInfo { region_options, .. } = &datanode_table_value.region_info;
430
431        let mut region_wal_options = get_region_wal_options(
432            &self.table_metadata_manager,
433            current_table_route_value,
434            table_id,
435        )
436        .await
437        .context(error::TableMetadataManagerSnafu)?;
438        // Legacy regions without a persisted WAL option use RaftEngine. Only
439        // existing routes get this default; allocated regions must supply one.
440        for route in current_table_route_value.region_routes().unwrap() {
441            region_wal_options
442                .entry(route.region.id.region_number())
443                .or_default();
444        }
445
446        // Merge and validate the new region wal options.
447        let validated_region_wal_options =
448            crate::procedure::repartition::utils::merge_and_validate_region_wal_options(
449                &region_wal_options,
450                new_region_wal_options,
451                &new_region_routes,
452                table_id,
453            )?;
454        info!(
455            "Updating table route for table: {}, new region routes: {:?}",
456            table_id, new_region_routes
457        );
458        self.table_metadata_manager
459            .update_table_route(
460                table_id,
461                datanode_table_value.region_info.clone(),
462                current_table_route_value,
463                new_region_routes,
464                region_options,
465                &validated_region_wal_options,
466            )
467            .await
468            .context(error::TableMetadataManagerSnafu)
469    }
470
471    /// Broadcasts the invalidate table cache message.
472    pub async fn invalidate_table_cache(&self) -> Result<()> {
473        let table_id = self.persistent_ctx.table_id;
474        let subject = format!(
475            "Invalidate table cache for repartition table, table: {}",
476            table_id,
477        );
478        let ctx = common_meta::cache_invalidator::Context {
479            subject: Some(subject),
480        };
481        let _ = self
482            .cache_invalidator
483            .invalidate(
484                &ctx,
485                &[
486                    CacheIdent::TableId(table_id),
487                    CacheIdent::TableName(TableName {
488                        catalog_name: self.persistent_ctx.catalog_name.clone(),
489                        schema_name: self.persistent_ctx.schema_name.clone(),
490                        table_name: self.persistent_ctx.table_name.clone(),
491                    }),
492                ],
493            )
494            .await;
495        Ok(())
496    }
497
498    pub fn register_operating_regions(
499        memory_region_keeper: &MemoryRegionKeeperRef,
500        region_routes: &[RegionRoute],
501    ) -> Result<Vec<OperatingRegionGuard>> {
502        let mut operating_guards = Vec::with_capacity(region_routes.len());
503        for (region_id, datanode_id, role) in operating_leader_region_roles(region_routes) {
504            let guard = memory_region_keeper
505                .register_with_role(datanode_id, region_id, role)
506                .context(error::RegionOperatingRaceSnafu {
507                    peer_id: datanode_id,
508                    region_id,
509                })?;
510            operating_guards.push(guard);
511        }
512        Ok(operating_guards)
513    }
514}
515
516#[async_trait::async_trait]
517#[typetag::serde(tag = "repartition_state")]
518pub(crate) trait State: Sync + Send + Debug {
519    fn name(&self) -> &'static str {
520        let type_name = std::any::type_name::<Self>();
521        // short name
522        type_name.split("::").last().unwrap_or(type_name)
523    }
524
525    /// Yields the next [State] and [Status].
526    async fn next(
527        &mut self,
528        ctx: &mut Context,
529        procedure_ctx: &ProcedureContext,
530    ) -> Result<(Box<dyn State>, Status)>;
531
532    fn as_any(&self) -> &dyn Any;
533}
534
535pub struct RepartitionProcedure {
536    state: Box<dyn State>,
537    context: Context,
538}
539
540#[derive(Debug, Serialize)]
541struct RepartitionData<'a> {
542    state: &'a dyn State,
543    persistent_ctx: &'a PersistentContext,
544}
545
546#[derive(Debug, Deserialize)]
547struct RepartitionDataOwned {
548    state: Box<dyn State>,
549    persistent_ctx: PersistentContext,
550}
551
552impl RepartitionProcedure {
553    const TYPE_NAME: &'static str = "metasrv-procedure::Repartition";
554
555    pub fn new(from: RepartitionFrom, to_exprs: Vec<PartitionExpr>, context: Context) -> Self {
556        let state = Box::new(RepartitionStart::new(from, to_exprs));
557
558        Self { state, context }
559    }
560
561    pub fn from_json<F>(json: &str, ctx_factory: F) -> ProcedureResult<Self>
562    where
563        F: FnOnce(PersistentContext) -> Context,
564    {
565        let RepartitionDataOwned {
566            state,
567            persistent_ctx,
568        } = serde_json::from_str(json).context(FromJsonSnafu)?;
569        let context = ctx_factory(persistent_ctx);
570
571        Ok(Self { state, context })
572    }
573
574    /// Returns whether parent rollback should run.
575    ///
576    /// This uses an "after repartition metadata update" semantic: once execution
577    /// reaches `UpdatePartitionMetadata` or any later rollback-active state,
578    /// rollback must try to clean metadata written by the repartition procedure.
579    ///
580    /// Notes:
581    /// - `RepartitionStart`: no-op, because no metadata has been updated yet.
582    /// - `UpdatePartitionMetadata`: rollback table partition metadata.
583    /// - `AllocateRegion` / `Dispatch` / `Collect`: rollback table partition metadata
584    ///   and allocated region metadata.
585    /// - `DeallocateRegion`: is not rollback-active.
586    /// - `RepartitionEnd`: no-op.
587    fn should_rollback(&self) -> bool {
588        self.state
589            .as_any()
590            .is::<update_partition_metadata::UpdatePartitionMetadata>()
591            || self.state.as_any().is::<allocate_region::AllocateRegion>()
592            || self.state.as_any().is::<dispatch::Dispatch>()
593            || self.state.as_any().is::<collect::Collect>()
594    }
595
596    fn rollback_plan_indices(&self) -> HashSet<usize> {
597        self.context
598            .persistent_ctx
599            .failed_procedures
600            .iter()
601            .chain(self.context.persistent_ctx.unknown_procedures.iter())
602            .map(|procedure_meta| procedure_meta.plan_index)
603            .collect()
604    }
605
606    /// Returns allocated region ids that parent rollback should remove.
607    ///
608    /// Rollback uses an "after region allocation" semantic:
609    /// - in `AllocateRegion` and `Dispatch`, all allocated regions belong to the
610    ///   current repartition attempt and must be cleaned up.
611    /// - in `Collect`, only the plans referenced by failed or unknown
612    ///   sub-procedures should be rolled back.
613    fn rollback_allocated_region_ids(&self) -> HashSet<store_api::storage::RegionId> {
614        if self.state.as_any().is::<allocate_region::AllocateRegion>()
615            || self.state.as_any().is::<dispatch::Dispatch>()
616        {
617            return self
618                .context
619                .persistent_ctx
620                .plans
621                .iter()
622                .flat_map(|plan| plan.allocated_region_ids.iter().copied())
623                .collect();
624        }
625
626        self.rollback_plan_indices()
627            .into_iter()
628            .flat_map(|plan_index| {
629                self.context.persistent_ctx.plans[plan_index]
630                    .allocated_region_ids
631                    .iter()
632                    .copied()
633            })
634            .collect()
635    }
636
637    /// Restores group-level staging metadata for failed/unknown plans.
638    ///
639    /// The helper mutates `region_routes` in memory.
640    async fn rollback_group_metadata_for_selected_plans(
641        &mut self,
642        region_routes: &mut [RegionRoute],
643    ) -> Result<()> {
644        let rollback_plan_indices = self.rollback_plan_indices();
645        if rollback_plan_indices.is_empty() {
646            return Ok(());
647        }
648
649        let mut region_routes_map = region_routes
650            .iter_mut()
651            .map(|route| (route.region.id, route))
652            .collect::<HashMap<_, _>>();
653        for plan_index in rollback_plan_indices {
654            let plan = &self.context.persistent_ctx.plans[plan_index];
655            rollback_group_metadata_routes(
656                plan.group_id,
657                &plan.source_regions,
658                &plan.original_target_routes,
659                &plan.allocated_region_ids,
660                &plan.pending_deallocate_region_ids,
661                &mut region_routes_map,
662            )?;
663        }
664
665        Ok(())
666    }
667
668    async fn rollback_partition_metadata(&mut self) -> Result<()> {
669        let Some(update) = self
670            .context
671            .persistent_ctx
672            .partition_metadata_update
673            .as_ref()
674        else {
675            return Ok(());
676        };
677        let table_info_value = self.context.get_raw_table_info_value().await?;
678        let current_partition_key_indices = &table_info_value.table_info.meta.partition_key_indices;
679        let Some(new_partition_key_indices) = update.rollback_partition_key_indices(
680            self.context.persistent_ctx.table_id,
681            current_partition_key_indices,
682        )?
683        else {
684            return Ok(());
685        };
686
687        let mut new_table_info = table_info_value.table_info.clone();
688        new_table_info.meta.partition_key_indices = new_partition_key_indices;
689        self.context
690            .update_table_info(&table_info_value, table_info_value.update(new_table_info))
691            .await?;
692
693        // Do not invalidate the table cache here. The table routes may still
694        // contain partition expressions until `rollback_inner` rolls them back.
695        // Exposing cleared partition columns with partitioned routes can build
696        // an inconsistent partition rule. The cache is invalidated once after
697        // both partition metadata and routes are rolled back.
698
699        Ok(())
700    }
701
702    async fn rollback_inner(&mut self, procedure_ctx: &ProcedureContext) -> Result<()> {
703        if !self.should_rollback() {
704            return Ok(());
705        }
706
707        let table_id = self.context.persistent_ctx.table_id;
708        let allocated_region_ids = self.rollback_allocated_region_ids();
709
710        let table_lock = TableLock::Write(table_id).into();
711        let _guard = procedure_ctx.provider.acquire_lock(&table_lock).await;
712
713        self.rollback_partition_metadata().await?;
714        let table_route_value = self.context.get_table_route_value().await?;
715        let original_region_routes = region_routes(table_id, table_route_value.get_inner_ref())?;
716        let mut current_region_routes = original_region_routes.clone();
717        self.rollback_group_metadata_for_selected_plans(&mut current_region_routes)
718            .await?;
719        let allocated_region_routes = DeallocateRegion::filter_deallocatable_region_routes(
720            table_id,
721            &current_region_routes,
722            &allocated_region_ids,
723        );
724        if !allocated_region_routes.is_empty() {
725            let table = TableName {
726                catalog_name: self.context.persistent_ctx.catalog_name.clone(),
727                schema_name: self.context.persistent_ctx.schema_name.clone(),
728                table_name: self.context.persistent_ctx.table_name.clone(),
729            };
730            // Memory guards are not required here,
731            // because the table metadata still contains routes for the deallocating regions.
732            if let Err(err) = DeallocateRegion::deallocate_regions(
733                &self.context.node_manager,
734                &self.context.leader_region_registry,
735                table,
736                table_id,
737                &allocated_region_routes,
738            )
739            .await
740            {
741                warn!(err; "Failed to drop allocated regions during repartition rollback, table_id: {}, regions: {:?}", table_id, allocated_region_ids);
742            }
743        }
744
745        let new_region_routes =
746            DeallocateRegion::generate_region_routes(&current_region_routes, &allocated_region_ids);
747
748        if new_region_routes != *original_region_routes {
749            self.context
750                .update_table_route(&table_route_value, new_region_routes, HashMap::new())
751                .await
752                .map_err(BoxedError::new)
753                .with_context(|_| error::RetryLaterWithSourceSnafu {
754                    reason: format!(
755                        "Failed to rollback allocated region routes for repartition table: {}",
756                        table_id
757                    ),
758                })?;
759        }
760
761        if let Err(err) = self.context.invalidate_table_cache().await {
762            warn!(err; "Failed to invalidate table cache during repartition rollback, table_id: {}", table_id);
763        }
764
765        Ok(())
766    }
767}
768
769#[async_trait::async_trait]
770impl Procedure for RepartitionProcedure {
771    fn type_name(&self) -> &str {
772        Self::TYPE_NAME
773    }
774
775    #[tracing::instrument(skip_all, fields(
776        state = %self.state.name(),
777        table_id = %self.context.persistent_ctx.table_id
778    ))]
779    async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
780        let state = &mut self.state;
781        let state_name = state.name();
782        // Log state transition
783        common_telemetry::info!(
784            "Repartition procedure executing state: {}, table_id: {}",
785            state_name,
786            self.context.persistent_ctx.table_id
787        );
788        match state.next(&mut self.context, _ctx).await {
789            Ok((next, status)) => {
790                *state = next;
791                Ok(status)
792            }
793            Err(e) => {
794                if e.is_retryable() {
795                    Err(ProcedureError::retry_later(e))
796                } else {
797                    error!(
798                        e;
799                        "Repartition procedure failed, table id: {}",
800                        self.context.persistent_ctx.table_id,
801                    );
802                    Err(ProcedureError::external(e))
803                }
804            }
805        }
806    }
807
808    async fn rollback(&mut self, ctx: &ProcedureContext) -> ProcedureResult<()> {
809        self.rollback_inner(ctx)
810            .await
811            .map_err(ProcedureError::external)
812    }
813
814    fn rollback_supported(&self) -> bool {
815        true
816    }
817
818    fn dump(&self) -> ProcedureResult<String> {
819        let data = RepartitionData {
820            state: self.state.as_ref(),
821            persistent_ctx: &self.context.persistent_ctx,
822        };
823        serde_json::to_string(&data).context(ToJsonSnafu)
824    }
825
826    fn lock_key(&self) -> LockKey {
827        LockKey::new(self.context.persistent_ctx.lock_key())
828    }
829
830    fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
831        if !ctx.event_type_filter.allows(REPARTITION_EVENT_TYPE) {
832            return None;
833        }
834
835        let event = if matches!(ctx.trigger, EventTrigger::Submitted) {
836            let start = self.state.as_any().downcast_ref::<RepartitionStart>()?;
837            RepartitionEvent::submitted(&self.context.persistent_ctx, start)
838        } else {
839            RepartitionEvent::lifecycle(&self.context.persistent_ctx)
840        };
841        Some(Box::new(event))
842    }
843}
844
845pub struct DefaultRepartitionProcedureFactory {
846    mailbox: MailboxRef,
847    server_addr: String,
848    gc_requirement_manager: RepartitionGcRequirementManagerRef,
849}
850
851impl DefaultRepartitionProcedureFactory {
852    pub fn new(
853        mailbox: MailboxRef,
854        server_addr: String,
855        gc_requirement_manager: RepartitionGcRequirementManagerRef,
856    ) -> Self {
857        Self {
858            mailbox,
859            server_addr,
860            gc_requirement_manager,
861        }
862    }
863}
864
865/// Rejects new repartition requests when metasrv GC is disabled.
866///
867/// Procedure loaders are still delegated to the enabled factory so procedures
868/// persisted before a metasrv restart remain recoverable after GC is re-enabled.
869pub struct GcDisabledRepartitionProcedureFactory {
870    enabled_factory: DefaultRepartitionProcedureFactory,
871}
872
873impl GcDisabledRepartitionProcedureFactory {
874    pub fn new(
875        mailbox: MailboxRef,
876        server_addr: String,
877        gc_requirement_manager: RepartitionGcRequirementManagerRef,
878    ) -> Self {
879        Self {
880            enabled_factory: DefaultRepartitionProcedureFactory::new(
881                mailbox,
882                server_addr,
883                gc_requirement_manager,
884            ),
885        }
886    }
887}
888
889#[async_trait::async_trait]
890impl RepartitionProcedureFactory for GcDisabledRepartitionProcedureFactory {
891    fn create(
892        &self,
893        _ddl_ctx: &DdlContext,
894        _table_name: TableName,
895        _table_id: TableId,
896        _source: RepartitionSource,
897        _to_exprs: Vec<String>,
898        _timeout: Option<Duration>,
899    ) -> std::result::Result<BoxedProcedure, BoxedError> {
900        Err(BoxedError::new(
901            error::InvalidArgumentsSnafu {
902                err_msg: "Repartition requires metasrv GC to be enabled".to_string(),
903            }
904            .build(),
905        ))
906    }
907
908    fn register_loaders(
909        &self,
910        ddl_ctx: &DdlContext,
911        procedure_manager: &ProcedureManagerRef,
912    ) -> std::result::Result<(), BoxedError> {
913        self.enabled_factory
914            .register_loaders(ddl_ctx, procedure_manager)
915    }
916
917    async fn ensure_gc_requirement(&self) -> std::result::Result<(), BoxedError> {
918        Err(BoxedError::new(
919            error::InvalidArgumentsSnafu {
920                err_msg: "Repartition requires metasrv GC to be enabled".to_string(),
921            }
922            .build(),
923        ))
924    }
925}
926
927#[async_trait::async_trait]
928impl RepartitionProcedureFactory for DefaultRepartitionProcedureFactory {
929    fn create(
930        &self,
931        ddl_ctx: &DdlContext,
932        table_name: TableName,
933        table_id: TableId,
934        source: RepartitionSource,
935        to_exprs: Vec<String>,
936        timeout: Option<Duration>,
937    ) -> std::result::Result<BoxedProcedure, BoxedError> {
938        let persistent_ctx = PersistentContext::new(table_name, table_id, timeout);
939        let from = match source {
940            RepartitionSource::Partitioned {
941                exprs,
942                target_partition_columns,
943            } => {
944                let exprs = exprs
945                    .iter()
946                    .map(|e| {
947                        PartitionExpr::from_json_str(e)
948                            .context(error::DeserializePartitionExprSnafu)?
949                            .context(error::EmptyPartitionExprSnafu)
950                    })
951                    .collect::<Result<Vec<_>>>()
952                    .map_err(BoxedError::new)?;
953                RepartitionFrom::Partitioned {
954                    exprs,
955                    target_partition_columns,
956                }
957            }
958            RepartitionSource::Unpartitioned { partition_columns } => {
959                RepartitionFrom::Unpartitioned { partition_columns }
960            }
961        };
962        let to_exprs = to_exprs
963            .iter()
964            .map(|e| {
965                PartitionExpr::from_json_str(e)
966                    .context(error::DeserializePartitionExprSnafu)?
967                    .context(error::EmptyPartitionExprSnafu)
968            })
969            .collect::<Result<Vec<_>>>()
970            .map_err(BoxedError::new)?;
971
972        let procedure = RepartitionProcedure::new(
973            from,
974            to_exprs,
975            Context::new(
976                ddl_ctx,
977                self.mailbox.clone(),
978                self.server_addr.clone(),
979                persistent_ctx,
980            ),
981        );
982
983        Ok(Box::new(procedure))
984    }
985
986    fn register_loaders(
987        &self,
988        ddl_ctx: &DdlContext,
989        procedure_manager: &ProcedureManagerRef,
990    ) -> std::result::Result<(), BoxedError> {
991        // Registers the repartition procedure loader.
992        let mailbox = self.mailbox.clone();
993        let server_addr = self.server_addr.clone();
994        let moved_ddl_ctx = ddl_ctx.clone();
995        procedure_manager
996            .register_loader(
997                RepartitionProcedure::TYPE_NAME,
998                Box::new(move |json| {
999                    let mailbox = mailbox.clone();
1000                    let server_addr = server_addr.clone();
1001                    let ddl_ctx = moved_ddl_ctx.clone();
1002                    let factory = move |persistent_ctx| {
1003                        Context::new(&ddl_ctx, mailbox, server_addr, persistent_ctx)
1004                    };
1005                    RepartitionProcedure::from_json(json, factory).map(|p| Box::new(p) as _)
1006                }),
1007            )
1008            .map_err(BoxedError::new)?;
1009
1010        // Registers the repartition group procedure loader.
1011        let mailbox = self.mailbox.clone();
1012        let server_addr = self.server_addr.clone();
1013        let moved_ddl_ctx = ddl_ctx.clone();
1014        procedure_manager
1015            .register_loader(
1016                RepartitionGroupProcedure::TYPE_NAME,
1017                Box::new(move |json| {
1018                    let mailbox = mailbox.clone();
1019                    let server_addr = server_addr.clone();
1020                    let ddl_ctx = moved_ddl_ctx.clone();
1021                    let factory = move |persistent_ctx| {
1022                        RepartitionGroupContext::new(&ddl_ctx, mailbox, server_addr, persistent_ctx)
1023                    };
1024                    RepartitionGroupProcedure::from_json(json, factory).map(|p| Box::new(p) as _)
1025                }),
1026            )
1027            .map_err(BoxedError::new)?;
1028
1029        Ok(())
1030    }
1031
1032    async fn ensure_gc_requirement(&self) -> std::result::Result<(), BoxedError> {
1033        self.gc_requirement_manager
1034            .require_gc()
1035            .await
1036            .map_err(BoxedError::new)
1037    }
1038}
1039
1040#[cfg(test)]
1041mod tests {
1042    use crate::procedure::test_util::{procedure_context_with_receivers, procedure_state_receiver};
1043    use std::collections::HashMap;
1044    use std::sync::Arc;
1045    use std::sync::atomic::{AtomicBool, Ordering};
1046
1047    use common_error::ext::{BoxedError, ErrorExt};
1048    use common_error::mock::MockError;
1049    use common_error::status_code::StatusCode;
1050    use common_event_recorder::EventTypeFilter;
1051    use common_meta::ddl::test_util::datanode_handler::{
1052        DatanodeWatcher, NaiveDatanodeHandler, UnexpectedErrorDatanodeHandler,
1053    };
1054    use common_meta::error;
1055    use common_meta::peer::Peer;
1056    use common_meta::region_keeper::MemoryRegionKeeper;
1057    use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
1058    use common_meta::state_store::KvStateStore;
1059    use common_meta::test_util::MockDatanodeManager;
1060    use common_procedure::local::{LocalManager, ManagerConfig};
1061    use common_procedure::{Error as ProcedureError, Procedure, ProcedureId, ProcedureState};
1062    use store_api::region_engine::RegionRole;
1063    use store_api::storage::RegionId;
1064    use table::table_name::TableName;
1065    use tokio::sync::mpsc;
1066    use uuid::Uuid;
1067
1068    use super::*;
1069    use crate::procedure::repartition::allocate_region::AllocateRegion;
1070    use crate::procedure::repartition::collect::Collect;
1071    use crate::procedure::repartition::deallocate_region::DeallocateRegion;
1072    use crate::procedure::repartition::dispatch::Dispatch;
1073    use crate::procedure::repartition::gc_requirement::RepartitionGcRequirementManager;
1074    use crate::procedure::repartition::group::update_metadata::UpdateMetadata;
1075    use crate::procedure::repartition::plan::{SourceRegionDescriptor, TargetRegionDescriptor};
1076    use crate::procedure::repartition::repartition_end::RepartitionEnd;
1077    use crate::procedure::repartition::test_util::{
1078        TestingEnv, assert_parent_state, current_parent_region_routes, extract_subprocedure_ids,
1079        new_parent_context, range_expr, test_region_route, test_region_wal_options,
1080    };
1081    use crate::procedure::repartition::update_partition_metadata::{
1082        PartitionMetadataUpdate, UpdatePartitionMetadata,
1083    };
1084
1085    fn test_plan(table_id: TableId) -> RepartitionPlanEntry {
1086        RepartitionPlanEntry {
1087            group_id: uuid::Uuid::new_v4(),
1088            source_regions: vec![SourceRegionDescriptor::partitioned(
1089                RegionId::new(table_id, 1),
1090                range_expr("x", 0, 100),
1091            )],
1092            target_regions: vec![
1093                TargetRegionDescriptor {
1094                    region_id: RegionId::new(table_id, 1),
1095                    partition_expr: range_expr("x", 0, 50),
1096                },
1097                TargetRegionDescriptor {
1098                    region_id: RegionId::new(table_id, 3),
1099                    partition_expr: range_expr("x", 50, 100),
1100                },
1101            ],
1102            allocated_region_ids: vec![RegionId::new(table_id, 3)],
1103            pending_deallocate_region_ids: vec![],
1104            transition_map: vec![vec![0, 1]],
1105            original_target_routes: vec![],
1106        }
1107    }
1108
1109    fn with_rollback_metadata(
1110        mut plan: RepartitionPlanEntry,
1111        original_target_routes: Vec<RegionRoute>,
1112    ) -> RepartitionPlanEntry {
1113        plan.original_target_routes = original_target_routes;
1114        plan
1115    }
1116
1117    fn apply_group_staging(
1118        plan: &RepartitionPlanEntry,
1119        current_region_routes: &[RegionRoute],
1120    ) -> Vec<RegionRoute> {
1121        UpdateMetadata::apply_staging_region_routes(
1122            plan.group_id,
1123            &plan.source_regions,
1124            &plan.target_regions,
1125            &plan.pending_deallocate_region_ids,
1126            current_region_routes,
1127        )
1128        .unwrap()
1129    }
1130
1131    fn exit_group_staging(
1132        plan: &RepartitionPlanEntry,
1133        current_region_routes: &[RegionRoute],
1134    ) -> Vec<RegionRoute> {
1135        UpdateMetadata::exit_staging_region_routes(
1136            plan.group_id,
1137            &plan.source_regions,
1138            &plan.target_regions,
1139            current_region_routes,
1140        )
1141        .unwrap()
1142    }
1143
1144    fn region_route_by_id(region_routes: &[RegionRoute], region_id: RegionId) -> &RegionRoute {
1145        region_routes
1146            .iter()
1147            .find(|route| route.region.id == region_id)
1148            .unwrap()
1149    }
1150
1151    async fn table_partition_key_indices(ctx: &Context) -> Vec<usize> {
1152        ctx.get_table_info_value()
1153            .await
1154            .unwrap()
1155            .table_info
1156            .meta
1157            .partition_key_indices
1158    }
1159
1160    fn test_procedure(state: Box<dyn State>, context: Context) -> RepartitionProcedure {
1161        RepartitionProcedure { state, context }
1162    }
1163
1164    fn test_context(env: &TestingEnv, table_id: TableId) -> Context {
1165        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1166        let ddl_ctx = env.ddl_context(node_manager);
1167        let persistent_ctx = PersistentContext::new(
1168            TableName::new("test_catalog", "test_schema", "test_table"),
1169            table_id,
1170            None,
1171        );
1172
1173        Context::new(
1174            &ddl_ctx,
1175            env.mailbox_ctx.mailbox().clone(),
1176            env.server_addr.clone(),
1177            persistent_ctx,
1178        )
1179    }
1180
1181    #[test]
1182    fn test_gc_disabled_factory_rejects_repartition_and_registers_loaders() {
1183        let env = TestingEnv::new();
1184        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1185        let ddl_ctx = env.ddl_context(node_manager);
1186        let factory = GcDisabledRepartitionProcedureFactory::new(
1187            env.mailbox_ctx.mailbox().clone(),
1188            env.server_addr.clone(),
1189            Arc::new(RepartitionGcRequirementManager::new(env.kv_backend.clone())),
1190        );
1191
1192        let err = factory
1193            .create(
1194                &ddl_ctx,
1195                TableName::new("test_catalog", "test_schema", "test_table"),
1196                1024,
1197                RepartitionSource::Unpartitioned {
1198                    partition_columns: vec![],
1199                },
1200                vec![],
1201                None,
1202            )
1203            .err()
1204            .expect("GC-disabled factory must reject repartition");
1205
1206        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1207        assert_eq!(
1208            "Invalid arguments: Repartition requires metasrv GC to be enabled",
1209            err.to_string()
1210        );
1211
1212        let state_store = Arc::new(KvStateStore::new(env.kv_backend));
1213        let procedure_manager = Arc::new(LocalManager::new(
1214            ManagerConfig::default(),
1215            state_store.clone(),
1216            state_store,
1217            None,
1218            None,
1219        ));
1220        let procedure_manager_ref: ProcedureManagerRef = procedure_manager.clone();
1221        factory
1222            .register_loaders(&ddl_ctx, &procedure_manager_ref)
1223            .unwrap();
1224        assert!(procedure_manager.contains_loader(RepartitionProcedure::TYPE_NAME));
1225        assert!(procedure_manager.contains_loader(RepartitionGroupProcedure::TYPE_NAME));
1226    }
1227
1228    #[test]
1229    fn test_filter_allocated_region_routes() {
1230        let table_id = 1024;
1231        let region_routes = vec![
1232            test_region_route(RegionId::new(table_id, 1), "a"),
1233            test_region_route(RegionId::new(table_id, 2), "b"),
1234        ];
1235        let allocated_region_ids = HashSet::from([RegionId::new(table_id, 2)]);
1236
1237        let new_region_routes =
1238            DeallocateRegion::generate_region_routes(&region_routes, &allocated_region_ids);
1239
1240        assert_eq!(new_region_routes.len(), 1);
1241        assert_eq!(new_region_routes[0].region.id, RegionId::new(table_id, 1));
1242    }
1243
1244    #[test]
1245    fn test_should_rollback_after_metadata_update() {
1246        let env = TestingEnv::new();
1247        let table_id = 1024;
1248
1249        let procedure = test_procedure(
1250            Box::new(RepartitionStart::new(
1251                RepartitionFrom::Partitioned {
1252                    exprs: vec![],
1253                    target_partition_columns: None,
1254                },
1255                vec![],
1256            )),
1257            test_context(&env, table_id),
1258        );
1259        assert!(!procedure.should_rollback());
1260
1261        let procedure = test_procedure(
1262            Box::new(UpdatePartitionMetadata::new(vec![])),
1263            test_context(&env, table_id),
1264        );
1265        assert!(procedure.should_rollback());
1266
1267        let procedure = test_procedure(
1268            Box::new(AllocateRegion::new(vec![])),
1269            test_context(&env, table_id),
1270        );
1271        assert!(procedure.should_rollback());
1272
1273        let procedure = test_procedure(Box::new(Dispatch), test_context(&env, table_id));
1274        assert!(procedure.should_rollback());
1275
1276        let procedure =
1277            test_procedure(Box::new(Collect::new(vec![])), test_context(&env, table_id));
1278        assert!(procedure.should_rollback());
1279
1280        let procedure = test_procedure(Box::new(DeallocateRegion), test_context(&env, table_id));
1281        assert!(!procedure.should_rollback());
1282
1283        let procedure = test_procedure(Box::new(RepartitionEnd), test_context(&env, table_id));
1284        assert!(!procedure.should_rollback());
1285    }
1286
1287    #[test]
1288    fn test_event_hook_records_submitted_and_lightweight_lifecycle_events() {
1289        let env = TestingEnv::new();
1290        let procedure = test_procedure(
1291            Box::new(RepartitionStart::new(
1292                RepartitionFrom::Unpartitioned {
1293                    partition_columns: vec!["x".to_string()],
1294                },
1295                vec![range_expr("x", 0, 100)],
1296            )),
1297            test_context(&env, 1024),
1298        );
1299        let state = ProcedureState::Running;
1300        let all = Arc::new(EventTypeFilter::All);
1301
1302        let submitted = procedure
1303            .event(&EventContext {
1304                procedure_id: ProcedureId::random(),
1305                lifecycle_state: &state,
1306                trigger: EventTrigger::Submitted,
1307                event_type_filter: all.clone(),
1308                event_context: None,
1309            })
1310            .unwrap();
1311        assert_eq!(submitted.event_type(), REPARTITION_EVENT_TYPE);
1312        assert_ne!(submitted.json_payload().unwrap(), serde_json::Value::Null);
1313
1314        let allowed = procedure
1315            .event(&EventContext {
1316                procedure_id: ProcedureId::random(),
1317                lifecycle_state: &state,
1318                trigger: EventTrigger::Submitted,
1319                event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::from([
1320                    REPARTITION_EVENT_TYPE.to_string(),
1321                ]))),
1322                event_context: None,
1323            })
1324            .unwrap();
1325        assert_eq!(allowed.event_type(), REPARTITION_EVENT_TYPE);
1326
1327        let succeeded = procedure
1328            .event(&EventContext {
1329                procedure_id: ProcedureId::random(),
1330                lifecycle_state: &state,
1331                trigger: EventTrigger::Succeeded,
1332                event_type_filter: all,
1333                event_context: None,
1334            })
1335            .unwrap();
1336        assert_eq!(succeeded.json_payload().unwrap(), serde_json::Value::Null);
1337
1338        let filtered = procedure.event(&EventContext {
1339            procedure_id: ProcedureId::random(),
1340            lifecycle_state: &state,
1341            trigger: EventTrigger::Submitted,
1342            event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::from([
1343                "another_event".to_string()
1344            ]))),
1345            event_context: None,
1346        });
1347        assert!(filtered.is_none());
1348
1349        let empty = procedure.event(&EventContext {
1350            procedure_id: ProcedureId::random(),
1351            lifecycle_state: &state,
1352            trigger: EventTrigger::Submitted,
1353            event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::new())),
1354            event_context: None,
1355        });
1356        assert!(empty.is_none());
1357    }
1358
1359    #[test]
1360    fn test_register_operating_regions_preserves_route_roles() {
1361        let keeper = Arc::new(MemoryRegionKeeper::new());
1362        let region_routes = vec![
1363            RegionRoute {
1364                region: Region::new_test(RegionId::new(1024, 1)),
1365                leader_peer: Some(Peer::empty(1)),
1366                follower_peers: vec![],
1367                leader_state: None,
1368                leader_down_since: None,
1369                write_route_policy: None,
1370            },
1371            RegionRoute {
1372                region: Region::new_test(RegionId::new(1024, 2)),
1373                leader_peer: Some(Peer::empty(2)),
1374                follower_peers: vec![],
1375                leader_state: Some(LeaderState::Staging),
1376                leader_down_since: None,
1377                write_route_policy: None,
1378            },
1379            RegionRoute {
1380                region: Region::new_test(RegionId::new(1024, 3)),
1381                leader_peer: Some(Peer::empty(3)),
1382                follower_peers: vec![],
1383                leader_state: Some(LeaderState::Downgrading),
1384                leader_down_since: None,
1385                write_route_policy: None,
1386            },
1387        ];
1388
1389        let _guards = Context::register_operating_regions(&keeper, &region_routes).unwrap();
1390
1391        let leader_regions =
1392            keeper.extract_operating_regions(1, &HashSet::from([RegionId::new(1024, 1)]));
1393        let staging_regions =
1394            keeper.extract_operating_regions(2, &HashSet::from([RegionId::new(1024, 2)]));
1395        let downgrading_regions =
1396            keeper.extract_operating_regions(3, &HashSet::from([RegionId::new(1024, 3)]));
1397
1398        assert_eq!(
1399            leader_regions
1400                .get(&RegionId::new(1024, 1))
1401                .map(|region| region.role),
1402            Some(RegionRole::Leader)
1403        );
1404        assert_eq!(
1405            staging_regions
1406                .get(&RegionId::new(1024, 2))
1407                .map(|region| region.role),
1408            Some(RegionRole::StagingLeader)
1409        );
1410        assert_eq!(
1411            downgrading_regions
1412                .get(&RegionId::new(1024, 3))
1413                .map(|region| region.role),
1414            Some(RegionRole::DowngradingLeader)
1415        );
1416    }
1417
1418    #[test]
1419    fn test_persistent_context_partition_metadata_update_serde_default() {
1420        let json = r#"{
1421            "catalog_name":"test_catalog",
1422            "schema_name":"test_schema",
1423            "table_name":"test_table",
1424            "table_id":1024,
1425            "plans":[],
1426            "timeout":"120s"
1427        }"#;
1428
1429        let persistent_ctx: PersistentContext = serde_json::from_str(json).unwrap();
1430
1431        assert!(persistent_ctx.partition_metadata_update.is_none());
1432    }
1433
1434    #[tokio::test]
1435    async fn test_repartition_rollback_removes_partition_metadata_indices() {
1436        let env = TestingEnv::new();
1437        let table_id = 1024;
1438        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1439        env.create_physical_table_metadata_for_repartition(
1440            table_id,
1441            vec![test_region_route(RegionId::new(table_id, 1), "")],
1442            test_region_wal_options(&[1]),
1443        )
1444        .await;
1445
1446        let mut context = new_parent_context(&env, node_manager, table_id);
1447        let current = context.get_raw_table_info_value().await.unwrap();
1448        let mut table_info = current.table_info.clone();
1449        table_info.meta.partition_key_indices = vec![0, 1];
1450        context
1451            .update_table_info(&current, current.update(table_info))
1452            .await
1453            .unwrap();
1454        context.persistent_ctx.partition_metadata_update = Some(
1455            PartitionMetadataUpdate::from_partitioned(vec![1], vec![0, 1]),
1456        );
1457        let mut procedure = RepartitionProcedure {
1458            state: Box::new(UpdatePartitionMetadata::new(vec![])),
1459            context,
1460        };
1461
1462        procedure
1463            .rollback(&TestingEnv::procedure_context())
1464            .await
1465            .unwrap();
1466
1467        assert_eq!(
1468            procedure
1469                .context
1470                .get_table_info_value()
1471                .await
1472                .unwrap()
1473                .table_info
1474                .meta
1475                .partition_key_indices,
1476            vec![1]
1477        );
1478    }
1479
1480    #[tokio::test]
1481    async fn test_repartition_rollback_removes_allocated_routes_from_dispatch() {
1482        let env = TestingEnv::new();
1483        let table_id = 1024;
1484        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1485        let ddl_ctx = env.ddl_context(node_manager);
1486        let original_region_routes = vec![
1487            test_region_route(
1488                RegionId::new(table_id, 1),
1489                &range_expr("x", 0, 100).as_json_str().unwrap(),
1490            ),
1491            test_region_route(
1492                RegionId::new(table_id, 2),
1493                &range_expr("x", 50, 100).as_json_str().unwrap(),
1494            ),
1495            test_region_route(RegionId::new(table_id, 3), ""),
1496        ];
1497        env.create_physical_table_metadata_with_wal_options(
1498            table_id,
1499            original_region_routes,
1500            test_region_wal_options(&[1, 2]),
1501        )
1502        .await;
1503
1504        let mut persistent_ctx = PersistentContext::new(
1505            TableName::new("test_catalog", "test_schema", "test_table"),
1506            table_id,
1507            None,
1508        );
1509        persistent_ctx.plans = vec![with_rollback_metadata(
1510            test_plan(table_id),
1511            vec![
1512                test_region_route(
1513                    RegionId::new(table_id, 1),
1514                    &range_expr("x", 0, 100).as_json_str().unwrap(),
1515                ),
1516                test_region_route(RegionId::new(table_id, 3), ""),
1517            ],
1518        )];
1519        persistent_ctx.failed_procedures = vec![ProcedureMeta {
1520            plan_index: 0,
1521            group_id: Uuid::new_v4(),
1522            procedure_id: ProcedureId::random(),
1523        }];
1524        let context = Context::new(
1525            &ddl_ctx,
1526            env.mailbox_ctx.mailbox().clone(),
1527            env.server_addr.clone(),
1528            persistent_ctx,
1529        );
1530        let mut procedure = RepartitionProcedure {
1531            state: Box::new(Dispatch),
1532            context,
1533        };
1534
1535        procedure
1536            .rollback(&TestingEnv::procedure_context())
1537            .await
1538            .unwrap();
1539
1540        let region_routes = current_parent_region_routes(&procedure.context).await;
1541        assert_eq!(region_routes.len(), 2);
1542        assert_eq!(region_routes[0].region.id, RegionId::new(table_id, 1));
1543        assert_eq!(region_routes[1].region.id, RegionId::new(table_id, 2));
1544    }
1545
1546    #[tokio::test]
1547    async fn test_repartition_rollback_removes_allocated_routes_from_allocate() {
1548        let env = TestingEnv::new();
1549        let table_id = 1024;
1550        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1551        let ddl_ctx = env.ddl_context(node_manager);
1552        let original_region_routes = vec![
1553            test_region_route(
1554                RegionId::new(table_id, 1),
1555                &range_expr("x", 0, 100).as_json_str().unwrap(),
1556            ),
1557            test_region_route(
1558                RegionId::new(table_id, 2),
1559                &range_expr("x", 50, 100).as_json_str().unwrap(),
1560            ),
1561            test_region_route(RegionId::new(table_id, 3), ""),
1562        ];
1563        env.create_physical_table_metadata_with_wal_options(
1564            table_id,
1565            original_region_routes,
1566            test_region_wal_options(&[1, 2]),
1567        )
1568        .await;
1569
1570        let mut persistent_ctx = PersistentContext::new(
1571            TableName::new("test_catalog", "test_schema", "test_table"),
1572            table_id,
1573            None,
1574        );
1575        persistent_ctx.plans = vec![test_plan(table_id)];
1576        let context = Context::new(
1577            &ddl_ctx,
1578            env.mailbox_ctx.mailbox().clone(),
1579            env.server_addr.clone(),
1580            persistent_ctx,
1581        );
1582        let mut procedure = RepartitionProcedure {
1583            state: Box::new(AllocateRegion::new(vec![])),
1584            context,
1585        };
1586
1587        procedure
1588            .rollback(&TestingEnv::procedure_context())
1589            .await
1590            .unwrap();
1591
1592        let region_routes = current_parent_region_routes(&procedure.context).await;
1593        assert_eq!(region_routes.len(), 2);
1594        assert_eq!(region_routes[0].region.id, RegionId::new(table_id, 1));
1595        assert_eq!(region_routes[1].region.id, RegionId::new(table_id, 2));
1596    }
1597
1598    #[tokio::test]
1599    async fn test_repartition_rollback_from_collect_only_removes_failed_allocated_routes() {
1600        let env = TestingEnv::new();
1601        let table_id = 1024;
1602        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1603        let ddl_ctx = env.ddl_context(node_manager);
1604        let original_region_routes = vec![
1605            test_region_route(
1606                RegionId::new(table_id, 1),
1607                &range_expr("x", 0, 100).as_json_str().unwrap(),
1608            ),
1609            test_region_route(
1610                RegionId::new(table_id, 2),
1611                &range_expr("x", 100, 200).as_json_str().unwrap(),
1612            ),
1613            test_region_route(RegionId::new(table_id, 3), ""),
1614            test_region_route(RegionId::new(table_id, 4), ""),
1615        ];
1616        env.create_physical_table_metadata_with_wal_options(
1617            table_id,
1618            original_region_routes,
1619            test_region_wal_options(&[1, 2, 3, 4]),
1620        )
1621        .await;
1622
1623        let mut persistent_ctx = PersistentContext::new(
1624            TableName::new("test_catalog", "test_schema", "test_table"),
1625            table_id,
1626            None,
1627        );
1628        let failed_plan = test_plan(table_id);
1629        let failed_plan = with_rollback_metadata(
1630            failed_plan,
1631            vec![
1632                test_region_route(
1633                    RegionId::new(table_id, 1),
1634                    &range_expr("x", 0, 100).as_json_str().unwrap(),
1635                ),
1636                test_region_route(RegionId::new(table_id, 3), ""),
1637            ],
1638        );
1639        let succeeded_plan = RepartitionPlanEntry {
1640            group_id: Uuid::new_v4(),
1641            source_regions: vec![SourceRegionDescriptor::partitioned(
1642                RegionId::new(table_id, 2),
1643                range_expr("x", 100, 200),
1644            )],
1645            target_regions: vec![
1646                TargetRegionDescriptor {
1647                    region_id: RegionId::new(table_id, 2),
1648                    partition_expr: range_expr("x", 100, 150),
1649                },
1650                TargetRegionDescriptor {
1651                    region_id: RegionId::new(table_id, 4),
1652                    partition_expr: range_expr("x", 150, 200),
1653                },
1654            ],
1655            allocated_region_ids: vec![RegionId::new(table_id, 4)],
1656            pending_deallocate_region_ids: vec![],
1657            transition_map: vec![vec![0]],
1658            original_target_routes: vec![
1659                test_region_route(
1660                    RegionId::new(table_id, 2),
1661                    &range_expr("x", 100, 200).as_json_str().unwrap(),
1662                ),
1663                test_region_route(RegionId::new(table_id, 4), ""),
1664            ],
1665        };
1666        persistent_ctx.plans = vec![failed_plan, succeeded_plan];
1667        persistent_ctx.failed_procedures = vec![ProcedureMeta {
1668            plan_index: 0,
1669            group_id: persistent_ctx.plans[0].group_id,
1670            procedure_id: ProcedureId::random(),
1671        }];
1672
1673        let context = Context::new(
1674            &ddl_ctx,
1675            env.mailbox_ctx.mailbox().clone(),
1676            env.server_addr.clone(),
1677            persistent_ctx,
1678        );
1679        let mut procedure = RepartitionProcedure {
1680            state: Box::new(Collect::new(vec![])),
1681            context,
1682        };
1683
1684        procedure
1685            .rollback(&TestingEnv::procedure_context())
1686            .await
1687            .unwrap();
1688
1689        let region_routes = current_parent_region_routes(&procedure.context).await;
1690        assert_eq!(region_routes.len(), 3);
1691        assert_eq!(region_routes[0].region.id, RegionId::new(table_id, 1));
1692        assert_eq!(region_routes[1].region.id, RegionId::new(table_id, 2));
1693        assert_eq!(region_routes[2].region.id, RegionId::new(table_id, 4));
1694    }
1695
1696    #[tokio::test]
1697    async fn test_repartition_rollback_from_collect_restores_failed_group_metadata_only() {
1698        let env = TestingEnv::new();
1699        let table_id = 1024;
1700        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1701        let ddl_ctx = env.ddl_context(node_manager);
1702        let original_region_routes = vec![
1703            test_region_route(
1704                RegionId::new(table_id, 1),
1705                &range_expr("x", 0, 100).as_json_str().unwrap(),
1706            ),
1707            test_region_route(
1708                RegionId::new(table_id, 2),
1709                &range_expr("x", 100, 200).as_json_str().unwrap(),
1710            ),
1711            test_region_route(RegionId::new(table_id, 3), ""),
1712            test_region_route(RegionId::new(table_id, 4), ""),
1713        ];
1714
1715        let failed_plan = with_rollback_metadata(
1716            test_plan(table_id),
1717            vec![
1718                original_region_routes[0].clone(),
1719                original_region_routes[2].clone(),
1720            ],
1721        );
1722        let succeeded_plan = RepartitionPlanEntry {
1723            group_id: Uuid::new_v4(),
1724            source_regions: vec![SourceRegionDescriptor::partitioned(
1725                RegionId::new(table_id, 2),
1726                range_expr("x", 100, 200),
1727            )],
1728            target_regions: vec![
1729                TargetRegionDescriptor {
1730                    region_id: RegionId::new(table_id, 2),
1731                    partition_expr: range_expr("x", 100, 150),
1732                },
1733                TargetRegionDescriptor {
1734                    region_id: RegionId::new(table_id, 4),
1735                    partition_expr: range_expr("x", 150, 200),
1736                },
1737            ],
1738            allocated_region_ids: vec![RegionId::new(table_id, 4)],
1739            pending_deallocate_region_ids: vec![],
1740            transition_map: vec![vec![0, 1]],
1741            original_target_routes: vec![
1742                original_region_routes[1].clone(),
1743                original_region_routes[3].clone(),
1744            ],
1745        };
1746        let current_region_routes = apply_group_staging(&failed_plan, &original_region_routes);
1747        let current_region_routes = apply_group_staging(&succeeded_plan, &current_region_routes);
1748        let current_region_routes = exit_group_staging(&succeeded_plan, &current_region_routes);
1749        env.create_physical_table_metadata_with_wal_options(
1750            table_id,
1751            current_region_routes,
1752            test_region_wal_options(&[1, 2, 3, 4]),
1753        )
1754        .await;
1755
1756        let mut persistent_ctx = PersistentContext::new(
1757            TableName::new("test_catalog", "test_schema", "test_table"),
1758            table_id,
1759            None,
1760        );
1761        persistent_ctx.plans = vec![failed_plan, succeeded_plan.clone()];
1762        persistent_ctx.failed_procedures = vec![ProcedureMeta {
1763            plan_index: 0,
1764            group_id: persistent_ctx.plans[0].group_id,
1765            procedure_id: ProcedureId::random(),
1766        }];
1767
1768        let context = Context::new(
1769            &ddl_ctx,
1770            env.mailbox_ctx.mailbox().clone(),
1771            env.server_addr.clone(),
1772            persistent_ctx,
1773        );
1774        let mut procedure = RepartitionProcedure {
1775            state: Box::new(Collect::new(vec![])),
1776            context,
1777        };
1778
1779        procedure
1780            .rollback(&TestingEnv::procedure_context())
1781            .await
1782            .unwrap();
1783
1784        assert_eq!(
1785            current_parent_region_routes(&procedure.context).await,
1786            vec![
1787                test_region_route(
1788                    RegionId::new(table_id, 1),
1789                    &range_expr("x", 0, 100).as_json_str().unwrap(),
1790                ),
1791                RegionRoute {
1792                    region: Region {
1793                        id: RegionId::new(table_id, 2),
1794                        partition_expr: range_expr("x", 100, 150).as_json_str().unwrap(),
1795                        ..Default::default()
1796                    },
1797                    leader_peer: Some(Peer::empty(1)),
1798                    ..Default::default()
1799                },
1800                RegionRoute {
1801                    region: Region {
1802                        id: RegionId::new(table_id, 4),
1803                        partition_expr: range_expr("x", 150, 200).as_json_str().unwrap(),
1804                        ..Default::default()
1805                    },
1806                    leader_peer: Some(Peer::empty(1)),
1807                    ..Default::default()
1808                },
1809            ]
1810        );
1811    }
1812
1813    #[tokio::test]
1814    async fn test_repartition_rollback_from_collect_restores_unknown_group_metadata() {
1815        let env = TestingEnv::new();
1816        let table_id = 1024;
1817        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1818        let ddl_ctx = env.ddl_context(node_manager);
1819        let original_region_routes = vec![
1820            test_region_route(
1821                RegionId::new(table_id, 1),
1822                &range_expr("x", 0, 100).as_json_str().unwrap(),
1823            ),
1824            test_region_route(
1825                RegionId::new(table_id, 2),
1826                &range_expr("x", 100, 200).as_json_str().unwrap(),
1827            ),
1828            test_region_route(RegionId::new(table_id, 3), ""),
1829        ];
1830        let plan = with_rollback_metadata(
1831            test_plan(table_id),
1832            vec![
1833                original_region_routes[0].clone(),
1834                original_region_routes[2].clone(),
1835            ],
1836        );
1837        let staged_region_routes = apply_group_staging(&plan, &original_region_routes);
1838        assert_eq!(
1839            region_route_by_id(&staged_region_routes, RegionId::new(table_id, 1))
1840                .region
1841                .partition_expr(),
1842            range_expr("x", 0, 50).as_json_str().unwrap()
1843        );
1844        assert!(
1845            region_route_by_id(&staged_region_routes, RegionId::new(table_id, 1))
1846                .is_leader_staging()
1847        );
1848        assert_eq!(
1849            region_route_by_id(&staged_region_routes, RegionId::new(table_id, 3))
1850                .region
1851                .partition_expr(),
1852            range_expr("x", 50, 100).as_json_str().unwrap()
1853        );
1854        assert!(
1855            region_route_by_id(&staged_region_routes, RegionId::new(table_id, 3))
1856                .is_leader_staging()
1857        );
1858        env.create_physical_table_metadata_with_wal_options(
1859            table_id,
1860            staged_region_routes,
1861            test_region_wal_options(&[1, 2, 3]),
1862        )
1863        .await;
1864
1865        let mut persistent_ctx = PersistentContext::new(
1866            TableName::new("test_catalog", "test_schema", "test_table"),
1867            table_id,
1868            None,
1869        );
1870        persistent_ctx.plans = vec![plan.clone()];
1871        persistent_ctx.unknown_procedures = vec![ProcedureMeta {
1872            plan_index: 0,
1873            group_id: plan.group_id,
1874            procedure_id: ProcedureId::random(),
1875        }];
1876
1877        let context = Context::new(
1878            &ddl_ctx,
1879            env.mailbox_ctx.mailbox().clone(),
1880            env.server_addr.clone(),
1881            persistent_ctx,
1882        );
1883        let mut procedure = RepartitionProcedure {
1884            state: Box::new(Collect::new(vec![])),
1885            context,
1886        };
1887
1888        procedure
1889            .rollback(&TestingEnv::procedure_context())
1890            .await
1891            .unwrap();
1892
1893        assert_eq!(
1894            current_parent_region_routes(&procedure.context).await,
1895            vec![
1896                original_region_routes[0].clone(),
1897                original_region_routes[1].clone()
1898            ]
1899        );
1900    }
1901
1902    #[tokio::test]
1903    async fn test_repartition_rollback_is_idempotent() {
1904        let env = TestingEnv::new();
1905        let table_id = 1024;
1906        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1907        let ddl_ctx = env.ddl_context(node_manager);
1908        let original_region_routes = vec![
1909            test_region_route(
1910                RegionId::new(table_id, 1),
1911                &range_expr("x", 0, 100).as_json_str().unwrap(),
1912            ),
1913            test_region_route(
1914                RegionId::new(table_id, 2),
1915                &range_expr("x", 50, 100).as_json_str().unwrap(),
1916            ),
1917            test_region_route(RegionId::new(table_id, 3), ""),
1918        ];
1919        env.create_physical_table_metadata_with_wal_options(
1920            table_id,
1921            original_region_routes,
1922            test_region_wal_options(&[1, 2]),
1923        )
1924        .await;
1925
1926        let mut persistent_ctx = PersistentContext::new(
1927            TableName::new("test_catalog", "test_schema", "test_table"),
1928            table_id,
1929            None,
1930        );
1931        persistent_ctx.plans = vec![with_rollback_metadata(
1932            test_plan(table_id),
1933            vec![
1934                test_region_route(
1935                    RegionId::new(table_id, 1),
1936                    &range_expr("x", 0, 100).as_json_str().unwrap(),
1937                ),
1938                test_region_route(RegionId::new(table_id, 3), ""),
1939            ],
1940        )];
1941        persistent_ctx.failed_procedures = vec![ProcedureMeta {
1942            plan_index: 0,
1943            group_id: Uuid::new_v4(),
1944            procedure_id: ProcedureId::random(),
1945        }];
1946        let context = Context::new(
1947            &ddl_ctx,
1948            env.mailbox_ctx.mailbox().clone(),
1949            env.server_addr.clone(),
1950            persistent_ctx,
1951        );
1952        let mut procedure = RepartitionProcedure {
1953            state: Box::new(Dispatch),
1954            context,
1955        };
1956
1957        procedure
1958            .rollback(&TestingEnv::procedure_context())
1959            .await
1960            .unwrap();
1961        let once = current_parent_region_routes(&procedure.context).await;
1962
1963        procedure
1964            .rollback(&TestingEnv::procedure_context())
1965            .await
1966            .unwrap();
1967        let twice = current_parent_region_routes(&procedure.context).await;
1968
1969        assert_eq!(once, twice);
1970        assert_eq!(once.len(), 2);
1971        assert_eq!(once[0].region.id, RegionId::new(table_id, 1));
1972        assert_eq!(once[1].region.id, RegionId::new(table_id, 2));
1973    }
1974
1975    #[tokio::test]
1976    async fn test_repartition_rollback_from_collect_restores_failed_merge_group_metadata_only() {
1977        let env = TestingEnv::new();
1978        let table_id = 1024;
1979        let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1980        let ddl_ctx = env.ddl_context(node_manager);
1981        let original_region_routes = vec![
1982            test_region_route(
1983                RegionId::new(table_id, 1),
1984                &range_expr("x", 0, 100).as_json_str().unwrap(),
1985            ),
1986            test_region_route(
1987                RegionId::new(table_id, 2),
1988                &range_expr("x", 100, 200).as_json_str().unwrap(),
1989            ),
1990            test_region_route(
1991                RegionId::new(table_id, 3),
1992                &range_expr("x", 200, 300).as_json_str().unwrap(),
1993            ),
1994            test_region_route(RegionId::new(table_id, 4), ""),
1995        ];
1996        let failed_merge_plan = RepartitionPlanEntry {
1997            group_id: Uuid::new_v4(),
1998            source_regions: vec![
1999                SourceRegionDescriptor::partitioned(
2000                    RegionId::new(table_id, 1),
2001                    range_expr("x", 0, 100),
2002                ),
2003                SourceRegionDescriptor::partitioned(
2004                    RegionId::new(table_id, 2),
2005                    range_expr("x", 100, 200),
2006                ),
2007            ],
2008            target_regions: vec![TargetRegionDescriptor {
2009                region_id: RegionId::new(table_id, 1),
2010                partition_expr: range_expr("x", 0, 200),
2011            }],
2012            allocated_region_ids: vec![],
2013            pending_deallocate_region_ids: vec![RegionId::new(table_id, 2)],
2014            transition_map: vec![vec![0], vec![0]],
2015            original_target_routes: vec![original_region_routes[0].clone()],
2016        };
2017        let succeeded_split_plan = RepartitionPlanEntry {
2018            group_id: Uuid::new_v4(),
2019            source_regions: vec![SourceRegionDescriptor::partitioned(
2020                RegionId::new(table_id, 3),
2021                range_expr("x", 200, 300),
2022            )],
2023            target_regions: vec![
2024                TargetRegionDescriptor {
2025                    region_id: RegionId::new(table_id, 3),
2026                    partition_expr: range_expr("x", 200, 250),
2027                },
2028                TargetRegionDescriptor {
2029                    region_id: RegionId::new(table_id, 4),
2030                    partition_expr: range_expr("x", 250, 300),
2031                },
2032            ],
2033            allocated_region_ids: vec![RegionId::new(table_id, 4)],
2034            pending_deallocate_region_ids: vec![],
2035            transition_map: vec![vec![0, 1]],
2036            original_target_routes: vec![
2037                original_region_routes[2].clone(),
2038                original_region_routes[3].clone(),
2039            ],
2040        };
2041        let current_region_routes =
2042            apply_group_staging(&failed_merge_plan, &original_region_routes);
2043        let current_region_routes =
2044            apply_group_staging(&succeeded_split_plan, &current_region_routes);
2045        let staged_region_routes =
2046            exit_group_staging(&succeeded_split_plan, &current_region_routes);
2047        env.create_physical_table_metadata_with_wal_options(
2048            table_id,
2049            staged_region_routes,
2050            test_region_wal_options(&[1, 2, 3, 4]),
2051        )
2052        .await;
2053
2054        let mut persistent_ctx = PersistentContext::new(
2055            TableName::new("test_catalog", "test_schema", "test_table"),
2056            table_id,
2057            None,
2058        );
2059        persistent_ctx.plans = vec![failed_merge_plan, succeeded_split_plan.clone()];
2060        persistent_ctx.failed_procedures = vec![ProcedureMeta {
2061            plan_index: 0,
2062            group_id: persistent_ctx.plans[0].group_id,
2063            procedure_id: ProcedureId::random(),
2064        }];
2065
2066        let context = Context::new(
2067            &ddl_ctx,
2068            env.mailbox_ctx.mailbox().clone(),
2069            env.server_addr.clone(),
2070            persistent_ctx,
2071        );
2072        let mut procedure = RepartitionProcedure {
2073            state: Box::new(Collect::new(vec![])),
2074            context,
2075        };
2076
2077        procedure
2078            .rollback(&TestingEnv::procedure_context())
2079            .await
2080            .unwrap();
2081
2082        let region_routes = current_parent_region_routes(&procedure.context).await;
2083        assert_eq!(
2084            region_routes,
2085            vec![
2086                test_region_route(
2087                    RegionId::new(table_id, 1),
2088                    &range_expr("x", 0, 100).as_json_str().unwrap(),
2089                ),
2090                test_region_route(
2091                    RegionId::new(table_id, 2),
2092                    &range_expr("x", 100, 200).as_json_str().unwrap(),
2093                ),
2094                RegionRoute {
2095                    region: Region {
2096                        id: RegionId::new(table_id, 3),
2097                        partition_expr: range_expr("x", 200, 250).as_json_str().unwrap(),
2098                        ..Default::default()
2099                    },
2100                    leader_peer: Some(Peer::empty(1)),
2101                    ..Default::default()
2102                },
2103                RegionRoute {
2104                    region: Region {
2105                        id: RegionId::new(table_id, 4),
2106                        partition_expr: range_expr("x", 250, 300).as_json_str().unwrap(),
2107                        ..Default::default()
2108                    },
2109                    leader_peer: Some(Peer::empty(1)),
2110                    ..Default::default()
2111                },
2112            ]
2113        );
2114    }
2115
2116    #[tokio::test]
2117    async fn test_repartition_after_partial_create_failure() {
2118        check_partial_create_failure(false).await;
2119    }
2120
2121    #[tokio::test]
2122    async fn test_repartition_after_pre_upgrade_partial_create_failure() {
2123        check_partial_create_failure(true).await;
2124    }
2125
2126    async fn check_partial_create_failure(pre_upgrade: bool) {
2127        use api::v1::region::region_request::Body;
2128        use common_meta::key::MetadataKey;
2129        use common_meta::key::table_route::TableRouteKey;
2130        use common_meta::rpc::store::PutRequest;
2131
2132        let env = TestingEnv::new();
2133        let table_id = 1024;
2134        let source = range_expr("x", 0, 90);
2135        let route = test_region_route(RegionId::new(table_id, 1), &source.as_json_str().unwrap());
2136        env.create_physical_table_metadata_for_repartition(
2137            table_id,
2138            vec![route.clone()],
2139            test_region_wal_options(&[1]),
2140        )
2141        .await;
2142
2143        let (tx, mut requests) = mpsc::channel(1);
2144        let fail_once = AtomicBool::new(true);
2145        let nodes = Arc::new(MockDatanodeManager::new(
2146            DatanodeWatcher::new(tx).with_handler(move |_, request| {
2147                if let Some(Body::Create(create)) = request.body
2148                    && create.region_id == RegionId::new(table_id, 3).as_u64()
2149                    && fail_once.swap(false, Ordering::SeqCst)
2150                {
2151                    return error::UnexpectedSnafu {
2152                        err_msg: "injected create failure",
2153                    }
2154                    .fail();
2155                }
2156                Ok(api::region::RegionResponse::new(0))
2157            }),
2158        ));
2159        let new_split = || {
2160            RepartitionProcedure::new(
2161                RepartitionFrom::Partitioned {
2162                    exprs: vec![source.clone()],
2163                    target_partition_columns: None,
2164                },
2165                vec![
2166                    range_expr("x", 0, 30),
2167                    range_expr("x", 30, 60),
2168                    range_expr("x", 60, 90),
2169                ],
2170                new_parent_context(&env, nodes.clone(), table_id),
2171            )
2172        };
2173
2174        let ctx = TestingEnv::procedure_context();
2175        let mut first = new_split();
2176        first.execute(&ctx).await.unwrap();
2177        let original = first.context.get_table_route_value().await.unwrap();
2178        first.execute(&ctx).await.unwrap();
2179        let reserved = first.context.get_table_route_value().await.unwrap();
2180        if pre_upgrade {
2181            // Old BuildPlan persisted the same Execute checkpoint without updating the route.
2182            env.kv_backend
2183                .put(
2184                    PutRequest::new()
2185                        .with_key(TableRouteKey::new(table_id).to_bytes())
2186                        .with_value(original.get_raw_bytes()),
2187                )
2188                .await
2189                .unwrap();
2190            first =
2191                RepartitionProcedure::from_json(&first.dump().unwrap(), |persistent_ctx| Context {
2192                    persistent_ctx,
2193                    ..new_parent_context(&env, nodes.clone(), table_id)
2194                })
2195                .unwrap();
2196        }
2197
2198        let observe_create = async {
2199            // The one-slot channel holds the second create until we observe the first.
2200            let (_, request) = requests.recv().await.unwrap();
2201            let Some(Body::Create(create)) = request.body else {
2202                panic!("expected create")
2203            };
2204            let current = new_split().context.get_table_route_value().await.unwrap();
2205            assert_eq!(current.max_region_number().unwrap(), 3);
2206            assert_eq!(current.version().unwrap(), reserved.version().unwrap());
2207            RegionId::from_u64(create.region_id).region_number()
2208        };
2209        let create_timeout = Duration::from_secs(5);
2210        let (result, first_number) = tokio::time::timeout(create_timeout, async {
2211            tokio::join!(first.execute(&ctx), observe_create)
2212        })
2213        .await
2214        .expect("first split timed out waiting for a create request");
2215        assert!(!result.unwrap_err().is_retry_later());
2216        let (_, request) = requests.try_recv().unwrap();
2217        let Some(Body::Create(create)) = request.body else {
2218            panic!("expected create")
2219        };
2220        let mut creates = [
2221            first_number,
2222            RegionId::from_u64(create.region_id).region_number(),
2223        ];
2224        creates.sort_unstable();
2225        assert_eq!(creates, [2, 3]);
2226        first.rollback(&ctx).await.unwrap();
2227        assert_eq!(
2228            current_parent_region_routes(&first.context).await,
2229            vec![route]
2230        );
2231        assert_eq!(
2232            first
2233                .context
2234                .get_table_route_value()
2235                .await
2236                .unwrap()
2237                .max_region_number()
2238                .unwrap(),
2239            3
2240        );
2241
2242        let mut second = new_split();
2243        second.execute(&ctx).await.unwrap();
2244        second.execute(&ctx).await.unwrap();
2245        let (result, _) = tokio::time::timeout(create_timeout, async {
2246            tokio::join!(second.execute(&ctx), requests.recv())
2247        })
2248        .await
2249        .expect("second split timed out waiting for a create request");
2250        result.unwrap();
2251        assert_parent_state::<Dispatch>(&second);
2252        let numbers: Vec<_> = current_parent_region_routes(&second.context)
2253            .await
2254            .iter()
2255            .map(|route| route.region.id.region_number())
2256            .collect();
2257        assert_eq!(numbers, [1, 4, 5]);
2258    }
2259
2260    #[tokio::test]
2261    async fn test_repartition_procedure_flow_split_failed_and_full_rollback() {
2262        let env = TestingEnv::new();
2263        let table_id = 1024;
2264        let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
2265
2266        env.create_physical_table_metadata_for_repartition(
2267            table_id,
2268            vec![
2269                test_region_route(
2270                    RegionId::new(table_id, 1),
2271                    &range_expr("x", 0, 100).as_json_str().unwrap(),
2272                ),
2273                test_region_route(
2274                    RegionId::new(table_id, 2),
2275                    &range_expr("x", 100, 200).as_json_str().unwrap(),
2276                ),
2277            ],
2278            test_region_wal_options(&[1, 2]),
2279        )
2280        .await;
2281
2282        let context = new_parent_context(&env, node_manager, table_id);
2283        let mut procedure = RepartitionProcedure::new(
2284            RepartitionFrom::Partitioned {
2285                exprs: vec![range_expr("x", 0, 100)],
2286                target_partition_columns: None,
2287            },
2288            vec![range_expr("x", 0, 50), range_expr("x", 50, 100)],
2289            context,
2290        );
2291
2292        let start_status = procedure
2293            .execute(&TestingEnv::procedure_context())
2294            .await
2295            .unwrap();
2296        assert!(!start_status.need_persist());
2297        let start_status = procedure
2298            .execute(&TestingEnv::procedure_context())
2299            .await
2300            .unwrap();
2301        assert!(start_status.need_persist());
2302        assert_parent_state::<AllocateRegion>(&procedure);
2303
2304        let allocate_status = procedure
2305            .execute(&TestingEnv::procedure_context())
2306            .await
2307            .unwrap();
2308        assert!(allocate_status.need_persist());
2309        assert_parent_state::<Dispatch>(&procedure);
2310        assert_eq!(procedure.context.persistent_ctx.plans.len(), 1);
2311        let plan = &procedure.context.persistent_ctx.plans[0];
2312        let expected_plan = test_plan(table_id);
2313        assert_eq!(plan.source_regions, expected_plan.source_regions);
2314        assert_eq!(plan.target_regions, expected_plan.target_regions);
2315        assert_eq!(
2316            plan.allocated_region_ids,
2317            expected_plan.allocated_region_ids
2318        );
2319        assert_eq!(
2320            plan.pending_deallocate_region_ids,
2321            expected_plan.pending_deallocate_region_ids
2322        );
2323        assert_eq!(plan.transition_map, expected_plan.transition_map);
2324        assert_eq!(
2325            current_parent_region_routes(&procedure.context).await,
2326            vec![
2327                test_region_route(
2328                    RegionId::new(table_id, 1),
2329                    &range_expr("x", 0, 100).as_json_str().unwrap(),
2330                ),
2331                test_region_route(
2332                    RegionId::new(table_id, 2),
2333                    &range_expr("x", 100, 200).as_json_str().unwrap(),
2334                ),
2335                RegionRoute {
2336                    region: Region {
2337                        id: RegionId::new(table_id, 3),
2338                        partition_expr: range_expr("x", 50, 100).as_json_str().unwrap(),
2339                        ..Default::default()
2340                    },
2341                    leader_peer: Some(Peer::empty(0)),
2342                    ..Default::default()
2343                },
2344            ]
2345        );
2346
2347        let dispatch_status = procedure
2348            .execute(&TestingEnv::procedure_context())
2349            .await
2350            .unwrap();
2351        assert!(!dispatch_status.need_persist());
2352        let subprocedure_ids = extract_subprocedure_ids(dispatch_status);
2353        assert_eq!(subprocedure_ids.len(), 1);
2354        assert_parent_state::<Collect>(&procedure);
2355
2356        let failed_state = ProcedureState::failed(Arc::new(ProcedureError::external(
2357            MockError::new(StatusCode::Internal),
2358        )));
2359        let collect_ctx = procedure_context_with_receivers(HashMap::from([(
2360            subprocedure_ids[0],
2361            procedure_state_receiver(failed_state),
2362        )]));
2363
2364        let err = procedure.execute(&collect_ctx).await.unwrap_err();
2365        assert!(!err.is_retry_later());
2366        assert_parent_state::<Collect>(&procedure);
2367
2368        procedure
2369            .rollback(&TestingEnv::procedure_context())
2370            .await
2371            .unwrap();
2372
2373        let region_routes = current_parent_region_routes(&procedure.context).await;
2374        assert_eq!(
2375            region_routes,
2376            vec![
2377                test_region_route(
2378                    RegionId::new(table_id, 1),
2379                    &range_expr("x", 0, 100).as_json_str().unwrap(),
2380                ),
2381                test_region_route(
2382                    RegionId::new(table_id, 2),
2383                    &range_expr("x", 100, 200).as_json_str().unwrap(),
2384                ),
2385            ]
2386        );
2387    }
2388
2389    #[tokio::test]
2390    async fn test_repartition_procedure_flow_unpartitioned_failed_and_full_rollback() {
2391        let env = TestingEnv::new();
2392        let table_id = 1024;
2393        let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
2394
2395        env.create_physical_table_metadata_for_repartition(
2396            table_id,
2397            vec![test_region_route(RegionId::new(table_id, 1), "")],
2398            test_region_wal_options(&[1]),
2399        )
2400        .await;
2401
2402        let context = new_parent_context(&env, node_manager, table_id);
2403        let to_exprs = vec![range_expr("col1", 0, 50), range_expr("col1", 50, 100)];
2404        let mut procedure = RepartitionProcedure::new(
2405            RepartitionFrom::Unpartitioned {
2406                partition_columns: vec!["col1".to_string()],
2407            },
2408            to_exprs.clone(),
2409            context,
2410        );
2411
2412        let start_status = procedure
2413            .execute(&TestingEnv::procedure_context())
2414            .await
2415            .unwrap();
2416        assert!(start_status.need_persist());
2417        assert_parent_state::<UpdatePartitionMetadata>(&procedure);
2418        assert_eq!(
2419            procedure
2420                .context
2421                .persistent_ctx
2422                .partition_metadata_update
2423                .as_ref()
2424                .unwrap()
2425                .target_partition_key_indices,
2426            vec![0]
2427        );
2428
2429        let update_status = procedure
2430            .execute(&TestingEnv::procedure_context())
2431            .await
2432            .unwrap();
2433        assert!(update_status.need_persist());
2434        assert_parent_state::<AllocateRegion>(&procedure);
2435        assert_eq!(
2436            table_partition_key_indices(&procedure.context).await,
2437            vec![0]
2438        );
2439
2440        let build_allocate_status = procedure
2441            .execute(&TestingEnv::procedure_context())
2442            .await
2443            .unwrap();
2444        assert!(build_allocate_status.need_persist());
2445        assert_parent_state::<AllocateRegion>(&procedure);
2446        assert_eq!(procedure.context.persistent_ctx.plans.len(), 1);
2447        let plan = &procedure.context.persistent_ctx.plans[0];
2448        assert_eq!(
2449            plan.source_regions,
2450            vec![SourceRegionDescriptor::Default {
2451                region_id: RegionId::new(table_id, 1)
2452            }]
2453        );
2454        assert_eq!(plan.target_regions.len(), 2);
2455        assert_eq!(plan.target_regions[0].region_id, RegionId::new(table_id, 1));
2456        assert_eq!(plan.target_regions[0].partition_expr, to_exprs[0]);
2457        assert_eq!(
2458            plan.allocated_region_ids,
2459            vec![plan.target_regions[1].region_id]
2460        );
2461        assert!(plan.pending_deallocate_region_ids.is_empty());
2462        assert_eq!(plan.transition_map, vec![vec![0, 1]]);
2463        let target_regions = plan.target_regions.clone();
2464
2465        let execute_allocate_status = procedure
2466            .execute(&TestingEnv::procedure_context())
2467            .await
2468            .unwrap();
2469        assert!(execute_allocate_status.need_persist());
2470        assert_parent_state::<Dispatch>(&procedure);
2471        let region_routes = current_parent_region_routes(&procedure.context).await;
2472        assert_eq!(region_routes.len(), 2);
2473        assert_eq!(
2474            region_route_by_id(&region_routes, target_regions[0].region_id)
2475                .region
2476                .partition_expr(),
2477            ""
2478        );
2479        assert_eq!(
2480            region_route_by_id(&region_routes, target_regions[1].region_id)
2481                .region
2482                .partition_expr(),
2483            to_exprs[1].as_json_str().unwrap()
2484        );
2485
2486        let dispatch_status = procedure
2487            .execute(&TestingEnv::procedure_context())
2488            .await
2489            .unwrap();
2490        let subprocedure_ids = extract_subprocedure_ids(dispatch_status);
2491        assert_eq!(subprocedure_ids.len(), 1);
2492        assert_parent_state::<Collect>(&procedure);
2493
2494        let failed_state = ProcedureState::failed(Arc::new(ProcedureError::external(
2495            MockError::new(StatusCode::Internal),
2496        )));
2497        let collect_ctx = procedure_context_with_receivers(HashMap::from([(
2498            subprocedure_ids[0],
2499            procedure_state_receiver(failed_state),
2500        )]));
2501        let err = procedure.execute(&collect_ctx).await.unwrap_err();
2502        assert!(!err.is_retry_later());
2503        assert_parent_state::<Collect>(&procedure);
2504
2505        procedure
2506            .rollback(&TestingEnv::procedure_context())
2507            .await
2508            .unwrap();
2509
2510        assert!(
2511            table_partition_key_indices(&procedure.context)
2512                .await
2513                .is_empty()
2514        );
2515        assert_eq!(
2516            current_parent_region_routes(&procedure.context).await,
2517            vec![test_region_route(RegionId::new(table_id, 1), "")]
2518        );
2519    }
2520
2521    #[tokio::test]
2522    async fn test_repartition_procedure_flow_unpartitioned_rollback_is_idempotent() {
2523        let env = TestingEnv::new();
2524        let table_id = 1024;
2525        let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
2526
2527        env.create_physical_table_metadata_for_repartition(
2528            table_id,
2529            vec![test_region_route(RegionId::new(table_id, 1), "")],
2530            test_region_wal_options(&[1]),
2531        )
2532        .await;
2533
2534        let context = new_parent_context(&env, node_manager, table_id);
2535        let mut procedure = RepartitionProcedure::new(
2536            RepartitionFrom::Unpartitioned {
2537                partition_columns: vec!["col1".to_string()],
2538            },
2539            vec![range_expr("col1", 0, 50), range_expr("col1", 50, 100)],
2540            context,
2541        );
2542
2543        procedure
2544            .execute(&TestingEnv::procedure_context())
2545            .await
2546            .unwrap();
2547        procedure
2548            .execute(&TestingEnv::procedure_context())
2549            .await
2550            .unwrap();
2551        procedure
2552            .execute(&TestingEnv::procedure_context())
2553            .await
2554            .unwrap();
2555        procedure
2556            .execute(&TestingEnv::procedure_context())
2557            .await
2558            .unwrap();
2559        assert_eq!(
2560            table_partition_key_indices(&procedure.context).await,
2561            vec![0]
2562        );
2563        assert_eq!(
2564            current_parent_region_routes(&procedure.context).await.len(),
2565            2
2566        );
2567
2568        let dispatch_status = procedure
2569            .execute(&TestingEnv::procedure_context())
2570            .await
2571            .unwrap();
2572        let subprocedure_ids = extract_subprocedure_ids(dispatch_status);
2573        assert_eq!(subprocedure_ids.len(), 1);
2574        assert_parent_state::<Collect>(&procedure);
2575
2576        let failed_state = ProcedureState::failed(Arc::new(ProcedureError::external(
2577            MockError::new(StatusCode::Internal),
2578        )));
2579        let collect_ctx = procedure_context_with_receivers(HashMap::from([(
2580            subprocedure_ids[0],
2581            procedure_state_receiver(failed_state),
2582        )]));
2583        let err = procedure.execute(&collect_ctx).await.unwrap_err();
2584        assert!(!err.is_retry_later());
2585
2586        procedure
2587            .rollback(&TestingEnv::procedure_context())
2588            .await
2589            .unwrap();
2590        let once_indices = table_partition_key_indices(&procedure.context).await;
2591        let once_routes = current_parent_region_routes(&procedure.context).await;
2592
2593        procedure
2594            .rollback(&TestingEnv::procedure_context())
2595            .await
2596            .unwrap();
2597        let twice_indices = table_partition_key_indices(&procedure.context).await;
2598        let twice_routes = current_parent_region_routes(&procedure.context).await;
2599
2600        assert_eq!(once_indices, twice_indices);
2601        assert_eq!(once_routes, twice_routes);
2602        assert!(twice_indices.is_empty());
2603        assert_eq!(
2604            twice_routes,
2605            vec![test_region_route(RegionId::new(table_id, 1), "")]
2606        );
2607    }
2608
2609    #[tokio::test]
2610    async fn test_repartition_procedure_flow_split_allocate_retryable_then_resume() {
2611        common_telemetry::init_default_ut_logging();
2612        let env = TestingEnv::new();
2613        let table_id = 1024;
2614        let (tx, _rx) = mpsc::channel(8);
2615        let should_retry = Arc::new(AtomicBool::new(true));
2616        let datanode_handler = DatanodeWatcher::new(tx).with_handler(move |_, _| {
2617            if should_retry.swap(false, Ordering::SeqCst) {
2618                return Err(error::Error::RetryLater {
2619                    source: BoxedError::new(
2620                        error::UnexpectedSnafu {
2621                            err_msg: "retry later",
2622                        }
2623                        .build(),
2624                    ),
2625                    clean_poisons: false,
2626                });
2627            }
2628
2629            Ok(api::region::RegionResponse::new(0))
2630        });
2631        let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
2632
2633        env.create_physical_table_metadata_for_repartition(
2634            table_id,
2635            vec![
2636                test_region_route(
2637                    RegionId::new(table_id, 1),
2638                    &range_expr("x", 0, 100).as_json_str().unwrap(),
2639                ),
2640                test_region_route(
2641                    RegionId::new(table_id, 2),
2642                    &range_expr("x", 100, 200).as_json_str().unwrap(),
2643                ),
2644            ],
2645            test_region_wal_options(&[1, 2]),
2646        )
2647        .await;
2648
2649        let context = new_parent_context(&env, node_manager, table_id);
2650        let mut procedure = RepartitionProcedure::new(
2651            RepartitionFrom::Partitioned {
2652                exprs: vec![range_expr("x", 0, 100)],
2653                target_partition_columns: None,
2654            },
2655            vec![range_expr("x", 0, 50), range_expr("x", 50, 100)],
2656            context,
2657        );
2658
2659        let start_status = procedure
2660            .execute(&TestingEnv::procedure_context())
2661            .await
2662            .unwrap();
2663        assert!(!start_status.need_persist());
2664        let start_status = procedure
2665            .execute(&TestingEnv::procedure_context())
2666            .await
2667            .unwrap();
2668        assert!(start_status.need_persist());
2669        assert_parent_state::<AllocateRegion>(&procedure);
2670
2671        let err = procedure
2672            .execute(&TestingEnv::procedure_context())
2673            .await
2674            .unwrap_err();
2675        assert!(err.is_retry_later());
2676        assert_parent_state::<AllocateRegion>(&procedure);
2677        assert!(!procedure.context.persistent_ctx.plans.is_empty());
2678        assert_eq!(
2679            current_parent_region_routes(&procedure.context).await,
2680            vec![
2681                test_region_route(
2682                    RegionId::new(table_id, 1),
2683                    &range_expr("x", 0, 100).as_json_str().unwrap(),
2684                ),
2685                test_region_route(
2686                    RegionId::new(table_id, 2),
2687                    &range_expr("x", 100, 200).as_json_str().unwrap(),
2688                ),
2689            ]
2690        );
2691
2692        let allocate_status = procedure
2693            .execute(&TestingEnv::procedure_context())
2694            .await
2695            .unwrap();
2696        assert!(allocate_status.need_persist());
2697        assert_parent_state::<Dispatch>(&procedure);
2698
2699        assert_eq!(procedure.context.persistent_ctx.plans.len(), 1);
2700        let plan = &procedure.context.persistent_ctx.plans[0];
2701        let expected_plan = test_plan(table_id);
2702        assert_eq!(plan.source_regions, expected_plan.source_regions);
2703        assert_eq!(plan.target_regions, expected_plan.target_regions);
2704        assert_eq!(
2705            plan.allocated_region_ids,
2706            expected_plan.allocated_region_ids
2707        );
2708        assert_eq!(plan.transition_map, expected_plan.transition_map);
2709        assert_eq!(
2710            current_parent_region_routes(&procedure.context).await,
2711            vec![
2712                test_region_route(
2713                    RegionId::new(table_id, 1),
2714                    &range_expr("x", 0, 100).as_json_str().unwrap(),
2715                ),
2716                test_region_route(
2717                    RegionId::new(table_id, 2),
2718                    &range_expr("x", 100, 200).as_json_str().unwrap(),
2719                ),
2720                RegionRoute {
2721                    region: Region {
2722                        id: RegionId::new(table_id, 3),
2723                        partition_expr: range_expr("x", 50, 100).as_json_str().unwrap(),
2724                        ..Default::default()
2725                    },
2726                    leader_peer: Some(Peer::empty(0)),
2727                    ..Default::default()
2728                },
2729            ]
2730        );
2731
2732        let dispatch_status = procedure
2733            .execute(&TestingEnv::procedure_context())
2734            .await
2735            .unwrap();
2736        assert!(!dispatch_status.need_persist());
2737        let subprocedure_ids = extract_subprocedure_ids(dispatch_status);
2738        assert_eq!(subprocedure_ids.len(), 1);
2739        assert_parent_state::<Collect>(&procedure);
2740    }
2741}