1pub mod allocate_region;
16pub mod collect;
17pub mod deallocate_region;
18pub mod dispatch;
19pub mod gc_requirement;
20pub mod group;
21pub mod plan;
22pub mod repartition_end;
23pub mod repartition_start;
24pub mod update_partition_metadata;
25pub mod utils;
26
27use std::any::Any;
28use std::collections::{HashMap, HashSet};
29use std::fmt::{Debug, Display};
30use std::time::{Duration, Instant};
31
32use common_error::ext::BoxedError;
33use common_meta::affinity::{AffinityFallback, AffinityResolverRef};
34use common_meta::cache_invalidator::CacheInvalidatorRef;
35use common_meta::ddl::DdlContext;
36use common_meta::ddl::allocator::region_routes::RegionRoutesAllocatorRef;
37use common_meta::ddl::allocator::wal_options::WalOptionsAllocatorRef;
38use common_meta::ddl::utils::get_region_wal_options;
39use common_meta::ddl_manager::{RepartitionProcedureFactory, RepartitionSource};
40use common_meta::instruction::CacheIdent;
41use common_meta::key::datanode_table::RegionInfo;
42use common_meta::key::table_info::TableInfoValue;
43use common_meta::key::table_route::TableRouteValue;
44use common_meta::key::{DeserializedValueWithBytes, TableMetadataManagerRef};
45use common_meta::lock_key::{CatalogLock, SchemaLock, TableLock, TableNameLock};
46use common_meta::node_manager::NodeManagerRef;
47use common_meta::region_keeper::{MemoryRegionKeeperRef, OperatingRegionGuard};
48use common_meta::region_registry::LeaderRegionRegistryRef;
49use common_meta::rpc::router::{RegionRoute, operating_leader_region_roles};
50use common_meta::wal_provider::RegionWalOptions;
51use common_procedure::error::{FromJsonSnafu, ToJsonSnafu};
52use common_procedure::{
53 BoxedProcedure, Context as ProcedureContext, Error as ProcedureError, EventContext,
54 EventTrigger, LockKey, Procedure, ProcedureManagerRef, Result as ProcedureResult, Status,
55 StringKey,
56};
57use common_telemetry::{error, info, warn};
58use partition::expr::PartitionExpr;
59use serde::{Deserialize, Serialize};
60use snafu::{OptionExt, ResultExt};
61use store_api::storage::{RegionNumber, TableId};
62use table::table_name::TableName;
63
64use crate::error::{self, Result};
65use crate::event::repartition::{REPARTITION_EVENT_TYPE, RepartitionEvent};
66use crate::procedure::repartition::collect::ProcedureMeta;
67use crate::procedure::repartition::deallocate_region::DeallocateRegion;
68use crate::procedure::repartition::gc_requirement::RepartitionGcRequirementManagerRef;
69use crate::procedure::repartition::group::{
70 Context as RepartitionGroupContext, RepartitionGroupProcedure, region_routes,
71};
72use crate::procedure::repartition::plan::RepartitionPlanEntry;
73use crate::procedure::repartition::repartition_start::{RepartitionFrom, RepartitionStart};
74use crate::procedure::repartition::update_partition_metadata::PartitionMetadataUpdate;
75use crate::procedure::repartition::utils::{
76 get_datanode_table_value, rollback_group_metadata_routes,
77};
78use crate::service::mailbox::MailboxRef;
79
80#[cfg(test)]
81pub mod test_util;
82
83#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
84pub struct PersistentContext {
85 pub catalog_name: String,
86 pub schema_name: String,
87 pub table_name: String,
88 pub table_id: TableId,
89 pub plans: Vec<RepartitionPlanEntry>,
90 #[serde(default)]
91 pub failed_procedures: Vec<ProcedureMeta>,
96 #[serde(default)]
97 pub unknown_procedures: Vec<ProcedureMeta>,
102 #[serde(with = "humantime_serde", default = "default_timeout")]
104 pub timeout: Duration,
105 #[serde(default)]
106 pub partition_metadata_update: Option<PartitionMetadataUpdate>,
108}
109
110fn default_timeout() -> Duration {
111 Duration::from_secs(2 * 60)
112}
113
114impl PersistentContext {
115 pub fn new(
119 TableName {
120 catalog_name,
121 schema_name,
122 table_name,
123 }: TableName,
124 table_id: TableId,
125 timeout: Option<Duration>,
126 ) -> Self {
127 Self {
128 catalog_name,
129 schema_name,
130 table_name,
131 table_id,
132 plans: vec![],
133 failed_procedures: vec![],
134 unknown_procedures: vec![],
135 timeout: timeout.unwrap_or_else(default_timeout),
136 partition_metadata_update: None,
137 }
138 }
139
140 pub fn lock_key(&self) -> Vec<StringKey> {
141 vec![
142 CatalogLock::Read(&self.catalog_name).into(),
143 SchemaLock::read(&self.catalog_name, &self.schema_name).into(),
144 TableLock::Write(self.table_id).into(),
145 TableNameLock::new(&self.catalog_name, &self.schema_name, &self.table_name).into(),
146 ]
147 }
148}
149
150#[derive(Clone)]
151pub struct Context {
152 pub affinity_resolver: Option<AffinityResolverRef>,
153 pub affinity_fallback: AffinityFallback,
154 pub persistent_ctx: PersistentContext,
155 pub volatile_ctx: VolatileContext,
156 pub table_metadata_manager: TableMetadataManagerRef,
157 pub memory_region_keeper: MemoryRegionKeeperRef,
158 pub node_manager: NodeManagerRef,
159 pub leader_region_registry: LeaderRegionRegistryRef,
160 pub mailbox: MailboxRef,
161 pub server_addr: String,
162 pub cache_invalidator: CacheInvalidatorRef,
163 pub region_routes_allocator: RegionRoutesAllocatorRef,
164 pub wal_options_allocator: WalOptionsAllocatorRef,
165 pub start_time: Instant,
166}
167
168#[derive(Debug, Clone, Default)]
169pub struct VolatileContext {
170 pub metrics: Metrics,
171 pub dispatch_start_time: Option<Instant>,
172}
173
174#[derive(Debug, Clone, Default)]
176pub struct Metrics {
177 build_plan_elapsed: Duration,
179 allocate_region_elapsed: Duration,
181 finish_groups_elapsed: Duration,
183 deallocate_region_elapsed: Duration,
185}
186
187impl Display for Metrics {
188 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
189 let total = self.build_plan_elapsed
190 + self.allocate_region_elapsed
191 + self.finish_groups_elapsed
192 + self.deallocate_region_elapsed;
193 write!(f, "total: {:?}", total)?;
194 let mut parts = Vec::with_capacity(4);
195 if self.build_plan_elapsed > Duration::ZERO {
196 parts.push(format!("build_plan_elapsed: {:?}", self.build_plan_elapsed));
197 }
198 if self.allocate_region_elapsed > Duration::ZERO {
199 parts.push(format!(
200 "allocate_region_elapsed: {:?}",
201 self.allocate_region_elapsed
202 ));
203 }
204 if self.finish_groups_elapsed > Duration::ZERO {
205 parts.push(format!(
206 "finish_groups_elapsed: {:?}",
207 self.finish_groups_elapsed
208 ));
209 }
210 if self.deallocate_region_elapsed > Duration::ZERO {
211 parts.push(format!(
212 "deallocate_region_elapsed: {:?}",
213 self.deallocate_region_elapsed
214 ));
215 }
216
217 if !parts.is_empty() {
218 write!(f, ", {}", parts.join(", "))?;
219 }
220 Ok(())
221 }
222}
223
224impl Metrics {
225 pub fn update_build_plan_elapsed(&mut self, elapsed: Duration) {
227 self.build_plan_elapsed += elapsed;
228 }
229
230 pub fn update_allocate_region_elapsed(&mut self, elapsed: Duration) {
232 self.allocate_region_elapsed += elapsed;
233 }
234
235 pub fn update_finish_groups_elapsed(&mut self, elapsed: Duration) {
237 self.finish_groups_elapsed += elapsed;
238 }
239
240 pub fn update_deallocate_region_elapsed(&mut self, elapsed: Duration) {
242 self.deallocate_region_elapsed += elapsed;
243 }
244}
245
246impl Context {
247 pub fn new(
248 ddl_ctx: &DdlContext,
249 mailbox: MailboxRef,
250 server_addr: String,
251 persistent_ctx: PersistentContext,
252 ) -> Self {
253 Self {
254 affinity_resolver: ddl_ctx.affinity_resolver.clone(),
255 affinity_fallback: ddl_ctx.affinity_fallback,
256 persistent_ctx,
257 table_metadata_manager: ddl_ctx.table_metadata_manager.clone(),
258 memory_region_keeper: ddl_ctx.memory_region_keeper.clone(),
259 node_manager: ddl_ctx.node_manager.clone(),
260 leader_region_registry: ddl_ctx.leader_region_registry.clone(),
261 mailbox,
262 server_addr,
263 cache_invalidator: ddl_ctx.cache_invalidator.clone(),
264 region_routes_allocator: ddl_ctx.table_metadata_allocator.region_routes_allocator(),
265 wal_options_allocator: ddl_ctx.table_metadata_allocator.wal_options_allocator(),
266 start_time: Instant::now(),
267 volatile_ctx: VolatileContext::default(),
268 }
269 }
270
271 pub fn next_operation_timeout(&self) -> Option<Duration> {
273 self.persistent_ctx
274 .timeout
275 .checked_sub(self.start_time.elapsed())
276 }
277
278 pub fn update_build_plan_elapsed(&mut self, elapsed: Duration) {
280 self.volatile_ctx.metrics.update_build_plan_elapsed(elapsed);
281 }
282
283 pub fn update_allocate_region_elapsed(&mut self, elapsed: Duration) {
285 self.volatile_ctx
286 .metrics
287 .update_allocate_region_elapsed(elapsed);
288 }
289
290 pub fn update_finish_groups_elapsed(&mut self, elapsed: Duration) {
292 self.volatile_ctx
293 .metrics
294 .update_finish_groups_elapsed(elapsed);
295 }
296
297 pub fn update_deallocate_region_elapsed(&mut self, elapsed: Duration) {
299 self.volatile_ctx
300 .metrics
301 .update_deallocate_region_elapsed(elapsed);
302 }
303
304 pub async fn get_table_route_value(
312 &self,
313 ) -> Result<DeserializedValueWithBytes<TableRouteValue>> {
314 let table_id = self.persistent_ctx.table_id;
315 let table_route_value = self
316 .table_metadata_manager
317 .table_route_manager()
318 .table_route_storage()
319 .get_with_raw_bytes(table_id)
320 .await
321 .map_err(BoxedError::new)
322 .with_context(|_| error::RetryLaterWithSourceSnafu {
323 reason: format!("Failed to get table route for table: {}", table_id),
324 })?
325 .context(error::TableRouteNotFoundSnafu { table_id })?;
326
327 Ok(table_route_value)
328 }
329
330 async fn reserve_region_numbers(
331 &self,
332 current: &DeserializedValueWithBytes<TableRouteValue>,
333 count: usize,
334 ) -> Result<RegionNumber> {
335 let table_id = self.persistent_ctx.table_id;
336 self.table_metadata_manager
337 .table_route_manager()
338 .try_reserve_region_numbers(table_id, current, count)
339 .await
340 .context(error::TableMetadataManagerSnafu)?
341 .with_context(|| error::RetryLaterSnafu {
342 reason: format!(
343 "Table route changed while reserving region numbers for table {table_id}"
344 ),
345 })
346 }
347
348 pub async fn get_raw_table_info_value(
356 &self,
357 ) -> Result<DeserializedValueWithBytes<TableInfoValue>> {
358 let table_id = self.persistent_ctx.table_id;
359 let table_info_value = self
360 .table_metadata_manager
361 .table_info_manager()
362 .get(table_id)
363 .await
364 .map_err(BoxedError::new)
365 .with_context(|_| error::RetryLaterWithSourceSnafu {
366 reason: format!("Failed to get table info for table: {}", table_id),
367 })?
368 .context(error::TableInfoNotFoundSnafu { table_id })?;
369
370 Ok(table_info_value)
371 }
372
373 pub async fn get_table_info_value(&self) -> Result<TableInfoValue> {
374 let table_info_value = self.get_raw_table_info_value().await?.into_inner();
375 Ok(table_info_value)
376 }
377
378 pub async fn update_table_info(
380 &self,
381 current_table_info_value: &DeserializedValueWithBytes<TableInfoValue>,
382 new_table_info_value: TableInfoValue,
383 ) -> Result<()> {
384 let table_id = self.persistent_ctx.table_id;
385 self.table_metadata_manager
386 .update_table_info(
387 current_table_info_value,
388 None,
389 new_table_info_value.table_info,
390 )
391 .await
392 .map_err(BoxedError::new)
393 .with_context(|_| error::RetryLaterWithSourceSnafu {
394 reason: format!("Failed to update table info for table: {}", table_id),
395 })
396 }
397
398 pub async fn update_table_route(
407 &self,
408 current_table_route_value: &DeserializedValueWithBytes<TableRouteValue>,
409 new_region_routes: Vec<RegionRoute>,
410 new_region_wal_options: RegionWalOptions,
411 ) -> Result<()> {
412 let table_id = self.persistent_ctx.table_id;
413 if new_region_routes.is_empty() {
414 return error::UnexpectedSnafu {
415 violated: format!("new_region_routes is empty for table: {}", table_id),
416 }
417 .fail();
418 }
419 let datanode_id = new_region_routes
420 .first()
421 .unwrap()
422 .leader_peer
423 .as_ref()
424 .context(error::NoLeaderSnafu)?
425 .id;
426 let datanode_table_value =
427 get_datanode_table_value(&self.table_metadata_manager, table_id, datanode_id).await?;
428
429 let RegionInfo { region_options, .. } = &datanode_table_value.region_info;
430
431 let mut region_wal_options = get_region_wal_options(
432 &self.table_metadata_manager,
433 current_table_route_value,
434 table_id,
435 )
436 .await
437 .context(error::TableMetadataManagerSnafu)?;
438 for route in current_table_route_value.region_routes().unwrap() {
441 region_wal_options
442 .entry(route.region.id.region_number())
443 .or_default();
444 }
445
446 let validated_region_wal_options =
448 crate::procedure::repartition::utils::merge_and_validate_region_wal_options(
449 ®ion_wal_options,
450 new_region_wal_options,
451 &new_region_routes,
452 table_id,
453 )?;
454 info!(
455 "Updating table route for table: {}, new region routes: {:?}",
456 table_id, new_region_routes
457 );
458 self.table_metadata_manager
459 .update_table_route(
460 table_id,
461 datanode_table_value.region_info.clone(),
462 current_table_route_value,
463 new_region_routes,
464 region_options,
465 &validated_region_wal_options,
466 )
467 .await
468 .context(error::TableMetadataManagerSnafu)
469 }
470
471 pub async fn invalidate_table_cache(&self) -> Result<()> {
473 let table_id = self.persistent_ctx.table_id;
474 let subject = format!(
475 "Invalidate table cache for repartition table, table: {}",
476 table_id,
477 );
478 let ctx = common_meta::cache_invalidator::Context {
479 subject: Some(subject),
480 };
481 let _ = self
482 .cache_invalidator
483 .invalidate(
484 &ctx,
485 &[
486 CacheIdent::TableId(table_id),
487 CacheIdent::TableName(TableName {
488 catalog_name: self.persistent_ctx.catalog_name.clone(),
489 schema_name: self.persistent_ctx.schema_name.clone(),
490 table_name: self.persistent_ctx.table_name.clone(),
491 }),
492 ],
493 )
494 .await;
495 Ok(())
496 }
497
498 pub fn register_operating_regions(
499 memory_region_keeper: &MemoryRegionKeeperRef,
500 region_routes: &[RegionRoute],
501 ) -> Result<Vec<OperatingRegionGuard>> {
502 let mut operating_guards = Vec::with_capacity(region_routes.len());
503 for (region_id, datanode_id, role) in operating_leader_region_roles(region_routes) {
504 let guard = memory_region_keeper
505 .register_with_role(datanode_id, region_id, role)
506 .context(error::RegionOperatingRaceSnafu {
507 peer_id: datanode_id,
508 region_id,
509 })?;
510 operating_guards.push(guard);
511 }
512 Ok(operating_guards)
513 }
514}
515
516#[async_trait::async_trait]
517#[typetag::serde(tag = "repartition_state")]
518pub(crate) trait State: Sync + Send + Debug {
519 fn name(&self) -> &'static str {
520 let type_name = std::any::type_name::<Self>();
521 type_name.split("::").last().unwrap_or(type_name)
523 }
524
525 async fn next(
527 &mut self,
528 ctx: &mut Context,
529 procedure_ctx: &ProcedureContext,
530 ) -> Result<(Box<dyn State>, Status)>;
531
532 fn as_any(&self) -> &dyn Any;
533}
534
535pub struct RepartitionProcedure {
536 state: Box<dyn State>,
537 context: Context,
538}
539
540#[derive(Debug, Serialize)]
541struct RepartitionData<'a> {
542 state: &'a dyn State,
543 persistent_ctx: &'a PersistentContext,
544}
545
546#[derive(Debug, Deserialize)]
547struct RepartitionDataOwned {
548 state: Box<dyn State>,
549 persistent_ctx: PersistentContext,
550}
551
552impl RepartitionProcedure {
553 const TYPE_NAME: &'static str = "metasrv-procedure::Repartition";
554
555 pub fn new(from: RepartitionFrom, to_exprs: Vec<PartitionExpr>, context: Context) -> Self {
556 let state = Box::new(RepartitionStart::new(from, to_exprs));
557
558 Self { state, context }
559 }
560
561 pub fn from_json<F>(json: &str, ctx_factory: F) -> ProcedureResult<Self>
562 where
563 F: FnOnce(PersistentContext) -> Context,
564 {
565 let RepartitionDataOwned {
566 state,
567 persistent_ctx,
568 } = serde_json::from_str(json).context(FromJsonSnafu)?;
569 let context = ctx_factory(persistent_ctx);
570
571 Ok(Self { state, context })
572 }
573
574 fn should_rollback(&self) -> bool {
588 self.state
589 .as_any()
590 .is::<update_partition_metadata::UpdatePartitionMetadata>()
591 || self.state.as_any().is::<allocate_region::AllocateRegion>()
592 || self.state.as_any().is::<dispatch::Dispatch>()
593 || self.state.as_any().is::<collect::Collect>()
594 }
595
596 fn rollback_plan_indices(&self) -> HashSet<usize> {
597 self.context
598 .persistent_ctx
599 .failed_procedures
600 .iter()
601 .chain(self.context.persistent_ctx.unknown_procedures.iter())
602 .map(|procedure_meta| procedure_meta.plan_index)
603 .collect()
604 }
605
606 fn rollback_allocated_region_ids(&self) -> HashSet<store_api::storage::RegionId> {
614 if self.state.as_any().is::<allocate_region::AllocateRegion>()
615 || self.state.as_any().is::<dispatch::Dispatch>()
616 {
617 return self
618 .context
619 .persistent_ctx
620 .plans
621 .iter()
622 .flat_map(|plan| plan.allocated_region_ids.iter().copied())
623 .collect();
624 }
625
626 self.rollback_plan_indices()
627 .into_iter()
628 .flat_map(|plan_index| {
629 self.context.persistent_ctx.plans[plan_index]
630 .allocated_region_ids
631 .iter()
632 .copied()
633 })
634 .collect()
635 }
636
637 async fn rollback_group_metadata_for_selected_plans(
641 &mut self,
642 region_routes: &mut [RegionRoute],
643 ) -> Result<()> {
644 let rollback_plan_indices = self.rollback_plan_indices();
645 if rollback_plan_indices.is_empty() {
646 return Ok(());
647 }
648
649 let mut region_routes_map = region_routes
650 .iter_mut()
651 .map(|route| (route.region.id, route))
652 .collect::<HashMap<_, _>>();
653 for plan_index in rollback_plan_indices {
654 let plan = &self.context.persistent_ctx.plans[plan_index];
655 rollback_group_metadata_routes(
656 plan.group_id,
657 &plan.source_regions,
658 &plan.original_target_routes,
659 &plan.allocated_region_ids,
660 &plan.pending_deallocate_region_ids,
661 &mut region_routes_map,
662 )?;
663 }
664
665 Ok(())
666 }
667
668 async fn rollback_partition_metadata(&mut self) -> Result<()> {
669 let Some(update) = self
670 .context
671 .persistent_ctx
672 .partition_metadata_update
673 .as_ref()
674 else {
675 return Ok(());
676 };
677 let table_info_value = self.context.get_raw_table_info_value().await?;
678 let current_partition_key_indices = &table_info_value.table_info.meta.partition_key_indices;
679 let Some(new_partition_key_indices) = update.rollback_partition_key_indices(
680 self.context.persistent_ctx.table_id,
681 current_partition_key_indices,
682 )?
683 else {
684 return Ok(());
685 };
686
687 let mut new_table_info = table_info_value.table_info.clone();
688 new_table_info.meta.partition_key_indices = new_partition_key_indices;
689 self.context
690 .update_table_info(&table_info_value, table_info_value.update(new_table_info))
691 .await?;
692
693 Ok(())
700 }
701
702 async fn rollback_inner(&mut self, procedure_ctx: &ProcedureContext) -> Result<()> {
703 if !self.should_rollback() {
704 return Ok(());
705 }
706
707 let table_id = self.context.persistent_ctx.table_id;
708 let allocated_region_ids = self.rollback_allocated_region_ids();
709
710 let table_lock = TableLock::Write(table_id).into();
711 let _guard = procedure_ctx.provider.acquire_lock(&table_lock).await;
712
713 self.rollback_partition_metadata().await?;
714 let table_route_value = self.context.get_table_route_value().await?;
715 let original_region_routes = region_routes(table_id, table_route_value.get_inner_ref())?;
716 let mut current_region_routes = original_region_routes.clone();
717 self.rollback_group_metadata_for_selected_plans(&mut current_region_routes)
718 .await?;
719 let allocated_region_routes = DeallocateRegion::filter_deallocatable_region_routes(
720 table_id,
721 ¤t_region_routes,
722 &allocated_region_ids,
723 );
724 if !allocated_region_routes.is_empty() {
725 let table = TableName {
726 catalog_name: self.context.persistent_ctx.catalog_name.clone(),
727 schema_name: self.context.persistent_ctx.schema_name.clone(),
728 table_name: self.context.persistent_ctx.table_name.clone(),
729 };
730 if let Err(err) = DeallocateRegion::deallocate_regions(
733 &self.context.node_manager,
734 &self.context.leader_region_registry,
735 table,
736 table_id,
737 &allocated_region_routes,
738 )
739 .await
740 {
741 warn!(err; "Failed to drop allocated regions during repartition rollback, table_id: {}, regions: {:?}", table_id, allocated_region_ids);
742 }
743 }
744
745 let new_region_routes =
746 DeallocateRegion::generate_region_routes(¤t_region_routes, &allocated_region_ids);
747
748 if new_region_routes != *original_region_routes {
749 self.context
750 .update_table_route(&table_route_value, new_region_routes, HashMap::new())
751 .await
752 .map_err(BoxedError::new)
753 .with_context(|_| error::RetryLaterWithSourceSnafu {
754 reason: format!(
755 "Failed to rollback allocated region routes for repartition table: {}",
756 table_id
757 ),
758 })?;
759 }
760
761 if let Err(err) = self.context.invalidate_table_cache().await {
762 warn!(err; "Failed to invalidate table cache during repartition rollback, table_id: {}", table_id);
763 }
764
765 Ok(())
766 }
767}
768
769#[async_trait::async_trait]
770impl Procedure for RepartitionProcedure {
771 fn type_name(&self) -> &str {
772 Self::TYPE_NAME
773 }
774
775 #[tracing::instrument(skip_all, fields(
776 state = %self.state.name(),
777 table_id = %self.context.persistent_ctx.table_id
778 ))]
779 async fn execute(&mut self, _ctx: &ProcedureContext) -> ProcedureResult<Status> {
780 let state = &mut self.state;
781 let state_name = state.name();
782 common_telemetry::info!(
784 "Repartition procedure executing state: {}, table_id: {}",
785 state_name,
786 self.context.persistent_ctx.table_id
787 );
788 match state.next(&mut self.context, _ctx).await {
789 Ok((next, status)) => {
790 *state = next;
791 Ok(status)
792 }
793 Err(e) => {
794 if e.is_retryable() {
795 Err(ProcedureError::retry_later(e))
796 } else {
797 error!(
798 e;
799 "Repartition procedure failed, table id: {}",
800 self.context.persistent_ctx.table_id,
801 );
802 Err(ProcedureError::external(e))
803 }
804 }
805 }
806 }
807
808 async fn rollback(&mut self, ctx: &ProcedureContext) -> ProcedureResult<()> {
809 self.rollback_inner(ctx)
810 .await
811 .map_err(ProcedureError::external)
812 }
813
814 fn rollback_supported(&self) -> bool {
815 true
816 }
817
818 fn dump(&self) -> ProcedureResult<String> {
819 let data = RepartitionData {
820 state: self.state.as_ref(),
821 persistent_ctx: &self.context.persistent_ctx,
822 };
823 serde_json::to_string(&data).context(ToJsonSnafu)
824 }
825
826 fn lock_key(&self) -> LockKey {
827 LockKey::new(self.context.persistent_ctx.lock_key())
828 }
829
830 fn event(&self, ctx: &EventContext<'_>) -> Option<Box<dyn common_event_recorder::Event>> {
831 if !ctx.event_type_filter.allows(REPARTITION_EVENT_TYPE) {
832 return None;
833 }
834
835 let event = if matches!(ctx.trigger, EventTrigger::Submitted) {
836 let start = self.state.as_any().downcast_ref::<RepartitionStart>()?;
837 RepartitionEvent::submitted(&self.context.persistent_ctx, start)
838 } else {
839 RepartitionEvent::lifecycle(&self.context.persistent_ctx)
840 };
841 Some(Box::new(event))
842 }
843}
844
845pub struct DefaultRepartitionProcedureFactory {
846 mailbox: MailboxRef,
847 server_addr: String,
848 gc_requirement_manager: RepartitionGcRequirementManagerRef,
849}
850
851impl DefaultRepartitionProcedureFactory {
852 pub fn new(
853 mailbox: MailboxRef,
854 server_addr: String,
855 gc_requirement_manager: RepartitionGcRequirementManagerRef,
856 ) -> Self {
857 Self {
858 mailbox,
859 server_addr,
860 gc_requirement_manager,
861 }
862 }
863}
864
865pub struct GcDisabledRepartitionProcedureFactory {
870 enabled_factory: DefaultRepartitionProcedureFactory,
871}
872
873impl GcDisabledRepartitionProcedureFactory {
874 pub fn new(
875 mailbox: MailboxRef,
876 server_addr: String,
877 gc_requirement_manager: RepartitionGcRequirementManagerRef,
878 ) -> Self {
879 Self {
880 enabled_factory: DefaultRepartitionProcedureFactory::new(
881 mailbox,
882 server_addr,
883 gc_requirement_manager,
884 ),
885 }
886 }
887}
888
889#[async_trait::async_trait]
890impl RepartitionProcedureFactory for GcDisabledRepartitionProcedureFactory {
891 fn create(
892 &self,
893 _ddl_ctx: &DdlContext,
894 _table_name: TableName,
895 _table_id: TableId,
896 _source: RepartitionSource,
897 _to_exprs: Vec<String>,
898 _timeout: Option<Duration>,
899 ) -> std::result::Result<BoxedProcedure, BoxedError> {
900 Err(BoxedError::new(
901 error::InvalidArgumentsSnafu {
902 err_msg: "Repartition requires metasrv GC to be enabled".to_string(),
903 }
904 .build(),
905 ))
906 }
907
908 fn register_loaders(
909 &self,
910 ddl_ctx: &DdlContext,
911 procedure_manager: &ProcedureManagerRef,
912 ) -> std::result::Result<(), BoxedError> {
913 self.enabled_factory
914 .register_loaders(ddl_ctx, procedure_manager)
915 }
916
917 async fn ensure_gc_requirement(&self) -> std::result::Result<(), BoxedError> {
918 Err(BoxedError::new(
919 error::InvalidArgumentsSnafu {
920 err_msg: "Repartition requires metasrv GC to be enabled".to_string(),
921 }
922 .build(),
923 ))
924 }
925}
926
927#[async_trait::async_trait]
928impl RepartitionProcedureFactory for DefaultRepartitionProcedureFactory {
929 fn create(
930 &self,
931 ddl_ctx: &DdlContext,
932 table_name: TableName,
933 table_id: TableId,
934 source: RepartitionSource,
935 to_exprs: Vec<String>,
936 timeout: Option<Duration>,
937 ) -> std::result::Result<BoxedProcedure, BoxedError> {
938 let persistent_ctx = PersistentContext::new(table_name, table_id, timeout);
939 let from = match source {
940 RepartitionSource::Partitioned {
941 exprs,
942 target_partition_columns,
943 } => {
944 let exprs = exprs
945 .iter()
946 .map(|e| {
947 PartitionExpr::from_json_str(e)
948 .context(error::DeserializePartitionExprSnafu)?
949 .context(error::EmptyPartitionExprSnafu)
950 })
951 .collect::<Result<Vec<_>>>()
952 .map_err(BoxedError::new)?;
953 RepartitionFrom::Partitioned {
954 exprs,
955 target_partition_columns,
956 }
957 }
958 RepartitionSource::Unpartitioned { partition_columns } => {
959 RepartitionFrom::Unpartitioned { partition_columns }
960 }
961 };
962 let to_exprs = to_exprs
963 .iter()
964 .map(|e| {
965 PartitionExpr::from_json_str(e)
966 .context(error::DeserializePartitionExprSnafu)?
967 .context(error::EmptyPartitionExprSnafu)
968 })
969 .collect::<Result<Vec<_>>>()
970 .map_err(BoxedError::new)?;
971
972 let procedure = RepartitionProcedure::new(
973 from,
974 to_exprs,
975 Context::new(
976 ddl_ctx,
977 self.mailbox.clone(),
978 self.server_addr.clone(),
979 persistent_ctx,
980 ),
981 );
982
983 Ok(Box::new(procedure))
984 }
985
986 fn register_loaders(
987 &self,
988 ddl_ctx: &DdlContext,
989 procedure_manager: &ProcedureManagerRef,
990 ) -> std::result::Result<(), BoxedError> {
991 let mailbox = self.mailbox.clone();
993 let server_addr = self.server_addr.clone();
994 let moved_ddl_ctx = ddl_ctx.clone();
995 procedure_manager
996 .register_loader(
997 RepartitionProcedure::TYPE_NAME,
998 Box::new(move |json| {
999 let mailbox = mailbox.clone();
1000 let server_addr = server_addr.clone();
1001 let ddl_ctx = moved_ddl_ctx.clone();
1002 let factory = move |persistent_ctx| {
1003 Context::new(&ddl_ctx, mailbox, server_addr, persistent_ctx)
1004 };
1005 RepartitionProcedure::from_json(json, factory).map(|p| Box::new(p) as _)
1006 }),
1007 )
1008 .map_err(BoxedError::new)?;
1009
1010 let mailbox = self.mailbox.clone();
1012 let server_addr = self.server_addr.clone();
1013 let moved_ddl_ctx = ddl_ctx.clone();
1014 procedure_manager
1015 .register_loader(
1016 RepartitionGroupProcedure::TYPE_NAME,
1017 Box::new(move |json| {
1018 let mailbox = mailbox.clone();
1019 let server_addr = server_addr.clone();
1020 let ddl_ctx = moved_ddl_ctx.clone();
1021 let factory = move |persistent_ctx| {
1022 RepartitionGroupContext::new(&ddl_ctx, mailbox, server_addr, persistent_ctx)
1023 };
1024 RepartitionGroupProcedure::from_json(json, factory).map(|p| Box::new(p) as _)
1025 }),
1026 )
1027 .map_err(BoxedError::new)?;
1028
1029 Ok(())
1030 }
1031
1032 async fn ensure_gc_requirement(&self) -> std::result::Result<(), BoxedError> {
1033 self.gc_requirement_manager
1034 .require_gc()
1035 .await
1036 .map_err(BoxedError::new)
1037 }
1038}
1039
1040#[cfg(test)]
1041mod tests {
1042 use crate::procedure::test_util::{procedure_context_with_receivers, procedure_state_receiver};
1043 use std::collections::HashMap;
1044 use std::sync::Arc;
1045 use std::sync::atomic::{AtomicBool, Ordering};
1046
1047 use common_error::ext::{BoxedError, ErrorExt};
1048 use common_error::mock::MockError;
1049 use common_error::status_code::StatusCode;
1050 use common_event_recorder::EventTypeFilter;
1051 use common_meta::ddl::test_util::datanode_handler::{
1052 DatanodeWatcher, NaiveDatanodeHandler, UnexpectedErrorDatanodeHandler,
1053 };
1054 use common_meta::error;
1055 use common_meta::peer::Peer;
1056 use common_meta::region_keeper::MemoryRegionKeeper;
1057 use common_meta::rpc::router::{LeaderState, Region, RegionRoute};
1058 use common_meta::state_store::KvStateStore;
1059 use common_meta::test_util::MockDatanodeManager;
1060 use common_procedure::local::{LocalManager, ManagerConfig};
1061 use common_procedure::{Error as ProcedureError, Procedure, ProcedureId, ProcedureState};
1062 use store_api::region_engine::RegionRole;
1063 use store_api::storage::RegionId;
1064 use table::table_name::TableName;
1065 use tokio::sync::mpsc;
1066 use uuid::Uuid;
1067
1068 use super::*;
1069 use crate::procedure::repartition::allocate_region::AllocateRegion;
1070 use crate::procedure::repartition::collect::Collect;
1071 use crate::procedure::repartition::deallocate_region::DeallocateRegion;
1072 use crate::procedure::repartition::dispatch::Dispatch;
1073 use crate::procedure::repartition::gc_requirement::RepartitionGcRequirementManager;
1074 use crate::procedure::repartition::group::update_metadata::UpdateMetadata;
1075 use crate::procedure::repartition::plan::{SourceRegionDescriptor, TargetRegionDescriptor};
1076 use crate::procedure::repartition::repartition_end::RepartitionEnd;
1077 use crate::procedure::repartition::test_util::{
1078 TestingEnv, assert_parent_state, current_parent_region_routes, extract_subprocedure_ids,
1079 new_parent_context, range_expr, test_region_route, test_region_wal_options,
1080 };
1081 use crate::procedure::repartition::update_partition_metadata::{
1082 PartitionMetadataUpdate, UpdatePartitionMetadata,
1083 };
1084
1085 fn test_plan(table_id: TableId) -> RepartitionPlanEntry {
1086 RepartitionPlanEntry {
1087 group_id: uuid::Uuid::new_v4(),
1088 source_regions: vec![SourceRegionDescriptor::partitioned(
1089 RegionId::new(table_id, 1),
1090 range_expr("x", 0, 100),
1091 )],
1092 target_regions: vec![
1093 TargetRegionDescriptor {
1094 region_id: RegionId::new(table_id, 1),
1095 partition_expr: range_expr("x", 0, 50),
1096 },
1097 TargetRegionDescriptor {
1098 region_id: RegionId::new(table_id, 3),
1099 partition_expr: range_expr("x", 50, 100),
1100 },
1101 ],
1102 allocated_region_ids: vec![RegionId::new(table_id, 3)],
1103 pending_deallocate_region_ids: vec![],
1104 transition_map: vec![vec![0, 1]],
1105 original_target_routes: vec![],
1106 }
1107 }
1108
1109 fn with_rollback_metadata(
1110 mut plan: RepartitionPlanEntry,
1111 original_target_routes: Vec<RegionRoute>,
1112 ) -> RepartitionPlanEntry {
1113 plan.original_target_routes = original_target_routes;
1114 plan
1115 }
1116
1117 fn apply_group_staging(
1118 plan: &RepartitionPlanEntry,
1119 current_region_routes: &[RegionRoute],
1120 ) -> Vec<RegionRoute> {
1121 UpdateMetadata::apply_staging_region_routes(
1122 plan.group_id,
1123 &plan.source_regions,
1124 &plan.target_regions,
1125 &plan.pending_deallocate_region_ids,
1126 current_region_routes,
1127 )
1128 .unwrap()
1129 }
1130
1131 fn exit_group_staging(
1132 plan: &RepartitionPlanEntry,
1133 current_region_routes: &[RegionRoute],
1134 ) -> Vec<RegionRoute> {
1135 UpdateMetadata::exit_staging_region_routes(
1136 plan.group_id,
1137 &plan.source_regions,
1138 &plan.target_regions,
1139 current_region_routes,
1140 )
1141 .unwrap()
1142 }
1143
1144 fn region_route_by_id(region_routes: &[RegionRoute], region_id: RegionId) -> &RegionRoute {
1145 region_routes
1146 .iter()
1147 .find(|route| route.region.id == region_id)
1148 .unwrap()
1149 }
1150
1151 async fn table_partition_key_indices(ctx: &Context) -> Vec<usize> {
1152 ctx.get_table_info_value()
1153 .await
1154 .unwrap()
1155 .table_info
1156 .meta
1157 .partition_key_indices
1158 }
1159
1160 fn test_procedure(state: Box<dyn State>, context: Context) -> RepartitionProcedure {
1161 RepartitionProcedure { state, context }
1162 }
1163
1164 fn test_context(env: &TestingEnv, table_id: TableId) -> Context {
1165 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1166 let ddl_ctx = env.ddl_context(node_manager);
1167 let persistent_ctx = PersistentContext::new(
1168 TableName::new("test_catalog", "test_schema", "test_table"),
1169 table_id,
1170 None,
1171 );
1172
1173 Context::new(
1174 &ddl_ctx,
1175 env.mailbox_ctx.mailbox().clone(),
1176 env.server_addr.clone(),
1177 persistent_ctx,
1178 )
1179 }
1180
1181 #[test]
1182 fn test_gc_disabled_factory_rejects_repartition_and_registers_loaders() {
1183 let env = TestingEnv::new();
1184 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1185 let ddl_ctx = env.ddl_context(node_manager);
1186 let factory = GcDisabledRepartitionProcedureFactory::new(
1187 env.mailbox_ctx.mailbox().clone(),
1188 env.server_addr.clone(),
1189 Arc::new(RepartitionGcRequirementManager::new(env.kv_backend.clone())),
1190 );
1191
1192 let err = factory
1193 .create(
1194 &ddl_ctx,
1195 TableName::new("test_catalog", "test_schema", "test_table"),
1196 1024,
1197 RepartitionSource::Unpartitioned {
1198 partition_columns: vec![],
1199 },
1200 vec![],
1201 None,
1202 )
1203 .err()
1204 .expect("GC-disabled factory must reject repartition");
1205
1206 assert_eq!(StatusCode::InvalidArguments, err.status_code());
1207 assert_eq!(
1208 "Invalid arguments: Repartition requires metasrv GC to be enabled",
1209 err.to_string()
1210 );
1211
1212 let state_store = Arc::new(KvStateStore::new(env.kv_backend));
1213 let procedure_manager = Arc::new(LocalManager::new(
1214 ManagerConfig::default(),
1215 state_store.clone(),
1216 state_store,
1217 None,
1218 None,
1219 ));
1220 let procedure_manager_ref: ProcedureManagerRef = procedure_manager.clone();
1221 factory
1222 .register_loaders(&ddl_ctx, &procedure_manager_ref)
1223 .unwrap();
1224 assert!(procedure_manager.contains_loader(RepartitionProcedure::TYPE_NAME));
1225 assert!(procedure_manager.contains_loader(RepartitionGroupProcedure::TYPE_NAME));
1226 }
1227
1228 #[test]
1229 fn test_filter_allocated_region_routes() {
1230 let table_id = 1024;
1231 let region_routes = vec![
1232 test_region_route(RegionId::new(table_id, 1), "a"),
1233 test_region_route(RegionId::new(table_id, 2), "b"),
1234 ];
1235 let allocated_region_ids = HashSet::from([RegionId::new(table_id, 2)]);
1236
1237 let new_region_routes =
1238 DeallocateRegion::generate_region_routes(®ion_routes, &allocated_region_ids);
1239
1240 assert_eq!(new_region_routes.len(), 1);
1241 assert_eq!(new_region_routes[0].region.id, RegionId::new(table_id, 1));
1242 }
1243
1244 #[test]
1245 fn test_should_rollback_after_metadata_update() {
1246 let env = TestingEnv::new();
1247 let table_id = 1024;
1248
1249 let procedure = test_procedure(
1250 Box::new(RepartitionStart::new(
1251 RepartitionFrom::Partitioned {
1252 exprs: vec![],
1253 target_partition_columns: None,
1254 },
1255 vec![],
1256 )),
1257 test_context(&env, table_id),
1258 );
1259 assert!(!procedure.should_rollback());
1260
1261 let procedure = test_procedure(
1262 Box::new(UpdatePartitionMetadata::new(vec![])),
1263 test_context(&env, table_id),
1264 );
1265 assert!(procedure.should_rollback());
1266
1267 let procedure = test_procedure(
1268 Box::new(AllocateRegion::new(vec![])),
1269 test_context(&env, table_id),
1270 );
1271 assert!(procedure.should_rollback());
1272
1273 let procedure = test_procedure(Box::new(Dispatch), test_context(&env, table_id));
1274 assert!(procedure.should_rollback());
1275
1276 let procedure =
1277 test_procedure(Box::new(Collect::new(vec![])), test_context(&env, table_id));
1278 assert!(procedure.should_rollback());
1279
1280 let procedure = test_procedure(Box::new(DeallocateRegion), test_context(&env, table_id));
1281 assert!(!procedure.should_rollback());
1282
1283 let procedure = test_procedure(Box::new(RepartitionEnd), test_context(&env, table_id));
1284 assert!(!procedure.should_rollback());
1285 }
1286
1287 #[test]
1288 fn test_event_hook_records_submitted_and_lightweight_lifecycle_events() {
1289 let env = TestingEnv::new();
1290 let procedure = test_procedure(
1291 Box::new(RepartitionStart::new(
1292 RepartitionFrom::Unpartitioned {
1293 partition_columns: vec!["x".to_string()],
1294 },
1295 vec![range_expr("x", 0, 100)],
1296 )),
1297 test_context(&env, 1024),
1298 );
1299 let state = ProcedureState::Running;
1300 let all = Arc::new(EventTypeFilter::All);
1301
1302 let submitted = procedure
1303 .event(&EventContext {
1304 procedure_id: ProcedureId::random(),
1305 lifecycle_state: &state,
1306 trigger: EventTrigger::Submitted,
1307 event_type_filter: all.clone(),
1308 event_context: None,
1309 })
1310 .unwrap();
1311 assert_eq!(submitted.event_type(), REPARTITION_EVENT_TYPE);
1312 assert_ne!(submitted.json_payload().unwrap(), serde_json::Value::Null);
1313
1314 let allowed = procedure
1315 .event(&EventContext {
1316 procedure_id: ProcedureId::random(),
1317 lifecycle_state: &state,
1318 trigger: EventTrigger::Submitted,
1319 event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::from([
1320 REPARTITION_EVENT_TYPE.to_string(),
1321 ]))),
1322 event_context: None,
1323 })
1324 .unwrap();
1325 assert_eq!(allowed.event_type(), REPARTITION_EVENT_TYPE);
1326
1327 let succeeded = procedure
1328 .event(&EventContext {
1329 procedure_id: ProcedureId::random(),
1330 lifecycle_state: &state,
1331 trigger: EventTrigger::Succeeded,
1332 event_type_filter: all,
1333 event_context: None,
1334 })
1335 .unwrap();
1336 assert_eq!(succeeded.json_payload().unwrap(), serde_json::Value::Null);
1337
1338 let filtered = procedure.event(&EventContext {
1339 procedure_id: ProcedureId::random(),
1340 lifecycle_state: &state,
1341 trigger: EventTrigger::Submitted,
1342 event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::from([
1343 "another_event".to_string()
1344 ]))),
1345 event_context: None,
1346 });
1347 assert!(filtered.is_none());
1348
1349 let empty = procedure.event(&EventContext {
1350 procedure_id: ProcedureId::random(),
1351 lifecycle_state: &state,
1352 trigger: EventTrigger::Submitted,
1353 event_type_filter: Arc::new(EventTypeFilter::Only(HashSet::new())),
1354 event_context: None,
1355 });
1356 assert!(empty.is_none());
1357 }
1358
1359 #[test]
1360 fn test_register_operating_regions_preserves_route_roles() {
1361 let keeper = Arc::new(MemoryRegionKeeper::new());
1362 let region_routes = vec![
1363 RegionRoute {
1364 region: Region::new_test(RegionId::new(1024, 1)),
1365 leader_peer: Some(Peer::empty(1)),
1366 follower_peers: vec![],
1367 leader_state: None,
1368 leader_down_since: None,
1369 write_route_policy: None,
1370 },
1371 RegionRoute {
1372 region: Region::new_test(RegionId::new(1024, 2)),
1373 leader_peer: Some(Peer::empty(2)),
1374 follower_peers: vec![],
1375 leader_state: Some(LeaderState::Staging),
1376 leader_down_since: None,
1377 write_route_policy: None,
1378 },
1379 RegionRoute {
1380 region: Region::new_test(RegionId::new(1024, 3)),
1381 leader_peer: Some(Peer::empty(3)),
1382 follower_peers: vec![],
1383 leader_state: Some(LeaderState::Downgrading),
1384 leader_down_since: None,
1385 write_route_policy: None,
1386 },
1387 ];
1388
1389 let _guards = Context::register_operating_regions(&keeper, ®ion_routes).unwrap();
1390
1391 let leader_regions =
1392 keeper.extract_operating_regions(1, &HashSet::from([RegionId::new(1024, 1)]));
1393 let staging_regions =
1394 keeper.extract_operating_regions(2, &HashSet::from([RegionId::new(1024, 2)]));
1395 let downgrading_regions =
1396 keeper.extract_operating_regions(3, &HashSet::from([RegionId::new(1024, 3)]));
1397
1398 assert_eq!(
1399 leader_regions
1400 .get(&RegionId::new(1024, 1))
1401 .map(|region| region.role),
1402 Some(RegionRole::Leader)
1403 );
1404 assert_eq!(
1405 staging_regions
1406 .get(&RegionId::new(1024, 2))
1407 .map(|region| region.role),
1408 Some(RegionRole::StagingLeader)
1409 );
1410 assert_eq!(
1411 downgrading_regions
1412 .get(&RegionId::new(1024, 3))
1413 .map(|region| region.role),
1414 Some(RegionRole::DowngradingLeader)
1415 );
1416 }
1417
1418 #[test]
1419 fn test_persistent_context_partition_metadata_update_serde_default() {
1420 let json = r#"{
1421 "catalog_name":"test_catalog",
1422 "schema_name":"test_schema",
1423 "table_name":"test_table",
1424 "table_id":1024,
1425 "plans":[],
1426 "timeout":"120s"
1427 }"#;
1428
1429 let persistent_ctx: PersistentContext = serde_json::from_str(json).unwrap();
1430
1431 assert!(persistent_ctx.partition_metadata_update.is_none());
1432 }
1433
1434 #[tokio::test]
1435 async fn test_repartition_rollback_removes_partition_metadata_indices() {
1436 let env = TestingEnv::new();
1437 let table_id = 1024;
1438 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1439 env.create_physical_table_metadata_for_repartition(
1440 table_id,
1441 vec![test_region_route(RegionId::new(table_id, 1), "")],
1442 test_region_wal_options(&[1]),
1443 )
1444 .await;
1445
1446 let mut context = new_parent_context(&env, node_manager, table_id);
1447 let current = context.get_raw_table_info_value().await.unwrap();
1448 let mut table_info = current.table_info.clone();
1449 table_info.meta.partition_key_indices = vec![0, 1];
1450 context
1451 .update_table_info(¤t, current.update(table_info))
1452 .await
1453 .unwrap();
1454 context.persistent_ctx.partition_metadata_update = Some(
1455 PartitionMetadataUpdate::from_partitioned(vec![1], vec![0, 1]),
1456 );
1457 let mut procedure = RepartitionProcedure {
1458 state: Box::new(UpdatePartitionMetadata::new(vec![])),
1459 context,
1460 };
1461
1462 procedure
1463 .rollback(&TestingEnv::procedure_context())
1464 .await
1465 .unwrap();
1466
1467 assert_eq!(
1468 procedure
1469 .context
1470 .get_table_info_value()
1471 .await
1472 .unwrap()
1473 .table_info
1474 .meta
1475 .partition_key_indices,
1476 vec![1]
1477 );
1478 }
1479
1480 #[tokio::test]
1481 async fn test_repartition_rollback_removes_allocated_routes_from_dispatch() {
1482 let env = TestingEnv::new();
1483 let table_id = 1024;
1484 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1485 let ddl_ctx = env.ddl_context(node_manager);
1486 let original_region_routes = vec![
1487 test_region_route(
1488 RegionId::new(table_id, 1),
1489 &range_expr("x", 0, 100).as_json_str().unwrap(),
1490 ),
1491 test_region_route(
1492 RegionId::new(table_id, 2),
1493 &range_expr("x", 50, 100).as_json_str().unwrap(),
1494 ),
1495 test_region_route(RegionId::new(table_id, 3), ""),
1496 ];
1497 env.create_physical_table_metadata_with_wal_options(
1498 table_id,
1499 original_region_routes,
1500 test_region_wal_options(&[1, 2]),
1501 )
1502 .await;
1503
1504 let mut persistent_ctx = PersistentContext::new(
1505 TableName::new("test_catalog", "test_schema", "test_table"),
1506 table_id,
1507 None,
1508 );
1509 persistent_ctx.plans = vec![with_rollback_metadata(
1510 test_plan(table_id),
1511 vec![
1512 test_region_route(
1513 RegionId::new(table_id, 1),
1514 &range_expr("x", 0, 100).as_json_str().unwrap(),
1515 ),
1516 test_region_route(RegionId::new(table_id, 3), ""),
1517 ],
1518 )];
1519 persistent_ctx.failed_procedures = vec![ProcedureMeta {
1520 plan_index: 0,
1521 group_id: Uuid::new_v4(),
1522 procedure_id: ProcedureId::random(),
1523 }];
1524 let context = Context::new(
1525 &ddl_ctx,
1526 env.mailbox_ctx.mailbox().clone(),
1527 env.server_addr.clone(),
1528 persistent_ctx,
1529 );
1530 let mut procedure = RepartitionProcedure {
1531 state: Box::new(Dispatch),
1532 context,
1533 };
1534
1535 procedure
1536 .rollback(&TestingEnv::procedure_context())
1537 .await
1538 .unwrap();
1539
1540 let region_routes = current_parent_region_routes(&procedure.context).await;
1541 assert_eq!(region_routes.len(), 2);
1542 assert_eq!(region_routes[0].region.id, RegionId::new(table_id, 1));
1543 assert_eq!(region_routes[1].region.id, RegionId::new(table_id, 2));
1544 }
1545
1546 #[tokio::test]
1547 async fn test_repartition_rollback_removes_allocated_routes_from_allocate() {
1548 let env = TestingEnv::new();
1549 let table_id = 1024;
1550 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1551 let ddl_ctx = env.ddl_context(node_manager);
1552 let original_region_routes = vec![
1553 test_region_route(
1554 RegionId::new(table_id, 1),
1555 &range_expr("x", 0, 100).as_json_str().unwrap(),
1556 ),
1557 test_region_route(
1558 RegionId::new(table_id, 2),
1559 &range_expr("x", 50, 100).as_json_str().unwrap(),
1560 ),
1561 test_region_route(RegionId::new(table_id, 3), ""),
1562 ];
1563 env.create_physical_table_metadata_with_wal_options(
1564 table_id,
1565 original_region_routes,
1566 test_region_wal_options(&[1, 2]),
1567 )
1568 .await;
1569
1570 let mut persistent_ctx = PersistentContext::new(
1571 TableName::new("test_catalog", "test_schema", "test_table"),
1572 table_id,
1573 None,
1574 );
1575 persistent_ctx.plans = vec![test_plan(table_id)];
1576 let context = Context::new(
1577 &ddl_ctx,
1578 env.mailbox_ctx.mailbox().clone(),
1579 env.server_addr.clone(),
1580 persistent_ctx,
1581 );
1582 let mut procedure = RepartitionProcedure {
1583 state: Box::new(AllocateRegion::new(vec![])),
1584 context,
1585 };
1586
1587 procedure
1588 .rollback(&TestingEnv::procedure_context())
1589 .await
1590 .unwrap();
1591
1592 let region_routes = current_parent_region_routes(&procedure.context).await;
1593 assert_eq!(region_routes.len(), 2);
1594 assert_eq!(region_routes[0].region.id, RegionId::new(table_id, 1));
1595 assert_eq!(region_routes[1].region.id, RegionId::new(table_id, 2));
1596 }
1597
1598 #[tokio::test]
1599 async fn test_repartition_rollback_from_collect_only_removes_failed_allocated_routes() {
1600 let env = TestingEnv::new();
1601 let table_id = 1024;
1602 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1603 let ddl_ctx = env.ddl_context(node_manager);
1604 let original_region_routes = vec![
1605 test_region_route(
1606 RegionId::new(table_id, 1),
1607 &range_expr("x", 0, 100).as_json_str().unwrap(),
1608 ),
1609 test_region_route(
1610 RegionId::new(table_id, 2),
1611 &range_expr("x", 100, 200).as_json_str().unwrap(),
1612 ),
1613 test_region_route(RegionId::new(table_id, 3), ""),
1614 test_region_route(RegionId::new(table_id, 4), ""),
1615 ];
1616 env.create_physical_table_metadata_with_wal_options(
1617 table_id,
1618 original_region_routes,
1619 test_region_wal_options(&[1, 2, 3, 4]),
1620 )
1621 .await;
1622
1623 let mut persistent_ctx = PersistentContext::new(
1624 TableName::new("test_catalog", "test_schema", "test_table"),
1625 table_id,
1626 None,
1627 );
1628 let failed_plan = test_plan(table_id);
1629 let failed_plan = with_rollback_metadata(
1630 failed_plan,
1631 vec![
1632 test_region_route(
1633 RegionId::new(table_id, 1),
1634 &range_expr("x", 0, 100).as_json_str().unwrap(),
1635 ),
1636 test_region_route(RegionId::new(table_id, 3), ""),
1637 ],
1638 );
1639 let succeeded_plan = RepartitionPlanEntry {
1640 group_id: Uuid::new_v4(),
1641 source_regions: vec![SourceRegionDescriptor::partitioned(
1642 RegionId::new(table_id, 2),
1643 range_expr("x", 100, 200),
1644 )],
1645 target_regions: vec![
1646 TargetRegionDescriptor {
1647 region_id: RegionId::new(table_id, 2),
1648 partition_expr: range_expr("x", 100, 150),
1649 },
1650 TargetRegionDescriptor {
1651 region_id: RegionId::new(table_id, 4),
1652 partition_expr: range_expr("x", 150, 200),
1653 },
1654 ],
1655 allocated_region_ids: vec![RegionId::new(table_id, 4)],
1656 pending_deallocate_region_ids: vec![],
1657 transition_map: vec![vec![0]],
1658 original_target_routes: vec![
1659 test_region_route(
1660 RegionId::new(table_id, 2),
1661 &range_expr("x", 100, 200).as_json_str().unwrap(),
1662 ),
1663 test_region_route(RegionId::new(table_id, 4), ""),
1664 ],
1665 };
1666 persistent_ctx.plans = vec![failed_plan, succeeded_plan];
1667 persistent_ctx.failed_procedures = vec![ProcedureMeta {
1668 plan_index: 0,
1669 group_id: persistent_ctx.plans[0].group_id,
1670 procedure_id: ProcedureId::random(),
1671 }];
1672
1673 let context = Context::new(
1674 &ddl_ctx,
1675 env.mailbox_ctx.mailbox().clone(),
1676 env.server_addr.clone(),
1677 persistent_ctx,
1678 );
1679 let mut procedure = RepartitionProcedure {
1680 state: Box::new(Collect::new(vec![])),
1681 context,
1682 };
1683
1684 procedure
1685 .rollback(&TestingEnv::procedure_context())
1686 .await
1687 .unwrap();
1688
1689 let region_routes = current_parent_region_routes(&procedure.context).await;
1690 assert_eq!(region_routes.len(), 3);
1691 assert_eq!(region_routes[0].region.id, RegionId::new(table_id, 1));
1692 assert_eq!(region_routes[1].region.id, RegionId::new(table_id, 2));
1693 assert_eq!(region_routes[2].region.id, RegionId::new(table_id, 4));
1694 }
1695
1696 #[tokio::test]
1697 async fn test_repartition_rollback_from_collect_restores_failed_group_metadata_only() {
1698 let env = TestingEnv::new();
1699 let table_id = 1024;
1700 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1701 let ddl_ctx = env.ddl_context(node_manager);
1702 let original_region_routes = vec![
1703 test_region_route(
1704 RegionId::new(table_id, 1),
1705 &range_expr("x", 0, 100).as_json_str().unwrap(),
1706 ),
1707 test_region_route(
1708 RegionId::new(table_id, 2),
1709 &range_expr("x", 100, 200).as_json_str().unwrap(),
1710 ),
1711 test_region_route(RegionId::new(table_id, 3), ""),
1712 test_region_route(RegionId::new(table_id, 4), ""),
1713 ];
1714
1715 let failed_plan = with_rollback_metadata(
1716 test_plan(table_id),
1717 vec![
1718 original_region_routes[0].clone(),
1719 original_region_routes[2].clone(),
1720 ],
1721 );
1722 let succeeded_plan = RepartitionPlanEntry {
1723 group_id: Uuid::new_v4(),
1724 source_regions: vec![SourceRegionDescriptor::partitioned(
1725 RegionId::new(table_id, 2),
1726 range_expr("x", 100, 200),
1727 )],
1728 target_regions: vec![
1729 TargetRegionDescriptor {
1730 region_id: RegionId::new(table_id, 2),
1731 partition_expr: range_expr("x", 100, 150),
1732 },
1733 TargetRegionDescriptor {
1734 region_id: RegionId::new(table_id, 4),
1735 partition_expr: range_expr("x", 150, 200),
1736 },
1737 ],
1738 allocated_region_ids: vec![RegionId::new(table_id, 4)],
1739 pending_deallocate_region_ids: vec![],
1740 transition_map: vec![vec![0, 1]],
1741 original_target_routes: vec![
1742 original_region_routes[1].clone(),
1743 original_region_routes[3].clone(),
1744 ],
1745 };
1746 let current_region_routes = apply_group_staging(&failed_plan, &original_region_routes);
1747 let current_region_routes = apply_group_staging(&succeeded_plan, ¤t_region_routes);
1748 let current_region_routes = exit_group_staging(&succeeded_plan, ¤t_region_routes);
1749 env.create_physical_table_metadata_with_wal_options(
1750 table_id,
1751 current_region_routes,
1752 test_region_wal_options(&[1, 2, 3, 4]),
1753 )
1754 .await;
1755
1756 let mut persistent_ctx = PersistentContext::new(
1757 TableName::new("test_catalog", "test_schema", "test_table"),
1758 table_id,
1759 None,
1760 );
1761 persistent_ctx.plans = vec![failed_plan, succeeded_plan.clone()];
1762 persistent_ctx.failed_procedures = vec![ProcedureMeta {
1763 plan_index: 0,
1764 group_id: persistent_ctx.plans[0].group_id,
1765 procedure_id: ProcedureId::random(),
1766 }];
1767
1768 let context = Context::new(
1769 &ddl_ctx,
1770 env.mailbox_ctx.mailbox().clone(),
1771 env.server_addr.clone(),
1772 persistent_ctx,
1773 );
1774 let mut procedure = RepartitionProcedure {
1775 state: Box::new(Collect::new(vec![])),
1776 context,
1777 };
1778
1779 procedure
1780 .rollback(&TestingEnv::procedure_context())
1781 .await
1782 .unwrap();
1783
1784 assert_eq!(
1785 current_parent_region_routes(&procedure.context).await,
1786 vec![
1787 test_region_route(
1788 RegionId::new(table_id, 1),
1789 &range_expr("x", 0, 100).as_json_str().unwrap(),
1790 ),
1791 RegionRoute {
1792 region: Region {
1793 id: RegionId::new(table_id, 2),
1794 partition_expr: range_expr("x", 100, 150).as_json_str().unwrap(),
1795 ..Default::default()
1796 },
1797 leader_peer: Some(Peer::empty(1)),
1798 ..Default::default()
1799 },
1800 RegionRoute {
1801 region: Region {
1802 id: RegionId::new(table_id, 4),
1803 partition_expr: range_expr("x", 150, 200).as_json_str().unwrap(),
1804 ..Default::default()
1805 },
1806 leader_peer: Some(Peer::empty(1)),
1807 ..Default::default()
1808 },
1809 ]
1810 );
1811 }
1812
1813 #[tokio::test]
1814 async fn test_repartition_rollback_from_collect_restores_unknown_group_metadata() {
1815 let env = TestingEnv::new();
1816 let table_id = 1024;
1817 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1818 let ddl_ctx = env.ddl_context(node_manager);
1819 let original_region_routes = vec![
1820 test_region_route(
1821 RegionId::new(table_id, 1),
1822 &range_expr("x", 0, 100).as_json_str().unwrap(),
1823 ),
1824 test_region_route(
1825 RegionId::new(table_id, 2),
1826 &range_expr("x", 100, 200).as_json_str().unwrap(),
1827 ),
1828 test_region_route(RegionId::new(table_id, 3), ""),
1829 ];
1830 let plan = with_rollback_metadata(
1831 test_plan(table_id),
1832 vec![
1833 original_region_routes[0].clone(),
1834 original_region_routes[2].clone(),
1835 ],
1836 );
1837 let staged_region_routes = apply_group_staging(&plan, &original_region_routes);
1838 assert_eq!(
1839 region_route_by_id(&staged_region_routes, RegionId::new(table_id, 1))
1840 .region
1841 .partition_expr(),
1842 range_expr("x", 0, 50).as_json_str().unwrap()
1843 );
1844 assert!(
1845 region_route_by_id(&staged_region_routes, RegionId::new(table_id, 1))
1846 .is_leader_staging()
1847 );
1848 assert_eq!(
1849 region_route_by_id(&staged_region_routes, RegionId::new(table_id, 3))
1850 .region
1851 .partition_expr(),
1852 range_expr("x", 50, 100).as_json_str().unwrap()
1853 );
1854 assert!(
1855 region_route_by_id(&staged_region_routes, RegionId::new(table_id, 3))
1856 .is_leader_staging()
1857 );
1858 env.create_physical_table_metadata_with_wal_options(
1859 table_id,
1860 staged_region_routes,
1861 test_region_wal_options(&[1, 2, 3]),
1862 )
1863 .await;
1864
1865 let mut persistent_ctx = PersistentContext::new(
1866 TableName::new("test_catalog", "test_schema", "test_table"),
1867 table_id,
1868 None,
1869 );
1870 persistent_ctx.plans = vec![plan.clone()];
1871 persistent_ctx.unknown_procedures = vec![ProcedureMeta {
1872 plan_index: 0,
1873 group_id: plan.group_id,
1874 procedure_id: ProcedureId::random(),
1875 }];
1876
1877 let context = Context::new(
1878 &ddl_ctx,
1879 env.mailbox_ctx.mailbox().clone(),
1880 env.server_addr.clone(),
1881 persistent_ctx,
1882 );
1883 let mut procedure = RepartitionProcedure {
1884 state: Box::new(Collect::new(vec![])),
1885 context,
1886 };
1887
1888 procedure
1889 .rollback(&TestingEnv::procedure_context())
1890 .await
1891 .unwrap();
1892
1893 assert_eq!(
1894 current_parent_region_routes(&procedure.context).await,
1895 vec![
1896 original_region_routes[0].clone(),
1897 original_region_routes[1].clone()
1898 ]
1899 );
1900 }
1901
1902 #[tokio::test]
1903 async fn test_repartition_rollback_is_idempotent() {
1904 let env = TestingEnv::new();
1905 let table_id = 1024;
1906 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1907 let ddl_ctx = env.ddl_context(node_manager);
1908 let original_region_routes = vec![
1909 test_region_route(
1910 RegionId::new(table_id, 1),
1911 &range_expr("x", 0, 100).as_json_str().unwrap(),
1912 ),
1913 test_region_route(
1914 RegionId::new(table_id, 2),
1915 &range_expr("x", 50, 100).as_json_str().unwrap(),
1916 ),
1917 test_region_route(RegionId::new(table_id, 3), ""),
1918 ];
1919 env.create_physical_table_metadata_with_wal_options(
1920 table_id,
1921 original_region_routes,
1922 test_region_wal_options(&[1, 2]),
1923 )
1924 .await;
1925
1926 let mut persistent_ctx = PersistentContext::new(
1927 TableName::new("test_catalog", "test_schema", "test_table"),
1928 table_id,
1929 None,
1930 );
1931 persistent_ctx.plans = vec![with_rollback_metadata(
1932 test_plan(table_id),
1933 vec![
1934 test_region_route(
1935 RegionId::new(table_id, 1),
1936 &range_expr("x", 0, 100).as_json_str().unwrap(),
1937 ),
1938 test_region_route(RegionId::new(table_id, 3), ""),
1939 ],
1940 )];
1941 persistent_ctx.failed_procedures = vec![ProcedureMeta {
1942 plan_index: 0,
1943 group_id: Uuid::new_v4(),
1944 procedure_id: ProcedureId::random(),
1945 }];
1946 let context = Context::new(
1947 &ddl_ctx,
1948 env.mailbox_ctx.mailbox().clone(),
1949 env.server_addr.clone(),
1950 persistent_ctx,
1951 );
1952 let mut procedure = RepartitionProcedure {
1953 state: Box::new(Dispatch),
1954 context,
1955 };
1956
1957 procedure
1958 .rollback(&TestingEnv::procedure_context())
1959 .await
1960 .unwrap();
1961 let once = current_parent_region_routes(&procedure.context).await;
1962
1963 procedure
1964 .rollback(&TestingEnv::procedure_context())
1965 .await
1966 .unwrap();
1967 let twice = current_parent_region_routes(&procedure.context).await;
1968
1969 assert_eq!(once, twice);
1970 assert_eq!(once.len(), 2);
1971 assert_eq!(once[0].region.id, RegionId::new(table_id, 1));
1972 assert_eq!(once[1].region.id, RegionId::new(table_id, 2));
1973 }
1974
1975 #[tokio::test]
1976 async fn test_repartition_rollback_from_collect_restores_failed_merge_group_metadata_only() {
1977 let env = TestingEnv::new();
1978 let table_id = 1024;
1979 let node_manager = Arc::new(MockDatanodeManager::new(UnexpectedErrorDatanodeHandler));
1980 let ddl_ctx = env.ddl_context(node_manager);
1981 let original_region_routes = vec![
1982 test_region_route(
1983 RegionId::new(table_id, 1),
1984 &range_expr("x", 0, 100).as_json_str().unwrap(),
1985 ),
1986 test_region_route(
1987 RegionId::new(table_id, 2),
1988 &range_expr("x", 100, 200).as_json_str().unwrap(),
1989 ),
1990 test_region_route(
1991 RegionId::new(table_id, 3),
1992 &range_expr("x", 200, 300).as_json_str().unwrap(),
1993 ),
1994 test_region_route(RegionId::new(table_id, 4), ""),
1995 ];
1996 let failed_merge_plan = RepartitionPlanEntry {
1997 group_id: Uuid::new_v4(),
1998 source_regions: vec![
1999 SourceRegionDescriptor::partitioned(
2000 RegionId::new(table_id, 1),
2001 range_expr("x", 0, 100),
2002 ),
2003 SourceRegionDescriptor::partitioned(
2004 RegionId::new(table_id, 2),
2005 range_expr("x", 100, 200),
2006 ),
2007 ],
2008 target_regions: vec![TargetRegionDescriptor {
2009 region_id: RegionId::new(table_id, 1),
2010 partition_expr: range_expr("x", 0, 200),
2011 }],
2012 allocated_region_ids: vec![],
2013 pending_deallocate_region_ids: vec![RegionId::new(table_id, 2)],
2014 transition_map: vec![vec![0], vec![0]],
2015 original_target_routes: vec![original_region_routes[0].clone()],
2016 };
2017 let succeeded_split_plan = RepartitionPlanEntry {
2018 group_id: Uuid::new_v4(),
2019 source_regions: vec![SourceRegionDescriptor::partitioned(
2020 RegionId::new(table_id, 3),
2021 range_expr("x", 200, 300),
2022 )],
2023 target_regions: vec![
2024 TargetRegionDescriptor {
2025 region_id: RegionId::new(table_id, 3),
2026 partition_expr: range_expr("x", 200, 250),
2027 },
2028 TargetRegionDescriptor {
2029 region_id: RegionId::new(table_id, 4),
2030 partition_expr: range_expr("x", 250, 300),
2031 },
2032 ],
2033 allocated_region_ids: vec![RegionId::new(table_id, 4)],
2034 pending_deallocate_region_ids: vec![],
2035 transition_map: vec![vec![0, 1]],
2036 original_target_routes: vec![
2037 original_region_routes[2].clone(),
2038 original_region_routes[3].clone(),
2039 ],
2040 };
2041 let current_region_routes =
2042 apply_group_staging(&failed_merge_plan, &original_region_routes);
2043 let current_region_routes =
2044 apply_group_staging(&succeeded_split_plan, ¤t_region_routes);
2045 let staged_region_routes =
2046 exit_group_staging(&succeeded_split_plan, ¤t_region_routes);
2047 env.create_physical_table_metadata_with_wal_options(
2048 table_id,
2049 staged_region_routes,
2050 test_region_wal_options(&[1, 2, 3, 4]),
2051 )
2052 .await;
2053
2054 let mut persistent_ctx = PersistentContext::new(
2055 TableName::new("test_catalog", "test_schema", "test_table"),
2056 table_id,
2057 None,
2058 );
2059 persistent_ctx.plans = vec![failed_merge_plan, succeeded_split_plan.clone()];
2060 persistent_ctx.failed_procedures = vec![ProcedureMeta {
2061 plan_index: 0,
2062 group_id: persistent_ctx.plans[0].group_id,
2063 procedure_id: ProcedureId::random(),
2064 }];
2065
2066 let context = Context::new(
2067 &ddl_ctx,
2068 env.mailbox_ctx.mailbox().clone(),
2069 env.server_addr.clone(),
2070 persistent_ctx,
2071 );
2072 let mut procedure = RepartitionProcedure {
2073 state: Box::new(Collect::new(vec![])),
2074 context,
2075 };
2076
2077 procedure
2078 .rollback(&TestingEnv::procedure_context())
2079 .await
2080 .unwrap();
2081
2082 let region_routes = current_parent_region_routes(&procedure.context).await;
2083 assert_eq!(
2084 region_routes,
2085 vec![
2086 test_region_route(
2087 RegionId::new(table_id, 1),
2088 &range_expr("x", 0, 100).as_json_str().unwrap(),
2089 ),
2090 test_region_route(
2091 RegionId::new(table_id, 2),
2092 &range_expr("x", 100, 200).as_json_str().unwrap(),
2093 ),
2094 RegionRoute {
2095 region: Region {
2096 id: RegionId::new(table_id, 3),
2097 partition_expr: range_expr("x", 200, 250).as_json_str().unwrap(),
2098 ..Default::default()
2099 },
2100 leader_peer: Some(Peer::empty(1)),
2101 ..Default::default()
2102 },
2103 RegionRoute {
2104 region: Region {
2105 id: RegionId::new(table_id, 4),
2106 partition_expr: range_expr("x", 250, 300).as_json_str().unwrap(),
2107 ..Default::default()
2108 },
2109 leader_peer: Some(Peer::empty(1)),
2110 ..Default::default()
2111 },
2112 ]
2113 );
2114 }
2115
2116 #[tokio::test]
2117 async fn test_repartition_after_partial_create_failure() {
2118 check_partial_create_failure(false).await;
2119 }
2120
2121 #[tokio::test]
2122 async fn test_repartition_after_pre_upgrade_partial_create_failure() {
2123 check_partial_create_failure(true).await;
2124 }
2125
2126 async fn check_partial_create_failure(pre_upgrade: bool) {
2127 use api::v1::region::region_request::Body;
2128 use common_meta::key::MetadataKey;
2129 use common_meta::key::table_route::TableRouteKey;
2130 use common_meta::rpc::store::PutRequest;
2131
2132 let env = TestingEnv::new();
2133 let table_id = 1024;
2134 let source = range_expr("x", 0, 90);
2135 let route = test_region_route(RegionId::new(table_id, 1), &source.as_json_str().unwrap());
2136 env.create_physical_table_metadata_for_repartition(
2137 table_id,
2138 vec![route.clone()],
2139 test_region_wal_options(&[1]),
2140 )
2141 .await;
2142
2143 let (tx, mut requests) = mpsc::channel(1);
2144 let fail_once = AtomicBool::new(true);
2145 let nodes = Arc::new(MockDatanodeManager::new(
2146 DatanodeWatcher::new(tx).with_handler(move |_, request| {
2147 if let Some(Body::Create(create)) = request.body
2148 && create.region_id == RegionId::new(table_id, 3).as_u64()
2149 && fail_once.swap(false, Ordering::SeqCst)
2150 {
2151 return error::UnexpectedSnafu {
2152 err_msg: "injected create failure",
2153 }
2154 .fail();
2155 }
2156 Ok(api::region::RegionResponse::new(0))
2157 }),
2158 ));
2159 let new_split = || {
2160 RepartitionProcedure::new(
2161 RepartitionFrom::Partitioned {
2162 exprs: vec![source.clone()],
2163 target_partition_columns: None,
2164 },
2165 vec![
2166 range_expr("x", 0, 30),
2167 range_expr("x", 30, 60),
2168 range_expr("x", 60, 90),
2169 ],
2170 new_parent_context(&env, nodes.clone(), table_id),
2171 )
2172 };
2173
2174 let ctx = TestingEnv::procedure_context();
2175 let mut first = new_split();
2176 first.execute(&ctx).await.unwrap();
2177 let original = first.context.get_table_route_value().await.unwrap();
2178 first.execute(&ctx).await.unwrap();
2179 let reserved = first.context.get_table_route_value().await.unwrap();
2180 if pre_upgrade {
2181 env.kv_backend
2183 .put(
2184 PutRequest::new()
2185 .with_key(TableRouteKey::new(table_id).to_bytes())
2186 .with_value(original.get_raw_bytes()),
2187 )
2188 .await
2189 .unwrap();
2190 first =
2191 RepartitionProcedure::from_json(&first.dump().unwrap(), |persistent_ctx| Context {
2192 persistent_ctx,
2193 ..new_parent_context(&env, nodes.clone(), table_id)
2194 })
2195 .unwrap();
2196 }
2197
2198 let observe_create = async {
2199 let (_, request) = requests.recv().await.unwrap();
2201 let Some(Body::Create(create)) = request.body else {
2202 panic!("expected create")
2203 };
2204 let current = new_split().context.get_table_route_value().await.unwrap();
2205 assert_eq!(current.max_region_number().unwrap(), 3);
2206 assert_eq!(current.version().unwrap(), reserved.version().unwrap());
2207 RegionId::from_u64(create.region_id).region_number()
2208 };
2209 let create_timeout = Duration::from_secs(5);
2210 let (result, first_number) = tokio::time::timeout(create_timeout, async {
2211 tokio::join!(first.execute(&ctx), observe_create)
2212 })
2213 .await
2214 .expect("first split timed out waiting for a create request");
2215 assert!(!result.unwrap_err().is_retry_later());
2216 let (_, request) = requests.try_recv().unwrap();
2217 let Some(Body::Create(create)) = request.body else {
2218 panic!("expected create")
2219 };
2220 let mut creates = [
2221 first_number,
2222 RegionId::from_u64(create.region_id).region_number(),
2223 ];
2224 creates.sort_unstable();
2225 assert_eq!(creates, [2, 3]);
2226 first.rollback(&ctx).await.unwrap();
2227 assert_eq!(
2228 current_parent_region_routes(&first.context).await,
2229 vec![route]
2230 );
2231 assert_eq!(
2232 first
2233 .context
2234 .get_table_route_value()
2235 .await
2236 .unwrap()
2237 .max_region_number()
2238 .unwrap(),
2239 3
2240 );
2241
2242 let mut second = new_split();
2243 second.execute(&ctx).await.unwrap();
2244 second.execute(&ctx).await.unwrap();
2245 let (result, _) = tokio::time::timeout(create_timeout, async {
2246 tokio::join!(second.execute(&ctx), requests.recv())
2247 })
2248 .await
2249 .expect("second split timed out waiting for a create request");
2250 result.unwrap();
2251 assert_parent_state::<Dispatch>(&second);
2252 let numbers: Vec<_> = current_parent_region_routes(&second.context)
2253 .await
2254 .iter()
2255 .map(|route| route.region.id.region_number())
2256 .collect();
2257 assert_eq!(numbers, [1, 4, 5]);
2258 }
2259
2260 #[tokio::test]
2261 async fn test_repartition_procedure_flow_split_failed_and_full_rollback() {
2262 let env = TestingEnv::new();
2263 let table_id = 1024;
2264 let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
2265
2266 env.create_physical_table_metadata_for_repartition(
2267 table_id,
2268 vec![
2269 test_region_route(
2270 RegionId::new(table_id, 1),
2271 &range_expr("x", 0, 100).as_json_str().unwrap(),
2272 ),
2273 test_region_route(
2274 RegionId::new(table_id, 2),
2275 &range_expr("x", 100, 200).as_json_str().unwrap(),
2276 ),
2277 ],
2278 test_region_wal_options(&[1, 2]),
2279 )
2280 .await;
2281
2282 let context = new_parent_context(&env, node_manager, table_id);
2283 let mut procedure = RepartitionProcedure::new(
2284 RepartitionFrom::Partitioned {
2285 exprs: vec![range_expr("x", 0, 100)],
2286 target_partition_columns: None,
2287 },
2288 vec![range_expr("x", 0, 50), range_expr("x", 50, 100)],
2289 context,
2290 );
2291
2292 let start_status = procedure
2293 .execute(&TestingEnv::procedure_context())
2294 .await
2295 .unwrap();
2296 assert!(!start_status.need_persist());
2297 let start_status = procedure
2298 .execute(&TestingEnv::procedure_context())
2299 .await
2300 .unwrap();
2301 assert!(start_status.need_persist());
2302 assert_parent_state::<AllocateRegion>(&procedure);
2303
2304 let allocate_status = procedure
2305 .execute(&TestingEnv::procedure_context())
2306 .await
2307 .unwrap();
2308 assert!(allocate_status.need_persist());
2309 assert_parent_state::<Dispatch>(&procedure);
2310 assert_eq!(procedure.context.persistent_ctx.plans.len(), 1);
2311 let plan = &procedure.context.persistent_ctx.plans[0];
2312 let expected_plan = test_plan(table_id);
2313 assert_eq!(plan.source_regions, expected_plan.source_regions);
2314 assert_eq!(plan.target_regions, expected_plan.target_regions);
2315 assert_eq!(
2316 plan.allocated_region_ids,
2317 expected_plan.allocated_region_ids
2318 );
2319 assert_eq!(
2320 plan.pending_deallocate_region_ids,
2321 expected_plan.pending_deallocate_region_ids
2322 );
2323 assert_eq!(plan.transition_map, expected_plan.transition_map);
2324 assert_eq!(
2325 current_parent_region_routes(&procedure.context).await,
2326 vec![
2327 test_region_route(
2328 RegionId::new(table_id, 1),
2329 &range_expr("x", 0, 100).as_json_str().unwrap(),
2330 ),
2331 test_region_route(
2332 RegionId::new(table_id, 2),
2333 &range_expr("x", 100, 200).as_json_str().unwrap(),
2334 ),
2335 RegionRoute {
2336 region: Region {
2337 id: RegionId::new(table_id, 3),
2338 partition_expr: range_expr("x", 50, 100).as_json_str().unwrap(),
2339 ..Default::default()
2340 },
2341 leader_peer: Some(Peer::empty(0)),
2342 ..Default::default()
2343 },
2344 ]
2345 );
2346
2347 let dispatch_status = procedure
2348 .execute(&TestingEnv::procedure_context())
2349 .await
2350 .unwrap();
2351 assert!(!dispatch_status.need_persist());
2352 let subprocedure_ids = extract_subprocedure_ids(dispatch_status);
2353 assert_eq!(subprocedure_ids.len(), 1);
2354 assert_parent_state::<Collect>(&procedure);
2355
2356 let failed_state = ProcedureState::failed(Arc::new(ProcedureError::external(
2357 MockError::new(StatusCode::Internal),
2358 )));
2359 let collect_ctx = procedure_context_with_receivers(HashMap::from([(
2360 subprocedure_ids[0],
2361 procedure_state_receiver(failed_state),
2362 )]));
2363
2364 let err = procedure.execute(&collect_ctx).await.unwrap_err();
2365 assert!(!err.is_retry_later());
2366 assert_parent_state::<Collect>(&procedure);
2367
2368 procedure
2369 .rollback(&TestingEnv::procedure_context())
2370 .await
2371 .unwrap();
2372
2373 let region_routes = current_parent_region_routes(&procedure.context).await;
2374 assert_eq!(
2375 region_routes,
2376 vec![
2377 test_region_route(
2378 RegionId::new(table_id, 1),
2379 &range_expr("x", 0, 100).as_json_str().unwrap(),
2380 ),
2381 test_region_route(
2382 RegionId::new(table_id, 2),
2383 &range_expr("x", 100, 200).as_json_str().unwrap(),
2384 ),
2385 ]
2386 );
2387 }
2388
2389 #[tokio::test]
2390 async fn test_repartition_procedure_flow_unpartitioned_failed_and_full_rollback() {
2391 let env = TestingEnv::new();
2392 let table_id = 1024;
2393 let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
2394
2395 env.create_physical_table_metadata_for_repartition(
2396 table_id,
2397 vec![test_region_route(RegionId::new(table_id, 1), "")],
2398 test_region_wal_options(&[1]),
2399 )
2400 .await;
2401
2402 let context = new_parent_context(&env, node_manager, table_id);
2403 let to_exprs = vec![range_expr("col1", 0, 50), range_expr("col1", 50, 100)];
2404 let mut procedure = RepartitionProcedure::new(
2405 RepartitionFrom::Unpartitioned {
2406 partition_columns: vec!["col1".to_string()],
2407 },
2408 to_exprs.clone(),
2409 context,
2410 );
2411
2412 let start_status = procedure
2413 .execute(&TestingEnv::procedure_context())
2414 .await
2415 .unwrap();
2416 assert!(start_status.need_persist());
2417 assert_parent_state::<UpdatePartitionMetadata>(&procedure);
2418 assert_eq!(
2419 procedure
2420 .context
2421 .persistent_ctx
2422 .partition_metadata_update
2423 .as_ref()
2424 .unwrap()
2425 .target_partition_key_indices,
2426 vec![0]
2427 );
2428
2429 let update_status = procedure
2430 .execute(&TestingEnv::procedure_context())
2431 .await
2432 .unwrap();
2433 assert!(update_status.need_persist());
2434 assert_parent_state::<AllocateRegion>(&procedure);
2435 assert_eq!(
2436 table_partition_key_indices(&procedure.context).await,
2437 vec![0]
2438 );
2439
2440 let build_allocate_status = procedure
2441 .execute(&TestingEnv::procedure_context())
2442 .await
2443 .unwrap();
2444 assert!(build_allocate_status.need_persist());
2445 assert_parent_state::<AllocateRegion>(&procedure);
2446 assert_eq!(procedure.context.persistent_ctx.plans.len(), 1);
2447 let plan = &procedure.context.persistent_ctx.plans[0];
2448 assert_eq!(
2449 plan.source_regions,
2450 vec![SourceRegionDescriptor::Default {
2451 region_id: RegionId::new(table_id, 1)
2452 }]
2453 );
2454 assert_eq!(plan.target_regions.len(), 2);
2455 assert_eq!(plan.target_regions[0].region_id, RegionId::new(table_id, 1));
2456 assert_eq!(plan.target_regions[0].partition_expr, to_exprs[0]);
2457 assert_eq!(
2458 plan.allocated_region_ids,
2459 vec![plan.target_regions[1].region_id]
2460 );
2461 assert!(plan.pending_deallocate_region_ids.is_empty());
2462 assert_eq!(plan.transition_map, vec![vec![0, 1]]);
2463 let target_regions = plan.target_regions.clone();
2464
2465 let execute_allocate_status = procedure
2466 .execute(&TestingEnv::procedure_context())
2467 .await
2468 .unwrap();
2469 assert!(execute_allocate_status.need_persist());
2470 assert_parent_state::<Dispatch>(&procedure);
2471 let region_routes = current_parent_region_routes(&procedure.context).await;
2472 assert_eq!(region_routes.len(), 2);
2473 assert_eq!(
2474 region_route_by_id(®ion_routes, target_regions[0].region_id)
2475 .region
2476 .partition_expr(),
2477 ""
2478 );
2479 assert_eq!(
2480 region_route_by_id(®ion_routes, target_regions[1].region_id)
2481 .region
2482 .partition_expr(),
2483 to_exprs[1].as_json_str().unwrap()
2484 );
2485
2486 let dispatch_status = procedure
2487 .execute(&TestingEnv::procedure_context())
2488 .await
2489 .unwrap();
2490 let subprocedure_ids = extract_subprocedure_ids(dispatch_status);
2491 assert_eq!(subprocedure_ids.len(), 1);
2492 assert_parent_state::<Collect>(&procedure);
2493
2494 let failed_state = ProcedureState::failed(Arc::new(ProcedureError::external(
2495 MockError::new(StatusCode::Internal),
2496 )));
2497 let collect_ctx = procedure_context_with_receivers(HashMap::from([(
2498 subprocedure_ids[0],
2499 procedure_state_receiver(failed_state),
2500 )]));
2501 let err = procedure.execute(&collect_ctx).await.unwrap_err();
2502 assert!(!err.is_retry_later());
2503 assert_parent_state::<Collect>(&procedure);
2504
2505 procedure
2506 .rollback(&TestingEnv::procedure_context())
2507 .await
2508 .unwrap();
2509
2510 assert!(
2511 table_partition_key_indices(&procedure.context)
2512 .await
2513 .is_empty()
2514 );
2515 assert_eq!(
2516 current_parent_region_routes(&procedure.context).await,
2517 vec![test_region_route(RegionId::new(table_id, 1), "")]
2518 );
2519 }
2520
2521 #[tokio::test]
2522 async fn test_repartition_procedure_flow_unpartitioned_rollback_is_idempotent() {
2523 let env = TestingEnv::new();
2524 let table_id = 1024;
2525 let node_manager = Arc::new(MockDatanodeManager::new(NaiveDatanodeHandler));
2526
2527 env.create_physical_table_metadata_for_repartition(
2528 table_id,
2529 vec![test_region_route(RegionId::new(table_id, 1), "")],
2530 test_region_wal_options(&[1]),
2531 )
2532 .await;
2533
2534 let context = new_parent_context(&env, node_manager, table_id);
2535 let mut procedure = RepartitionProcedure::new(
2536 RepartitionFrom::Unpartitioned {
2537 partition_columns: vec!["col1".to_string()],
2538 },
2539 vec![range_expr("col1", 0, 50), range_expr("col1", 50, 100)],
2540 context,
2541 );
2542
2543 procedure
2544 .execute(&TestingEnv::procedure_context())
2545 .await
2546 .unwrap();
2547 procedure
2548 .execute(&TestingEnv::procedure_context())
2549 .await
2550 .unwrap();
2551 procedure
2552 .execute(&TestingEnv::procedure_context())
2553 .await
2554 .unwrap();
2555 procedure
2556 .execute(&TestingEnv::procedure_context())
2557 .await
2558 .unwrap();
2559 assert_eq!(
2560 table_partition_key_indices(&procedure.context).await,
2561 vec![0]
2562 );
2563 assert_eq!(
2564 current_parent_region_routes(&procedure.context).await.len(),
2565 2
2566 );
2567
2568 let dispatch_status = procedure
2569 .execute(&TestingEnv::procedure_context())
2570 .await
2571 .unwrap();
2572 let subprocedure_ids = extract_subprocedure_ids(dispatch_status);
2573 assert_eq!(subprocedure_ids.len(), 1);
2574 assert_parent_state::<Collect>(&procedure);
2575
2576 let failed_state = ProcedureState::failed(Arc::new(ProcedureError::external(
2577 MockError::new(StatusCode::Internal),
2578 )));
2579 let collect_ctx = procedure_context_with_receivers(HashMap::from([(
2580 subprocedure_ids[0],
2581 procedure_state_receiver(failed_state),
2582 )]));
2583 let err = procedure.execute(&collect_ctx).await.unwrap_err();
2584 assert!(!err.is_retry_later());
2585
2586 procedure
2587 .rollback(&TestingEnv::procedure_context())
2588 .await
2589 .unwrap();
2590 let once_indices = table_partition_key_indices(&procedure.context).await;
2591 let once_routes = current_parent_region_routes(&procedure.context).await;
2592
2593 procedure
2594 .rollback(&TestingEnv::procedure_context())
2595 .await
2596 .unwrap();
2597 let twice_indices = table_partition_key_indices(&procedure.context).await;
2598 let twice_routes = current_parent_region_routes(&procedure.context).await;
2599
2600 assert_eq!(once_indices, twice_indices);
2601 assert_eq!(once_routes, twice_routes);
2602 assert!(twice_indices.is_empty());
2603 assert_eq!(
2604 twice_routes,
2605 vec![test_region_route(RegionId::new(table_id, 1), "")]
2606 );
2607 }
2608
2609 #[tokio::test]
2610 async fn test_repartition_procedure_flow_split_allocate_retryable_then_resume() {
2611 common_telemetry::init_default_ut_logging();
2612 let env = TestingEnv::new();
2613 let table_id = 1024;
2614 let (tx, _rx) = mpsc::channel(8);
2615 let should_retry = Arc::new(AtomicBool::new(true));
2616 let datanode_handler = DatanodeWatcher::new(tx).with_handler(move |_, _| {
2617 if should_retry.swap(false, Ordering::SeqCst) {
2618 return Err(error::Error::RetryLater {
2619 source: BoxedError::new(
2620 error::UnexpectedSnafu {
2621 err_msg: "retry later",
2622 }
2623 .build(),
2624 ),
2625 clean_poisons: false,
2626 });
2627 }
2628
2629 Ok(api::region::RegionResponse::new(0))
2630 });
2631 let node_manager = Arc::new(MockDatanodeManager::new(datanode_handler));
2632
2633 env.create_physical_table_metadata_for_repartition(
2634 table_id,
2635 vec![
2636 test_region_route(
2637 RegionId::new(table_id, 1),
2638 &range_expr("x", 0, 100).as_json_str().unwrap(),
2639 ),
2640 test_region_route(
2641 RegionId::new(table_id, 2),
2642 &range_expr("x", 100, 200).as_json_str().unwrap(),
2643 ),
2644 ],
2645 test_region_wal_options(&[1, 2]),
2646 )
2647 .await;
2648
2649 let context = new_parent_context(&env, node_manager, table_id);
2650 let mut procedure = RepartitionProcedure::new(
2651 RepartitionFrom::Partitioned {
2652 exprs: vec![range_expr("x", 0, 100)],
2653 target_partition_columns: None,
2654 },
2655 vec![range_expr("x", 0, 50), range_expr("x", 50, 100)],
2656 context,
2657 );
2658
2659 let start_status = procedure
2660 .execute(&TestingEnv::procedure_context())
2661 .await
2662 .unwrap();
2663 assert!(!start_status.need_persist());
2664 let start_status = procedure
2665 .execute(&TestingEnv::procedure_context())
2666 .await
2667 .unwrap();
2668 assert!(start_status.need_persist());
2669 assert_parent_state::<AllocateRegion>(&procedure);
2670
2671 let err = procedure
2672 .execute(&TestingEnv::procedure_context())
2673 .await
2674 .unwrap_err();
2675 assert!(err.is_retry_later());
2676 assert_parent_state::<AllocateRegion>(&procedure);
2677 assert!(!procedure.context.persistent_ctx.plans.is_empty());
2678 assert_eq!(
2679 current_parent_region_routes(&procedure.context).await,
2680 vec![
2681 test_region_route(
2682 RegionId::new(table_id, 1),
2683 &range_expr("x", 0, 100).as_json_str().unwrap(),
2684 ),
2685 test_region_route(
2686 RegionId::new(table_id, 2),
2687 &range_expr("x", 100, 200).as_json_str().unwrap(),
2688 ),
2689 ]
2690 );
2691
2692 let allocate_status = procedure
2693 .execute(&TestingEnv::procedure_context())
2694 .await
2695 .unwrap();
2696 assert!(allocate_status.need_persist());
2697 assert_parent_state::<Dispatch>(&procedure);
2698
2699 assert_eq!(procedure.context.persistent_ctx.plans.len(), 1);
2700 let plan = &procedure.context.persistent_ctx.plans[0];
2701 let expected_plan = test_plan(table_id);
2702 assert_eq!(plan.source_regions, expected_plan.source_regions);
2703 assert_eq!(plan.target_regions, expected_plan.target_regions);
2704 assert_eq!(
2705 plan.allocated_region_ids,
2706 expected_plan.allocated_region_ids
2707 );
2708 assert_eq!(plan.transition_map, expected_plan.transition_map);
2709 assert_eq!(
2710 current_parent_region_routes(&procedure.context).await,
2711 vec![
2712 test_region_route(
2713 RegionId::new(table_id, 1),
2714 &range_expr("x", 0, 100).as_json_str().unwrap(),
2715 ),
2716 test_region_route(
2717 RegionId::new(table_id, 2),
2718 &range_expr("x", 100, 200).as_json_str().unwrap(),
2719 ),
2720 RegionRoute {
2721 region: Region {
2722 id: RegionId::new(table_id, 3),
2723 partition_expr: range_expr("x", 50, 100).as_json_str().unwrap(),
2724 ..Default::default()
2725 },
2726 leader_peer: Some(Peer::empty(0)),
2727 ..Default::default()
2728 },
2729 ]
2730 );
2731
2732 let dispatch_status = procedure
2733 .execute(&TestingEnv::procedure_context())
2734 .await
2735 .unwrap();
2736 assert!(!dispatch_status.need_persist());
2737 let subprocedure_ids = extract_subprocedure_ids(dispatch_status);
2738 assert_eq!(subprocedure_ids.len(), 1);
2739 assert_parent_state::<Collect>(&procedure);
2740 }
2741}