1use std::any::Any;
16use std::ops::Div;
17
18use api::v1::meta::MailboxMessage;
19use common_meta::RegionIdent;
20use common_meta::distributed_time_constants::default_distributed_time_constants;
21use common_meta::instruction::{
22 Instruction, InstructionReply, OpenRegion, OpenRegionReason, SimpleReply,
23};
24use common_meta::key::datanode_table::RegionInfo;
25use common_procedure::{Context as ProcedureContext, Status};
26use common_telemetry::info;
27use common_telemetry::tracing_context::TracingContext;
28use serde::{Deserialize, Serialize};
29use snafu::{OptionExt, ResultExt};
30use store_api::region_request::RegionRequirements;
31use tokio::time::Instant;
32
33use crate::error::{self, Result};
34use crate::handler::HeartbeatMailbox;
35use crate::procedure::region_migration::flush_leader_region::PreFlushRegion;
36use crate::procedure::region_migration::{Context, RegionMigrationTriggerReason, State};
37use crate::procedure::utils::instruction_error_result;
38use crate::service::mailbox::Channel;
39
40#[derive(Debug, Serialize, Deserialize)]
41pub struct OpenCandidateRegion;
42
43#[async_trait::async_trait]
44#[typetag::serde]
45impl State for OpenCandidateRegion {
46 async fn next(
47 &mut self,
48 ctx: &mut Context,
49 procedure_ctx: &ProcedureContext,
50 ) -> Result<(Box<dyn State>, Status)> {
51 let trigger_reason = ctx.trigger_reason(procedure_ctx.event_context.as_ref());
52 let instruction = self
53 .build_open_region_instruction(ctx, trigger_reason)
54 .await?;
55 let now = Instant::now();
56 self.open_candidate_region(ctx, instruction).await?;
57 ctx.update_open_candidate_region_elapsed(now);
58
59 Ok((Box::new(PreFlushRegion), Status::executing(false)))
60 }
61
62 fn as_any(&self) -> &dyn Any {
63 self
64 }
65}
66
67impl OpenCandidateRegion {
68 async fn build_open_region_instruction(
73 &self,
74 ctx: &mut Context,
75 trigger_reason: RegionMigrationTriggerReason,
76 ) -> Result<Instruction> {
77 let region_ids = ctx.persistent_ctx.region_ids.clone();
78 let from_peer_id = ctx.persistent_ctx.from_peer.id;
79 let to_peer_id = ctx.persistent_ctx.to_peer.id;
80 let reason = match trigger_reason {
81 RegionMigrationTriggerReason::Failover => OpenRegionReason::RegionFailover,
82 _ => OpenRegionReason::RegionMigration,
83 };
84 let datanode_table_values = ctx.get_from_peer_datanode_table_values().await?;
85 let mut open_regions = Vec::with_capacity(region_ids.len());
86
87 for region_id in region_ids {
88 let table_id = region_id.table_id();
89 let region_number = region_id.region_number();
90 let datanode_table_value = datanode_table_values.get(&table_id).context(
91 error::DatanodeTableNotFoundSnafu {
92 table_id,
93 datanode_id: from_peer_id,
94 },
95 )?;
96 let RegionInfo {
97 region_storage_path,
98 region_options,
99 region_wal_options,
100 engine,
101 } = datanode_table_value.region_info.clone();
102
103 open_regions.push(OpenRegion::new(
104 RegionIdent {
105 datanode_id: to_peer_id,
106 table_id,
107 region_number,
108 engine,
109 },
110 ®ion_storage_path,
111 region_options,
112 region_wal_options,
113 true,
114 Some(reason),
115 RegionRequirements::object_storage(),
116 ));
117 }
118
119 Ok(Instruction::OpenRegions(open_regions))
120 }
121
122 async fn open_candidate_region(
133 &self,
134 ctx: &mut Context,
135 open_instruction: Instruction,
136 ) -> Result<()> {
137 let pc = &ctx.persistent_ctx;
138 let vc = &mut ctx.volatile_ctx;
139 let region_ids = &pc.region_ids;
140 let candidate = &pc.to_peer;
141
142 if vc.opening_region_guards.is_empty() {
145 for region_id in region_ids {
146 let guard = ctx
148 .opening_region_keeper
149 .register_candidate_region(candidate.id, *region_id)
150 .context(error::RegionOperatingRaceSnafu {
151 peer_id: candidate.id,
152 region_id: *region_id,
153 })?;
154 vc.opening_region_guards.push(guard);
155 }
156 }
157
158 let tracing_ctx = TracingContext::from_current_span();
159 let msg = MailboxMessage::json_message(
160 &format!("Open candidate regions: {:?}", region_ids),
161 &format!("Metasrv@{}", ctx.server_addr()),
162 &format!("Datanode-{}@{}", candidate.id, candidate.addr),
163 common_time::util::current_time_millis(),
164 &open_instruction,
165 Some(tracing_ctx.to_w3c()),
166 )
167 .with_context(|_| error::SerializeToJsonSnafu {
168 input: open_instruction.to_string(),
169 })?;
170
171 let operation_timeout =
172 ctx.next_operation_timeout()
173 .context(error::ExceededDeadlineSnafu {
174 operation: "Open candidate region",
175 })?;
176 let operation_timeout = operation_timeout
177 .div(2)
178 .max(default_distributed_time_constants().region_lease);
179 let ch = Channel::Datanode(candidate.id);
180 let now = Instant::now();
181 let receiver = ctx.mailbox.send(&ch, msg, operation_timeout).await?;
182
183 match receiver.await {
184 Ok(msg) => {
185 let reply = HeartbeatMailbox::json_reply(&msg)?;
186 info!(
187 "Received open region reply: {:?}, region: {:?}, elapsed: {:?}",
188 reply,
189 region_ids,
190 now.elapsed()
191 );
192 let InstructionReply::OpenRegions(SimpleReply { result, error }) = reply else {
193 return error::UnexpectedInstructionReplySnafu {
194 mailbox_message: msg.to_string(),
195 reason: "expect open region reply",
196 }
197 .fail();
198 };
199
200 if result {
201 Ok(())
202 } else if let Some(error) = error {
203 instruction_error_result(
204 &error,
205 format!(
206 "Region {region_ids:?} is not opened by datanode {:?}, error: {error:?}, elapsed: {:?}",
207 candidate,
208 now.elapsed()
209 ),
210 )
211 } else {
212 error::UnexpectedSnafu {
213 violated: format!(
214 "Region {region_ids:?} is not opened by datanode {:?}, but error is absent, elapsed: {:?}",
215 candidate,
216 now.elapsed()
217 ),
218 }
219 .fail()
220 }
221 }
222 Err(error::Error::MailboxTimeout { .. }) => {
223 let reason = format!(
224 "Mailbox received timeout for open candidate region {region_ids:?} on datanode {:?}, elapsed: {:?}",
225 candidate,
226 now.elapsed()
227 );
228 error::RetryLaterSnafu { reason }.fail()
229 }
230 Err(e) => Err(e),
231 }
232 }
233}
234
235#[cfg(test)]
236mod tests {
237 use std::assert_matches;
238 use std::collections::{HashMap, HashSet};
239
240 use common_catalog::consts::MITO2_ENGINE;
241 use common_error::ext::RetryHint;
242 use common_error::status_code::StatusCode;
243 use common_meta::DatanodeId;
244 use common_meta::instruction::InstructionError;
245 use common_meta::key::table_route::TableRouteValue;
246 use common_meta::key::test_utils::new_test_table_info;
247 use common_meta::peer::Peer;
248 use common_meta::rpc::router::{Region, RegionRoute};
249 use store_api::region_engine::RegionRole;
250 use store_api::storage::RegionId;
251
252 use super::*;
253 use crate::error::Error;
254 use crate::procedure::region_migration::test_util::{self, TestingEnv, new_procedure_context};
255 use crate::procedure::region_migration::{ContextFactory, PersistentContext};
256 use crate::procedure::test_util::{
257 new_close_region_reply, new_open_region_reply, new_open_region_reply_with_error,
258 send_mock_reply,
259 };
260
261 fn new_persistent_context() -> PersistentContext {
262 test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
263 }
264
265 fn new_mock_open_instruction(datanode_id: DatanodeId, region_id: RegionId) -> Instruction {
266 Instruction::OpenRegions(vec![OpenRegion::new(
267 RegionIdent {
268 datanode_id,
269 table_id: region_id.table_id(),
270 region_number: region_id.region_number(),
271 engine: MITO2_ENGINE.to_string(),
272 },
273 "/bar/foo/region/",
274 Default::default(),
275 Default::default(),
276 true,
277 Some(OpenRegionReason::RegionMigration),
278 RegionRequirements::object_storage(),
279 )])
280 }
281
282 #[tokio::test]
283 async fn test_datanode_table_is_not_found_error() {
284 let state = OpenCandidateRegion;
285 let persistent_context = new_persistent_context();
286 let env = TestingEnv::new();
287 let mut ctx = env.context_factory().new_context(persistent_context);
288
289 let err = state
290 .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Unknown)
291 .await
292 .unwrap_err();
293
294 assert_matches!(err, Error::DatanodeTableNotFound { .. });
295 assert!(!err.is_retryable());
296 }
297
298 #[tokio::test]
299 async fn test_build_open_region_instruction_reason() {
300 let state = OpenCandidateRegion;
301 let persistent_context = new_persistent_context();
302 let from_peer_id = persistent_context.from_peer.id;
303 let region_id = persistent_context.region_ids[0];
304 let env = TestingEnv::new();
305
306 let table_info = new_test_table_info(1024);
307 let region_routes = vec![RegionRoute {
308 region: Region::new_test(region_id),
309 leader_peer: Some(Peer::empty(from_peer_id)),
310 ..Default::default()
311 }];
312 env.table_metadata_manager()
313 .create_table_metadata(
314 table_info,
315 TableRouteValue::physical(region_routes),
316 HashMap::default(),
317 )
318 .await
319 .unwrap();
320
321 let mut ctx = env
322 .context_factory()
323 .new_context(persistent_context.clone());
324 let instruction = state
325 .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Unknown)
326 .await
327 .unwrap();
328 let open_regions = instruction.into_open_regions().unwrap();
329 assert_eq!(
330 Some(OpenRegionReason::RegionMigration),
331 open_regions[0].reason
332 );
333 assert_eq!(
334 RegionRequirements::object_storage(),
335 open_regions[0].requirements
336 );
337
338 let mut ctx = env.context_factory().new_context(persistent_context);
339 let instruction = state
340 .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Failover)
341 .await
342 .unwrap();
343 let open_regions = instruction.into_open_regions().unwrap();
344 assert_eq!(
345 Some(OpenRegionReason::RegionFailover),
346 open_regions[0].reason
347 );
348 assert_eq!(
349 RegionRequirements::object_storage(),
350 open_regions[0].requirements
351 );
352 }
353
354 #[tokio::test]
355 async fn test_datanode_is_unreachable() {
356 let state = OpenCandidateRegion;
357 let persistent_context = new_persistent_context();
360 let region_id = persistent_context.region_ids[0];
361 let to_peer_id = persistent_context.to_peer.id;
362 let env = TestingEnv::new();
363 let mut ctx = env.context_factory().new_context(persistent_context);
364
365 let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
366 let err = state
367 .open_candidate_region(&mut ctx, open_instruction)
368 .await
369 .unwrap_err();
370
371 assert_matches!(err, Error::PusherNotFound { .. });
372 assert!(!err.is_retryable());
373 }
374
375 #[tokio::test]
376 async fn test_candidate_region_opening_error() {
377 let state = OpenCandidateRegion;
378 let persistent_context = new_persistent_context();
381 let region_id = persistent_context.region_ids[0];
382 let to_peer_id = persistent_context.to_peer.id;
383
384 let env = TestingEnv::new();
385 let mut ctx = env.context_factory().new_context(persistent_context);
386 let opening_region_keeper = env.opening_region_keeper();
387 let _guard = opening_region_keeper
388 .register_with_role(to_peer_id, region_id, RegionRole::Follower)
389 .unwrap();
390
391 let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
392 let err = state
393 .open_candidate_region(&mut ctx, open_instruction)
394 .await
395 .unwrap_err();
396
397 assert_matches!(err, Error::RegionOperatingRace { .. });
398 assert!(!err.is_retryable());
399 }
400
401 #[tokio::test]
402 async fn test_unexpected_instruction_reply() {
403 let state = OpenCandidateRegion;
404 let persistent_context = new_persistent_context();
407 let region_id = persistent_context.region_ids[0];
408 let to_peer_id = persistent_context.to_peer.id;
409
410 let mut env = TestingEnv::new();
411 let mut ctx = env.context_factory().new_context(persistent_context);
412 let mailbox_ctx = env.mailbox_context();
413 let mailbox = mailbox_ctx.mailbox().clone();
414
415 let (tx, rx) = tokio::sync::mpsc::channel(1);
416
417 mailbox_ctx
418 .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
419 .await;
420
421 send_mock_reply(mailbox, rx, |id| Ok(new_close_region_reply(id)));
423
424 let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
425 let err = state
426 .open_candidate_region(&mut ctx, open_instruction)
427 .await
428 .unwrap_err();
429
430 assert_matches!(err, Error::UnexpectedInstructionReply { .. });
431 assert!(!err.is_retryable());
432 }
433
434 #[tokio::test]
435 async fn test_instruction_exceeded_deadline() {
436 let state = OpenCandidateRegion;
437 let persistent_context = new_persistent_context();
440 let region_id = persistent_context.region_ids[0];
441 let to_peer_id = persistent_context.to_peer.id;
442
443 let mut env = TestingEnv::new();
444 let mut ctx = env.context_factory().new_context(persistent_context);
445 let mailbox_ctx = env.mailbox_context();
446 let mailbox = mailbox_ctx.mailbox().clone();
447
448 let (tx, rx) = tokio::sync::mpsc::channel(1);
449
450 mailbox_ctx
451 .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
452 .await;
453
454 send_mock_reply(mailbox, rx, |id| {
456 Err(error::MailboxTimeoutSnafu { id }.build())
457 });
458
459 let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
460 let err = state
461 .open_candidate_region(&mut ctx, open_instruction)
462 .await
463 .unwrap_err();
464
465 assert_matches!(err, Error::RetryLater { .. });
466 assert!(err.is_retryable());
467 }
468
469 #[tokio::test]
470 async fn test_open_candidate_region_failed() {
471 let state = OpenCandidateRegion;
472 let persistent_context = new_persistent_context();
475 let region_id = persistent_context.region_ids[0];
476 let to_peer_id = persistent_context.to_peer.id;
477 let mut env = TestingEnv::new();
478
479 let mut ctx = env.context_factory().new_context(persistent_context);
480 let mailbox_ctx = env.mailbox_context();
481 let mailbox = mailbox_ctx.mailbox().clone();
482
483 let (tx, rx) = tokio::sync::mpsc::channel(1);
484
485 mailbox_ctx
486 .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
487 .await;
488
489 send_mock_reply(mailbox, rx, |id| {
490 Ok(new_open_region_reply_with_error(
491 id,
492 false,
493 Some(InstructionError {
494 code: StatusCode::StorageUnavailable,
495 message: "test mocked".to_string(),
496 retry_hint: RetryHint::Retryable,
497 }),
498 ))
499 });
500
501 let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
502 let err = state
503 .open_candidate_region(&mut ctx, open_instruction)
504 .await
505 .unwrap_err();
506
507 assert_matches!(err, Error::RetryLater { .. });
508 assert!(err.is_retryable());
509 assert!(format!("{err:?}").contains("test mocked"));
510 }
511
512 #[tokio::test]
513 async fn test_open_candidate_region_non_retryable_instruction_error() {
514 let state = OpenCandidateRegion;
515 let persistent_context = new_persistent_context();
516 let region_id = persistent_context.region_ids[0];
517 let to_peer_id = persistent_context.to_peer.id;
518 let mut env = TestingEnv::new();
519
520 let mut ctx = env.context_factory().new_context(persistent_context);
521 let mailbox_ctx = env.mailbox_context();
522 let mailbox = mailbox_ctx.mailbox().clone();
523
524 let (tx, rx) = tokio::sync::mpsc::channel(1);
525
526 mailbox_ctx
527 .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
528 .await;
529
530 send_mock_reply(mailbox, rx, |id| {
531 Ok(new_open_region_reply_with_error(
532 id,
533 false,
534 Some(InstructionError {
535 code: StatusCode::Internal,
536 message: "non retryable mocked".to_string(),
537 retry_hint: RetryHint::NonRetryable,
538 }),
539 ))
540 });
541
542 let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
543 let err = state
544 .open_candidate_region(&mut ctx, open_instruction)
545 .await
546 .unwrap_err();
547
548 assert_matches!(err, Error::Unexpected { .. });
549 assert!(!err.is_retryable());
550 assert!(format!("{err:?}").contains("non retryable mocked"));
551 }
552
553 #[tokio::test]
554 async fn test_open_candidate_region_false_without_error_is_unexpected() {
555 let state = OpenCandidateRegion;
556 let persistent_context = new_persistent_context();
557 let region_id = persistent_context.region_ids[0];
558 let to_peer_id = persistent_context.to_peer.id;
559 let mut env = TestingEnv::new();
560
561 let mut ctx = env.context_factory().new_context(persistent_context);
562 let mailbox_ctx = env.mailbox_context();
563 let mailbox = mailbox_ctx.mailbox().clone();
564
565 let (tx, rx) = tokio::sync::mpsc::channel(1);
566
567 mailbox_ctx
568 .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
569 .await;
570
571 send_mock_reply(mailbox, rx, |id| {
572 Ok(new_open_region_reply_with_error(id, false, None))
573 });
574
575 let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
576 let err = state
577 .open_candidate_region(&mut ctx, open_instruction)
578 .await
579 .unwrap_err();
580
581 assert_matches!(err, Error::Unexpected { .. });
582 assert!(!err.is_retryable());
583 }
584
585 #[tokio::test]
586 async fn test_next_flush_leader_region_state() {
587 let mut state = Box::new(OpenCandidateRegion);
588 let persistent_context = new_persistent_context();
591 let from_peer_id = persistent_context.from_peer.id;
592 let region_id = persistent_context.region_ids[0];
593 let to_peer_id = persistent_context.to_peer.id;
594 let mut env = TestingEnv::new();
595
596 let table_info = new_test_table_info(1024);
598 let region_routes = vec![RegionRoute {
599 region: Region::new_test(region_id),
600 leader_peer: Some(Peer::empty(from_peer_id)),
601 ..Default::default()
602 }];
603
604 env.table_metadata_manager()
605 .create_table_metadata(
606 table_info,
607 TableRouteValue::physical(region_routes),
608 HashMap::default(),
609 )
610 .await
611 .unwrap();
612
613 let mut ctx = env.context_factory().new_context(persistent_context);
614 let mailbox_ctx = env.mailbox_context();
615 let mailbox = mailbox_ctx.mailbox().clone();
616
617 let (tx, rx) = tokio::sync::mpsc::channel(1);
618
619 mailbox_ctx
620 .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
621 .await;
622
623 send_mock_reply(mailbox, rx, |id| Ok(new_open_region_reply(id, true, None)));
624 let procedure_ctx = new_procedure_context();
625 let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
626 let candidate_lease_id = {
627 let vc = &ctx.volatile_ctx;
628 assert_eq!(vc.opening_region_guards[0].info(), (to_peer_id, region_id));
629 vc.opening_region_guards[0].candidate_lease_id()
630 };
631 assert!(candidate_lease_id.is_some_and(|id| id != 0));
632 let regions = HashSet::from([region_id]);
633 let registered = |ctx: &Context| {
634 ctx.opening_region_keeper
635 .extract_operating_regions(to_peer_id, ®ions)[®ion_id]
636 .candidate_lease_id
637 };
638 assert_eq!(registered(&ctx), candidate_lease_id);
639
640 state.next(&mut ctx, &procedure_ctx).await.unwrap();
642 assert_eq!(ctx.volatile_ctx.opening_region_guards.len(), 1);
643 assert_eq!(registered(&ctx), candidate_lease_id);
644
645 let flush_leader_region = next.as_any().downcast_ref::<PreFlushRegion>().unwrap();
646 assert_matches!(flush_leader_region, PreFlushRegion);
647 }
648}