Skip to main content

meta_srv/procedure/region_migration/
open_candidate_region.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::ops::Div;
17
18use api::v1::meta::MailboxMessage;
19use common_meta::RegionIdent;
20use common_meta::distributed_time_constants::default_distributed_time_constants;
21use common_meta::instruction::{
22    Instruction, InstructionReply, OpenRegion, OpenRegionReason, SimpleReply,
23};
24use common_meta::key::datanode_table::RegionInfo;
25use common_procedure::{Context as ProcedureContext, Status};
26use common_telemetry::info;
27use common_telemetry::tracing_context::TracingContext;
28use serde::{Deserialize, Serialize};
29use snafu::{OptionExt, ResultExt};
30use store_api::region_request::RegionRequirements;
31use tokio::time::Instant;
32
33use crate::error::{self, Result};
34use crate::handler::HeartbeatMailbox;
35use crate::procedure::region_migration::flush_leader_region::PreFlushRegion;
36use crate::procedure::region_migration::{Context, RegionMigrationTriggerReason, State};
37use crate::procedure::utils::instruction_error_result;
38use crate::service::mailbox::Channel;
39
40#[derive(Debug, Serialize, Deserialize)]
41pub struct OpenCandidateRegion;
42
43#[async_trait::async_trait]
44#[typetag::serde]
45impl State for OpenCandidateRegion {
46    async fn next(
47        &mut self,
48        ctx: &mut Context,
49        procedure_ctx: &ProcedureContext,
50    ) -> Result<(Box<dyn State>, Status)> {
51        let trigger_reason = ctx.trigger_reason(procedure_ctx.event_context.as_ref());
52        let instruction = self
53            .build_open_region_instruction(ctx, trigger_reason)
54            .await?;
55        let now = Instant::now();
56        self.open_candidate_region(ctx, instruction).await?;
57        ctx.update_open_candidate_region_elapsed(now);
58
59        Ok((Box::new(PreFlushRegion), Status::executing(false)))
60    }
61
62    fn as_any(&self) -> &dyn Any {
63        self
64    }
65}
66
67impl OpenCandidateRegion {
68    /// Builds open region instructions
69    ///
70    /// Abort(non-retry):
71    /// - Datanode Table is not found.
72    async fn build_open_region_instruction(
73        &self,
74        ctx: &mut Context,
75        trigger_reason: RegionMigrationTriggerReason,
76    ) -> Result<Instruction> {
77        let region_ids = ctx.persistent_ctx.region_ids.clone();
78        let from_peer_id = ctx.persistent_ctx.from_peer.id;
79        let to_peer_id = ctx.persistent_ctx.to_peer.id;
80        let reason = match trigger_reason {
81            RegionMigrationTriggerReason::Failover => OpenRegionReason::RegionFailover,
82            _ => OpenRegionReason::RegionMigration,
83        };
84        let datanode_table_values = ctx.get_from_peer_datanode_table_values().await?;
85        let mut open_regions = Vec::with_capacity(region_ids.len());
86
87        for region_id in region_ids {
88            let table_id = region_id.table_id();
89            let region_number = region_id.region_number();
90            let datanode_table_value = datanode_table_values.get(&table_id).context(
91                error::DatanodeTableNotFoundSnafu {
92                    table_id,
93                    datanode_id: from_peer_id,
94                },
95            )?;
96            let RegionInfo {
97                region_storage_path,
98                region_options,
99                region_wal_options,
100                engine,
101            } = datanode_table_value.region_info.clone();
102
103            open_regions.push(OpenRegion::new(
104                RegionIdent {
105                    datanode_id: to_peer_id,
106                    table_id,
107                    region_number,
108                    engine,
109                },
110                &region_storage_path,
111                region_options,
112                region_wal_options,
113                true,
114                Some(reason),
115                RegionRequirements::object_storage(),
116            ));
117        }
118
119        Ok(Instruction::OpenRegions(open_regions))
120    }
121
122    /// Opens the candidate region.
123    ///
124    /// Abort(non-retry):
125    /// - The Datanode is unreachable(e.g., Candidate pusher is not found).
126    /// - Unexpected instruction reply.
127    /// - Another procedure is opening the candidate region.
128    ///
129    /// Retry:
130    /// - Exceeded deadline of open instruction.
131    /// - Datanode failed to open the candidate region.
132    async fn open_candidate_region(
133        &self,
134        ctx: &mut Context,
135        open_instruction: Instruction,
136    ) -> Result<()> {
137        let pc = &ctx.persistent_ctx;
138        let vc = &mut ctx.volatile_ctx;
139        let region_ids = &pc.region_ids;
140        let candidate = &pc.to_peer;
141
142        // This method might be invoked multiple times.
143        // Only registers the guard if `opening_region_guard` is absent.
144        if vc.opening_region_guards.is_empty() {
145            for region_id in region_ids {
146                // Registers the opening region.
147                let guard = ctx
148                    .opening_region_keeper
149                    .register_candidate_region(candidate.id, *region_id)
150                    .context(error::RegionOperatingRaceSnafu {
151                        peer_id: candidate.id,
152                        region_id: *region_id,
153                    })?;
154                vc.opening_region_guards.push(guard);
155            }
156        }
157
158        let tracing_ctx = TracingContext::from_current_span();
159        let msg = MailboxMessage::json_message(
160            &format!("Open candidate regions: {:?}", region_ids),
161            &format!("Metasrv@{}", ctx.server_addr()),
162            &format!("Datanode-{}@{}", candidate.id, candidate.addr),
163            common_time::util::current_time_millis(),
164            &open_instruction,
165            Some(tracing_ctx.to_w3c()),
166        )
167        .with_context(|_| error::SerializeToJsonSnafu {
168            input: open_instruction.to_string(),
169        })?;
170
171        let operation_timeout =
172            ctx.next_operation_timeout()
173                .context(error::ExceededDeadlineSnafu {
174                    operation: "Open candidate region",
175                })?;
176        let operation_timeout = operation_timeout
177            .div(2)
178            .max(default_distributed_time_constants().region_lease);
179        let ch = Channel::Datanode(candidate.id);
180        let now = Instant::now();
181        let receiver = ctx.mailbox.send(&ch, msg, operation_timeout).await?;
182
183        match receiver.await {
184            Ok(msg) => {
185                let reply = HeartbeatMailbox::json_reply(&msg)?;
186                info!(
187                    "Received open region reply: {:?}, region: {:?}, elapsed: {:?}",
188                    reply,
189                    region_ids,
190                    now.elapsed()
191                );
192                let InstructionReply::OpenRegions(SimpleReply { result, error }) = reply else {
193                    return error::UnexpectedInstructionReplySnafu {
194                        mailbox_message: msg.to_string(),
195                        reason: "expect open region reply",
196                    }
197                    .fail();
198                };
199
200                if result {
201                    Ok(())
202                } else if let Some(error) = error {
203                    instruction_error_result(
204                        &error,
205                        format!(
206                            "Region {region_ids:?} is not opened by datanode {:?}, error: {error:?}, elapsed: {:?}",
207                            candidate,
208                            now.elapsed()
209                        ),
210                    )
211                } else {
212                    error::UnexpectedSnafu {
213                        violated: format!(
214                            "Region {region_ids:?} is not opened by datanode {:?}, but error is absent, elapsed: {:?}",
215                            candidate,
216                            now.elapsed()
217                        ),
218                    }
219                    .fail()
220                }
221            }
222            Err(error::Error::MailboxTimeout { .. }) => {
223                let reason = format!(
224                    "Mailbox received timeout for open candidate region {region_ids:?} on datanode {:?}, elapsed: {:?}",
225                    candidate,
226                    now.elapsed()
227                );
228                error::RetryLaterSnafu { reason }.fail()
229            }
230            Err(e) => Err(e),
231        }
232    }
233}
234
235#[cfg(test)]
236mod tests {
237    use std::assert_matches;
238    use std::collections::{HashMap, HashSet};
239
240    use common_catalog::consts::MITO2_ENGINE;
241    use common_error::ext::RetryHint;
242    use common_error::status_code::StatusCode;
243    use common_meta::DatanodeId;
244    use common_meta::instruction::InstructionError;
245    use common_meta::key::table_route::TableRouteValue;
246    use common_meta::key::test_utils::new_test_table_info;
247    use common_meta::peer::Peer;
248    use common_meta::rpc::router::{Region, RegionRoute};
249    use store_api::region_engine::RegionRole;
250    use store_api::storage::RegionId;
251
252    use super::*;
253    use crate::error::Error;
254    use crate::procedure::region_migration::test_util::{self, TestingEnv, new_procedure_context};
255    use crate::procedure::region_migration::{ContextFactory, PersistentContext};
256    use crate::procedure::test_util::{
257        new_close_region_reply, new_open_region_reply, new_open_region_reply_with_error,
258        send_mock_reply,
259    };
260
261    fn new_persistent_context() -> PersistentContext {
262        test_util::new_persistent_context(1, 2, RegionId::new(1024, 1))
263    }
264
265    fn new_mock_open_instruction(datanode_id: DatanodeId, region_id: RegionId) -> Instruction {
266        Instruction::OpenRegions(vec![OpenRegion::new(
267            RegionIdent {
268                datanode_id,
269                table_id: region_id.table_id(),
270                region_number: region_id.region_number(),
271                engine: MITO2_ENGINE.to_string(),
272            },
273            "/bar/foo/region/",
274            Default::default(),
275            Default::default(),
276            true,
277            Some(OpenRegionReason::RegionMigration),
278            RegionRequirements::object_storage(),
279        )])
280    }
281
282    #[tokio::test]
283    async fn test_datanode_table_is_not_found_error() {
284        let state = OpenCandidateRegion;
285        let persistent_context = new_persistent_context();
286        let env = TestingEnv::new();
287        let mut ctx = env.context_factory().new_context(persistent_context);
288
289        let err = state
290            .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Unknown)
291            .await
292            .unwrap_err();
293
294        assert_matches!(err, Error::DatanodeTableNotFound { .. });
295        assert!(!err.is_retryable());
296    }
297
298    #[tokio::test]
299    async fn test_build_open_region_instruction_reason() {
300        let state = OpenCandidateRegion;
301        let persistent_context = new_persistent_context();
302        let from_peer_id = persistent_context.from_peer.id;
303        let region_id = persistent_context.region_ids[0];
304        let env = TestingEnv::new();
305
306        let table_info = new_test_table_info(1024);
307        let region_routes = vec![RegionRoute {
308            region: Region::new_test(region_id),
309            leader_peer: Some(Peer::empty(from_peer_id)),
310            ..Default::default()
311        }];
312        env.table_metadata_manager()
313            .create_table_metadata(
314                table_info,
315                TableRouteValue::physical(region_routes),
316                HashMap::default(),
317            )
318            .await
319            .unwrap();
320
321        let mut ctx = env
322            .context_factory()
323            .new_context(persistent_context.clone());
324        let instruction = state
325            .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Unknown)
326            .await
327            .unwrap();
328        let open_regions = instruction.into_open_regions().unwrap();
329        assert_eq!(
330            Some(OpenRegionReason::RegionMigration),
331            open_regions[0].reason
332        );
333        assert_eq!(
334            RegionRequirements::object_storage(),
335            open_regions[0].requirements
336        );
337
338        let mut ctx = env.context_factory().new_context(persistent_context);
339        let instruction = state
340            .build_open_region_instruction(&mut ctx, RegionMigrationTriggerReason::Failover)
341            .await
342            .unwrap();
343        let open_regions = instruction.into_open_regions().unwrap();
344        assert_eq!(
345            Some(OpenRegionReason::RegionFailover),
346            open_regions[0].reason
347        );
348        assert_eq!(
349            RegionRequirements::object_storage(),
350            open_regions[0].requirements
351        );
352    }
353
354    #[tokio::test]
355    async fn test_datanode_is_unreachable() {
356        let state = OpenCandidateRegion;
357        // from_peer: 1
358        // to_peer: 2
359        let persistent_context = new_persistent_context();
360        let region_id = persistent_context.region_ids[0];
361        let to_peer_id = persistent_context.to_peer.id;
362        let env = TestingEnv::new();
363        let mut ctx = env.context_factory().new_context(persistent_context);
364
365        let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
366        let err = state
367            .open_candidate_region(&mut ctx, open_instruction)
368            .await
369            .unwrap_err();
370
371        assert_matches!(err, Error::PusherNotFound { .. });
372        assert!(!err.is_retryable());
373    }
374
375    #[tokio::test]
376    async fn test_candidate_region_opening_error() {
377        let state = OpenCandidateRegion;
378        // from_peer: 1
379        // to_peer: 2
380        let persistent_context = new_persistent_context();
381        let region_id = persistent_context.region_ids[0];
382        let to_peer_id = persistent_context.to_peer.id;
383
384        let env = TestingEnv::new();
385        let mut ctx = env.context_factory().new_context(persistent_context);
386        let opening_region_keeper = env.opening_region_keeper();
387        let _guard = opening_region_keeper
388            .register_with_role(to_peer_id, region_id, RegionRole::Follower)
389            .unwrap();
390
391        let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
392        let err = state
393            .open_candidate_region(&mut ctx, open_instruction)
394            .await
395            .unwrap_err();
396
397        assert_matches!(err, Error::RegionOperatingRace { .. });
398        assert!(!err.is_retryable());
399    }
400
401    #[tokio::test]
402    async fn test_unexpected_instruction_reply() {
403        let state = OpenCandidateRegion;
404        // from_peer: 1
405        // to_peer: 2
406        let persistent_context = new_persistent_context();
407        let region_id = persistent_context.region_ids[0];
408        let to_peer_id = persistent_context.to_peer.id;
409
410        let mut env = TestingEnv::new();
411        let mut ctx = env.context_factory().new_context(persistent_context);
412        let mailbox_ctx = env.mailbox_context();
413        let mailbox = mailbox_ctx.mailbox().clone();
414
415        let (tx, rx) = tokio::sync::mpsc::channel(1);
416
417        mailbox_ctx
418            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
419            .await;
420
421        // Sends an incorrect reply.
422        send_mock_reply(mailbox, rx, |id| Ok(new_close_region_reply(id)));
423
424        let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
425        let err = state
426            .open_candidate_region(&mut ctx, open_instruction)
427            .await
428            .unwrap_err();
429
430        assert_matches!(err, Error::UnexpectedInstructionReply { .. });
431        assert!(!err.is_retryable());
432    }
433
434    #[tokio::test]
435    async fn test_instruction_exceeded_deadline() {
436        let state = OpenCandidateRegion;
437        // from_peer: 1
438        // to_peer: 2
439        let persistent_context = new_persistent_context();
440        let region_id = persistent_context.region_ids[0];
441        let to_peer_id = persistent_context.to_peer.id;
442
443        let mut env = TestingEnv::new();
444        let mut ctx = env.context_factory().new_context(persistent_context);
445        let mailbox_ctx = env.mailbox_context();
446        let mailbox = mailbox_ctx.mailbox().clone();
447
448        let (tx, rx) = tokio::sync::mpsc::channel(1);
449
450        mailbox_ctx
451            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
452            .await;
453
454        // Sends an timeout error.
455        send_mock_reply(mailbox, rx, |id| {
456            Err(error::MailboxTimeoutSnafu { id }.build())
457        });
458
459        let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
460        let err = state
461            .open_candidate_region(&mut ctx, open_instruction)
462            .await
463            .unwrap_err();
464
465        assert_matches!(err, Error::RetryLater { .. });
466        assert!(err.is_retryable());
467    }
468
469    #[tokio::test]
470    async fn test_open_candidate_region_failed() {
471        let state = OpenCandidateRegion;
472        // from_peer: 1
473        // to_peer: 2
474        let persistent_context = new_persistent_context();
475        let region_id = persistent_context.region_ids[0];
476        let to_peer_id = persistent_context.to_peer.id;
477        let mut env = TestingEnv::new();
478
479        let mut ctx = env.context_factory().new_context(persistent_context);
480        let mailbox_ctx = env.mailbox_context();
481        let mailbox = mailbox_ctx.mailbox().clone();
482
483        let (tx, rx) = tokio::sync::mpsc::channel(1);
484
485        mailbox_ctx
486            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
487            .await;
488
489        send_mock_reply(mailbox, rx, |id| {
490            Ok(new_open_region_reply_with_error(
491                id,
492                false,
493                Some(InstructionError {
494                    code: StatusCode::StorageUnavailable,
495                    message: "test mocked".to_string(),
496                    retry_hint: RetryHint::Retryable,
497                }),
498            ))
499        });
500
501        let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
502        let err = state
503            .open_candidate_region(&mut ctx, open_instruction)
504            .await
505            .unwrap_err();
506
507        assert_matches!(err, Error::RetryLater { .. });
508        assert!(err.is_retryable());
509        assert!(format!("{err:?}").contains("test mocked"));
510    }
511
512    #[tokio::test]
513    async fn test_open_candidate_region_non_retryable_instruction_error() {
514        let state = OpenCandidateRegion;
515        let persistent_context = new_persistent_context();
516        let region_id = persistent_context.region_ids[0];
517        let to_peer_id = persistent_context.to_peer.id;
518        let mut env = TestingEnv::new();
519
520        let mut ctx = env.context_factory().new_context(persistent_context);
521        let mailbox_ctx = env.mailbox_context();
522        let mailbox = mailbox_ctx.mailbox().clone();
523
524        let (tx, rx) = tokio::sync::mpsc::channel(1);
525
526        mailbox_ctx
527            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
528            .await;
529
530        send_mock_reply(mailbox, rx, |id| {
531            Ok(new_open_region_reply_with_error(
532                id,
533                false,
534                Some(InstructionError {
535                    code: StatusCode::Internal,
536                    message: "non retryable mocked".to_string(),
537                    retry_hint: RetryHint::NonRetryable,
538                }),
539            ))
540        });
541
542        let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
543        let err = state
544            .open_candidate_region(&mut ctx, open_instruction)
545            .await
546            .unwrap_err();
547
548        assert_matches!(err, Error::Unexpected { .. });
549        assert!(!err.is_retryable());
550        assert!(format!("{err:?}").contains("non retryable mocked"));
551    }
552
553    #[tokio::test]
554    async fn test_open_candidate_region_false_without_error_is_unexpected() {
555        let state = OpenCandidateRegion;
556        let persistent_context = new_persistent_context();
557        let region_id = persistent_context.region_ids[0];
558        let to_peer_id = persistent_context.to_peer.id;
559        let mut env = TestingEnv::new();
560
561        let mut ctx = env.context_factory().new_context(persistent_context);
562        let mailbox_ctx = env.mailbox_context();
563        let mailbox = mailbox_ctx.mailbox().clone();
564
565        let (tx, rx) = tokio::sync::mpsc::channel(1);
566
567        mailbox_ctx
568            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
569            .await;
570
571        send_mock_reply(mailbox, rx, |id| {
572            Ok(new_open_region_reply_with_error(id, false, None))
573        });
574
575        let open_instruction = new_mock_open_instruction(to_peer_id, region_id);
576        let err = state
577            .open_candidate_region(&mut ctx, open_instruction)
578            .await
579            .unwrap_err();
580
581        assert_matches!(err, Error::Unexpected { .. });
582        assert!(!err.is_retryable());
583    }
584
585    #[tokio::test]
586    async fn test_next_flush_leader_region_state() {
587        let mut state = Box::new(OpenCandidateRegion);
588        // from_peer: 1
589        // to_peer: 2
590        let persistent_context = new_persistent_context();
591        let from_peer_id = persistent_context.from_peer.id;
592        let region_id = persistent_context.region_ids[0];
593        let to_peer_id = persistent_context.to_peer.id;
594        let mut env = TestingEnv::new();
595
596        // Prepares table
597        let table_info = new_test_table_info(1024);
598        let region_routes = vec![RegionRoute {
599            region: Region::new_test(region_id),
600            leader_peer: Some(Peer::empty(from_peer_id)),
601            ..Default::default()
602        }];
603
604        env.table_metadata_manager()
605            .create_table_metadata(
606                table_info,
607                TableRouteValue::physical(region_routes),
608                HashMap::default(),
609            )
610            .await
611            .unwrap();
612
613        let mut ctx = env.context_factory().new_context(persistent_context);
614        let mailbox_ctx = env.mailbox_context();
615        let mailbox = mailbox_ctx.mailbox().clone();
616
617        let (tx, rx) = tokio::sync::mpsc::channel(1);
618
619        mailbox_ctx
620            .insert_heartbeat_response_receiver(Channel::Datanode(to_peer_id), tx)
621            .await;
622
623        send_mock_reply(mailbox, rx, |id| Ok(new_open_region_reply(id, true, None)));
624        let procedure_ctx = new_procedure_context();
625        let (next, _) = state.next(&mut ctx, &procedure_ctx).await.unwrap();
626        let candidate_lease_id = {
627            let vc = &ctx.volatile_ctx;
628            assert_eq!(vc.opening_region_guards[0].info(), (to_peer_id, region_id));
629            vc.opening_region_guards[0].candidate_lease_id()
630        };
631        assert!(candidate_lease_id.is_some_and(|id| id != 0));
632        let regions = HashSet::from([region_id]);
633        let registered = |ctx: &Context| {
634            ctx.opening_region_keeper
635                .extract_operating_regions(to_peer_id, &regions)[&region_id]
636                .candidate_lease_id
637        };
638        assert_eq!(registered(&ctx), candidate_lease_id);
639
640        // A retry of the step keeps the registration and its id.
641        state.next(&mut ctx, &procedure_ctx).await.unwrap();
642        assert_eq!(ctx.volatile_ctx.opening_region_guards.len(), 1);
643        assert_eq!(registered(&ctx), candidate_lease_id);
644
645        let flush_leader_region = next.as_any().downcast_ref::<PreFlushRegion>().unwrap();
646        assert_matches!(flush_leader_region, PreFlushRegion);
647    }
648}