Skip to main content

meta_srv/handler/
failure_handler.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use api::v1::meta::{HeartbeatRequest, Role};
16use async_trait::async_trait;
17use common_telemetry::info;
18use store_api::storage::RegionId;
19
20use crate::error::Result;
21use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
22use crate::metasrv::Context;
23use crate::region::supervisor::{DatanodeHeartbeat, HeartbeatAcceptor, RegionSupervisor};
24
25pub struct RegionFailureHandler {
26    heartbeat_acceptor: HeartbeatAcceptor,
27}
28
29impl RegionFailureHandler {
30    pub(crate) fn new(
31        mut region_supervisor: RegionSupervisor,
32        heartbeat_acceptor: HeartbeatAcceptor,
33    ) -> Self {
34        info!("Starting region supervisor");
35        common_runtime::spawn_global(async move { region_supervisor.run().await });
36        Self { heartbeat_acceptor }
37    }
38}
39
40#[async_trait]
41impl HeartbeatHandler for RegionFailureHandler {
42    fn is_acceptable(&self, role: Role) -> bool {
43        role == Role::Datanode
44    }
45
46    async fn handle(
47        &self,
48        req: &HeartbeatRequest,
49        _ctx: &mut Context,
50        acc: &mut HeartbeatAccumulator,
51    ) -> Result<HandleControl> {
52        let Some(stat) = acc.stat.as_ref() else {
53            return Ok(HandleControl::Continue);
54        };
55
56        // Never blocks: the region lease has already been decided earlier in the
57        // chain, so a busy supervisor must not withhold the heartbeat response.
58        let mut heartbeat = DatanodeHeartbeat::from(stat);
59        heartbeat.opening_regions = req
60            .opening_regions
61            .iter()
62            .map(|region| RegionId::from_u64(region.region_id))
63            .collect();
64        self.heartbeat_acceptor.accept(heartbeat);
65
66        Ok(HandleControl::Continue)
67    }
68}
69
70#[cfg(test)]
71mod tests {
72    use std::time::Duration;
73
74    use api::v1::meta::HeartbeatRequest;
75    use common_catalog::consts::default_engine;
76    use common_meta::datanode::{RegionManifestInfo, RegionStat, Stat};
77    use store_api::region_engine::RegionRole;
78    use store_api::storage::RegionId;
79    use tokio::sync::oneshot;
80
81    use crate::handler::failure_handler::RegionFailureHandler;
82    use crate::handler::{HeartbeatAccumulator, HeartbeatHandler};
83    use crate::metasrv::builder::MetasrvBuilder;
84    use crate::region::supervisor::tests::new_test_supervisor;
85    use crate::region::supervisor::{DatanodeHeartbeat, Event, HeartbeatAcceptor};
86
87    fn new_region_stat(region_id: u64) -> RegionStat {
88        RegionStat {
89            id: RegionId::from_u64(region_id),
90            rcus: 0,
91            wcus: 0,
92            approximate_bytes: 0,
93            engine: default_engine().to_string(),
94            role: RegionRole::Follower,
95            num_rows: 0,
96            memtable_size: 0,
97            manifest_size: 0,
98            sst_size: 0,
99            sst_num: 0,
100            index_size: 0,
101            region_manifest: RegionManifestInfo::Mito {
102                manifest_version: 0,
103                flushed_entry_id: 0,
104                file_removed_cnt: 0,
105            },
106            data_topic_latest_entry_id: 0,
107            metadata_topic_latest_entry_id: 0,
108            written_bytes: 0,
109            ingested_parquet_bytes: 0,
110            query_cpu_time: 0,
111            query_scanned_bytes: 0,
112            min_timestamp: None,
113            max_timestamp: None,
114            has_pending_tails: false,
115        }
116    }
117
118    fn new_stat() -> Stat {
119        Stat {
120            id: 42,
121            region_stats: vec![new_region_stat(1), new_region_stat(2), new_region_stat(3)],
122            timestamp_millis: 1000,
123            ..Default::default()
124        }
125    }
126
127    #[tokio::test]
128    async fn test_handle_heartbeat() {
129        let (supervisor, sender) = new_test_supervisor();
130        let heartbeat_acceptor = HeartbeatAcceptor::new(sender.clone());
131        let handler = RegionFailureHandler::new(supervisor, heartbeat_acceptor);
132        let req = &HeartbeatRequest::default();
133        let builder = MetasrvBuilder::new();
134        let metasrv = builder.build().await.unwrap();
135        let mut ctx = metasrv.new_ctx();
136        let acc = &mut HeartbeatAccumulator::default();
137        acc.stat = Some(new_stat());
138
139        handler.handle(req, &mut ctx, acc).await.unwrap();
140        let (tx, rx) = oneshot::channel();
141        sender.send(Event::Dump(tx)).await.unwrap();
142        let detector = rx.await.unwrap();
143        assert_eq!(detector.iter().collect::<Vec<_>>().len(), 3);
144    }
145
146    #[tokio::test]
147    async fn test_handle_heartbeat_does_not_block_on_full_supervisor_queue() {
148        // Regression test for #9419: a full queue used to park the handler.
149        // Keep the supervisor alive, or the queue reports Closed.
150        let (_supervisor, sender) = new_test_supervisor();
151        // Built directly, so the supervisor stays undrained.
152        let handler = RegionFailureHandler {
153            heartbeat_acceptor: HeartbeatAcceptor::new(sender.clone()),
154        };
155
156        let stat = new_stat();
157        let mut queued = 0;
158        while sender
159            .try_send(Event::HeartbeatArrived(DatanodeHeartbeat::from(&stat)))
160            .is_ok()
161        {
162            queued += 1;
163        }
164        assert_eq!(queued, sender.max_capacity());
165
166        let builder = MetasrvBuilder::new();
167        let metasrv = builder.build().await.unwrap();
168        let mut ctx = metasrv.new_ctx();
169        let acc = &mut HeartbeatAccumulator::default();
170        // Without a stat the handler returns before the acceptor.
171        acc.stat = Some(stat);
172
173        let result = tokio::time::timeout(
174            Duration::from_secs(5),
175            handler.handle(&HeartbeatRequest::default(), &mut ctx, acc),
176        )
177        .await;
178        assert!(result.is_ok());
179    }
180}