Skip to main content

meta_srv/handler/
utils.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::future::Future;
16use std::sync::Arc;
17
18use api::v1::meta::{Peer, Role};
19use common_meta::instruction::CacheIdent;
20use common_meta::key::MetadataValue;
21use common_meta::key::node_address::NodeAddressValue;
22use common_meta::rpc::store::PutRequest;
23use common_telemetry::{error, info};
24use dashmap::DashMap;
25use snafu::ResultExt;
26use tokio::sync::Mutex;
27
28use crate::Result;
29use crate::error::{InvalidNodeInfoFormatSnafu, KvBackendSnafu};
30use crate::metasrv::Context;
31
32/// Serializes address changes per node and acknowledges only successful updates.
33/// Node information refreshed on every heartbeat has a separate lifecycle.
34#[derive(Debug, Default)]
35pub(crate) struct NodeAddressUpdater {
36    states: DashMap<(Role, u64), Arc<Mutex<NodeAddressEpochs>>>,
37}
38
39#[derive(Debug, Default)]
40struct NodeAddressEpochs {
41    /// Rejects stale heartbeats even after a newer epoch failed to save.
42    seen_epoch: Option<u64>,
43    /// Suppresses retries only after persistence succeeds.
44    saved_epoch: Option<u64>,
45}
46
47/// The outcome of a successful address save.
48pub(crate) struct SavedAddress<T> {
49    pub(crate) output: T,
50    /// An earlier save of this epoch failed or was cancelled, but its write
51    /// may still have reached the store.
52    pub(crate) retried: bool,
53}
54
55impl NodeAddressUpdater {
56    /// Saves a new or previously failed epoch. Returns `None` when the
57    /// heartbeat is stale or already saved.
58    /// Cache invalidation is separate and does not affect acknowledgement.
59    pub(crate) async fn update_if_needed<T, F, Fut>(
60        &self,
61        node: (Role, u64),
62        epoch: u64,
63        save: F,
64    ) -> Result<Option<SavedAddress<T>>>
65    where
66        F: FnOnce() -> Fut,
67        Fut: Future<Output = Result<T>>,
68    {
69        // Never retain a DashMap guard across await, even while waiting for a
70        // different node: its entry may live in the same shard.
71        let state = {
72            let entry = self.states.entry(node).or_default();
73            Arc::clone(entry.value())
74        };
75        let mut state = state.lock().await;
76        if state.seen_epoch.is_some_and(|seen| epoch < seen) || state.saved_epoch == Some(epoch) {
77            return Ok(None);
78        }
79
80        // Seeing an epoch rejects stale heartbeats, but only a successful save
81        // suppresses retries of that same epoch. Cancellation also leaves it retryable.
82        let retried = state.seen_epoch == Some(epoch);
83        state.seen_epoch = Some(epoch);
84        let output = save().await?;
85        state.saved_epoch = Some(epoch);
86        Ok(Some(SavedAddress { output, retried }))
87    }
88}
89
90/// Persists an address through the leader cache and returns the one it
91/// replaced, treating an undecodable value as absent.
92pub(crate) async fn save_node_address(
93    ctx: &Context,
94    key: Vec<u8>,
95    peer: Peer,
96) -> Result<Option<NodeAddressValue>> {
97    let address = NodeAddressValue::new(peer);
98    let value = address
99        .try_as_raw_value()
100        .context(InvalidNodeInfoFormatSnafu)?;
101    let resp = ctx
102        .leader_cached_kv_backend
103        .put(PutRequest {
104            key,
105            value,
106            prev_kv: true,
107        })
108        .await
109        .context(KvBackendSnafu)?;
110    info!("Successfully updated node address: {:?}", address.peer);
111    Ok(resp
112        .prev_kv
113        .and_then(|kv| NodeAddressValue::try_from_raw_value(&kv.value).ok()))
114}
115
116/// Preserves the existing best-effort notification behavior. Notification
117/// failures do not undo a persisted address or cause same-epoch rewrites.
118pub(crate) async fn invalidate_address_caches(
119    ctx: &Context,
120    node_id: u64,
121    cache_idents: &[CacheIdent],
122) {
123    if let Err(e) = ctx
124        .cache_invalidator
125        .invalidate(&Default::default(), cache_idents)
126        .await
127    {
128        error!(e; "Failed to invalidate address caches for node {}: {:?}", node_id, cache_idents);
129    }
130}
131
132#[cfg(test)]
133mod tests {
134    use std::time::Duration;
135
136    use tokio::sync::Notify;
137    use tokio::time::timeout;
138
139    use super::*;
140
141    #[tokio::test]
142    async fn test_failed_save_retries_same_epoch_and_rejects_stale_heartbeats() {
143        let updater = NodeAddressUpdater::default();
144        let key = (Role::Datanode, 1);
145        assert!(
146            updater
147                .update_if_needed::<(), _, _>(key, 2, || async {
148                    crate::error::UnexpectedSnafu {
149                        violated: "injected save failure",
150                    }
151                    .fail()
152                })
153                .await
154                .is_err()
155        );
156        assert!(
157            updater
158                .update_if_needed::<(), _, _>(key, 1, || async {
159                    panic!("stale heartbeat must not save");
160                })
161                .await
162                .unwrap()
163                .is_none()
164        );
165        assert!(
166            updater
167                .update_if_needed(key, 2, || async { Ok(()) })
168                .await
169                .unwrap()
170                .is_some_and(|saved| saved.retried)
171        );
172        assert!(
173            updater
174                .update_if_needed::<(), _, _>(key, 2, || async {
175                    panic!("successful epoch must not save again");
176                })
177                .await
178                .unwrap()
179                .is_none()
180        );
181        assert!(
182            updater
183                .update_if_needed(key, 3, || async { Ok(()) })
184                .await
185                .unwrap()
186                .is_some_and(|saved| !saved.retried)
187        );
188    }
189
190    #[tokio::test]
191    async fn test_cancelled_save_can_be_retried() {
192        let updater = NodeAddressUpdater::default();
193        let key = (Role::Datanode, 1);
194        let entered = Notify::new();
195        {
196            let first = updater.update_if_needed(key, 0, || async {
197                entered.notify_one();
198                std::future::pending::<Result<()>>().await
199            });
200            tokio::pin!(first);
201            tokio::select! {
202                _ = entered.notified() => {},
203                _ = &mut first => panic!("save must remain pending"),
204            }
205        }
206        assert!(
207            timeout(
208                Duration::from_secs(2),
209                updater.update_if_needed(key, 0, || async { Ok(()) })
210            )
211            .await
212            .unwrap()
213            .unwrap()
214            .is_some()
215        );
216    }
217
218    #[test]
219    fn test_same_node_serialized_other_nodes_progress() {
220        // A watchdog outside the runtime detects synchronous DashMap deadlocks;
221        // a Tokio timeout alone cannot fire if the only worker is blocked.
222        let (progress_tx, progress_rx) = std::sync::mpsc::sync_channel(1);
223        let release = Arc::new(Notify::new());
224        let finish = Arc::new(Notify::new());
225        let worker = std::thread::spawn({
226            let release = release.clone();
227            let finish = finish.clone();
228            move || {
229                tokio::runtime::Builder::new_current_thread()
230                    .enable_time()
231                    .build()
232                    .unwrap()
233                    .block_on(async move {
234                        let updater = Arc::new(NodeAddressUpdater::default());
235                        let entered = Arc::new(Notify::new());
236
237                        let first = tokio::spawn({
238                            let updater = updater.clone();
239                            let entered = entered.clone();
240                            let release = release.clone();
241                            async move {
242                                updater
243                                    .update_if_needed((Role::Datanode, 1), 1, || async {
244                                        entered.notify_one();
245                                        release.notified().await;
246                                        Ok(())
247                                    })
248                                    .await
249                            }
250                        });
251                        entered.notified().await;
252                        let second = tokio::spawn({
253                            let updater = updater.clone();
254                            async move {
255                                updater
256                                    .update_if_needed((Role::Datanode, 1), 2, || async { Ok(()) })
257                                    .await
258                            }
259                        });
260                        tokio::task::yield_now().await;
261                        assert!(!second.is_finished());
262                        // Exercise shared DashMap shards on a single-thread runtime while the
263                        // first node is suspended in external I/O.
264                        timeout(Duration::from_secs(2), async {
265                            for id in 0_u64..128 {
266                                assert!(
267                                    updater
268                                        .update_if_needed((Role::Datanode, id + 2), 1, || async {
269                                            Ok(())
270                                        })
271                                        .await
272                                        .unwrap()
273                                        .is_some()
274                                );
275                            }
276                        })
277                        .await
278                        .unwrap();
279                        progress_tx.send(()).unwrap();
280                        finish.notified().await;
281                        timeout(Duration::from_secs(2), async {
282                            assert!(first.await.unwrap().unwrap().is_some());
283                            assert!(second.await.unwrap().unwrap().is_some());
284                        })
285                        .await
286                        .unwrap();
287                        assert!(
288                            updater
289                                .update_if_needed::<(), _, _>((Role::Datanode, 1), 1, || async {
290                                    panic!("old epoch must not overwrite a newer one");
291                                })
292                                .await
293                                .unwrap()
294                                .is_none()
295                        );
296                    });
297            }
298        });
299        let progress = progress_rx.recv_timeout(Duration::from_secs(5));
300        release.notify_one();
301        finish.notify_one();
302        progress.expect("other node updates must progress while one node save is blocked");
303        worker.join().unwrap();
304    }
305}