meta_srv/handler/
utils.rs1use std::future::Future;
16use std::sync::Arc;
17
18use api::v1::meta::{Peer, Role};
19use common_meta::instruction::CacheIdent;
20use common_meta::key::MetadataValue;
21use common_meta::key::node_address::NodeAddressValue;
22use common_meta::rpc::store::PutRequest;
23use common_telemetry::{error, info};
24use dashmap::DashMap;
25use snafu::ResultExt;
26use tokio::sync::Mutex;
27
28use crate::Result;
29use crate::error::{InvalidNodeInfoFormatSnafu, KvBackendSnafu};
30use crate::metasrv::Context;
31
32#[derive(Debug, Default)]
35pub(crate) struct NodeAddressUpdater {
36 states: DashMap<(Role, u64), Arc<Mutex<NodeAddressEpochs>>>,
37}
38
39#[derive(Debug, Default)]
40struct NodeAddressEpochs {
41 seen_epoch: Option<u64>,
43 saved_epoch: Option<u64>,
45}
46
47pub(crate) struct SavedAddress<T> {
49 pub(crate) output: T,
50 pub(crate) retried: bool,
53}
54
55impl NodeAddressUpdater {
56 pub(crate) async fn update_if_needed<T, F, Fut>(
60 &self,
61 node: (Role, u64),
62 epoch: u64,
63 save: F,
64 ) -> Result<Option<SavedAddress<T>>>
65 where
66 F: FnOnce() -> Fut,
67 Fut: Future<Output = Result<T>>,
68 {
69 let state = {
72 let entry = self.states.entry(node).or_default();
73 Arc::clone(entry.value())
74 };
75 let mut state = state.lock().await;
76 if state.seen_epoch.is_some_and(|seen| epoch < seen) || state.saved_epoch == Some(epoch) {
77 return Ok(None);
78 }
79
80 let retried = state.seen_epoch == Some(epoch);
83 state.seen_epoch = Some(epoch);
84 let output = save().await?;
85 state.saved_epoch = Some(epoch);
86 Ok(Some(SavedAddress { output, retried }))
87 }
88}
89
90pub(crate) async fn save_node_address(
93 ctx: &Context,
94 key: Vec<u8>,
95 peer: Peer,
96) -> Result<Option<NodeAddressValue>> {
97 let address = NodeAddressValue::new(peer);
98 let value = address
99 .try_as_raw_value()
100 .context(InvalidNodeInfoFormatSnafu)?;
101 let resp = ctx
102 .leader_cached_kv_backend
103 .put(PutRequest {
104 key,
105 value,
106 prev_kv: true,
107 })
108 .await
109 .context(KvBackendSnafu)?;
110 info!("Successfully updated node address: {:?}", address.peer);
111 Ok(resp
112 .prev_kv
113 .and_then(|kv| NodeAddressValue::try_from_raw_value(&kv.value).ok()))
114}
115
116pub(crate) async fn invalidate_address_caches(
119 ctx: &Context,
120 node_id: u64,
121 cache_idents: &[CacheIdent],
122) {
123 if let Err(e) = ctx
124 .cache_invalidator
125 .invalidate(&Default::default(), cache_idents)
126 .await
127 {
128 error!(e; "Failed to invalidate address caches for node {}: {:?}", node_id, cache_idents);
129 }
130}
131
132#[cfg(test)]
133mod tests {
134 use std::time::Duration;
135
136 use tokio::sync::Notify;
137 use tokio::time::timeout;
138
139 use super::*;
140
141 #[tokio::test]
142 async fn test_failed_save_retries_same_epoch_and_rejects_stale_heartbeats() {
143 let updater = NodeAddressUpdater::default();
144 let key = (Role::Datanode, 1);
145 assert!(
146 updater
147 .update_if_needed::<(), _, _>(key, 2, || async {
148 crate::error::UnexpectedSnafu {
149 violated: "injected save failure",
150 }
151 .fail()
152 })
153 .await
154 .is_err()
155 );
156 assert!(
157 updater
158 .update_if_needed::<(), _, _>(key, 1, || async {
159 panic!("stale heartbeat must not save");
160 })
161 .await
162 .unwrap()
163 .is_none()
164 );
165 assert!(
166 updater
167 .update_if_needed(key, 2, || async { Ok(()) })
168 .await
169 .unwrap()
170 .is_some_and(|saved| saved.retried)
171 );
172 assert!(
173 updater
174 .update_if_needed::<(), _, _>(key, 2, || async {
175 panic!("successful epoch must not save again");
176 })
177 .await
178 .unwrap()
179 .is_none()
180 );
181 assert!(
182 updater
183 .update_if_needed(key, 3, || async { Ok(()) })
184 .await
185 .unwrap()
186 .is_some_and(|saved| !saved.retried)
187 );
188 }
189
190 #[tokio::test]
191 async fn test_cancelled_save_can_be_retried() {
192 let updater = NodeAddressUpdater::default();
193 let key = (Role::Datanode, 1);
194 let entered = Notify::new();
195 {
196 let first = updater.update_if_needed(key, 0, || async {
197 entered.notify_one();
198 std::future::pending::<Result<()>>().await
199 });
200 tokio::pin!(first);
201 tokio::select! {
202 _ = entered.notified() => {},
203 _ = &mut first => panic!("save must remain pending"),
204 }
205 }
206 assert!(
207 timeout(
208 Duration::from_secs(2),
209 updater.update_if_needed(key, 0, || async { Ok(()) })
210 )
211 .await
212 .unwrap()
213 .unwrap()
214 .is_some()
215 );
216 }
217
218 #[test]
219 fn test_same_node_serialized_other_nodes_progress() {
220 let (progress_tx, progress_rx) = std::sync::mpsc::sync_channel(1);
223 let release = Arc::new(Notify::new());
224 let finish = Arc::new(Notify::new());
225 let worker = std::thread::spawn({
226 let release = release.clone();
227 let finish = finish.clone();
228 move || {
229 tokio::runtime::Builder::new_current_thread()
230 .enable_time()
231 .build()
232 .unwrap()
233 .block_on(async move {
234 let updater = Arc::new(NodeAddressUpdater::default());
235 let entered = Arc::new(Notify::new());
236
237 let first = tokio::spawn({
238 let updater = updater.clone();
239 let entered = entered.clone();
240 let release = release.clone();
241 async move {
242 updater
243 .update_if_needed((Role::Datanode, 1), 1, || async {
244 entered.notify_one();
245 release.notified().await;
246 Ok(())
247 })
248 .await
249 }
250 });
251 entered.notified().await;
252 let second = tokio::spawn({
253 let updater = updater.clone();
254 async move {
255 updater
256 .update_if_needed((Role::Datanode, 1), 2, || async { Ok(()) })
257 .await
258 }
259 });
260 tokio::task::yield_now().await;
261 assert!(!second.is_finished());
262 timeout(Duration::from_secs(2), async {
265 for id in 0_u64..128 {
266 assert!(
267 updater
268 .update_if_needed((Role::Datanode, id + 2), 1, || async {
269 Ok(())
270 })
271 .await
272 .unwrap()
273 .is_some()
274 );
275 }
276 })
277 .await
278 .unwrap();
279 progress_tx.send(()).unwrap();
280 finish.notified().await;
281 timeout(Duration::from_secs(2), async {
282 assert!(first.await.unwrap().unwrap().is_some());
283 assert!(second.await.unwrap().unwrap().is_some());
284 })
285 .await
286 .unwrap();
287 assert!(
288 updater
289 .update_if_needed::<(), _, _>((Role::Datanode, 1), 1, || async {
290 panic!("old epoch must not overwrite a newer one");
291 })
292 .await
293 .unwrap()
294 .is_none()
295 );
296 });
297 }
298 });
299 let progress = progress_rx.recv_timeout(Duration::from_secs(5));
300 release.notify_one();
301 finish.notify_one();
302 progress.expect("other node updates must progress while one node save is blocked");
303 worker.join().unwrap();
304 }
305}