Skip to main content

meta_srv/handler/
collect_stats_handler.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::cmp::Ordering;
16use std::sync::Arc;
17
18use api::v1::meta::{HeartbeatRequest, Peer, Role};
19use common_meta::DatanodeId;
20use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue, Stat};
21use common_meta::instruction::CacheIdent;
22use common_meta::key::MetadataKey;
23use common_meta::key::node_address::{NodeAddressKey, NodeAddressValue};
24use common_meta::rpc::store::PutRequest;
25use common_telemetry::{error, warn};
26use dashmap::DashMap;
27use futures::TryStreamExt;
28use snafu::ResultExt;
29use store_api::storage::TableId;
30use tokio::sync::Mutex;
31
32use crate::error::{self, Result};
33use crate::handler::utils::{self, NodeAddressUpdater};
34use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
35use crate::metasrv::Context;
36
37#[derive(Debug, Default)]
38struct EpochStats {
39    stats: Vec<Stat>,
40    epoch: Option<u64>,
41}
42
43impl EpochStats {
44    #[inline]
45    fn drain_all(&mut self) -> Vec<Stat> {
46        std::mem::take(&mut self.stats)
47    }
48
49    #[inline]
50    fn clear_stats(&mut self) {
51        self.stats.clear();
52    }
53
54    #[inline]
55    fn push_stat(&mut self, stat: Stat) {
56        self.stats.push(stat);
57    }
58
59    #[inline]
60    fn len(&self) -> usize {
61        self.stats.len()
62    }
63
64    #[inline]
65    fn epoch(&self) -> Option<u64> {
66        self.epoch
67    }
68
69    #[inline]
70    fn set_epoch(&mut self, epoch: u64) {
71        self.epoch = Some(epoch);
72    }
73}
74
75const DEFAULT_FLUSH_STATS_FACTOR: usize = 3;
76
77pub struct CollectStatsHandler {
78    stats_cache: DashMap<DatanodeStatKey, Arc<Mutex<EpochStats>>>,
79    flush_stats_factor: usize,
80    address_updater: NodeAddressUpdater,
81}
82
83impl Default for CollectStatsHandler {
84    fn default() -> Self {
85        Self::new(None)
86    }
87}
88
89impl CollectStatsHandler {
90    pub fn new(flush_stats_factor: Option<usize>) -> Self {
91        Self {
92            flush_stats_factor: flush_stats_factor.unwrap_or(DEFAULT_FLUSH_STATS_FACTOR),
93            stats_cache: DashMap::default(),
94            address_updater: NodeAddressUpdater::default(),
95        }
96    }
97}
98
99#[async_trait::async_trait]
100impl HeartbeatHandler for CollectStatsHandler {
101    fn is_acceptable(&self, role: Role) -> bool {
102        role == Role::Datanode
103    }
104
105    async fn handle(
106        &self,
107        _req: &HeartbeatRequest,
108        ctx: &mut Context,
109        acc: &mut HeartbeatAccumulator,
110    ) -> Result<HandleControl> {
111        let Some(current_stat) = acc.stat.take() else {
112            return Ok(HandleControl::Continue);
113        };
114
115        let key = current_stat.stat_key();
116        let state = {
117            let entry = self
118                .stats_cache
119                .entry(key)
120                .or_insert_with(|| Arc::new(Mutex::new(EpochStats::default())));
121            Arc::clone(entry.value())
122        };
123        let mut epoch_stats = state.lock().await;
124
125        let key: Vec<u8> = key.into();
126
127        let current_epoch = current_stat.node_epoch;
128        let refresh = if let Some(epoch) = epoch_stats.epoch() {
129            match current_stat.node_epoch.cmp(&epoch) {
130                Ordering::Greater => {
131                    // This node may have been redeployed.
132                    epoch_stats.clear_stats();
133                    epoch_stats.set_epoch(current_stat.node_epoch);
134                    epoch_stats.push_stat(current_stat);
135                    true
136                }
137                Ordering::Equal => {
138                    epoch_stats.push_stat(current_stat);
139                    false
140                }
141                Ordering::Less => {
142                    warn!("Ignore stale heartbeat: {:?}", current_stat);
143                    false
144                }
145            }
146        } else {
147            epoch_stats.set_epoch(current_stat.node_epoch);
148            epoch_stats.push_stat(current_stat);
149            // If the epoch is empty, it indicates that the current node sending the heartbeat
150            // for the first time to the current meta leader, so it is necessary to save
151            // the data to the KV store as soon as possible.
152            true
153        };
154
155        // Keep address updates and statistics in the original per-node order.
156        // Address acknowledgement is independent of the statistics epoch so a
157        // failed save can be retried by the next heartbeat of the same epoch.
158        if epoch_stats.epoch() == Some(current_epoch)
159            && let Some(stat) = epoch_stats.stats.last()
160        {
161            update_datanode_address(&self.address_updater, ctx, stat).await;
162        }
163
164        if !refresh && epoch_stats.len() < self.flush_stats_factor {
165            return Ok(HandleControl::Continue);
166        }
167
168        let value: Vec<u8> = DatanodeStatValue {
169            stats: epoch_stats.drain_all(),
170        }
171        .try_into()
172        .context(error::InvalidDatanodeStatFormatSnafu {})?;
173        let put = PutRequest {
174            key,
175            value,
176            prev_kv: false,
177        };
178
179        let _ = ctx
180            .in_memory
181            .put(put)
182            .await
183            .context(error::KvBackendSnafu)?;
184
185        Ok(HandleControl::Continue)
186    }
187}
188
189/// Applies datanode-specific address and table-cache updates. The caller keeps
190/// its statistics lock so notifications and statistics retain their ordering.
191async fn update_datanode_address(updater: &NodeAddressUpdater, ctx: &Context, stat: &Stat) {
192    let updated = updater
193        .update_if_needed((Role::Datanode, stat.id), stat.node_epoch, || async {
194            let peer = Peer { id: stat.id, addr: stat.addr.clone() };
195            let key = NodeAddressKey::with_datanode(stat.id).to_bytes();
196            utils::save_node_address(ctx, key, peer).await
197        })
198        .await
199        .inspect_err(|e| {
200            error!(e; "Failed to update datanode address, node: {}, address: {}", stat.id, stat.addr);
201        });
202    let Ok(Some(saved)) = updated else {
203        return;
204    };
205
206    let mut table_ids = stat.table_ids();
207    // Cached table routes embed the address, and a restarted datanode's first
208    // heartbeat may report few regions, so invalidate every table on the node.
209    // A failed earlier attempt may already have stored the new address, which
210    // hides the change from this save.
211    if saved.retried || address_changed(saved.output.as_ref(), &stat.addr) {
212        match datanode_table_ids(ctx, stat.id).await {
213            Ok(ids) => table_ids.extend(ids),
214            Err(e) => {
215                error!(e; "Failed to list tables of datanode {} for address invalidation", stat.id)
216            }
217        }
218    }
219    let cache_idents = table_ids
220        .into_iter()
221        .map(CacheIdent::TableId)
222        .collect::<Vec<_>>();
223    utils::invalidate_address_caches(ctx, stat.id, &cache_idents).await;
224}
225
226/// A missing or undecodable previous address counts as changed.
227fn address_changed(previous: Option<&NodeAddressValue>, addr: &str) -> bool {
228    previous.is_none_or(|previous| previous.peer.addr != addr)
229}
230
231async fn datanode_table_ids(ctx: &Context, datanode_id: DatanodeId) -> Result<Vec<TableId>> {
232    ctx.table_metadata_manager
233        .datanode_table_manager()
234        .tables(datanode_id)
235        .map_ok(|value| value.table_id)
236        .try_collect()
237        .await
238        .context(error::TableMetadataManagerSnafu)
239}
240
241#[cfg(test)]
242mod tests {
243    use std::any::Any;
244    use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering};
245    use std::sync::{Arc, Mutex as StdMutex, mpsc};
246    use std::thread;
247    use std::time::Duration;
248
249    use common_meta::cache_invalidator::{CacheInvalidator, Context as InvalidationContext};
250    use common_meta::datanode::DatanodeStatKey;
251    use common_meta::error::{Error as MetaError, Result as MetaResult};
252    use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue, RegionInfo};
253    use common_meta::key::{MetadataValue, RegionRoleSet};
254    use common_meta::kv_backend::{KvBackend, KvBackendRef, ResettableKvBackend, TxnService};
255    use common_meta::rpc::store::{
256        BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse,
257        BatchPutRequest, BatchPutResponse, DeleteRangeRequest, DeleteRangeResponse, PutResponse,
258        RangeRequest, RangeResponse,
259    };
260    use tokio::sync::Semaphore;
261    use tokio::time::{sleep, timeout};
262
263    use super::*;
264    use crate::handler::test_utils::TestEnv;
265    use crate::service::store::cached_kv::LeaderCachedKvBackend;
266
267    struct ControlledKvBackend {
268        recorded_puts: StdMutex<Vec<PutRequest>>,
269        put_entered: Semaphore,
270        put_release: Semaphore,
271        block_next_put: AtomicBool,
272        delay_puts: AtomicBool,
273        active_puts: AtomicUsize,
274        max_active_puts: AtomicUsize,
275    }
276
277    impl ControlledKvBackend {
278        fn new() -> Self {
279            Self {
280                recorded_puts: StdMutex::new(Vec::new()),
281                put_entered: Semaphore::new(0),
282                put_release: Semaphore::new(0),
283                block_next_put: AtomicBool::new(false),
284                delay_puts: AtomicBool::new(false),
285                active_puts: AtomicUsize::new(0),
286                max_active_puts: AtomicUsize::new(0),
287            }
288        }
289
290        fn block_next_put(&self) {
291            self.block_next_put.store(true, AtomicOrdering::Relaxed);
292        }
293
294        async fn wait_for_blocked_put(&self) {
295            self.put_entered.acquire().await.unwrap().forget();
296        }
297
298        fn release_one_put(&self) {
299            self.put_release.add_permits(1);
300        }
301
302        fn set_delay_puts(&self, delay: bool) {
303            self.delay_puts.store(delay, AtomicOrdering::Relaxed);
304        }
305
306        fn clear_recorded_puts(&self) {
307            self.recorded_puts.lock().unwrap().clear();
308        }
309
310        fn recorded_puts(&self) -> Vec<PutRequest> {
311            self.recorded_puts.lock().unwrap().clone()
312        }
313
314        fn max_active_puts(&self) -> usize {
315            self.max_active_puts.load(AtomicOrdering::Relaxed)
316        }
317
318        fn start_put(&self, req: &PutRequest) -> PutGuard<'_> {
319            self.recorded_puts.lock().unwrap().push(req.clone());
320            let active = self.active_puts.fetch_add(1, AtomicOrdering::Relaxed) + 1;
321            self.max_active_puts
322                .fetch_max(active, AtomicOrdering::Relaxed);
323            PutGuard { backend: self }
324        }
325    }
326
327    struct PutGuard<'a> {
328        backend: &'a ControlledKvBackend,
329    }
330
331    impl Drop for PutGuard<'_> {
332        fn drop(&mut self) {
333            self.backend
334                .active_puts
335                .fetch_sub(1, AtomicOrdering::Relaxed);
336        }
337    }
338
339    #[async_trait::async_trait]
340    impl TxnService for ControlledKvBackend {
341        type Error = MetaError;
342    }
343
344    #[async_trait::async_trait]
345    impl KvBackend for ControlledKvBackend {
346        fn name(&self) -> &str {
347            "controlled"
348        }
349
350        fn as_any(&self) -> &dyn Any {
351            self
352        }
353
354        async fn range(&self, _req: RangeRequest) -> MetaResult<RangeResponse> {
355            unimplemented!()
356        }
357
358        async fn put(&self, req: PutRequest) -> MetaResult<PutResponse> {
359            let _guard = self.start_put(&req);
360            if self.block_next_put.swap(false, AtomicOrdering::Relaxed) {
361                self.put_entered.add_permits(1);
362                self.put_release.acquire().await.unwrap().forget();
363            }
364            if self.delay_puts.load(AtomicOrdering::Relaxed) {
365                sleep(Duration::from_millis(10)).await;
366            }
367            Ok(PutResponse::default())
368        }
369
370        async fn batch_put(&self, _req: BatchPutRequest) -> MetaResult<BatchPutResponse> {
371            unimplemented!()
372        }
373
374        async fn batch_get(&self, _req: BatchGetRequest) -> MetaResult<BatchGetResponse> {
375            unimplemented!()
376        }
377
378        async fn delete_range(&self, _req: DeleteRangeRequest) -> MetaResult<DeleteRangeResponse> {
379            unimplemented!()
380        }
381
382        async fn batch_delete(&self, _req: BatchDeleteRequest) -> MetaResult<BatchDeleteResponse> {
383            unimplemented!()
384        }
385    }
386
387    impl ResettableKvBackend for ControlledKvBackend {
388        fn reset(&self) {
389            self.clear_recorded_puts();
390        }
391
392        fn as_kv_backend_ref(self: Arc<Self>) -> KvBackendRef {
393            self
394        }
395    }
396
397    fn stat(node_id: u64, epoch: u64, marker: u64, addr: &str) -> Stat {
398        Stat {
399            timestamp_millis: marker as i64,
400            id: node_id,
401            addr: addr.to_string(),
402            region_num: marker,
403            node_epoch: epoch,
404            ..Default::default()
405        }
406    }
407
408    async fn handle_stat(
409        handler: Arc<CollectStatsHandler>,
410        mut ctx: Context,
411        stat: Stat,
412    ) -> Result<HandleControl> {
413        let mut acc = HeartbeatAccumulator {
414            stat: Some(stat),
415            ..Default::default()
416        };
417        handler
418            .handle(&HeartbeatRequest::default(), &mut ctx, &mut acc)
419            .await
420    }
421
422    fn use_controlled_address_backend(ctx: &mut Context) -> Arc<ControlledKvBackend> {
423        let backend = Arc::new(ControlledKvBackend::new());
424        ctx.leader_cached_kv_backend =
425            Arc::new(LeaderCachedKvBackend::with_always_leader(backend.clone()));
426        backend
427    }
428
429    #[tokio::test]
430    async fn test_handle_datanode_stats() {
431        let env = TestEnv::new();
432        let ctx = env.ctx();
433
434        let handler = CollectStatsHandler::default();
435        handle_request_many_times(ctx.clone(), &handler, 1).await;
436
437        let key = DatanodeStatKey { node_id: 101 };
438        let key: Vec<u8> = key.into();
439        let res = ctx.in_memory.get(&key).await.unwrap();
440        let kv = res.unwrap();
441        let key: DatanodeStatKey = kv.key.clone().try_into().unwrap();
442        assert_eq!(101, key.node_id);
443        let val: DatanodeStatValue = kv.value.try_into().unwrap();
444        // first new stat must be set in kv store immediately
445        assert_eq!(1, val.stats.len());
446        assert_eq!(1, val.stats[0].region_num);
447
448        handle_request_many_times(ctx.clone(), &handler, 10).await;
449
450        let key: Vec<u8> = key.into();
451        let res = ctx.in_memory.get(&key).await.unwrap();
452        let kv = res.unwrap();
453        let val: DatanodeStatValue = kv.value.try_into().unwrap();
454        assert_eq!(handler.flush_stats_factor, val.stats.len());
455    }
456
457    #[test]
458    fn test_same_datanode_wait_keeps_current_thread_runtime_responsive() {
459        let (backend_tx, backend_rx) = mpsc::sync_channel(1);
460        let (timer_tx, timer_rx) = mpsc::sync_channel(1);
461        let (done_tx, done_rx) = mpsc::sync_channel(1);
462
463        let worker = thread::spawn(move || {
464            tokio::runtime::Builder::new_current_thread()
465                .enable_time()
466                .build()
467                .unwrap()
468                .block_on(async move {
469                    let env = TestEnv::new();
470                    let mut ctx = env.ctx();
471                    let address_backend = use_controlled_address_backend(&mut ctx);
472                    address_backend.block_next_put();
473                    backend_tx.send(address_backend.clone()).unwrap();
474
475                    let handler = Arc::new(CollectStatsHandler::default());
476                    let first = tokio::spawn(handle_stat(
477                        handler.clone(),
478                        ctx.clone(),
479                        stat(101, 1, 1, "dn-101-v1"),
480                    ));
481                    address_backend.wait_for_blocked_put().await;
482
483                    let second =
484                        tokio::spawn(handle_stat(handler, ctx, stat(101, 1, 2, "dn-101-v1")));
485                    tokio::spawn(async move {
486                        sleep(Duration::from_millis(10)).await;
487                        timer_tx.send(()).unwrap();
488                    });
489                    tokio::task::yield_now().await;
490
491                    first.await.unwrap().unwrap();
492                    second.await.unwrap().unwrap();
493                    done_tx.send(()).unwrap();
494                });
495        });
496
497        let address_backend = backend_rx.recv_timeout(Duration::from_secs(1)).unwrap();
498        let timer_result = timer_rx.recv_timeout(Duration::from_secs(1));
499        address_backend.release_one_put();
500        timer_result.expect("the current-thread runtime must remain responsive");
501        done_rx
502            .recv_timeout(Duration::from_secs(1))
503            .expect("both heartbeat handlers must complete after releasing the write");
504        worker.join().unwrap();
505    }
506
507    #[tokio::test]
508    async fn test_concurrent_flush_persists_every_stat_once() {
509        let env = TestEnv::new();
510        let mut ctx = env.ctx();
511        let stats_backend = Arc::new(ControlledKvBackend::new());
512        ctx.in_memory = stats_backend.clone();
513        stats_backend.set_delay_puts(true);
514
515        let flush_stats_factor = 3;
516        let handler = Arc::new(CollectStatsHandler::new(Some(flush_stats_factor)));
517        handle_stat(handler.clone(), ctx.clone(), stat(101, 1, 0, "dn-101"))
518            .await
519            .unwrap();
520        stats_backend.clear_recorded_puts();
521
522        let mut tasks = Vec::with_capacity(2 * flush_stats_factor);
523        for marker in 1..=(2 * flush_stats_factor) {
524            tasks.push(tokio::spawn(handle_stat(
525                handler.clone(),
526                ctx.clone(),
527                stat(101, 1, marker as u64, "dn-101"),
528            )));
529        }
530        for task in tasks {
531            timeout(Duration::from_secs(1), task)
532                .await
533                .unwrap()
534                .unwrap()
535                .unwrap();
536        }
537
538        let puts = stats_backend.recorded_puts();
539        assert_eq!(2, puts.len());
540        let mut markers = puts
541            .into_iter()
542            .flat_map(|put| {
543                let value: DatanodeStatValue = put.value.try_into().unwrap();
544                value
545                    .stats
546                    .into_iter()
547                    .map(|stat| stat.region_num)
548                    .collect::<Vec<_>>()
549            })
550            .collect::<Vec<_>>();
551        markers.sort_unstable();
552        assert_eq!((1..=6).collect::<Vec<_>>(), markers);
553        assert_eq!(1, stats_backend.max_active_puts());
554    }
555
556    async fn handle_request_many_times(
557        mut ctx: Context,
558        handler: &CollectStatsHandler,
559        loop_times: i32,
560    ) {
561        let req = HeartbeatRequest::default();
562        for i in 1..=loop_times {
563            let mut acc = HeartbeatAccumulator {
564                stat: Some(Stat {
565                    id: 101,
566                    region_num: i as _,
567                    ..Default::default()
568                }),
569                ..Default::default()
570            };
571            handler.handle(&req, &mut ctx, &mut acc).await.unwrap();
572        }
573    }
574
575    #[derive(Default)]
576    struct RecordingCacheInvalidator {
577        calls: StdMutex<Vec<Vec<CacheIdent>>>,
578    }
579
580    impl RecordingCacheInvalidator {
581        fn table_ids_per_call(&self) -> Vec<Vec<TableId>> {
582            self.calls
583                .lock()
584                .unwrap()
585                .iter()
586                .map(|idents| {
587                    let mut ids = idents
588                        .iter()
589                        .map(|ident| match ident {
590                            CacheIdent::TableId(id) => *id,
591                            other => panic!("unexpected cache ident: {other:?}"),
592                        })
593                        .collect::<Vec<_>>();
594                    ids.sort_unstable();
595                    ids
596                })
597                .collect()
598        }
599    }
600
601    #[async_trait::async_trait]
602    impl CacheInvalidator for RecordingCacheInvalidator {
603        async fn invalidate(
604            &self,
605            _: &InvalidationContext,
606            idents: &[CacheIdent],
607        ) -> MetaResult<()> {
608            self.calls.lock().unwrap().push(idents.to_vec());
609            Ok(())
610        }
611
612        fn invalidate_all(&self) -> MetaResult<()> {
613            Ok(())
614        }
615    }
616
617    async fn register_datanode_tables(ctx: &Context, datanode_id: u64, table_ids: &[TableId]) {
618        for table_id in table_ids {
619            let value = DatanodeTableValue::new(
620                *table_id,
621                RegionRoleSet::new(vec![0], vec![]),
622                RegionInfo::default(),
623            );
624            ctx.kv_backend
625                .put(PutRequest {
626                    key: DatanodeTableKey::new(datanode_id, *table_id).to_bytes(),
627                    value: value.try_as_raw_value().unwrap(),
628                    prev_kv: false,
629                })
630                .await
631                .unwrap();
632        }
633    }
634
635    fn use_recording_invalidator(ctx: &mut Context) -> Arc<RecordingCacheInvalidator> {
636        let invalidator = Arc::new(RecordingCacheInvalidator::default());
637        ctx.cache_invalidator = invalidator.clone();
638        invalidator
639    }
640
641    #[tokio::test]
642    async fn test_address_change_invalidates_all_tables_on_datanode() {
643        let mut ctx = TestEnv::new().ctx();
644        let invalidator = use_recording_invalidator(&mut ctx);
645        register_datanode_tables(&ctx, 101, &[1024, 1025]).await;
646        register_datanode_tables(&ctx, 102, &[2048]).await;
647        let handler = Arc::new(CollectStatsHandler::default());
648
649        handle_stat(handler.clone(), ctx.clone(), stat(101, 1, 0, "dn-101-v1"))
650            .await
651            .unwrap();
652        // Restarted on a new address, reporting no regions yet.
653        handle_stat(handler.clone(), ctx.clone(), stat(101, 2, 0, "dn-101-v2"))
654            .await
655            .unwrap();
656        handle_stat(handler, ctx, stat(101, 2, 0, "dn-101-v2"))
657            .await
658            .unwrap();
659
660        assert_eq!(
661            vec![vec![1024, 1025], vec![1024, 1025]],
662            invalidator.table_ids_per_call()
663        );
664    }
665
666    #[tokio::test]
667    async fn test_unchanged_address_only_invalidates_reported_tables() {
668        let mut ctx = TestEnv::new().ctx();
669        register_datanode_tables(&ctx, 101, &[1024, 1025]).await;
670        let handler = Arc::new(CollectStatsHandler::default());
671        handle_stat(handler, ctx.clone(), stat(101, 1, 0, "dn-101"))
672            .await
673            .unwrap();
674
675        // A fresh handler sees the node as after a metasrv leader change.
676        let invalidator = use_recording_invalidator(&mut ctx);
677        let handler = Arc::new(CollectStatsHandler::default());
678        handle_stat(handler.clone(), ctx.clone(), stat(101, 1, 0, "dn-101"))
679            .await
680            .unwrap();
681        handle_stat(handler, ctx, stat(101, 2, 0, "dn-101"))
682            .await
683            .unwrap();
684
685        assert_eq!(
686            vec![Vec::<TableId>::new(), vec![]],
687            invalidator.table_ids_per_call()
688        );
689    }
690
691    #[tokio::test]
692    async fn test_retry_after_landed_write_invalidates_all_tables_on_datanode() {
693        use common_meta::kv_backend::test_util::MockKvBackendBuilder;
694        use common_meta::rpc::KeyValue;
695
696        let mut ctx = TestEnv::new().ctx();
697        let invalidator = use_recording_invalidator(&mut ctx);
698        register_datanode_tables(&ctx, 101, &[1024, 1025]).await;
699        let attempts = Arc::new(AtomicUsize::new(0));
700        let backend = MockKvBackendBuilder::default()
701            .put_fn(Arc::new({
702                let attempts = attempts.clone();
703                move |req: PutRequest| {
704                    // The first write lands but reports an error, so the retry
705                    // reads back its own address as the previous one.
706                    if attempts.fetch_add(1, AtomicOrdering::Relaxed) == 0 {
707                        return common_meta::error::UnexpectedSnafu {
708                            err_msg: "injected lost response",
709                        }
710                        .fail();
711                    }
712                    Ok(PutResponse {
713                        prev_kv: Some(KeyValue {
714                            key: req.key,
715                            value: req.value,
716                        }),
717                    })
718                }
719            }) as _)
720            .build()
721            .unwrap();
722        ctx.leader_cached_kv_backend =
723            Arc::new(LeaderCachedKvBackend::with_always_leader(Arc::new(backend)));
724        let handler = Arc::new(CollectStatsHandler::default());
725
726        handle_stat(handler.clone(), ctx.clone(), stat(101, 2, 0, "dn-101-v2"))
727            .await
728            .unwrap();
729        handle_stat(handler, ctx, stat(101, 2, 0, "dn-101-v2"))
730            .await
731            .unwrap();
732
733        assert_eq!(2, attempts.load(AtomicOrdering::Relaxed));
734        assert_eq!(vec![vec![1024, 1025]], invalidator.table_ids_per_call());
735    }
736
737    #[tokio::test]
738    async fn test_persistence_failure_retries_but_notification_failure_does_not() {
739        use crate::handler::collect_stats_handler::CollectStatsHandler;
740        use crate::handler::test_utils::FailingCacheInvalidator;
741        use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
742        use crate::service::store::cached_kv::LeaderCachedKvBackend;
743        use api::v1::meta::HeartbeatRequest;
744        use common_meta::key::MetadataValue;
745        use common_meta::key::node_address::NodeAddressValue;
746        use common_meta::kv_backend::test_util::MockKvBackendBuilder;
747        use common_meta::rpc::store::PutResponse;
748        let mut ctx = TestEnv::new().ctx();
749        let puts = Arc::new(std::sync::Mutex::new(Vec::<PutRequest>::new()));
750        let backend = MockKvBackendBuilder::default()
751            .put_fn(Arc::new({
752                let puts = puts.clone();
753                move |req: PutRequest| {
754                    let mut puts = puts.lock().unwrap();
755                    puts.push(req);
756                    if puts.len() == 1 {
757                        return common_meta::error::UnexpectedSnafu {
758                            err_msg: "injected persistence failure",
759                        }
760                        .fail();
761                    }
762                    Ok(PutResponse::default())
763                }
764            }) as _)
765            .build()
766            .unwrap();
767        ctx.leader_cached_kv_backend =
768            Arc::new(LeaderCachedKvBackend::with_always_leader(Arc::new(backend)));
769        let invalidator = Arc::new(FailingCacheInvalidator::default());
770        ctx.cache_invalidator = invalidator.clone();
771        let handler = CollectStatsHandler::default();
772        let mut stat = Stat {
773            id: 101,
774            addr: "dn-original".into(),
775            node_epoch: 7,
776            ..Default::default()
777        };
778        let req = HeartbeatRequest::default();
779
780        // A backend error must not interrupt subsequent heartbeat handlers.
781        assert!(matches!(
782            handler
783                .handle(
784                    &req,
785                    &mut ctx,
786                    &mut HeartbeatAccumulator {
787                        stat: Some(stat.clone()),
788                        ..Default::default()
789                    }
790                )
791                .await
792                .unwrap(),
793            HandleControl::Continue
794        ));
795        assert_eq!(puts.lock().unwrap().len(), 1);
796        assert_eq!(invalidator.attempts(), 0);
797
798        let stats_key: Vec<u8> = common_meta::datanode::DatanodeStatKey { node_id: 101 }.into();
799        assert!(ctx.in_memory.get(&stats_key).await.unwrap().is_some());
800
801        // Same epoch retries persistence using the current heartbeat's address.
802        stat.addr = "dn-retry".into();
803        handler
804            .handle(
805                &req,
806                &mut ctx,
807                &mut HeartbeatAccumulator {
808                    stat: Some(stat.clone()),
809                    ..Default::default()
810                },
811            )
812            .await
813            .unwrap();
814        {
815            let puts = puts.lock().unwrap();
816            assert_eq!(puts.len(), 2);
817            assert_eq!(puts[1].key, NodeAddressKey::with_datanode(101).to_bytes());
818            assert_eq!(
819                NodeAddressValue::try_from_raw_value(&puts[1].value)
820                    .unwrap()
821                    .peer
822                    .addr,
823                "dn-retry"
824            );
825        }
826        assert_eq!(invalidator.attempts(), 1);
827
828        // Notification remains best-effort: its failure neither rewrites nor retries.
829        handler
830            .handle(
831                &req,
832                &mut ctx,
833                &mut HeartbeatAccumulator {
834                    stat: Some(stat.clone()),
835                    ..Default::default()
836                },
837            )
838            .await
839            .unwrap();
840        assert_eq!(puts.lock().unwrap().len(), 2);
841        assert_eq!(invalidator.attempts(), 1);
842
843        stat.node_epoch = 8;
844        handler
845            .handle(
846                &req,
847                &mut ctx,
848                &mut HeartbeatAccumulator {
849                    stat: Some(stat.clone()),
850                    ..Default::default()
851                },
852            )
853            .await
854            .unwrap();
855        assert_eq!(puts.lock().unwrap().len(), 3);
856        assert_eq!(invalidator.attempts(), 2);
857    }
858}