1use std::cmp::Ordering;
16use std::sync::Arc;
17
18use api::v1::meta::{HeartbeatRequest, Peer, Role};
19use common_meta::DatanodeId;
20use common_meta::datanode::{DatanodeStatKey, DatanodeStatValue, Stat};
21use common_meta::instruction::CacheIdent;
22use common_meta::key::MetadataKey;
23use common_meta::key::node_address::{NodeAddressKey, NodeAddressValue};
24use common_meta::rpc::store::PutRequest;
25use common_telemetry::{error, warn};
26use dashmap::DashMap;
27use futures::TryStreamExt;
28use snafu::ResultExt;
29use store_api::storage::TableId;
30use tokio::sync::Mutex;
31
32use crate::error::{self, Result};
33use crate::handler::utils::{self, NodeAddressUpdater};
34use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
35use crate::metasrv::Context;
36
37#[derive(Debug, Default)]
38struct EpochStats {
39 stats: Vec<Stat>,
40 epoch: Option<u64>,
41}
42
43impl EpochStats {
44 #[inline]
45 fn drain_all(&mut self) -> Vec<Stat> {
46 std::mem::take(&mut self.stats)
47 }
48
49 #[inline]
50 fn clear_stats(&mut self) {
51 self.stats.clear();
52 }
53
54 #[inline]
55 fn push_stat(&mut self, stat: Stat) {
56 self.stats.push(stat);
57 }
58
59 #[inline]
60 fn len(&self) -> usize {
61 self.stats.len()
62 }
63
64 #[inline]
65 fn epoch(&self) -> Option<u64> {
66 self.epoch
67 }
68
69 #[inline]
70 fn set_epoch(&mut self, epoch: u64) {
71 self.epoch = Some(epoch);
72 }
73}
74
75const DEFAULT_FLUSH_STATS_FACTOR: usize = 3;
76
77pub struct CollectStatsHandler {
78 stats_cache: DashMap<DatanodeStatKey, Arc<Mutex<EpochStats>>>,
79 flush_stats_factor: usize,
80 address_updater: NodeAddressUpdater,
81}
82
83impl Default for CollectStatsHandler {
84 fn default() -> Self {
85 Self::new(None)
86 }
87}
88
89impl CollectStatsHandler {
90 pub fn new(flush_stats_factor: Option<usize>) -> Self {
91 Self {
92 flush_stats_factor: flush_stats_factor.unwrap_or(DEFAULT_FLUSH_STATS_FACTOR),
93 stats_cache: DashMap::default(),
94 address_updater: NodeAddressUpdater::default(),
95 }
96 }
97}
98
99#[async_trait::async_trait]
100impl HeartbeatHandler for CollectStatsHandler {
101 fn is_acceptable(&self, role: Role) -> bool {
102 role == Role::Datanode
103 }
104
105 async fn handle(
106 &self,
107 _req: &HeartbeatRequest,
108 ctx: &mut Context,
109 acc: &mut HeartbeatAccumulator,
110 ) -> Result<HandleControl> {
111 let Some(current_stat) = acc.stat.take() else {
112 return Ok(HandleControl::Continue);
113 };
114
115 let key = current_stat.stat_key();
116 let state = {
117 let entry = self
118 .stats_cache
119 .entry(key)
120 .or_insert_with(|| Arc::new(Mutex::new(EpochStats::default())));
121 Arc::clone(entry.value())
122 };
123 let mut epoch_stats = state.lock().await;
124
125 let key: Vec<u8> = key.into();
126
127 let current_epoch = current_stat.node_epoch;
128 let refresh = if let Some(epoch) = epoch_stats.epoch() {
129 match current_stat.node_epoch.cmp(&epoch) {
130 Ordering::Greater => {
131 epoch_stats.clear_stats();
133 epoch_stats.set_epoch(current_stat.node_epoch);
134 epoch_stats.push_stat(current_stat);
135 true
136 }
137 Ordering::Equal => {
138 epoch_stats.push_stat(current_stat);
139 false
140 }
141 Ordering::Less => {
142 warn!("Ignore stale heartbeat: {:?}", current_stat);
143 false
144 }
145 }
146 } else {
147 epoch_stats.set_epoch(current_stat.node_epoch);
148 epoch_stats.push_stat(current_stat);
149 true
153 };
154
155 if epoch_stats.epoch() == Some(current_epoch)
159 && let Some(stat) = epoch_stats.stats.last()
160 {
161 update_datanode_address(&self.address_updater, ctx, stat).await;
162 }
163
164 if !refresh && epoch_stats.len() < self.flush_stats_factor {
165 return Ok(HandleControl::Continue);
166 }
167
168 let value: Vec<u8> = DatanodeStatValue {
169 stats: epoch_stats.drain_all(),
170 }
171 .try_into()
172 .context(error::InvalidDatanodeStatFormatSnafu {})?;
173 let put = PutRequest {
174 key,
175 value,
176 prev_kv: false,
177 };
178
179 let _ = ctx
180 .in_memory
181 .put(put)
182 .await
183 .context(error::KvBackendSnafu)?;
184
185 Ok(HandleControl::Continue)
186 }
187}
188
189async fn update_datanode_address(updater: &NodeAddressUpdater, ctx: &Context, stat: &Stat) {
192 let updated = updater
193 .update_if_needed((Role::Datanode, stat.id), stat.node_epoch, || async {
194 let peer = Peer { id: stat.id, addr: stat.addr.clone() };
195 let key = NodeAddressKey::with_datanode(stat.id).to_bytes();
196 utils::save_node_address(ctx, key, peer).await
197 })
198 .await
199 .inspect_err(|e| {
200 error!(e; "Failed to update datanode address, node: {}, address: {}", stat.id, stat.addr);
201 });
202 let Ok(Some(saved)) = updated else {
203 return;
204 };
205
206 let mut table_ids = stat.table_ids();
207 if saved.retried || address_changed(saved.output.as_ref(), &stat.addr) {
212 match datanode_table_ids(ctx, stat.id).await {
213 Ok(ids) => table_ids.extend(ids),
214 Err(e) => {
215 error!(e; "Failed to list tables of datanode {} for address invalidation", stat.id)
216 }
217 }
218 }
219 let cache_idents = table_ids
220 .into_iter()
221 .map(CacheIdent::TableId)
222 .collect::<Vec<_>>();
223 utils::invalidate_address_caches(ctx, stat.id, &cache_idents).await;
224}
225
226fn address_changed(previous: Option<&NodeAddressValue>, addr: &str) -> bool {
228 previous.is_none_or(|previous| previous.peer.addr != addr)
229}
230
231async fn datanode_table_ids(ctx: &Context, datanode_id: DatanodeId) -> Result<Vec<TableId>> {
232 ctx.table_metadata_manager
233 .datanode_table_manager()
234 .tables(datanode_id)
235 .map_ok(|value| value.table_id)
236 .try_collect()
237 .await
238 .context(error::TableMetadataManagerSnafu)
239}
240
241#[cfg(test)]
242mod tests {
243 use std::any::Any;
244 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering};
245 use std::sync::{Arc, Mutex as StdMutex, mpsc};
246 use std::thread;
247 use std::time::Duration;
248
249 use common_meta::cache_invalidator::{CacheInvalidator, Context as InvalidationContext};
250 use common_meta::datanode::DatanodeStatKey;
251 use common_meta::error::{Error as MetaError, Result as MetaResult};
252 use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue, RegionInfo};
253 use common_meta::key::{MetadataValue, RegionRoleSet};
254 use common_meta::kv_backend::{KvBackend, KvBackendRef, ResettableKvBackend, TxnService};
255 use common_meta::rpc::store::{
256 BatchDeleteRequest, BatchDeleteResponse, BatchGetRequest, BatchGetResponse,
257 BatchPutRequest, BatchPutResponse, DeleteRangeRequest, DeleteRangeResponse, PutResponse,
258 RangeRequest, RangeResponse,
259 };
260 use tokio::sync::Semaphore;
261 use tokio::time::{sleep, timeout};
262
263 use super::*;
264 use crate::handler::test_utils::TestEnv;
265 use crate::service::store::cached_kv::LeaderCachedKvBackend;
266
267 struct ControlledKvBackend {
268 recorded_puts: StdMutex<Vec<PutRequest>>,
269 put_entered: Semaphore,
270 put_release: Semaphore,
271 block_next_put: AtomicBool,
272 delay_puts: AtomicBool,
273 active_puts: AtomicUsize,
274 max_active_puts: AtomicUsize,
275 }
276
277 impl ControlledKvBackend {
278 fn new() -> Self {
279 Self {
280 recorded_puts: StdMutex::new(Vec::new()),
281 put_entered: Semaphore::new(0),
282 put_release: Semaphore::new(0),
283 block_next_put: AtomicBool::new(false),
284 delay_puts: AtomicBool::new(false),
285 active_puts: AtomicUsize::new(0),
286 max_active_puts: AtomicUsize::new(0),
287 }
288 }
289
290 fn block_next_put(&self) {
291 self.block_next_put.store(true, AtomicOrdering::Relaxed);
292 }
293
294 async fn wait_for_blocked_put(&self) {
295 self.put_entered.acquire().await.unwrap().forget();
296 }
297
298 fn release_one_put(&self) {
299 self.put_release.add_permits(1);
300 }
301
302 fn set_delay_puts(&self, delay: bool) {
303 self.delay_puts.store(delay, AtomicOrdering::Relaxed);
304 }
305
306 fn clear_recorded_puts(&self) {
307 self.recorded_puts.lock().unwrap().clear();
308 }
309
310 fn recorded_puts(&self) -> Vec<PutRequest> {
311 self.recorded_puts.lock().unwrap().clone()
312 }
313
314 fn max_active_puts(&self) -> usize {
315 self.max_active_puts.load(AtomicOrdering::Relaxed)
316 }
317
318 fn start_put(&self, req: &PutRequest) -> PutGuard<'_> {
319 self.recorded_puts.lock().unwrap().push(req.clone());
320 let active = self.active_puts.fetch_add(1, AtomicOrdering::Relaxed) + 1;
321 self.max_active_puts
322 .fetch_max(active, AtomicOrdering::Relaxed);
323 PutGuard { backend: self }
324 }
325 }
326
327 struct PutGuard<'a> {
328 backend: &'a ControlledKvBackend,
329 }
330
331 impl Drop for PutGuard<'_> {
332 fn drop(&mut self) {
333 self.backend
334 .active_puts
335 .fetch_sub(1, AtomicOrdering::Relaxed);
336 }
337 }
338
339 #[async_trait::async_trait]
340 impl TxnService for ControlledKvBackend {
341 type Error = MetaError;
342 }
343
344 #[async_trait::async_trait]
345 impl KvBackend for ControlledKvBackend {
346 fn name(&self) -> &str {
347 "controlled"
348 }
349
350 fn as_any(&self) -> &dyn Any {
351 self
352 }
353
354 async fn range(&self, _req: RangeRequest) -> MetaResult<RangeResponse> {
355 unimplemented!()
356 }
357
358 async fn put(&self, req: PutRequest) -> MetaResult<PutResponse> {
359 let _guard = self.start_put(&req);
360 if self.block_next_put.swap(false, AtomicOrdering::Relaxed) {
361 self.put_entered.add_permits(1);
362 self.put_release.acquire().await.unwrap().forget();
363 }
364 if self.delay_puts.load(AtomicOrdering::Relaxed) {
365 sleep(Duration::from_millis(10)).await;
366 }
367 Ok(PutResponse::default())
368 }
369
370 async fn batch_put(&self, _req: BatchPutRequest) -> MetaResult<BatchPutResponse> {
371 unimplemented!()
372 }
373
374 async fn batch_get(&self, _req: BatchGetRequest) -> MetaResult<BatchGetResponse> {
375 unimplemented!()
376 }
377
378 async fn delete_range(&self, _req: DeleteRangeRequest) -> MetaResult<DeleteRangeResponse> {
379 unimplemented!()
380 }
381
382 async fn batch_delete(&self, _req: BatchDeleteRequest) -> MetaResult<BatchDeleteResponse> {
383 unimplemented!()
384 }
385 }
386
387 impl ResettableKvBackend for ControlledKvBackend {
388 fn reset(&self) {
389 self.clear_recorded_puts();
390 }
391
392 fn as_kv_backend_ref(self: Arc<Self>) -> KvBackendRef {
393 self
394 }
395 }
396
397 fn stat(node_id: u64, epoch: u64, marker: u64, addr: &str) -> Stat {
398 Stat {
399 timestamp_millis: marker as i64,
400 id: node_id,
401 addr: addr.to_string(),
402 region_num: marker,
403 node_epoch: epoch,
404 ..Default::default()
405 }
406 }
407
408 async fn handle_stat(
409 handler: Arc<CollectStatsHandler>,
410 mut ctx: Context,
411 stat: Stat,
412 ) -> Result<HandleControl> {
413 let mut acc = HeartbeatAccumulator {
414 stat: Some(stat),
415 ..Default::default()
416 };
417 handler
418 .handle(&HeartbeatRequest::default(), &mut ctx, &mut acc)
419 .await
420 }
421
422 fn use_controlled_address_backend(ctx: &mut Context) -> Arc<ControlledKvBackend> {
423 let backend = Arc::new(ControlledKvBackend::new());
424 ctx.leader_cached_kv_backend =
425 Arc::new(LeaderCachedKvBackend::with_always_leader(backend.clone()));
426 backend
427 }
428
429 #[tokio::test]
430 async fn test_handle_datanode_stats() {
431 let env = TestEnv::new();
432 let ctx = env.ctx();
433
434 let handler = CollectStatsHandler::default();
435 handle_request_many_times(ctx.clone(), &handler, 1).await;
436
437 let key = DatanodeStatKey { node_id: 101 };
438 let key: Vec<u8> = key.into();
439 let res = ctx.in_memory.get(&key).await.unwrap();
440 let kv = res.unwrap();
441 let key: DatanodeStatKey = kv.key.clone().try_into().unwrap();
442 assert_eq!(101, key.node_id);
443 let val: DatanodeStatValue = kv.value.try_into().unwrap();
444 assert_eq!(1, val.stats.len());
446 assert_eq!(1, val.stats[0].region_num);
447
448 handle_request_many_times(ctx.clone(), &handler, 10).await;
449
450 let key: Vec<u8> = key.into();
451 let res = ctx.in_memory.get(&key).await.unwrap();
452 let kv = res.unwrap();
453 let val: DatanodeStatValue = kv.value.try_into().unwrap();
454 assert_eq!(handler.flush_stats_factor, val.stats.len());
455 }
456
457 #[test]
458 fn test_same_datanode_wait_keeps_current_thread_runtime_responsive() {
459 let (backend_tx, backend_rx) = mpsc::sync_channel(1);
460 let (timer_tx, timer_rx) = mpsc::sync_channel(1);
461 let (done_tx, done_rx) = mpsc::sync_channel(1);
462
463 let worker = thread::spawn(move || {
464 tokio::runtime::Builder::new_current_thread()
465 .enable_time()
466 .build()
467 .unwrap()
468 .block_on(async move {
469 let env = TestEnv::new();
470 let mut ctx = env.ctx();
471 let address_backend = use_controlled_address_backend(&mut ctx);
472 address_backend.block_next_put();
473 backend_tx.send(address_backend.clone()).unwrap();
474
475 let handler = Arc::new(CollectStatsHandler::default());
476 let first = tokio::spawn(handle_stat(
477 handler.clone(),
478 ctx.clone(),
479 stat(101, 1, 1, "dn-101-v1"),
480 ));
481 address_backend.wait_for_blocked_put().await;
482
483 let second =
484 tokio::spawn(handle_stat(handler, ctx, stat(101, 1, 2, "dn-101-v1")));
485 tokio::spawn(async move {
486 sleep(Duration::from_millis(10)).await;
487 timer_tx.send(()).unwrap();
488 });
489 tokio::task::yield_now().await;
490
491 first.await.unwrap().unwrap();
492 second.await.unwrap().unwrap();
493 done_tx.send(()).unwrap();
494 });
495 });
496
497 let address_backend = backend_rx.recv_timeout(Duration::from_secs(1)).unwrap();
498 let timer_result = timer_rx.recv_timeout(Duration::from_secs(1));
499 address_backend.release_one_put();
500 timer_result.expect("the current-thread runtime must remain responsive");
501 done_rx
502 .recv_timeout(Duration::from_secs(1))
503 .expect("both heartbeat handlers must complete after releasing the write");
504 worker.join().unwrap();
505 }
506
507 #[tokio::test]
508 async fn test_concurrent_flush_persists_every_stat_once() {
509 let env = TestEnv::new();
510 let mut ctx = env.ctx();
511 let stats_backend = Arc::new(ControlledKvBackend::new());
512 ctx.in_memory = stats_backend.clone();
513 stats_backend.set_delay_puts(true);
514
515 let flush_stats_factor = 3;
516 let handler = Arc::new(CollectStatsHandler::new(Some(flush_stats_factor)));
517 handle_stat(handler.clone(), ctx.clone(), stat(101, 1, 0, "dn-101"))
518 .await
519 .unwrap();
520 stats_backend.clear_recorded_puts();
521
522 let mut tasks = Vec::with_capacity(2 * flush_stats_factor);
523 for marker in 1..=(2 * flush_stats_factor) {
524 tasks.push(tokio::spawn(handle_stat(
525 handler.clone(),
526 ctx.clone(),
527 stat(101, 1, marker as u64, "dn-101"),
528 )));
529 }
530 for task in tasks {
531 timeout(Duration::from_secs(1), task)
532 .await
533 .unwrap()
534 .unwrap()
535 .unwrap();
536 }
537
538 let puts = stats_backend.recorded_puts();
539 assert_eq!(2, puts.len());
540 let mut markers = puts
541 .into_iter()
542 .flat_map(|put| {
543 let value: DatanodeStatValue = put.value.try_into().unwrap();
544 value
545 .stats
546 .into_iter()
547 .map(|stat| stat.region_num)
548 .collect::<Vec<_>>()
549 })
550 .collect::<Vec<_>>();
551 markers.sort_unstable();
552 assert_eq!((1..=6).collect::<Vec<_>>(), markers);
553 assert_eq!(1, stats_backend.max_active_puts());
554 }
555
556 async fn handle_request_many_times(
557 mut ctx: Context,
558 handler: &CollectStatsHandler,
559 loop_times: i32,
560 ) {
561 let req = HeartbeatRequest::default();
562 for i in 1..=loop_times {
563 let mut acc = HeartbeatAccumulator {
564 stat: Some(Stat {
565 id: 101,
566 region_num: i as _,
567 ..Default::default()
568 }),
569 ..Default::default()
570 };
571 handler.handle(&req, &mut ctx, &mut acc).await.unwrap();
572 }
573 }
574
575 #[derive(Default)]
576 struct RecordingCacheInvalidator {
577 calls: StdMutex<Vec<Vec<CacheIdent>>>,
578 }
579
580 impl RecordingCacheInvalidator {
581 fn table_ids_per_call(&self) -> Vec<Vec<TableId>> {
582 self.calls
583 .lock()
584 .unwrap()
585 .iter()
586 .map(|idents| {
587 let mut ids = idents
588 .iter()
589 .map(|ident| match ident {
590 CacheIdent::TableId(id) => *id,
591 other => panic!("unexpected cache ident: {other:?}"),
592 })
593 .collect::<Vec<_>>();
594 ids.sort_unstable();
595 ids
596 })
597 .collect()
598 }
599 }
600
601 #[async_trait::async_trait]
602 impl CacheInvalidator for RecordingCacheInvalidator {
603 async fn invalidate(
604 &self,
605 _: &InvalidationContext,
606 idents: &[CacheIdent],
607 ) -> MetaResult<()> {
608 self.calls.lock().unwrap().push(idents.to_vec());
609 Ok(())
610 }
611
612 fn invalidate_all(&self) -> MetaResult<()> {
613 Ok(())
614 }
615 }
616
617 async fn register_datanode_tables(ctx: &Context, datanode_id: u64, table_ids: &[TableId]) {
618 for table_id in table_ids {
619 let value = DatanodeTableValue::new(
620 *table_id,
621 RegionRoleSet::new(vec![0], vec![]),
622 RegionInfo::default(),
623 );
624 ctx.kv_backend
625 .put(PutRequest {
626 key: DatanodeTableKey::new(datanode_id, *table_id).to_bytes(),
627 value: value.try_as_raw_value().unwrap(),
628 prev_kv: false,
629 })
630 .await
631 .unwrap();
632 }
633 }
634
635 fn use_recording_invalidator(ctx: &mut Context) -> Arc<RecordingCacheInvalidator> {
636 let invalidator = Arc::new(RecordingCacheInvalidator::default());
637 ctx.cache_invalidator = invalidator.clone();
638 invalidator
639 }
640
641 #[tokio::test]
642 async fn test_address_change_invalidates_all_tables_on_datanode() {
643 let mut ctx = TestEnv::new().ctx();
644 let invalidator = use_recording_invalidator(&mut ctx);
645 register_datanode_tables(&ctx, 101, &[1024, 1025]).await;
646 register_datanode_tables(&ctx, 102, &[2048]).await;
647 let handler = Arc::new(CollectStatsHandler::default());
648
649 handle_stat(handler.clone(), ctx.clone(), stat(101, 1, 0, "dn-101-v1"))
650 .await
651 .unwrap();
652 handle_stat(handler.clone(), ctx.clone(), stat(101, 2, 0, "dn-101-v2"))
654 .await
655 .unwrap();
656 handle_stat(handler, ctx, stat(101, 2, 0, "dn-101-v2"))
657 .await
658 .unwrap();
659
660 assert_eq!(
661 vec![vec![1024, 1025], vec![1024, 1025]],
662 invalidator.table_ids_per_call()
663 );
664 }
665
666 #[tokio::test]
667 async fn test_unchanged_address_only_invalidates_reported_tables() {
668 let mut ctx = TestEnv::new().ctx();
669 register_datanode_tables(&ctx, 101, &[1024, 1025]).await;
670 let handler = Arc::new(CollectStatsHandler::default());
671 handle_stat(handler, ctx.clone(), stat(101, 1, 0, "dn-101"))
672 .await
673 .unwrap();
674
675 let invalidator = use_recording_invalidator(&mut ctx);
677 let handler = Arc::new(CollectStatsHandler::default());
678 handle_stat(handler.clone(), ctx.clone(), stat(101, 1, 0, "dn-101"))
679 .await
680 .unwrap();
681 handle_stat(handler, ctx, stat(101, 2, 0, "dn-101"))
682 .await
683 .unwrap();
684
685 assert_eq!(
686 vec![Vec::<TableId>::new(), vec![]],
687 invalidator.table_ids_per_call()
688 );
689 }
690
691 #[tokio::test]
692 async fn test_retry_after_landed_write_invalidates_all_tables_on_datanode() {
693 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
694 use common_meta::rpc::KeyValue;
695
696 let mut ctx = TestEnv::new().ctx();
697 let invalidator = use_recording_invalidator(&mut ctx);
698 register_datanode_tables(&ctx, 101, &[1024, 1025]).await;
699 let attempts = Arc::new(AtomicUsize::new(0));
700 let backend = MockKvBackendBuilder::default()
701 .put_fn(Arc::new({
702 let attempts = attempts.clone();
703 move |req: PutRequest| {
704 if attempts.fetch_add(1, AtomicOrdering::Relaxed) == 0 {
707 return common_meta::error::UnexpectedSnafu {
708 err_msg: "injected lost response",
709 }
710 .fail();
711 }
712 Ok(PutResponse {
713 prev_kv: Some(KeyValue {
714 key: req.key,
715 value: req.value,
716 }),
717 })
718 }
719 }) as _)
720 .build()
721 .unwrap();
722 ctx.leader_cached_kv_backend =
723 Arc::new(LeaderCachedKvBackend::with_always_leader(Arc::new(backend)));
724 let handler = Arc::new(CollectStatsHandler::default());
725
726 handle_stat(handler.clone(), ctx.clone(), stat(101, 2, 0, "dn-101-v2"))
727 .await
728 .unwrap();
729 handle_stat(handler, ctx, stat(101, 2, 0, "dn-101-v2"))
730 .await
731 .unwrap();
732
733 assert_eq!(2, attempts.load(AtomicOrdering::Relaxed));
734 assert_eq!(vec![vec![1024, 1025]], invalidator.table_ids_per_call());
735 }
736
737 #[tokio::test]
738 async fn test_persistence_failure_retries_but_notification_failure_does_not() {
739 use crate::handler::collect_stats_handler::CollectStatsHandler;
740 use crate::handler::test_utils::FailingCacheInvalidator;
741 use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
742 use crate::service::store::cached_kv::LeaderCachedKvBackend;
743 use api::v1::meta::HeartbeatRequest;
744 use common_meta::key::MetadataValue;
745 use common_meta::key::node_address::NodeAddressValue;
746 use common_meta::kv_backend::test_util::MockKvBackendBuilder;
747 use common_meta::rpc::store::PutResponse;
748 let mut ctx = TestEnv::new().ctx();
749 let puts = Arc::new(std::sync::Mutex::new(Vec::<PutRequest>::new()));
750 let backend = MockKvBackendBuilder::default()
751 .put_fn(Arc::new({
752 let puts = puts.clone();
753 move |req: PutRequest| {
754 let mut puts = puts.lock().unwrap();
755 puts.push(req);
756 if puts.len() == 1 {
757 return common_meta::error::UnexpectedSnafu {
758 err_msg: "injected persistence failure",
759 }
760 .fail();
761 }
762 Ok(PutResponse::default())
763 }
764 }) as _)
765 .build()
766 .unwrap();
767 ctx.leader_cached_kv_backend =
768 Arc::new(LeaderCachedKvBackend::with_always_leader(Arc::new(backend)));
769 let invalidator = Arc::new(FailingCacheInvalidator::default());
770 ctx.cache_invalidator = invalidator.clone();
771 let handler = CollectStatsHandler::default();
772 let mut stat = Stat {
773 id: 101,
774 addr: "dn-original".into(),
775 node_epoch: 7,
776 ..Default::default()
777 };
778 let req = HeartbeatRequest::default();
779
780 assert!(matches!(
782 handler
783 .handle(
784 &req,
785 &mut ctx,
786 &mut HeartbeatAccumulator {
787 stat: Some(stat.clone()),
788 ..Default::default()
789 }
790 )
791 .await
792 .unwrap(),
793 HandleControl::Continue
794 ));
795 assert_eq!(puts.lock().unwrap().len(), 1);
796 assert_eq!(invalidator.attempts(), 0);
797
798 let stats_key: Vec<u8> = common_meta::datanode::DatanodeStatKey { node_id: 101 }.into();
799 assert!(ctx.in_memory.get(&stats_key).await.unwrap().is_some());
800
801 stat.addr = "dn-retry".into();
803 handler
804 .handle(
805 &req,
806 &mut ctx,
807 &mut HeartbeatAccumulator {
808 stat: Some(stat.clone()),
809 ..Default::default()
810 },
811 )
812 .await
813 .unwrap();
814 {
815 let puts = puts.lock().unwrap();
816 assert_eq!(puts.len(), 2);
817 assert_eq!(puts[1].key, NodeAddressKey::with_datanode(101).to_bytes());
818 assert_eq!(
819 NodeAddressValue::try_from_raw_value(&puts[1].value)
820 .unwrap()
821 .peer
822 .addr,
823 "dn-retry"
824 );
825 }
826 assert_eq!(invalidator.attempts(), 1);
827
828 handler
830 .handle(
831 &req,
832 &mut ctx,
833 &mut HeartbeatAccumulator {
834 stat: Some(stat.clone()),
835 ..Default::default()
836 },
837 )
838 .await
839 .unwrap();
840 assert_eq!(puts.lock().unwrap().len(), 2);
841 assert_eq!(invalidator.attempts(), 1);
842
843 stat.node_epoch = 8;
844 handler
845 .handle(
846 &req,
847 &mut ctx,
848 &mut HeartbeatAccumulator {
849 stat: Some(stat.clone()),
850 ..Default::default()
851 },
852 )
853 .await
854 .unwrap();
855 assert_eq!(puts.lock().unwrap().len(), 3);
856 assert_eq!(invalidator.attempts(), 2);
857 }
858}