1use std::time::{Duration, Instant};
16
17use api::v1::meta::{HeartbeatRequest, Role};
18use api::v1::value::ValueData;
19use api::v1::{ColumnSchema, Row, RowInsertRequest, RowInsertRequests, Rows, Value};
20use client::DEFAULT_CATALOG_NAME;
21use client::inserter::{Context as InserterContext, Inserter};
22use common_catalog::consts::DEFAULT_PRIVATE_SCHEMA_NAME;
23use common_macro::{Schema, ToRow};
24use common_meta::DatanodeId;
25use common_meta::datanode::{REGION_STATS_HISTORY_TABLE_NAME, RegionStat};
26use common_telemetry::warn;
27use dashmap::DashMap;
28use store_api::region_engine::RegionRole;
29use store_api::storage::RegionId;
30
31use crate::error::Result;
32use crate::handler::{HandleControl, HeartbeatAccumulator, HeartbeatHandler};
33use crate::metasrv::Context;
34
35pub struct PersistStatsHandler {
37 inserter: Box<dyn Inserter>,
38 last_persisted_region_stats: DashMap<RegionId, PersistedRegionStat>,
39 last_persisted_time: DashMap<DatanodeId, Instant>,
40 persist_interval: Duration,
41}
42
43const DEFAULT_CONTEXT: InserterContext = InserterContext {
45 catalog: DEFAULT_CATALOG_NAME,
46 schema: DEFAULT_PRIVATE_SCHEMA_NAME,
47};
48
49#[derive(Debug, Clone, Copy, PartialEq, Eq)]
50struct PersistedRegionStat {
51 region_id: RegionId,
52 written_bytes: u64,
53}
54
55impl From<&RegionStat> for PersistedRegionStat {
56 fn from(stat: &RegionStat) -> Self {
57 Self {
58 region_id: stat.id,
59 written_bytes: stat.written_bytes,
60 }
61 }
62}
63
64#[derive(ToRow, Schema)]
65struct PersistRegionStat<'a> {
66 table_id: u32,
67 region_id: u64,
68 region_number: u32,
69 manifest_size: u64,
70 datanode_id: u64,
71 #[col(datatype = "string")]
72 engine: &'a str,
73 num_rows: u64,
74 sst_num: u64,
75 sst_size: u64,
76 write_bytes_delta: u64,
77 #[col(
78 name = "greptime_timestamp",
80 semantic = "Timestamp",
81 datatype = "TimestampMillisecond"
82 )]
83 timestamp_millis: i64,
84}
85
86fn compute_persist_region_stat(
88 region_stat: &RegionStat,
89 datanode_id: DatanodeId,
90 timestamp_millis: i64,
91 persisted_region_stat: Option<PersistedRegionStat>,
92) -> PersistRegionStat<'_> {
93 let write_bytes_delta = persisted_region_stat
94 .and_then(|persisted_region_stat| {
95 region_stat
96 .written_bytes
97 .checked_sub(persisted_region_stat.written_bytes)
98 })
99 .unwrap_or_default();
100
101 PersistRegionStat {
102 table_id: region_stat.id.table_id(),
103 region_id: region_stat.id.as_u64(),
104 region_number: region_stat.id.region_number(),
105 manifest_size: region_stat.manifest_size,
106 datanode_id,
107 engine: region_stat.engine.as_str(),
108 num_rows: region_stat.num_rows,
109 sst_num: region_stat.sst_num,
110 sst_size: region_stat.sst_size,
111 write_bytes_delta,
112 timestamp_millis,
113 }
114}
115
116fn to_persisted_if_leader(
117 region_stat: &RegionStat,
118 last_persisted_region_stats: &DashMap<RegionId, PersistedRegionStat>,
119 datanode_id: DatanodeId,
120 timestamp_millis: i64,
121) -> Option<(Row, PersistedRegionStat)> {
122 if matches!(
123 region_stat.role,
124 RegionRole::Leader | RegionRole::StagingLeader
125 ) {
126 let persisted_region_stat = last_persisted_region_stats.get(®ion_stat.id).map(|s| *s);
127 Some((
128 compute_persist_region_stat(
129 region_stat,
130 datanode_id,
131 timestamp_millis,
132 persisted_region_stat,
133 )
134 .to_row(),
135 PersistedRegionStat::from(region_stat),
136 ))
137 } else {
138 None
139 }
140}
141
142fn align_ts(ts: i64, interval: Duration) -> i64 {
147 assert!(
148 interval.as_millis() != 0,
149 "interval must be greater than zero"
150 );
151 ts / interval.as_millis() as i64 * interval.as_millis() as i64
152}
153
154impl PersistStatsHandler {
155 pub fn new(inserter: Box<dyn Inserter>, mut persist_interval: Duration) -> Self {
157 if persist_interval < Duration::from_secs(10 * 60) {
158 warn!("persist_interval is less than 10 minutes, set to 10 minutes");
159 persist_interval = Duration::from_secs(10 * 60);
160 }
161
162 Self {
163 inserter,
164 last_persisted_region_stats: DashMap::new(),
165 last_persisted_time: DashMap::new(),
166 persist_interval,
167 }
168 }
169
170 fn should_persist(&self, datanode_id: DatanodeId) -> bool {
171 let Some(last_persisted_time) = self.last_persisted_time.get(&datanode_id) else {
172 return true;
173 };
174
175 last_persisted_time.elapsed() >= self.persist_interval
176 }
177
178 async fn persist(
179 &self,
180 timestamp_millis: i64,
181 datanode_id: DatanodeId,
182 region_stats: &[RegionStat],
183 ) {
184 let aligned_ts = align_ts(timestamp_millis, self.persist_interval);
186 let (rows, incoming_region_stats): (Vec<_>, Vec<_>) = region_stats
187 .iter()
188 .flat_map(|region_stat| {
189 to_persisted_if_leader(
190 region_stat,
191 &self.last_persisted_region_stats,
192 datanode_id,
193 aligned_ts,
194 )
195 })
196 .unzip();
197
198 if rows.is_empty() {
199 return;
200 }
201
202 if let Err(err) = self
203 .inserter
204 .insert_rows(
205 &DEFAULT_CONTEXT,
206 RowInsertRequests {
207 inserts: vec![RowInsertRequest {
208 table_name: REGION_STATS_HISTORY_TABLE_NAME.to_string(),
209 rows: Some(Rows {
210 schema: PersistRegionStat::schema(),
211 rows,
212 }),
213 }],
214 },
215 )
216 .await
217 {
218 warn!(
219 "Failed to persist region stats, datanode_id: {}, error: {:?}",
220 datanode_id, err
221 );
222 return;
223 }
224
225 self.last_persisted_time.insert(datanode_id, Instant::now());
226 for s in incoming_region_stats {
227 self.last_persisted_region_stats.insert(s.region_id, s);
228 }
229 }
230}
231
232#[async_trait::async_trait]
233impl HeartbeatHandler for PersistStatsHandler {
234 fn is_acceptable(&self, role: Role) -> bool {
235 role == Role::Datanode
236 }
237
238 async fn handle(
239 &self,
240 _req: &HeartbeatRequest,
241 _: &mut Context,
242 acc: &mut HeartbeatAccumulator,
243 ) -> Result<HandleControl> {
244 let Some(current_stat) = acc.stat.as_ref() else {
245 return Ok(HandleControl::Continue);
246 };
247
248 if !self.should_persist(current_stat.id) {
249 return Ok(HandleControl::Continue);
250 }
251
252 self.persist(
253 current_stat.timestamp_millis,
254 current_stat.id,
255 ¤t_stat.region_stats,
256 )
257 .await;
258
259 Ok(HandleControl::Continue)
260 }
261}
262
263#[cfg(test)]
264mod tests {
265 use std::sync::{Arc, Mutex};
266
267 use client::inserter::{Context as InserterContext, InsertOptions};
268 use common_meta::datanode::{RegionManifestInfo, RegionStat, Stat};
269 use store_api::region_engine::RegionRole;
270 use store_api::storage::RegionId;
271
272 use super::*;
273 use crate::handler::test_utils::TestEnv;
274
275 fn create_test_region_stat(
276 table_id: u32,
277 region_number: u32,
278 written_bytes: u64,
279 engine: &str,
280 ) -> RegionStat {
281 let region_id = RegionId::new(table_id, region_number);
282 RegionStat {
283 id: region_id,
284 rcus: 100,
285 wcus: 200,
286 approximate_bytes: 1024,
287 engine: engine.to_string(),
288 role: RegionRole::Leader,
289 num_rows: 1000,
290 memtable_size: 512,
291 manifest_size: 256,
292 sst_size: 2048,
293 sst_num: 5,
294 index_size: 128,
295 region_manifest: RegionManifestInfo::Mito {
296 manifest_version: 1,
297 flushed_entry_id: 100,
298 file_removed_cnt: 0,
299 },
300 written_bytes,
301 ingested_parquet_bytes: 0,
302 query_cpu_time: 0,
303 query_scanned_bytes: 0,
304 data_topic_latest_entry_id: 200,
305 metadata_topic_latest_entry_id: 200,
306 min_timestamp: None,
307 max_timestamp: None,
308 has_pending_tails: false,
309 }
310 }
311
312 #[test]
313 fn test_compute_persist_region_stat_with_no_persisted_stat() {
314 let region_stat = create_test_region_stat(1, 1, 1000, "mito");
315 let datanode_id = 123;
316 let timestamp_millis = 1640995200000; let result = compute_persist_region_stat(®ion_stat, datanode_id, timestamp_millis, None);
318 assert_eq!(result.table_id, 1);
319 assert_eq!(result.region_id, region_stat.id.as_u64());
320 assert_eq!(result.region_number, 1);
321 assert_eq!(result.manifest_size, 256);
322 assert_eq!(result.datanode_id, datanode_id);
323 assert_eq!(result.engine, "mito");
324 assert_eq!(result.num_rows, 1000);
325 assert_eq!(result.sst_num, 5);
326 assert_eq!(result.sst_size, 2048);
327 assert_eq!(result.write_bytes_delta, 0); assert_eq!(result.timestamp_millis, timestamp_millis);
329 }
330
331 #[test]
332 fn test_compute_persist_region_stat_with_persisted_stat_increase() {
333 let region_stat = create_test_region_stat(2, 3, 1500, "mito");
334 let datanode_id = 456;
335 let timestamp_millis = 1640995260000; let persisted_stat = PersistedRegionStat {
337 region_id: region_stat.id,
338 written_bytes: 1000, };
340 let result = compute_persist_region_stat(
341 ®ion_stat,
342 datanode_id,
343 timestamp_millis,
344 Some(persisted_stat),
345 );
346 assert_eq!(result.table_id, 2);
347 assert_eq!(result.region_id, region_stat.id.as_u64());
348 assert_eq!(result.region_number, 3);
349 assert_eq!(result.manifest_size, 256);
350 assert_eq!(result.datanode_id, datanode_id);
351 assert_eq!(result.engine, "mito");
352 assert_eq!(result.num_rows, 1000);
353 assert_eq!(result.sst_num, 5);
354 assert_eq!(result.sst_size, 2048);
355 assert_eq!(result.write_bytes_delta, 500); assert_eq!(result.timestamp_millis, timestamp_millis);
357 }
358
359 #[test]
360 fn test_compute_persist_region_stat_with_persisted_stat_decrease() {
361 let region_stat = create_test_region_stat(3, 5, 800, "mito");
362 let datanode_id = 789;
363 let timestamp_millis = 1640995320000; let persisted_stat = PersistedRegionStat {
365 region_id: region_stat.id,
366 written_bytes: 1200, };
368 let result = compute_persist_region_stat(
369 ®ion_stat,
370 datanode_id,
371 timestamp_millis,
372 Some(persisted_stat),
373 );
374 assert_eq!(result.table_id, 3);
375 assert_eq!(result.region_id, region_stat.id.as_u64());
376 assert_eq!(result.region_number, 5);
377 assert_eq!(result.manifest_size, 256);
378 assert_eq!(result.datanode_id, datanode_id);
379 assert_eq!(result.engine, "mito");
380 assert_eq!(result.num_rows, 1000);
381 assert_eq!(result.sst_num, 5);
382 assert_eq!(result.sst_size, 2048);
383 assert_eq!(result.write_bytes_delta, 0); assert_eq!(result.timestamp_millis, timestamp_millis);
385 }
386
387 #[test]
388 fn test_compute_persist_region_stat_with_persisted_stat_equal() {
389 let region_stat = create_test_region_stat(4, 7, 2000, "mito");
390 let datanode_id = 101;
391 let timestamp_millis = 1640995380000; let persisted_stat = PersistedRegionStat {
393 region_id: region_stat.id,
394 written_bytes: 2000, };
396 let result = compute_persist_region_stat(
397 ®ion_stat,
398 datanode_id,
399 timestamp_millis,
400 Some(persisted_stat),
401 );
402 assert_eq!(result.table_id, 4);
403 assert_eq!(result.region_id, region_stat.id.as_u64());
404 assert_eq!(result.region_number, 7);
405 assert_eq!(result.manifest_size, 256);
406 assert_eq!(result.datanode_id, datanode_id);
407 assert_eq!(result.engine, "mito");
408 assert_eq!(result.num_rows, 1000);
409 assert_eq!(result.sst_num, 5);
410 assert_eq!(result.sst_size, 2048);
411 assert_eq!(result.write_bytes_delta, 0); assert_eq!(result.timestamp_millis, timestamp_millis);
413 }
414
415 #[test]
416 fn test_compute_persist_region_stat_with_overflow_protection() {
417 let region_stat = create_test_region_stat(8, 15, 500, "mito");
418 let datanode_id = 505;
419 let timestamp_millis = 1640995620000; let persisted_stat = PersistedRegionStat {
421 region_id: region_stat.id,
422 written_bytes: 1000, };
424 let result = compute_persist_region_stat(
425 ®ion_stat,
426 datanode_id,
427 timestamp_millis,
428 Some(persisted_stat),
429 );
430 assert_eq!(result.table_id, 8);
431 assert_eq!(result.region_id, region_stat.id.as_u64());
432 assert_eq!(result.region_number, 15);
433 assert_eq!(result.manifest_size, 256);
434 assert_eq!(result.datanode_id, datanode_id);
435 assert_eq!(result.engine, "mito");
436 assert_eq!(result.num_rows, 1000);
437 assert_eq!(result.sst_num, 5);
438 assert_eq!(result.sst_size, 2048);
439 assert_eq!(result.write_bytes_delta, 0); assert_eq!(result.timestamp_millis, timestamp_millis);
441 }
442
443 struct MockInserter {
444 requests: Arc<Mutex<Vec<api::v1::RowInsertRequest>>>,
445 }
446
447 #[async_trait::async_trait]
448 impl Inserter for MockInserter {
449 async fn insert_rows(
450 &self,
451 _context: &InserterContext<'_>,
452 requests: api::v1::RowInsertRequests,
453 ) -> client::error::Result<()> {
454 self.requests.lock().unwrap().extend(requests.inserts);
455
456 Ok(())
457 }
458
459 fn set_options(&mut self, _options: &InsertOptions) {}
460 }
461
462 #[tokio::test]
463 async fn test_not_persist_region_stats() {
464 let env = TestEnv::new();
465 let mut ctx = env.ctx();
466
467 let requests = Arc::new(Mutex::new(vec![]));
468 let inserter = MockInserter {
469 requests: requests.clone(),
470 };
471 let handler = PersistStatsHandler::new(Box::new(inserter), Duration::from_secs(10));
472 let mut acc = HeartbeatAccumulator {
473 stat: Some(Stat {
474 id: 1,
475 timestamp_millis: 1640995200000,
476 region_stats: vec![create_test_region_stat(1, 1, 1000, "mito")],
477 ..Default::default()
478 }),
479 ..Default::default()
480 };
481 handler.last_persisted_time.insert(1, Instant::now());
482 handler
484 .handle(&HeartbeatRequest::default(), &mut ctx, &mut acc)
485 .await
486 .unwrap();
487 assert!(requests.lock().unwrap().is_empty());
488 }
489
490 #[tokio::test]
491 async fn test_persist_region_stats() {
492 let env = TestEnv::new();
493 let mut ctx = env.ctx();
494 let requests = Arc::new(Mutex::new(vec![]));
495 let inserter = MockInserter {
496 requests: requests.clone(),
497 };
498
499 let handler = PersistStatsHandler::new(Box::new(inserter), Duration::from_secs(10));
500
501 let region_stat = create_test_region_stat(1, 1, 1000, "mito");
502 let timestamp_millis = 1640995200000;
503 let datanode_id = 1;
504 let region_id = RegionId::new(1, 1);
505 let mut acc = HeartbeatAccumulator {
506 stat: Some(Stat {
507 id: datanode_id,
508 timestamp_millis,
509 region_stats: vec![region_stat.clone()],
510 ..Default::default()
511 }),
512 ..Default::default()
513 };
514
515 handler.last_persisted_region_stats.insert(
516 region_id,
517 PersistedRegionStat {
518 region_id,
519 written_bytes: 500,
520 },
521 );
522 let (expected_row, expected_persisted_region_stat) = to_persisted_if_leader(
523 ®ion_stat,
524 &handler.last_persisted_region_stats,
525 datanode_id,
526 timestamp_millis,
527 )
528 .unwrap();
529 let before_insert_time = Instant::now();
530 handler
532 .handle(&HeartbeatRequest::default(), &mut ctx, &mut acc)
533 .await
534 .unwrap();
535 let request = {
536 let mut requests = requests.lock().unwrap();
537 assert_eq!(requests.len(), 1);
538 requests.pop().unwrap()
539 };
540 assert_eq!(
541 request.table_name,
542 REGION_STATS_HISTORY_TABLE_NAME.to_string()
543 );
544 assert_eq!(request.rows.unwrap().rows, vec![expected_row]);
545
546 assert!(
548 handler
549 .last_persisted_time
550 .get(&datanode_id)
551 .unwrap()
552 .gt(&before_insert_time)
553 );
554
555 assert_eq!(
557 handler
558 .last_persisted_region_stats
559 .get(®ion_id)
560 .unwrap()
561 .value(),
562 &expected_persisted_region_stat
563 );
564 }
565}