1use std::time::Duration;
16
17use lazy_static::lazy_static;
18use prometheus::*;
19use puffin::puffin_manager::stager::StagerNotifier;
20
21pub const STAGE_LABEL: &str = "stage";
23pub const TYPE_LABEL: &str = "type";
25const CACHE_EVICTION_CAUSE: &str = "cause";
26pub const FLUSH_REASON: &str = "reason";
28pub const FILE_TYPE_LABEL: &str = "file_type";
30pub const WORKER_LABEL: &str = "worker";
32pub const PARTITION_LABEL: &str = "partition";
34pub const STAGING_TYPE: &str = "index_staging";
36pub const RECYCLE_TYPE: &str = "recycle_bin";
38
39lazy_static! {
41 pub static ref WRITE_BUFFER_BYTES: IntGauge =
43 register_int_gauge!("greptime_mito_write_buffer_bytes", "mito write buffer bytes").unwrap();
44 pub static ref MEMTABLE_DICT_BYTES: IntGauge =
46 register_int_gauge!("greptime_mito_memtable_dict_bytes", "mito memtable dictionary size in bytes").unwrap();
47 pub static ref REGION_COUNT: IntGaugeVec =
49 register_int_gauge_vec!(
50 "greptime_mito_region_count",
51 "mito region count in each worker",
52 &[WORKER_LABEL],
53 ).unwrap();
54 pub static ref HANDLE_REQUEST_ELAPSED: HistogramVec = register_histogram_vec!(
56 "greptime_mito_handle_request_elapsed",
57 "mito handle request elapsed",
58 &[TYPE_LABEL],
59 exponential_buckets(0.01, 10.0, 7).unwrap(),
61 )
62 .unwrap();
63
64 pub static ref FLUSH_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
68 "greptime_mito_flush_requests_total",
69 "mito flush requests total",
70 &[FLUSH_REASON]
71 )
72 .unwrap();
73 pub static ref FLUSH_FAILURE_TOTAL: IntCounter =
75 register_int_counter!("greptime_mito_flush_failure_total", "mito flush failure total").unwrap();
76 pub static ref FLUSH_ELAPSED: HistogramVec = register_histogram_vec!(
78 "greptime_mito_flush_elapsed",
79 "mito flush elapsed",
80 &[TYPE_LABEL],
81 exponential_buckets(1.0, 5.0, 6).unwrap(),
83 )
84 .unwrap();
85 pub static ref FLUSH_BYTES_TOTAL: IntCounter =
87 register_int_counter!("greptime_mito_flush_bytes_total", "mito flush bytes total").unwrap();
88 pub static ref INFLIGHT_FLUSH_COUNT: IntGauge =
90 register_int_gauge!(
91 "greptime_mito_inflight_flush_count",
92 "inflight flush count",
93 ).unwrap();
94 pub static ref WRITE_REJECT_TOTAL: IntCounter =
101 register_int_counter!("greptime_mito_write_reject_total", "mito write reject total").unwrap();
102 pub static ref WRITE_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
104 "greptime_mito_write_stage_elapsed",
105 "mito write stage elapsed",
106 &[STAGE_LABEL],
107 exponential_buckets(0.01, 10.0, 6).unwrap(),
109 )
110 .unwrap();
111 pub static ref WRITE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
113 "greptime_mito_write_rows_total",
114 "mito write rows total",
115 &[TYPE_LABEL]
116 )
117 .unwrap();
118}
119
120lazy_static! {
122 pub static ref COMPACTION_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
132 "greptime_mito_compaction_stage_elapsed",
133 "mito compaction stage elapsed",
134 &[STAGE_LABEL],
135 exponential_buckets(1.0, 10.0, 6).unwrap(),
137 )
138 .unwrap();
139 pub static ref COMPACTION_ELAPSED_TOTAL: Histogram =
141 register_histogram!(
142 "greptime_mito_compaction_total_elapsed",
143 "mito compaction total elapsed",
144 exponential_buckets(1.0, 10.0, 6).unwrap(),
146 ).unwrap();
147 pub static ref COMPACTION_REQUEST_COUNT: IntCounter =
149 register_int_counter!("greptime_mito_compaction_requests_total", "mito compaction requests total").unwrap();
150 pub static ref COMPACTION_FAILURE_COUNT: IntCounter =
152 register_int_counter!("greptime_mito_compaction_failure_total", "mito compaction failure total").unwrap();
153
154 pub static ref INFLIGHT_COMPACTION_COUNT: IntGauge =
156 register_int_gauge!(
157 "greptime_mito_inflight_compaction_count",
158 "inflight compaction count",
159 ).unwrap();
160}
161
162lazy_static! {
164 pub static ref READ_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
166 "greptime_mito_read_stage_elapsed",
167 "mito read stage elapsed",
168 &[STAGE_LABEL],
169 exponential_buckets(0.01, 10.0, 7).unwrap(),
171 )
172 .unwrap();
173 pub static ref READ_STAGE_FETCH_PAGES: Histogram = READ_STAGE_ELAPSED.with_label_values(&["fetch_pages"]);
174 pub static ref IN_PROGRESS_SCAN: IntGaugeVec = register_int_gauge_vec!(
176 "greptime_mito_in_progress_scan",
177 "mito in progress scan per partition",
178 &[TYPE_LABEL, PARTITION_LABEL]
179 )
180 .unwrap();
181 pub static ref READ_ROWS_TOTAL: IntCounterVec =
183 register_int_counter_vec!("greptime_mito_read_rows_total", "mito read rows total", &[TYPE_LABEL]).unwrap();
184 pub static ref MERGE_FILTER_ROWS_TOTAL: IntCounterVec =
186 register_int_counter_vec!("greptime_mito_merge_filter_rows_total", "mito merge filter rows total", &[TYPE_LABEL]).unwrap();
187 pub static ref READ_ROW_GROUPS_TOTAL: IntCounterVec =
189 register_int_counter_vec!("greptime_mito_read_row_groups_total", "mito read row groups total", &[TYPE_LABEL]).unwrap();
190 pub static ref PRECISE_FILTER_ROWS_TOTAL: IntCounterVec =
192 register_int_counter_vec!("greptime_mito_precise_filter_rows_total", "mito precise filter rows total", &[TYPE_LABEL]).unwrap();
193 pub static ref READ_ROWS_IN_ROW_GROUP_TOTAL: IntCounterVec =
194 register_int_counter_vec!("greptime_mito_read_rows_in_row_group_total", "mito read rows in row group total", &[TYPE_LABEL]).unwrap();
195 pub static ref READ_SST_COUNT: Histogram = register_histogram!(
197 "greptime_mito_read_sst_count",
198 "Number of SSTs to scan in a scan task",
199 vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 256.0, 1024.0],
200 ).unwrap();
201 pub static ref READ_ROWS_RETURN: Histogram = register_histogram!(
203 "greptime_mito_read_rows_return",
204 "Number of rows returned in a scan task",
205 exponential_buckets(100.0, 10.0, 8).unwrap(),
206 ).unwrap();
207 pub static ref READ_BATCHES_RETURN: Histogram = register_histogram!(
209 "greptime_mito_read_batches_return",
210 "Number of rows returned in a scan task",
211 exponential_buckets(100.0, 10.0, 7).unwrap(),
212 ).unwrap();
213 pub static ref SCAN_MEMORY_USAGE_BYTES: IntGauge = register_int_gauge!(
215 "greptime_mito_scan_memory_usage_bytes",
216 "current scan memory usage in bytes"
217 ).unwrap();
218 pub static ref SCAN_REQUESTS_REJECTED_TOTAL: IntCounter = register_int_counter!(
220 "greptime_mito_scan_requests_rejected_total",
221 "total number of scan requests rejected due to memory limit"
222 ).unwrap();
223}
224
225lazy_static! {
227 pub static ref CACHE_HIT: IntCounterVec = register_int_counter_vec!(
229 "greptime_mito_cache_hit",
230 "mito cache hit",
231 &[TYPE_LABEL]
232 )
233 .unwrap();
234 pub static ref CACHE_MISS: IntCounterVec = register_int_counter_vec!(
236 "greptime_mito_cache_miss",
237 "mito cache miss",
238 &[TYPE_LABEL]
239 )
240 .unwrap();
241 pub static ref CACHE_BYTES: IntGaugeVec = register_int_gauge_vec!(
243 "greptime_mito_cache_bytes",
244 "mito cache bytes",
245 &[TYPE_LABEL]
246 )
247 .unwrap();
248 pub static ref WRITE_CACHE_DOWNLOAD_BYTES_TOTAL: IntCounter = register_int_counter!(
250 "mito_write_cache_download_bytes_total",
251 "mito write cache download bytes total",
252 ).unwrap();
253 pub static ref WRITE_CACHE_DOWNLOAD_ELAPSED: HistogramVec = register_histogram_vec!(
255 "mito_write_cache_download_elapsed",
256 "mito write cache download elapsed",
257 &[TYPE_LABEL],
258 exponential_buckets(0.1, 10.0, 6).unwrap(),
260 ).unwrap();
261 pub static ref WRITE_CACHE_INFLIGHT_DOWNLOAD: IntGauge = register_int_gauge!(
263 "mito_write_cache_inflight_download_count",
264 "mito write cache inflight download tasks",
265 ).unwrap();
266 pub static ref UPLOAD_BYTES_TOTAL: IntCounter = register_int_counter!(
268 "mito_upload_bytes_total",
269 "mito upload bytes total",
270 )
271 .unwrap();
272 pub static ref CACHE_EVICTION: IntCounterVec = register_int_counter_vec!(
274 "greptime_mito_cache_eviction",
275 "mito cache eviction",
276 &[TYPE_LABEL, CACHE_EVICTION_CAUSE]
277 ).unwrap();
278}
279
280lazy_static! {
282 pub static ref INDEX_APPLY_ELAPSED: HistogramVec = register_histogram_vec!(
285 "greptime_index_apply_elapsed",
286 "index apply elapsed",
287 &[TYPE_LABEL],
288 exponential_buckets(0.01, 10.0, 6).unwrap(),
290 )
291 .unwrap();
292 pub static ref INDEX_APPLY_MEMORY_USAGE: IntGauge = register_int_gauge!(
294 "greptime_index_apply_memory_usage",
295 "index apply memory usage",
296 )
297 .unwrap();
298 pub static ref INDEX_CREATE_ELAPSED: HistogramVec = register_histogram_vec!(
300 "greptime_index_create_elapsed",
301 "index create elapsed",
302 &[STAGE_LABEL, TYPE_LABEL],
303 exponential_buckets(0.1, 10.0, 6).unwrap(),
305 )
306 .unwrap();
307 pub static ref INDEX_CREATE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
309 "greptime_index_create_rows_total",
310 "index create rows total",
311 &[TYPE_LABEL],
312 )
313 .unwrap();
314 pub static ref INDEX_CREATE_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
316 "greptime_index_create_bytes_total",
317 "index create bytes total",
318 &[TYPE_LABEL],
319 )
320 .unwrap();
321 pub static ref INDEX_CREATE_MEMORY_USAGE: IntGaugeVec = register_int_gauge_vec!(
323 "greptime_index_create_memory_usage",
324 "index create memory usage",
325 &[TYPE_LABEL],
326 ).unwrap();
327 pub static ref INDEX_IO_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
329 "greptime_index_io_bytes_total",
330 "index io bytes total",
331 &[TYPE_LABEL, FILE_TYPE_LABEL]
332 )
333 .unwrap();
334 pub static ref INDEX_PUFFIN_READ_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
336 .with_label_values(&["read", "puffin"]);
337 pub static ref INDEX_PUFFIN_WRITE_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
339 .with_label_values(&["write", "puffin"]);
340 pub static ref INDEX_INTERMEDIATE_READ_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
342 .with_label_values(&["read", "intermediate"]);
343 pub static ref INDEX_INTERMEDIATE_WRITE_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
345 .with_label_values(&["write", "intermediate"]);
346
347 pub static ref INDEX_IO_OP_TOTAL: IntCounterVec = register_int_counter_vec!(
349 "greptime_index_io_op_total",
350 "index io op total",
351 &[TYPE_LABEL, FILE_TYPE_LABEL]
352 )
353 .unwrap();
354 pub static ref INDEX_PUFFIN_READ_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
356 .with_label_values(&["read", "puffin"]);
357 pub static ref INDEX_PUFFIN_SEEK_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
359 .with_label_values(&["seek", "puffin"]);
360 pub static ref INDEX_PUFFIN_WRITE_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
362 .with_label_values(&["write", "puffin"]);
363 pub static ref INDEX_PUFFIN_FLUSH_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
365 .with_label_values(&["flush", "puffin"]);
366 pub static ref INDEX_INTERMEDIATE_READ_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
368 .with_label_values(&["read", "intermediate"]);
369 pub static ref INDEX_INTERMEDIATE_SEEK_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
371 .with_label_values(&["seek", "intermediate"]);
372 pub static ref INDEX_INTERMEDIATE_WRITE_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
374 .with_label_values(&["write", "intermediate"]);
375 pub static ref INDEX_INTERMEDIATE_FLUSH_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
377 .with_label_values(&["flush", "intermediate"]);
378}
379
380lazy_static! {
381 pub static ref PARTITION_TREE_DATA_BUFFER_FREEZE_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
383 "greptime_partition_tree_buffer_freeze_stage_elapsed",
384 "mito partition tree data buffer freeze stage elapsed",
385 &[STAGE_LABEL],
386 exponential_buckets(0.01, 10.0, 6).unwrap(),
388 )
389 .unwrap();
390
391 pub static ref PARTITION_TREE_READ_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
393 "greptime_partition_tree_read_stage_elapsed",
394 "mito partition tree read stage elapsed",
395 &[STAGE_LABEL],
396 exponential_buckets(0.01, 10.0, 6).unwrap(),
398 )
399 .unwrap();
400
401 pub static ref MANIFEST_OP_ELAPSED: HistogramVec = register_histogram_vec!(
408 "greptime_manifest_op_elapsed",
409 "mito manifest operation elapsed",
410 &["op"],
411 exponential_buckets(0.01, 10.0, 6).unwrap(),
413 ).unwrap();
414
415
416 pub static ref REGION_WORKER_HANDLE_WRITE_ELAPSED: HistogramVec = register_histogram_vec!(
417 "greptime_region_worker_handle_write",
418 "elapsed time for handling writes in region worker loop",
419 &["stage"],
420 exponential_buckets(0.001, 10.0, 5).unwrap()
421 ).unwrap();
422
423}
424
425lazy_static! {
426 pub static ref COMPACTION_INPUT_BYTES: Counter = register_counter!(
428 "greptime_mito_compaction_input_bytes",
429 "mito compaction input file size",
430 ).unwrap();
431
432 pub static ref COMPACTION_OUTPUT_BYTES: Counter = register_counter!(
434 "greptime_mito_compaction_output_bytes",
435 "mito compaction output file size",
436 ).unwrap();
437
438 pub static ref MEMTABLE_ACTIVE_SERIES_COUNT: IntGauge = register_int_gauge!(
440 "greptime_mito_memtable_active_series_count",
441 "active time series count in TimeSeriesMemtable",
442 ).unwrap();
443
444 pub static ref MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT: IntGauge = register_int_gauge!(
446 "greptime_mito_memtable_field_builder_count",
447 "active field builder count in TimeSeriesMemtable",
448 ).unwrap();
449
450 pub static ref WRITE_STALLING: IntGaugeVec = register_int_gauge_vec!(
452 "greptime_mito_write_stalling_count",
453 "mito stalled write request in each worker",
454 &[WORKER_LABEL]
455 ).unwrap();
456 pub static ref GC_REF_FILE_CNT: IntGauge = register_int_gauge!(
458 "greptime_gc_ref_file_count",
459 "gc ref file count",
460 ).unwrap();
461 pub static ref WRITE_STALL_TOTAL: IntCounter = register_int_counter!(
463 "greptime_mito_write_stall_total",
464 "Total number of stalled write requests"
465 ).unwrap();
466 pub static ref REQUEST_WAIT_TIME: HistogramVec = register_histogram_vec!(
468 "greptime_mito_request_wait_time",
469 "mito request wait time before being handled by region worker",
470 &[WORKER_LABEL],
471 exponential_buckets(0.001, 10.0, 8).unwrap(),
473 )
474 .unwrap();
475
476 pub static ref GC_DELETE_FILE_CNT: IntGauge =
478 register_int_gauge!(
479 "greptime_mito_gc_delete_file_count",
480 "mito gc deleted file count",
481 ).unwrap();
482
483 pub static ref CACHE_FILL_DOWNLOADED_FILES: IntCounter = register_int_counter!(
485 "mito_cache_fill_downloaded_files",
486 "mito cache fill downloaded files count",
487 ).unwrap();
488
489 pub static ref CACHE_FILL_PENDING_FILES: IntGauge = register_int_gauge!(
491 "mito_cache_fill_pending_files",
492 "mito cache fill pending files count",
493 ).unwrap();
494}
495
496pub struct StagerMetrics {
498 cache_hit: IntCounter,
499 cache_miss: IntCounter,
500 staging_cache_bytes: IntGauge,
501 recycle_cache_bytes: IntGauge,
502 cache_eviction: IntCounter,
503 staging_miss_read: Histogram,
504}
505
506impl StagerMetrics {
507 pub fn new() -> Self {
509 Self {
510 cache_hit: CACHE_HIT.with_label_values(&[STAGING_TYPE]),
511 cache_miss: CACHE_MISS.with_label_values(&[STAGING_TYPE]),
512 staging_cache_bytes: CACHE_BYTES.with_label_values(&[STAGING_TYPE]),
513 recycle_cache_bytes: CACHE_BYTES.with_label_values(&[RECYCLE_TYPE]),
514 cache_eviction: CACHE_EVICTION.with_label_values(&[STAGING_TYPE, "size"]),
515 staging_miss_read: READ_STAGE_ELAPSED.with_label_values(&["staging_miss_read"]),
516 }
517 }
518}
519
520impl Default for StagerMetrics {
521 fn default() -> Self {
522 Self::new()
523 }
524}
525
526impl StagerNotifier for StagerMetrics {
527 fn on_cache_hit(&self, _size: u64) {
528 self.cache_hit.inc();
529 }
530
531 fn on_cache_miss(&self, _size: u64) {
532 self.cache_miss.inc();
533 }
534
535 fn on_cache_insert(&self, size: u64) {
536 self.staging_cache_bytes.add(size as i64);
537 }
538
539 fn on_load_dir(&self, duration: Duration) {
540 self.staging_miss_read.observe(duration.as_secs_f64());
541 }
542
543 fn on_load_blob(&self, duration: Duration) {
544 self.staging_miss_read.observe(duration.as_secs_f64());
545 }
546
547 fn on_cache_evict(&self, size: u64) {
548 self.cache_eviction.inc();
549 self.staging_cache_bytes.sub(size as i64);
550 }
551
552 fn on_recycle_insert(&self, size: u64) {
553 self.recycle_cache_bytes.add(size as i64);
554 }
555
556 fn on_recycle_clear(&self, size: u64) {
557 self.recycle_cache_bytes.sub(size as i64);
558 }
559}