mito2/
metrics.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::time::Duration;
16
17use lazy_static::lazy_static;
18use prometheus::*;
19use puffin::puffin_manager::stager::StagerNotifier;
20
21/// Stage label.
22pub const STAGE_LABEL: &str = "stage";
23/// Type label.
24pub const TYPE_LABEL: &str = "type";
25const CACHE_EVICTION_CAUSE: &str = "cause";
26/// Reason to flush.
27pub const FLUSH_REASON: &str = "reason";
28/// File type label.
29pub const FILE_TYPE_LABEL: &str = "file_type";
30/// Region worker id label.
31pub const WORKER_LABEL: &str = "worker";
32/// Partition label.
33pub const PARTITION_LABEL: &str = "partition";
34/// Staging dir type label.
35pub const STAGING_TYPE: &str = "index_staging";
36/// Recycle bin type label.
37pub const RECYCLE_TYPE: &str = "recycle_bin";
38
39lazy_static! {
40    /// Global write buffer size in bytes.
41    pub static ref WRITE_BUFFER_BYTES: IntGauge =
42        register_int_gauge!("greptime_mito_write_buffer_bytes", "mito write buffer bytes").unwrap();
43    /// Global memtable dictionary size in bytes.
44    pub static ref MEMTABLE_DICT_BYTES: IntGauge =
45        register_int_gauge!("greptime_mito_memtable_dict_bytes", "mito memtable dictionary size in bytes").unwrap();
46    /// Gauge for open regions in each worker.
47    pub static ref REGION_COUNT: IntGaugeVec =
48        register_int_gauge_vec!(
49            "greptime_mito_region_count",
50            "mito region count in each worker",
51            &[WORKER_LABEL],
52        ).unwrap();
53    /// Elapsed time to handle requests.
54    pub static ref HANDLE_REQUEST_ELAPSED: HistogramVec = register_histogram_vec!(
55            "greptime_mito_handle_request_elapsed",
56            "mito handle request elapsed",
57            &[TYPE_LABEL],
58            // 0.01 ~ 10000
59            exponential_buckets(0.01, 10.0, 7).unwrap(),
60        )
61        .unwrap();
62
63    // ------ Flush related metrics
64    /// Counter of scheduled flush requests.
65    /// Note that the flush scheduler may merge some flush requests.
66    pub static ref FLUSH_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
67            "greptime_mito_flush_requests_total",
68            "mito flush requests total",
69            &[FLUSH_REASON]
70        )
71        .unwrap();
72    /// Counter of scheduled failed flush jobs.
73    pub static ref FLUSH_ERRORS_TOTAL: IntCounter =
74        register_int_counter!("greptime_mito_flush_errors_total", "mito flush errors total").unwrap();
75    /// Elapsed time of a flush job.
76    pub static ref FLUSH_ELAPSED: HistogramVec = register_histogram_vec!(
77            "greptime_mito_flush_elapsed",
78            "mito flush elapsed",
79            &[TYPE_LABEL],
80            // 1 ~ 625
81            exponential_buckets(1.0, 5.0, 6).unwrap(),
82        )
83        .unwrap();
84    /// Histogram of flushed bytes.
85    pub static ref FLUSH_BYTES_TOTAL: IntCounter =
86        register_int_counter!("greptime_mito_flush_bytes_total", "mito flush bytes total").unwrap();
87    /// Gauge for inflight compaction tasks.
88    pub static ref INFLIGHT_FLUSH_COUNT: IntGauge =
89        register_int_gauge!(
90            "greptime_mito_inflight_flush_count",
91            "inflight flush count",
92        ).unwrap();
93    // ------ End of flush related metrics
94
95
96    // ------ Write related metrics
97    /// Number of stalled write requests in each worker.
98    pub static ref WRITE_STALL_TOTAL: IntGaugeVec = register_int_gauge_vec!(
99            "greptime_mito_write_stall_total",
100            "mito stalled write request in each worker",
101            &[WORKER_LABEL]
102        ).unwrap();
103    /// Counter of rejected write requests.
104    pub static ref WRITE_REJECT_TOTAL: IntCounter =
105        register_int_counter!("greptime_mito_write_reject_total", "mito write reject total").unwrap();
106    /// Elapsed time of each write stage.
107    pub static ref WRITE_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
108            "greptime_mito_write_stage_elapsed",
109            "mito write stage elapsed",
110            &[STAGE_LABEL],
111            // 0.01 ~ 1000
112            exponential_buckets(0.01, 10.0, 6).unwrap(),
113        )
114        .unwrap();
115    /// Counter of rows to write.
116    pub static ref WRITE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
117        "greptime_mito_write_rows_total",
118        "mito write rows total",
119        &[TYPE_LABEL]
120    )
121    .unwrap();
122    // ------ End of write related metrics
123
124
125    // Compaction metrics
126    /// Timer of different stages in compaction.
127    pub static ref COMPACTION_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
128        "greptime_mito_compaction_stage_elapsed",
129        "mito compaction stage elapsed",
130        &[STAGE_LABEL],
131        // 1 ~ 100000
132        exponential_buckets(1.0, 10.0, 6).unwrap(),
133    )
134    .unwrap();
135    /// Timer of whole compaction task.
136    pub static ref COMPACTION_ELAPSED_TOTAL: Histogram =
137        register_histogram!(
138        "greptime_mito_compaction_total_elapsed",
139        "mito compaction total elapsed",
140        // 1 ~ 100000
141        exponential_buckets(1.0, 10.0, 6).unwrap(),
142    ).unwrap();
143    /// Counter of all requested compaction task.
144    pub static ref COMPACTION_REQUEST_COUNT: IntCounter =
145        register_int_counter!("greptime_mito_compaction_requests_total", "mito compaction requests total").unwrap();
146    /// Counter of failed compaction task.
147    pub static ref COMPACTION_FAILURE_COUNT: IntCounter =
148        register_int_counter!("greptime_mito_compaction_failure_total", "mito compaction failure total").unwrap();
149
150    /// Gauge for inflight compaction tasks.
151    pub static ref INFLIGHT_COMPACTION_COUNT: IntGauge =
152        register_int_gauge!(
153            "greptime_mito_inflight_compaction_count",
154            "inflight compaction count",
155        ).unwrap();
156    // ------- End of compaction metrics.
157
158    // Query metrics.
159    /// Timer of different stages in query.
160    pub static ref READ_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
161        "greptime_mito_read_stage_elapsed",
162        "mito read stage elapsed",
163        &[STAGE_LABEL],
164        // 0.01 ~ 10000
165        exponential_buckets(0.01, 10.0, 7).unwrap(),
166    )
167    .unwrap();
168    pub static ref READ_STAGE_FETCH_PAGES: Histogram = READ_STAGE_ELAPSED.with_label_values(&["fetch_pages"]);
169    /// Number of in-progress scan per partition.
170    pub static ref IN_PROGRESS_SCAN: IntGaugeVec = register_int_gauge_vec!(
171        "greptime_mito_in_progress_scan",
172        "mito in progress scan per partition",
173        &[TYPE_LABEL, PARTITION_LABEL]
174    )
175    .unwrap();
176    /// Counter of rows read from different source.
177    pub static ref READ_ROWS_TOTAL: IntCounterVec =
178        register_int_counter_vec!("greptime_mito_read_rows_total", "mito read rows total", &[TYPE_LABEL]).unwrap();
179    /// Counter of filtered rows during merge.
180    pub static ref MERGE_FILTER_ROWS_TOTAL: IntCounterVec =
181        register_int_counter_vec!("greptime_mito_merge_filter_rows_total", "mito merge filter rows total", &[TYPE_LABEL]).unwrap();
182    /// Counter of row groups read.
183    pub static ref READ_ROW_GROUPS_TOTAL: IntCounterVec =
184        register_int_counter_vec!("greptime_mito_read_row_groups_total", "mito read row groups total", &[TYPE_LABEL]).unwrap();
185    /// Counter of filtered rows by precise filter.
186    pub static ref PRECISE_FILTER_ROWS_TOTAL: IntCounterVec =
187        register_int_counter_vec!("greptime_mito_precise_filter_rows_total", "mito precise filter rows total", &[TYPE_LABEL]).unwrap();
188    pub static ref READ_ROWS_IN_ROW_GROUP_TOTAL: IntCounterVec =
189        register_int_counter_vec!("greptime_mito_read_rows_in_row_group_total", "mito read rows in row group total", &[TYPE_LABEL]).unwrap();
190    /// Histogram for the number of SSTs to scan per query.
191    pub static ref READ_SST_COUNT: Histogram = register_histogram!(
192        "greptime_mito_read_sst_count",
193        "Number of SSTs to scan in a scan task",
194        vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 256.0, 1024.0],
195    ).unwrap();
196    /// Histogram for the number of rows returned per query.
197    pub static ref READ_ROWS_RETURN: Histogram = register_histogram!(
198        "greptime_mito_read_rows_return",
199        "Number of rows returned in a scan task",
200        exponential_buckets(100.0, 10.0, 8).unwrap(),
201    ).unwrap();
202    /// Histogram for the number of batches returned per query.
203    pub static ref READ_BATCHES_RETURN: Histogram = register_histogram!(
204        "greptime_mito_read_batches_return",
205        "Number of rows returned in a scan task",
206        exponential_buckets(100.0, 10.0, 7).unwrap(),
207    ).unwrap();
208    // ------- End of query metrics.
209
210    // Cache related metrics.
211    /// Cache hit counter.
212    pub static ref CACHE_HIT: IntCounterVec = register_int_counter_vec!(
213        "greptime_mito_cache_hit",
214        "mito cache hit",
215        &[TYPE_LABEL]
216    )
217    .unwrap();
218    /// Cache miss counter.
219    pub static ref CACHE_MISS: IntCounterVec = register_int_counter_vec!(
220        "greptime_mito_cache_miss",
221        "mito cache miss",
222        &[TYPE_LABEL]
223    )
224    .unwrap();
225    /// Cache size in bytes.
226    pub static ref CACHE_BYTES: IntGaugeVec = register_int_gauge_vec!(
227        "greptime_mito_cache_bytes",
228        "mito cache bytes",
229        &[TYPE_LABEL]
230    )
231    .unwrap();
232    /// Download bytes counter in the write cache.
233    pub static ref WRITE_CACHE_DOWNLOAD_BYTES_TOTAL: IntCounter = register_int_counter!(
234        "mito_write_cache_download_bytes_total",
235        "mito write cache download bytes total",
236    ).unwrap();
237    /// Timer of the downloading task in the write cache.
238    pub static ref WRITE_CACHE_DOWNLOAD_ELAPSED: HistogramVec = register_histogram_vec!(
239        "mito_write_cache_download_elapsed",
240        "mito write cache download elapsed",
241        &[TYPE_LABEL],
242        // 0.1 ~ 10000
243        exponential_buckets(0.1, 10.0, 6).unwrap(),
244    ).unwrap();
245    /// Number of inflight download tasks.
246    pub static ref WRITE_CACHE_INFLIGHT_DOWNLOAD: IntGauge = register_int_gauge!(
247        "mito_write_cache_inflight_download_count",
248        "mito write cache inflight download tasks",
249    ).unwrap();
250    /// Upload bytes counter.
251    pub static ref UPLOAD_BYTES_TOTAL: IntCounter = register_int_counter!(
252        "mito_upload_bytes_total",
253        "mito upload bytes total",
254    )
255    .unwrap();
256    /// Cache eviction counter, labeled with cache type and eviction reason.
257    pub static ref CACHE_EVICTION: IntCounterVec = register_int_counter_vec!(
258        "greptime_mito_cache_eviction",
259        "mito cache eviction",
260        &[TYPE_LABEL, CACHE_EVICTION_CAUSE]
261    ).unwrap();
262    // ------- End of cache metrics.
263
264    // Index metrics.
265    /// Timer of index application.
266    pub static ref INDEX_APPLY_ELAPSED: HistogramVec = register_histogram_vec!(
267        "greptime_index_apply_elapsed",
268        "index apply elapsed",
269        &[TYPE_LABEL],
270        // 0.01 ~ 1000
271        exponential_buckets(0.01, 10.0, 6).unwrap(),
272    )
273    .unwrap();
274    /// Gauge of index apply memory usage.
275    pub static ref INDEX_APPLY_MEMORY_USAGE: IntGauge = register_int_gauge!(
276        "greptime_index_apply_memory_usage",
277        "index apply memory usage",
278    )
279    .unwrap();
280    /// Timer of index creation.
281    pub static ref INDEX_CREATE_ELAPSED: HistogramVec = register_histogram_vec!(
282        "greptime_index_create_elapsed",
283        "index create elapsed",
284        &[STAGE_LABEL, TYPE_LABEL],
285        // 0.1 ~ 10000
286        exponential_buckets(0.1, 10.0, 6).unwrap(),
287    )
288    .unwrap();
289    /// Counter of rows indexed.
290    pub static ref INDEX_CREATE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
291        "greptime_index_create_rows_total",
292        "index create rows total",
293        &[TYPE_LABEL],
294    )
295    .unwrap();
296    /// Counter of created index bytes.
297    pub static ref INDEX_CREATE_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
298        "greptime_index_create_bytes_total",
299        "index create bytes total",
300        &[TYPE_LABEL],
301    )
302    .unwrap();
303    /// Gauge of index create memory usage.
304    pub static ref INDEX_CREATE_MEMORY_USAGE: IntGaugeVec = register_int_gauge_vec!(
305        "greptime_index_create_memory_usage",
306        "index create memory usage",
307        &[TYPE_LABEL],
308    ).unwrap();
309    /// Counter of r/w bytes on index related IO operations.
310    pub static ref INDEX_IO_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
311        "greptime_index_io_bytes_total",
312        "index io bytes total",
313        &[TYPE_LABEL, FILE_TYPE_LABEL]
314    )
315    .unwrap();
316    /// Counter of read bytes on puffin files.
317    pub static ref INDEX_PUFFIN_READ_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
318        .with_label_values(&["read", "puffin"]);
319    /// Counter of write bytes on puffin files.
320    pub static ref INDEX_PUFFIN_WRITE_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
321        .with_label_values(&["write", "puffin"]);
322    /// Counter of read bytes on intermediate files.
323    pub static ref INDEX_INTERMEDIATE_READ_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
324        .with_label_values(&["read", "intermediate"]);
325    /// Counter of write bytes on intermediate files.
326    pub static ref INDEX_INTERMEDIATE_WRITE_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
327        .with_label_values(&["write", "intermediate"]);
328
329    /// Counter of r/w operations on index related IO operations, e.g. read, write, seek and flush.
330    pub static ref INDEX_IO_OP_TOTAL: IntCounterVec = register_int_counter_vec!(
331        "greptime_index_io_op_total",
332        "index io op total",
333        &[TYPE_LABEL, FILE_TYPE_LABEL]
334    )
335    .unwrap();
336    /// Counter of read operations on puffin files.
337    pub static ref INDEX_PUFFIN_READ_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
338        .with_label_values(&["read", "puffin"]);
339    /// Counter of seek operations on puffin files.
340    pub static ref INDEX_PUFFIN_SEEK_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
341        .with_label_values(&["seek", "puffin"]);
342    /// Counter of write operations on puffin files.
343    pub static ref INDEX_PUFFIN_WRITE_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
344        .with_label_values(&["write", "puffin"]);
345    /// Counter of flush operations on puffin files.
346    pub static ref INDEX_PUFFIN_FLUSH_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
347        .with_label_values(&["flush", "puffin"]);
348    /// Counter of read operations on intermediate files.
349    pub static ref INDEX_INTERMEDIATE_READ_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
350        .with_label_values(&["read", "intermediate"]);
351    /// Counter of seek operations on intermediate files.
352    pub static ref INDEX_INTERMEDIATE_SEEK_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
353        .with_label_values(&["seek", "intermediate"]);
354    /// Counter of write operations on intermediate files.
355    pub static ref INDEX_INTERMEDIATE_WRITE_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
356        .with_label_values(&["write", "intermediate"]);
357    /// Counter of flush operations on intermediate files.
358    pub static ref INDEX_INTERMEDIATE_FLUSH_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
359        .with_label_values(&["flush", "intermediate"]);
360    // ------- End of index metrics.
361
362    /// Partition tree memtable data buffer freeze metrics
363    pub static ref PARTITION_TREE_DATA_BUFFER_FREEZE_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
364        "greptime_partition_tree_buffer_freeze_stage_elapsed",
365        "mito partition tree data buffer freeze stage elapsed",
366        &[STAGE_LABEL],
367        // 0.01 ~ 1000
368        exponential_buckets(0.01, 10.0, 6).unwrap(),
369    )
370    .unwrap();
371
372    /// Partition tree memtable read path metrics
373    pub static ref PARTITION_TREE_READ_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
374        "greptime_partition_tree_read_stage_elapsed",
375        "mito partition tree read stage elapsed",
376        &[STAGE_LABEL],
377        // 0.01 ~ 1000
378        exponential_buckets(0.01, 10.0, 6).unwrap(),
379    )
380    .unwrap();
381
382    // ------- End of partition tree memtable metrics.
383
384
385    // Manifest related metrics:
386
387    /// Elapsed time of manifest operation. Labeled with "op".
388    pub static ref MANIFEST_OP_ELAPSED: HistogramVec = register_histogram_vec!(
389        "greptime_manifest_op_elapsed",
390        "mito manifest operation elapsed",
391        &["op"],
392        // 0.01 ~ 1000
393        exponential_buckets(0.01, 10.0, 6).unwrap(),
394    ).unwrap();
395}
396
397/// Stager notifier to collect metrics.
398pub struct StagerMetrics {
399    cache_hit: IntCounter,
400    cache_miss: IntCounter,
401    staging_cache_bytes: IntGauge,
402    recycle_cache_bytes: IntGauge,
403    cache_eviction: IntCounter,
404    staging_miss_read: Histogram,
405}
406
407impl StagerMetrics {
408    /// Creates a new stager notifier.
409    pub fn new() -> Self {
410        Self {
411            cache_hit: CACHE_HIT.with_label_values(&[STAGING_TYPE]),
412            cache_miss: CACHE_MISS.with_label_values(&[STAGING_TYPE]),
413            staging_cache_bytes: CACHE_BYTES.with_label_values(&[STAGING_TYPE]),
414            recycle_cache_bytes: CACHE_BYTES.with_label_values(&[RECYCLE_TYPE]),
415            cache_eviction: CACHE_EVICTION.with_label_values(&[STAGING_TYPE, "size"]),
416            staging_miss_read: READ_STAGE_ELAPSED.with_label_values(&["staging_miss_read"]),
417        }
418    }
419}
420
421impl Default for StagerMetrics {
422    fn default() -> Self {
423        Self::new()
424    }
425}
426
427impl StagerNotifier for StagerMetrics {
428    fn on_cache_hit(&self, _size: u64) {
429        self.cache_hit.inc();
430    }
431
432    fn on_cache_miss(&self, _size: u64) {
433        self.cache_miss.inc();
434    }
435
436    fn on_cache_insert(&self, size: u64) {
437        self.staging_cache_bytes.add(size as i64);
438    }
439
440    fn on_load_dir(&self, duration: Duration) {
441        self.staging_miss_read.observe(duration.as_secs_f64());
442    }
443
444    fn on_load_blob(&self, duration: Duration) {
445        self.staging_miss_read.observe(duration.as_secs_f64());
446    }
447
448    fn on_cache_evict(&self, size: u64) {
449        self.cache_eviction.inc();
450        self.staging_cache_bytes.sub(size as i64);
451    }
452
453    fn on_recycle_insert(&self, size: u64) {
454        self.recycle_cache_bytes.add(size as i64);
455    }
456
457    fn on_recycle_clear(&self, size: u64) {
458        self.recycle_cache_bytes.sub(size as i64);
459    }
460}
mito2/metrics.rs

mito2/
metrics.rs
