mito2/
metrics.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub(crate) mod manifest_size;
16
17use std::time::Duration;
18
19use lazy_static::lazy_static;
20use prometheus::*;
21use puffin::puffin_manager::stager::StagerNotifier;
22
23/// Stage label.
24pub const STAGE_LABEL: &str = "stage";
25/// Type label.
26pub const TYPE_LABEL: &str = "type";
27const CACHE_EVICTION_CAUSE: &str = "cause";
28/// Reason to flush.
29pub const FLUSH_REASON: &str = "reason";
30/// File type label.
31pub const FILE_TYPE_LABEL: &str = "file_type";
32/// Region worker id label.
33pub const WORKER_LABEL: &str = "worker";
34/// Partition label.
35pub const PARTITION_LABEL: &str = "partition";
36/// Staging dir type label.
37pub const STAGING_TYPE: &str = "index_staging";
38/// Recycle bin type label.
39pub const RECYCLE_TYPE: &str = "recycle_bin";
40
41// Write metrics.
42lazy_static! {
43    /// Global write buffer size in bytes.
44    pub static ref WRITE_BUFFER_BYTES: IntGauge =
45        register_int_gauge!("greptime_mito_write_buffer_bytes", "mito write buffer bytes").unwrap();
46    /// Global memtable dictionary size in bytes.
47    pub static ref MEMTABLE_DICT_BYTES: IntGauge =
48        register_int_gauge!("greptime_mito_memtable_dict_bytes", "mito memtable dictionary size in bytes").unwrap();
49    /// Gauge for open regions in each worker.
50    pub static ref REGION_COUNT: IntGaugeVec =
51        register_int_gauge_vec!(
52            "greptime_mito_region_count",
53            "mito region count in each worker",
54            &[WORKER_LABEL],
55        ).unwrap();
56    /// Elapsed time to handle requests.
57    pub static ref HANDLE_REQUEST_ELAPSED: HistogramVec = register_histogram_vec!(
58            "greptime_mito_handle_request_elapsed",
59            "mito handle request elapsed",
60            &[TYPE_LABEL],
61            // 0.01 ~ 10000
62            exponential_buckets(0.01, 10.0, 7).unwrap(),
63        )
64        .unwrap();
65
66    // ------ Flush related metrics
67    /// Counter of scheduled flush requests.
68    /// Note that the flush scheduler may merge some flush requests.
69    pub static ref FLUSH_REQUESTS_TOTAL: IntCounterVec = register_int_counter_vec!(
70            "greptime_mito_flush_requests_total",
71            "mito flush requests total",
72            &[FLUSH_REASON]
73        )
74        .unwrap();
75    /// Counter of scheduled failed flush jobs.
76    pub static ref FLUSH_FAILURE_TOTAL: IntCounter =
77        register_int_counter!("greptime_mito_flush_failure_total", "mito flush failure total").unwrap();
78    /// Elapsed time of a flush job.
79    pub static ref FLUSH_ELAPSED: HistogramVec = register_histogram_vec!(
80            "greptime_mito_flush_elapsed",
81            "mito flush elapsed",
82            &[TYPE_LABEL],
83            // 1 ~ 625
84            exponential_buckets(1.0, 5.0, 6).unwrap(),
85        )
86        .unwrap();
87    /// Histogram of flushed bytes.
88    pub static ref FLUSH_BYTES_TOTAL: IntCounter =
89        register_int_counter!("greptime_mito_flush_bytes_total", "mito flush bytes total").unwrap();
90    /// Gauge for inflight flush tasks.
91    pub static ref INFLIGHT_FLUSH_COUNT: IntGauge =
92        register_int_gauge!(
93            "greptime_mito_inflight_flush_count",
94            "inflight flush count",
95        ).unwrap();
96    // ------ End of flush related metrics
97
98
99    // ------ Write related metrics
100    //
101    /// Counter of rejected write requests.
102    pub static ref WRITE_REJECT_TOTAL: IntCounter =
103        register_int_counter!("greptime_mito_write_reject_total", "mito write reject total").unwrap();
104    /// Elapsed time of each write stage.
105    pub static ref WRITE_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
106            "greptime_mito_write_stage_elapsed",
107            "mito write stage elapsed",
108            &[STAGE_LABEL],
109            // 0.01 ~ 1000
110            exponential_buckets(0.01, 10.0, 6).unwrap(),
111        )
112        .unwrap();
113    /// Counter of rows to write.
114    pub static ref WRITE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
115        "greptime_mito_write_rows_total",
116        "mito write rows total",
117        &[TYPE_LABEL]
118    )
119    .unwrap();
120}
121
122// Compaction metrics.
123lazy_static! {
124    /// Timer of different stages in compaction.
125    /// - pick
126    /// - merge (in parallel)
127    ///   - iter_source
128    ///   - write_batch
129    ///   - update_index
130    ///   - upload_parquet
131    ///   - upload puffin
132    /// - write_manifest
133    pub static ref COMPACTION_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
134        "greptime_mito_compaction_stage_elapsed",
135        "mito compaction stage elapsed",
136        &[STAGE_LABEL],
137        // 1 ~ 100000
138        exponential_buckets(1.0, 10.0, 6).unwrap(),
139    )
140    .unwrap();
141    /// TWCS candidate planning wall time, including blocking-pool waits and
142    /// empty/error attempts. `mode` is fair or stateless, never a region ID.
143    pub static ref TWCS_PICK_ELAPSED: HistogramVec = register_histogram_vec!(
144        "greptime_mito_twcs_pick_elapsed_seconds",
145        "twcs candidate planning wall time including blocking-pool waits",
146        &["mode"],
147        // 100us ~ 105s, so sub-second planning regressions are visible too.
148        exponential_buckets(0.0001, 4.0, 11).unwrap(),
149    ).unwrap();
150    /// Windows whose candidate discovery actually started, including windows
151    /// with no eligible candidates; excludes windows skipped by early exit.
152    pub static ref TWCS_SCANNED_WINDOWS: IntCounterVec = register_int_counter_vec!(
153        "greptime_mito_twcs_scanned_windows_total",
154        "twcs windows evaluated during candidate discovery",
155        &["mode"],
156    ).unwrap();
157    /// Automatic seeds attempted within the output budget, including seeds
158    /// absorbed by or rejected with a LastNonNull closure. Excludes work left
159    /// behind by the budget; counts attempts, not admission or completion.
160    /// Promoted means the seed reached the waiting threshold at selection time.
161    pub static ref TWCS_SELECTED_CANDIDATES: IntCounterVec = register_int_counter_vec!(
162        "greptime_mito_twcs_selected_candidates_total",
163        "automatic twcs seeds selected by class and normal or promoted priority",
164        &["class", "priority"],
165    ).unwrap();
166    /// Elapsed time of each successfully published compaction unit, labeled by `type`.
167    /// - local: one dependency unit, from task start through worker apply acknowledgement
168    /// - remote: one remote job, from request start through publication
169    pub static ref COMPACTION_ELAPSED_TOTAL: HistogramVec =
170        register_histogram_vec!(
171        "greptime_mito_compaction_total_elapsed",
172        "mito compaction elapsed per published unit",
173        &[TYPE_LABEL],
174        // 1 ~ 100000
175        exponential_buckets(1.0, 10.0, 6).unwrap(),
176    ).unwrap();
177    /// Counter of all requested compaction task.
178    pub static ref COMPACTION_REQUEST_COUNT: IntCounter =
179        register_int_counter!("greptime_mito_compaction_requests_total", "mito compaction requests total").unwrap();
180    /// Counter of failed compaction task.
181    pub static ref COMPACTION_FAILURE_COUNT: IntCounter =
182        register_int_counter!("greptime_mito_compaction_failure_total", "mito compaction failure total").unwrap();
183
184    /// Prepared compaction units waiting for global admission.
185    pub static ref COMPACTION_READY_UNITS: IntGauge = register_int_gauge!(
186        "greptime_mito_compaction_ready_units", "prepared units waiting for global admission"
187    ).unwrap();
188    /// Includes authorized work not yet polled, through task exit and lease release.
189    pub static ref COMPACTION_ADMITTED_UNITS: IntGauge = register_int_gauge!(
190        "greptime_mito_compaction_admitted_units", "compaction units holding global execution capacity"
191    ).unwrap();
192    pub static ref COMPACTION_ADMISSION_WAIT: Histogram = register_histogram!(
193        "greptime_mito_compaction_admission_wait_seconds", "time prepared units wait for admission",
194        exponential_buckets(0.001, 4.0, 10).unwrap()
195    ).unwrap();
196    /// Gauge for inflight compaction tasks: local units and remote jobs.
197    pub static ref INFLIGHT_COMPACTION_COUNT: IntGauge =
198        register_int_gauge!(
199            "greptime_mito_inflight_compaction_count",
200            "inflight compaction count",
201        ).unwrap();
202
203    /// Bytes reserved by compaction memory manager.
204    pub static ref COMPACTION_MEMORY_IN_USE: IntGauge =
205        register_int_gauge!(
206            "greptime_mito_compaction_memory_in_use_bytes",
207            "bytes currently reserved for compaction tasks",
208        )
209        .unwrap();
210    /// Configured compaction memory limit.
211    pub static ref COMPACTION_MEMORY_LIMIT: IntGauge =
212        register_int_gauge!(
213            "greptime_mito_compaction_memory_limit_bytes",
214            "maximum bytes allowed for compaction tasks",
215        )
216        .unwrap();
217    /// Wait time to obtain compaction memory.
218    pub static ref COMPACTION_MEMORY_WAIT: Histogram = register_histogram!(
219        "greptime_mito_compaction_memory_wait_seconds",
220        "time waiting for compaction memory",
221        // 0.01s ~ ~10s
222        exponential_buckets(0.01, 2.0, 10).unwrap(),
223    ).unwrap();
224    /// Counter of rejected compaction memory allocations.
225    pub static ref COMPACTION_MEMORY_REJECTED: IntCounterVec =
226        register_int_counter_vec!(
227            "greptime_mito_compaction_memory_rejected_total",
228            "number of compaction tasks rejected due to memory limit",
229            &[TYPE_LABEL]
230        ).unwrap();
231}
232
233// Query metrics.
234lazy_static! {
235    /// Timer of different stages in query.
236    pub static ref READ_STAGE_ELAPSED: HistogramVec = register_histogram_vec!(
237        "greptime_mito_read_stage_elapsed",
238        "mito read stage elapsed",
239        &[STAGE_LABEL],
240        // 0.01 ~ 10000
241        exponential_buckets(0.01, 10.0, 7).unwrap(),
242    )
243    .unwrap();
244    pub static ref READ_STAGE_FETCH_PAGES: Histogram = READ_STAGE_ELAPSED.with_label_values(&["fetch_pages"]);
245    /// Number of in-progress scan per partition.
246    pub static ref IN_PROGRESS_SCAN: IntGaugeVec = register_int_gauge_vec!(
247        "greptime_mito_in_progress_scan",
248        "mito in progress scan per partition",
249        &[TYPE_LABEL, PARTITION_LABEL]
250    )
251    .unwrap();
252    /// Counter of rows read from different source.
253    pub static ref READ_ROWS_TOTAL: IntCounterVec =
254        register_int_counter_vec!("greptime_mito_read_rows_total", "mito read rows total", &[TYPE_LABEL]).unwrap();
255    /// Counter of filtered rows during merge.
256    pub static ref MERGE_FILTER_ROWS_TOTAL: IntCounterVec =
257        register_int_counter_vec!("greptime_mito_merge_filter_rows_total", "mito merge filter rows total", &[TYPE_LABEL]).unwrap();
258    /// Counter of row groups read.
259    pub static ref READ_ROW_GROUPS_TOTAL: IntCounterVec =
260        register_int_counter_vec!("greptime_mito_read_row_groups_total", "mito read row groups total", &[TYPE_LABEL]).unwrap();
261    /// Counter of filtered rows by precise filter.
262    pub static ref PRECISE_FILTER_ROWS_TOTAL: IntCounterVec =
263        register_int_counter_vec!("greptime_mito_precise_filter_rows_total", "mito precise filter rows total", &[TYPE_LABEL]).unwrap();
264    pub static ref READ_ROWS_IN_ROW_GROUP_TOTAL: IntCounterVec =
265        register_int_counter_vec!("greptime_mito_read_rows_in_row_group_total", "mito read rows in row group total", &[TYPE_LABEL]).unwrap();
266    /// Histogram for the number of SSTs to scan per query.
267    pub static ref READ_SST_COUNT: Histogram = register_histogram!(
268        "greptime_mito_read_sst_count",
269        "Number of SSTs to scan in a scan task",
270        vec![1.0, 4.0, 8.0, 16.0, 32.0, 64.0, 256.0, 1024.0],
271    ).unwrap();
272    /// Planned maximum concurrent sources per query scan limit evaluation or compaction reader.
273    /// Includes memtables counted by the file limit, rejected scans, and memory-budgeted scans.
274    pub static ref SCAN_MAX_CONCURRENT_FILES: HistogramVec = register_histogram_vec!(
275        "greptime_mito_scan_max_concurrent_files",
276        "Estimated maximum concurrent files (including memtable sources) per query scan limit evaluation or compaction reader",
277        &[TYPE_LABEL],
278        vec![0.0, 1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 384.0, 512.0, 1024.0, 2048.0, 4096.0, 8192.0],
279    ).unwrap();
280    /// Histogram for the number of rows returned per query.
281    pub static ref READ_ROWS_RETURN: Histogram = register_histogram!(
282        "greptime_mito_read_rows_return",
283        "Number of rows returned in a scan task",
284        exponential_buckets(100.0, 10.0, 8).unwrap(),
285    ).unwrap();
286    /// Histogram for the number of batches returned per query.
287    pub static ref READ_BATCHES_RETURN: Histogram = register_histogram!(
288        "greptime_mito_read_batches_return",
289        "Number of rows returned in a scan task",
290        exponential_buckets(100.0, 10.0, 7).unwrap(),
291    ).unwrap();
292    /// Gauge for scan memory usage in bytes.
293    pub static ref SCAN_MEMORY_USAGE_BYTES: IntGauge = register_int_gauge!(
294        "greptime_mito_scan_memory_usage_bytes",
295        "current scan memory usage in bytes"
296    ).unwrap();
297    /// Counter of scan allocation attempts that could not acquire memory immediately.
298    pub static ref SCAN_MEMORY_EXHAUSTED_TOTAL: IntCounter = register_int_counter!(
299        "greptime_mito_scan_memory_exhausted_total",
300        "total number of times scan memory was unavailable for immediate acquisition"
301    ).unwrap();
302    /// Counter of scan requests that ultimately failed due to memory pressure.
303    pub static ref SCAN_REQUESTS_REJECTED_TOTAL: IntCounter = register_int_counter!(
304        "greptime_mito_scan_requests_rejected_total",
305        "total number of scan requests that ultimately failed due to memory limit"
306    ).unwrap();
307    /// Gauge for active file range builders in the pruner.
308    pub static ref PRUNER_ACTIVE_BUILDERS: IntGauge = register_int_gauge!(
309        "greptime_mito_pruner_active_builders",
310        "number of active file range builders in the pruner"
311    ).unwrap();
312}
313
314pub(crate) fn cache_loading_metrics(
315    cache_type: &'static str,
316) -> std::sync::Arc<common_cache::metrics::CacheMetrics> {
317    use common_cache::metrics::{CacheMetrics, OperationMetrics};
318
319    let operation = |stage| OperationMetrics {
320        started: CACHE_NATIVE_LOAD_EVENTS.with_label_values(&[cache_type, stage, "started"]),
321        succeeded: CACHE_NATIVE_LOAD_EVENTS.with_label_values(&[cache_type, stage, "success"]),
322        failed: CACHE_NATIVE_LOAD_EVENTS.with_label_values(&[cache_type, stage, "error"]),
323        cancelled: CACHE_NATIVE_LOAD_EVENTS.with_label_values(&[cache_type, stage, "cancelled"]),
324        active: CACHE_NATIVE_LOAD_ACTIVE.with_label_values(&[cache_type, stage]),
325        elapsed: CACHE_NATIVE_LOAD_ELAPSED.with_label_values(&[cache_type, stage]),
326    };
327    std::sync::Arc::new(CacheMetrics {
328        requests: operation("request"),
329        loads: operation("loader"),
330    })
331}
332
333// Cache metrics.
334lazy_static! {
335    pub static ref RANGE_CACHE_STAGING_BYTES: IntGauge = register_int_gauge!(
336        "greptime_mito_range_cache_staging_bytes",
337        "Referenced batch bytes retained by active shared range buffers, including unread streaming batches"
338    )
339    .unwrap();
340    pub static ref RANGE_CACHE_STREAMING_BYTES: IntGauge = register_int_gauge!(
341        "greptime_mito_range_cache_streaming_bytes",
342        "Referenced bytes retained for active readers after full-result cache admission stops"
343    ).unwrap();
344}
345
346lazy_static! {
347    /// Native cache request and actual loader lifecycle; request is not a waiter count.
348    pub static ref CACHE_NATIVE_LOAD_EVENTS: IntCounterVec = register_int_counter_vec!(
349        "greptime_mito_cache_native_load_events_total",
350        "Native cache request and executed loader outcomes; requests include hits and shared loads",
351        &[TYPE_LABEL, STAGE_LABEL, "event"]
352    ).unwrap();
353    pub static ref CACHE_NATIVE_LOAD_ACTIVE: IntGaugeVec = register_int_gauge_vec!(
354        "greptime_mito_cache_native_load_active",
355        "Active native cache requests or executed loaders, not waiting requests",
356        &[TYPE_LABEL, STAGE_LABEL]
357    ).unwrap();
358    pub static ref CACHE_NATIVE_LOAD_ELAPSED: HistogramVec = register_histogram_vec!(
359        "greptime_mito_cache_native_load_seconds",
360        "Native cache request or executed loader wall time including suspension and cancellation, not CPU time",
361        &[TYPE_LABEL, STAGE_LABEL], exponential_buckets(0.0001, 10.0, 6).unwrap()
362    ).unwrap();
363
364    /// Cache hit counter.
365    pub static ref CACHE_HIT: IntCounterVec = register_int_counter_vec!(
366        "greptime_mito_cache_hit",
367        "mito cache hit",
368        &[TYPE_LABEL]
369    )
370    .unwrap();
371    /// Cache miss counter.
372    pub static ref CACHE_MISS: IntCounterVec = register_int_counter_vec!(
373        "greptime_mito_cache_miss",
374        "mito cache miss",
375        &[TYPE_LABEL]
376    )
377    .unwrap();
378    /// Cache size in bytes.
379    pub static ref CACHE_BYTES: common_cache::metrics::CacheSizeMetrics = {
380        let metrics = common_cache::metrics::CacheSizeMetrics::new(
381            prometheus::Opts::new("greptime_mito_cache_bytes", "mito cache bytes; backend cache weights are approximate"),
382            &[TYPE_LABEL],
383        ).unwrap();
384        prometheus::register(Box::new(metrics.clone())).unwrap();
385        metrics
386    };
387    /// Download bytes counter in the write cache.
388    pub static ref WRITE_CACHE_DOWNLOAD_BYTES_TOTAL: IntCounter = register_int_counter!(
389        "mito_write_cache_download_bytes_total",
390        "mito write cache download bytes total",
391    ).unwrap();
392    /// Timer of the downloading task in the write cache.
393    pub static ref WRITE_CACHE_DOWNLOAD_ELAPSED: HistogramVec = register_histogram_vec!(
394        "mito_write_cache_download_elapsed",
395        "mito write cache download elapsed",
396        &[TYPE_LABEL],
397        // 0.1 ~ 10000
398        exponential_buckets(0.1, 10.0, 6).unwrap(),
399    ).unwrap();
400    /// Number of inflight download tasks.
401    pub static ref WRITE_CACHE_INFLIGHT_DOWNLOAD: IntGauge = register_int_gauge!(
402        "mito_write_cache_inflight_download_count",
403        "mito write cache inflight download tasks",
404    ).unwrap();
405    /// Upload bytes counter.
406    pub static ref UPLOAD_BYTES_TOTAL: IntCounter = register_int_counter!(
407        "mito_upload_bytes_total",
408        "mito upload bytes total",
409    )
410    .unwrap();
411    /// Bounded event labels distinguish lookup misses from actual shared loads.
412    pub static ref RANGE_CACHE_WORKER_EVENTS: IntCounterVec = register_int_counter_vec!(
413        "greptime_mito_range_cache_worker_events_total",
414        "Range cache load coordination and SST retirement events",
415        &["event"]
416    )
417    .unwrap();
418
419    /// Cache eviction counter, labeled with cache type and eviction reason.
420    pub static ref CACHE_EVICTION: IntCounterVec = register_int_counter_vec!(
421        "greptime_mito_cache_eviction",
422        "mito cache eviction",
423        &[TYPE_LABEL, CACHE_EVICTION_CAUSE]
424    ).unwrap();
425}
426
427// Index metrics.
428lazy_static! {
429    /// Approximate published index bytes in open regions, refreshed by maintenance.
430    pub static ref SERIES_INDEX_DISK_BYTES: IntGauge = register_int_gauge!(
431        "greptime_mito_series_index_disk_bytes", "estimated series and range index bytes in open regions"
432    ).unwrap();
433    /// Maintenance passes deferred by the estimated disk usage.
434    pub static ref SERIES_INDEX_CAPACITY_DEFERRED: IntCounter = register_int_counter!(
435        "greptime_mito_series_index_capacity_deferred_total", "series-index capacity deferrals"
436    ).unwrap();
437    // Index metrics.
438    /// Outcomes of series-index reconciliation passes.
439    pub static ref SERIES_INDEX_RECONCILE_TOTAL: IntCounterVec = register_int_counter_vec!(
440        "greptime_mito_series_index_reconcile_total",
441        "series-index reconciliation passes",
442        &["result"],
443    ).unwrap();
444    /// Elapsed time of series-index reconciliation phases.
445    pub static ref SERIES_INDEX_RECONCILE_ELAPSED: HistogramVec = register_histogram_vec!(
446        "greptime_mito_series_index_reconcile_elapsed",
447        "series-index reconciliation elapsed time",
448        &["phase"],
449        exponential_buckets(0.01, 10.0, 7).unwrap(),
450    ).unwrap();
451    /// Series and range index file operations.
452    pub static ref SERIES_INDEX_FILE_OPERATION_TOTAL: IntCounterVec = register_int_counter_vec!(
453        "greptime_mito_series_index_file_operation_total",
454        "series-index file operations",
455        &["index_type", "operation", "result"],
456    ).unwrap();
457    /// Number of stale index publications rejected at each publication stage.
458    pub static ref INDEX_PUBLICATION_STALE_TOTAL: IntCounterVec =
459        register_int_counter_vec!(
460            "greptime_mito_index_publication_stale_total",
461            "stale index publications rejected",
462            &[STAGE_LABEL],
463        ).unwrap();
464    /// Number of failures while cleaning local state for stale index artifacts.
465    pub static ref INDEX_ARTIFACT_CLEANUP_FAILURE_TOTAL: IntCounter =
466        register_int_counter!(
467            "greptime_mito_index_artifact_cleanup_failure_total",
468            "failures while cleaning local state for stale index artifacts",
469        ).unwrap();
470    /// Timer of index application.
471    pub static ref INDEX_APPLY_ELAPSED: HistogramVec = register_histogram_vec!(
472        "greptime_index_apply_elapsed",
473        "index apply elapsed",
474        &[TYPE_LABEL],
475        // 0.01 ~ 1000
476        exponential_buckets(0.01, 10.0, 6).unwrap(),
477    )
478    .unwrap();
479    /// Gauge of index apply memory usage.
480    pub static ref INDEX_APPLY_MEMORY_USAGE: IntGauge = register_int_gauge!(
481        "greptime_index_apply_memory_usage",
482        "index apply memory usage",
483    )
484    .unwrap();
485    /// Timer of index creation.
486    pub static ref INDEX_CREATE_ELAPSED: HistogramVec = register_histogram_vec!(
487        "greptime_index_create_elapsed",
488        "index create elapsed",
489        &[STAGE_LABEL, TYPE_LABEL],
490        // 0.1 ~ 10000
491        exponential_buckets(0.1, 10.0, 6).unwrap(),
492    )
493    .unwrap();
494    /// Counter of rows indexed.
495    pub static ref INDEX_CREATE_ROWS_TOTAL: IntCounterVec = register_int_counter_vec!(
496        "greptime_index_create_rows_total",
497        "index create rows total",
498        &[TYPE_LABEL],
499    )
500    .unwrap();
501    /// Counter of created index bytes.
502    pub static ref INDEX_CREATE_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
503        "greptime_index_create_bytes_total",
504        "index create bytes total",
505        &[TYPE_LABEL],
506    )
507    .unwrap();
508    /// Gauge of index create memory usage.
509    pub static ref INDEX_CREATE_MEMORY_USAGE: IntGaugeVec = register_int_gauge_vec!(
510        "greptime_index_create_memory_usage",
511        "index create memory usage",
512        &[TYPE_LABEL],
513    ).unwrap();
514    /// Counter of r/w bytes on index related IO operations.
515    pub static ref INDEX_IO_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
516        "greptime_index_io_bytes_total",
517        "index io bytes total",
518        &[TYPE_LABEL, FILE_TYPE_LABEL]
519    )
520    .unwrap();
521    /// Counter of read bytes on puffin files.
522    pub static ref INDEX_PUFFIN_READ_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
523        .with_label_values(&["read", "puffin"]);
524    /// Counter of write bytes on puffin files.
525    pub static ref INDEX_PUFFIN_WRITE_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
526        .with_label_values(&["write", "puffin"]);
527    /// Counter of read bytes on intermediate files.
528    pub static ref INDEX_INTERMEDIATE_READ_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
529        .with_label_values(&["read", "intermediate"]);
530    /// Counter of write bytes on intermediate files.
531    pub static ref INDEX_INTERMEDIATE_WRITE_BYTES_TOTAL: IntCounter = INDEX_IO_BYTES_TOTAL
532        .with_label_values(&["write", "intermediate"]);
533
534    /// Counter of r/w operations on index related IO operations, e.g. read, write, seek and flush.
535    pub static ref INDEX_IO_OP_TOTAL: IntCounterVec = register_int_counter_vec!(
536        "greptime_index_io_op_total",
537        "index io op total",
538        &[TYPE_LABEL, FILE_TYPE_LABEL]
539    )
540    .unwrap();
541    /// Counter of read operations on puffin files.
542    pub static ref INDEX_PUFFIN_READ_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
543        .with_label_values(&["read", "puffin"]);
544    /// Counter of seek operations on puffin files.
545    pub static ref INDEX_PUFFIN_SEEK_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
546        .with_label_values(&["seek", "puffin"]);
547    /// Counter of write operations on puffin files.
548    pub static ref INDEX_PUFFIN_WRITE_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
549        .with_label_values(&["write", "puffin"]);
550    /// Counter of flush operations on puffin files.
551    pub static ref INDEX_PUFFIN_FLUSH_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
552        .with_label_values(&["flush", "puffin"]);
553    /// Counter of read operations on intermediate files.
554    pub static ref INDEX_INTERMEDIATE_READ_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
555        .with_label_values(&["read", "intermediate"]);
556    /// Counter of seek operations on intermediate files.
557    pub static ref INDEX_INTERMEDIATE_SEEK_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
558        .with_label_values(&["seek", "intermediate"]);
559    /// Counter of write operations on intermediate files.
560    pub static ref INDEX_INTERMEDIATE_WRITE_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
561        .with_label_values(&["write", "intermediate"]);
562    /// Counter of flush operations on intermediate files.
563    pub static ref INDEX_INTERMEDIATE_FLUSH_OP_TOTAL: IntCounter = INDEX_IO_OP_TOTAL
564        .with_label_values(&["flush", "intermediate"]);
565}
566
567lazy_static! {
568    // Manifest related metrics:
569
570    /// Elapsed time of manifest operation. Labeled with "op".
571    pub static ref MANIFEST_OP_ELAPSED: HistogramVec = register_histogram_vec!(
572        "greptime_manifest_op_elapsed",
573        "mito manifest operation elapsed",
574        &["op"],
575        // 0.01 ~ 1000
576        exponential_buckets(0.01, 10.0, 6).unwrap(),
577    ).unwrap();
578
579
580    pub static ref REGION_WORKER_HANDLE_WRITE_ELAPSED: HistogramVec = register_histogram_vec!(
581        "greptime_region_worker_handle_write",
582        "elapsed time for handling writes in region worker loop",
583        &["stage"],
584        exponential_buckets(0.001, 10.0, 5).unwrap()
585    ).unwrap();
586
587}
588
589lazy_static! {
590    /// Counter for compaction input file size.
591    pub static ref COMPACTION_INPUT_BYTES: Counter = register_counter!(
592        "greptime_mito_compaction_input_bytes",
593        "mito compaction input file size",
594        ).unwrap();
595
596    /// Counter for compaction output file size.
597    pub static ref COMPACTION_OUTPUT_BYTES: Counter = register_counter!(
598        "greptime_mito_compaction_output_bytes",
599        "mito compaction output file size",
600        ).unwrap();
601
602    /// Active series count in TimeSeriesMemtable
603    pub static ref MEMTABLE_ACTIVE_SERIES_COUNT: IntGauge = register_int_gauge!(
604        "greptime_mito_memtable_active_series_count",
605        "active time series count in TimeSeriesMemtable",
606        ).unwrap();
607
608    /// Active field builder count in TimeSeriesMemtable
609    pub static ref MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT: IntGauge = register_int_gauge!(
610        "greptime_mito_memtable_field_builder_count",
611        "active field builder count in TimeSeriesMemtable",
612        ).unwrap();
613
614    /// Number of stalling write requests in each worker.
615    pub static ref WRITE_STALLING: IntGaugeVec = register_int_gauge_vec!(
616            "greptime_mito_write_stalling_count",
617            "mito stalled write request in each worker",
618            &[WORKER_LABEL]
619        ).unwrap();
620    /// Number of ref files
621    pub static ref GC_REF_FILE_CNT: IntGauge = register_int_gauge!(
622            "greptime_gc_ref_file_count",
623            "gc ref file count",
624        ).unwrap();
625    /// Total number of stalled write requests.
626    pub static ref WRITE_STALL_TOTAL: IntCounter = register_int_counter!(
627        "greptime_mito_write_stall_total",
628        "Total number of stalled write requests"
629    ).unwrap();
630    /// Time waiting for requests to be handled by the region worker.
631    pub static ref REQUEST_WAIT_TIME: HistogramVec = register_histogram_vec!(
632            "greptime_mito_request_wait_time",
633            "mito request wait time before being handled by region worker",
634            &[WORKER_LABEL],
635            // 0.001 ~ 10000
636            exponential_buckets(0.001, 10.0, 8).unwrap(),
637        )
638        .unwrap();
639
640    /// Counter for the number of files deleted by the GC worker.
641    pub static ref GC_DELETE_FILE_CNT: IntCounter =
642        register_int_counter!(
643            "greptime_mito_gc_delete_file_count",
644            "mito gc deleted file count",
645        ).unwrap();
646
647    /// Counter for the number of unparsable files skipped by GC.
648    pub static ref GC_SKIPPED_UNPARSABLE_FILES: IntCounter =
649        register_int_counter!(
650            "greptime_mito_gc_skipped_unparsable_files",
651            "mito gc skipped unparsable files count",
652        ).unwrap();
653
654    /// Counter for the number of orphaned index files found by GC.
655    pub static ref GC_ORPHANED_INDEX_FILES: IntCounter =
656        register_int_counter!(
657            "greptime_mito_gc_orphaned_index_files",
658            "mito gc orphaned index files count",
659        ).unwrap();
660
661    /// Histogram for GC operation duration by stage.
662    pub static ref GC_DURATION_SECONDS: HistogramVec = register_histogram_vec!(
663        "greptime_mito_gc_duration_seconds",
664        "GC operation duration by stage",
665        &[STAGE_LABEL],
666        exponential_buckets(0.01, 10.0, 6).unwrap(),
667    ).unwrap();
668
669    /// Counter for GC runs by mode.
670    pub static ref GC_RUNS_TOTAL: IntCounterVec = register_int_counter_vec!(
671        "greptime_mito_gc_runs_total",
672        "Total GC runs by mode",
673        &["mode"],
674    ).unwrap();
675
676    /// Counter for GC errors by type.
677    pub static ref GC_ERRORS_TOTAL: IntCounterVec = register_int_counter_vec!(
678        "greptime_mito_gc_errors_total",
679        "Total GC errors by type",
680        &["error_type"],
681    ).unwrap();
682
683    /// Counter for total files deleted by GC, labeled by file type.
684    pub static ref GC_FILES_DELETED_TOTAL: IntCounterVec = register_int_counter_vec!(
685        "greptime_mito_gc_files_deleted_total",
686        "Total files deleted by GC",
687        &[FILE_TYPE_LABEL],
688    ).unwrap();
689
690    /// Total number of files downloaded during cache fill on region open.
691    pub static ref CACHE_FILL_DOWNLOADED_FILES: IntCounter = register_int_counter!(
692        "mito_cache_fill_downloaded_files",
693        "mito cache fill downloaded files count",
694    ).unwrap();
695
696    /// Number of files pending download during cache fill on region open.
697    pub static ref CACHE_FILL_PENDING_FILES: IntGauge = register_int_gauge!(
698        "mito_cache_fill_pending_files",
699        "mito cache fill pending files count",
700    ).unwrap();
701
702    /// Counter of flush files.
703    pub static ref FLUSH_FILE_TOTAL: IntCounter =
704        register_int_counter!("greptime_mito_flush_file_total", "mito flushed file count").unwrap();
705}
706
707#[cfg(feature = "hdfs-object-store")]
708lazy_static! {
709    /// Counter for the number of temporary files deleted by the sweeper.
710    pub static ref SWEEP_FILES_DELETED_TOTAL: IntCounter = register_int_counter!(
711        "greptime_mito_sweep_files_deleted_total",
712        "Total temporary files deleted by the sweeper",
713    ).unwrap();
714
715    /// Counter for the number of failures while sweeping temporary files.
716    pub static ref SWEEP_ERRORS_TOTAL: IntCounter = register_int_counter!(
717        "greptime_mito_sweep_errors_total",
718        "Total failures while sweeping temporary files",
719    ).unwrap();
720
721    /// Counter for the number of temporary file sweeps run.
722    pub static ref SWEEP_RUNS_TOTAL: IntCounter = register_int_counter!(
723        "greptime_mito_sweep_runs_total",
724        "Total temporary file sweeps run",
725    ).unwrap();
726}
727
728/// Stager notifier to collect metrics.
729pub struct StagerMetrics {
730    cache_hit: IntCounter,
731    cache_miss: IntCounter,
732    staging_cache_bytes: IntGauge,
733    recycle_cache_bytes: IntGauge,
734    cache_eviction: IntCounter,
735    staging_miss_read: Histogram,
736}
737
738impl StagerMetrics {
739    /// Creates a new stager notifier.
740    pub fn new() -> Self {
741        Self {
742            cache_hit: CACHE_HIT.with_label_values(&[STAGING_TYPE]),
743            cache_miss: CACHE_MISS.with_label_values(&[STAGING_TYPE]),
744            staging_cache_bytes: CACHE_BYTES.with_label_values(&[STAGING_TYPE]),
745            recycle_cache_bytes: CACHE_BYTES.with_label_values(&[RECYCLE_TYPE]),
746            cache_eviction: CACHE_EVICTION.with_label_values(&[STAGING_TYPE, "size"]),
747            staging_miss_read: READ_STAGE_ELAPSED.with_label_values(&["staging_miss_read"]),
748        }
749    }
750}
751
752impl Default for StagerMetrics {
753    fn default() -> Self {
754        Self::new()
755    }
756}
757
758impl StagerNotifier for StagerMetrics {
759    fn on_cache_hit(&self, _size: u64) {
760        self.cache_hit.inc();
761    }
762
763    fn on_cache_miss(&self, _size: u64) {
764        self.cache_miss.inc();
765    }
766
767    fn on_cache_insert(&self, size: u64) {
768        self.staging_cache_bytes.add(size as i64);
769    }
770
771    fn on_load_dir(&self, duration: Duration) {
772        self.staging_miss_read.observe(duration.as_secs_f64());
773    }
774
775    fn on_load_blob(&self, duration: Duration) {
776        self.staging_miss_read.observe(duration.as_secs_f64());
777    }
778
779    fn on_cache_evict(&self, size: u64) {
780        self.cache_eviction.inc();
781        self.staging_cache_bytes.sub(size as i64);
782    }
783
784    fn on_recycle_insert(&self, size: u64) {
785        self.recycle_cache_bytes.add(size as i64);
786    }
787
788    fn on_recycle_clear(&self, size: u64) {
789        self.recycle_cache_bytes.sub(size as i64);
790    }
791}
792
793lazy_static! {
794    pub static ref CACHE_PAGE_LOADED_BYTES: IntCounterVec = register_int_counter_vec!(
795        "greptime_mito_cache_page_loaded_bytes_total",
796        "Bytes produced by successful page loaders, excluding waiters",
797        &[TYPE_LABEL]
798    )
799    .unwrap();
800    /// Configured capacity summed across live page caches in each category.
801    pub static ref CACHE_CAPACITY_BYTES: common_cache::metrics::CacheSizeMetrics = {
802        let metrics = common_cache::metrics::CacheSizeMetrics::new(
803            prometheus::Opts::new("greptime_mito_cache_capacity_bytes", "Configured capacity of live page caches in bytes"),
804            &[TYPE_LABEL],
805        ).unwrap();
806        prometheus::register(Box::new(metrics.clone())).unwrap();
807        metrics
808    };
809}
810
811/// Binds shared page/cache I/O metrics while retaining the business hit/miss labels.
812pub(crate) fn cache_page_observer(
813    cache_name_label: &'static str,
814    cache_category_label: &'static str,
815    lookup_counts: fn(usize, usize) -> (usize, usize),
816) -> common_cache::paged::PageCacheObserver {
817    use common_cache::paged::PageEvent;
818    let hits = CACHE_HIT.with_label_values(&[cache_category_label]);
819    let misses = CACHE_MISS.with_label_values(&[cache_category_label]);
820    let loaded = CACHE_PAGE_LOADED_BYTES.with_label_values(&[cache_name_label]);
821    common_cache::paged::PageCacheObserver {
822        bytes: CACHE_BYTES.bind(&[cache_category_label]),
823        capacity: CACHE_CAPACITY_BYTES.bind(&[cache_category_label]),
824        loads: Some(cache_loading_metrics(cache_name_label)),
825        events: std::sync::Arc::new(move |event| match event {
826            PageEvent::Lookup {
827                hits: count,
828                misses: missing,
829            } => {
830                let (count, missing) = lookup_counts(count, missing);
831                hits.inc_by(count as u64);
832                misses.inc_by(missing as u64);
833            }
834            PageEvent::Loaded { bytes } => loaded.inc_by(bytes),
835            PageEvent::Evicted { cause } => CACHE_EVICTION
836                .with_label_values(&[cache_category_label, cause.as_str()])
837                .inc(),
838        }),
839    }
840}
841
842lazy_static! {
843    pub static ref CACHE_PAGE_BYTES: IntCounterVec = register_int_counter_vec!(
844        "greptime_mito_cache_page_bytes_total",
845        "Logical page fetch bytes by kind; not physical backend bytes",
846        &[TYPE_LABEL, "kind"]
847    )
848    .unwrap();
849}
850
851/// Records logical byte amplification independently of physical batch reads.
852pub(crate) fn cache_page_fetch_metrics(
853    cache_type: &'static str,
854    stats: &common_cache::paged::PageFetchStats,
855) {
856    for (kind, value) in [
857        ("requested", stats.requested_bytes),
858        ("aligned", stats.aligned_bytes),
859        ("unique_requested", stats.unique_requested_bytes),
860        ("cached", stats.cached_bytes),
861    ] {
862        CACHE_PAGE_BYTES
863            .with_label_values(&[cache_type, kind])
864            .inc_by(value);
865    }
866}
867
868#[cfg(test)]
869mod tests {
870
871    use super::*;
872    use common_cache::paged::{PageEvent, PageFetchStats};
873
874    #[test]
875    fn page_cache_capacity_sums_live_instances_by_category() {
876        use bytes::Bytes;
877        use common_cache::paged::PagedCache;
878        use prometheus::core::Collector;
879
880        let category = "test_shared_page_category";
881        let sample = || {
882            CACHE_CAPACITY_BYTES
883                .collect()
884                .into_iter()
885                .flat_map(|family| family.get_metric().to_vec())
886                .find(|metric| {
887                    metric
888                        .get_label()
889                        .iter()
890                        .any(|label| label.value() == category)
891                })
892                .unwrap()
893                .get_gauge()
894                .as_ref()
895                .unwrap()
896                .value()
897        };
898        let make_cache = |name, capacity| {
899            PagedCache::<u64>::new(
900                name,
901                capacity,
902                64,
903                |_, bytes: &Bytes| bytes.len() as u32,
904                cache_page_observer(name, category, |hits, misses| (hits, misses)),
905            )
906        };
907        let first = make_cache("test_first_pages", 128);
908        let second = make_cache("test_second_pages", 256);
909        assert_eq!(sample(), 384.0);
910        drop(first);
911        assert_eq!(sample(), 256.0);
912        drop(second);
913        assert_eq!(sample(), 0.0);
914    }
915
916    #[test]
917    fn test_page_observer_retains_business_lookup_units() {
918        let page = cache_page_observer("test_page_unit", "test_page_unit", |hits, misses| {
919            (hits, misses)
920        });
921        let request =
922            cache_page_observer("test_request_unit", "test_request_unit", |hits, misses| {
923                (usize::from(hits > 0), usize::from(misses > 0))
924            });
925        for observer in [&page, &request] {
926            (observer.events)(PageEvent::Lookup { hits: 3, misses: 2 });
927            (observer.events)(PageEvent::Loaded { bytes: 4096 });
928        }
929        assert_eq!(CACHE_HIT.with_label_values(&["test_page_unit"]).get(), 3);
930        assert_eq!(CACHE_MISS.with_label_values(&["test_page_unit"]).get(), 2);
931        assert_eq!(CACHE_HIT.with_label_values(&["test_request_unit"]).get(), 1);
932        assert_eq!(
933            CACHE_MISS.with_label_values(&["test_request_unit"]).get(),
934            1
935        );
936        cache_page_fetch_metrics(
937            "test_page_unit",
938            &PageFetchStats {
939                requested_bytes: 7,
940                aligned_bytes: 12,
941                unique_requested_bytes: 6,
942                cached_bytes: 3,
943                ..Default::default()
944            },
945        );
946        for (kind, expected) in [
947            ("requested", 7),
948            ("aligned", 12),
949            ("unique_requested", 6),
950            ("cached", 3),
951        ] {
952            assert_eq!(
953                CACHE_PAGE_BYTES
954                    .with_label_values(&["test_page_unit", kind])
955                    .get(),
956                expected
957            );
958        }
959    }
960}
mito2/metrics.rs

mito2/
metrics.rs
