log_store/
metrics.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use lazy_static::lazy_static;
16use prometheus::*;
17
18/// Logstore label.
19pub const LOGSTORE_LABEL: &str = "logstore";
20/// Operation type label.
21pub const OPTYPE_LABEL: &str = "optype";
22/// Kafka topic label.
23pub const TOPIC_LABEL: &str = "topic";
24/// Kafka partition label.
25pub const PARTITION_LABEL: &str = "partition";
26
27lazy_static! {
28    /// Counters of bytes of each operation on a logstore.
29    pub static ref METRIC_LOGSTORE_OP_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
30        "greptime_logstore_op_bytes_total",
31        "logstore operation bytes total",
32        &[LOGSTORE_LABEL, OPTYPE_LABEL],
33    )
34    .unwrap();
35    /// Counter of bytes of the append_batch operation on the kafka logstore.
36    pub static ref METRIC_KAFKA_APPEND_BATCH_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
37        &["kafka", "append_batch"],
38    );
39    /// Counter of bytes of the read operation on the kafka logstore.
40    pub static ref METRIC_KAFKA_READ_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
41        &["kafka", "read"],
42    );
43    /// Counter of bytes of the append_batch operation on the raft-engine logstore.
44    pub static ref METRIC_RAFT_ENGINE_APPEND_BATCH_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
45        &["raft-engine", "append_batch"],
46    );
47    /// Counter of bytes of the read operation on the raft-engine logstore.
48    pub static ref METRIC_RAFT_ENGINE_READ_BYTES_TOTAL: IntCounter = METRIC_LOGSTORE_OP_BYTES_TOTAL.with_label_values(
49        &["raft-engine", "read"],
50    );
51
52    /// Timer of operations on a logstore.
53    pub static ref METRIC_LOGSTORE_OP_ELAPSED: HistogramVec = register_histogram_vec!(
54        "greptime_logstore_op_elapsed",
55        "logstore operation elapsed",
56        &[LOGSTORE_LABEL, OPTYPE_LABEL],
57        vec![
58            0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
59            45.0, 60.0,
60        ],
61    )
62    .unwrap();
63    /// Timer of the append_batch operation on the kafka logstore.
64    pub static ref METRIC_KAFKA_APPEND_BATCH_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["kafka", "append_batch"]);
65    /// Timer of the append_batch operation on the kafka logstore.
66    /// This timer only measures the duration of the read operation, not measures the total duration of replay.
67    pub static ref METRIC_KAFKA_READ_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["kafka", "read"]);
68    /// Timer of the append_batch operation on the raft-engine logstore.
69    pub static ref METRIC_RAFT_ENGINE_APPEND_BATCH_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["raft-engine", "append_batch"]);
70    /// Timer of the append_batch operation on the raft-engine logstore.
71    /// This timer only measures the duration of the read operation, not measures the total duration of replay.
72    pub static ref METRIC_RAFT_ENGINE_READ_ELAPSED: Histogram = METRIC_LOGSTORE_OP_ELAPSED.with_label_values(&["raft-engine", "read"]);
73
74    pub static ref METRIC_KAFKA_CLIENT_BYTES_TOTAL: IntCounterVec = register_int_counter_vec!(
75        "greptime_logstore_kafka_client_bytes_total",
76        "kafka logstore's bytes traffic total",
77        &[LOGSTORE_LABEL, PARTITION_LABEL],
78    )
79    .unwrap();
80    pub static ref METRIC_KAFKA_CLIENT_TRAFFIC_TOTAL: IntCounterVec = register_int_counter_vec!(
81        "greptime_logstore_kafka_client_traffic_total",
82        "kafka logstore's request count traffic total",
83        &[LOGSTORE_LABEL, PARTITION_LABEL],
84    )
85    .unwrap();
86    pub static ref METRIC_KAFKA_CLIENT_PRODUCE_ELAPSED: HistogramVec = register_histogram_vec!(
87        "greptime_logstore_kafka_client_produce_elapsed",
88        "kafka logstore produce operation elapsed",
89        &[LOGSTORE_LABEL, PARTITION_LABEL],
90    )
91    .unwrap();
92
93    /// Counter of segments a read of the object store logstore skipped because
94    /// they did not decode.
95    pub static ref METRIC_OBJECT_STORE_WAL_SKIPPED_SEGMENTS_TOTAL: IntCounter = register_int_counter!(
96        "greptime_logstore_object_store_wal_skipped_segments_total",
97        "object store logstore skipped corrupted segments total",
98    )
99    .unwrap();
100
101    /// Counter of regions the object store logstore let open although its
102    /// prefix does not hold their WAL history, because
103    /// `accept_missing_history` is set; their writes that were not flushed
104    /// are lost.
105    pub static ref METRIC_OBJECT_STORE_WAL_MISSING_HISTORY_ACCEPTED_TOTAL: IntCounter = register_int_counter!(
106        "greptime_logstore_object_store_wal_missing_history_accepted_total",
107        "object store logstore regions opened without their WAL history total",
108    )
109    .unwrap();
110
111    /// Counter of objects the object store logstore deleted because no
112    /// region needs them any more.
113    pub static ref METRIC_OBJECT_STORE_WAL_DELETED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
114        "greptime_logstore_object_store_wal_deleted_objects_total",
115        "object store logstore deleted objects total",
116    )
117    .unwrap();
118    /// Counter of object deletes of the object store logstore that failed and
119    /// are attempted again at the next collection.
120    pub static ref METRIC_OBJECT_STORE_WAL_FAILED_DELETES_TOTAL: IntCounter = register_int_counter!(
121        "greptime_logstore_object_store_wal_failed_deletes_total",
122        "object store logstore failed object deletes total",
123    )
124    .unwrap();
125    /// Counter of release records in the objects the object store logstore
126    /// indexed, including records carried over from batches that failed.
127    pub static ref METRIC_OBJECT_STORE_WAL_RELEASE_RECORDS_TOTAL: IntCounter = register_int_counter!(
128        "greptime_logstore_object_store_wal_release_records_total",
129        "object store logstore release records made durable total",
130    )
131    .unwrap();
132
133    /// How long an object of the object store logstore takes to become durable
134    /// once its batch is sealed, sampled for every conditional create that
135    /// succeeds, an identical retry included. The time runs from the seal, so
136    /// it includes any wait for a create slot and, in the `enqueued` mode,
137    /// every repeated attempt and the delay before it.
138    ///
139    /// A create is a few milliseconds on a local object store and tens of
140    /// milliseconds on a cloud one, so the buckets start at 1ms and reach 8s.
141    pub static ref METRIC_OBJECT_STORE_WAL_SEAL_TO_DURABLE_SECONDS: Histogram = register_histogram!(
142        "greptime_logstore_object_store_wal_seal_to_durable_seconds",
143        "object store logstore seconds from sealing a batch to the success of its conditional create, slot waits and retries included",
144        exponential_buckets(0.001, 2.0, 14).unwrap(),
145    )
146    .unwrap();
147
148    /// How long an append of the object store logstore waits for its
149    /// acknowledgement in the `durable` mode, which is the write latency a
150    /// caller sees.
151    ///
152    /// The wait is one seal plus every earlier object plus one create, so with
153    /// the default 100ms flush interval it sits in the tens of milliseconds;
154    /// the buckets straddle that and leave room for a stalled uploader.
155    pub static ref METRIC_OBJECT_STORE_WAL_APPEND_ACK_SECONDS: Histogram = register_histogram!(
156        "greptime_logstore_object_store_wal_append_ack_seconds",
157        "object store logstore seconds an append waits for its acknowledgement in the durable mode",
158        exponential_buckets(0.002, 2.0, 14).unwrap(),
159    )
160    .unwrap();
161
162    /// How large the objects of the object store logstore are, which says
163    /// whether batches seal on size or on the flush interval.
164    ///
165    /// The buckets are powers of two from 1KiB, the size of a timer-sealed
166    /// batch of one small entry, to 16MiB, twice the 8MiB default batch size,
167    /// which is a threshold rather than an upper bound.
168    pub static ref METRIC_OBJECT_STORE_WAL_OBJECT_BYTES: Histogram = register_histogram!(
169        "greptime_logstore_object_store_wal_object_bytes",
170        "object store logstore bytes of each object sealed",
171        exponential_buckets(1024.0, 2.0, 15).unwrap(),
172    )
173    .unwrap();
174
175    /// How many entries the objects of the object store logstore hold, which
176    /// says how many writes one request amortises.
177    ///
178    /// The largest finite bucket is 2^20, the position limit of one region in
179    /// one object. An object holds the entries of many regions, so its total
180    /// can pass that; such an object falls in the overflow bucket.
181    pub static ref METRIC_OBJECT_STORE_WAL_OBJECT_ENTRIES: Histogram = register_histogram!(
182        "greptime_logstore_object_store_wal_object_entries",
183        "object store logstore entries of each object sealed",
184        exponential_buckets(1.0, 4.0, 11).unwrap(),
185    )
186    .unwrap();
187
188    /// Counter of conditional creates of the object store logstore that
189    /// succeeded, an identical retry that found its own object included. A
190    /// create that failed is counted in the create failure or conflict counter
191    /// instead.
192    pub static ref METRIC_OBJECT_STORE_WAL_CREATED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
193        "greptime_logstore_object_store_wal_created_objects_total",
194        "object store logstore conditional creates that succeeded, identical retries included",
195    )
196    .unwrap();
197
198    /// Number of objects the object store logstore has indexed, which is what
199    /// a recovery of the prefix would have to read.
200    pub static ref METRIC_OBJECT_STORE_WAL_INDEXED_OBJECTS: IntGauge = register_int_gauge!(
201        "greptime_logstore_object_store_wal_indexed_objects",
202        "object store logstore objects currently indexed",
203    )
204    .unwrap();
205
206    /// Bytes of the objects the object store logstore has indexed, which is
207    /// what the WAL occupies under its prefix.
208    pub static ref METRIC_OBJECT_STORE_WAL_INDEXED_BYTES: IntGauge = register_int_gauge!(
209        "greptime_logstore_object_store_wal_indexed_bytes",
210        "object store logstore bytes currently indexed",
211    )
212    .unwrap();
213
214    /// How long a recovery of the object store logstore that succeeded took to
215    /// rebuild its catalog, which is the part of a restart the WAL is
216    /// responsible for. A recovery that was abandoned records nothing.
217    ///
218    /// Recovery reads a few small ranges per object, so the buckets run from
219    /// 10ms, an empty prefix, to minutes on a prefix with many objects.
220    pub static ref METRIC_OBJECT_STORE_WAL_RECOVERY_SECONDS: Histogram = register_histogram!(
221        "greptime_logstore_object_store_wal_recovery_seconds",
222        "object store logstore seconds spent by a recovery that succeeded",
223        exponential_buckets(0.01, 3.0, 10).unwrap(),
224    )
225    .unwrap();
226
227    /// Counter of objects a recovery of the object store logstore that
228    /// succeeded read footers from, which is what its time is spent on. It
229    /// counts the objects of the same recoveries the timer above measures, so
230    /// an abandoned one contributes to neither.
231    pub static ref METRIC_OBJECT_STORE_WAL_RECOVERED_OBJECTS_TOTAL: IntCounter = register_int_counter!(
232        "greptime_logstore_object_store_wal_recovered_objects_total",
233        "object store logstore objects read by a recovery that succeeded total",
234    )
235    .unwrap();
236
237    /// How long a read stream of the object store logstore took to serve the
238    /// replay of one region, recorded once the stream has been read to its
239    /// end; one that fails or is dropped part way records nothing.
240    ///
241    /// A read fetches one segment per object of the region, so the buckets run
242    /// from 5ms, a region with nothing to replay, to minutes.
243    pub static ref METRIC_OBJECT_STORE_WAL_READ_SECONDS: Histogram = register_histogram!(
244        "greptime_logstore_object_store_wal_read_seconds",
245        "object store logstore seconds spent serving a region read that ran to its end",
246        exponential_buckets(0.005, 3.0, 10).unwrap(),
247    )
248    .unwrap();
249
250    /// Counter of conditional creates of the object store logstore that found
251    /// an object with different content. In the `durable` mode an object of an
252    /// earlier epoch fails the batch like a transient failure; any other
253    /// conflict poisons the store.
254    pub static ref METRIC_OBJECT_STORE_WAL_CREATE_CONFLICTS_TOTAL: IntCounter = register_int_counter!(
255        "greptime_logstore_object_store_wal_create_conflicts_total",
256        "object store logstore conflicting conditional creates total",
257    )
258    .unwrap();
259
260    /// Counter of conditional creates of the object store logstore the object
261    /// store did not confirm, which fail the appends behind them in the
262    /// `durable` mode and are repeated in the `enqueued` mode.
263    pub static ref METRIC_OBJECT_STORE_WAL_CREATE_FAILURES_TOTAL: IntCounter = register_int_counter!(
264        "greptime_logstore_object_store_wal_create_failures_total",
265        "object store logstore conditional creates the object store did not confirm total",
266    )
267    .unwrap();
268
269    /// Counter of object store logstores that hit an error they cannot recover
270    /// from, after which every operation of the store fails until it reopens.
271    pub static ref METRIC_OBJECT_STORE_WAL_POISONED_TOTAL: IntCounter = register_int_counter!(
272        "greptime_logstore_object_store_wal_poisoned_total",
273        "object store logstores poisoned total",
274    )
275    .unwrap();
276
277    /// Counter of appends the object store logstore held back until an upload
278    /// completed: in either mode because the sealed batches waiting to become
279    /// durable left no room for another, and in the `enqueued` mode also
280    /// because the unpersisted backlog reached a threshold. Either says the
281    /// write rate is above what the uploads sustain.
282    pub static ref METRIC_OBJECT_STORE_WAL_STALLED_APPENDS_TOTAL: IntCounter = register_int_counter!(
283        "greptime_logstore_object_store_wal_stalled_appends_total",
284        "object store logstore appends stalled by full sealed batches or the backlog thresholds total",
285    )
286    .unwrap();
287
288    /// How long such an append stays held back, until it is admitted or until
289    /// it is refused, a stop, a terminal error or the teardown of the store
290    /// releasing it included, which is the latency the `enqueued` mode pays
291    /// once the backlog is at a threshold.
292    ///
293    /// A stalled append waits for one upload to complete, so the buckets run
294    /// from 5ms to half a minute of a struggling object store.
295    pub static ref METRIC_OBJECT_STORE_WAL_STALLED_APPEND_SECONDS: Histogram = register_histogram!(
296        "greptime_logstore_object_store_wal_stalled_append_seconds",
297        "object store logstore seconds a stalled append is held back before it is admitted or refused",
298        exponential_buckets(0.005, 3.0, 9).unwrap(),
299    )
300    .unwrap();
301
302    /// Number of object store logstores in the `enqueued` mode whose backlog
303    /// the object store refuses with an error that repeating the create
304    /// cannot settle, such as denied access or a missing bucket. Such a store
305    /// keeps its backlog, attempts it again with backoff and rejects appends
306    /// until a create succeeds.
307    pub static ref METRIC_OBJECT_STORE_WAL_FAILING: IntGauge = register_int_gauge!(
308        "greptime_logstore_object_store_wal_failing",
309        "object store logstores rejecting appends because the object store refuses their backlog with an error that requires intervention",
310    )
311    .unwrap();
312
313    /// Counter of appends the object store logstore rejected while its
314    /// backlog is refused, see the gauge above. They take no entry id and
315    /// fail with a retryable error. Releases of dropped regions failed in
316    /// that state count too; their records wait for the next object.
317    pub static ref METRIC_OBJECT_STORE_WAL_REJECTED_APPENDS_TOTAL: IntCounter = register_int_counter!(
318        "greptime_logstore_object_store_wal_rejected_appends_total",
319        "object store logstore appends rejected while the object store refuses the backlog total",
320    )
321    .unwrap();
322}
log_store/metrics.rs

log_store/
metrics.rs
