common_telemetry/
panic_hook.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::panic;
16#[cfg(feature = "deadlock_detection")]
17use std::time::Duration;
18
19use backtrace::Backtrace;
20use lazy_static::lazy_static;
21use prometheus::*;
22
23lazy_static! {
24    pub static ref PANIC_COUNTER: IntCounter =
25        register_int_counter!("greptime_panic_counter", "panic_counter").unwrap();
26}
27
28pub fn set_panic_hook() {
29    // Set a panic hook that records the panic as a `tracing` event at the
30    // `ERROR` verbosity level.
31    //
32    // If we are currently in a span when the panic occurred, the logged event
33    // will include the current span, allowing the context in which the panic
34    // occurred to be recorded.
35    let default_hook = panic::take_hook();
36    panic::set_hook(Box::new(move |panic| {
37        let backtrace = Backtrace::new();
38        let backtrace = format!("{backtrace:?}");
39        let build_info = common_version::build_info();
40        if let Some(location) = panic.location() {
41            tracing::error!(
42                message = %panic,
43                backtrace = %backtrace,
44                panic.file = location.file(),
45                panic.line = location.line(),
46                panic.column = location.column(),
47                branch = %build_info.branch,
48                version = %build_info.version,
49                commit = %build_info.commit,
50            );
51        } else {
52            tracing::error!(
53                message = %panic,
54                backtrace = %backtrace,
55                branch = %build_info.branch,
56                version = %build_info.version,
57                commit = %build_info.commit,
58            );
59        }
60        PANIC_COUNTER.inc();
61        default_hook(panic);
62    }));
63
64    #[cfg(feature = "deadlock_detection")]
65    let _ = std::thread::spawn(move || {
66        loop {
67            std::thread::sleep(Duration::from_secs(5));
68            let deadlocks = parking_lot::deadlock::check_deadlock();
69            if deadlocks.is_empty() {
70                continue;
71            }
72
73            tracing::info!("{} deadlocks detected", deadlocks.len());
74            for (i, threads) in deadlocks.iter().enumerate() {
75                tracing::info!("Deadlock #{}", i);
76                for t in threads {
77                    tracing::info!("Thread Id {:#?}", t.thread_id());
78                    tracing::info!("{:#?}", t.backtrace());
79                }
80            }
81        }
82    });
83}
common_telemetry/panic_hook.rs

common_telemetry/
panic_hook.rs
