common_telemetry/
panic_hook.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::panic;
16#[cfg(feature = "deadlock_detection")]
17use std::time::Duration;
18
19use backtrace::Backtrace;
20use lazy_static::lazy_static;
21use prometheus::*;
22
23lazy_static! {
24    pub static ref PANIC_COUNTER: IntCounter =
25        register_int_counter!("greptime_panic_counter", "panic_counter").unwrap();
26}
27
28pub fn set_panic_hook() {
29    // Set a panic hook that records the panic as a `tracing` event at the
30    // `ERROR` verbosity level.
31    //
32    // If we are currently in a span when the panic occurred, the logged event
33    // will include the current span, allowing the context in which the panic
34    // occurred to be recorded.
35    let default_hook = panic::take_hook();
36    panic::set_hook(Box::new(move |panic| {
37        let backtrace = Backtrace::new();
38        let backtrace = format!("{backtrace:?}");
39        if let Some(location) = panic.location() {
40            tracing::error!(
41                message = %panic,
42                backtrace = %backtrace,
43                panic.file = location.file(),
44                panic.line = location.line(),
45                panic.column = location.column(),
46            );
47        } else {
48            tracing::error!(message = %panic, backtrace = %backtrace);
49        }
50        PANIC_COUNTER.inc();
51        default_hook(panic);
52    }));
53
54    #[cfg(feature = "deadlock_detection")]
55    let _ = std::thread::spawn(move || loop {
56        std::thread::sleep(Duration::from_secs(5));
57        let deadlocks = parking_lot::deadlock::check_deadlock();
58        if deadlocks.is_empty() {
59            continue;
60        }
61
62        tracing::info!("{} deadlocks detected", deadlocks.len());
63        for (i, threads) in deadlocks.iter().enumerate() {
64            tracing::info!("Deadlock #{}", i);
65            for t in threads {
66                tracing::info!("Thread Id {:#?}", t.thread_id());
67                tracing::info!("{:#?}", t.backtrace());
68            }
69        }
70    });
71}
common_telemetry/panic_hook.rs

common_telemetry/
panic_hook.rs
