Skip to main content

meta_srv/
metasrv.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod builder;
16
17use std::fmt::{self, Display};
18use std::sync::atomic::{AtomicBool, Ordering};
19use std::sync::{Arc, Mutex, RwLock};
20use std::time::Duration;
21
22use api::v1::meta::{HeartbeatConfig, Role};
23use clap::ValueEnum;
24use common_base::Plugins;
25use common_base::readable_size::ReadableSize;
26use common_config::{Configurable, DEFAULT_DATA_HOME};
27use common_event_recorder::EventRecorderOptions;
28use common_greptimedb_telemetry::GreptimeDBTelemetryTask;
29use common_meta::affinity::{AffinityFallback, AffinityRef, AffinityResolverRef};
30use common_meta::cache_invalidator::CacheInvalidatorRef;
31use common_meta::ddl::allocator::resource_id::ResourceIdAllocatorRef;
32use common_meta::ddl_manager::DdlManagerRef;
33use common_meta::distributed_time_constants::{
34    self, BASE_HEARTBEAT_INTERVAL, default_distributed_time_constants, frontend_heartbeat_interval,
35};
36use common_meta::election::LeaderChangeMessage;
37pub use common_meta::election::{ElectionRef, MetasrvNodeInfo};
38use common_meta::key::TableMetadataManagerRef;
39use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
40use common_meta::kv_backend::{KvBackendRef, ResettableKvBackend, ResettableKvBackendRef};
41use common_meta::leadership_notifier::{
42    LeadershipChangeNotifier, LeadershipChangeNotifierCustomizerRef,
43};
44use common_meta::node_expiry_listener::NodeExpiryListener;
45use common_meta::peer::{Peer, PeerDiscoveryRef};
46use common_meta::reconciliation::manager::ReconciliationManagerRef;
47use common_meta::region_keeper::MemoryRegionKeeperRef;
48use common_meta::region_registry::LeaderRegionRegistryRef;
49use common_meta::stats::topic::TopicStatsRegistryRef;
50use common_meta::wal_provider::WalProviderRef;
51use common_options::datanode::DatanodeClientOptions;
52use common_options::memory::MemoryOptions;
53use common_procedure::ProcedureManagerRef;
54use common_procedure::options::ProcedureConfig;
55use common_stat::ResourceStatRef;
56use common_telemetry::logging::{LoggingOptions, TracingOptions};
57use common_telemetry::{error, info, warn};
58use common_time::util::DefaultSystemTimer;
59use common_wal::config::MetasrvWalConfig;
60use serde::{Deserialize, Serialize};
61use servers::grpc::GrpcOptions;
62use servers::http::HttpOptions;
63use servers::tls::TlsOption;
64use snafu::{OptionExt, ResultExt};
65use store_api::storage::RegionId;
66use tokio::sync::broadcast::error::RecvError;
67
68use crate::cluster::MetaPeerClientRef;
69use crate::discovery;
70use crate::discovery::node_info::NodeInfoAccessor;
71use crate::error::{
72    self, InitMetadataSnafu, KvBackendSnafu, Result, StartProcedureManagerSnafu,
73    StartTelemetryTaskSnafu, StopProcedureManagerSnafu,
74};
75use crate::failure_detector::PhiAccrualFailureDetectorOptions;
76use crate::gc::{GcSchedulerOptions, GcTickerRef};
77use crate::handler::{HeartbeatHandlerGroupBuilder, HeartbeatHandlerGroupRef};
78use crate::node_scheduling::NodeSchedulingManagerRef;
79use crate::procedure::ProcedureManagerListenerAdapter;
80use crate::procedure::drain_datanode::manager::DrainDatanodeManagerRef;
81use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
82use crate::procedure::repartition::gc_requirement::RepartitionGcRequirementManagerRef;
83use crate::procedure::wal_prune::manager::WalPruneTickerRef;
84use crate::pubsub::{PublisherRef, SubscriptionManagerRef};
85use crate::region::flush_trigger::RegionFlushTickerRef;
86use crate::region::supervisor::RegionSupervisorTickerRef;
87use crate::selector::{RegionStatAwareSelector, Selector, SelectorType};
88use crate::service::mailbox::MailboxRef;
89use crate::service::store::cached_kv::LeaderCachedKvBackend;
90use crate::state::{StateRef, become_follower, become_leader};
91use crate::utils::database::DatabaseOperatorRef;
92
93pub const TABLE_ID_SEQ: &str = "table_id";
94pub const FLOW_ID_SEQ: &str = "flow_id";
95pub const METASRV_DATA_DIR: &str = "metasrv";
96
97// The datastores that implements metadata kvbackend.
98#[derive(Clone, Debug, PartialEq, Serialize, Default, Deserialize, ValueEnum)]
99#[serde(rename_all = "snake_case")]
100pub enum BackendImpl {
101    // Etcd as metadata storage.
102    #[default]
103    EtcdStore,
104    // In memory metadata storage - mostly used for testing.
105    MemoryStore,
106    #[cfg(feature = "pg_kvbackend")]
107    // Postgres as metadata storage.
108    PostgresStore,
109    #[cfg(feature = "mysql_kvbackend")]
110    // MySql as metadata storage.
111    MysqlStore,
112}
113
114/// Configuration options for the stats persistence.
115#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
116pub struct StatsPersistenceOptions {
117    /// TTL for the stats table that will be used to store the stats.
118    #[serde(with = "humantime_serde")]
119    pub ttl: Duration,
120    /// The interval to persist the stats.
121    #[serde(with = "humantime_serde")]
122    pub interval: Duration,
123}
124
125impl Default for StatsPersistenceOptions {
126    fn default() -> Self {
127        Self {
128            ttl: Duration::ZERO,
129            interval: Duration::from_secs(10 * 60),
130        }
131    }
132}
133
134/// Heartbeat configuration for a single node type.
135#[derive(Clone, PartialEq, Serialize, Deserialize, Debug)]
136#[serde(default)]
137pub struct HeartbeatOptions {
138    /// Heartbeat interval.
139    #[serde(with = "humantime_serde")]
140    pub interval: Duration,
141    /// Retry interval when heartbeat connection fails.
142    #[serde(with = "humantime_serde")]
143    pub retry_interval: Duration,
144}
145
146impl Default for HeartbeatOptions {
147    fn default() -> Self {
148        Self {
149            interval: BASE_HEARTBEAT_INTERVAL,
150            retry_interval: BASE_HEARTBEAT_INTERVAL,
151        }
152    }
153}
154
155impl HeartbeatOptions {
156    pub fn datanode_from(base_interval: Duration) -> Self {
157        Self {
158            interval: base_interval,
159            retry_interval: base_interval,
160        }
161    }
162
163    pub fn frontend_from(base_interval: Duration) -> Self {
164        Self {
165            interval: frontend_heartbeat_interval(base_interval),
166            retry_interval: base_interval,
167        }
168    }
169
170    pub fn flownode_from(base_interval: Duration) -> Self {
171        Self {
172            interval: base_interval,
173            retry_interval: base_interval,
174        }
175    }
176}
177
178impl From<HeartbeatOptions> for HeartbeatConfig {
179    fn from(opts: HeartbeatOptions) -> Self {
180        Self {
181            heartbeat_interval_ms: opts.interval.as_millis() as u64,
182            retry_interval_ms: opts.retry_interval.as_millis() as u64,
183            gc_enabled: false,
184        }
185    }
186}
187
188#[derive(Clone, PartialEq, Serialize, Deserialize, Debug)]
189#[serde(default)]
190pub struct BackendClientOptions {
191    #[serde(with = "humantime_serde")]
192    pub keep_alive_timeout: Duration,
193    #[serde(with = "humantime_serde")]
194    pub keep_alive_interval: Duration,
195    #[serde(with = "humantime_serde")]
196    pub connect_timeout: Duration,
197}
198
199impl Default for BackendClientOptions {
200    fn default() -> Self {
201        Self {
202            keep_alive_interval: Duration::from_secs(10),
203            keep_alive_timeout: Duration::from_secs(3),
204            connect_timeout: Duration::from_secs(3),
205        }
206    }
207}
208
209#[derive(Clone, PartialEq, Serialize, Deserialize)]
210#[serde(default)]
211pub struct MetasrvOptions {
212    /// The address the server listens on.
213    #[deprecated(note = "Use grpc.bind_addr instead")]
214    pub bind_addr: String,
215    /// The address the server advertises to the clients.
216    #[deprecated(note = "Use grpc.server_addr instead")]
217    pub server_addr: String,
218    /// The address of the store, e.g., etcd.
219    #[serde(serialize_with = "serialize_store_addrs")]
220    pub store_addrs: Vec<String>,
221    /// TLS configuration for kv store backend (PostgreSQL/MySQL)
222    /// Only applicable when using PostgreSQL or MySQL as the metadata store
223    #[serde(default)]
224    pub backend_tls: Option<TlsOption>,
225    /// The backend client options.
226    /// Currently, only applicable when using etcd as the metadata store.
227    #[serde(default)]
228    pub backend_client: BackendClientOptions,
229    /// The type of selector.
230    pub selector: SelectorType,
231    /// Whether to enable region failover.
232    pub enable_region_failover: bool,
233    /// The base heartbeat interval.
234    ///
235    /// This value is used to calculate the distributed time constants for components.
236    /// e.g., the region lease time is `heartbeat_interval * 3 + Duration::from_secs(1)`.
237    #[serde(with = "humantime_serde")]
238    pub heartbeat_interval: Duration,
239    /// The delay before starting region failure detection.
240    /// This delay helps prevent Metasrv from triggering unnecessary region failovers before all Datanodes are fully started.
241    /// Especially useful when the cluster is not deployed with GreptimeDB Operator and maintenance mode is not enabled.
242    #[serde(with = "humantime_serde")]
243    pub region_failure_detector_initialization_delay: Duration,
244    /// Whether to allow region failover on local WAL.
245    ///
246    /// If it's true, the region failover will be allowed even if the local WAL is used.
247    /// Note that this option is not recommended to be set to true, because it may lead to data loss during failover.
248    pub allow_region_failover_on_local_wal: bool,
249    pub grpc: GrpcOptions,
250    /// The HTTP server options.
251    pub http: HttpOptions,
252    /// The logging options.
253    pub logging: LoggingOptions,
254    /// The procedure options.
255    pub procedure: ProcedureConfig,
256    /// The failure detector options.
257    pub failure_detector: PhiAccrualFailureDetectorOptions,
258    /// The datanode options.
259    pub datanode: DatanodeClientOptions,
260    /// Whether to enable telemetry.
261    pub enable_telemetry: bool,
262    /// The data home directory.
263    pub data_home: String,
264    /// The WAL options.
265    pub wal: MetasrvWalConfig,
266    /// The store key prefix. If it is not empty, all keys in the store will be prefixed with it.
267    /// This is useful when multiple metasrv clusters share the same store.
268    pub store_key_prefix: String,
269    /// The max operations per txn
270    ///
271    /// This value is usually limited by which store is used for the `KvBackend`.
272    /// For example, if using etcd, this value should ensure that it is less than
273    /// or equal to the `--max-txn-ops` option value of etcd.
274    ///
275    /// TODO(jeremy): Currently, this option only affects the etcd store, but it may
276    /// also affect other stores in the future. In other words, each store needs to
277    /// limit the number of operations in a txn because an infinitely large txn could
278    /// potentially block other operations.
279    pub max_txn_ops: usize,
280    /// The factor that determines how often statistics should be flushed,
281    /// based on the number of received heartbeats. When the number of heartbeats
282    /// reaches this factor, a flush operation is triggered.
283    pub flush_stats_factor: usize,
284    /// The tracing options.
285    pub tracing: TracingOptions,
286    /// The memory options.
287    pub memory: MemoryOptions,
288    /// The datastore for kv metadata.
289    pub backend: BackendImpl,
290    #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
291    /// Table name of rds kv backend.
292    pub meta_table_name: String,
293    #[cfg(feature = "pg_kvbackend")]
294    /// Lock id for meta kv election. Only effect when using pg_kvbackend.
295    pub meta_election_lock_id: u64,
296    #[cfg(feature = "pg_kvbackend")]
297    /// Optional PostgreSQL schema for metadata table (defaults to current search_path if empty).
298    pub meta_schema_name: Option<String>,
299    #[cfg(feature = "pg_kvbackend")]
300    /// Automatically create PostgreSQL schema if it doesn't exist (default: true).
301    pub auto_create_schema: bool,
302    #[serde(with = "humantime_serde")]
303    pub node_max_idle_time: Duration,
304    /// The event recorder options.
305    pub event_recorder: EventRecorderOptions,
306    /// The stats persistence options.
307    pub stats_persistence: StatsPersistenceOptions,
308    /// The GC scheduler options.
309    pub gc: GcSchedulerOptions,
310}
311
312impl fmt::Debug for MetasrvOptions {
313    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
314        let mut debug_struct = f.debug_struct("MetasrvOptions");
315        debug_struct
316            .field("store_addrs", &sanitize_store_addrs(&self.store_addrs))
317            .field("backend_tls", &self.backend_tls)
318            .field("selector", &self.selector)
319            .field("enable_region_failover", &self.enable_region_failover)
320            .field(
321                "allow_region_failover_on_local_wal",
322                &self.allow_region_failover_on_local_wal,
323            )
324            .field("grpc", &self.grpc)
325            .field("http", &self.http)
326            .field("logging", &self.logging)
327            .field("procedure", &self.procedure)
328            .field("failure_detector", &self.failure_detector)
329            .field("datanode", &self.datanode)
330            .field("enable_telemetry", &self.enable_telemetry)
331            .field("data_home", &self.data_home)
332            .field("wal", &self.wal)
333            .field("store_key_prefix", &self.store_key_prefix)
334            .field("max_txn_ops", &self.max_txn_ops)
335            .field("flush_stats_factor", &self.flush_stats_factor)
336            .field("tracing", &self.tracing)
337            .field("backend", &self.backend)
338            .field("event_recorder", &self.event_recorder)
339            .field("stats_persistence", &self.stats_persistence)
340            .field("heartbeat_interval", &self.heartbeat_interval)
341            .field("backend_client", &self.backend_client);
342
343        #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
344        debug_struct.field("meta_table_name", &self.meta_table_name);
345
346        #[cfg(feature = "pg_kvbackend")]
347        debug_struct.field("meta_election_lock_id", &self.meta_election_lock_id);
348        #[cfg(feature = "pg_kvbackend")]
349        debug_struct.field("meta_schema_name", &self.meta_schema_name);
350
351        debug_struct
352            .field("node_max_idle_time", &self.node_max_idle_time)
353            .finish()
354    }
355}
356
357const DEFAULT_METASRV_ADDR_PORT: &str = "3002";
358
359impl Default for MetasrvOptions {
360    fn default() -> Self {
361        Self {
362            #[allow(deprecated)]
363            bind_addr: String::new(),
364            #[allow(deprecated)]
365            server_addr: String::new(),
366            store_addrs: vec!["127.0.0.1:2379".to_string()],
367            backend_tls: Some(TlsOption::prefer()),
368            selector: SelectorType::default(),
369            enable_region_failover: false,
370            heartbeat_interval: distributed_time_constants::BASE_HEARTBEAT_INTERVAL,
371            region_failure_detector_initialization_delay: Duration::from_secs(10 * 60),
372            allow_region_failover_on_local_wal: false,
373            grpc: GrpcOptions {
374                bind_addr: format!("127.0.0.1:{}", DEFAULT_METASRV_ADDR_PORT),
375                ..Default::default()
376            },
377            http: HttpOptions::default(),
378            logging: LoggingOptions::default(),
379            procedure: ProcedureConfig {
380                max_retry_times: 12,
381                retry_delay: Duration::from_millis(500),
382                // The etcd the maximum size of any request is 1.5 MiB
383                // 1500KiB = 1536KiB (1.5MiB) - 36KiB (reserved size of key)
384                max_metadata_value_size: Some(ReadableSize::kb(1500)),
385                max_running_procedures: 128,
386            },
387            failure_detector: PhiAccrualFailureDetectorOptions::default(),
388            datanode: DatanodeClientOptions::default(),
389            enable_telemetry: true,
390            data_home: DEFAULT_DATA_HOME.to_string(),
391            wal: MetasrvWalConfig::default(),
392            store_key_prefix: String::new(),
393            max_txn_ops: 128,
394            flush_stats_factor: 3,
395            tracing: TracingOptions::default(),
396            memory: MemoryOptions::default(),
397            backend: BackendImpl::EtcdStore,
398            #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
399            meta_table_name: common_meta::kv_backend::DEFAULT_META_TABLE_NAME.to_string(),
400            #[cfg(feature = "pg_kvbackend")]
401            meta_election_lock_id: common_meta::kv_backend::DEFAULT_META_ELECTION_LOCK_ID,
402            #[cfg(feature = "pg_kvbackend")]
403            meta_schema_name: None,
404            #[cfg(feature = "pg_kvbackend")]
405            auto_create_schema: true,
406            node_max_idle_time: Duration::from_secs(24 * 60 * 60),
407            event_recorder: EventRecorderOptions::default(),
408            stats_persistence: StatsPersistenceOptions::default(),
409            gc: GcSchedulerOptions::default(),
410            backend_client: BackendClientOptions::default(),
411        }
412    }
413}
414
415impl Configurable for MetasrvOptions {
416    fn env_list_keys() -> Option<&'static [&'static str]> {
417        Some(&[
418            "wal.broker_endpoints",
419            "store_addrs",
420            "event_recorder.event_types",
421        ])
422    }
423}
424
425fn sanitize_store_addrs(addrs: &[String]) -> Vec<String> {
426    addrs
427        .iter()
428        .map(|addr| common_meta::kv_backend::util::sanitize_connection_string(addr))
429        .collect()
430}
431
432fn serialize_store_addrs<S>(addrs: &[String], serializer: S) -> std::result::Result<S::Ok, S::Error>
433where
434    S: serde::Serializer,
435{
436    sanitize_store_addrs(addrs).serialize(serializer)
437}
438
439pub struct MetasrvInfo {
440    pub server_addr: String,
441}
442#[derive(Clone)]
443pub struct Context {
444    pub server_addr: String,
445    pub in_memory: ResettableKvBackendRef,
446    pub kv_backend: KvBackendRef,
447    pub leader_cached_kv_backend: ResettableKvBackendRef,
448    pub meta_peer_client: MetaPeerClientRef,
449    pub mailbox: MailboxRef,
450    pub election: Option<ElectionRef>,
451    pub is_infancy: bool,
452    pub table_metadata_manager: TableMetadataManagerRef,
453    pub cache_invalidator: CacheInvalidatorRef,
454    pub leader_region_registry: LeaderRegionRegistryRef,
455    pub topic_stats_registry: TopicStatsRegistryRef,
456    pub heartbeat_interval: Duration,
457    pub gc_enabled: bool,
458    pub is_handshake: bool,
459}
460
461impl Context {
462    pub fn reset_in_memory(&self) {
463        self.in_memory.reset();
464        self.leader_region_registry.reset();
465    }
466
467    pub fn with_handshake(mut self, is_handshake: bool) -> Self {
468        self.is_handshake = is_handshake;
469        self
470    }
471
472    pub fn heartbeat_options_for(&self, role: Role) -> HeartbeatOptions {
473        match role {
474            Role::Datanode => HeartbeatOptions::datanode_from(self.heartbeat_interval),
475            Role::Frontend => HeartbeatOptions::frontend_from(self.heartbeat_interval),
476            Role::Flownode => HeartbeatOptions::flownode_from(self.heartbeat_interval),
477        }
478    }
479}
480
481#[derive(Clone, Copy)]
482pub enum SelectTarget {
483    Datanode,
484    Flownode,
485}
486
487impl Display for SelectTarget {
488    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
489        match self {
490            SelectTarget::Datanode => write!(f, "datanode"),
491            SelectTarget::Flownode => write!(f, "flownode"),
492        }
493    }
494}
495
496#[derive(Clone)]
497pub struct SelectorContext {
498    pub peer_discovery: PeerDiscoveryRef,
499    /// Evaluator shared by datanode and flownode selection.
500    pub affinity: AffinityRef,
501    /// Whether selection may retry without affinity when candidates are insufficient.
502    pub affinity_fallback: AffinityFallback,
503}
504
505pub type SelectorRef = Arc<dyn Selector<Context = SelectorContext, Output = Vec<Peer>>>;
506
507/// Context passed to a selector factory during metasrv bootstrap.
508///
509/// The factory runs after bootstrap has constructed the selector configured by
510/// [`MetasrvOptions::selector`], so plugins can either decorate `base_selector` or
511/// build a completely different selector using bootstrap-only dependencies like
512/// [`MetaPeerClientRef`].
513pub struct SelectorFactoryContext {
514    pub metasrv_options: MetasrvOptions,
515    pub meta_peer_client: MetaPeerClientRef,
516    pub in_memory: ResettableKvBackendRef,
517    pub election: Option<ElectionRef>,
518    pub base_selector: SelectorRef,
519}
520
521/// Builds the final datanode selector metasrv should use.
522pub trait SelectorFactory: Send + Sync {
523    fn build(&self, ctx: SelectorFactoryContext) -> SelectorRef;
524
525    /// Builds the resolver shared by allocation and failover callers.
526    ///
527    /// The accessor includes last-known records of inactive source nodes; discovery
528    /// is the shared scheduling-aware view of active nodes.
529    fn build_affinity_resolver(
530        &self,
531        _node_info_accessor: Arc<dyn NodeInfoAccessor>,
532        _peer_discovery: PeerDiscoveryRef,
533    ) -> Option<AffinityResolverRef> {
534        None
535    }
536}
537
538/// Shared selector factory plugin registered through [`common_base::Plugins`].
539pub type SelectorFactoryRef = Arc<dyn SelectorFactory>;
540pub type RegionStatAwareSelectorRef =
541    Arc<dyn RegionStatAwareSelector<Context = SelectorContext, Output = Vec<(RegionId, Peer)>>>;
542
543pub struct MetaStateHandler {
544    subscribe_manager: Option<SubscriptionManagerRef>,
545    greptimedb_telemetry_task: Arc<GreptimeDBTelemetryTask>,
546    leader_cached_kv_backend: Arc<LeaderCachedKvBackend>,
547    leadership_change_notifier: LeadershipChangeNotifier,
548    state: StateRef,
549}
550
551impl MetaStateHandler {
552    pub async fn on_leader_start(&self) {
553        self.state.write().unwrap().next_state(become_leader(false));
554
555        if let Err(e) = self.leader_cached_kv_backend.load().await {
556            error!(e; "Failed to load kv into leader cache kv store");
557        } else {
558            self.state.write().unwrap().next_state(become_leader(true));
559        }
560
561        self.leadership_change_notifier
562            .notify_on_leader_start()
563            .await;
564
565        self.greptimedb_telemetry_task.should_report(true);
566    }
567
568    pub async fn on_leader_stop(&self) {
569        self.state.write().unwrap().next_state(become_follower());
570
571        self.leadership_change_notifier
572            .notify_on_leader_stop()
573            .await;
574
575        // Suspends reporting.
576        self.greptimedb_telemetry_task.should_report(false);
577
578        if let Some(sub_manager) = self.subscribe_manager.clone() {
579            info!("Leader changed, un_subscribe all");
580            if let Err(e) = sub_manager.unsubscribe_all() {
581                error!(e; "Failed to un_subscribe all");
582            }
583        }
584    }
585}
586
587pub struct Metasrv {
588    state: StateRef,
589    started: Arc<AtomicBool>,
590    start_time_ms: u64,
591    options: MetasrvOptions,
592    // It is only valid at the leader node and is used to temporarily
593    // store some data that will not be persisted.
594    in_memory: ResettableKvBackendRef,
595    kv_backend: KvBackendRef,
596    leader_cached_kv_backend: Arc<LeaderCachedKvBackend>,
597    meta_peer_client: MetaPeerClientRef,
598    // The selector is used to select a target datanode.
599    selector: SelectorRef,
600    selector_ctx: SelectorContext,
601    // The flow selector is used to select a target flownode.
602    flow_selector: SelectorRef,
603    handler_group: RwLock<Option<HeartbeatHandlerGroupRef>>,
604    handler_group_builder: Mutex<Option<HeartbeatHandlerGroupBuilder>>,
605    election: Option<ElectionRef>,
606    procedure_manager: ProcedureManagerRef,
607    mailbox: MailboxRef,
608    ddl_manager: DdlManagerRef,
609    wal_provider: WalProviderRef,
610    table_metadata_manager: TableMetadataManagerRef,
611    runtime_switch_manager: RuntimeSwitchManagerRef,
612    repartition_gc_requirement_manager: RepartitionGcRequirementManagerRef,
613    memory_region_keeper: MemoryRegionKeeperRef,
614    greptimedb_telemetry_task: Arc<GreptimeDBTelemetryTask>,
615    region_migration_manager: RegionMigrationManagerRef,
616    drain_datanode_manager: DrainDatanodeManagerRef,
617    region_supervisor_ticker: Option<RegionSupervisorTickerRef>,
618    cache_invalidator: CacheInvalidatorRef,
619    leader_region_registry: LeaderRegionRegistryRef,
620    topic_stats_registry: TopicStatsRegistryRef,
621    wal_prune_ticker: Option<WalPruneTickerRef>,
622    region_flush_ticker: Option<RegionFlushTickerRef>,
623    table_id_allocator: ResourceIdAllocatorRef,
624    reconciliation_manager: ReconciliationManagerRef,
625    resource_stat: ResourceStatRef,
626    gc_ticker: Option<GcTickerRef>,
627    database_operator: DatabaseOperatorRef,
628    node_scheduling_manager: NodeSchedulingManagerRef,
629
630    plugins: Plugins,
631}
632
633impl Metasrv {
634    pub(crate) async fn ensure_repartition_gc_enabled(&self) -> Result<()> {
635        self.repartition_gc_requirement_manager
636            .ensure_gc_enabled(self.options.gc.enable, &self.procedure_manager)
637            .await
638    }
639
640    pub async fn try_start(&self) -> Result<()> {
641        self.ensure_repartition_gc_enabled().await?;
642        self.try_start_after_gc_check().await
643    }
644
645    pub(crate) async fn try_start_after_gc_check(&self) -> Result<()> {
646        if self
647            .started
648            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
649            .is_err()
650        {
651            warn!("Metasrv already started");
652            return Ok(());
653        }
654
655        let handler_group_builder =
656            self.handler_group_builder
657                .lock()
658                .unwrap()
659                .take()
660                .context(error::UnexpectedSnafu {
661                    violated: "expected heartbeat handler group builder",
662                })?;
663        *self.handler_group.write().unwrap() = Some(Arc::new(handler_group_builder.build()?));
664
665        // Creates default schema if not exists
666        self.table_metadata_manager
667            .init()
668            .await
669            .context(InitMetadataSnafu)?;
670
671        if let Some(election) = self.election() {
672            let procedure_manager = self.procedure_manager.clone();
673            let in_memory = self.in_memory.clone();
674            let leader_cached_kv_backend = self.leader_cached_kv_backend.clone();
675            let subscribe_manager = self.subscription_manager();
676            let mut rx = election.subscribe_leader_change();
677            let greptimedb_telemetry_task = self.greptimedb_telemetry_task.clone();
678            greptimedb_telemetry_task
679                .start()
680                .context(StartTelemetryTaskSnafu)?;
681
682            // Builds leadership change notifier.
683            let mut leadership_change_notifier = LeadershipChangeNotifier::default();
684            leadership_change_notifier.add_listener(self.wal_provider.clone());
685            leadership_change_notifier
686                .add_listener(Arc::new(ProcedureManagerListenerAdapter(procedure_manager)));
687            leadership_change_notifier.add_listener(Arc::new(NodeExpiryListener::new(
688                self.options.node_max_idle_time,
689                self.in_memory.clone(),
690            )));
691            if let Some(region_supervisor_ticker) = &self.region_supervisor_ticker {
692                leadership_change_notifier.add_listener(region_supervisor_ticker.clone() as _);
693            }
694            if let Some(wal_prune_ticker) = &self.wal_prune_ticker {
695                leadership_change_notifier.add_listener(wal_prune_ticker.clone() as _);
696            }
697            if let Some(region_flush_trigger) = &self.region_flush_ticker {
698                leadership_change_notifier.add_listener(region_flush_trigger.clone() as _);
699            }
700            if let Some(gc_ticker) = &self.gc_ticker {
701                leadership_change_notifier.add_listener(gc_ticker.clone() as _);
702            }
703            if let Some(customizer) = self.plugins.get::<LeadershipChangeNotifierCustomizerRef>() {
704                customizer.customize(&mut leadership_change_notifier);
705            }
706
707            let state_handler = MetaStateHandler {
708                greptimedb_telemetry_task,
709                subscribe_manager,
710                state: self.state.clone(),
711                leader_cached_kv_backend: leader_cached_kv_backend.clone(),
712                leadership_change_notifier,
713            };
714            let _handle = common_runtime::spawn_global(async move {
715                loop {
716                    match rx.recv().await {
717                        Ok(msg) => {
718                            in_memory.reset();
719                            leader_cached_kv_backend.reset();
720                            info!("Leader's cache has bean cleared on leader change: {msg}");
721                            match msg {
722                                LeaderChangeMessage::Elected(_) => {
723                                    state_handler.on_leader_start().await;
724                                }
725                                LeaderChangeMessage::StepDown(leader) => {
726                                    error!("Leader :{:?} step down", leader);
727
728                                    state_handler.on_leader_stop().await;
729                                }
730                            }
731                        }
732                        Err(RecvError::Closed) => {
733                            error!("Not expected, is leader election loop still running?");
734                            break;
735                        }
736                        Err(RecvError::Lagged(_)) => {
737                            break;
738                        }
739                    }
740                }
741
742                state_handler.on_leader_stop().await;
743            });
744
745            // Register candidate and keep lease in background.
746            {
747                let election = election.clone();
748                let started = self.started.clone();
749                let node_info = self.node_info();
750                let _handle = common_runtime::spawn_global(async move {
751                    while started.load(Ordering::Acquire) {
752                        let res = election.register_candidate(&node_info).await;
753                        if let Err(e) = res {
754                            warn!(e; "Metasrv register candidate error");
755                        }
756                    }
757                });
758            }
759
760            // Campaign
761            {
762                let election = election.clone();
763                let started = self.started.clone();
764                let _handle = common_runtime::spawn_global(async move {
765                    while started.load(Ordering::Acquire) {
766                        let res = election.campaign().await;
767                        if let Err(e) = res {
768                            warn!(e; "Metasrv election error");
769                        }
770                        election.reset_campaign().await;
771                        info!("Metasrv re-initiate election");
772                    }
773                    info!("Metasrv stopped");
774                });
775            }
776        } else {
777            warn!(
778                "Ensure only one instance of Metasrv is running, as there is no election service."
779            );
780
781            if let Err(e) = self.wal_provider.start().await {
782                error!(e; "Failed to start wal provider");
783            }
784            // Always load kv into cached kv store.
785            self.leader_cached_kv_backend
786                .load()
787                .await
788                .context(KvBackendSnafu)?;
789            self.procedure_manager
790                .start()
791                .await
792                .context(StartProcedureManagerSnafu)?;
793        }
794
795        info!("Metasrv started");
796
797        Ok(())
798    }
799
800    pub async fn shutdown(&self) -> Result<()> {
801        if self
802            .started
803            .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire)
804            .is_err()
805        {
806            warn!("Metasrv already stopped");
807            return Ok(());
808        }
809
810        self.procedure_manager
811            .stop()
812            .await
813            .context(StopProcedureManagerSnafu)?;
814
815        info!("Metasrv stopped");
816
817        Ok(())
818    }
819
820    pub fn start_time_ms(&self) -> u64 {
821        self.start_time_ms
822    }
823
824    pub fn resource_stat(&self) -> &ResourceStatRef {
825        &self.resource_stat
826    }
827
828    pub fn node_info(&self) -> MetasrvNodeInfo {
829        let build_info = common_version::build_info();
830        MetasrvNodeInfo {
831            addr: self.options().grpc.server_addr.clone(),
832            version: build_info.version.to_string(),
833            git_commit: build_info.commit_short.to_string(),
834            start_time_ms: self.start_time_ms(),
835            total_cpu_millicores: self.resource_stat.get_total_cpu_millicores(),
836            total_memory_bytes: self.resource_stat.get_total_memory_bytes(),
837            cpu_usage_millicores: self.resource_stat.get_cpu_usage_millicores(),
838            memory_usage_bytes: self.resource_stat.get_memory_usage_bytes(),
839            hostname: hostname::get()
840                .unwrap_or_default()
841                .to_string_lossy()
842                .to_string(),
843        }
844    }
845
846    /// Looks up a datanode peer by peer_id, returning it only when it's alive.
847    /// A datanode is considered alive when it's still within the lease period.
848    pub(crate) async fn lookup_datanode_peer(&self, peer_id: u64) -> Result<Option<Peer>> {
849        discovery::utils::alive_datanode(
850            &DefaultSystemTimer,
851            self.meta_peer_client.as_ref(),
852            peer_id,
853            default_distributed_time_constants().datanode_lease,
854        )
855        .await
856    }
857
858    pub fn options(&self) -> &MetasrvOptions {
859        &self.options
860    }
861
862    pub fn in_memory(&self) -> &ResettableKvBackendRef {
863        &self.in_memory
864    }
865
866    pub fn kv_backend(&self) -> &KvBackendRef {
867        &self.kv_backend
868    }
869
870    /// Returns the shared KV backend that caches metadata while the leader cache is enabled.
871    pub fn leader_cached_kv_backend(&self) -> &Arc<LeaderCachedKvBackend> {
872        &self.leader_cached_kv_backend
873    }
874
875    /// Returns the undecorated peer client.
876    ///
877    /// Its `active_datanodes` does not know about node scheduling states. Use
878    /// [`Metasrv::selector_ctx`] for any consumer that places regions or picks datanodes.
879    pub fn meta_peer_client(&self) -> &MetaPeerClientRef {
880        &self.meta_peer_client
881    }
882
883    pub fn selector(&self) -> &SelectorRef {
884        &self.selector
885    }
886
887    /// Returns the selector context, whose `peer_discovery` stamps node scheduling states.
888    pub fn selector_ctx(&self) -> &SelectorContext {
889        &self.selector_ctx
890    }
891
892    pub fn flow_selector(&self) -> &SelectorRef {
893        &self.flow_selector
894    }
895
896    pub fn handler_group(&self) -> Option<HeartbeatHandlerGroupRef> {
897        self.handler_group.read().unwrap().clone()
898    }
899
900    pub fn election(&self) -> Option<&ElectionRef> {
901        self.election.as_ref()
902    }
903
904    pub fn mailbox(&self) -> &MailboxRef {
905        &self.mailbox
906    }
907
908    pub fn ddl_manager(&self) -> &DdlManagerRef {
909        &self.ddl_manager
910    }
911
912    pub fn procedure_manager(&self) -> &ProcedureManagerRef {
913        &self.procedure_manager
914    }
915
916    pub fn table_metadata_manager(&self) -> &TableMetadataManagerRef {
917        &self.table_metadata_manager
918    }
919
920    pub fn runtime_switch_manager(&self) -> &RuntimeSwitchManagerRef {
921        &self.runtime_switch_manager
922    }
923
924    pub fn memory_region_keeper(&self) -> &MemoryRegionKeeperRef {
925        &self.memory_region_keeper
926    }
927
928    pub fn region_migration_manager(&self) -> &RegionMigrationManagerRef {
929        &self.region_migration_manager
930    }
931
932    /// Returns the [`DrainDatanodeManagerRef`].
933    pub fn drain_datanode_manager(&self) -> &DrainDatanodeManagerRef {
934        &self.drain_datanode_manager
935    }
936
937    pub fn publish(&self) -> Option<PublisherRef> {
938        self.plugins.get::<PublisherRef>()
939    }
940
941    pub fn subscription_manager(&self) -> Option<SubscriptionManagerRef> {
942        self.plugins.get::<SubscriptionManagerRef>()
943    }
944
945    pub fn table_id_allocator(&self) -> &ResourceIdAllocatorRef {
946        &self.table_id_allocator
947    }
948
949    pub fn reconciliation_manager(&self) -> &ReconciliationManagerRef {
950        &self.reconciliation_manager
951    }
952
953    pub fn database_operator(&self) -> &DatabaseOperatorRef {
954        &self.database_operator
955    }
956
957    pub fn node_scheduling_manager(&self) -> &NodeSchedulingManagerRef {
958        &self.node_scheduling_manager
959    }
960
961    pub fn plugins(&self) -> &Plugins {
962        &self.plugins
963    }
964
965    pub fn started(&self) -> Arc<AtomicBool> {
966        self.started.clone()
967    }
968
969    pub fn gc_ticker(&self) -> Option<GcTickerRef> {
970        self.gc_ticker.as_ref().cloned()
971    }
972
973    #[inline]
974    pub fn new_ctx(&self) -> Context {
975        let server_addr = self.options().grpc.server_addr.clone();
976        let in_memory = self.in_memory.clone();
977        let kv_backend = self.kv_backend.clone();
978        let leader_cached_kv_backend = self.leader_cached_kv_backend.clone();
979        let meta_peer_client = self.meta_peer_client.clone();
980        let mailbox = self.mailbox.clone();
981        let election = self.election.clone();
982        let table_metadata_manager = self.table_metadata_manager.clone();
983        let cache_invalidator = self.cache_invalidator.clone();
984        let leader_region_registry = self.leader_region_registry.clone();
985        let topic_stats_registry = self.topic_stats_registry.clone();
986
987        Context {
988            server_addr,
989            in_memory,
990            kv_backend,
991            leader_cached_kv_backend,
992            meta_peer_client,
993            mailbox,
994            election,
995            is_infancy: false,
996            table_metadata_manager,
997            cache_invalidator,
998            leader_region_registry,
999            topic_stats_registry,
1000            heartbeat_interval: self.options().heartbeat_interval,
1001            gc_enabled: self.options().gc.enable,
1002            is_handshake: false,
1003        }
1004    }
1005}
1006
1007#[cfg(test)]
1008mod tests {
1009    use common_event_recorder::EventTypeFilter;
1010
1011    use super::*;
1012    use crate::metasrv::MetasrvNodeInfo;
1013
1014    #[test]
1015    fn test_store_addresses_are_redacted() {
1016        let address = "mysql://alice:store-secret@localhost/database?password=query-secret";
1017        let options: MetasrvOptions =
1018            toml::from_str(&format!("store_addrs = ['{address}']")).unwrap();
1019        assert_eq!(options.store_addrs, [address]);
1020        for output in [options.to_toml().unwrap(), format!("{options:#?}")] {
1021            assert!(!output.contains("store-secret"));
1022            assert!(!output.contains("query-secret"));
1023            assert!(output.contains("alice"));
1024        }
1025    }
1026
1027    #[test]
1028    fn test_deserialize_metasrv_node_info() {
1029        let str = r#"{"addr":"127.0.0.1:4002","version":"0.1.0","git_commit":"1234567890","start_time_ms":1715145600}"#;
1030        let node_info: MetasrvNodeInfo = serde_json::from_str(str).unwrap();
1031        assert_eq!(node_info.addr, "127.0.0.1:4002");
1032        assert_eq!(node_info.version, "0.1.0");
1033        assert_eq!(node_info.git_commit, "1234567890");
1034        assert_eq!(node_info.start_time_ms, 1715145600);
1035    }
1036
1037    #[test]
1038    fn test_metasrv_event_recorder_options_preserve_event_type_filter_semantics() {
1039        let all = MetasrvOptions::default().event_recorder;
1040        let none: EventRecorderOptions = toml::from_str("ttl = '90d'\nevent_types = []").unwrap();
1041        let selected: EventRecorderOptions =
1042            toml::from_str("ttl = '90d'\nevent_types = ['create_database']").unwrap();
1043
1044        assert!(all.event_types.allows("future_event"));
1045        assert_eq!(
1046            none.event_types.as_ref(),
1047            &EventTypeFilter::Only(Default::default())
1048        );
1049        assert!(selected.event_types.allows("create_database"));
1050        assert!(!selected.event_types.allows("drop_database"));
1051    }
1052}