1pub mod builder;
16
17use std::fmt::{self, Display};
18use std::sync::atomic::{AtomicBool, Ordering};
19use std::sync::{Arc, Mutex, RwLock};
20use std::time::Duration;
21
22use api::v1::meta::{HeartbeatConfig, Role};
23use clap::ValueEnum;
24use common_base::Plugins;
25use common_base::readable_size::ReadableSize;
26use common_config::{Configurable, DEFAULT_DATA_HOME};
27use common_event_recorder::EventRecorderOptions;
28use common_greptimedb_telemetry::GreptimeDBTelemetryTask;
29use common_meta::affinity::{AffinityFallback, AffinityRef, AffinityResolverRef};
30use common_meta::cache_invalidator::CacheInvalidatorRef;
31use common_meta::ddl::allocator::resource_id::ResourceIdAllocatorRef;
32use common_meta::ddl_manager::DdlManagerRef;
33use common_meta::distributed_time_constants::{
34 self, BASE_HEARTBEAT_INTERVAL, default_distributed_time_constants, frontend_heartbeat_interval,
35};
36use common_meta::election::LeaderChangeMessage;
37pub use common_meta::election::{ElectionRef, MetasrvNodeInfo};
38use common_meta::key::TableMetadataManagerRef;
39use common_meta::key::runtime_switch::RuntimeSwitchManagerRef;
40use common_meta::kv_backend::{KvBackendRef, ResettableKvBackend, ResettableKvBackendRef};
41use common_meta::leadership_notifier::{
42 LeadershipChangeNotifier, LeadershipChangeNotifierCustomizerRef,
43};
44use common_meta::node_expiry_listener::NodeExpiryListener;
45use common_meta::peer::{Peer, PeerDiscoveryRef};
46use common_meta::reconciliation::manager::ReconciliationManagerRef;
47use common_meta::region_keeper::MemoryRegionKeeperRef;
48use common_meta::region_registry::LeaderRegionRegistryRef;
49use common_meta::stats::topic::TopicStatsRegistryRef;
50use common_meta::wal_provider::WalProviderRef;
51use common_options::datanode::DatanodeClientOptions;
52use common_options::memory::MemoryOptions;
53use common_procedure::ProcedureManagerRef;
54use common_procedure::options::ProcedureConfig;
55use common_stat::ResourceStatRef;
56use common_telemetry::logging::{LoggingOptions, TracingOptions};
57use common_telemetry::{error, info, warn};
58use common_time::util::DefaultSystemTimer;
59use common_wal::config::MetasrvWalConfig;
60use serde::{Deserialize, Serialize};
61use servers::grpc::GrpcOptions;
62use servers::http::HttpOptions;
63use servers::tls::TlsOption;
64use snafu::{OptionExt, ResultExt};
65use store_api::storage::RegionId;
66use tokio::sync::broadcast::error::RecvError;
67
68use crate::cluster::MetaPeerClientRef;
69use crate::discovery;
70use crate::discovery::node_info::NodeInfoAccessor;
71use crate::error::{
72 self, InitMetadataSnafu, KvBackendSnafu, Result, StartProcedureManagerSnafu,
73 StartTelemetryTaskSnafu, StopProcedureManagerSnafu,
74};
75use crate::failure_detector::PhiAccrualFailureDetectorOptions;
76use crate::gc::{GcSchedulerOptions, GcTickerRef};
77use crate::handler::{HeartbeatHandlerGroupBuilder, HeartbeatHandlerGroupRef};
78use crate::node_scheduling::NodeSchedulingManagerRef;
79use crate::procedure::ProcedureManagerListenerAdapter;
80use crate::procedure::drain_datanode::manager::DrainDatanodeManagerRef;
81use crate::procedure::region_migration::manager::RegionMigrationManagerRef;
82use crate::procedure::repartition::gc_requirement::RepartitionGcRequirementManagerRef;
83use crate::procedure::wal_prune::manager::WalPruneTickerRef;
84use crate::pubsub::{PublisherRef, SubscriptionManagerRef};
85use crate::region::flush_trigger::RegionFlushTickerRef;
86use crate::region::supervisor::RegionSupervisorTickerRef;
87use crate::selector::{RegionStatAwareSelector, Selector, SelectorType};
88use crate::service::mailbox::MailboxRef;
89use crate::service::store::cached_kv::LeaderCachedKvBackend;
90use crate::state::{StateRef, become_follower, become_leader};
91use crate::utils::database::DatabaseOperatorRef;
92
93pub const TABLE_ID_SEQ: &str = "table_id";
94pub const FLOW_ID_SEQ: &str = "flow_id";
95pub const METASRV_DATA_DIR: &str = "metasrv";
96
97#[derive(Clone, Debug, PartialEq, Serialize, Default, Deserialize, ValueEnum)]
99#[serde(rename_all = "snake_case")]
100pub enum BackendImpl {
101 #[default]
103 EtcdStore,
104 MemoryStore,
106 #[cfg(feature = "pg_kvbackend")]
107 PostgresStore,
109 #[cfg(feature = "mysql_kvbackend")]
110 MysqlStore,
112}
113
114#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
116pub struct StatsPersistenceOptions {
117 #[serde(with = "humantime_serde")]
119 pub ttl: Duration,
120 #[serde(with = "humantime_serde")]
122 pub interval: Duration,
123}
124
125impl Default for StatsPersistenceOptions {
126 fn default() -> Self {
127 Self {
128 ttl: Duration::ZERO,
129 interval: Duration::from_secs(10 * 60),
130 }
131 }
132}
133
134#[derive(Clone, PartialEq, Serialize, Deserialize, Debug)]
136#[serde(default)]
137pub struct HeartbeatOptions {
138 #[serde(with = "humantime_serde")]
140 pub interval: Duration,
141 #[serde(with = "humantime_serde")]
143 pub retry_interval: Duration,
144}
145
146impl Default for HeartbeatOptions {
147 fn default() -> Self {
148 Self {
149 interval: BASE_HEARTBEAT_INTERVAL,
150 retry_interval: BASE_HEARTBEAT_INTERVAL,
151 }
152 }
153}
154
155impl HeartbeatOptions {
156 pub fn datanode_from(base_interval: Duration) -> Self {
157 Self {
158 interval: base_interval,
159 retry_interval: base_interval,
160 }
161 }
162
163 pub fn frontend_from(base_interval: Duration) -> Self {
164 Self {
165 interval: frontend_heartbeat_interval(base_interval),
166 retry_interval: base_interval,
167 }
168 }
169
170 pub fn flownode_from(base_interval: Duration) -> Self {
171 Self {
172 interval: base_interval,
173 retry_interval: base_interval,
174 }
175 }
176}
177
178impl From<HeartbeatOptions> for HeartbeatConfig {
179 fn from(opts: HeartbeatOptions) -> Self {
180 Self {
181 heartbeat_interval_ms: opts.interval.as_millis() as u64,
182 retry_interval_ms: opts.retry_interval.as_millis() as u64,
183 gc_enabled: false,
184 }
185 }
186}
187
188#[derive(Clone, PartialEq, Serialize, Deserialize, Debug)]
189#[serde(default)]
190pub struct BackendClientOptions {
191 #[serde(with = "humantime_serde")]
192 pub keep_alive_timeout: Duration,
193 #[serde(with = "humantime_serde")]
194 pub keep_alive_interval: Duration,
195 #[serde(with = "humantime_serde")]
196 pub connect_timeout: Duration,
197}
198
199impl Default for BackendClientOptions {
200 fn default() -> Self {
201 Self {
202 keep_alive_interval: Duration::from_secs(10),
203 keep_alive_timeout: Duration::from_secs(3),
204 connect_timeout: Duration::from_secs(3),
205 }
206 }
207}
208
209#[derive(Clone, PartialEq, Serialize, Deserialize)]
210#[serde(default)]
211pub struct MetasrvOptions {
212 #[deprecated(note = "Use grpc.bind_addr instead")]
214 pub bind_addr: String,
215 #[deprecated(note = "Use grpc.server_addr instead")]
217 pub server_addr: String,
218 #[serde(serialize_with = "serialize_store_addrs")]
220 pub store_addrs: Vec<String>,
221 #[serde(default)]
224 pub backend_tls: Option<TlsOption>,
225 #[serde(default)]
228 pub backend_client: BackendClientOptions,
229 pub selector: SelectorType,
231 pub enable_region_failover: bool,
233 #[serde(with = "humantime_serde")]
238 pub heartbeat_interval: Duration,
239 #[serde(with = "humantime_serde")]
243 pub region_failure_detector_initialization_delay: Duration,
244 pub allow_region_failover_on_local_wal: bool,
249 pub grpc: GrpcOptions,
250 pub http: HttpOptions,
252 pub logging: LoggingOptions,
254 pub procedure: ProcedureConfig,
256 pub failure_detector: PhiAccrualFailureDetectorOptions,
258 pub datanode: DatanodeClientOptions,
260 pub enable_telemetry: bool,
262 pub data_home: String,
264 pub wal: MetasrvWalConfig,
266 pub store_key_prefix: String,
269 pub max_txn_ops: usize,
280 pub flush_stats_factor: usize,
284 pub tracing: TracingOptions,
286 pub memory: MemoryOptions,
288 pub backend: BackendImpl,
290 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
291 pub meta_table_name: String,
293 #[cfg(feature = "pg_kvbackend")]
294 pub meta_election_lock_id: u64,
296 #[cfg(feature = "pg_kvbackend")]
297 pub meta_schema_name: Option<String>,
299 #[cfg(feature = "pg_kvbackend")]
300 pub auto_create_schema: bool,
302 #[serde(with = "humantime_serde")]
303 pub node_max_idle_time: Duration,
304 pub event_recorder: EventRecorderOptions,
306 pub stats_persistence: StatsPersistenceOptions,
308 pub gc: GcSchedulerOptions,
310}
311
312impl fmt::Debug for MetasrvOptions {
313 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
314 let mut debug_struct = f.debug_struct("MetasrvOptions");
315 debug_struct
316 .field("store_addrs", &sanitize_store_addrs(&self.store_addrs))
317 .field("backend_tls", &self.backend_tls)
318 .field("selector", &self.selector)
319 .field("enable_region_failover", &self.enable_region_failover)
320 .field(
321 "allow_region_failover_on_local_wal",
322 &self.allow_region_failover_on_local_wal,
323 )
324 .field("grpc", &self.grpc)
325 .field("http", &self.http)
326 .field("logging", &self.logging)
327 .field("procedure", &self.procedure)
328 .field("failure_detector", &self.failure_detector)
329 .field("datanode", &self.datanode)
330 .field("enable_telemetry", &self.enable_telemetry)
331 .field("data_home", &self.data_home)
332 .field("wal", &self.wal)
333 .field("store_key_prefix", &self.store_key_prefix)
334 .field("max_txn_ops", &self.max_txn_ops)
335 .field("flush_stats_factor", &self.flush_stats_factor)
336 .field("tracing", &self.tracing)
337 .field("backend", &self.backend)
338 .field("event_recorder", &self.event_recorder)
339 .field("stats_persistence", &self.stats_persistence)
340 .field("heartbeat_interval", &self.heartbeat_interval)
341 .field("backend_client", &self.backend_client);
342
343 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
344 debug_struct.field("meta_table_name", &self.meta_table_name);
345
346 #[cfg(feature = "pg_kvbackend")]
347 debug_struct.field("meta_election_lock_id", &self.meta_election_lock_id);
348 #[cfg(feature = "pg_kvbackend")]
349 debug_struct.field("meta_schema_name", &self.meta_schema_name);
350
351 debug_struct
352 .field("node_max_idle_time", &self.node_max_idle_time)
353 .finish()
354 }
355}
356
357const DEFAULT_METASRV_ADDR_PORT: &str = "3002";
358
359impl Default for MetasrvOptions {
360 fn default() -> Self {
361 Self {
362 #[allow(deprecated)]
363 bind_addr: String::new(),
364 #[allow(deprecated)]
365 server_addr: String::new(),
366 store_addrs: vec!["127.0.0.1:2379".to_string()],
367 backend_tls: Some(TlsOption::prefer()),
368 selector: SelectorType::default(),
369 enable_region_failover: false,
370 heartbeat_interval: distributed_time_constants::BASE_HEARTBEAT_INTERVAL,
371 region_failure_detector_initialization_delay: Duration::from_secs(10 * 60),
372 allow_region_failover_on_local_wal: false,
373 grpc: GrpcOptions {
374 bind_addr: format!("127.0.0.1:{}", DEFAULT_METASRV_ADDR_PORT),
375 ..Default::default()
376 },
377 http: HttpOptions::default(),
378 logging: LoggingOptions::default(),
379 procedure: ProcedureConfig {
380 max_retry_times: 12,
381 retry_delay: Duration::from_millis(500),
382 max_metadata_value_size: Some(ReadableSize::kb(1500)),
385 max_running_procedures: 128,
386 },
387 failure_detector: PhiAccrualFailureDetectorOptions::default(),
388 datanode: DatanodeClientOptions::default(),
389 enable_telemetry: true,
390 data_home: DEFAULT_DATA_HOME.to_string(),
391 wal: MetasrvWalConfig::default(),
392 store_key_prefix: String::new(),
393 max_txn_ops: 128,
394 flush_stats_factor: 3,
395 tracing: TracingOptions::default(),
396 memory: MemoryOptions::default(),
397 backend: BackendImpl::EtcdStore,
398 #[cfg(any(feature = "pg_kvbackend", feature = "mysql_kvbackend"))]
399 meta_table_name: common_meta::kv_backend::DEFAULT_META_TABLE_NAME.to_string(),
400 #[cfg(feature = "pg_kvbackend")]
401 meta_election_lock_id: common_meta::kv_backend::DEFAULT_META_ELECTION_LOCK_ID,
402 #[cfg(feature = "pg_kvbackend")]
403 meta_schema_name: None,
404 #[cfg(feature = "pg_kvbackend")]
405 auto_create_schema: true,
406 node_max_idle_time: Duration::from_secs(24 * 60 * 60),
407 event_recorder: EventRecorderOptions::default(),
408 stats_persistence: StatsPersistenceOptions::default(),
409 gc: GcSchedulerOptions::default(),
410 backend_client: BackendClientOptions::default(),
411 }
412 }
413}
414
415impl Configurable for MetasrvOptions {
416 fn env_list_keys() -> Option<&'static [&'static str]> {
417 Some(&[
418 "wal.broker_endpoints",
419 "store_addrs",
420 "event_recorder.event_types",
421 ])
422 }
423}
424
425fn sanitize_store_addrs(addrs: &[String]) -> Vec<String> {
426 addrs
427 .iter()
428 .map(|addr| common_meta::kv_backend::util::sanitize_connection_string(addr))
429 .collect()
430}
431
432fn serialize_store_addrs<S>(addrs: &[String], serializer: S) -> std::result::Result<S::Ok, S::Error>
433where
434 S: serde::Serializer,
435{
436 sanitize_store_addrs(addrs).serialize(serializer)
437}
438
439pub struct MetasrvInfo {
440 pub server_addr: String,
441}
442#[derive(Clone)]
443pub struct Context {
444 pub server_addr: String,
445 pub in_memory: ResettableKvBackendRef,
446 pub kv_backend: KvBackendRef,
447 pub leader_cached_kv_backend: ResettableKvBackendRef,
448 pub meta_peer_client: MetaPeerClientRef,
449 pub mailbox: MailboxRef,
450 pub election: Option<ElectionRef>,
451 pub is_infancy: bool,
452 pub table_metadata_manager: TableMetadataManagerRef,
453 pub cache_invalidator: CacheInvalidatorRef,
454 pub leader_region_registry: LeaderRegionRegistryRef,
455 pub topic_stats_registry: TopicStatsRegistryRef,
456 pub heartbeat_interval: Duration,
457 pub gc_enabled: bool,
458 pub is_handshake: bool,
459}
460
461impl Context {
462 pub fn reset_in_memory(&self) {
463 self.in_memory.reset();
464 self.leader_region_registry.reset();
465 }
466
467 pub fn with_handshake(mut self, is_handshake: bool) -> Self {
468 self.is_handshake = is_handshake;
469 self
470 }
471
472 pub fn heartbeat_options_for(&self, role: Role) -> HeartbeatOptions {
473 match role {
474 Role::Datanode => HeartbeatOptions::datanode_from(self.heartbeat_interval),
475 Role::Frontend => HeartbeatOptions::frontend_from(self.heartbeat_interval),
476 Role::Flownode => HeartbeatOptions::flownode_from(self.heartbeat_interval),
477 }
478 }
479}
480
481#[derive(Clone, Copy)]
482pub enum SelectTarget {
483 Datanode,
484 Flownode,
485}
486
487impl Display for SelectTarget {
488 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
489 match self {
490 SelectTarget::Datanode => write!(f, "datanode"),
491 SelectTarget::Flownode => write!(f, "flownode"),
492 }
493 }
494}
495
496#[derive(Clone)]
497pub struct SelectorContext {
498 pub peer_discovery: PeerDiscoveryRef,
499 pub affinity: AffinityRef,
501 pub affinity_fallback: AffinityFallback,
503}
504
505pub type SelectorRef = Arc<dyn Selector<Context = SelectorContext, Output = Vec<Peer>>>;
506
507pub struct SelectorFactoryContext {
514 pub metasrv_options: MetasrvOptions,
515 pub meta_peer_client: MetaPeerClientRef,
516 pub in_memory: ResettableKvBackendRef,
517 pub election: Option<ElectionRef>,
518 pub base_selector: SelectorRef,
519}
520
521pub trait SelectorFactory: Send + Sync {
523 fn build(&self, ctx: SelectorFactoryContext) -> SelectorRef;
524
525 fn build_affinity_resolver(
530 &self,
531 _node_info_accessor: Arc<dyn NodeInfoAccessor>,
532 _peer_discovery: PeerDiscoveryRef,
533 ) -> Option<AffinityResolverRef> {
534 None
535 }
536}
537
538pub type SelectorFactoryRef = Arc<dyn SelectorFactory>;
540pub type RegionStatAwareSelectorRef =
541 Arc<dyn RegionStatAwareSelector<Context = SelectorContext, Output = Vec<(RegionId, Peer)>>>;
542
543pub struct MetaStateHandler {
544 subscribe_manager: Option<SubscriptionManagerRef>,
545 greptimedb_telemetry_task: Arc<GreptimeDBTelemetryTask>,
546 leader_cached_kv_backend: Arc<LeaderCachedKvBackend>,
547 leadership_change_notifier: LeadershipChangeNotifier,
548 state: StateRef,
549}
550
551impl MetaStateHandler {
552 pub async fn on_leader_start(&self) {
553 self.state.write().unwrap().next_state(become_leader(false));
554
555 if let Err(e) = self.leader_cached_kv_backend.load().await {
556 error!(e; "Failed to load kv into leader cache kv store");
557 } else {
558 self.state.write().unwrap().next_state(become_leader(true));
559 }
560
561 self.leadership_change_notifier
562 .notify_on_leader_start()
563 .await;
564
565 self.greptimedb_telemetry_task.should_report(true);
566 }
567
568 pub async fn on_leader_stop(&self) {
569 self.state.write().unwrap().next_state(become_follower());
570
571 self.leadership_change_notifier
572 .notify_on_leader_stop()
573 .await;
574
575 self.greptimedb_telemetry_task.should_report(false);
577
578 if let Some(sub_manager) = self.subscribe_manager.clone() {
579 info!("Leader changed, un_subscribe all");
580 if let Err(e) = sub_manager.unsubscribe_all() {
581 error!(e; "Failed to un_subscribe all");
582 }
583 }
584 }
585}
586
587pub struct Metasrv {
588 state: StateRef,
589 started: Arc<AtomicBool>,
590 start_time_ms: u64,
591 options: MetasrvOptions,
592 in_memory: ResettableKvBackendRef,
595 kv_backend: KvBackendRef,
596 leader_cached_kv_backend: Arc<LeaderCachedKvBackend>,
597 meta_peer_client: MetaPeerClientRef,
598 selector: SelectorRef,
600 selector_ctx: SelectorContext,
601 flow_selector: SelectorRef,
603 handler_group: RwLock<Option<HeartbeatHandlerGroupRef>>,
604 handler_group_builder: Mutex<Option<HeartbeatHandlerGroupBuilder>>,
605 election: Option<ElectionRef>,
606 procedure_manager: ProcedureManagerRef,
607 mailbox: MailboxRef,
608 ddl_manager: DdlManagerRef,
609 wal_provider: WalProviderRef,
610 table_metadata_manager: TableMetadataManagerRef,
611 runtime_switch_manager: RuntimeSwitchManagerRef,
612 repartition_gc_requirement_manager: RepartitionGcRequirementManagerRef,
613 memory_region_keeper: MemoryRegionKeeperRef,
614 greptimedb_telemetry_task: Arc<GreptimeDBTelemetryTask>,
615 region_migration_manager: RegionMigrationManagerRef,
616 drain_datanode_manager: DrainDatanodeManagerRef,
617 region_supervisor_ticker: Option<RegionSupervisorTickerRef>,
618 cache_invalidator: CacheInvalidatorRef,
619 leader_region_registry: LeaderRegionRegistryRef,
620 topic_stats_registry: TopicStatsRegistryRef,
621 wal_prune_ticker: Option<WalPruneTickerRef>,
622 region_flush_ticker: Option<RegionFlushTickerRef>,
623 table_id_allocator: ResourceIdAllocatorRef,
624 reconciliation_manager: ReconciliationManagerRef,
625 resource_stat: ResourceStatRef,
626 gc_ticker: Option<GcTickerRef>,
627 database_operator: DatabaseOperatorRef,
628 node_scheduling_manager: NodeSchedulingManagerRef,
629
630 plugins: Plugins,
631}
632
633impl Metasrv {
634 pub(crate) async fn ensure_repartition_gc_enabled(&self) -> Result<()> {
635 self.repartition_gc_requirement_manager
636 .ensure_gc_enabled(self.options.gc.enable, &self.procedure_manager)
637 .await
638 }
639
640 pub async fn try_start(&self) -> Result<()> {
641 self.ensure_repartition_gc_enabled().await?;
642 self.try_start_after_gc_check().await
643 }
644
645 pub(crate) async fn try_start_after_gc_check(&self) -> Result<()> {
646 if self
647 .started
648 .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
649 .is_err()
650 {
651 warn!("Metasrv already started");
652 return Ok(());
653 }
654
655 let handler_group_builder =
656 self.handler_group_builder
657 .lock()
658 .unwrap()
659 .take()
660 .context(error::UnexpectedSnafu {
661 violated: "expected heartbeat handler group builder",
662 })?;
663 *self.handler_group.write().unwrap() = Some(Arc::new(handler_group_builder.build()?));
664
665 self.table_metadata_manager
667 .init()
668 .await
669 .context(InitMetadataSnafu)?;
670
671 if let Some(election) = self.election() {
672 let procedure_manager = self.procedure_manager.clone();
673 let in_memory = self.in_memory.clone();
674 let leader_cached_kv_backend = self.leader_cached_kv_backend.clone();
675 let subscribe_manager = self.subscription_manager();
676 let mut rx = election.subscribe_leader_change();
677 let greptimedb_telemetry_task = self.greptimedb_telemetry_task.clone();
678 greptimedb_telemetry_task
679 .start()
680 .context(StartTelemetryTaskSnafu)?;
681
682 let mut leadership_change_notifier = LeadershipChangeNotifier::default();
684 leadership_change_notifier.add_listener(self.wal_provider.clone());
685 leadership_change_notifier
686 .add_listener(Arc::new(ProcedureManagerListenerAdapter(procedure_manager)));
687 leadership_change_notifier.add_listener(Arc::new(NodeExpiryListener::new(
688 self.options.node_max_idle_time,
689 self.in_memory.clone(),
690 )));
691 if let Some(region_supervisor_ticker) = &self.region_supervisor_ticker {
692 leadership_change_notifier.add_listener(region_supervisor_ticker.clone() as _);
693 }
694 if let Some(wal_prune_ticker) = &self.wal_prune_ticker {
695 leadership_change_notifier.add_listener(wal_prune_ticker.clone() as _);
696 }
697 if let Some(region_flush_trigger) = &self.region_flush_ticker {
698 leadership_change_notifier.add_listener(region_flush_trigger.clone() as _);
699 }
700 if let Some(gc_ticker) = &self.gc_ticker {
701 leadership_change_notifier.add_listener(gc_ticker.clone() as _);
702 }
703 if let Some(customizer) = self.plugins.get::<LeadershipChangeNotifierCustomizerRef>() {
704 customizer.customize(&mut leadership_change_notifier);
705 }
706
707 let state_handler = MetaStateHandler {
708 greptimedb_telemetry_task,
709 subscribe_manager,
710 state: self.state.clone(),
711 leader_cached_kv_backend: leader_cached_kv_backend.clone(),
712 leadership_change_notifier,
713 };
714 let _handle = common_runtime::spawn_global(async move {
715 loop {
716 match rx.recv().await {
717 Ok(msg) => {
718 in_memory.reset();
719 leader_cached_kv_backend.reset();
720 info!("Leader's cache has bean cleared on leader change: {msg}");
721 match msg {
722 LeaderChangeMessage::Elected(_) => {
723 state_handler.on_leader_start().await;
724 }
725 LeaderChangeMessage::StepDown(leader) => {
726 error!("Leader :{:?} step down", leader);
727
728 state_handler.on_leader_stop().await;
729 }
730 }
731 }
732 Err(RecvError::Closed) => {
733 error!("Not expected, is leader election loop still running?");
734 break;
735 }
736 Err(RecvError::Lagged(_)) => {
737 break;
738 }
739 }
740 }
741
742 state_handler.on_leader_stop().await;
743 });
744
745 {
747 let election = election.clone();
748 let started = self.started.clone();
749 let node_info = self.node_info();
750 let _handle = common_runtime::spawn_global(async move {
751 while started.load(Ordering::Acquire) {
752 let res = election.register_candidate(&node_info).await;
753 if let Err(e) = res {
754 warn!(e; "Metasrv register candidate error");
755 }
756 }
757 });
758 }
759
760 {
762 let election = election.clone();
763 let started = self.started.clone();
764 let _handle = common_runtime::spawn_global(async move {
765 while started.load(Ordering::Acquire) {
766 let res = election.campaign().await;
767 if let Err(e) = res {
768 warn!(e; "Metasrv election error");
769 }
770 election.reset_campaign().await;
771 info!("Metasrv re-initiate election");
772 }
773 info!("Metasrv stopped");
774 });
775 }
776 } else {
777 warn!(
778 "Ensure only one instance of Metasrv is running, as there is no election service."
779 );
780
781 if let Err(e) = self.wal_provider.start().await {
782 error!(e; "Failed to start wal provider");
783 }
784 self.leader_cached_kv_backend
786 .load()
787 .await
788 .context(KvBackendSnafu)?;
789 self.procedure_manager
790 .start()
791 .await
792 .context(StartProcedureManagerSnafu)?;
793 }
794
795 info!("Metasrv started");
796
797 Ok(())
798 }
799
800 pub async fn shutdown(&self) -> Result<()> {
801 if self
802 .started
803 .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire)
804 .is_err()
805 {
806 warn!("Metasrv already stopped");
807 return Ok(());
808 }
809
810 self.procedure_manager
811 .stop()
812 .await
813 .context(StopProcedureManagerSnafu)?;
814
815 info!("Metasrv stopped");
816
817 Ok(())
818 }
819
820 pub fn start_time_ms(&self) -> u64 {
821 self.start_time_ms
822 }
823
824 pub fn resource_stat(&self) -> &ResourceStatRef {
825 &self.resource_stat
826 }
827
828 pub fn node_info(&self) -> MetasrvNodeInfo {
829 let build_info = common_version::build_info();
830 MetasrvNodeInfo {
831 addr: self.options().grpc.server_addr.clone(),
832 version: build_info.version.to_string(),
833 git_commit: build_info.commit_short.to_string(),
834 start_time_ms: self.start_time_ms(),
835 total_cpu_millicores: self.resource_stat.get_total_cpu_millicores(),
836 total_memory_bytes: self.resource_stat.get_total_memory_bytes(),
837 cpu_usage_millicores: self.resource_stat.get_cpu_usage_millicores(),
838 memory_usage_bytes: self.resource_stat.get_memory_usage_bytes(),
839 hostname: hostname::get()
840 .unwrap_or_default()
841 .to_string_lossy()
842 .to_string(),
843 }
844 }
845
846 pub(crate) async fn lookup_datanode_peer(&self, peer_id: u64) -> Result<Option<Peer>> {
849 discovery::utils::alive_datanode(
850 &DefaultSystemTimer,
851 self.meta_peer_client.as_ref(),
852 peer_id,
853 default_distributed_time_constants().datanode_lease,
854 )
855 .await
856 }
857
858 pub fn options(&self) -> &MetasrvOptions {
859 &self.options
860 }
861
862 pub fn in_memory(&self) -> &ResettableKvBackendRef {
863 &self.in_memory
864 }
865
866 pub fn kv_backend(&self) -> &KvBackendRef {
867 &self.kv_backend
868 }
869
870 pub fn leader_cached_kv_backend(&self) -> &Arc<LeaderCachedKvBackend> {
872 &self.leader_cached_kv_backend
873 }
874
875 pub fn meta_peer_client(&self) -> &MetaPeerClientRef {
880 &self.meta_peer_client
881 }
882
883 pub fn selector(&self) -> &SelectorRef {
884 &self.selector
885 }
886
887 pub fn selector_ctx(&self) -> &SelectorContext {
889 &self.selector_ctx
890 }
891
892 pub fn flow_selector(&self) -> &SelectorRef {
893 &self.flow_selector
894 }
895
896 pub fn handler_group(&self) -> Option<HeartbeatHandlerGroupRef> {
897 self.handler_group.read().unwrap().clone()
898 }
899
900 pub fn election(&self) -> Option<&ElectionRef> {
901 self.election.as_ref()
902 }
903
904 pub fn mailbox(&self) -> &MailboxRef {
905 &self.mailbox
906 }
907
908 pub fn ddl_manager(&self) -> &DdlManagerRef {
909 &self.ddl_manager
910 }
911
912 pub fn procedure_manager(&self) -> &ProcedureManagerRef {
913 &self.procedure_manager
914 }
915
916 pub fn table_metadata_manager(&self) -> &TableMetadataManagerRef {
917 &self.table_metadata_manager
918 }
919
920 pub fn runtime_switch_manager(&self) -> &RuntimeSwitchManagerRef {
921 &self.runtime_switch_manager
922 }
923
924 pub fn memory_region_keeper(&self) -> &MemoryRegionKeeperRef {
925 &self.memory_region_keeper
926 }
927
928 pub fn region_migration_manager(&self) -> &RegionMigrationManagerRef {
929 &self.region_migration_manager
930 }
931
932 pub fn drain_datanode_manager(&self) -> &DrainDatanodeManagerRef {
934 &self.drain_datanode_manager
935 }
936
937 pub fn publish(&self) -> Option<PublisherRef> {
938 self.plugins.get::<PublisherRef>()
939 }
940
941 pub fn subscription_manager(&self) -> Option<SubscriptionManagerRef> {
942 self.plugins.get::<SubscriptionManagerRef>()
943 }
944
945 pub fn table_id_allocator(&self) -> &ResourceIdAllocatorRef {
946 &self.table_id_allocator
947 }
948
949 pub fn reconciliation_manager(&self) -> &ReconciliationManagerRef {
950 &self.reconciliation_manager
951 }
952
953 pub fn database_operator(&self) -> &DatabaseOperatorRef {
954 &self.database_operator
955 }
956
957 pub fn node_scheduling_manager(&self) -> &NodeSchedulingManagerRef {
958 &self.node_scheduling_manager
959 }
960
961 pub fn plugins(&self) -> &Plugins {
962 &self.plugins
963 }
964
965 pub fn started(&self) -> Arc<AtomicBool> {
966 self.started.clone()
967 }
968
969 pub fn gc_ticker(&self) -> Option<GcTickerRef> {
970 self.gc_ticker.as_ref().cloned()
971 }
972
973 #[inline]
974 pub fn new_ctx(&self) -> Context {
975 let server_addr = self.options().grpc.server_addr.clone();
976 let in_memory = self.in_memory.clone();
977 let kv_backend = self.kv_backend.clone();
978 let leader_cached_kv_backend = self.leader_cached_kv_backend.clone();
979 let meta_peer_client = self.meta_peer_client.clone();
980 let mailbox = self.mailbox.clone();
981 let election = self.election.clone();
982 let table_metadata_manager = self.table_metadata_manager.clone();
983 let cache_invalidator = self.cache_invalidator.clone();
984 let leader_region_registry = self.leader_region_registry.clone();
985 let topic_stats_registry = self.topic_stats_registry.clone();
986
987 Context {
988 server_addr,
989 in_memory,
990 kv_backend,
991 leader_cached_kv_backend,
992 meta_peer_client,
993 mailbox,
994 election,
995 is_infancy: false,
996 table_metadata_manager,
997 cache_invalidator,
998 leader_region_registry,
999 topic_stats_registry,
1000 heartbeat_interval: self.options().heartbeat_interval,
1001 gc_enabled: self.options().gc.enable,
1002 is_handshake: false,
1003 }
1004 }
1005}
1006
1007#[cfg(test)]
1008mod tests {
1009 use common_event_recorder::EventTypeFilter;
1010
1011 use super::*;
1012 use crate::metasrv::MetasrvNodeInfo;
1013
1014 #[test]
1015 fn test_store_addresses_are_redacted() {
1016 let address = "mysql://alice:store-secret@localhost/database?password=query-secret";
1017 let options: MetasrvOptions =
1018 toml::from_str(&format!("store_addrs = ['{address}']")).unwrap();
1019 assert_eq!(options.store_addrs, [address]);
1020 for output in [options.to_toml().unwrap(), format!("{options:#?}")] {
1021 assert!(!output.contains("store-secret"));
1022 assert!(!output.contains("query-secret"));
1023 assert!(output.contains("alice"));
1024 }
1025 }
1026
1027 #[test]
1028 fn test_deserialize_metasrv_node_info() {
1029 let str = r#"{"addr":"127.0.0.1:4002","version":"0.1.0","git_commit":"1234567890","start_time_ms":1715145600}"#;
1030 let node_info: MetasrvNodeInfo = serde_json::from_str(str).unwrap();
1031 assert_eq!(node_info.addr, "127.0.0.1:4002");
1032 assert_eq!(node_info.version, "0.1.0");
1033 assert_eq!(node_info.git_commit, "1234567890");
1034 assert_eq!(node_info.start_time_ms, 1715145600);
1035 }
1036
1037 #[test]
1038 fn test_metasrv_event_recorder_options_preserve_event_type_filter_semantics() {
1039 let all = MetasrvOptions::default().event_recorder;
1040 let none: EventRecorderOptions = toml::from_str("ttl = '90d'\nevent_types = []").unwrap();
1041 let selected: EventRecorderOptions =
1042 toml::from_str("ttl = '90d'\nevent_types = ['create_database']").unwrap();
1043
1044 assert!(all.event_types.allows("future_event"));
1045 assert_eq!(
1046 none.event_types.as_ref(),
1047 &EventTypeFilter::Only(Default::default())
1048 );
1049 assert!(selected.event_types.allows("create_database"));
1050 assert!(!selected.event_types.allows("drop_database"));
1051 }
1052}