1use std::path::Path;
16use std::sync::atomic::AtomicBool;
17use std::sync::{Arc, Mutex, RwLock};
18use std::time::Duration;
19
20use client::client_manager::NodeClients;
21use client::inserter::InsertOptions;
22use common_base::Plugins;
23use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
24use common_event_recorder::{DEFAULT_COMPACTION_TIME_WINDOW, EventRecorderImpl, EventRecorderRef};
25use common_meta::affinity::{AffinityFallback, AffinityRef, NoopAffinity};
26use common_meta::ddl::flow_meta::FlowMetadataAllocator;
27use common_meta::ddl::table_meta::{TableMetadataAllocator, TableMetadataAllocatorRef};
28use common_meta::ddl::{
29 DdlContext, NoopRegionFailureDetectorControl, RegionFailureDetectorControllerRef,
30};
31use common_meta::ddl_manager::{
32 DdlManager, DdlManagerConfiguratorRef, RepartitionProcedureFactoryRef,
33};
34use common_meta::distributed_time_constants::default_distributed_time_constants;
35use common_meta::key::TableMetadataManager;
36use common_meta::key::flow::FlowMetadataManager;
37use common_meta::key::flow::flow_state::FlowStateManager;
38use common_meta::key::runtime_switch::{RuntimeSwitchManager, RuntimeSwitchManagerRef};
39use common_meta::kv_backend::memory::MemoryKvBackend;
40use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
41use common_meta::node_manager::NodeManagerRef;
42use common_meta::peer::PeerDiscoveryRef;
43use common_meta::reconciliation::manager::ReconciliationManager;
44use common_meta::region_keeper::MemoryRegionKeeper;
45use common_meta::region_registry::LeaderRegionRegistry;
46use common_meta::sequence::SequenceBuilder;
47use common_meta::state_store::KvStateStore;
48use common_meta::stats::topic::TopicStatsRegistry;
49use common_meta::wal_provider::{WalProvider, build_kafka_client, build_wal_provider};
50use common_procedure::ProcedureManagerRef;
51use common_procedure::local::{LocalManager, ManagerConfig};
52use common_stat::ResourceStatImpl;
53use common_telemetry::{info, warn};
54use common_wal::config::object_store::EnableObjectStoreWal;
55use snafu::{ResultExt, ensure};
56use store_api::storage::MAX_REGION_SEQ;
57
58use crate::bootstrap::build_default_meta_peer_client;
59use crate::cache_invalidator::MetasrvCacheInvalidator;
60use crate::cluster::MetaPeerClientRef;
61use crate::discovery::node_scheduling::SchedulingAwarePeerDiscovery;
62use crate::error::{self, BuildWalProviderSnafu, OtherSnafu, Result};
63use crate::event::EventHandlerImpl;
64use crate::gc::{DefaultGcSchedulerCtx, GcScheduler};
65use crate::greptimedb_telemetry::get_greptimedb_telemetry_task;
66use crate::handler::failure_handler::RegionFailureHandler;
67use crate::handler::flow_state_handler::FlowStateHandler;
68use crate::handler::persist_stats_handler::PersistStatsHandler;
69use crate::handler::region_lease_handler::{CustomizedRegionLeaseRenewerRef, RegionLeaseHandler};
70use crate::handler::{HeartbeatHandlerGroupBuilder, HeartbeatMailbox, Pushers};
71use crate::metasrv::{
72 ElectionRef, FLOW_ID_SEQ, METASRV_DATA_DIR, Metasrv, MetasrvInfo, MetasrvOptions,
73 RegionStatAwareSelectorRef, SelectTarget, SelectorContext, SelectorFactoryRef, SelectorRef,
74 TABLE_ID_SEQ,
75};
76use crate::node_scheduling::NodeSchedulingManager;
77use crate::peer::MetasrvPeerAllocator;
78use crate::procedure::drain_datanode::manager::DrainDatanodeManager;
79use crate::procedure::region_migration::DefaultContextFactory;
80use crate::procedure::region_migration::manager::RegionMigrationManager;
81use crate::procedure::repartition::gc_requirement::RepartitionGcRequirementManager;
82use crate::procedure::repartition::{
83 DefaultRepartitionProcedureFactory, GcDisabledRepartitionProcedureFactory,
84};
85use crate::procedure::wal_prune::Context as WalPruneContext;
86use crate::procedure::wal_prune::manager::{WalPruneManager, WalPruneTicker};
87use crate::region::flush_trigger::RegionFlushTrigger;
88use crate::region::supervisor::{
89 DEFAULT_INITIALIZATION_RETRY_PERIOD, DEFAULT_TICK_INTERVAL, HeartbeatAcceptor,
90 RegionFailureDetectorControl, RegionSupervisor, RegionSupervisorSelector,
91 RegionSupervisorTicker,
92};
93use crate::selector::lease_based::LeaseBasedSelector;
94use crate::selector::round_robin::RoundRobinSelector;
95use crate::service::mailbox::MailboxRef;
96use crate::service::store::cached_kv::LeaderCachedKvBackend;
97use crate::state::State;
98use crate::utils::database::DatabaseOperator;
99use crate::utils::insert_forwarder::InsertForwarder;
100
101const REGION_STATS_TABLE_TWCS_COMPACTION_TIME_WINDOW: Duration = Duration::from_secs(86400);
103
104pub struct MetasrvBuilder {
106 options: Option<MetasrvOptions>,
107 kv_backend: Option<KvBackendRef>,
108 in_memory: Option<ResettableKvBackendRef>,
109 selector: Option<SelectorRef>,
110 handler_group_builder: Option<HeartbeatHandlerGroupBuilder>,
111 election: Option<ElectionRef>,
112 meta_peer_client: Option<MetaPeerClientRef>,
113 node_manager: Option<NodeManagerRef>,
114 plugins: Option<Plugins>,
115 table_metadata_allocator: Option<TableMetadataAllocatorRef>,
116}
117
118impl MetasrvBuilder {
119 pub fn new() -> Self {
120 Self {
121 kv_backend: None,
122 in_memory: None,
123 selector: None,
124 handler_group_builder: None,
125 meta_peer_client: None,
126 election: None,
127 options: None,
128 node_manager: None,
129 plugins: None,
130 table_metadata_allocator: None,
131 }
132 }
133
134 pub fn options(mut self, options: MetasrvOptions) -> Self {
135 self.options = Some(options);
136 self
137 }
138
139 pub fn kv_backend(mut self, kv_backend: KvBackendRef) -> Self {
140 self.kv_backend = Some(kv_backend);
141 self
142 }
143
144 pub fn in_memory(mut self, in_memory: ResettableKvBackendRef) -> Self {
145 self.in_memory = Some(in_memory);
146 self
147 }
148
149 pub fn selector(mut self, selector: SelectorRef) -> Self {
150 self.selector = Some(selector);
151 self
152 }
153
154 pub fn heartbeat_handler(
155 mut self,
156 handler_group_builder: HeartbeatHandlerGroupBuilder,
157 ) -> Self {
158 self.handler_group_builder = Some(handler_group_builder);
159 self
160 }
161
162 pub fn meta_peer_client(mut self, meta_peer_client: MetaPeerClientRef) -> Self {
163 self.meta_peer_client = Some(meta_peer_client);
164 self
165 }
166
167 pub fn election(mut self, election: Option<ElectionRef>) -> Self {
168 self.election = election;
169 self
170 }
171
172 pub fn node_manager(mut self, node_manager: NodeManagerRef) -> Self {
173 self.node_manager = Some(node_manager);
174 self
175 }
176
177 pub fn plugins(mut self, plugins: Plugins) -> Self {
178 self.plugins = Some(plugins);
179 self
180 }
181
182 pub fn table_metadata_allocator(
183 mut self,
184 table_metadata_allocator: TableMetadataAllocatorRef,
185 ) -> Self {
186 self.table_metadata_allocator = Some(table_metadata_allocator);
187 self
188 }
189
190 pub fn options_ref(&self) -> Option<&MetasrvOptions> {
191 self.options.as_ref()
192 }
193
194 pub fn kv_backend_ref(&self) -> Option<&KvBackendRef> {
195 self.kv_backend.as_ref()
196 }
197
198 pub fn in_memory_ref(&self) -> Option<&ResettableKvBackendRef> {
199 self.in_memory.as_ref()
200 }
201
202 pub fn election_ref(&self) -> Option<&ElectionRef> {
203 self.election.as_ref()
204 }
205
206 pub fn meta_peer_client_ref(&self) -> Option<&MetaPeerClientRef> {
207 self.meta_peer_client.as_ref()
208 }
209
210 pub fn node_manager_ref(&self) -> Option<&NodeManagerRef> {
211 self.node_manager.as_ref()
212 }
213
214 pub async fn build(self) -> Result<Metasrv> {
215 let MetasrvBuilder {
216 election,
217 meta_peer_client,
218 options,
219 kv_backend,
220 in_memory,
221 selector,
222 handler_group_builder,
223 node_manager,
224 plugins,
225 table_metadata_allocator,
226 } = self;
227
228 let options = options.unwrap_or_default();
229 options.gc.validate()?;
230
231 let kv_backend = kv_backend.unwrap_or_else(|| Arc::new(MemoryKvBackend::new()));
232 let in_memory = in_memory.unwrap_or_else(|| Arc::new(MemoryKvBackend::new()));
233
234 let state = Arc::new(RwLock::new(match election {
235 None => State::leader(options.grpc.server_addr.clone(), true),
236 Some(_) => State::follower(options.grpc.server_addr.clone()),
237 }));
238
239 let leader_cached_kv_backend = Arc::new(LeaderCachedKvBackend::new(
240 state.clone(),
241 kv_backend.clone(),
242 ));
243
244 let meta_peer_client = meta_peer_client
245 .unwrap_or_else(|| build_default_meta_peer_client(&election, &in_memory));
246
247 let node_scheduling_manager = Arc::new(NodeSchedulingManager::new(kv_backend.clone()));
248
249 let selector_ctx = SelectorContext {
255 affinity_fallback: plugins
256 .as_ref()
257 .and_then(|plugins| plugins.get::<AffinityFallback>())
258 .unwrap_or_default(),
259 affinity: plugins
260 .as_ref()
261 .and_then(|plugins| plugins.get::<AffinityRef>())
262 .unwrap_or_else(|| Arc::new(NoopAffinity)),
263 peer_discovery: Arc::new(SchedulingAwarePeerDiscovery::new(
264 node_scheduling_manager.clone(),
265 meta_peer_client.clone(),
266 )),
267 };
268 let database_operator =
270 Arc::new(DatabaseOperator::new(selector_ctx.peer_discovery.clone()));
271
272 let event_inserter = Box::new(InsertForwarder::new(
273 database_operator.clone(),
274 Some(InsertOptions {
275 ttl: options.event_recorder.ttl,
276 append_mode: true,
277 twcs_compaction_time_window: Some(DEFAULT_COMPACTION_TIME_WINDOW),
278 }),
279 ));
280 let event_recorder = Arc::new(EventRecorderImpl::with_event_type_filter(
282 Box::new(EventHandlerImpl::new(event_inserter)),
283 options.event_recorder.event_types.clone(),
284 options.event_recorder.flush_interval,
285 ));
286
287 let selector = selector.unwrap_or_else(|| Arc::new(LeaseBasedSelector));
288 let pushers = Pushers::default();
289 let mailbox = build_mailbox(&kv_backend, &pushers);
290 let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone()));
291 let repartition_gc_requirement_manager =
292 Arc::new(RepartitionGcRequirementManager::new(kv_backend.clone()));
293 let procedure_manager = build_procedure_manager(
294 &options,
295 &kv_backend,
296 &runtime_switch_manager,
297 event_recorder,
298 );
299
300 let table_metadata_manager = Arc::new(TableMetadataManager::new(
301 leader_cached_kv_backend.clone() as _,
302 ));
303 let flow_metadata_manager = Arc::new(FlowMetadataManager::new(
304 leader_cached_kv_backend.clone() as _,
305 ));
306
307 let wal_provider = build_wal_provider(&options.wal, kv_backend.clone())
308 .await
309 .context(BuildWalProviderSnafu)?;
310 if matches!(wal_provider, WalProvider::ObjectStore { .. }) {
311 ensure!(
312 plugins
313 .as_ref()
314 .and_then(|plugins| plugins.get::<EnableObjectStoreWal>())
315 .is_some(),
316 error::InvalidArgumentsSnafu {
317 err_msg: "WAL provider `experimental_object_store` requires the `EnableObjectStoreWal` plugin",
318 }
319 );
320 }
321
322 let wal_provider = Arc::new(wal_provider);
323 let is_remote_wal = wal_provider.is_remote_wal();
324 let table_metadata_allocator = table_metadata_allocator.unwrap_or_else(|| {
325 let sequence = Arc::new(
326 SequenceBuilder::new(TABLE_ID_SEQ, kv_backend.clone())
327 .initial(MIN_USER_TABLE_ID as u64)
328 .step(10)
329 .build(),
330 );
331 let peer_allocator = Arc::new(
332 MetasrvPeerAllocator::new(selector_ctx.clone(), selector.clone())
333 .with_max_items(MAX_REGION_SEQ),
334 );
335 Arc::new(TableMetadataAllocator::with_peer_allocator(
336 sequence,
337 wal_provider.clone(),
338 peer_allocator,
339 ))
340 });
341 let table_id_allocator = table_metadata_allocator.table_id_allocator();
342
343 let flow_selector =
344 Arc::new(RoundRobinSelector::new(SelectTarget::Flownode)) as SelectorRef;
345
346 let flow_metadata_allocator = {
347 let flow_selector_ctx = selector_ctx.clone();
349 let peer_allocator = Arc::new(MetasrvPeerAllocator::new(
350 flow_selector_ctx,
351 flow_selector.clone(),
352 ));
353 let seq = Arc::new(
354 SequenceBuilder::new(FLOW_ID_SEQ, kv_backend.clone())
355 .initial(MIN_USER_FLOW_ID as u64)
356 .step(10)
357 .build(),
358 );
359
360 Arc::new(FlowMetadataAllocator::with_peer_allocator(
361 seq,
362 peer_allocator,
363 ))
364 };
365 let flow_state_handler =
366 FlowStateHandler::new(FlowStateManager::new(in_memory.clone().as_kv_backend_ref()));
367
368 let memory_region_keeper = Arc::new(MemoryRegionKeeper::default());
369 let node_manager = node_manager.unwrap_or_else(|| {
370 let datanode_client_channel_config = options.datanode.client.channel_config();
371 Arc::new(NodeClients::new(datanode_client_channel_config))
372 });
373 let cache_invalidator = Arc::new(MetasrvCacheInvalidator::new(
374 mailbox.clone(),
375 MetasrvInfo {
376 server_addr: options.grpc.server_addr.clone(),
377 },
378 ));
379
380 if !is_remote_wal && options.enable_region_failover {
381 ensure!(
382 options.allow_region_failover_on_local_wal,
383 error::UnexpectedSnafu {
384 violated: "Region failover is not supported in the local WAL implementation!
385 If you want to enable region failover for local WAL, please set `allow_region_failover_on_local_wal` to true.",
386 }
387 );
388 if options.allow_region_failover_on_local_wal {
389 warn!(
390 "Region failover is force enabled in the local WAL implementation! This may lead to data loss during failover!"
391 );
392 }
393 }
394
395 let (tx, rx) = RegionSupervisor::channel();
396 let (region_failure_detector_controller, region_supervisor_ticker): (
397 RegionFailureDetectorControllerRef,
398 Option<std::sync::Arc<RegionSupervisorTicker>>,
399 ) = if options.enable_region_failover {
400 (
401 Arc::new(RegionFailureDetectorControl::new(tx.clone())) as _,
402 Some(Arc::new(RegionSupervisorTicker::new(
403 DEFAULT_TICK_INTERVAL,
404 options.region_failure_detector_initialization_delay,
405 DEFAULT_INITIALIZATION_RETRY_PERIOD,
406 tx.clone(),
407 ))),
408 )
409 } else {
410 (Arc::new(NoopRegionFailureDetectorControl) as _, None as _)
411 };
412
413 let region_migration_manager = Arc::new(RegionMigrationManager::new(
415 procedure_manager.clone(),
416 DefaultContextFactory::new(
417 in_memory.clone(),
418 table_metadata_manager.clone(),
419 memory_region_keeper.clone(),
420 region_failure_detector_controller.clone(),
421 mailbox.clone(),
422 options.grpc.server_addr.clone(),
423 cache_invalidator.clone(),
424 ),
425 ));
426 region_migration_manager.try_start()?;
427
428 let drain_datanode_manager = Arc::new(DrainDatanodeManager::new(
429 procedure_manager.clone(),
430 table_metadata_manager.clone(),
431 node_scheduling_manager.clone(),
432 region_migration_manager.clone(),
433 selector_ctx.clone(),
434 ));
435 drain_datanode_manager.try_start()?;
436 let affinity_resolver = plugins
437 .as_ref()
438 .and_then(|plugins| plugins.get::<SelectorFactoryRef>())
439 .and_then(|factory| {
440 factory.build_affinity_resolver(
441 meta_peer_client.clone(),
442 selector_ctx.peer_discovery.clone(),
443 )
444 });
445 let region_supervisor_selector = plugins
446 .as_ref()
447 .and_then(|plugins| plugins.get::<RegionStatAwareSelectorRef>());
448
449 let supervisor_selector = match region_supervisor_selector {
450 Some(selector) => {
451 info!("Using region stat aware selector");
452 RegionSupervisorSelector::RegionStatAwareSelector(selector)
453 }
454 None => RegionSupervisorSelector::NaiveSelector(selector.clone()),
455 };
456
457 let region_failover_handler = if options.enable_region_failover {
458 let region_supervisor = RegionSupervisor::new(
459 rx,
460 options.failure_detector,
461 selector_ctx.clone(),
462 supervisor_selector,
463 region_migration_manager.clone(),
464 runtime_switch_manager.clone(),
465 meta_peer_client.clone(),
466 leader_cached_kv_backend.clone(),
467 )
468 .with_state(state.clone());
469 let region_supervisor = if let Some(resolver) = &affinity_resolver {
470 region_supervisor.with_affinity_resolver(resolver.clone())
471 } else {
472 region_supervisor
473 };
474
475 Some(RegionFailureHandler::new(
476 region_supervisor,
477 HeartbeatAcceptor::new(tx),
478 ))
479 } else {
480 None
481 };
482
483 let leader_region_registry = Arc::new(LeaderRegionRegistry::default());
484 let topic_stats_registry = Arc::new(TopicStatsRegistry::default());
485
486 let ddl_context = DdlContext {
487 peer_alloc_context_provider: None,
488 affinity_fallback: selector_ctx.affinity_fallback,
489 affinity_resolver,
490 node_manager: node_manager.clone(),
491 cache_invalidator: cache_invalidator.clone(),
492 memory_region_keeper: memory_region_keeper.clone(),
493 leader_region_registry: leader_region_registry.clone(),
494 table_metadata_manager: table_metadata_manager.clone(),
495 table_metadata_allocator: table_metadata_allocator.clone(),
496 flow_metadata_manager: flow_metadata_manager.clone(),
497 flow_metadata_allocator: flow_metadata_allocator.clone(),
498 region_failure_detector_controller,
499 soft_drop_enabled: ddl_soft_drop_enabled(&options),
500 soft_drop_retention: ddl_soft_drop_retention(&options),
501 create_database_metadata_committer: None,
502 };
503 let procedure_manager_c = procedure_manager.clone();
504 let repartition_procedure_factory: RepartitionProcedureFactoryRef = if options.gc.enable {
505 Arc::new(DefaultRepartitionProcedureFactory::new(
506 mailbox.clone(),
507 options.grpc.server_addr.clone(),
508 repartition_gc_requirement_manager.clone(),
509 ))
510 } else {
511 Arc::new(GcDisabledRepartitionProcedureFactory::new(
512 mailbox.clone(),
513 options.grpc.server_addr.clone(),
514 repartition_gc_requirement_manager.clone(),
515 ))
516 };
517 let ddl_manager = DdlManager::new(
518 ddl_context,
519 procedure_manager_c,
520 repartition_procedure_factory,
521 );
522
523 let ddl_manager = if let Some(configurator) = plugins
524 .as_ref()
525 .and_then(|p| p.get::<DdlManagerConfiguratorRef<DdlManagerConfigureContext>>())
526 {
527 let ctx = DdlManagerConfigureContext {
528 kv_backend: kv_backend.clone(),
529 peer_discovery: selector_ctx.peer_discovery.clone(),
530 };
531 configurator
532 .configure(ddl_manager, ctx)
533 .await
534 .context(OtherSnafu)?
535 } else {
536 ddl_manager
537 };
538 ddl_manager
539 .register_loaders()
540 .context(error::InitDdlManagerSnafu)?;
541
542 let ddl_manager = Arc::new(ddl_manager);
543
544 let region_flush_ticker = if let Some(remote_wal_options) = options.wal.remote_wal_options()
545 {
546 let (region_flush_trigger, region_flush_ticker) = RegionFlushTrigger::new(
547 table_metadata_manager.clone(),
548 leader_region_registry.clone(),
549 topic_stats_registry.clone(),
550 mailbox.clone(),
551 options.grpc.server_addr.clone(),
552 remote_wal_options.flush_trigger_size,
553 remote_wal_options.checkpoint_trigger_size,
554 remote_wal_options.region_flush_trigger_interval,
555 remote_wal_options.periodic_checkpoint_persist_interval,
556 );
557 region_flush_trigger.try_start()?;
558
559 Some(Arc::new(region_flush_ticker))
560 } else {
561 None
562 };
563
564 let wal_prune_ticker = if let Some(remote_wal_options) = options.wal.remote_wal_options()
566 && options.wal.enable_active_wal_pruning()
567 {
568 let (tx, rx) = WalPruneManager::channel();
569 let kafka_client = build_kafka_client(&remote_wal_options.connection)
570 .await
571 .context(error::BuildKafkaClientSnafu)?;
572 let wal_prune_context = WalPruneContext {
573 client: Arc::new(kafka_client),
574 table_metadata_manager: table_metadata_manager.clone(),
575 leader_region_registry: leader_region_registry.clone(),
576 };
577 let wal_prune_manager = WalPruneManager::new(
578 remote_wal_options.auto_prune_parallelism,
579 remote_wal_options.auto_prune_logical_delete,
580 rx,
581 procedure_manager.clone(),
582 wal_prune_context,
583 );
584 wal_prune_manager.try_start().await?;
586 let wal_prune_ticker = Arc::new(WalPruneTicker::new(
587 remote_wal_options.auto_prune_interval,
588 tx.clone(),
589 ));
590 Some(wal_prune_ticker)
591 } else {
592 None
593 };
594
595 let gc_ticker = if options.gc.enable {
596 let gc_scheduler_ctx = DefaultGcSchedulerCtx::try_new(
597 table_metadata_manager.clone(),
598 procedure_manager.clone(),
599 runtime_switch_manager.clone(),
600 #[cfg(feature = "enterprise")]
601 ddl_manager.clone(),
602 meta_peer_client.clone(),
603 mailbox.clone(),
604 options.grpc.server_addr.clone(),
605 )?;
606 let (gc_scheduler, gc_ticker) = GcScheduler::new_with_config(
607 gc_scheduler_ctx,
608 runtime_switch_manager.clone(),
609 options.gc.clone(),
610 )?;
611 gc_scheduler.try_start()?;
612
613 Some(Arc::new(gc_ticker))
614 } else {
615 None
616 };
617
618 let customized_region_lease_renewer = plugins
619 .as_ref()
620 .and_then(|plugins| plugins.get::<CustomizedRegionLeaseRenewerRef>());
621
622 let persist_region_stats_handler = if !options.stats_persistence.ttl.is_zero() {
623 let inserter = Box::new(InsertForwarder::new(
624 database_operator.clone(),
625 Some(InsertOptions {
626 ttl: options.stats_persistence.ttl,
627 append_mode: true,
628 twcs_compaction_time_window: Some(
629 REGION_STATS_TABLE_TWCS_COMPACTION_TIME_WINDOW,
630 ),
631 }),
632 ));
633
634 Some(PersistStatsHandler::new(
635 inserter,
636 options.stats_persistence.interval,
637 ))
638 } else {
639 None
640 };
641
642 let handler_group_builder = match handler_group_builder {
643 Some(handler_group_builder) => handler_group_builder,
644 None => {
645 let region_lease_handler = RegionLeaseHandler::new(
646 default_distributed_time_constants().region_lease.as_secs(),
647 table_metadata_manager.clone(),
648 memory_region_keeper.clone(),
649 customized_region_lease_renewer,
650 );
651
652 HeartbeatHandlerGroupBuilder::new(pushers)
653 .with_plugins(plugins.clone())
654 .with_region_failure_handler(region_failover_handler)
655 .with_region_lease_handler(Some(region_lease_handler))
656 .with_flush_stats_factor(Some(options.flush_stats_factor))
657 .with_flow_state_handler(Some(flow_state_handler))
658 .with_persist_stats_handler(persist_region_stats_handler)
659 .add_default_handlers()
660 }
661 };
662
663 let enable_telemetry = options.enable_telemetry;
664 let metasrv_home = Path::new(&options.data_home)
665 .join(METASRV_DATA_DIR)
666 .to_string_lossy()
667 .to_string();
668
669 let reconciliation_manager = Arc::new(ReconciliationManager::new(
670 node_manager.clone(),
671 table_metadata_manager.clone(),
672 cache_invalidator.clone(),
673 procedure_manager.clone(),
674 ));
675 reconciliation_manager
676 .try_start()
677 .context(error::InitReconciliationManagerSnafu)?;
678
679 let mut resource_stat = ResourceStatImpl::default();
680 resource_stat.start_collect_cpu_usage();
681
682 Ok(Metasrv {
683 state,
684 started: Arc::new(AtomicBool::new(false)),
685 start_time_ms: common_time::util::current_time_millis() as u64,
686 options,
687 in_memory,
688 kv_backend,
689 leader_cached_kv_backend,
690 meta_peer_client: meta_peer_client.clone(),
691 selector,
692 selector_ctx,
693 flow_selector,
695 handler_group: RwLock::new(None),
696 handler_group_builder: Mutex::new(Some(handler_group_builder)),
697 election,
698 procedure_manager,
699 mailbox,
700 ddl_manager,
701 wal_provider,
702 table_metadata_manager,
703 runtime_switch_manager,
704 repartition_gc_requirement_manager,
705 greptimedb_telemetry_task: get_greptimedb_telemetry_task(
706 Some(metasrv_home),
707 meta_peer_client,
708 enable_telemetry,
709 )
710 .await,
711 plugins: plugins.unwrap_or_else(Plugins::default),
712 memory_region_keeper,
713 region_migration_manager,
714 drain_datanode_manager,
715 region_supervisor_ticker,
716 cache_invalidator,
717 leader_region_registry,
718 wal_prune_ticker,
719 region_flush_ticker,
720 table_id_allocator,
721 reconciliation_manager,
722 topic_stats_registry,
723 resource_stat: Arc::new(resource_stat),
724 gc_ticker,
725 database_operator,
726 node_scheduling_manager,
727 })
728 }
729}
730
731fn build_mailbox(kv_backend: &KvBackendRef, pushers: &Pushers) -> MailboxRef {
732 let mailbox_sequence = SequenceBuilder::new("heartbeat_mailbox", kv_backend.clone())
733 .initial(1)
734 .step(100)
735 .build();
736
737 HeartbeatMailbox::create(pushers.clone(), mailbox_sequence)
738}
739
740fn build_procedure_manager(
741 options: &MetasrvOptions,
742 kv_backend: &KvBackendRef,
743 runtime_switch_manager: &RuntimeSwitchManagerRef,
744 event_recorder: EventRecorderRef,
745) -> ProcedureManagerRef {
746 let manager_config = ManagerConfig {
747 max_retry_times: options.procedure.max_retry_times,
748 retry_delay: options.procedure.retry_delay,
749 max_running_procedures: options.procedure.max_running_procedures,
750 ..Default::default()
751 };
752 let kv_state_store = Arc::new(
753 KvStateStore::new(kv_backend.clone()).with_max_value_size(
754 options
755 .procedure
756 .max_metadata_value_size
757 .map(|v| v.as_bytes() as usize),
758 ),
759 );
760
761 Arc::new(LocalManager::new(
762 manager_config,
763 kv_state_store.clone(),
764 kv_state_store,
765 Some(runtime_switch_manager.clone()),
766 Some(event_recorder),
767 ))
768}
769
770fn ddl_soft_drop_enabled(options: &MetasrvOptions) -> bool {
775 cfg!(feature = "enterprise") && options.gc.experimental_soft_drop.enable
776}
777
778fn ddl_soft_drop_retention(options: &MetasrvOptions) -> Option<Duration> {
780 Some(options.gc.experimental_soft_drop.retention)
781}
782
783impl Default for MetasrvBuilder {
784 fn default() -> Self {
785 Self::new()
786 }
787}
788
789pub struct DdlManagerConfigureContext {
791 pub kv_backend: KvBackendRef,
792 pub peer_discovery: PeerDiscoveryRef,
793}
794
795#[cfg(test)]
796mod tests {
797 use super::*;
798
799 #[cfg(feature = "enterprise")]
800 #[tokio::test]
801 async fn test_object_store_wal_requires_enablement() {
802 use common_wal::config::MetasrvWalConfig;
803
804 let options = MetasrvOptions {
805 wal: MetasrvWalConfig::ObjectStore(Default::default()),
806 enable_region_failover: true,
807 enable_telemetry: false,
808 ..Default::default()
809 };
810 let result = MetasrvBuilder::new().options(options.clone()).build().await;
811 assert!(matches!(
812 result,
813 Err(error::Error::InvalidArguments { err_msg, .. })
814 if err_msg == "WAL provider `experimental_object_store` requires the `EnableObjectStoreWal` plugin"
815 ));
816 let plugins = Plugins::new();
817 plugins.insert(EnableObjectStoreWal);
818 let metasrv = MetasrvBuilder::new()
819 .options(options)
820 .plugins(plugins)
821 .build()
822 .await
823 .unwrap();
824 assert!(metasrv.wal_provider.is_remote_wal());
825 assert!(metasrv.region_supervisor_ticker.is_some());
826 assert!(metasrv.region_flush_ticker.is_none());
827 assert!(metasrv.wal_prune_ticker.is_none());
828 }
829
830 #[cfg(feature = "enterprise")]
831 #[test]
832 fn test_ddl_soft_drop_gate_preserves_retention_for_recovery() {
833 let mut options = MetasrvOptions::default();
834 options.gc.enable = true;
835 options.gc.experimental_soft_drop.enable = true;
836 options.gc.experimental_soft_drop.retention = Duration::from_secs(123);
837
838 assert!(ddl_soft_drop_enabled(&options));
839 assert_eq!(
840 Some(Duration::from_secs(123)),
841 ddl_soft_drop_retention(&options)
842 );
843 }
844
845 #[cfg(not(feature = "enterprise"))]
846 #[test]
847 fn test_ddl_soft_drop_is_always_disabled_in_non_enterprise_build() {
848 let mut options = MetasrvOptions::default();
849 options.gc.enable = true;
850 options.gc.experimental_soft_drop.enable = true;
851 options.gc.experimental_soft_drop.retention = Duration::from_secs(123);
852
853 assert!(!ddl_soft_drop_enabled(&options));
854 assert_eq!(
856 Some(Duration::from_secs(123)),
857 ddl_soft_drop_retention(&options)
858 );
859 }
860
861 #[test]
862 fn test_ddl_soft_drop_is_disabled_by_default() {
863 assert!(!ddl_soft_drop_enabled(&MetasrvOptions::default()));
864 }
865
866 #[test]
867 fn test_soft_drop_options_are_validated() {
868 let mut options = MetasrvOptions::default();
869 options.gc.enable = true;
870 options.gc.experimental_soft_drop.enable = true;
871 options.gc.experimental_soft_drop.retention = Duration::ZERO;
872
873 assert!(options.gc.validate().is_err());
874 }
875
876 #[tokio::test]
877 async fn test_builder_rejects_soft_drop_when_gc_is_disabled() {
878 let mut options = MetasrvOptions::default();
879 options.gc.experimental_soft_drop.enable = true;
880
881 assert!(
882 MetasrvBuilder::new()
883 .options(options)
884 .build()
885 .await
886 .is_err()
887 );
888 }
889
890 #[tokio::test]
891 async fn test_builder_skips_gc_validation_when_gc_is_disabled() {
892 let mut options = MetasrvOptions::default();
893 options.gc.enable = false;
894 options.gc.max_concurrent_tables = 0;
895
896 let metasrv = MetasrvBuilder::new()
897 .options(options)
898 .build()
899 .await
900 .unwrap();
901 let context = metasrv.ddl_manager().create_context();
902 assert_eq!(context.affinity_fallback, AffinityFallback::Error);
903 assert!(context.affinity_resolver.is_none());
904 assert!(context.peer_alloc_context_provider.is_none());
905 }
906
907 #[tokio::test]
908 async fn test_builder_injects_affinity_components() {
909 use std::sync::atomic::{AtomicUsize, Ordering};
910
911 use common_meta::affinity::{
912 Affinity, AffinityRef, AffinityReference, AffinityRequirements, AffinityResolver,
913 AffinityResolverRef,
914 };
915 use common_meta::cluster::NodeInfo;
916 use common_meta::error::Result as MetaResult;
917
918 use common_meta::ddl_manager::DdlManagerConfigurator;
919
920 use crate::discovery::node_info::NodeInfoAccessor;
921 use crate::metasrv::{SelectorFactory, SelectorFactoryContext};
922
923 struct TestAffinity;
924
925 impl Affinity for TestAffinity {
926 fn matches(&self, _: &AffinityRequirements, _: &NodeInfo) -> bool {
927 true
928 }
929 }
930
931 struct TestResolver;
932
933 #[async_trait::async_trait]
934 impl AffinityResolver for TestResolver {
935 async fn resolve(
936 &self,
937 references: &[AffinityReference],
938 ) -> MetaResult<Vec<Option<AffinityRequirements>>> {
939 Ok(vec![None; references.len()])
940 }
941 }
942
943 struct RecordingFactory {
944 calls: Arc<AtomicUsize>,
945 resolver: AffinityResolverRef,
946 discovery: Arc<std::sync::Mutex<Option<PeerDiscoveryRef>>>,
947 }
948
949 struct RecordingConfigurator {
950 discovery: Arc<std::sync::Mutex<Option<PeerDiscoveryRef>>>,
951 }
952
953 #[async_trait::async_trait]
954 impl DdlManagerConfigurator<DdlManagerConfigureContext> for RecordingConfigurator {
955 async fn configure(
956 &self,
957 ddl_manager: DdlManager,
958 ctx: DdlManagerConfigureContext,
959 ) -> std::result::Result<DdlManager, common_error::ext::BoxedError> {
960 *self.discovery.lock().unwrap() = Some(ctx.peer_discovery);
961 Ok(ddl_manager)
962 }
963 }
964
965 impl SelectorFactory for RecordingFactory {
966 fn build_affinity_resolver(
967 &self,
968 _: Arc<dyn NodeInfoAccessor>,
969 peer_discovery: PeerDiscoveryRef,
970 ) -> Option<AffinityResolverRef> {
971 *self.discovery.lock().unwrap() = Some(peer_discovery);
972 self.calls.fetch_add(1, Ordering::Relaxed);
973 Some(self.resolver.clone())
974 }
975
976 fn build(&self, ctx: SelectorFactoryContext) -> SelectorRef {
977 ctx.base_selector
978 }
979 }
980
981 let factory_discovery = Arc::new(std::sync::Mutex::new(None));
982 let configurator_discovery = Arc::new(std::sync::Mutex::new(None));
983 let calls = Arc::new(AtomicUsize::new(0));
984 let resolver: AffinityResolverRef = Arc::new(TestResolver);
985 let affinity: AffinityRef = Arc::new(TestAffinity);
986 let plugins = Plugins::new();
987 plugins.insert(affinity.clone());
988 plugins.insert(AffinityFallback::Unconstrained);
989 plugins.insert(Arc::new(RecordingFactory {
990 calls: calls.clone(),
991 resolver: resolver.clone(),
992 discovery: factory_discovery.clone(),
993 }) as SelectorFactoryRef);
994 plugins.insert(Arc::new(RecordingConfigurator {
995 discovery: configurator_discovery.clone(),
996 })
997 as DdlManagerConfiguratorRef<DdlManagerConfigureContext>);
998 let metasrv = MetasrvBuilder::new()
999 .options(MetasrvOptions {
1000 enable_telemetry: false,
1001 ..Default::default()
1002 })
1003 .plugins(plugins)
1004 .build()
1005 .await
1006 .unwrap();
1007
1008 assert_eq!(
1009 metasrv.selector_ctx().affinity_fallback,
1010 AffinityFallback::Unconstrained
1011 );
1012 assert_eq!(
1013 metasrv.ddl_manager().create_context().affinity_fallback,
1014 AffinityFallback::Unconstrained
1015 );
1016 assert_eq!(calls.load(Ordering::Relaxed), 1);
1017 for discovery in [factory_discovery, configurator_discovery] {
1018 assert!(Arc::ptr_eq(
1019 discovery.lock().unwrap().as_ref().unwrap(),
1020 &metasrv.selector_ctx().peer_discovery,
1021 ));
1022 }
1023 assert!(Arc::ptr_eq(&metasrv.selector_ctx().affinity, &affinity));
1024 assert!(Arc::ptr_eq(
1025 metasrv
1026 .ddl_manager()
1027 .create_context()
1028 .affinity_resolver
1029 .as_ref()
1030 .unwrap(),
1031 &resolver
1032 ));
1033 }
1034
1035 #[tokio::test]
1036 async fn test_builder_wires_node_scheduling_discovery() {
1037 use common_meta::peer::Peer;
1038
1039 use crate::key::NodeSchedulingRole;
1040 use crate::test_util::put_datanodes;
1041
1042 let metasrv = MetasrvBuilder::new()
1043 .options(MetasrvOptions {
1044 enable_telemetry: false,
1045 ..Default::default()
1046 })
1047 .build()
1048 .await
1049 .unwrap();
1050
1051 put_datanodes(
1052 metasrv.meta_peer_client(),
1053 vec![Peer::empty(1), Peer::empty(2)],
1054 )
1055 .await;
1056 metasrv
1057 .node_scheduling_manager()
1058 .mark_unschedulable(NodeSchedulingRole::Datanode, 2)
1059 .await
1060 .unwrap();
1061
1062 let datanodes = metasrv
1063 .selector_ctx()
1064 .peer_discovery
1065 .active_datanodes(None)
1066 .await
1067 .unwrap();
1068 assert_eq!(2, datanodes.len());
1069 for datanode in datanodes {
1070 let expected = datanode.peer.id != 2;
1071 assert_eq!(Some(expected), datanode.schedulable);
1072 }
1073 }
1074}