Skip to main content

meta_srv/metasrv/
builder.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::path::Path;
16use std::sync::atomic::AtomicBool;
17use std::sync::{Arc, Mutex, RwLock};
18use std::time::Duration;
19
20use client::client_manager::NodeClients;
21use client::inserter::InsertOptions;
22use common_base::Plugins;
23use common_catalog::consts::{MIN_USER_FLOW_ID, MIN_USER_TABLE_ID};
24use common_event_recorder::{DEFAULT_COMPACTION_TIME_WINDOW, EventRecorderImpl, EventRecorderRef};
25use common_meta::affinity::{AffinityFallback, AffinityRef, NoopAffinity};
26use common_meta::ddl::flow_meta::FlowMetadataAllocator;
27use common_meta::ddl::table_meta::{TableMetadataAllocator, TableMetadataAllocatorRef};
28use common_meta::ddl::{
29    DdlContext, NoopRegionFailureDetectorControl, RegionFailureDetectorControllerRef,
30};
31use common_meta::ddl_manager::{
32    DdlManager, DdlManagerConfiguratorRef, RepartitionProcedureFactoryRef,
33};
34use common_meta::distributed_time_constants::default_distributed_time_constants;
35use common_meta::key::TableMetadataManager;
36use common_meta::key::flow::FlowMetadataManager;
37use common_meta::key::flow::flow_state::FlowStateManager;
38use common_meta::key::runtime_switch::{RuntimeSwitchManager, RuntimeSwitchManagerRef};
39use common_meta::kv_backend::memory::MemoryKvBackend;
40use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef};
41use common_meta::node_manager::NodeManagerRef;
42use common_meta::peer::PeerDiscoveryRef;
43use common_meta::reconciliation::manager::ReconciliationManager;
44use common_meta::region_keeper::MemoryRegionKeeper;
45use common_meta::region_registry::LeaderRegionRegistry;
46use common_meta::sequence::SequenceBuilder;
47use common_meta::state_store::KvStateStore;
48use common_meta::stats::topic::TopicStatsRegistry;
49use common_meta::wal_provider::{WalProvider, build_kafka_client, build_wal_provider};
50use common_procedure::ProcedureManagerRef;
51use common_procedure::local::{LocalManager, ManagerConfig};
52use common_stat::ResourceStatImpl;
53use common_telemetry::{info, warn};
54use common_wal::config::object_store::EnableObjectStoreWal;
55use snafu::{ResultExt, ensure};
56use store_api::storage::MAX_REGION_SEQ;
57
58use crate::bootstrap::build_default_meta_peer_client;
59use crate::cache_invalidator::MetasrvCacheInvalidator;
60use crate::cluster::MetaPeerClientRef;
61use crate::discovery::node_scheduling::SchedulingAwarePeerDiscovery;
62use crate::error::{self, BuildWalProviderSnafu, OtherSnafu, Result};
63use crate::event::EventHandlerImpl;
64use crate::gc::{DefaultGcSchedulerCtx, GcScheduler};
65use crate::greptimedb_telemetry::get_greptimedb_telemetry_task;
66use crate::handler::failure_handler::RegionFailureHandler;
67use crate::handler::flow_state_handler::FlowStateHandler;
68use crate::handler::persist_stats_handler::PersistStatsHandler;
69use crate::handler::region_lease_handler::{CustomizedRegionLeaseRenewerRef, RegionLeaseHandler};
70use crate::handler::{HeartbeatHandlerGroupBuilder, HeartbeatMailbox, Pushers};
71use crate::metasrv::{
72    ElectionRef, FLOW_ID_SEQ, METASRV_DATA_DIR, Metasrv, MetasrvInfo, MetasrvOptions,
73    RegionStatAwareSelectorRef, SelectTarget, SelectorContext, SelectorFactoryRef, SelectorRef,
74    TABLE_ID_SEQ,
75};
76use crate::node_scheduling::NodeSchedulingManager;
77use crate::peer::MetasrvPeerAllocator;
78use crate::procedure::drain_datanode::manager::DrainDatanodeManager;
79use crate::procedure::region_migration::DefaultContextFactory;
80use crate::procedure::region_migration::manager::RegionMigrationManager;
81use crate::procedure::repartition::gc_requirement::RepartitionGcRequirementManager;
82use crate::procedure::repartition::{
83    DefaultRepartitionProcedureFactory, GcDisabledRepartitionProcedureFactory,
84};
85use crate::procedure::wal_prune::Context as WalPruneContext;
86use crate::procedure::wal_prune::manager::{WalPruneManager, WalPruneTicker};
87use crate::region::flush_trigger::RegionFlushTrigger;
88use crate::region::supervisor::{
89    DEFAULT_INITIALIZATION_RETRY_PERIOD, DEFAULT_TICK_INTERVAL, HeartbeatAcceptor,
90    RegionFailureDetectorControl, RegionSupervisor, RegionSupervisorSelector,
91    RegionSupervisorTicker,
92};
93use crate::selector::lease_based::LeaseBasedSelector;
94use crate::selector::round_robin::RoundRobinSelector;
95use crate::service::mailbox::MailboxRef;
96use crate::service::store::cached_kv::LeaderCachedKvBackend;
97use crate::state::State;
98use crate::utils::database::DatabaseOperator;
99use crate::utils::insert_forwarder::InsertForwarder;
100
101/// The time window for twcs compaction of the region stats table.
102const REGION_STATS_TABLE_TWCS_COMPACTION_TIME_WINDOW: Duration = Duration::from_secs(86400);
103
104// TODO(fys): try use derive_builder macro
105pub struct MetasrvBuilder {
106    options: Option<MetasrvOptions>,
107    kv_backend: Option<KvBackendRef>,
108    in_memory: Option<ResettableKvBackendRef>,
109    selector: Option<SelectorRef>,
110    handler_group_builder: Option<HeartbeatHandlerGroupBuilder>,
111    election: Option<ElectionRef>,
112    meta_peer_client: Option<MetaPeerClientRef>,
113    node_manager: Option<NodeManagerRef>,
114    plugins: Option<Plugins>,
115    table_metadata_allocator: Option<TableMetadataAllocatorRef>,
116}
117
118impl MetasrvBuilder {
119    pub fn new() -> Self {
120        Self {
121            kv_backend: None,
122            in_memory: None,
123            selector: None,
124            handler_group_builder: None,
125            meta_peer_client: None,
126            election: None,
127            options: None,
128            node_manager: None,
129            plugins: None,
130            table_metadata_allocator: None,
131        }
132    }
133
134    pub fn options(mut self, options: MetasrvOptions) -> Self {
135        self.options = Some(options);
136        self
137    }
138
139    pub fn kv_backend(mut self, kv_backend: KvBackendRef) -> Self {
140        self.kv_backend = Some(kv_backend);
141        self
142    }
143
144    pub fn in_memory(mut self, in_memory: ResettableKvBackendRef) -> Self {
145        self.in_memory = Some(in_memory);
146        self
147    }
148
149    pub fn selector(mut self, selector: SelectorRef) -> Self {
150        self.selector = Some(selector);
151        self
152    }
153
154    pub fn heartbeat_handler(
155        mut self,
156        handler_group_builder: HeartbeatHandlerGroupBuilder,
157    ) -> Self {
158        self.handler_group_builder = Some(handler_group_builder);
159        self
160    }
161
162    pub fn meta_peer_client(mut self, meta_peer_client: MetaPeerClientRef) -> Self {
163        self.meta_peer_client = Some(meta_peer_client);
164        self
165    }
166
167    pub fn election(mut self, election: Option<ElectionRef>) -> Self {
168        self.election = election;
169        self
170    }
171
172    pub fn node_manager(mut self, node_manager: NodeManagerRef) -> Self {
173        self.node_manager = Some(node_manager);
174        self
175    }
176
177    pub fn plugins(mut self, plugins: Plugins) -> Self {
178        self.plugins = Some(plugins);
179        self
180    }
181
182    pub fn table_metadata_allocator(
183        mut self,
184        table_metadata_allocator: TableMetadataAllocatorRef,
185    ) -> Self {
186        self.table_metadata_allocator = Some(table_metadata_allocator);
187        self
188    }
189
190    pub fn options_ref(&self) -> Option<&MetasrvOptions> {
191        self.options.as_ref()
192    }
193
194    pub fn kv_backend_ref(&self) -> Option<&KvBackendRef> {
195        self.kv_backend.as_ref()
196    }
197
198    pub fn in_memory_ref(&self) -> Option<&ResettableKvBackendRef> {
199        self.in_memory.as_ref()
200    }
201
202    pub fn election_ref(&self) -> Option<&ElectionRef> {
203        self.election.as_ref()
204    }
205
206    pub fn meta_peer_client_ref(&self) -> Option<&MetaPeerClientRef> {
207        self.meta_peer_client.as_ref()
208    }
209
210    pub fn node_manager_ref(&self) -> Option<&NodeManagerRef> {
211        self.node_manager.as_ref()
212    }
213
214    pub async fn build(self) -> Result<Metasrv> {
215        let MetasrvBuilder {
216            election,
217            meta_peer_client,
218            options,
219            kv_backend,
220            in_memory,
221            selector,
222            handler_group_builder,
223            node_manager,
224            plugins,
225            table_metadata_allocator,
226        } = self;
227
228        let options = options.unwrap_or_default();
229        options.gc.validate()?;
230
231        let kv_backend = kv_backend.unwrap_or_else(|| Arc::new(MemoryKvBackend::new()));
232        let in_memory = in_memory.unwrap_or_else(|| Arc::new(MemoryKvBackend::new()));
233
234        let state = Arc::new(RwLock::new(match election {
235            None => State::leader(options.grpc.server_addr.clone(), true),
236            Some(_) => State::follower(options.grpc.server_addr.clone()),
237        }));
238
239        let leader_cached_kv_backend = Arc::new(LeaderCachedKvBackend::new(
240            state.clone(),
241            kv_backend.clone(),
242        ));
243
244        let meta_peer_client = meta_peer_client
245            .unwrap_or_else(|| build_default_meta_peer_client(&election, &in_memory));
246
247        let node_scheduling_manager = Arc::new(NodeSchedulingManager::new(kv_backend.clone()));
248
249        // The only place a node scheduling decorator is built. Every consumer that reads the
250        // datanode list (region placement, failover exclusions) must go through
251        // `Metasrv::selector_ctx().peer_discovery`, and building a `PeerDiscoveryRef` from the raw
252        // client is impossible by construction. Bypassing this decorator silently drops the
253        // drain fence, so do not build a second instance.
254        let selector_ctx = SelectorContext {
255            affinity_fallback: plugins
256                .as_ref()
257                .and_then(|plugins| plugins.get::<AffinityFallback>())
258                .unwrap_or_default(),
259            affinity: plugins
260                .as_ref()
261                .and_then(|plugins| plugins.get::<AffinityRef>())
262                .unwrap_or_else(|| Arc::new(NoopAffinity)),
263            peer_discovery: Arc::new(SchedulingAwarePeerDiscovery::new(
264                node_scheduling_manager.clone(),
265                meta_peer_client.clone(),
266            )),
267        };
268        // Frontends only; the decorator passes them through unchanged.
269        let database_operator =
270            Arc::new(DatabaseOperator::new(selector_ctx.peer_discovery.clone()));
271
272        let event_inserter = Box::new(InsertForwarder::new(
273            database_operator.clone(),
274            Some(InsertOptions {
275                ttl: options.event_recorder.ttl,
276                append_mode: true,
277                twcs_compaction_time_window: Some(DEFAULT_COMPACTION_TIME_WINDOW),
278            }),
279        ));
280        // Builds the event recorder to record important events and persist them as the system table.
281        let event_recorder = Arc::new(EventRecorderImpl::with_event_type_filter(
282            Box::new(EventHandlerImpl::new(event_inserter)),
283            options.event_recorder.event_types.clone(),
284            options.event_recorder.flush_interval,
285        ));
286
287        let selector = selector.unwrap_or_else(|| Arc::new(LeaseBasedSelector));
288        let pushers = Pushers::default();
289        let mailbox = build_mailbox(&kv_backend, &pushers);
290        let runtime_switch_manager = Arc::new(RuntimeSwitchManager::new(kv_backend.clone()));
291        let repartition_gc_requirement_manager =
292            Arc::new(RepartitionGcRequirementManager::new(kv_backend.clone()));
293        let procedure_manager = build_procedure_manager(
294            &options,
295            &kv_backend,
296            &runtime_switch_manager,
297            event_recorder,
298        );
299
300        let table_metadata_manager = Arc::new(TableMetadataManager::new(
301            leader_cached_kv_backend.clone() as _,
302        ));
303        let flow_metadata_manager = Arc::new(FlowMetadataManager::new(
304            leader_cached_kv_backend.clone() as _,
305        ));
306
307        let wal_provider = build_wal_provider(&options.wal, kv_backend.clone())
308            .await
309            .context(BuildWalProviderSnafu)?;
310        if matches!(wal_provider, WalProvider::ObjectStore { .. }) {
311            ensure!(
312                plugins
313                    .as_ref()
314                    .and_then(|plugins| plugins.get::<EnableObjectStoreWal>())
315                    .is_some(),
316                error::InvalidArgumentsSnafu {
317                    err_msg: "WAL provider `experimental_object_store` requires the `EnableObjectStoreWal` plugin",
318                }
319            );
320        }
321
322        let wal_provider = Arc::new(wal_provider);
323        let is_remote_wal = wal_provider.is_remote_wal();
324        let table_metadata_allocator = table_metadata_allocator.unwrap_or_else(|| {
325            let sequence = Arc::new(
326                SequenceBuilder::new(TABLE_ID_SEQ, kv_backend.clone())
327                    .initial(MIN_USER_TABLE_ID as u64)
328                    .step(10)
329                    .build(),
330            );
331            let peer_allocator = Arc::new(
332                MetasrvPeerAllocator::new(selector_ctx.clone(), selector.clone())
333                    .with_max_items(MAX_REGION_SEQ),
334            );
335            Arc::new(TableMetadataAllocator::with_peer_allocator(
336                sequence,
337                wal_provider.clone(),
338                peer_allocator,
339            ))
340        });
341        let table_id_allocator = table_metadata_allocator.table_id_allocator();
342
343        let flow_selector =
344            Arc::new(RoundRobinSelector::new(SelectTarget::Flownode)) as SelectorRef;
345
346        let flow_metadata_allocator = {
347            // for now flownode just use round-robin selector
348            let flow_selector_ctx = selector_ctx.clone();
349            let peer_allocator = Arc::new(MetasrvPeerAllocator::new(
350                flow_selector_ctx,
351                flow_selector.clone(),
352            ));
353            let seq = Arc::new(
354                SequenceBuilder::new(FLOW_ID_SEQ, kv_backend.clone())
355                    .initial(MIN_USER_FLOW_ID as u64)
356                    .step(10)
357                    .build(),
358            );
359
360            Arc::new(FlowMetadataAllocator::with_peer_allocator(
361                seq,
362                peer_allocator,
363            ))
364        };
365        let flow_state_handler =
366            FlowStateHandler::new(FlowStateManager::new(in_memory.clone().as_kv_backend_ref()));
367
368        let memory_region_keeper = Arc::new(MemoryRegionKeeper::default());
369        let node_manager = node_manager.unwrap_or_else(|| {
370            let datanode_client_channel_config = options.datanode.client.channel_config();
371            Arc::new(NodeClients::new(datanode_client_channel_config))
372        });
373        let cache_invalidator = Arc::new(MetasrvCacheInvalidator::new(
374            mailbox.clone(),
375            MetasrvInfo {
376                server_addr: options.grpc.server_addr.clone(),
377            },
378        ));
379
380        if !is_remote_wal && options.enable_region_failover {
381            ensure!(
382                options.allow_region_failover_on_local_wal,
383                error::UnexpectedSnafu {
384                    violated: "Region failover is not supported in the local WAL implementation!
385                    If you want to enable region failover for local WAL, please set `allow_region_failover_on_local_wal` to true.",
386                }
387            );
388            if options.allow_region_failover_on_local_wal {
389                warn!(
390                    "Region failover is force enabled in the local WAL implementation! This may lead to data loss during failover!"
391                );
392            }
393        }
394
395        let (tx, rx) = RegionSupervisor::channel();
396        let (region_failure_detector_controller, region_supervisor_ticker): (
397            RegionFailureDetectorControllerRef,
398            Option<std::sync::Arc<RegionSupervisorTicker>>,
399        ) = if options.enable_region_failover {
400            (
401                Arc::new(RegionFailureDetectorControl::new(tx.clone())) as _,
402                Some(Arc::new(RegionSupervisorTicker::new(
403                    DEFAULT_TICK_INTERVAL,
404                    options.region_failure_detector_initialization_delay,
405                    DEFAULT_INITIALIZATION_RETRY_PERIOD,
406                    tx.clone(),
407                ))),
408            )
409        } else {
410            (Arc::new(NoopRegionFailureDetectorControl) as _, None as _)
411        };
412
413        // region migration manager
414        let region_migration_manager = Arc::new(RegionMigrationManager::new(
415            procedure_manager.clone(),
416            DefaultContextFactory::new(
417                in_memory.clone(),
418                table_metadata_manager.clone(),
419                memory_region_keeper.clone(),
420                region_failure_detector_controller.clone(),
421                mailbox.clone(),
422                options.grpc.server_addr.clone(),
423                cache_invalidator.clone(),
424            ),
425        ));
426        region_migration_manager.try_start()?;
427
428        let drain_datanode_manager = Arc::new(DrainDatanodeManager::new(
429            procedure_manager.clone(),
430            table_metadata_manager.clone(),
431            node_scheduling_manager.clone(),
432            region_migration_manager.clone(),
433            selector_ctx.clone(),
434        ));
435        drain_datanode_manager.try_start()?;
436        let affinity_resolver = plugins
437            .as_ref()
438            .and_then(|plugins| plugins.get::<SelectorFactoryRef>())
439            .and_then(|factory| {
440                factory.build_affinity_resolver(
441                    meta_peer_client.clone(),
442                    selector_ctx.peer_discovery.clone(),
443                )
444            });
445        let region_supervisor_selector = plugins
446            .as_ref()
447            .and_then(|plugins| plugins.get::<RegionStatAwareSelectorRef>());
448
449        let supervisor_selector = match region_supervisor_selector {
450            Some(selector) => {
451                info!("Using region stat aware selector");
452                RegionSupervisorSelector::RegionStatAwareSelector(selector)
453            }
454            None => RegionSupervisorSelector::NaiveSelector(selector.clone()),
455        };
456
457        let region_failover_handler = if options.enable_region_failover {
458            let region_supervisor = RegionSupervisor::new(
459                rx,
460                options.failure_detector,
461                selector_ctx.clone(),
462                supervisor_selector,
463                region_migration_manager.clone(),
464                runtime_switch_manager.clone(),
465                meta_peer_client.clone(),
466                leader_cached_kv_backend.clone(),
467            )
468            .with_state(state.clone());
469            let region_supervisor = if let Some(resolver) = &affinity_resolver {
470                region_supervisor.with_affinity_resolver(resolver.clone())
471            } else {
472                region_supervisor
473            };
474
475            Some(RegionFailureHandler::new(
476                region_supervisor,
477                HeartbeatAcceptor::new(tx),
478            ))
479        } else {
480            None
481        };
482
483        let leader_region_registry = Arc::new(LeaderRegionRegistry::default());
484        let topic_stats_registry = Arc::new(TopicStatsRegistry::default());
485
486        let ddl_context = DdlContext {
487            peer_alloc_context_provider: None,
488            affinity_fallback: selector_ctx.affinity_fallback,
489            affinity_resolver,
490            node_manager: node_manager.clone(),
491            cache_invalidator: cache_invalidator.clone(),
492            memory_region_keeper: memory_region_keeper.clone(),
493            leader_region_registry: leader_region_registry.clone(),
494            table_metadata_manager: table_metadata_manager.clone(),
495            table_metadata_allocator: table_metadata_allocator.clone(),
496            flow_metadata_manager: flow_metadata_manager.clone(),
497            flow_metadata_allocator: flow_metadata_allocator.clone(),
498            region_failure_detector_controller,
499            soft_drop_enabled: ddl_soft_drop_enabled(&options),
500            soft_drop_retention: ddl_soft_drop_retention(&options),
501            create_database_metadata_committer: None,
502        };
503        let procedure_manager_c = procedure_manager.clone();
504        let repartition_procedure_factory: RepartitionProcedureFactoryRef = if options.gc.enable {
505            Arc::new(DefaultRepartitionProcedureFactory::new(
506                mailbox.clone(),
507                options.grpc.server_addr.clone(),
508                repartition_gc_requirement_manager.clone(),
509            ))
510        } else {
511            Arc::new(GcDisabledRepartitionProcedureFactory::new(
512                mailbox.clone(),
513                options.grpc.server_addr.clone(),
514                repartition_gc_requirement_manager.clone(),
515            ))
516        };
517        let ddl_manager = DdlManager::new(
518            ddl_context,
519            procedure_manager_c,
520            repartition_procedure_factory,
521        );
522
523        let ddl_manager = if let Some(configurator) = plugins
524            .as_ref()
525            .and_then(|p| p.get::<DdlManagerConfiguratorRef<DdlManagerConfigureContext>>())
526        {
527            let ctx = DdlManagerConfigureContext {
528                kv_backend: kv_backend.clone(),
529                peer_discovery: selector_ctx.peer_discovery.clone(),
530            };
531            configurator
532                .configure(ddl_manager, ctx)
533                .await
534                .context(OtherSnafu)?
535        } else {
536            ddl_manager
537        };
538        ddl_manager
539            .register_loaders()
540            .context(error::InitDdlManagerSnafu)?;
541
542        let ddl_manager = Arc::new(ddl_manager);
543
544        let region_flush_ticker = if let Some(remote_wal_options) = options.wal.remote_wal_options()
545        {
546            let (region_flush_trigger, region_flush_ticker) = RegionFlushTrigger::new(
547                table_metadata_manager.clone(),
548                leader_region_registry.clone(),
549                topic_stats_registry.clone(),
550                mailbox.clone(),
551                options.grpc.server_addr.clone(),
552                remote_wal_options.flush_trigger_size,
553                remote_wal_options.checkpoint_trigger_size,
554                remote_wal_options.region_flush_trigger_interval,
555                remote_wal_options.periodic_checkpoint_persist_interval,
556            );
557            region_flush_trigger.try_start()?;
558
559            Some(Arc::new(region_flush_ticker))
560        } else {
561            None
562        };
563
564        // Kafka WAL prune ticker and manager.
565        let wal_prune_ticker = if let Some(remote_wal_options) = options.wal.remote_wal_options()
566            && options.wal.enable_active_wal_pruning()
567        {
568            let (tx, rx) = WalPruneManager::channel();
569            let kafka_client = build_kafka_client(&remote_wal_options.connection)
570                .await
571                .context(error::BuildKafkaClientSnafu)?;
572            let wal_prune_context = WalPruneContext {
573                client: Arc::new(kafka_client),
574                table_metadata_manager: table_metadata_manager.clone(),
575                leader_region_registry: leader_region_registry.clone(),
576            };
577            let wal_prune_manager = WalPruneManager::new(
578                remote_wal_options.auto_prune_parallelism,
579                remote_wal_options.auto_prune_logical_delete,
580                rx,
581                procedure_manager.clone(),
582                wal_prune_context,
583            );
584            // Start manager in background. Ticker will be started in the main thread to send ticks.
585            wal_prune_manager.try_start().await?;
586            let wal_prune_ticker = Arc::new(WalPruneTicker::new(
587                remote_wal_options.auto_prune_interval,
588                tx.clone(),
589            ));
590            Some(wal_prune_ticker)
591        } else {
592            None
593        };
594
595        let gc_ticker = if options.gc.enable {
596            let gc_scheduler_ctx = DefaultGcSchedulerCtx::try_new(
597                table_metadata_manager.clone(),
598                procedure_manager.clone(),
599                runtime_switch_manager.clone(),
600                #[cfg(feature = "enterprise")]
601                ddl_manager.clone(),
602                meta_peer_client.clone(),
603                mailbox.clone(),
604                options.grpc.server_addr.clone(),
605            )?;
606            let (gc_scheduler, gc_ticker) = GcScheduler::new_with_config(
607                gc_scheduler_ctx,
608                runtime_switch_manager.clone(),
609                options.gc.clone(),
610            )?;
611            gc_scheduler.try_start()?;
612
613            Some(Arc::new(gc_ticker))
614        } else {
615            None
616        };
617
618        let customized_region_lease_renewer = plugins
619            .as_ref()
620            .and_then(|plugins| plugins.get::<CustomizedRegionLeaseRenewerRef>());
621
622        let persist_region_stats_handler = if !options.stats_persistence.ttl.is_zero() {
623            let inserter = Box::new(InsertForwarder::new(
624                database_operator.clone(),
625                Some(InsertOptions {
626                    ttl: options.stats_persistence.ttl,
627                    append_mode: true,
628                    twcs_compaction_time_window: Some(
629                        REGION_STATS_TABLE_TWCS_COMPACTION_TIME_WINDOW,
630                    ),
631                }),
632            ));
633
634            Some(PersistStatsHandler::new(
635                inserter,
636                options.stats_persistence.interval,
637            ))
638        } else {
639            None
640        };
641
642        let handler_group_builder = match handler_group_builder {
643            Some(handler_group_builder) => handler_group_builder,
644            None => {
645                let region_lease_handler = RegionLeaseHandler::new(
646                    default_distributed_time_constants().region_lease.as_secs(),
647                    table_metadata_manager.clone(),
648                    memory_region_keeper.clone(),
649                    customized_region_lease_renewer,
650                );
651
652                HeartbeatHandlerGroupBuilder::new(pushers)
653                    .with_plugins(plugins.clone())
654                    .with_region_failure_handler(region_failover_handler)
655                    .with_region_lease_handler(Some(region_lease_handler))
656                    .with_flush_stats_factor(Some(options.flush_stats_factor))
657                    .with_flow_state_handler(Some(flow_state_handler))
658                    .with_persist_stats_handler(persist_region_stats_handler)
659                    .add_default_handlers()
660            }
661        };
662
663        let enable_telemetry = options.enable_telemetry;
664        let metasrv_home = Path::new(&options.data_home)
665            .join(METASRV_DATA_DIR)
666            .to_string_lossy()
667            .to_string();
668
669        let reconciliation_manager = Arc::new(ReconciliationManager::new(
670            node_manager.clone(),
671            table_metadata_manager.clone(),
672            cache_invalidator.clone(),
673            procedure_manager.clone(),
674        ));
675        reconciliation_manager
676            .try_start()
677            .context(error::InitReconciliationManagerSnafu)?;
678
679        let mut resource_stat = ResourceStatImpl::default();
680        resource_stat.start_collect_cpu_usage();
681
682        Ok(Metasrv {
683            state,
684            started: Arc::new(AtomicBool::new(false)),
685            start_time_ms: common_time::util::current_time_millis() as u64,
686            options,
687            in_memory,
688            kv_backend,
689            leader_cached_kv_backend,
690            meta_peer_client: meta_peer_client.clone(),
691            selector,
692            selector_ctx,
693            // TODO(jeremy): We do not allow configuring the flow selector.
694            flow_selector,
695            handler_group: RwLock::new(None),
696            handler_group_builder: Mutex::new(Some(handler_group_builder)),
697            election,
698            procedure_manager,
699            mailbox,
700            ddl_manager,
701            wal_provider,
702            table_metadata_manager,
703            runtime_switch_manager,
704            repartition_gc_requirement_manager,
705            greptimedb_telemetry_task: get_greptimedb_telemetry_task(
706                Some(metasrv_home),
707                meta_peer_client,
708                enable_telemetry,
709            )
710            .await,
711            plugins: plugins.unwrap_or_else(Plugins::default),
712            memory_region_keeper,
713            region_migration_manager,
714            drain_datanode_manager,
715            region_supervisor_ticker,
716            cache_invalidator,
717            leader_region_registry,
718            wal_prune_ticker,
719            region_flush_ticker,
720            table_id_allocator,
721            reconciliation_manager,
722            topic_stats_registry,
723            resource_stat: Arc::new(resource_stat),
724            gc_ticker,
725            database_operator,
726            node_scheduling_manager,
727        })
728    }
729}
730
731fn build_mailbox(kv_backend: &KvBackendRef, pushers: &Pushers) -> MailboxRef {
732    let mailbox_sequence = SequenceBuilder::new("heartbeat_mailbox", kv_backend.clone())
733        .initial(1)
734        .step(100)
735        .build();
736
737    HeartbeatMailbox::create(pushers.clone(), mailbox_sequence)
738}
739
740fn build_procedure_manager(
741    options: &MetasrvOptions,
742    kv_backend: &KvBackendRef,
743    runtime_switch_manager: &RuntimeSwitchManagerRef,
744    event_recorder: EventRecorderRef,
745) -> ProcedureManagerRef {
746    let manager_config = ManagerConfig {
747        max_retry_times: options.procedure.max_retry_times,
748        retry_delay: options.procedure.retry_delay,
749        max_running_procedures: options.procedure.max_running_procedures,
750        ..Default::default()
751    };
752    let kv_state_store = Arc::new(
753        KvStateStore::new(kv_backend.clone()).with_max_value_size(
754            options
755                .procedure
756                .max_metadata_value_size
757                .map(|v| v.as_bytes() as usize),
758        ),
759    );
760
761    Arc::new(LocalManager::new(
762        manager_config,
763        kv_state_store.clone(),
764        kv_state_store,
765        Some(runtime_switch_manager.clone()),
766        Some(event_recorder),
767    ))
768}
769
770/// Resolves if soft-drop is enabled from metasrv options.
771///
772/// Soft drop is an enterprise-only feature; it is always disabled in
773/// non-enterprise builds regardless of the configuration.
774fn ddl_soft_drop_enabled(options: &MetasrvOptions) -> bool {
775    cfg!(feature = "enterprise") && options.gc.experimental_soft_drop.enable
776}
777
778/// Returns soft-drop retention for recovering persisted procedures.
779fn ddl_soft_drop_retention(options: &MetasrvOptions) -> Option<Duration> {
780    Some(options.gc.experimental_soft_drop.retention)
781}
782
783impl Default for MetasrvBuilder {
784    fn default() -> Self {
785        Self::new()
786    }
787}
788
789/// The context for [`DdlManagerConfiguratorRef`].
790pub struct DdlManagerConfigureContext {
791    pub kv_backend: KvBackendRef,
792    pub peer_discovery: PeerDiscoveryRef,
793}
794
795#[cfg(test)]
796mod tests {
797    use super::*;
798
799    #[cfg(feature = "enterprise")]
800    #[tokio::test]
801    async fn test_object_store_wal_requires_enablement() {
802        use common_wal::config::MetasrvWalConfig;
803
804        let options = MetasrvOptions {
805            wal: MetasrvWalConfig::ObjectStore(Default::default()),
806            enable_region_failover: true,
807            enable_telemetry: false,
808            ..Default::default()
809        };
810        let result = MetasrvBuilder::new().options(options.clone()).build().await;
811        assert!(matches!(
812            result,
813            Err(error::Error::InvalidArguments { err_msg, .. })
814                if err_msg == "WAL provider `experimental_object_store` requires the `EnableObjectStoreWal` plugin"
815        ));
816        let plugins = Plugins::new();
817        plugins.insert(EnableObjectStoreWal);
818        let metasrv = MetasrvBuilder::new()
819            .options(options)
820            .plugins(plugins)
821            .build()
822            .await
823            .unwrap();
824        assert!(metasrv.wal_provider.is_remote_wal());
825        assert!(metasrv.region_supervisor_ticker.is_some());
826        assert!(metasrv.region_flush_ticker.is_none());
827        assert!(metasrv.wal_prune_ticker.is_none());
828    }
829
830    #[cfg(feature = "enterprise")]
831    #[test]
832    fn test_ddl_soft_drop_gate_preserves_retention_for_recovery() {
833        let mut options = MetasrvOptions::default();
834        options.gc.enable = true;
835        options.gc.experimental_soft_drop.enable = true;
836        options.gc.experimental_soft_drop.retention = Duration::from_secs(123);
837
838        assert!(ddl_soft_drop_enabled(&options));
839        assert_eq!(
840            Some(Duration::from_secs(123)),
841            ddl_soft_drop_retention(&options)
842        );
843    }
844
845    #[cfg(not(feature = "enterprise"))]
846    #[test]
847    fn test_ddl_soft_drop_is_always_disabled_in_non_enterprise_build() {
848        let mut options = MetasrvOptions::default();
849        options.gc.enable = true;
850        options.gc.experimental_soft_drop.enable = true;
851        options.gc.experimental_soft_drop.retention = Duration::from_secs(123);
852
853        assert!(!ddl_soft_drop_enabled(&options));
854        // Retention is preserved for recovering persisted procedures.
855        assert_eq!(
856            Some(Duration::from_secs(123)),
857            ddl_soft_drop_retention(&options)
858        );
859    }
860
861    #[test]
862    fn test_ddl_soft_drop_is_disabled_by_default() {
863        assert!(!ddl_soft_drop_enabled(&MetasrvOptions::default()));
864    }
865
866    #[test]
867    fn test_soft_drop_options_are_validated() {
868        let mut options = MetasrvOptions::default();
869        options.gc.enable = true;
870        options.gc.experimental_soft_drop.enable = true;
871        options.gc.experimental_soft_drop.retention = Duration::ZERO;
872
873        assert!(options.gc.validate().is_err());
874    }
875
876    #[tokio::test]
877    async fn test_builder_rejects_soft_drop_when_gc_is_disabled() {
878        let mut options = MetasrvOptions::default();
879        options.gc.experimental_soft_drop.enable = true;
880
881        assert!(
882            MetasrvBuilder::new()
883                .options(options)
884                .build()
885                .await
886                .is_err()
887        );
888    }
889
890    #[tokio::test]
891    async fn test_builder_skips_gc_validation_when_gc_is_disabled() {
892        let mut options = MetasrvOptions::default();
893        options.gc.enable = false;
894        options.gc.max_concurrent_tables = 0;
895
896        let metasrv = MetasrvBuilder::new()
897            .options(options)
898            .build()
899            .await
900            .unwrap();
901        let context = metasrv.ddl_manager().create_context();
902        assert_eq!(context.affinity_fallback, AffinityFallback::Error);
903        assert!(context.affinity_resolver.is_none());
904        assert!(context.peer_alloc_context_provider.is_none());
905    }
906
907    #[tokio::test]
908    async fn test_builder_injects_affinity_components() {
909        use std::sync::atomic::{AtomicUsize, Ordering};
910
911        use common_meta::affinity::{
912            Affinity, AffinityRef, AffinityReference, AffinityRequirements, AffinityResolver,
913            AffinityResolverRef,
914        };
915        use common_meta::cluster::NodeInfo;
916        use common_meta::error::Result as MetaResult;
917
918        use common_meta::ddl_manager::DdlManagerConfigurator;
919
920        use crate::discovery::node_info::NodeInfoAccessor;
921        use crate::metasrv::{SelectorFactory, SelectorFactoryContext};
922
923        struct TestAffinity;
924
925        impl Affinity for TestAffinity {
926            fn matches(&self, _: &AffinityRequirements, _: &NodeInfo) -> bool {
927                true
928            }
929        }
930
931        struct TestResolver;
932
933        #[async_trait::async_trait]
934        impl AffinityResolver for TestResolver {
935            async fn resolve(
936                &self,
937                references: &[AffinityReference],
938            ) -> MetaResult<Vec<Option<AffinityRequirements>>> {
939                Ok(vec![None; references.len()])
940            }
941        }
942
943        struct RecordingFactory {
944            calls: Arc<AtomicUsize>,
945            resolver: AffinityResolverRef,
946            discovery: Arc<std::sync::Mutex<Option<PeerDiscoveryRef>>>,
947        }
948
949        struct RecordingConfigurator {
950            discovery: Arc<std::sync::Mutex<Option<PeerDiscoveryRef>>>,
951        }
952
953        #[async_trait::async_trait]
954        impl DdlManagerConfigurator<DdlManagerConfigureContext> for RecordingConfigurator {
955            async fn configure(
956                &self,
957                ddl_manager: DdlManager,
958                ctx: DdlManagerConfigureContext,
959            ) -> std::result::Result<DdlManager, common_error::ext::BoxedError> {
960                *self.discovery.lock().unwrap() = Some(ctx.peer_discovery);
961                Ok(ddl_manager)
962            }
963        }
964
965        impl SelectorFactory for RecordingFactory {
966            fn build_affinity_resolver(
967                &self,
968                _: Arc<dyn NodeInfoAccessor>,
969                peer_discovery: PeerDiscoveryRef,
970            ) -> Option<AffinityResolverRef> {
971                *self.discovery.lock().unwrap() = Some(peer_discovery);
972                self.calls.fetch_add(1, Ordering::Relaxed);
973                Some(self.resolver.clone())
974            }
975
976            fn build(&self, ctx: SelectorFactoryContext) -> SelectorRef {
977                ctx.base_selector
978            }
979        }
980
981        let factory_discovery = Arc::new(std::sync::Mutex::new(None));
982        let configurator_discovery = Arc::new(std::sync::Mutex::new(None));
983        let calls = Arc::new(AtomicUsize::new(0));
984        let resolver: AffinityResolverRef = Arc::new(TestResolver);
985        let affinity: AffinityRef = Arc::new(TestAffinity);
986        let plugins = Plugins::new();
987        plugins.insert(affinity.clone());
988        plugins.insert(AffinityFallback::Unconstrained);
989        plugins.insert(Arc::new(RecordingFactory {
990            calls: calls.clone(),
991            resolver: resolver.clone(),
992            discovery: factory_discovery.clone(),
993        }) as SelectorFactoryRef);
994        plugins.insert(Arc::new(RecordingConfigurator {
995            discovery: configurator_discovery.clone(),
996        })
997            as DdlManagerConfiguratorRef<DdlManagerConfigureContext>);
998        let metasrv = MetasrvBuilder::new()
999            .options(MetasrvOptions {
1000                enable_telemetry: false,
1001                ..Default::default()
1002            })
1003            .plugins(plugins)
1004            .build()
1005            .await
1006            .unwrap();
1007
1008        assert_eq!(
1009            metasrv.selector_ctx().affinity_fallback,
1010            AffinityFallback::Unconstrained
1011        );
1012        assert_eq!(
1013            metasrv.ddl_manager().create_context().affinity_fallback,
1014            AffinityFallback::Unconstrained
1015        );
1016        assert_eq!(calls.load(Ordering::Relaxed), 1);
1017        for discovery in [factory_discovery, configurator_discovery] {
1018            assert!(Arc::ptr_eq(
1019                discovery.lock().unwrap().as_ref().unwrap(),
1020                &metasrv.selector_ctx().peer_discovery,
1021            ));
1022        }
1023        assert!(Arc::ptr_eq(&metasrv.selector_ctx().affinity, &affinity));
1024        assert!(Arc::ptr_eq(
1025            metasrv
1026                .ddl_manager()
1027                .create_context()
1028                .affinity_resolver
1029                .as_ref()
1030                .unwrap(),
1031            &resolver
1032        ));
1033    }
1034
1035    #[tokio::test]
1036    async fn test_builder_wires_node_scheduling_discovery() {
1037        use common_meta::peer::Peer;
1038
1039        use crate::key::NodeSchedulingRole;
1040        use crate::test_util::put_datanodes;
1041
1042        let metasrv = MetasrvBuilder::new()
1043            .options(MetasrvOptions {
1044                enable_telemetry: false,
1045                ..Default::default()
1046            })
1047            .build()
1048            .await
1049            .unwrap();
1050
1051        put_datanodes(
1052            metasrv.meta_peer_client(),
1053            vec![Peer::empty(1), Peer::empty(2)],
1054        )
1055        .await;
1056        metasrv
1057            .node_scheduling_manager()
1058            .mark_unschedulable(NodeSchedulingRole::Datanode, 2)
1059            .await
1060            .unwrap();
1061
1062        let datanodes = metasrv
1063            .selector_ctx()
1064            .peer_discovery
1065            .active_datanodes(None)
1066            .await
1067            .unwrap();
1068        assert_eq!(2, datanodes.len());
1069        for datanode in datanodes {
1070            let expected = datanode.peer.id != 2;
1071            assert_eq!(Some(expected), datanode.schedulable);
1072        }
1073    }
1074}