common_wal/config/kafka/
datanode.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::time::Duration;
16
17use common_base::readable_size::ReadableSize;
18use serde::{Deserialize, Serialize};
19
20use crate::config::kafka::common::{KafkaConnectionConfig, KafkaTopicConfig};
21
22/// Kafka wal configurations for datanode.
23#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
24#[serde(default)]
25pub struct DatanodeKafkaConfig {
26    /// The kafka connection config.
27    #[serde(flatten)]
28    pub connection: KafkaConnectionConfig,
29    /// TODO(weny): Remove the alias once we release v0.9.
30    /// The max size of a single producer batch.
31    #[serde(alias = "max_batch_size")]
32    pub max_batch_bytes: ReadableSize,
33    /// The consumer wait timeout.
34    #[serde(with = "humantime_serde")]
35    pub consumer_wait_timeout: Duration,
36    /// The kafka topic config.
37    #[serde(flatten)]
38    pub kafka_topic: KafkaTopicConfig,
39    // Automatically create topics for WAL.
40    pub auto_create_topics: bool,
41    // Create index for WAL.
42    pub create_index: bool,
43    #[serde(with = "humantime_serde")]
44    pub dump_index_interval: Duration,
45    /// Ignore missing entries during read WAL.
46    pub overwrite_entry_start_id: bool,
47}
48
49impl Default for DatanodeKafkaConfig {
50    fn default() -> Self {
51        Self {
52            connection: KafkaConnectionConfig::default(),
53            // Warning: Kafka has a default limit of 1MB per message in a topic.
54            max_batch_bytes: ReadableSize::mb(1),
55            consumer_wait_timeout: Duration::from_millis(100),
56            kafka_topic: KafkaTopicConfig::default(),
57            auto_create_topics: true,
58            create_index: true,
59            dump_index_interval: Duration::from_secs(60),
60            overwrite_entry_start_id: false,
61        }
62    }
63}
common_wal/config/kafka/datanode.rs

common_wal/config/kafka/
datanode.rs
