store_api/
mito_engine_options.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Option keys for the mito engine.
16//! We define them in this mod so the create parser can use it to validate table options.
17
18/// Option key for all WAL options.
19pub use common_wal::options::WAL_OPTIONS_KEY;
20/// Option key for append mode.
21pub const APPEND_MODE_KEY: &str = "append_mode";
22/// Option key for the per-region write buffer size.
23pub const WRITE_BUFFER_SIZE_KEY: &str = "write_buffer_size";
24/// Option key for merge mode.
25pub const MERGE_MODE_KEY: &str = "merge_mode";
26/// Option key for TTL(time-to-live)
27pub const TTL_KEY: &str = "ttl";
28/// Option key for the per-table auto flush interval.
29pub const AUTO_FLUSH_INTERVAL_KEY: &str = "auto_flush_interval";
30/// Option key for snapshot read.
31pub const SNAPSHOT_READ: &str = "snapshot_read";
32/// Option key for compaction type.
33pub const COMPACTION_TYPE: &str = "compaction.type";
34/// Option key for forcing compaction options override.
35pub const COMPACTION_OVERRIDE: &str = "compaction.override";
36/// TWCS compaction strategy.
37pub const COMPACTION_TYPE_TWCS: &str = "twcs";
38/// Option key for twcs min file num to trigger a compaction.
39pub const TWCS_TRIGGER_FILE_NUM: &str = "compaction.twcs.trigger_file_num";
40/// Option key for twcs max output file size.
41pub const TWCS_MAX_OUTPUT_FILE_SIZE: &str = "compaction.twcs.max_output_file_size";
42/// Option key for twcs time window.
43pub const TWCS_TIME_WINDOW: &str = "compaction.twcs.time_window";
44/// Option key for twcs remote compaction.
45pub const TWCS_REMOTE_COMPACTION: &str = "compaction.twcs.remote_compaction";
46/// Option key for twcs fallback to local.
47pub const TWCS_FALLBACK_TO_LOCAL: &str = "compaction.twcs.fallback_to_local";
48/// Option key for memtable type.
49pub const MEMTABLE_TYPE: &str = "memtable.type";
50/// Option key for bulk memtable merge threshold.
51pub const MEMTABLE_BULK_MERGE_THRESHOLD: &str = "memtable.bulk.merge_threshold";
52/// Option key for bulk memtable encode row threshold.
53pub const MEMTABLE_BULK_ENCODE_ROW_THRESHOLD: &str = "memtable.bulk.encode_row_threshold";
54/// Option key for bulk memtable encode bytes threshold.
55pub const MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD: &str = "memtable.bulk.encode_bytes_threshold";
56/// Option key for bulk memtable max merge groups.
57pub const MEMTABLE_BULK_MAX_MERGE_GROUPS: &str = "memtable.bulk.max_merge_groups";
58/// Option key for memtable partition tree index max keys per shard.
59pub const MEMTABLE_PARTITION_TREE_INDEX_MAX_KEYS_PER_SHARD: &str =
60    "memtable.partition_tree.index_max_keys_per_shard";
61/// Option key for memtable partition tree data freeze threshold.
62pub const MEMTABLE_PARTITION_TREE_DATA_FREEZE_THRESHOLD: &str =
63    "memtable.partition_tree.data_freeze_threshold";
64/// Option key for memtable partition tree fork dictionary bytes.
65pub const MEMTABLE_PARTITION_TREE_FORK_DICTIONARY_BYTES: &str =
66    "memtable.partition_tree.fork_dictionary_bytes";
67/// Option key for skipping WAL.
68pub const SKIP_WAL_KEY: &str = "skip_wal";
69/// Option key for sst format.
70pub const SST_FORMAT_KEY: &str = "sst_format";
71/// Option key for the max number of rows in a parquet row group.
72pub const MAX_ROW_GROUP_ROW_COUNT: &str = "max_row_group_row_count";
73/// Upper bound for [`MAX_ROW_GROUP_ROW_COUNT`].
74pub const MAX_ROW_GROUP_ROW_COUNT_LIMIT: usize = 10 * 1024 * 1024;
75// Note: Adding new options here should also check if this option should be removed in [metric_engine::engine::create::region_options_for_metadata_region].
76
77/// Returns true if the `key` is a valid option key for the mito engine.
78pub fn is_mito_engine_option_key(key: &str) -> bool {
79    [
80        "ttl",
81        WRITE_BUFFER_SIZE_KEY,
82        AUTO_FLUSH_INTERVAL_KEY,
83        COMPACTION_TYPE,
84        COMPACTION_OVERRIDE,
85        TWCS_TRIGGER_FILE_NUM,
86        TWCS_MAX_OUTPUT_FILE_SIZE,
87        TWCS_TIME_WINDOW,
88        TWCS_REMOTE_COMPACTION,
89        TWCS_FALLBACK_TO_LOCAL,
90        "storage",
91        "index.inverted_index.ignore_column_ids",
92        "index.inverted_index.segment_row_count",
93        WAL_OPTIONS_KEY,
94        MEMTABLE_TYPE,
95        MEMTABLE_BULK_MERGE_THRESHOLD,
96        MEMTABLE_BULK_ENCODE_ROW_THRESHOLD,
97        MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD,
98        MEMTABLE_BULK_MAX_MERGE_GROUPS,
99        MEMTABLE_PARTITION_TREE_INDEX_MAX_KEYS_PER_SHARD,
100        MEMTABLE_PARTITION_TREE_DATA_FREEZE_THRESHOLD,
101        MEMTABLE_PARTITION_TREE_FORK_DICTIONARY_BYTES,
102        // We don't allow to create a mito table with sparse primary key encoding directly.
103        APPEND_MODE_KEY,
104        MERGE_MODE_KEY,
105        SST_FORMAT_KEY,
106        MAX_ROW_GROUP_ROW_COUNT,
107    ]
108    .contains(&key)
109}
110
111#[cfg(test)]
112mod tests {
113    use super::*;
114
115    #[test]
116    fn test_is_mito_engine_option_key() {
117        assert!(is_mito_engine_option_key("ttl"));
118        assert!(is_mito_engine_option_key("write_buffer_size"));
119        assert!(is_mito_engine_option_key("auto_flush_interval"));
120        assert!(is_mito_engine_option_key("compaction.type"));
121        assert!(is_mito_engine_option_key("compaction.override"));
122        assert!(is_mito_engine_option_key(
123            "compaction.twcs.trigger_file_num"
124        ));
125        assert!(is_mito_engine_option_key("compaction.twcs.time_window"));
126        assert!(is_mito_engine_option_key("storage"));
127        assert!(is_mito_engine_option_key(
128            "index.inverted_index.ignore_column_ids"
129        ));
130        assert!(is_mito_engine_option_key(
131            "index.inverted_index.segment_row_count"
132        ));
133        assert!(is_mito_engine_option_key("wal_options"));
134        assert!(is_mito_engine_option_key("memtable.type"));
135        assert!(is_mito_engine_option_key("memtable.bulk.merge_threshold"));
136        assert!(is_mito_engine_option_key(
137            "memtable.bulk.encode_row_threshold"
138        ));
139        assert!(is_mito_engine_option_key(
140            "memtable.bulk.encode_bytes_threshold"
141        ));
142        assert!(is_mito_engine_option_key("memtable.bulk.max_merge_groups"));
143        assert!(is_mito_engine_option_key(
144            "memtable.partition_tree.index_max_keys_per_shard"
145        ));
146        assert!(is_mito_engine_option_key(
147            "memtable.partition_tree.data_freeze_threshold"
148        ));
149        assert!(is_mito_engine_option_key(
150            "memtable.partition_tree.fork_dictionary_bytes"
151        ));
152        assert!(is_mito_engine_option_key("append_mode"));
153        assert!(is_mito_engine_option_key("max_row_group_row_count"));
154        assert!(!is_mito_engine_option_key("foo"));
155    }
156}
store_api/mito_engine_options.rs

store_api/
mito_engine_options.rs
