metric_engine/engine/
options.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Specific options for the metric engine to create or open a region.
16
17use std::collections::HashMap;
18
19use store_api::metric_engine_consts::{
20    MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING,
21    METRIC_ENGINE_INDEX_SKIPPING_INDEX_FALSE_POSITIVE_RATE_OPTION,
22    METRIC_ENGINE_INDEX_SKIPPING_INDEX_FALSE_POSITIVE_RATE_OPTION_DEFAULT,
23    METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION,
24    METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION_DEFAULT, METRIC_ENGINE_INDEX_TYPE_OPTION,
25    PRIMARY_KEY_ENCODING,
26};
27use store_api::mito_engine_options::{COMPACTION_TYPE, COMPACTION_TYPE_TWCS, TWCS_TIME_WINDOW};
28
29/// Prefix for legacy `memtable.partition_tree.*` option keys. These keys are
30/// silently dropped by the metric engine; the partition tree memtable is gone.
31const LEGACY_PARTITION_TREE_OPTION_PREFIX: &str = "memtable.partition_tree.";
32
33use crate::error::{Error, ParseRegionOptionsSnafu, Result};
34
35/// The empirical value for the seg row count of the metric data region.
36/// Compared to the mito engine, the pattern of the metric engine constructs smaller indices.
37/// Therefore, compared to the default seg row count of 1024, by adjusting it to a smaller
38/// value and appropriately increasing the size of the index, it results in an improved indexing effect.
39const SEG_ROW_COUNT_FOR_DATA_REGION: u32 = 256;
40
41/// The default compaction time window for metric engine data regions.
42const DEFAULT_DATA_REGION_COMPACTION_TIME_WINDOW: &str = "1d";
43
44/// Physical region options.
45#[derive(Debug, Clone, Copy, PartialEq)]
46pub struct PhysicalRegionOptions {
47    pub index: IndexOptions,
48}
49
50/// Index options for auto created columns
51#[derive(Debug, Clone, Copy, Default, PartialEq)]
52pub enum IndexOptions {
53    #[default]
54    None,
55    Inverted,
56    Skipping {
57        granularity: u32,
58        false_positive_rate: f64,
59    },
60}
61
62/// Sets data region specific options.
63pub fn set_data_region_options(options: &mut HashMap<String, String>) {
64    options.remove(METRIC_ENGINE_INDEX_TYPE_OPTION);
65    options.remove(METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION);
66    options.remove(METRIC_ENGINE_INDEX_SKIPPING_INDEX_FALSE_POSITIVE_RATE_OPTION);
67    options.insert(
68        "index.inverted_index.segment_row_count".to_string(),
69        SEG_ROW_COUNT_FOR_DATA_REGION.to_string(),
70    );
71
72    // Extract primary key encoding from the legacy nested key before dropping
73    // all `memtable.partition_tree.*` keys.
74    let legacy_encoding = options.remove(MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING);
75    options.retain(|k, _| !k.starts_with(LEGACY_PARTITION_TREE_OPTION_PREFIX));
76
77    // Set memtable options for the data region. Bulk memtable produces
78    // flat-encoded ranges, so the SST format must be flat to match.
79    options.insert("memtable.type".to_string(), "bulk".to_string());
80    options.insert("sst_format".to_string(), "flat".to_string());
81
82    // Decide the top-level primary key encoding: caller-supplied top-level key wins,
83    // then extracted legacy value, then the `sparse` default.
84    if !options.contains_key(PRIMARY_KEY_ENCODING) {
85        if let Some(encoding) = legacy_encoding {
86            options.insert(PRIMARY_KEY_ENCODING.to_string(), encoding);
87        } else {
88            options.insert(PRIMARY_KEY_ENCODING.to_string(), "sparse".to_string());
89        }
90    }
91
92    if !options.contains_key(TWCS_TIME_WINDOW) {
93        options.insert(
94            COMPACTION_TYPE.to_string(),
95            COMPACTION_TYPE_TWCS.to_string(),
96        );
97        options.insert(
98            TWCS_TIME_WINDOW.to_string(),
99            DEFAULT_DATA_REGION_COMPACTION_TIME_WINDOW.to_string(),
100        );
101    }
102}
103
104impl TryFrom<&HashMap<String, String>> for PhysicalRegionOptions {
105    type Error = Error;
106
107    fn try_from(value: &HashMap<String, String>) -> Result<Self> {
108        let index = match value
109            .get(METRIC_ENGINE_INDEX_TYPE_OPTION)
110            .map(|s| s.to_lowercase())
111        {
112            Some(ref index_type) if index_type == "inverted" => Ok(IndexOptions::Inverted),
113            Some(ref index_type) if index_type == "skipping" => {
114                let granularity = value
115                    .get(METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION)
116                    .map_or(
117                        Ok(METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION_DEFAULT),
118                        |g| {
119                            g.parse().map_err(|_| {
120                                ParseRegionOptionsSnafu {
121                                    reason: format!("Invalid granularity: {}", g),
122                                }
123                                .build()
124                            })
125                        },
126                    )?;
127                let false_positive_rate = value
128                    .get(METRIC_ENGINE_INDEX_SKIPPING_INDEX_FALSE_POSITIVE_RATE_OPTION)
129                    .map_or(
130                        Ok(METRIC_ENGINE_INDEX_SKIPPING_INDEX_FALSE_POSITIVE_RATE_OPTION_DEFAULT),
131                        |f| {
132                            f.parse().ok().filter(|f| *f > 0.0 && *f <= 1.0).ok_or(
133                                ParseRegionOptionsSnafu {
134                                    reason: format!("Invalid false positive rate: {}", f),
135                                }
136                                .build(),
137                            )
138                        },
139                    )?;
140                Ok(IndexOptions::Skipping {
141                    granularity,
142                    false_positive_rate,
143                })
144            }
145            Some(index_type) => ParseRegionOptionsSnafu {
146                reason: format!("Invalid index type: {}", index_type),
147            }
148            .fail(),
149            None => Ok(IndexOptions::default()),
150        }?;
151
152        Ok(PhysicalRegionOptions { index })
153    }
154}
155
156#[cfg(test)]
157mod tests {
158    use super::*;
159
160    #[test]
161    fn test_set_data_region_options_should_remove_metric_engine_options() {
162        let mut options = HashMap::new();
163        options.insert(
164            METRIC_ENGINE_INDEX_TYPE_OPTION.to_string(),
165            "inverted".to_string(),
166        );
167        options.insert(
168            METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION.to_string(),
169            "102400".to_string(),
170        );
171        options.insert(
172            METRIC_ENGINE_INDEX_SKIPPING_INDEX_FALSE_POSITIVE_RATE_OPTION.to_string(),
173            "0.01".to_string(),
174        );
175        set_data_region_options(&mut options);
176
177        for key in [
178            METRIC_ENGINE_INDEX_TYPE_OPTION,
179            METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION,
180            METRIC_ENGINE_INDEX_SKIPPING_INDEX_FALSE_POSITIVE_RATE_OPTION,
181        ] {
182            assert_eq!(options.get(key), None);
183        }
184    }
185
186    #[test]
187    fn test_deserialize_physical_region_options_from_hashmap() {
188        let mut options = HashMap::new();
189        options.insert(
190            METRIC_ENGINE_INDEX_TYPE_OPTION.to_string(),
191            "inverted".to_string(),
192        );
193        options.insert(
194            METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION.to_string(),
195            "102400".to_string(),
196        );
197        let physical_region_options = PhysicalRegionOptions::try_from(&options).unwrap();
198        assert_eq!(physical_region_options.index, IndexOptions::Inverted);
199
200        let mut options = HashMap::new();
201        options.insert(
202            METRIC_ENGINE_INDEX_TYPE_OPTION.to_string(),
203            "skipping".to_string(),
204        );
205        options.insert(
206            METRIC_ENGINE_INDEX_SKIPPING_INDEX_GRANULARITY_OPTION.to_string(),
207            "102400".to_string(),
208        );
209        options.insert(
210            METRIC_ENGINE_INDEX_SKIPPING_INDEX_FALSE_POSITIVE_RATE_OPTION.to_string(),
211            "0.01".to_string(),
212        );
213        let physical_region_options = PhysicalRegionOptions::try_from(&options).unwrap();
214        assert_eq!(
215            physical_region_options.index,
216            IndexOptions::Skipping {
217                granularity: 102400,
218                false_positive_rate: 0.01,
219            }
220        );
221    }
222
223    #[test]
224    fn test_set_data_region_options_default_compaction_time_window() {
225        // Test that default time window is set when not specified
226        let mut options = HashMap::new();
227        set_data_region_options(&mut options);
228
229        assert_eq!(options.get("memtable.type"), Some(&"bulk".to_string()));
230        assert_eq!(options.get("sst_format"), Some(&"flat".to_string()));
231        assert_eq!(
232            options.get(COMPACTION_TYPE),
233            Some(&COMPACTION_TYPE_TWCS.to_string())
234        );
235        assert_eq!(options.get(TWCS_TIME_WINDOW), Some(&"1d".to_string()));
236    }
237
238    #[test]
239    fn test_set_data_region_options_sparse_primary_key_encoding() {
240        let mut options = HashMap::new();
241        set_data_region_options(&mut options);
242
243        assert_eq!(options.get("memtable.type"), Some(&"bulk".to_string()));
244        assert_eq!(options.get("sst_format"), Some(&"flat".to_string()));
245        assert_eq!(
246            options.get(PRIMARY_KEY_ENCODING),
247            Some(&"sparse".to_string())
248        );
249        assert!(!options.contains_key(MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING));
250    }
251
252    #[test]
253    fn test_set_data_region_options_migrates_legacy_partition_tree_options() {
254        let mut options = HashMap::new();
255        options.insert("memtable.type".to_string(), "partition_tree".to_string());
256        options.insert(
257            MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING.to_string(),
258            "sparse".to_string(),
259        );
260        options.insert(
261            "memtable.partition_tree.index_max_keys_per_shard".to_string(),
262            "2048".to_string(),
263        );
264        set_data_region_options(&mut options);
265
266        assert_eq!(options.get("memtable.type"), Some(&"bulk".to_string()));
267        assert_eq!(options.get("sst_format"), Some(&"flat".to_string()));
268        assert_eq!(
269            options.get(PRIMARY_KEY_ENCODING),
270            Some(&"sparse".to_string())
271        );
272        // All legacy partition-tree-specific keys should be stripped.
273        assert!(!options.contains_key(MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING));
274        assert!(!options.contains_key("memtable.partition_tree.index_max_keys_per_shard"));
275    }
276
277    #[test]
278    fn test_set_data_region_options_preserves_existing_top_level_encoding() {
279        let mut options = HashMap::new();
280        options.insert(PRIMARY_KEY_ENCODING.to_string(), "dense".to_string());
281        // Sparse flag is on but caller already specified dense.
282        set_data_region_options(&mut options);
283
284        assert_eq!(
285            options.get(PRIMARY_KEY_ENCODING),
286            Some(&"dense".to_string())
287        );
288    }
289
290    #[test]
291    fn test_set_data_region_options_respects_user_compaction_time_window() {
292        // Test that user-specified time window is preserved
293        let mut options = HashMap::new();
294        options.insert(TWCS_TIME_WINDOW.to_string(), "2h".to_string());
295        options.insert(COMPACTION_TYPE.to_string(), "twcs".to_string());
296        set_data_region_options(&mut options);
297
298        // User's time window should be preserved
299        assert_eq!(options.get(TWCS_TIME_WINDOW), Some(&"2h".to_string()));
300    }
301}
metric_engine/engine/options.rs

metric_engine/engine/
options.rs
