Skip to main content

mito2/region/
options.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Options for a region.
16//!
17//! If we add options in this mod, we also need to modify [store_api::mito_engine_options].
18
19use std::collections::HashMap;
20use std::time::Duration;
21
22use api::v1::SemanticType;
23use common_base::readable_size::ReadableSize;
24use common_telemetry::info;
25use common_time::TimeToLive;
26use common_wal::options::{WAL_OPTIONS_KEY, WalOptions};
27use datatypes::schema::COMPRESSION_KEY;
28use serde::de::Error as _;
29use serde::{Deserialize, Deserializer, Serialize};
30use serde_json::Value;
31use serde_with::{DisplayFromStr, NoneAsEmptyString, serde_as, with_prefix};
32use snafu::{ResultExt, ensure};
33use store_api::codec::PrimaryKeyEncoding;
34use store_api::metadata::RegionMetadata;
35use store_api::metric_engine_consts::{
36    MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING, PRIMARY_KEY_ENCODING,
37};
38use store_api::mito_engine_options::{
39    COMPACTION_OVERRIDE, COMPACTION_TWCS_PREFIX, FloatFieldEncoding, MAX_ROW_GROUP_ROW_COUNT_LIMIT,
40};
41use store_api::storage::{ColumnId, RegionId};
42use strum::EnumString;
43
44use crate::error::{InvalidRegionOptionsSnafu, JsonOptionsSnafu, Result};
45use crate::memtable::bulk::BulkMemtableConfig;
46use crate::sst::FormatType;
47use crate::sst::parquet::DEFAULT_ROW_GROUP_SIZE;
48
49const DEFAULT_INDEX_SEGMENT_ROW_COUNT: usize = 1024;
50const MEMTABLE_PARTITION_TREE_PREFIX: &str = "memtable.partition_tree.";
51const MEMTABLE_BULK_PREFIX: &str = "memtable.bulk.";
52const MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD: &str = "memtable.bulk.encode_bytes_threshold";
53
54/// Legacy memtable type identifier accepted for backward compatibility.
55/// The partition tree memtable has been removed; parsing this value falls
56/// back to the default (bulk) memtable at runtime.
57const LEGACY_PARTITION_TREE_MEMTABLE_TYPE: &str = "partition_tree";
58
59pub(crate) fn parse_wal_options(
60    options_map: &HashMap<String, String>,
61) -> std::result::Result<WalOptions, serde_json::Error> {
62    options_map
63        .get(WAL_OPTIONS_KEY)
64        .map_or(Ok(WalOptions::default()), |encoded_wal_options| {
65            serde_json::from_str(encoded_wal_options)
66        })
67}
68
69/// Mode to handle duplicate rows while merging.
70#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, EnumString)]
71#[serde(rename_all = "snake_case")]
72#[strum(serialize_all = "snake_case")]
73pub enum MergeMode {
74    /// Keeps the last row.
75    #[default]
76    LastRow,
77    /// Keeps the last non-null field for each row.
78    LastNonNull,
79}
80
81// Note: We need to update [store_api::mito_engine_options::is_mito_engine_option_key()]
82// if we want expose the option to table options.
83/// Options that affect the entire region.
84///
85/// Users need to specify the options while creating/opening a region.
86#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
87#[serde(default)]
88pub struct RegionOptions {
89    /// Region SST files TTL.
90    pub ttl: Option<TimeToLive>,
91    /// Per-region auto flush interval override. Falls back to the global
92    /// `auto_flush_interval` engine config when unset.
93    #[serde(with = "humantime_serde")]
94    pub auto_flush_interval: Option<Duration>,
95    /// Compaction options.
96    pub compaction: CompactionOptions,
97    pub compaction_override: bool,
98    /// Per-output decode and encode parallelism. Falls back to the engine config
99    /// when unset; 1 disables parallelism without disabling the merge pipeline.
100    /// Independent of TWCS options and their database-level inheritance.
101    #[serde(skip_serializing_if = "Option::is_none")]
102    pub compaction_output_parallelism: Option<usize>,
103    /// Custom storage. Uses default storage if it is `None`.
104    pub storage: Option<String>,
105    /// If append mode is enabled, the region keeps duplicate rows.
106    pub append_mode: bool,
107    /// Whether to skip writing new WAL entries.
108    pub skip_wal: bool,
109    /// Wal options.
110    pub wal_options: WalOptions,
111    /// Index options.
112    pub index_options: IndexOptions,
113    /// Memtable options.
114    pub memtable: Option<MemtableOptions>,
115    /// The mode to merge duplicate rows.
116    /// Only takes effect when `append_mode` is `false`.
117    pub merge_mode: Option<MergeMode>,
118    /// SST format type.
119    pub sst_format: Option<FormatType>,
120    /// Max number of rows in a parquet row group. Uses [DEFAULT_ROW_GROUP_SIZE] if `None`.
121    #[serde(skip_serializing_if = "Option::is_none")]
122    pub max_row_group_row_count: Option<usize>,
123    /// Primary key encoding. Sparse regions require reserved table ID and TSID columns.
124    #[serde(skip_serializing_if = "Option::is_none")]
125    pub primary_key_encoding: Option<PrimaryKeyEncoding>,
126    /// Per-region write buffer size. A positive size flushes/stalls this region
127    /// independently of the global write buffer limit and rejects writes at twice
128    /// the configured size; zero disables both limits.
129    #[serde(skip_serializing_if = "Option::is_none")]
130    pub write_buffer_size: Option<ReadableSize>,
131    /// Whether to preserve original per-row sequence numbers when flushing.
132    /// Compaction always retains effective input sequences, regardless of this option.
133    ///
134    /// Only meaningful for append-only tables (`append_mode = true`): when enabled,
135    /// every row keeps its exact sequence number in memtables, flushed SSTs and
136    /// compacted SSTs, and scans with an exact `(checkpoint, upper_bound]` sequence
137    /// range can filter SST rows row-level instead of falling back to file-level
138    /// sequence metadata.
139    #[serde(default, skip_serializing_if = "is_false")]
140    pub preserve_row_sequence: bool,
141    /// Encoding for direct floating-point field columns in Parquet SSTs.
142    #[serde(default)]
143    pub float_field_encoding: FloatFieldEncoding,
144}
145
146fn is_false(value: &bool) -> bool {
147    !*value
148}
149
150impl RegionOptions {
151    /// Validates options.
152    pub fn validate(&self) -> Result<()> {
153        if self.append_mode {
154            ensure!(
155                self.merge_mode
156                    .is_none_or(|mode| mode == MergeMode::LastRow),
157                InvalidRegionOptionsSnafu {
158                    reason: "only last_row merge_mode is allowed when append_mode is enabled",
159                }
160            );
161        }
162        if let Some(auto_flush_interval) = self.auto_flush_interval {
163            ensure!(
164                auto_flush_interval > Duration::ZERO,
165                InvalidRegionOptionsSnafu {
166                    reason: "auto_flush_interval must be greater than 0",
167                }
168            );
169        }
170        if let Some(parallelism) = self.compaction_output_parallelism {
171            ensure!(
172                parallelism > 0,
173                InvalidRegionOptionsSnafu {
174                    reason: "compaction.experimental_output_parallelism must be greater than 0",
175                }
176            );
177        }
178        if let Some(row_count) = self.max_row_group_row_count {
179            ensure!(
180                row_count > 0 && row_count <= MAX_ROW_GROUP_ROW_COUNT_LIMIT,
181                InvalidRegionOptionsSnafu {
182                    reason: format!(
183                        "max_row_group_row_count must be in (0, {MAX_ROW_GROUP_ROW_COUNT_LIMIT}], got {row_count}",
184                    ),
185                }
186            );
187        }
188        if self.preserve_row_sequence {
189            ensure!(
190                self.append_mode,
191                InvalidRegionOptionsSnafu {
192                    reason: "preserve_row_sequence is only supported for append-only tables (append_mode must be true)",
193                }
194            );
195        }
196        let CompactionOptions::Twcs(options) = &self.compaction;
197        ensure!(
198            options.active_window_l1_merge_trigger >= 2,
199            InvalidRegionOptionsSnafu {
200                reason: "active_window.l1_merge_trigger must be at least 2",
201            }
202        );
203        ensure!(
204            options.inactive_window_l1_merge_trigger >= 2,
205            InvalidRegionOptionsSnafu {
206                reason: "inactive_window.l1_merge_trigger must be at least 2",
207            }
208        );
209        Ok(())
210    }
211
212    /// Rejects tag compression when the SST layout packs tags into `__primary_key`.
213    /// The caller must resolve the SST format before validation.
214    pub(crate) fn validate_column_compression(&self, metadata: &RegionMetadata) -> Result<()> {
215        if self.sst_format == Some(FormatType::Flat)
216            && metadata.primary_key_encoding == PrimaryKeyEncoding::Dense
217        {
218            return Ok(());
219        }
220        for column in &metadata.column_metadatas {
221            ensure!(
222                column.semantic_type != SemanticType::Tag
223                    || !column
224                        .column_schema
225                        .metadata()
226                        .contains_key(COMPRESSION_KEY),
227                InvalidRegionOptionsSnafu {
228                    reason: format!(
229                        "COMPRESSION on tag column {} requires flat SST format and dense primary key encoding",
230                        column.column_schema.name
231                    ),
232                }
233            );
234        }
235        Ok(())
236    }
237
238    /// Returns the configured row group size, falling back to [DEFAULT_ROW_GROUP_SIZE].
239    pub fn row_group_size(&self) -> usize {
240        self.max_row_group_row_count
241            .unwrap_or(DEFAULT_ROW_GROUP_SIZE)
242    }
243
244    /// Returns `true` if deduplication is needed.
245    pub fn need_dedup(&self) -> bool {
246        !self.append_mode
247    }
248
249    /// Returns the `merge_mode` if it is set, otherwise returns the default [`MergeMode`].
250    pub fn merge_mode(&self) -> MergeMode {
251        self.merge_mode.unwrap_or_default()
252    }
253
254    /// Returns the `auto_flush_interval` if it is set, otherwise returns `default`.
255    pub fn auto_flush_interval_or(&self, default: Duration) -> Duration {
256        self.auto_flush_interval.unwrap_or(default)
257    }
258
259    /// Returns the per-output compaction parallelism, falling back to the engine default.
260    pub fn compaction_output_parallelism_or(&self, default: usize) -> usize {
261        self.compaction_output_parallelism.unwrap_or(default)
262    }
263
264    /// Returns the `primary_key_encoding` if it is set, otherwise returns the default [`PrimaryKeyEncoding`].
265    pub fn primary_key_encoding(&self) -> PrimaryKeyEncoding {
266        self.primary_key_encoding.unwrap_or_default()
267    }
268
269    /// Returns whether the manifest of this region is published conditionally, which it is when
270    /// the region's writes require a lease and its WAL is the object store WAL.
271    pub(crate) fn requires_conditional_manifest_publication(&self, lease_enabled: bool) -> bool {
272        lease_enabled && matches!(self.wal_options, WalOptions::ObjectStore(_))
273    }
274}
275
276impl RegionOptions {
277    /// Parses [RegionOptions] from the raw `options_map`.
278    pub fn try_from_options(
279        region_id: RegionId,
280        options_map: &HashMap<String, String>,
281    ) -> Result<Self> {
282        let value = options_map_to_value(options_map);
283        let json = serde_json::to_string(&value).context(JsonOptionsSnafu)?;
284
285        // #[serde(flatten)] doesn't work with #[serde(default)] so we need to parse
286        // each field manually instead of using #[serde(flatten)] for `compaction`.
287        // See https://github.com/serde-rs/serde/issues/1626
288        let options: RegionOptionsWithoutEnum =
289            serde_json::from_str(&json).context(JsonOptionsSnafu)?;
290        let has_compaction_type =
291            validate_enum_options(options_map, "compaction.type", &[COMPACTION_TWCS_PREFIX])?;
292        let compaction = if has_compaction_type {
293            serde_json::from_str(&json).context(JsonOptionsSnafu)?
294        } else {
295            CompactionOptions::default()
296        };
297
298        let wal_options = parse_wal_options(options_map).context(JsonOptionsSnafu)?;
299
300        let index_options: IndexOptions = serde_json::from_str(&json).context(JsonOptionsSnafu)?;
301        let is_legacy_partition_tree = options_map
302            .get("memtable.type")
303            .map(|s| s.eq_ignore_ascii_case(LEGACY_PARTITION_TREE_MEMTABLE_TYPE))
304            .unwrap_or(false);
305        let memtable = if validate_enum_options(
306            options_map,
307            "memtable.type",
308            &[MEMTABLE_PARTITION_TREE_PREFIX, MEMTABLE_BULK_PREFIX],
309        )? {
310            if is_legacy_partition_tree {
311                // The partition tree memtable has been removed. Fall back to the
312                // default memtable; the primary key encoding (if any) is still
313                // read separately below from the legacy nested key.
314                None
315            } else {
316                Some(serde_json::from_str(&json).context(JsonOptionsSnafu)?)
317            }
318        } else {
319            None
320        };
321
322        let primary_key_encoding = options_map
323            .get(PRIMARY_KEY_ENCODING)
324            .or_else(|| options_map.get(MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING))
325            .map(|v| match v.to_lowercase().as_str() {
326                "dense" => Ok(PrimaryKeyEncoding::Dense),
327                "sparse" => Ok(PrimaryKeyEncoding::Sparse),
328                _ => Err(InvalidRegionOptionsSnafu {
329                    reason: format!("Invalid primary key encoding: {v}"),
330                }
331                .build()),
332            })
333            .transpose()?;
334
335        // The partition tree memtable has been removed. Besides falling back to
336        // the default memtable, also override the SST format to flat.
337        let mut sst_format = options.sst_format;
338        if is_legacy_partition_tree {
339            info!(
340                "Region {} specified the removed partition_tree memtable; \
341                 overriding memtable to the default and SST format to flat",
342                region_id
343            );
344            sst_format = Some(FormatType::Flat);
345        }
346
347        // Bulk memtable produces flat-encoded ranges and flushes them through
348        // `put_sst()`, so the SST format must be flat to match.
349        if matches!(memtable, Some(MemtableOptions::Bulk(_)))
350            || primary_key_encoding == Some(PrimaryKeyEncoding::Sparse)
351        {
352            if let Some(format) = sst_format
353                && format != FormatType::Flat
354            {
355                info!(
356                    "Region {} uses bulk memtable; overriding sst_format from {:?} to flat",
357                    region_id, format
358                );
359            }
360            sst_format = Some(FormatType::Flat);
361        }
362
363        let float_field_encoding = options.float_field_encoding.unwrap_or_default();
364
365        let compaction_override_flag = options_map
366            .get(COMPACTION_OVERRIDE)
367            .map(|v| matches!(v.to_lowercase().as_str(), "true" | "1"))
368            .unwrap_or(false);
369        let compaction_override = has_compaction_type || compaction_override_flag;
370
371        let opts = RegionOptions {
372            ttl: options.ttl,
373            auto_flush_interval: options.auto_flush_interval,
374            compaction,
375            compaction_override,
376            compaction_output_parallelism: options.compaction_output_parallelism,
377            storage: options.storage,
378            append_mode: options.append_mode,
379            skip_wal: options.skip_wal,
380            wal_options,
381            index_options,
382            memtable,
383            merge_mode: options.merge_mode,
384            sst_format,
385            max_row_group_row_count: options.max_row_group_row_count,
386            primary_key_encoding,
387            write_buffer_size: options.write_buffer_size,
388            preserve_row_sequence: options.preserve_row_sequence,
389            float_field_encoding,
390        };
391        opts.validate()?;
392
393        Ok(opts)
394    }
395
396    /// Parses region options using `default_bulk_config` for an implicit bulk threshold.
397    pub(crate) fn try_from_options_with_bulk_config(
398        region_id: RegionId,
399        options_map: &HashMap<String, String>,
400        default_bulk_config: &BulkMemtableConfig,
401    ) -> Result<Self> {
402        let mut options = Self::try_from_options(region_id, options_map)?;
403        if !options_map.contains_key(MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD)
404            && let Some(MemtableOptions::Bulk(config)) = &mut options.memtable
405        {
406            config.encode_bytes_threshold = default_bulk_config.encode_bytes_threshold;
407        }
408        Ok(options)
409    }
410}
411
412/// Options for compactions
413#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
414#[serde(tag = "compaction.type")]
415#[serde(rename_all = "snake_case")]
416pub enum CompactionOptions {
417    /// Time window compaction strategy.
418    #[serde(with = "prefix_twcs")]
419    Twcs(TwcsOptions),
420}
421
422impl CompactionOptions {
423    pub(crate) fn time_window(&self) -> Option<Duration> {
424        match self {
425            CompactionOptions::Twcs(opts) => opts.time_window,
426        }
427    }
428
429    pub(crate) fn remote_compaction(&self) -> bool {
430        match self {
431            CompactionOptions::Twcs(opts) => opts.remote_compaction,
432        }
433    }
434
435    pub(crate) fn fallback_to_local(&self) -> bool {
436        match self {
437            CompactionOptions::Twcs(opts) => opts.fallback_to_local,
438        }
439    }
440}
441
442impl Default for CompactionOptions {
443    fn default() -> Self {
444        Self::Twcs(TwcsOptions::default())
445    }
446}
447
448/// Time window compaction options.
449#[serde_as]
450#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
451#[serde(default)]
452pub struct TwcsOptions {
453    /// Minimum file num in the active time window to trigger a compaction.
454    #[serde_as(as = "DisplayFromStr")]
455    #[serde(rename = "active_window.trigger_file_num", alias = "trigger_file_num")]
456    pub active_window_trigger_file_num: usize,
457    /// Minimum L1 file num in the active window to allow a safety compaction.
458    #[serde_as(as = "DisplayFromStr")]
459    #[serde(rename = "active_window.l1_merge_trigger")]
460    pub active_window_l1_merge_trigger: usize,
461    /// Minimum file num in an inactive time window to trigger a compaction.
462    #[serde_as(as = "DisplayFromStr")]
463    #[serde(rename = "inactive_window.trigger_file_num")]
464    pub inactive_window_trigger_file_num: usize,
465    /// Minimum L1 file num to trigger a compaction in an inactive window.
466    #[serde_as(as = "DisplayFromStr")]
467    #[serde(rename = "inactive_window.l1_merge_trigger")]
468    pub inactive_window_l1_merge_trigger: usize,
469    /// Compaction time window defined when creating tables.
470    #[serde(with = "humantime_serde")]
471    pub time_window: Option<Duration>,
472    /// Soft output SST size threshold for TWCS and strict-window compaction.
473    /// `None` or zero disables size-based splitting. With a primary key, files
474    /// split at series boundaries, so a single series may exceed the threshold.
475    pub max_output_file_size: Option<ReadableSize>,
476    /// Whether to use remote compaction.
477    #[serde_as(as = "DisplayFromStr")]
478    pub remote_compaction: bool,
479    /// Whether to fall back to local compaction if remote compaction fails.
480    #[serde_as(as = "DisplayFromStr")]
481    pub fallback_to_local: bool,
482}
483
484with_prefix!(prefix_twcs "compaction.twcs.");
485
486impl TwcsOptions {
487    /// Returns time window in second resolution.
488    pub fn time_window_seconds(&self) -> Option<i64> {
489        self.time_window.and_then(|window| {
490            let window_secs = window.as_secs();
491            if window_secs == 0 {
492                None
493            } else {
494                window_secs.try_into().ok()
495            }
496        })
497    }
498}
499
500impl Default for TwcsOptions {
501    fn default() -> Self {
502        Self {
503            active_window_trigger_file_num: 4,
504            active_window_l1_merge_trigger: 16,
505            inactive_window_trigger_file_num: 2,
506            inactive_window_l1_merge_trigger: 8,
507            time_window: None,
508            max_output_file_size: Some(ReadableSize::mb(512)),
509            remote_compaction: false,
510            fallback_to_local: true,
511        }
512    }
513}
514
515/// We need to define a new struct without enum fields as `#[serde(default)]` does not
516/// support external tagging.
517#[serde_as]
518#[derive(Debug, Deserialize)]
519#[serde(default)]
520struct RegionOptionsWithoutEnum {
521    write_buffer_size: Option<ReadableSize>,
522    /// Region SST files TTL.
523    ttl: Option<TimeToLive>,
524    #[serde(with = "humantime_serde")]
525    auto_flush_interval: Option<Duration>,
526    #[serde(rename = "compaction.experimental_output_parallelism")]
527    #[serde_as(as = "NoneAsEmptyString")]
528    compaction_output_parallelism: Option<usize>,
529    storage: Option<String>,
530    #[serde_as(as = "DisplayFromStr")]
531    append_mode: bool,
532    #[serde_as(as = "DisplayFromStr")]
533    skip_wal: bool,
534    #[serde_as(as = "NoneAsEmptyString")]
535    merge_mode: Option<MergeMode>,
536    #[serde_as(as = "NoneAsEmptyString")]
537    sst_format: Option<FormatType>,
538    #[serde_as(as = "NoneAsEmptyString")]
539    max_row_group_row_count: Option<usize>,
540    #[serde_as(as = "DisplayFromStr")]
541    preserve_row_sequence: bool,
542    #[serde(rename = "experimental_sst_float_field_encoding")]
543    #[serde_as(as = "NoneAsEmptyString")]
544    float_field_encoding: Option<FloatFieldEncoding>,
545}
546
547impl Default for RegionOptionsWithoutEnum {
548    fn default() -> Self {
549        let options = RegionOptions::default();
550        RegionOptionsWithoutEnum {
551            write_buffer_size: options.write_buffer_size,
552            ttl: options.ttl,
553            auto_flush_interval: options.auto_flush_interval,
554            compaction_output_parallelism: options.compaction_output_parallelism,
555            storage: options.storage,
556            append_mode: options.append_mode,
557            skip_wal: options.skip_wal,
558            merge_mode: options.merge_mode,
559            sst_format: options.sst_format,
560            max_row_group_row_count: options.max_row_group_row_count,
561            preserve_row_sequence: options.preserve_row_sequence,
562            float_field_encoding: Some(options.float_field_encoding),
563        }
564    }
565}
566
567with_prefix!(prefix_inverted_index "index.inverted_index.");
568
569/// Options for index.
570#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
571#[serde(default)]
572pub struct IndexOptions {
573    /// Options for the inverted index.
574    #[serde(flatten, with = "prefix_inverted_index")]
575    pub inverted_index: InvertedIndexOptions,
576}
577
578/// Options for the inverted index.
579#[serde_as]
580#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
581#[serde(default)]
582pub struct InvertedIndexOptions {
583    /// The column ids that should be ignored when building the inverted index.
584    /// The column ids are separated by commas. For example, "1,2,3".
585    #[serde(deserialize_with = "deserialize_ignore_column_ids")]
586    #[serde(serialize_with = "serialize_ignore_column_ids")]
587    pub ignore_column_ids: Vec<ColumnId>,
588
589    /// The number of rows in a segment.
590    #[serde_as(as = "DisplayFromStr")]
591    pub segment_row_count: usize,
592}
593
594impl Default for InvertedIndexOptions {
595    fn default() -> Self {
596        Self {
597            ignore_column_ids: Vec::new(),
598            segment_row_count: DEFAULT_INDEX_SEGMENT_ROW_COUNT,
599        }
600    }
601}
602
603/// Options for region level memtable.
604#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
605#[serde(tag = "memtable.type", rename_all = "snake_case")]
606pub enum MemtableOptions {
607    TimeSeries,
608    #[serde(with = "prefix_bulk")]
609    Bulk(BulkMemtableConfig),
610}
611
612with_prefix!(prefix_bulk "memtable.bulk.");
613
614fn deserialize_ignore_column_ids<'de, D>(deserializer: D) -> Result<Vec<ColumnId>, D::Error>
615where
616    D: Deserializer<'de>,
617{
618    let s: String = Deserialize::deserialize(deserializer)?;
619    let mut column_ids = Vec::new();
620    if s.is_empty() {
621        return Ok(column_ids);
622    }
623    for item in s.split(',') {
624        let column_id = item.parse().map_err(D::Error::custom)?;
625        column_ids.push(column_id);
626    }
627    Ok(column_ids)
628}
629
630fn serialize_ignore_column_ids<S>(column_ids: &[ColumnId], serializer: S) -> Result<S::Ok, S::Error>
631where
632    S: serde::Serializer,
633{
634    let s = column_ids
635        .iter()
636        .map(|id| id.to_string())
637        .collect::<Vec<_>>()
638        .join(",");
639    serializer.serialize_str(&s)
640}
641
642/// Converts the `options` map to a json object.
643///
644/// Replaces "null" strings by `null` json values.
645fn options_map_to_value(options: &HashMap<String, String>) -> Value {
646    let map = options
647        .iter()
648        .map(|(key, value)| {
649            // Only convert the key to lowercase.
650            if value.eq_ignore_ascii_case("null") {
651                (key.clone(), Value::Null)
652            } else {
653                (key.clone(), Value::from(value.clone()))
654            }
655        })
656        .collect();
657    Value::Object(map)
658}
659
660// `#[serde(default)]` doesn't support enum (https://github.com/serde-rs/serde/issues/1799) so we
661// check the type key first.
662/// Validates whether the `options_map` has valid options for specific `enum_tag_key`
663/// and returns `true` if the map contains the enum tag.
664///
665/// Variant options must start with one of `enum_option_prefixes`. If variant options
666/// are provided, the tagged enum type key must also be provided.
667fn validate_enum_options(
668    options_map: &HashMap<String, String>,
669    enum_tag_key: &str,
670    enum_option_prefixes: &[&str],
671) -> Result<bool> {
672    let mut has_enum_options = false;
673    let mut has_tag = false;
674    for key in options_map.keys() {
675        if key == enum_tag_key {
676            has_tag = true;
677        } else if !has_enum_options
678            && enum_option_prefixes
679                .iter()
680                .any(|prefix| key.starts_with(prefix))
681        {
682            has_enum_options = true;
683        }
684
685        if has_tag && has_enum_options {
686            break;
687        }
688    }
689
690    // If tag is not provided, then other options for the enum should not exist.
691    ensure!(
692        has_tag || !has_enum_options,
693        InvalidRegionOptionsSnafu {
694            reason: format!("missing key {} in options", enum_tag_key),
695        }
696    );
697
698    Ok(has_tag)
699}
700
701#[cfg(test)]
702mod tests {
703    use common_error::ext::ErrorExt;
704    use common_error::status_code::StatusCode;
705    use common_wal::options::{KafkaWalOptions, ObjectStoreWalOptions};
706    use store_api::mito_engine_options::{
707        EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, SKIP_WAL_KEY, WRITE_BUFFER_SIZE_KEY,
708    };
709
710    use super::*;
711
712    fn make_map(options: &[(&str, &str)]) -> HashMap<String, String> {
713        options
714            .iter()
715            .map(|(k, v)| (k.to_string(), v.to_string()))
716            .collect()
717    }
718
719    #[test]
720    fn test_empty_region_options() {
721        let map = make_map(&[]);
722        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
723        assert_eq!(RegionOptions::default(), options);
724    }
725
726    #[test]
727    fn test_conditional_manifest_requires_lease_and_object_store_wal() {
728        let options = |wal_options| RegionOptions {
729            wal_options,
730            ..Default::default()
731        };
732        let object_store_wal = options(WalOptions::ObjectStore(ObjectStoreWalOptions::new(
733            "wal".to_string(),
734        )));
735
736        assert!(object_store_wal.requires_conditional_manifest_publication(true));
737        assert!(!object_store_wal.requires_conditional_manifest_publication(false));
738        assert!(!options(WalOptions::RaftEngine).requires_conditional_manifest_publication(true));
739    }
740
741    #[test]
742    fn test_float_field_encoding_defaults_and_parses() {
743        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &make_map(&[])).unwrap();
744        assert_eq!(FloatFieldEncoding::Default, options.float_field_encoding);
745
746        let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "default")]);
747        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
748        assert_eq!(FloatFieldEncoding::Default, options.float_field_encoding);
749
750        let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "byte_stream_split")]);
751        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
752        assert_eq!(
753            FloatFieldEncoding::ByteStreamSplit,
754            options.float_field_encoding
755        );
756
757        let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "invalid")]);
758        assert!(RegionOptions::try_from_options(RegionId::new(0, 0), &map).is_err());
759    }
760
761    #[test]
762    fn test_with_ttl() {
763        let map = make_map(&[("ttl", "7d")]);
764        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
765        let expect = RegionOptions {
766            ttl: Some(Duration::from_secs(3600 * 24 * 7).into()),
767            ..Default::default()
768        };
769        assert_eq!(expect, options);
770    }
771
772    #[test]
773    fn test_with_skip_wal() {
774        let map = make_map(&[(SKIP_WAL_KEY, "true")]);
775        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
776        assert!(options.skip_wal);
777    }
778
779    #[test]
780    fn test_with_auto_flush_interval() {
781        let map = make_map(&[("auto_flush_interval", "5m")]);
782        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
783        let expect = RegionOptions {
784            auto_flush_interval: Some(Duration::from_secs(5 * 60)),
785            ..Default::default()
786        };
787        assert_eq!(expect, options);
788    }
789
790    #[test]
791    fn test_with_zero_auto_flush_interval() {
792        let map = make_map(&[("auto_flush_interval", "0s")]);
793        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
794        assert!(
795            err.to_string().contains("auto_flush_interval"),
796            "unexpected error: {err}"
797        );
798    }
799
800    #[test]
801    fn test_with_write_buffer_size() {
802        let map = make_map(&[(WRITE_BUFFER_SIZE_KEY, "128MiB")]);
803        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
804        let expect = RegionOptions {
805            write_buffer_size: Some(ReadableSize::mb(128)),
806            ..Default::default()
807        };
808        assert_eq!(expect, options);
809    }
810
811    #[test]
812    fn test_with_zero_write_buffer_size() {
813        let map = make_map(&[(WRITE_BUFFER_SIZE_KEY, "0")]);
814        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
815        let expect = RegionOptions {
816            write_buffer_size: Some(ReadableSize::mb(0)),
817            ..Default::default()
818        };
819        assert_eq!(expect, options);
820    }
821
822    #[test]
823    fn test_with_storage() {
824        let map = make_map(&[("storage", "S3")]);
825        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
826        let expect = RegionOptions {
827            storage: Some("S3".to_string()),
828            ..Default::default()
829        };
830        assert_eq!(expect, options);
831    }
832
833    #[test]
834    fn test_without_compaction_type() {
835        let map = make_map(&[
836            ("compaction.twcs.trigger_file_num", "8"),
837            ("compaction.twcs.time_window", "2h"),
838        ]);
839        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
840        assert_eq!(StatusCode::InvalidArguments, err.status_code());
841    }
842
843    #[test]
844    fn test_with_compaction_type() {
845        let map = make_map(&[
846            ("compaction.twcs.active_window.trigger_file_num", "8"),
847            ("compaction.twcs.active_window.l1_merge_trigger", "16"),
848            ("compaction.twcs.inactive_window.trigger_file_num", "2"),
849            ("compaction.twcs.inactive_window.l1_merge_trigger", "12"),
850            ("compaction.twcs.time_window", "2h"),
851            ("compaction.type", "twcs"),
852        ]);
853        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
854        let expect = RegionOptions {
855            compaction: CompactionOptions::Twcs(TwcsOptions {
856                active_window_trigger_file_num: 8,
857                active_window_l1_merge_trigger: 16,
858                inactive_window_trigger_file_num: 2,
859                inactive_window_l1_merge_trigger: 12,
860                time_window: Some(Duration::from_secs(3600 * 2)),
861                ..Default::default()
862            }),
863            compaction_override: true,
864            ..Default::default()
865        };
866        assert_eq!(expect, options);
867    }
868
869    #[test]
870    fn test_twcs_window_trigger_below_two_is_accepted_for_compatibility() {
871        // Tables created before the >= 2 ALTER-time check may have persisted
872        // trigger values of 1; region open must still accept them.
873        let map = make_map(&[
874            ("compaction.twcs.active_window.trigger_file_num", "1"),
875            ("compaction.type", "twcs"),
876        ]);
877        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
878        let CompactionOptions::Twcs(twcs) = &options.compaction;
879        assert_eq!(1, twcs.active_window_trigger_file_num);
880
881        let map = make_map(&[
882            ("compaction.twcs.inactive_window.trigger_file_num", "1"),
883            ("compaction.type", "twcs"),
884        ]);
885        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
886        let CompactionOptions::Twcs(twcs) = &options.compaction;
887        assert_eq!(1, twcs.inactive_window_trigger_file_num);
888    }
889
890    #[test]
891    fn test_active_window_l1_merge_trigger_below_two_is_rejected() {
892        let map = make_map(&[
893            ("compaction.twcs.active_window.l1_merge_trigger", "1"),
894            ("compaction.type", "twcs"),
895        ]);
896
897        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
898        assert_eq!(StatusCode::InvalidArguments, err.status_code());
899    }
900
901    #[test]
902    fn test_inactive_window_l1_merge_trigger_defaults_to_eight() {
903        let value = serde_json::to_value(TwcsOptions::default()).unwrap();
904        assert_eq!(
905            Some("8"),
906            value
907                .get("inactive_window.l1_merge_trigger")
908                .and_then(|value| value.as_str())
909        );
910    }
911
912    #[test]
913    fn test_inactive_window_l1_merge_trigger_below_two_is_rejected() {
914        let map = make_map(&[
915            ("compaction.twcs.inactive_window.l1_merge_trigger", "1"),
916            ("compaction.type", "twcs"),
917        ]);
918
919        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
920        assert_eq!(StatusCode::InvalidArguments, err.status_code());
921    }
922
923    #[test]
924    fn test_with_compaction_override_true_without_compaction_type() {
925        let map = make_map(&[(COMPACTION_OVERRIDE, "true")]);
926        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
927        let expect = RegionOptions {
928            compaction_override: true,
929            ..Default::default()
930        };
931        assert_eq!(expect, options);
932    }
933
934    #[test]
935    fn test_with_compaction_override_false_without_compaction_type() {
936        let map = make_map(&[(COMPACTION_OVERRIDE, "false")]);
937        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
938        assert_eq!(RegionOptions::default(), options);
939    }
940
941    #[test]
942    fn test_compaction_twcs_options_still_require_compaction_type_with_override() {
943        let map = make_map(&[
944            (COMPACTION_OVERRIDE, "true"),
945            ("compaction.twcs.time_window", "2h"),
946        ]);
947        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
948        assert_eq!(StatusCode::InvalidArguments, err.status_code());
949    }
950
951    fn test_with_wal_options(wal_options: &WalOptions) -> bool {
952        let encoded_wal_options = serde_json::to_string(&wal_options).unwrap();
953        let map = make_map(&[(WAL_OPTIONS_KEY, &encoded_wal_options)]);
954        let got = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
955        let expect = RegionOptions {
956            wal_options: wal_options.clone(),
957            ..Default::default()
958        };
959        expect == got
960    }
961
962    #[test]
963    fn test_with_index() {
964        let map = make_map(&[
965            ("index.inverted_index.ignore_column_ids", "1,2,3"),
966            ("index.inverted_index.segment_row_count", "512"),
967        ]);
968        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
969        let expect = RegionOptions {
970            index_options: IndexOptions {
971                inverted_index: InvertedIndexOptions {
972                    ignore_column_ids: vec![1, 2, 3],
973                    segment_row_count: 512,
974                },
975            },
976            ..Default::default()
977        };
978        assert_eq!(expect, options);
979    }
980
981    // No need to add compatible tests for RegionOptions since the above tests already check for compatibility.
982    #[test]
983    fn test_with_any_wal_options() {
984        let all_wal_options = [
985            WalOptions::RaftEngine,
986            WalOptions::Kafka(KafkaWalOptions::new("test_topic".to_string())),
987        ];
988        all_wal_options.iter().all(test_with_wal_options);
989    }
990
991    #[test]
992    fn test_with_memtable() {
993        let map = make_map(&[("memtable.type", "time_series")]);
994        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
995        let expect = RegionOptions {
996            memtable: Some(MemtableOptions::TimeSeries),
997            ..Default::default()
998        };
999        assert_eq!(expect, options);
1000
1001        let map = make_map(&[("memtable.type", "bulk")]);
1002        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1003        let expect = RegionOptions {
1004            memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig::default())),
1005            sst_format: Some(FormatType::Flat),
1006            ..Default::default()
1007        };
1008        assert_eq!(expect, options);
1009
1010        let map = make_map(&[
1011            ("memtable.type", "bulk"),
1012            ("memtable.bulk.merge_threshold", "7"),
1013            ("memtable.bulk.encode_row_threshold", "11"),
1014            ("memtable.bulk.encode_bytes_threshold", "13"),
1015            ("memtable.bulk.max_merge_groups", "17"),
1016        ]);
1017        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1018        let expect = RegionOptions {
1019            memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig {
1020                merge_threshold: 7,
1021                encode_row_threshold: 11,
1022                encode_bytes_threshold: 13,
1023                max_merge_groups: 17,
1024            })),
1025            sst_format: Some(FormatType::Flat),
1026            ..Default::default()
1027        };
1028        assert_eq!(expect, options);
1029
1030        // Legacy partition_tree memtable falls back to the default memtable and
1031        // overrides the SST format to flat.
1032        let map = make_map(&[("memtable.type", "partition_tree")]);
1033        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1034        let expect = RegionOptions {
1035            memtable: None,
1036            sst_format: Some(FormatType::Flat),
1037            ..Default::default()
1038        };
1039        assert_eq!(expect, options);
1040
1041        // Legacy partition_tree options are tolerated alongside the type tag.
1042        let map = make_map(&[
1043            ("memtable.type", "partition_tree"),
1044            ("memtable.partition_tree.index_max_keys_per_shard", "2048"),
1045            ("memtable.partition_tree.fork_dictionary_bytes", "128M"),
1046        ]);
1047        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1048        let expect = RegionOptions {
1049            memtable: None,
1050            sst_format: Some(FormatType::Flat),
1051            ..Default::default()
1052        };
1053        assert_eq!(expect, options);
1054    }
1055
1056    #[test]
1057    fn test_sparse_forces_flat_sst() {
1058        for format in [None, Some("primary_key"), Some("flat")] {
1059            let mut map = make_map(&[("primary_key_encoding", "sparse")]);
1060            if let Some(format) = format {
1061                map.insert("sst_format".into(), format.into());
1062            }
1063            let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1064            assert_eq!(options.primary_key_encoding(), PrimaryKeyEncoding::Sparse);
1065            assert_eq!(options.sst_format, Some(FormatType::Flat));
1066        }
1067    }
1068
1069    #[test]
1070    fn test_primary_key_encoding() {
1071        // New top-level key.
1072        let map = make_map(&[("primary_key_encoding", "sparse")]);
1073        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1074        assert_eq!(options.primary_key_encoding(), PrimaryKeyEncoding::Sparse);
1075        assert_eq!(
1076            options.primary_key_encoding,
1077            Some(PrimaryKeyEncoding::Sparse)
1078        );
1079
1080        // Legacy memtable.type=partition_tree + legacy encoding.
1081        let map = make_map(&[
1082            ("memtable.type", "partition_tree"),
1083            ("memtable.partition_tree.primary_key_encoding", "sparse"),
1084        ]);
1085        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1086        assert_eq!(options.memtable, None);
1087        assert_eq!(options.sst_format, Some(FormatType::Flat));
1088        assert_eq!(options.primary_key_encoding(), PrimaryKeyEncoding::Sparse);
1089
1090        // Invalid value rejected.
1091        let map = make_map(&[("primary_key_encoding", "bogus")]);
1092        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1093        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1094    }
1095
1096    #[test]
1097    fn test_legacy_partition_tree_overrides_sst_format() {
1098        // Legacy partition_tree memtable falls back to the default memtable and
1099        // overrides the SST format to flat, even when a different format was set.
1100        let map = make_map(&[
1101            ("memtable.type", "partition_tree"),
1102            ("sst_format", "primary_key"),
1103        ]);
1104        let options = RegionOptions::try_from_options(RegionId::new(1, 1), &map).unwrap();
1105        assert_eq!(options.memtable, None);
1106        assert_eq!(options.sst_format, Some(FormatType::Flat));
1107    }
1108
1109    #[test]
1110    fn test_bulk_memtable_overrides_sst_format() {
1111        // Bulk memtable produces flat-encoded ranges, so an explicit
1112        // `sst_format=primary_key` must be overridden to flat to keep the
1113        // in-memory and on-disk encodings in sync.
1114        let map = make_map(&[("memtable.type", "bulk"), ("sst_format", "primary_key")]);
1115        let options = RegionOptions::try_from_options(RegionId::new(1, 1), &map).unwrap();
1116        assert_eq!(
1117            options.memtable,
1118            Some(MemtableOptions::Bulk(BulkMemtableConfig::default()))
1119        );
1120        assert_eq!(options.sst_format, Some(FormatType::Flat));
1121    }
1122
1123    #[test]
1124    fn test_unknown_memtable_type() {
1125        let map = make_map(&[("memtable.type", "no_such_memtable")]);
1126        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1127        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1128    }
1129
1130    #[test]
1131    fn test_without_memtable_type() {
1132        let map = make_map(&[("memtable.partition_tree.index_max_keys_per_shard", "2048")]);
1133        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1134        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1135
1136        let map = make_map(&[("memtable.bulk.merge_threshold", "7")]);
1137        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1138        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1139    }
1140
1141    #[test]
1142    fn test_with_merge_mode() {
1143        let map = make_map(&[("merge_mode", "last_row")]);
1144        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1145        assert_eq!(MergeMode::LastRow, options.merge_mode());
1146
1147        let map = make_map(&[("merge_mode", "last_non_null")]);
1148        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1149        assert_eq!(MergeMode::LastNonNull, options.merge_mode());
1150
1151        let map = make_map(&[("merge_mode", "unknown")]);
1152        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1153        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1154    }
1155
1156    #[test]
1157    fn test_append_mode_allows_last_row_merge_mode() {
1158        let map = make_map(&[("append_mode", "true"), ("merge_mode", "last_row")]);
1159        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1160        assert!(options.append_mode);
1161        assert_eq!(MergeMode::LastRow, options.merge_mode());
1162
1163        let map = make_map(&[("append_mode", "true"), ("merge_mode", "last_non_null")]);
1164        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1165        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1166    }
1167
1168    #[test]
1169    fn test_with_all() {
1170        let wal_options = WalOptions::Kafka(KafkaWalOptions::new("test_topic".to_string()));
1171        let map = make_map(&[
1172            ("ttl", "7d"),
1173            ("compaction.twcs.trigger_file_num", "8"),
1174            ("compaction.twcs.max_output_file_size", "1GB"),
1175            ("compaction.twcs.time_window", "2h"),
1176            ("compaction.type", "twcs"),
1177            ("compaction.twcs.remote_compaction", "false"),
1178            ("compaction.twcs.fallback_to_local", "true"),
1179            ("storage", "S3"),
1180            ("append_mode", "false"),
1181            ("index.inverted_index.ignore_column_ids", "1,2,3"),
1182            ("index.inverted_index.segment_row_count", "512"),
1183            (
1184                WAL_OPTIONS_KEY,
1185                &serde_json::to_string(&wal_options).unwrap(),
1186            ),
1187            ("memtable.type", "bulk"),
1188            ("memtable.bulk.merge_threshold", "7"),
1189            ("memtable.bulk.encode_row_threshold", "11"),
1190            ("memtable.bulk.encode_bytes_threshold", "13"),
1191            ("memtable.bulk.max_merge_groups", "17"),
1192            ("merge_mode", "last_non_null"),
1193        ]);
1194        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1195        let expect = RegionOptions {
1196            ttl: Some(Duration::from_secs(3600 * 24 * 7).into()),
1197            auto_flush_interval: None,
1198            compaction: CompactionOptions::Twcs(TwcsOptions {
1199                active_window_trigger_file_num: 8,
1200                active_window_l1_merge_trigger: 16,
1201                inactive_window_trigger_file_num: 2,
1202                inactive_window_l1_merge_trigger: 8,
1203                time_window: Some(Duration::from_secs(3600 * 2)),
1204                max_output_file_size: Some(ReadableSize::gb(1)),
1205                remote_compaction: false,
1206                fallback_to_local: true,
1207            }),
1208            compaction_override: true,
1209            compaction_output_parallelism: None,
1210            storage: Some("S3".to_string()),
1211            append_mode: false,
1212            skip_wal: false,
1213            wal_options,
1214            index_options: IndexOptions {
1215                inverted_index: InvertedIndexOptions {
1216                    ignore_column_ids: vec![1, 2, 3],
1217                    segment_row_count: 512,
1218                },
1219            },
1220            memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig {
1221                merge_threshold: 7,
1222                encode_row_threshold: 11,
1223                encode_bytes_threshold: 13,
1224                max_merge_groups: 17,
1225            })),
1226            merge_mode: Some(MergeMode::LastNonNull),
1227            sst_format: Some(FormatType::Flat),
1228            max_row_group_row_count: None,
1229            primary_key_encoding: None,
1230            write_buffer_size: None,
1231            preserve_row_sequence: false,
1232            float_field_encoding: FloatFieldEncoding::default(),
1233        };
1234        assert_eq!(expect, options);
1235    }
1236
1237    #[test]
1238    fn test_with_preserve_row_sequence() {
1239        let map = make_map(&[("append_mode", "false"), ("preserve_row_sequence", "true")]);
1240        let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1241        assert_eq!(StatusCode::InvalidArguments, err.status_code());
1242        assert!(err.to_string().contains("preserve_row_sequence"));
1243
1244        let map = make_map(&[("append_mode", "true"), ("preserve_row_sequence", "true")]);
1245        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1246        assert!(options.append_mode);
1247        assert!(options.preserve_row_sequence);
1248
1249        let map = make_map(&[("append_mode", "true")]);
1250        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1251        assert!(!options.preserve_row_sequence);
1252
1253        let map = make_map(&[]);
1254        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1255        assert_eq!(RegionOptions::default(), options);
1256    }
1257
1258    #[test]
1259    fn test_region_options_serde() {
1260        let options = RegionOptions {
1261            ttl: Some(Duration::from_secs(3600 * 24 * 7).into()),
1262            auto_flush_interval: None,
1263            compaction: CompactionOptions::Twcs(TwcsOptions {
1264                active_window_trigger_file_num: 8,
1265                active_window_l1_merge_trigger: 8,
1266                inactive_window_trigger_file_num: 2,
1267                inactive_window_l1_merge_trigger: 8,
1268                time_window: Some(Duration::from_secs(3600 * 2)),
1269                max_output_file_size: None,
1270                remote_compaction: false,
1271                fallback_to_local: true,
1272            }),
1273            compaction_override: false,
1274            storage: Some("S3".to_string()),
1275            append_mode: false,
1276            skip_wal: false,
1277            wal_options: WalOptions::Kafka(KafkaWalOptions::new("test_topic".to_string())),
1278            index_options: IndexOptions {
1279                inverted_index: InvertedIndexOptions {
1280                    ignore_column_ids: vec![1, 2, 3],
1281                    segment_row_count: 512,
1282                },
1283            },
1284            memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig::default())),
1285            merge_mode: Some(MergeMode::LastNonNull),
1286            sst_format: None,
1287            max_row_group_row_count: None,
1288            primary_key_encoding: None,
1289            write_buffer_size: Some(ReadableSize::mb(128)),
1290            compaction_output_parallelism: Some(4),
1291            preserve_row_sequence: true,
1292            float_field_encoding: FloatFieldEncoding::default(),
1293        };
1294        let region_options_json_str = serde_json::to_string(&options).unwrap();
1295        assert!(region_options_json_str.contains("preserve_row_sequence"));
1296        let got: RegionOptions = serde_json::from_str(&region_options_json_str).unwrap();
1297        assert_eq!(options, got);
1298
1299        // Old manifests without the key default to false.
1300        let old_region_options_json_str = r#"{"ttl":null}"#;
1301        let got: RegionOptions = serde_json::from_str(old_region_options_json_str).unwrap();
1302        assert_eq!(None, got.write_buffer_size);
1303        assert_eq!(None, got.compaction_output_parallelism);
1304        assert_eq!(4, got.compaction_output_parallelism_or(4));
1305        assert!(!got.preserve_row_sequence);
1306        assert_eq!(FloatFieldEncoding::Default, got.float_field_encoding);
1307        let CompactionOptions::Twcs(twcs) = got.compaction;
1308        assert_eq!(16, twcs.active_window_l1_merge_trigger);
1309        assert_eq!(8, twcs.inactive_window_l1_merge_trigger);
1310
1311        let default_json = serde_json::to_value(RegionOptions::default()).unwrap();
1312        assert!(default_json.get(WRITE_BUFFER_SIZE_KEY).is_none());
1313    }
1314
1315    #[test]
1316    fn test_region_options_str_serde() {
1317        // Notes: use empty string for `ignore_column_ids` to test the empty string case.
1318        let region_options_json_str = r#"{
1319  "ttl": "7days",
1320  "compaction": {
1321    "compaction.type": "twcs",
1322    "compaction.twcs.trigger_file_num": "8",
1323    "compaction.twcs.max_output_file_size": "7MB",
1324    "compaction.twcs.time_window": "2h"
1325  },
1326  "storage": "S3",
1327  "append_mode": false,
1328  "wal_options": {
1329    "wal.provider": "kafka",
1330    "wal.kafka.topic": "test_topic"
1331  },
1332  "index_options": {
1333    "index.inverted_index.ignore_column_ids": "",
1334    "index.inverted_index.segment_row_count": "512"
1335  },
1336  "memtable": {
1337    "memtable.type": "bulk"
1338  },
1339  "merge_mode": "last_non_null"
1340}"#;
1341        let got: RegionOptions = serde_json::from_str(region_options_json_str).unwrap();
1342        let options = RegionOptions {
1343            ttl: Some(Duration::from_secs(3600 * 24 * 7).into()),
1344            auto_flush_interval: None,
1345            compaction: CompactionOptions::Twcs(TwcsOptions {
1346                active_window_trigger_file_num: 8,
1347                active_window_l1_merge_trigger: 16,
1348                inactive_window_trigger_file_num: 2,
1349                inactive_window_l1_merge_trigger: 8,
1350                time_window: Some(Duration::from_secs(3600 * 2)),
1351                max_output_file_size: Some(ReadableSize::mb(7)),
1352                remote_compaction: false,
1353                fallback_to_local: true,
1354            }),
1355            compaction_override: false,
1356            storage: Some("S3".to_string()),
1357            append_mode: false,
1358            skip_wal: false,
1359            wal_options: WalOptions::Kafka(KafkaWalOptions::new("test_topic".to_string())),
1360            index_options: IndexOptions {
1361                inverted_index: InvertedIndexOptions {
1362                    ignore_column_ids: vec![],
1363                    segment_row_count: 512,
1364                },
1365            },
1366            memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig::default())),
1367            merge_mode: Some(MergeMode::LastNonNull),
1368            sst_format: None,
1369            max_row_group_row_count: None,
1370            primary_key_encoding: None,
1371            write_buffer_size: None,
1372            compaction_output_parallelism: None,
1373            preserve_row_sequence: false,
1374            float_field_encoding: FloatFieldEncoding::default(),
1375        };
1376        assert_eq!(options, got);
1377    }
1378
1379    #[test]
1380    fn test_compaction_output_parallelism_preserves_strategy_inheritance() {
1381        for parallelism in [None, Some(1), Some(4)] {
1382            let mut map = make_map(&[]);
1383            if let Some(parallelism) = parallelism {
1384                map.insert(
1385                    "compaction.experimental_output_parallelism".to_string(),
1386                    parallelism.to_string(),
1387                );
1388            }
1389            let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1390            assert_eq!(parallelism, options.compaction_output_parallelism);
1391            assert_eq!(
1392                parallelism.unwrap_or(8),
1393                options.compaction_output_parallelism_or(8)
1394            );
1395            assert!(!options.compaction_override);
1396
1397            map.insert("compaction.type".to_string(), "twcs".to_string());
1398            map.insert("compaction.twcs.time_window".to_string(), "2h".to_string());
1399            let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1400            assert_eq!(parallelism, options.compaction_output_parallelism);
1401            assert_eq!(
1402                Some(Duration::from_secs(7200)),
1403                options.compaction.time_window()
1404            );
1405        }
1406    }
1407
1408    #[test]
1409    fn test_invalid_compaction_output_parallelism() {
1410        for parallelism in ["0", "-1", "1.5", "invalid", "18446744073709551616"] {
1411            let map = make_map(&[("compaction.experimental_output_parallelism", parallelism)]);
1412            assert!(RegionOptions::try_from_options(RegionId::new(0, 0), &map).is_err());
1413        }
1414    }
1415
1416    #[test]
1417    fn test_max_row_group_row_count() {
1418        // Default falls back to DEFAULT_ROW_GROUP_SIZE.
1419        assert_eq!(None, RegionOptions::default().max_row_group_row_count);
1420        assert_eq!(
1421            DEFAULT_ROW_GROUP_SIZE,
1422            RegionOptions::default().row_group_size()
1423        );
1424
1425        // A configured value is parsed and used as the row group size.
1426        let map = make_map(&[("max_row_group_row_count", "51200")]);
1427        let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1428        assert_eq!(Some(51200), options.max_row_group_row_count);
1429        assert_eq!(51200, options.row_group_size());
1430
1431        // Zero is rejected.
1432        let map = make_map(&[("max_row_group_row_count", "0")]);
1433        assert!(RegionOptions::try_from_options(RegionId::new(0, 0), &map).is_err());
1434    }
1435}