1use std::collections::HashMap;
20use std::time::Duration;
21
22use api::v1::SemanticType;
23use common_base::readable_size::ReadableSize;
24use common_telemetry::info;
25use common_time::TimeToLive;
26use common_wal::options::{WAL_OPTIONS_KEY, WalOptions};
27use datatypes::schema::COMPRESSION_KEY;
28use serde::de::Error as _;
29use serde::{Deserialize, Deserializer, Serialize};
30use serde_json::Value;
31use serde_with::{DisplayFromStr, NoneAsEmptyString, serde_as, with_prefix};
32use snafu::{ResultExt, ensure};
33use store_api::codec::PrimaryKeyEncoding;
34use store_api::metadata::RegionMetadata;
35use store_api::metric_engine_consts::{
36 MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING, PRIMARY_KEY_ENCODING,
37};
38use store_api::mito_engine_options::{
39 COMPACTION_OVERRIDE, COMPACTION_TWCS_PREFIX, FloatFieldEncoding, MAX_ROW_GROUP_ROW_COUNT_LIMIT,
40};
41use store_api::storage::{ColumnId, RegionId};
42use strum::EnumString;
43
44use crate::error::{InvalidRegionOptionsSnafu, JsonOptionsSnafu, Result};
45use crate::memtable::bulk::BulkMemtableConfig;
46use crate::sst::FormatType;
47use crate::sst::parquet::DEFAULT_ROW_GROUP_SIZE;
48
49const DEFAULT_INDEX_SEGMENT_ROW_COUNT: usize = 1024;
50const MEMTABLE_PARTITION_TREE_PREFIX: &str = "memtable.partition_tree.";
51const MEMTABLE_BULK_PREFIX: &str = "memtable.bulk.";
52const MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD: &str = "memtable.bulk.encode_bytes_threshold";
53
54const LEGACY_PARTITION_TREE_MEMTABLE_TYPE: &str = "partition_tree";
58
59pub(crate) fn parse_wal_options(
60 options_map: &HashMap<String, String>,
61) -> std::result::Result<WalOptions, serde_json::Error> {
62 options_map
63 .get(WAL_OPTIONS_KEY)
64 .map_or(Ok(WalOptions::default()), |encoded_wal_options| {
65 serde_json::from_str(encoded_wal_options)
66 })
67}
68
69#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, EnumString)]
71#[serde(rename_all = "snake_case")]
72#[strum(serialize_all = "snake_case")]
73pub enum MergeMode {
74 #[default]
76 LastRow,
77 LastNonNull,
79}
80
81#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]
87#[serde(default)]
88pub struct RegionOptions {
89 pub ttl: Option<TimeToLive>,
91 #[serde(with = "humantime_serde")]
94 pub auto_flush_interval: Option<Duration>,
95 pub compaction: CompactionOptions,
97 pub compaction_override: bool,
98 #[serde(skip_serializing_if = "Option::is_none")]
102 pub compaction_output_parallelism: Option<usize>,
103 pub storage: Option<String>,
105 pub append_mode: bool,
107 pub skip_wal: bool,
109 pub wal_options: WalOptions,
111 pub index_options: IndexOptions,
113 pub memtable: Option<MemtableOptions>,
115 pub merge_mode: Option<MergeMode>,
118 pub sst_format: Option<FormatType>,
120 #[serde(skip_serializing_if = "Option::is_none")]
122 pub max_row_group_row_count: Option<usize>,
123 #[serde(skip_serializing_if = "Option::is_none")]
125 pub primary_key_encoding: Option<PrimaryKeyEncoding>,
126 #[serde(skip_serializing_if = "Option::is_none")]
130 pub write_buffer_size: Option<ReadableSize>,
131 #[serde(default, skip_serializing_if = "is_false")]
140 pub preserve_row_sequence: bool,
141 #[serde(default)]
143 pub float_field_encoding: FloatFieldEncoding,
144}
145
146fn is_false(value: &bool) -> bool {
147 !*value
148}
149
150impl RegionOptions {
151 pub fn validate(&self) -> Result<()> {
153 if self.append_mode {
154 ensure!(
155 self.merge_mode
156 .is_none_or(|mode| mode == MergeMode::LastRow),
157 InvalidRegionOptionsSnafu {
158 reason: "only last_row merge_mode is allowed when append_mode is enabled",
159 }
160 );
161 }
162 if let Some(auto_flush_interval) = self.auto_flush_interval {
163 ensure!(
164 auto_flush_interval > Duration::ZERO,
165 InvalidRegionOptionsSnafu {
166 reason: "auto_flush_interval must be greater than 0",
167 }
168 );
169 }
170 if let Some(parallelism) = self.compaction_output_parallelism {
171 ensure!(
172 parallelism > 0,
173 InvalidRegionOptionsSnafu {
174 reason: "compaction.experimental_output_parallelism must be greater than 0",
175 }
176 );
177 }
178 if let Some(row_count) = self.max_row_group_row_count {
179 ensure!(
180 row_count > 0 && row_count <= MAX_ROW_GROUP_ROW_COUNT_LIMIT,
181 InvalidRegionOptionsSnafu {
182 reason: format!(
183 "max_row_group_row_count must be in (0, {MAX_ROW_GROUP_ROW_COUNT_LIMIT}], got {row_count}",
184 ),
185 }
186 );
187 }
188 if self.preserve_row_sequence {
189 ensure!(
190 self.append_mode,
191 InvalidRegionOptionsSnafu {
192 reason: "preserve_row_sequence is only supported for append-only tables (append_mode must be true)",
193 }
194 );
195 }
196 let CompactionOptions::Twcs(options) = &self.compaction;
197 ensure!(
198 options.active_window_l1_merge_trigger >= 2,
199 InvalidRegionOptionsSnafu {
200 reason: "active_window.l1_merge_trigger must be at least 2",
201 }
202 );
203 ensure!(
204 options.inactive_window_l1_merge_trigger >= 2,
205 InvalidRegionOptionsSnafu {
206 reason: "inactive_window.l1_merge_trigger must be at least 2",
207 }
208 );
209 Ok(())
210 }
211
212 pub(crate) fn validate_column_compression(&self, metadata: &RegionMetadata) -> Result<()> {
215 if self.sst_format == Some(FormatType::Flat)
216 && metadata.primary_key_encoding == PrimaryKeyEncoding::Dense
217 {
218 return Ok(());
219 }
220 for column in &metadata.column_metadatas {
221 ensure!(
222 column.semantic_type != SemanticType::Tag
223 || !column
224 .column_schema
225 .metadata()
226 .contains_key(COMPRESSION_KEY),
227 InvalidRegionOptionsSnafu {
228 reason: format!(
229 "COMPRESSION on tag column {} requires flat SST format and dense primary key encoding",
230 column.column_schema.name
231 ),
232 }
233 );
234 }
235 Ok(())
236 }
237
238 pub fn row_group_size(&self) -> usize {
240 self.max_row_group_row_count
241 .unwrap_or(DEFAULT_ROW_GROUP_SIZE)
242 }
243
244 pub fn need_dedup(&self) -> bool {
246 !self.append_mode
247 }
248
249 pub fn merge_mode(&self) -> MergeMode {
251 self.merge_mode.unwrap_or_default()
252 }
253
254 pub fn auto_flush_interval_or(&self, default: Duration) -> Duration {
256 self.auto_flush_interval.unwrap_or(default)
257 }
258
259 pub fn compaction_output_parallelism_or(&self, default: usize) -> usize {
261 self.compaction_output_parallelism.unwrap_or(default)
262 }
263
264 pub fn primary_key_encoding(&self) -> PrimaryKeyEncoding {
266 self.primary_key_encoding.unwrap_or_default()
267 }
268
269 pub(crate) fn requires_conditional_manifest_publication(&self, lease_enabled: bool) -> bool {
272 lease_enabled && matches!(self.wal_options, WalOptions::ObjectStore(_))
273 }
274}
275
276impl RegionOptions {
277 pub fn try_from_options(
279 region_id: RegionId,
280 options_map: &HashMap<String, String>,
281 ) -> Result<Self> {
282 let value = options_map_to_value(options_map);
283 let json = serde_json::to_string(&value).context(JsonOptionsSnafu)?;
284
285 let options: RegionOptionsWithoutEnum =
289 serde_json::from_str(&json).context(JsonOptionsSnafu)?;
290 let has_compaction_type =
291 validate_enum_options(options_map, "compaction.type", &[COMPACTION_TWCS_PREFIX])?;
292 let compaction = if has_compaction_type {
293 serde_json::from_str(&json).context(JsonOptionsSnafu)?
294 } else {
295 CompactionOptions::default()
296 };
297
298 let wal_options = parse_wal_options(options_map).context(JsonOptionsSnafu)?;
299
300 let index_options: IndexOptions = serde_json::from_str(&json).context(JsonOptionsSnafu)?;
301 let is_legacy_partition_tree = options_map
302 .get("memtable.type")
303 .map(|s| s.eq_ignore_ascii_case(LEGACY_PARTITION_TREE_MEMTABLE_TYPE))
304 .unwrap_or(false);
305 let memtable = if validate_enum_options(
306 options_map,
307 "memtable.type",
308 &[MEMTABLE_PARTITION_TREE_PREFIX, MEMTABLE_BULK_PREFIX],
309 )? {
310 if is_legacy_partition_tree {
311 None
315 } else {
316 Some(serde_json::from_str(&json).context(JsonOptionsSnafu)?)
317 }
318 } else {
319 None
320 };
321
322 let primary_key_encoding = options_map
323 .get(PRIMARY_KEY_ENCODING)
324 .or_else(|| options_map.get(MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING))
325 .map(|v| match v.to_lowercase().as_str() {
326 "dense" => Ok(PrimaryKeyEncoding::Dense),
327 "sparse" => Ok(PrimaryKeyEncoding::Sparse),
328 _ => Err(InvalidRegionOptionsSnafu {
329 reason: format!("Invalid primary key encoding: {v}"),
330 }
331 .build()),
332 })
333 .transpose()?;
334
335 let mut sst_format = options.sst_format;
338 if is_legacy_partition_tree {
339 info!(
340 "Region {} specified the removed partition_tree memtable; \
341 overriding memtable to the default and SST format to flat",
342 region_id
343 );
344 sst_format = Some(FormatType::Flat);
345 }
346
347 if matches!(memtable, Some(MemtableOptions::Bulk(_)))
350 || primary_key_encoding == Some(PrimaryKeyEncoding::Sparse)
351 {
352 if let Some(format) = sst_format
353 && format != FormatType::Flat
354 {
355 info!(
356 "Region {} uses bulk memtable; overriding sst_format from {:?} to flat",
357 region_id, format
358 );
359 }
360 sst_format = Some(FormatType::Flat);
361 }
362
363 let float_field_encoding = options.float_field_encoding.unwrap_or_default();
364
365 let compaction_override_flag = options_map
366 .get(COMPACTION_OVERRIDE)
367 .map(|v| matches!(v.to_lowercase().as_str(), "true" | "1"))
368 .unwrap_or(false);
369 let compaction_override = has_compaction_type || compaction_override_flag;
370
371 let opts = RegionOptions {
372 ttl: options.ttl,
373 auto_flush_interval: options.auto_flush_interval,
374 compaction,
375 compaction_override,
376 compaction_output_parallelism: options.compaction_output_parallelism,
377 storage: options.storage,
378 append_mode: options.append_mode,
379 skip_wal: options.skip_wal,
380 wal_options,
381 index_options,
382 memtable,
383 merge_mode: options.merge_mode,
384 sst_format,
385 max_row_group_row_count: options.max_row_group_row_count,
386 primary_key_encoding,
387 write_buffer_size: options.write_buffer_size,
388 preserve_row_sequence: options.preserve_row_sequence,
389 float_field_encoding,
390 };
391 opts.validate()?;
392
393 Ok(opts)
394 }
395
396 pub(crate) fn try_from_options_with_bulk_config(
398 region_id: RegionId,
399 options_map: &HashMap<String, String>,
400 default_bulk_config: &BulkMemtableConfig,
401 ) -> Result<Self> {
402 let mut options = Self::try_from_options(region_id, options_map)?;
403 if !options_map.contains_key(MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD)
404 && let Some(MemtableOptions::Bulk(config)) = &mut options.memtable
405 {
406 config.encode_bytes_threshold = default_bulk_config.encode_bytes_threshold;
407 }
408 Ok(options)
409 }
410}
411
412#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
414#[serde(tag = "compaction.type")]
415#[serde(rename_all = "snake_case")]
416pub enum CompactionOptions {
417 #[serde(with = "prefix_twcs")]
419 Twcs(TwcsOptions),
420}
421
422impl CompactionOptions {
423 pub(crate) fn time_window(&self) -> Option<Duration> {
424 match self {
425 CompactionOptions::Twcs(opts) => opts.time_window,
426 }
427 }
428
429 pub(crate) fn remote_compaction(&self) -> bool {
430 match self {
431 CompactionOptions::Twcs(opts) => opts.remote_compaction,
432 }
433 }
434
435 pub(crate) fn fallback_to_local(&self) -> bool {
436 match self {
437 CompactionOptions::Twcs(opts) => opts.fallback_to_local,
438 }
439 }
440}
441
442impl Default for CompactionOptions {
443 fn default() -> Self {
444 Self::Twcs(TwcsOptions::default())
445 }
446}
447
448#[serde_as]
450#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
451#[serde(default)]
452pub struct TwcsOptions {
453 #[serde_as(as = "DisplayFromStr")]
455 #[serde(rename = "active_window.trigger_file_num", alias = "trigger_file_num")]
456 pub active_window_trigger_file_num: usize,
457 #[serde_as(as = "DisplayFromStr")]
459 #[serde(rename = "active_window.l1_merge_trigger")]
460 pub active_window_l1_merge_trigger: usize,
461 #[serde_as(as = "DisplayFromStr")]
463 #[serde(rename = "inactive_window.trigger_file_num")]
464 pub inactive_window_trigger_file_num: usize,
465 #[serde_as(as = "DisplayFromStr")]
467 #[serde(rename = "inactive_window.l1_merge_trigger")]
468 pub inactive_window_l1_merge_trigger: usize,
469 #[serde(with = "humantime_serde")]
471 pub time_window: Option<Duration>,
472 pub max_output_file_size: Option<ReadableSize>,
476 #[serde_as(as = "DisplayFromStr")]
478 pub remote_compaction: bool,
479 #[serde_as(as = "DisplayFromStr")]
481 pub fallback_to_local: bool,
482}
483
484with_prefix!(prefix_twcs "compaction.twcs.");
485
486impl TwcsOptions {
487 pub fn time_window_seconds(&self) -> Option<i64> {
489 self.time_window.and_then(|window| {
490 let window_secs = window.as_secs();
491 if window_secs == 0 {
492 None
493 } else {
494 window_secs.try_into().ok()
495 }
496 })
497 }
498}
499
500impl Default for TwcsOptions {
501 fn default() -> Self {
502 Self {
503 active_window_trigger_file_num: 4,
504 active_window_l1_merge_trigger: 16,
505 inactive_window_trigger_file_num: 2,
506 inactive_window_l1_merge_trigger: 8,
507 time_window: None,
508 max_output_file_size: Some(ReadableSize::mb(512)),
509 remote_compaction: false,
510 fallback_to_local: true,
511 }
512 }
513}
514
515#[serde_as]
518#[derive(Debug, Deserialize)]
519#[serde(default)]
520struct RegionOptionsWithoutEnum {
521 write_buffer_size: Option<ReadableSize>,
522 ttl: Option<TimeToLive>,
524 #[serde(with = "humantime_serde")]
525 auto_flush_interval: Option<Duration>,
526 #[serde(rename = "compaction.experimental_output_parallelism")]
527 #[serde_as(as = "NoneAsEmptyString")]
528 compaction_output_parallelism: Option<usize>,
529 storage: Option<String>,
530 #[serde_as(as = "DisplayFromStr")]
531 append_mode: bool,
532 #[serde_as(as = "DisplayFromStr")]
533 skip_wal: bool,
534 #[serde_as(as = "NoneAsEmptyString")]
535 merge_mode: Option<MergeMode>,
536 #[serde_as(as = "NoneAsEmptyString")]
537 sst_format: Option<FormatType>,
538 #[serde_as(as = "NoneAsEmptyString")]
539 max_row_group_row_count: Option<usize>,
540 #[serde_as(as = "DisplayFromStr")]
541 preserve_row_sequence: bool,
542 #[serde(rename = "experimental_sst_float_field_encoding")]
543 #[serde_as(as = "NoneAsEmptyString")]
544 float_field_encoding: Option<FloatFieldEncoding>,
545}
546
547impl Default for RegionOptionsWithoutEnum {
548 fn default() -> Self {
549 let options = RegionOptions::default();
550 RegionOptionsWithoutEnum {
551 write_buffer_size: options.write_buffer_size,
552 ttl: options.ttl,
553 auto_flush_interval: options.auto_flush_interval,
554 compaction_output_parallelism: options.compaction_output_parallelism,
555 storage: options.storage,
556 append_mode: options.append_mode,
557 skip_wal: options.skip_wal,
558 merge_mode: options.merge_mode,
559 sst_format: options.sst_format,
560 max_row_group_row_count: options.max_row_group_row_count,
561 preserve_row_sequence: options.preserve_row_sequence,
562 float_field_encoding: Some(options.float_field_encoding),
563 }
564 }
565}
566
567with_prefix!(prefix_inverted_index "index.inverted_index.");
568
569#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
571#[serde(default)]
572pub struct IndexOptions {
573 #[serde(flatten, with = "prefix_inverted_index")]
575 pub inverted_index: InvertedIndexOptions,
576}
577
578#[serde_as]
580#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
581#[serde(default)]
582pub struct InvertedIndexOptions {
583 #[serde(deserialize_with = "deserialize_ignore_column_ids")]
586 #[serde(serialize_with = "serialize_ignore_column_ids")]
587 pub ignore_column_ids: Vec<ColumnId>,
588
589 #[serde_as(as = "DisplayFromStr")]
591 pub segment_row_count: usize,
592}
593
594impl Default for InvertedIndexOptions {
595 fn default() -> Self {
596 Self {
597 ignore_column_ids: Vec::new(),
598 segment_row_count: DEFAULT_INDEX_SEGMENT_ROW_COUNT,
599 }
600 }
601}
602
603#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
605#[serde(tag = "memtable.type", rename_all = "snake_case")]
606pub enum MemtableOptions {
607 TimeSeries,
608 #[serde(with = "prefix_bulk")]
609 Bulk(BulkMemtableConfig),
610}
611
612with_prefix!(prefix_bulk "memtable.bulk.");
613
614fn deserialize_ignore_column_ids<'de, D>(deserializer: D) -> Result<Vec<ColumnId>, D::Error>
615where
616 D: Deserializer<'de>,
617{
618 let s: String = Deserialize::deserialize(deserializer)?;
619 let mut column_ids = Vec::new();
620 if s.is_empty() {
621 return Ok(column_ids);
622 }
623 for item in s.split(',') {
624 let column_id = item.parse().map_err(D::Error::custom)?;
625 column_ids.push(column_id);
626 }
627 Ok(column_ids)
628}
629
630fn serialize_ignore_column_ids<S>(column_ids: &[ColumnId], serializer: S) -> Result<S::Ok, S::Error>
631where
632 S: serde::Serializer,
633{
634 let s = column_ids
635 .iter()
636 .map(|id| id.to_string())
637 .collect::<Vec<_>>()
638 .join(",");
639 serializer.serialize_str(&s)
640}
641
642fn options_map_to_value(options: &HashMap<String, String>) -> Value {
646 let map = options
647 .iter()
648 .map(|(key, value)| {
649 if value.eq_ignore_ascii_case("null") {
651 (key.clone(), Value::Null)
652 } else {
653 (key.clone(), Value::from(value.clone()))
654 }
655 })
656 .collect();
657 Value::Object(map)
658}
659
660fn validate_enum_options(
668 options_map: &HashMap<String, String>,
669 enum_tag_key: &str,
670 enum_option_prefixes: &[&str],
671) -> Result<bool> {
672 let mut has_enum_options = false;
673 let mut has_tag = false;
674 for key in options_map.keys() {
675 if key == enum_tag_key {
676 has_tag = true;
677 } else if !has_enum_options
678 && enum_option_prefixes
679 .iter()
680 .any(|prefix| key.starts_with(prefix))
681 {
682 has_enum_options = true;
683 }
684
685 if has_tag && has_enum_options {
686 break;
687 }
688 }
689
690 ensure!(
692 has_tag || !has_enum_options,
693 InvalidRegionOptionsSnafu {
694 reason: format!("missing key {} in options", enum_tag_key),
695 }
696 );
697
698 Ok(has_tag)
699}
700
701#[cfg(test)]
702mod tests {
703 use common_error::ext::ErrorExt;
704 use common_error::status_code::StatusCode;
705 use common_wal::options::{KafkaWalOptions, ObjectStoreWalOptions};
706 use store_api::mito_engine_options::{
707 EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, SKIP_WAL_KEY, WRITE_BUFFER_SIZE_KEY,
708 };
709
710 use super::*;
711
712 fn make_map(options: &[(&str, &str)]) -> HashMap<String, String> {
713 options
714 .iter()
715 .map(|(k, v)| (k.to_string(), v.to_string()))
716 .collect()
717 }
718
719 #[test]
720 fn test_empty_region_options() {
721 let map = make_map(&[]);
722 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
723 assert_eq!(RegionOptions::default(), options);
724 }
725
726 #[test]
727 fn test_conditional_manifest_requires_lease_and_object_store_wal() {
728 let options = |wal_options| RegionOptions {
729 wal_options,
730 ..Default::default()
731 };
732 let object_store_wal = options(WalOptions::ObjectStore(ObjectStoreWalOptions::new(
733 "wal".to_string(),
734 )));
735
736 assert!(object_store_wal.requires_conditional_manifest_publication(true));
737 assert!(!object_store_wal.requires_conditional_manifest_publication(false));
738 assert!(!options(WalOptions::RaftEngine).requires_conditional_manifest_publication(true));
739 }
740
741 #[test]
742 fn test_float_field_encoding_defaults_and_parses() {
743 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &make_map(&[])).unwrap();
744 assert_eq!(FloatFieldEncoding::Default, options.float_field_encoding);
745
746 let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "default")]);
747 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
748 assert_eq!(FloatFieldEncoding::Default, options.float_field_encoding);
749
750 let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "byte_stream_split")]);
751 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
752 assert_eq!(
753 FloatFieldEncoding::ByteStreamSplit,
754 options.float_field_encoding
755 );
756
757 let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "invalid")]);
758 assert!(RegionOptions::try_from_options(RegionId::new(0, 0), &map).is_err());
759 }
760
761 #[test]
762 fn test_with_ttl() {
763 let map = make_map(&[("ttl", "7d")]);
764 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
765 let expect = RegionOptions {
766 ttl: Some(Duration::from_secs(3600 * 24 * 7).into()),
767 ..Default::default()
768 };
769 assert_eq!(expect, options);
770 }
771
772 #[test]
773 fn test_with_skip_wal() {
774 let map = make_map(&[(SKIP_WAL_KEY, "true")]);
775 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
776 assert!(options.skip_wal);
777 }
778
779 #[test]
780 fn test_with_auto_flush_interval() {
781 let map = make_map(&[("auto_flush_interval", "5m")]);
782 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
783 let expect = RegionOptions {
784 auto_flush_interval: Some(Duration::from_secs(5 * 60)),
785 ..Default::default()
786 };
787 assert_eq!(expect, options);
788 }
789
790 #[test]
791 fn test_with_zero_auto_flush_interval() {
792 let map = make_map(&[("auto_flush_interval", "0s")]);
793 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
794 assert!(
795 err.to_string().contains("auto_flush_interval"),
796 "unexpected error: {err}"
797 );
798 }
799
800 #[test]
801 fn test_with_write_buffer_size() {
802 let map = make_map(&[(WRITE_BUFFER_SIZE_KEY, "128MiB")]);
803 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
804 let expect = RegionOptions {
805 write_buffer_size: Some(ReadableSize::mb(128)),
806 ..Default::default()
807 };
808 assert_eq!(expect, options);
809 }
810
811 #[test]
812 fn test_with_zero_write_buffer_size() {
813 let map = make_map(&[(WRITE_BUFFER_SIZE_KEY, "0")]);
814 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
815 let expect = RegionOptions {
816 write_buffer_size: Some(ReadableSize::mb(0)),
817 ..Default::default()
818 };
819 assert_eq!(expect, options);
820 }
821
822 #[test]
823 fn test_with_storage() {
824 let map = make_map(&[("storage", "S3")]);
825 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
826 let expect = RegionOptions {
827 storage: Some("S3".to_string()),
828 ..Default::default()
829 };
830 assert_eq!(expect, options);
831 }
832
833 #[test]
834 fn test_without_compaction_type() {
835 let map = make_map(&[
836 ("compaction.twcs.trigger_file_num", "8"),
837 ("compaction.twcs.time_window", "2h"),
838 ]);
839 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
840 assert_eq!(StatusCode::InvalidArguments, err.status_code());
841 }
842
843 #[test]
844 fn test_with_compaction_type() {
845 let map = make_map(&[
846 ("compaction.twcs.active_window.trigger_file_num", "8"),
847 ("compaction.twcs.active_window.l1_merge_trigger", "16"),
848 ("compaction.twcs.inactive_window.trigger_file_num", "2"),
849 ("compaction.twcs.inactive_window.l1_merge_trigger", "12"),
850 ("compaction.twcs.time_window", "2h"),
851 ("compaction.type", "twcs"),
852 ]);
853 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
854 let expect = RegionOptions {
855 compaction: CompactionOptions::Twcs(TwcsOptions {
856 active_window_trigger_file_num: 8,
857 active_window_l1_merge_trigger: 16,
858 inactive_window_trigger_file_num: 2,
859 inactive_window_l1_merge_trigger: 12,
860 time_window: Some(Duration::from_secs(3600 * 2)),
861 ..Default::default()
862 }),
863 compaction_override: true,
864 ..Default::default()
865 };
866 assert_eq!(expect, options);
867 }
868
869 #[test]
870 fn test_twcs_window_trigger_below_two_is_accepted_for_compatibility() {
871 let map = make_map(&[
874 ("compaction.twcs.active_window.trigger_file_num", "1"),
875 ("compaction.type", "twcs"),
876 ]);
877 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
878 let CompactionOptions::Twcs(twcs) = &options.compaction;
879 assert_eq!(1, twcs.active_window_trigger_file_num);
880
881 let map = make_map(&[
882 ("compaction.twcs.inactive_window.trigger_file_num", "1"),
883 ("compaction.type", "twcs"),
884 ]);
885 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
886 let CompactionOptions::Twcs(twcs) = &options.compaction;
887 assert_eq!(1, twcs.inactive_window_trigger_file_num);
888 }
889
890 #[test]
891 fn test_active_window_l1_merge_trigger_below_two_is_rejected() {
892 let map = make_map(&[
893 ("compaction.twcs.active_window.l1_merge_trigger", "1"),
894 ("compaction.type", "twcs"),
895 ]);
896
897 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
898 assert_eq!(StatusCode::InvalidArguments, err.status_code());
899 }
900
901 #[test]
902 fn test_inactive_window_l1_merge_trigger_defaults_to_eight() {
903 let value = serde_json::to_value(TwcsOptions::default()).unwrap();
904 assert_eq!(
905 Some("8"),
906 value
907 .get("inactive_window.l1_merge_trigger")
908 .and_then(|value| value.as_str())
909 );
910 }
911
912 #[test]
913 fn test_inactive_window_l1_merge_trigger_below_two_is_rejected() {
914 let map = make_map(&[
915 ("compaction.twcs.inactive_window.l1_merge_trigger", "1"),
916 ("compaction.type", "twcs"),
917 ]);
918
919 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
920 assert_eq!(StatusCode::InvalidArguments, err.status_code());
921 }
922
923 #[test]
924 fn test_with_compaction_override_true_without_compaction_type() {
925 let map = make_map(&[(COMPACTION_OVERRIDE, "true")]);
926 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
927 let expect = RegionOptions {
928 compaction_override: true,
929 ..Default::default()
930 };
931 assert_eq!(expect, options);
932 }
933
934 #[test]
935 fn test_with_compaction_override_false_without_compaction_type() {
936 let map = make_map(&[(COMPACTION_OVERRIDE, "false")]);
937 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
938 assert_eq!(RegionOptions::default(), options);
939 }
940
941 #[test]
942 fn test_compaction_twcs_options_still_require_compaction_type_with_override() {
943 let map = make_map(&[
944 (COMPACTION_OVERRIDE, "true"),
945 ("compaction.twcs.time_window", "2h"),
946 ]);
947 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
948 assert_eq!(StatusCode::InvalidArguments, err.status_code());
949 }
950
951 fn test_with_wal_options(wal_options: &WalOptions) -> bool {
952 let encoded_wal_options = serde_json::to_string(&wal_options).unwrap();
953 let map = make_map(&[(WAL_OPTIONS_KEY, &encoded_wal_options)]);
954 let got = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
955 let expect = RegionOptions {
956 wal_options: wal_options.clone(),
957 ..Default::default()
958 };
959 expect == got
960 }
961
962 #[test]
963 fn test_with_index() {
964 let map = make_map(&[
965 ("index.inverted_index.ignore_column_ids", "1,2,3"),
966 ("index.inverted_index.segment_row_count", "512"),
967 ]);
968 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
969 let expect = RegionOptions {
970 index_options: IndexOptions {
971 inverted_index: InvertedIndexOptions {
972 ignore_column_ids: vec![1, 2, 3],
973 segment_row_count: 512,
974 },
975 },
976 ..Default::default()
977 };
978 assert_eq!(expect, options);
979 }
980
981 #[test]
983 fn test_with_any_wal_options() {
984 let all_wal_options = [
985 WalOptions::RaftEngine,
986 WalOptions::Kafka(KafkaWalOptions::new("test_topic".to_string())),
987 ];
988 all_wal_options.iter().all(test_with_wal_options);
989 }
990
991 #[test]
992 fn test_with_memtable() {
993 let map = make_map(&[("memtable.type", "time_series")]);
994 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
995 let expect = RegionOptions {
996 memtable: Some(MemtableOptions::TimeSeries),
997 ..Default::default()
998 };
999 assert_eq!(expect, options);
1000
1001 let map = make_map(&[("memtable.type", "bulk")]);
1002 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1003 let expect = RegionOptions {
1004 memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig::default())),
1005 sst_format: Some(FormatType::Flat),
1006 ..Default::default()
1007 };
1008 assert_eq!(expect, options);
1009
1010 let map = make_map(&[
1011 ("memtable.type", "bulk"),
1012 ("memtable.bulk.merge_threshold", "7"),
1013 ("memtable.bulk.encode_row_threshold", "11"),
1014 ("memtable.bulk.encode_bytes_threshold", "13"),
1015 ("memtable.bulk.max_merge_groups", "17"),
1016 ]);
1017 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1018 let expect = RegionOptions {
1019 memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig {
1020 merge_threshold: 7,
1021 encode_row_threshold: 11,
1022 encode_bytes_threshold: 13,
1023 max_merge_groups: 17,
1024 })),
1025 sst_format: Some(FormatType::Flat),
1026 ..Default::default()
1027 };
1028 assert_eq!(expect, options);
1029
1030 let map = make_map(&[("memtable.type", "partition_tree")]);
1033 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1034 let expect = RegionOptions {
1035 memtable: None,
1036 sst_format: Some(FormatType::Flat),
1037 ..Default::default()
1038 };
1039 assert_eq!(expect, options);
1040
1041 let map = make_map(&[
1043 ("memtable.type", "partition_tree"),
1044 ("memtable.partition_tree.index_max_keys_per_shard", "2048"),
1045 ("memtable.partition_tree.fork_dictionary_bytes", "128M"),
1046 ]);
1047 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1048 let expect = RegionOptions {
1049 memtable: None,
1050 sst_format: Some(FormatType::Flat),
1051 ..Default::default()
1052 };
1053 assert_eq!(expect, options);
1054 }
1055
1056 #[test]
1057 fn test_sparse_forces_flat_sst() {
1058 for format in [None, Some("primary_key"), Some("flat")] {
1059 let mut map = make_map(&[("primary_key_encoding", "sparse")]);
1060 if let Some(format) = format {
1061 map.insert("sst_format".into(), format.into());
1062 }
1063 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1064 assert_eq!(options.primary_key_encoding(), PrimaryKeyEncoding::Sparse);
1065 assert_eq!(options.sst_format, Some(FormatType::Flat));
1066 }
1067 }
1068
1069 #[test]
1070 fn test_primary_key_encoding() {
1071 let map = make_map(&[("primary_key_encoding", "sparse")]);
1073 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1074 assert_eq!(options.primary_key_encoding(), PrimaryKeyEncoding::Sparse);
1075 assert_eq!(
1076 options.primary_key_encoding,
1077 Some(PrimaryKeyEncoding::Sparse)
1078 );
1079
1080 let map = make_map(&[
1082 ("memtable.type", "partition_tree"),
1083 ("memtable.partition_tree.primary_key_encoding", "sparse"),
1084 ]);
1085 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1086 assert_eq!(options.memtable, None);
1087 assert_eq!(options.sst_format, Some(FormatType::Flat));
1088 assert_eq!(options.primary_key_encoding(), PrimaryKeyEncoding::Sparse);
1089
1090 let map = make_map(&[("primary_key_encoding", "bogus")]);
1092 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1093 assert_eq!(StatusCode::InvalidArguments, err.status_code());
1094 }
1095
1096 #[test]
1097 fn test_legacy_partition_tree_overrides_sst_format() {
1098 let map = make_map(&[
1101 ("memtable.type", "partition_tree"),
1102 ("sst_format", "primary_key"),
1103 ]);
1104 let options = RegionOptions::try_from_options(RegionId::new(1, 1), &map).unwrap();
1105 assert_eq!(options.memtable, None);
1106 assert_eq!(options.sst_format, Some(FormatType::Flat));
1107 }
1108
1109 #[test]
1110 fn test_bulk_memtable_overrides_sst_format() {
1111 let map = make_map(&[("memtable.type", "bulk"), ("sst_format", "primary_key")]);
1115 let options = RegionOptions::try_from_options(RegionId::new(1, 1), &map).unwrap();
1116 assert_eq!(
1117 options.memtable,
1118 Some(MemtableOptions::Bulk(BulkMemtableConfig::default()))
1119 );
1120 assert_eq!(options.sst_format, Some(FormatType::Flat));
1121 }
1122
1123 #[test]
1124 fn test_unknown_memtable_type() {
1125 let map = make_map(&[("memtable.type", "no_such_memtable")]);
1126 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1127 assert_eq!(StatusCode::InvalidArguments, err.status_code());
1128 }
1129
1130 #[test]
1131 fn test_without_memtable_type() {
1132 let map = make_map(&[("memtable.partition_tree.index_max_keys_per_shard", "2048")]);
1133 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1134 assert_eq!(StatusCode::InvalidArguments, err.status_code());
1135
1136 let map = make_map(&[("memtable.bulk.merge_threshold", "7")]);
1137 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1138 assert_eq!(StatusCode::InvalidArguments, err.status_code());
1139 }
1140
1141 #[test]
1142 fn test_with_merge_mode() {
1143 let map = make_map(&[("merge_mode", "last_row")]);
1144 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1145 assert_eq!(MergeMode::LastRow, options.merge_mode());
1146
1147 let map = make_map(&[("merge_mode", "last_non_null")]);
1148 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1149 assert_eq!(MergeMode::LastNonNull, options.merge_mode());
1150
1151 let map = make_map(&[("merge_mode", "unknown")]);
1152 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1153 assert_eq!(StatusCode::InvalidArguments, err.status_code());
1154 }
1155
1156 #[test]
1157 fn test_append_mode_allows_last_row_merge_mode() {
1158 let map = make_map(&[("append_mode", "true"), ("merge_mode", "last_row")]);
1159 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1160 assert!(options.append_mode);
1161 assert_eq!(MergeMode::LastRow, options.merge_mode());
1162
1163 let map = make_map(&[("append_mode", "true"), ("merge_mode", "last_non_null")]);
1164 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1165 assert_eq!(StatusCode::InvalidArguments, err.status_code());
1166 }
1167
1168 #[test]
1169 fn test_with_all() {
1170 let wal_options = WalOptions::Kafka(KafkaWalOptions::new("test_topic".to_string()));
1171 let map = make_map(&[
1172 ("ttl", "7d"),
1173 ("compaction.twcs.trigger_file_num", "8"),
1174 ("compaction.twcs.max_output_file_size", "1GB"),
1175 ("compaction.twcs.time_window", "2h"),
1176 ("compaction.type", "twcs"),
1177 ("compaction.twcs.remote_compaction", "false"),
1178 ("compaction.twcs.fallback_to_local", "true"),
1179 ("storage", "S3"),
1180 ("append_mode", "false"),
1181 ("index.inverted_index.ignore_column_ids", "1,2,3"),
1182 ("index.inverted_index.segment_row_count", "512"),
1183 (
1184 WAL_OPTIONS_KEY,
1185 &serde_json::to_string(&wal_options).unwrap(),
1186 ),
1187 ("memtable.type", "bulk"),
1188 ("memtable.bulk.merge_threshold", "7"),
1189 ("memtable.bulk.encode_row_threshold", "11"),
1190 ("memtable.bulk.encode_bytes_threshold", "13"),
1191 ("memtable.bulk.max_merge_groups", "17"),
1192 ("merge_mode", "last_non_null"),
1193 ]);
1194 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1195 let expect = RegionOptions {
1196 ttl: Some(Duration::from_secs(3600 * 24 * 7).into()),
1197 auto_flush_interval: None,
1198 compaction: CompactionOptions::Twcs(TwcsOptions {
1199 active_window_trigger_file_num: 8,
1200 active_window_l1_merge_trigger: 16,
1201 inactive_window_trigger_file_num: 2,
1202 inactive_window_l1_merge_trigger: 8,
1203 time_window: Some(Duration::from_secs(3600 * 2)),
1204 max_output_file_size: Some(ReadableSize::gb(1)),
1205 remote_compaction: false,
1206 fallback_to_local: true,
1207 }),
1208 compaction_override: true,
1209 compaction_output_parallelism: None,
1210 storage: Some("S3".to_string()),
1211 append_mode: false,
1212 skip_wal: false,
1213 wal_options,
1214 index_options: IndexOptions {
1215 inverted_index: InvertedIndexOptions {
1216 ignore_column_ids: vec![1, 2, 3],
1217 segment_row_count: 512,
1218 },
1219 },
1220 memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig {
1221 merge_threshold: 7,
1222 encode_row_threshold: 11,
1223 encode_bytes_threshold: 13,
1224 max_merge_groups: 17,
1225 })),
1226 merge_mode: Some(MergeMode::LastNonNull),
1227 sst_format: Some(FormatType::Flat),
1228 max_row_group_row_count: None,
1229 primary_key_encoding: None,
1230 write_buffer_size: None,
1231 preserve_row_sequence: false,
1232 float_field_encoding: FloatFieldEncoding::default(),
1233 };
1234 assert_eq!(expect, options);
1235 }
1236
1237 #[test]
1238 fn test_with_preserve_row_sequence() {
1239 let map = make_map(&[("append_mode", "false"), ("preserve_row_sequence", "true")]);
1240 let err = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap_err();
1241 assert_eq!(StatusCode::InvalidArguments, err.status_code());
1242 assert!(err.to_string().contains("preserve_row_sequence"));
1243
1244 let map = make_map(&[("append_mode", "true"), ("preserve_row_sequence", "true")]);
1245 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1246 assert!(options.append_mode);
1247 assert!(options.preserve_row_sequence);
1248
1249 let map = make_map(&[("append_mode", "true")]);
1250 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1251 assert!(!options.preserve_row_sequence);
1252
1253 let map = make_map(&[]);
1254 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1255 assert_eq!(RegionOptions::default(), options);
1256 }
1257
1258 #[test]
1259 fn test_region_options_serde() {
1260 let options = RegionOptions {
1261 ttl: Some(Duration::from_secs(3600 * 24 * 7).into()),
1262 auto_flush_interval: None,
1263 compaction: CompactionOptions::Twcs(TwcsOptions {
1264 active_window_trigger_file_num: 8,
1265 active_window_l1_merge_trigger: 8,
1266 inactive_window_trigger_file_num: 2,
1267 inactive_window_l1_merge_trigger: 8,
1268 time_window: Some(Duration::from_secs(3600 * 2)),
1269 max_output_file_size: None,
1270 remote_compaction: false,
1271 fallback_to_local: true,
1272 }),
1273 compaction_override: false,
1274 storage: Some("S3".to_string()),
1275 append_mode: false,
1276 skip_wal: false,
1277 wal_options: WalOptions::Kafka(KafkaWalOptions::new("test_topic".to_string())),
1278 index_options: IndexOptions {
1279 inverted_index: InvertedIndexOptions {
1280 ignore_column_ids: vec![1, 2, 3],
1281 segment_row_count: 512,
1282 },
1283 },
1284 memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig::default())),
1285 merge_mode: Some(MergeMode::LastNonNull),
1286 sst_format: None,
1287 max_row_group_row_count: None,
1288 primary_key_encoding: None,
1289 write_buffer_size: Some(ReadableSize::mb(128)),
1290 compaction_output_parallelism: Some(4),
1291 preserve_row_sequence: true,
1292 float_field_encoding: FloatFieldEncoding::default(),
1293 };
1294 let region_options_json_str = serde_json::to_string(&options).unwrap();
1295 assert!(region_options_json_str.contains("preserve_row_sequence"));
1296 let got: RegionOptions = serde_json::from_str(®ion_options_json_str).unwrap();
1297 assert_eq!(options, got);
1298
1299 let old_region_options_json_str = r#"{"ttl":null}"#;
1301 let got: RegionOptions = serde_json::from_str(old_region_options_json_str).unwrap();
1302 assert_eq!(None, got.write_buffer_size);
1303 assert_eq!(None, got.compaction_output_parallelism);
1304 assert_eq!(4, got.compaction_output_parallelism_or(4));
1305 assert!(!got.preserve_row_sequence);
1306 assert_eq!(FloatFieldEncoding::Default, got.float_field_encoding);
1307 let CompactionOptions::Twcs(twcs) = got.compaction;
1308 assert_eq!(16, twcs.active_window_l1_merge_trigger);
1309 assert_eq!(8, twcs.inactive_window_l1_merge_trigger);
1310
1311 let default_json = serde_json::to_value(RegionOptions::default()).unwrap();
1312 assert!(default_json.get(WRITE_BUFFER_SIZE_KEY).is_none());
1313 }
1314
1315 #[test]
1316 fn test_region_options_str_serde() {
1317 let region_options_json_str = r#"{
1319 "ttl": "7days",
1320 "compaction": {
1321 "compaction.type": "twcs",
1322 "compaction.twcs.trigger_file_num": "8",
1323 "compaction.twcs.max_output_file_size": "7MB",
1324 "compaction.twcs.time_window": "2h"
1325 },
1326 "storage": "S3",
1327 "append_mode": false,
1328 "wal_options": {
1329 "wal.provider": "kafka",
1330 "wal.kafka.topic": "test_topic"
1331 },
1332 "index_options": {
1333 "index.inverted_index.ignore_column_ids": "",
1334 "index.inverted_index.segment_row_count": "512"
1335 },
1336 "memtable": {
1337 "memtable.type": "bulk"
1338 },
1339 "merge_mode": "last_non_null"
1340}"#;
1341 let got: RegionOptions = serde_json::from_str(region_options_json_str).unwrap();
1342 let options = RegionOptions {
1343 ttl: Some(Duration::from_secs(3600 * 24 * 7).into()),
1344 auto_flush_interval: None,
1345 compaction: CompactionOptions::Twcs(TwcsOptions {
1346 active_window_trigger_file_num: 8,
1347 active_window_l1_merge_trigger: 16,
1348 inactive_window_trigger_file_num: 2,
1349 inactive_window_l1_merge_trigger: 8,
1350 time_window: Some(Duration::from_secs(3600 * 2)),
1351 max_output_file_size: Some(ReadableSize::mb(7)),
1352 remote_compaction: false,
1353 fallback_to_local: true,
1354 }),
1355 compaction_override: false,
1356 storage: Some("S3".to_string()),
1357 append_mode: false,
1358 skip_wal: false,
1359 wal_options: WalOptions::Kafka(KafkaWalOptions::new("test_topic".to_string())),
1360 index_options: IndexOptions {
1361 inverted_index: InvertedIndexOptions {
1362 ignore_column_ids: vec![],
1363 segment_row_count: 512,
1364 },
1365 },
1366 memtable: Some(MemtableOptions::Bulk(BulkMemtableConfig::default())),
1367 merge_mode: Some(MergeMode::LastNonNull),
1368 sst_format: None,
1369 max_row_group_row_count: None,
1370 primary_key_encoding: None,
1371 write_buffer_size: None,
1372 compaction_output_parallelism: None,
1373 preserve_row_sequence: false,
1374 float_field_encoding: FloatFieldEncoding::default(),
1375 };
1376 assert_eq!(options, got);
1377 }
1378
1379 #[test]
1380 fn test_compaction_output_parallelism_preserves_strategy_inheritance() {
1381 for parallelism in [None, Some(1), Some(4)] {
1382 let mut map = make_map(&[]);
1383 if let Some(parallelism) = parallelism {
1384 map.insert(
1385 "compaction.experimental_output_parallelism".to_string(),
1386 parallelism.to_string(),
1387 );
1388 }
1389 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1390 assert_eq!(parallelism, options.compaction_output_parallelism);
1391 assert_eq!(
1392 parallelism.unwrap_or(8),
1393 options.compaction_output_parallelism_or(8)
1394 );
1395 assert!(!options.compaction_override);
1396
1397 map.insert("compaction.type".to_string(), "twcs".to_string());
1398 map.insert("compaction.twcs.time_window".to_string(), "2h".to_string());
1399 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1400 assert_eq!(parallelism, options.compaction_output_parallelism);
1401 assert_eq!(
1402 Some(Duration::from_secs(7200)),
1403 options.compaction.time_window()
1404 );
1405 }
1406 }
1407
1408 #[test]
1409 fn test_invalid_compaction_output_parallelism() {
1410 for parallelism in ["0", "-1", "1.5", "invalid", "18446744073709551616"] {
1411 let map = make_map(&[("compaction.experimental_output_parallelism", parallelism)]);
1412 assert!(RegionOptions::try_from_options(RegionId::new(0, 0), &map).is_err());
1413 }
1414 }
1415
1416 #[test]
1417 fn test_max_row_group_row_count() {
1418 assert_eq!(None, RegionOptions::default().max_row_group_row_count);
1420 assert_eq!(
1421 DEFAULT_ROW_GROUP_SIZE,
1422 RegionOptions::default().row_group_size()
1423 );
1424
1425 let map = make_map(&[("max_row_group_row_count", "51200")]);
1427 let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
1428 assert_eq!(Some(51200), options.max_row_group_row_count);
1429 assert_eq!(51200, options.row_group_size());
1430
1431 let map = make_map(&[("max_row_group_row_count", "0")]);
1433 assert!(RegionOptions::try_from_options(RegionId::new(0, 0), &map).is_err());
1434 }
1435}