1use std::collections::{BTreeMap, HashMap, HashSet};
18use std::fmt;
19use std::str::FromStr;
20
21use common_base::readable_size::ReadableSize;
22use common_datasource::object_store::oss::is_supported_in_oss;
23use common_datasource::object_store::s3::is_supported_in_s3;
24use common_query::AddColumnLocation;
25use common_time::TimeToLive;
26use common_time::range::TimestampRange;
27use datatypes::data_type::ConcreteDataType;
28use datatypes::json::JsonSettings;
29use datatypes::prelude::VectorRef;
30use datatypes::schema::{
31 ColumnDefaultConstraint, ColumnSchema, FulltextOptions, Schema, SkippingIndexOptions,
32};
33use greptime_proto::v1::region::{build_index_request, compact_request};
34use once_cell::sync::Lazy;
35use serde::{Deserialize, Serialize};
36use store_api::metric_engine_consts::{
37 LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY, is_metric_engine_option_key,
38};
39use store_api::mito_engine_options::{
40 APPEND_MODE_KEY, COMPACTION_TYPE, EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, FloatFieldEncoding,
41 MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD, MEMTABLE_BULK_ENCODE_ROW_THRESHOLD,
42 MEMTABLE_BULK_MAX_MERGE_GROUPS, MEMTABLE_BULK_MERGE_THRESHOLD, MEMTABLE_TYPE, MERGE_MODE_KEY,
43 SST_FORMAT_KEY, TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER, TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
44 TWCS_FALLBACK_TO_LOCAL, TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER,
45 TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM, TWCS_MAX_OUTPUT_FILE_SIZE, TWCS_TIME_WINDOW,
46 TWCS_TRIGGER_FILE_NUM, is_mito_engine_option_key, normalize_twcs_trigger_options,
47};
48use store_api::region_request::{SetRegionOption, UnsetRegionOption};
49
50use crate::error::{ConflictingTableOptionsSnafu, ParseTableOptionSnafu, Result};
51use crate::metadata::{TableId, TableVersion};
52
53mod semantic;
54pub use semantic::*;
55
56pub const FILE_TABLE_META_KEY: &str = "__private.file_table_meta";
57pub const RELATIONAL_META_KEY: &str = "__private.relational_meta";
58pub const FILE_TABLE_LOCATION_KEY: &str = "location";
59pub const FILE_TABLE_PATTERN_KEY: &str = "pattern";
60pub const FILE_TABLE_FORMAT_KEY: &str = "format";
61pub const RELATIONAL_CONNECTION_KEY: &str = "connection";
62pub const RELATIONAL_REMOTE_TABLE_KEY: &str = "remote_table";
63
64#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)]
70pub struct RelationalOptions {
71 pub connection: String,
73 pub remote_table: String,
74}
75
76impl RelationalOptions {
77 pub fn redacted(&self) -> Self {
82 Self {
83 connection: redact_connection(&self.connection),
84 remote_table: self.remote_table.clone(),
85 }
86 }
87}
88
89pub fn redact_connection(connection: &str) -> String {
93 match url::Url::parse(connection) {
94 Ok(mut url) => {
95 if url.password().is_some() {
96 let _ = url.set_password(Some(REDACTED_OPTION_VALUE));
97 }
98 url.to_string()
99 }
100 Err(_) => REDACTED_OPTION_VALUE.to_string(),
101 }
102}
103
104pub fn redact_relational_options(encoded: &str) -> String {
105 serde_json::from_str::<RelationalOptions>(encoded)
106 .ok()
107 .and_then(|opts| serde_json::to_string(&opts.redacted()).ok())
108 .unwrap_or_else(|| REDACTED_OPTION_VALUE.to_string())
109}
110
111pub fn redact_option_value(key: &str, value: &str) -> Option<String> {
112 if is_credential_option(key) {
113 Some(REDACTED_OPTION_VALUE.to_string())
114 } else if key == RELATIONAL_META_KEY {
115 Some(redact_relational_options(value))
116 } else {
117 None
118 }
119}
120
121impl std::fmt::Debug for RelationalOptions {
122 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
123 f.debug_struct("RelationalOptions")
124 .field("connection", &redact_connection(&self.connection))
125 .field("remote_table", &self.remote_table)
126 .finish()
127 }
128}
129
130pub const TABLE_DATA_MODEL: &str = "table_data_model";
131pub const TABLE_DATA_MODEL_TRACE_V1: &str = "greptime_trace_v1";
132pub const TABLE_DATA_MODEL_TRACE_V2: &str = "greptime_trace_v2";
134pub const TABLE_DATA_MODEL_LOG_V1: &str = "greptime_log_v1";
137
138pub fn is_trace_table(table_info: &crate::metadata::TableInfo) -> bool {
141 let table_data_model = table_info.meta.options.data_model();
142 matches!(
143 table_data_model,
144 Some(TABLE_DATA_MODEL_TRACE_V1 | TABLE_DATA_MODEL_TRACE_V2)
145 )
146}
147
148pub const OTLP_METRIC_COMPAT_KEY: &str = "otlp_metric_compat";
150pub const OTLP_METRIC_COMPAT_PROM: &str = "prom";
151
152pub const VALID_TABLE_OPTION_KEYS: [&str; 17] = [
153 WRITE_BUFFER_SIZE_KEY,
155 TTL_KEY,
156 STORAGE_KEY,
157 COMMENT_KEY,
158 SKIP_WAL_KEY,
159 SST_FORMAT_KEY,
160 FILE_TABLE_LOCATION_KEY,
162 FILE_TABLE_FORMAT_KEY,
163 FILE_TABLE_PATTERN_KEY,
164 RELATIONAL_CONNECTION_KEY,
166 RELATIONAL_REMOTE_TABLE_KEY,
167 PHYSICAL_TABLE_METADATA_KEY,
169 LOGICAL_TABLE_METADATA_KEY,
170 TABLE_DATA_MODEL,
172 OTLP_METRIC_COMPAT_KEY,
173 REPARTITION_COLUMN_HINT_KEY,
174 REPARTITION_PARTITION_NUM_HINT_KEY,
175];
176
177pub const DDL_TIMEOUT: &str = "timeout";
178pub const DDL_WAIT: &str = "wait";
179
180pub const VALID_DDL_OPTION_KEYS: [&str; 2] = [DDL_TIMEOUT, DDL_WAIT];
181
182pub const INGEST_ROWS_RATE_LIMIT_KEY: &str = "ingest_rows_rate_limit";
184
185static VALID_DB_OPT_KEYS: Lazy<HashSet<&str>> = Lazy::new(|| {
187 let mut set = HashSet::new();
188 set.insert(TTL_KEY);
189 set.insert(STORAGE_KEY);
190 set.insert(MEMTABLE_TYPE);
191 set.insert(MEMTABLE_BULK_MERGE_THRESHOLD);
192 set.insert(MEMTABLE_BULK_ENCODE_ROW_THRESHOLD);
193 set.insert(MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD);
194 set.insert(MEMTABLE_BULK_MAX_MERGE_GROUPS);
195 set.insert(APPEND_MODE_KEY);
196 set.insert(MERGE_MODE_KEY);
197 set.insert(SKIP_WAL_KEY);
198 set.insert(COMPACTION_TYPE);
199 set.insert(TWCS_FALLBACK_TO_LOCAL);
200 set.insert(TWCS_TIME_WINDOW);
201 set.insert(TWCS_TRIGGER_FILE_NUM);
202 set.insert(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM);
203 set.insert(TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER);
204 set.insert(TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM);
205 set.insert(TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER);
206 set.insert(TWCS_MAX_OUTPUT_FILE_SIZE);
207 set.insert(SST_FORMAT_KEY);
208 set.insert(INGEST_ROWS_RATE_LIMIT_KEY);
209 set
210});
211
212pub fn validate_database_option(key: &str) -> bool {
214 VALID_DB_OPT_KEYS.contains(&key)
215}
216
217pub fn validate_database_option_value(
219 key: &str,
220 value: Option<&str>,
221) -> std::result::Result<(), &'static str> {
222 if key == INGEST_ROWS_RATE_LIMIT_KEY {
223 return value
224 .and_then(|value| value.parse::<u64>().ok())
225 .map(|_| ())
226 .ok_or("expected a non-negative integer fitting in u64");
227 }
228 let (minimum, constraint) = match key {
229 TWCS_TRIGGER_FILE_NUM
230 | TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM
231 | TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM => {
232 (0, "expected a non-negative integer fitting in usize")
233 }
234 TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER | TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER => {
235 (2, "expected an integer greater than or equal to 2")
236 }
237 _ => return Ok(()),
238 };
239 if value
240 .and_then(|value| value.parse::<usize>().ok())
241 .is_some_and(|files| files >= minimum)
242 {
243 Ok(())
244 } else {
245 Err(constraint)
246 }
247}
248
249pub fn validate_table_option(key: &str) -> bool {
251 if is_supported_in_s3(key) {
252 return true;
253 }
254
255 if is_supported_in_oss(key) {
256 return true;
257 }
258
259 if is_mito_engine_option_key(key) {
260 return true;
261 }
262
263 if is_metric_engine_option_key(key) {
264 return true;
265 }
266
267 if is_semantic_option_key(key) {
270 return true;
271 }
272
273 VALID_TABLE_OPTION_KEYS.contains(&key) || VALID_DDL_OPTION_KEYS.contains(&key)
274}
275
276#[derive(Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
277#[serde(default)]
278pub struct TableOptions {
279 pub write_buffer_size: Option<ReadableSize>,
281 pub ttl: Option<TimeToLive>,
283 pub skip_wal: bool,
285 pub extra_options: HashMap<String, String>,
287}
288
289pub const WRITE_BUFFER_SIZE_KEY: &str = store_api::mito_engine_options::WRITE_BUFFER_SIZE_KEY;
290pub const TTL_KEY: &str = store_api::mito_engine_options::TTL_KEY;
291pub const STORAGE_KEY: &str = "storage";
292pub const COMMENT_KEY: &str = "comment";
293pub const AUTO_CREATE_TABLE_KEY: &str = "auto_create_table";
294pub const SKIP_WAL_KEY: &str = store_api::mito_engine_options::SKIP_WAL_KEY;
295pub const TRACE_TABLE_PARTITIONS_HINT_KEY: &str = "trace_table_partitions";
296pub const REPARTITION_COLUMN_HINT_KEY: &str = "repartition.column.hint";
297
298pub const REPARTITION_PARTITION_NUM_HINT_KEY: &str = "repartition.partition.num.hint";
300
301impl TableOptions {
302 pub fn data_model(&self) -> Option<&str> {
304 self.extra_options.get(TABLE_DATA_MODEL).map(String::as_str)
305 }
306
307 pub fn try_from_iter<T: ToString, U: IntoIterator<Item = (T, T)>>(
308 iter: U,
309 ) -> Result<TableOptions> {
310 let mut options = TableOptions::default();
311
312 let mut kvs: HashMap<String, String> = iter
313 .into_iter()
314 .map(|(k, v)| (k.to_string(), v.to_string()))
315 .collect();
316
317 normalize_twcs_trigger_options(&mut kvs).map_err(|conflict| {
318 ConflictingTableOptionsSnafu {
319 first_key: TWCS_TRIGGER_FILE_NUM,
320 first_value: conflict.legacy_value,
321 second_key: TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
322 second_value: conflict.canonical_value,
323 }
324 .build()
325 })?;
326
327 if let Some(write_buffer_size) = kvs.get(WRITE_BUFFER_SIZE_KEY) {
328 let size = ReadableSize::from_str(write_buffer_size).map_err(|_| {
329 ParseTableOptionSnafu {
330 key: WRITE_BUFFER_SIZE_KEY,
331 value: write_buffer_size,
332 }
333 .build()
334 })?;
335 options.write_buffer_size = Some(size)
336 }
337
338 if let Some(ttl) = kvs.get(TTL_KEY) {
339 let ttl_value = TimeToLive::from_humantime_or_str(ttl).map_err(|_| {
340 ParseTableOptionSnafu {
341 key: TTL_KEY,
342 value: ttl,
343 }
344 .build()
345 })?;
346 options.ttl = Some(ttl_value);
347 }
348
349 if let Some(encoding) = kvs.get(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING) {
350 encoding.parse::<FloatFieldEncoding>().map_err(|_| {
351 ParseTableOptionSnafu {
352 key: EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
353 value: encoding,
354 }
355 .build()
356 })?;
357 }
358
359 if let Some(skip_wal) = kvs.get(SKIP_WAL_KEY) {
360 options.skip_wal = skip_wal.parse().map_err(|_| {
361 ParseTableOptionSnafu {
362 key: SKIP_WAL_KEY,
363 value: skip_wal,
364 }
365 .build()
366 })?;
367 }
368
369 options.extra_options = HashMap::from_iter(
370 kvs.into_iter()
371 .filter(|(k, _)| k != WRITE_BUFFER_SIZE_KEY && k != TTL_KEY),
372 );
373
374 Ok(options)
375 }
376}
377
378const CREDENTIAL_OPTION_KEYS: [&str; 5] = [
380 "access_key_id",
381 "connection",
382 "secret_access_key",
383 "session_token",
384 "access_key_secret",
385];
386
387pub const REDACTED_OPTION_VALUE: &str = "******";
389
390pub fn is_credential_option(key: &str) -> bool {
391 CREDENTIAL_OPTION_KEYS.contains(&key)
392}
393
394impl fmt::Display for TableOptions {
397 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
398 let mut key_vals = vec![];
399 if let Some(size) = self.write_buffer_size {
400 key_vals.push(format!("{}={}", WRITE_BUFFER_SIZE_KEY, size));
401 }
402
403 if let Some(ttl) = self.ttl.map(|ttl| ttl.to_string()) {
404 key_vals.push(format!("{}={}", TTL_KEY, ttl));
405 }
406
407 if self.skip_wal && !self.extra_options.contains_key(SKIP_WAL_KEY) {
408 key_vals.push(format!("{}={}", SKIP_WAL_KEY, self.skip_wal));
409 }
410
411 let extra_options: BTreeMap<_, _> = self.extra_options.iter().collect();
413 for (k, v) in extra_options {
414 let value = redact_option_value(k, v).unwrap_or_else(|| v.clone());
415 key_vals.push(format!("{}={}", k, value));
416 }
417
418 write!(f, "{}", key_vals.join(" "))
419 }
420}
421
422impl fmt::Debug for TableOptions {
423 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
424 let extra_options = self
425 .extra_options
426 .iter()
427 .map(|(key, value)| {
428 let redacted = redact_option_value(key, value).unwrap_or_else(|| value.clone());
429 (key.clone(), redacted)
430 })
431 .collect::<HashMap<_, _>>();
432 f.debug_struct("TableOptions")
433 .field("write_buffer_size", &self.write_buffer_size)
434 .field("ttl", &self.ttl)
435 .field("skip_wal", &self.skip_wal)
436 .field("extra_options", &extra_options)
437 .finish()
438 }
439}
440
441impl From<&TableOptions> for HashMap<String, String> {
442 fn from(opts: &TableOptions) -> Self {
443 let mut res = HashMap::with_capacity(3 + opts.extra_options.len());
444 if let Some(write_buffer_size) = opts.write_buffer_size {
445 let _ = res.insert(
446 WRITE_BUFFER_SIZE_KEY.to_string(),
447 write_buffer_size.to_string(),
448 );
449 }
450 if let Some(ttl_str) = opts.ttl.map(|ttl| ttl.to_string()) {
451 let _ = res.insert(TTL_KEY.to_string(), ttl_str);
452 }
453 if opts.skip_wal {
454 let _ = res.insert(SKIP_WAL_KEY.to_string(), true.to_string());
455 }
456 res.extend(opts.extra_options.clone());
457 res
458 }
459}
460
461#[derive(Debug, Clone, Serialize, Deserialize)]
463pub struct AlterTableRequest {
464 pub catalog_name: String,
465 pub schema_name: String,
466 pub table_name: String,
467 pub table_id: TableId,
468 pub alter_kind: AlterKind,
469 pub table_version: Option<TableVersion>,
471}
472
473#[derive(Debug, Clone, Serialize, Deserialize)]
475pub struct AddColumnRequest {
476 pub column_schema: ColumnSchema,
477 pub is_key: bool,
478 pub location: Option<AddColumnLocation>,
479 pub add_if_not_exists: bool,
481}
482
483#[derive(Debug, Clone, Serialize, Deserialize)]
485pub struct ModifyColumnTypeRequest {
486 pub column_name: String,
487 pub target_type: ConcreteDataType,
488}
489
490#[derive(Debug, Clone, Serialize, Deserialize)]
492pub struct SetJsonSettingsRequest {
493 pub column_name: String,
494 pub settings: JsonSettings,
495}
496
497#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
501pub enum AnnotationFamily {
502 Semantic,
504 RepartitionHint,
506}
507
508impl AnnotationFamily {
509 pub fn namespace(self) -> &'static str {
511 match self {
512 Self::Semantic => SEMANTIC_PREFIX,
513 Self::RepartitionHint => "repartition.",
514 }
515 }
516
517 pub fn of_key(key: &str) -> Option<Self> {
518 if key.starts_with(SEMANTIC_PREFIX) {
519 Some(Self::Semantic)
520 } else if matches!(
521 key,
522 REPARTITION_COLUMN_HINT_KEY | REPARTITION_PARTITION_NUM_HINT_KEY
523 ) {
524 Some(Self::RepartitionHint)
525 } else {
526 None
527 }
528 }
529
530 pub fn allows_logical_tables(self) -> bool {
534 match self {
535 Self::Semantic => true,
536 Self::RepartitionHint => false,
537 }
538 }
539
540 pub fn requires_unique_keys(self) -> bool {
542 matches!(self, Self::RepartitionHint)
543 }
544
545 pub fn mixed_batch_error(self) -> String {
547 match self {
548 Self::Semantic => format!(
549 "`{SEMANTIC_PREFIX}*` options must be altered separately from other table options"
550 ),
551 Self::RepartitionHint => {
552 "repartition hints must be altered separately from other table options".to_string()
553 }
554 }
555 }
556}
557
558#[derive(Debug, Clone, PartialEq, Eq)]
560pub enum AnnotationKeyError {
561 MixedFamilies { family: AnnotationFamily },
562 DuplicateKey,
563}
564
565impl fmt::Display for AnnotationKeyError {
566 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
567 match self {
568 Self::MixedFamilies { family } => f.write_str(&family.mixed_batch_error()),
569 Self::DuplicateKey => f.write_str("duplicate repartition hint keys"),
570 }
571 }
572}
573
574pub fn validate_annotation_keys<'a>(
580 keys: impl IntoIterator<Item = &'a str>,
581) -> std::result::Result<Option<AnnotationFamily>, AnnotationKeyError> {
582 let mut keys = keys.into_iter();
583 let Some(first) = keys.next() else {
584 return Ok(None);
585 };
586 let family = AnnotationFamily::of_key(first);
587 let reject_duplicates = family.is_some_and(|family| family.requires_unique_keys());
588 let mut seen = HashSet::new();
589 if reject_duplicates {
590 seen.insert(first);
591 }
592 for key in keys {
593 let this = AnnotationFamily::of_key(key);
594 if this != family
595 && let Some(family) = family.or(this)
596 {
597 return Err(AnnotationKeyError::MixedFamilies { family });
598 }
599 if reject_duplicates && !seen.insert(key) {
600 return Err(AnnotationKeyError::DuplicateKey);
601 }
602 }
603 Ok(family)
604}
605
606pub struct AnnotationContext<'a> {
608 pub data_model: Option<&'a str>,
609 pub schema: &'a Schema,
610 pub partition_key_indices: &'a [usize],
611}
612
613#[derive(Debug, Clone, PartialEq, Eq)]
618pub enum AnnotationValidationError {
619 UnknownKey {
620 key: String,
621 },
622 InvalidValue {
623 key: String,
624 value: String,
625 },
626 ColumnNotFound {
627 column: String,
628 },
629 ColumnNotStringForm {
630 key: String,
631 column: String,
632 ty: ConcreteDataType,
633 },
634 InvalidPartitionNumHint {
635 value: String,
636 },
637 NotSingleColumn,
638 PartitionMetadataConflict,
639 TimeIndexConflict,
640}
641
642impl fmt::Display for AnnotationValidationError {
643 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
644 match self {
645 Self::UnknownKey { key } => write!(f, "unknown semantic option `{key}`"),
646 Self::InvalidValue { key, value } => {
647 write!(f, "invalid value `{value}` for semantic option `{key}`")
648 }
649 Self::ColumnNotFound { column } => write!(f, "column `{column}` not found"),
650 Self::ColumnNotStringForm { key, column, ty } => write!(
651 f,
652 "entity column `{column}` (option `{key}`) has type `{ty}`, \
653 which cannot render as a string"
654 ),
655 Self::InvalidPartitionNumHint { value } => write!(
656 f,
657 "{REPARTITION_PARTITION_NUM_HINT_KEY} expects a positive integer within u32 range, got `{value}`"
658 ),
659 Self::NotSingleColumn => write!(
660 f,
661 "{REPARTITION_COLUMN_HINT_KEY} expects exactly one column name"
662 ),
663 Self::PartitionMetadataConflict => write!(
664 f,
665 "cannot set {REPARTITION_COLUMN_HINT_KEY} on a table with partition metadata"
666 ),
667 Self::TimeIndexConflict => write!(
668 f,
669 "cannot set {REPARTITION_COLUMN_HINT_KEY} to the time index column"
670 ),
671 }
672 }
673}
674
675pub(crate) fn validate_and_normalize_annotation(
679 family: AnnotationFamily,
680 cx: &AnnotationContext<'_>,
681 key: &str,
682 value: &str,
683) -> std::result::Result<String, AnnotationValidationError> {
684 match family {
685 AnnotationFamily::Semantic => {
686 if !is_semantic_option_key(key) {
687 return Err(AnnotationValidationError::UnknownKey {
688 key: key.to_string(),
689 });
690 }
691 if !validate_semantic_option(key, value) {
692 return Err(AnnotationValidationError::InvalidValue {
693 key: key.to_string(),
694 value: value.to_string(),
695 });
696 }
697 if parse_entity_option_key(key).is_some() {
698 for column in parse_entity_columns(value) {
699 if trace_v2_attribute(cx.schema, cx.data_model, &column).is_some() {
700 continue;
701 }
702 let schema = cx.schema.column_schema_by_name(&column).ok_or_else(|| {
703 AnnotationValidationError::ColumnNotFound {
704 column: column.clone(),
705 }
706 })?;
707 if !has_stable_string_form(&schema.data_type) {
708 return Err(AnnotationValidationError::ColumnNotStringForm {
709 key: key.to_string(),
710 column,
711 ty: schema.data_type.clone(),
712 });
713 }
714 }
715 }
716 Ok(value.to_string())
717 }
718 AnnotationFamily::RepartitionHint if key == REPARTITION_PARTITION_NUM_HINT_KEY => {
719 let value = value.trim();
720 if !matches!(value.parse::<u32>(), Ok(1..)) {
721 return Err(AnnotationValidationError::InvalidPartitionNumHint {
722 value: value.to_string(),
723 });
724 }
725 Ok(value.to_string())
726 }
727 AnnotationFamily::RepartitionHint => {
728 let column_name = value.trim();
729 if column_name.is_empty() || column_name.contains(',') {
730 return Err(AnnotationValidationError::NotSingleColumn);
731 }
732 if !cx.partition_key_indices.is_empty() {
733 return Err(AnnotationValidationError::PartitionMetadataConflict);
734 }
735 let column_index = cx.schema.column_index_by_name(column_name).ok_or_else(|| {
736 AnnotationValidationError::ColumnNotFound {
737 column: column_name.to_string(),
738 }
739 })?;
740 if cx.schema.timestamp_index() == Some(column_index) {
741 return Err(AnnotationValidationError::TimeIndexConflict);
742 }
743 Ok(column_name.to_string())
744 }
745 }
746}
747
748pub fn validate_and_normalize_annotation_options(
751 options: &mut TableOptions,
752 schema: &Schema,
753 partition_key_indices: &[usize],
754) -> std::result::Result<(), AnnotationValidationError> {
755 let cx = AnnotationContext {
756 data_model: options.data_model(),
757 schema,
758 partition_key_indices,
759 };
760 let mut normalized = Vec::new();
761 for (key, value) in &options.extra_options {
762 let Some(family) = AnnotationFamily::of_key(key) else {
763 continue;
764 };
765 let checked = validate_and_normalize_annotation(family, &cx, key, value)?;
766 if checked != *value {
767 normalized.push((key.clone(), checked));
768 }
769 }
770 for (key, value) in normalized {
771 options.extra_options.insert(key, value);
772 }
773 Ok(())
774}
775
776#[derive(Debug, Clone, Serialize, Deserialize)]
777pub enum AlterKind {
778 AddColumns {
779 columns: Vec<AddColumnRequest>,
780 },
781 DropColumns {
782 names: Vec<String>,
783 },
784 ModifyColumnTypes {
785 columns: Vec<ModifyColumnTypeRequest>,
786 },
787 SetJsonSettings {
788 request: SetJsonSettingsRequest,
789 },
790 RenameTable {
791 new_table_name: String,
792 },
793 SetTableOptions {
794 options: Vec<SetRegionOption>,
795 },
796 UnsetTableOptions {
797 keys: Vec<UnsetRegionOption>,
798 },
799 SetAnnotations {
800 family: AnnotationFamily,
801 options: Vec<(String, String)>,
802 },
803 UnsetAnnotations {
804 family: AnnotationFamily,
805 keys: Vec<String>,
806 },
807 SetIndexes {
808 options: Vec<SetIndexOption>,
809 },
810 UnsetIndexes {
811 options: Vec<UnsetIndexOption>,
812 },
813 DropDefaults {
814 names: Vec<String>,
815 },
816 SetDefaults {
817 defaults: Vec<SetDefaultRequest>,
818 },
819}
820
821#[derive(Debug, Clone, Serialize, Deserialize)]
822pub struct SetDefaultRequest {
823 pub column_name: String,
824 pub default_constraint: Option<ColumnDefaultConstraint>,
825}
826
827#[derive(Debug, Clone, Serialize, Deserialize)]
828pub enum SetIndexOption {
829 Fulltext {
830 column_name: String,
831 options: FulltextOptions,
832 },
833 Inverted {
834 column_name: String,
835 },
836 Skipping {
837 column_name: String,
838 options: SkippingIndexOptions,
839 },
840}
841
842impl SetIndexOption {
843 pub fn column_name(&self) -> &str {
845 match self {
846 SetIndexOption::Fulltext { column_name, .. } => column_name,
847 SetIndexOption::Inverted { column_name, .. } => column_name,
848 SetIndexOption::Skipping { column_name, .. } => column_name,
849 }
850 }
851}
852
853#[derive(Debug, Clone, Serialize, Deserialize)]
854pub enum UnsetIndexOption {
855 Fulltext { column_name: String },
856 Inverted { column_name: String },
857 Skipping { column_name: String },
858}
859
860impl UnsetIndexOption {
861 pub fn column_name(&self) -> &str {
863 match self {
864 UnsetIndexOption::Fulltext { column_name, .. } => column_name,
865 UnsetIndexOption::Inverted { column_name, .. } => column_name,
866 UnsetIndexOption::Skipping { column_name, .. } => column_name,
867 }
868 }
869}
870
871#[derive(Debug)]
872pub struct InsertRequest {
873 pub catalog_name: String,
874 pub schema_name: String,
875 pub table_name: String,
876 pub columns_values: HashMap<String, VectorRef>,
877 pub skip_wal: bool,
879}
880
881#[derive(Debug)]
883pub struct DeleteRequest {
884 pub catalog_name: String,
885 pub schema_name: String,
886 pub table_name: String,
887 pub key_column_values: HashMap<String, VectorRef>,
891}
892
893#[derive(Debug)]
894pub enum CopyDirection {
895 Export,
896 Import,
897}
898
899#[derive(Debug)]
901pub struct CopyTableRequest {
902 pub catalog_name: String,
903 pub schema_name: String,
904 pub table_name: String,
905 pub location: String,
906 pub with: HashMap<String, String>,
907 pub connection: HashMap<String, String>,
908 pub pattern: Option<String>,
909 pub direction: CopyDirection,
910 pub timestamp_range: Option<TimestampRange>,
911 pub limit: Option<u64>,
912}
913
914#[derive(Debug, Clone, Default)]
915pub struct FlushTableRequest {
916 pub catalog_name: String,
917 pub schema_name: String,
918 pub table_name: String,
919}
920
921#[derive(Debug, Clone, Default)]
922pub struct BuildIndexTableRequest {
923 pub options: Option<build_index_request::Options>,
925 pub catalog_name: String,
926 pub schema_name: String,
927 pub table_name: String,
928}
929
930#[derive(Debug, Clone, PartialEq)]
931pub struct CompactTableRequest {
932 pub catalog_name: String,
933 pub schema_name: String,
934 pub table_name: String,
935 pub compact_options: compact_request::Options,
936 pub parallelism: u32,
937 pub time_range: Option<TimestampRange>,
938}
939
940impl Default for CompactTableRequest {
941 fn default() -> Self {
942 Self {
943 catalog_name: Default::default(),
944 schema_name: Default::default(),
945 table_name: Default::default(),
946 compact_options: compact_request::Options::Regular(Default::default()),
947 parallelism: 1,
948 time_range: None,
949 }
950 }
951}
952
953#[derive(Debug, Clone, Default, Deserialize, Serialize)]
954pub struct CopyDatabaseRequest {
955 pub catalog_name: String,
956 pub schema_name: String,
957 pub location: String,
958 pub with: HashMap<String, String>,
959 pub connection: HashMap<String, String>,
960 pub time_range: Option<TimestampRange>,
961}
962
963#[derive(Debug, Clone, Default, Deserialize, Serialize)]
964pub struct CopyQueryToRequest {
965 pub location: String,
966 pub with: HashMap<String, String>,
967 pub connection: HashMap<String, String>,
968}
969
970#[cfg(test)]
971mod tests {
972 use std::time::Duration;
973
974 use common_error::ext::ErrorExt;
975 use common_error::status_code::StatusCode;
976
977 use super::*;
978
979 #[test]
980 fn test_redact_connection() {
981 assert_eq!(
982 redact_connection("mysql://user:secret@127.0.0.1:3306/db"),
983 "mysql://user:******@127.0.0.1:3306/db"
984 );
985 assert_eq!(
987 redact_connection("mysql://user@127.0.0.1:3306/db"),
988 "mysql://user@127.0.0.1:3306/db"
989 );
990 assert_eq!(redact_connection("not-a-url"), "******");
992 }
993
994 #[test]
995 fn test_relational_options_redacted() {
996 let options = RelationalOptions {
997 connection: "mysql://user:secret@127.0.0.1:3306/db".to_string(),
998 remote_table: "sample".to_string(),
999 };
1000 let redacted = options.redacted();
1001 assert_eq!(redacted.connection, "mysql://user:******@127.0.0.1:3306/db");
1002 assert_eq!(redacted.remote_table, "sample");
1003 }
1004
1005 #[test]
1006 fn test_relational_options_debug_redacts_connection() {
1007 let options = RelationalOptions {
1008 connection: "mysql://user:secret@127.0.0.1:3306/db".to_string(),
1009 remote_table: "sample".to_string(),
1010 };
1011 let debug = format!("{options:?}");
1012 assert!(!debug.contains("secret"));
1013 assert!(debug.contains("******"));
1014 assert!(debug.contains("sample"));
1015 }
1016
1017 #[test]
1018 fn test_table_options_display_redacts_relational_meta() {
1019 let options = TableOptions {
1020 extra_options: HashMap::from([(
1021 RELATIONAL_META_KEY.to_string(),
1022 r#"{"connection":"mysql://user:secret@127.0.0.1:3306/db","remote_table":"sample"}"#
1023 .to_string(),
1024 )]),
1025 ..Default::default()
1026 };
1027 let rendered = options.to_string();
1028 assert!(!rendered.contains("secret"));
1029 assert!(rendered.contains("******"));
1030 }
1031
1032 #[test]
1033 fn test_table_options_debug_redacts_relational_meta() {
1034 let options = TableOptions {
1035 extra_options: HashMap::from([(
1036 RELATIONAL_META_KEY.to_string(),
1037 r#"{"connection":"mysql://user:secret@127.0.0.1:3306/db","remote_table":"sample"}"#
1038 .to_string(),
1039 )]),
1040 ..Default::default()
1041 };
1042 let rendered = format!("{options:?}");
1043 assert!(!rendered.contains("secret"));
1044 assert!(rendered.contains("******"));
1045 }
1046
1047 #[test]
1048 fn test_table_options_debug_redacts_credential() {
1049 let options = TableOptions {
1050 extra_options: HashMap::from([
1051 ("secret_access_key".to_string(), "my-secret".to_string()),
1052 ("session_token".to_string(), "my-token".to_string()),
1053 ("location".to_string(), "s3://bucket/file.csv".to_string()),
1054 ]),
1055 ..Default::default()
1056 };
1057 let rendered = format!("{options:?}");
1058 assert!(!rendered.contains("my-secret"));
1059 assert!(!rendered.contains("my-token"));
1060 assert!(rendered.contains("******"));
1061 assert!(rendered.contains("s3://bucket/file.csv"));
1062 }
1063
1064 #[test]
1065 fn test_validate_annotation_keys() {
1066 let column = REPARTITION_COLUMN_HINT_KEY;
1067 let count = REPARTITION_PARTITION_NUM_HINT_KEY;
1068 for (keys, expected) in [
1069 (vec![], None),
1070 (vec![TTL_KEY, TTL_KEY], None),
1071 (vec!["repartition.unknown.hint"], None),
1072 (vec![column], Some(AnnotationFamily::RepartitionHint)),
1073 (vec![count], Some(AnnotationFamily::RepartitionHint)),
1074 (vec![column, count], Some(AnnotationFamily::RepartitionHint)),
1075 (vec![count, column], Some(AnnotationFamily::RepartitionHint)),
1076 (
1077 vec!["greptime.semantic.source", "greptime.semantic.source"],
1078 Some(AnnotationFamily::Semantic),
1079 ),
1080 ] {
1081 assert_eq!(validate_annotation_keys(keys), Ok(expected));
1082 }
1083 for keys in [
1084 vec![column, column],
1085 vec![count, count],
1086 vec![column, count, column],
1087 ] {
1088 assert_eq!(
1089 validate_annotation_keys(keys),
1090 Err(AnnotationKeyError::DuplicateKey)
1091 );
1092 }
1093 for keys in [
1094 vec![column, TTL_KEY],
1095 vec![TTL_KEY, count],
1096 vec![column, "greptime.semantic.source"],
1097 ] {
1098 assert_eq!(
1099 validate_annotation_keys(keys),
1100 Err(AnnotationKeyError::MixedFamilies {
1101 family: AnnotationFamily::RepartitionHint,
1102 })
1103 );
1104 }
1105 }
1106
1107 #[test]
1108 fn test_validate_table_option() {
1109 assert!(validate_table_option(FILE_TABLE_LOCATION_KEY));
1110 assert!(validate_table_option(FILE_TABLE_FORMAT_KEY));
1111 assert!(validate_table_option(FILE_TABLE_PATTERN_KEY));
1112 assert!(validate_table_option(RELATIONAL_CONNECTION_KEY));
1113 assert!(validate_table_option(RELATIONAL_REMOTE_TABLE_KEY));
1114 assert!(validate_table_option(TTL_KEY));
1115 assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY));
1116 assert!(validate_table_option(STORAGE_KEY));
1117 assert!(validate_table_option(MEMTABLE_BULK_MERGE_THRESHOLD));
1118 assert!(validate_table_option(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING));
1119 assert!(validate_table_option(REPARTITION_COLUMN_HINT_KEY));
1120 assert!(validate_table_option(REPARTITION_PARTITION_NUM_HINT_KEY));
1121 assert_eq!(AnnotationFamily::of_key("repartition.unknown.hint"), None);
1122 assert!(!validate_table_option("foo"));
1123
1124 assert!(validate_table_option(SEMANTIC_SIGNAL_TYPE));
1126 assert!(validate_table_option(SEMANTIC_METRIC_TYPE));
1127 assert!(!validate_table_option("greptime.semantic.future.key"));
1129 assert!(!validate_table_option("greptime.semanticx"));
1130 assert!(!validate_table_option(SEMANTIC_PER_TABLE_INDEX_KEY));
1131 }
1132
1133 #[test]
1134 fn test_validate_database_option() {
1135 assert!(validate_database_option(MEMTABLE_TYPE));
1136 assert!(validate_database_option(MEMTABLE_BULK_MERGE_THRESHOLD));
1137 assert!(validate_database_option(MEMTABLE_BULK_ENCODE_ROW_THRESHOLD));
1138 assert!(validate_database_option(
1139 MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD
1140 ));
1141 assert!(validate_database_option(MEMTABLE_BULK_MAX_MERGE_GROUPS));
1142 assert!(validate_database_option(
1143 "compaction.twcs.active_window.trigger_file_num"
1144 ));
1145 assert!(validate_database_option(
1146 "compaction.twcs.active_window.l1_merge_trigger"
1147 ));
1148 assert!(validate_database_option(
1149 "compaction.twcs.inactive_window.trigger_file_num"
1150 ));
1151 assert!(validate_database_option(
1152 "compaction.twcs.inactive_window.l1_merge_trigger"
1153 ));
1154 assert!(validate_database_option(INGEST_ROWS_RATE_LIMIT_KEY));
1155 assert!(validate_database_option("ingest_rows_rate_limit"));
1156 assert!(!validate_database_option("foo"));
1157 }
1158
1159 #[test]
1160 fn test_parse_float_field_encoding_option() {
1161 let options = TableOptions::try_from_iter([(
1162 EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
1163 "byte_stream_split",
1164 )])
1165 .unwrap();
1166 assert_eq!(
1167 options
1168 .extra_options
1169 .get(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING),
1170 Some(&"byte_stream_split".to_string())
1171 );
1172 assert!(
1173 TableOptions::try_from_iter([(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "invalid",)])
1174 .is_err()
1175 );
1176 }
1177
1178 #[test]
1179 fn test_database_trigger_value_boundaries() {
1180 let maximum = usize::MAX.to_string();
1181 let overflow = format!("{maximum}0");
1182 for (key, minimum) in [
1183 (TWCS_TRIGGER_FILE_NUM, 0),
1184 (TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, 0),
1185 (TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM, 0),
1186 (TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER, 2),
1187 (TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER, 2),
1188 ] {
1189 for invalid in [
1190 None,
1191 Some(""),
1192 Some("invalid"),
1193 Some("-1"),
1194 Some(overflow.as_str()),
1195 ] {
1196 assert!(
1197 validate_database_option_value(key, invalid).is_err(),
1198 "{key}: {invalid:?}"
1199 );
1200 }
1201 for valid in ["2", maximum.as_str()] {
1202 assert!(
1203 validate_database_option_value(key, Some(valid)).is_ok(),
1204 "{key}: {valid}"
1205 );
1206 }
1207 for boundary in ["0", "1"] {
1208 assert_eq!(
1209 validate_database_option_value(key, Some(boundary)).is_ok(),
1210 minimum == 0,
1211 "{key}: {boundary}"
1212 );
1213 }
1214 }
1215 }
1216
1217 #[test]
1218 fn test_database_ingest_rate_limit_value_boundaries() {
1219 for invalid in [
1220 None,
1221 Some(""),
1222 Some("abc"),
1223 Some("1000/s"),
1224 Some("-1"),
1225 Some("1.5"),
1226 Some("18446744073709551616"),
1227 ] {
1228 assert!(validate_database_option_value(INGEST_ROWS_RATE_LIMIT_KEY, invalid).is_err());
1229 }
1230 let maximum = u64::MAX.to_string();
1231 for valid in ["0", "1", maximum.as_str()] {
1232 assert!(
1233 validate_database_option_value(INGEST_ROWS_RATE_LIMIT_KEY, Some(valid)).is_ok()
1234 );
1235 }
1236 }
1237
1238 #[test]
1239 fn test_serialize_table_options() {
1240 let options = TableOptions {
1241 write_buffer_size: None,
1242 ttl: Some(Duration::from_secs(1000).into()),
1243 extra_options: HashMap::new(),
1244 skip_wal: false,
1245 };
1246 let serialized = serde_json::to_string(&options).unwrap();
1247 let deserialized: TableOptions = serde_json::from_str(&serialized).unwrap();
1248 assert_eq!(options, deserialized);
1249 }
1250
1251 #[test]
1252 fn test_convert_hashmap_between_table_options() {
1253 let options = TableOptions {
1254 write_buffer_size: Some(ReadableSize::mb(128)),
1255 ttl: Some(Duration::from_secs(1000).into()),
1256 extra_options: HashMap::new(),
1257 skip_wal: false,
1258 };
1259 let serialized_map = HashMap::from(&options);
1260 let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1261 assert_eq!(options, serialized);
1262
1263 let options = TableOptions {
1264 write_buffer_size: None,
1265 ttl: None,
1266 extra_options: HashMap::from([(SKIP_WAL_KEY.to_string(), true.to_string())]),
1267 skip_wal: true,
1268 };
1269 let serialized_map = HashMap::from(&options);
1270 assert_eq!(
1271 Some("true"),
1272 serialized_map.get(SKIP_WAL_KEY).map(String::as_str)
1273 );
1274 let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1275 assert_eq!(options, serialized);
1276
1277 let options = TableOptions {
1278 write_buffer_size: None,
1279 ttl: Default::default(),
1280 extra_options: HashMap::new(),
1281 skip_wal: false,
1282 };
1283 let serialized_map = HashMap::from(&options);
1284 let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1285 assert_eq!(options, serialized);
1286
1287 let options = TableOptions {
1288 write_buffer_size: Some(ReadableSize::mb(128)),
1289 ttl: Some(Duration::from_secs(1000).into()),
1290 extra_options: HashMap::from([("a".to_string(), "A".to_string())]),
1291 skip_wal: false,
1292 };
1293 let serialized_map = HashMap::from(&options);
1294 let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1295 assert_eq!(options, serialized);
1296
1297 let options = TableOptions {
1298 extra_options: HashMap::from([(SKIP_WAL_KEY.to_string(), false.to_string())]),
1299 skip_wal: false,
1300 ..Default::default()
1301 };
1302 let serialized_map = HashMap::from(&options);
1303 let serialized = TableOptions::try_from_iter(&serialized_map).unwrap();
1304 assert_eq!(options, serialized);
1305 }
1306
1307 #[test]
1308 fn test_table_options_normalizes_twcs_trigger_aliases() {
1309 for options in [
1310 vec![(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, "4")],
1311 vec![
1312 (TWCS_TRIGGER_FILE_NUM, "4"),
1313 (TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, "4"),
1314 ],
1315 ] {
1316 let table_options = TableOptions::try_from_iter(options).unwrap();
1317 assert_eq!(
1318 HashMap::from([(TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string())]),
1319 table_options.extra_options
1320 );
1321 }
1322 }
1323
1324 #[test]
1325 fn test_table_options_rejects_conflicting_twcs_trigger_aliases() {
1326 let error = TableOptions::try_from_iter([
1327 (TWCS_TRIGGER_FILE_NUM, "4"),
1328 (TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, "8"),
1329 ])
1330 .unwrap_err();
1331 assert_eq!(StatusCode::InvalidArguments, error.status_code());
1332 assert_eq!(
1333 "Conflicting table options: compaction.twcs.trigger_file_num=4 and compaction.twcs.active_window.trigger_file_num=8",
1334 error.to_string()
1335 );
1336 }
1337
1338 #[test]
1339 fn test_table_options_to_string() {
1340 let options = TableOptions {
1341 write_buffer_size: Some(ReadableSize::mb(128)),
1342 ttl: Some(Duration::from_secs(1000).into()),
1343 extra_options: HashMap::new(),
1344 skip_wal: false,
1345 };
1346
1347 assert_eq!(
1348 "write_buffer_size=128.0MiB ttl=16m 40s",
1349 options.to_string()
1350 );
1351
1352 let options = TableOptions {
1353 write_buffer_size: Some(ReadableSize::mb(128)),
1354 ttl: Some(Duration::from_secs(1000).into()),
1355 extra_options: HashMap::from([("a".to_string(), "A".to_string())]),
1356 skip_wal: false,
1357 };
1358
1359 assert_eq!(
1360 "write_buffer_size=128.0MiB ttl=16m 40s a=A",
1361 options.to_string()
1362 );
1363
1364 let options = TableOptions {
1365 write_buffer_size: Some(ReadableSize::mb(128)),
1366 ttl: Some(Duration::from_secs(1000).into()),
1367 extra_options: HashMap::new(),
1368 skip_wal: true,
1369 };
1370 assert_eq!(
1371 "write_buffer_size=128.0MiB ttl=16m 40s skip_wal=true",
1372 options.to_string()
1373 );
1374
1375 let options = TableOptions {
1376 write_buffer_size: None,
1377 ttl: None,
1378 extra_options: HashMap::from([(SKIP_WAL_KEY.to_string(), "false".to_string())]),
1379 skip_wal: false,
1380 };
1381 assert_eq!("skip_wal=false", options.to_string());
1382 }
1383}