1use std::collections::HashMap;
16use std::fmt::{self, Display};
17use std::time::Duration;
18
19use api::helper::{ColumnDataTypeWrapper, from_pb_time_ranges, from_pb_time_unit};
20use api::v1::add_column_location::LocationType;
21use api::v1::column_def::{
22 as_fulltext_option_analyzer, as_fulltext_option_backend, as_skipping_index_type,
23};
24use api::v1::region::bulk_insert_request::Body;
25use api::v1::region::{
26 AlterRequest, AlterRequests, BuildIndexRequest, BulkInsertRequest, CancelCompactionRequest,
27 CleanUpRequest as PbCleanUpRequest, CloseRequest, CompactRequest, CreateRequest,
28 CreateRequests, DeleteRequests, DropRequest, DropRequests, FlushRequest, InsertRequests,
29 OpenRequest, TruncateRequest, alter_request, build_index_request, compact_request,
30 region_request, truncate_request,
31};
32use api::v1::{
33 self, Analyzer, ArrowIpc, FulltextBackend as PbFulltextBackend, Option as PbOption, Rows,
34 SemanticType, SkippingIndexType as PbSkippingIndexType, WriteHint,
35};
36use arrow_schema::extension::ExtensionType;
37pub use common_base::AffectedRows;
38use common_base::readable_size::ReadableSize;
39use common_grpc::flight::FlightDecoder;
40use common_recordbatch::DfRecordBatch;
41use common_time::range::TimestampRange;
42use common_time::{TimeToLive, Timestamp};
43use datatypes::error::time_index_not_widening_error;
44use datatypes::extension::json::Json2ExtensionType;
45use datatypes::json::{JsonSettings, JsonTypeHint};
46use datatypes::prelude::ConcreteDataType;
47use datatypes::schema::{FulltextOptions, SkippingIndexOptions};
48use num_enum::TryFromPrimitive;
49use serde::{Deserialize, Deserializer, Serialize, Serializer};
50use snafu::{OptionExt, ResultExt, ensure};
51use strum::{AsRefStr, IntoStaticStr};
52
53use crate::logstore::entry;
54use crate::metadata::{
55 ColumnMetadata, ConvertTimeRangesSnafu, DecodeProtoSnafu, FlightCodecSnafu,
56 InvalidIndexOptionSnafu, InvalidRawRegionRequestSnafu, InvalidRegionRequestSnafu,
57 InvalidSetRegionOptionRequestSnafu, InvalidUnsetRegionOptionRequestSnafu, MetadataError,
58 RegionMetadata, Result, UnexpectedSnafu,
59};
60use crate::metric_engine_consts::PHYSICAL_TABLE_METADATA_KEY;
61use crate::metrics;
62use crate::mito_engine_options::{
63 APPEND_MODE_KEY, AUTO_FLUSH_INTERVAL_KEY, COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM,
64 MAX_ROW_GROUP_ROW_COUNT, MAX_ROW_GROUP_ROW_COUNT_LIMIT, PRESERVE_ROW_SEQUENCE, SKIP_WAL_KEY,
65 SST_FORMAT_KEY, TTL_KEY, TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER,
66 TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER,
67 TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM, TWCS_MAX_OUTPUT_FILE_SIZE, TWCS_TIME_WINDOW,
68 TWCS_TRIGGER_FILE_NUM, WRITE_BUFFER_SIZE_KEY,
69};
70use crate::path_utils::table_dir;
71use crate::storage::{ColumnId, RegionId};
72
73#[derive(Debug, Clone, Copy, PartialEq, TryFromPrimitive)]
75#[repr(u8)]
76pub enum PathType {
77 Bare,
81 Data,
85 Metadata,
89}
90
91#[derive(Debug, IntoStaticStr)]
92pub enum BatchRegionDdlRequest {
93 Create(Vec<(RegionId, RegionCreateRequest)>),
94 Drop(Vec<(RegionId, RegionDropRequest)>),
95 Alter(Vec<(RegionId, RegionAlterRequest)>),
96}
97
98impl BatchRegionDdlRequest {
99 pub fn try_from_request_body(body: region_request::Body) -> Result<Option<Self>> {
101 match body {
102 region_request::Body::Creates(creates) => {
103 let requests = creates
104 .requests
105 .into_iter()
106 .map(parse_region_create)
107 .collect::<Result<Vec<_>>>()?;
108 Ok(Some(Self::Create(requests)))
109 }
110 region_request::Body::Drops(drops) => {
111 let requests = drops
112 .requests
113 .into_iter()
114 .map(parse_region_drop)
115 .collect::<Result<Vec<_>>>()?;
116 Ok(Some(Self::Drop(requests)))
117 }
118 region_request::Body::Alters(alters) => {
119 let requests = alters
120 .requests
121 .into_iter()
122 .map(parse_region_alter)
123 .collect::<Result<Vec<_>>>()?;
124 Ok(Some(Self::Alter(requests)))
125 }
126 _ => Ok(None),
127 }
128 }
129
130 pub fn request_type(&self) -> &'static str {
131 self.into()
132 }
133
134 pub fn into_region_requests(self) -> Vec<(RegionId, RegionRequest)> {
135 match self {
136 Self::Create(requests) => requests
137 .into_iter()
138 .map(|(region_id, request)| (region_id, RegionRequest::Create(request)))
139 .collect(),
140 Self::Drop(requests) => requests
141 .into_iter()
142 .map(|(region_id, request)| (region_id, RegionRequest::Drop(request)))
143 .collect(),
144 Self::Alter(requests) => requests
145 .into_iter()
146 .map(|(region_id, request)| (region_id, RegionRequest::Alter(request)))
147 .collect(),
148 }
149 }
150}
151
152#[derive(Debug, IntoStaticStr)]
153pub enum RegionRequest {
154 Put(RegionPutRequest),
155 Delete(RegionDeleteRequest),
156 Create(RegionCreateRequest),
157 Drop(RegionDropRequest),
158 Open(RegionOpenRequest),
159 CleanUp(RegionCleanUpRequest),
160 Close(RegionCloseRequest),
161 Alter(RegionAlterRequest),
162 Flush(RegionFlushRequest),
163 Compact(RegionCompactRequest),
164 CancelCompaction(RegionCancelCompactionRequest),
165 BuildIndex(RegionBuildIndexRequest),
166 Truncate(RegionTruncateRequest),
167 Catchup(RegionCatchupRequest),
168 BulkInserts(RegionBulkInsertsRequest),
169 EnterStaging(EnterStagingRequest),
170 ApplyStagingManifest(ApplyStagingManifestRequest),
171}
172
173impl RegionRequest {
174 pub fn try_from_request_body(body: region_request::Body) -> Result<Vec<(RegionId, Self)>> {
177 match body {
178 region_request::Body::Inserts(inserts) => make_region_puts(inserts),
179 region_request::Body::Deletes(deletes) => make_region_deletes(deletes),
180 region_request::Body::Create(create) => make_region_create(create),
181 region_request::Body::Drop(drop) => make_region_drop(drop),
182 region_request::Body::Open(open) => make_region_open(open),
183 region_request::Body::CleanUp(clean_up) => make_region_clean_up(clean_up),
184 region_request::Body::Close(close) => make_region_close(close),
185 region_request::Body::Alter(alter) => make_region_alter(alter),
186 region_request::Body::Flush(flush) => make_region_flush(flush),
187 region_request::Body::Compact(compact) => make_region_compact(compact),
188 region_request::Body::CancelCompaction(cancel) => make_region_cancel_compaction(cancel),
189 region_request::Body::BuildIndex(index) => make_region_build_index(index),
190 region_request::Body::Truncate(truncate) => make_region_truncate(truncate),
191 region_request::Body::Creates(creates) => make_region_creates(creates),
192 region_request::Body::Drops(drops) => make_region_drops(drops),
193 region_request::Body::Alters(alters) => make_region_alters(alters),
194 region_request::Body::BulkInsert(bulk) => make_region_bulk_inserts(bulk),
195 region_request::Body::Sync(_) => UnexpectedSnafu {
196 reason: "Sync request should be handled separately by RegionServer",
197 }
198 .fail(),
199 region_request::Body::ListMetadata(_) => UnexpectedSnafu {
200 reason: "ListMetadata request should be handled separately by RegionServer",
201 }
202 .fail(),
203 region_request::Body::RemoteDynFilter(_) => UnexpectedSnafu {
204 reason: "RemoteDynFilter request should be handled separately by RegionServer",
205 }
206 .fail(),
207 region_request::Body::ApplyStagingManifest(apply) => {
208 make_region_apply_staging_manifest(apply)
209 }
210 }
211 }
212
213 pub fn request_type(&self) -> &'static str {
215 self.into()
216 }
217}
218
219pub fn make_region_put(
221 request: api::v1::region::InsertRequest,
222) -> Option<(RegionId, RegionPutRequest)> {
223 request.rows.map(|rows| {
224 (
225 request.region_id.into(),
226 RegionPutRequest {
227 rows,
228 hint: None,
229 skip_wal: request.skip_wal,
230 partition_expr_version: request.partition_expr_version.map(|version| version.value),
231 },
232 )
233 })
234}
235
236fn make_region_puts(inserts: InsertRequests) -> Result<Vec<(RegionId, RegionRequest)>> {
237 Ok(inserts
238 .requests
239 .into_iter()
240 .filter_map(make_region_put)
241 .map(|(region_id, request)| (region_id, RegionRequest::Put(request)))
242 .collect())
243}
244
245pub fn make_region_delete(
247 request: api::v1::region::DeleteRequest,
248) -> Option<(RegionId, RegionDeleteRequest)> {
249 request.rows.map(|rows| {
250 (
251 request.region_id.into(),
252 RegionDeleteRequest {
253 rows,
254 hint: None,
255 partition_expr_version: request.partition_expr_version.map(|version| version.value),
256 },
257 )
258 })
259}
260
261fn make_region_deletes(deletes: DeleteRequests) -> Result<Vec<(RegionId, RegionRequest)>> {
262 Ok(deletes
263 .requests
264 .into_iter()
265 .filter_map(make_region_delete)
266 .map(|(region_id, request)| (region_id, RegionRequest::Delete(request)))
267 .collect())
268}
269
270fn parse_region_create(create: CreateRequest) -> Result<(RegionId, RegionCreateRequest)> {
271 let column_metadatas = create
272 .column_defs
273 .into_iter()
274 .map(ColumnMetadata::try_from_column_def)
275 .collect::<Result<Vec<_>>>()?;
276 let region_id = RegionId::from(create.region_id);
277 let table_dir = table_dir(&create.path, region_id.table_id());
278 let partition_expr_json = create.partition.as_ref().map(|p| p.expression.clone());
279 Ok((
280 region_id,
281 RegionCreateRequest {
282 engine: create.engine,
283 column_metadatas,
284 primary_key: create.primary_key,
285 options: create.options,
286 table_dir,
287 path_type: PathType::Bare,
288 partition_expr_json,
289 requirements: create
290 .requirements
291 .map(RegionRequirements::from)
292 .unwrap_or_default(),
293 },
294 ))
295}
296
297fn make_region_create(create: CreateRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
298 let (region_id, request) = parse_region_create(create)?;
299 Ok(vec![(region_id, RegionRequest::Create(request))])
300}
301
302fn make_region_creates(creates: CreateRequests) -> Result<Vec<(RegionId, RegionRequest)>> {
303 let mut requests = Vec::with_capacity(creates.requests.len());
304 for create in creates.requests {
305 requests.extend(make_region_create(create)?);
306 }
307 Ok(requests)
308}
309
310fn parse_region_drop(drop: DropRequest) -> Result<(RegionId, RegionDropRequest)> {
311 let region_id = drop.region_id.into();
312 Ok((
313 region_id,
314 RegionDropRequest {
315 fast_path: drop.fast_path,
316 force: drop.force,
317 partial_drop: drop.partial_drop,
318 },
319 ))
320}
321
322fn make_region_drop(drop: DropRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
323 let (region_id, request) = parse_region_drop(drop)?;
324 Ok(vec![(region_id, RegionRequest::Drop(request))])
325}
326
327fn make_region_drops(drops: DropRequests) -> Result<Vec<(RegionId, RegionRequest)>> {
328 let mut requests = Vec::with_capacity(drops.requests.len());
329 for drop in drops.requests {
330 requests.extend(make_region_drop(drop)?);
331 }
332 Ok(requests)
333}
334
335fn make_region_open(open: OpenRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
336 let region_id = RegionId::from(open.region_id);
337 let table_dir = table_dir(&open.path, region_id.table_id());
338 Ok(vec![(
339 region_id,
340 RegionRequest::Open(RegionOpenRequest {
341 engine: open.engine,
342 table_dir,
343 path_type: PathType::Bare,
344 options: open.options,
345 skip_wal_replay: false,
346 checkpoint: None,
347 requirements: Default::default(),
348 }),
349 )])
350}
351
352fn make_region_clean_up(clean_up: PbCleanUpRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
353 let region_id = RegionId::from(clean_up.region_id);
354 let table_dir = table_dir(&clean_up.path, region_id.table_id());
355 Ok(vec![(
356 region_id,
357 RegionRequest::CleanUp(RegionCleanUpRequest {
358 engine: clean_up.engine,
359 table_dir,
360 path_type: PathType::Bare,
361 options: clean_up.options,
362 }),
363 )])
364}
365
366fn make_region_close(close: CloseRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
367 let region_id = close.region_id.into();
368 Ok(vec![(
369 region_id,
370 RegionRequest::Close(RegionCloseRequest {
371 flush_on_close: close.flush_on_close,
372 }),
373 )])
374}
375
376fn parse_region_alter(alter: AlterRequest) -> Result<(RegionId, RegionAlterRequest)> {
377 let region_id = alter.region_id.into();
378 let request = RegionAlterRequest::try_from(alter)?;
379 Ok((region_id, request))
380}
381
382fn make_region_alter(alter: AlterRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
383 let (region_id, request) = parse_region_alter(alter)?;
384 Ok(vec![(region_id, RegionRequest::Alter(request))])
385}
386
387fn make_region_alters(alters: AlterRequests) -> Result<Vec<(RegionId, RegionRequest)>> {
388 let mut requests = Vec::with_capacity(alters.requests.len());
389 for alter in alters.requests {
390 requests.extend(make_region_alter(alter)?);
391 }
392 Ok(requests)
393}
394
395fn make_region_flush(flush: FlushRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
396 let region_id = flush.region_id.into();
397 Ok(vec![(
398 region_id,
399 RegionRequest::Flush(RegionFlushRequest::default()),
400 )])
401}
402
403fn make_region_compact(compact: CompactRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
404 let region_id = compact.region_id.into();
405 let options = compact
406 .options
407 .unwrap_or(compact_request::Options::Regular(Default::default()));
408 let parallelism = if compact.parallelism == 0 {
410 None
411 } else {
412 Some(compact.parallelism)
413 };
414 let time_range = compact
415 .time_range
416 .map(|range| {
417 let time_unit = v1::TimeUnit::try_from(range.time_unit).map_err(|_| {
418 InvalidRegionRequestSnafu {
419 region_id,
420 err: format!("invalid compaction time unit: {}", range.time_unit),
421 }
422 .build()
423 })?;
424 let time_unit = from_pb_time_unit(time_unit);
425 let start = Timestamp::new(range.start, time_unit);
426 let end = Timestamp::new(range.end, time_unit);
427 ensure!(
428 start < end,
429 InvalidRegionRequestSnafu {
430 region_id,
431 err: format!(
432 "compaction start time must be earlier than end time: {} >= {}",
433 range.start, range.end
434 ),
435 }
436 );
437 TimestampRange::new(start, end).with_context(|| InvalidRegionRequestSnafu {
438 region_id,
439 err: "invalid compaction time range".to_string(),
440 })
441 })
442 .transpose()?;
443 Ok(vec![(
444 region_id,
445 RegionRequest::Compact(RegionCompactRequest {
446 options,
447 parallelism,
448 time_range,
449 }),
450 )])
451}
452
453fn make_region_cancel_compaction(
454 cancel: CancelCompactionRequest,
455) -> Result<Vec<(RegionId, RegionRequest)>> {
456 let region_id = cancel.region_id.into();
457 ensure!(
458 !cancel.task_id.trim().is_empty(),
459 InvalidRegionRequestSnafu {
460 region_id,
461 err: "compaction task_id must not be empty".to_string(),
462 }
463 );
464 Ok(vec![(
465 region_id,
466 RegionRequest::CancelCompaction(RegionCancelCompactionRequest {
467 task_id: cancel.task_id,
468 }),
469 )])
470}
471
472fn make_region_build_index(index: BuildIndexRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
473 let region_id = index.region_id.into();
474 Ok(vec![(
475 region_id,
476 RegionRequest::BuildIndex(RegionBuildIndexRequest {
477 options: index.options,
478 }),
479 )])
480}
481
482fn make_region_truncate(truncate: TruncateRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
483 let region_id = truncate.region_id.into();
484 match truncate.kind {
485 None => InvalidRawRegionRequestSnafu {
486 err: "missing kind in TruncateRequest".to_string(),
487 }
488 .fail(),
489 Some(truncate_request::Kind::All(_)) => Ok(vec![(
490 region_id,
491 RegionRequest::Truncate(RegionTruncateRequest::All),
492 )]),
493 Some(truncate_request::Kind::TimeRanges(time_ranges)) => {
494 let time_ranges = from_pb_time_ranges(time_ranges).context(ConvertTimeRangesSnafu)?;
495
496 Ok(vec![(
497 region_id,
498 RegionRequest::Truncate(RegionTruncateRequest::ByTimeRanges { time_ranges }),
499 )])
500 }
501 Some(truncate_request::Kind::Unflushed(_)) => Ok(vec![(
502 region_id,
503 RegionRequest::Truncate(RegionTruncateRequest::Unflushed),
504 )]),
505 }
506}
507
508fn make_region_bulk_inserts(request: BulkInsertRequest) -> Result<Vec<(RegionId, RegionRequest)>> {
510 let region_id = request.region_id.into();
511 let skip_wal = request.skip_wal;
512 let partition_expr_version = request.partition_expr_version.map(|v| v.value);
513 let aligned_schema_version = request.aligned_schema_version.map(|v| v.schema_version);
514 let Some(Body::ArrowIpc(request)) = request.body else {
515 return Ok(vec![]);
516 };
517
518 let decoder_timer = metrics::CONVERT_REGION_BULK_REQUEST
519 .with_label_values(&["decode"])
520 .start_timer();
521 let mut decoder =
522 FlightDecoder::try_from_schema_bytes(&request.schema).context(FlightCodecSnafu)?;
523 let payload = decoder
524 .try_decode_record_batch(&request.data_header, &request.payload)
525 .context(FlightCodecSnafu)?;
526 decoder_timer.observe_duration();
527 Ok(vec![(
528 region_id,
529 RegionRequest::BulkInserts(RegionBulkInsertsRequest {
530 region_id,
531 payload,
532 raw_data: request,
533 skip_wal,
534 partition_expr_version,
535 aligned_schema_version,
536 }),
537 )])
538}
539
540fn make_region_apply_staging_manifest(
541 api::v1::region::ApplyStagingManifestRequest {
542 region_id,
543 partition_expr,
544 central_region_id,
545 manifest_path,
546 }: api::v1::region::ApplyStagingManifestRequest,
547) -> Result<Vec<(RegionId, RegionRequest)>> {
548 let region_id = region_id.into();
549 Ok(vec![(
550 region_id,
551 RegionRequest::ApplyStagingManifest(ApplyStagingManifestRequest {
552 partition_expr,
553 central_region_id: central_region_id.into(),
554 manifest_path,
555 }),
556 )])
557}
558
559#[derive(Debug)]
561pub struct RegionPutRequest {
562 pub rows: Rows,
564 pub hint: Option<WriteHint>,
566 pub skip_wal: bool,
569 pub partition_expr_version: Option<u64>,
571}
572
573#[derive(Debug)]
575pub struct RegionDeleteRequest {
576 pub rows: Rows,
580 pub hint: Option<WriteHint>,
582 pub partition_expr_version: Option<u64>,
584}
585
586#[derive(Debug, Clone)]
587pub struct RegionCreateRequest {
588 pub engine: String,
590 pub column_metadatas: Vec<ColumnMetadata>,
592 pub primary_key: Vec<ColumnId>,
594 pub options: HashMap<String, String>,
596 pub table_dir: String,
598 pub path_type: PathType,
600 pub partition_expr_json: Option<String>,
603 pub requirements: RegionRequirements,
605}
606
607impl RegionCreateRequest {
608 pub fn validate(&self) -> Result<()> {
610 ensure!(
612 self.column_metadatas
613 .iter()
614 .any(|x| x.semantic_type == SemanticType::Timestamp),
615 InvalidRegionRequestSnafu {
616 region_id: RegionId::new(0, 0),
617 err: "missing timestamp column in create region request".to_string(),
618 }
619 );
620
621 let mut column_id_to_indices = HashMap::with_capacity(self.column_metadatas.len());
623 for (i, c) in self.column_metadatas.iter().enumerate() {
624 if let Some(previous) = column_id_to_indices.insert(c.column_id, i) {
625 return InvalidRegionRequestSnafu {
626 region_id: RegionId::new(0, 0),
627 err: format!(
628 "duplicate column id {} (at position {} and {}) in create region request",
629 c.column_id, previous, i
630 ),
631 }
632 .fail();
633 }
634 }
635
636 for column_id in &self.primary_key {
638 ensure!(
639 column_id_to_indices.contains_key(column_id),
640 InvalidRegionRequestSnafu {
641 region_id: RegionId::new(0, 0),
642 err: format!(
643 "missing primary key column {} in create region request",
644 column_id
645 ),
646 }
647 );
648 }
649
650 Ok(())
651 }
652
653 pub fn is_physical_table(&self) -> bool {
655 self.options.contains_key(PHYSICAL_TABLE_METADATA_KEY)
656 }
657}
658
659#[derive(Debug, Clone)]
660pub struct RegionDropRequest {
661 pub fast_path: bool,
664
665 pub force: bool,
668
669 pub partial_drop: bool,
672}
673
674#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
676#[serde(default)]
677pub struct RegionRequirements {
678 pub object_storage: bool,
680}
681
682impl RegionRequirements {
683 pub fn empty() -> Self {
685 Self::default()
686 }
687
688 pub fn object_storage() -> Self {
690 Self {
691 object_storage: true,
692 }
693 }
694}
695
696impl From<api::v1::region::RegionRequirements> for RegionRequirements {
697 fn from(value: api::v1::region::RegionRequirements) -> Self {
698 Self {
699 object_storage: value.object_storage,
700 }
701 }
702}
703
704impl From<RegionRequirements> for api::v1::region::RegionRequirements {
705 fn from(value: RegionRequirements) -> Self {
706 Self {
707 object_storage: value.object_storage,
708 }
709 }
710}
711
712#[derive(Debug, Clone)]
714pub struct RegionOpenRequest {
715 pub engine: String,
717 pub table_dir: String,
719 pub path_type: PathType,
721 pub options: HashMap<String, String>,
723 pub skip_wal_replay: bool,
725 pub checkpoint: Option<ReplayCheckpoint>,
727 pub requirements: RegionRequirements,
729}
730
731#[derive(Debug, Clone, Copy, PartialEq, Eq)]
732pub struct ReplayCheckpoint {
733 pub entry_id: u64,
734 pub metadata_entry_id: Option<u64>,
735}
736
737impl RegionOpenRequest {
738 pub fn is_physical_table(&self) -> bool {
740 self.options.contains_key(PHYSICAL_TABLE_METADATA_KEY)
741 }
742}
743
744#[derive(Debug, Clone)]
746pub struct RegionCleanUpRequest {
747 pub engine: String,
749 pub table_dir: String,
751 pub path_type: PathType,
753 pub options: HashMap<String, String>,
755}
756
757impl RegionCleanUpRequest {
758 pub fn is_physical_table(&self) -> bool {
760 self.options.contains_key(PHYSICAL_TABLE_METADATA_KEY)
761 }
762}
763
764#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
766pub struct RegionCloseRequest {
767 pub flush_on_close: bool,
769}
770
771#[derive(Debug, PartialEq, Eq, Clone)]
773pub struct RegionAlterRequest {
774 pub kind: AlterKind,
776}
777
778impl RegionAlterRequest {
779 pub fn validate(&self, metadata: &RegionMetadata) -> Result<()> {
781 self.kind.validate(metadata)?;
782
783 Ok(())
784 }
785
786 pub fn need_alter(&self, metadata: &RegionMetadata) -> bool {
790 debug_assert!(self.validate(metadata).is_ok());
791 self.kind.need_alter(metadata)
792 }
793}
794
795impl TryFrom<AlterRequest> for RegionAlterRequest {
796 type Error = MetadataError;
797
798 fn try_from(value: AlterRequest) -> Result<Self> {
799 let kind = value.kind.context(InvalidRawRegionRequestSnafu {
800 err: "missing kind in AlterRequest",
801 })?;
802
803 let kind = AlterKind::try_from(kind)?;
804 Ok(RegionAlterRequest { kind })
805 }
806}
807
808#[derive(Debug, PartialEq, Eq, Clone, AsRefStr)]
810pub enum AlterKind {
811 AddColumns {
813 columns: Vec<AddColumn>,
815 },
816 DropColumns {
818 names: Vec<String>,
820 },
821 ModifyColumnTypes {
826 columns: Vec<ModifyColumnType>,
828 },
829 SetJsonSettings {
831 column_name: String,
833 settings: JsonSettings,
835 },
836 SetRegionOptions { options: Vec<SetRegionOption> },
838 UnsetRegionOptions { keys: Vec<UnsetRegionOption> },
840 SetIndexes { options: Vec<SetIndexOption> },
842 UnsetIndexes { options: Vec<UnsetIndexOption> },
844 DropDefaults {
846 names: Vec<String>,
848 },
849 SetDefaults {
851 columns: Vec<SetDefault>,
853 },
854 SyncColumns {
856 column_metadatas: Vec<ColumnMetadata>,
857 },
858}
859#[derive(Debug, PartialEq, Eq, Clone)]
860pub struct SetDefault {
861 pub name: String,
862 pub default_constraint: Vec<u8>,
863}
864
865#[derive(Debug, PartialEq, Eq, Clone)]
866pub enum SetIndexOption {
867 Fulltext {
868 column_name: String,
869 options: FulltextOptions,
870 },
871 Inverted {
872 column_name: String,
873 },
874 Skipping {
875 column_name: String,
876 options: SkippingIndexOptions,
877 },
878}
879
880impl SetIndexOption {
881 pub fn column_name(&self) -> &String {
883 match self {
884 SetIndexOption::Fulltext { column_name, .. } => column_name,
885 SetIndexOption::Inverted { column_name } => column_name,
886 SetIndexOption::Skipping { column_name, .. } => column_name,
887 }
888 }
889
890 pub fn is_fulltext(&self) -> bool {
892 match self {
893 SetIndexOption::Fulltext { .. } => true,
894 SetIndexOption::Inverted { .. } => false,
895 SetIndexOption::Skipping { .. } => false,
896 }
897 }
898}
899
900impl TryFrom<v1::SetIndex> for SetIndexOption {
901 type Error = MetadataError;
902
903 fn try_from(value: v1::SetIndex) -> Result<Self> {
904 let option = value.options.context(InvalidRawRegionRequestSnafu {
905 err: "missing options in SetIndex",
906 })?;
907
908 let opt = match option {
909 v1::set_index::Options::Fulltext(x) => SetIndexOption::Fulltext {
910 column_name: x.column_name.clone(),
911 options: FulltextOptions::new(
912 x.enable,
913 as_fulltext_option_analyzer(
914 Analyzer::try_from(x.analyzer).context(DecodeProtoSnafu)?,
915 ),
916 x.case_sensitive,
917 as_fulltext_option_backend(
918 PbFulltextBackend::try_from(x.backend).context(DecodeProtoSnafu)?,
919 ),
920 x.granularity as u32,
921 x.false_positive_rate,
922 )
923 .context(InvalidIndexOptionSnafu)?,
924 },
925 v1::set_index::Options::Inverted(i) => SetIndexOption::Inverted {
926 column_name: i.column_name,
927 },
928 v1::set_index::Options::Skipping(s) => SetIndexOption::Skipping {
929 column_name: s.column_name,
930 options: SkippingIndexOptions::new(
931 s.granularity as u32,
932 s.false_positive_rate,
933 as_skipping_index_type(
934 PbSkippingIndexType::try_from(s.skipping_index_type)
935 .context(DecodeProtoSnafu)?,
936 ),
937 )
938 .context(InvalidIndexOptionSnafu)?,
939 },
940 };
941
942 Ok(opt)
943 }
944}
945
946#[derive(Debug, PartialEq, Eq, Clone)]
947pub enum UnsetIndexOption {
948 Fulltext { column_name: String },
949 Inverted { column_name: String },
950 Skipping { column_name: String },
951}
952
953impl UnsetIndexOption {
954 pub fn column_name(&self) -> &String {
955 match self {
956 UnsetIndexOption::Fulltext { column_name } => column_name,
957 UnsetIndexOption::Inverted { column_name } => column_name,
958 UnsetIndexOption::Skipping { column_name } => column_name,
959 }
960 }
961
962 pub fn is_fulltext(&self) -> bool {
963 match self {
964 UnsetIndexOption::Fulltext { .. } => true,
965 UnsetIndexOption::Inverted { .. } => false,
966 UnsetIndexOption::Skipping { .. } => false,
967 }
968 }
969}
970
971impl TryFrom<v1::UnsetIndex> for UnsetIndexOption {
972 type Error = MetadataError;
973
974 fn try_from(value: v1::UnsetIndex) -> Result<Self> {
975 let option = value.options.context(InvalidRawRegionRequestSnafu {
976 err: "missing options in UnsetIndex",
977 })?;
978
979 let opt = match option {
980 v1::unset_index::Options::Fulltext(f) => UnsetIndexOption::Fulltext {
981 column_name: f.column_name,
982 },
983 v1::unset_index::Options::Inverted(i) => UnsetIndexOption::Inverted {
984 column_name: i.column_name,
985 },
986 v1::unset_index::Options::Skipping(s) => UnsetIndexOption::Skipping {
987 column_name: s.column_name,
988 },
989 };
990
991 Ok(opt)
992 }
993}
994
995impl AlterKind {
996 pub fn validate(&self, metadata: &RegionMetadata) -> Result<()> {
1000 match self {
1001 AlterKind::AddColumns { columns } => {
1002 for col_to_add in columns {
1003 col_to_add.validate(metadata)?;
1004 }
1005 }
1006 AlterKind::DropColumns { names } => {
1007 for name in names {
1008 Self::validate_column_to_drop(name, metadata)?;
1009 }
1010 }
1011 AlterKind::ModifyColumnTypes { columns } => {
1012 for col_to_change in columns {
1013 col_to_change.validate(metadata)?;
1014 }
1015 }
1016 AlterKind::SetJsonSettings { column_name, .. } => {
1017 Self::validate_set_json_settings(column_name, metadata)?
1018 }
1019 AlterKind::SetRegionOptions { .. } => {}
1020 AlterKind::UnsetRegionOptions { .. } => {}
1021 AlterKind::SetIndexes { options } => {
1022 for option in options {
1023 Self::validate_column_alter_index_option(
1024 option.column_name(),
1025 metadata,
1026 option.is_fulltext(),
1027 )?;
1028 }
1029 }
1030 AlterKind::UnsetIndexes { options } => {
1031 for option in options {
1032 Self::validate_column_alter_index_option(
1033 option.column_name(),
1034 metadata,
1035 option.is_fulltext(),
1036 )?;
1037 }
1038 }
1039 AlterKind::DropDefaults { names } => {
1040 names
1041 .iter()
1042 .try_for_each(|name| Self::validate_column_existence(name, metadata))?;
1043 }
1044 AlterKind::SetDefaults { columns } => {
1045 columns
1046 .iter()
1047 .try_for_each(|col| Self::validate_column_existence(&col.name, metadata))?;
1048 }
1049 AlterKind::SyncColumns { column_metadatas } => {
1050 let new_primary_keys = column_metadatas
1051 .iter()
1052 .filter(|c| c.semantic_type == SemanticType::Tag)
1053 .map(|c| (c.column_schema.name.as_str(), c.column_id))
1054 .collect::<HashMap<_, _>>();
1055
1056 let old_primary_keys = metadata
1057 .column_metadatas
1058 .iter()
1059 .filter(|c| c.semantic_type == SemanticType::Tag)
1060 .map(|c| (c.column_schema.name.as_str(), c.column_id));
1061
1062 for (name, id) in old_primary_keys {
1063 let primary_key =
1064 new_primary_keys
1065 .get(name)
1066 .with_context(|| InvalidRegionRequestSnafu {
1067 region_id: metadata.region_id,
1068 err: format!("column {} is not a primary key", name),
1069 })?;
1070
1071 ensure!(
1072 *primary_key == id,
1073 InvalidRegionRequestSnafu {
1074 region_id: metadata.region_id,
1075 err: format!(
1076 "column with same name {} has different id, existing: {}, got: {}",
1077 name, id, primary_key
1078 ),
1079 }
1080 );
1081 }
1082
1083 let new_ts_column = column_metadatas
1084 .iter()
1085 .find(|c| c.semantic_type == SemanticType::Timestamp)
1086 .map(|c| (c.column_schema.name.as_str(), c.column_id))
1087 .context(InvalidRegionRequestSnafu {
1088 region_id: metadata.region_id,
1089 err: "timestamp column not found",
1090 })?;
1091
1092 let old_ts_column = metadata
1094 .column_metadatas
1095 .iter()
1096 .find(|c| c.semantic_type == SemanticType::Timestamp)
1097 .map(|c| (c.column_schema.name.as_str(), c.column_id))
1098 .unwrap();
1099
1100 ensure!(
1101 new_ts_column == old_ts_column,
1102 InvalidRegionRequestSnafu {
1103 region_id: metadata.region_id,
1104 err: format!(
1105 "timestamp column {} has different id, existing: {}, got: {}",
1106 old_ts_column.0, old_ts_column.1, new_ts_column.1
1107 ),
1108 }
1109 );
1110 }
1111 }
1112 Ok(())
1113 }
1114
1115 pub fn need_alter(&self, metadata: &RegionMetadata) -> bool {
1117 debug_assert!(self.validate(metadata).is_ok());
1118 match self {
1119 AlterKind::AddColumns { columns } => columns
1120 .iter()
1121 .any(|col_to_add| col_to_add.need_alter(metadata)),
1122 AlterKind::DropColumns { names } => names
1123 .iter()
1124 .any(|name| metadata.column_by_name(name).is_some()),
1125 AlterKind::ModifyColumnTypes { columns } => columns
1126 .iter()
1127 .any(|col_to_change| col_to_change.need_alter(metadata)),
1128 AlterKind::SetJsonSettings {
1129 column_name,
1130 settings,
1131 } => metadata.column_by_name(column_name).is_some_and(|col| {
1132 col.column_schema
1133 .extension_type::<Json2ExtensionType>()
1134 .ok()
1135 .flatten()
1136 .is_none_or(|extension| {
1137 !extension.metadata().json_settings().equivalent(settings)
1138 })
1139 }),
1140 AlterKind::SetRegionOptions { .. } => true,
1141 AlterKind::UnsetRegionOptions { .. } => true,
1142 AlterKind::SetIndexes { options, .. } => options
1143 .iter()
1144 .any(|option| metadata.column_by_name(option.column_name()).is_some()),
1145 AlterKind::UnsetIndexes { options } => options
1146 .iter()
1147 .any(|option| metadata.column_by_name(option.column_name()).is_some()),
1148 AlterKind::DropDefaults { names } => names
1149 .iter()
1150 .any(|name| metadata.column_by_name(name).is_some()),
1151
1152 AlterKind::SetDefaults { columns } => columns
1153 .iter()
1154 .any(|x| metadata.column_by_name(&x.name).is_some()),
1155 AlterKind::SyncColumns { column_metadatas } => {
1156 metadata.column_metadatas != *column_metadatas
1157 }
1158 }
1159 }
1160
1161 fn validate_column_to_drop(name: &str, metadata: &RegionMetadata) -> Result<()> {
1163 let Some(column) = metadata.column_by_name(name) else {
1164 return Ok(());
1165 };
1166 ensure!(
1167 column.semantic_type == SemanticType::Field,
1168 InvalidRegionRequestSnafu {
1169 region_id: metadata.region_id,
1170 err: format!("column {} is not a field and could not be dropped", name),
1171 }
1172 );
1173 Ok(())
1174 }
1175
1176 fn validate_column_alter_index_option(
1178 column_name: &String,
1179 metadata: &RegionMetadata,
1180 is_fulltext: bool,
1181 ) -> Result<()> {
1182 let column = metadata
1183 .column_by_name(column_name)
1184 .context(InvalidRegionRequestSnafu {
1185 region_id: metadata.region_id,
1186 err: format!("column {} not found", column_name),
1187 })?;
1188
1189 if is_fulltext {
1190 ensure!(
1191 column.column_schema.data_type.is_string(),
1192 InvalidRegionRequestSnafu {
1193 region_id: metadata.region_id,
1194 err: format!(
1195 "cannot change alter index options for non-string column {}",
1196 column_name
1197 ),
1198 }
1199 );
1200 }
1201
1202 Ok(())
1203 }
1204
1205 fn validate_column_existence(column_name: &String, metadata: &RegionMetadata) -> Result<()> {
1207 metadata
1208 .column_by_name(column_name)
1209 .context(InvalidRegionRequestSnafu {
1210 region_id: metadata.region_id,
1211 err: format!("column {} not found", column_name),
1212 })?;
1213
1214 Ok(())
1215 }
1216
1217 fn validate_set_json_settings(col_name: &String, metadata: &RegionMetadata) -> Result<()> {
1218 let region_id = metadata.region_id;
1219
1220 let col = metadata
1221 .column_by_name(col_name)
1222 .with_context(|| InvalidRegionRequestSnafu {
1223 region_id,
1224 err: format!("column {} not found", col_name),
1225 })?;
1226
1227 ensure!(
1228 col.semantic_type == SemanticType::Field,
1229 InvalidRegionRequestSnafu {
1230 region_id,
1231 err: format!("column {} is not a field column", col_name),
1232 }
1233 );
1234 ensure!(
1235 col.column_schema.data_type.is_json2(),
1236 InvalidRegionRequestSnafu {
1237 region_id,
1238 err: format!("column {} is not a JSON2 column", col_name),
1239 }
1240 );
1241
1242 Ok(())
1243 }
1244}
1245
1246impl TryFrom<alter_request::Kind> for AlterKind {
1247 type Error = MetadataError;
1248
1249 fn try_from(kind: alter_request::Kind) -> Result<Self> {
1250 let alter_kind = match kind {
1251 alter_request::Kind::AddColumns(x) => {
1252 let columns = x
1253 .add_columns
1254 .into_iter()
1255 .map(|x| x.try_into())
1256 .collect::<Result<Vec<_>>>()?;
1257 AlterKind::AddColumns { columns }
1258 }
1259 alter_request::Kind::ModifyColumnTypes(x) => {
1260 let columns = x
1261 .modify_column_types
1262 .into_iter()
1263 .map(|x| x.into())
1264 .collect::<Vec<_>>();
1265 AlterKind::ModifyColumnTypes { columns }
1266 }
1267 alter_request::Kind::SetJsonSettings(x) => {
1268 let settings = x.settings.context(InvalidRawRegionRequestSnafu {
1269 err: "missing settings in SetJsonSettings",
1270 })?;
1271 AlterKind::SetJsonSettings {
1272 column_name: x.column_name,
1273 settings: json_settings_from_proto(settings)?,
1274 }
1275 }
1276 alter_request::Kind::DropColumns(x) => {
1277 let names = x.drop_columns.into_iter().map(|x| x.name).collect();
1278 AlterKind::DropColumns { names }
1279 }
1280 alter_request::Kind::SetTableOptions(options) => AlterKind::SetRegionOptions {
1281 options: options
1282 .table_options
1283 .iter()
1284 .map(TryFrom::try_from)
1285 .collect::<Result<Vec<_>>>()?,
1286 },
1287 alter_request::Kind::UnsetTableOptions(options) => AlterKind::UnsetRegionOptions {
1288 keys: options
1289 .keys
1290 .iter()
1291 .map(|key| UnsetRegionOption::try_from(key.as_str()))
1292 .collect::<Result<Vec<_>>>()?,
1293 },
1294 alter_request::Kind::SetIndex(o) => AlterKind::SetIndexes {
1295 options: vec![SetIndexOption::try_from(o)?],
1296 },
1297 alter_request::Kind::UnsetIndex(o) => AlterKind::UnsetIndexes {
1298 options: vec![UnsetIndexOption::try_from(o)?],
1299 },
1300 alter_request::Kind::SetIndexes(o) => AlterKind::SetIndexes {
1301 options: o
1302 .set_indexes
1303 .into_iter()
1304 .map(SetIndexOption::try_from)
1305 .collect::<Result<Vec<_>>>()?,
1306 },
1307 alter_request::Kind::UnsetIndexes(o) => AlterKind::UnsetIndexes {
1308 options: o
1309 .unset_indexes
1310 .into_iter()
1311 .map(UnsetIndexOption::try_from)
1312 .collect::<Result<Vec<_>>>()?,
1313 },
1314 alter_request::Kind::DropDefaults(x) => AlterKind::DropDefaults {
1315 names: x.drop_defaults.into_iter().map(|x| x.column_name).collect(),
1316 },
1317 alter_request::Kind::SetDefaults(x) => AlterKind::SetDefaults {
1318 columns: x
1319 .set_defaults
1320 .into_iter()
1321 .map(|x| {
1322 Ok(SetDefault {
1323 name: x.column_name,
1324 default_constraint: x.default_constraint.clone(),
1325 })
1326 })
1327 .collect::<Result<Vec<_>>>()?,
1328 },
1329 alter_request::Kind::SyncColumns(x) => AlterKind::SyncColumns {
1330 column_metadatas: x
1331 .column_defs
1332 .into_iter()
1333 .map(ColumnMetadata::try_from_column_def)
1334 .collect::<Result<Vec<_>>>()?,
1335 },
1336 };
1337
1338 Ok(alter_kind)
1339 }
1340}
1341
1342#[derive(Debug, PartialEq, Eq, Clone)]
1344pub struct AddColumn {
1345 pub column_metadata: ColumnMetadata,
1347 pub location: Option<AddColumnLocation>,
1350}
1351
1352impl AddColumn {
1353 pub fn validate(&self, metadata: &RegionMetadata) -> Result<()> {
1358 ensure!(
1359 self.column_metadata.column_schema.is_nullable()
1360 || self
1361 .column_metadata
1362 .column_schema
1363 .default_constraint()
1364 .is_some(),
1365 InvalidRegionRequestSnafu {
1366 region_id: metadata.region_id,
1367 err: format!(
1368 "no default value for column {}",
1369 self.column_metadata.column_schema.name
1370 ),
1371 }
1372 );
1373
1374 if let Some(existing_column) =
1375 metadata.column_by_name(&self.column_metadata.column_schema.name)
1376 {
1377 ensure!(
1379 *existing_column == self.column_metadata,
1380 InvalidRegionRequestSnafu {
1381 region_id: metadata.region_id,
1382 err: format!(
1383 "column {} already exists with different metadata, existing: {:?}, got: {:?}",
1384 self.column_metadata.column_schema.name,
1385 existing_column,
1386 self.column_metadata,
1387 ),
1388 }
1389 );
1390 ensure!(
1391 self.location.is_none(),
1392 InvalidRegionRequestSnafu {
1393 region_id: metadata.region_id,
1394 err: format!(
1395 "column {} already exists, but location is specified",
1396 self.column_metadata.column_schema.name
1397 ),
1398 }
1399 );
1400 }
1401
1402 if let Some(existing_column) = metadata.column_by_id(self.column_metadata.column_id) {
1403 ensure!(
1405 existing_column.column_schema.name == self.column_metadata.column_schema.name,
1406 InvalidRegionRequestSnafu {
1407 region_id: metadata.region_id,
1408 err: format!(
1409 "column id {} already exists with different name {}",
1410 self.column_metadata.column_id, existing_column.column_schema.name
1411 ),
1412 }
1413 );
1414 }
1415
1416 Ok(())
1417 }
1418
1419 pub fn need_alter(&self, metadata: &RegionMetadata) -> bool {
1421 debug_assert!(self.validate(metadata).is_ok());
1422 metadata
1423 .column_by_name(&self.column_metadata.column_schema.name)
1424 .is_none()
1425 }
1426}
1427
1428impl TryFrom<v1::region::AddColumn> for AddColumn {
1429 type Error = MetadataError;
1430
1431 fn try_from(add_column: v1::region::AddColumn) -> Result<Self> {
1432 let column_def = add_column
1433 .column_def
1434 .context(InvalidRawRegionRequestSnafu {
1435 err: "missing column_def in AddColumn",
1436 })?;
1437
1438 let column_metadata = ColumnMetadata::try_from_column_def(column_def)?;
1439 let location = add_column
1440 .location
1441 .map(AddColumnLocation::try_from)
1442 .transpose()?;
1443
1444 Ok(AddColumn {
1445 column_metadata,
1446 location,
1447 })
1448 }
1449}
1450
1451#[derive(Debug, PartialEq, Eq, Clone)]
1453pub enum AddColumnLocation {
1454 First,
1456 After {
1458 column_name: String,
1460 },
1461}
1462
1463impl TryFrom<v1::AddColumnLocation> for AddColumnLocation {
1464 type Error = MetadataError;
1465
1466 fn try_from(location: v1::AddColumnLocation) -> Result<Self> {
1467 let location_type = LocationType::try_from(location.location_type)
1468 .map_err(|e| InvalidRawRegionRequestSnafu { err: e.to_string() }.build())?;
1469 let add_column_location = match location_type {
1470 LocationType::First => AddColumnLocation::First,
1471 LocationType::After => AddColumnLocation::After {
1472 column_name: location.after_column_name,
1473 },
1474 };
1475
1476 Ok(add_column_location)
1477 }
1478}
1479
1480#[derive(Debug, PartialEq, Eq, Clone)]
1482pub struct ModifyColumnType {
1483 pub column_name: String,
1485 pub target_type: ConcreteDataType,
1487}
1488
1489impl ModifyColumnType {
1490 pub fn validate(&self, metadata: &RegionMetadata) -> Result<()> {
1492 let column_meta = metadata
1493 .column_by_name(&self.column_name)
1494 .with_context(|| InvalidRegionRequestSnafu {
1495 region_id: metadata.region_id,
1496 err: format!("column {} not found", self.column_name),
1497 })?;
1498
1499 match column_meta.semantic_type {
1500 SemanticType::Field => {
1501 ensure!(
1502 column_meta
1503 .column_schema
1504 .data_type
1505 .can_arrow_type_cast_to(&self.target_type),
1506 InvalidRegionRequestSnafu {
1507 region_id: metadata.region_id,
1508 err: format!(
1509 "column '{}' cannot be cast automatically to type '{}'",
1510 self.column_name, self.target_type
1511 ),
1512 }
1513 );
1514 }
1515 SemanticType::Timestamp => {
1520 ensure!(
1521 column_meta.column_schema.data_type == self.target_type
1522 || column_meta
1523 .column_schema
1524 .data_type
1525 .is_timestamp_unit_widening_to(&self.target_type),
1526 InvalidRegionRequestSnafu {
1527 region_id: metadata.region_id,
1528 err: time_index_not_widening_error(
1529 &column_meta.column_schema.name,
1530 &column_meta.column_schema.data_type,
1531 &self.target_type,
1532 ),
1533 }
1534 );
1535 }
1536 SemanticType::Tag => {
1537 return InvalidRegionRequestSnafu {
1538 region_id: metadata.region_id,
1539 err: format!(
1540 "tag column '{}' cannot change type, it is part of the primary key",
1541 self.column_name
1542 ),
1543 }
1544 .fail();
1545 }
1546 }
1547
1548 Ok(())
1549 }
1550
1551 pub fn need_alter(&self, metadata: &RegionMetadata) -> bool {
1555 debug_assert!(self.validate(metadata).is_ok());
1556 metadata
1557 .column_by_name(&self.column_name)
1558 .is_some_and(|column| column.column_schema.data_type != self.target_type)
1559 }
1560}
1561
1562impl From<v1::ModifyColumnType> for ModifyColumnType {
1563 fn from(modify_column_type: v1::ModifyColumnType) -> Self {
1564 let target_type = ColumnDataTypeWrapper::new(
1565 modify_column_type.target_type(),
1566 modify_column_type.target_type_extension,
1567 )
1568 .into();
1569
1570 ModifyColumnType {
1571 column_name: modify_column_type.column_name,
1572 target_type,
1573 }
1574 }
1575}
1576
1577fn json_settings_from_proto(settings: v1::JsonSettings) -> Result<JsonSettings> {
1578 let type_hints = settings
1579 .type_hints
1580 .into_iter()
1581 .map(|hint| {
1582 let wrapper = ColumnDataTypeWrapper::try_new(hint.data_type, hint.datatype_extension)
1583 .map_err(|err| {
1584 InvalidRawRegionRequestSnafu {
1585 err: err.to_string(),
1586 }
1587 .build()
1588 })?;
1589 let data_type = ConcreteDataType::from(wrapper);
1590
1591 Ok(JsonTypeHint {
1592 path: hint.path,
1593 data_type,
1594 inverted_index: hint.inverted_index,
1595 fulltext_index: hint
1596 .fulltext_index
1597 .map(api::v1::column_def::json_fulltext_options_from_proto)
1598 .transpose()
1599 .map_err(|err| {
1600 InvalidRawRegionRequestSnafu {
1601 err: err.to_string(),
1602 }
1603 .build()
1604 })?,
1605 skipping_index: hint
1606 .skipping_index
1607 .map(api::v1::column_def::json_skipping_options_from_proto)
1608 .transpose()
1609 .map_err(|err| {
1610 InvalidRawRegionRequestSnafu {
1611 err: err.to_string(),
1612 }
1613 .build()
1614 })?,
1615 })
1616 })
1617 .collect::<Result<Vec<_>>>()?;
1618
1619 JsonSettings::try_new(type_hints, settings.max_auto_expanded_paths).map_err(|err| {
1620 InvalidRawRegionRequestSnafu {
1621 err: err.to_string(),
1622 }
1623 .build()
1624 })
1625}
1626
1627#[derive(Debug, Eq, PartialEq, Clone)]
1632pub enum SetRegionOption {
1633 WriteBufferSize(Option<ReadableSize>),
1634 Ttl(Option<TimeToLive>),
1635 Twsc(String, String),
1637 Format(String),
1639 AppendMode(bool),
1641 AutoFlushInterval(Option<Duration>),
1643 MaxRowGroupRowCount(Option<usize>),
1645 PreserveRowSequence(bool),
1646 SkipWal(bool),
1648 CompactionOutputParallelism(Option<usize>),
1650}
1651
1652#[derive(Serialize, Deserialize)]
1653enum SetRegionOptionSerde {
1654 WriteBufferSize(Option<ReadableSize>),
1655 Ttl(Option<TimeToLive>),
1656 Twsc(String, String),
1657 Format(String),
1658 AppendMode(bool),
1659 AutoFlushInterval(Option<Duration>),
1660 MaxRowGroupRowCount(Option<usize>),
1661 PreserveRowSequence(bool),
1662 SkipWal(bool),
1663 CompactionOutputParallelism(Option<usize>),
1664}
1665
1666impl Serialize for SetRegionOption {
1667 fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
1668 where
1669 S: Serializer,
1670 {
1671 if matches!(self, Self::SkipWal(true)) {
1673 return serializer.serialize_unit_variant("SetRegionOption", 8, "SkipWal");
1674 }
1675
1676 let option = match self {
1677 Self::WriteBufferSize(value) => SetRegionOptionSerde::WriteBufferSize(*value),
1678 Self::Ttl(value) => SetRegionOptionSerde::Ttl(*value),
1679 Self::Twsc(key, value) => SetRegionOptionSerde::Twsc(key.clone(), value.clone()),
1680 Self::Format(value) => SetRegionOptionSerde::Format(value.clone()),
1681 Self::AppendMode(value) => SetRegionOptionSerde::AppendMode(*value),
1682 Self::AutoFlushInterval(value) => SetRegionOptionSerde::AutoFlushInterval(*value),
1683 Self::MaxRowGroupRowCount(value) => SetRegionOptionSerde::MaxRowGroupRowCount(*value),
1684 Self::PreserveRowSequence(value) => SetRegionOptionSerde::PreserveRowSequence(*value),
1685 Self::SkipWal(value) => SetRegionOptionSerde::SkipWal(*value),
1686 Self::CompactionOutputParallelism(value) => {
1687 SetRegionOptionSerde::CompactionOutputParallelism(*value)
1688 }
1689 };
1690 option.serialize(serializer)
1691 }
1692}
1693
1694#[derive(Deserialize)]
1695enum LegacySetRegionOption {
1696 SkipWal,
1697}
1698
1699#[derive(Deserialize)]
1700#[serde(untagged)]
1701enum BackwardCompatibleSetRegionOption {
1702 Current(SetRegionOptionSerde),
1703 Legacy(LegacySetRegionOption),
1704}
1705
1706impl From<SetRegionOptionSerde> for SetRegionOption {
1707 fn from(option: SetRegionOptionSerde) -> Self {
1708 match option {
1709 SetRegionOptionSerde::WriteBufferSize(value) => Self::WriteBufferSize(value),
1710 SetRegionOptionSerde::Ttl(value) => Self::Ttl(value),
1711 SetRegionOptionSerde::Twsc(key, value) => Self::Twsc(key, value),
1712 SetRegionOptionSerde::Format(value) => Self::Format(value),
1713 SetRegionOptionSerde::AppendMode(value) => Self::AppendMode(value),
1714 SetRegionOptionSerde::AutoFlushInterval(value) => Self::AutoFlushInterval(value),
1715 SetRegionOptionSerde::MaxRowGroupRowCount(value) => Self::MaxRowGroupRowCount(value),
1716 SetRegionOptionSerde::PreserveRowSequence(value) => Self::PreserveRowSequence(value),
1717 SetRegionOptionSerde::SkipWal(value) => Self::SkipWal(value),
1718 SetRegionOptionSerde::CompactionOutputParallelism(value) => {
1719 Self::CompactionOutputParallelism(value)
1720 }
1721 }
1722 }
1723}
1724
1725impl<'de> Deserialize<'de> for SetRegionOption {
1726 fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
1727 where
1728 D: Deserializer<'de>,
1729 {
1730 Ok(
1731 match BackwardCompatibleSetRegionOption::deserialize(deserializer)? {
1732 BackwardCompatibleSetRegionOption::Current(option) => option.into(),
1733 BackwardCompatibleSetRegionOption::Legacy(LegacySetRegionOption::SkipWal) => {
1734 Self::SkipWal(true)
1735 }
1736 },
1737 )
1738 }
1739}
1740
1741impl TryFrom<&PbOption> for SetRegionOption {
1742 type Error = MetadataError;
1743
1744 fn try_from(value: &PbOption) -> std::result::Result<Self, Self::Error> {
1745 let PbOption { key, value } = value;
1746 match key.as_str() {
1747 WRITE_BUFFER_SIZE_KEY => {
1748 let size = value
1749 .parse::<ReadableSize>()
1750 .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?;
1751 Ok(Self::WriteBufferSize(Some(size)))
1752 }
1753 TTL_KEY => {
1754 let ttl = TimeToLive::from_humantime_or_str(value)
1755 .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?;
1756
1757 Ok(Self::Ttl(Some(ttl)))
1758 }
1759 TWCS_TRIGGER_FILE_NUM
1760 | TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM
1761 | TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER
1762 | TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM
1763 | TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER
1764 | TWCS_MAX_OUTPUT_FILE_SIZE
1765 | TWCS_TIME_WINDOW => Ok(Self::Twsc(key.clone(), value.clone())),
1766 SST_FORMAT_KEY => Ok(Self::Format(value.clone())),
1767 APPEND_MODE_KEY => {
1768 let append_mode = value
1769 .parse::<bool>()
1770 .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?;
1771 Ok(Self::AppendMode(append_mode))
1772 }
1773 AUTO_FLUSH_INTERVAL_KEY => {
1774 if value.is_empty() {
1775 return Ok(Self::AutoFlushInterval(None));
1779 }
1780 let interval = humantime::parse_duration(value)
1781 .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?;
1782 if interval <= Duration::ZERO {
1783 return InvalidSetRegionOptionRequestSnafu { key, value }.fail();
1784 }
1785 Ok(Self::AutoFlushInterval(Some(interval)))
1786 }
1787 COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM => {
1788 if value.is_empty() {
1789 return Ok(Self::CompactionOutputParallelism(None));
1790 }
1791 let parallelism = value
1792 .parse::<usize>()
1793 .ok()
1794 .filter(|parallelism| *parallelism > 0)
1795 .ok_or_else(|| InvalidSetRegionOptionRequestSnafu { key, value }.build())?;
1796 Ok(Self::CompactionOutputParallelism(Some(parallelism)))
1797 }
1798 MAX_ROW_GROUP_ROW_COUNT => {
1799 if value.is_empty() {
1800 return Ok(Self::MaxRowGroupRowCount(None));
1801 }
1802 let row_count = value
1803 .parse::<usize>()
1804 .ok()
1805 .filter(|row_count| {
1806 *row_count > 0 && *row_count <= MAX_ROW_GROUP_ROW_COUNT_LIMIT
1807 })
1808 .ok_or_else(|| InvalidSetRegionOptionRequestSnafu { key, value }.build())?;
1809 Ok(Self::MaxRowGroupRowCount(Some(row_count)))
1810 }
1811 PRESERVE_ROW_SEQUENCE => {
1812 let preserve = value
1813 .parse::<bool>()
1814 .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?;
1815 Ok(Self::PreserveRowSequence(preserve))
1816 }
1817 SKIP_WAL_KEY => {
1818 let skip_wal = value
1819 .parse::<bool>()
1820 .map_err(|_| InvalidSetRegionOptionRequestSnafu { key, value }.build())?;
1821 Ok(Self::SkipWal(skip_wal))
1822 }
1823 _ => InvalidSetRegionOptionRequestSnafu { key, value }.fail(),
1824 }
1825 }
1826}
1827
1828impl From<&UnsetRegionOption> for SetRegionOption {
1829 fn from(unset_option: &UnsetRegionOption) -> Self {
1830 match unset_option {
1831 UnsetRegionOption::TwcsTriggerFileNum => {
1832 SetRegionOption::Twsc(unset_option.to_string(), String::new())
1833 }
1834 UnsetRegionOption::TwcsActiveWindowTriggerFileNum => {
1835 SetRegionOption::Twsc(unset_option.to_string(), String::new())
1836 }
1837 UnsetRegionOption::TwcsActiveWindowL1MergeTrigger => {
1838 SetRegionOption::Twsc(unset_option.to_string(), String::new())
1839 }
1840 UnsetRegionOption::TwcsInactiveWindowTriggerFileNum => {
1841 SetRegionOption::Twsc(unset_option.to_string(), String::new())
1842 }
1843 UnsetRegionOption::TwcsInactiveWindowL1MergeTrigger => {
1844 SetRegionOption::Twsc(unset_option.to_string(), String::new())
1845 }
1846 UnsetRegionOption::TwcsMaxOutputFileSize => {
1847 SetRegionOption::Twsc(unset_option.to_string(), String::new())
1848 }
1849 UnsetRegionOption::TwcsTimeWindow => {
1850 SetRegionOption::Twsc(unset_option.to_string(), String::new())
1851 }
1852 UnsetRegionOption::Ttl => SetRegionOption::Ttl(Default::default()),
1853 UnsetRegionOption::MaxRowGroupRowCount => SetRegionOption::MaxRowGroupRowCount(None),
1854 UnsetRegionOption::WriteBufferSize => SetRegionOption::WriteBufferSize(None),
1855 UnsetRegionOption::PreserveRowSequence => SetRegionOption::PreserveRowSequence(false),
1856 UnsetRegionOption::CompactionOutputParallelism => {
1857 SetRegionOption::CompactionOutputParallelism(None)
1858 }
1859 }
1860 }
1861}
1862
1863impl TryFrom<&str> for UnsetRegionOption {
1864 type Error = MetadataError;
1865
1866 fn try_from(key: &str) -> Result<Self> {
1867 match key.to_ascii_lowercase().as_str() {
1868 TTL_KEY => Ok(Self::Ttl),
1869 WRITE_BUFFER_SIZE_KEY => Ok(Self::WriteBufferSize),
1870 TWCS_TRIGGER_FILE_NUM => Ok(Self::TwcsTriggerFileNum),
1871 TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM => Ok(Self::TwcsActiveWindowTriggerFileNum),
1872 TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER => Ok(Self::TwcsActiveWindowL1MergeTrigger),
1873 TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM => Ok(Self::TwcsInactiveWindowTriggerFileNum),
1874 TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER => Ok(Self::TwcsInactiveWindowL1MergeTrigger),
1875 TWCS_MAX_OUTPUT_FILE_SIZE => Ok(Self::TwcsMaxOutputFileSize),
1876 TWCS_TIME_WINDOW => Ok(Self::TwcsTimeWindow),
1877 MAX_ROW_GROUP_ROW_COUNT => Ok(Self::MaxRowGroupRowCount),
1878 PRESERVE_ROW_SEQUENCE => Ok(Self::PreserveRowSequence),
1879 COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM => Ok(Self::CompactionOutputParallelism),
1880 _ => InvalidUnsetRegionOptionRequestSnafu { key }.fail(),
1881 }
1882 }
1883}
1884
1885#[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]
1886pub enum UnsetRegionOption {
1887 TwcsTriggerFileNum,
1888 TwcsActiveWindowTriggerFileNum,
1889 TwcsInactiveWindowTriggerFileNum,
1890 TwcsInactiveWindowL1MergeTrigger,
1891 TwcsMaxOutputFileSize,
1892 TwcsTimeWindow,
1893 Ttl,
1894 MaxRowGroupRowCount,
1895 WriteBufferSize,
1896 PreserveRowSequence,
1897 TwcsActiveWindowL1MergeTrigger,
1898 CompactionOutputParallelism,
1899}
1900
1901impl UnsetRegionOption {
1902 pub fn as_str(&self) -> &str {
1903 match self {
1904 Self::Ttl => TTL_KEY,
1905 Self::WriteBufferSize => WRITE_BUFFER_SIZE_KEY,
1906 Self::TwcsTriggerFileNum => TWCS_TRIGGER_FILE_NUM,
1907 Self::TwcsActiveWindowTriggerFileNum => TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
1908 Self::TwcsActiveWindowL1MergeTrigger => TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER,
1909 Self::TwcsInactiveWindowTriggerFileNum => TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM,
1910 Self::TwcsInactiveWindowL1MergeTrigger => TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER,
1911 Self::TwcsMaxOutputFileSize => TWCS_MAX_OUTPUT_FILE_SIZE,
1912 Self::TwcsTimeWindow => TWCS_TIME_WINDOW,
1913 Self::MaxRowGroupRowCount => MAX_ROW_GROUP_ROW_COUNT,
1914 Self::PreserveRowSequence => PRESERVE_ROW_SEQUENCE,
1915 Self::CompactionOutputParallelism => COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM,
1916 }
1917 }
1918}
1919
1920impl Display for UnsetRegionOption {
1921 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1922 write!(f, "{}", self.as_str())
1923 }
1924}
1925
1926#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
1927pub enum RegionFlushReason {
1928 RegionMigration,
1930 Repartition,
1932 RemoteWalPrune,
1934 Closing,
1936 Downgrading,
1938}
1939
1940#[derive(Debug, Clone, Default)]
1941pub struct RegionFlushRequest {
1942 pub row_group_size: Option<usize>,
1943 pub reason: Option<RegionFlushReason>,
1944}
1945
1946#[derive(Debug)]
1948pub struct RegionCancelCompactionRequest {
1949 pub task_id: String,
1950}
1951
1952#[derive(Debug)]
1953pub struct RegionCompactRequest {
1954 pub options: compact_request::Options,
1955 pub parallelism: Option<u32>,
1956 pub time_range: Option<TimestampRange>,
1957}
1958
1959impl Default for RegionCompactRequest {
1960 fn default() -> Self {
1961 Self {
1962 options: compact_request::Options::Regular(Default::default()),
1964 parallelism: None,
1965 time_range: None,
1966 }
1967 }
1968}
1969
1970#[derive(Debug, Clone, Default)]
1971pub struct RegionBuildIndexRequest {
1972 pub options: Option<build_index_request::Options>,
1974}
1975
1976#[derive(Debug)]
1978pub enum RegionTruncateRequest {
1979 All,
1984 Unflushed,
1994 ByTimeRanges {
1997 time_ranges: Vec<(Timestamp, Timestamp)>,
2001 },
2002}
2003
2004#[derive(Debug, Clone, Copy, Default)]
2009pub struct RegionCatchupRequest {
2010 pub set_writable: bool,
2012 pub entry_id: Option<entry::Id>,
2015 pub metadata_entry_id: Option<entry::Id>,
2019 pub checkpoint: Option<ReplayCheckpoint>,
2021}
2022
2023#[derive(Debug, Clone)]
2024pub struct RegionBulkInsertsRequest {
2025 pub skip_wal: bool,
2027 pub region_id: RegionId,
2028 pub payload: DfRecordBatch,
2029 pub raw_data: ArrowIpc,
2030 pub partition_expr_version: Option<u64>,
2031 pub aligned_schema_version: Option<u64>,
2032}
2033
2034impl RegionBulkInsertsRequest {
2035 pub fn estimated_size(&self) -> usize {
2036 self.payload.get_array_memory_size()
2037 }
2038}
2039
2040#[derive(Debug, Clone, PartialEq, Eq)]
2046pub enum StagingPartitionDirective {
2047 UpdatePartitionExpr(String),
2048 RejectAllWrites,
2049}
2050
2051impl StagingPartitionDirective {
2052 pub fn partition_expr(&self) -> Option<&str> {
2054 match self {
2055 Self::UpdatePartitionExpr(expr) => Some(expr),
2056 Self::RejectAllWrites => None,
2057 }
2058 }
2059}
2060
2061#[derive(Debug, Clone)]
2062pub struct EnterStagingRequest {
2063 pub partition_directive: StagingPartitionDirective,
2065}
2066
2067impl EnterStagingRequest {
2068 pub fn with_partition_expr(partition_expr: String) -> Self {
2070 Self {
2071 partition_directive: StagingPartitionDirective::UpdatePartitionExpr(partition_expr),
2072 }
2073 }
2074}
2075
2076#[derive(Debug, Clone)]
2095pub struct ApplyStagingManifestRequest {
2096 pub partition_expr: String,
2098 pub central_region_id: RegionId,
2100 pub manifest_path: String,
2103}
2104
2105impl fmt::Display for RegionRequest {
2106 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
2107 match self {
2108 RegionRequest::Put(_) => write!(f, "Put"),
2109 RegionRequest::Delete(_) => write!(f, "Delete"),
2110 RegionRequest::Create(_) => write!(f, "Create"),
2111 RegionRequest::Drop(_) => write!(f, "Drop"),
2112 RegionRequest::Open(_) => write!(f, "Open"),
2113 RegionRequest::CleanUp(_) => write!(f, "CleanUp"),
2114 RegionRequest::Close(_) => write!(f, "Close"),
2115 RegionRequest::Alter(_) => write!(f, "Alter"),
2116 RegionRequest::Flush(_) => write!(f, "Flush"),
2117 RegionRequest::Compact(_) => write!(f, "Compact"),
2118 RegionRequest::CancelCompaction(_) => write!(f, "CancelCompaction"),
2119 RegionRequest::BuildIndex(_) => write!(f, "BuildIndex"),
2120 RegionRequest::Truncate(_) => write!(f, "Truncate"),
2121 RegionRequest::Catchup(_) => write!(f, "Catchup"),
2122 RegionRequest::BulkInserts(_) => write!(f, "BulkInserts"),
2123 RegionRequest::EnterStaging(_) => write!(f, "EnterStaging"),
2124 RegionRequest::ApplyStagingManifest(_) => write!(f, "ApplyStagingManifest"),
2125 }
2126 }
2127}
2128
2129#[cfg(test)]
2130mod tests {
2131
2132 use api::v1::region::RegionColumnDef;
2133 use api::v1::{ColumnDataType, ColumnDef};
2134 use common_time::range::TimestampRange;
2135 use datatypes::extension::json::{Json2ExtensionType, JsonMetadata};
2136 use datatypes::json::JsonSettings;
2137 use datatypes::prelude::ConcreteDataType;
2138 use datatypes::schema::{ColumnSchema, FulltextAnalyzer, FulltextBackend};
2139 use datatypes::types::JsonType;
2140
2141 use super::*;
2142 use crate::metadata::RegionMetadataBuilder;
2143
2144 #[test]
2145 fn test_cancel_compaction_request_conversion() {
2146 let region_id = RegionId::new(7, 3);
2147 let requests = RegionRequest::try_from_request_body(
2148 region_request::Body::CancelCompaction(CancelCompactionRequest {
2149 region_id: region_id.as_u64(),
2150 task_id: "task-1".to_string(),
2151 }),
2152 )
2153 .unwrap();
2154 assert_eq!(requests.len(), 1);
2155 assert_eq!(requests[0].0, region_id);
2156 let RegionRequest::CancelCompaction(request) = &requests[0].1 else {
2157 panic!("expected cancellation")
2158 };
2159 assert_eq!(request.task_id, "task-1");
2160 for task_id in ["", " ", "\t\n"] {
2161 assert!(
2162 RegionRequest::try_from_request_body(region_request::Body::CancelCompaction(
2163 CancelCompactionRequest {
2164 region_id: region_id.as_u64(),
2165 task_id: task_id.to_string()
2166 }
2167 ))
2168 .is_err()
2169 );
2170 }
2171 }
2172
2173 #[test]
2174 fn test_build_index_options_round_trip() {
2175 use prost::Message;
2176
2177 for options in [
2178 None,
2179 Some(build_index_request::Options::SstIndex(Default::default())),
2180 Some(build_index_request::Options::SeriesIndex(Default::default())),
2181 ] {
2182 let request = BuildIndexRequest {
2183 region_id: 42,
2184 options,
2185 };
2186 let decoded = BuildIndexRequest::decode(request.encode_to_vec().as_slice()).unwrap();
2187 let requests = make_region_build_index(decoded).unwrap();
2188 assert_eq!(1, requests.len());
2189 assert_eq!(RegionId::from_u64(42), requests[0].0);
2190 let RegionRequest::BuildIndex(request) = &requests[0].1 else {
2191 panic!("expected build-index request");
2192 };
2193 assert_eq!(options, request.options);
2194 }
2195 let legacy = BuildIndexRequest::decode(&[0x08, 42][..]).unwrap();
2197 assert!(legacy.options.is_none());
2198 }
2199
2200 #[test]
2201 fn test_make_region_puts_preserves_skip_wal() {
2202 let region_id = RegionId::new(42, 3);
2203 let rows = Rows::default();
2204 let requests = make_region_puts(InsertRequests {
2205 requests: [false, true, false]
2206 .into_iter()
2207 .map(|skip_wal| api::v1::region::InsertRequest {
2208 region_id: region_id.as_u64(),
2209 rows: Some(rows.clone()),
2210 partition_expr_version: Some(api::v1::PartitionExprVersion { value: 7 }),
2211 skip_wal,
2212 })
2213 .collect(),
2214 })
2215 .unwrap();
2216
2217 assert_eq!(3, requests.len());
2218 for ((id, request), skip_wal) in requests.into_iter().zip([false, true, false]) {
2219 assert_eq!(region_id, id);
2220 let RegionRequest::Put(request) = request else {
2221 panic!("expected a put request");
2222 };
2223 assert_eq!(rows, request.rows);
2224 assert_eq!(skip_wal, request.skip_wal);
2225 assert_eq!(Some(7), request.partition_expr_version);
2226 assert!(request.hint.is_none());
2227 }
2228 }
2229
2230 #[test]
2231 fn test_make_region_compact_with_time_range() {
2232 let requests = make_region_compact(CompactRequest {
2233 region_id: 42,
2234 time_range: Some(api::v1::region::CompactionTimeRange {
2235 start: 1_000,
2236 end: 2_000,
2237 time_unit: api::v1::TimeUnit::Microsecond as i32,
2238 }),
2239 ..Default::default()
2240 })
2241 .unwrap();
2242
2243 let RegionRequest::Compact(request) = &requests[0].1 else {
2244 unreachable!();
2245 };
2246 assert_eq!(
2247 Some(
2248 TimestampRange::new(
2249 Timestamp::new_microsecond(1_000),
2250 Timestamp::new_microsecond(2_000),
2251 )
2252 .unwrap()
2253 ),
2254 request.time_range
2255 );
2256 }
2257
2258 #[test]
2259 fn test_make_region_truncate_unflushed() {
2260 let region_id = RegionId::new(42, 3);
2261 let requests =
2262 RegionRequest::try_from_request_body(region_request::Body::Truncate(TruncateRequest {
2263 region_id: region_id.as_u64(),
2264 kind: Some(truncate_request::Kind::Unflushed(
2265 api::v1::region::Unflushed {},
2266 )),
2267 }))
2268 .unwrap();
2269
2270 assert_eq!(region_id, requests[0].0);
2271 assert!(matches!(
2272 requests[0].1,
2273 RegionRequest::Truncate(RegionTruncateRequest::Unflushed)
2274 ));
2275 }
2276
2277 #[test]
2278 fn test_make_region_truncate_requires_kind() {
2279 let error =
2280 RegionRequest::try_from_request_body(region_request::Body::Truncate(TruncateRequest {
2281 region_id: RegionId::new(42, 3).as_u64(),
2282 kind: None,
2283 }))
2284 .unwrap_err();
2285
2286 assert!(
2287 error
2288 .to_string()
2289 .contains("missing kind in TruncateRequest")
2290 );
2291 }
2292
2293 #[test]
2294 fn test_from_proto_location() {
2295 let proto_location = v1::AddColumnLocation {
2296 location_type: LocationType::First as i32,
2297 after_column_name: String::default(),
2298 };
2299 let location = AddColumnLocation::try_from(proto_location).unwrap();
2300 assert_eq!(location, AddColumnLocation::First);
2301
2302 let proto_location = v1::AddColumnLocation {
2303 location_type: 10,
2304 after_column_name: String::default(),
2305 };
2306 AddColumnLocation::try_from(proto_location).unwrap_err();
2307
2308 let proto_location = v1::AddColumnLocation {
2309 location_type: LocationType::After as i32,
2310 after_column_name: "a".to_string(),
2311 };
2312 let location = AddColumnLocation::try_from(proto_location).unwrap();
2313 assert_eq!(
2314 location,
2315 AddColumnLocation::After {
2316 column_name: "a".to_string()
2317 }
2318 );
2319 }
2320
2321 #[test]
2322 fn test_from_none_proto_add_column() {
2323 AddColumn::try_from(v1::region::AddColumn {
2324 column_def: None,
2325 location: None,
2326 })
2327 .unwrap_err();
2328 }
2329
2330 #[test]
2331 fn test_compaction_output_parallelism_set_unset_and_serde() {
2332 for (raw, expected) in [("", None), ("1", Some(1)), ("4", Some(4))] {
2333 let option = SetRegionOption::try_from(&PbOption {
2334 key: COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM.to_string(),
2335 value: raw.to_string(),
2336 })
2337 .unwrap();
2338 assert_eq!(
2339 SetRegionOption::CompactionOutputParallelism(expected),
2340 option
2341 );
2342 let json = serde_json::to_string(&option).unwrap();
2343 assert_eq!(
2344 option,
2345 serde_json::from_str::<SetRegionOption>(&json).unwrap()
2346 );
2347 }
2348 for raw in ["0", "-1", "1.5", "invalid", "18446744073709551616"] {
2349 assert!(
2350 SetRegionOption::try_from(&PbOption {
2351 key: COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM.to_string(),
2352 value: raw.to_string(),
2353 })
2354 .is_err()
2355 );
2356 }
2357 let unset =
2358 UnsetRegionOption::try_from(COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM).unwrap();
2359 assert_eq!(COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM, unset.as_str());
2360 assert_eq!(
2361 SetRegionOption::CompactionOutputParallelism(None),
2362 (&unset).into()
2363 );
2364 }
2365
2366 #[test]
2367 fn test_set_region_option_auto_flush_interval_try_from() {
2368 use std::time::Duration;
2369
2370 let pb = PbOption {
2372 key: "auto_flush_interval".to_string(),
2373 value: "5m".to_string(),
2374 };
2375 let opt = SetRegionOption::try_from(&pb).unwrap();
2376 assert_eq!(
2377 opt,
2378 SetRegionOption::AutoFlushInterval(Some(Duration::from_secs(300)))
2379 );
2380
2381 let pb = PbOption {
2383 key: "auto_flush_interval".to_string(),
2384 value: String::new(),
2385 };
2386 let opt = SetRegionOption::try_from(&pb).unwrap();
2387 assert_eq!(opt, SetRegionOption::AutoFlushInterval(None));
2388
2389 let pb = PbOption {
2391 key: "auto_flush_interval".to_string(),
2392 value: "0s".to_string(),
2393 };
2394 assert!(SetRegionOption::try_from(&pb).is_err());
2395
2396 let pb = PbOption {
2398 key: "auto_flush_interval".to_string(),
2399 value: "not_a_duration".to_string(),
2400 };
2401 assert!(SetRegionOption::try_from(&pb).is_err());
2402 }
2403
2404 #[test]
2405 fn test_set_region_option_skip_wal_try_from() {
2406 let pb = PbOption {
2407 key: SKIP_WAL_KEY.to_string(),
2408 value: "true".to_string(),
2409 };
2410 assert_eq!(
2411 SetRegionOption::SkipWal(true),
2412 SetRegionOption::try_from(&pb).unwrap()
2413 );
2414
2415 let pb = PbOption {
2416 key: SKIP_WAL_KEY.to_string(),
2417 value: "false".to_string(),
2418 };
2419 assert_eq!(
2420 SetRegionOption::SkipWal(false),
2421 SetRegionOption::try_from(&pb).unwrap()
2422 );
2423
2424 for value in ["", "invalid"] {
2425 let pb = PbOption {
2426 key: SKIP_WAL_KEY.to_string(),
2427 value: value.to_string(),
2428 };
2429 assert!(SetRegionOption::try_from(&pb).is_err());
2430 }
2431
2432 assert!(UnsetRegionOption::try_from(SKIP_WAL_KEY).is_err());
2433 }
2434
2435 #[test]
2436 fn test_set_region_option_skip_wal_serde_compatibility() {
2437 let legacy = serde_json::from_str::<SetRegionOption>(r#""SkipWal""#).unwrap();
2438 assert_eq!(SetRegionOption::SkipWal(true), legacy);
2439
2440 check_set_region_option_skip_wal_serde_compatibility(false);
2441 check_set_region_option_skip_wal_serde_compatibility(true);
2442 }
2443
2444 fn check_set_region_option_skip_wal_serde_compatibility(skip_wal: bool) {
2445 let option = SetRegionOption::SkipWal(skip_wal);
2446 let serialized = serde_json::to_string(&option).unwrap();
2447 let expected = if skip_wal {
2448 r#""SkipWal""#
2449 } else {
2450 r#"{"SkipWal":false}"#
2451 };
2452 assert_eq!(expected, serialized);
2453 assert_eq!(
2454 option,
2455 serde_json::from_str::<SetRegionOption>(&serialized).unwrap()
2456 );
2457 if skip_wal {
2458 assert!(serde_json::from_str::<LegacySetRegionOption>(&serialized).is_ok());
2459 assert_eq!(
2460 option,
2461 serde_json::from_str::<SetRegionOption>(r#"{"SkipWal":true}"#).unwrap()
2462 );
2463 }
2464 }
2465
2466 #[test]
2467 fn test_set_region_option_max_row_group_row_count_try_from() {
2468 let pb = PbOption {
2469 key: MAX_ROW_GROUP_ROW_COUNT.to_string(),
2470 value: "512".to_string(),
2471 };
2472 assert_eq!(
2473 SetRegionOption::MaxRowGroupRowCount(Some(512)),
2474 SetRegionOption::try_from(&pb).unwrap()
2475 );
2476
2477 let pb = PbOption {
2478 key: MAX_ROW_GROUP_ROW_COUNT.to_string(),
2479 value: String::new(),
2480 };
2481 assert_eq!(
2482 SetRegionOption::MaxRowGroupRowCount(None),
2483 SetRegionOption::try_from(&pb).unwrap()
2484 );
2485
2486 for value in [
2487 "0".to_string(),
2488 (MAX_ROW_GROUP_ROW_COUNT_LIMIT + 1).to_string(),
2489 "invalid".to_string(),
2490 ] {
2491 let pb = PbOption {
2492 key: MAX_ROW_GROUP_ROW_COUNT.to_string(),
2493 value,
2494 };
2495 assert!(SetRegionOption::try_from(&pb).is_err());
2496 }
2497
2498 assert_eq!(
2499 UnsetRegionOption::MaxRowGroupRowCount,
2500 UnsetRegionOption::try_from(MAX_ROW_GROUP_ROW_COUNT).unwrap()
2501 );
2502 }
2503
2504 #[test]
2505 fn test_set_region_option_preserve_row_sequence_try_from() {
2506 for (value, expected) in [("true", true), ("false", false)] {
2507 let pb = PbOption {
2508 key: PRESERVE_ROW_SEQUENCE.to_string(),
2509 value: value.to_string(),
2510 };
2511 assert_eq!(
2512 SetRegionOption::PreserveRowSequence(expected),
2513 SetRegionOption::try_from(&pb).unwrap()
2514 );
2515 }
2516
2517 for value in ["1", "invalid"] {
2518 let pb = PbOption {
2519 key: PRESERVE_ROW_SEQUENCE.to_string(),
2520 value: value.to_string(),
2521 };
2522 assert!(SetRegionOption::try_from(&pb).is_err());
2523 }
2524
2525 assert_eq!(
2526 UnsetRegionOption::PreserveRowSequence,
2527 UnsetRegionOption::try_from(PRESERVE_ROW_SEQUENCE).unwrap()
2528 );
2529 assert_eq!(
2530 SetRegionOption::PreserveRowSequence(false),
2531 (&UnsetRegionOption::PreserveRowSequence).into()
2532 );
2533 }
2534
2535 #[test]
2536 fn test_set_twcs_window_trigger_options_try_from() {
2537 for key in [
2538 "compaction.twcs.active_window.trigger_file_num",
2539 "compaction.twcs.active_window.l1_merge_trigger",
2540 "compaction.twcs.inactive_window.trigger_file_num",
2541 "compaction.twcs.inactive_window.l1_merge_trigger",
2542 ] {
2543 let option = PbOption {
2544 key: key.to_string(),
2545 value: "8".to_string(),
2546 };
2547 assert_eq!(
2548 SetRegionOption::Twsc(key.to_string(), "8".to_string()),
2549 SetRegionOption::try_from(&option).unwrap(),
2550 "{key}"
2551 );
2552 }
2553 }
2554
2555 #[test]
2556 fn test_unset_twcs_window_trigger_options_try_from() {
2557 for (key, expected) in [
2558 (
2559 "compaction.twcs.active_window.trigger_file_num",
2560 UnsetRegionOption::TwcsActiveWindowTriggerFileNum,
2561 ),
2562 (
2563 "compaction.twcs.active_window.l1_merge_trigger",
2564 UnsetRegionOption::TwcsActiveWindowL1MergeTrigger,
2565 ),
2566 (
2567 "compaction.twcs.inactive_window.trigger_file_num",
2568 UnsetRegionOption::TwcsInactiveWindowTriggerFileNum,
2569 ),
2570 ] {
2571 assert_eq!(expected, UnsetRegionOption::try_from(key).unwrap());
2572 assert_eq!(
2573 SetRegionOption::Twsc(key.to_string(), String::new()),
2574 SetRegionOption::from(&expected)
2575 );
2576 }
2577
2578 let key = "compaction.twcs.inactive_window.l1_merge_trigger";
2579 let option = UnsetRegionOption::try_from(key).unwrap();
2580 assert_eq!(key, option.to_string());
2581 assert_eq!(
2582 SetRegionOption::Twsc(key.to_string(), String::new()),
2583 SetRegionOption::from(&option)
2584 );
2585 }
2586
2587 #[test]
2588 fn test_json_settings_path_validation_from_proto() {
2589 for (path, valid_path) in [
2590 (vec!["nested", "leaf"], true),
2591 (vec![""], false),
2592 (vec!["nested", ""], false),
2593 (
2594 vec!["nested", datatypes::json::JSON2_REMAINDER_FIELD_NAME],
2595 false,
2596 ),
2597 ] {
2598 for inverted_index in [false, true] {
2599 let settings = api::v1::SetJsonSettings {
2600 column_name: "payload".into(),
2601 settings: Some(api::v1::JsonSettings {
2602 type_hints: vec![api::v1::JsonTypeHint {
2603 path: path.iter().map(|part| (*part).to_string()).collect(),
2604 data_type: ColumnDataType::String as i32,
2605 inverted_index,
2606 skipping_index: None,
2607 fulltext_index: None,
2608 ..Default::default()
2609 }],
2610 max_auto_expanded_paths: Some(0),
2611 }),
2612 };
2613 let result = RegionAlterRequest::try_from(AlterRequest {
2614 region_id: 0,
2615 schema_version: 1,
2616 kind: Some(alter_request::Kind::SetJsonSettings(settings)),
2617 });
2618 assert_eq!(
2619 result.is_ok(),
2620 valid_path,
2621 "path: {path:?}, inverted_index: {inverted_index}"
2622 );
2623 }
2624 }
2625 }
2626
2627 #[test]
2628 fn test_from_proto_alter_request() {
2629 RegionAlterRequest::try_from(AlterRequest {
2630 region_id: 0,
2631 schema_version: 1,
2632 kind: None,
2633 })
2634 .unwrap_err();
2635
2636 let request = RegionAlterRequest::try_from(AlterRequest {
2637 region_id: 0,
2638 schema_version: 1,
2639 kind: Some(alter_request::Kind::AddColumns(v1::region::AddColumns {
2640 add_columns: vec![v1::region::AddColumn {
2641 column_def: Some(RegionColumnDef {
2642 column_def: Some(ColumnDef {
2643 name: "a".to_string(),
2644 data_type: ColumnDataType::String as i32,
2645 is_nullable: true,
2646 default_constraint: vec![],
2647 semantic_type: SemanticType::Field as i32,
2648 comment: String::new(),
2649 ..Default::default()
2650 }),
2651 column_id: 1,
2652 }),
2653 location: Some(v1::AddColumnLocation {
2654 location_type: LocationType::First as i32,
2655 after_column_name: String::default(),
2656 }),
2657 }],
2658 })),
2659 })
2660 .unwrap();
2661
2662 assert_eq!(
2663 request,
2664 RegionAlterRequest {
2665 kind: AlterKind::AddColumns {
2666 columns: vec![AddColumn {
2667 column_metadata: ColumnMetadata {
2668 column_schema: ColumnSchema::new(
2669 "a",
2670 ConcreteDataType::string_datatype(),
2671 true,
2672 ),
2673 semantic_type: SemanticType::Field,
2674 column_id: 1,
2675 },
2676 location: Some(AddColumnLocation::First),
2677 }]
2678 },
2679 }
2680 );
2681
2682 let request = RegionAlterRequest::try_from(AlterRequest {
2683 region_id: 0,
2684 schema_version: 1,
2685 kind: Some(alter_request::Kind::SetJsonSettings(v1::SetJsonSettings {
2686 column_name: "payload".to_string(),
2687 settings: Some(v1::JsonSettings {
2688 type_hints: vec![v1::JsonTypeHint {
2689 path: vec!["service".to_string()],
2690 data_type: ColumnDataType::String as i32,
2691 datatype_extension: None,
2692 inverted_index: true,
2693 fulltext_index: None,
2694 skipping_index: Some(v1::JsonSkippingIndexOptions {
2695 granularity: 8,
2696 index_type: 0,
2697 false_positive_rate: 0.05,
2698 }),
2699 }],
2700 max_auto_expanded_paths: Some(10),
2701 }),
2702 })),
2703 })
2704 .unwrap();
2705
2706 let AlterKind::SetJsonSettings {
2707 column_name,
2708 settings,
2709 } = request.kind
2710 else {
2711 unreachable!()
2712 };
2713 assert_eq!("payload", column_name);
2714 assert_eq!(Some(10), settings.max_auto_expanded_paths());
2715 assert_eq!(1, settings.type_hints().len());
2716 assert!(settings.type_hints()[0].inverted_index);
2717 assert_eq!(
2718 settings.type_hints()[0]
2719 .skipping_index
2720 .as_ref()
2721 .unwrap()
2722 .granularity,
2723 8
2724 );
2725 }
2726
2727 #[test]
2728 fn test_write_buffer_size_region_options() {
2729 let option = PbOption {
2730 key: WRITE_BUFFER_SIZE_KEY.to_string(),
2731 value: "128MiB".to_string(),
2732 };
2733 assert_eq!(
2734 SetRegionOption::WriteBufferSize(Some(ReadableSize::mb(128))),
2735 SetRegionOption::try_from(&option).unwrap()
2736 );
2737
2738 let option = PbOption {
2739 key: WRITE_BUFFER_SIZE_KEY.to_string(),
2740 value: "invalid".to_string(),
2741 };
2742 SetRegionOption::try_from(&option).unwrap_err();
2743
2744 let option = PbOption {
2747 key: WRITE_BUFFER_SIZE_KEY.to_string(),
2748 value: String::new(),
2749 };
2750 SetRegionOption::try_from(&option).unwrap_err();
2751
2752 assert_eq!(
2753 UnsetRegionOption::WriteBufferSize,
2754 UnsetRegionOption::try_from(WRITE_BUFFER_SIZE_KEY).unwrap()
2755 );
2756 assert_eq!(
2757 SetRegionOption::WriteBufferSize(None),
2758 SetRegionOption::from(&UnsetRegionOption::WriteBufferSize)
2759 );
2760 }
2761
2762 fn new_metadata() -> RegionMetadata {
2765 let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 1));
2766 builder
2767 .push_column_metadata(ColumnMetadata {
2768 column_schema: ColumnSchema::new(
2769 "ts",
2770 ConcreteDataType::timestamp_millisecond_datatype(),
2771 false,
2772 ),
2773 semantic_type: SemanticType::Timestamp,
2774 column_id: 1,
2775 })
2776 .push_column_metadata(ColumnMetadata {
2777 column_schema: ColumnSchema::new(
2778 "tag_0",
2779 ConcreteDataType::string_datatype(),
2780 true,
2781 ),
2782 semantic_type: SemanticType::Tag,
2783 column_id: 2,
2784 })
2785 .push_column_metadata(ColumnMetadata {
2786 column_schema: ColumnSchema::new(
2787 "field_0",
2788 ConcreteDataType::string_datatype(),
2789 true,
2790 ),
2791 semantic_type: SemanticType::Field,
2792 column_id: 3,
2793 })
2794 .push_column_metadata(ColumnMetadata {
2795 column_schema: ColumnSchema::new(
2796 "field_1",
2797 ConcreteDataType::boolean_datatype(),
2798 true,
2799 ),
2800 semantic_type: SemanticType::Field,
2801 column_id: 4,
2802 })
2803 .primary_key(vec![2]);
2804 builder.build().unwrap()
2805 }
2806
2807 fn json2_column_schema(name: &str, settings: JsonSettings) -> ColumnSchema {
2808 let mut column_schema =
2809 ColumnSchema::new(name, ConcreteDataType::Json(JsonType::null()), true);
2810 column_schema.with_extension_type(&Json2ExtensionType::new(std::sync::Arc::new(
2811 JsonMetadata::new(settings),
2812 )));
2813 column_schema
2814 }
2815
2816 #[test]
2817 fn test_add_column_validate() {
2818 let metadata = new_metadata();
2819 let add_column = AddColumn {
2820 column_metadata: ColumnMetadata {
2821 column_schema: ColumnSchema::new(
2822 "tag_1",
2823 ConcreteDataType::string_datatype(),
2824 true,
2825 ),
2826 semantic_type: SemanticType::Tag,
2827 column_id: 5,
2828 },
2829 location: None,
2830 };
2831 add_column.validate(&metadata).unwrap();
2832 assert!(add_column.need_alter(&metadata));
2833
2834 AddColumn {
2836 column_metadata: ColumnMetadata {
2837 column_schema: ColumnSchema::new(
2838 "tag_1",
2839 ConcreteDataType::string_datatype(),
2840 false,
2841 ),
2842 semantic_type: SemanticType::Tag,
2843 column_id: 5,
2844 },
2845 location: None,
2846 }
2847 .validate(&metadata)
2848 .unwrap_err();
2849
2850 let add_column = AddColumn {
2852 column_metadata: ColumnMetadata {
2853 column_schema: ColumnSchema::new(
2854 "tag_0",
2855 ConcreteDataType::string_datatype(),
2856 true,
2857 ),
2858 semantic_type: SemanticType::Tag,
2859 column_id: 2,
2860 },
2861 location: None,
2862 };
2863 add_column.validate(&metadata).unwrap();
2864 assert!(!add_column.need_alter(&metadata));
2865 }
2866
2867 #[test]
2868 fn test_add_duplicate_columns() {
2869 let kind = AlterKind::AddColumns {
2870 columns: vec![
2871 AddColumn {
2872 column_metadata: ColumnMetadata {
2873 column_schema: ColumnSchema::new(
2874 "tag_1",
2875 ConcreteDataType::string_datatype(),
2876 true,
2877 ),
2878 semantic_type: SemanticType::Tag,
2879 column_id: 5,
2880 },
2881 location: None,
2882 },
2883 AddColumn {
2884 column_metadata: ColumnMetadata {
2885 column_schema: ColumnSchema::new(
2886 "tag_1",
2887 ConcreteDataType::string_datatype(),
2888 true,
2889 ),
2890 semantic_type: SemanticType::Field,
2891 column_id: 6,
2892 },
2893 location: None,
2894 },
2895 ],
2896 };
2897 let metadata = new_metadata();
2898 kind.validate(&metadata).unwrap();
2899 assert!(kind.need_alter(&metadata));
2900 }
2901
2902 #[test]
2903 fn test_add_existing_column_different_metadata() {
2904 let metadata = new_metadata();
2905
2906 let kind = AlterKind::AddColumns {
2908 columns: vec![AddColumn {
2909 column_metadata: ColumnMetadata {
2910 column_schema: ColumnSchema::new(
2911 "tag_0",
2912 ConcreteDataType::string_datatype(),
2913 true,
2914 ),
2915 semantic_type: SemanticType::Tag,
2916 column_id: 4,
2917 },
2918 location: None,
2919 }],
2920 };
2921 kind.validate(&metadata).unwrap_err();
2922
2923 let kind = AlterKind::AddColumns {
2925 columns: vec![AddColumn {
2926 column_metadata: ColumnMetadata {
2927 column_schema: ColumnSchema::new(
2928 "tag_0",
2929 ConcreteDataType::int64_datatype(),
2930 true,
2931 ),
2932 semantic_type: SemanticType::Tag,
2933 column_id: 2,
2934 },
2935 location: None,
2936 }],
2937 };
2938 kind.validate(&metadata).unwrap_err();
2939
2940 let kind = AlterKind::AddColumns {
2942 columns: vec![AddColumn {
2943 column_metadata: ColumnMetadata {
2944 column_schema: ColumnSchema::new(
2945 "tag_1",
2946 ConcreteDataType::string_datatype(),
2947 true,
2948 ),
2949 semantic_type: SemanticType::Tag,
2950 column_id: 2,
2951 },
2952 location: None,
2953 }],
2954 };
2955 kind.validate(&metadata).unwrap_err();
2956 }
2957
2958 #[test]
2959 fn test_add_existing_column_with_location() {
2960 let metadata = new_metadata();
2961 let kind = AlterKind::AddColumns {
2962 columns: vec![AddColumn {
2963 column_metadata: ColumnMetadata {
2964 column_schema: ColumnSchema::new(
2965 "tag_0",
2966 ConcreteDataType::string_datatype(),
2967 true,
2968 ),
2969 semantic_type: SemanticType::Tag,
2970 column_id: 2,
2971 },
2972 location: Some(AddColumnLocation::First),
2973 }],
2974 };
2975 kind.validate(&metadata).unwrap_err();
2976 }
2977
2978 #[test]
2979 fn test_validate_drop_column() {
2980 let metadata = new_metadata();
2981 let kind = AlterKind::DropColumns {
2982 names: vec!["xxxx".to_string()],
2983 };
2984 kind.validate(&metadata).unwrap();
2985 assert!(!kind.need_alter(&metadata));
2986
2987 AlterKind::DropColumns {
2988 names: vec!["tag_0".to_string()],
2989 }
2990 .validate(&metadata)
2991 .unwrap_err();
2992
2993 let kind = AlterKind::DropColumns {
2994 names: vec!["field_0".to_string()],
2995 };
2996 kind.validate(&metadata).unwrap();
2997 assert!(kind.need_alter(&metadata));
2998 }
2999
3000 #[test]
3001 fn test_validate_modify_column_type() {
3002 let metadata = new_metadata();
3003 AlterKind::ModifyColumnTypes {
3004 columns: vec![ModifyColumnType {
3005 column_name: "xxxx".to_string(),
3006 target_type: ConcreteDataType::string_datatype(),
3007 }],
3008 }
3009 .validate(&metadata)
3010 .unwrap_err();
3011
3012 AlterKind::ModifyColumnTypes {
3013 columns: vec![ModifyColumnType {
3014 column_name: "field_1".to_string(),
3015 target_type: ConcreteDataType::date_datatype(),
3016 }],
3017 }
3018 .validate(&metadata)
3019 .unwrap_err();
3020
3021 AlterKind::ModifyColumnTypes {
3022 columns: vec![ModifyColumnType {
3023 column_name: "ts".to_string(),
3024 target_type: ConcreteDataType::date_datatype(),
3025 }],
3026 }
3027 .validate(&metadata)
3028 .unwrap_err();
3029
3030 let kind = AlterKind::ModifyColumnTypes {
3032 columns: vec![ModifyColumnType {
3033 column_name: "ts".to_string(),
3034 target_type: ConcreteDataType::timestamp_microsecond_datatype(),
3035 }],
3036 };
3037 kind.validate(&metadata).unwrap();
3038 assert!(kind.need_alter(&metadata));
3039
3040 let metadata_nano = {
3042 let mut metadata = new_metadata();
3043 for col in metadata.column_metadatas.iter_mut() {
3044 if col.column_schema.name == "ts" {
3045 col.column_schema.data_type = ConcreteDataType::timestamp_nanosecond_datatype();
3046 }
3047 }
3048 metadata
3049 };
3050 AlterKind::ModifyColumnTypes {
3051 columns: vec![ModifyColumnType {
3052 column_name: "ts".to_string(),
3053 target_type: ConcreteDataType::timestamp_millisecond_datatype(),
3054 }],
3055 }
3056 .validate(&metadata_nano)
3057 .unwrap_err();
3058
3059 let same_type = AlterKind::ModifyColumnTypes {
3063 columns: vec![ModifyColumnType {
3064 column_name: "ts".to_string(),
3065 target_type: ConcreteDataType::timestamp_millisecond_datatype(),
3066 }],
3067 };
3068 same_type.validate(&metadata).unwrap();
3069 assert!(!same_type.need_alter(&metadata));
3070
3071 AlterKind::ModifyColumnTypes {
3073 columns: vec![ModifyColumnType {
3074 column_name: "ts".to_string(),
3075 target_type: ConcreteDataType::string_datatype(),
3076 }],
3077 }
3078 .validate(&metadata)
3079 .unwrap_err();
3080
3081 AlterKind::ModifyColumnTypes {
3082 columns: vec![ModifyColumnType {
3083 column_name: "tag_0".to_string(),
3084 target_type: ConcreteDataType::date_datatype(),
3085 }],
3086 }
3087 .validate(&metadata)
3088 .unwrap_err();
3089
3090 let kind = AlterKind::ModifyColumnTypes {
3091 columns: vec![ModifyColumnType {
3092 column_name: "field_0".to_string(),
3093 target_type: ConcreteDataType::int32_datatype(),
3094 }],
3095 };
3096 kind.validate(&metadata).unwrap();
3097 assert!(kind.need_alter(&metadata));
3098 }
3099
3100 #[test]
3101 fn test_validate_set_json_settings() {
3102 let mut metadata = new_metadata();
3103 let current_schema = json2_column_schema("field_0", JsonSettings::new_v2());
3104 let target_settings = JsonSettings::try_new(vec![], Some(10)).unwrap();
3105 metadata
3106 .column_metadatas
3107 .iter_mut()
3108 .find(|column| column.column_schema.name == "field_0")
3109 .unwrap()
3110 .column_schema = current_schema.clone();
3111
3112 let kind = AlterKind::SetJsonSettings {
3113 column_name: "field_0".to_string(),
3114 settings: target_settings.clone(),
3115 };
3116 kind.validate(&metadata).unwrap();
3117 assert!(kind.need_alter(&metadata));
3118
3119 let no_op = AlterKind::SetJsonSettings {
3120 column_name: "field_0".to_string(),
3121 settings: JsonSettings::new_v2(),
3122 };
3123 no_op.validate(&metadata).unwrap();
3124 assert!(!no_op.need_alter(&metadata));
3125
3126 AlterKind::SetJsonSettings {
3127 column_name: "tag_0".to_string(),
3128 settings: target_settings,
3129 }
3130 .validate(&metadata)
3131 .unwrap_err();
3132 }
3133
3134 #[test]
3135 fn test_validate_add_columns() {
3136 let kind = AlterKind::AddColumns {
3137 columns: vec![
3138 AddColumn {
3139 column_metadata: ColumnMetadata {
3140 column_schema: ColumnSchema::new(
3141 "tag_1",
3142 ConcreteDataType::string_datatype(),
3143 true,
3144 ),
3145 semantic_type: SemanticType::Tag,
3146 column_id: 5,
3147 },
3148 location: None,
3149 },
3150 AddColumn {
3151 column_metadata: ColumnMetadata {
3152 column_schema: ColumnSchema::new(
3153 "field_2",
3154 ConcreteDataType::string_datatype(),
3155 true,
3156 ),
3157 semantic_type: SemanticType::Field,
3158 column_id: 6,
3159 },
3160 location: None,
3161 },
3162 ],
3163 };
3164 let request = RegionAlterRequest { kind };
3165 let mut metadata = new_metadata();
3166 metadata.schema_version = 1;
3167 request.validate(&metadata).unwrap();
3168 }
3169
3170 #[test]
3171 fn test_validate_create_region() {
3172 let column_metadatas = vec![
3173 ColumnMetadata {
3174 column_schema: ColumnSchema::new(
3175 "ts",
3176 ConcreteDataType::timestamp_millisecond_datatype(),
3177 false,
3178 ),
3179 semantic_type: SemanticType::Timestamp,
3180 column_id: 1,
3181 },
3182 ColumnMetadata {
3183 column_schema: ColumnSchema::new(
3184 "tag_0",
3185 ConcreteDataType::string_datatype(),
3186 true,
3187 ),
3188 semantic_type: SemanticType::Tag,
3189 column_id: 2,
3190 },
3191 ColumnMetadata {
3192 column_schema: ColumnSchema::new(
3193 "field_0",
3194 ConcreteDataType::string_datatype(),
3195 true,
3196 ),
3197 semantic_type: SemanticType::Field,
3198 column_id: 3,
3199 },
3200 ];
3201 let create = RegionCreateRequest {
3202 engine: "mito".to_string(),
3203 column_metadatas,
3204 primary_key: vec![3, 4],
3205 options: HashMap::new(),
3206 table_dir: "path".to_string(),
3207 path_type: PathType::Bare,
3208 partition_expr_json: Some("".to_string()),
3209 requirements: Default::default(),
3210 };
3211
3212 assert!(create.validate().is_err());
3213 }
3214
3215 #[test]
3216 fn test_parse_create_region_requirements_defaults_to_empty() {
3217 let create = CreateRequest {
3218 region_id: RegionId::new(42, 0).as_u64(),
3219 engine: "mito".to_string(),
3220 column_defs: vec![],
3221 primary_key: vec![],
3222 path: "test".to_string(),
3223 options: HashMap::new(),
3224 partition: None,
3225 requirements: None,
3226 };
3227
3228 let requests =
3229 RegionRequest::try_from_request_body(region_request::Body::Create(create)).unwrap();
3230 let RegionRequest::Create(request) = &requests[0].1 else {
3231 unreachable!()
3232 };
3233
3234 assert_eq!(request.requirements, RegionRequirements::empty());
3235 }
3236
3237 #[test]
3238 fn test_parse_create_region_requirements_from_proto() {
3239 let create = CreateRequest {
3240 region_id: RegionId::new(42, 0).as_u64(),
3241 engine: "mito".to_string(),
3242 column_defs: vec![],
3243 primary_key: vec![],
3244 path: "test".to_string(),
3245 options: HashMap::new(),
3246 partition: None,
3247 requirements: Some(api::v1::region::RegionRequirements {
3248 object_storage: true,
3249 }),
3250 };
3251
3252 let requests =
3253 RegionRequest::try_from_request_body(region_request::Body::Create(create)).unwrap();
3254 let RegionRequest::Create(request) = &requests[0].1 else {
3255 unreachable!()
3256 };
3257
3258 assert_eq!(request.requirements, RegionRequirements::object_storage());
3259 }
3260
3261 #[test]
3262 fn test_parse_region_cleanup_from_proto() {
3263 let clean_up = api::v1::region::CleanUpRequest {
3264 region_id: RegionId::new(42, 3).as_u64(),
3265 engine: "mito".to_string(),
3266 path: "test".to_string(),
3267 options: HashMap::from([("k".to_string(), "v".to_string())]),
3268 };
3269
3270 let requests =
3271 RegionRequest::try_from_request_body(region_request::Body::CleanUp(clean_up)).unwrap();
3272 let RegionRequest::CleanUp(request) = &requests[0].1 else {
3273 unreachable!()
3274 };
3275
3276 assert_eq!(requests[0].0, RegionId::new(42, 3));
3277 assert_eq!(request.engine, "mito");
3278 assert_eq!(request.table_dir, "data/test/42/");
3279 assert_eq!(request.path_type, PathType::Bare);
3280 assert_eq!(request.options.get("k"), Some(&"v".to_string()));
3281 }
3282
3283 #[test]
3284 fn test_validate_modify_column_fulltext_options() {
3285 let kind = AlterKind::SetIndexes {
3286 options: vec![SetIndexOption::Fulltext {
3287 column_name: "tag_0".to_string(),
3288 options: FulltextOptions::new_unchecked(
3289 true,
3290 FulltextAnalyzer::Chinese,
3291 false,
3292 FulltextBackend::Bloom,
3293 1000,
3294 0.01,
3295 ),
3296 }],
3297 };
3298 let request = RegionAlterRequest { kind };
3299 let mut metadata = new_metadata();
3300 metadata.schema_version = 1;
3301 request.validate(&metadata).unwrap();
3302
3303 let kind = AlterKind::UnsetIndexes {
3304 options: vec![UnsetIndexOption::Fulltext {
3305 column_name: "tag_0".to_string(),
3306 }],
3307 };
3308 let request = RegionAlterRequest { kind };
3309 let mut metadata = new_metadata();
3310 metadata.schema_version = 1;
3311 request.validate(&metadata).unwrap();
3312 }
3313
3314 #[test]
3315 fn test_validate_sync_columns() {
3316 let metadata = new_metadata();
3317 let kind = AlterKind::SyncColumns {
3318 column_metadatas: vec![
3319 ColumnMetadata {
3320 column_schema: ColumnSchema::new(
3321 "tag_1",
3322 ConcreteDataType::string_datatype(),
3323 true,
3324 ),
3325 semantic_type: SemanticType::Tag,
3326 column_id: 5,
3327 },
3328 ColumnMetadata {
3329 column_schema: ColumnSchema::new(
3330 "field_2",
3331 ConcreteDataType::string_datatype(),
3332 true,
3333 ),
3334 semantic_type: SemanticType::Field,
3335 column_id: 6,
3336 },
3337 ],
3338 };
3339 let err = kind.validate(&metadata).unwrap_err();
3340 assert!(err.to_string().contains("not a primary key"));
3341
3342 let mut column_metadatas_with_different_ts_column = metadata.column_metadatas.clone();
3344 let ts_column = column_metadatas_with_different_ts_column
3345 .iter_mut()
3346 .find(|c| c.semantic_type == SemanticType::Timestamp)
3347 .unwrap();
3348 ts_column.column_schema.name = "ts1".to_string();
3349
3350 let kind = AlterKind::SyncColumns {
3351 column_metadatas: column_metadatas_with_different_ts_column,
3352 };
3353 let err = kind.validate(&metadata).unwrap_err();
3354 assert!(
3355 err.to_string()
3356 .contains("timestamp column ts has different id")
3357 );
3358
3359 let mut column_metadatas_with_different_pk_column = metadata.column_metadatas.clone();
3361 let pk_column = column_metadatas_with_different_pk_column
3362 .iter_mut()
3363 .find(|c| c.column_schema.name == "tag_0")
3364 .unwrap();
3365 pk_column.column_id = 100;
3366 let kind = AlterKind::SyncColumns {
3367 column_metadatas: column_metadatas_with_different_pk_column,
3368 };
3369 let err = kind.validate(&metadata).unwrap_err();
3370 assert!(
3371 err.to_string()
3372 .contains("column with same name tag_0 has different id")
3373 );
3374
3375 let mut column_metadatas_with_new_field_column = metadata.column_metadatas.clone();
3377 column_metadatas_with_new_field_column.push(ColumnMetadata {
3378 column_schema: ColumnSchema::new("field_2", ConcreteDataType::string_datatype(), true),
3379 semantic_type: SemanticType::Field,
3380 column_id: 4,
3381 });
3382 let kind = AlterKind::SyncColumns {
3383 column_metadatas: column_metadatas_with_new_field_column,
3384 };
3385 kind.validate(&metadata).unwrap();
3386 }
3387
3388 #[test]
3389 fn test_cast_path_type_to_primitive() {
3390 assert_eq!(PathType::Bare as u8, 0);
3391 assert_eq!(PathType::Data as u8, 1);
3392 assert_eq!(PathType::Metadata as u8, 2);
3393 assert_eq!(
3394 PathType::try_from(PathType::Bare as u8).unwrap(),
3395 PathType::Bare
3396 );
3397 assert_eq!(
3398 PathType::try_from(PathType::Data as u8).unwrap(),
3399 PathType::Data
3400 );
3401 assert_eq!(
3402 PathType::try_from(PathType::Metadata as u8).unwrap(),
3403 PathType::Metadata
3404 );
3405 }
3406}