1use std::any::Any;
16use std::sync::Arc;
17
18use common_datasource::compression::CompressionType;
19use common_error::ext::{BoxedError, ErrorExt, RetryHint};
20use common_error::status_code::StatusCode;
21use common_macro::stack_trace_debug;
22use common_memory_manager;
23use common_runtime::JoinError;
24use common_time::Timestamp;
25use common_time::timestamp::TimeUnit;
26use datatypes::arrow::error::ArrowError;
27use datatypes::prelude::ConcreteDataType;
28use object_store::ErrorKind;
29use object_store::error::retry_hint_from_opendal_error;
30use partition::error::Error as PartitionError;
31use prost::DecodeError;
32use snafu::{Location, Snafu};
33use store_api::ManifestVersion;
34use store_api::logstore::provider::Provider;
35use store_api::storage::{FileId, RegionId};
36use tokio::time::error::Elapsed;
37
38use crate::cache::file_cache::FileType;
39use crate::region::RegionRoleState;
40use crate::schedule::remote_job_scheduler::JobId;
41use crate::worker::WorkerId;
42
43#[derive(Snafu)]
44#[snafu(visibility(pub))]
45#[stack_trace_debug]
46pub enum Error {
47 #[cfg(feature = "enterprise")]
48 #[snafu(display("Failed to route multi-region compaction input"))]
49 RouteMultiRegion {
50 #[snafu(source)]
51 source: PartitionError,
52 #[snafu(implicit)]
53 location: Location,
54 },
55
56 #[cfg(feature = "enterprise")]
57 #[snafu(display("Invalid multi-region compaction: {reason}"))]
58 InvalidMultiRegionCompaction {
59 reason: String,
60 #[snafu(implicit)]
61 location: Location,
62 },
63
64 #[cfg(feature = "enterprise")]
65 #[snafu(display("Multi-region compaction resource limit: {reason}"))]
66 MultiRegionCompactionResource {
67 reason: String,
68 #[snafu(implicit)]
69 location: Location,
70 },
71
72 #[snafu(display("SST writer {kind} needs {bytes} bytes, budget is {limit}"))]
73 SstWriteMemoryExceeded {
74 kind: &'static str,
75 bytes: usize,
76 limit: usize,
77 #[snafu(implicit)]
78 location: Location,
79 },
80
81 #[snafu(display("Unexpected data type"))]
82 DataTypeMismatch {
83 source: datatypes::error::Error,
84 #[snafu(implicit)]
85 location: Location,
86 },
87
88 #[snafu(display("External error, context: {}", context))]
89 External {
90 source: BoxedError,
91 context: String,
92 #[snafu(implicit)]
93 location: Location,
94 },
95
96 #[snafu(display("OpenDAL operator failed"))]
97 OpenDal {
98 #[snafu(implicit)]
99 location: Location,
100 #[snafu(source)]
101 error: object_store::Error,
102 },
103
104 #[snafu(display(
105 "Manifest delta {} disappeared after it was listed, path: {}",
106 version,
107 path
108 ))]
109 ManifestDeltaNotFound {
110 version: ManifestVersion,
111 path: String,
112 #[snafu(source)]
113 error: object_store::Error,
114 #[snafu(implicit)]
115 location: Location,
116 },
117
118 #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
119 CompressObject {
120 compress_type: CompressionType,
121 path: String,
122 #[snafu(source)]
123 error: std::io::Error,
124 },
125
126 #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
127 DecompressObject {
128 compress_type: CompressionType,
129 path: String,
130 #[snafu(source)]
131 error: std::io::Error,
132 },
133
134 #[snafu(display("Failed to ser/de json object"))]
135 SerdeJson {
136 #[snafu(implicit)]
137 location: Location,
138 #[snafu(source)]
139 error: serde_json::Error,
140 },
141
142 #[snafu(display("Failed to serialize column metadata"))]
143 SerializeColumnMetadata {
144 #[snafu(source)]
145 error: serde_json::Error,
146 #[snafu(implicit)]
147 location: Location,
148 },
149
150 #[snafu(display("Manifest version {version} conflicts in {manifest_dir}: {reason}"))]
151 ManifestVersionConflict {
152 version: ManifestVersion,
153 manifest_dir: String,
154 reason: String,
155 #[snafu(implicit)]
156 location: Location,
157 },
158
159 #[snafu(display(
160 "Manifest checkpoint {version} in {manifest_dir} was removed after it was listed"
161 ))]
162 ManifestCheckpointVanished {
163 version: ManifestVersion,
164 manifest_dir: String,
165 #[snafu(implicit)]
166 location: Location,
167 },
168
169 #[snafu(display("Failed to serialize manifest, region_id: {}", region_id))]
170 SerializeManifest {
171 region_id: RegionId,
172 #[snafu(source)]
173 error: serde_json::Error,
174 #[snafu(implicit)]
175 location: Location,
176 },
177
178 #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
179 InvalidScanIndex {
180 start: ManifestVersion,
181 end: ManifestVersion,
182 #[snafu(implicit)]
183 location: Location,
184 },
185
186 #[snafu(display("Invalid UTF-8 content"))]
187 Utf8 {
188 #[snafu(implicit)]
189 location: Location,
190 #[snafu(source)]
191 error: std::str::Utf8Error,
192 },
193
194 #[snafu(display("Cannot find RegionMetadata"))]
195 RegionMetadataNotFound {
196 #[snafu(implicit)]
197 location: Location,
198 },
199
200 #[snafu(display("Failed to join handle"))]
201 Join {
202 #[snafu(source)]
203 error: common_runtime::JoinError,
204 #[snafu(implicit)]
205 location: Location,
206 },
207
208 #[snafu(display("Worker {} is stopped", id))]
209 WorkerStopped {
210 id: WorkerId,
211 #[snafu(implicit)]
212 location: Location,
213 },
214
215 #[snafu(display("Engine is stopped"))]
216 EngineStopped {
217 #[snafu(implicit)]
218 location: Location,
219 },
220
221 #[snafu(display("Failed to recv result"))]
222 Recv {
223 #[snafu(source)]
224 error: tokio::sync::oneshot::error::RecvError,
225 #[snafu(implicit)]
226 location: Location,
227 },
228
229 #[snafu(display("Invalid metadata, {}", reason))]
230 InvalidMeta {
231 reason: String,
232 #[snafu(implicit)]
233 location: Location,
234 },
235
236 #[snafu(display("Invalid region metadata"))]
237 InvalidMetadata {
238 source: store_api::metadata::MetadataError,
239 #[snafu(implicit)]
240 location: Location,
241 },
242
243 #[snafu(display("Failed to create RecordBatch from vectors"))]
244 NewRecordBatch {
245 #[snafu(implicit)]
246 location: Location,
247 #[snafu(source)]
248 error: ArrowError,
249 },
250
251 #[snafu(display("Failed to read parquet file, path: {}", path))]
252 ReadParquet {
253 path: String,
254 #[snafu(source)]
255 error: parquet::errors::ParquetError,
256 #[snafu(implicit)]
257 location: Location,
258 },
259
260 #[snafu(display("Failed to load range result"))]
261 LoadRangeResult {
262 source: Arc<Error>,
263 #[snafu(implicit)]
264 location: Location,
265 },
266
267 #[snafu(display("Failed to load SST metadata"))]
268 LoadSstMeta {
269 source: Arc<Error>,
270 #[snafu(implicit)]
271 location: Location,
272 },
273
274 #[snafu(display("Failed to load Parquet page"))]
275 LoadParquetPage {
276 source: Arc<Error>,
277 #[snafu(implicit)]
278 location: Location,
279 },
280
281 #[snafu(display("Failed to write parquet file"))]
282 WriteParquet {
283 #[snafu(source)]
284 error: parquet::errors::ParquetError,
285 #[snafu(implicit)]
286 location: Location,
287 },
288
289 #[snafu(display(
290 "Cannot assign a stable field id to native histogram sub-field '{}' of column id {} (unknown sub-field name or derived id overflows i32)",
291 field_name,
292 column_id
293 ))]
294 InvalidNativeHistogramSubfield {
295 column_id: i32,
296 field_name: String,
297 #[snafu(implicit)]
298 location: Location,
299 },
300
301 #[snafu(display(
302 "Native histogram column '{}' has no usable PARQUET:field_id to namespace its sub-field ids (missing, malformed, or exceeds i32::MAX)",
303 field_name
304 ))]
305 InvalidNativeHistogramFieldId {
306 field_name: String,
307 #[snafu(implicit)]
308 location: Location,
309 },
310
311 #[snafu(display("Region {} not found", region_id))]
312 RegionNotFound {
313 region_id: RegionId,
314 #[snafu(implicit)]
315 location: Location,
316 },
317
318 #[snafu(display("Object store not found: {}", object_store))]
319 ObjectStoreNotFound {
320 object_store: String,
321 #[snafu(implicit)]
322 location: Location,
323 },
324
325 #[snafu(display("Region {} is corrupted, reason: {}", region_id, reason))]
326 RegionCorrupted {
327 region_id: RegionId,
328 reason: String,
329 #[snafu(implicit)]
330 location: Location,
331 },
332
333 #[snafu(display("Invalid request to region {}, reason: {}", region_id, reason))]
334 InvalidRequest {
335 region_id: RegionId,
336 reason: String,
337 #[snafu(implicit)]
338 location: Location,
339 },
340
341 #[snafu(display(
342 "STALE_CURSOR: incremental query stale, region: {}, given_seq: {}, min_readable_seq: {}, retry_hint: FALLBACK_FULL_RECOMPUTE",
343 region_id,
344 given_seq,
345 min_readable_seq
346 ))]
347 IncrementalQueryStale {
348 region_id: RegionId,
349 given_seq: u64,
350 min_readable_seq: u64,
351 #[snafu(implicit)]
352 location: Location,
353 },
354
355 #[snafu(display(
356 "STALE_SNAPSHOT_FENCE: snapshot upper bound stale, region: {}, given_seq: {}, min_enforceable_seq: {}, retry_hint: REBIND_SNAPSHOT_FENCE",
357 region_id,
358 given_seq,
359 min_enforceable_seq
360 ))]
361 SnapshotFenceStale {
362 region_id: RegionId,
363 given_seq: u64,
364 min_enforceable_seq: u64,
365 #[snafu(implicit)]
366 location: Location,
367 },
368
369 #[snafu(display(
370 "SEQUENCE_RANGE_UNSUPPORTED: exact sequence-range read unsupported, region: {}, min_seq: {}, max_seq: {}, reason: {}, retry_hint: FALLBACK_MEMTABLE_ONLY_OR_FULL_RECOMPUTE",
371 region_id,
372 min_seq,
373 max_seq,
374 reason
375 ))]
376 SequenceRangeUnsupported {
377 region_id: RegionId,
378 min_seq: u64,
379 max_seq: u64,
380 reason: String,
381 #[snafu(implicit)]
382 location: Location,
383 },
384
385 #[snafu(display(
386 "region {} is unusable for sequence reads: file {} declares region {}",
387 region_id,
388 file_id,
389 file_region_id
390 ))]
391 RegionSequenceDomainBroken {
392 region_id: RegionId,
393 file_region_id: RegionId,
394 file_id: FileId,
395 #[snafu(implicit)]
396 location: Location,
397 },
398
399 #[snafu(display("Old manifest missing for region {}", region_id))]
400 MissingOldManifest {
401 region_id: RegionId,
402 #[snafu(implicit)]
403 location: Location,
404 },
405
406 #[snafu(display("New manifest missing for region {}", region_id))]
407 MissingNewManifest {
408 region_id: RegionId,
409 #[snafu(implicit)]
410 location: Location,
411 },
412
413 #[snafu(display("Manifest missing for region {}", region_id))]
414 MissingManifest {
415 region_id: RegionId,
416 #[snafu(implicit)]
417 location: Location,
418 },
419
420 #[snafu(display("File consistency check failed for file {}: {}", file_id, reason))]
421 InconsistentFile {
422 file_id: FileId,
423 reason: String,
424 #[snafu(implicit)]
425 location: Location,
426 },
427
428 #[snafu(display("Files lost during remapping: old={}, new={}", old_count, new_count))]
429 FilesLost {
430 old_count: usize,
431 new_count: usize,
432 #[snafu(implicit)]
433 location: Location,
434 },
435
436 #[snafu(display("No old manifests provided (need at least one for template)"))]
437 NoOldManifests {
438 #[snafu(implicit)]
439 location: Location,
440 },
441
442 #[snafu(display("Failed to fetch manifests"))]
443 FetchManifests {
444 #[snafu(implicit)]
445 location: Location,
446 source: BoxedError,
447 },
448
449 #[snafu(display("Partition expression missing for region {}", region_id))]
450 MissingPartitionExpr {
451 region_id: RegionId,
452 #[snafu(implicit)]
453 location: Location,
454 },
455
456 #[snafu(display("Failed to serialize partition expression: {}", source))]
457 SerializePartitionExpr {
458 #[snafu(source)]
459 source: PartitionError,
460 #[snafu(implicit)]
461 location: Location,
462 },
463
464 #[snafu(display(
465 "Failed to convert ConcreteDataType to ColumnDataType, reason: {}",
466 reason
467 ))]
468 ConvertColumnDataType {
469 reason: String,
470 source: api::error::Error,
471 #[snafu(implicit)]
472 location: Location,
473 },
474
475 #[snafu(display("Need to fill default value for region {}", region_id))]
478 FillDefault {
479 region_id: RegionId,
480 },
482
483 #[snafu(display(
484 "Failed to create default value for column {} of region {}",
485 column,
486 region_id
487 ))]
488 CreateDefault {
489 region_id: RegionId,
490 column: String,
491 source: datatypes::Error,
492 #[snafu(implicit)]
493 location: Location,
494 },
495
496 #[snafu(display("Failed to build entry, region_id: {}", region_id))]
497 BuildEntry {
498 region_id: RegionId,
499 #[snafu(implicit)]
500 location: Location,
501 source: BoxedError,
502 },
503
504 #[snafu(display("Failed to write WAL"))]
505 WriteWal {
506 #[snafu(implicit)]
507 location: Location,
508 source: BoxedError,
509 },
510
511 #[snafu(display("Failed to read WAL, provider: {}", provider))]
512 ReadWal {
513 provider: Provider,
514 #[snafu(implicit)]
515 location: Location,
516 source: BoxedError,
517 },
518
519 #[snafu(display("Failed to read the WAL of node {} for region {}", node_id, region_id))]
520 ReadNodeWal {
521 region_id: RegionId,
522 node_id: u64,
523 #[snafu(implicit)]
524 location: Location,
525 source: BoxedError,
526 },
527
528 #[snafu(display("Failed to decode WAL entry, region_id: {}", region_id))]
529 DecodeWal {
530 region_id: RegionId,
531 #[snafu(implicit)]
532 location: Location,
533 #[snafu(source)]
534 error: DecodeError,
535 },
536
537 #[snafu(display("Failed to delete WAL, region_id: {}", region_id))]
538 DeleteWal {
539 region_id: RegionId,
540 #[snafu(implicit)]
541 location: Location,
542 source: BoxedError,
543 },
544
545 #[snafu(display("Failed to wait for the WAL to be durable, region_id: {}", region_id))]
546 WaitWalDurable {
547 region_id: RegionId,
548 #[snafu(implicit)]
549 location: Location,
550 source: BoxedError,
551 },
552
553 #[snafu(display("Failed to check the WAL history, region_id: {}", region_id))]
554 CheckWalHistory {
555 region_id: RegionId,
556 #[snafu(implicit)]
557 location: Location,
558 source: BoxedError,
559 },
560
561 #[snafu(display("Failed to resolve the WAL provider, region_id: {}", region_id))]
562 ResolveWalProvider {
563 region_id: RegionId,
564 #[snafu(implicit)]
565 location: Location,
566 source: BoxedError,
567 },
568
569 #[snafu(display("Failed to write region"))]
571 WriteGroup { source: Arc<Error> },
572
573 #[snafu(display("Invalid parquet SST file {}, reason: {}", file, reason))]
574 InvalidParquet {
575 file: String,
576 reason: String,
577 #[snafu(implicit)]
578 location: Location,
579 },
580
581 #[snafu(display("Invalid batch, {}", reason))]
582 InvalidBatch {
583 reason: String,
584 #[snafu(implicit)]
585 location: Location,
586 },
587
588 #[snafu(display("Invalid arrow record batch, {}", reason))]
589 InvalidRecordBatch {
590 reason: String,
591 #[snafu(implicit)]
592 location: Location,
593 },
594
595 #[snafu(display("Invalid wal read request, {}", reason))]
596 InvalidWalReadRequest {
597 reason: String,
598 #[snafu(implicit)]
599 location: Location,
600 },
601
602 #[snafu(display("Failed to convert array to vector"))]
603 ConvertVector {
604 #[snafu(implicit)]
605 location: Location,
606 source: datatypes::error::Error,
607 },
608
609 #[snafu(display("Failed to compute arrow arrays"))]
610 ComputeArrow {
611 #[snafu(implicit)]
612 location: Location,
613 #[snafu(source)]
614 error: datatypes::arrow::error::ArrowError,
615 },
616
617 #[snafu(display("Failed to evaluate partition filter"))]
618 EvalPartitionFilter {
619 #[snafu(implicit)]
620 location: Location,
621 #[snafu(source)]
622 error: datafusion::error::DataFusionError,
623 },
624
625 #[snafu(display("Failed to merge candidate series"))]
626 MergeCandidateSeries {
627 #[snafu(implicit)]
628 location: Location,
629 #[snafu(source)]
630 error: datafusion::error::DataFusionError,
631 },
632
633 #[snafu(display("Scan memory exhausted"))]
634 ScanMemoryExhausted {
635 #[snafu(implicit)]
636 location: Location,
637 #[snafu(source)]
638 error: datafusion::error::DataFusionError,
639 },
640
641 #[snafu(display("Failed to compute vector"))]
642 ComputeVector {
643 #[snafu(implicit)]
644 location: Location,
645 source: datatypes::error::Error,
646 },
647
648 #[snafu(display("Primary key length mismatch, expect: {}, actual: {}", expect, actual))]
649 PrimaryKeyLengthMismatch {
650 expect: usize,
651 actual: usize,
652 #[snafu(implicit)]
653 location: Location,
654 },
655
656 #[snafu(display("Invalid sender",))]
657 InvalidSender {
658 #[snafu(implicit)]
659 location: Location,
660 },
661
662 #[snafu(display("Invalid scheduler state"))]
663 InvalidSchedulerState {
664 #[snafu(implicit)]
665 location: Location,
666 },
667
668 #[snafu(display("Failed to stop scheduler"))]
669 StopScheduler {
670 #[snafu(source)]
671 error: JoinError,
672 #[snafu(implicit)]
673 location: Location,
674 },
675
676 #[snafu(display(
677 "Failed to batch delete SST files, region id: {}, file ids: {:?}",
678 region_id,
679 file_ids
680 ))]
681 DeleteSsts {
682 region_id: RegionId,
683 file_ids: Vec<FileId>,
684 #[snafu(source)]
685 error: object_store::Error,
686 #[snafu(implicit)]
687 location: Location,
688 },
689
690 #[snafu(display("Failed to delete index file, file id: {}", file_id))]
691 DeleteIndex {
692 file_id: FileId,
693 #[snafu(source)]
694 error: object_store::Error,
695 #[snafu(implicit)]
696 location: Location,
697 },
698
699 #[snafu(display("Failed to batch delete index files, file ids: {:?}", file_ids))]
700 DeleteIndexes {
701 file_ids: Vec<FileId>,
702 #[snafu(source)]
703 error: object_store::Error,
704 #[snafu(implicit)]
705 location: Location,
706 },
707
708 #[snafu(display("Failed to flush region {}", region_id))]
709 FlushRegion {
710 region_id: RegionId,
711 source: Arc<Error>,
712 #[snafu(implicit)]
713 location: Location,
714 },
715
716 #[snafu(display("Region {} is dropped", region_id))]
717 RegionDropped {
718 region_id: RegionId,
719 #[snafu(implicit)]
720 location: Location,
721 },
722
723 #[snafu(display("Region {} is closed", region_id))]
724 RegionClosed {
725 region_id: RegionId,
726 #[snafu(implicit)]
727 location: Location,
728 },
729
730 #[snafu(display(
731 "Stale compaction execution for region {}, the region may have been reopened, truncated or the compaction was superseded",
732 region_id
733 ))]
734 StaleCompactionExecution {
735 region_id: RegionId,
736 #[snafu(implicit)]
737 location: Location,
738 },
739
740 #[snafu(display("Region {} is truncated", region_id))]
741 RegionTruncated {
742 region_id: RegionId,
743 #[snafu(implicit)]
744 location: Location,
745 },
746
747 #[snafu(display(
748 "Engine write buffer is full, rejecting write requests of region {}",
749 region_id,
750 ))]
751 RejectWrite {
752 region_id: RegionId,
753 #[snafu(implicit)]
754 location: Location,
755 },
756
757 #[snafu(display("Region {} rejects writes while in staging", region_id))]
758 RejectWriteInStaging {
759 region_id: RegionId,
760 #[snafu(implicit)]
761 location: Location,
762 },
763
764 #[snafu(display("Region {} still has WAL tails to recover", region_id))]
765 RegionPendingTails {
766 region_id: RegionId,
767 #[snafu(implicit)]
768 location: Location,
769 },
770
771 #[snafu(display(
772 "Region {} cannot read pending WAL tails of node {}",
773 region_id,
774 node_id
775 ))]
776 RemoteWalUnsupported {
777 region_id: RegionId,
778 node_id: u64,
779 #[snafu(implicit)]
780 location: Location,
781 },
782
783 #[snafu(display(
784 "Region {} cannot read entry {} of the WAL of node {}, where its pending tail ends",
785 region_id,
786 entry_id,
787 node_id
788 ))]
789 TailEntryMissing {
790 region_id: RegionId,
791 node_id: u64,
792 entry_id: u64,
793 #[snafu(implicit)]
794 location: Location,
795 },
796
797 #[snafu(display("Region {} does not hold a valid leader lease", region_id))]
798 RegionLeaseExpired {
799 region_id: RegionId,
800 #[snafu(implicit)]
801 location: Location,
802 },
803
804 #[snafu(display(
805 "Region {} lost its leader lease while writing to the WAL, the write may have been applied",
806 region_id
807 ))]
808 UnacknowledgedWrite {
809 region_id: RegionId,
810 #[snafu(implicit)]
811 location: Location,
812 },
813
814 #[snafu(display("Failed to compact region {}", region_id))]
815 CompactRegion {
816 region_id: RegionId,
817 source: Arc<Error>,
818 #[snafu(implicit)]
819 location: Location,
820 },
821
822 #[snafu(display("Failed to edit region {}", region_id))]
823 EditRegion {
824 region_id: RegionId,
825 source: Arc<Error>,
826 #[snafu(implicit)]
827 location: Location,
828 },
829
830 #[snafu(display(
831 "Failed to end staging of region {}, which is still staging",
832 region_id
833 ))]
834 EndStaging {
835 region_id: RegionId,
836 source: Arc<Error>,
837 #[snafu(implicit)]
838 location: Location,
839 },
840
841 #[snafu(display("Region {} waits for a retry of its end-of-staging commit", region_id))]
842 EndStagingPending {
843 region_id: RegionId,
844 #[snafu(implicit)]
845 location: Location,
846 },
847
848 #[snafu(display(
849 "Failed to compat readers for region {}, reason: {}",
850 region_id,
851 reason,
852 ))]
853 CompatReader {
854 region_id: RegionId,
855 reason: String,
856 #[snafu(implicit)]
857 location: Location,
858 },
859
860 #[snafu(display("Invalue region req"))]
861 InvalidRegionRequest {
862 source: store_api::metadata::MetadataError,
863 #[snafu(implicit)]
864 location: Location,
865 },
866
867 #[snafu(display(
868 "Region {} is in {:?} state, which does not permit manifest updates.",
869 region_id,
870 state
871 ))]
872 UpdateManifest {
873 region_id: RegionId,
874 state: RegionRoleState,
875 #[snafu(implicit)]
876 location: Location,
877 },
878
879 #[snafu(display("Region {} is in {:?} state, expect: {:?}", region_id, state, expect))]
880 RegionState {
881 region_id: RegionId,
882 state: RegionRoleState,
883 expect: RegionRoleState,
884 #[snafu(implicit)]
885 location: Location,
886 },
887
888 #[snafu(display(
889 "Partition expr version mismatch for region {}: request {}, expected {}",
890 region_id,
891 request_version,
892 expected_version
893 ))]
894 PartitionExprVersionMismatch {
895 region_id: RegionId,
896 request_version: u64,
897 expected_version: u64,
898 #[snafu(implicit)]
899 location: Location,
900 },
901
902 #[snafu(display("Invalid options"))]
903 JsonOptions {
904 #[snafu(source)]
905 error: serde_json::Error,
906 #[snafu(implicit)]
907 location: Location,
908 },
909
910 #[snafu(display(
911 "Empty region directory, region_id: {}, region_dir: {}",
912 region_id,
913 region_dir,
914 ))]
915 EmptyRegionDir {
916 region_id: RegionId,
917 region_dir: String,
918 #[snafu(implicit)]
919 location: Location,
920 },
921
922 #[snafu(display("Empty manifest directory, manifest_dir: {}", manifest_dir,))]
923 EmptyManifestDir {
924 manifest_dir: String,
925 #[snafu(implicit)]
926 location: Location,
927 },
928
929 #[snafu(display("Column not found, column: {column}"))]
930 ColumnNotFound {
931 column: String,
932 #[snafu(implicit)]
933 location: Location,
934 },
935
936 #[snafu(display("Failed to build index applier"))]
937 BuildIndexApplier {
938 source: index::inverted_index::error::Error,
939 #[snafu(implicit)]
940 location: Location,
941 },
942
943 #[snafu(display("Failed to build index asynchronously in region {}", region_id))]
944 BuildIndexAsync {
945 region_id: RegionId,
946 source: Arc<Error>,
947 #[snafu(implicit)]
948 location: Location,
949 },
950
951 #[snafu(display(
952 "Index build did not complete for files {:?} in region {}",
953 file_ids,
954 region_id
955 ))]
956 IndexBuildIncomplete {
957 region_id: RegionId,
958 file_ids: Vec<FileId>,
959 #[snafu(implicit)]
960 location: Location,
961 },
962
963 #[snafu(display("Failed to convert value"))]
964 ConvertValue {
965 source: datatypes::error::Error,
966 #[snafu(implicit)]
967 location: Location,
968 },
969
970 #[snafu(display("Failed to apply inverted index"))]
971 ApplyInvertedIndex {
972 source: index::inverted_index::error::Error,
973 #[snafu(implicit)]
974 location: Location,
975 },
976
977 #[snafu(display("Failed to apply bloom filter index"))]
978 ApplyBloomFilterIndex {
979 source: index::bloom_filter::error::Error,
980 #[snafu(implicit)]
981 location: Location,
982 },
983
984 #[snafu(display("Failed to push index value"))]
985 PushIndexValue {
986 source: index::inverted_index::error::Error,
987 #[snafu(implicit)]
988 location: Location,
989 },
990
991 #[snafu(display("Failed to write index completely"))]
992 IndexFinish {
993 source: index::inverted_index::error::Error,
994 #[snafu(implicit)]
995 location: Location,
996 },
997
998 #[snafu(display("Operate on aborted index"))]
999 OperateAbortedIndex {
1000 #[snafu(implicit)]
1001 location: Location,
1002 },
1003
1004 #[snafu(display("Failed to read puffin blob"))]
1005 PuffinReadBlob {
1006 source: puffin::error::Error,
1007 #[snafu(implicit)]
1008 location: Location,
1009 },
1010
1011 #[snafu(display("Failed to add blob to puffin file"))]
1012 PuffinAddBlob {
1013 source: puffin::error::Error,
1014 #[snafu(implicit)]
1015 location: Location,
1016 },
1017
1018 #[snafu(display("Failed to clean dir {dir}"))]
1019 CleanDir {
1020 dir: String,
1021 #[snafu(source)]
1022 error: std::io::Error,
1023 #[snafu(implicit)]
1024 location: Location,
1025 },
1026
1027 #[snafu(display("Invalid config, {reason}"))]
1028 InvalidConfig {
1029 reason: String,
1030 #[snafu(implicit)]
1031 location: Location,
1032 },
1033
1034 #[snafu(display(
1035 "Stale log entry found during replay, region: {}, flushed: {}, replayed: {}",
1036 region_id,
1037 flushed_entry_id,
1038 unexpected_entry_id
1039 ))]
1040 StaleLogEntry {
1041 region_id: RegionId,
1042 flushed_entry_id: u64,
1043 unexpected_entry_id: u64,
1044 },
1045
1046 #[snafu(display(
1047 "Failed to download file, region_id: {}, file_id: {}, file_type: {:?}",
1048 region_id,
1049 file_id,
1050 file_type,
1051 ))]
1052 Download {
1053 region_id: RegionId,
1054 file_id: FileId,
1055 file_type: FileType,
1056 #[snafu(source)]
1057 error: std::io::Error,
1058 #[snafu(implicit)]
1059 location: Location,
1060 },
1061
1062 #[snafu(display(
1063 "Failed to upload file, region_id: {}, file_id: {}, file_type: {:?}",
1064 region_id,
1065 file_id,
1066 file_type,
1067 ))]
1068 Upload {
1069 region_id: RegionId,
1070 file_id: FileId,
1071 file_type: FileType,
1072 #[snafu(source)]
1073 error: std::io::Error,
1074 #[snafu(implicit)]
1075 location: Location,
1076 },
1077
1078 #[snafu(display("Failed to create directory {}", dir))]
1079 CreateDir {
1080 dir: String,
1081 #[snafu(source)]
1082 error: std::io::Error,
1083 },
1084
1085 #[snafu(display("Record batch error"))]
1086 RecordBatch {
1087 source: common_recordbatch::error::Error,
1088 #[snafu(implicit)]
1089 location: Location,
1090 },
1091
1092 #[snafu(display("BiErrors, first: {first}, second: {second}"))]
1093 BiErrors {
1094 first: Box<Error>,
1095 second: Box<Error>,
1096 #[snafu(implicit)]
1097 location: Location,
1098 },
1099
1100 #[snafu(display("Encode null value"))]
1101 IndexEncodeNull {
1102 #[snafu(implicit)]
1103 location: Location,
1104 },
1105
1106 #[snafu(display("Failed to encode memtable to Parquet bytes"))]
1107 EncodeMemtable {
1108 #[snafu(source)]
1109 error: parquet::errors::ParquetError,
1110 #[snafu(implicit)]
1111 location: Location,
1112 },
1113
1114 #[snafu(display("Partition {} out of range, {} in total", given, all))]
1115 PartitionOutOfRange {
1116 given: usize,
1117 all: usize,
1118 #[snafu(implicit)]
1119 location: Location,
1120 },
1121
1122 #[snafu(display("Failed to iter data part"))]
1123 ReadDataPart {
1124 #[snafu(implicit)]
1125 location: Location,
1126 #[snafu(source)]
1127 error: parquet::errors::ParquetError,
1128 },
1129
1130 #[snafu(display("Failed to read row group in memtable"))]
1131 DecodeArrowRowGroup {
1132 #[snafu(source)]
1133 error: ArrowError,
1134 #[snafu(implicit)]
1135 location: Location,
1136 },
1137
1138 #[snafu(display("Invalid region options, {}", reason))]
1139 InvalidRegionOptions {
1140 reason: String,
1141 #[snafu(implicit)]
1142 location: Location,
1143 },
1144
1145 #[snafu(display("checksum mismatch (actual: {}, expected: {})", actual, expected))]
1146 ChecksumMismatch { actual: u32, expected: u32 },
1147
1148 #[snafu(display(
1149 "No checkpoint found, region: {}, last_version: {}",
1150 region_id,
1151 last_version
1152 ))]
1153 NoCheckpoint {
1154 region_id: RegionId,
1155 last_version: ManifestVersion,
1156 #[snafu(implicit)]
1157 location: Location,
1158 },
1159
1160 #[snafu(display(
1161 "No manifests found in range: [{}..{}), region: {}, last_version: {}",
1162 start_version,
1163 end_version,
1164 region_id,
1165 last_version
1166 ))]
1167 NoManifests {
1168 region_id: RegionId,
1169 start_version: ManifestVersion,
1170 end_version: ManifestVersion,
1171 last_version: ManifestVersion,
1172 #[snafu(implicit)]
1173 location: Location,
1174 },
1175
1176 #[snafu(display(
1177 "Failed to install manifest to {}, region: {}, available manifest version: {}, last version: {}",
1178 target_version,
1179 region_id,
1180 available_version,
1181 last_version
1182 ))]
1183 InstallManifestTo {
1184 region_id: RegionId,
1185 target_version: ManifestVersion,
1186 available_version: ManifestVersion,
1187 #[snafu(implicit)]
1188 location: Location,
1189 last_version: ManifestVersion,
1190 },
1191
1192 #[snafu(display("Region {} is stopped", region_id))]
1193 RegionStopped {
1194 region_id: RegionId,
1195 #[snafu(implicit)]
1196 location: Location,
1197 },
1198
1199 #[snafu(display(
1200 "Time range predicate overflows, timestamp: {:?}, target unit: {}",
1201 timestamp,
1202 unit
1203 ))]
1204 TimeRangePredicateOverflow {
1205 timestamp: Timestamp,
1206 unit: TimeUnit,
1207 #[snafu(implicit)]
1208 location: Location,
1209 },
1210
1211 #[snafu(display("Failed to open region"))]
1212 OpenRegion {
1213 #[snafu(implicit)]
1214 location: Location,
1215 source: Arc<Error>,
1216 },
1217
1218 #[snafu(display(
1219 "Region {} does not satisfy requirement '{}': {}",
1220 region_id,
1221 requirement,
1222 reason
1223 ))]
1224 RegionRequirement {
1225 region_id: RegionId,
1226 requirement: &'static str,
1227 reason: &'static str,
1228 #[snafu(implicit)]
1229 location: Location,
1230 },
1231
1232 #[snafu(display("Failed to parse job id"))]
1233 ParseJobId {
1234 #[snafu(implicit)]
1235 location: Location,
1236 #[snafu(source)]
1237 error: uuid::Error,
1238 },
1239
1240 #[snafu(display("Operation is not supported: {}", err_msg))]
1241 UnsupportedOperation {
1242 err_msg: String,
1243 #[snafu(implicit)]
1244 location: Location,
1245 },
1246
1247 #[snafu(display(
1248 "Failed to remotely compact region {} by job {:?} due to {}",
1249 region_id,
1250 job_id,
1251 reason
1252 ))]
1253 RemoteCompaction {
1254 region_id: RegionId,
1255 job_id: Option<JobId>,
1256 reason: String,
1257 #[snafu(implicit)]
1258 location: Location,
1259 },
1260
1261 #[snafu(display("Failed to initialize puffin stager"))]
1262 PuffinInitStager {
1263 source: puffin::error::Error,
1264 #[snafu(implicit)]
1265 location: Location,
1266 },
1267
1268 #[snafu(display("Failed to purge puffin stager"))]
1269 PuffinPurgeStager {
1270 source: puffin::error::Error,
1271 #[snafu(implicit)]
1272 location: Location,
1273 },
1274
1275 #[snafu(display("Failed to build puffin reader"))]
1276 PuffinBuildReader {
1277 source: puffin::error::Error,
1278 #[snafu(implicit)]
1279 location: Location,
1280 },
1281
1282 #[snafu(display("Failed to retrieve index options from column metadata"))]
1283 IndexOptions {
1284 #[snafu(implicit)]
1285 location: Location,
1286 source: datatypes::error::Error,
1287 column_name: String,
1288 },
1289
1290 #[snafu(display("Failed to retrieve compression options of column {column_name}"))]
1291 CompressionOptions {
1292 #[snafu(implicit)]
1293 location: Location,
1294 source: datatypes::error::Error,
1295 column_name: String,
1296 },
1297
1298 #[snafu(display("Failed to create fulltext index creator"))]
1299 CreateFulltextCreator {
1300 source: index::fulltext_index::error::Error,
1301 #[snafu(implicit)]
1302 location: Location,
1303 },
1304
1305 #[snafu(display("Failed to cast vector of {from} to {to}"))]
1306 CastVector {
1307 #[snafu(implicit)]
1308 location: Location,
1309 from: ConcreteDataType,
1310 to: ConcreteDataType,
1311 source: datatypes::error::Error,
1312 },
1313
1314 #[snafu(display("Failed to push text to fulltext index"))]
1315 FulltextPushText {
1316 source: index::fulltext_index::error::Error,
1317 #[snafu(implicit)]
1318 location: Location,
1319 },
1320
1321 #[snafu(display("Failed to finalize fulltext index creator"))]
1322 FulltextFinish {
1323 source: index::fulltext_index::error::Error,
1324 #[snafu(implicit)]
1325 location: Location,
1326 },
1327
1328 #[snafu(display("Failed to apply fulltext index"))]
1329 ApplyFulltextIndex {
1330 source: index::fulltext_index::error::Error,
1331 #[snafu(implicit)]
1332 location: Location,
1333 },
1334
1335 #[snafu(display("SST file {} does not contain valid stats info", file_path))]
1336 StatsNotPresent {
1337 file_path: String,
1338 #[snafu(implicit)]
1339 location: Location,
1340 },
1341
1342 #[snafu(display("Failed to decode stats of file {}", file_path))]
1343 DecodeStats {
1344 file_path: String,
1345 #[snafu(implicit)]
1346 location: Location,
1347 },
1348
1349 #[snafu(display("Invalid SST primary key range: {reason}"))]
1350 InvalidPrimaryKeyRange {
1351 reason: String,
1352 #[snafu(implicit)]
1353 location: Location,
1354 },
1355
1356 #[snafu(display("Failed to decode SST primary key range {endpoint} endpoint"))]
1357 DecodePrimaryKeyRange {
1358 endpoint: &'static str,
1359 source: mito_codec::error::Error,
1360 #[snafu(implicit)]
1361 location: Location,
1362 },
1363
1364 #[snafu(display("Region {} is busy", region_id))]
1365 RegionBusy {
1366 region_id: RegionId,
1367 #[snafu(implicit)]
1368 location: Location,
1369 },
1370
1371 #[snafu(display("Failed to get schema metadata"))]
1372 GetSchemaMetadata {
1373 source: common_meta::error::Error,
1374 #[snafu(implicit)]
1375 location: Location,
1376 },
1377
1378 #[snafu(display("Timeout"))]
1379 Timeout {
1380 #[snafu(source)]
1381 error: Elapsed,
1382 #[snafu(implicit)]
1383 location: Location,
1384 },
1385
1386 #[snafu(display("Failed to read file metadata"))]
1387 Metadata {
1388 #[snafu(source)]
1389 error: std::io::Error,
1390 #[snafu(implicit)]
1391 location: Location,
1392 },
1393
1394 #[snafu(display("Failed to push value to bloom filter"))]
1395 PushBloomFilterValue {
1396 source: index::bloom_filter::error::Error,
1397 #[snafu(implicit)]
1398 location: Location,
1399 },
1400
1401 #[snafu(display("Failed to finish bloom filter"))]
1402 BloomFilterFinish {
1403 source: index::bloom_filter::error::Error,
1404 #[snafu(implicit)]
1405 location: Location,
1406 },
1407
1408 #[snafu(display("Manual compaction is override by following operations."))]
1409 ManualCompactionOverride {},
1410
1411 #[snafu(display("Manual compaction is already running for region {region_id}."))]
1412 ManualCompactionAlreadyRunning { region_id: RegionId },
1413
1414 #[snafu(display("Compaction is cancelled."))]
1415 CompactionCancelled {},
1416
1417 #[snafu(display("SST write is cancelled."))]
1418 SstWriteCancelled {},
1419
1420 #[snafu(display("Flush is cancelled."))]
1421 FlushCancelled {},
1422
1423 #[snafu(display("Compaction memory exhausted for region {region_id} (policy: {policy})",))]
1424 CompactionMemoryExhausted {
1425 region_id: RegionId,
1426 policy: String,
1427 #[snafu(source)]
1428 source: common_memory_manager::Error,
1429 #[snafu(implicit)]
1430 location: Location,
1431 },
1432
1433 #[snafu(display(
1434 "Incompatible WAL provider change. This is typically caused by changing WAL provider in database config file without completely cleaning existing files. Global provider: {}, region provider: {}",
1435 global,
1436 region
1437 ))]
1438 IncompatibleWalProviderChange { global: String, region: String },
1439
1440 #[snafu(display("Expected mito manifest info"))]
1441 MitoManifestInfo {
1442 #[snafu(implicit)]
1443 location: Location,
1444 },
1445
1446 #[snafu(display("Failed to scan series"))]
1447 ScanSeries {
1448 #[snafu(implicit)]
1449 location: Location,
1450 source: Arc<Error>,
1451 },
1452
1453 #[snafu(display("Partition {} scan multiple times", partition))]
1454 ScanMultiTimes {
1455 partition: usize,
1456 #[snafu(implicit)]
1457 location: Location,
1458 },
1459
1460 #[snafu(display("Invalid partition expression: {}", expr))]
1461 InvalidPartitionExpr {
1462 expr: String,
1463 #[snafu(implicit)]
1464 location: Location,
1465 source: partition::error::Error,
1466 },
1467
1468 #[snafu(display("Failed to decode bulk wal entry"))]
1469 ConvertBulkWalEntry {
1470 #[snafu(implicit)]
1471 location: Location,
1472 source: common_grpc::Error,
1473 },
1474
1475 #[snafu(display("Failed to encode"))]
1476 Encode {
1477 #[snafu(implicit)]
1478 location: Location,
1479 source: mito_codec::error::Error,
1480 },
1481
1482 #[snafu(display("Failed to decode"))]
1483 Decode {
1484 #[snafu(implicit)]
1485 location: Location,
1486 source: mito_codec::error::Error,
1487 },
1488
1489 #[snafu(display("Unexpected: {reason}"))]
1490 Unexpected {
1491 reason: String,
1492 #[snafu(implicit)]
1493 location: Location,
1494 },
1495
1496 #[cfg(feature = "enterprise")]
1497 #[snafu(display("Failed to scan external range"))]
1498 ScanExternalRange {
1499 source: BoxedError,
1500 #[snafu(implicit)]
1501 location: Location,
1502 },
1503
1504 #[snafu(display(
1505 "Inconsistent timestamp column length, expect: {}, actual: {}",
1506 expected,
1507 actual
1508 ))]
1509 InconsistentTimestampLength {
1510 expected: usize,
1511 actual: usize,
1512 #[snafu(implicit)]
1513 location: Location,
1514 },
1515
1516 #[snafu(display(
1517 "Too many files to read concurrently: {}, max allowed: {}",
1518 actual,
1519 max
1520 ))]
1521 TooManyFilesToRead {
1522 actual: usize,
1523 max: usize,
1524 #[snafu(implicit)]
1525 location: Location,
1526 },
1527
1528 #[snafu(display("Duration out of range: {input:?}"))]
1529 DurationOutOfRange {
1530 input: std::time::Duration,
1531 #[snafu(source)]
1532 error: chrono::OutOfRangeError,
1533 #[snafu(implicit)]
1534 location: Location,
1535 },
1536
1537 #[snafu(display("GC job permit exhausted"))]
1538 TooManyGcJobs {
1539 #[snafu(implicit)]
1540 location: Location,
1541 },
1542
1543 #[snafu(display(
1544 "Staging partition expr mismatch, manifest: {:?}, request: {}",
1545 manifest_expr,
1546 request_expr
1547 ))]
1548 StagingPartitionExprMismatch {
1549 manifest_expr: Option<String>,
1550 request_expr: String,
1551 #[snafu(implicit)]
1552 location: Location,
1553 },
1554
1555 #[snafu(display(
1556 "Invalid source and target region, source: {}, target: {}",
1557 source_region_id,
1558 target_region_id
1559 ))]
1560 InvalidSourceAndTargetRegion {
1561 source_region_id: RegionId,
1562 target_region_id: RegionId,
1563 #[snafu(implicit)]
1564 location: Location,
1565 },
1566
1567 #[snafu(display("Failed to prune file"))]
1568 PruneFile {
1569 source: Arc<Error>,
1570 #[snafu(implicit)]
1571 location: Location,
1572 },
1573
1574 #[snafu(display("Failed to cast column"))]
1575 CastColumn {
1576 #[snafu(source)]
1577 error: datafusion::error::DataFusionError,
1578 #[snafu(implicit)]
1579 location: Location,
1580 },
1581
1582 #[snafu(display("Failed to generate Arrow schema from Parquet file: {}", file))]
1583 ParquetToArrowSchema {
1584 file: String,
1585 #[snafu(source)]
1586 error: parquet::errors::ParquetError,
1587 #[snafu(implicit)]
1588 location: Location,
1589 },
1590
1591 #[snafu(display(
1592 "Region {} is in {:?} state, expect: Writable, Staging or Downgrading",
1593 region_id,
1594 state
1595 ))]
1596 FlushableRegionState {
1597 region_id: RegionId,
1598 state: RegionRoleState,
1599 #[snafu(implicit)]
1600 location: Location,
1601 },
1602}
1603
1604pub type Result<T, E = Error> = std::result::Result<T, E>;
1605
1606impl Error {
1607 pub(crate) fn is_fill_default(&self) -> bool {
1609 matches!(self, Error::FillDefault { .. })
1610 }
1611
1612 pub(crate) fn is_object_not_found(&self) -> bool {
1614 match self {
1615 Error::OpenDal { error, .. } => error.kind() == ErrorKind::NotFound,
1616 Error::ManifestDeltaNotFound { .. } => true,
1617 Error::LoadSstMeta { source, .. }
1618 | Error::LoadParquetPage { source, .. }
1619 | Error::LoadRangeResult { source, .. } => source.is_object_not_found(),
1620 _ => false,
1621 }
1622 }
1623
1624 pub(crate) fn may_have_persisted_manifest_update(&self) -> bool {
1631 match self {
1632 Error::InvalidRequest { .. }
1634 | Error::UpdateManifest { .. }
1635 | Error::RegionState { .. }
1636 | Error::RegionTruncated { .. }
1637 | Error::RegionStopped { .. }
1638 | Error::RegionLeaseExpired { .. }
1639 | Error::EndStagingPending { .. }
1640 | Error::SerdeJson { .. }
1641 | Error::CompressObject { .. } => false,
1642 Error::OpenDal { error, .. } => !matches!(
1643 error.kind(),
1644 ErrorKind::Unsupported
1645 | ErrorKind::ConfigInvalid
1646 | ErrorKind::NotFound
1647 | ErrorKind::IsADirectory
1648 | ErrorKind::NotADirectory
1649 | ErrorKind::AlreadyExists
1650 | ErrorKind::IsSameFile
1651 | ErrorKind::ConditionNotMatch
1652 | ErrorKind::RangeNotSatisfied
1653 ),
1654 _ => true,
1655 }
1656 }
1657}
1658
1659impl ErrorExt for Error {
1660 fn status_code(&self) -> StatusCode {
1661 use Error::*;
1662
1663 match self {
1664 #[cfg(feature = "enterprise")]
1665 RouteMultiRegion { source, .. } => source.status_code(),
1666 #[cfg(feature = "enterprise")]
1667 InvalidMultiRegionCompaction { .. } => StatusCode::InvalidArguments,
1668 #[cfg(feature = "enterprise")]
1669 MultiRegionCompactionResource { .. } => StatusCode::RuntimeResourcesExhausted,
1670 SstWriteMemoryExceeded { .. } => StatusCode::RuntimeResourcesExhausted,
1671 DataTypeMismatch { source, .. } => source.status_code(),
1672 OpenDal { .. }
1673 | ManifestDeltaNotFound { .. }
1674 | ManifestVersionConflict { .. }
1675 | ManifestCheckpointVanished { .. }
1676 | ReadParquet { .. } => StatusCode::StorageUnavailable,
1677 WriteWal { source, .. }
1678 | ReadWal { source, .. }
1679 | ReadNodeWal { source, .. }
1680 | DeleteWal { source, .. }
1681 | WaitWalDurable { source, .. }
1682 | ResolveWalProvider { source, .. }
1683 | CheckWalHistory { source, .. } => source.status_code(),
1684 CompressObject { .. }
1685 | DecompressObject { .. }
1686 | SerdeJson { .. }
1687 | Utf8 { .. }
1688 | NewRecordBatch { .. }
1689 | RegionCorrupted { .. }
1690 | InconsistentFile { .. }
1691 | CreateDefault { .. }
1692 | InvalidParquet { .. }
1693 | OperateAbortedIndex { .. }
1694 | IndexEncodeNull { .. }
1695 | NoCheckpoint { .. }
1696 | NoManifests { .. }
1697 | FilesLost { .. }
1698 | InstallManifestTo { .. }
1699 | Unexpected { .. }
1700 | SerializeColumnMetadata { .. }
1701 | SerializeManifest { .. }
1702 | StagingPartitionExprMismatch { .. } => StatusCode::Unexpected,
1703
1704 RegionNotFound { .. } => StatusCode::RegionNotFound,
1705 ObjectStoreNotFound { .. }
1706 | InvalidScanIndex { .. }
1707 | InvalidMeta { .. }
1708 | InvalidRequest { .. }
1709 | PartitionExprVersionMismatch { .. }
1710 | FillDefault { .. }
1711 | ConvertColumnDataType { .. }
1712 | ColumnNotFound { .. }
1713 | InvalidMetadata { .. }
1714 | InvalidRegionOptions { .. }
1715 | InvalidWalReadRequest { .. }
1716 | PartitionOutOfRange { .. }
1717 | ParseJobId { .. }
1718 | DurationOutOfRange { .. }
1719 | MissingOldManifest { .. }
1720 | MissingNewManifest { .. }
1721 | MissingManifest { .. }
1722 | NoOldManifests { .. }
1723 | MissingPartitionExpr { .. }
1724 | SerializePartitionExpr { .. }
1725 | InvalidSourceAndTargetRegion { .. } => StatusCode::InvalidArguments,
1726
1727 IncrementalQueryStale { .. } | SnapshotFenceStale { .. } => StatusCode::RequestOutdated,
1728
1729 SequenceRangeUnsupported { .. } => StatusCode::Unsupported,
1730
1731 RegionSequenceDomainBroken { .. }
1732 | RegionMetadataNotFound { .. }
1733 | Join { .. }
1734 | WorkerStopped { .. }
1735 | EngineStopped { .. }
1736 | Recv { .. }
1737 | DecodeWal { .. }
1738 | ComputeArrow { .. }
1739 | EvalPartitionFilter { .. }
1740 | MergeCandidateSeries { .. }
1741 | BiErrors { .. }
1742 | StopScheduler { .. }
1743 | ComputeVector { .. }
1744 | EncodeMemtable { .. }
1745 | CreateDir { .. }
1746 | ReadDataPart { .. }
1747 | BuildEntry { .. }
1748 | Metadata { .. }
1749 | CastColumn { .. }
1750 | MitoManifestInfo { .. }
1751 | ParquetToArrowSchema { .. } => StatusCode::Internal,
1752
1753 FetchManifests { source, .. } => source.status_code(),
1754
1755 OpenRegion { source, .. } => source.status_code(),
1756
1757 WriteParquet { .. } => StatusCode::StorageUnavailable,
1758 WriteGroup { source, .. }
1759 | LoadSstMeta { source, .. }
1760 | LoadParquetPage { source, .. }
1761 | LoadRangeResult { source, .. } => source.status_code(),
1762 InvalidBatch { .. } => StatusCode::InvalidArguments,
1763 InvalidRecordBatch { .. } => StatusCode::InvalidArguments,
1764 ConvertVector { source, .. } => source.status_code(),
1765
1766 PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
1767 InvalidSender { .. } => StatusCode::InvalidArguments,
1768 InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
1769 RegionRequirement { .. } => StatusCode::InvalidArguments,
1770 DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => {
1771 StatusCode::StorageUnavailable
1772 }
1773 FlushRegion { source, .. } | BuildIndexAsync { source, .. } => source.status_code(),
1774 IndexBuildIncomplete { .. } => StatusCode::IllegalState,
1775 RegionDropped { .. } => StatusCode::Cancelled,
1776 RegionClosed { .. } => StatusCode::Cancelled,
1777 StaleCompactionExecution { .. } => StatusCode::Cancelled,
1778 RegionTruncated { .. } => StatusCode::Cancelled,
1779 RejectWrite { .. } | RejectWriteInStaging { .. } => StatusCode::StorageUnavailable,
1780 CompactRegion { source, .. } => source.status_code(),
1781 EditRegion { source, .. } | EndStaging { source, .. } => source.status_code(),
1782 CompatReader { .. } => StatusCode::Unexpected,
1783 InvalidRegionRequest { source, .. } => source.status_code(),
1784 RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady,
1785 RegionLeaseExpired { .. } | UnacknowledgedWrite { .. } => StatusCode::RegionNotReady,
1786 RegionPendingTails { .. } => StatusCode::RegionNotReady,
1787 RemoteWalUnsupported { .. } => StatusCode::Unsupported,
1788 TailEntryMissing { .. } => StatusCode::StorageUnavailable,
1789 JsonOptions { .. } => StatusCode::InvalidArguments,
1790 EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
1791 ConvertValue { source, .. } => source.status_code(),
1792 ApplyBloomFilterIndex { source, .. } => source.status_code(),
1793 InvalidPartitionExpr { source, .. } => source.status_code(),
1794 BuildIndexApplier { source, .. }
1795 | PushIndexValue { source, .. }
1796 | ApplyInvertedIndex { source, .. }
1797 | IndexFinish { source, .. } => source.status_code(),
1798 PuffinReadBlob { source, .. }
1799 | PuffinAddBlob { source, .. }
1800 | PuffinInitStager { source, .. }
1801 | PuffinBuildReader { source, .. }
1802 | PuffinPurgeStager { source, .. } => source.status_code(),
1803 CleanDir { .. } => StatusCode::Unexpected,
1804 InvalidConfig { .. } => StatusCode::InvalidArguments,
1805 StaleLogEntry { .. }
1806 | InvalidNativeHistogramSubfield { .. }
1807 | InvalidNativeHistogramFieldId { .. } => StatusCode::Unexpected,
1808
1809 External { source, .. } => source.status_code(),
1810
1811 RecordBatch { source, .. } => source.status_code(),
1812
1813 Download { .. } | Upload { .. } => StatusCode::StorageUnavailable,
1814 ChecksumMismatch { .. } => StatusCode::Unexpected,
1815 RegionStopped { .. } => StatusCode::RegionNotReady,
1816 TimeRangePredicateOverflow { .. } => StatusCode::InvalidArguments,
1817 UnsupportedOperation { .. } => StatusCode::Unsupported,
1818 RemoteCompaction { .. } => StatusCode::Unexpected,
1819
1820 IndexOptions { source, .. } => source.status_code(),
1821 CompressionOptions { source, .. } => source.status_code(),
1822 CreateFulltextCreator { source, .. } => source.status_code(),
1823 CastVector { source, .. } => source.status_code(),
1824 FulltextPushText { source, .. }
1825 | FulltextFinish { source, .. }
1826 | ApplyFulltextIndex { source, .. } => source.status_code(),
1827 DecodeStats { .. }
1828 | StatsNotPresent { .. }
1829 | InvalidPrimaryKeyRange { .. }
1830 | DecodePrimaryKeyRange { .. } => StatusCode::Internal,
1831 RegionBusy { .. } | EndStagingPending { .. } => StatusCode::RegionBusy,
1832 GetSchemaMetadata { source, .. } => source.status_code(),
1833 Timeout { .. } => StatusCode::Cancelled,
1834
1835 DecodeArrowRowGroup { .. } => StatusCode::Internal,
1836
1837 PushBloomFilterValue { source, .. } | BloomFilterFinish { source, .. } => {
1838 source.status_code()
1839 }
1840
1841 ManualCompactionOverride {}
1842 | CompactionCancelled {}
1843 | FlushCancelled {}
1844 | SstWriteCancelled {} => StatusCode::Cancelled,
1845
1846 ManualCompactionAlreadyRunning { .. } => StatusCode::RegionBusy,
1850
1851 ScanMemoryExhausted { .. } => StatusCode::RuntimeResourcesExhausted,
1852
1853 CompactionMemoryExhausted { source, .. } => source.status_code(),
1854
1855 IncompatibleWalProviderChange { .. } => StatusCode::InvalidArguments,
1856
1857 ScanSeries { source, .. } => source.status_code(),
1858
1859 ScanMultiTimes { .. } => StatusCode::InvalidArguments,
1860 ConvertBulkWalEntry { source, .. } => source.status_code(),
1861
1862 Encode { source, .. } | Decode { source, .. } => source.status_code(),
1863
1864 #[cfg(feature = "enterprise")]
1865 ScanExternalRange { source, .. } => source.status_code(),
1866
1867 InconsistentTimestampLength { .. } => StatusCode::InvalidArguments,
1868
1869 TooManyFilesToRead { .. } | TooManyGcJobs { .. } => StatusCode::RateLimited,
1870
1871 PruneFile { source, .. } => source.status_code(),
1872
1873 FlushableRegionState { .. } => StatusCode::RegionNotReady,
1874 }
1875 }
1876
1877 fn as_any(&self) -> &dyn Any {
1878 self
1879 }
1880
1881 fn retry_hint(&self) -> RetryHint {
1882 use Error::*;
1883
1884 match self {
1885 PartitionExprVersionMismatch { .. }
1886 | RegionNotFound { .. }
1887 | RegionClosed { .. }
1888 | RejectWriteInStaging { .. }
1889 | RegionLeaseExpired { .. }
1890 | UnacknowledgedWrite { .. }
1891 | RegionState {
1892 state: RegionRoleState::Follower,
1893 ..
1894 } => RetryHint::RetryAfterRouteRefresh,
1895 ReadParquet { .. }
1896 | WriteParquet { .. }
1897 | RejectWrite { .. }
1898 | Download { .. }
1899 | Upload { .. }
1900 | RegionState { .. }
1901 | UpdateManifest { .. }
1902 | RegionStopped { .. }
1903 | RegionBusy { .. }
1904 | RegionPendingTails { .. }
1905 | EndStaging { .. }
1906 | EndStagingPending { .. }
1907 | ManualCompactionAlreadyRunning { .. }
1908 | FlushableRegionState { .. }
1909 | ManifestDeltaNotFound { .. }
1910 | ManifestVersionConflict { .. }
1911 | ManifestCheckpointVanished { .. } => RetryHint::Retryable,
1912
1913 OpenDal { error, .. }
1914 | DeleteSsts { error, .. }
1915 | DeleteIndex { error, .. }
1916 | DeleteIndexes { error, .. } => retry_hint_from_opendal_error(error),
1917
1918 WriteWal { source, .. }
1919 | ReadWal { source, .. }
1920 | ReadNodeWal { source, .. }
1921 | DeleteWal { source, .. }
1922 | WaitWalDurable { source, .. }
1923 | ResolveWalProvider { source, .. }
1924 | CheckWalHistory { source, .. }
1925 | FetchManifests { source, .. }
1926 | External { source, .. } => source.retry_hint(),
1927
1928 OpenRegion { source, .. }
1929 | LoadSstMeta { source, .. }
1930 | LoadParquetPage { source, .. }
1931 | LoadRangeResult { source, .. }
1932 | WriteGroup { source, .. }
1933 | FlushRegion { source, .. }
1934 | BuildIndexAsync { source, .. }
1935 | CompactRegion { source, .. }
1936 | EditRegion { source, .. }
1937 | ScanSeries { source, .. }
1938 | PruneFile { source, .. } => source.retry_hint(),
1939
1940 DataTypeMismatch { source, .. }
1941 | ConvertVector { source, .. }
1942 | ConvertValue { source, .. }
1943 | IndexOptions { source, .. }
1944 | CompressionOptions { source, .. }
1945 | CastVector { source, .. } => source.retry_hint(),
1946
1947 BuildIndexApplier { source, .. }
1948 | PushIndexValue { source, .. }
1949 | ApplyInvertedIndex { source, .. }
1950 | IndexFinish { source, .. } => source.retry_hint(),
1951
1952 ApplyBloomFilterIndex { source, .. }
1953 | PushBloomFilterValue { source, .. }
1954 | BloomFilterFinish { source, .. } => source.retry_hint(),
1955
1956 PuffinReadBlob { source, .. }
1957 | PuffinAddBlob { source, .. }
1958 | PuffinInitStager { source, .. }
1959 | PuffinBuildReader { source, .. }
1960 | PuffinPurgeStager { source, .. } => source.retry_hint(),
1961
1962 CreateFulltextCreator { source, .. }
1963 | FulltextPushText { source, .. }
1964 | FulltextFinish { source, .. }
1965 | ApplyFulltextIndex { source, .. } => source.retry_hint(),
1966
1967 InvalidRegionRequest { source, .. } => source.retry_hint(),
1968 InvalidPartitionExpr { source, .. } => source.retry_hint(),
1969 RecordBatch { source, .. } => source.retry_hint(),
1970 GetSchemaMetadata { source, .. } => source.retry_hint(),
1971 CompactionMemoryExhausted { source, .. } => source.retry_hint(),
1972 ConvertBulkWalEntry { source, .. } => source.retry_hint(),
1973 Encode { source, .. } | Decode { source, .. } => source.retry_hint(),
1974
1975 #[cfg(feature = "enterprise")]
1976 ScanExternalRange { source, .. } => source.retry_hint(),
1977
1978 _ => RetryHint::NonRetryable,
1979 }
1980 }
1981}
1982
1983#[cfg(test)]
1984mod tests {
1985 use snafu::IntoError;
1986
1987 use super::*;
1988 use crate::region::RegionLeaderState;
1989
1990 #[test]
1991 fn test_partition_version_mismatch_retry_hint() {
1992 let error = PartitionExprVersionMismatchSnafu {
1993 region_id: RegionId::new(1, 1),
1994 request_version: 1u64,
1995 expected_version: 2u64,
1996 }
1997 .build();
1998 assert_eq!(error.status_code(), StatusCode::InvalidArguments);
1999 assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
2000 }
2001
2002 #[test]
2003 fn test_region_state_retry_hint() {
2004 let region_id = RegionId::new(1, 1);
2005 let writable = RegionRoleState::Leader(RegionLeaderState::Writable);
2006 let follower_error = RegionStateSnafu {
2007 region_id,
2008 state: RegionRoleState::Follower,
2009 expect: writable,
2010 }
2011 .build();
2012 assert_eq!(
2013 follower_error.retry_hint(),
2014 RetryHint::RetryAfterRouteRefresh
2015 );
2016 assert_eq!(follower_error.status_code(), StatusCode::RegionNotReady);
2017 let leader_error = RegionStateSnafu {
2018 region_id,
2019 state: writable,
2020 expect: RegionRoleState::Follower,
2021 }
2022 .build();
2023 assert_eq!(leader_error.retry_hint(), RetryHint::Retryable);
2024 for (error, status, hint) in [
2025 (
2026 RejectWriteSnafu { region_id }.build(),
2027 StatusCode::StorageUnavailable,
2028 RetryHint::Retryable,
2029 ),
2030 (
2031 RejectWriteInStagingSnafu { region_id }.build(),
2032 StatusCode::StorageUnavailable,
2033 RetryHint::RetryAfterRouteRefresh,
2034 ),
2035 (
2036 RegionNotFoundSnafu { region_id }.build(),
2037 StatusCode::RegionNotFound,
2038 RetryHint::RetryAfterRouteRefresh,
2039 ),
2040 (
2041 RegionClosedSnafu { region_id }.build(),
2042 StatusCode::Cancelled,
2043 RetryHint::RetryAfterRouteRefresh,
2044 ),
2045 (
2046 RegionDroppedSnafu { region_id }.build(),
2047 StatusCode::Cancelled,
2048 RetryHint::NonRetryable,
2049 ),
2050 (
2051 RegionTruncatedSnafu { region_id }.build(),
2052 StatusCode::Cancelled,
2053 RetryHint::NonRetryable,
2054 ),
2055 ] {
2056 assert_eq!(error.status_code(), status);
2057 assert_eq!(error.retry_hint(), hint);
2058 }
2059 }
2060
2061 #[test]
2062 fn test_shared_sst_metadata_preserves_error_semantics() {
2063 for source in [
2064 OpenDalSnafu {}
2065 .into_error(object_store::Error::new(ErrorKind::NotFound, "missing SST")),
2066 OpenDalSnafu {}.into_error(object_store::Error::new(
2067 ErrorKind::Unexpected,
2068 "read failed",
2069 )),
2070 RejectWriteSnafu {
2071 region_id: RegionId::new(1, 1),
2072 }
2073 .build(),
2074 ] {
2075 let expected_not_found = source.is_object_not_found();
2076 let expected_status = source.status_code();
2077 let expected_retry = source.retry_hint();
2078 let source = Arc::new(source);
2079 for wrapped in [
2080 LoadSstMetaSnafu.into_error(source.clone()),
2081 LoadRangeResultSnafu.into_error(source),
2082 ] {
2083 assert_eq!(expected_not_found, wrapped.is_object_not_found());
2084 assert_eq!(expected_status, wrapped.status_code());
2085 assert_eq!(expected_retry, wrapped.retry_hint());
2086 }
2087 }
2088 }
2089
2090 #[test]
2091 fn test_manifest_update_persistence() {
2092 let rejected_kinds = [
2093 ErrorKind::Unsupported,
2094 ErrorKind::ConfigInvalid,
2095 ErrorKind::NotFound,
2096 ErrorKind::IsADirectory,
2097 ErrorKind::NotADirectory,
2098 ErrorKind::AlreadyExists,
2099 ErrorKind::IsSameFile,
2100 ErrorKind::ConditionNotMatch,
2101 ErrorKind::RangeNotSatisfied,
2102 ];
2103 for kind in rejected_kinds {
2104 let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
2105 assert!(
2106 !error.may_have_persisted_manifest_update(),
2107 "error kind {kind:?} should prove the manifest was not persisted"
2108 );
2109 }
2110
2111 for kind in [
2113 ErrorKind::Unexpected,
2114 ErrorKind::RateLimited,
2115 ErrorKind::PermissionDenied,
2116 ] {
2117 let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
2118 assert!(
2119 error.may_have_persisted_manifest_update(),
2120 "error kind {kind:?} should not prove the manifest was not persisted"
2121 );
2122 }
2123
2124 let region_id = RegionId::new(1, 1);
2125 let error = RegionTruncatedSnafu { region_id }.build();
2126 assert!(!error.may_have_persisted_manifest_update());
2127 let error = InvalidRequestSnafu {
2128 region_id,
2129 reason: "manifest applicability check rejected the edit",
2130 }
2131 .build();
2132 assert!(!error.may_have_persisted_manifest_update());
2133 let error = EndStagingPendingSnafu { region_id }.build();
2135 assert!(!error.may_have_persisted_manifest_update());
2136 assert!(error.retry_hint().is_retryable());
2137 assert_eq!(error.status_code(), StatusCode::RegionBusy);
2138 let error = RegionBusySnafu { region_id }.build();
2140 assert!(error.may_have_persisted_manifest_update());
2141
2142 let error = RegionLeaseExpiredSnafu { region_id }.build();
2144 assert!(!error.may_have_persisted_manifest_update());
2145
2146 let error = RegionMetadataNotFoundSnafu {}.build();
2148 assert!(error.may_have_persisted_manifest_update());
2149
2150 let error = ManifestVersionConflictSnafu {
2151 version: 1u64,
2152 manifest_dir: "manifest/",
2153 reason: "test",
2154 }
2155 .build();
2156 assert!(error.may_have_persisted_manifest_update());
2157 assert!(error.retry_hint().is_retryable());
2158 }
2159}