mito2/
error.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::sync::Arc;
17
18use common_datasource::compression::CompressionType;
19use common_error::ext::{BoxedError, ErrorExt, RetryHint};
20use common_error::status_code::StatusCode;
21use common_macro::stack_trace_debug;
22use common_memory_manager;
23use common_runtime::JoinError;
24use common_time::Timestamp;
25use common_time::timestamp::TimeUnit;
26use datatypes::arrow::error::ArrowError;
27use datatypes::prelude::ConcreteDataType;
28use object_store::ErrorKind;
29use object_store::error::retry_hint_from_opendal_error;
30use partition::error::Error as PartitionError;
31use prost::DecodeError;
32use snafu::{Location, Snafu};
33use store_api::ManifestVersion;
34use store_api::logstore::provider::Provider;
35use store_api::storage::{FileId, RegionId};
36use tokio::time::error::Elapsed;
37
38use crate::cache::file_cache::FileType;
39use crate::region::RegionRoleState;
40use crate::schedule::remote_job_scheduler::JobId;
41use crate::worker::WorkerId;
42
43#[derive(Snafu)]
44#[snafu(visibility(pub))]
45#[stack_trace_debug]
46pub enum Error {
47    #[snafu(display("Unexpected data type"))]
48    DataTypeMismatch {
49        source: datatypes::error::Error,
50        #[snafu(implicit)]
51        location: Location,
52    },
53
54    #[snafu(display("External error, context: {}", context))]
55    External {
56        source: BoxedError,
57        context: String,
58        #[snafu(implicit)]
59        location: Location,
60    },
61
62    #[snafu(display("OpenDAL operator failed"))]
63    OpenDal {
64        #[snafu(implicit)]
65        location: Location,
66        #[snafu(source)]
67        error: object_store::Error,
68    },
69
70    #[snafu(display(
71        "Manifest delta {} disappeared after it was listed, path: {}",
72        version,
73        path
74    ))]
75    ManifestDeltaNotFound {
76        version: ManifestVersion,
77        path: String,
78        #[snafu(source)]
79        error: object_store::Error,
80        #[snafu(implicit)]
81        location: Location,
82    },
83
84    #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
85    CompressObject {
86        compress_type: CompressionType,
87        path: String,
88        #[snafu(source)]
89        error: std::io::Error,
90    },
91
92    #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
93    DecompressObject {
94        compress_type: CompressionType,
95        path: String,
96        #[snafu(source)]
97        error: std::io::Error,
98    },
99
100    #[snafu(display("Failed to ser/de json object"))]
101    SerdeJson {
102        #[snafu(implicit)]
103        location: Location,
104        #[snafu(source)]
105        error: serde_json::Error,
106    },
107
108    #[snafu(display("Failed to serialize column metadata"))]
109    SerializeColumnMetadata {
110        #[snafu(source)]
111        error: serde_json::Error,
112        #[snafu(implicit)]
113        location: Location,
114    },
115
116    #[snafu(display("Failed to serialize manifest, region_id: {}", region_id))]
117    SerializeManifest {
118        region_id: RegionId,
119        #[snafu(source)]
120        error: serde_json::Error,
121        #[snafu(implicit)]
122        location: Location,
123    },
124
125    #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
126    InvalidScanIndex {
127        start: ManifestVersion,
128        end: ManifestVersion,
129        #[snafu(implicit)]
130        location: Location,
131    },
132
133    #[snafu(display("Invalid UTF-8 content"))]
134    Utf8 {
135        #[snafu(implicit)]
136        location: Location,
137        #[snafu(source)]
138        error: std::str::Utf8Error,
139    },
140
141    #[snafu(display("Cannot find RegionMetadata"))]
142    RegionMetadataNotFound {
143        #[snafu(implicit)]
144        location: Location,
145    },
146
147    #[snafu(display("Failed to join handle"))]
148    Join {
149        #[snafu(source)]
150        error: common_runtime::JoinError,
151        #[snafu(implicit)]
152        location: Location,
153    },
154
155    #[snafu(display("Worker {} is stopped", id))]
156    WorkerStopped {
157        id: WorkerId,
158        #[snafu(implicit)]
159        location: Location,
160    },
161
162    #[snafu(display("Failed to recv result"))]
163    Recv {
164        #[snafu(source)]
165        error: tokio::sync::oneshot::error::RecvError,
166        #[snafu(implicit)]
167        location: Location,
168    },
169
170    #[snafu(display("Invalid metadata, {}", reason))]
171    InvalidMeta {
172        reason: String,
173        #[snafu(implicit)]
174        location: Location,
175    },
176
177    #[snafu(display("Invalid region metadata"))]
178    InvalidMetadata {
179        source: store_api::metadata::MetadataError,
180        #[snafu(implicit)]
181        location: Location,
182    },
183
184    #[snafu(display("Failed to create RecordBatch from vectors"))]
185    NewRecordBatch {
186        #[snafu(implicit)]
187        location: Location,
188        #[snafu(source)]
189        error: ArrowError,
190    },
191
192    #[snafu(display("Failed to read parquet file, path: {}", path))]
193    ReadParquet {
194        path: String,
195        #[snafu(source)]
196        error: parquet::errors::ParquetError,
197        #[snafu(implicit)]
198        location: Location,
199    },
200
201    #[snafu(display("Failed to write parquet file"))]
202    WriteParquet {
203        #[snafu(source)]
204        error: parquet::errors::ParquetError,
205        #[snafu(implicit)]
206        location: Location,
207    },
208
209    #[snafu(display(
210        "Cannot assign a stable field id to native histogram sub-field '{}' of column id {} (unknown sub-field name or derived id overflows i32)",
211        field_name,
212        column_id
213    ))]
214    InvalidNativeHistogramSubfield {
215        column_id: i32,
216        field_name: String,
217        #[snafu(implicit)]
218        location: Location,
219    },
220
221    #[snafu(display(
222        "Native histogram column '{}' has no usable PARQUET:field_id to namespace its sub-field ids (missing, malformed, or exceeds i32::MAX)",
223        field_name
224    ))]
225    InvalidNativeHistogramFieldId {
226        field_name: String,
227        #[snafu(implicit)]
228        location: Location,
229    },
230
231    #[snafu(display("Region {} not found", region_id))]
232    RegionNotFound {
233        region_id: RegionId,
234        #[snafu(implicit)]
235        location: Location,
236    },
237
238    #[snafu(display("Object store not found: {}", object_store))]
239    ObjectStoreNotFound {
240        object_store: String,
241        #[snafu(implicit)]
242        location: Location,
243    },
244
245    #[snafu(display("Region {} is corrupted, reason: {}", region_id, reason))]
246    RegionCorrupted {
247        region_id: RegionId,
248        reason: String,
249        #[snafu(implicit)]
250        location: Location,
251    },
252
253    #[snafu(display("Invalid request to region {}, reason: {}", region_id, reason))]
254    InvalidRequest {
255        region_id: RegionId,
256        reason: String,
257        #[snafu(implicit)]
258        location: Location,
259    },
260
261    #[snafu(display(
262        "STALE_CURSOR: incremental query stale, region: {}, given_seq: {}, min_readable_seq: {}, retry_hint: FALLBACK_FULL_RECOMPUTE",
263        region_id,
264        given_seq,
265        min_readable_seq
266    ))]
267    IncrementalQueryStale {
268        region_id: RegionId,
269        given_seq: u64,
270        min_readable_seq: u64,
271        #[snafu(implicit)]
272        location: Location,
273    },
274
275    #[snafu(display(
276        "STALE_SNAPSHOT_FENCE: snapshot upper bound stale, region: {}, given_seq: {}, min_enforceable_seq: {}, retry_hint: REBIND_SNAPSHOT_FENCE",
277        region_id,
278        given_seq,
279        min_enforceable_seq
280    ))]
281    SnapshotFenceStale {
282        region_id: RegionId,
283        given_seq: u64,
284        min_enforceable_seq: u64,
285        #[snafu(implicit)]
286        location: Location,
287    },
288
289    #[snafu(display(
290        "SEQUENCE_RANGE_UNSUPPORTED: exact sequence-range read unsupported, region: {}, min_seq: {}, max_seq: {}, reason: {}, retry_hint: FALLBACK_MEMTABLE_ONLY_OR_FULL_RECOMPUTE",
291        region_id,
292        min_seq,
293        max_seq,
294        reason
295    ))]
296    SequenceRangeUnsupported {
297        region_id: RegionId,
298        min_seq: u64,
299        max_seq: u64,
300        reason: String,
301        #[snafu(implicit)]
302        location: Location,
303    },
304
305    #[snafu(display(
306        "region {} is unusable for sequence reads: file {} declares region {}",
307        region_id,
308        file_id,
309        file_region_id
310    ))]
311    RegionSequenceDomainBroken {
312        region_id: RegionId,
313        file_region_id: RegionId,
314        file_id: FileId,
315        #[snafu(implicit)]
316        location: Location,
317    },
318
319    #[snafu(display("Old manifest missing for region {}", region_id))]
320    MissingOldManifest {
321        region_id: RegionId,
322        #[snafu(implicit)]
323        location: Location,
324    },
325
326    #[snafu(display("New manifest missing for region {}", region_id))]
327    MissingNewManifest {
328        region_id: RegionId,
329        #[snafu(implicit)]
330        location: Location,
331    },
332
333    #[snafu(display("Manifest missing for region {}", region_id))]
334    MissingManifest {
335        region_id: RegionId,
336        #[snafu(implicit)]
337        location: Location,
338    },
339
340    #[snafu(display("File consistency check failed for file {}: {}", file_id, reason))]
341    InconsistentFile {
342        file_id: FileId,
343        reason: String,
344        #[snafu(implicit)]
345        location: Location,
346    },
347
348    #[snafu(display("Files lost during remapping: old={}, new={}", old_count, new_count))]
349    FilesLost {
350        old_count: usize,
351        new_count: usize,
352        #[snafu(implicit)]
353        location: Location,
354    },
355
356    #[snafu(display("No old manifests provided (need at least one for template)"))]
357    NoOldManifests {
358        #[snafu(implicit)]
359        location: Location,
360    },
361
362    #[snafu(display("Failed to fetch manifests"))]
363    FetchManifests {
364        #[snafu(implicit)]
365        location: Location,
366        source: BoxedError,
367    },
368
369    #[snafu(display("Partition expression missing for region {}", region_id))]
370    MissingPartitionExpr {
371        region_id: RegionId,
372        #[snafu(implicit)]
373        location: Location,
374    },
375
376    #[snafu(display("Failed to serialize partition expression: {}", source))]
377    SerializePartitionExpr {
378        #[snafu(source)]
379        source: PartitionError,
380        #[snafu(implicit)]
381        location: Location,
382    },
383
384    #[snafu(display(
385        "Failed to convert ConcreteDataType to ColumnDataType, reason: {}",
386        reason
387    ))]
388    ConvertColumnDataType {
389        reason: String,
390        source: api::error::Error,
391        #[snafu(implicit)]
392        location: Location,
393    },
394
395    /// An error type to indicate that schema is changed and we need
396    /// to fill default values again.
397    #[snafu(display("Need to fill default value for region {}", region_id))]
398    FillDefault {
399        region_id: RegionId,
400        // The error is for internal use so we don't need a location.
401    },
402
403    #[snafu(display(
404        "Failed to create default value for column {} of region {}",
405        column,
406        region_id
407    ))]
408    CreateDefault {
409        region_id: RegionId,
410        column: String,
411        source: datatypes::Error,
412        #[snafu(implicit)]
413        location: Location,
414    },
415
416    #[snafu(display("Failed to build entry, region_id: {}", region_id))]
417    BuildEntry {
418        region_id: RegionId,
419        #[snafu(implicit)]
420        location: Location,
421        source: BoxedError,
422    },
423
424    #[snafu(display("Failed to write WAL"))]
425    WriteWal {
426        #[snafu(implicit)]
427        location: Location,
428        source: BoxedError,
429    },
430
431    #[snafu(display("Failed to read WAL, provider: {}", provider))]
432    ReadWal {
433        provider: Provider,
434        #[snafu(implicit)]
435        location: Location,
436        source: BoxedError,
437    },
438
439    #[snafu(display("Failed to decode WAL entry, region_id: {}", region_id))]
440    DecodeWal {
441        region_id: RegionId,
442        #[snafu(implicit)]
443        location: Location,
444        #[snafu(source)]
445        error: DecodeError,
446    },
447
448    #[snafu(display("Failed to delete WAL, region_id: {}", region_id))]
449    DeleteWal {
450        region_id: RegionId,
451        #[snafu(implicit)]
452        location: Location,
453        source: BoxedError,
454    },
455
456    #[snafu(display("Failed to wait for the WAL to be durable, region_id: {}", region_id))]
457    WaitWalDurable {
458        region_id: RegionId,
459        #[snafu(implicit)]
460        location: Location,
461        source: BoxedError,
462    },
463
464    #[snafu(display("Failed to check the WAL history, region_id: {}", region_id))]
465    CheckWalHistory {
466        region_id: RegionId,
467        #[snafu(implicit)]
468        location: Location,
469        source: BoxedError,
470    },
471
472    // Shared error for each writer in the write group.
473    #[snafu(display("Failed to write region"))]
474    WriteGroup { source: Arc<Error> },
475
476    #[snafu(display("Invalid parquet SST file {}, reason: {}", file, reason))]
477    InvalidParquet {
478        file: String,
479        reason: String,
480        #[snafu(implicit)]
481        location: Location,
482    },
483
484    #[snafu(display("Invalid batch, {}", reason))]
485    InvalidBatch {
486        reason: String,
487        #[snafu(implicit)]
488        location: Location,
489    },
490
491    #[snafu(display("Invalid arrow record batch, {}", reason))]
492    InvalidRecordBatch {
493        reason: String,
494        #[snafu(implicit)]
495        location: Location,
496    },
497
498    #[snafu(display("Invalid wal read request, {}", reason))]
499    InvalidWalReadRequest {
500        reason: String,
501        #[snafu(implicit)]
502        location: Location,
503    },
504
505    #[snafu(display("Failed to convert array to vector"))]
506    ConvertVector {
507        #[snafu(implicit)]
508        location: Location,
509        source: datatypes::error::Error,
510    },
511
512    #[snafu(display("Failed to compute arrow arrays"))]
513    ComputeArrow {
514        #[snafu(implicit)]
515        location: Location,
516        #[snafu(source)]
517        error: datatypes::arrow::error::ArrowError,
518    },
519
520    #[snafu(display("Failed to evaluate partition filter"))]
521    EvalPartitionFilter {
522        #[snafu(implicit)]
523        location: Location,
524        #[snafu(source)]
525        error: datafusion::error::DataFusionError,
526    },
527
528    #[snafu(display("Failed to merge candidate series"))]
529    MergeCandidateSeries {
530        #[snafu(implicit)]
531        location: Location,
532        #[snafu(source)]
533        error: datafusion::error::DataFusionError,
534    },
535
536    #[snafu(display("Failed to compute vector"))]
537    ComputeVector {
538        #[snafu(implicit)]
539        location: Location,
540        source: datatypes::error::Error,
541    },
542
543    #[snafu(display("Primary key length mismatch, expect: {}, actual: {}", expect, actual))]
544    PrimaryKeyLengthMismatch {
545        expect: usize,
546        actual: usize,
547        #[snafu(implicit)]
548        location: Location,
549    },
550
551    #[snafu(display("Invalid sender",))]
552    InvalidSender {
553        #[snafu(implicit)]
554        location: Location,
555    },
556
557    #[snafu(display("Invalid scheduler state"))]
558    InvalidSchedulerState {
559        #[snafu(implicit)]
560        location: Location,
561    },
562
563    #[snafu(display("Failed to stop scheduler"))]
564    StopScheduler {
565        #[snafu(source)]
566        error: JoinError,
567        #[snafu(implicit)]
568        location: Location,
569    },
570
571    #[snafu(display(
572        "Failed to batch delete SST files, region id: {}, file ids: {:?}",
573        region_id,
574        file_ids
575    ))]
576    DeleteSsts {
577        region_id: RegionId,
578        file_ids: Vec<FileId>,
579        #[snafu(source)]
580        error: object_store::Error,
581        #[snafu(implicit)]
582        location: Location,
583    },
584
585    #[snafu(display("Failed to delete index file, file id: {}", file_id))]
586    DeleteIndex {
587        file_id: FileId,
588        #[snafu(source)]
589        error: object_store::Error,
590        #[snafu(implicit)]
591        location: Location,
592    },
593
594    #[snafu(display("Failed to batch delete index files, file ids: {:?}", file_ids))]
595    DeleteIndexes {
596        file_ids: Vec<FileId>,
597        #[snafu(source)]
598        error: object_store::Error,
599        #[snafu(implicit)]
600        location: Location,
601    },
602
603    #[snafu(display("Failed to flush region {}", region_id))]
604    FlushRegion {
605        region_id: RegionId,
606        source: Arc<Error>,
607        #[snafu(implicit)]
608        location: Location,
609    },
610
611    #[snafu(display("Region {} is dropped", region_id))]
612    RegionDropped {
613        region_id: RegionId,
614        #[snafu(implicit)]
615        location: Location,
616    },
617
618    #[snafu(display("Region {} is closed", region_id))]
619    RegionClosed {
620        region_id: RegionId,
621        #[snafu(implicit)]
622        location: Location,
623    },
624
625    #[snafu(display(
626        "Stale compaction execution for region {}, the region may have been reopened, truncated or the compaction was superseded",
627        region_id
628    ))]
629    StaleCompactionExecution {
630        region_id: RegionId,
631        #[snafu(implicit)]
632        location: Location,
633    },
634
635    #[snafu(display("Region {} is truncated", region_id))]
636    RegionTruncated {
637        region_id: RegionId,
638        #[snafu(implicit)]
639        location: Location,
640    },
641
642    #[snafu(display(
643        "Engine write buffer is full, rejecting write requests of region {}",
644        region_id,
645    ))]
646    RejectWrite {
647        region_id: RegionId,
648        #[snafu(implicit)]
649        location: Location,
650    },
651
652    #[snafu(display("Region {} rejects writes while in staging", region_id))]
653    RejectWriteInStaging {
654        region_id: RegionId,
655        #[snafu(implicit)]
656        location: Location,
657    },
658
659    #[snafu(display("Failed to compact region {}", region_id))]
660    CompactRegion {
661        region_id: RegionId,
662        source: Arc<Error>,
663        #[snafu(implicit)]
664        location: Location,
665    },
666
667    #[snafu(display("Failed to edit region {}", region_id))]
668    EditRegion {
669        region_id: RegionId,
670        source: Arc<Error>,
671        #[snafu(implicit)]
672        location: Location,
673    },
674
675    #[snafu(display(
676        "Failed to compat readers for region {}, reason: {}",
677        region_id,
678        reason,
679    ))]
680    CompatReader {
681        region_id: RegionId,
682        reason: String,
683        #[snafu(implicit)]
684        location: Location,
685    },
686
687    #[snafu(display("Invalue region req"))]
688    InvalidRegionRequest {
689        source: store_api::metadata::MetadataError,
690        #[snafu(implicit)]
691        location: Location,
692    },
693
694    #[snafu(display(
695        "Region {} is in {:?} state, which does not permit manifest updates.",
696        region_id,
697        state
698    ))]
699    UpdateManifest {
700        region_id: RegionId,
701        state: RegionRoleState,
702        #[snafu(implicit)]
703        location: Location,
704    },
705
706    #[snafu(display("Region {} is in {:?} state, expect: {:?}", region_id, state, expect))]
707    RegionState {
708        region_id: RegionId,
709        state: RegionRoleState,
710        expect: RegionRoleState,
711        #[snafu(implicit)]
712        location: Location,
713    },
714
715    #[snafu(display(
716        "Partition expr version mismatch for region {}: request {}, expected {}",
717        region_id,
718        request_version,
719        expected_version
720    ))]
721    PartitionExprVersionMismatch {
722        region_id: RegionId,
723        request_version: u64,
724        expected_version: u64,
725        #[snafu(implicit)]
726        location: Location,
727    },
728
729    #[snafu(display("Invalid options"))]
730    JsonOptions {
731        #[snafu(source)]
732        error: serde_json::Error,
733        #[snafu(implicit)]
734        location: Location,
735    },
736
737    #[snafu(display(
738        "Empty region directory, region_id: {}, region_dir: {}",
739        region_id,
740        region_dir,
741    ))]
742    EmptyRegionDir {
743        region_id: RegionId,
744        region_dir: String,
745        #[snafu(implicit)]
746        location: Location,
747    },
748
749    #[snafu(display("Empty manifest directory, manifest_dir: {}", manifest_dir,))]
750    EmptyManifestDir {
751        manifest_dir: String,
752        #[snafu(implicit)]
753        location: Location,
754    },
755
756    #[snafu(display("Column not found, column: {column}"))]
757    ColumnNotFound {
758        column: String,
759        #[snafu(implicit)]
760        location: Location,
761    },
762
763    #[snafu(display("Failed to build index applier"))]
764    BuildIndexApplier {
765        source: index::inverted_index::error::Error,
766        #[snafu(implicit)]
767        location: Location,
768    },
769
770    #[snafu(display("Failed to build index asynchronously in region {}", region_id))]
771    BuildIndexAsync {
772        region_id: RegionId,
773        source: Arc<Error>,
774        #[snafu(implicit)]
775        location: Location,
776    },
777
778    #[snafu(display(
779        "Index build did not complete for files {:?} in region {}",
780        file_ids,
781        region_id
782    ))]
783    IndexBuildIncomplete {
784        region_id: RegionId,
785        file_ids: Vec<FileId>,
786        #[snafu(implicit)]
787        location: Location,
788    },
789
790    #[snafu(display("Failed to convert value"))]
791    ConvertValue {
792        source: datatypes::error::Error,
793        #[snafu(implicit)]
794        location: Location,
795    },
796
797    #[snafu(display("Failed to apply inverted index"))]
798    ApplyInvertedIndex {
799        source: index::inverted_index::error::Error,
800        #[snafu(implicit)]
801        location: Location,
802    },
803
804    #[snafu(display("Failed to apply bloom filter index"))]
805    ApplyBloomFilterIndex {
806        source: index::bloom_filter::error::Error,
807        #[snafu(implicit)]
808        location: Location,
809    },
810
811    #[snafu(display("Failed to push index value"))]
812    PushIndexValue {
813        source: index::inverted_index::error::Error,
814        #[snafu(implicit)]
815        location: Location,
816    },
817
818    #[snafu(display("Failed to write index completely"))]
819    IndexFinish {
820        source: index::inverted_index::error::Error,
821        #[snafu(implicit)]
822        location: Location,
823    },
824
825    #[snafu(display("Operate on aborted index"))]
826    OperateAbortedIndex {
827        #[snafu(implicit)]
828        location: Location,
829    },
830
831    #[snafu(display("Failed to read puffin blob"))]
832    PuffinReadBlob {
833        source: puffin::error::Error,
834        #[snafu(implicit)]
835        location: Location,
836    },
837
838    #[snafu(display("Failed to add blob to puffin file"))]
839    PuffinAddBlob {
840        source: puffin::error::Error,
841        #[snafu(implicit)]
842        location: Location,
843    },
844
845    #[snafu(display("Failed to clean dir {dir}"))]
846    CleanDir {
847        dir: String,
848        #[snafu(source)]
849        error: std::io::Error,
850        #[snafu(implicit)]
851        location: Location,
852    },
853
854    #[snafu(display("Invalid config, {reason}"))]
855    InvalidConfig {
856        reason: String,
857        #[snafu(implicit)]
858        location: Location,
859    },
860
861    #[snafu(display(
862        "Stale log entry found during replay, region: {}, flushed: {}, replayed: {}",
863        region_id,
864        flushed_entry_id,
865        unexpected_entry_id
866    ))]
867    StaleLogEntry {
868        region_id: RegionId,
869        flushed_entry_id: u64,
870        unexpected_entry_id: u64,
871    },
872
873    #[snafu(display(
874        "Failed to download file, region_id: {}, file_id: {}, file_type: {:?}",
875        region_id,
876        file_id,
877        file_type,
878    ))]
879    Download {
880        region_id: RegionId,
881        file_id: FileId,
882        file_type: FileType,
883        #[snafu(source)]
884        error: std::io::Error,
885        #[snafu(implicit)]
886        location: Location,
887    },
888
889    #[snafu(display(
890        "Failed to upload file, region_id: {}, file_id: {}, file_type: {:?}",
891        region_id,
892        file_id,
893        file_type,
894    ))]
895    Upload {
896        region_id: RegionId,
897        file_id: FileId,
898        file_type: FileType,
899        #[snafu(source)]
900        error: std::io::Error,
901        #[snafu(implicit)]
902        location: Location,
903    },
904
905    #[snafu(display("Failed to create directory {}", dir))]
906    CreateDir {
907        dir: String,
908        #[snafu(source)]
909        error: std::io::Error,
910    },
911
912    #[snafu(display("Record batch error"))]
913    RecordBatch {
914        source: common_recordbatch::error::Error,
915        #[snafu(implicit)]
916        location: Location,
917    },
918
919    #[snafu(display("BiErrors, first: {first}, second: {second}"))]
920    BiErrors {
921        first: Box<Error>,
922        second: Box<Error>,
923        #[snafu(implicit)]
924        location: Location,
925    },
926
927    #[snafu(display("Encode null value"))]
928    IndexEncodeNull {
929        #[snafu(implicit)]
930        location: Location,
931    },
932
933    #[snafu(display("Failed to encode memtable to Parquet bytes"))]
934    EncodeMemtable {
935        #[snafu(source)]
936        error: parquet::errors::ParquetError,
937        #[snafu(implicit)]
938        location: Location,
939    },
940
941    #[snafu(display("Partition {} out of range, {} in total", given, all))]
942    PartitionOutOfRange {
943        given: usize,
944        all: usize,
945        #[snafu(implicit)]
946        location: Location,
947    },
948
949    #[snafu(display("Failed to iter data part"))]
950    ReadDataPart {
951        #[snafu(implicit)]
952        location: Location,
953        #[snafu(source)]
954        error: parquet::errors::ParquetError,
955    },
956
957    #[snafu(display("Failed to read row group in memtable"))]
958    DecodeArrowRowGroup {
959        #[snafu(source)]
960        error: ArrowError,
961        #[snafu(implicit)]
962        location: Location,
963    },
964
965    #[snafu(display("Invalid region options, {}", reason))]
966    InvalidRegionOptions {
967        reason: String,
968        #[snafu(implicit)]
969        location: Location,
970    },
971
972    #[snafu(display("checksum mismatch (actual: {}, expected: {})", actual, expected))]
973    ChecksumMismatch { actual: u32, expected: u32 },
974
975    #[snafu(display(
976        "No checkpoint found, region: {}, last_version: {}",
977        region_id,
978        last_version
979    ))]
980    NoCheckpoint {
981        region_id: RegionId,
982        last_version: ManifestVersion,
983        #[snafu(implicit)]
984        location: Location,
985    },
986
987    #[snafu(display(
988        "No manifests found in range: [{}..{}), region: {}, last_version: {}",
989        start_version,
990        end_version,
991        region_id,
992        last_version
993    ))]
994    NoManifests {
995        region_id: RegionId,
996        start_version: ManifestVersion,
997        end_version: ManifestVersion,
998        last_version: ManifestVersion,
999        #[snafu(implicit)]
1000        location: Location,
1001    },
1002
1003    #[snafu(display(
1004        "Failed to install manifest to {}, region: {}, available manifest version: {}, last version: {}",
1005        target_version,
1006        region_id,
1007        available_version,
1008        last_version
1009    ))]
1010    InstallManifestTo {
1011        region_id: RegionId,
1012        target_version: ManifestVersion,
1013        available_version: ManifestVersion,
1014        #[snafu(implicit)]
1015        location: Location,
1016        last_version: ManifestVersion,
1017    },
1018
1019    #[snafu(display("Region {} is stopped", region_id))]
1020    RegionStopped {
1021        region_id: RegionId,
1022        #[snafu(implicit)]
1023        location: Location,
1024    },
1025
1026    #[snafu(display(
1027        "Time range predicate overflows, timestamp: {:?}, target unit: {}",
1028        timestamp,
1029        unit
1030    ))]
1031    TimeRangePredicateOverflow {
1032        timestamp: Timestamp,
1033        unit: TimeUnit,
1034        #[snafu(implicit)]
1035        location: Location,
1036    },
1037
1038    #[snafu(display("Failed to open region"))]
1039    OpenRegion {
1040        #[snafu(implicit)]
1041        location: Location,
1042        source: Arc<Error>,
1043    },
1044
1045    #[snafu(display(
1046        "Region {} does not satisfy requirement '{}': {}",
1047        region_id,
1048        requirement,
1049        reason
1050    ))]
1051    RegionRequirement {
1052        region_id: RegionId,
1053        requirement: &'static str,
1054        reason: &'static str,
1055        #[snafu(implicit)]
1056        location: Location,
1057    },
1058
1059    #[snafu(display("Failed to parse job id"))]
1060    ParseJobId {
1061        #[snafu(implicit)]
1062        location: Location,
1063        #[snafu(source)]
1064        error: uuid::Error,
1065    },
1066
1067    #[snafu(display("Operation is not supported: {}", err_msg))]
1068    UnsupportedOperation {
1069        err_msg: String,
1070        #[snafu(implicit)]
1071        location: Location,
1072    },
1073
1074    #[snafu(display(
1075        "Failed to remotely compact region {} by job {:?} due to {}",
1076        region_id,
1077        job_id,
1078        reason
1079    ))]
1080    RemoteCompaction {
1081        region_id: RegionId,
1082        job_id: Option<JobId>,
1083        reason: String,
1084        #[snafu(implicit)]
1085        location: Location,
1086    },
1087
1088    #[snafu(display("Failed to initialize puffin stager"))]
1089    PuffinInitStager {
1090        source: puffin::error::Error,
1091        #[snafu(implicit)]
1092        location: Location,
1093    },
1094
1095    #[snafu(display("Failed to purge puffin stager"))]
1096    PuffinPurgeStager {
1097        source: puffin::error::Error,
1098        #[snafu(implicit)]
1099        location: Location,
1100    },
1101
1102    #[snafu(display("Failed to build puffin reader"))]
1103    PuffinBuildReader {
1104        source: puffin::error::Error,
1105        #[snafu(implicit)]
1106        location: Location,
1107    },
1108
1109    #[snafu(display("Failed to retrieve index options from column metadata"))]
1110    IndexOptions {
1111        #[snafu(implicit)]
1112        location: Location,
1113        source: datatypes::error::Error,
1114        column_name: String,
1115    },
1116
1117    #[snafu(display("Failed to retrieve compression options of column {column_name}"))]
1118    CompressionOptions {
1119        #[snafu(implicit)]
1120        location: Location,
1121        source: datatypes::error::Error,
1122        column_name: String,
1123    },
1124
1125    #[snafu(display("Failed to create fulltext index creator"))]
1126    CreateFulltextCreator {
1127        source: index::fulltext_index::error::Error,
1128        #[snafu(implicit)]
1129        location: Location,
1130    },
1131
1132    #[snafu(display("Failed to cast vector of {from} to {to}"))]
1133    CastVector {
1134        #[snafu(implicit)]
1135        location: Location,
1136        from: ConcreteDataType,
1137        to: ConcreteDataType,
1138        source: datatypes::error::Error,
1139    },
1140
1141    #[snafu(display("Failed to push text to fulltext index"))]
1142    FulltextPushText {
1143        source: index::fulltext_index::error::Error,
1144        #[snafu(implicit)]
1145        location: Location,
1146    },
1147
1148    #[snafu(display("Failed to finalize fulltext index creator"))]
1149    FulltextFinish {
1150        source: index::fulltext_index::error::Error,
1151        #[snafu(implicit)]
1152        location: Location,
1153    },
1154
1155    #[snafu(display("Failed to apply fulltext index"))]
1156    ApplyFulltextIndex {
1157        source: index::fulltext_index::error::Error,
1158        #[snafu(implicit)]
1159        location: Location,
1160    },
1161
1162    #[snafu(display("SST file {} does not contain valid stats info", file_path))]
1163    StatsNotPresent {
1164        file_path: String,
1165        #[snafu(implicit)]
1166        location: Location,
1167    },
1168
1169    #[snafu(display("Failed to decode stats of file {}", file_path))]
1170    DecodeStats {
1171        file_path: String,
1172        #[snafu(implicit)]
1173        location: Location,
1174    },
1175
1176    #[snafu(display("Invalid SST primary key range: {reason}"))]
1177    InvalidPrimaryKeyRange {
1178        reason: String,
1179        #[snafu(implicit)]
1180        location: Location,
1181    },
1182
1183    #[snafu(display("Failed to decode SST primary key range {endpoint} endpoint"))]
1184    DecodePrimaryKeyRange {
1185        endpoint: &'static str,
1186        source: mito_codec::error::Error,
1187        #[snafu(implicit)]
1188        location: Location,
1189    },
1190
1191    #[snafu(display("Region {} is busy", region_id))]
1192    RegionBusy {
1193        region_id: RegionId,
1194        #[snafu(implicit)]
1195        location: Location,
1196    },
1197
1198    #[snafu(display("Failed to get schema metadata"))]
1199    GetSchemaMetadata {
1200        source: common_meta::error::Error,
1201        #[snafu(implicit)]
1202        location: Location,
1203    },
1204
1205    #[snafu(display("Timeout"))]
1206    Timeout {
1207        #[snafu(source)]
1208        error: Elapsed,
1209        #[snafu(implicit)]
1210        location: Location,
1211    },
1212
1213    #[snafu(display("Failed to read file metadata"))]
1214    Metadata {
1215        #[snafu(source)]
1216        error: std::io::Error,
1217        #[snafu(implicit)]
1218        location: Location,
1219    },
1220
1221    #[snafu(display("Failed to push value to bloom filter"))]
1222    PushBloomFilterValue {
1223        source: index::bloom_filter::error::Error,
1224        #[snafu(implicit)]
1225        location: Location,
1226    },
1227
1228    #[snafu(display("Failed to finish bloom filter"))]
1229    BloomFilterFinish {
1230        source: index::bloom_filter::error::Error,
1231        #[snafu(implicit)]
1232        location: Location,
1233    },
1234
1235    #[snafu(display("Manual compaction is override by following operations."))]
1236    ManualCompactionOverride {},
1237
1238    #[snafu(display("Manual compaction is already running for region {region_id}."))]
1239    ManualCompactionAlreadyRunning { region_id: RegionId },
1240
1241    #[snafu(display("Compaction is cancelled."))]
1242    CompactionCancelled {},
1243
1244    #[snafu(display("Flush is cancelled."))]
1245    FlushCancelled {},
1246
1247    #[snafu(display("Compaction memory exhausted for region {region_id} (policy: {policy})",))]
1248    CompactionMemoryExhausted {
1249        region_id: RegionId,
1250        policy: String,
1251        #[snafu(source)]
1252        source: common_memory_manager::Error,
1253        #[snafu(implicit)]
1254        location: Location,
1255    },
1256
1257    #[snafu(display(
1258        "Incompatible WAL provider change. This is typically caused by changing WAL provider in database config file without completely cleaning existing files. Global provider: {}, region provider: {}",
1259        global,
1260        region
1261    ))]
1262    IncompatibleWalProviderChange { global: String, region: String },
1263
1264    #[snafu(display("Expected mito manifest info"))]
1265    MitoManifestInfo {
1266        #[snafu(implicit)]
1267        location: Location,
1268    },
1269
1270    #[snafu(display("Failed to scan series"))]
1271    ScanSeries {
1272        #[snafu(implicit)]
1273        location: Location,
1274        source: Arc<Error>,
1275    },
1276
1277    #[snafu(display("Partition {} scan multiple times", partition))]
1278    ScanMultiTimes {
1279        partition: usize,
1280        #[snafu(implicit)]
1281        location: Location,
1282    },
1283
1284    #[snafu(display("Invalid partition expression: {}", expr))]
1285    InvalidPartitionExpr {
1286        expr: String,
1287        #[snafu(implicit)]
1288        location: Location,
1289        source: partition::error::Error,
1290    },
1291
1292    #[snafu(display("Failed to decode bulk wal entry"))]
1293    ConvertBulkWalEntry {
1294        #[snafu(implicit)]
1295        location: Location,
1296        source: common_grpc::Error,
1297    },
1298
1299    #[snafu(display("Failed to encode"))]
1300    Encode {
1301        #[snafu(implicit)]
1302        location: Location,
1303        source: mito_codec::error::Error,
1304    },
1305
1306    #[snafu(display("Failed to decode"))]
1307    Decode {
1308        #[snafu(implicit)]
1309        location: Location,
1310        source: mito_codec::error::Error,
1311    },
1312
1313    #[snafu(display("Unexpected: {reason}"))]
1314    Unexpected {
1315        reason: String,
1316        #[snafu(implicit)]
1317        location: Location,
1318    },
1319
1320    #[cfg(feature = "enterprise")]
1321    #[snafu(display("Failed to scan external range"))]
1322    ScanExternalRange {
1323        source: BoxedError,
1324        #[snafu(implicit)]
1325        location: Location,
1326    },
1327
1328    #[snafu(display(
1329        "Inconsistent timestamp column length, expect: {}, actual: {}",
1330        expected,
1331        actual
1332    ))]
1333    InconsistentTimestampLength {
1334        expected: usize,
1335        actual: usize,
1336        #[snafu(implicit)]
1337        location: Location,
1338    },
1339
1340    #[snafu(display(
1341        "Too many files to read concurrently: {}, max allowed: {}",
1342        actual,
1343        max
1344    ))]
1345    TooManyFilesToRead {
1346        actual: usize,
1347        max: usize,
1348        #[snafu(implicit)]
1349        location: Location,
1350    },
1351
1352    #[snafu(display("Duration out of range: {input:?}"))]
1353    DurationOutOfRange {
1354        input: std::time::Duration,
1355        #[snafu(source)]
1356        error: chrono::OutOfRangeError,
1357        #[snafu(implicit)]
1358        location: Location,
1359    },
1360
1361    #[snafu(display("GC job permit exhausted"))]
1362    TooManyGcJobs {
1363        #[snafu(implicit)]
1364        location: Location,
1365    },
1366
1367    #[snafu(display(
1368        "Staging partition expr mismatch, manifest: {:?}, request: {}",
1369        manifest_expr,
1370        request_expr
1371    ))]
1372    StagingPartitionExprMismatch {
1373        manifest_expr: Option<String>,
1374        request_expr: String,
1375        #[snafu(implicit)]
1376        location: Location,
1377    },
1378
1379    #[snafu(display(
1380        "Invalid source and target region, source: {}, target: {}",
1381        source_region_id,
1382        target_region_id
1383    ))]
1384    InvalidSourceAndTargetRegion {
1385        source_region_id: RegionId,
1386        target_region_id: RegionId,
1387        #[snafu(implicit)]
1388        location: Location,
1389    },
1390
1391    #[snafu(display("Failed to prune file"))]
1392    PruneFile {
1393        source: Arc<Error>,
1394        #[snafu(implicit)]
1395        location: Location,
1396    },
1397
1398    #[snafu(display("Failed to cast column"))]
1399    CastColumn {
1400        #[snafu(source)]
1401        error: datafusion::error::DataFusionError,
1402        #[snafu(implicit)]
1403        location: Location,
1404    },
1405
1406    #[snafu(display("Failed to generate Arrow schema from Parquet file: {}", file))]
1407    ParquetToArrowSchema {
1408        file: String,
1409        #[snafu(source)]
1410        error: parquet::errors::ParquetError,
1411        #[snafu(implicit)]
1412        location: Location,
1413    },
1414
1415    #[snafu(display(
1416        "Region {} is in {:?} state, expect: Writable, Staging or Downgrading",
1417        region_id,
1418        state
1419    ))]
1420    FlushableRegionState {
1421        region_id: RegionId,
1422        state: RegionRoleState,
1423        #[snafu(implicit)]
1424        location: Location,
1425    },
1426}
1427
1428pub type Result<T, E = Error> = std::result::Result<T, E>;
1429
1430impl Error {
1431    /// Returns true if we need to fill default value for a region.
1432    pub(crate) fn is_fill_default(&self) -> bool {
1433        matches!(self, Error::FillDefault { .. })
1434    }
1435
1436    /// Returns true if the file is not found on the object store.
1437    pub(crate) fn is_object_not_found(&self) -> bool {
1438        match self {
1439            Error::OpenDal { error, .. } => error.kind() == ErrorKind::NotFound,
1440            Error::ManifestDeltaNotFound { .. } => true,
1441            _ => false,
1442        }
1443    }
1444
1445    /// Returns whether a failed manifest update may have been persisted.
1446    ///
1447    /// Unknown errors are treated conservatively because deleting output SSTs after an ambiguous
1448    /// manifest write could leave the manifest referencing missing files.
1449    pub(crate) fn may_have_persisted_manifest_update(&self) -> bool {
1450        match self {
1451            Error::UpdateManifest { .. }
1452            | Error::RegionState { .. }
1453            | Error::RegionTruncated { .. }
1454            | Error::RegionStopped { .. }
1455            | Error::SerdeJson { .. }
1456            | Error::CompressObject { .. } => false,
1457            Error::OpenDal { error, .. } => !matches!(
1458                error.kind(),
1459                ErrorKind::Unsupported
1460                    | ErrorKind::ConfigInvalid
1461                    | ErrorKind::NotFound
1462                    | ErrorKind::PermissionDenied
1463                    | ErrorKind::IsADirectory
1464                    | ErrorKind::NotADirectory
1465                    | ErrorKind::AlreadyExists
1466                    | ErrorKind::RateLimited
1467                    | ErrorKind::IsSameFile
1468                    | ErrorKind::ConditionNotMatch
1469                    | ErrorKind::RangeNotSatisfied
1470            ),
1471            _ => true,
1472        }
1473    }
1474}
1475
1476impl ErrorExt for Error {
1477    fn status_code(&self) -> StatusCode {
1478        use Error::*;
1479
1480        match self {
1481            DataTypeMismatch { source, .. } => source.status_code(),
1482            OpenDal { .. } | ManifestDeltaNotFound { .. } | ReadParquet { .. } => {
1483                StatusCode::StorageUnavailable
1484            }
1485            WriteWal { source, .. }
1486            | ReadWal { source, .. }
1487            | DeleteWal { source, .. }
1488            | WaitWalDurable { source, .. }
1489            | CheckWalHistory { source, .. } => source.status_code(),
1490            CompressObject { .. }
1491            | DecompressObject { .. }
1492            | SerdeJson { .. }
1493            | Utf8 { .. }
1494            | NewRecordBatch { .. }
1495            | RegionCorrupted { .. }
1496            | InconsistentFile { .. }
1497            | CreateDefault { .. }
1498            | InvalidParquet { .. }
1499            | OperateAbortedIndex { .. }
1500            | IndexEncodeNull { .. }
1501            | NoCheckpoint { .. }
1502            | NoManifests { .. }
1503            | FilesLost { .. }
1504            | InstallManifestTo { .. }
1505            | Unexpected { .. }
1506            | SerializeColumnMetadata { .. }
1507            | SerializeManifest { .. }
1508            | StagingPartitionExprMismatch { .. } => StatusCode::Unexpected,
1509
1510            RegionNotFound { .. } => StatusCode::RegionNotFound,
1511            ObjectStoreNotFound { .. }
1512            | InvalidScanIndex { .. }
1513            | InvalidMeta { .. }
1514            | InvalidRequest { .. }
1515            | PartitionExprVersionMismatch { .. }
1516            | FillDefault { .. }
1517            | ConvertColumnDataType { .. }
1518            | ColumnNotFound { .. }
1519            | InvalidMetadata { .. }
1520            | InvalidRegionOptions { .. }
1521            | InvalidWalReadRequest { .. }
1522            | PartitionOutOfRange { .. }
1523            | ParseJobId { .. }
1524            | DurationOutOfRange { .. }
1525            | MissingOldManifest { .. }
1526            | MissingNewManifest { .. }
1527            | MissingManifest { .. }
1528            | NoOldManifests { .. }
1529            | MissingPartitionExpr { .. }
1530            | SerializePartitionExpr { .. }
1531            | InvalidSourceAndTargetRegion { .. } => StatusCode::InvalidArguments,
1532
1533            IncrementalQueryStale { .. } | SnapshotFenceStale { .. } => StatusCode::RequestOutdated,
1534
1535            SequenceRangeUnsupported { .. } => StatusCode::Unsupported,
1536
1537            RegionSequenceDomainBroken { .. }
1538            | RegionMetadataNotFound { .. }
1539            | Join { .. }
1540            | WorkerStopped { .. }
1541            | Recv { .. }
1542            | DecodeWal { .. }
1543            | ComputeArrow { .. }
1544            | EvalPartitionFilter { .. }
1545            | MergeCandidateSeries { .. }
1546            | BiErrors { .. }
1547            | StopScheduler { .. }
1548            | ComputeVector { .. }
1549            | EncodeMemtable { .. }
1550            | CreateDir { .. }
1551            | ReadDataPart { .. }
1552            | BuildEntry { .. }
1553            | Metadata { .. }
1554            | CastColumn { .. }
1555            | MitoManifestInfo { .. }
1556            | ParquetToArrowSchema { .. } => StatusCode::Internal,
1557
1558            FetchManifests { source, .. } => source.status_code(),
1559
1560            OpenRegion { source, .. } => source.status_code(),
1561
1562            WriteParquet { .. } => StatusCode::StorageUnavailable,
1563            WriteGroup { source, .. } => source.status_code(),
1564            InvalidBatch { .. } => StatusCode::InvalidArguments,
1565            InvalidRecordBatch { .. } => StatusCode::InvalidArguments,
1566            ConvertVector { source, .. } => source.status_code(),
1567
1568            PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
1569            InvalidSender { .. } => StatusCode::InvalidArguments,
1570            InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
1571            RegionRequirement { .. } => StatusCode::InvalidArguments,
1572            DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => {
1573                StatusCode::StorageUnavailable
1574            }
1575            FlushRegion { source, .. } | BuildIndexAsync { source, .. } => source.status_code(),
1576            IndexBuildIncomplete { .. } => StatusCode::IllegalState,
1577            RegionDropped { .. } => StatusCode::Cancelled,
1578            RegionClosed { .. } => StatusCode::Cancelled,
1579            StaleCompactionExecution { .. } => StatusCode::Cancelled,
1580            RegionTruncated { .. } => StatusCode::Cancelled,
1581            RejectWrite { .. } | RejectWriteInStaging { .. } => StatusCode::StorageUnavailable,
1582            CompactRegion { source, .. } => source.status_code(),
1583            EditRegion { source, .. } => source.status_code(),
1584            CompatReader { .. } => StatusCode::Unexpected,
1585            InvalidRegionRequest { source, .. } => source.status_code(),
1586            RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady,
1587            JsonOptions { .. } => StatusCode::InvalidArguments,
1588            EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
1589            ConvertValue { source, .. } => source.status_code(),
1590            ApplyBloomFilterIndex { source, .. } => source.status_code(),
1591            InvalidPartitionExpr { source, .. } => source.status_code(),
1592            BuildIndexApplier { source, .. }
1593            | PushIndexValue { source, .. }
1594            | ApplyInvertedIndex { source, .. }
1595            | IndexFinish { source, .. } => source.status_code(),
1596            PuffinReadBlob { source, .. }
1597            | PuffinAddBlob { source, .. }
1598            | PuffinInitStager { source, .. }
1599            | PuffinBuildReader { source, .. }
1600            | PuffinPurgeStager { source, .. } => source.status_code(),
1601            CleanDir { .. } => StatusCode::Unexpected,
1602            InvalidConfig { .. } => StatusCode::InvalidArguments,
1603            StaleLogEntry { .. }
1604            | InvalidNativeHistogramSubfield { .. }
1605            | InvalidNativeHistogramFieldId { .. } => StatusCode::Unexpected,
1606
1607            External { source, .. } => source.status_code(),
1608
1609            RecordBatch { source, .. } => source.status_code(),
1610
1611            Download { .. } | Upload { .. } => StatusCode::StorageUnavailable,
1612            ChecksumMismatch { .. } => StatusCode::Unexpected,
1613            RegionStopped { .. } => StatusCode::RegionNotReady,
1614            TimeRangePredicateOverflow { .. } => StatusCode::InvalidArguments,
1615            UnsupportedOperation { .. } => StatusCode::Unsupported,
1616            RemoteCompaction { .. } => StatusCode::Unexpected,
1617
1618            IndexOptions { source, .. } => source.status_code(),
1619            CompressionOptions { source, .. } => source.status_code(),
1620            CreateFulltextCreator { source, .. } => source.status_code(),
1621            CastVector { source, .. } => source.status_code(),
1622            FulltextPushText { source, .. }
1623            | FulltextFinish { source, .. }
1624            | ApplyFulltextIndex { source, .. } => source.status_code(),
1625            DecodeStats { .. }
1626            | StatsNotPresent { .. }
1627            | InvalidPrimaryKeyRange { .. }
1628            | DecodePrimaryKeyRange { .. } => StatusCode::Internal,
1629            RegionBusy { .. } => StatusCode::RegionBusy,
1630            GetSchemaMetadata { source, .. } => source.status_code(),
1631            Timeout { .. } => StatusCode::Cancelled,
1632
1633            DecodeArrowRowGroup { .. } => StatusCode::Internal,
1634
1635            PushBloomFilterValue { source, .. } | BloomFilterFinish { source, .. } => {
1636                source.status_code()
1637            }
1638
1639            ManualCompactionOverride {} | CompactionCancelled {} | FlushCancelled {} => {
1640                StatusCode::Cancelled
1641            }
1642
1643            // A concurrent manual compaction fails fast instead of being queued;
1644            // the conflict is reported to the caller, which decides whether to
1645            // issue a new request after the running one finishes.
1646            ManualCompactionAlreadyRunning { .. } => StatusCode::RegionBusy,
1647
1648            CompactionMemoryExhausted { source, .. } => source.status_code(),
1649
1650            IncompatibleWalProviderChange { .. } => StatusCode::InvalidArguments,
1651
1652            ScanSeries { source, .. } => source.status_code(),
1653
1654            ScanMultiTimes { .. } => StatusCode::InvalidArguments,
1655            ConvertBulkWalEntry { source, .. } => source.status_code(),
1656
1657            Encode { source, .. } | Decode { source, .. } => source.status_code(),
1658
1659            #[cfg(feature = "enterprise")]
1660            ScanExternalRange { source, .. } => source.status_code(),
1661
1662            InconsistentTimestampLength { .. } => StatusCode::InvalidArguments,
1663
1664            TooManyFilesToRead { .. } | TooManyGcJobs { .. } => StatusCode::RateLimited,
1665
1666            PruneFile { source, .. } => source.status_code(),
1667
1668            FlushableRegionState { .. } => StatusCode::RegionNotReady,
1669        }
1670    }
1671
1672    fn as_any(&self) -> &dyn Any {
1673        self
1674    }
1675
1676    fn retry_hint(&self) -> RetryHint {
1677        use Error::*;
1678
1679        match self {
1680            PartitionExprVersionMismatch { .. }
1681            | RegionNotFound { .. }
1682            | RejectWriteInStaging { .. }
1683            | RegionState {
1684                state: RegionRoleState::Follower,
1685                ..
1686            } => RetryHint::RetryAfterRouteRefresh,
1687            ReadParquet { .. }
1688            | WriteParquet { .. }
1689            | RejectWrite { .. }
1690            | Download { .. }
1691            | Upload { .. }
1692            | RegionState { .. }
1693            | UpdateManifest { .. }
1694            | RegionStopped { .. }
1695            | RegionBusy { .. }
1696            | ManualCompactionAlreadyRunning { .. }
1697            | FlushableRegionState { .. }
1698            | ManifestDeltaNotFound { .. } => RetryHint::Retryable,
1699
1700            OpenDal { error, .. }
1701            | DeleteSsts { error, .. }
1702            | DeleteIndex { error, .. }
1703            | DeleteIndexes { error, .. } => retry_hint_from_opendal_error(error),
1704
1705            WriteWal { source, .. }
1706            | ReadWal { source, .. }
1707            | DeleteWal { source, .. }
1708            | WaitWalDurable { source, .. }
1709            | CheckWalHistory { source, .. }
1710            | FetchManifests { source, .. }
1711            | External { source, .. } => source.retry_hint(),
1712
1713            OpenRegion { source, .. }
1714            | WriteGroup { source, .. }
1715            | FlushRegion { source, .. }
1716            | BuildIndexAsync { source, .. }
1717            | CompactRegion { source, .. }
1718            | EditRegion { source, .. }
1719            | ScanSeries { source, .. }
1720            | PruneFile { source, .. } => source.retry_hint(),
1721
1722            DataTypeMismatch { source, .. }
1723            | ConvertVector { source, .. }
1724            | ConvertValue { source, .. }
1725            | IndexOptions { source, .. }
1726            | CompressionOptions { source, .. }
1727            | CastVector { source, .. } => source.retry_hint(),
1728
1729            BuildIndexApplier { source, .. }
1730            | PushIndexValue { source, .. }
1731            | ApplyInvertedIndex { source, .. }
1732            | IndexFinish { source, .. } => source.retry_hint(),
1733
1734            ApplyBloomFilterIndex { source, .. }
1735            | PushBloomFilterValue { source, .. }
1736            | BloomFilterFinish { source, .. } => source.retry_hint(),
1737
1738            PuffinReadBlob { source, .. }
1739            | PuffinAddBlob { source, .. }
1740            | PuffinInitStager { source, .. }
1741            | PuffinBuildReader { source, .. }
1742            | PuffinPurgeStager { source, .. } => source.retry_hint(),
1743
1744            CreateFulltextCreator { source, .. }
1745            | FulltextPushText { source, .. }
1746            | FulltextFinish { source, .. }
1747            | ApplyFulltextIndex { source, .. } => source.retry_hint(),
1748
1749            InvalidRegionRequest { source, .. } => source.retry_hint(),
1750            InvalidPartitionExpr { source, .. } => source.retry_hint(),
1751            RecordBatch { source, .. } => source.retry_hint(),
1752            GetSchemaMetadata { source, .. } => source.retry_hint(),
1753            CompactionMemoryExhausted { source, .. } => source.retry_hint(),
1754            ConvertBulkWalEntry { source, .. } => source.retry_hint(),
1755            Encode { source, .. } | Decode { source, .. } => source.retry_hint(),
1756
1757            #[cfg(feature = "enterprise")]
1758            ScanExternalRange { source, .. } => source.retry_hint(),
1759
1760            _ => RetryHint::NonRetryable,
1761        }
1762    }
1763}
1764
1765#[cfg(test)]
1766mod tests {
1767    use snafu::IntoError;
1768
1769    use super::*;
1770    use crate::region::RegionLeaderState;
1771
1772    #[test]
1773    fn test_partition_version_mismatch_retry_hint() {
1774        let error = PartitionExprVersionMismatchSnafu {
1775            region_id: RegionId::new(1, 1),
1776            request_version: 1u64,
1777            expected_version: 2u64,
1778        }
1779        .build();
1780        assert_eq!(error.status_code(), StatusCode::InvalidArguments);
1781        assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
1782    }
1783
1784    #[test]
1785    fn test_region_state_retry_hint() {
1786        let region_id = RegionId::new(1, 1);
1787        let writable = RegionRoleState::Leader(RegionLeaderState::Writable);
1788        let follower_error = RegionStateSnafu {
1789            region_id,
1790            state: RegionRoleState::Follower,
1791            expect: writable,
1792        }
1793        .build();
1794        assert_eq!(
1795            follower_error.retry_hint(),
1796            RetryHint::RetryAfterRouteRefresh
1797        );
1798        assert_eq!(follower_error.status_code(), StatusCode::RegionNotReady);
1799        let leader_error = RegionStateSnafu {
1800            region_id,
1801            state: writable,
1802            expect: RegionRoleState::Follower,
1803        }
1804        .build();
1805        assert_eq!(leader_error.retry_hint(), RetryHint::Retryable);
1806        for (error, status, hint) in [
1807            (
1808                RejectWriteSnafu { region_id }.build(),
1809                StatusCode::StorageUnavailable,
1810                RetryHint::Retryable,
1811            ),
1812            (
1813                RejectWriteInStagingSnafu { region_id }.build(),
1814                StatusCode::StorageUnavailable,
1815                RetryHint::RetryAfterRouteRefresh,
1816            ),
1817            (
1818                RegionNotFoundSnafu { region_id }.build(),
1819                StatusCode::RegionNotFound,
1820                RetryHint::RetryAfterRouteRefresh,
1821            ),
1822        ] {
1823            assert_eq!(error.status_code(), status);
1824            assert_eq!(error.retry_hint(), hint);
1825        }
1826    }
1827
1828    #[test]
1829    fn test_manifest_update_persistence() {
1830        let rejected_kinds = [
1831            ErrorKind::Unsupported,
1832            ErrorKind::ConfigInvalid,
1833            ErrorKind::NotFound,
1834            ErrorKind::PermissionDenied,
1835            ErrorKind::IsADirectory,
1836            ErrorKind::NotADirectory,
1837            ErrorKind::AlreadyExists,
1838            ErrorKind::RateLimited,
1839            ErrorKind::IsSameFile,
1840            ErrorKind::ConditionNotMatch,
1841            ErrorKind::RangeNotSatisfied,
1842        ];
1843        for kind in rejected_kinds {
1844            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
1845            assert!(
1846                !error.may_have_persisted_manifest_update(),
1847                "error kind {kind:?} should prove the manifest was not persisted"
1848            );
1849        }
1850
1851        let error =
1852            OpenDalSnafu {}.into_error(object_store::Error::new(ErrorKind::Unexpected, "test"));
1853        assert!(error.may_have_persisted_manifest_update());
1854
1855        let region_id = RegionId::new(1, 1);
1856        let error = RegionTruncatedSnafu { region_id }.build();
1857        assert!(!error.may_have_persisted_manifest_update());
1858
1859        // This can be raised while applying an edit after its manifest file was saved.
1860        let error = RegionMetadataNotFoundSnafu {}.build();
1861        assert!(error.may_have_persisted_manifest_update());
1862    }
1863}
mito2/error.rs

mito2/
error.rs
