Skip to main content

mito2/
error.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::sync::Arc;
17
18use common_datasource::compression::CompressionType;
19use common_error::ext::{BoxedError, ErrorExt, RetryHint};
20use common_error::status_code::StatusCode;
21use common_macro::stack_trace_debug;
22use common_memory_manager;
23use common_runtime::JoinError;
24use common_time::Timestamp;
25use common_time::timestamp::TimeUnit;
26use datatypes::arrow::error::ArrowError;
27use datatypes::prelude::ConcreteDataType;
28use object_store::ErrorKind;
29use object_store::error::retry_hint_from_opendal_error;
30use partition::error::Error as PartitionError;
31use prost::DecodeError;
32use snafu::{Location, Snafu};
33use store_api::ManifestVersion;
34use store_api::logstore::provider::Provider;
35use store_api::storage::{FileId, RegionId};
36use tokio::time::error::Elapsed;
37
38use crate::cache::file_cache::FileType;
39use crate::region::RegionRoleState;
40use crate::schedule::remote_job_scheduler::JobId;
41use crate::worker::WorkerId;
42
43#[derive(Snafu)]
44#[snafu(visibility(pub))]
45#[stack_trace_debug]
46pub enum Error {
47    #[cfg(feature = "enterprise")]
48    #[snafu(display("Failed to route multi-region compaction input"))]
49    RouteMultiRegion {
50        #[snafu(source)]
51        source: PartitionError,
52        #[snafu(implicit)]
53        location: Location,
54    },
55
56    #[cfg(feature = "enterprise")]
57    #[snafu(display("Invalid multi-region compaction: {reason}"))]
58    InvalidMultiRegionCompaction {
59        reason: String,
60        #[snafu(implicit)]
61        location: Location,
62    },
63
64    #[cfg(feature = "enterprise")]
65    #[snafu(display("Multi-region compaction resource limit: {reason}"))]
66    MultiRegionCompactionResource {
67        reason: String,
68        #[snafu(implicit)]
69        location: Location,
70    },
71
72    #[snafu(display("SST writer {kind} needs {bytes} bytes, budget is {limit}"))]
73    SstWriteMemoryExceeded {
74        kind: &'static str,
75        bytes: usize,
76        limit: usize,
77        #[snafu(implicit)]
78        location: Location,
79    },
80
81    #[snafu(display("Unexpected data type"))]
82    DataTypeMismatch {
83        source: datatypes::error::Error,
84        #[snafu(implicit)]
85        location: Location,
86    },
87
88    #[snafu(display("External error, context: {}", context))]
89    External {
90        source: BoxedError,
91        context: String,
92        #[snafu(implicit)]
93        location: Location,
94    },
95
96    #[snafu(display("OpenDAL operator failed"))]
97    OpenDal {
98        #[snafu(implicit)]
99        location: Location,
100        #[snafu(source)]
101        error: object_store::Error,
102    },
103
104    #[snafu(display(
105        "Manifest delta {} disappeared after it was listed, path: {}",
106        version,
107        path
108    ))]
109    ManifestDeltaNotFound {
110        version: ManifestVersion,
111        path: String,
112        #[snafu(source)]
113        error: object_store::Error,
114        #[snafu(implicit)]
115        location: Location,
116    },
117
118    #[snafu(display("Fail to compress object by {}, path: {}", compress_type, path))]
119    CompressObject {
120        compress_type: CompressionType,
121        path: String,
122        #[snafu(source)]
123        error: std::io::Error,
124    },
125
126    #[snafu(display("Fail to decompress object by {}, path: {}", compress_type, path))]
127    DecompressObject {
128        compress_type: CompressionType,
129        path: String,
130        #[snafu(source)]
131        error: std::io::Error,
132    },
133
134    #[snafu(display("Failed to ser/de json object"))]
135    SerdeJson {
136        #[snafu(implicit)]
137        location: Location,
138        #[snafu(source)]
139        error: serde_json::Error,
140    },
141
142    #[snafu(display("Failed to serialize column metadata"))]
143    SerializeColumnMetadata {
144        #[snafu(source)]
145        error: serde_json::Error,
146        #[snafu(implicit)]
147        location: Location,
148    },
149
150    #[snafu(display("Manifest version {version} conflicts in {manifest_dir}: {reason}"))]
151    ManifestVersionConflict {
152        version: ManifestVersion,
153        manifest_dir: String,
154        reason: String,
155        #[snafu(implicit)]
156        location: Location,
157    },
158
159    #[snafu(display(
160        "Manifest checkpoint {version} in {manifest_dir} was removed after it was listed"
161    ))]
162    ManifestCheckpointVanished {
163        version: ManifestVersion,
164        manifest_dir: String,
165        #[snafu(implicit)]
166        location: Location,
167    },
168
169    #[snafu(display("Failed to serialize manifest, region_id: {}", region_id))]
170    SerializeManifest {
171        region_id: RegionId,
172        #[snafu(source)]
173        error: serde_json::Error,
174        #[snafu(implicit)]
175        location: Location,
176    },
177
178    #[snafu(display("Invalid scan index, start: {}, end: {}", start, end))]
179    InvalidScanIndex {
180        start: ManifestVersion,
181        end: ManifestVersion,
182        #[snafu(implicit)]
183        location: Location,
184    },
185
186    #[snafu(display("Invalid UTF-8 content"))]
187    Utf8 {
188        #[snafu(implicit)]
189        location: Location,
190        #[snafu(source)]
191        error: std::str::Utf8Error,
192    },
193
194    #[snafu(display("Cannot find RegionMetadata"))]
195    RegionMetadataNotFound {
196        #[snafu(implicit)]
197        location: Location,
198    },
199
200    #[snafu(display("Failed to join handle"))]
201    Join {
202        #[snafu(source)]
203        error: common_runtime::JoinError,
204        #[snafu(implicit)]
205        location: Location,
206    },
207
208    #[snafu(display("Worker {} is stopped", id))]
209    WorkerStopped {
210        id: WorkerId,
211        #[snafu(implicit)]
212        location: Location,
213    },
214
215    #[snafu(display("Engine is stopped"))]
216    EngineStopped {
217        #[snafu(implicit)]
218        location: Location,
219    },
220
221    #[snafu(display("Failed to recv result"))]
222    Recv {
223        #[snafu(source)]
224        error: tokio::sync::oneshot::error::RecvError,
225        #[snafu(implicit)]
226        location: Location,
227    },
228
229    #[snafu(display("Invalid metadata, {}", reason))]
230    InvalidMeta {
231        reason: String,
232        #[snafu(implicit)]
233        location: Location,
234    },
235
236    #[snafu(display("Invalid region metadata"))]
237    InvalidMetadata {
238        source: store_api::metadata::MetadataError,
239        #[snafu(implicit)]
240        location: Location,
241    },
242
243    #[snafu(display("Failed to create RecordBatch from vectors"))]
244    NewRecordBatch {
245        #[snafu(implicit)]
246        location: Location,
247        #[snafu(source)]
248        error: ArrowError,
249    },
250
251    #[snafu(display("Failed to read parquet file, path: {}", path))]
252    ReadParquet {
253        path: String,
254        #[snafu(source)]
255        error: parquet::errors::ParquetError,
256        #[snafu(implicit)]
257        location: Location,
258    },
259
260    #[snafu(display("Failed to load range result"))]
261    LoadRangeResult {
262        source: Arc<Error>,
263        #[snafu(implicit)]
264        location: Location,
265    },
266
267    #[snafu(display("Failed to load SST metadata"))]
268    LoadSstMeta {
269        source: Arc<Error>,
270        #[snafu(implicit)]
271        location: Location,
272    },
273
274    #[snafu(display("Failed to load Parquet page"))]
275    LoadParquetPage {
276        source: Arc<Error>,
277        #[snafu(implicit)]
278        location: Location,
279    },
280
281    #[snafu(display("Failed to write parquet file"))]
282    WriteParquet {
283        #[snafu(source)]
284        error: parquet::errors::ParquetError,
285        #[snafu(implicit)]
286        location: Location,
287    },
288
289    #[snafu(display(
290        "Cannot assign a stable field id to native histogram sub-field '{}' of column id {} (unknown sub-field name or derived id overflows i32)",
291        field_name,
292        column_id
293    ))]
294    InvalidNativeHistogramSubfield {
295        column_id: i32,
296        field_name: String,
297        #[snafu(implicit)]
298        location: Location,
299    },
300
301    #[snafu(display(
302        "Native histogram column '{}' has no usable PARQUET:field_id to namespace its sub-field ids (missing, malformed, or exceeds i32::MAX)",
303        field_name
304    ))]
305    InvalidNativeHistogramFieldId {
306        field_name: String,
307        #[snafu(implicit)]
308        location: Location,
309    },
310
311    #[snafu(display("Region {} not found", region_id))]
312    RegionNotFound {
313        region_id: RegionId,
314        #[snafu(implicit)]
315        location: Location,
316    },
317
318    #[snafu(display("Object store not found: {}", object_store))]
319    ObjectStoreNotFound {
320        object_store: String,
321        #[snafu(implicit)]
322        location: Location,
323    },
324
325    #[snafu(display("Region {} is corrupted, reason: {}", region_id, reason))]
326    RegionCorrupted {
327        region_id: RegionId,
328        reason: String,
329        #[snafu(implicit)]
330        location: Location,
331    },
332
333    #[snafu(display("Invalid request to region {}, reason: {}", region_id, reason))]
334    InvalidRequest {
335        region_id: RegionId,
336        reason: String,
337        #[snafu(implicit)]
338        location: Location,
339    },
340
341    #[snafu(display(
342        "STALE_CURSOR: incremental query stale, region: {}, given_seq: {}, min_readable_seq: {}, retry_hint: FALLBACK_FULL_RECOMPUTE",
343        region_id,
344        given_seq,
345        min_readable_seq
346    ))]
347    IncrementalQueryStale {
348        region_id: RegionId,
349        given_seq: u64,
350        min_readable_seq: u64,
351        #[snafu(implicit)]
352        location: Location,
353    },
354
355    #[snafu(display(
356        "STALE_SNAPSHOT_FENCE: snapshot upper bound stale, region: {}, given_seq: {}, min_enforceable_seq: {}, retry_hint: REBIND_SNAPSHOT_FENCE",
357        region_id,
358        given_seq,
359        min_enforceable_seq
360    ))]
361    SnapshotFenceStale {
362        region_id: RegionId,
363        given_seq: u64,
364        min_enforceable_seq: u64,
365        #[snafu(implicit)]
366        location: Location,
367    },
368
369    #[snafu(display(
370        "SEQUENCE_RANGE_UNSUPPORTED: exact sequence-range read unsupported, region: {}, min_seq: {}, max_seq: {}, reason: {}, retry_hint: FALLBACK_MEMTABLE_ONLY_OR_FULL_RECOMPUTE",
371        region_id,
372        min_seq,
373        max_seq,
374        reason
375    ))]
376    SequenceRangeUnsupported {
377        region_id: RegionId,
378        min_seq: u64,
379        max_seq: u64,
380        reason: String,
381        #[snafu(implicit)]
382        location: Location,
383    },
384
385    #[snafu(display(
386        "region {} is unusable for sequence reads: file {} declares region {}",
387        region_id,
388        file_id,
389        file_region_id
390    ))]
391    RegionSequenceDomainBroken {
392        region_id: RegionId,
393        file_region_id: RegionId,
394        file_id: FileId,
395        #[snafu(implicit)]
396        location: Location,
397    },
398
399    #[snafu(display("Old manifest missing for region {}", region_id))]
400    MissingOldManifest {
401        region_id: RegionId,
402        #[snafu(implicit)]
403        location: Location,
404    },
405
406    #[snafu(display("New manifest missing for region {}", region_id))]
407    MissingNewManifest {
408        region_id: RegionId,
409        #[snafu(implicit)]
410        location: Location,
411    },
412
413    #[snafu(display("Manifest missing for region {}", region_id))]
414    MissingManifest {
415        region_id: RegionId,
416        #[snafu(implicit)]
417        location: Location,
418    },
419
420    #[snafu(display("File consistency check failed for file {}: {}", file_id, reason))]
421    InconsistentFile {
422        file_id: FileId,
423        reason: String,
424        #[snafu(implicit)]
425        location: Location,
426    },
427
428    #[snafu(display("Files lost during remapping: old={}, new={}", old_count, new_count))]
429    FilesLost {
430        old_count: usize,
431        new_count: usize,
432        #[snafu(implicit)]
433        location: Location,
434    },
435
436    #[snafu(display("No old manifests provided (need at least one for template)"))]
437    NoOldManifests {
438        #[snafu(implicit)]
439        location: Location,
440    },
441
442    #[snafu(display("Failed to fetch manifests"))]
443    FetchManifests {
444        #[snafu(implicit)]
445        location: Location,
446        source: BoxedError,
447    },
448
449    #[snafu(display("Partition expression missing for region {}", region_id))]
450    MissingPartitionExpr {
451        region_id: RegionId,
452        #[snafu(implicit)]
453        location: Location,
454    },
455
456    #[snafu(display("Failed to serialize partition expression: {}", source))]
457    SerializePartitionExpr {
458        #[snafu(source)]
459        source: PartitionError,
460        #[snafu(implicit)]
461        location: Location,
462    },
463
464    #[snafu(display(
465        "Failed to convert ConcreteDataType to ColumnDataType, reason: {}",
466        reason
467    ))]
468    ConvertColumnDataType {
469        reason: String,
470        source: api::error::Error,
471        #[snafu(implicit)]
472        location: Location,
473    },
474
475    /// An error type to indicate that schema is changed and we need
476    /// to fill default values again.
477    #[snafu(display("Need to fill default value for region {}", region_id))]
478    FillDefault {
479        region_id: RegionId,
480        // The error is for internal use so we don't need a location.
481    },
482
483    #[snafu(display(
484        "Failed to create default value for column {} of region {}",
485        column,
486        region_id
487    ))]
488    CreateDefault {
489        region_id: RegionId,
490        column: String,
491        source: datatypes::Error,
492        #[snafu(implicit)]
493        location: Location,
494    },
495
496    #[snafu(display("Failed to build entry, region_id: {}", region_id))]
497    BuildEntry {
498        region_id: RegionId,
499        #[snafu(implicit)]
500        location: Location,
501        source: BoxedError,
502    },
503
504    #[snafu(display("Failed to write WAL"))]
505    WriteWal {
506        #[snafu(implicit)]
507        location: Location,
508        source: BoxedError,
509    },
510
511    #[snafu(display("Failed to read WAL, provider: {}", provider))]
512    ReadWal {
513        provider: Provider,
514        #[snafu(implicit)]
515        location: Location,
516        source: BoxedError,
517    },
518
519    #[snafu(display("Failed to read the WAL of node {} for region {}", node_id, region_id))]
520    ReadNodeWal {
521        region_id: RegionId,
522        node_id: u64,
523        #[snafu(implicit)]
524        location: Location,
525        source: BoxedError,
526    },
527
528    #[snafu(display("Failed to decode WAL entry, region_id: {}", region_id))]
529    DecodeWal {
530        region_id: RegionId,
531        #[snafu(implicit)]
532        location: Location,
533        #[snafu(source)]
534        error: DecodeError,
535    },
536
537    #[snafu(display("Failed to delete WAL, region_id: {}", region_id))]
538    DeleteWal {
539        region_id: RegionId,
540        #[snafu(implicit)]
541        location: Location,
542        source: BoxedError,
543    },
544
545    #[snafu(display("Failed to wait for the WAL to be durable, region_id: {}", region_id))]
546    WaitWalDurable {
547        region_id: RegionId,
548        #[snafu(implicit)]
549        location: Location,
550        source: BoxedError,
551    },
552
553    #[snafu(display("Failed to check the WAL history, region_id: {}", region_id))]
554    CheckWalHistory {
555        region_id: RegionId,
556        #[snafu(implicit)]
557        location: Location,
558        source: BoxedError,
559    },
560
561    #[snafu(display("Failed to resolve the WAL provider, region_id: {}", region_id))]
562    ResolveWalProvider {
563        region_id: RegionId,
564        #[snafu(implicit)]
565        location: Location,
566        source: BoxedError,
567    },
568
569    // Shared error for each writer in the write group.
570    #[snafu(display("Failed to write region"))]
571    WriteGroup { source: Arc<Error> },
572
573    #[snafu(display("Invalid parquet SST file {}, reason: {}", file, reason))]
574    InvalidParquet {
575        file: String,
576        reason: String,
577        #[snafu(implicit)]
578        location: Location,
579    },
580
581    #[snafu(display("Invalid batch, {}", reason))]
582    InvalidBatch {
583        reason: String,
584        #[snafu(implicit)]
585        location: Location,
586    },
587
588    #[snafu(display("Invalid arrow record batch, {}", reason))]
589    InvalidRecordBatch {
590        reason: String,
591        #[snafu(implicit)]
592        location: Location,
593    },
594
595    #[snafu(display("Invalid wal read request, {}", reason))]
596    InvalidWalReadRequest {
597        reason: String,
598        #[snafu(implicit)]
599        location: Location,
600    },
601
602    #[snafu(display("Failed to convert array to vector"))]
603    ConvertVector {
604        #[snafu(implicit)]
605        location: Location,
606        source: datatypes::error::Error,
607    },
608
609    #[snafu(display("Failed to compute arrow arrays"))]
610    ComputeArrow {
611        #[snafu(implicit)]
612        location: Location,
613        #[snafu(source)]
614        error: datatypes::arrow::error::ArrowError,
615    },
616
617    #[snafu(display("Failed to evaluate partition filter"))]
618    EvalPartitionFilter {
619        #[snafu(implicit)]
620        location: Location,
621        #[snafu(source)]
622        error: datafusion::error::DataFusionError,
623    },
624
625    #[snafu(display("Failed to merge candidate series"))]
626    MergeCandidateSeries {
627        #[snafu(implicit)]
628        location: Location,
629        #[snafu(source)]
630        error: datafusion::error::DataFusionError,
631    },
632
633    #[snafu(display("Scan memory exhausted"))]
634    ScanMemoryExhausted {
635        #[snafu(implicit)]
636        location: Location,
637        #[snafu(source)]
638        error: datafusion::error::DataFusionError,
639    },
640
641    #[snafu(display("Failed to compute vector"))]
642    ComputeVector {
643        #[snafu(implicit)]
644        location: Location,
645        source: datatypes::error::Error,
646    },
647
648    #[snafu(display("Primary key length mismatch, expect: {}, actual: {}", expect, actual))]
649    PrimaryKeyLengthMismatch {
650        expect: usize,
651        actual: usize,
652        #[snafu(implicit)]
653        location: Location,
654    },
655
656    #[snafu(display("Invalid sender",))]
657    InvalidSender {
658        #[snafu(implicit)]
659        location: Location,
660    },
661
662    #[snafu(display("Invalid scheduler state"))]
663    InvalidSchedulerState {
664        #[snafu(implicit)]
665        location: Location,
666    },
667
668    #[snafu(display("Failed to stop scheduler"))]
669    StopScheduler {
670        #[snafu(source)]
671        error: JoinError,
672        #[snafu(implicit)]
673        location: Location,
674    },
675
676    #[snafu(display(
677        "Failed to batch delete SST files, region id: {}, file ids: {:?}",
678        region_id,
679        file_ids
680    ))]
681    DeleteSsts {
682        region_id: RegionId,
683        file_ids: Vec<FileId>,
684        #[snafu(source)]
685        error: object_store::Error,
686        #[snafu(implicit)]
687        location: Location,
688    },
689
690    #[snafu(display("Failed to delete index file, file id: {}", file_id))]
691    DeleteIndex {
692        file_id: FileId,
693        #[snafu(source)]
694        error: object_store::Error,
695        #[snafu(implicit)]
696        location: Location,
697    },
698
699    #[snafu(display("Failed to batch delete index files, file ids: {:?}", file_ids))]
700    DeleteIndexes {
701        file_ids: Vec<FileId>,
702        #[snafu(source)]
703        error: object_store::Error,
704        #[snafu(implicit)]
705        location: Location,
706    },
707
708    #[snafu(display("Failed to flush region {}", region_id))]
709    FlushRegion {
710        region_id: RegionId,
711        source: Arc<Error>,
712        #[snafu(implicit)]
713        location: Location,
714    },
715
716    #[snafu(display("Region {} is dropped", region_id))]
717    RegionDropped {
718        region_id: RegionId,
719        #[snafu(implicit)]
720        location: Location,
721    },
722
723    #[snafu(display("Region {} is closed", region_id))]
724    RegionClosed {
725        region_id: RegionId,
726        #[snafu(implicit)]
727        location: Location,
728    },
729
730    #[snafu(display(
731        "Stale compaction execution for region {}, the region may have been reopened, truncated or the compaction was superseded",
732        region_id
733    ))]
734    StaleCompactionExecution {
735        region_id: RegionId,
736        #[snafu(implicit)]
737        location: Location,
738    },
739
740    #[snafu(display("Region {} is truncated", region_id))]
741    RegionTruncated {
742        region_id: RegionId,
743        #[snafu(implicit)]
744        location: Location,
745    },
746
747    #[snafu(display(
748        "Engine write buffer is full, rejecting write requests of region {}",
749        region_id,
750    ))]
751    RejectWrite {
752        region_id: RegionId,
753        #[snafu(implicit)]
754        location: Location,
755    },
756
757    #[snafu(display("Region {} rejects writes while in staging", region_id))]
758    RejectWriteInStaging {
759        region_id: RegionId,
760        #[snafu(implicit)]
761        location: Location,
762    },
763
764    #[snafu(display("Region {} still has WAL tails to recover", region_id))]
765    RegionPendingTails {
766        region_id: RegionId,
767        #[snafu(implicit)]
768        location: Location,
769    },
770
771    #[snafu(display(
772        "Region {} cannot read pending WAL tails of node {}",
773        region_id,
774        node_id
775    ))]
776    RemoteWalUnsupported {
777        region_id: RegionId,
778        node_id: u64,
779        #[snafu(implicit)]
780        location: Location,
781    },
782
783    #[snafu(display(
784        "Region {} cannot read entry {} of the WAL of node {}, where its pending tail ends",
785        region_id,
786        entry_id,
787        node_id
788    ))]
789    TailEntryMissing {
790        region_id: RegionId,
791        node_id: u64,
792        entry_id: u64,
793        #[snafu(implicit)]
794        location: Location,
795    },
796
797    #[snafu(display("Region {} does not hold a valid leader lease", region_id))]
798    RegionLeaseExpired {
799        region_id: RegionId,
800        #[snafu(implicit)]
801        location: Location,
802    },
803
804    #[snafu(display(
805        "Region {} lost its leader lease while writing to the WAL, the write may have been applied",
806        region_id
807    ))]
808    UnacknowledgedWrite {
809        region_id: RegionId,
810        #[snafu(implicit)]
811        location: Location,
812    },
813
814    #[snafu(display("Failed to compact region {}", region_id))]
815    CompactRegion {
816        region_id: RegionId,
817        source: Arc<Error>,
818        #[snafu(implicit)]
819        location: Location,
820    },
821
822    #[snafu(display("Failed to edit region {}", region_id))]
823    EditRegion {
824        region_id: RegionId,
825        source: Arc<Error>,
826        #[snafu(implicit)]
827        location: Location,
828    },
829
830    #[snafu(display(
831        "Failed to end staging of region {}, which is still staging",
832        region_id
833    ))]
834    EndStaging {
835        region_id: RegionId,
836        source: Arc<Error>,
837        #[snafu(implicit)]
838        location: Location,
839    },
840
841    #[snafu(display("Region {} waits for a retry of its end-of-staging commit", region_id))]
842    EndStagingPending {
843        region_id: RegionId,
844        #[snafu(implicit)]
845        location: Location,
846    },
847
848    #[snafu(display(
849        "Failed to compat readers for region {}, reason: {}",
850        region_id,
851        reason,
852    ))]
853    CompatReader {
854        region_id: RegionId,
855        reason: String,
856        #[snafu(implicit)]
857        location: Location,
858    },
859
860    #[snafu(display("Invalue region req"))]
861    InvalidRegionRequest {
862        source: store_api::metadata::MetadataError,
863        #[snafu(implicit)]
864        location: Location,
865    },
866
867    #[snafu(display(
868        "Region {} is in {:?} state, which does not permit manifest updates.",
869        region_id,
870        state
871    ))]
872    UpdateManifest {
873        region_id: RegionId,
874        state: RegionRoleState,
875        #[snafu(implicit)]
876        location: Location,
877    },
878
879    #[snafu(display("Region {} is in {:?} state, expect: {:?}", region_id, state, expect))]
880    RegionState {
881        region_id: RegionId,
882        state: RegionRoleState,
883        expect: RegionRoleState,
884        #[snafu(implicit)]
885        location: Location,
886    },
887
888    #[snafu(display(
889        "Partition expr version mismatch for region {}: request {}, expected {}",
890        region_id,
891        request_version,
892        expected_version
893    ))]
894    PartitionExprVersionMismatch {
895        region_id: RegionId,
896        request_version: u64,
897        expected_version: u64,
898        #[snafu(implicit)]
899        location: Location,
900    },
901
902    #[snafu(display("Invalid options"))]
903    JsonOptions {
904        #[snafu(source)]
905        error: serde_json::Error,
906        #[snafu(implicit)]
907        location: Location,
908    },
909
910    #[snafu(display(
911        "Empty region directory, region_id: {}, region_dir: {}",
912        region_id,
913        region_dir,
914    ))]
915    EmptyRegionDir {
916        region_id: RegionId,
917        region_dir: String,
918        #[snafu(implicit)]
919        location: Location,
920    },
921
922    #[snafu(display("Empty manifest directory, manifest_dir: {}", manifest_dir,))]
923    EmptyManifestDir {
924        manifest_dir: String,
925        #[snafu(implicit)]
926        location: Location,
927    },
928
929    #[snafu(display("Column not found, column: {column}"))]
930    ColumnNotFound {
931        column: String,
932        #[snafu(implicit)]
933        location: Location,
934    },
935
936    #[snafu(display("Failed to build index applier"))]
937    BuildIndexApplier {
938        source: index::inverted_index::error::Error,
939        #[snafu(implicit)]
940        location: Location,
941    },
942
943    #[snafu(display("Failed to build index asynchronously in region {}", region_id))]
944    BuildIndexAsync {
945        region_id: RegionId,
946        source: Arc<Error>,
947        #[snafu(implicit)]
948        location: Location,
949    },
950
951    #[snafu(display(
952        "Index build did not complete for files {:?} in region {}",
953        file_ids,
954        region_id
955    ))]
956    IndexBuildIncomplete {
957        region_id: RegionId,
958        file_ids: Vec<FileId>,
959        #[snafu(implicit)]
960        location: Location,
961    },
962
963    #[snafu(display("Failed to convert value"))]
964    ConvertValue {
965        source: datatypes::error::Error,
966        #[snafu(implicit)]
967        location: Location,
968    },
969
970    #[snafu(display("Failed to apply inverted index"))]
971    ApplyInvertedIndex {
972        source: index::inverted_index::error::Error,
973        #[snafu(implicit)]
974        location: Location,
975    },
976
977    #[snafu(display("Failed to apply bloom filter index"))]
978    ApplyBloomFilterIndex {
979        source: index::bloom_filter::error::Error,
980        #[snafu(implicit)]
981        location: Location,
982    },
983
984    #[snafu(display("Failed to push index value"))]
985    PushIndexValue {
986        source: index::inverted_index::error::Error,
987        #[snafu(implicit)]
988        location: Location,
989    },
990
991    #[snafu(display("Failed to write index completely"))]
992    IndexFinish {
993        source: index::inverted_index::error::Error,
994        #[snafu(implicit)]
995        location: Location,
996    },
997
998    #[snafu(display("Operate on aborted index"))]
999    OperateAbortedIndex {
1000        #[snafu(implicit)]
1001        location: Location,
1002    },
1003
1004    #[snafu(display("Failed to read puffin blob"))]
1005    PuffinReadBlob {
1006        source: puffin::error::Error,
1007        #[snafu(implicit)]
1008        location: Location,
1009    },
1010
1011    #[snafu(display("Failed to add blob to puffin file"))]
1012    PuffinAddBlob {
1013        source: puffin::error::Error,
1014        #[snafu(implicit)]
1015        location: Location,
1016    },
1017
1018    #[snafu(display("Failed to clean dir {dir}"))]
1019    CleanDir {
1020        dir: String,
1021        #[snafu(source)]
1022        error: std::io::Error,
1023        #[snafu(implicit)]
1024        location: Location,
1025    },
1026
1027    #[snafu(display("Invalid config, {reason}"))]
1028    InvalidConfig {
1029        reason: String,
1030        #[snafu(implicit)]
1031        location: Location,
1032    },
1033
1034    #[snafu(display(
1035        "Stale log entry found during replay, region: {}, flushed: {}, replayed: {}",
1036        region_id,
1037        flushed_entry_id,
1038        unexpected_entry_id
1039    ))]
1040    StaleLogEntry {
1041        region_id: RegionId,
1042        flushed_entry_id: u64,
1043        unexpected_entry_id: u64,
1044    },
1045
1046    #[snafu(display(
1047        "Failed to download file, region_id: {}, file_id: {}, file_type: {:?}",
1048        region_id,
1049        file_id,
1050        file_type,
1051    ))]
1052    Download {
1053        region_id: RegionId,
1054        file_id: FileId,
1055        file_type: FileType,
1056        #[snafu(source)]
1057        error: std::io::Error,
1058        #[snafu(implicit)]
1059        location: Location,
1060    },
1061
1062    #[snafu(display(
1063        "Failed to upload file, region_id: {}, file_id: {}, file_type: {:?}",
1064        region_id,
1065        file_id,
1066        file_type,
1067    ))]
1068    Upload {
1069        region_id: RegionId,
1070        file_id: FileId,
1071        file_type: FileType,
1072        #[snafu(source)]
1073        error: std::io::Error,
1074        #[snafu(implicit)]
1075        location: Location,
1076    },
1077
1078    #[snafu(display("Failed to create directory {}", dir))]
1079    CreateDir {
1080        dir: String,
1081        #[snafu(source)]
1082        error: std::io::Error,
1083    },
1084
1085    #[snafu(display("Record batch error"))]
1086    RecordBatch {
1087        source: common_recordbatch::error::Error,
1088        #[snafu(implicit)]
1089        location: Location,
1090    },
1091
1092    #[snafu(display("BiErrors, first: {first}, second: {second}"))]
1093    BiErrors {
1094        first: Box<Error>,
1095        second: Box<Error>,
1096        #[snafu(implicit)]
1097        location: Location,
1098    },
1099
1100    #[snafu(display("Encode null value"))]
1101    IndexEncodeNull {
1102        #[snafu(implicit)]
1103        location: Location,
1104    },
1105
1106    #[snafu(display("Failed to encode memtable to Parquet bytes"))]
1107    EncodeMemtable {
1108        #[snafu(source)]
1109        error: parquet::errors::ParquetError,
1110        #[snafu(implicit)]
1111        location: Location,
1112    },
1113
1114    #[snafu(display("Partition {} out of range, {} in total", given, all))]
1115    PartitionOutOfRange {
1116        given: usize,
1117        all: usize,
1118        #[snafu(implicit)]
1119        location: Location,
1120    },
1121
1122    #[snafu(display("Failed to iter data part"))]
1123    ReadDataPart {
1124        #[snafu(implicit)]
1125        location: Location,
1126        #[snafu(source)]
1127        error: parquet::errors::ParquetError,
1128    },
1129
1130    #[snafu(display("Failed to read row group in memtable"))]
1131    DecodeArrowRowGroup {
1132        #[snafu(source)]
1133        error: ArrowError,
1134        #[snafu(implicit)]
1135        location: Location,
1136    },
1137
1138    #[snafu(display("Invalid region options, {}", reason))]
1139    InvalidRegionOptions {
1140        reason: String,
1141        #[snafu(implicit)]
1142        location: Location,
1143    },
1144
1145    #[snafu(display("checksum mismatch (actual: {}, expected: {})", actual, expected))]
1146    ChecksumMismatch { actual: u32, expected: u32 },
1147
1148    #[snafu(display(
1149        "No checkpoint found, region: {}, last_version: {}",
1150        region_id,
1151        last_version
1152    ))]
1153    NoCheckpoint {
1154        region_id: RegionId,
1155        last_version: ManifestVersion,
1156        #[snafu(implicit)]
1157        location: Location,
1158    },
1159
1160    #[snafu(display(
1161        "No manifests found in range: [{}..{}), region: {}, last_version: {}",
1162        start_version,
1163        end_version,
1164        region_id,
1165        last_version
1166    ))]
1167    NoManifests {
1168        region_id: RegionId,
1169        start_version: ManifestVersion,
1170        end_version: ManifestVersion,
1171        last_version: ManifestVersion,
1172        #[snafu(implicit)]
1173        location: Location,
1174    },
1175
1176    #[snafu(display(
1177        "Failed to install manifest to {}, region: {}, available manifest version: {}, last version: {}",
1178        target_version,
1179        region_id,
1180        available_version,
1181        last_version
1182    ))]
1183    InstallManifestTo {
1184        region_id: RegionId,
1185        target_version: ManifestVersion,
1186        available_version: ManifestVersion,
1187        #[snafu(implicit)]
1188        location: Location,
1189        last_version: ManifestVersion,
1190    },
1191
1192    #[snafu(display("Region {} is stopped", region_id))]
1193    RegionStopped {
1194        region_id: RegionId,
1195        #[snafu(implicit)]
1196        location: Location,
1197    },
1198
1199    #[snafu(display(
1200        "Time range predicate overflows, timestamp: {:?}, target unit: {}",
1201        timestamp,
1202        unit
1203    ))]
1204    TimeRangePredicateOverflow {
1205        timestamp: Timestamp,
1206        unit: TimeUnit,
1207        #[snafu(implicit)]
1208        location: Location,
1209    },
1210
1211    #[snafu(display("Failed to open region"))]
1212    OpenRegion {
1213        #[snafu(implicit)]
1214        location: Location,
1215        source: Arc<Error>,
1216    },
1217
1218    #[snafu(display(
1219        "Region {} does not satisfy requirement '{}': {}",
1220        region_id,
1221        requirement,
1222        reason
1223    ))]
1224    RegionRequirement {
1225        region_id: RegionId,
1226        requirement: &'static str,
1227        reason: &'static str,
1228        #[snafu(implicit)]
1229        location: Location,
1230    },
1231
1232    #[snafu(display("Failed to parse job id"))]
1233    ParseJobId {
1234        #[snafu(implicit)]
1235        location: Location,
1236        #[snafu(source)]
1237        error: uuid::Error,
1238    },
1239
1240    #[snafu(display("Operation is not supported: {}", err_msg))]
1241    UnsupportedOperation {
1242        err_msg: String,
1243        #[snafu(implicit)]
1244        location: Location,
1245    },
1246
1247    #[snafu(display(
1248        "Failed to remotely compact region {} by job {:?} due to {}",
1249        region_id,
1250        job_id,
1251        reason
1252    ))]
1253    RemoteCompaction {
1254        region_id: RegionId,
1255        job_id: Option<JobId>,
1256        reason: String,
1257        #[snafu(implicit)]
1258        location: Location,
1259    },
1260
1261    #[snafu(display("Failed to initialize puffin stager"))]
1262    PuffinInitStager {
1263        source: puffin::error::Error,
1264        #[snafu(implicit)]
1265        location: Location,
1266    },
1267
1268    #[snafu(display("Failed to purge puffin stager"))]
1269    PuffinPurgeStager {
1270        source: puffin::error::Error,
1271        #[snafu(implicit)]
1272        location: Location,
1273    },
1274
1275    #[snafu(display("Failed to build puffin reader"))]
1276    PuffinBuildReader {
1277        source: puffin::error::Error,
1278        #[snafu(implicit)]
1279        location: Location,
1280    },
1281
1282    #[snafu(display("Failed to retrieve index options from column metadata"))]
1283    IndexOptions {
1284        #[snafu(implicit)]
1285        location: Location,
1286        source: datatypes::error::Error,
1287        column_name: String,
1288    },
1289
1290    #[snafu(display("Failed to retrieve compression options of column {column_name}"))]
1291    CompressionOptions {
1292        #[snafu(implicit)]
1293        location: Location,
1294        source: datatypes::error::Error,
1295        column_name: String,
1296    },
1297
1298    #[snafu(display("Failed to create fulltext index creator"))]
1299    CreateFulltextCreator {
1300        source: index::fulltext_index::error::Error,
1301        #[snafu(implicit)]
1302        location: Location,
1303    },
1304
1305    #[snafu(display("Failed to cast vector of {from} to {to}"))]
1306    CastVector {
1307        #[snafu(implicit)]
1308        location: Location,
1309        from: ConcreteDataType,
1310        to: ConcreteDataType,
1311        source: datatypes::error::Error,
1312    },
1313
1314    #[snafu(display("Failed to push text to fulltext index"))]
1315    FulltextPushText {
1316        source: index::fulltext_index::error::Error,
1317        #[snafu(implicit)]
1318        location: Location,
1319    },
1320
1321    #[snafu(display("Failed to finalize fulltext index creator"))]
1322    FulltextFinish {
1323        source: index::fulltext_index::error::Error,
1324        #[snafu(implicit)]
1325        location: Location,
1326    },
1327
1328    #[snafu(display("Failed to apply fulltext index"))]
1329    ApplyFulltextIndex {
1330        source: index::fulltext_index::error::Error,
1331        #[snafu(implicit)]
1332        location: Location,
1333    },
1334
1335    #[snafu(display("SST file {} does not contain valid stats info", file_path))]
1336    StatsNotPresent {
1337        file_path: String,
1338        #[snafu(implicit)]
1339        location: Location,
1340    },
1341
1342    #[snafu(display("Failed to decode stats of file {}", file_path))]
1343    DecodeStats {
1344        file_path: String,
1345        #[snafu(implicit)]
1346        location: Location,
1347    },
1348
1349    #[snafu(display("Invalid SST primary key range: {reason}"))]
1350    InvalidPrimaryKeyRange {
1351        reason: String,
1352        #[snafu(implicit)]
1353        location: Location,
1354    },
1355
1356    #[snafu(display("Failed to decode SST primary key range {endpoint} endpoint"))]
1357    DecodePrimaryKeyRange {
1358        endpoint: &'static str,
1359        source: mito_codec::error::Error,
1360        #[snafu(implicit)]
1361        location: Location,
1362    },
1363
1364    #[snafu(display("Region {} is busy", region_id))]
1365    RegionBusy {
1366        region_id: RegionId,
1367        #[snafu(implicit)]
1368        location: Location,
1369    },
1370
1371    #[snafu(display("Failed to get schema metadata"))]
1372    GetSchemaMetadata {
1373        source: common_meta::error::Error,
1374        #[snafu(implicit)]
1375        location: Location,
1376    },
1377
1378    #[snafu(display("Timeout"))]
1379    Timeout {
1380        #[snafu(source)]
1381        error: Elapsed,
1382        #[snafu(implicit)]
1383        location: Location,
1384    },
1385
1386    #[snafu(display("Failed to read file metadata"))]
1387    Metadata {
1388        #[snafu(source)]
1389        error: std::io::Error,
1390        #[snafu(implicit)]
1391        location: Location,
1392    },
1393
1394    #[snafu(display("Failed to push value to bloom filter"))]
1395    PushBloomFilterValue {
1396        source: index::bloom_filter::error::Error,
1397        #[snafu(implicit)]
1398        location: Location,
1399    },
1400
1401    #[snafu(display("Failed to finish bloom filter"))]
1402    BloomFilterFinish {
1403        source: index::bloom_filter::error::Error,
1404        #[snafu(implicit)]
1405        location: Location,
1406    },
1407
1408    #[snafu(display("Manual compaction is override by following operations."))]
1409    ManualCompactionOverride {},
1410
1411    #[snafu(display("Manual compaction is already running for region {region_id}."))]
1412    ManualCompactionAlreadyRunning { region_id: RegionId },
1413
1414    #[snafu(display("Compaction is cancelled."))]
1415    CompactionCancelled {},
1416
1417    #[snafu(display("SST write is cancelled."))]
1418    SstWriteCancelled {},
1419
1420    #[snafu(display("Flush is cancelled."))]
1421    FlushCancelled {},
1422
1423    #[snafu(display("Compaction memory exhausted for region {region_id} (policy: {policy})",))]
1424    CompactionMemoryExhausted {
1425        region_id: RegionId,
1426        policy: String,
1427        #[snafu(source)]
1428        source: common_memory_manager::Error,
1429        #[snafu(implicit)]
1430        location: Location,
1431    },
1432
1433    #[snafu(display(
1434        "Incompatible WAL provider change. This is typically caused by changing WAL provider in database config file without completely cleaning existing files. Global provider: {}, region provider: {}",
1435        global,
1436        region
1437    ))]
1438    IncompatibleWalProviderChange { global: String, region: String },
1439
1440    #[snafu(display("Expected mito manifest info"))]
1441    MitoManifestInfo {
1442        #[snafu(implicit)]
1443        location: Location,
1444    },
1445
1446    #[snafu(display("Failed to scan series"))]
1447    ScanSeries {
1448        #[snafu(implicit)]
1449        location: Location,
1450        source: Arc<Error>,
1451    },
1452
1453    #[snafu(display("Partition {} scan multiple times", partition))]
1454    ScanMultiTimes {
1455        partition: usize,
1456        #[snafu(implicit)]
1457        location: Location,
1458    },
1459
1460    #[snafu(display("Invalid partition expression: {}", expr))]
1461    InvalidPartitionExpr {
1462        expr: String,
1463        #[snafu(implicit)]
1464        location: Location,
1465        source: partition::error::Error,
1466    },
1467
1468    #[snafu(display("Failed to decode bulk wal entry"))]
1469    ConvertBulkWalEntry {
1470        #[snafu(implicit)]
1471        location: Location,
1472        source: common_grpc::Error,
1473    },
1474
1475    #[snafu(display("Failed to encode"))]
1476    Encode {
1477        #[snafu(implicit)]
1478        location: Location,
1479        source: mito_codec::error::Error,
1480    },
1481
1482    #[snafu(display("Failed to decode"))]
1483    Decode {
1484        #[snafu(implicit)]
1485        location: Location,
1486        source: mito_codec::error::Error,
1487    },
1488
1489    #[snafu(display("Unexpected: {reason}"))]
1490    Unexpected {
1491        reason: String,
1492        #[snafu(implicit)]
1493        location: Location,
1494    },
1495
1496    #[cfg(feature = "enterprise")]
1497    #[snafu(display("Failed to scan external range"))]
1498    ScanExternalRange {
1499        source: BoxedError,
1500        #[snafu(implicit)]
1501        location: Location,
1502    },
1503
1504    #[snafu(display(
1505        "Inconsistent timestamp column length, expect: {}, actual: {}",
1506        expected,
1507        actual
1508    ))]
1509    InconsistentTimestampLength {
1510        expected: usize,
1511        actual: usize,
1512        #[snafu(implicit)]
1513        location: Location,
1514    },
1515
1516    #[snafu(display(
1517        "Too many files to read concurrently: {}, max allowed: {}",
1518        actual,
1519        max
1520    ))]
1521    TooManyFilesToRead {
1522        actual: usize,
1523        max: usize,
1524        #[snafu(implicit)]
1525        location: Location,
1526    },
1527
1528    #[snafu(display("Duration out of range: {input:?}"))]
1529    DurationOutOfRange {
1530        input: std::time::Duration,
1531        #[snafu(source)]
1532        error: chrono::OutOfRangeError,
1533        #[snafu(implicit)]
1534        location: Location,
1535    },
1536
1537    #[snafu(display("GC job permit exhausted"))]
1538    TooManyGcJobs {
1539        #[snafu(implicit)]
1540        location: Location,
1541    },
1542
1543    #[snafu(display(
1544        "Staging partition expr mismatch, manifest: {:?}, request: {}",
1545        manifest_expr,
1546        request_expr
1547    ))]
1548    StagingPartitionExprMismatch {
1549        manifest_expr: Option<String>,
1550        request_expr: String,
1551        #[snafu(implicit)]
1552        location: Location,
1553    },
1554
1555    #[snafu(display(
1556        "Invalid source and target region, source: {}, target: {}",
1557        source_region_id,
1558        target_region_id
1559    ))]
1560    InvalidSourceAndTargetRegion {
1561        source_region_id: RegionId,
1562        target_region_id: RegionId,
1563        #[snafu(implicit)]
1564        location: Location,
1565    },
1566
1567    #[snafu(display("Failed to prune file"))]
1568    PruneFile {
1569        source: Arc<Error>,
1570        #[snafu(implicit)]
1571        location: Location,
1572    },
1573
1574    #[snafu(display("Failed to cast column"))]
1575    CastColumn {
1576        #[snafu(source)]
1577        error: datafusion::error::DataFusionError,
1578        #[snafu(implicit)]
1579        location: Location,
1580    },
1581
1582    #[snafu(display("Failed to generate Arrow schema from Parquet file: {}", file))]
1583    ParquetToArrowSchema {
1584        file: String,
1585        #[snafu(source)]
1586        error: parquet::errors::ParquetError,
1587        #[snafu(implicit)]
1588        location: Location,
1589    },
1590
1591    #[snafu(display(
1592        "Region {} is in {:?} state, expect: Writable, Staging or Downgrading",
1593        region_id,
1594        state
1595    ))]
1596    FlushableRegionState {
1597        region_id: RegionId,
1598        state: RegionRoleState,
1599        #[snafu(implicit)]
1600        location: Location,
1601    },
1602}
1603
1604pub type Result<T, E = Error> = std::result::Result<T, E>;
1605
1606impl Error {
1607    /// Returns true if we need to fill default value for a region.
1608    pub(crate) fn is_fill_default(&self) -> bool {
1609        matches!(self, Error::FillDefault { .. })
1610    }
1611
1612    /// Returns true if the file is not found on the object store.
1613    pub(crate) fn is_object_not_found(&self) -> bool {
1614        match self {
1615            Error::OpenDal { error, .. } => error.kind() == ErrorKind::NotFound,
1616            Error::ManifestDeltaNotFound { .. } => true,
1617            Error::LoadSstMeta { source, .. }
1618            | Error::LoadParquetPage { source, .. }
1619            | Error::LoadRangeResult { source, .. } => source.is_object_not_found(),
1620            _ => false,
1621        }
1622    }
1623
1624    /// Returns whether a failed manifest update may have been persisted.
1625    ///
1626    /// Unknown errors are treated conservatively because deleting output SSTs after an ambiguous
1627    /// manifest write could leave the manifest referencing missing files. The retry layer returns
1628    /// the last attempt's error, so an error that can follow a stored attempt (throttling, or a 403
1629    /// from a revoked key, a changed bucket policy or clock skew) never proves nothing was written.
1630    pub(crate) fn may_have_persisted_manifest_update(&self) -> bool {
1631        match self {
1632            // Applicability and capability checks reject the edit before storage I/O.
1633            Error::InvalidRequest { .. }
1634            | Error::UpdateManifest { .. }
1635            | Error::RegionState { .. }
1636            | Error::RegionTruncated { .. }
1637            | Error::RegionStopped { .. }
1638            | Error::RegionLeaseExpired { .. }
1639            | Error::EndStagingPending { .. }
1640            | Error::SerdeJson { .. }
1641            | Error::CompressObject { .. } => false,
1642            Error::OpenDal { error, .. } => !matches!(
1643                error.kind(),
1644                ErrorKind::Unsupported
1645                    | ErrorKind::ConfigInvalid
1646                    | ErrorKind::NotFound
1647                    | ErrorKind::IsADirectory
1648                    | ErrorKind::NotADirectory
1649                    | ErrorKind::AlreadyExists
1650                    | ErrorKind::IsSameFile
1651                    | ErrorKind::ConditionNotMatch
1652                    | ErrorKind::RangeNotSatisfied
1653            ),
1654            _ => true,
1655        }
1656    }
1657}
1658
1659impl ErrorExt for Error {
1660    fn status_code(&self) -> StatusCode {
1661        use Error::*;
1662
1663        match self {
1664            #[cfg(feature = "enterprise")]
1665            RouteMultiRegion { source, .. } => source.status_code(),
1666            #[cfg(feature = "enterprise")]
1667            InvalidMultiRegionCompaction { .. } => StatusCode::InvalidArguments,
1668            #[cfg(feature = "enterprise")]
1669            MultiRegionCompactionResource { .. } => StatusCode::RuntimeResourcesExhausted,
1670            SstWriteMemoryExceeded { .. } => StatusCode::RuntimeResourcesExhausted,
1671            DataTypeMismatch { source, .. } => source.status_code(),
1672            OpenDal { .. }
1673            | ManifestDeltaNotFound { .. }
1674            | ManifestVersionConflict { .. }
1675            | ManifestCheckpointVanished { .. }
1676            | ReadParquet { .. } => StatusCode::StorageUnavailable,
1677            WriteWal { source, .. }
1678            | ReadWal { source, .. }
1679            | ReadNodeWal { source, .. }
1680            | DeleteWal { source, .. }
1681            | WaitWalDurable { source, .. }
1682            | ResolveWalProvider { source, .. }
1683            | CheckWalHistory { source, .. } => source.status_code(),
1684            CompressObject { .. }
1685            | DecompressObject { .. }
1686            | SerdeJson { .. }
1687            | Utf8 { .. }
1688            | NewRecordBatch { .. }
1689            | RegionCorrupted { .. }
1690            | InconsistentFile { .. }
1691            | CreateDefault { .. }
1692            | InvalidParquet { .. }
1693            | OperateAbortedIndex { .. }
1694            | IndexEncodeNull { .. }
1695            | NoCheckpoint { .. }
1696            | NoManifests { .. }
1697            | FilesLost { .. }
1698            | InstallManifestTo { .. }
1699            | Unexpected { .. }
1700            | SerializeColumnMetadata { .. }
1701            | SerializeManifest { .. }
1702            | StagingPartitionExprMismatch { .. } => StatusCode::Unexpected,
1703
1704            RegionNotFound { .. } => StatusCode::RegionNotFound,
1705            ObjectStoreNotFound { .. }
1706            | InvalidScanIndex { .. }
1707            | InvalidMeta { .. }
1708            | InvalidRequest { .. }
1709            | PartitionExprVersionMismatch { .. }
1710            | FillDefault { .. }
1711            | ConvertColumnDataType { .. }
1712            | ColumnNotFound { .. }
1713            | InvalidMetadata { .. }
1714            | InvalidRegionOptions { .. }
1715            | InvalidWalReadRequest { .. }
1716            | PartitionOutOfRange { .. }
1717            | ParseJobId { .. }
1718            | DurationOutOfRange { .. }
1719            | MissingOldManifest { .. }
1720            | MissingNewManifest { .. }
1721            | MissingManifest { .. }
1722            | NoOldManifests { .. }
1723            | MissingPartitionExpr { .. }
1724            | SerializePartitionExpr { .. }
1725            | InvalidSourceAndTargetRegion { .. } => StatusCode::InvalidArguments,
1726
1727            IncrementalQueryStale { .. } | SnapshotFenceStale { .. } => StatusCode::RequestOutdated,
1728
1729            SequenceRangeUnsupported { .. } => StatusCode::Unsupported,
1730
1731            RegionSequenceDomainBroken { .. }
1732            | RegionMetadataNotFound { .. }
1733            | Join { .. }
1734            | WorkerStopped { .. }
1735            | EngineStopped { .. }
1736            | Recv { .. }
1737            | DecodeWal { .. }
1738            | ComputeArrow { .. }
1739            | EvalPartitionFilter { .. }
1740            | MergeCandidateSeries { .. }
1741            | BiErrors { .. }
1742            | StopScheduler { .. }
1743            | ComputeVector { .. }
1744            | EncodeMemtable { .. }
1745            | CreateDir { .. }
1746            | ReadDataPart { .. }
1747            | BuildEntry { .. }
1748            | Metadata { .. }
1749            | CastColumn { .. }
1750            | MitoManifestInfo { .. }
1751            | ParquetToArrowSchema { .. } => StatusCode::Internal,
1752
1753            FetchManifests { source, .. } => source.status_code(),
1754
1755            OpenRegion { source, .. } => source.status_code(),
1756
1757            WriteParquet { .. } => StatusCode::StorageUnavailable,
1758            WriteGroup { source, .. }
1759            | LoadSstMeta { source, .. }
1760            | LoadParquetPage { source, .. }
1761            | LoadRangeResult { source, .. } => source.status_code(),
1762            InvalidBatch { .. } => StatusCode::InvalidArguments,
1763            InvalidRecordBatch { .. } => StatusCode::InvalidArguments,
1764            ConvertVector { source, .. } => source.status_code(),
1765
1766            PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments,
1767            InvalidSender { .. } => StatusCode::InvalidArguments,
1768            InvalidSchedulerState { .. } => StatusCode::InvalidArguments,
1769            RegionRequirement { .. } => StatusCode::InvalidArguments,
1770            DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => {
1771                StatusCode::StorageUnavailable
1772            }
1773            FlushRegion { source, .. } | BuildIndexAsync { source, .. } => source.status_code(),
1774            IndexBuildIncomplete { .. } => StatusCode::IllegalState,
1775            RegionDropped { .. } => StatusCode::Cancelled,
1776            RegionClosed { .. } => StatusCode::Cancelled,
1777            StaleCompactionExecution { .. } => StatusCode::Cancelled,
1778            RegionTruncated { .. } => StatusCode::Cancelled,
1779            RejectWrite { .. } | RejectWriteInStaging { .. } => StatusCode::StorageUnavailable,
1780            CompactRegion { source, .. } => source.status_code(),
1781            EditRegion { source, .. } | EndStaging { source, .. } => source.status_code(),
1782            CompatReader { .. } => StatusCode::Unexpected,
1783            InvalidRegionRequest { source, .. } => source.status_code(),
1784            RegionState { .. } | UpdateManifest { .. } => StatusCode::RegionNotReady,
1785            RegionLeaseExpired { .. } | UnacknowledgedWrite { .. } => StatusCode::RegionNotReady,
1786            RegionPendingTails { .. } => StatusCode::RegionNotReady,
1787            RemoteWalUnsupported { .. } => StatusCode::Unsupported,
1788            TailEntryMissing { .. } => StatusCode::StorageUnavailable,
1789            JsonOptions { .. } => StatusCode::InvalidArguments,
1790            EmptyRegionDir { .. } | EmptyManifestDir { .. } => StatusCode::RegionNotFound,
1791            ConvertValue { source, .. } => source.status_code(),
1792            ApplyBloomFilterIndex { source, .. } => source.status_code(),
1793            InvalidPartitionExpr { source, .. } => source.status_code(),
1794            BuildIndexApplier { source, .. }
1795            | PushIndexValue { source, .. }
1796            | ApplyInvertedIndex { source, .. }
1797            | IndexFinish { source, .. } => source.status_code(),
1798            PuffinReadBlob { source, .. }
1799            | PuffinAddBlob { source, .. }
1800            | PuffinInitStager { source, .. }
1801            | PuffinBuildReader { source, .. }
1802            | PuffinPurgeStager { source, .. } => source.status_code(),
1803            CleanDir { .. } => StatusCode::Unexpected,
1804            InvalidConfig { .. } => StatusCode::InvalidArguments,
1805            StaleLogEntry { .. }
1806            | InvalidNativeHistogramSubfield { .. }
1807            | InvalidNativeHistogramFieldId { .. } => StatusCode::Unexpected,
1808
1809            External { source, .. } => source.status_code(),
1810
1811            RecordBatch { source, .. } => source.status_code(),
1812
1813            Download { .. } | Upload { .. } => StatusCode::StorageUnavailable,
1814            ChecksumMismatch { .. } => StatusCode::Unexpected,
1815            RegionStopped { .. } => StatusCode::RegionNotReady,
1816            TimeRangePredicateOverflow { .. } => StatusCode::InvalidArguments,
1817            UnsupportedOperation { .. } => StatusCode::Unsupported,
1818            RemoteCompaction { .. } => StatusCode::Unexpected,
1819
1820            IndexOptions { source, .. } => source.status_code(),
1821            CompressionOptions { source, .. } => source.status_code(),
1822            CreateFulltextCreator { source, .. } => source.status_code(),
1823            CastVector { source, .. } => source.status_code(),
1824            FulltextPushText { source, .. }
1825            | FulltextFinish { source, .. }
1826            | ApplyFulltextIndex { source, .. } => source.status_code(),
1827            DecodeStats { .. }
1828            | StatsNotPresent { .. }
1829            | InvalidPrimaryKeyRange { .. }
1830            | DecodePrimaryKeyRange { .. } => StatusCode::Internal,
1831            RegionBusy { .. } | EndStagingPending { .. } => StatusCode::RegionBusy,
1832            GetSchemaMetadata { source, .. } => source.status_code(),
1833            Timeout { .. } => StatusCode::Cancelled,
1834
1835            DecodeArrowRowGroup { .. } => StatusCode::Internal,
1836
1837            PushBloomFilterValue { source, .. } | BloomFilterFinish { source, .. } => {
1838                source.status_code()
1839            }
1840
1841            ManualCompactionOverride {}
1842            | CompactionCancelled {}
1843            | FlushCancelled {}
1844            | SstWriteCancelled {} => StatusCode::Cancelled,
1845
1846            // A concurrent manual compaction fails fast instead of being queued;
1847            // the conflict is reported to the caller, which decides whether to
1848            // issue a new request after the running one finishes.
1849            ManualCompactionAlreadyRunning { .. } => StatusCode::RegionBusy,
1850
1851            ScanMemoryExhausted { .. } => StatusCode::RuntimeResourcesExhausted,
1852
1853            CompactionMemoryExhausted { source, .. } => source.status_code(),
1854
1855            IncompatibleWalProviderChange { .. } => StatusCode::InvalidArguments,
1856
1857            ScanSeries { source, .. } => source.status_code(),
1858
1859            ScanMultiTimes { .. } => StatusCode::InvalidArguments,
1860            ConvertBulkWalEntry { source, .. } => source.status_code(),
1861
1862            Encode { source, .. } | Decode { source, .. } => source.status_code(),
1863
1864            #[cfg(feature = "enterprise")]
1865            ScanExternalRange { source, .. } => source.status_code(),
1866
1867            InconsistentTimestampLength { .. } => StatusCode::InvalidArguments,
1868
1869            TooManyFilesToRead { .. } | TooManyGcJobs { .. } => StatusCode::RateLimited,
1870
1871            PruneFile { source, .. } => source.status_code(),
1872
1873            FlushableRegionState { .. } => StatusCode::RegionNotReady,
1874        }
1875    }
1876
1877    fn as_any(&self) -> &dyn Any {
1878        self
1879    }
1880
1881    fn retry_hint(&self) -> RetryHint {
1882        use Error::*;
1883
1884        match self {
1885            PartitionExprVersionMismatch { .. }
1886            | RegionNotFound { .. }
1887            | RegionClosed { .. }
1888            | RejectWriteInStaging { .. }
1889            | RegionLeaseExpired { .. }
1890            | UnacknowledgedWrite { .. }
1891            | RegionState {
1892                state: RegionRoleState::Follower,
1893                ..
1894            } => RetryHint::RetryAfterRouteRefresh,
1895            ReadParquet { .. }
1896            | WriteParquet { .. }
1897            | RejectWrite { .. }
1898            | Download { .. }
1899            | Upload { .. }
1900            | RegionState { .. }
1901            | UpdateManifest { .. }
1902            | RegionStopped { .. }
1903            | RegionBusy { .. }
1904            | RegionPendingTails { .. }
1905            | EndStaging { .. }
1906            | EndStagingPending { .. }
1907            | ManualCompactionAlreadyRunning { .. }
1908            | FlushableRegionState { .. }
1909            | ManifestDeltaNotFound { .. }
1910            | ManifestVersionConflict { .. }
1911            | ManifestCheckpointVanished { .. } => RetryHint::Retryable,
1912
1913            OpenDal { error, .. }
1914            | DeleteSsts { error, .. }
1915            | DeleteIndex { error, .. }
1916            | DeleteIndexes { error, .. } => retry_hint_from_opendal_error(error),
1917
1918            WriteWal { source, .. }
1919            | ReadWal { source, .. }
1920            | ReadNodeWal { source, .. }
1921            | DeleteWal { source, .. }
1922            | WaitWalDurable { source, .. }
1923            | ResolveWalProvider { source, .. }
1924            | CheckWalHistory { source, .. }
1925            | FetchManifests { source, .. }
1926            | External { source, .. } => source.retry_hint(),
1927
1928            OpenRegion { source, .. }
1929            | LoadSstMeta { source, .. }
1930            | LoadParquetPage { source, .. }
1931            | LoadRangeResult { source, .. }
1932            | WriteGroup { source, .. }
1933            | FlushRegion { source, .. }
1934            | BuildIndexAsync { source, .. }
1935            | CompactRegion { source, .. }
1936            | EditRegion { source, .. }
1937            | ScanSeries { source, .. }
1938            | PruneFile { source, .. } => source.retry_hint(),
1939
1940            DataTypeMismatch { source, .. }
1941            | ConvertVector { source, .. }
1942            | ConvertValue { source, .. }
1943            | IndexOptions { source, .. }
1944            | CompressionOptions { source, .. }
1945            | CastVector { source, .. } => source.retry_hint(),
1946
1947            BuildIndexApplier { source, .. }
1948            | PushIndexValue { source, .. }
1949            | ApplyInvertedIndex { source, .. }
1950            | IndexFinish { source, .. } => source.retry_hint(),
1951
1952            ApplyBloomFilterIndex { source, .. }
1953            | PushBloomFilterValue { source, .. }
1954            | BloomFilterFinish { source, .. } => source.retry_hint(),
1955
1956            PuffinReadBlob { source, .. }
1957            | PuffinAddBlob { source, .. }
1958            | PuffinInitStager { source, .. }
1959            | PuffinBuildReader { source, .. }
1960            | PuffinPurgeStager { source, .. } => source.retry_hint(),
1961
1962            CreateFulltextCreator { source, .. }
1963            | FulltextPushText { source, .. }
1964            | FulltextFinish { source, .. }
1965            | ApplyFulltextIndex { source, .. } => source.retry_hint(),
1966
1967            InvalidRegionRequest { source, .. } => source.retry_hint(),
1968            InvalidPartitionExpr { source, .. } => source.retry_hint(),
1969            RecordBatch { source, .. } => source.retry_hint(),
1970            GetSchemaMetadata { source, .. } => source.retry_hint(),
1971            CompactionMemoryExhausted { source, .. } => source.retry_hint(),
1972            ConvertBulkWalEntry { source, .. } => source.retry_hint(),
1973            Encode { source, .. } | Decode { source, .. } => source.retry_hint(),
1974
1975            #[cfg(feature = "enterprise")]
1976            ScanExternalRange { source, .. } => source.retry_hint(),
1977
1978            _ => RetryHint::NonRetryable,
1979        }
1980    }
1981}
1982
1983#[cfg(test)]
1984mod tests {
1985    use snafu::IntoError;
1986
1987    use super::*;
1988    use crate::region::RegionLeaderState;
1989
1990    #[test]
1991    fn test_partition_version_mismatch_retry_hint() {
1992        let error = PartitionExprVersionMismatchSnafu {
1993            region_id: RegionId::new(1, 1),
1994            request_version: 1u64,
1995            expected_version: 2u64,
1996        }
1997        .build();
1998        assert_eq!(error.status_code(), StatusCode::InvalidArguments);
1999        assert_eq!(error.retry_hint(), RetryHint::RetryAfterRouteRefresh);
2000    }
2001
2002    #[test]
2003    fn test_region_state_retry_hint() {
2004        let region_id = RegionId::new(1, 1);
2005        let writable = RegionRoleState::Leader(RegionLeaderState::Writable);
2006        let follower_error = RegionStateSnafu {
2007            region_id,
2008            state: RegionRoleState::Follower,
2009            expect: writable,
2010        }
2011        .build();
2012        assert_eq!(
2013            follower_error.retry_hint(),
2014            RetryHint::RetryAfterRouteRefresh
2015        );
2016        assert_eq!(follower_error.status_code(), StatusCode::RegionNotReady);
2017        let leader_error = RegionStateSnafu {
2018            region_id,
2019            state: writable,
2020            expect: RegionRoleState::Follower,
2021        }
2022        .build();
2023        assert_eq!(leader_error.retry_hint(), RetryHint::Retryable);
2024        for (error, status, hint) in [
2025            (
2026                RejectWriteSnafu { region_id }.build(),
2027                StatusCode::StorageUnavailable,
2028                RetryHint::Retryable,
2029            ),
2030            (
2031                RejectWriteInStagingSnafu { region_id }.build(),
2032                StatusCode::StorageUnavailable,
2033                RetryHint::RetryAfterRouteRefresh,
2034            ),
2035            (
2036                RegionNotFoundSnafu { region_id }.build(),
2037                StatusCode::RegionNotFound,
2038                RetryHint::RetryAfterRouteRefresh,
2039            ),
2040            (
2041                RegionClosedSnafu { region_id }.build(),
2042                StatusCode::Cancelled,
2043                RetryHint::RetryAfterRouteRefresh,
2044            ),
2045            (
2046                RegionDroppedSnafu { region_id }.build(),
2047                StatusCode::Cancelled,
2048                RetryHint::NonRetryable,
2049            ),
2050            (
2051                RegionTruncatedSnafu { region_id }.build(),
2052                StatusCode::Cancelled,
2053                RetryHint::NonRetryable,
2054            ),
2055        ] {
2056            assert_eq!(error.status_code(), status);
2057            assert_eq!(error.retry_hint(), hint);
2058        }
2059    }
2060
2061    #[test]
2062    fn test_shared_sst_metadata_preserves_error_semantics() {
2063        for source in [
2064            OpenDalSnafu {}
2065                .into_error(object_store::Error::new(ErrorKind::NotFound, "missing SST")),
2066            OpenDalSnafu {}.into_error(object_store::Error::new(
2067                ErrorKind::Unexpected,
2068                "read failed",
2069            )),
2070            RejectWriteSnafu {
2071                region_id: RegionId::new(1, 1),
2072            }
2073            .build(),
2074        ] {
2075            let expected_not_found = source.is_object_not_found();
2076            let expected_status = source.status_code();
2077            let expected_retry = source.retry_hint();
2078            let source = Arc::new(source);
2079            for wrapped in [
2080                LoadSstMetaSnafu.into_error(source.clone()),
2081                LoadRangeResultSnafu.into_error(source),
2082            ] {
2083                assert_eq!(expected_not_found, wrapped.is_object_not_found());
2084                assert_eq!(expected_status, wrapped.status_code());
2085                assert_eq!(expected_retry, wrapped.retry_hint());
2086            }
2087        }
2088    }
2089
2090    #[test]
2091    fn test_manifest_update_persistence() {
2092        let rejected_kinds = [
2093            ErrorKind::Unsupported,
2094            ErrorKind::ConfigInvalid,
2095            ErrorKind::NotFound,
2096            ErrorKind::IsADirectory,
2097            ErrorKind::NotADirectory,
2098            ErrorKind::AlreadyExists,
2099            ErrorKind::IsSameFile,
2100            ErrorKind::ConditionNotMatch,
2101            ErrorKind::RangeNotSatisfied,
2102        ];
2103        for kind in rejected_kinds {
2104            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
2105            assert!(
2106                !error.may_have_persisted_manifest_update(),
2107                "error kind {kind:?} should prove the manifest was not persisted"
2108            );
2109        }
2110
2111        // The last retry can fail with these kinds even though an earlier attempt was stored.
2112        for kind in [
2113            ErrorKind::Unexpected,
2114            ErrorKind::RateLimited,
2115            ErrorKind::PermissionDenied,
2116        ] {
2117            let error = OpenDalSnafu {}.into_error(object_store::Error::new(kind, "test"));
2118            assert!(
2119                error.may_have_persisted_manifest_update(),
2120                "error kind {kind:?} should not prove the manifest was not persisted"
2121            );
2122        }
2123
2124        let region_id = RegionId::new(1, 1);
2125        let error = RegionTruncatedSnafu { region_id }.build();
2126        assert!(!error.may_have_persisted_manifest_update());
2127        let error = InvalidRequestSnafu {
2128            region_id,
2129            reason: "manifest applicability check rejected the edit",
2130        }
2131        .build();
2132        assert!(!error.may_have_persisted_manifest_update());
2133        // Staging that waits for a retried end commit refuses before storage I/O.
2134        let error = EndStagingPendingSnafu { region_id }.build();
2135        assert!(!error.may_have_persisted_manifest_update());
2136        assert!(error.retry_hint().is_retryable());
2137        assert_eq!(error.status_code(), StatusCode::RegionBusy);
2138        // Other busy errors keep the conservative classification.
2139        let error = RegionBusySnafu { region_id }.build();
2140        assert!(error.may_have_persisted_manifest_update());
2141
2142        // A commit refused before its PUT because the Leader deadline has passed.
2143        let error = RegionLeaseExpiredSnafu { region_id }.build();
2144        assert!(!error.may_have_persisted_manifest_update());
2145
2146        // This can be raised while applying an edit after its manifest file was saved.
2147        let error = RegionMetadataNotFoundSnafu {}.build();
2148        assert!(error.may_have_persisted_manifest_update());
2149
2150        let error = ManifestVersionConflictSnafu {
2151            version: 1u64,
2152            manifest_dir: "manifest/",
2153            reason: "test",
2154        }
2155        .build();
2156        assert!(error.may_have_persisted_manifest_update());
2157        assert!(error.retry_hint().is_retryable());
2158    }
2159}