Skip to main content

mito2/cache/
write_cache.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! A write-through cache for remote object stores.
16
17use std::sync::Arc;
18use std::time::{Duration, Instant};
19
20use common_base::readable_size::ReadableSize;
21use common_telemetry::{debug, info};
22use object_store::ObjectStore;
23use snafu::{IntoError, ResultExt};
24use store_api::storage::RegionId;
25use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
26use tokio_util::sync::CancellationToken;
27
28use crate::access_layer::{
29    EncodedSstIndexBuildParams, FilePathProvider, Metrics, OperationType, RegionFilePathFactory,
30    SstInfoArray, SstWriteRequest, TempFileCleaner, WriteCachePathProvider, new_fs_cache_store,
31};
32use crate::cache::file_cache::{FileCache, FileCacheRef, FileType, IndexKey, IndexValue};
33use crate::cache::manifest_cache::ManifestCache;
34use crate::error::{self, Result};
35use crate::metrics::UPLOAD_BYTES_TOTAL;
36use crate::region::opener::RegionLoadCacheTask;
37use crate::sst::file::RegionFileId;
38use crate::sst::index::intermediate::IntermediateManager;
39use crate::sst::index::puffin_manager::{PuffinManagerFactory, SstPuffinManager};
40use crate::sst::index::registry::IndexRegistry;
41use crate::sst::index::{IndexerBuilderImpl, build_index_from_parquet_bytes};
42use crate::sst::parquet::writer::{ParquetWriter, SstWriteContext};
43use crate::sst::parquet::{SstInfo, WriteOptions};
44use crate::sst::upload::{self, UploadError};
45use crate::sst::{DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY};
46
47/// Wraps the remote object store used by write cache uploads.
48pub trait WriteCacheUploadStoreWrapper: Send + Sync {
49    /// Wraps an object store before uploading a cached file.
50    ///
51    /// `op_type` identifies the origin of the upload so implementations can
52    /// apply different policies per operation, e.g. throttling compaction
53    /// uploads but not flush uploads. Index rebuild uploads are reported as
54    /// [`OperationType::Compact`].
55    fn wrap(&self, store: ObjectStore, op_type: OperationType) -> ObjectStore;
56}
57
58pub type WriteCacheUploadStoreWrapperRef = Arc<dyn WriteCacheUploadStoreWrapper>;
59
60/// A cache for uploading files to remote object stores.
61///
62/// It keeps files in local disk and then sends files to object stores.
63pub struct WriteCache {
64    /// Local file cache.
65    file_cache: FileCacheRef,
66    /// Puffin manager factory for index.
67    puffin_manager_factory: PuffinManagerFactory,
68    /// Intermediate manager for index.
69    intermediate_manager: IntermediateManager,
70    /// Sender for region load cache tasks.
71    task_sender: UnboundedSender<RegionLoadCacheTask>,
72    /// Optional cache for manifest files.
73    manifest_cache: Option<ManifestCache>,
74    /// Optional wrapper for remote stores used by uploads.
75    upload_store_wrapper: Option<WriteCacheUploadStoreWrapperRef>,
76}
77
78pub type WriteCacheRef = Arc<WriteCache>;
79
80impl WriteCache {
81    /// Create the cache with a `local_store` to cache files and a
82    /// `object_store_manager` for all object stores.
83    #[allow(clippy::too_many_arguments)]
84    pub async fn new(
85        local_store: ObjectStore,
86        cache_capacity: ReadableSize,
87        ttl: Option<Duration>,
88        index_cache_percent: Option<u8>,
89        enable_background_worker: bool,
90        puffin_manager_factory: PuffinManagerFactory,
91        intermediate_manager: IntermediateManager,
92        manifest_cache: Option<ManifestCache>,
93    ) -> Result<Self> {
94        let (task_sender, task_receiver) = unbounded_channel();
95
96        let file_cache = Arc::new(FileCache::new(
97            local_store,
98            cache_capacity,
99            ttl,
100            index_cache_percent,
101            enable_background_worker,
102        ));
103        file_cache.recover(false, Some(task_receiver)).await;
104
105        Ok(Self {
106            file_cache,
107            puffin_manager_factory,
108            intermediate_manager,
109            task_sender,
110            manifest_cache,
111            upload_store_wrapper: None,
112        })
113    }
114
115    /// Creates a write cache based on local fs.
116    #[allow(clippy::too_many_arguments)]
117    pub async fn new_fs(
118        cache_dir: &str,
119        cache_capacity: ReadableSize,
120        ttl: Option<Duration>,
121        index_cache_percent: Option<u8>,
122        enable_background_worker: bool,
123        puffin_manager_factory: PuffinManagerFactory,
124        intermediate_manager: IntermediateManager,
125        manifest_cache_capacity: ReadableSize,
126    ) -> Result<Self> {
127        info!("Init write cache on {cache_dir}, capacity: {cache_capacity}");
128
129        let local_store = new_fs_cache_store(cache_dir).await?;
130
131        // Create manifest cache if capacity is non-zero
132        let manifest_cache = if manifest_cache_capacity.as_bytes() > 0 {
133            Some(ManifestCache::new(local_store.clone(), manifest_cache_capacity, ttl, false).await)
134        } else {
135            None
136        };
137
138        Self::new(
139            local_store,
140            cache_capacity,
141            ttl,
142            index_cache_percent,
143            enable_background_worker,
144            puffin_manager_factory,
145            intermediate_manager,
146            manifest_cache,
147        )
148        .await
149    }
150
151    /// Returns the file cache of the write cache.
152    pub(crate) fn file_cache(&self) -> FileCacheRef {
153        self.file_cache.clone()
154    }
155
156    /// Returns the manifest cache if available.
157    pub(crate) fn manifest_cache(&self) -> Option<ManifestCache> {
158        self.manifest_cache.clone()
159    }
160
161    /// Sets the wrapper for remote stores used by uploads.
162    pub(crate) fn with_upload_store_wrapper(
163        mut self,
164        upload_store_wrapper: Option<WriteCacheUploadStoreWrapperRef>,
165    ) -> Self {
166        self.upload_store_wrapper = upload_store_wrapper;
167        self
168    }
169
170    /// Build the puffin manager
171    pub(crate) fn build_puffin_manager(&self) -> SstPuffinManager {
172        let store = self.file_cache.local_store();
173        let path_provider = WriteCachePathProvider::new(self.file_cache.clone());
174        self.puffin_manager_factory.build(store, path_provider)
175    }
176
177    /// Put encoded SST data to the cache and upload to the remote object store.
178    ///
179    /// Builds the SST's indexes from `data` when `index_build_params` is set and uploads them
180    /// with the SST.
181    #[allow(clippy::too_many_arguments)]
182    pub(crate) async fn put_and_upload_sst(
183        &self,
184        data: &bytes::Bytes,
185        region_id: RegionId,
186        sst_info: &mut SstInfo,
187        upload_request: SstUploadRequest,
188        write_buffer_size: ReadableSize,
189        index_build_params: Option<EncodedSstIndexBuildParams>,
190        metrics: &mut Metrics,
191    ) -> Result<()> {
192        let file_id = sst_info.file_id;
193        let region_file_id = RegionFileId::new(region_id, file_id);
194
195        // Create index key for the SST file
196        let parquet_key = IndexKey::new(region_id, file_id, FileType::Parquet);
197
198        // Write to cache first
199        let cache_start = Instant::now();
200        let cache_path = self.file_cache.cache_file_path(parquet_key);
201        let store = self.file_cache.local_store();
202        let cleaner = TempFileCleaner::new(region_id, store.clone());
203        let write_res = store
204            .write_with(&cache_path, data.clone())
205            .chunk(write_buffer_size.as_bytes() as usize)
206            .await
207            .context(crate::error::OpenDalSnafu);
208        if let Err(e) = write_res {
209            cleaner.clean_by_file_id(file_id).await;
210            return Err(e);
211        }
212
213        metrics.write_batch = cache_start.elapsed();
214
215        if let Some(params) = index_build_params {
216            let start = Instant::now();
217            let builder = params.indexer_builder(
218                self.build_puffin_manager(),
219                true,
220                self.intermediate_manager.clone(),
221            );
222            sst_info.index_metadata =
223                build_index_from_parquet_bytes(&builder, region_file_id, data).await;
224            metrics.update_index = start.elapsed();
225        }
226
227        // Upload to remote store
228        let remote_store = &upload_request.remote_store;
229        let mut upload_tracker = UploadTracker::new(region_id);
230        let upload_start = Instant::now();
231        let remote_path = upload_request
232            .dest_path_provider
233            .build_sst_file_path(region_file_id);
234        let mut result = self
235            .upload(
236                parquet_key,
237                &remote_path,
238                remote_store,
239                UploadOptions {
240                    // `put_and_upload_sst` is only used by the flush path.
241                    op_type: OperationType::Flush,
242                    write_buffer_size,
243                },
244            )
245            .await;
246        metrics.upload_parquet = upload_start.elapsed();
247
248        if result.is_ok() && sst_info.index_metadata.file_size > 0 {
249            upload_tracker.push_uploaded_file(remote_path);
250            let puffin_key = IndexKey::new(region_id, file_id, FileType::Puffin(0));
251            let puffin_path = upload_request
252                .dest_path_provider
253                .build_index_file_path(region_file_id);
254            let start = Instant::now();
255            result = self
256                .upload(
257                    puffin_key,
258                    &puffin_path,
259                    remote_store,
260                    UploadOptions {
261                        op_type: OperationType::Flush,
262                        write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
263                    },
264                )
265                .await;
266            metrics.upload_puffin = start.elapsed();
267        }
268
269        if let Err(e) = result {
270            upload_tracker
271                .clean(
272                    std::slice::from_ref(sst_info),
273                    &self.file_cache,
274                    remote_store,
275                )
276                .await;
277            return Err(e);
278        }
279        Ok(())
280    }
281
282    /// Returns the intermediate manager of the write cache.
283    pub(crate) fn intermediate_manager(&self) -> &IntermediateManager {
284        &self.intermediate_manager
285    }
286
287    /// Preserves write ownership and cancellation through local encoding and upload.
288    pub(crate) async fn write_and_upload_sst_with_context(
289        &self,
290        write_request: SstWriteRequest,
291        upload_request: SstUploadRequest,
292        write_opts: &WriteOptions,
293        metrics: &mut Metrics,
294        context: SstWriteContext,
295    ) -> Result<SstInfoArray> {
296        let region_id = write_request.metadata.region_id;
297        let override_sequence = if write_request.preserve_row_sequence {
298            None
299        } else {
300            write_request.max_sequence
301        };
302
303        let store = self.file_cache.local_store();
304        let path_provider = WriteCachePathProvider::new(self.file_cache.clone());
305        let indexer = IndexerBuilderImpl {
306            index_build_inputs: write_request.index_build_inputs,
307            build_type: write_request.op_type.into(),
308            metadata: write_request.metadata.clone(),
309            puffin_manager: self
310                .puffin_manager_factory
311                .build(store.clone(), path_provider.clone()),
312            write_cache_enabled: true,
313            intermediate_manager: self.intermediate_manager.clone(),
314            index_options: write_request.index_options,
315            registry: Arc::new(IndexRegistry::new(
316                write_request.inverted_index_config,
317                write_request.fulltext_index_config,
318                write_request.bloom_filter_index_config,
319            )),
320        };
321
322        let cleaner = TempFileCleaner::new(region_id, store.clone());
323        // Write to FileCache.
324        let mut writer = ParquetWriter::new_with_object_store(
325            store.clone(),
326            write_request.metadata,
327            write_request.index_config,
328            indexer,
329            path_provider.clone(),
330            metrics,
331        )
332        .await
333        .with_file_cleaner(cleaner)
334        .with_write_guard(context.guard)
335        .with_cancellation(context.cancellation.clone());
336
337        let sst_info = match write_request.sst_write_format {
338            crate::sst::FormatType::PrimaryKey => {
339                writer
340                    .write_all_flat_as_primary_key(
341                        write_request.source,
342                        override_sequence,
343                        write_opts,
344                    )
345                    .await?
346            }
347            crate::sst::FormatType::Flat => {
348                writer
349                    .write_all_flat(write_request.source, override_sequence, write_opts)
350                    .await?
351            }
352        };
353
354        // Upload sst file to remote object store.
355        if sst_info.is_empty() {
356            return Ok(sst_info);
357        }
358
359        let mut upload_tracker = UploadTracker::new(region_id);
360        let mut err = None;
361        let op_type = write_request.op_type;
362        let remote_store = &upload_request.remote_store;
363        for sst in &sst_info {
364            if context
365                .cancellation
366                .as_ref()
367                .is_some_and(|handle| handle.is_cancelled())
368            {
369                err = Some(error::SstWriteCancelledSnafu.build());
370                break;
371            }
372            let parquet_key = IndexKey::new(region_id, sst.file_id, FileType::Parquet);
373            let parquet_path = upload_request
374                .dest_path_provider
375                .build_sst_file_path(RegionFileId::new(region_id, sst.file_id));
376            let start = Instant::now();
377            if let Err(e) = self
378                .upload_with_cancellation(
379                    parquet_key,
380                    &parquet_path,
381                    remote_store,
382                    UploadOptions {
383                        op_type,
384                        write_buffer_size: write_opts.write_buffer_size,
385                    },
386                    context.cancellation.as_ref(),
387                )
388                .await
389            {
390                err = Some(e);
391                break;
392            }
393            metrics.upload_parquet += start.elapsed();
394            upload_tracker.push_uploaded_file(parquet_path);
395
396            if sst.index_metadata.file_size > 0 {
397                let puffin_key = IndexKey::new(region_id, sst.file_id, FileType::Puffin(0));
398                let puffin_path = upload_request
399                    .dest_path_provider
400                    .build_index_file_path(RegionFileId::new(region_id, sst.file_id));
401                let start = Instant::now();
402                if let Err(e) = self
403                    .upload_with_cancellation(
404                        puffin_key,
405                        &puffin_path,
406                        remote_store,
407                        UploadOptions {
408                            op_type,
409                            write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
410                        },
411                        context.cancellation.as_ref(),
412                    )
413                    .await
414                {
415                    err = Some(e);
416                    break;
417                }
418                metrics.upload_puffin += start.elapsed();
419                upload_tracker.push_uploaded_file(puffin_path);
420            }
421        }
422
423        if let Some(err) = err {
424            // Cleans files on failure.
425            upload_tracker
426                .clean(&sst_info, &self.file_cache, remote_store)
427                .await;
428            return Err(err);
429        }
430
431        Ok(sst_info)
432    }
433
434    /// Removes a file from the cache by `index_key`.
435    pub(crate) async fn remove(&self, index_key: IndexKey) {
436        self.file_cache.remove(index_key).await
437    }
438
439    /// Downloads a file in `remote_path` from the remote object store to the local cache
440    /// (specified by `index_key`).
441    pub(crate) async fn download(
442        &self,
443        index_key: IndexKey,
444        remote_path: &str,
445        remote_store: &ObjectStore,
446        file_size: u64,
447    ) -> Result<()> {
448        self.file_cache
449            .download(index_key, remote_path, remote_store, file_size)
450            .await
451    }
452
453    /// Downloads the target file into write cache only when it is not cached.
454    ///
455    /// Returns `Ok(true)` if this call performs a download, or `Ok(false)` if the
456    /// file is already present in write cache and download is skipped.
457    pub(crate) async fn download_if_absent(
458        &self,
459        index_key: IndexKey,
460        remote_path: &str,
461        remote_store: &ObjectStore,
462        file_size: u64,
463    ) -> Result<bool> {
464        if self.file_cache.contains_key(&index_key) {
465            debug!(
466                "Skip downloading file already in write cache, region: {}, file: {}",
467                index_key.region_id, index_key.file_id
468            );
469            return Ok(false);
470        }
471
472        self.download(index_key, remote_path, remote_store, file_size)
473            .await?;
474        Ok(true)
475    }
476
477    /// Uploads a Parquet file or a Puffin file to the remote object store.
478    pub(crate) async fn upload(
479        &self,
480        index_key: IndexKey,
481        upload_path: &str,
482        remote_store: &ObjectStore,
483        options: UploadOptions,
484    ) -> Result<()> {
485        self.upload_with_cancellation(index_key, upload_path, remote_store, options, None)
486            .await
487    }
488
489    async fn upload_with_cancellation(
490        &self,
491        index_key: IndexKey,
492        upload_path: &str,
493        remote_store: &ObjectStore,
494        options: UploadOptions,
495        cancellation: Option<&CancellationToken>,
496    ) -> Result<()> {
497        let region_id = index_key.region_id;
498        let file_id = index_key.file_id;
499        let cache_path = self.file_cache.cache_file_path(index_key);
500
501        let start = Instant::now();
502        let cached_value = self
503            .file_cache
504            .local_store()
505            .stat(&cache_path)
506            .await
507            .context(error::OpenDalSnafu)?;
508        let mut reader = self
509            .file_cache
510            .local_store()
511            .reader_with(&cache_path)
512            .chunk(options.write_buffer_size.as_bytes().max(1) as usize)
513            .await
514            .context(error::OpenDalSnafu)?
515            .into_stream(0..cached_value.content_length())
516            .await
517            .context(error::OpenDalSnafu)?;
518
519        let upload_store = self.upload_store_wrapper.as_ref().map_or_else(
520            || remote_store.clone(),
521            |wrapper| wrapper.wrap(remote_store.clone(), options.op_type),
522        );
523        let mut writer = upload_store
524            .writer_with(upload_path)
525            .chunk(options.write_buffer_size.as_bytes() as usize)
526            .concurrent(DEFAULT_WRITE_CONCURRENCY)
527            .await
528            .context(error::OpenDalSnafu)?;
529
530        let result = async {
531            let bytes_written = upload::copy(
532                &mut reader,
533                &mut writer,
534                options.write_buffer_size.as_bytes() as usize,
535                cancellation,
536            )
537            .await
538            .map_err(|error| match error {
539                UploadError::Cancelled => error::SstWriteCancelledSnafu.build(),
540                UploadError::Read { source } => error::OpenDalSnafu.into_error(source),
541                UploadError::Write { source } => error::UploadSnafu {
542                    region_id,
543                    file_id,
544                    file_type: index_key.file_type,
545                }
546                .into_error(std::io::Error::from(source)),
547            })?;
548            if cancellation.is_some_and(CancellationToken::is_cancelled) {
549                return error::SstWriteCancelledSnafu.fail();
550            }
551            writer
552                .close()
553                .await
554                .map_err(std::io::Error::from)
555                .context(error::UploadSnafu {
556                    region_id,
557                    file_id,
558                    file_type: index_key.file_type,
559                })?;
560            Ok(bytes_written)
561        }
562        .await;
563        let bytes_written = match result {
564            Ok(bytes_written) => bytes_written,
565            Err(error) => {
566                if let Err(abort_error) = writer.abort().await {
567                    common_telemetry::warn!(abort_error; "Failed to abort SST upload, region: {}, file: {}", region_id, file_id);
568                }
569                return Err(error);
570            }
571        };
572
573        UPLOAD_BYTES_TOTAL.inc_by(bytes_written);
574
575        debug!(
576            "Successfully upload file to remote, region: {}, file: {}, upload_path: {}, cost: {:?}",
577            region_id,
578            file_id,
579            upload_path,
580            start.elapsed(),
581        );
582
583        let index_value = IndexValue {
584            file_size: bytes_written as _,
585        };
586        // Register to file cache
587        self.file_cache.put(index_key, index_value).await;
588
589        Ok(())
590    }
591
592    /// Sends a region load cache task to the background processing queue.
593    ///
594    /// If the receiver has been dropped, the error is ignored.
595    pub(crate) fn load_region_cache(&self, task: RegionLoadCacheTask) {
596        let _ = self.task_sender.send(task);
597    }
598}
599
600/// Request to write and upload a SST.
601pub struct SstUploadRequest {
602    /// Destination path provider of which SST files in write cache should be uploaded to.
603    pub dest_path_provider: RegionFilePathFactory,
604    /// Remote object store to upload.
605    pub remote_store: ObjectStore,
606}
607
608/// Policies for uploading a cached SST or index file.
609pub(crate) struct UploadOptions {
610    /// Origin of the upload, used by the remote store wrapper.
611    pub op_type: OperationType,
612    /// Buffer size for the remote writer.
613    pub write_buffer_size: ReadableSize,
614}
615
616/// A structs to track files to upload and clean them if upload failed.
617pub(crate) struct UploadTracker {
618    /// Id of the region to track.
619    region_id: RegionId,
620    /// Paths of files uploaded successfully.
621    files_uploaded: Vec<String>,
622}
623
624impl UploadTracker {
625    /// Creates a new instance of `UploadTracker` for a given region.
626    pub(crate) fn new(region_id: RegionId) -> Self {
627        Self {
628            region_id,
629            files_uploaded: Vec::new(),
630        }
631    }
632
633    /// Add a file path to the list of uploaded files.
634    pub(crate) fn push_uploaded_file(&mut self, path: String) {
635        self.files_uploaded.push(path);
636    }
637
638    /// Cleans uploaded files and files in the file cache at best effort.
639    pub(crate) async fn clean(
640        &self,
641        sst_info: &[SstInfo],
642        file_cache: &FileCacheRef,
643        remote_store: &ObjectStore,
644    ) {
645        common_telemetry::info!(
646            "Start cleaning files on upload failure, region: {}, num_ssts: {}",
647            self.region_id,
648            sst_info.len()
649        );
650
651        // Cleans files in the file cache first.
652        for sst in sst_info {
653            let parquet_key = IndexKey::new(self.region_id, sst.file_id, FileType::Parquet);
654            file_cache.remove(parquet_key).await;
655
656            if sst.index_metadata.file_size > 0 {
657                let puffin_key = IndexKey::new(
658                    self.region_id,
659                    sst.file_id,
660                    FileType::Puffin(sst.index_metadata.version),
661                );
662                file_cache.remove(puffin_key).await;
663            }
664        }
665
666        // Cleans uploaded files.
667        for file_path in &self.files_uploaded {
668            if let Err(e) = remote_store.delete(file_path).await {
669                common_telemetry::error!(e; "Failed to delete file {}", file_path);
670            }
671        }
672    }
673}
674
675#[cfg(test)]
676mod tests {
677    use std::sync::Mutex;
678
679    use bytes::Bytes;
680    use common_test_util::temp_dir::create_temp_dir;
681    use object_store::services::Memory;
682    use object_store::{ATOMIC_WRITE_DIR, ObjectStore};
683    use parquet::file::metadata::PageIndexPolicy;
684    use store_api::region_request::PathType;
685    use store_api::storage::FileId;
686
687    use super::*;
688    use crate::access_layer::{OperationType, WriteType};
689    use crate::cache::file_cache::IndexValue;
690    use crate::cache::test_util::{assert_parquet_metadata_equal, new_fs_store};
691    use crate::cache::{CacheManager, CacheStrategy};
692    use crate::error::InvalidBatchSnafu;
693    use crate::read::FlatSource;
694    use crate::region::options::IndexOptions;
695    use crate::sst::parquet::reader::ParquetReaderBuilder;
696    use crate::test_util::TestEnv;
697    use crate::test_util::sst_util::{
698        WriteChunkRecorder, new_flat_source_from_record_batches, new_record_batch_by_range,
699        sst_file_handle_with_file_id, sst_region_metadata,
700    };
701
702    struct CancellingUploadWriter {
703        inner: object_store::layers::mock::oio::Writer,
704        cancellation: CancellationToken,
705        bytes_written: Arc<std::sync::atomic::AtomicUsize>,
706        aborted: Arc<std::sync::atomic::AtomicBool>,
707    }
708
709    impl object_store::layers::mock::oio::Write for CancellingUploadWriter {
710        async fn write(&mut self, buffer: object_store::Buffer) -> object_store::Result<()> {
711            let len = buffer.len();
712            self.inner.write(buffer).await?;
713            self.bytes_written
714                .fetch_add(len, std::sync::atomic::Ordering::Relaxed);
715            self.cancellation.cancel();
716            Ok(())
717        }
718        async fn close(
719            &mut self,
720        ) -> object_store::layers::mock::Result<object_store::layers::mock::Metadata> {
721            self.inner.close().await
722        }
723        async fn abort(&mut self) -> object_store::Result<()> {
724            self.aborted
725                .store(true, std::sync::atomic::Ordering::Relaxed);
726            self.inner.abort().await
727        }
728    }
729
730    #[tokio::test]
731    async fn test_cancel_upload_aborts_incomplete_object() {
732        use object_store::layers::mock::MockLayerBuilder;
733        use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
734
735        let env = TestEnv::new().await;
736        let local_store = ObjectStore::new(Memory::default()).unwrap();
737        let cancellation = CancellationToken::new();
738        let bytes_written = Arc::new(AtomicUsize::new(0));
739        let aborted = Arc::new(AtomicBool::new(false));
740        let signal = cancellation.clone();
741        let bytes = bytes_written.clone();
742        let abort = aborted.clone();
743        let layer = MockLayerBuilder::default()
744            .writer_factory(Arc::new(move |_, _, inner| {
745                Box::new(CancellingUploadWriter {
746                    inner,
747                    cancellation: signal.clone(),
748                    bytes_written: bytes.clone(),
749                    aborted: abort.clone(),
750                })
751            }))
752            .build()
753            .unwrap();
754        let remote_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
755        let cache = env
756            .create_write_cache(local_store.clone(), ReadableSize::mb(10))
757            .await;
758        let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
759        let data = vec![42; 1024 * 1024];
760        local_store
761            .write(&cache.file_cache.cache_file_path(key), data.clone())
762            .await
763            .unwrap();
764        let result = cache
765            .upload_with_cancellation(
766                key,
767                "cancelled.parquet",
768                &remote_store,
769                UploadOptions {
770                    op_type: OperationType::Compact,
771                    write_buffer_size: ReadableSize::kb(1),
772                },
773                Some(&cancellation),
774            )
775            .await;
776        assert!(matches!(
777            result,
778            Err(error::Error::SstWriteCancelled { .. })
779        ));
780        assert!(aborted.load(Ordering::Relaxed));
781        let bytes = bytes_written.load(Ordering::Relaxed);
782        assert!(
783            bytes > 0 && bytes < data.len(),
784            "wrote {bytes} bytes before cancelling"
785        );
786        assert!(!remote_store.exists("cancelled.parquet").await.unwrap());
787    }
788
789    struct FailingUploadWriter {
790        inner: object_store::layers::mock::oio::Writer,
791        fail_close: bool,
792        fail_abort: bool,
793        successful_writes: usize,
794    }
795
796    impl object_store::layers::mock::oio::Write for FailingUploadWriter {
797        async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
798            if !self.fail_close && self.successful_writes == 1 {
799                return Err(object_store::Error::new(
800                    object_store::ErrorKind::NotFound,
801                    "injected write failure",
802                ));
803            }
804            self.inner.write(bytes).await?;
805            self.successful_writes += 1;
806            Ok(())
807        }
808
809        async fn close(&mut self) -> object_store::Result<object_store::layers::mock::Metadata> {
810            Err(object_store::Error::new(
811                object_store::ErrorKind::NotFound,
812                "injected close failure",
813            ))
814        }
815
816        async fn abort(&mut self) -> object_store::Result<()> {
817            if self.fail_abort {
818                return Err(object_store::Error::new(
819                    object_store::ErrorKind::Unexpected,
820                    "injected abort failure",
821                ));
822            }
823            self.inner.abort().await
824        }
825    }
826
827    #[rstest::rstest]
828    #[case::write(false, false)]
829    #[case::close(true, false)]
830    #[case::write_and_abort(false, true)]
831    #[tokio::test]
832    async fn test_upload_error_preserves_file_context(
833        #[case] fail_close: bool,
834        #[case] fail_abort: bool,
835    ) {
836        use object_store::layers::mock::{MockLayerBuilder, ObservedWriter, WriterObservations};
837
838        let env = TestEnv::new().await;
839        let local_store = ObjectStore::new(Memory::default()).unwrap();
840        let observations = Arc::new(std::sync::Mutex::new(WriterObservations::default()));
841        let observed = observations.clone();
842        let layer = MockLayerBuilder::default()
843            .writer_factory(Arc::new(move |_, _, inner| {
844                Box::new(ObservedWriter::new(
845                    Box::new(FailingUploadWriter {
846                        inner,
847                        fail_close,
848                        fail_abort,
849                        successful_writes: 0,
850                    }),
851                    observed.clone(),
852                ))
853            }))
854            .build()
855            .unwrap();
856        let remote_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
857        let cache = env
858            .create_write_cache(local_store.clone(), ReadableSize::mb(10))
859            .await;
860        let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
861        local_store
862            .write(&cache.file_cache.cache_file_path(key), vec![42; 4096])
863            .await
864            .unwrap();
865        assert!(!cache.file_cache.contains_key(&key));
866        let err = cache
867            .upload(
868                key,
869                "failed.parquet",
870                &remote_store,
871                UploadOptions {
872                    op_type: OperationType::Compact,
873                    write_buffer_size: ReadableSize::kb(1),
874                },
875            )
876            .await
877            .unwrap_err();
878        assert!(!err.is_object_not_found());
879        assert!(err.may_have_persisted_manifest_update());
880        match err {
881            error::Error::Upload {
882                region_id,
883                file_id,
884                file_type,
885                error,
886                ..
887            } => {
888                assert_eq!(key.region_id, region_id);
889                assert_eq!(key.file_id, file_id);
890                assert_eq!(key.file_type, file_type);
891                assert_eq!(std::io::ErrorKind::NotFound, error.kind());
892                let message = if fail_close {
893                    "injected close failure"
894                } else {
895                    "injected write failure"
896                };
897                assert!(error.to_string().contains(message));
898            }
899            err => panic!("expected Upload error, got {err:?}"),
900        }
901        assert!(!cache.file_cache.contains_key(&key));
902        assert!(!remote_store.exists("failed.parquet").await.unwrap());
903        let observations = observations.lock().unwrap();
904        // The write failure follows one completed chunk, so partial progress is exercised.
905        assert_eq!(
906            vec![1024; if fail_close { 4 } else { 2 }],
907            observations.write_sizes
908        );
909        assert_eq!(1, observations.abort_calls);
910        assert_eq!(usize::from(fail_close), observations.close_calls);
911    }
912
913    #[tokio::test]
914    async fn test_upload_local_read_error() {
915        use object_store::layers::mock::MockLayerBuilder;
916
917        let env = TestEnv::new().await;
918        // The unit reader rejects reads after stat and reader creation succeed.
919        let layer = MockLayerBuilder::default()
920            .reader_factory(Arc::new(|_, _, _| Box::new(())))
921            .build()
922            .unwrap();
923        let local_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
924        let remote_store = ObjectStore::new(Memory::default()).unwrap();
925        let cache = env
926            .create_write_cache(local_store.clone(), ReadableSize::mb(10))
927            .await;
928        let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
929        local_store
930            .write(&cache.file_cache.cache_file_path(key), vec![42; 4096])
931            .await
932            .unwrap();
933        let err = cache
934            .upload(
935                key,
936                "failed.parquet",
937                &remote_store,
938                UploadOptions {
939                    op_type: OperationType::Compact,
940                    write_buffer_size: ReadableSize::kb(1),
941                },
942            )
943            .await
944            .unwrap_err();
945        assert!(matches!(err, error::Error::OpenDal { .. }));
946        assert!(!remote_store.exists("failed.parquet").await.unwrap());
947    }
948
949    struct RedirectUploadStoreWrapper {
950        target_store: ObjectStore,
951        last_op_type: Mutex<Option<OperationType>>,
952    }
953
954    impl WriteCacheUploadStoreWrapper for RedirectUploadStoreWrapper {
955        fn wrap(&self, _store: ObjectStore, op_type: OperationType) -> ObjectStore {
956            *self.last_op_type.lock().unwrap() = Some(op_type);
957            self.target_store.clone()
958        }
959    }
960
961    #[tokio::test]
962    async fn test_upload_uses_wrapped_remote_store() {
963        let env = TestEnv::new().await;
964        let local_store = ObjectStore::new(Memory::default()).unwrap();
965        let original_store = ObjectStore::new(Memory::default()).unwrap();
966        let target_store = ObjectStore::new(Memory::default()).unwrap();
967        let wrapper = Arc::new(RedirectUploadStoreWrapper {
968            target_store: target_store.clone(),
969            last_op_type: Mutex::new(None),
970        });
971        let write_cache = WriteCache::new(
972            local_store.clone(),
973            ReadableSize::mb(10),
974            None,
975            None,
976            false,
977            env.get_puffin_manager(),
978            env.get_intermediate_manager(),
979            None,
980        )
981        .await
982        .unwrap()
983        .with_upload_store_wrapper(Some(wrapper.clone()));
984
985        let region_id = RegionId::new(1024, 1);
986        let file_id = FileId::random();
987        let key = IndexKey::new(region_id, file_id, FileType::Parquet);
988        let cache_path = write_cache.file_cache.cache_file_path(key);
989        let upload_path = "wrapped-upload.parquet";
990        let data = Bytes::from_static(b"wrapped upload data");
991        local_store.write(&cache_path, data.clone()).await.unwrap();
992
993        write_cache
994            .upload(
995                key,
996                upload_path,
997                &original_store,
998                UploadOptions {
999                    op_type: OperationType::Compact,
1000                    write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
1001                },
1002            )
1003            .await
1004            .unwrap();
1005
1006        assert_eq!(
1007            *wrapper.last_op_type.lock().unwrap(),
1008            Some(OperationType::Compact)
1009        );
1010
1011        assert!(original_store.stat(upload_path).await.is_err());
1012        assert_eq!(
1013            target_store.read(upload_path).await.unwrap().to_vec(),
1014            data.to_vec()
1015        );
1016    }
1017
1018    #[rstest::rstest]
1019    #[tokio::test]
1020    async fn test_write_and_upload_sst(
1021        #[values(OperationType::Flush, OperationType::Compact)] op_type: OperationType,
1022    ) {
1023        // TODO(QuenKar): maybe find a way to create some object server for testing,
1024        // and now just use local file system to mock.
1025        let mut env = TestEnv::new().await;
1026        let remote_chunks = WriteChunkRecorder::default();
1027        let mock_store = env.init_object_store_manager().layer(remote_chunks.layer());
1028        let path_provider = RegionFilePathFactory::new("test".to_string(), PathType::Bare);
1029
1030        let local_dir = create_temp_dir("");
1031        let local_chunks = WriteChunkRecorder::default();
1032        let local_store =
1033            new_fs_store(local_dir.path().to_str().unwrap()).layer(local_chunks.layer());
1034
1035        let write_cache = env
1036            .create_write_cache(local_store.clone(), ReadableSize::mb(10))
1037            .await;
1038
1039        // Create source.
1040        let metadata = Arc::new(sst_region_metadata());
1041        let region_id = metadata.region_id;
1042        let source = new_flat_source_from_record_batches(vec![
1043            new_record_batch_by_range(&["a", "d"], 0, 60),
1044            new_record_batch_by_range(&["b", "f"], 0, 40),
1045            new_record_batch_by_range(&["b", "h"], 100, 200),
1046        ]);
1047
1048        let write_request = SstWriteRequest {
1049            index_build_inputs: None,
1050            op_type,
1051            metadata,
1052            source,
1053            max_sequence: None,
1054            sst_write_format: Default::default(),
1055            cache_manager: Default::default(),
1056            preserve_row_sequence: false,
1057            index_options: IndexOptions::default(),
1058            index_config: Default::default(),
1059            inverted_index_config: Default::default(),
1060            fulltext_index_config: Default::default(),
1061            bloom_filter_index_config: Default::default(),
1062        };
1063
1064        let upload_request = SstUploadRequest {
1065            dest_path_provider: path_provider.clone(),
1066            remote_store: mock_store.clone(),
1067        };
1068
1069        let write_opts = WriteOptions {
1070            write_buffer_size: ReadableSize::kb(1),
1071            row_group_size: 512,
1072            ..Default::default()
1073        };
1074
1075        // Write to cache and upload sst to mock remote store
1076        let mut metrics = Metrics::new(WriteType::Flush);
1077        let mut sst_infos = write_cache
1078            .write_and_upload_sst_with_context(
1079                write_request,
1080                upload_request,
1081                &write_opts,
1082                &mut metrics,
1083                SstWriteContext::default(),
1084            )
1085            .await
1086            .unwrap();
1087        let sst_info = sst_infos.remove(0);
1088
1089        let file_id = sst_info.file_id;
1090        let sst_upload_path =
1091            path_provider.build_sst_file_path(RegionFileId::new(region_id, file_id));
1092        let index_upload_path =
1093            path_provider.build_index_file_path(RegionFileId::new(region_id, file_id));
1094
1095        // Check write cache contains the key
1096        let key = IndexKey::new(region_id, file_id, FileType::Parquet);
1097        assert!(write_cache.file_cache.contains_key(&key));
1098
1099        // Check file data
1100        let remote_data = mock_store.read(&sst_upload_path).await.unwrap();
1101        let cache_data = local_store
1102            .read(&write_cache.file_cache.cache_file_path(key))
1103            .await
1104            .unwrap();
1105        assert_eq!(remote_data.to_vec(), cache_data.to_vec());
1106        let chunk_size = write_opts.write_buffer_size.as_bytes() as usize;
1107        assert!(remote_data.len() > chunk_size);
1108        remote_chunks.assert_chunks(&sst_upload_path, chunk_size, remote_data.len());
1109        local_chunks.assert_chunks(
1110            &write_cache.file_cache.cache_file_path(key),
1111            chunk_size,
1112            cache_data.len(),
1113        );
1114
1115        // Check write cache contains the index key
1116        let index_key = IndexKey::new(region_id, file_id, FileType::Puffin(0));
1117        assert!(write_cache.file_cache.contains_key(&index_key));
1118
1119        let remote_index_data = mock_store.read(&index_upload_path).await.unwrap();
1120        let cache_index_data = local_store
1121            .read(&write_cache.file_cache.cache_file_path(index_key))
1122            .await
1123            .unwrap();
1124        assert_eq!(remote_index_data.to_vec(), cache_index_data.to_vec());
1125        remote_chunks.assert_chunks(
1126            &index_upload_path,
1127            DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize,
1128            remote_index_data.len(),
1129        );
1130
1131        // Removes the file from the cache.
1132        let sst_index_key = IndexKey::new(region_id, file_id, FileType::Parquet);
1133        write_cache.remove(sst_index_key).await;
1134        assert!(!write_cache.file_cache.contains_key(&sst_index_key));
1135        write_cache.remove(index_key).await;
1136        assert!(!write_cache.file_cache.contains_key(&index_key));
1137    }
1138
1139    #[tokio::test]
1140    async fn test_read_metadata_from_write_cache() {
1141        common_telemetry::init_default_ut_logging();
1142        let mut env = TestEnv::new().await;
1143        // Object keys are relative to the mock store root, not native paths.
1144        let data_home = "write-cache-test".to_string();
1145        let mock_store = env.init_object_store_manager();
1146
1147        let local_dir = create_temp_dir("");
1148        let local_path = local_dir.path().to_str().unwrap();
1149        let local_store = new_fs_store(local_path);
1150
1151        // Create a cache manager using only write cache
1152        let write_cache = env
1153            .create_write_cache(local_store.clone(), ReadableSize::mb(10))
1154            .await;
1155        let cache_manager = Arc::new(
1156            CacheManager::builder()
1157                .write_cache(Some(write_cache.clone()))
1158                .build(),
1159        );
1160        assert!(!cache_manager.sst_meta_cache_enabled());
1161
1162        // Create source
1163        let metadata = Arc::new(sst_region_metadata());
1164
1165        let source = new_flat_source_from_record_batches(vec![
1166            new_record_batch_by_range(&["a", "d"], 0, 60),
1167            new_record_batch_by_range(&["b", "f"], 0, 40),
1168            new_record_batch_by_range(&["b", "h"], 100, 200),
1169        ]);
1170
1171        // Write to local cache and upload sst to mock remote store
1172        let write_request = SstWriteRequest {
1173            index_build_inputs: None,
1174            op_type: OperationType::Flush,
1175            metadata,
1176            source,
1177            max_sequence: None,
1178            sst_write_format: Default::default(),
1179            cache_manager: cache_manager.clone(),
1180            preserve_row_sequence: false,
1181            index_options: IndexOptions::default(),
1182            index_config: Default::default(),
1183            inverted_index_config: Default::default(),
1184            fulltext_index_config: Default::default(),
1185            bloom_filter_index_config: Default::default(),
1186        };
1187        let write_opts = WriteOptions {
1188            row_group_size: 512,
1189            ..Default::default()
1190        };
1191        let upload_request = SstUploadRequest {
1192            dest_path_provider: RegionFilePathFactory::new(data_home.clone(), PathType::Bare),
1193            remote_store: mock_store.clone(),
1194        };
1195
1196        let mut metrics = Metrics::new(WriteType::Flush);
1197        let mut sst_infos = write_cache
1198            .write_and_upload_sst_with_context(
1199                write_request,
1200                upload_request,
1201                &write_opts,
1202                &mut metrics,
1203                SstWriteContext::default(),
1204            )
1205            .await
1206            .unwrap();
1207        let sst_info = sst_infos.remove(0);
1208        let write_parquet_metadata = sst_info.file_metadata.unwrap();
1209
1210        // Read metadata from write cache without preparing an in-memory metadata cache entry.
1211        let handle = sst_file_handle_with_file_id(sst_info.file_id, 0, 1000);
1212        let builder = ParquetReaderBuilder::new(
1213            data_home,
1214            PathType::Bare,
1215            handle.clone(),
1216            mock_store.clone(),
1217        )
1218        .cache(CacheStrategy::EnableAll(cache_manager.clone()))
1219        .page_index_policy(PageIndexPolicy::Optional);
1220        let reader = builder.build().await.unwrap().unwrap();
1221        let cached_write_parquet_metadata = crate::cache::CachedSstMeta::try_new(
1222            "test.sst",
1223            Arc::unwrap_or_clone(write_parquet_metadata),
1224        )
1225        .unwrap()
1226        .parquet_metadata();
1227
1228        // Check parquet metadata
1229        assert_parquet_metadata_equal(cached_write_parquet_metadata, reader.parquet_metadata());
1230    }
1231
1232    #[tokio::test]
1233    async fn test_write_cache_clean_tmp_files() {
1234        common_telemetry::init_default_ut_logging();
1235        let mut env = TestEnv::new().await;
1236        let data_home = env.data_home().display().to_string();
1237        let mock_store = env.init_object_store_manager();
1238
1239        let write_cache_dir = create_temp_dir("");
1240        let write_cache_path = write_cache_dir.path().to_str().unwrap();
1241        let write_cache = env
1242            .create_write_cache_from_path(write_cache_path, ReadableSize::mb(10))
1243            .await;
1244
1245        // Create a cache manager using only write cache
1246        let cache_manager = Arc::new(
1247            CacheManager::builder()
1248                .write_cache(Some(write_cache.clone()))
1249                .build(),
1250        );
1251
1252        // Create source
1253        let metadata = Arc::new(sst_region_metadata());
1254
1255        // Creates a source that can return an error to abort the writer.
1256        let record_batch = new_record_batch_by_range(&["a", "d"], 0, 60);
1257        let schema = record_batch.schema();
1258        let iter = Box::new(
1259            [
1260                Ok(record_batch),
1261                InvalidBatchSnafu {
1262                    reason: "Abort the writer",
1263                }
1264                .fail(),
1265            ]
1266            .into_iter(),
1267        );
1268        let source = FlatSource::new_iter(schema, iter);
1269
1270        // Write to local cache and upload sst to mock remote store
1271        let write_request = SstWriteRequest {
1272            index_build_inputs: None,
1273            op_type: OperationType::Flush,
1274            metadata,
1275            source,
1276            max_sequence: None,
1277            sst_write_format: Default::default(),
1278            cache_manager: cache_manager.clone(),
1279            preserve_row_sequence: false,
1280            index_options: IndexOptions::default(),
1281            index_config: Default::default(),
1282            inverted_index_config: Default::default(),
1283            fulltext_index_config: Default::default(),
1284            bloom_filter_index_config: Default::default(),
1285        };
1286        let write_opts = WriteOptions {
1287            row_group_size: 512,
1288            ..Default::default()
1289        };
1290        let upload_request = SstUploadRequest {
1291            dest_path_provider: RegionFilePathFactory::new(data_home.clone(), PathType::Bare),
1292            remote_store: mock_store.clone(),
1293        };
1294
1295        let mut metrics = Metrics::new(WriteType::Flush);
1296        write_cache
1297            .write_and_upload_sst_with_context(
1298                write_request,
1299                upload_request,
1300                &write_opts,
1301                &mut metrics,
1302                SstWriteContext::default(),
1303            )
1304            .await
1305            .unwrap_err();
1306        let atomic_write_dir = write_cache_dir.path().join(ATOMIC_WRITE_DIR);
1307        let mut entries = tokio::fs::read_dir(&atomic_write_dir).await.unwrap();
1308        let mut has_files = false;
1309        while let Some(entry) = entries.next_entry().await.unwrap() {
1310            if entry.file_type().await.unwrap().is_dir() {
1311                continue;
1312            }
1313            has_files = true;
1314            common_telemetry::warn!(
1315                "Found remaining temporary file in atomic dir: {}",
1316                entry.path().display()
1317            );
1318        }
1319
1320        assert!(!has_files);
1321    }
1322
1323    #[tokio::test]
1324    async fn test_download_if_absent_skips_when_cached() {
1325        let mut env = TestEnv::new().await;
1326        let remote_store = env.init_object_store_manager();
1327
1328        let local_dir = create_temp_dir("");
1329        let local_store = new_fs_store(local_dir.path().to_str().unwrap());
1330        let write_cache = env
1331            .create_write_cache(local_store.clone(), ReadableSize::mb(10))
1332            .await;
1333
1334        let region_id = RegionId::new(1024, 1);
1335        let file_id = FileId::random();
1336        let key = IndexKey::new(region_id, file_id, FileType::Parquet);
1337        write_cache
1338            .file_cache()
1339            .put(key, IndexValue { file_size: 1 })
1340            .await;
1341
1342        let downloaded = write_cache
1343            .download_if_absent(key, "missing/path.parquet", &remote_store, 1)
1344            .await
1345            .unwrap();
1346
1347        assert!(!downloaded);
1348    }
1349
1350    #[tokio::test]
1351    async fn test_download_if_absent_downloads_when_missing() {
1352        let mut env = TestEnv::new().await;
1353        let remote_store = env.init_object_store_manager();
1354
1355        let local_dir = create_temp_dir("");
1356        let local_store = new_fs_store(local_dir.path().to_str().unwrap());
1357        let write_cache = env
1358            .create_write_cache(local_store.clone(), ReadableSize::mb(10))
1359            .await;
1360
1361        let region_id = RegionId::new(1024, 2);
1362        let file_id = FileId::random();
1363        let key = IndexKey::new(region_id, file_id, FileType::Parquet);
1364        let remote_path = format!("download-if-absent/{file_id}.parquet");
1365        let remote_data = Bytes::from_static(b"download-if-absent-test");
1366        remote_store
1367            .write(&remote_path, remote_data.clone())
1368            .await
1369            .unwrap();
1370
1371        let downloaded = write_cache
1372            .download_if_absent(key, &remote_path, &remote_store, remote_data.len() as u64)
1373            .await
1374            .unwrap();
1375
1376        assert!(downloaded);
1377        assert!(write_cache.file_cache().contains_key(&key));
1378
1379        let cached_data = local_store
1380            .read(&write_cache.file_cache().cache_file_path(key))
1381            .await
1382            .unwrap();
1383        assert_eq!(cached_data.to_vec(), remote_data.to_vec());
1384    }
1385}