1use std::sync::Arc;
18use std::time::{Duration, Instant};
19
20use common_base::readable_size::ReadableSize;
21use common_telemetry::{debug, info};
22use object_store::ObjectStore;
23use snafu::{IntoError, ResultExt};
24use store_api::storage::RegionId;
25use tokio::sync::mpsc::{UnboundedSender, unbounded_channel};
26use tokio_util::sync::CancellationToken;
27
28use crate::access_layer::{
29 EncodedSstIndexBuildParams, FilePathProvider, Metrics, OperationType, RegionFilePathFactory,
30 SstInfoArray, SstWriteRequest, TempFileCleaner, WriteCachePathProvider, new_fs_cache_store,
31};
32use crate::cache::file_cache::{FileCache, FileCacheRef, FileType, IndexKey, IndexValue};
33use crate::cache::manifest_cache::ManifestCache;
34use crate::error::{self, Result};
35use crate::metrics::UPLOAD_BYTES_TOTAL;
36use crate::region::opener::RegionLoadCacheTask;
37use crate::sst::file::RegionFileId;
38use crate::sst::index::intermediate::IntermediateManager;
39use crate::sst::index::puffin_manager::{PuffinManagerFactory, SstPuffinManager};
40use crate::sst::index::registry::IndexRegistry;
41use crate::sst::index::{IndexerBuilderImpl, build_index_from_parquet_bytes};
42use crate::sst::parquet::writer::{ParquetWriter, SstWriteContext};
43use crate::sst::parquet::{SstInfo, WriteOptions};
44use crate::sst::upload::{self, UploadError};
45use crate::sst::{DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY};
46
47pub trait WriteCacheUploadStoreWrapper: Send + Sync {
49 fn wrap(&self, store: ObjectStore, op_type: OperationType) -> ObjectStore;
56}
57
58pub type WriteCacheUploadStoreWrapperRef = Arc<dyn WriteCacheUploadStoreWrapper>;
59
60pub struct WriteCache {
64 file_cache: FileCacheRef,
66 puffin_manager_factory: PuffinManagerFactory,
68 intermediate_manager: IntermediateManager,
70 task_sender: UnboundedSender<RegionLoadCacheTask>,
72 manifest_cache: Option<ManifestCache>,
74 upload_store_wrapper: Option<WriteCacheUploadStoreWrapperRef>,
76}
77
78pub type WriteCacheRef = Arc<WriteCache>;
79
80impl WriteCache {
81 #[allow(clippy::too_many_arguments)]
84 pub async fn new(
85 local_store: ObjectStore,
86 cache_capacity: ReadableSize,
87 ttl: Option<Duration>,
88 index_cache_percent: Option<u8>,
89 enable_background_worker: bool,
90 puffin_manager_factory: PuffinManagerFactory,
91 intermediate_manager: IntermediateManager,
92 manifest_cache: Option<ManifestCache>,
93 ) -> Result<Self> {
94 let (task_sender, task_receiver) = unbounded_channel();
95
96 let file_cache = Arc::new(FileCache::new(
97 local_store,
98 cache_capacity,
99 ttl,
100 index_cache_percent,
101 enable_background_worker,
102 ));
103 file_cache.recover(false, Some(task_receiver)).await;
104
105 Ok(Self {
106 file_cache,
107 puffin_manager_factory,
108 intermediate_manager,
109 task_sender,
110 manifest_cache,
111 upload_store_wrapper: None,
112 })
113 }
114
115 #[allow(clippy::too_many_arguments)]
117 pub async fn new_fs(
118 cache_dir: &str,
119 cache_capacity: ReadableSize,
120 ttl: Option<Duration>,
121 index_cache_percent: Option<u8>,
122 enable_background_worker: bool,
123 puffin_manager_factory: PuffinManagerFactory,
124 intermediate_manager: IntermediateManager,
125 manifest_cache_capacity: ReadableSize,
126 ) -> Result<Self> {
127 info!("Init write cache on {cache_dir}, capacity: {cache_capacity}");
128
129 let local_store = new_fs_cache_store(cache_dir).await?;
130
131 let manifest_cache = if manifest_cache_capacity.as_bytes() > 0 {
133 Some(ManifestCache::new(local_store.clone(), manifest_cache_capacity, ttl, false).await)
134 } else {
135 None
136 };
137
138 Self::new(
139 local_store,
140 cache_capacity,
141 ttl,
142 index_cache_percent,
143 enable_background_worker,
144 puffin_manager_factory,
145 intermediate_manager,
146 manifest_cache,
147 )
148 .await
149 }
150
151 pub(crate) fn file_cache(&self) -> FileCacheRef {
153 self.file_cache.clone()
154 }
155
156 pub(crate) fn manifest_cache(&self) -> Option<ManifestCache> {
158 self.manifest_cache.clone()
159 }
160
161 pub(crate) fn with_upload_store_wrapper(
163 mut self,
164 upload_store_wrapper: Option<WriteCacheUploadStoreWrapperRef>,
165 ) -> Self {
166 self.upload_store_wrapper = upload_store_wrapper;
167 self
168 }
169
170 pub(crate) fn build_puffin_manager(&self) -> SstPuffinManager {
172 let store = self.file_cache.local_store();
173 let path_provider = WriteCachePathProvider::new(self.file_cache.clone());
174 self.puffin_manager_factory.build(store, path_provider)
175 }
176
177 #[allow(clippy::too_many_arguments)]
182 pub(crate) async fn put_and_upload_sst(
183 &self,
184 data: &bytes::Bytes,
185 region_id: RegionId,
186 sst_info: &mut SstInfo,
187 upload_request: SstUploadRequest,
188 write_buffer_size: ReadableSize,
189 index_build_params: Option<EncodedSstIndexBuildParams>,
190 metrics: &mut Metrics,
191 ) -> Result<()> {
192 let file_id = sst_info.file_id;
193 let region_file_id = RegionFileId::new(region_id, file_id);
194
195 let parquet_key = IndexKey::new(region_id, file_id, FileType::Parquet);
197
198 let cache_start = Instant::now();
200 let cache_path = self.file_cache.cache_file_path(parquet_key);
201 let store = self.file_cache.local_store();
202 let cleaner = TempFileCleaner::new(region_id, store.clone());
203 let write_res = store
204 .write_with(&cache_path, data.clone())
205 .chunk(write_buffer_size.as_bytes() as usize)
206 .await
207 .context(crate::error::OpenDalSnafu);
208 if let Err(e) = write_res {
209 cleaner.clean_by_file_id(file_id).await;
210 return Err(e);
211 }
212
213 metrics.write_batch = cache_start.elapsed();
214
215 if let Some(params) = index_build_params {
216 let start = Instant::now();
217 let builder = params.indexer_builder(
218 self.build_puffin_manager(),
219 true,
220 self.intermediate_manager.clone(),
221 );
222 sst_info.index_metadata =
223 build_index_from_parquet_bytes(&builder, region_file_id, data).await;
224 metrics.update_index = start.elapsed();
225 }
226
227 let remote_store = &upload_request.remote_store;
229 let mut upload_tracker = UploadTracker::new(region_id);
230 let upload_start = Instant::now();
231 let remote_path = upload_request
232 .dest_path_provider
233 .build_sst_file_path(region_file_id);
234 let mut result = self
235 .upload(
236 parquet_key,
237 &remote_path,
238 remote_store,
239 UploadOptions {
240 op_type: OperationType::Flush,
242 write_buffer_size,
243 },
244 )
245 .await;
246 metrics.upload_parquet = upload_start.elapsed();
247
248 if result.is_ok() && sst_info.index_metadata.file_size > 0 {
249 upload_tracker.push_uploaded_file(remote_path);
250 let puffin_key = IndexKey::new(region_id, file_id, FileType::Puffin(0));
251 let puffin_path = upload_request
252 .dest_path_provider
253 .build_index_file_path(region_file_id);
254 let start = Instant::now();
255 result = self
256 .upload(
257 puffin_key,
258 &puffin_path,
259 remote_store,
260 UploadOptions {
261 op_type: OperationType::Flush,
262 write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
263 },
264 )
265 .await;
266 metrics.upload_puffin = start.elapsed();
267 }
268
269 if let Err(e) = result {
270 upload_tracker
271 .clean(
272 std::slice::from_ref(sst_info),
273 &self.file_cache,
274 remote_store,
275 )
276 .await;
277 return Err(e);
278 }
279 Ok(())
280 }
281
282 pub(crate) fn intermediate_manager(&self) -> &IntermediateManager {
284 &self.intermediate_manager
285 }
286
287 pub(crate) async fn write_and_upload_sst_with_context(
289 &self,
290 write_request: SstWriteRequest,
291 upload_request: SstUploadRequest,
292 write_opts: &WriteOptions,
293 metrics: &mut Metrics,
294 context: SstWriteContext,
295 ) -> Result<SstInfoArray> {
296 let region_id = write_request.metadata.region_id;
297 let override_sequence = if write_request.preserve_row_sequence {
298 None
299 } else {
300 write_request.max_sequence
301 };
302
303 let store = self.file_cache.local_store();
304 let path_provider = WriteCachePathProvider::new(self.file_cache.clone());
305 let indexer = IndexerBuilderImpl {
306 index_build_inputs: write_request.index_build_inputs,
307 build_type: write_request.op_type.into(),
308 metadata: write_request.metadata.clone(),
309 puffin_manager: self
310 .puffin_manager_factory
311 .build(store.clone(), path_provider.clone()),
312 write_cache_enabled: true,
313 intermediate_manager: self.intermediate_manager.clone(),
314 index_options: write_request.index_options,
315 registry: Arc::new(IndexRegistry::new(
316 write_request.inverted_index_config,
317 write_request.fulltext_index_config,
318 write_request.bloom_filter_index_config,
319 )),
320 };
321
322 let cleaner = TempFileCleaner::new(region_id, store.clone());
323 let mut writer = ParquetWriter::new_with_object_store(
325 store.clone(),
326 write_request.metadata,
327 write_request.index_config,
328 indexer,
329 path_provider.clone(),
330 metrics,
331 )
332 .await
333 .with_file_cleaner(cleaner)
334 .with_write_guard(context.guard)
335 .with_cancellation(context.cancellation.clone());
336
337 let sst_info = match write_request.sst_write_format {
338 crate::sst::FormatType::PrimaryKey => {
339 writer
340 .write_all_flat_as_primary_key(
341 write_request.source,
342 override_sequence,
343 write_opts,
344 )
345 .await?
346 }
347 crate::sst::FormatType::Flat => {
348 writer
349 .write_all_flat(write_request.source, override_sequence, write_opts)
350 .await?
351 }
352 };
353
354 if sst_info.is_empty() {
356 return Ok(sst_info);
357 }
358
359 let mut upload_tracker = UploadTracker::new(region_id);
360 let mut err = None;
361 let op_type = write_request.op_type;
362 let remote_store = &upload_request.remote_store;
363 for sst in &sst_info {
364 if context
365 .cancellation
366 .as_ref()
367 .is_some_and(|handle| handle.is_cancelled())
368 {
369 err = Some(error::SstWriteCancelledSnafu.build());
370 break;
371 }
372 let parquet_key = IndexKey::new(region_id, sst.file_id, FileType::Parquet);
373 let parquet_path = upload_request
374 .dest_path_provider
375 .build_sst_file_path(RegionFileId::new(region_id, sst.file_id));
376 let start = Instant::now();
377 if let Err(e) = self
378 .upload_with_cancellation(
379 parquet_key,
380 &parquet_path,
381 remote_store,
382 UploadOptions {
383 op_type,
384 write_buffer_size: write_opts.write_buffer_size,
385 },
386 context.cancellation.as_ref(),
387 )
388 .await
389 {
390 err = Some(e);
391 break;
392 }
393 metrics.upload_parquet += start.elapsed();
394 upload_tracker.push_uploaded_file(parquet_path);
395
396 if sst.index_metadata.file_size > 0 {
397 let puffin_key = IndexKey::new(region_id, sst.file_id, FileType::Puffin(0));
398 let puffin_path = upload_request
399 .dest_path_provider
400 .build_index_file_path(RegionFileId::new(region_id, sst.file_id));
401 let start = Instant::now();
402 if let Err(e) = self
403 .upload_with_cancellation(
404 puffin_key,
405 &puffin_path,
406 remote_store,
407 UploadOptions {
408 op_type,
409 write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
410 },
411 context.cancellation.as_ref(),
412 )
413 .await
414 {
415 err = Some(e);
416 break;
417 }
418 metrics.upload_puffin += start.elapsed();
419 upload_tracker.push_uploaded_file(puffin_path);
420 }
421 }
422
423 if let Some(err) = err {
424 upload_tracker
426 .clean(&sst_info, &self.file_cache, remote_store)
427 .await;
428 return Err(err);
429 }
430
431 Ok(sst_info)
432 }
433
434 pub(crate) async fn remove(&self, index_key: IndexKey) {
436 self.file_cache.remove(index_key).await
437 }
438
439 pub(crate) async fn download(
442 &self,
443 index_key: IndexKey,
444 remote_path: &str,
445 remote_store: &ObjectStore,
446 file_size: u64,
447 ) -> Result<()> {
448 self.file_cache
449 .download(index_key, remote_path, remote_store, file_size)
450 .await
451 }
452
453 pub(crate) async fn download_if_absent(
458 &self,
459 index_key: IndexKey,
460 remote_path: &str,
461 remote_store: &ObjectStore,
462 file_size: u64,
463 ) -> Result<bool> {
464 if self.file_cache.contains_key(&index_key) {
465 debug!(
466 "Skip downloading file already in write cache, region: {}, file: {}",
467 index_key.region_id, index_key.file_id
468 );
469 return Ok(false);
470 }
471
472 self.download(index_key, remote_path, remote_store, file_size)
473 .await?;
474 Ok(true)
475 }
476
477 pub(crate) async fn upload(
479 &self,
480 index_key: IndexKey,
481 upload_path: &str,
482 remote_store: &ObjectStore,
483 options: UploadOptions,
484 ) -> Result<()> {
485 self.upload_with_cancellation(index_key, upload_path, remote_store, options, None)
486 .await
487 }
488
489 async fn upload_with_cancellation(
490 &self,
491 index_key: IndexKey,
492 upload_path: &str,
493 remote_store: &ObjectStore,
494 options: UploadOptions,
495 cancellation: Option<&CancellationToken>,
496 ) -> Result<()> {
497 let region_id = index_key.region_id;
498 let file_id = index_key.file_id;
499 let cache_path = self.file_cache.cache_file_path(index_key);
500
501 let start = Instant::now();
502 let cached_value = self
503 .file_cache
504 .local_store()
505 .stat(&cache_path)
506 .await
507 .context(error::OpenDalSnafu)?;
508 let mut reader = self
509 .file_cache
510 .local_store()
511 .reader_with(&cache_path)
512 .chunk(options.write_buffer_size.as_bytes().max(1) as usize)
513 .await
514 .context(error::OpenDalSnafu)?
515 .into_stream(0..cached_value.content_length())
516 .await
517 .context(error::OpenDalSnafu)?;
518
519 let upload_store = self.upload_store_wrapper.as_ref().map_or_else(
520 || remote_store.clone(),
521 |wrapper| wrapper.wrap(remote_store.clone(), options.op_type),
522 );
523 let mut writer = upload_store
524 .writer_with(upload_path)
525 .chunk(options.write_buffer_size.as_bytes() as usize)
526 .concurrent(DEFAULT_WRITE_CONCURRENCY)
527 .await
528 .context(error::OpenDalSnafu)?;
529
530 let result = async {
531 let bytes_written = upload::copy(
532 &mut reader,
533 &mut writer,
534 options.write_buffer_size.as_bytes() as usize,
535 cancellation,
536 )
537 .await
538 .map_err(|error| match error {
539 UploadError::Cancelled => error::SstWriteCancelledSnafu.build(),
540 UploadError::Read { source } => error::OpenDalSnafu.into_error(source),
541 UploadError::Write { source } => error::UploadSnafu {
542 region_id,
543 file_id,
544 file_type: index_key.file_type,
545 }
546 .into_error(std::io::Error::from(source)),
547 })?;
548 if cancellation.is_some_and(CancellationToken::is_cancelled) {
549 return error::SstWriteCancelledSnafu.fail();
550 }
551 writer
552 .close()
553 .await
554 .map_err(std::io::Error::from)
555 .context(error::UploadSnafu {
556 region_id,
557 file_id,
558 file_type: index_key.file_type,
559 })?;
560 Ok(bytes_written)
561 }
562 .await;
563 let bytes_written = match result {
564 Ok(bytes_written) => bytes_written,
565 Err(error) => {
566 if let Err(abort_error) = writer.abort().await {
567 common_telemetry::warn!(abort_error; "Failed to abort SST upload, region: {}, file: {}", region_id, file_id);
568 }
569 return Err(error);
570 }
571 };
572
573 UPLOAD_BYTES_TOTAL.inc_by(bytes_written);
574
575 debug!(
576 "Successfully upload file to remote, region: {}, file: {}, upload_path: {}, cost: {:?}",
577 region_id,
578 file_id,
579 upload_path,
580 start.elapsed(),
581 );
582
583 let index_value = IndexValue {
584 file_size: bytes_written as _,
585 };
586 self.file_cache.put(index_key, index_value).await;
588
589 Ok(())
590 }
591
592 pub(crate) fn load_region_cache(&self, task: RegionLoadCacheTask) {
596 let _ = self.task_sender.send(task);
597 }
598}
599
600pub struct SstUploadRequest {
602 pub dest_path_provider: RegionFilePathFactory,
604 pub remote_store: ObjectStore,
606}
607
608pub(crate) struct UploadOptions {
610 pub op_type: OperationType,
612 pub write_buffer_size: ReadableSize,
614}
615
616pub(crate) struct UploadTracker {
618 region_id: RegionId,
620 files_uploaded: Vec<String>,
622}
623
624impl UploadTracker {
625 pub(crate) fn new(region_id: RegionId) -> Self {
627 Self {
628 region_id,
629 files_uploaded: Vec::new(),
630 }
631 }
632
633 pub(crate) fn push_uploaded_file(&mut self, path: String) {
635 self.files_uploaded.push(path);
636 }
637
638 pub(crate) async fn clean(
640 &self,
641 sst_info: &[SstInfo],
642 file_cache: &FileCacheRef,
643 remote_store: &ObjectStore,
644 ) {
645 common_telemetry::info!(
646 "Start cleaning files on upload failure, region: {}, num_ssts: {}",
647 self.region_id,
648 sst_info.len()
649 );
650
651 for sst in sst_info {
653 let parquet_key = IndexKey::new(self.region_id, sst.file_id, FileType::Parquet);
654 file_cache.remove(parquet_key).await;
655
656 if sst.index_metadata.file_size > 0 {
657 let puffin_key = IndexKey::new(
658 self.region_id,
659 sst.file_id,
660 FileType::Puffin(sst.index_metadata.version),
661 );
662 file_cache.remove(puffin_key).await;
663 }
664 }
665
666 for file_path in &self.files_uploaded {
668 if let Err(e) = remote_store.delete(file_path).await {
669 common_telemetry::error!(e; "Failed to delete file {}", file_path);
670 }
671 }
672 }
673}
674
675#[cfg(test)]
676mod tests {
677 use std::sync::Mutex;
678
679 use bytes::Bytes;
680 use common_test_util::temp_dir::create_temp_dir;
681 use object_store::services::Memory;
682 use object_store::{ATOMIC_WRITE_DIR, ObjectStore};
683 use parquet::file::metadata::PageIndexPolicy;
684 use store_api::region_request::PathType;
685 use store_api::storage::FileId;
686
687 use super::*;
688 use crate::access_layer::{OperationType, WriteType};
689 use crate::cache::file_cache::IndexValue;
690 use crate::cache::test_util::{assert_parquet_metadata_equal, new_fs_store};
691 use crate::cache::{CacheManager, CacheStrategy};
692 use crate::error::InvalidBatchSnafu;
693 use crate::read::FlatSource;
694 use crate::region::options::IndexOptions;
695 use crate::sst::parquet::reader::ParquetReaderBuilder;
696 use crate::test_util::TestEnv;
697 use crate::test_util::sst_util::{
698 WriteChunkRecorder, new_flat_source_from_record_batches, new_record_batch_by_range,
699 sst_file_handle_with_file_id, sst_region_metadata,
700 };
701
702 struct CancellingUploadWriter {
703 inner: object_store::layers::mock::oio::Writer,
704 cancellation: CancellationToken,
705 bytes_written: Arc<std::sync::atomic::AtomicUsize>,
706 aborted: Arc<std::sync::atomic::AtomicBool>,
707 }
708
709 impl object_store::layers::mock::oio::Write for CancellingUploadWriter {
710 async fn write(&mut self, buffer: object_store::Buffer) -> object_store::Result<()> {
711 let len = buffer.len();
712 self.inner.write(buffer).await?;
713 self.bytes_written
714 .fetch_add(len, std::sync::atomic::Ordering::Relaxed);
715 self.cancellation.cancel();
716 Ok(())
717 }
718 async fn close(
719 &mut self,
720 ) -> object_store::layers::mock::Result<object_store::layers::mock::Metadata> {
721 self.inner.close().await
722 }
723 async fn abort(&mut self) -> object_store::Result<()> {
724 self.aborted
725 .store(true, std::sync::atomic::Ordering::Relaxed);
726 self.inner.abort().await
727 }
728 }
729
730 #[tokio::test]
731 async fn test_cancel_upload_aborts_incomplete_object() {
732 use object_store::layers::mock::MockLayerBuilder;
733 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
734
735 let env = TestEnv::new().await;
736 let local_store = ObjectStore::new(Memory::default()).unwrap();
737 let cancellation = CancellationToken::new();
738 let bytes_written = Arc::new(AtomicUsize::new(0));
739 let aborted = Arc::new(AtomicBool::new(false));
740 let signal = cancellation.clone();
741 let bytes = bytes_written.clone();
742 let abort = aborted.clone();
743 let layer = MockLayerBuilder::default()
744 .writer_factory(Arc::new(move |_, _, inner| {
745 Box::new(CancellingUploadWriter {
746 inner,
747 cancellation: signal.clone(),
748 bytes_written: bytes.clone(),
749 aborted: abort.clone(),
750 })
751 }))
752 .build()
753 .unwrap();
754 let remote_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
755 let cache = env
756 .create_write_cache(local_store.clone(), ReadableSize::mb(10))
757 .await;
758 let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
759 let data = vec![42; 1024 * 1024];
760 local_store
761 .write(&cache.file_cache.cache_file_path(key), data.clone())
762 .await
763 .unwrap();
764 let result = cache
765 .upload_with_cancellation(
766 key,
767 "cancelled.parquet",
768 &remote_store,
769 UploadOptions {
770 op_type: OperationType::Compact,
771 write_buffer_size: ReadableSize::kb(1),
772 },
773 Some(&cancellation),
774 )
775 .await;
776 assert!(matches!(
777 result,
778 Err(error::Error::SstWriteCancelled { .. })
779 ));
780 assert!(aborted.load(Ordering::Relaxed));
781 let bytes = bytes_written.load(Ordering::Relaxed);
782 assert!(
783 bytes > 0 && bytes < data.len(),
784 "wrote {bytes} bytes before cancelling"
785 );
786 assert!(!remote_store.exists("cancelled.parquet").await.unwrap());
787 }
788
789 struct FailingUploadWriter {
790 inner: object_store::layers::mock::oio::Writer,
791 fail_close: bool,
792 fail_abort: bool,
793 successful_writes: usize,
794 }
795
796 impl object_store::layers::mock::oio::Write for FailingUploadWriter {
797 async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
798 if !self.fail_close && self.successful_writes == 1 {
799 return Err(object_store::Error::new(
800 object_store::ErrorKind::NotFound,
801 "injected write failure",
802 ));
803 }
804 self.inner.write(bytes).await?;
805 self.successful_writes += 1;
806 Ok(())
807 }
808
809 async fn close(&mut self) -> object_store::Result<object_store::layers::mock::Metadata> {
810 Err(object_store::Error::new(
811 object_store::ErrorKind::NotFound,
812 "injected close failure",
813 ))
814 }
815
816 async fn abort(&mut self) -> object_store::Result<()> {
817 if self.fail_abort {
818 return Err(object_store::Error::new(
819 object_store::ErrorKind::Unexpected,
820 "injected abort failure",
821 ));
822 }
823 self.inner.abort().await
824 }
825 }
826
827 #[rstest::rstest]
828 #[case::write(false, false)]
829 #[case::close(true, false)]
830 #[case::write_and_abort(false, true)]
831 #[tokio::test]
832 async fn test_upload_error_preserves_file_context(
833 #[case] fail_close: bool,
834 #[case] fail_abort: bool,
835 ) {
836 use object_store::layers::mock::{MockLayerBuilder, ObservedWriter, WriterObservations};
837
838 let env = TestEnv::new().await;
839 let local_store = ObjectStore::new(Memory::default()).unwrap();
840 let observations = Arc::new(std::sync::Mutex::new(WriterObservations::default()));
841 let observed = observations.clone();
842 let layer = MockLayerBuilder::default()
843 .writer_factory(Arc::new(move |_, _, inner| {
844 Box::new(ObservedWriter::new(
845 Box::new(FailingUploadWriter {
846 inner,
847 fail_close,
848 fail_abort,
849 successful_writes: 0,
850 }),
851 observed.clone(),
852 ))
853 }))
854 .build()
855 .unwrap();
856 let remote_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
857 let cache = env
858 .create_write_cache(local_store.clone(), ReadableSize::mb(10))
859 .await;
860 let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
861 local_store
862 .write(&cache.file_cache.cache_file_path(key), vec![42; 4096])
863 .await
864 .unwrap();
865 assert!(!cache.file_cache.contains_key(&key));
866 let err = cache
867 .upload(
868 key,
869 "failed.parquet",
870 &remote_store,
871 UploadOptions {
872 op_type: OperationType::Compact,
873 write_buffer_size: ReadableSize::kb(1),
874 },
875 )
876 .await
877 .unwrap_err();
878 assert!(!err.is_object_not_found());
879 assert!(err.may_have_persisted_manifest_update());
880 match err {
881 error::Error::Upload {
882 region_id,
883 file_id,
884 file_type,
885 error,
886 ..
887 } => {
888 assert_eq!(key.region_id, region_id);
889 assert_eq!(key.file_id, file_id);
890 assert_eq!(key.file_type, file_type);
891 assert_eq!(std::io::ErrorKind::NotFound, error.kind());
892 let message = if fail_close {
893 "injected close failure"
894 } else {
895 "injected write failure"
896 };
897 assert!(error.to_string().contains(message));
898 }
899 err => panic!("expected Upload error, got {err:?}"),
900 }
901 assert!(!cache.file_cache.contains_key(&key));
902 assert!(!remote_store.exists("failed.parquet").await.unwrap());
903 let observations = observations.lock().unwrap();
904 assert_eq!(
906 vec![1024; if fail_close { 4 } else { 2 }],
907 observations.write_sizes
908 );
909 assert_eq!(1, observations.abort_calls);
910 assert_eq!(usize::from(fail_close), observations.close_calls);
911 }
912
913 #[tokio::test]
914 async fn test_upload_local_read_error() {
915 use object_store::layers::mock::MockLayerBuilder;
916
917 let env = TestEnv::new().await;
918 let layer = MockLayerBuilder::default()
920 .reader_factory(Arc::new(|_, _, _| Box::new(())))
921 .build()
922 .unwrap();
923 let local_store = ObjectStore::new(Memory::default()).unwrap().layer(layer);
924 let remote_store = ObjectStore::new(Memory::default()).unwrap();
925 let cache = env
926 .create_write_cache(local_store.clone(), ReadableSize::mb(10))
927 .await;
928 let key = IndexKey::new(RegionId::new(1024, 1), FileId::random(), FileType::Parquet);
929 local_store
930 .write(&cache.file_cache.cache_file_path(key), vec![42; 4096])
931 .await
932 .unwrap();
933 let err = cache
934 .upload(
935 key,
936 "failed.parquet",
937 &remote_store,
938 UploadOptions {
939 op_type: OperationType::Compact,
940 write_buffer_size: ReadableSize::kb(1),
941 },
942 )
943 .await
944 .unwrap_err();
945 assert!(matches!(err, error::Error::OpenDal { .. }));
946 assert!(!remote_store.exists("failed.parquet").await.unwrap());
947 }
948
949 struct RedirectUploadStoreWrapper {
950 target_store: ObjectStore,
951 last_op_type: Mutex<Option<OperationType>>,
952 }
953
954 impl WriteCacheUploadStoreWrapper for RedirectUploadStoreWrapper {
955 fn wrap(&self, _store: ObjectStore, op_type: OperationType) -> ObjectStore {
956 *self.last_op_type.lock().unwrap() = Some(op_type);
957 self.target_store.clone()
958 }
959 }
960
961 #[tokio::test]
962 async fn test_upload_uses_wrapped_remote_store() {
963 let env = TestEnv::new().await;
964 let local_store = ObjectStore::new(Memory::default()).unwrap();
965 let original_store = ObjectStore::new(Memory::default()).unwrap();
966 let target_store = ObjectStore::new(Memory::default()).unwrap();
967 let wrapper = Arc::new(RedirectUploadStoreWrapper {
968 target_store: target_store.clone(),
969 last_op_type: Mutex::new(None),
970 });
971 let write_cache = WriteCache::new(
972 local_store.clone(),
973 ReadableSize::mb(10),
974 None,
975 None,
976 false,
977 env.get_puffin_manager(),
978 env.get_intermediate_manager(),
979 None,
980 )
981 .await
982 .unwrap()
983 .with_upload_store_wrapper(Some(wrapper.clone()));
984
985 let region_id = RegionId::new(1024, 1);
986 let file_id = FileId::random();
987 let key = IndexKey::new(region_id, file_id, FileType::Parquet);
988 let cache_path = write_cache.file_cache.cache_file_path(key);
989 let upload_path = "wrapped-upload.parquet";
990 let data = Bytes::from_static(b"wrapped upload data");
991 local_store.write(&cache_path, data.clone()).await.unwrap();
992
993 write_cache
994 .upload(
995 key,
996 upload_path,
997 &original_store,
998 UploadOptions {
999 op_type: OperationType::Compact,
1000 write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
1001 },
1002 )
1003 .await
1004 .unwrap();
1005
1006 assert_eq!(
1007 *wrapper.last_op_type.lock().unwrap(),
1008 Some(OperationType::Compact)
1009 );
1010
1011 assert!(original_store.stat(upload_path).await.is_err());
1012 assert_eq!(
1013 target_store.read(upload_path).await.unwrap().to_vec(),
1014 data.to_vec()
1015 );
1016 }
1017
1018 #[rstest::rstest]
1019 #[tokio::test]
1020 async fn test_write_and_upload_sst(
1021 #[values(OperationType::Flush, OperationType::Compact)] op_type: OperationType,
1022 ) {
1023 let mut env = TestEnv::new().await;
1026 let remote_chunks = WriteChunkRecorder::default();
1027 let mock_store = env.init_object_store_manager().layer(remote_chunks.layer());
1028 let path_provider = RegionFilePathFactory::new("test".to_string(), PathType::Bare);
1029
1030 let local_dir = create_temp_dir("");
1031 let local_chunks = WriteChunkRecorder::default();
1032 let local_store =
1033 new_fs_store(local_dir.path().to_str().unwrap()).layer(local_chunks.layer());
1034
1035 let write_cache = env
1036 .create_write_cache(local_store.clone(), ReadableSize::mb(10))
1037 .await;
1038
1039 let metadata = Arc::new(sst_region_metadata());
1041 let region_id = metadata.region_id;
1042 let source = new_flat_source_from_record_batches(vec![
1043 new_record_batch_by_range(&["a", "d"], 0, 60),
1044 new_record_batch_by_range(&["b", "f"], 0, 40),
1045 new_record_batch_by_range(&["b", "h"], 100, 200),
1046 ]);
1047
1048 let write_request = SstWriteRequest {
1049 index_build_inputs: None,
1050 op_type,
1051 metadata,
1052 source,
1053 max_sequence: None,
1054 sst_write_format: Default::default(),
1055 cache_manager: Default::default(),
1056 preserve_row_sequence: false,
1057 index_options: IndexOptions::default(),
1058 index_config: Default::default(),
1059 inverted_index_config: Default::default(),
1060 fulltext_index_config: Default::default(),
1061 bloom_filter_index_config: Default::default(),
1062 };
1063
1064 let upload_request = SstUploadRequest {
1065 dest_path_provider: path_provider.clone(),
1066 remote_store: mock_store.clone(),
1067 };
1068
1069 let write_opts = WriteOptions {
1070 write_buffer_size: ReadableSize::kb(1),
1071 row_group_size: 512,
1072 ..Default::default()
1073 };
1074
1075 let mut metrics = Metrics::new(WriteType::Flush);
1077 let mut sst_infos = write_cache
1078 .write_and_upload_sst_with_context(
1079 write_request,
1080 upload_request,
1081 &write_opts,
1082 &mut metrics,
1083 SstWriteContext::default(),
1084 )
1085 .await
1086 .unwrap();
1087 let sst_info = sst_infos.remove(0);
1088
1089 let file_id = sst_info.file_id;
1090 let sst_upload_path =
1091 path_provider.build_sst_file_path(RegionFileId::new(region_id, file_id));
1092 let index_upload_path =
1093 path_provider.build_index_file_path(RegionFileId::new(region_id, file_id));
1094
1095 let key = IndexKey::new(region_id, file_id, FileType::Parquet);
1097 assert!(write_cache.file_cache.contains_key(&key));
1098
1099 let remote_data = mock_store.read(&sst_upload_path).await.unwrap();
1101 let cache_data = local_store
1102 .read(&write_cache.file_cache.cache_file_path(key))
1103 .await
1104 .unwrap();
1105 assert_eq!(remote_data.to_vec(), cache_data.to_vec());
1106 let chunk_size = write_opts.write_buffer_size.as_bytes() as usize;
1107 assert!(remote_data.len() > chunk_size);
1108 remote_chunks.assert_chunks(&sst_upload_path, chunk_size, remote_data.len());
1109 local_chunks.assert_chunks(
1110 &write_cache.file_cache.cache_file_path(key),
1111 chunk_size,
1112 cache_data.len(),
1113 );
1114
1115 let index_key = IndexKey::new(region_id, file_id, FileType::Puffin(0));
1117 assert!(write_cache.file_cache.contains_key(&index_key));
1118
1119 let remote_index_data = mock_store.read(&index_upload_path).await.unwrap();
1120 let cache_index_data = local_store
1121 .read(&write_cache.file_cache.cache_file_path(index_key))
1122 .await
1123 .unwrap();
1124 assert_eq!(remote_index_data.to_vec(), cache_index_data.to_vec());
1125 remote_chunks.assert_chunks(
1126 &index_upload_path,
1127 DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize,
1128 remote_index_data.len(),
1129 );
1130
1131 let sst_index_key = IndexKey::new(region_id, file_id, FileType::Parquet);
1133 write_cache.remove(sst_index_key).await;
1134 assert!(!write_cache.file_cache.contains_key(&sst_index_key));
1135 write_cache.remove(index_key).await;
1136 assert!(!write_cache.file_cache.contains_key(&index_key));
1137 }
1138
1139 #[tokio::test]
1140 async fn test_read_metadata_from_write_cache() {
1141 common_telemetry::init_default_ut_logging();
1142 let mut env = TestEnv::new().await;
1143 let data_home = "write-cache-test".to_string();
1145 let mock_store = env.init_object_store_manager();
1146
1147 let local_dir = create_temp_dir("");
1148 let local_path = local_dir.path().to_str().unwrap();
1149 let local_store = new_fs_store(local_path);
1150
1151 let write_cache = env
1153 .create_write_cache(local_store.clone(), ReadableSize::mb(10))
1154 .await;
1155 let cache_manager = Arc::new(
1156 CacheManager::builder()
1157 .write_cache(Some(write_cache.clone()))
1158 .build(),
1159 );
1160 assert!(!cache_manager.sst_meta_cache_enabled());
1161
1162 let metadata = Arc::new(sst_region_metadata());
1164
1165 let source = new_flat_source_from_record_batches(vec![
1166 new_record_batch_by_range(&["a", "d"], 0, 60),
1167 new_record_batch_by_range(&["b", "f"], 0, 40),
1168 new_record_batch_by_range(&["b", "h"], 100, 200),
1169 ]);
1170
1171 let write_request = SstWriteRequest {
1173 index_build_inputs: None,
1174 op_type: OperationType::Flush,
1175 metadata,
1176 source,
1177 max_sequence: None,
1178 sst_write_format: Default::default(),
1179 cache_manager: cache_manager.clone(),
1180 preserve_row_sequence: false,
1181 index_options: IndexOptions::default(),
1182 index_config: Default::default(),
1183 inverted_index_config: Default::default(),
1184 fulltext_index_config: Default::default(),
1185 bloom_filter_index_config: Default::default(),
1186 };
1187 let write_opts = WriteOptions {
1188 row_group_size: 512,
1189 ..Default::default()
1190 };
1191 let upload_request = SstUploadRequest {
1192 dest_path_provider: RegionFilePathFactory::new(data_home.clone(), PathType::Bare),
1193 remote_store: mock_store.clone(),
1194 };
1195
1196 let mut metrics = Metrics::new(WriteType::Flush);
1197 let mut sst_infos = write_cache
1198 .write_and_upload_sst_with_context(
1199 write_request,
1200 upload_request,
1201 &write_opts,
1202 &mut metrics,
1203 SstWriteContext::default(),
1204 )
1205 .await
1206 .unwrap();
1207 let sst_info = sst_infos.remove(0);
1208 let write_parquet_metadata = sst_info.file_metadata.unwrap();
1209
1210 let handle = sst_file_handle_with_file_id(sst_info.file_id, 0, 1000);
1212 let builder = ParquetReaderBuilder::new(
1213 data_home,
1214 PathType::Bare,
1215 handle.clone(),
1216 mock_store.clone(),
1217 )
1218 .cache(CacheStrategy::EnableAll(cache_manager.clone()))
1219 .page_index_policy(PageIndexPolicy::Optional);
1220 let reader = builder.build().await.unwrap().unwrap();
1221 let cached_write_parquet_metadata = crate::cache::CachedSstMeta::try_new(
1222 "test.sst",
1223 Arc::unwrap_or_clone(write_parquet_metadata),
1224 )
1225 .unwrap()
1226 .parquet_metadata();
1227
1228 assert_parquet_metadata_equal(cached_write_parquet_metadata, reader.parquet_metadata());
1230 }
1231
1232 #[tokio::test]
1233 async fn test_write_cache_clean_tmp_files() {
1234 common_telemetry::init_default_ut_logging();
1235 let mut env = TestEnv::new().await;
1236 let data_home = env.data_home().display().to_string();
1237 let mock_store = env.init_object_store_manager();
1238
1239 let write_cache_dir = create_temp_dir("");
1240 let write_cache_path = write_cache_dir.path().to_str().unwrap();
1241 let write_cache = env
1242 .create_write_cache_from_path(write_cache_path, ReadableSize::mb(10))
1243 .await;
1244
1245 let cache_manager = Arc::new(
1247 CacheManager::builder()
1248 .write_cache(Some(write_cache.clone()))
1249 .build(),
1250 );
1251
1252 let metadata = Arc::new(sst_region_metadata());
1254
1255 let record_batch = new_record_batch_by_range(&["a", "d"], 0, 60);
1257 let schema = record_batch.schema();
1258 let iter = Box::new(
1259 [
1260 Ok(record_batch),
1261 InvalidBatchSnafu {
1262 reason: "Abort the writer",
1263 }
1264 .fail(),
1265 ]
1266 .into_iter(),
1267 );
1268 let source = FlatSource::new_iter(schema, iter);
1269
1270 let write_request = SstWriteRequest {
1272 index_build_inputs: None,
1273 op_type: OperationType::Flush,
1274 metadata,
1275 source,
1276 max_sequence: None,
1277 sst_write_format: Default::default(),
1278 cache_manager: cache_manager.clone(),
1279 preserve_row_sequence: false,
1280 index_options: IndexOptions::default(),
1281 index_config: Default::default(),
1282 inverted_index_config: Default::default(),
1283 fulltext_index_config: Default::default(),
1284 bloom_filter_index_config: Default::default(),
1285 };
1286 let write_opts = WriteOptions {
1287 row_group_size: 512,
1288 ..Default::default()
1289 };
1290 let upload_request = SstUploadRequest {
1291 dest_path_provider: RegionFilePathFactory::new(data_home.clone(), PathType::Bare),
1292 remote_store: mock_store.clone(),
1293 };
1294
1295 let mut metrics = Metrics::new(WriteType::Flush);
1296 write_cache
1297 .write_and_upload_sst_with_context(
1298 write_request,
1299 upload_request,
1300 &write_opts,
1301 &mut metrics,
1302 SstWriteContext::default(),
1303 )
1304 .await
1305 .unwrap_err();
1306 let atomic_write_dir = write_cache_dir.path().join(ATOMIC_WRITE_DIR);
1307 let mut entries = tokio::fs::read_dir(&atomic_write_dir).await.unwrap();
1308 let mut has_files = false;
1309 while let Some(entry) = entries.next_entry().await.unwrap() {
1310 if entry.file_type().await.unwrap().is_dir() {
1311 continue;
1312 }
1313 has_files = true;
1314 common_telemetry::warn!(
1315 "Found remaining temporary file in atomic dir: {}",
1316 entry.path().display()
1317 );
1318 }
1319
1320 assert!(!has_files);
1321 }
1322
1323 #[tokio::test]
1324 async fn test_download_if_absent_skips_when_cached() {
1325 let mut env = TestEnv::new().await;
1326 let remote_store = env.init_object_store_manager();
1327
1328 let local_dir = create_temp_dir("");
1329 let local_store = new_fs_store(local_dir.path().to_str().unwrap());
1330 let write_cache = env
1331 .create_write_cache(local_store.clone(), ReadableSize::mb(10))
1332 .await;
1333
1334 let region_id = RegionId::new(1024, 1);
1335 let file_id = FileId::random();
1336 let key = IndexKey::new(region_id, file_id, FileType::Parquet);
1337 write_cache
1338 .file_cache()
1339 .put(key, IndexValue { file_size: 1 })
1340 .await;
1341
1342 let downloaded = write_cache
1343 .download_if_absent(key, "missing/path.parquet", &remote_store, 1)
1344 .await
1345 .unwrap();
1346
1347 assert!(!downloaded);
1348 }
1349
1350 #[tokio::test]
1351 async fn test_download_if_absent_downloads_when_missing() {
1352 let mut env = TestEnv::new().await;
1353 let remote_store = env.init_object_store_manager();
1354
1355 let local_dir = create_temp_dir("");
1356 let local_store = new_fs_store(local_dir.path().to_str().unwrap());
1357 let write_cache = env
1358 .create_write_cache(local_store.clone(), ReadableSize::mb(10))
1359 .await;
1360
1361 let region_id = RegionId::new(1024, 2);
1362 let file_id = FileId::random();
1363 let key = IndexKey::new(region_id, file_id, FileType::Parquet);
1364 let remote_path = format!("download-if-absent/{file_id}.parquet");
1365 let remote_data = Bytes::from_static(b"download-if-absent-test");
1366 remote_store
1367 .write(&remote_path, remote_data.clone())
1368 .await
1369 .unwrap();
1370
1371 let downloaded = write_cache
1372 .download_if_absent(key, &remote_path, &remote_store, remote_data.len() as u64)
1373 .await
1374 .unwrap();
1375
1376 assert!(downloaded);
1377 assert!(write_cache.file_cache().contains_key(&key));
1378
1379 let cached_data = local_store
1380 .read(&write_cache.file_cache().cache_file_path(key))
1381 .await
1382 .unwrap();
1383 assert_eq!(cached_data.to_vec(), remote_data.to_vec());
1384 }
1385}