1use std::path::Path;
16use std::sync::{Arc, Mutex};
17use std::time::Duration;
18
19use async_trait::async_trait;
20use common_error::ext::BoxedError;
21use object_store::{FuturesAsyncWriter, ObjectStore, Service, ServiceInfo, Servicer};
22use puffin::error::{self as puffin_error, Result as PuffinResult};
23use puffin::puffin_manager::file_accessor::PuffinFileAccessor;
24use puffin::puffin_manager::file_key::{PuffinFileKey, PuffinStorageScope};
25use puffin::puffin_manager::fs_puffin_manager::FsPuffinManager;
26use puffin::puffin_manager::stager::{BoundedStager, Stager};
27use puffin::puffin_manager::{BlobGuard, PuffinManager, PuffinReader};
28use snafu::ResultExt;
29
30use crate::access_layer::FilePathProvider;
31use crate::error::{PuffinInitStagerSnafu, PuffinPurgeStagerSnafu, Result};
32use crate::metrics::{
33 INDEX_PUFFIN_FLUSH_OP_TOTAL, INDEX_PUFFIN_READ_BYTES_TOTAL, INDEX_PUFFIN_READ_OP_TOTAL,
34 INDEX_PUFFIN_WRITE_BYTES_TOTAL, INDEX_PUFFIN_WRITE_OP_TOTAL, StagerMetrics,
35};
36use crate::sst::file::RegionIndexId;
37use crate::sst::index::store::{self, InstrumentedStore};
38
39type InstrumentedRangeReader = store::InstrumentedRangeReader<'static>;
40type InstrumentedAsyncWrite = store::InstrumentedAsyncWrite<'static, FuturesAsyncWriter>;
41
42pub type SstPuffinManager =
43 FsPuffinManager<Arc<BoundedStager<RegionIndexId>>, ObjectStorePuffinFileAccessor>;
44pub type SstPuffinReader = <SstPuffinManager as PuffinManager>::Reader;
45pub type SstPuffinWriter = <SstPuffinManager as PuffinManager>::Writer;
46pub type SstPuffinBlob = <SstPuffinReader as PuffinReader>::Blob;
47pub type SstPuffinDir = <SstPuffinReader as PuffinReader>::Dir;
48pub type BlobReader = <SstPuffinBlob as BlobGuard>::Reader;
49
50const STAGING_DIR: &str = "staging";
51
52#[derive(Default)]
56struct BackendScopes {
57 entries: Mutex<Vec<BackendScope>>,
58}
59
60struct BackendScope {
61 service: Servicer,
62 location: ServiceInfo,
63 scope: Arc<PuffinStorageScope>,
64}
65
66impl BackendScopes {
67 fn get(&self, store: &ObjectStore) -> Arc<PuffinStorageScope> {
68 let service = store.base_service();
69 let location = store.service().info();
70 let mut entries = self
71 .entries
72 .lock()
73 .unwrap_or_else(|error| error.into_inner());
74 if let Some(entry) = entries
75 .iter()
76 .find(|entry| Arc::ptr_eq(&entry.service, service) && entry.location == location)
77 {
78 return entry.scope.clone();
79 }
80 let scope = Arc::new(PuffinStorageScope::unique(
81 location.scheme().to_string(),
82 location.name().to_string(),
83 location.root().to_string(),
84 ));
85 entries.push(BackendScope {
86 service: service.clone(),
87 location,
88 scope: scope.clone(),
89 });
90 scope
91 }
92}
93
94#[derive(Clone)]
96pub struct PuffinManagerFactory {
97 stager: Arc<BoundedStager<RegionIndexId>>,
99
100 write_buffer_size: Option<usize>,
102 backend_scopes: Arc<BackendScopes>,
103}
104
105impl PuffinManagerFactory {
106 pub async fn new(
108 aux_path: impl AsRef<Path>,
109 staging_capacity: u64,
110 write_buffer_size: Option<usize>,
111 staging_ttl: Option<Duration>,
112 ) -> Result<Self> {
113 let staging_dir = aux_path.as_ref().join(STAGING_DIR);
114 let stager = BoundedStager::new(
115 staging_dir,
116 staging_capacity,
117 Some(Arc::new(StagerMetrics::default())),
118 staging_ttl,
119 )
120 .await
121 .context(PuffinInitStagerSnafu)?;
122 Ok(Self {
123 stager: Arc::new(stager),
124 write_buffer_size,
125 backend_scopes: Arc::default(),
126 })
127 }
128
129 pub(crate) fn with_write_buffer_size(mut self, bytes: usize) -> Self {
131 self.write_buffer_size = Some(bytes);
132 self
133 }
134
135 pub fn build(
136 &self,
137 store: ObjectStore,
138 path_provider: impl FilePathProvider + 'static,
139 ) -> SstPuffinManager {
140 let scope = self.backend_scopes.get(&store);
141 let store = InstrumentedStore::new(store).with_write_buffer_size(self.write_buffer_size);
142 let puffin_file_accessor =
143 ObjectStorePuffinFileAccessor::new(store, Arc::new(path_provider), scope);
144 SstPuffinManager::new(self.stager.clone(), puffin_file_accessor)
145 }
146
147 pub(crate) fn build_cached(
153 &self,
154 origin: &SstPuffinManager,
155 store: ObjectStore,
156 path_provider: impl FilePathProvider + 'static,
157 ) -> SstPuffinManager {
158 let origin = origin.file_accessor();
159 let accessor = ObjectStorePuffinFileAccessor {
160 object_store: InstrumentedStore::new(store)
161 .with_write_buffer_size(self.write_buffer_size),
162 path_provider: Arc::new(path_provider),
163 cache_path_provider: origin.cache_path_provider.clone(),
164 cache_scope: origin.cache_scope.clone(),
165 };
166 SstPuffinManager::new(self.stager.clone(), accessor)
167 }
168
169 pub(crate) async fn purge_stager(&self, file_id: RegionIndexId) -> Result<()> {
170 self.stager
171 .purge(&file_id)
172 .await
173 .context(PuffinPurgeStagerSnafu)
174 }
175}
176
177#[cfg(test)]
178impl PuffinManagerFactory {
179 pub(crate) async fn new_for_test_async(
180 prefix: &str,
181 ) -> (common_test_util::temp_dir::TempDir, Self) {
182 let tempdir = common_test_util::temp_dir::create_temp_dir(prefix);
183 let factory = Self::new(tempdir.path().to_path_buf(), 1024, None, None)
184 .await
185 .unwrap();
186 (tempdir, factory)
187 }
188
189 pub(crate) fn new_for_test_block(prefix: &str) -> (common_test_util::temp_dir::TempDir, Self) {
190 let tempdir = common_test_util::temp_dir::create_temp_dir(prefix);
191
192 let f = Self::new(tempdir.path().to_path_buf(), 1024, None, None);
193 let factory = common_runtime::block_on_global(f).unwrap();
194
195 (tempdir, factory)
196 }
197}
198
199#[derive(Clone)]
201pub struct ObjectStorePuffinFileAccessor {
202 object_store: InstrumentedStore,
203 path_provider: Arc<dyn FilePathProvider>,
204 cache_path_provider: Arc<dyn FilePathProvider>,
205 cache_scope: Arc<PuffinStorageScope>,
206}
207
208impl ObjectStorePuffinFileAccessor {
209 pub(crate) fn new(
210 object_store: InstrumentedStore,
211 path_provider: Arc<dyn FilePathProvider>,
212 cache_scope: Arc<PuffinStorageScope>,
213 ) -> Self {
214 Self {
215 object_store,
216 cache_path_provider: path_provider.clone(),
217 path_provider,
218 cache_scope,
219 }
220 }
221
222 pub(crate) fn store(&self) -> &InstrumentedStore {
223 &self.object_store
224 }
225}
226
227#[async_trait]
228impl PuffinFileAccessor for ObjectStorePuffinFileAccessor {
229 type Reader = InstrumentedRangeReader;
230 type Writer = InstrumentedAsyncWrite;
231 type FileHandle = RegionIndexId;
232
233 fn file_key(&self, handle: &RegionIndexId) -> PuffinFileKey<RegionIndexId> {
234 PuffinFileKey::new(
235 self.cache_scope.clone(),
236 self.cache_path_provider
237 .build_index_file_path_with_version(*handle),
238 *handle,
239 )
240 }
241
242 async fn reader(&self, handle: &RegionIndexId) -> PuffinResult<Self::Reader> {
243 let file_path = self
244 .path_provider
245 .build_index_file_path_with_version(*handle);
246 self.object_store
247 .range_reader(
248 &file_path,
249 &INDEX_PUFFIN_READ_BYTES_TOTAL,
250 &INDEX_PUFFIN_READ_OP_TOTAL,
251 )
252 .await
253 .map_err(BoxedError::new)
254 .context(puffin_error::ExternalSnafu)
255 }
256
257 async fn writer(&self, handle: &RegionIndexId) -> PuffinResult<Self::Writer> {
258 let file_path = self
259 .path_provider
260 .build_index_file_path_with_version(*handle);
261 self.object_store
262 .writer(
263 &file_path,
264 &INDEX_PUFFIN_WRITE_BYTES_TOTAL,
265 &INDEX_PUFFIN_WRITE_OP_TOTAL,
266 &INDEX_PUFFIN_FLUSH_OP_TOTAL,
267 )
268 .await
269 .map_err(BoxedError::new)
270 .context(puffin_error::ExternalSnafu)
271 }
272}
273
274#[cfg(test)]
275mod tests {
276
277 use common_base::range_read::RangeReader;
278 use common_test_util::temp_dir::create_temp_dir;
279 use futures::io::Cursor;
280 use object_store::services::Memory;
281 use puffin::blob_metadata::CompressionCodec;
282 use puffin::puffin_manager::{PuffinManager, PuffinReader, PuffinWriter, PutOptions};
283 use store_api::storage::FileId;
284
285 use super::*;
286 use crate::sst::file::{RegionFileId, RegionIndexId};
287
288 struct TestFilePathProvider;
289
290 impl FilePathProvider for TestFilePathProvider {
291 fn build_index_file_path(&self, file_id: RegionFileId) -> String {
292 file_id.file_id().to_string()
293 }
294
295 fn build_index_file_path_with_version(&self, index_id: RegionIndexId) -> String {
296 index_id.file_id.file_id().to_string()
297 }
298
299 fn build_sst_file_path(&self, file_id: RegionFileId) -> String {
300 file_id.file_id().to_string()
301 }
302 }
303
304 #[tokio::test]
305 async fn test_file_key_tracks_storage_path_and_version() {
306 let (_dir, factory) = PuffinManagerFactory::new_for_test_async("test_file_key_").await;
307 let store = ObjectStore::new(Memory::default()).unwrap();
308 let index_id = RegionIndexId::new(RegionFileId::new(0.into(), FileId::random()), 0);
309 let manager = factory.build(store.clone(), TestFilePathProvider);
310 let rebuilt = factory.build(store, TestFilePathProvider);
311 assert_eq!(manager.file_key(&index_id), rebuilt.file_key(&index_id));
312 let next = RegionIndexId::new(index_id.file_id, 1);
314 assert_ne!(manager.file_key(&index_id), manager.file_key(&next));
315
316 let other_store = ObjectStore::new(Memory::default()).unwrap();
317 let other = factory.build(other_store, TestFilePathProvider);
318 assert_ne!(manager.file_key(&index_id), other.file_key(&index_id));
319
320 struct PrefixedPath;
321 impl FilePathProvider for PrefixedPath {
322 fn build_index_file_path(&self, id: RegionFileId) -> String {
323 format!("other/{}", id.file_id())
324 }
325 fn build_index_file_path_with_version(&self, id: RegionIndexId) -> String {
326 format!("other/{}", id.file_id())
327 }
328 fn build_sst_file_path(&self, id: RegionFileId) -> String {
329 format!("other/{}", id.file_id())
330 }
331 }
332 let same_store = manager.file_accessor().store().store().clone();
333 let moved = factory.build(same_store, PrefixedPath);
334 assert_ne!(manager.file_key(&index_id), moved.file_key(&index_id));
335 }
336
337 #[tokio::test]
338 async fn test_cached_replica_reuses_origin_metadata_and_reads_local_bytes() {
339 use api::v1::index::InvertedIndexMetas;
340 use puffin::puffin_manager::cache::PuffinMetadataCache;
341 use store_api::region_request::PathType;
342
343 use crate::access_layer::RegionFilePathFactory;
344 use crate::cache::index::inverted_index::InvertedIndexCache;
345 use crate::metrics::CACHE_BYTES;
346
347 let (_dir, factory) = PuffinManagerFactory::new_for_test_async("test_replica_key_").await;
348 let remote_store = ObjectStore::new(Memory::default()).unwrap();
349 let local_store = ObjectStore::new(Memory::default()).unwrap();
350 let metadata_cache = Arc::new(PuffinMetadataCache::new(1024 * 1024, &CACHE_BYTES));
351 let manager = factory
352 .build(remote_store.clone(), TestFilePathProvider)
353 .with_puffin_metadata_cache(Some(metadata_cache.clone()));
354 let replica = factory
355 .build_cached(
356 &manager,
357 local_store.clone(),
358 RegionFilePathFactory::new("cached/".into(), PathType::Bare),
359 )
360 .with_puffin_metadata_cache(Some(metadata_cache));
361 let isolated_replica = factory.build(
363 local_store.clone(),
364 RegionFilePathFactory::new("cached/".into(), PathType::Bare),
365 );
366 let id = RegionIndexId::new(RegionFileId::new(0.into(), FileId::random()), 0);
367 assert_ne!(manager.file_key(&id), isolated_replica.file_key(&id));
368 assert_eq!(manager.file_key(&id), replica.file_key(&id));
369 assert_ne!(
370 manager.file_key(&id),
371 replica.file_key(&RegionIndexId::new(id.file_id, 1))
372 );
373
374 let bytes = b"read these bytes from the local replica";
375 let mut writer = manager.writer(&id).await.unwrap();
376 writer
377 .put_blob(
378 "blob",
379 Cursor::new(bytes),
380 PutOptions::default(),
381 Default::default(),
382 )
383 .await
384 .unwrap();
385 writer.finish().await.unwrap();
386 let remote_path = TestFilePathProvider.build_index_file_path_with_version(id);
387 let local_path = RegionFilePathFactory::new("cached/".into(), PathType::Bare)
388 .build_index_file_path_with_version(id);
389 assert_ne!(remote_path, local_path);
390 local_store
391 .write(&local_path, remote_store.read(&remote_path).await.unwrap())
392 .await
393 .unwrap();
394
395 let metadata = manager.reader(&id).await.unwrap().metadata().await.unwrap();
396 remote_store.delete(&remote_path).await.unwrap();
397 let reader = replica.reader(&id).await.unwrap();
398 assert!(Arc::ptr_eq(&metadata, &reader.metadata().await.unwrap()));
399 let blob = reader.blob("blob").await.unwrap().reader().await.unwrap();
400 assert_eq!(&*blob.read(0..bytes.len() as u64).await.unwrap(), bytes);
401
402 let index_cache = InvertedIndexCache::new(1024 * 1024, 1024 * 1024, 1024);
403 let index_metadata = Arc::new(InvertedIndexMetas::default());
404 index_cache
405 .put_metadata(manager.file_key(&id), index_metadata.clone())
406 .await;
407 assert!(
408 index_cache
409 .get_metadata(isolated_replica.file_key(&id))
410 .await
411 .is_none()
412 );
413 assert!(Arc::ptr_eq(
414 &index_metadata,
415 &index_cache
416 .get_metadata(replica.file_key(&id))
417 .await
418 .unwrap()
419 ));
420 }
421
422 #[tokio::test]
423 async fn test_file_key_isolates_s3_endpoints_without_io() {
424 let (_dir, factory) = PuffinManagerFactory::new_for_test_async("test_s3_scope_").await;
425 let build = |endpoint| {
426 ObjectStore::new(
427 object_store::services::S3::default()
428 .bucket("same-bucket")
429 .root("same-root")
430 .region("us-east-1")
431 .endpoint(endpoint),
432 )
433 .unwrap()
434 };
435 let first = build("http://127.0.0.1:19001");
436 let second = build("http://127.0.0.1:19002");
437 assert_eq!(first.info().scheme(), second.info().scheme());
438 assert_eq!(first.info().name(), second.info().name());
439 assert_eq!(first.info().root(), second.info().root());
440 let index_id = RegionIndexId::new(RegionFileId::new(0.into(), FileId::random()), 0);
441 let manager = factory.build(first.clone(), TestFilePathProvider);
442 let reconstructed = factory.clone().build(first.clone(), TestFilePathProvider);
443 let other = factory.build(second, TestFilePathProvider);
444 assert_eq!(
445 manager.file_key(&index_id),
446 reconstructed.file_key(&index_id)
447 );
448 assert_ne!(manager.file_key(&index_id), other.file_key(&index_id));
449 let (_other_dir, other_factory) =
450 PuffinManagerFactory::new_for_test_async("test_other_scope_").await;
451 let independent = other_factory.build(first, TestFilePathProvider);
452 assert_ne!(manager.file_key(&index_id), independent.file_key(&index_id));
453 }
454
455 #[tokio::test]
456 async fn test_puffin_manager_factory() {
457 let (_dir, factory) =
458 PuffinManagerFactory::new_for_test_async("test_puffin_manager_factory_").await;
459
460 let object_store = ObjectStore::new(Memory::default()).unwrap();
461 let manager = factory.build(object_store, TestFilePathProvider);
462
463 let file_id = RegionIndexId::new(RegionFileId::new(0.into(), FileId::random()), 0);
464 let blob_key = "blob-key";
465 let dir_key = "dir-key";
466 let raw_data = b"hello world!";
467
468 let mut writer = manager.writer(&file_id).await.unwrap();
469 writer
470 .put_blob(
471 blob_key,
472 Cursor::new(raw_data),
473 PutOptions::default(),
474 Default::default(),
475 )
476 .await
477 .unwrap();
478 let dir_data = create_temp_dir("test_puffin_manager_factory_dir_data_");
479 tokio::fs::write(dir_data.path().join("hello"), raw_data)
480 .await
481 .unwrap();
482 writer
483 .put_dir(
484 dir_key,
485 dir_data.path().into(),
486 PutOptions {
487 compression: Some(CompressionCodec::Zstd),
488 },
489 Default::default(),
490 )
491 .await
492 .unwrap();
493 writer.finish().await.unwrap();
494
495 let reader = manager.reader(&file_id).await.unwrap();
496 let blob_guard = reader.blob(blob_key).await.unwrap();
497 let blob_reader = blob_guard.reader().await.unwrap();
498 let meta = blob_reader.metadata().await.unwrap();
499 let bs = blob_reader.read(0..meta.content_length).await.unwrap();
500 assert_eq!(&*bs, raw_data);
501
502 let (dir_guard, _metrics) = reader.dir(dir_key).await.unwrap();
503 let file = dir_guard.path().join("hello");
504 let data = tokio::fs::read(file).await.unwrap();
505 assert_eq!(data, raw_data);
506 }
507}