Skip to main content

mito2/sst/index/
puffin_manager.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::path::Path;
16use std::sync::{Arc, Mutex};
17use std::time::Duration;
18
19use async_trait::async_trait;
20use common_error::ext::BoxedError;
21use object_store::{FuturesAsyncWriter, ObjectStore, Service, ServiceInfo, Servicer};
22use puffin::error::{self as puffin_error, Result as PuffinResult};
23use puffin::puffin_manager::file_accessor::PuffinFileAccessor;
24use puffin::puffin_manager::file_key::{PuffinFileKey, PuffinStorageScope};
25use puffin::puffin_manager::fs_puffin_manager::FsPuffinManager;
26use puffin::puffin_manager::stager::{BoundedStager, Stager};
27use puffin::puffin_manager::{BlobGuard, PuffinManager, PuffinReader};
28use snafu::ResultExt;
29
30use crate::access_layer::FilePathProvider;
31use crate::error::{PuffinInitStagerSnafu, PuffinPurgeStagerSnafu, Result};
32use crate::metrics::{
33    INDEX_PUFFIN_FLUSH_OP_TOTAL, INDEX_PUFFIN_READ_BYTES_TOTAL, INDEX_PUFFIN_READ_OP_TOTAL,
34    INDEX_PUFFIN_WRITE_BYTES_TOTAL, INDEX_PUFFIN_WRITE_OP_TOTAL, StagerMetrics,
35};
36use crate::sst::file::RegionIndexId;
37use crate::sst::index::store::{self, InstrumentedStore};
38
39type InstrumentedRangeReader = store::InstrumentedRangeReader<'static>;
40type InstrumentedAsyncWrite = store::InstrumentedAsyncWrite<'static, FuturesAsyncWriter>;
41
42pub type SstPuffinManager =
43    FsPuffinManager<Arc<BoundedStager<RegionIndexId>>, ObjectStorePuffinFileAccessor>;
44pub type SstPuffinReader = <SstPuffinManager as PuffinManager>::Reader;
45pub type SstPuffinWriter = <SstPuffinManager as PuffinManager>::Writer;
46pub type SstPuffinBlob = <SstPuffinReader as PuffinReader>::Blob;
47pub type SstPuffinDir = <SstPuffinReader as PuffinReader>::Dir;
48pub type BlobReader = <SstPuffinBlob as BlobGuard>::Reader;
49
50const STAGING_DIR: &str = "staging";
51
52/// Reuses a scope across managers built from the same backend instance.
53/// Keeping the service alive prevents pointer reuse from aliasing a cached identity.
54/// Entries live with the factory and grow with distinct backend instances it encounters.
55#[derive(Default)]
56struct BackendScopes {
57    entries: Mutex<Vec<BackendScope>>,
58}
59
60struct BackendScope {
61    service: Servicer,
62    location: ServiceInfo,
63    scope: Arc<PuffinStorageScope>,
64}
65
66impl BackendScopes {
67    fn get(&self, store: &ObjectStore) -> Arc<PuffinStorageScope> {
68        let service = store.base_service();
69        let location = store.service().info();
70        let mut entries = self
71            .entries
72            .lock()
73            .unwrap_or_else(|error| error.into_inner());
74        if let Some(entry) = entries
75            .iter()
76            .find(|entry| Arc::ptr_eq(&entry.service, service) && entry.location == location)
77        {
78            return entry.scope.clone();
79        }
80        let scope = Arc::new(PuffinStorageScope::unique(
81            location.scheme().to_string(),
82            location.name().to_string(),
83            location.root().to_string(),
84        ));
85        entries.push(BackendScope {
86            service: service.clone(),
87            location,
88            scope: scope.clone(),
89        });
90        scope
91    }
92}
93
94/// A factory for creating `SstPuffinManager` instances.
95#[derive(Clone)]
96pub struct PuffinManagerFactory {
97    /// The stager used by the puffin manager.
98    stager: Arc<BoundedStager<RegionIndexId>>,
99
100    /// The size of the write buffer used to create object store.
101    write_buffer_size: Option<usize>,
102    backend_scopes: Arc<BackendScopes>,
103}
104
105impl PuffinManagerFactory {
106    /// Creates a new `PuffinManagerFactory` instance.
107    pub async fn new(
108        aux_path: impl AsRef<Path>,
109        staging_capacity: u64,
110        write_buffer_size: Option<usize>,
111        staging_ttl: Option<Duration>,
112    ) -> Result<Self> {
113        let staging_dir = aux_path.as_ref().join(STAGING_DIR);
114        let stager = BoundedStager::new(
115            staging_dir,
116            staging_capacity,
117            Some(Arc::new(StagerMetrics::default())),
118            staging_ttl,
119        )
120        .await
121        .context(PuffinInitStagerSnafu)?;
122        Ok(Self {
123            stager: Arc::new(stager),
124            write_buffer_size,
125            backend_scopes: Arc::default(),
126        })
127    }
128
129    /// Overrides only this task's upload buffers, sharing the existing stager.
130    pub(crate) fn with_write_buffer_size(mut self, bytes: usize) -> Self {
131        self.write_buffer_size = Some(bytes);
132        self
133    }
134
135    pub fn build(
136        &self,
137        store: ObjectStore,
138        path_provider: impl FilePathProvider + 'static,
139    ) -> SstPuffinManager {
140        let scope = self.backend_scopes.get(&store);
141        let store = InstrumentedStore::new(store).with_write_buffer_size(self.write_buffer_size);
142        let puffin_file_accessor =
143            ObjectStorePuffinFileAccessor::new(store, Arc::new(path_provider), scope);
144        SstPuffinManager::new(self.stager.clone(), puffin_file_accessor)
145    }
146
147    /// Builds a reader manager for an exact local replica of an origin file.
148    ///
149    /// `origin` must come from this factory or one of its clones so both managers
150    /// share the same stager. The replica must contain the origin's exact bytes.
151    /// Reads use the replica location; cache keys retain the origin's identity.
152    pub(crate) fn build_cached(
153        &self,
154        origin: &SstPuffinManager,
155        store: ObjectStore,
156        path_provider: impl FilePathProvider + 'static,
157    ) -> SstPuffinManager {
158        let origin = origin.file_accessor();
159        let accessor = ObjectStorePuffinFileAccessor {
160            object_store: InstrumentedStore::new(store)
161                .with_write_buffer_size(self.write_buffer_size),
162            path_provider: Arc::new(path_provider),
163            cache_path_provider: origin.cache_path_provider.clone(),
164            cache_scope: origin.cache_scope.clone(),
165        };
166        SstPuffinManager::new(self.stager.clone(), accessor)
167    }
168
169    pub(crate) async fn purge_stager(&self, file_id: RegionIndexId) -> Result<()> {
170        self.stager
171            .purge(&file_id)
172            .await
173            .context(PuffinPurgeStagerSnafu)
174    }
175}
176
177#[cfg(test)]
178impl PuffinManagerFactory {
179    pub(crate) async fn new_for_test_async(
180        prefix: &str,
181    ) -> (common_test_util::temp_dir::TempDir, Self) {
182        let tempdir = common_test_util::temp_dir::create_temp_dir(prefix);
183        let factory = Self::new(tempdir.path().to_path_buf(), 1024, None, None)
184            .await
185            .unwrap();
186        (tempdir, factory)
187    }
188
189    pub(crate) fn new_for_test_block(prefix: &str) -> (common_test_util::temp_dir::TempDir, Self) {
190        let tempdir = common_test_util::temp_dir::create_temp_dir(prefix);
191
192        let f = Self::new(tempdir.path().to_path_buf(), 1024, None, None);
193        let factory = common_runtime::block_on_global(f).unwrap();
194
195        (tempdir, factory)
196    }
197}
198
199/// A `PuffinFileAccessor` implementation that uses an object store as the underlying storage.
200#[derive(Clone)]
201pub struct ObjectStorePuffinFileAccessor {
202    object_store: InstrumentedStore,
203    path_provider: Arc<dyn FilePathProvider>,
204    cache_path_provider: Arc<dyn FilePathProvider>,
205    cache_scope: Arc<PuffinStorageScope>,
206}
207
208impl ObjectStorePuffinFileAccessor {
209    pub(crate) fn new(
210        object_store: InstrumentedStore,
211        path_provider: Arc<dyn FilePathProvider>,
212        cache_scope: Arc<PuffinStorageScope>,
213    ) -> Self {
214        Self {
215            object_store,
216            cache_path_provider: path_provider.clone(),
217            path_provider,
218            cache_scope,
219        }
220    }
221
222    pub(crate) fn store(&self) -> &InstrumentedStore {
223        &self.object_store
224    }
225}
226
227#[async_trait]
228impl PuffinFileAccessor for ObjectStorePuffinFileAccessor {
229    type Reader = InstrumentedRangeReader;
230    type Writer = InstrumentedAsyncWrite;
231    type FileHandle = RegionIndexId;
232
233    fn file_key(&self, handle: &RegionIndexId) -> PuffinFileKey<RegionIndexId> {
234        PuffinFileKey::new(
235            self.cache_scope.clone(),
236            self.cache_path_provider
237                .build_index_file_path_with_version(*handle),
238            *handle,
239        )
240    }
241
242    async fn reader(&self, handle: &RegionIndexId) -> PuffinResult<Self::Reader> {
243        let file_path = self
244            .path_provider
245            .build_index_file_path_with_version(*handle);
246        self.object_store
247            .range_reader(
248                &file_path,
249                &INDEX_PUFFIN_READ_BYTES_TOTAL,
250                &INDEX_PUFFIN_READ_OP_TOTAL,
251            )
252            .await
253            .map_err(BoxedError::new)
254            .context(puffin_error::ExternalSnafu)
255    }
256
257    async fn writer(&self, handle: &RegionIndexId) -> PuffinResult<Self::Writer> {
258        let file_path = self
259            .path_provider
260            .build_index_file_path_with_version(*handle);
261        self.object_store
262            .writer(
263                &file_path,
264                &INDEX_PUFFIN_WRITE_BYTES_TOTAL,
265                &INDEX_PUFFIN_WRITE_OP_TOTAL,
266                &INDEX_PUFFIN_FLUSH_OP_TOTAL,
267            )
268            .await
269            .map_err(BoxedError::new)
270            .context(puffin_error::ExternalSnafu)
271    }
272}
273
274#[cfg(test)]
275mod tests {
276
277    use common_base::range_read::RangeReader;
278    use common_test_util::temp_dir::create_temp_dir;
279    use futures::io::Cursor;
280    use object_store::services::Memory;
281    use puffin::blob_metadata::CompressionCodec;
282    use puffin::puffin_manager::{PuffinManager, PuffinReader, PuffinWriter, PutOptions};
283    use store_api::storage::FileId;
284
285    use super::*;
286    use crate::sst::file::{RegionFileId, RegionIndexId};
287
288    struct TestFilePathProvider;
289
290    impl FilePathProvider for TestFilePathProvider {
291        fn build_index_file_path(&self, file_id: RegionFileId) -> String {
292            file_id.file_id().to_string()
293        }
294
295        fn build_index_file_path_with_version(&self, index_id: RegionIndexId) -> String {
296            index_id.file_id.file_id().to_string()
297        }
298
299        fn build_sst_file_path(&self, file_id: RegionFileId) -> String {
300            file_id.file_id().to_string()
301        }
302    }
303
304    #[tokio::test]
305    async fn test_file_key_tracks_storage_path_and_version() {
306        let (_dir, factory) = PuffinManagerFactory::new_for_test_async("test_file_key_").await;
307        let store = ObjectStore::new(Memory::default()).unwrap();
308        let index_id = RegionIndexId::new(RegionFileId::new(0.into(), FileId::random()), 0);
309        let manager = factory.build(store.clone(), TestFilePathProvider);
310        let rebuilt = factory.build(store, TestFilePathProvider);
311        assert_eq!(manager.file_key(&index_id), rebuilt.file_key(&index_id));
312        // Even a path provider that omits the version cannot alias generations.
313        let next = RegionIndexId::new(index_id.file_id, 1);
314        assert_ne!(manager.file_key(&index_id), manager.file_key(&next));
315
316        let other_store = ObjectStore::new(Memory::default()).unwrap();
317        let other = factory.build(other_store, TestFilePathProvider);
318        assert_ne!(manager.file_key(&index_id), other.file_key(&index_id));
319
320        struct PrefixedPath;
321        impl FilePathProvider for PrefixedPath {
322            fn build_index_file_path(&self, id: RegionFileId) -> String {
323                format!("other/{}", id.file_id())
324            }
325            fn build_index_file_path_with_version(&self, id: RegionIndexId) -> String {
326                format!("other/{}", id.file_id())
327            }
328            fn build_sst_file_path(&self, id: RegionFileId) -> String {
329                format!("other/{}", id.file_id())
330            }
331        }
332        let same_store = manager.file_accessor().store().store().clone();
333        let moved = factory.build(same_store, PrefixedPath);
334        assert_ne!(manager.file_key(&index_id), moved.file_key(&index_id));
335    }
336
337    #[tokio::test]
338    async fn test_cached_replica_reuses_origin_metadata_and_reads_local_bytes() {
339        use api::v1::index::InvertedIndexMetas;
340        use puffin::puffin_manager::cache::PuffinMetadataCache;
341        use store_api::region_request::PathType;
342
343        use crate::access_layer::RegionFilePathFactory;
344        use crate::cache::index::inverted_index::InvertedIndexCache;
345        use crate::metrics::CACHE_BYTES;
346
347        let (_dir, factory) = PuffinManagerFactory::new_for_test_async("test_replica_key_").await;
348        let remote_store = ObjectStore::new(Memory::default()).unwrap();
349        let local_store = ObjectStore::new(Memory::default()).unwrap();
350        let metadata_cache = Arc::new(PuffinMetadataCache::new(1024 * 1024, &CACHE_BYTES));
351        let manager = factory
352            .build(remote_store.clone(), TestFilePathProvider)
353            .with_puffin_metadata_cache(Some(metadata_cache.clone()));
354        let replica = factory
355            .build_cached(
356                &manager,
357                local_store.clone(),
358                RegionFilePathFactory::new("cached/".into(), PathType::Bare),
359            )
360            .with_puffin_metadata_cache(Some(metadata_cache));
361        // Hold the local bytes and path fixed; only cache identity differs.
362        let isolated_replica = factory.build(
363            local_store.clone(),
364            RegionFilePathFactory::new("cached/".into(), PathType::Bare),
365        );
366        let id = RegionIndexId::new(RegionFileId::new(0.into(), FileId::random()), 0);
367        assert_ne!(manager.file_key(&id), isolated_replica.file_key(&id));
368        assert_eq!(manager.file_key(&id), replica.file_key(&id));
369        assert_ne!(
370            manager.file_key(&id),
371            replica.file_key(&RegionIndexId::new(id.file_id, 1))
372        );
373
374        let bytes = b"read these bytes from the local replica";
375        let mut writer = manager.writer(&id).await.unwrap();
376        writer
377            .put_blob(
378                "blob",
379                Cursor::new(bytes),
380                PutOptions::default(),
381                Default::default(),
382            )
383            .await
384            .unwrap();
385        writer.finish().await.unwrap();
386        let remote_path = TestFilePathProvider.build_index_file_path_with_version(id);
387        let local_path = RegionFilePathFactory::new("cached/".into(), PathType::Bare)
388            .build_index_file_path_with_version(id);
389        assert_ne!(remote_path, local_path);
390        local_store
391            .write(&local_path, remote_store.read(&remote_path).await.unwrap())
392            .await
393            .unwrap();
394
395        let metadata = manager.reader(&id).await.unwrap().metadata().await.unwrap();
396        remote_store.delete(&remote_path).await.unwrap();
397        let reader = replica.reader(&id).await.unwrap();
398        assert!(Arc::ptr_eq(&metadata, &reader.metadata().await.unwrap()));
399        let blob = reader.blob("blob").await.unwrap().reader().await.unwrap();
400        assert_eq!(&*blob.read(0..bytes.len() as u64).await.unwrap(), bytes);
401
402        let index_cache = InvertedIndexCache::new(1024 * 1024, 1024 * 1024, 1024);
403        let index_metadata = Arc::new(InvertedIndexMetas::default());
404        index_cache
405            .put_metadata(manager.file_key(&id), index_metadata.clone())
406            .await;
407        assert!(
408            index_cache
409                .get_metadata(isolated_replica.file_key(&id))
410                .await
411                .is_none()
412        );
413        assert!(Arc::ptr_eq(
414            &index_metadata,
415            &index_cache
416                .get_metadata(replica.file_key(&id))
417                .await
418                .unwrap()
419        ));
420    }
421
422    #[tokio::test]
423    async fn test_file_key_isolates_s3_endpoints_without_io() {
424        let (_dir, factory) = PuffinManagerFactory::new_for_test_async("test_s3_scope_").await;
425        let build = |endpoint| {
426            ObjectStore::new(
427                object_store::services::S3::default()
428                    .bucket("same-bucket")
429                    .root("same-root")
430                    .region("us-east-1")
431                    .endpoint(endpoint),
432            )
433            .unwrap()
434        };
435        let first = build("http://127.0.0.1:19001");
436        let second = build("http://127.0.0.1:19002");
437        assert_eq!(first.info().scheme(), second.info().scheme());
438        assert_eq!(first.info().name(), second.info().name());
439        assert_eq!(first.info().root(), second.info().root());
440        let index_id = RegionIndexId::new(RegionFileId::new(0.into(), FileId::random()), 0);
441        let manager = factory.build(first.clone(), TestFilePathProvider);
442        let reconstructed = factory.clone().build(first.clone(), TestFilePathProvider);
443        let other = factory.build(second, TestFilePathProvider);
444        assert_eq!(
445            manager.file_key(&index_id),
446            reconstructed.file_key(&index_id)
447        );
448        assert_ne!(manager.file_key(&index_id), other.file_key(&index_id));
449        let (_other_dir, other_factory) =
450            PuffinManagerFactory::new_for_test_async("test_other_scope_").await;
451        let independent = other_factory.build(first, TestFilePathProvider);
452        assert_ne!(manager.file_key(&index_id), independent.file_key(&index_id));
453    }
454
455    #[tokio::test]
456    async fn test_puffin_manager_factory() {
457        let (_dir, factory) =
458            PuffinManagerFactory::new_for_test_async("test_puffin_manager_factory_").await;
459
460        let object_store = ObjectStore::new(Memory::default()).unwrap();
461        let manager = factory.build(object_store, TestFilePathProvider);
462
463        let file_id = RegionIndexId::new(RegionFileId::new(0.into(), FileId::random()), 0);
464        let blob_key = "blob-key";
465        let dir_key = "dir-key";
466        let raw_data = b"hello world!";
467
468        let mut writer = manager.writer(&file_id).await.unwrap();
469        writer
470            .put_blob(
471                blob_key,
472                Cursor::new(raw_data),
473                PutOptions::default(),
474                Default::default(),
475            )
476            .await
477            .unwrap();
478        let dir_data = create_temp_dir("test_puffin_manager_factory_dir_data_");
479        tokio::fs::write(dir_data.path().join("hello"), raw_data)
480            .await
481            .unwrap();
482        writer
483            .put_dir(
484                dir_key,
485                dir_data.path().into(),
486                PutOptions {
487                    compression: Some(CompressionCodec::Zstd),
488                },
489                Default::default(),
490            )
491            .await
492            .unwrap();
493        writer.finish().await.unwrap();
494
495        let reader = manager.reader(&file_id).await.unwrap();
496        let blob_guard = reader.blob(blob_key).await.unwrap();
497        let blob_reader = blob_guard.reader().await.unwrap();
498        let meta = blob_reader.metadata().await.unwrap();
499        let bs = blob_reader.read(0..meta.content_length).await.unwrap();
500        assert_eq!(&*bs, raw_data);
501
502        let (dir_guard, _metrics) = reader.dir(dir_key).await.unwrap();
503        let file = dir_guard.path().join("hello");
504        let data = tokio::fs::read(file).await.unwrap();
505        assert_eq!(data, raw_data);
506    }
507}