puffin/
puffin_manager.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod cache;
16pub mod file_accessor;
17pub mod fs_puffin_manager;
18pub mod stager;
19
20#[cfg(test)]
21mod tests;
22
23use std::collections::HashMap;
24use std::path::PathBuf;
25use std::sync::Arc;
26
27use async_trait::async_trait;
28use common_base::range_read::RangeReader;
29use futures::AsyncRead;
30
31use crate::blob_metadata::{BlobMetadata, CompressionCodec};
32use crate::error::Result;
33use crate::file_metadata::FileMetadata;
34
35/// Metrics returned by `PuffinReader::dir` operations.
36#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
37pub struct DirMetrics {
38    /// Whether this was a cache hit (true) or cache miss (false).
39    pub cache_hit: bool,
40    /// Size of the directory in bytes.
41    pub dir_size: u64,
42}
43
44/// The `PuffinManager` trait provides a unified interface for creating `PuffinReader` and `PuffinWriter`.
45#[async_trait]
46pub trait PuffinManager {
47    type Reader: PuffinReader;
48    type Writer: PuffinWriter;
49    type FileHandle: ToString + Clone + Send + Sync;
50
51    /// Creates a `PuffinReader` for the specified `handle`.
52    async fn reader(&self, handle: &Self::FileHandle) -> Result<Self::Reader>;
53
54    /// Creates a `PuffinWriter` for the specified `handle`.
55    async fn writer(&self, handle: &Self::FileHandle) -> Result<Self::Writer>;
56}
57
58/// The `PuffinWriter` trait provides methods for writing blobs and directories to a Puffin file.
59#[async_trait]
60pub trait PuffinWriter {
61    /// Writes a blob associated with the specified `key` to the Puffin file.
62    /// Returns the number of bytes written.
63    async fn put_blob<R>(
64        &mut self,
65        key: &str,
66        raw_data: R,
67        options: PutOptions,
68        properties: HashMap<String, String>,
69    ) -> Result<u64>
70    where
71        R: AsyncRead + Send;
72
73    /// Writes a directory associated with the specified `key` to the Puffin file.
74    /// Returns the number of bytes written.
75    ///
76    /// The specified `dir` should be accessible from the filesystem.
77    async fn put_dir(
78        &mut self,
79        key: &str,
80        dir: PathBuf,
81        options: PutOptions,
82        properties: HashMap<String, String>,
83    ) -> Result<u64>;
84
85    /// Sets whether the footer should be LZ4 compressed.
86    fn set_footer_lz4_compressed(&mut self, lz4_compressed: bool);
87
88    /// Finalizes the Puffin file after writing.
89    async fn finish(self) -> Result<u64>;
90}
91
92/// Options available for `put_blob` and `put_dir` methods.
93#[derive(Debug, Clone, Default)]
94pub struct PutOptions {
95    /// The compression codec to use for blob data.
96    pub compression: Option<CompressionCodec>,
97}
98
99/// The `PuffinReader` trait provides methods for reading blobs and directories from a Puffin file.
100#[async_trait]
101pub trait PuffinReader {
102    type Blob: BlobGuard;
103    type Dir: DirGuard;
104
105    fn with_file_size_hint(self, file_size_hint: Option<u64>) -> Self;
106
107    /// Returns the metadata of the Puffin file.
108    async fn metadata(&self) -> Result<Arc<FileMetadata>>;
109
110    /// Reads a blob from the Puffin file.
111    ///
112    /// The returned `GuardWithMetadata` is used to access the blob data and its metadata.
113    /// Users should hold the `GuardWithMetadata` until they are done with the blob data.
114    async fn blob(&self, key: &str) -> Result<GuardWithMetadata<Self::Blob>>;
115
116    /// Reads a directory from the Puffin file.
117    ///
118    /// The returned tuple contains `GuardWithMetadata` and `DirMetrics`.
119    /// The `GuardWithMetadata` is used to access the directory data and its metadata.
120    /// Users should hold the `GuardWithMetadata` until they are done with the directory data.
121    async fn dir(&self, key: &str) -> Result<(GuardWithMetadata<Self::Dir>, DirMetrics)>;
122}
123
124/// `BlobGuard` is provided by the `PuffinReader` to access the blob data.
125/// Users should hold the `BlobGuard` until they are done with the blob data.
126#[async_trait]
127#[auto_impl::auto_impl(Arc)]
128pub trait BlobGuard {
129    type Reader: RangeReader;
130    async fn reader(&self) -> Result<Self::Reader>;
131}
132
133/// `DirGuard` is provided by the `PuffinReader` to access the directory in the filesystem.
134/// Users should hold the `DirGuard` until they are done with the directory.
135#[auto_impl::auto_impl(Arc)]
136pub trait DirGuard {
137    fn path(&self) -> &PathBuf;
138}
139
140/// `GuardWithMetadata` provides access to the blob or directory data and its metadata.
141pub struct GuardWithMetadata<G> {
142    guard: G,
143    metadata: BlobMetadata,
144}
145
146impl<G> GuardWithMetadata<G> {
147    /// Creates a new `GuardWithMetadata` instance.
148    pub fn new(guard: G, metadata: BlobMetadata) -> Self {
149        Self { guard, metadata }
150    }
151
152    /// Returns the metadata of the directory.
153    pub fn metadata(&self) -> &BlobMetadata {
154        &self.metadata
155    }
156}
157
158impl<G: BlobGuard> GuardWithMetadata<G> {
159    /// Returns the reader for the blob data.
160    pub async fn reader(&self) -> Result<G::Reader> {
161        self.guard.reader().await
162    }
163}
164
165impl<G: DirGuard> GuardWithMetadata<G> {
166    /// Returns the path of the directory.
167    pub fn path(&self) -> &PathBuf {
168        self.guard.path()
169    }
170}
puffin/puffin_manager.rs

puffin/
puffin_manager.rs
