mito2/manifest/storage/
utils.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use object_store::{Entry, ObjectStore};
16use snafu::ResultExt;
17use store_api::ManifestVersion;
18
19use crate::cache::manifest_cache::ManifestCache;
20use crate::error::{OpenDalSnafu, Result};
21
22const CHUNK_SIZE: usize = 64 * 1024 * 1024; // 64MB
23
24/// Gets a manifest file from cache.
25/// Returns the file data if found in cache, None otherwise.
26pub(crate) async fn get_from_cache(cache: Option<&ManifestCache>, key: &str) -> Option<Vec<u8>> {
27    let cache = cache?;
28    cache.get_file(key).await
29}
30
31/// Puts a manifest file into cache.
32pub(crate) async fn put_to_cache(cache: Option<&ManifestCache>, key: String, data: &[u8]) {
33    let Some(cache) = cache else {
34        return;
35    };
36    cache.put_file(key, data.to_vec()).await
37}
38
39/// Removes a manifest file from cache.
40pub(crate) async fn remove_from_cache(cache: Option<&ManifestCache>, key: &str) {
41    let Some(cache) = cache else {
42        return;
43    };
44    cache.remove(key).await
45}
46
47/// Writes data to object store and puts it into cache.
48pub(crate) async fn write_and_put_cache(
49    object_store: &ObjectStore,
50    cache: Option<&ManifestCache>,
51    path: &str,
52    data: Vec<u8>,
53) -> Result<()> {
54    // Clone data for cache before writing, only if cache is enabled.
55    let cache_data = if cache.is_some() {
56        Some(data.clone())
57    } else {
58        None
59    };
60
61    // Write to object store
62    object_store
63        .write_with(path, data)
64        .chunk(CHUNK_SIZE) // 64MB
65        .await
66        .context(OpenDalSnafu)?;
67
68    // Put to cache if we cloned the data
69    if let Some(data) = cache_data {
70        put_to_cache(cache, path.to_string(), &data).await;
71    }
72
73    Ok(())
74}
75
76/// Sorts the manifest files.
77pub(crate) fn sort_manifests(entries: &mut [(ManifestVersion, Entry)]) {
78    entries.sort_unstable_by_key(|(version, _)| *version);
79}
mito2/manifest/storage/utils.rs

mito2/manifest/storage/
utils.rs
