Skip to main content

store_api/storage/
file.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{HashMap, HashSet};
16use std::fmt;
17use std::fmt::Debug;
18use std::str::FromStr;
19
20use serde::{Deserialize, Serialize};
21use snafu::{ResultExt, Snafu};
22use uuid::Uuid;
23
24use crate::ManifestVersion;
25use crate::storage::RegionId;
26
27/// Index version
28pub type IndexVersion = u64;
29
30#[derive(Debug, Snafu, PartialEq)]
31pub struct ParseIdError {
32    source: uuid::Error,
33}
34
35/// Unique id for [SST File].
36#[derive(
37    Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default,
38)]
39pub struct FileId(Uuid);
40
41impl FileId {
42    /// Returns a new unique [FileId] randomly.
43    pub fn random() -> FileId {
44        FileId(Uuid::new_v4())
45    }
46
47    /// Parses id from string.
48    pub fn parse_str(input: &str) -> std::result::Result<FileId, ParseIdError> {
49        Uuid::parse_str(input).map(FileId).context(ParseIdSnafu)
50    }
51
52    /// Converts [FileId] as byte slice.
53    pub fn as_bytes(&self) -> &[u8] {
54        self.0.as_bytes()
55    }
56
57    /// Constructs a file id from its packed 16-byte UUID representation.
58    pub fn from_bytes(bytes: [u8; 16]) -> FileId {
59        FileId(Uuid::from_bytes(bytes))
60    }
61}
62
63impl From<FileId> for Uuid {
64    fn from(value: FileId) -> Self {
65        value.0
66    }
67}
68
69impl fmt::Display for FileId {
70    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
71        write!(f, "{}", self.0)
72    }
73}
74
75impl FromStr for FileId {
76    type Err = ParseIdError;
77
78    fn from_str(s: &str) -> std::result::Result<FileId, ParseIdError> {
79        FileId::parse_str(s)
80    }
81}
82
83#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
84pub struct FileRef {
85    pub region_id: RegionId,
86    pub file_id: FileId,
87    pub index_version: Option<IndexVersion>,
88}
89
90impl FileRef {
91    pub fn new(region_id: RegionId, file_id: FileId, index_version: Option<IndexVersion>) -> Self {
92        Self {
93            region_id,
94            file_id,
95            index_version,
96        }
97    }
98}
99
100/// The tmp file manifest which record a table's file references.
101/// Also record the manifest version when these tmp files are read.
102#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
103pub struct FileRefsManifest {
104    pub file_refs: HashMap<RegionId, HashSet<FileRef>>,
105    /// Manifest version when this manifest is read for its files
106    pub manifest_version: HashMap<RegionId, ManifestVersion>,
107    /// Cross-region file ownership mapping.
108    ///
109    /// Key is the source/original region id (before repartition); value is the set of
110    /// target/destination region ids (after repartition) that currently hold files
111    /// originally coming from that source region.
112    ///
113    pub cross_region_refs: HashMap<RegionId, HashSet<RegionId>>,
114}
115
116#[derive(Clone, Default, Debug, PartialEq, Eq, Serialize, Deserialize)]
117pub struct GcReport {
118    /// Deleted SST/parquet file ids per region. Index-only deletions are reported via
119    /// `deleted_indexes` because a naked `FileId` cannot distinguish index versions.
120    /// TODO(discord9): change to `RemovedFile`?
121    pub deleted_files: HashMap<RegionId, Vec<FileId>>,
122    pub deleted_indexes: HashMap<RegionId, Vec<(FileId, IndexVersion)>>,
123    /// Regions that need retry in next gc round, usually because their tmp ref files are outdated
124    pub need_retry_regions: HashSet<RegionId>,
125    /// Regions successfully processed in this GC run
126    pub processed_regions: HashSet<RegionId>,
127}
128
129impl GcReport {
130    pub fn new(
131        deleted_files: HashMap<RegionId, Vec<FileId>>,
132        deleted_indexes: HashMap<RegionId, Vec<(FileId, IndexVersion)>>,
133        need_retry_regions: HashSet<RegionId>,
134    ) -> Self {
135        Self {
136            deleted_files,
137            deleted_indexes,
138            need_retry_regions,
139            processed_regions: HashSet::new(),
140        }
141    }
142
143    pub fn merge(&mut self, other: GcReport) {
144        for (region, files) in other.deleted_files {
145            let self_files = self.deleted_files.entry(region).or_default();
146            let dedup: HashSet<FileId> = HashSet::from_iter(
147                std::mem::take(self_files)
148                    .into_iter()
149                    .chain(files.iter().cloned()),
150            );
151            *self_files = dedup.into_iter().collect();
152        }
153        for (region, files) in other.deleted_indexes {
154            let self_files = self.deleted_indexes.entry(region).or_default();
155            let dedup: HashSet<(FileId, IndexVersion)> = HashSet::from_iter(
156                std::mem::take(self_files)
157                    .into_iter()
158                    .chain(files.iter().cloned()),
159            );
160            *self_files = dedup.into_iter().collect();
161        }
162        self.need_retry_regions.extend(other.need_retry_regions);
163        self.processed_regions.extend(other.processed_regions);
164        // Remove regions that have succeeded from need_retry_regions
165        self.need_retry_regions
166            .retain(|region| !self.deleted_files.contains_key(region));
167    }
168}
169
170#[cfg(test)]
171mod tests {
172
173    use super::*;
174
175    #[test]
176    fn test_file_id() {
177        let id = FileId::random();
178        let uuid_str = id.to_string();
179        assert_eq!(id.0.to_string(), uuid_str);
180
181        let parsed = FileId::parse_str(&uuid_str).unwrap();
182        assert_eq!(id, parsed);
183        let parsed = uuid_str.parse().unwrap();
184        assert_eq!(id, parsed);
185    }
186
187    #[test]
188    fn test_file_id_serialization() {
189        let id = FileId::random();
190        let json = serde_json::to_string(&id).unwrap();
191        assert_eq!(format!("\"{id}\""), json);
192
193        let parsed = serde_json::from_str(&json).unwrap();
194        assert_eq!(id, parsed);
195    }
196
197    #[test]
198    fn test_file_refs_manifest_serialization() {
199        let mut manifest = FileRefsManifest::default();
200        let r0 = RegionId::new(1024, 1);
201        let r1 = RegionId::new(1024, 2);
202        manifest
203            .file_refs
204            .insert(r0, [FileRef::new(r0, FileId::random(), None)].into());
205        manifest
206            .file_refs
207            .insert(r1, [FileRef::new(r1, FileId::random(), None)].into());
208        manifest.manifest_version.insert(r0, 10);
209        manifest.manifest_version.insert(r1, 20);
210        manifest.cross_region_refs.insert(r0, [r1].into());
211        manifest.cross_region_refs.insert(r1, [r0].into());
212
213        let json = serde_json::to_string(&manifest).unwrap();
214        let parsed: FileRefsManifest = serde_json::from_str(&json).unwrap();
215        assert_eq!(manifest, parsed);
216    }
217
218    #[test]
219    fn test_file_ref_new() {
220        let region_id = RegionId::new(1024, 1);
221        let file_id = FileId::random();
222
223        // Test with Some(index_version)
224        let index_version: IndexVersion = 42;
225        let file_ref = FileRef::new(region_id, file_id, Some(index_version));
226        assert_eq!(file_ref.region_id, region_id);
227        assert_eq!(file_ref.file_id, file_id);
228        assert_eq!(file_ref.index_version, Some(index_version));
229
230        // Test with None
231        let file_ref_none = FileRef::new(region_id, file_id, None);
232        assert_eq!(file_ref_none.region_id, region_id);
233        assert_eq!(file_ref_none.file_id, file_id);
234        assert_eq!(file_ref_none.index_version, None);
235    }
236
237    #[test]
238    fn test_file_ref_equality() {
239        let region_id = RegionId::new(1024, 1);
240        let file_id = FileId::random();
241
242        let file_ref1 = FileRef::new(region_id, file_id, Some(10));
243        let file_ref2 = FileRef::new(region_id, file_id, Some(10));
244        let file_ref3 = FileRef::new(region_id, file_id, Some(20));
245        let file_ref4 = FileRef::new(region_id, file_id, None);
246
247        assert_eq!(file_ref1, file_ref2);
248        assert_ne!(file_ref1, file_ref3);
249        assert_ne!(file_ref1, file_ref4);
250        assert_ne!(file_ref3, file_ref4);
251
252        // Test equality with Some(0) vs None
253        let file_ref_zero = FileRef::new(region_id, file_id, Some(0));
254        assert_ne!(file_ref_zero, file_ref4);
255    }
256
257    #[test]
258    fn test_file_ref_serialization() {
259        let region_id = RegionId::new(1024, 1);
260        let file_id = FileId::random();
261
262        // Test with Some(index_version)
263        let index_version: IndexVersion = 12345;
264        let file_ref = FileRef::new(region_id, file_id, Some(index_version));
265
266        let json = serde_json::to_string(&file_ref).unwrap();
267        let parsed: FileRef = serde_json::from_str(&json).unwrap();
268
269        assert_eq!(file_ref, parsed);
270        assert_eq!(parsed.index_version, Some(index_version));
271
272        // Test with None
273        let file_ref_none = FileRef::new(region_id, file_id, None);
274        let json_none = serde_json::to_string(&file_ref_none).unwrap();
275        let parsed_none: FileRef = serde_json::from_str(&json_none).unwrap();
276
277        assert_eq!(file_ref_none, parsed_none);
278        assert_eq!(parsed_none.index_version, None);
279    }
280}