mito2/sst/
location.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use object_store::util;
16use snafu::OptionExt as _;
17use store_api::metric_engine_consts::{DATA_REGION_SUBDIR, METADATA_REGION_SUBDIR};
18use store_api::path_utils::region_name;
19use store_api::region_request::PathType;
20use store_api::storage::{FileId, RegionId};
21
22use crate::cache::file_cache::FileType;
23use crate::error::UnexpectedSnafu;
24use crate::sst::file::{RegionFileId, RegionIndexId};
25
26/// Generate region dir from table_dir, region_id and path_type
27pub fn region_dir_from_table_dir(
28    table_dir: &str,
29    region_id: RegionId,
30    path_type: PathType,
31) -> String {
32    let region_name = region_name(region_id.table_id(), region_id.region_sequence());
33    let base_region_dir = util::join_dir(table_dir, &region_name);
34
35    match path_type {
36        PathType::Bare => base_region_dir,
37        PathType::Data => util::join_dir(&base_region_dir, DATA_REGION_SUBDIR),
38        PathType::Metadata => util::join_dir(&base_region_dir, METADATA_REGION_SUBDIR),
39    }
40}
41
42/// Generate index dir from region_dir
43pub fn index_dir(region_dir: &str) -> String {
44    util::join_dir(region_dir, "index")
45}
46
47pub fn sst_file_path(table_dir: &str, region_file_id: RegionFileId, path_type: PathType) -> String {
48    let region_dir = region_dir_from_table_dir(table_dir, region_file_id.region_id(), path_type);
49    util::join_path(
50        &region_dir,
51        &format!("{}.parquet", region_file_id.file_id()),
52    )
53}
54
55pub fn index_file_path(table_dir: &str, index_id: RegionIndexId, path_type: PathType) -> String {
56    let region_dir = region_dir_from_table_dir(table_dir, index_id.file_id.region_id(), path_type);
57    let index_dir = index_dir(&region_dir);
58
59    let filename = if index_id.version == 0 {
60        format!("{}.puffin", index_id.file_id.file_id())
61    } else {
62        format!("{}.{}.puffin", index_id.file_id.file_id(), index_id.version)
63    };
64
65    util::join_path(&index_dir, &filename)
66}
67
68/// Legacy function for backward compatibility - creates index file path using RegionFileId with version 0
69pub fn index_file_path_legacy(
70    table_dir: &str,
71    region_file_id: RegionFileId,
72    path_type: PathType,
73) -> String {
74    let index_id = RegionIndexId::new(region_file_id, 0);
75    index_file_path(table_dir, index_id, path_type)
76}
77
78/// Parse file ID and version from index filename
79pub fn parse_index_file_info(filepath: &str) -> crate::error::Result<(FileId, u64)> {
80    let filename = filepath.rsplit('/').next().context(UnexpectedSnafu {
81        reason: format!("invalid file path: {}", filepath),
82    })?;
83    let parts: Vec<&str> = filename.split('.').collect();
84
85    if parts.len() == 2 && parts[1] == "puffin" {
86        // Legacy format: {file_id}.puffin (version 0)
87        let file_id = parts[0];
88        FileId::parse_str(file_id).map(|id| (id, 0)).map_err(|e| {
89            UnexpectedSnafu {
90                reason: format!("invalid file id: {}, err: {}", file_id, e),
91            }
92            .build()
93        })
94    } else if parts.len() == 3 && parts[2] == "puffin" {
95        // New format: {file_id}.{version}.puffin
96        let file_id = parts[0];
97        let version = parts[1].parse::<u64>().map_err(|_| {
98            UnexpectedSnafu {
99                reason: format!("invalid version in file name: {}", filename),
100            }
101            .build()
102        })?;
103        FileId::parse_str(file_id)
104            .map(|id| (id, version))
105            .map_err(|e| {
106                UnexpectedSnafu {
107                    reason: format!("invalid file id: {}, err: {}", file_id, e),
108                }
109                .build()
110            })
111    } else {
112        UnexpectedSnafu {
113            reason: format!("invalid index file name: {}", filename),
114        }
115        .fail()
116    }
117}
118
119pub fn parse_file_id_type_from_path(filepath: &str) -> crate::error::Result<(FileId, FileType)> {
120    let filename = filepath.rsplit('/').next().context(UnexpectedSnafu {
121        reason: format!("invalid file path: {}", filepath),
122    })?;
123    // get part before first '.'
124    let parts: Vec<&str> = filename.split('.').collect();
125    if parts.len() < 2 {
126        return UnexpectedSnafu {
127            reason: format!("invalid file name: {}", filename),
128        }
129        .fail();
130    }
131    let file_id = parts[0];
132    let file_id = FileId::parse_str(file_id).map_err(|e| {
133        UnexpectedSnafu {
134            reason: format!("invalid file id: {}, err: {}", file_id, e),
135        }
136        .build()
137    })?;
138    let file_type = FileType::parse(parts[1..].join(".").as_str()).context(UnexpectedSnafu {
139        reason: format!("invalid file type in file name: {}", filename),
140    })?;
141    Ok((file_id, file_type))
142}
143
144#[cfg(test)]
145mod tests {
146    use store_api::storage::{FileId, RegionId};
147
148    use super::*;
149
150    #[test]
151    fn test_sst_file_path() {
152        let file_id = FileId::random();
153        let region_file_id = RegionFileId::new(RegionId::new(1, 2), file_id);
154        assert_eq!(
155            sst_file_path("table_dir", region_file_id, PathType::Bare),
156            format!("table_dir/1_0000000002/{}.parquet", file_id)
157        );
158        assert_eq!(
159            sst_file_path("table_dir", region_file_id, PathType::Data),
160            format!("table_dir/1_0000000002/data/{}.parquet", file_id)
161        );
162        assert_eq!(
163            sst_file_path("table_dir", region_file_id, PathType::Metadata),
164            format!("table_dir/1_0000000002/metadata/{}.parquet", file_id)
165        );
166    }
167
168    #[test]
169    fn test_index_file_path() {
170        let file_id = FileId::random();
171        let region_file_id = RegionFileId::new(RegionId::new(1, 2), file_id);
172        let index_id = RegionIndexId::new(region_file_id, 0);
173        assert_eq!(
174            index_file_path("table_dir", index_id, PathType::Bare),
175            format!("table_dir/1_0000000002/index/{}.puffin", file_id)
176        );
177        assert_eq!(
178            index_file_path("table_dir", index_id, PathType::Data),
179            format!("table_dir/1_0000000002/data/index/{}.puffin", file_id)
180        );
181        assert_eq!(
182            index_file_path("table_dir", index_id, PathType::Metadata),
183            format!("table_dir/1_0000000002/metadata/index/{}.puffin", file_id)
184        );
185    }
186
187    #[test]
188    fn test_index_file_path_versioned() {
189        let file_id = FileId::random();
190        let region_file_id = RegionFileId::new(RegionId::new(1, 2), file_id);
191        let index_id_v1 = RegionIndexId::new(region_file_id, 1);
192        let index_id_v2 = RegionIndexId::new(region_file_id, 2);
193
194        assert_eq!(
195            index_file_path("table_dir", index_id_v1, PathType::Bare),
196            format!("table_dir/1_0000000002/index/{}.1.puffin", file_id)
197        );
198        assert_eq!(
199            index_file_path("table_dir", index_id_v2, PathType::Bare),
200            format!("table_dir/1_0000000002/index/{}.2.puffin", file_id)
201        );
202    }
203
204    #[test]
205    fn test_parse_index_file_info() {
206        // Test legacy format
207        let file_id = FileId::random();
208        let result =
209            parse_index_file_info(&format!("table_dir/1_0000000002/index/{file_id}.puffin"))
210                .unwrap();
211        assert_eq!(result.0.to_string(), file_id.to_string());
212        assert_eq!(result.1, 0);
213
214        // Test versioned format
215        let result =
216            parse_index_file_info(&format!("table_dir/1_0000000002/index/{file_id}.1.puffin"))
217                .unwrap();
218        assert_eq!(result.0.to_string(), file_id.to_string());
219        assert_eq!(result.1, 1);
220
221        let result =
222            parse_index_file_info(&format!("table_dir/1_0000000002/index/{file_id}.42.puffin"))
223                .unwrap();
224        assert_eq!(result.0.to_string(), file_id.to_string());
225        assert_eq!(result.1, 42);
226    }
227
228    #[test]
229    fn test_parse_file_id_type_from_path() {
230        use crate::cache::file_cache::FileType;
231
232        // Test parquet file
233        let file_id = FileId::random();
234        let path = format!("table_dir/1_0000000002/data/{}.parquet", file_id);
235        let result = parse_file_id_type_from_path(&path).unwrap();
236        assert_eq!(result.0.to_string(), file_id.to_string());
237        assert_eq!(result.1, FileType::Parquet);
238
239        // Test puffin file (legacy format, version 0)
240        let file_id = FileId::random();
241        let path = format!("table_dir/1_0000000002/index/{}.puffin", file_id);
242        let result = parse_file_id_type_from_path(&path).unwrap();
243        assert_eq!(result.0.to_string(), file_id.to_string());
244        assert_eq!(result.1, FileType::Puffin(0));
245
246        // Test versioned puffin file
247        let file_id = FileId::random();
248        let path = format!("table_dir/1_0000000002/index/{}.1.puffin", file_id);
249        let result = parse_file_id_type_from_path(&path).unwrap();
250        assert_eq!(result.0.to_string(), file_id.to_string());
251        assert_eq!(result.1, FileType::Puffin(1));
252
253        // Test with different path types
254        let file_id = FileId::random();
255        let path = format!("table_dir/1_0000000002/metadata/{}.parquet", file_id);
256        let result = parse_file_id_type_from_path(&path).unwrap();
257        assert_eq!(result.0.to_string(), file_id.to_string());
258        assert_eq!(result.1, FileType::Parquet);
259
260        // Test with bare path type
261        let file_id = FileId::random();
262        let path = format!("table_dir/1_0000000002/{}.parquet", file_id);
263        let result = parse_file_id_type_from_path(&path).unwrap();
264        assert_eq!(result.0.to_string(), file_id.to_string());
265        assert_eq!(result.1, FileType::Parquet);
266
267        // Test error cases
268        // Invalid file extension
269        let result = parse_file_id_type_from_path("table_dir/1_0000000002/data/test.invalid");
270        assert!(result.is_err());
271
272        // Invalid file ID
273        let result =
274            parse_file_id_type_from_path("table_dir/1_0000000002/data/invalid-file-id.parquet");
275        assert!(result.is_err());
276
277        // No file extension
278        let result = parse_file_id_type_from_path("table_dir/1_0000000002/data/test");
279        assert!(result.is_err());
280
281        // Empty filename
282        let result = parse_file_id_type_from_path("table_dir/1_0000000002/data/");
283        assert!(result.is_err());
284    }
285}
mito2/sst/location.rs

mito2/sst/
location.rs
