mito2/sst/
location.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use object_store::util;
16use snafu::OptionExt as _;
17use store_api::metric_engine_consts::{DATA_REGION_SUBDIR, METADATA_REGION_SUBDIR};
18use store_api::path_utils::region_name;
19use store_api::region_request::PathType;
20use store_api::storage::{FileId, RegionId};
21
22use crate::error::UnexpectedSnafu;
23use crate::sst::file::{RegionFileId, RegionIndexId};
24
25/// Generate region dir from table_dir, region_id and path_type
26pub fn region_dir_from_table_dir(
27    table_dir: &str,
28    region_id: RegionId,
29    path_type: PathType,
30) -> String {
31    let region_name = region_name(region_id.table_id(), region_id.region_sequence());
32    let base_region_dir = util::join_dir(table_dir, &region_name);
33
34    match path_type {
35        PathType::Bare => base_region_dir,
36        PathType::Data => util::join_dir(&base_region_dir, DATA_REGION_SUBDIR),
37        PathType::Metadata => util::join_dir(&base_region_dir, METADATA_REGION_SUBDIR),
38    }
39}
40
41pub fn sst_file_path(table_dir: &str, region_file_id: RegionFileId, path_type: PathType) -> String {
42    let region_dir = region_dir_from_table_dir(table_dir, region_file_id.region_id(), path_type);
43    util::join_path(
44        &region_dir,
45        &format!("{}.parquet", region_file_id.file_id()),
46    )
47}
48
49pub fn index_file_path(table_dir: &str, index_id: RegionIndexId, path_type: PathType) -> String {
50    let region_dir = region_dir_from_table_dir(table_dir, index_id.file_id.region_id(), path_type);
51    let index_dir = util::join_dir(&region_dir, "index");
52
53    let filename = if index_id.version == 0 {
54        format!("{}.puffin", index_id.file_id.file_id())
55    } else {
56        format!("{}.{}.puffin", index_id.file_id.file_id(), index_id.version)
57    };
58
59    util::join_path(&index_dir, &filename)
60}
61
62/// Legacy function for backward compatibility - creates index file path using RegionFileId with version 0
63pub fn index_file_path_legacy(
64    table_dir: &str,
65    region_file_id: RegionFileId,
66    path_type: PathType,
67) -> String {
68    let index_id = RegionIndexId::new(region_file_id, 0);
69    index_file_path(table_dir, index_id, path_type)
70}
71
72/// Parse file ID and version from index filename
73pub fn parse_index_file_info(filepath: &str) -> crate::error::Result<(FileId, u64)> {
74    let filename = filepath.rsplit('/').next().context(UnexpectedSnafu {
75        reason: format!("invalid file path: {}", filepath),
76    })?;
77    let parts: Vec<&str> = filename.split('.').collect();
78
79    if parts.len() == 2 && parts[1] == "puffin" {
80        // Legacy format: {file_id}.puffin (version 0)
81        let file_id = parts[0];
82        FileId::parse_str(file_id).map(|id| (id, 0)).map_err(|e| {
83            UnexpectedSnafu {
84                reason: format!("invalid file id: {}, err: {}", file_id, e),
85            }
86            .build()
87        })
88    } else if parts.len() == 3 && parts[2] == "puffin" {
89        // New format: {file_id}.{version}.puffin
90        let file_id = parts[0];
91        let version = parts[1].parse::<u64>().map_err(|_| {
92            UnexpectedSnafu {
93                reason: format!("invalid version in file name: {}", filename),
94            }
95            .build()
96        })?;
97        FileId::parse_str(file_id)
98            .map(|id| (id, version))
99            .map_err(|e| {
100                UnexpectedSnafu {
101                    reason: format!("invalid file id: {}, err: {}", file_id, e),
102                }
103                .build()
104            })
105    } else {
106        UnexpectedSnafu {
107            reason: format!("invalid index file name: {}", filename),
108        }
109        .fail()
110    }
111}
112
113/// Get RegionFileId from sst or index filename
114pub fn parse_file_id_from_path(filepath: &str) -> crate::error::Result<FileId> {
115    let filename = filepath.rsplit('/').next().context(UnexpectedSnafu {
116        reason: format!("invalid file path: {}", filepath),
117    })?;
118    let parts: Vec<&str> = filename.split('.').collect();
119    if parts.len() != 2 {
120        return UnexpectedSnafu {
121            reason: format!("invalid file name: {}", filename),
122        }
123        .fail();
124    }
125    if parts[1] != "parquet" && parts[1] != "puffin" {
126        return UnexpectedSnafu {
127            reason: format!("invalid file extension: {}", parts[1]),
128        }
129        .fail();
130    }
131    let file_id = parts[0];
132    FileId::parse_str(file_id).map_err(|e| {
133        UnexpectedSnafu {
134            reason: format!("invalid file id: {}, err: {}", file_id, e),
135        }
136        .build()
137    })
138}
139
140#[cfg(test)]
141mod tests {
142    use store_api::storage::{FileId, RegionId};
143
144    use super::*;
145
146    #[test]
147    fn test_sst_file_path() {
148        let file_id = FileId::random();
149        let region_file_id = RegionFileId::new(RegionId::new(1, 2), file_id);
150        assert_eq!(
151            sst_file_path("table_dir", region_file_id, PathType::Bare),
152            format!("table_dir/1_0000000002/{}.parquet", file_id)
153        );
154        assert_eq!(
155            sst_file_path("table_dir", region_file_id, PathType::Data),
156            format!("table_dir/1_0000000002/data/{}.parquet", file_id)
157        );
158        assert_eq!(
159            sst_file_path("table_dir", region_file_id, PathType::Metadata),
160            format!("table_dir/1_0000000002/metadata/{}.parquet", file_id)
161        );
162    }
163
164    #[test]
165    fn test_index_file_path() {
166        let file_id = FileId::random();
167        let region_file_id = RegionFileId::new(RegionId::new(1, 2), file_id);
168        let index_id = RegionIndexId::new(region_file_id, 0);
169        assert_eq!(
170            index_file_path("table_dir", index_id, PathType::Bare),
171            format!("table_dir/1_0000000002/index/{}.puffin", file_id)
172        );
173        assert_eq!(
174            index_file_path("table_dir", index_id, PathType::Data),
175            format!("table_dir/1_0000000002/data/index/{}.puffin", file_id)
176        );
177        assert_eq!(
178            index_file_path("table_dir", index_id, PathType::Metadata),
179            format!("table_dir/1_0000000002/metadata/index/{}.puffin", file_id)
180        );
181    }
182
183    #[test]
184    fn test_index_file_path_versioned() {
185        let file_id = FileId::random();
186        let region_file_id = RegionFileId::new(RegionId::new(1, 2), file_id);
187        let index_id_v1 = RegionIndexId::new(region_file_id, 1);
188        let index_id_v2 = RegionIndexId::new(region_file_id, 2);
189
190        assert_eq!(
191            index_file_path("table_dir", index_id_v1, PathType::Bare),
192            format!("table_dir/1_0000000002/index/{}.1.puffin", file_id)
193        );
194        assert_eq!(
195            index_file_path("table_dir", index_id_v2, PathType::Bare),
196            format!("table_dir/1_0000000002/index/{}.2.puffin", file_id)
197        );
198    }
199
200    #[test]
201    fn test_parse_index_file_info() {
202        // Test legacy format
203        let file_id = FileId::random();
204        let result =
205            parse_index_file_info(&format!("table_dir/1_0000000002/index/{file_id}.puffin"))
206                .unwrap();
207        assert_eq!(result.0.to_string(), file_id.to_string());
208        assert_eq!(result.1, 0);
209
210        // Test versioned format
211        let result =
212            parse_index_file_info(&format!("table_dir/1_0000000002/index/{file_id}.1.puffin"))
213                .unwrap();
214        assert_eq!(result.0.to_string(), file_id.to_string());
215        assert_eq!(result.1, 1);
216
217        let result =
218            parse_index_file_info(&format!("table_dir/1_0000000002/index/{file_id}.42.puffin"))
219                .unwrap();
220        assert_eq!(result.0.to_string(), file_id.to_string());
221        assert_eq!(result.1, 42);
222    }
223}
mito2/sst/location.rs

mito2/sst/
location.rs
