object_store/
config.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::time::Duration;
16
17use common_base::readable_size::ReadableSize;
18use common_base::secrets::{ExposeSecret, SecretString};
19use serde::{Deserialize, Serialize};
20
21/// Object storage config
22#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
23#[serde(tag = "type")]
24pub enum ObjectStoreConfig {
25    File(FileConfig),
26    S3(S3Config),
27    Oss(OssConfig),
28    Azblob(AzblobConfig),
29    Gcs(GcsConfig),
30}
31
32impl Default for ObjectStoreConfig {
33    fn default() -> Self {
34        ObjectStoreConfig::File(FileConfig {})
35    }
36}
37
38impl ObjectStoreConfig {
39    /// Returns the object storage type name, such as `S3`, `Oss` etc.
40    pub fn provider_name(&self) -> &'static str {
41        match self {
42            Self::File(_) => "File",
43            Self::S3(_) => "S3",
44            Self::Oss(_) => "Oss",
45            Self::Azblob(_) => "Azblob",
46            Self::Gcs(_) => "Gcs",
47        }
48    }
49
50    /// Returns true when it's a remote object storage such as AWS s3 etc.
51    pub fn is_object_storage(&self) -> bool {
52        !matches!(self, Self::File(_))
53    }
54
55    /// Returns the object storage configuration name, return the provider name if it's empty.
56    pub fn config_name(&self) -> &str {
57        let name = match self {
58            // file storage doesn't support name
59            Self::File(_) => self.provider_name(),
60            Self::S3(s3) => &s3.name,
61            Self::Oss(oss) => &oss.name,
62            Self::Azblob(az) => &az.name,
63            Self::Gcs(gcs) => &gcs.name,
64        };
65
66        if name.trim().is_empty() {
67            return self.provider_name();
68        }
69
70        name
71    }
72}
73
74#[derive(Debug, Clone, Serialize, Default, Deserialize, Eq, PartialEq)]
75#[serde(default)]
76pub struct FileConfig {}
77
78#[derive(Debug, Clone, Serialize, Deserialize)]
79#[serde(default)]
80pub struct S3Config {
81    pub name: String,
82    pub bucket: String,
83    pub root: String,
84    #[serde(skip_serializing)]
85    pub access_key_id: SecretString,
86    #[serde(skip_serializing)]
87    pub secret_access_key: SecretString,
88    pub endpoint: Option<String>,
89    pub region: Option<String>,
90    /// Enable virtual host style so that opendal will send API requests in virtual host style instead of path style.
91    /// By default, opendal will send API to https://s3.us-east-1.amazonaws.com/bucket_name
92    /// Enabled, opendal will send API to https://bucket_name.s3.us-east-1.amazonaws.com
93    pub enable_virtual_host_style: bool,
94    #[serde(flatten)]
95    pub cache: ObjectStorageCacheConfig,
96    pub http_client: HttpClientConfig,
97}
98
99impl Default for S3Config {
100    fn default() -> Self {
101        Self {
102            name: String::default(),
103            bucket: String::default(),
104            root: String::default(),
105            access_key_id: SecretString::from(String::default()),
106            secret_access_key: SecretString::from(String::default()),
107            enable_virtual_host_style: false,
108            endpoint: Option::default(),
109            region: Option::default(),
110            cache: ObjectStorageCacheConfig::default(),
111            http_client: HttpClientConfig::default(),
112        }
113    }
114}
115
116impl PartialEq for S3Config {
117    fn eq(&self, other: &Self) -> bool {
118        self.name == other.name
119            && self.bucket == other.bucket
120            && self.root == other.root
121            && self.access_key_id.expose_secret() == other.access_key_id.expose_secret()
122            && self.secret_access_key.expose_secret() == other.secret_access_key.expose_secret()
123            && self.endpoint == other.endpoint
124            && self.region == other.region
125            && self.enable_virtual_host_style == other.enable_virtual_host_style
126            && self.cache == other.cache
127            && self.http_client == other.http_client
128    }
129}
130
131#[derive(Debug, Clone, Serialize, Deserialize)]
132#[serde(default)]
133pub struct OssConfig {
134    pub name: String,
135    pub bucket: String,
136    pub root: String,
137    #[serde(skip_serializing)]
138    pub access_key_id: SecretString,
139    #[serde(skip_serializing)]
140    pub access_key_secret: SecretString,
141    pub endpoint: String,
142    #[serde(flatten)]
143    pub cache: ObjectStorageCacheConfig,
144    pub http_client: HttpClientConfig,
145}
146
147impl PartialEq for OssConfig {
148    fn eq(&self, other: &Self) -> bool {
149        self.name == other.name
150            && self.bucket == other.bucket
151            && self.root == other.root
152            && self.access_key_id.expose_secret() == other.access_key_id.expose_secret()
153            && self.access_key_secret.expose_secret() == other.access_key_secret.expose_secret()
154            && self.endpoint == other.endpoint
155            && self.cache == other.cache
156            && self.http_client == other.http_client
157    }
158}
159
160impl Default for OssConfig {
161    fn default() -> Self {
162        Self {
163            name: String::default(),
164            bucket: String::default(),
165            root: String::default(),
166            access_key_id: SecretString::from(String::default()),
167            access_key_secret: SecretString::from(String::default()),
168            endpoint: String::default(),
169            cache: ObjectStorageCacheConfig::default(),
170            http_client: HttpClientConfig::default(),
171        }
172    }
173}
174
175#[derive(Debug, Clone, Serialize, Deserialize)]
176#[serde(default)]
177pub struct AzblobConfig {
178    pub name: String,
179    pub container: String,
180    pub root: String,
181    #[serde(skip_serializing)]
182    pub account_name: SecretString,
183    #[serde(skip_serializing)]
184    pub account_key: SecretString,
185    pub endpoint: String,
186    pub sas_token: Option<String>,
187    #[serde(flatten)]
188    pub cache: ObjectStorageCacheConfig,
189    pub http_client: HttpClientConfig,
190}
191
192impl PartialEq for AzblobConfig {
193    fn eq(&self, other: &Self) -> bool {
194        self.name == other.name
195            && self.container == other.container
196            && self.root == other.root
197            && self.account_name.expose_secret() == other.account_name.expose_secret()
198            && self.account_key.expose_secret() == other.account_key.expose_secret()
199            && self.endpoint == other.endpoint
200            && self.sas_token == other.sas_token
201            && self.cache == other.cache
202            && self.http_client == other.http_client
203    }
204}
205impl Default for AzblobConfig {
206    fn default() -> Self {
207        Self {
208            name: String::default(),
209            container: String::default(),
210            root: String::default(),
211            account_name: SecretString::from(String::default()),
212            account_key: SecretString::from(String::default()),
213            endpoint: String::default(),
214            sas_token: Option::default(),
215            cache: ObjectStorageCacheConfig::default(),
216            http_client: HttpClientConfig::default(),
217        }
218    }
219}
220
221#[derive(Debug, Clone, Serialize, Deserialize)]
222#[serde(default)]
223pub struct GcsConfig {
224    pub name: String,
225    pub root: String,
226    pub bucket: String,
227    pub scope: String,
228    #[serde(skip_serializing)]
229    pub credential_path: SecretString,
230    #[serde(skip_serializing)]
231    pub credential: SecretString,
232    pub endpoint: String,
233    #[serde(flatten)]
234    pub cache: ObjectStorageCacheConfig,
235    pub http_client: HttpClientConfig,
236}
237
238impl Default for GcsConfig {
239    fn default() -> Self {
240        Self {
241            name: String::default(),
242            root: String::default(),
243            bucket: String::default(),
244            scope: String::default(),
245            credential_path: SecretString::from(String::default()),
246            credential: SecretString::from(String::default()),
247            endpoint: String::default(),
248            cache: ObjectStorageCacheConfig::default(),
249            http_client: HttpClientConfig::default(),
250        }
251    }
252}
253
254impl PartialEq for GcsConfig {
255    fn eq(&self, other: &Self) -> bool {
256        self.name == other.name
257            && self.root == other.root
258            && self.bucket == other.bucket
259            && self.scope == other.scope
260            && self.credential_path.expose_secret() == other.credential_path.expose_secret()
261            && self.credential.expose_secret() == other.credential.expose_secret()
262            && self.endpoint == other.endpoint
263            && self.cache == other.cache
264            && self.http_client == other.http_client
265    }
266}
267
268/// The http client options to the storage.
269#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
270#[serde(default)]
271pub struct HttpClientConfig {
272    /// The maximum idle connection per host allowed in the pool.
273    pub(crate) pool_max_idle_per_host: u32,
274
275    /// The timeout for only the connect phase of a http client.
276    #[serde(with = "humantime_serde")]
277    pub(crate) connect_timeout: Duration,
278
279    /// The total request timeout, applied from when the request starts connecting until the response body has finished.
280    /// Also considered a total deadline.
281    #[serde(with = "humantime_serde")]
282    pub(crate) timeout: Duration,
283
284    /// The timeout for idle sockets being kept-alive.
285    #[serde(with = "humantime_serde")]
286    pub(crate) pool_idle_timeout: Duration,
287
288    /// Skip SSL certificate validation (insecure)
289    pub skip_ssl_validation: bool,
290}
291
292impl Default for HttpClientConfig {
293    fn default() -> Self {
294        Self {
295            pool_max_idle_per_host: 1024,
296            connect_timeout: Duration::from_secs(30),
297            timeout: Duration::from_secs(30),
298            pool_idle_timeout: Duration::from_secs(90),
299            skip_ssl_validation: false,
300        }
301    }
302}
303
304#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
305#[serde(default)]
306pub struct ObjectStorageCacheConfig {
307    /// The local file cache directory
308    pub cache_path: Option<String>,
309    /// The cache capacity in bytes
310    pub cache_capacity: Option<ReadableSize>,
311}
312
313#[cfg(test)]
314mod tests {
315    use super::*;
316    use crate::config::ObjectStoreConfig;
317
318    #[test]
319    fn test_config_name() {
320        let object_store_config = ObjectStoreConfig::default();
321        assert_eq!("File", object_store_config.config_name());
322
323        let s3_config = ObjectStoreConfig::S3(S3Config::default());
324        assert_eq!("S3", s3_config.config_name());
325        assert_eq!("S3", s3_config.provider_name());
326
327        let s3_config = ObjectStoreConfig::S3(S3Config {
328            name: "test".to_string(),
329            ..Default::default()
330        });
331        assert_eq!("test", s3_config.config_name());
332        assert_eq!("S3", s3_config.provider_name());
333    }
334
335    #[test]
336    fn test_is_object_storage() {
337        let store = ObjectStoreConfig::default();
338        assert!(!store.is_object_storage());
339        let s3_config = ObjectStoreConfig::S3(S3Config::default());
340        assert!(s3_config.is_object_storage());
341        let oss_config = ObjectStoreConfig::Oss(OssConfig::default());
342        assert!(oss_config.is_object_storage());
343        let gcs_config = ObjectStoreConfig::Gcs(GcsConfig::default());
344        assert!(gcs_config.is_object_storage());
345        let azblob_config = ObjectStoreConfig::Azblob(AzblobConfig::default());
346        assert!(azblob_config.is_object_storage());
347    }
348}
object_store/config.rs

object_store/
config.rs
