Skip to main content

common_datasource/
compression.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::fmt::Display;
16use std::io;
17use std::str::FromStr;
18
19use async_compression::Level;
20use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder};
21use async_compression::tokio::write;
22use bytes::Bytes;
23use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
24use futures::Stream;
25use serde::{Deserialize, Serialize};
26use strum::EnumIter;
27use tokio::io::{AsyncRead, AsyncWriteExt, BufReader};
28use tokio_util::io::{ReaderStream, StreamReader};
29
30use crate::error::{self, Error, Result};
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter, Serialize, Deserialize)]
33#[serde(rename_all = "lowercase")]
34pub enum CompressionType {
35    /// Gzip-ed file
36    Gzip,
37    /// Bzip2-ed file
38    Bzip2,
39    /// Xz-ed file (liblzma)
40    Xz,
41    /// Zstd-ed file,
42    Zstd,
43    /// Uncompressed file
44    Uncompressed,
45}
46
47impl FromStr for CompressionType {
48    type Err = Error;
49
50    fn from_str(s: &str) -> Result<Self> {
51        let s = s.to_uppercase();
52        match s.as_str() {
53            "GZIP" | "GZ" => Ok(Self::Gzip),
54            "BZIP2" | "BZ2" => Ok(Self::Bzip2),
55            "XZ" => Ok(Self::Xz),
56            "ZST" | "ZSTD" => Ok(Self::Zstd),
57            "" => Ok(Self::Uncompressed),
58            _ => error::UnsupportedCompressionTypeSnafu {
59                compression_type: s,
60            }
61            .fail(),
62        }
63    }
64}
65
66impl Display for CompressionType {
67    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
68        f.write_str(match self {
69            Self::Gzip => "GZIP",
70            Self::Bzip2 => "BZIP2",
71            Self::Xz => "XZ",
72            Self::Zstd => "ZSTD",
73            Self::Uncompressed => "",
74        })
75    }
76}
77
78impl CompressionType {
79    pub const fn is_compressed(&self) -> bool {
80        !matches!(self, &Self::Uncompressed)
81    }
82
83    pub const fn file_extension(&self) -> &'static str {
84        match self {
85            Self::Gzip => "gz",
86            Self::Bzip2 => "bz2",
87            Self::Xz => "xz",
88            Self::Zstd => "zst",
89            Self::Uncompressed => "",
90        }
91    }
92
93    /// Encodes `content` as a gzip stream at level 0, which stores it without compression.
94    pub async fn encode_gzip_stored(content: &[u8]) -> io::Result<Vec<u8>> {
95        let mut buffer = Vec::with_capacity(content.len());
96        let mut encoder = write::GzipEncoder::with_quality(&mut buffer, Level::Precise(0));
97        encoder.write_all(content).await?;
98        encoder.shutdown().await?;
99        Ok(buffer)
100    }
101}
102
103macro_rules! impl_compression_type {
104    ($(($enum_item:ident, $prefix:ident)),*) => {
105        paste::item! {
106            use bytes::{Buf, BufMut, BytesMut};
107
108            impl CompressionType {
109                pub async fn encode<B: Buf>(&self, mut content: B) -> io::Result<Vec<u8>> {
110                    match self {
111                        $(
112                            CompressionType::$enum_item => {
113                                let mut buffer = Vec::with_capacity(content.remaining());
114                                let mut encoder = write::[<$prefix Encoder>]::new(&mut buffer);
115                                encoder.write_all_buf(&mut content).await?;
116                                encoder.shutdown().await?;
117                                Ok(buffer)
118                            }
119                        )*
120                        CompressionType::Uncompressed => {
121                            let mut bs = BytesMut::with_capacity(content.remaining());
122                            bs.put(content);
123                            Ok(bs.to_vec())
124                        },
125                    }
126                }
127
128                pub async fn decode<B: Buf>(&self, mut content: B) -> io::Result<Vec<u8>> {
129                    match self {
130                        $(
131                            CompressionType::$enum_item => {
132                                let mut buffer = Vec::with_capacity(content.remaining() * 2);
133                                let mut encoder = write::[<$prefix Decoder>]::new(&mut buffer);
134                                encoder.write_all_buf(&mut content).await?;
135                                encoder.shutdown().await?;
136                                Ok(buffer)
137                            }
138                        )*
139                        CompressionType::Uncompressed => {
140                            let mut bs = BytesMut::with_capacity(content.remaining());
141                            bs.put(content);
142                            Ok(bs.to_vec())
143                        },
144                    }
145                }
146
147                pub fn convert_async_read<T: AsyncRead + Unpin + Send + 'static>(
148                    &self,
149                    s: T,
150                ) -> Box<dyn AsyncRead + Unpin + Send> {
151                    match self {
152                        $(CompressionType::$enum_item => Box::new([<$prefix Decoder>]::new(BufReader::new(s))),)*
153                        CompressionType::Uncompressed => Box::new(s),
154                    }
155                }
156
157                pub fn convert_stream<T: Stream<Item = io::Result<Bytes>> + Unpin + Send + 'static>(
158                    &self,
159                    s: T,
160                ) -> Box<dyn Stream<Item = io::Result<Bytes>> + Send + Unpin> {
161                    match self {
162                        $(CompressionType::$enum_item => Box::new(ReaderStream::new([<$prefix Decoder>]::new(StreamReader::new(s)))),)*
163                        CompressionType::Uncompressed => Box::new(s),
164                    }
165                }
166            }
167
168            #[cfg(test)]
169            mod tests {
170                use super::CompressionType;
171
172                $(
173                #[tokio::test]
174                async fn [<test_ $enum_item:lower _compression>]() {
175                    let string = "foo_bar".as_bytes();
176                    let compress = CompressionType::$enum_item
177                        .encode(string)
178                        .await
179                        .unwrap();
180                    let decompress = CompressionType::$enum_item
181                        .decode(compress.as_slice())
182                        .await
183                        .unwrap();
184                    assert_eq!(decompress, string);
185                })*
186
187                #[tokio::test]
188                async fn test_uncompression() {
189                    let string = "foo_bar".as_bytes();
190                    let compress = CompressionType::Uncompressed
191                        .encode(string)
192                        .await
193                        .unwrap();
194                    let decompress = CompressionType::Uncompressed
195                        .decode(compress.as_slice())
196                        .await
197                        .unwrap();
198                    assert_eq!(decompress, string);
199                }
200            }
201        }
202    };
203}
204
205impl_compression_type!((Gzip, Gzip), (Bzip2, Bz), (Xz, Xz), (Zstd, Zstd));
206
207impl From<CompressionType> for FileCompressionType {
208    fn from(t: CompressionType) -> Self {
209        match t {
210            CompressionType::Gzip => FileCompressionType::GZIP,
211            CompressionType::Bzip2 => FileCompressionType::BZIP2,
212            CompressionType::Xz => FileCompressionType::XZ,
213            CompressionType::Zstd => FileCompressionType::ZSTD,
214            CompressionType::Uncompressed => FileCompressionType::UNCOMPRESSED,
215        }
216    }
217}