common_datasource/
compression.rs1use std::fmt::Display;
16use std::io;
17use std::str::FromStr;
18
19use async_compression::Level;
20use async_compression::tokio::bufread::{BzDecoder, GzipDecoder, XzDecoder, ZstdDecoder};
21use async_compression::tokio::write;
22use bytes::Bytes;
23use datafusion::datasource::file_format::file_compression_type::FileCompressionType;
24use futures::Stream;
25use serde::{Deserialize, Serialize};
26use strum::EnumIter;
27use tokio::io::{AsyncRead, AsyncWriteExt, BufReader};
28use tokio_util::io::{ReaderStream, StreamReader};
29
30use crate::error::{self, Error, Result};
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EnumIter, Serialize, Deserialize)]
33#[serde(rename_all = "lowercase")]
34pub enum CompressionType {
35 Gzip,
37 Bzip2,
39 Xz,
41 Zstd,
43 Uncompressed,
45}
46
47impl FromStr for CompressionType {
48 type Err = Error;
49
50 fn from_str(s: &str) -> Result<Self> {
51 let s = s.to_uppercase();
52 match s.as_str() {
53 "GZIP" | "GZ" => Ok(Self::Gzip),
54 "BZIP2" | "BZ2" => Ok(Self::Bzip2),
55 "XZ" => Ok(Self::Xz),
56 "ZST" | "ZSTD" => Ok(Self::Zstd),
57 "" => Ok(Self::Uncompressed),
58 _ => error::UnsupportedCompressionTypeSnafu {
59 compression_type: s,
60 }
61 .fail(),
62 }
63 }
64}
65
66impl Display for CompressionType {
67 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
68 f.write_str(match self {
69 Self::Gzip => "GZIP",
70 Self::Bzip2 => "BZIP2",
71 Self::Xz => "XZ",
72 Self::Zstd => "ZSTD",
73 Self::Uncompressed => "",
74 })
75 }
76}
77
78impl CompressionType {
79 pub const fn is_compressed(&self) -> bool {
80 !matches!(self, &Self::Uncompressed)
81 }
82
83 pub const fn file_extension(&self) -> &'static str {
84 match self {
85 Self::Gzip => "gz",
86 Self::Bzip2 => "bz2",
87 Self::Xz => "xz",
88 Self::Zstd => "zst",
89 Self::Uncompressed => "",
90 }
91 }
92
93 pub async fn encode_gzip_stored(content: &[u8]) -> io::Result<Vec<u8>> {
95 let mut buffer = Vec::with_capacity(content.len());
96 let mut encoder = write::GzipEncoder::with_quality(&mut buffer, Level::Precise(0));
97 encoder.write_all(content).await?;
98 encoder.shutdown().await?;
99 Ok(buffer)
100 }
101}
102
103macro_rules! impl_compression_type {
104 ($(($enum_item:ident, $prefix:ident)),*) => {
105 paste::item! {
106 use bytes::{Buf, BufMut, BytesMut};
107
108 impl CompressionType {
109 pub async fn encode<B: Buf>(&self, mut content: B) -> io::Result<Vec<u8>> {
110 match self {
111 $(
112 CompressionType::$enum_item => {
113 let mut buffer = Vec::with_capacity(content.remaining());
114 let mut encoder = write::[<$prefix Encoder>]::new(&mut buffer);
115 encoder.write_all_buf(&mut content).await?;
116 encoder.shutdown().await?;
117 Ok(buffer)
118 }
119 )*
120 CompressionType::Uncompressed => {
121 let mut bs = BytesMut::with_capacity(content.remaining());
122 bs.put(content);
123 Ok(bs.to_vec())
124 },
125 }
126 }
127
128 pub async fn decode<B: Buf>(&self, mut content: B) -> io::Result<Vec<u8>> {
129 match self {
130 $(
131 CompressionType::$enum_item => {
132 let mut buffer = Vec::with_capacity(content.remaining() * 2);
133 let mut encoder = write::[<$prefix Decoder>]::new(&mut buffer);
134 encoder.write_all_buf(&mut content).await?;
135 encoder.shutdown().await?;
136 Ok(buffer)
137 }
138 )*
139 CompressionType::Uncompressed => {
140 let mut bs = BytesMut::with_capacity(content.remaining());
141 bs.put(content);
142 Ok(bs.to_vec())
143 },
144 }
145 }
146
147 pub fn convert_async_read<T: AsyncRead + Unpin + Send + 'static>(
148 &self,
149 s: T,
150 ) -> Box<dyn AsyncRead + Unpin + Send> {
151 match self {
152 $(CompressionType::$enum_item => Box::new([<$prefix Decoder>]::new(BufReader::new(s))),)*
153 CompressionType::Uncompressed => Box::new(s),
154 }
155 }
156
157 pub fn convert_stream<T: Stream<Item = io::Result<Bytes>> + Unpin + Send + 'static>(
158 &self,
159 s: T,
160 ) -> Box<dyn Stream<Item = io::Result<Bytes>> + Send + Unpin> {
161 match self {
162 $(CompressionType::$enum_item => Box::new(ReaderStream::new([<$prefix Decoder>]::new(StreamReader::new(s)))),)*
163 CompressionType::Uncompressed => Box::new(s),
164 }
165 }
166 }
167
168 #[cfg(test)]
169 mod tests {
170 use super::CompressionType;
171
172 $(
173 #[tokio::test]
174 async fn [<test_ $enum_item:lower _compression>]() {
175 let string = "foo_bar".as_bytes();
176 let compress = CompressionType::$enum_item
177 .encode(string)
178 .await
179 .unwrap();
180 let decompress = CompressionType::$enum_item
181 .decode(compress.as_slice())
182 .await
183 .unwrap();
184 assert_eq!(decompress, string);
185 })*
186
187 #[tokio::test]
188 async fn test_uncompression() {
189 let string = "foo_bar".as_bytes();
190 let compress = CompressionType::Uncompressed
191 .encode(string)
192 .await
193 .unwrap();
194 let decompress = CompressionType::Uncompressed
195 .decode(compress.as_slice())
196 .await
197 .unwrap();
198 assert_eq!(decompress, string);
199 }
200 }
201 }
202 };
203}
204
205impl_compression_type!((Gzip, Gzip), (Bzip2, Bz), (Xz, Xz), (Zstd, Zstd));
206
207impl From<CompressionType> for FileCompressionType {
208 fn from(t: CompressionType) -> Self {
209 match t {
210 CompressionType::Gzip => FileCompressionType::GZIP,
211 CompressionType::Bzip2 => FileCompressionType::BZIP2,
212 CompressionType::Xz => FileCompressionType::XZ,
213 CompressionType::Zstd => FileCompressionType::ZSTD,
214 CompressionType::Uncompressed => FileCompressionType::UNCOMPRESSED,
215 }
216 }
217}