mito2/sst/parquet/
metadata.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::result::Result as StdResult;
16use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
17
18use bytes::Bytes;
19use futures::FutureExt;
20use futures::future::BoxFuture;
21use object_store::ObjectStore;
22use parquet::arrow::async_reader::MetadataFetch;
23use parquet::errors::{ParquetError, Result as ParquetResult};
24use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader};
25use parquet::file::statistics::Statistics;
26use snafu::{IntoError as _, ResultExt};
27use store_api::metadata::RegionMetadata;
28use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME;
29
30use crate::error::{self, Result};
31use crate::sst::parquet::reader::MetadataCacheMetrics;
32
33/// The estimated size of the footer and metadata need to read from the end of parquet file.
34const DEFAULT_PREFETCH_SIZE: u64 = 64 * 1024;
35
36pub struct MetadataLoader<'a> {
37    // An object store that supports async read
38    object_store: ObjectStore,
39    // The path of parquet file
40    file_path: &'a str,
41    // The size of parquet file
42    file_size: u64,
43    page_index_policy: PageIndexPolicy,
44}
45
46impl<'a> MetadataLoader<'a> {
47    /// Create a new parquet metadata loader.
48    pub fn new(
49        object_store: ObjectStore,
50        file_path: &'a str,
51        file_size: u64,
52    ) -> MetadataLoader<'a> {
53        Self {
54            object_store,
55            file_path,
56            file_size,
57            page_index_policy: Default::default(),
58        }
59    }
60
61    pub(crate) fn with_page_index_policy(&mut self, page_index_policy: PageIndexPolicy) {
62        self.page_index_policy = page_index_policy;
63    }
64
65    /// Get the size of parquet file. If file_size is 0, stat the object store to get the size.
66    async fn get_file_size(&self) -> Result<u64> {
67        let file_size = match self.file_size {
68            0 => self
69                .object_store
70                .stat(self.file_path)
71                .await
72                .context(error::OpenDalSnafu)?
73                .content_length(),
74            other => other,
75        };
76        Ok(file_size)
77    }
78
79    pub async fn load(&self, cache_metrics: &mut MetadataCacheMetrics) -> Result<ParquetMetaData> {
80        let path = self.file_path;
81        let file_size = self.get_file_size().await?;
82        let reader = ParquetMetaDataReader::new()
83            .with_prefetch_hint(Some(DEFAULT_PREFETCH_SIZE as usize))
84            // Mito uses offset indexes to translate row selections into byte ranges. It does not
85            // consume Parquet column indexes, so decoding them only bloats the metadata cache.
86            .with_column_index_policy(PageIndexPolicy::Skip)
87            .with_offset_index_policy(self.page_index_policy);
88
89        let num_reads = AtomicUsize::new(0);
90        let bytes_read = AtomicU64::new(0);
91        let fetch = ObjectStoreFetch {
92            object_store: &self.object_store,
93            file_path: self.file_path,
94            num_reads: &num_reads,
95            bytes_read: &bytes_read,
96        };
97
98        let metadata = reader
99            .load_and_finish(fetch, file_size)
100            .await
101            .map_err(|e| match unbox_external_error(e) {
102                Ok(os_err) => error::OpenDalSnafu {}.into_error(os_err),
103                Err(parquet_err) => error::ReadParquetSnafu { path }.into_error(parquet_err),
104            })?;
105
106        cache_metrics.num_reads = num_reads.into_inner();
107        cache_metrics.bytes_read = bytes_read.into_inner();
108
109        Ok(metadata)
110    }
111}
112
113/// Unpack ParquetError to get object_store::Error if possible.
114fn unbox_external_error(e: ParquetError) -> StdResult<object_store::Error, ParquetError> {
115    match e {
116        ParquetError::External(boxed_err) => match boxed_err.downcast::<object_store::Error>() {
117            Ok(os_err) => Ok(*os_err),
118            Err(parquet_error) => Err(ParquetError::External(parquet_error)),
119        },
120        other => Err(other),
121    }
122}
123
124pub(crate) fn extract_primary_key_range(
125    parquet_meta: &ParquetMetaData,
126    region_metadata: &RegionMetadata,
127) -> Option<(Bytes, Bytes)> {
128    if region_metadata.primary_key.is_empty() {
129        return None;
130    }
131
132    let pk_column_idx = parquet_meta
133        .file_metadata()
134        .schema_descr()
135        .columns()
136        .iter()
137        .position(|column| column.name() == PRIMARY_KEY_COLUMN_NAME)?;
138
139    let mut min: Option<Bytes> = None;
140    let mut max: Option<Bytes> = None;
141
142    for row_group in parquet_meta.row_groups() {
143        let Statistics::ByteArray(stats) = row_group.column(pk_column_idx).statistics()? else {
144            return None;
145        };
146
147        let row_group_min = Bytes::copy_from_slice(stats.min_bytes_opt()?);
148        let row_group_max = Bytes::copy_from_slice(stats.max_bytes_opt()?);
149        min = Some(match min {
150            Some(current) => current.min(row_group_min),
151            None => row_group_min,
152        });
153        max = Some(match max {
154            Some(current) => current.max(row_group_max),
155            None => row_group_max,
156        });
157    }
158
159    min.zip(max)
160}
161
162struct ObjectStoreFetch<'a> {
163    object_store: &'a ObjectStore,
164    file_path: &'a str,
165    num_reads: &'a AtomicUsize,
166    bytes_read: &'a AtomicU64,
167}
168
169impl MetadataFetch for ObjectStoreFetch<'_> {
170    fn fetch(&mut self, range: std::ops::Range<u64>) -> BoxFuture<'_, ParquetResult<Bytes>> {
171        let bytes_to_read = range.end - range.start;
172        async move {
173            let data = self
174                .object_store
175                .read_with(self.file_path)
176                .range(range)
177                .await
178                .map_err(|e| ParquetError::External(Box::new(e)))?;
179            self.num_reads.fetch_add(1, Ordering::Relaxed);
180            self.bytes_read.fetch_add(bytes_to_read, Ordering::Relaxed);
181            Ok(data.to_bytes())
182        }
183        .boxed()
184    }
185}
186
187#[cfg(test)]
188mod tests {
189    use std::sync::Arc;
190
191    use datatypes::arrow::array::{
192        ArrayRef, BinaryArray, DictionaryArray, Int64Array, UInt32Array,
193    };
194    use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field, Schema};
195    use datatypes::arrow::record_batch::RecordBatch;
196    use object_store::services::Memory;
197    use parquet::arrow::ArrowWriter;
198    use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
199    use parquet::file::metadata::{KeyValue, ParquetMetaData};
200    use parquet::file::properties::{EnabledStatistics, WriterProperties};
201
202    use super::*;
203    use crate::sst::parquet::PARQUET_METADATA_KEY;
204    use crate::test_util::sst_util::sst_region_metadata;
205
206    fn build_test_parquet_bytes(
207        include_primary_key: bool,
208        primary_keys: &[&[u8]],
209        row_group_sizes: &[usize],
210        stats_enabled: EnabledStatistics,
211    ) -> Vec<u8> {
212        let total_rows = row_group_sizes.iter().sum::<usize>();
213        let mut fields = vec![Field::new("field", ArrowDataType::Int64, true)];
214        let mut columns: Vec<ArrayRef> =
215            vec![Arc::new(Int64Array::from_iter_values(0..total_rows as i64))];
216        if include_primary_key {
217            assert_eq!(total_rows, primary_keys.len());
218            fields.push(Field::new(
219                "__primary_key",
220                ArrowDataType::Dictionary(
221                    Box::new(ArrowDataType::UInt32),
222                    Box::new(ArrowDataType::Binary),
223                ),
224                false,
225            ));
226            let values = Arc::new(BinaryArray::from_iter_values(primary_keys.iter().copied()));
227            let keys = UInt32Array::from_iter_values(0..primary_keys.len() as u32);
228            columns.push(Arc::new(DictionaryArray::new(keys, values)));
229        }
230
231        let schema = Arc::new(Schema::new(fields));
232        let region_metadata = Arc::new(sst_region_metadata());
233        let key_value = KeyValue::new(
234            PARQUET_METADATA_KEY.to_string(),
235            region_metadata.to_json().unwrap(),
236        );
237        let props = WriterProperties::builder()
238            .set_key_value_metadata(Some(vec![key_value]))
239            .set_statistics_enabled(stats_enabled)
240            .build();
241
242        let mut parquet_bytes = Vec::new();
243        let mut writer =
244            ArrowWriter::try_new(&mut parquet_bytes, schema.clone(), Some(props)).unwrap();
245        let mut offset = 0;
246        for row_group_size in row_group_sizes {
247            let batch = RecordBatch::try_new(
248                schema.clone(),
249                columns
250                    .iter()
251                    .map(|column| column.slice(offset, *row_group_size))
252                    .collect(),
253            )
254            .unwrap();
255            writer.write(&batch).unwrap();
256            offset += row_group_size;
257        }
258        writer.close().unwrap();
259
260        parquet_bytes
261    }
262
263    fn build_test_metadata(
264        include_primary_key: bool,
265        primary_keys: &[&[u8]],
266        row_group_sizes: &[usize],
267        stats_enabled: EnabledStatistics,
268    ) -> ParquetMetaData {
269        ParquetRecordBatchReaderBuilder::try_new(Bytes::from(build_test_parquet_bytes(
270            include_primary_key,
271            primary_keys,
272            row_group_sizes,
273            stats_enabled,
274        )))
275        .unwrap()
276        .metadata()
277        .as_ref()
278        .clone()
279    }
280
281    #[tokio::test]
282    async fn test_metadata_loader_only_loads_offset_indexes() {
283        let parquet_bytes = build_test_parquet_bytes(false, &[], &[4], EnabledStatistics::Page);
284        let file_size = parquet_bytes.len() as u64;
285        let file_path = "test.parquet";
286        let object_store = ObjectStore::new(Memory::default()).unwrap().finish();
287        object_store.write(file_path, parquet_bytes).await.unwrap();
288
289        let mut loader = MetadataLoader::new(object_store, file_path, file_size);
290        loader.with_page_index_policy(PageIndexPolicy::Required);
291        let metadata = loader
292            .load(&mut MetadataCacheMetrics::default())
293            .await
294            .unwrap();
295
296        assert!(metadata.column_index().is_none());
297        assert!(metadata.offset_index().is_some());
298    }
299
300    #[test]
301    fn test_extract_primary_key_range_returns_none_when_column_absent() {
302        let metadata = build_test_metadata(false, &[], &[1], EnabledStatistics::Page);
303        let region_metadata = sst_region_metadata();
304
305        assert_eq!(None, extract_primary_key_range(&metadata, &region_metadata));
306    }
307
308    #[test]
309    fn test_extract_primary_key_range_folds_row_group_stats() {
310        let metadata = build_test_metadata(
311            true,
312            &[b"bbb", b"ccc", b"aaa", b"zzz"],
313            &[2, 2],
314            EnabledStatistics::Page,
315        );
316        let region_metadata = sst_region_metadata();
317
318        assert_eq!(
319            Some((Bytes::from_static(b"aaa"), Bytes::from_static(b"zzz"))),
320            extract_primary_key_range(&metadata, &region_metadata)
321        );
322    }
323
324    #[test]
325    fn test_extract_primary_key_range_returns_none_when_any_rg_stats_missing() {
326        let metadata = build_test_metadata(
327            true,
328            &[b"bbb", b"ccc", b"aaa", b"zzz"],
329            &[2, 2],
330            EnabledStatistics::None,
331        );
332        let region_metadata = sst_region_metadata();
333
334        assert_eq!(None, extract_primary_key_range(&metadata, &region_metadata));
335    }
336}
mito2/sst/parquet/metadata.rs

mito2/sst/parquet/
metadata.rs
