Skip to main content

store_api/
sst_entry.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16
17use bytes::Bytes;
18use common_recordbatch::DfRecordBatch;
19use common_time::Timestamp;
20use common_time::timestamp::TimeUnit;
21use datafusion_common::DataFusionError;
22use datafusion_expr::LogicalPlan;
23use datatypes::arrow::array::{
24    ArrayRef, BinaryArray, BooleanArray, TimestampMillisecondArray, TimestampNanosecondArray,
25    UInt8Array, UInt32Array, UInt64Array,
26};
27use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef;
28use datatypes::arrow::error::ArrowError;
29use datatypes::arrow_array::StringArray;
30use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
31use serde::{Deserialize, Serialize};
32
33use crate::inspect::{self, ProjectedBatchBuilder};
34use crate::storage::{RegionGroup, RegionId, RegionNumber, RegionSeq, ScanRequest, TableId};
35
36// Manifest SST column names.
37pub const TABLE_DIR: &str = "table_dir";
38pub const REGION_ID: &str = "region_id";
39pub const TABLE_ID: &str = "table_id";
40pub const REGION_NUMBER: &str = "region_number";
41pub const REGION_GROUP: &str = "region_group";
42pub const REGION_SEQUENCE: &str = "region_sequence";
43pub const FILE_ID: &str = "file_id";
44pub const INDEX_VERSION: &str = "index_version";
45pub const LEVEL: &str = "level";
46pub const FILE_PATH: &str = "file_path";
47pub const FILE_SIZE: &str = "file_size";
48pub const INDEX_FILE_PATH: &str = "index_file_path";
49pub const INDEX_FILE_SIZE: &str = "index_file_size";
50pub const NUM_ROWS: &str = "num_rows";
51pub const NUM_ROW_GROUPS: &str = "num_row_groups";
52pub const NUM_SERIES: &str = "num_series";
53pub const MIN_TS: &str = "min_ts";
54pub const MAX_TS: &str = "max_ts";
55pub const SEQUENCE: &str = "sequence";
56pub const ORIGIN_REGION_ID: &str = "origin_region_id";
57pub const NODE_ID: &str = "node_id";
58pub const VISIBLE: &str = "visible";
59pub const PRIMARY_KEY_MIN: &str = "primary_key_min";
60pub const PRIMARY_KEY_MAX: &str = "primary_key_max";
61pub const MAX_ROW_GROUP_UNCOMPRESSED_SIZE: &str = "max_row_group_uncompressed_size";
62pub const PARTITION_EXPR: &str = "partition_expr";
63
64/// Puffin index type for bloom filter indexes.
65pub const PUFFIN_INDEX_TYPE_BLOOM_FILTER: &str = "bloom_filter";
66/// Puffin index type for bloom-backed full-text indexes.
67pub const PUFFIN_INDEX_TYPE_FULLTEXT_BLOOM: &str = "fulltext_bloom";
68/// Puffin index type for Tantivy-backed full-text indexes.
69pub const PUFFIN_INDEX_TYPE_FULLTEXT_TANTIVY: &str = "fulltext_tantivy";
70/// Puffin index type for inverted indexes.
71pub const PUFFIN_INDEX_TYPE_INVERTED: &str = "inverted";
72
73/// An entry describing a SST file known by the engine's manifest.
74#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
75pub struct ManifestSstEntry {
76    /// The table directory this file belongs to.
77    pub table_dir: String,
78    /// The region id of region that refers to the file.
79    pub region_id: RegionId,
80    /// The table id this file belongs to.
81    pub table_id: TableId,
82    /// The region number this file belongs to.
83    pub region_number: RegionNumber,
84    /// The region group this file belongs to.
85    pub region_group: RegionGroup,
86    /// The region sequence this file belongs to.
87    pub region_sequence: RegionSeq,
88    /// Engine-specific file identifier (string form).
89    pub file_id: String,
90    /// Index version, increment when the index file is rebuilt.
91    pub index_version: u64,
92    /// SST level.
93    pub level: u8,
94    /// Full path of the SST file in object store.
95    pub file_path: String,
96    /// File size in bytes.
97    pub file_size: u64,
98    /// Maximum uncompressed row group size in bytes. Zero means unknown.
99    pub max_row_group_uncompressed_size: u64,
100    /// Full path of the index file in object store.
101    pub index_file_path: Option<String>,
102    /// File size of the index file in object store.
103    pub index_file_size: Option<u64>,
104    /// Number of rows in the SST.
105    pub num_rows: u64,
106    /// Number of row groups in the SST.
107    pub num_row_groups: u64,
108    /// Number of series in the SST.
109    pub num_series: Option<u64>,
110    /// Min timestamp.
111    pub min_ts: Timestamp,
112    /// Max timestamp.
113    pub max_ts: Timestamp,
114    /// The sequence number associated with this file.
115    pub sequence: Option<u64>,
116    /// Human-readable partition expression associated with this file.
117    pub partition_expr: Option<String>,
118    /// The region id of region that creates the file.
119    pub origin_region_id: RegionId,
120    /// The node id fetched from the manifest.
121    pub node_id: Option<u64>,
122    /// Whether this file is visible in current version.
123    pub visible: bool,
124    /// Minimum encoded primary key in the SST.
125    pub primary_key_min: Option<Bytes>,
126    /// Maximum encoded primary key in the SST.
127    pub primary_key_max: Option<Bytes>,
128}
129
130impl ManifestSstEntry {
131    /// Returns the schema of the manifest sst entry.
132    pub fn schema() -> SchemaRef {
133        use datatypes::prelude::ConcreteDataType as Ty;
134        Arc::new(Schema::new(vec![
135            ColumnSchema::new(TABLE_DIR, Ty::string_datatype(), false),
136            ColumnSchema::new(REGION_ID, Ty::uint64_datatype(), false),
137            ColumnSchema::new(TABLE_ID, Ty::uint32_datatype(), false),
138            ColumnSchema::new(REGION_NUMBER, Ty::uint32_datatype(), false),
139            ColumnSchema::new(REGION_GROUP, Ty::uint8_datatype(), false),
140            ColumnSchema::new(REGION_SEQUENCE, Ty::uint32_datatype(), false),
141            ColumnSchema::new(FILE_ID, Ty::string_datatype(), false),
142            ColumnSchema::new(INDEX_VERSION, Ty::uint64_datatype(), false),
143            ColumnSchema::new(LEVEL, Ty::uint8_datatype(), false),
144            ColumnSchema::new(FILE_PATH, Ty::string_datatype(), false),
145            ColumnSchema::new(FILE_SIZE, Ty::uint64_datatype(), false),
146            ColumnSchema::new(INDEX_FILE_PATH, Ty::string_datatype(), true),
147            ColumnSchema::new(INDEX_FILE_SIZE, Ty::uint64_datatype(), true),
148            ColumnSchema::new(NUM_ROWS, Ty::uint64_datatype(), false),
149            ColumnSchema::new(NUM_ROW_GROUPS, Ty::uint64_datatype(), false),
150            ColumnSchema::new(NUM_SERIES, Ty::uint64_datatype(), true),
151            ColumnSchema::new(MIN_TS, Ty::timestamp_nanosecond_datatype(), true),
152            ColumnSchema::new(MAX_TS, Ty::timestamp_nanosecond_datatype(), true),
153            ColumnSchema::new(SEQUENCE, Ty::uint64_datatype(), true),
154            ColumnSchema::new(ORIGIN_REGION_ID, Ty::uint64_datatype(), false),
155            ColumnSchema::new(NODE_ID, Ty::uint64_datatype(), true),
156            ColumnSchema::new(VISIBLE, Ty::boolean_datatype(), false),
157            ColumnSchema::new(PRIMARY_KEY_MIN, Ty::binary_datatype(), true),
158            ColumnSchema::new(PRIMARY_KEY_MAX, Ty::binary_datatype(), true),
159            ColumnSchema::new(
160                MAX_ROW_GROUP_UNCOMPRESSED_SIZE,
161                Ty::uint64_datatype(),
162                false,
163            ),
164            ColumnSchema::new(PARTITION_EXPR, Ty::string_datatype(), true),
165        ]))
166    }
167
168    /// Converts a list of manifest sst entries to a record batch.
169    pub fn to_record_batch(entries: &[Self]) -> std::result::Result<DfRecordBatch, ArrowError> {
170        Self::to_projected_record_batch(entries, None)
171    }
172
173    /// Builds only the requested columns, including empty and repeated projections.
174    fn to_projected_record_batch(
175        entries: &[Self],
176        projection: Option<&[usize]>,
177    ) -> std::result::Result<DfRecordBatch, ArrowError> {
178        let schema = Self::schema().arrow_schema().clone();
179        let schema = match projection {
180            Some(projection) => Arc::new(schema.project(projection)?),
181            None => schema,
182        };
183        ManifestSstBatchBuilder::new(schema).build(entries)
184    }
185
186    /// Reserved internal inspect table name.
187    ///
188    /// This table name is used only for building logical plans on the
189    /// frontend -> datanode path. It is not user-visible and cannot be
190    /// referenced by user queries.
191    pub fn reserved_table_name_for_inspection() -> &'static str {
192        "__inspect/__mito/__sst_manifest"
193    }
194
195    /// Builds a logical plan for scanning the manifest sst entries.
196    pub fn build_plan(scan_request: ScanRequest) -> Result<LogicalPlan, DataFusionError> {
197        inspect::build_plan(
198            scan_request,
199            Self::reserved_table_name_for_inspection(),
200            Self::schema(),
201        )
202    }
203}
204
205/// Converts manifest SST batches using a schema prepared once per scan.
206/// Repeated projected columns share the same Arrow array.
207pub struct ManifestSstBatchBuilder {
208    projected: ProjectedBatchBuilder,
209}
210
211impl ManifestSstBatchBuilder {
212    /// Creates a builder for a projected manifest SST schema.
213    pub fn new(schema: ArrowSchemaRef) -> Self {
214        Self {
215            projected: ProjectedBatchBuilder::new(schema),
216        }
217    }
218
219    /// Builds the projected columns, copying string contents only into Arrow buffers.
220    pub fn build(
221        &self,
222        entries: &[ManifestSstEntry],
223    ) -> std::result::Result<DfRecordBatch, ArrowError> {
224        self.projected.build(entries.len(), |name| {
225            let array: ArrayRef = match name {
226                TABLE_DIR => Arc::new(StringArray::from_iter_values(
227                    entries.iter().map(|e| e.table_dir.as_str()),
228                )),
229                REGION_ID => Arc::new(UInt64Array::from_iter_values(
230                    entries.iter().map(|e| e.region_id.as_u64()),
231                )),
232                TABLE_ID => Arc::new(UInt32Array::from_iter_values(
233                    entries.iter().map(|e| e.table_id),
234                )),
235                REGION_NUMBER => Arc::new(UInt32Array::from_iter_values(
236                    entries.iter().map(|e| e.region_number),
237                )),
238                REGION_GROUP => Arc::new(UInt8Array::from_iter_values(
239                    entries.iter().map(|e| e.region_group),
240                )),
241                REGION_SEQUENCE => Arc::new(UInt32Array::from_iter_values(
242                    entries.iter().map(|e| e.region_sequence),
243                )),
244                FILE_ID => Arc::new(StringArray::from_iter_values(
245                    entries.iter().map(|e| e.file_id.as_str()),
246                )),
247                INDEX_VERSION => Arc::new(UInt64Array::from_iter(
248                    entries.iter().map(|e| e.index_version),
249                )),
250                LEVEL => Arc::new(UInt8Array::from_iter_values(
251                    entries.iter().map(|e| e.level),
252                )),
253                FILE_PATH => Arc::new(StringArray::from_iter_values(
254                    entries.iter().map(|e| e.file_path.as_str()),
255                )),
256                FILE_SIZE => Arc::new(UInt64Array::from_iter_values(
257                    entries.iter().map(|e| e.file_size),
258                )),
259                INDEX_FILE_PATH => Arc::new(StringArray::from_iter(
260                    entries.iter().map(|e| e.index_file_path.as_ref()),
261                )),
262                INDEX_FILE_SIZE => Arc::new(UInt64Array::from_iter(
263                    entries.iter().map(|e| e.index_file_size),
264                )),
265                NUM_ROWS => Arc::new(UInt64Array::from_iter_values(
266                    entries.iter().map(|e| e.num_rows),
267                )),
268                NUM_ROW_GROUPS => Arc::new(UInt64Array::from_iter_values(
269                    entries.iter().map(|e| e.num_row_groups),
270                )),
271                NUM_SERIES => {
272                    Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.num_series)))
273                }
274                MIN_TS => Arc::new(TimestampNanosecondArray::from_iter(entries.iter().map(
275                    |e| {
276                        e.min_ts
277                            .convert_to(TimeUnit::Nanosecond)
278                            .map(|ts| ts.value())
279                    },
280                ))),
281                MAX_TS => Arc::new(TimestampNanosecondArray::from_iter(entries.iter().map(
282                    |e| {
283                        e.max_ts
284                            .convert_to(TimeUnit::Nanosecond)
285                            .map(|ts| ts.value())
286                    },
287                ))),
288                SEQUENCE => Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.sequence))),
289                ORIGIN_REGION_ID => Arc::new(UInt64Array::from_iter_values(
290                    entries.iter().map(|e| e.origin_region_id.as_u64()),
291                )),
292                NODE_ID => Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.node_id))),
293                VISIBLE => Arc::new(BooleanArray::from_iter(
294                    entries.iter().map(|e| Some(e.visible)),
295                )),
296                PRIMARY_KEY_MIN => Arc::new(BinaryArray::from_iter(
297                    entries.iter().map(|e| e.primary_key_min.as_deref()),
298                )),
299                PRIMARY_KEY_MAX => Arc::new(BinaryArray::from_iter(
300                    entries.iter().map(|e| e.primary_key_max.as_deref()),
301                )),
302                MAX_ROW_GROUP_UNCOMPRESSED_SIZE => Arc::new(UInt64Array::from_iter_values(
303                    entries.iter().map(|e| e.max_row_group_uncompressed_size),
304                )),
305                PARTITION_EXPR => Arc::new(StringArray::from_iter(
306                    entries.iter().map(|e| e.partition_expr.as_ref()),
307                )),
308                name => {
309                    return Err(ArrowError::SchemaError(format!(
310                        "Unknown manifest SST column: {name}"
311                    )));
312                }
313            };
314            Ok(array)
315        })
316    }
317}
318
319/// An entry describing a SST file listed from storage layer directly.
320#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
321pub struct StorageSstEntry {
322    /// Full path of the SST file in object store.
323    pub file_path: String,
324    /// File size in bytes.
325    pub file_size: Option<u64>,
326    /// Last modified time in milliseconds since epoch, if available from storage.
327    pub last_modified_ms: Option<Timestamp>,
328    /// The node id fetched from the manifest.
329    pub node_id: Option<u64>,
330}
331
332impl StorageSstEntry {
333    /// Returns the schema of the storage sst entry.
334    pub fn schema() -> SchemaRef {
335        use datatypes::prelude::ConcreteDataType as Ty;
336        Arc::new(Schema::new(vec![
337            ColumnSchema::new("file_path", Ty::string_datatype(), false),
338            ColumnSchema::new("file_size", Ty::uint64_datatype(), true),
339            ColumnSchema::new(
340                "last_modified_ms",
341                Ty::timestamp_millisecond_datatype(),
342                true,
343            ),
344            ColumnSchema::new("node_id", Ty::uint64_datatype(), true),
345        ]))
346    }
347
348    /// Converts a list of storage sst entries to a record batch.
349    pub fn to_record_batch(entries: &[Self]) -> std::result::Result<DfRecordBatch, ArrowError> {
350        let schema = Self::schema();
351        let file_paths = entries.iter().map(|e| e.file_path.as_str());
352        let file_sizes = entries.iter().map(|e| e.file_size);
353        let last_modified_ms = entries.iter().map(|e| {
354            e.last_modified_ms
355                .and_then(|ts| ts.convert_to(TimeUnit::Millisecond).map(|ts| ts.value()))
356        });
357        let node_ids = entries.iter().map(|e| e.node_id);
358
359        let columns: Vec<ArrayRef> = vec![
360            Arc::new(StringArray::from_iter_values(file_paths)),
361            Arc::new(UInt64Array::from_iter(file_sizes)),
362            Arc::new(TimestampMillisecondArray::from_iter(last_modified_ms)),
363            Arc::new(UInt64Array::from_iter(node_ids)),
364        ];
365
366        DfRecordBatch::try_new(schema.arrow_schema().clone(), columns)
367    }
368
369    /// Reserved internal inspect table name.
370    ///
371    /// This table name is used only for building logical plans on the
372    /// frontend -> datanode path. It is not user-visible and cannot be
373    /// referenced by user queries.
374    pub fn reserved_table_name_for_inspection() -> &'static str {
375        "__inspect/__mito/__sst_storage"
376    }
377
378    /// Builds a logical plan for scanning the storage sst entries.
379    pub fn build_plan(scan_request: ScanRequest) -> Result<LogicalPlan, DataFusionError> {
380        inspect::build_plan(
381            scan_request,
382            Self::reserved_table_name_for_inspection(),
383            Self::schema(),
384        )
385    }
386}
387
388/// An entry describing puffin index metadata for inspection.
389#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
390pub struct PuffinIndexMetaEntry {
391    /// The table directory this index belongs to.
392    pub table_dir: String,
393    /// The full path of the index file in object store.
394    pub index_file_path: String,
395    /// The region id referencing the index file.
396    pub region_id: RegionId,
397    /// The table id referencing the index file.
398    pub table_id: TableId,
399    /// The region number referencing the index file.
400    pub region_number: RegionNumber,
401    /// The region group referencing the index file.
402    pub region_group: RegionGroup,
403    /// The region sequence referencing the index file.
404    pub region_sequence: RegionSeq,
405    /// Engine-specific file identifier (string form).
406    pub file_id: String,
407    /// Size of the index file in object store (if available).
408    pub index_file_size: Option<u64>,
409    /// Logical index type (`bloom_filter`, `fulltext_bloom`, `fulltext_tantivy`, `inverted`).
410    pub index_type: String,
411    /// Target type (`column`, ...).
412    pub target_type: String,
413    /// Encoded target key string.
414    pub target_key: String,
415    /// Structured JSON describing the target.
416    pub target_json: String,
417    /// Size of the blob storing this target.
418    pub blob_size: u64,
419    /// Structured JSON describing index-specific metadata (if available).
420    pub meta_json: Option<String>,
421    /// Node id associated with the index file (if known).
422    pub node_id: Option<u64>,
423}
424
425impl PuffinIndexMetaEntry {
426    /// Returns the schema describing puffin index metadata entries.
427    pub fn schema() -> SchemaRef {
428        use datatypes::prelude::ConcreteDataType as Ty;
429        Arc::new(Schema::new(vec![
430            ColumnSchema::new("table_dir", Ty::string_datatype(), false),
431            ColumnSchema::new("index_file_path", Ty::string_datatype(), false),
432            ColumnSchema::new("region_id", Ty::uint64_datatype(), false),
433            ColumnSchema::new("table_id", Ty::uint32_datatype(), false),
434            ColumnSchema::new("region_number", Ty::uint32_datatype(), false),
435            ColumnSchema::new("region_group", Ty::uint8_datatype(), false),
436            ColumnSchema::new("region_sequence", Ty::uint32_datatype(), false),
437            ColumnSchema::new("file_id", Ty::string_datatype(), false),
438            ColumnSchema::new("index_file_size", Ty::uint64_datatype(), true),
439            ColumnSchema::new("index_type", Ty::string_datatype(), false),
440            ColumnSchema::new("target_type", Ty::string_datatype(), false),
441            ColumnSchema::new("target_key", Ty::string_datatype(), false),
442            ColumnSchema::new("target_json", Ty::string_datatype(), false),
443            ColumnSchema::new("blob_size", Ty::uint64_datatype(), false),
444            ColumnSchema::new("meta_json", Ty::string_datatype(), true),
445            ColumnSchema::new("node_id", Ty::uint64_datatype(), true),
446        ]))
447    }
448
449    /// Converts a list of puffin index metadata entries to a record batch.
450    pub fn to_record_batch(entries: &[Self]) -> std::result::Result<DfRecordBatch, ArrowError> {
451        let schema = Self::schema();
452        let table_dirs = entries.iter().map(|e| e.table_dir.as_str());
453        let index_file_paths = entries.iter().map(|e| e.index_file_path.as_str());
454        let region_ids = entries.iter().map(|e| e.region_id.as_u64());
455        let table_ids = entries.iter().map(|e| e.table_id);
456        let region_numbers = entries.iter().map(|e| e.region_number);
457        let region_groups = entries.iter().map(|e| e.region_group);
458        let region_sequences = entries.iter().map(|e| e.region_sequence);
459        let file_ids = entries.iter().map(|e| e.file_id.as_str());
460        let index_file_sizes = entries.iter().map(|e| e.index_file_size);
461        let index_types = entries.iter().map(|e| e.index_type.as_str());
462        let target_types = entries.iter().map(|e| e.target_type.as_str());
463        let target_keys = entries.iter().map(|e| e.target_key.as_str());
464        let target_jsons = entries.iter().map(|e| e.target_json.as_str());
465        let blob_sizes = entries.iter().map(|e| e.blob_size);
466        let meta_jsons = entries.iter().map(|e| e.meta_json.as_deref());
467        let node_ids = entries.iter().map(|e| e.node_id);
468
469        let columns: Vec<ArrayRef> = vec![
470            Arc::new(StringArray::from_iter_values(table_dirs)),
471            Arc::new(StringArray::from_iter_values(index_file_paths)),
472            Arc::new(UInt64Array::from_iter_values(region_ids)),
473            Arc::new(UInt32Array::from_iter_values(table_ids)),
474            Arc::new(UInt32Array::from_iter_values(region_numbers)),
475            Arc::new(UInt8Array::from_iter_values(region_groups)),
476            Arc::new(UInt32Array::from_iter_values(region_sequences)),
477            Arc::new(StringArray::from_iter_values(file_ids)),
478            Arc::new(UInt64Array::from_iter(index_file_sizes)),
479            Arc::new(StringArray::from_iter_values(index_types)),
480            Arc::new(StringArray::from_iter_values(target_types)),
481            Arc::new(StringArray::from_iter_values(target_keys)),
482            Arc::new(StringArray::from_iter_values(target_jsons)),
483            Arc::new(UInt64Array::from_iter_values(blob_sizes)),
484            Arc::new(StringArray::from_iter(meta_jsons)),
485            Arc::new(UInt64Array::from_iter(node_ids)),
486        ];
487
488        DfRecordBatch::try_new(schema.arrow_schema().clone(), columns)
489    }
490
491    /// Reserved internal inspect table name for puffin index metadata.
492    pub fn reserved_table_name_for_inspection() -> &'static str {
493        "__inspect/__mito/__puffin_index_meta"
494    }
495
496    /// Builds a logical plan for scanning puffin index metadata entries.
497    pub fn build_plan(scan_request: ScanRequest) -> Result<LogicalPlan, DataFusionError> {
498        inspect::build_plan(
499            scan_request,
500            Self::reserved_table_name_for_inspection(),
501            Self::schema(),
502        )
503    }
504}
505
506#[cfg(test)]
507mod tests {
508    use datafusion_common::TableReference;
509    use datafusion_expr::{Expr, LogicalPlan, Operator, binary_expr, col, lit};
510    use datatypes::arrow::array::{
511        Array, BinaryArray, TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array,
512        UInt32Array, UInt64Array,
513    };
514    use datatypes::arrow_array::StringArray;
515
516    use super::*;
517
518    #[test]
519    fn test_sst_entry_manifest_to_record_batch() {
520        // Prepare entries
521        let table_id1: TableId = 1;
522        let region_group1: RegionGroup = 2;
523        let region_seq1: RegionSeq = 3;
524        let region_number1: RegionNumber = ((region_group1 as u32) << 24) | region_seq1;
525        let region_id1 = RegionId::with_group_and_seq(table_id1, region_group1, region_seq1);
526
527        let table_id2: TableId = 5;
528        let region_group2: RegionGroup = 1;
529        let region_seq2: RegionSeq = 42;
530        let region_number2: RegionNumber = ((region_group2 as u32) << 24) | region_seq2;
531        let region_id2 = RegionId::with_group_and_seq(table_id2, region_group2, region_seq2);
532
533        let entries = vec![
534            ManifestSstEntry {
535                table_dir: "tdir1".to_string(),
536                region_id: region_id1,
537                table_id: table_id1,
538                region_number: region_number1,
539                region_group: region_group1,
540                region_sequence: region_seq1,
541                file_id: "f1".to_string(),
542                index_version: 0,
543                level: 1,
544                file_path: "/p1".to_string(),
545                file_size: 100,
546                max_row_group_uncompressed_size: 80,
547                index_file_path: None,
548                index_file_size: None,
549                num_rows: 10,
550                num_row_groups: 2,
551                num_series: Some(5),
552                min_ts: Timestamp::new_millisecond(1000), // 1s -> 1_000_000_000ns
553                max_ts: Timestamp::new_second(2),         // 2s -> 2_000_000_000ns
554                sequence: None,
555                partition_expr: Some("a < 10".to_string()),
556                origin_region_id: region_id1,
557                node_id: Some(1),
558                visible: false,
559                primary_key_min: Some(Bytes::from_static(b"aaa")),
560                primary_key_max: Some(Bytes::from_static(b"zzz")),
561            },
562            ManifestSstEntry {
563                table_dir: "tdir2".to_string(),
564                region_id: region_id2,
565                table_id: table_id2,
566                region_number: region_number2,
567                region_group: region_group2,
568                region_sequence: region_seq2,
569                file_id: "f2".to_string(),
570                index_version: 1,
571                level: 3,
572                file_path: "/p2".to_string(),
573                file_size: 200,
574                max_row_group_uncompressed_size: 160,
575                index_file_path: Some("idx".to_string()),
576                index_file_size: Some(11),
577                num_rows: 20,
578                num_row_groups: 4,
579                num_series: None,
580                min_ts: Timestamp::new_nanosecond(5),     // 5ns
581                max_ts: Timestamp::new_microsecond(2000), // 2ms -> 2_000_000ns
582                sequence: Some(9),
583                partition_expr: None,
584                origin_region_id: region_id2,
585                node_id: None,
586                visible: true,
587                primary_key_min: None,
588                primary_key_max: None,
589            },
590        ];
591
592        let schema = ManifestSstEntry::schema();
593        let batch = ManifestSstEntry::to_record_batch(&entries).unwrap();
594        for projection in [vec![], vec![16, 6, 10], vec![1, 1], (0..26).rev().collect()] {
595            let projected =
596                ManifestSstEntry::to_projected_record_batch(&entries, Some(&projection)).unwrap();
597            assert_eq!(projected, batch.project(&projection).unwrap());
598            assert_eq!(projected.num_rows(), entries.len());
599        }
600        assert!(ManifestSstEntry::to_projected_record_batch(&entries, Some(&[26])).is_err());
601
602        let projected_schema = Arc::new(batch.schema().project(&[6, 9, 6, 11, 25, 9]).unwrap());
603        let builder = ManifestSstBatchBuilder::new(projected_schema.clone());
604        for rows in [entries.as_slice(), &entries[..1], &entries[..0]] {
605            let projected = builder.build(rows).unwrap();
606            assert_eq!(
607                projected,
608                ManifestSstEntry::to_record_batch(rows)
609                    .unwrap()
610                    .project(&[6, 9, 6, 11, 25, 9])
611                    .unwrap()
612            );
613            assert!(Arc::ptr_eq(&projected.schema(), &projected_schema));
614            assert!(Arc::ptr_eq(projected.column(0), projected.column(2)));
615            assert!(Arc::ptr_eq(projected.column(1), projected.column(5)));
616        }
617
618        // Schema checks
619        assert_eq!(schema.arrow_schema().fields().len(), batch.num_columns());
620        assert_eq!(2, batch.num_rows());
621        let expected_columns = [
622            "table_dir",
623            "region_id",
624            "table_id",
625            "region_number",
626            "region_group",
627            "region_sequence",
628            "file_id",
629            "index_version",
630            "level",
631            "file_path",
632            "file_size",
633            "index_file_path",
634            "index_file_size",
635            "num_rows",
636            "num_row_groups",
637            "num_series",
638            "min_ts",
639            "max_ts",
640            "sequence",
641            "origin_region_id",
642            "node_id",
643            "visible",
644            "primary_key_min",
645            "primary_key_max",
646            "max_row_group_uncompressed_size",
647            "partition_expr",
648        ];
649        assert_eq!(
650            expected_columns,
651            schema
652                .arrow_schema()
653                .fields()
654                .iter()
655                .map(|field| field.name().as_str())
656                .collect::<Vec<_>>()
657                .as_slice()
658        );
659        for (i, f) in schema.arrow_schema().fields().iter().enumerate() {
660            assert_eq!(f.name(), batch.schema().field(i).name());
661            assert_eq!(f.is_nullable(), batch.schema().field(i).is_nullable());
662            assert_eq!(f.data_type(), batch.schema().field(i).data_type());
663        }
664
665        // Column asserts
666        let table_dirs = batch
667            .column(0)
668            .as_any()
669            .downcast_ref::<StringArray>()
670            .unwrap();
671        assert_eq!("tdir1", table_dirs.value(0));
672        assert_eq!("tdir2", table_dirs.value(1));
673
674        let region_ids = batch
675            .column(1)
676            .as_any()
677            .downcast_ref::<UInt64Array>()
678            .unwrap();
679        assert_eq!(region_id1.as_u64(), region_ids.value(0));
680        assert_eq!(region_id2.as_u64(), region_ids.value(1));
681
682        let table_ids = batch
683            .column(2)
684            .as_any()
685            .downcast_ref::<UInt32Array>()
686            .unwrap();
687        assert_eq!(table_id1, table_ids.value(0));
688        assert_eq!(table_id2, table_ids.value(1));
689
690        let region_numbers = batch
691            .column(3)
692            .as_any()
693            .downcast_ref::<UInt32Array>()
694            .unwrap();
695        assert_eq!(region_number1, region_numbers.value(0));
696        assert_eq!(region_number2, region_numbers.value(1));
697
698        let region_groups = batch
699            .column(4)
700            .as_any()
701            .downcast_ref::<UInt8Array>()
702            .unwrap();
703        assert_eq!(region_group1, region_groups.value(0));
704        assert_eq!(region_group2, region_groups.value(1));
705
706        let region_sequences = batch
707            .column(5)
708            .as_any()
709            .downcast_ref::<UInt32Array>()
710            .unwrap();
711        assert_eq!(region_seq1, region_sequences.value(0));
712        assert_eq!(region_seq2, region_sequences.value(1));
713
714        let file_ids = batch
715            .column(6)
716            .as_any()
717            .downcast_ref::<StringArray>()
718            .unwrap();
719        assert_eq!("f1", file_ids.value(0));
720        assert_eq!("f2", file_ids.value(1));
721
722        let index_versions = batch
723            .column(7)
724            .as_any()
725            .downcast_ref::<UInt64Array>()
726            .unwrap();
727        assert_eq!(0, index_versions.value(0));
728        assert_eq!(1, index_versions.value(1));
729
730        let levels = batch
731            .column(8)
732            .as_any()
733            .downcast_ref::<UInt8Array>()
734            .unwrap();
735        assert_eq!(1, levels.value(0));
736        assert_eq!(3, levels.value(1));
737
738        let file_paths = batch
739            .column(9)
740            .as_any()
741            .downcast_ref::<StringArray>()
742            .unwrap();
743        assert_eq!("/p1", file_paths.value(0));
744        assert_eq!("/p2", file_paths.value(1));
745
746        let file_sizes = batch
747            .column(10)
748            .as_any()
749            .downcast_ref::<UInt64Array>()
750            .unwrap();
751        assert_eq!(100, file_sizes.value(0));
752        assert_eq!(200, file_sizes.value(1));
753
754        let index_file_paths = batch
755            .column(11)
756            .as_any()
757            .downcast_ref::<StringArray>()
758            .unwrap();
759        assert!(index_file_paths.is_null(0));
760        assert_eq!("idx", index_file_paths.value(1));
761
762        let index_file_sizes = batch
763            .column(12)
764            .as_any()
765            .downcast_ref::<UInt64Array>()
766            .unwrap();
767        assert!(index_file_sizes.is_null(0));
768        assert_eq!(11, index_file_sizes.value(1));
769
770        let num_rows = batch
771            .column(13)
772            .as_any()
773            .downcast_ref::<UInt64Array>()
774            .unwrap();
775        assert_eq!(10, num_rows.value(0));
776        assert_eq!(20, num_rows.value(1));
777
778        let num_row_groups = batch
779            .column(14)
780            .as_any()
781            .downcast_ref::<UInt64Array>()
782            .unwrap();
783        assert_eq!(2, num_row_groups.value(0));
784        assert_eq!(4, num_row_groups.value(1));
785
786        let num_series = batch
787            .column(15)
788            .as_any()
789            .downcast_ref::<UInt64Array>()
790            .unwrap();
791        assert_eq!(5, num_series.value(0));
792        assert!(num_series.is_null(1));
793
794        let min_ts = batch
795            .column(16)
796            .as_any()
797            .downcast_ref::<TimestampNanosecondArray>()
798            .unwrap();
799        assert_eq!(1_000_000_000, min_ts.value(0));
800        assert_eq!(5, min_ts.value(1));
801
802        let max_ts = batch
803            .column(17)
804            .as_any()
805            .downcast_ref::<TimestampNanosecondArray>()
806            .unwrap();
807        assert_eq!(2_000_000_000, max_ts.value(0));
808        assert_eq!(2_000_000, max_ts.value(1));
809
810        let sequences = batch
811            .column(18)
812            .as_any()
813            .downcast_ref::<UInt64Array>()
814            .unwrap();
815        assert!(sequences.is_null(0));
816        assert_eq!(9, sequences.value(1));
817
818        let origin_region_ids = batch
819            .column(19)
820            .as_any()
821            .downcast_ref::<UInt64Array>()
822            .unwrap();
823        assert_eq!(region_id1.as_u64(), origin_region_ids.value(0));
824        assert_eq!(region_id2.as_u64(), origin_region_ids.value(1));
825
826        let node_ids = batch
827            .column(20)
828            .as_any()
829            .downcast_ref::<UInt64Array>()
830            .unwrap();
831        assert_eq!(1, node_ids.value(0));
832        assert!(node_ids.is_null(1));
833
834        let visible = batch
835            .column(21)
836            .as_any()
837            .downcast_ref::<BooleanArray>()
838            .unwrap();
839        assert!(!visible.value(0));
840        assert!(visible.value(1));
841
842        let primary_key_min = batch
843            .column(22)
844            .as_any()
845            .downcast_ref::<BinaryArray>()
846            .unwrap();
847        assert_eq!(b"aaa", primary_key_min.value(0));
848        assert!(primary_key_min.is_null(1));
849
850        let primary_key_max = batch
851            .column(23)
852            .as_any()
853            .downcast_ref::<BinaryArray>()
854            .unwrap();
855        assert_eq!(b"zzz", primary_key_max.value(0));
856        assert!(primary_key_max.is_null(1));
857
858        let max_row_group_uncompressed_sizes = batch
859            .column(24)
860            .as_any()
861            .downcast_ref::<UInt64Array>()
862            .unwrap();
863        assert_eq!(80, max_row_group_uncompressed_sizes.value(0));
864        assert_eq!(160, max_row_group_uncompressed_sizes.value(1));
865
866        let partition_exprs = batch
867            .column(25)
868            .as_any()
869            .downcast_ref::<StringArray>()
870            .unwrap();
871        assert_eq!("a < 10", partition_exprs.value(0));
872        assert!(partition_exprs.is_null(1));
873    }
874
875    #[test]
876    fn test_sst_entry_storage_to_record_batch() {
877        let entries = vec![
878            StorageSstEntry {
879                file_path: "/s1".to_string(),
880                file_size: None,
881                last_modified_ms: None,
882                node_id: Some(1),
883            },
884            StorageSstEntry {
885                file_path: "/s2".to_string(),
886                file_size: Some(123),
887                last_modified_ms: Some(Timestamp::new_millisecond(456)),
888                node_id: None,
889            },
890        ];
891
892        let schema = StorageSstEntry::schema();
893        let batch = StorageSstEntry::to_record_batch(&entries).unwrap();
894
895        assert_eq!(schema.arrow_schema().fields().len(), batch.num_columns());
896        assert_eq!(2, batch.num_rows());
897
898        let file_paths = batch
899            .column(0)
900            .as_any()
901            .downcast_ref::<StringArray>()
902            .unwrap();
903        assert_eq!("/s1", file_paths.value(0));
904        assert_eq!("/s2", file_paths.value(1));
905
906        let file_sizes = batch
907            .column(1)
908            .as_any()
909            .downcast_ref::<UInt64Array>()
910            .unwrap();
911        assert!(file_sizes.is_null(0));
912        assert_eq!(123, file_sizes.value(1));
913
914        let last_modified = batch
915            .column(2)
916            .as_any()
917            .downcast_ref::<TimestampMillisecondArray>()
918            .unwrap();
919        assert!(last_modified.is_null(0));
920        assert_eq!(456, last_modified.value(1));
921
922        let node_ids = batch
923            .column(3)
924            .as_any()
925            .downcast_ref::<UInt64Array>()
926            .unwrap();
927        assert_eq!(1, node_ids.value(0));
928        assert!(node_ids.is_null(1));
929    }
930
931    #[test]
932    fn test_puffin_index_meta_to_record_batch() {
933        let entries = vec![
934            PuffinIndexMetaEntry {
935                table_dir: "table1".to_string(),
936                index_file_path: "index1".to_string(),
937                region_id: RegionId::with_group_and_seq(10, 0, 20),
938                table_id: 10,
939                region_number: 20,
940                region_group: 0,
941                region_sequence: 20,
942                file_id: "file1".to_string(),
943                index_file_size: Some(1024),
944                index_type: "bloom_filter".to_string(),
945                target_type: "column".to_string(),
946                target_key: "1".to_string(),
947                target_json: "{\"column\":1}".to_string(),
948                blob_size: 256,
949                meta_json: Some("{\"bloom\":{}}".to_string()),
950                node_id: Some(42),
951            },
952            PuffinIndexMetaEntry {
953                table_dir: "table2".to_string(),
954                index_file_path: "index2".to_string(),
955                region_id: RegionId::with_group_and_seq(11, 0, 21),
956                table_id: 11,
957                region_number: 21,
958                region_group: 0,
959                region_sequence: 21,
960                file_id: "file2".to_string(),
961                index_file_size: None,
962                index_type: "inverted".to_string(),
963                target_type: "unknown".to_string(),
964                target_key: "legacy".to_string(),
965                target_json: "{}".to_string(),
966                blob_size: 0,
967                meta_json: None,
968                node_id: None,
969            },
970        ];
971
972        let schema = PuffinIndexMetaEntry::schema();
973        let batch = PuffinIndexMetaEntry::to_record_batch(&entries).unwrap();
974
975        assert_eq!(schema.arrow_schema().fields().len(), batch.num_columns());
976        assert_eq!(2, batch.num_rows());
977
978        let table_dirs = batch
979            .column(0)
980            .as_any()
981            .downcast_ref::<StringArray>()
982            .unwrap();
983        assert_eq!("table1", table_dirs.value(0));
984        assert_eq!("table2", table_dirs.value(1));
985
986        let index_file_paths = batch
987            .column(1)
988            .as_any()
989            .downcast_ref::<StringArray>()
990            .unwrap();
991        assert_eq!("index1", index_file_paths.value(0));
992        assert_eq!("index2", index_file_paths.value(1));
993
994        let region_ids = batch
995            .column(2)
996            .as_any()
997            .downcast_ref::<UInt64Array>()
998            .unwrap();
999        assert_eq!(
1000            RegionId::with_group_and_seq(10, 0, 20).as_u64(),
1001            region_ids.value(0)
1002        );
1003        assert_eq!(
1004            RegionId::with_group_and_seq(11, 0, 21).as_u64(),
1005            region_ids.value(1)
1006        );
1007
1008        let table_ids = batch
1009            .column(3)
1010            .as_any()
1011            .downcast_ref::<UInt32Array>()
1012            .unwrap();
1013        assert_eq!(10, table_ids.value(0));
1014        assert_eq!(11, table_ids.value(1));
1015
1016        let region_numbers = batch
1017            .column(4)
1018            .as_any()
1019            .downcast_ref::<UInt32Array>()
1020            .unwrap();
1021        assert_eq!(20, region_numbers.value(0));
1022        assert_eq!(21, region_numbers.value(1));
1023
1024        let region_groups = batch
1025            .column(5)
1026            .as_any()
1027            .downcast_ref::<UInt8Array>()
1028            .unwrap();
1029        assert_eq!(0, region_groups.value(0));
1030        assert_eq!(0, region_groups.value(1));
1031
1032        let region_sequences = batch
1033            .column(6)
1034            .as_any()
1035            .downcast_ref::<UInt32Array>()
1036            .unwrap();
1037        assert_eq!(20, region_sequences.value(0));
1038        assert_eq!(21, region_sequences.value(1));
1039
1040        let file_ids = batch
1041            .column(7)
1042            .as_any()
1043            .downcast_ref::<StringArray>()
1044            .unwrap();
1045        assert_eq!("file1", file_ids.value(0));
1046        assert_eq!("file2", file_ids.value(1));
1047
1048        let index_file_sizes = batch
1049            .column(8)
1050            .as_any()
1051            .downcast_ref::<UInt64Array>()
1052            .unwrap();
1053        assert_eq!(1024, index_file_sizes.value(0));
1054        assert!(index_file_sizes.is_null(1));
1055
1056        let index_types = batch
1057            .column(9)
1058            .as_any()
1059            .downcast_ref::<StringArray>()
1060            .unwrap();
1061        assert_eq!("bloom_filter", index_types.value(0));
1062        assert_eq!("inverted", index_types.value(1));
1063
1064        let target_types = batch
1065            .column(10)
1066            .as_any()
1067            .downcast_ref::<StringArray>()
1068            .unwrap();
1069        assert_eq!("column", target_types.value(0));
1070        assert_eq!("unknown", target_types.value(1));
1071
1072        let target_keys = batch
1073            .column(11)
1074            .as_any()
1075            .downcast_ref::<StringArray>()
1076            .unwrap();
1077        assert_eq!("1", target_keys.value(0));
1078        assert_eq!("legacy", target_keys.value(1));
1079
1080        let target_json = batch
1081            .column(12)
1082            .as_any()
1083            .downcast_ref::<StringArray>()
1084            .unwrap();
1085        assert_eq!("{\"column\":1}", target_json.value(0));
1086        assert_eq!("{}", target_json.value(1));
1087
1088        let blob_sizes = batch
1089            .column(13)
1090            .as_any()
1091            .downcast_ref::<UInt64Array>()
1092            .unwrap();
1093        assert_eq!(256, blob_sizes.value(0));
1094        assert_eq!(0, blob_sizes.value(1));
1095
1096        let meta_jsons = batch
1097            .column(14)
1098            .as_any()
1099            .downcast_ref::<StringArray>()
1100            .unwrap();
1101        assert_eq!("{\"bloom\":{}}", meta_jsons.value(0));
1102        assert!(meta_jsons.is_null(1));
1103
1104        let node_ids = batch
1105            .column(15)
1106            .as_any()
1107            .downcast_ref::<UInt64Array>()
1108            .unwrap();
1109        assert_eq!(42, node_ids.value(0));
1110        assert!(node_ids.is_null(1));
1111    }
1112
1113    #[test]
1114    fn test_manifest_build_plan() {
1115        let request = ScanRequest {
1116            projection: Some(vec![0, 1]),
1117            filters: vec![col("table_id").gt(lit(0u32))],
1118            limit: Some(5),
1119            ..Default::default()
1120        };
1121
1122        let plan = ManifestSstEntry::build_plan(request).unwrap();
1123
1124        // The hidden table_id predicate must run before projection and limit.
1125        let LogicalPlan::Limit(limit) = &plan else {
1126            panic!("expected limit")
1127        };
1128        let LogicalPlan::Projection(projection) = limit.input.as_ref() else {
1129            panic!("expected projection")
1130        };
1131        let LogicalPlan::Filter(filter) = projection.input.as_ref() else {
1132            panic!("expected filter")
1133        };
1134        let LogicalPlan::TableScan(scan) = filter.input.as_ref() else {
1135            panic!("expected table scan")
1136        };
1137        assert_eq!(
1138            scan.table_name,
1139            TableReference::bare(ManifestSstEntry::reserved_table_name_for_inspection())
1140        );
1141        assert_eq!(scan.projection, None);
1142        assert_eq!(
1143            filter.predicate,
1144            Expr::Column(datafusion_common::Column::new(
1145                Some(scan.table_name.clone()),
1146                "table_id",
1147            ))
1148            .gt(lit(0u32))
1149        );
1150
1151        let fields = plan.schema().fields();
1152        assert_eq!(fields.len(), 2);
1153        assert_eq!(fields[0].name(), "table_dir");
1154        assert_eq!(fields[1].name(), "region_id");
1155    }
1156
1157    #[test]
1158    fn test_storage_build_plan() {
1159        let projection = Some(vec![0, 2]);
1160        let request = ScanRequest {
1161            projection,
1162            filters: vec![binary_expr(col("file_path"), Operator::Eq, lit("/a"))],
1163            limit: Some(1),
1164            ..Default::default()
1165        };
1166
1167        let plan = StorageSstEntry::build_plan(request).unwrap();
1168        let (scan, has_filter, has_limit) = extract_scan(&plan);
1169        assert!(has_filter);
1170        assert!(has_limit);
1171        assert_eq!(
1172            scan.table_name,
1173            TableReference::bare(StorageSstEntry::reserved_table_name_for_inspection())
1174        );
1175        assert_eq!(scan.projection, None);
1176
1177        let fields = plan.schema().fields();
1178        assert_eq!(fields.len(), 2);
1179        assert_eq!(fields[0].name(), "file_path");
1180        assert_eq!(fields[1].name(), "last_modified_ms");
1181    }
1182
1183    // Helper to reach TableScan and detect presence of Filter/Limit in plan
1184    fn extract_scan(plan: &LogicalPlan) -> (&datafusion_expr::logical_plan::TableScan, bool, bool) {
1185        use datafusion_expr::logical_plan::Limit;
1186
1187        match plan {
1188            LogicalPlan::Filter(f) => {
1189                let (scan, _, has_limit) = extract_scan(&f.input);
1190                (scan, true, has_limit)
1191            }
1192            LogicalPlan::Limit(Limit { input, .. }) => {
1193                let (scan, has_filter, _) = extract_scan(input);
1194                (scan, has_filter, true)
1195            }
1196            LogicalPlan::Projection(projection) => extract_scan(&projection.input),
1197            LogicalPlan::TableScan(scan) => (scan, false, false),
1198            other => panic!("unexpected plan: {other:?}"),
1199        }
1200    }
1201}