1use std::sync::Arc;
16
17use bytes::Bytes;
18use common_recordbatch::DfRecordBatch;
19use common_time::Timestamp;
20use common_time::timestamp::TimeUnit;
21use datafusion_common::DataFusionError;
22use datafusion_expr::LogicalPlan;
23use datatypes::arrow::array::{
24 ArrayRef, BinaryArray, BooleanArray, TimestampMillisecondArray, TimestampNanosecondArray,
25 UInt8Array, UInt32Array, UInt64Array,
26};
27use datatypes::arrow::datatypes::SchemaRef as ArrowSchemaRef;
28use datatypes::arrow::error::ArrowError;
29use datatypes::arrow_array::StringArray;
30use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
31use serde::{Deserialize, Serialize};
32
33use crate::inspect::{self, ProjectedBatchBuilder};
34use crate::storage::{RegionGroup, RegionId, RegionNumber, RegionSeq, ScanRequest, TableId};
35
36pub const TABLE_DIR: &str = "table_dir";
38pub const REGION_ID: &str = "region_id";
39pub const TABLE_ID: &str = "table_id";
40pub const REGION_NUMBER: &str = "region_number";
41pub const REGION_GROUP: &str = "region_group";
42pub const REGION_SEQUENCE: &str = "region_sequence";
43pub const FILE_ID: &str = "file_id";
44pub const INDEX_VERSION: &str = "index_version";
45pub const LEVEL: &str = "level";
46pub const FILE_PATH: &str = "file_path";
47pub const FILE_SIZE: &str = "file_size";
48pub const INDEX_FILE_PATH: &str = "index_file_path";
49pub const INDEX_FILE_SIZE: &str = "index_file_size";
50pub const NUM_ROWS: &str = "num_rows";
51pub const NUM_ROW_GROUPS: &str = "num_row_groups";
52pub const NUM_SERIES: &str = "num_series";
53pub const MIN_TS: &str = "min_ts";
54pub const MAX_TS: &str = "max_ts";
55pub const SEQUENCE: &str = "sequence";
56pub const ORIGIN_REGION_ID: &str = "origin_region_id";
57pub const NODE_ID: &str = "node_id";
58pub const VISIBLE: &str = "visible";
59pub const PRIMARY_KEY_MIN: &str = "primary_key_min";
60pub const PRIMARY_KEY_MAX: &str = "primary_key_max";
61pub const MAX_ROW_GROUP_UNCOMPRESSED_SIZE: &str = "max_row_group_uncompressed_size";
62pub const PARTITION_EXPR: &str = "partition_expr";
63
64pub const PUFFIN_INDEX_TYPE_BLOOM_FILTER: &str = "bloom_filter";
66pub const PUFFIN_INDEX_TYPE_FULLTEXT_BLOOM: &str = "fulltext_bloom";
68pub const PUFFIN_INDEX_TYPE_FULLTEXT_TANTIVY: &str = "fulltext_tantivy";
70pub const PUFFIN_INDEX_TYPE_INVERTED: &str = "inverted";
72
73#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
75pub struct ManifestSstEntry {
76 pub table_dir: String,
78 pub region_id: RegionId,
80 pub table_id: TableId,
82 pub region_number: RegionNumber,
84 pub region_group: RegionGroup,
86 pub region_sequence: RegionSeq,
88 pub file_id: String,
90 pub index_version: u64,
92 pub level: u8,
94 pub file_path: String,
96 pub file_size: u64,
98 pub max_row_group_uncompressed_size: u64,
100 pub index_file_path: Option<String>,
102 pub index_file_size: Option<u64>,
104 pub num_rows: u64,
106 pub num_row_groups: u64,
108 pub num_series: Option<u64>,
110 pub min_ts: Timestamp,
112 pub max_ts: Timestamp,
114 pub sequence: Option<u64>,
116 pub partition_expr: Option<String>,
118 pub origin_region_id: RegionId,
120 pub node_id: Option<u64>,
122 pub visible: bool,
124 pub primary_key_min: Option<Bytes>,
126 pub primary_key_max: Option<Bytes>,
128}
129
130impl ManifestSstEntry {
131 pub fn schema() -> SchemaRef {
133 use datatypes::prelude::ConcreteDataType as Ty;
134 Arc::new(Schema::new(vec![
135 ColumnSchema::new(TABLE_DIR, Ty::string_datatype(), false),
136 ColumnSchema::new(REGION_ID, Ty::uint64_datatype(), false),
137 ColumnSchema::new(TABLE_ID, Ty::uint32_datatype(), false),
138 ColumnSchema::new(REGION_NUMBER, Ty::uint32_datatype(), false),
139 ColumnSchema::new(REGION_GROUP, Ty::uint8_datatype(), false),
140 ColumnSchema::new(REGION_SEQUENCE, Ty::uint32_datatype(), false),
141 ColumnSchema::new(FILE_ID, Ty::string_datatype(), false),
142 ColumnSchema::new(INDEX_VERSION, Ty::uint64_datatype(), false),
143 ColumnSchema::new(LEVEL, Ty::uint8_datatype(), false),
144 ColumnSchema::new(FILE_PATH, Ty::string_datatype(), false),
145 ColumnSchema::new(FILE_SIZE, Ty::uint64_datatype(), false),
146 ColumnSchema::new(INDEX_FILE_PATH, Ty::string_datatype(), true),
147 ColumnSchema::new(INDEX_FILE_SIZE, Ty::uint64_datatype(), true),
148 ColumnSchema::new(NUM_ROWS, Ty::uint64_datatype(), false),
149 ColumnSchema::new(NUM_ROW_GROUPS, Ty::uint64_datatype(), false),
150 ColumnSchema::new(NUM_SERIES, Ty::uint64_datatype(), true),
151 ColumnSchema::new(MIN_TS, Ty::timestamp_nanosecond_datatype(), true),
152 ColumnSchema::new(MAX_TS, Ty::timestamp_nanosecond_datatype(), true),
153 ColumnSchema::new(SEQUENCE, Ty::uint64_datatype(), true),
154 ColumnSchema::new(ORIGIN_REGION_ID, Ty::uint64_datatype(), false),
155 ColumnSchema::new(NODE_ID, Ty::uint64_datatype(), true),
156 ColumnSchema::new(VISIBLE, Ty::boolean_datatype(), false),
157 ColumnSchema::new(PRIMARY_KEY_MIN, Ty::binary_datatype(), true),
158 ColumnSchema::new(PRIMARY_KEY_MAX, Ty::binary_datatype(), true),
159 ColumnSchema::new(
160 MAX_ROW_GROUP_UNCOMPRESSED_SIZE,
161 Ty::uint64_datatype(),
162 false,
163 ),
164 ColumnSchema::new(PARTITION_EXPR, Ty::string_datatype(), true),
165 ]))
166 }
167
168 pub fn to_record_batch(entries: &[Self]) -> std::result::Result<DfRecordBatch, ArrowError> {
170 Self::to_projected_record_batch(entries, None)
171 }
172
173 fn to_projected_record_batch(
175 entries: &[Self],
176 projection: Option<&[usize]>,
177 ) -> std::result::Result<DfRecordBatch, ArrowError> {
178 let schema = Self::schema().arrow_schema().clone();
179 let schema = match projection {
180 Some(projection) => Arc::new(schema.project(projection)?),
181 None => schema,
182 };
183 ManifestSstBatchBuilder::new(schema).build(entries)
184 }
185
186 pub fn reserved_table_name_for_inspection() -> &'static str {
192 "__inspect/__mito/__sst_manifest"
193 }
194
195 pub fn build_plan(scan_request: ScanRequest) -> Result<LogicalPlan, DataFusionError> {
197 inspect::build_plan(
198 scan_request,
199 Self::reserved_table_name_for_inspection(),
200 Self::schema(),
201 )
202 }
203}
204
205pub struct ManifestSstBatchBuilder {
208 projected: ProjectedBatchBuilder,
209}
210
211impl ManifestSstBatchBuilder {
212 pub fn new(schema: ArrowSchemaRef) -> Self {
214 Self {
215 projected: ProjectedBatchBuilder::new(schema),
216 }
217 }
218
219 pub fn build(
221 &self,
222 entries: &[ManifestSstEntry],
223 ) -> std::result::Result<DfRecordBatch, ArrowError> {
224 self.projected.build(entries.len(), |name| {
225 let array: ArrayRef = match name {
226 TABLE_DIR => Arc::new(StringArray::from_iter_values(
227 entries.iter().map(|e| e.table_dir.as_str()),
228 )),
229 REGION_ID => Arc::new(UInt64Array::from_iter_values(
230 entries.iter().map(|e| e.region_id.as_u64()),
231 )),
232 TABLE_ID => Arc::new(UInt32Array::from_iter_values(
233 entries.iter().map(|e| e.table_id),
234 )),
235 REGION_NUMBER => Arc::new(UInt32Array::from_iter_values(
236 entries.iter().map(|e| e.region_number),
237 )),
238 REGION_GROUP => Arc::new(UInt8Array::from_iter_values(
239 entries.iter().map(|e| e.region_group),
240 )),
241 REGION_SEQUENCE => Arc::new(UInt32Array::from_iter_values(
242 entries.iter().map(|e| e.region_sequence),
243 )),
244 FILE_ID => Arc::new(StringArray::from_iter_values(
245 entries.iter().map(|e| e.file_id.as_str()),
246 )),
247 INDEX_VERSION => Arc::new(UInt64Array::from_iter(
248 entries.iter().map(|e| e.index_version),
249 )),
250 LEVEL => Arc::new(UInt8Array::from_iter_values(
251 entries.iter().map(|e| e.level),
252 )),
253 FILE_PATH => Arc::new(StringArray::from_iter_values(
254 entries.iter().map(|e| e.file_path.as_str()),
255 )),
256 FILE_SIZE => Arc::new(UInt64Array::from_iter_values(
257 entries.iter().map(|e| e.file_size),
258 )),
259 INDEX_FILE_PATH => Arc::new(StringArray::from_iter(
260 entries.iter().map(|e| e.index_file_path.as_ref()),
261 )),
262 INDEX_FILE_SIZE => Arc::new(UInt64Array::from_iter(
263 entries.iter().map(|e| e.index_file_size),
264 )),
265 NUM_ROWS => Arc::new(UInt64Array::from_iter_values(
266 entries.iter().map(|e| e.num_rows),
267 )),
268 NUM_ROW_GROUPS => Arc::new(UInt64Array::from_iter_values(
269 entries.iter().map(|e| e.num_row_groups),
270 )),
271 NUM_SERIES => {
272 Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.num_series)))
273 }
274 MIN_TS => Arc::new(TimestampNanosecondArray::from_iter(entries.iter().map(
275 |e| {
276 e.min_ts
277 .convert_to(TimeUnit::Nanosecond)
278 .map(|ts| ts.value())
279 },
280 ))),
281 MAX_TS => Arc::new(TimestampNanosecondArray::from_iter(entries.iter().map(
282 |e| {
283 e.max_ts
284 .convert_to(TimeUnit::Nanosecond)
285 .map(|ts| ts.value())
286 },
287 ))),
288 SEQUENCE => Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.sequence))),
289 ORIGIN_REGION_ID => Arc::new(UInt64Array::from_iter_values(
290 entries.iter().map(|e| e.origin_region_id.as_u64()),
291 )),
292 NODE_ID => Arc::new(UInt64Array::from_iter(entries.iter().map(|e| e.node_id))),
293 VISIBLE => Arc::new(BooleanArray::from_iter(
294 entries.iter().map(|e| Some(e.visible)),
295 )),
296 PRIMARY_KEY_MIN => Arc::new(BinaryArray::from_iter(
297 entries.iter().map(|e| e.primary_key_min.as_deref()),
298 )),
299 PRIMARY_KEY_MAX => Arc::new(BinaryArray::from_iter(
300 entries.iter().map(|e| e.primary_key_max.as_deref()),
301 )),
302 MAX_ROW_GROUP_UNCOMPRESSED_SIZE => Arc::new(UInt64Array::from_iter_values(
303 entries.iter().map(|e| e.max_row_group_uncompressed_size),
304 )),
305 PARTITION_EXPR => Arc::new(StringArray::from_iter(
306 entries.iter().map(|e| e.partition_expr.as_ref()),
307 )),
308 name => {
309 return Err(ArrowError::SchemaError(format!(
310 "Unknown manifest SST column: {name}"
311 )));
312 }
313 };
314 Ok(array)
315 })
316 }
317}
318
319#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
321pub struct StorageSstEntry {
322 pub file_path: String,
324 pub file_size: Option<u64>,
326 pub last_modified_ms: Option<Timestamp>,
328 pub node_id: Option<u64>,
330}
331
332impl StorageSstEntry {
333 pub fn schema() -> SchemaRef {
335 use datatypes::prelude::ConcreteDataType as Ty;
336 Arc::new(Schema::new(vec![
337 ColumnSchema::new("file_path", Ty::string_datatype(), false),
338 ColumnSchema::new("file_size", Ty::uint64_datatype(), true),
339 ColumnSchema::new(
340 "last_modified_ms",
341 Ty::timestamp_millisecond_datatype(),
342 true,
343 ),
344 ColumnSchema::new("node_id", Ty::uint64_datatype(), true),
345 ]))
346 }
347
348 pub fn to_record_batch(entries: &[Self]) -> std::result::Result<DfRecordBatch, ArrowError> {
350 let schema = Self::schema();
351 let file_paths = entries.iter().map(|e| e.file_path.as_str());
352 let file_sizes = entries.iter().map(|e| e.file_size);
353 let last_modified_ms = entries.iter().map(|e| {
354 e.last_modified_ms
355 .and_then(|ts| ts.convert_to(TimeUnit::Millisecond).map(|ts| ts.value()))
356 });
357 let node_ids = entries.iter().map(|e| e.node_id);
358
359 let columns: Vec<ArrayRef> = vec![
360 Arc::new(StringArray::from_iter_values(file_paths)),
361 Arc::new(UInt64Array::from_iter(file_sizes)),
362 Arc::new(TimestampMillisecondArray::from_iter(last_modified_ms)),
363 Arc::new(UInt64Array::from_iter(node_ids)),
364 ];
365
366 DfRecordBatch::try_new(schema.arrow_schema().clone(), columns)
367 }
368
369 pub fn reserved_table_name_for_inspection() -> &'static str {
375 "__inspect/__mito/__sst_storage"
376 }
377
378 pub fn build_plan(scan_request: ScanRequest) -> Result<LogicalPlan, DataFusionError> {
380 inspect::build_plan(
381 scan_request,
382 Self::reserved_table_name_for_inspection(),
383 Self::schema(),
384 )
385 }
386}
387
388#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
390pub struct PuffinIndexMetaEntry {
391 pub table_dir: String,
393 pub index_file_path: String,
395 pub region_id: RegionId,
397 pub table_id: TableId,
399 pub region_number: RegionNumber,
401 pub region_group: RegionGroup,
403 pub region_sequence: RegionSeq,
405 pub file_id: String,
407 pub index_file_size: Option<u64>,
409 pub index_type: String,
411 pub target_type: String,
413 pub target_key: String,
415 pub target_json: String,
417 pub blob_size: u64,
419 pub meta_json: Option<String>,
421 pub node_id: Option<u64>,
423}
424
425impl PuffinIndexMetaEntry {
426 pub fn schema() -> SchemaRef {
428 use datatypes::prelude::ConcreteDataType as Ty;
429 Arc::new(Schema::new(vec![
430 ColumnSchema::new("table_dir", Ty::string_datatype(), false),
431 ColumnSchema::new("index_file_path", Ty::string_datatype(), false),
432 ColumnSchema::new("region_id", Ty::uint64_datatype(), false),
433 ColumnSchema::new("table_id", Ty::uint32_datatype(), false),
434 ColumnSchema::new("region_number", Ty::uint32_datatype(), false),
435 ColumnSchema::new("region_group", Ty::uint8_datatype(), false),
436 ColumnSchema::new("region_sequence", Ty::uint32_datatype(), false),
437 ColumnSchema::new("file_id", Ty::string_datatype(), false),
438 ColumnSchema::new("index_file_size", Ty::uint64_datatype(), true),
439 ColumnSchema::new("index_type", Ty::string_datatype(), false),
440 ColumnSchema::new("target_type", Ty::string_datatype(), false),
441 ColumnSchema::new("target_key", Ty::string_datatype(), false),
442 ColumnSchema::new("target_json", Ty::string_datatype(), false),
443 ColumnSchema::new("blob_size", Ty::uint64_datatype(), false),
444 ColumnSchema::new("meta_json", Ty::string_datatype(), true),
445 ColumnSchema::new("node_id", Ty::uint64_datatype(), true),
446 ]))
447 }
448
449 pub fn to_record_batch(entries: &[Self]) -> std::result::Result<DfRecordBatch, ArrowError> {
451 let schema = Self::schema();
452 let table_dirs = entries.iter().map(|e| e.table_dir.as_str());
453 let index_file_paths = entries.iter().map(|e| e.index_file_path.as_str());
454 let region_ids = entries.iter().map(|e| e.region_id.as_u64());
455 let table_ids = entries.iter().map(|e| e.table_id);
456 let region_numbers = entries.iter().map(|e| e.region_number);
457 let region_groups = entries.iter().map(|e| e.region_group);
458 let region_sequences = entries.iter().map(|e| e.region_sequence);
459 let file_ids = entries.iter().map(|e| e.file_id.as_str());
460 let index_file_sizes = entries.iter().map(|e| e.index_file_size);
461 let index_types = entries.iter().map(|e| e.index_type.as_str());
462 let target_types = entries.iter().map(|e| e.target_type.as_str());
463 let target_keys = entries.iter().map(|e| e.target_key.as_str());
464 let target_jsons = entries.iter().map(|e| e.target_json.as_str());
465 let blob_sizes = entries.iter().map(|e| e.blob_size);
466 let meta_jsons = entries.iter().map(|e| e.meta_json.as_deref());
467 let node_ids = entries.iter().map(|e| e.node_id);
468
469 let columns: Vec<ArrayRef> = vec![
470 Arc::new(StringArray::from_iter_values(table_dirs)),
471 Arc::new(StringArray::from_iter_values(index_file_paths)),
472 Arc::new(UInt64Array::from_iter_values(region_ids)),
473 Arc::new(UInt32Array::from_iter_values(table_ids)),
474 Arc::new(UInt32Array::from_iter_values(region_numbers)),
475 Arc::new(UInt8Array::from_iter_values(region_groups)),
476 Arc::new(UInt32Array::from_iter_values(region_sequences)),
477 Arc::new(StringArray::from_iter_values(file_ids)),
478 Arc::new(UInt64Array::from_iter(index_file_sizes)),
479 Arc::new(StringArray::from_iter_values(index_types)),
480 Arc::new(StringArray::from_iter_values(target_types)),
481 Arc::new(StringArray::from_iter_values(target_keys)),
482 Arc::new(StringArray::from_iter_values(target_jsons)),
483 Arc::new(UInt64Array::from_iter_values(blob_sizes)),
484 Arc::new(StringArray::from_iter(meta_jsons)),
485 Arc::new(UInt64Array::from_iter(node_ids)),
486 ];
487
488 DfRecordBatch::try_new(schema.arrow_schema().clone(), columns)
489 }
490
491 pub fn reserved_table_name_for_inspection() -> &'static str {
493 "__inspect/__mito/__puffin_index_meta"
494 }
495
496 pub fn build_plan(scan_request: ScanRequest) -> Result<LogicalPlan, DataFusionError> {
498 inspect::build_plan(
499 scan_request,
500 Self::reserved_table_name_for_inspection(),
501 Self::schema(),
502 )
503 }
504}
505
506#[cfg(test)]
507mod tests {
508 use datafusion_common::TableReference;
509 use datafusion_expr::{Expr, LogicalPlan, Operator, binary_expr, col, lit};
510 use datatypes::arrow::array::{
511 Array, BinaryArray, TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array,
512 UInt32Array, UInt64Array,
513 };
514 use datatypes::arrow_array::StringArray;
515
516 use super::*;
517
518 #[test]
519 fn test_sst_entry_manifest_to_record_batch() {
520 let table_id1: TableId = 1;
522 let region_group1: RegionGroup = 2;
523 let region_seq1: RegionSeq = 3;
524 let region_number1: RegionNumber = ((region_group1 as u32) << 24) | region_seq1;
525 let region_id1 = RegionId::with_group_and_seq(table_id1, region_group1, region_seq1);
526
527 let table_id2: TableId = 5;
528 let region_group2: RegionGroup = 1;
529 let region_seq2: RegionSeq = 42;
530 let region_number2: RegionNumber = ((region_group2 as u32) << 24) | region_seq2;
531 let region_id2 = RegionId::with_group_and_seq(table_id2, region_group2, region_seq2);
532
533 let entries = vec![
534 ManifestSstEntry {
535 table_dir: "tdir1".to_string(),
536 region_id: region_id1,
537 table_id: table_id1,
538 region_number: region_number1,
539 region_group: region_group1,
540 region_sequence: region_seq1,
541 file_id: "f1".to_string(),
542 index_version: 0,
543 level: 1,
544 file_path: "/p1".to_string(),
545 file_size: 100,
546 max_row_group_uncompressed_size: 80,
547 index_file_path: None,
548 index_file_size: None,
549 num_rows: 10,
550 num_row_groups: 2,
551 num_series: Some(5),
552 min_ts: Timestamp::new_millisecond(1000), max_ts: Timestamp::new_second(2), sequence: None,
555 partition_expr: Some("a < 10".to_string()),
556 origin_region_id: region_id1,
557 node_id: Some(1),
558 visible: false,
559 primary_key_min: Some(Bytes::from_static(b"aaa")),
560 primary_key_max: Some(Bytes::from_static(b"zzz")),
561 },
562 ManifestSstEntry {
563 table_dir: "tdir2".to_string(),
564 region_id: region_id2,
565 table_id: table_id2,
566 region_number: region_number2,
567 region_group: region_group2,
568 region_sequence: region_seq2,
569 file_id: "f2".to_string(),
570 index_version: 1,
571 level: 3,
572 file_path: "/p2".to_string(),
573 file_size: 200,
574 max_row_group_uncompressed_size: 160,
575 index_file_path: Some("idx".to_string()),
576 index_file_size: Some(11),
577 num_rows: 20,
578 num_row_groups: 4,
579 num_series: None,
580 min_ts: Timestamp::new_nanosecond(5), max_ts: Timestamp::new_microsecond(2000), sequence: Some(9),
583 partition_expr: None,
584 origin_region_id: region_id2,
585 node_id: None,
586 visible: true,
587 primary_key_min: None,
588 primary_key_max: None,
589 },
590 ];
591
592 let schema = ManifestSstEntry::schema();
593 let batch = ManifestSstEntry::to_record_batch(&entries).unwrap();
594 for projection in [vec![], vec![16, 6, 10], vec![1, 1], (0..26).rev().collect()] {
595 let projected =
596 ManifestSstEntry::to_projected_record_batch(&entries, Some(&projection)).unwrap();
597 assert_eq!(projected, batch.project(&projection).unwrap());
598 assert_eq!(projected.num_rows(), entries.len());
599 }
600 assert!(ManifestSstEntry::to_projected_record_batch(&entries, Some(&[26])).is_err());
601
602 let projected_schema = Arc::new(batch.schema().project(&[6, 9, 6, 11, 25, 9]).unwrap());
603 let builder = ManifestSstBatchBuilder::new(projected_schema.clone());
604 for rows in [entries.as_slice(), &entries[..1], &entries[..0]] {
605 let projected = builder.build(rows).unwrap();
606 assert_eq!(
607 projected,
608 ManifestSstEntry::to_record_batch(rows)
609 .unwrap()
610 .project(&[6, 9, 6, 11, 25, 9])
611 .unwrap()
612 );
613 assert!(Arc::ptr_eq(&projected.schema(), &projected_schema));
614 assert!(Arc::ptr_eq(projected.column(0), projected.column(2)));
615 assert!(Arc::ptr_eq(projected.column(1), projected.column(5)));
616 }
617
618 assert_eq!(schema.arrow_schema().fields().len(), batch.num_columns());
620 assert_eq!(2, batch.num_rows());
621 let expected_columns = [
622 "table_dir",
623 "region_id",
624 "table_id",
625 "region_number",
626 "region_group",
627 "region_sequence",
628 "file_id",
629 "index_version",
630 "level",
631 "file_path",
632 "file_size",
633 "index_file_path",
634 "index_file_size",
635 "num_rows",
636 "num_row_groups",
637 "num_series",
638 "min_ts",
639 "max_ts",
640 "sequence",
641 "origin_region_id",
642 "node_id",
643 "visible",
644 "primary_key_min",
645 "primary_key_max",
646 "max_row_group_uncompressed_size",
647 "partition_expr",
648 ];
649 assert_eq!(
650 expected_columns,
651 schema
652 .arrow_schema()
653 .fields()
654 .iter()
655 .map(|field| field.name().as_str())
656 .collect::<Vec<_>>()
657 .as_slice()
658 );
659 for (i, f) in schema.arrow_schema().fields().iter().enumerate() {
660 assert_eq!(f.name(), batch.schema().field(i).name());
661 assert_eq!(f.is_nullable(), batch.schema().field(i).is_nullable());
662 assert_eq!(f.data_type(), batch.schema().field(i).data_type());
663 }
664
665 let table_dirs = batch
667 .column(0)
668 .as_any()
669 .downcast_ref::<StringArray>()
670 .unwrap();
671 assert_eq!("tdir1", table_dirs.value(0));
672 assert_eq!("tdir2", table_dirs.value(1));
673
674 let region_ids = batch
675 .column(1)
676 .as_any()
677 .downcast_ref::<UInt64Array>()
678 .unwrap();
679 assert_eq!(region_id1.as_u64(), region_ids.value(0));
680 assert_eq!(region_id2.as_u64(), region_ids.value(1));
681
682 let table_ids = batch
683 .column(2)
684 .as_any()
685 .downcast_ref::<UInt32Array>()
686 .unwrap();
687 assert_eq!(table_id1, table_ids.value(0));
688 assert_eq!(table_id2, table_ids.value(1));
689
690 let region_numbers = batch
691 .column(3)
692 .as_any()
693 .downcast_ref::<UInt32Array>()
694 .unwrap();
695 assert_eq!(region_number1, region_numbers.value(0));
696 assert_eq!(region_number2, region_numbers.value(1));
697
698 let region_groups = batch
699 .column(4)
700 .as_any()
701 .downcast_ref::<UInt8Array>()
702 .unwrap();
703 assert_eq!(region_group1, region_groups.value(0));
704 assert_eq!(region_group2, region_groups.value(1));
705
706 let region_sequences = batch
707 .column(5)
708 .as_any()
709 .downcast_ref::<UInt32Array>()
710 .unwrap();
711 assert_eq!(region_seq1, region_sequences.value(0));
712 assert_eq!(region_seq2, region_sequences.value(1));
713
714 let file_ids = batch
715 .column(6)
716 .as_any()
717 .downcast_ref::<StringArray>()
718 .unwrap();
719 assert_eq!("f1", file_ids.value(0));
720 assert_eq!("f2", file_ids.value(1));
721
722 let index_versions = batch
723 .column(7)
724 .as_any()
725 .downcast_ref::<UInt64Array>()
726 .unwrap();
727 assert_eq!(0, index_versions.value(0));
728 assert_eq!(1, index_versions.value(1));
729
730 let levels = batch
731 .column(8)
732 .as_any()
733 .downcast_ref::<UInt8Array>()
734 .unwrap();
735 assert_eq!(1, levels.value(0));
736 assert_eq!(3, levels.value(1));
737
738 let file_paths = batch
739 .column(9)
740 .as_any()
741 .downcast_ref::<StringArray>()
742 .unwrap();
743 assert_eq!("/p1", file_paths.value(0));
744 assert_eq!("/p2", file_paths.value(1));
745
746 let file_sizes = batch
747 .column(10)
748 .as_any()
749 .downcast_ref::<UInt64Array>()
750 .unwrap();
751 assert_eq!(100, file_sizes.value(0));
752 assert_eq!(200, file_sizes.value(1));
753
754 let index_file_paths = batch
755 .column(11)
756 .as_any()
757 .downcast_ref::<StringArray>()
758 .unwrap();
759 assert!(index_file_paths.is_null(0));
760 assert_eq!("idx", index_file_paths.value(1));
761
762 let index_file_sizes = batch
763 .column(12)
764 .as_any()
765 .downcast_ref::<UInt64Array>()
766 .unwrap();
767 assert!(index_file_sizes.is_null(0));
768 assert_eq!(11, index_file_sizes.value(1));
769
770 let num_rows = batch
771 .column(13)
772 .as_any()
773 .downcast_ref::<UInt64Array>()
774 .unwrap();
775 assert_eq!(10, num_rows.value(0));
776 assert_eq!(20, num_rows.value(1));
777
778 let num_row_groups = batch
779 .column(14)
780 .as_any()
781 .downcast_ref::<UInt64Array>()
782 .unwrap();
783 assert_eq!(2, num_row_groups.value(0));
784 assert_eq!(4, num_row_groups.value(1));
785
786 let num_series = batch
787 .column(15)
788 .as_any()
789 .downcast_ref::<UInt64Array>()
790 .unwrap();
791 assert_eq!(5, num_series.value(0));
792 assert!(num_series.is_null(1));
793
794 let min_ts = batch
795 .column(16)
796 .as_any()
797 .downcast_ref::<TimestampNanosecondArray>()
798 .unwrap();
799 assert_eq!(1_000_000_000, min_ts.value(0));
800 assert_eq!(5, min_ts.value(1));
801
802 let max_ts = batch
803 .column(17)
804 .as_any()
805 .downcast_ref::<TimestampNanosecondArray>()
806 .unwrap();
807 assert_eq!(2_000_000_000, max_ts.value(0));
808 assert_eq!(2_000_000, max_ts.value(1));
809
810 let sequences = batch
811 .column(18)
812 .as_any()
813 .downcast_ref::<UInt64Array>()
814 .unwrap();
815 assert!(sequences.is_null(0));
816 assert_eq!(9, sequences.value(1));
817
818 let origin_region_ids = batch
819 .column(19)
820 .as_any()
821 .downcast_ref::<UInt64Array>()
822 .unwrap();
823 assert_eq!(region_id1.as_u64(), origin_region_ids.value(0));
824 assert_eq!(region_id2.as_u64(), origin_region_ids.value(1));
825
826 let node_ids = batch
827 .column(20)
828 .as_any()
829 .downcast_ref::<UInt64Array>()
830 .unwrap();
831 assert_eq!(1, node_ids.value(0));
832 assert!(node_ids.is_null(1));
833
834 let visible = batch
835 .column(21)
836 .as_any()
837 .downcast_ref::<BooleanArray>()
838 .unwrap();
839 assert!(!visible.value(0));
840 assert!(visible.value(1));
841
842 let primary_key_min = batch
843 .column(22)
844 .as_any()
845 .downcast_ref::<BinaryArray>()
846 .unwrap();
847 assert_eq!(b"aaa", primary_key_min.value(0));
848 assert!(primary_key_min.is_null(1));
849
850 let primary_key_max = batch
851 .column(23)
852 .as_any()
853 .downcast_ref::<BinaryArray>()
854 .unwrap();
855 assert_eq!(b"zzz", primary_key_max.value(0));
856 assert!(primary_key_max.is_null(1));
857
858 let max_row_group_uncompressed_sizes = batch
859 .column(24)
860 .as_any()
861 .downcast_ref::<UInt64Array>()
862 .unwrap();
863 assert_eq!(80, max_row_group_uncompressed_sizes.value(0));
864 assert_eq!(160, max_row_group_uncompressed_sizes.value(1));
865
866 let partition_exprs = batch
867 .column(25)
868 .as_any()
869 .downcast_ref::<StringArray>()
870 .unwrap();
871 assert_eq!("a < 10", partition_exprs.value(0));
872 assert!(partition_exprs.is_null(1));
873 }
874
875 #[test]
876 fn test_sst_entry_storage_to_record_batch() {
877 let entries = vec![
878 StorageSstEntry {
879 file_path: "/s1".to_string(),
880 file_size: None,
881 last_modified_ms: None,
882 node_id: Some(1),
883 },
884 StorageSstEntry {
885 file_path: "/s2".to_string(),
886 file_size: Some(123),
887 last_modified_ms: Some(Timestamp::new_millisecond(456)),
888 node_id: None,
889 },
890 ];
891
892 let schema = StorageSstEntry::schema();
893 let batch = StorageSstEntry::to_record_batch(&entries).unwrap();
894
895 assert_eq!(schema.arrow_schema().fields().len(), batch.num_columns());
896 assert_eq!(2, batch.num_rows());
897
898 let file_paths = batch
899 .column(0)
900 .as_any()
901 .downcast_ref::<StringArray>()
902 .unwrap();
903 assert_eq!("/s1", file_paths.value(0));
904 assert_eq!("/s2", file_paths.value(1));
905
906 let file_sizes = batch
907 .column(1)
908 .as_any()
909 .downcast_ref::<UInt64Array>()
910 .unwrap();
911 assert!(file_sizes.is_null(0));
912 assert_eq!(123, file_sizes.value(1));
913
914 let last_modified = batch
915 .column(2)
916 .as_any()
917 .downcast_ref::<TimestampMillisecondArray>()
918 .unwrap();
919 assert!(last_modified.is_null(0));
920 assert_eq!(456, last_modified.value(1));
921
922 let node_ids = batch
923 .column(3)
924 .as_any()
925 .downcast_ref::<UInt64Array>()
926 .unwrap();
927 assert_eq!(1, node_ids.value(0));
928 assert!(node_ids.is_null(1));
929 }
930
931 #[test]
932 fn test_puffin_index_meta_to_record_batch() {
933 let entries = vec![
934 PuffinIndexMetaEntry {
935 table_dir: "table1".to_string(),
936 index_file_path: "index1".to_string(),
937 region_id: RegionId::with_group_and_seq(10, 0, 20),
938 table_id: 10,
939 region_number: 20,
940 region_group: 0,
941 region_sequence: 20,
942 file_id: "file1".to_string(),
943 index_file_size: Some(1024),
944 index_type: "bloom_filter".to_string(),
945 target_type: "column".to_string(),
946 target_key: "1".to_string(),
947 target_json: "{\"column\":1}".to_string(),
948 blob_size: 256,
949 meta_json: Some("{\"bloom\":{}}".to_string()),
950 node_id: Some(42),
951 },
952 PuffinIndexMetaEntry {
953 table_dir: "table2".to_string(),
954 index_file_path: "index2".to_string(),
955 region_id: RegionId::with_group_and_seq(11, 0, 21),
956 table_id: 11,
957 region_number: 21,
958 region_group: 0,
959 region_sequence: 21,
960 file_id: "file2".to_string(),
961 index_file_size: None,
962 index_type: "inverted".to_string(),
963 target_type: "unknown".to_string(),
964 target_key: "legacy".to_string(),
965 target_json: "{}".to_string(),
966 blob_size: 0,
967 meta_json: None,
968 node_id: None,
969 },
970 ];
971
972 let schema = PuffinIndexMetaEntry::schema();
973 let batch = PuffinIndexMetaEntry::to_record_batch(&entries).unwrap();
974
975 assert_eq!(schema.arrow_schema().fields().len(), batch.num_columns());
976 assert_eq!(2, batch.num_rows());
977
978 let table_dirs = batch
979 .column(0)
980 .as_any()
981 .downcast_ref::<StringArray>()
982 .unwrap();
983 assert_eq!("table1", table_dirs.value(0));
984 assert_eq!("table2", table_dirs.value(1));
985
986 let index_file_paths = batch
987 .column(1)
988 .as_any()
989 .downcast_ref::<StringArray>()
990 .unwrap();
991 assert_eq!("index1", index_file_paths.value(0));
992 assert_eq!("index2", index_file_paths.value(1));
993
994 let region_ids = batch
995 .column(2)
996 .as_any()
997 .downcast_ref::<UInt64Array>()
998 .unwrap();
999 assert_eq!(
1000 RegionId::with_group_and_seq(10, 0, 20).as_u64(),
1001 region_ids.value(0)
1002 );
1003 assert_eq!(
1004 RegionId::with_group_and_seq(11, 0, 21).as_u64(),
1005 region_ids.value(1)
1006 );
1007
1008 let table_ids = batch
1009 .column(3)
1010 .as_any()
1011 .downcast_ref::<UInt32Array>()
1012 .unwrap();
1013 assert_eq!(10, table_ids.value(0));
1014 assert_eq!(11, table_ids.value(1));
1015
1016 let region_numbers = batch
1017 .column(4)
1018 .as_any()
1019 .downcast_ref::<UInt32Array>()
1020 .unwrap();
1021 assert_eq!(20, region_numbers.value(0));
1022 assert_eq!(21, region_numbers.value(1));
1023
1024 let region_groups = batch
1025 .column(5)
1026 .as_any()
1027 .downcast_ref::<UInt8Array>()
1028 .unwrap();
1029 assert_eq!(0, region_groups.value(0));
1030 assert_eq!(0, region_groups.value(1));
1031
1032 let region_sequences = batch
1033 .column(6)
1034 .as_any()
1035 .downcast_ref::<UInt32Array>()
1036 .unwrap();
1037 assert_eq!(20, region_sequences.value(0));
1038 assert_eq!(21, region_sequences.value(1));
1039
1040 let file_ids = batch
1041 .column(7)
1042 .as_any()
1043 .downcast_ref::<StringArray>()
1044 .unwrap();
1045 assert_eq!("file1", file_ids.value(0));
1046 assert_eq!("file2", file_ids.value(1));
1047
1048 let index_file_sizes = batch
1049 .column(8)
1050 .as_any()
1051 .downcast_ref::<UInt64Array>()
1052 .unwrap();
1053 assert_eq!(1024, index_file_sizes.value(0));
1054 assert!(index_file_sizes.is_null(1));
1055
1056 let index_types = batch
1057 .column(9)
1058 .as_any()
1059 .downcast_ref::<StringArray>()
1060 .unwrap();
1061 assert_eq!("bloom_filter", index_types.value(0));
1062 assert_eq!("inverted", index_types.value(1));
1063
1064 let target_types = batch
1065 .column(10)
1066 .as_any()
1067 .downcast_ref::<StringArray>()
1068 .unwrap();
1069 assert_eq!("column", target_types.value(0));
1070 assert_eq!("unknown", target_types.value(1));
1071
1072 let target_keys = batch
1073 .column(11)
1074 .as_any()
1075 .downcast_ref::<StringArray>()
1076 .unwrap();
1077 assert_eq!("1", target_keys.value(0));
1078 assert_eq!("legacy", target_keys.value(1));
1079
1080 let target_json = batch
1081 .column(12)
1082 .as_any()
1083 .downcast_ref::<StringArray>()
1084 .unwrap();
1085 assert_eq!("{\"column\":1}", target_json.value(0));
1086 assert_eq!("{}", target_json.value(1));
1087
1088 let blob_sizes = batch
1089 .column(13)
1090 .as_any()
1091 .downcast_ref::<UInt64Array>()
1092 .unwrap();
1093 assert_eq!(256, blob_sizes.value(0));
1094 assert_eq!(0, blob_sizes.value(1));
1095
1096 let meta_jsons = batch
1097 .column(14)
1098 .as_any()
1099 .downcast_ref::<StringArray>()
1100 .unwrap();
1101 assert_eq!("{\"bloom\":{}}", meta_jsons.value(0));
1102 assert!(meta_jsons.is_null(1));
1103
1104 let node_ids = batch
1105 .column(15)
1106 .as_any()
1107 .downcast_ref::<UInt64Array>()
1108 .unwrap();
1109 assert_eq!(42, node_ids.value(0));
1110 assert!(node_ids.is_null(1));
1111 }
1112
1113 #[test]
1114 fn test_manifest_build_plan() {
1115 let request = ScanRequest {
1116 projection: Some(vec![0, 1]),
1117 filters: vec![col("table_id").gt(lit(0u32))],
1118 limit: Some(5),
1119 ..Default::default()
1120 };
1121
1122 let plan = ManifestSstEntry::build_plan(request).unwrap();
1123
1124 let LogicalPlan::Limit(limit) = &plan else {
1126 panic!("expected limit")
1127 };
1128 let LogicalPlan::Projection(projection) = limit.input.as_ref() else {
1129 panic!("expected projection")
1130 };
1131 let LogicalPlan::Filter(filter) = projection.input.as_ref() else {
1132 panic!("expected filter")
1133 };
1134 let LogicalPlan::TableScan(scan) = filter.input.as_ref() else {
1135 panic!("expected table scan")
1136 };
1137 assert_eq!(
1138 scan.table_name,
1139 TableReference::bare(ManifestSstEntry::reserved_table_name_for_inspection())
1140 );
1141 assert_eq!(scan.projection, None);
1142 assert_eq!(
1143 filter.predicate,
1144 Expr::Column(datafusion_common::Column::new(
1145 Some(scan.table_name.clone()),
1146 "table_id",
1147 ))
1148 .gt(lit(0u32))
1149 );
1150
1151 let fields = plan.schema().fields();
1152 assert_eq!(fields.len(), 2);
1153 assert_eq!(fields[0].name(), "table_dir");
1154 assert_eq!(fields[1].name(), "region_id");
1155 }
1156
1157 #[test]
1158 fn test_storage_build_plan() {
1159 let projection = Some(vec![0, 2]);
1160 let request = ScanRequest {
1161 projection,
1162 filters: vec![binary_expr(col("file_path"), Operator::Eq, lit("/a"))],
1163 limit: Some(1),
1164 ..Default::default()
1165 };
1166
1167 let plan = StorageSstEntry::build_plan(request).unwrap();
1168 let (scan, has_filter, has_limit) = extract_scan(&plan);
1169 assert!(has_filter);
1170 assert!(has_limit);
1171 assert_eq!(
1172 scan.table_name,
1173 TableReference::bare(StorageSstEntry::reserved_table_name_for_inspection())
1174 );
1175 assert_eq!(scan.projection, None);
1176
1177 let fields = plan.schema().fields();
1178 assert_eq!(fields.len(), 2);
1179 assert_eq!(fields[0].name(), "file_path");
1180 assert_eq!(fields[1].name(), "last_modified_ms");
1181 }
1182
1183 fn extract_scan(plan: &LogicalPlan) -> (&datafusion_expr::logical_plan::TableScan, bool, bool) {
1185 use datafusion_expr::logical_plan::Limit;
1186
1187 match plan {
1188 LogicalPlan::Filter(f) => {
1189 let (scan, _, has_limit) = extract_scan(&f.input);
1190 (scan, true, has_limit)
1191 }
1192 LogicalPlan::Limit(Limit { input, .. }) => {
1193 let (scan, has_filter, _) = extract_scan(input);
1194 (scan, has_filter, true)
1195 }
1196 LogicalPlan::Projection(projection) => extract_scan(&projection.input),
1197 LogicalPlan::TableScan(scan) => (scan, false, false),
1198 other => panic!("unexpected plan: {other:?}"),
1199 }
1200 }
1201}