1use core::pin::pin;
16use std::sync::{Arc, Weak};
17
18use arrow_schema::SchemaRef as ArrowSchemaRef;
19use common_catalog::consts::INFORMATION_SCHEMA_PARTITIONS_TABLE_ID;
20use common_error::ext::BoxedError;
21use common_recordbatch::adapter::RecordBatchStreamAdapter;
22use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
23use datafusion::execution::TaskContext;
24use datafusion::physical_plan::SendableRecordBatchStream as DfSendableRecordBatchStream;
25use datafusion::physical_plan::stream::RecordBatchStreamAdapter as DfRecordBatchStreamAdapter;
26use datafusion::physical_plan::streaming::PartitionStream as DfPartitionStream;
27use datatypes::prelude::{ConcreteDataType, ScalarVectorBuilder, VectorRef};
28use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
29use datatypes::timestamp::TimestampSecond;
30use datatypes::value::Value;
31use datatypes::vectors::{
32 Int64Vector, Int64VectorBuilder, MutableVector, StringVector, StringVectorBuilder,
33 TimestampSecondVector, TimestampSecondVectorBuilder, UInt64VectorBuilder,
34};
35use futures::{StreamExt, TryStreamExt};
36use partition::manager::PartitionInfo;
37use snafu::{OptionExt, ResultExt};
38use store_api::storage::{ScanRequest, TableId};
39use table::metadata::{TableInfo, TableType};
40
41use crate::CatalogManager;
42use crate::error::{
43 CreateRecordBatchSnafu, FindPartitionsSnafu, InternalSnafu, PartitionManagerNotFoundSnafu,
44 Result, UpgradeWeakCatalogManagerRefSnafu,
45};
46use crate::kvbackend::KvBackendCatalogManager;
47use crate::system_schema::SystemTable;
48use crate::system_schema::information_schema::{
49 InformationSchemaTableScope, InformationTable, PARTITIONS, Predicates,
50};
51
52const TABLE_CATALOG: &str = "table_catalog";
53const TABLE_SCHEMA: &str = "table_schema";
54const TABLE_NAME: &str = "table_name";
55const PARTITION_NAME: &str = "partition_name";
56const PARTITION_EXPRESSION: &str = "partition_expression";
57const GREPTIME_PARTITION_ID: &str = "greptime_partition_id";
59const INIT_CAPACITY: usize = 42;
60
61#[derive(Debug)]
65pub(super) struct InformationSchemaPartitions {
66 schema: SchemaRef,
67 catalog_name: String,
68 catalog_manager: Weak<dyn CatalogManager>,
69}
70
71impl InformationSchemaPartitions {
72 pub(super) fn new(catalog_name: String, catalog_manager: Weak<dyn CatalogManager>) -> Self {
73 Self {
74 schema: Self::schema(),
75 catalog_name,
76 catalog_manager,
77 }
78 }
79
80 pub(crate) fn schema() -> SchemaRef {
81 Arc::new(Schema::new(vec![
82 ColumnSchema::new(TABLE_CATALOG, ConcreteDataType::string_datatype(), false),
83 ColumnSchema::new(TABLE_SCHEMA, ConcreteDataType::string_datatype(), false),
84 ColumnSchema::new(TABLE_NAME, ConcreteDataType::string_datatype(), false),
85 ColumnSchema::new(PARTITION_NAME, ConcreteDataType::string_datatype(), false),
86 ColumnSchema::new(
87 "subpartition_name",
88 ConcreteDataType::string_datatype(),
89 true,
90 ),
91 ColumnSchema::new(
92 "partition_ordinal_position",
93 ConcreteDataType::int64_datatype(),
94 true,
95 ),
96 ColumnSchema::new(
97 "subpartition_ordinal_position",
98 ConcreteDataType::int64_datatype(),
99 true,
100 ),
101 ColumnSchema::new(
102 "partition_method",
103 ConcreteDataType::string_datatype(),
104 true,
105 ),
106 ColumnSchema::new(
107 "subpartition_method",
108 ConcreteDataType::string_datatype(),
109 true,
110 ),
111 ColumnSchema::new(
112 PARTITION_EXPRESSION,
113 ConcreteDataType::string_datatype(),
114 true,
115 ),
116 ColumnSchema::new(
117 "subpartition_expression",
118 ConcreteDataType::string_datatype(),
119 true,
120 ),
121 ColumnSchema::new(
122 "partition_description",
123 ConcreteDataType::string_datatype(),
124 true,
125 ),
126 ColumnSchema::new("table_rows", ConcreteDataType::int64_datatype(), true),
127 ColumnSchema::new("avg_row_length", ConcreteDataType::int64_datatype(), true),
128 ColumnSchema::new("data_length", ConcreteDataType::int64_datatype(), true),
129 ColumnSchema::new("max_data_length", ConcreteDataType::int64_datatype(), true),
130 ColumnSchema::new("index_length", ConcreteDataType::int64_datatype(), true),
131 ColumnSchema::new("data_free", ConcreteDataType::int64_datatype(), true),
132 ColumnSchema::new(
133 "create_time",
134 ConcreteDataType::timestamp_second_datatype(),
135 true,
136 ),
137 ColumnSchema::new(
138 "update_time",
139 ConcreteDataType::timestamp_second_datatype(),
140 true,
141 ),
142 ColumnSchema::new(
143 "check_time",
144 ConcreteDataType::timestamp_second_datatype(),
145 true,
146 ),
147 ColumnSchema::new("checksum", ConcreteDataType::int64_datatype(), true),
148 ColumnSchema::new(
149 "partition_comment",
150 ConcreteDataType::string_datatype(),
151 true,
152 ),
153 ColumnSchema::new("nodegroup", ConcreteDataType::string_datatype(), true),
154 ColumnSchema::new("tablespace_name", ConcreteDataType::string_datatype(), true),
155 ColumnSchema::new(
156 GREPTIME_PARTITION_ID,
157 ConcreteDataType::uint64_datatype(),
158 true,
159 ),
160 ]))
161 }
162
163 fn builder(&self) -> InformationSchemaPartitionsBuilder {
164 InformationSchemaPartitionsBuilder::new(
165 self.schema.clone(),
166 self.catalog_name.clone(),
167 self.catalog_manager.clone(),
168 )
169 }
170}
171
172impl InformationTable for InformationSchemaPartitions {
173 fn scope(&self) -> InformationSchemaTableScope {
174 InformationSchemaTableScope::Catalog
175 }
176}
177
178impl SystemTable for InformationSchemaPartitions {
179 fn table_id(&self) -> TableId {
180 INFORMATION_SCHEMA_PARTITIONS_TABLE_ID
181 }
182
183 fn table_name(&self) -> &'static str {
184 PARTITIONS
185 }
186
187 fn schema(&self) -> SchemaRef {
188 self.schema.clone()
189 }
190
191 fn to_stream(&self, request: ScanRequest) -> Result<SendableRecordBatchStream> {
192 let schema = self.schema.arrow_schema().clone();
193 let mut builder = self.builder();
194 let stream = Box::pin(DfRecordBatchStreamAdapter::new(
195 schema,
196 futures::stream::once(async move {
197 builder
198 .make_partitions(Some(request))
199 .await
200 .map(|x| x.into_df_record_batch())
201 .map_err(Into::into)
202 }),
203 ));
204 Ok(Box::pin(
205 RecordBatchStreamAdapter::try_new(stream)
206 .map_err(BoxedError::new)
207 .context(InternalSnafu)?,
208 ))
209 }
210}
211
212struct InformationSchemaPartitionsBuilder {
213 schema: SchemaRef,
214 catalog_name: String,
215 catalog_manager: Weak<dyn CatalogManager>,
216
217 catalog_names: StringVectorBuilder,
218 schema_names: StringVectorBuilder,
219 table_names: StringVectorBuilder,
220 partition_names: StringVectorBuilder,
221 partition_ordinal_positions: Int64VectorBuilder,
222 partition_expressions: StringVectorBuilder,
223 partition_descriptions: StringVectorBuilder,
224 create_times: TimestampSecondVectorBuilder,
225 partition_ids: UInt64VectorBuilder,
226}
227
228impl InformationSchemaPartitionsBuilder {
229 fn new(
230 schema: SchemaRef,
231 catalog_name: String,
232 catalog_manager: Weak<dyn CatalogManager>,
233 ) -> Self {
234 Self {
235 schema,
236 catalog_name,
237 catalog_manager,
238 catalog_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
239 schema_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
240 table_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
241 partition_names: StringVectorBuilder::with_capacity(INIT_CAPACITY),
242 partition_ordinal_positions: Int64VectorBuilder::with_capacity(INIT_CAPACITY),
243 partition_expressions: StringVectorBuilder::with_capacity(INIT_CAPACITY),
244 partition_descriptions: StringVectorBuilder::with_capacity(INIT_CAPACITY),
245 create_times: TimestampSecondVectorBuilder::with_capacity(INIT_CAPACITY),
246 partition_ids: UInt64VectorBuilder::with_capacity(INIT_CAPACITY),
247 }
248 }
249
250 async fn make_partitions(&mut self, request: Option<ScanRequest>) -> Result<RecordBatch> {
252 let catalog_name = self.catalog_name.clone();
253 let catalog_manager = self
254 .catalog_manager
255 .upgrade()
256 .context(UpgradeWeakCatalogManagerRefSnafu)?;
257
258 let partition_manager = catalog_manager
259 .as_any()
260 .downcast_ref::<KvBackendCatalogManager>()
261 .map(|catalog_manager| catalog_manager.partition_manager())
262 .context(PartitionManagerNotFoundSnafu)?;
263
264 let predicates = Predicates::from_scan_request(&request);
265
266 if let (Some(schema_name), Some(table_name)) = (
270 predicates
271 .find_eq_value(TABLE_SCHEMA)
272 .and_then(Value::as_string),
273 predicates
274 .find_eq_value(TABLE_NAME)
275 .and_then(Value::as_string),
276 ) {
277 if let Some(table) = catalog_manager
278 .table(&catalog_name, &schema_name, &table_name, None)
279 .await?
280 {
281 let table_info = table.table_info();
282
283 if table_info.table_type != TableType::Temporary {
286 let mut table_partitions = partition_manager
287 .batch_find_table_partitions(&[table_info.table_id()])
288 .await
289 .context(FindPartitionsSnafu)?;
290
291 let partitions = table_partitions
292 .remove(&table_info.table_id())
293 .unwrap_or_default();
294
295 self.add_partitions(
296 &predicates,
297 &table_info,
298 &catalog_name,
299 &schema_name,
300 &table_info.name,
301 &partitions,
302 );
303 }
304 }
305
306 return self.finish();
309 }
310
311 for schema_name in catalog_manager.schema_names(&catalog_name, None).await? {
312 let table_info_stream = catalog_manager
313 .tables(&catalog_name, &schema_name, None)
314 .try_filter_map(|t| async move {
315 let table_info = t.table_info();
316 if table_info.table_type == TableType::Temporary {
317 Ok(None)
318 } else {
319 Ok(Some(table_info))
320 }
321 });
322
323 const BATCH_SIZE: usize = 128;
324
325 let mut table_info_chunks = pin!(table_info_stream.ready_chunks(BATCH_SIZE));
327
328 while let Some(table_infos) = table_info_chunks.next().await {
329 let table_infos = table_infos.into_iter().collect::<Result<Vec<_>>>()?;
330 let table_ids: Vec<TableId> =
331 table_infos.iter().map(|info| info.ident.table_id).collect();
332
333 let mut table_partitions = partition_manager
334 .batch_find_table_partitions(&table_ids)
335 .await
336 .context(FindPartitionsSnafu)?;
337
338 for table_info in table_infos {
339 let partitions = table_partitions
340 .remove(&table_info.ident.table_id)
341 .unwrap_or(vec![]);
342
343 self.add_partitions(
344 &predicates,
345 &table_info,
346 &catalog_name,
347 &schema_name,
348 &table_info.name,
349 &partitions,
350 );
351 }
352 }
353 }
354
355 self.finish()
356 }
357
358 #[allow(clippy::too_many_arguments)]
359 fn add_partitions(
360 &mut self,
361 predicates: &Predicates,
362 table_info: &TableInfo,
363 catalog_name: &str,
364 schema_name: &str,
365 table_name: &str,
366 partitions: &[PartitionInfo],
367 ) {
368 let row = [
369 (TABLE_CATALOG, &Value::from(catalog_name)),
370 (TABLE_SCHEMA, &Value::from(schema_name)),
371 (TABLE_NAME, &Value::from(table_name)),
372 ];
373
374 if !predicates.eval(&row) {
375 return;
376 }
377
378 let partition_columns: String = table_info
381 .meta
382 .partition_column_names()
383 .cloned()
384 .collect::<Vec<_>>()
385 .join(", ");
386
387 let partition_expr_str = if partition_columns.is_empty() {
388 None
389 } else {
390 Some(partition_columns)
391 };
392
393 for (index, partition) in partitions.iter().enumerate() {
394 let partition_name = format!("p{index}");
395
396 self.catalog_names.push(Some(catalog_name));
397 self.schema_names.push(Some(schema_name));
398 self.table_names.push(Some(table_name));
399 self.partition_names.push(Some(&partition_name));
400 self.partition_ordinal_positions
401 .push(Some((index + 1) as i64));
402 self.partition_expressions
404 .push(partition_expr_str.as_deref());
405 let description = partition.partition_expr.as_ref().map(|e| e.to_string());
407 self.partition_descriptions.push(description.as_deref());
408 self.create_times.push(Some(TimestampSecond::from(
409 table_info.meta.created_on.timestamp(),
410 )));
411 self.partition_ids.push(Some(partition.id.as_u64()));
412 }
413 }
414
415 fn finish(&mut self) -> Result<RecordBatch> {
416 let rows_num = self.catalog_names.len();
417
418 let null_string_vector = Arc::new(StringVector::from(vec![None as Option<&str>; rows_num]));
419 let null_i64_vector = Arc::new(Int64Vector::from(vec![None; rows_num]));
420 let null_timestamp_second_vector =
421 Arc::new(TimestampSecondVector::from(vec![None; rows_num]));
422 let partition_methods = Arc::new(StringVector::from(vec![Some("RANGE"); rows_num]));
423
424 let columns: Vec<VectorRef> = vec![
425 Arc::new(self.catalog_names.finish()),
426 Arc::new(self.schema_names.finish()),
427 Arc::new(self.table_names.finish()),
428 Arc::new(self.partition_names.finish()),
429 null_string_vector.clone(),
430 Arc::new(self.partition_ordinal_positions.finish()),
431 null_i64_vector.clone(),
432 partition_methods,
433 null_string_vector.clone(),
434 Arc::new(self.partition_expressions.finish()),
435 null_string_vector.clone(),
436 Arc::new(self.partition_descriptions.finish()),
437 null_i64_vector.clone(),
439 null_i64_vector.clone(),
440 null_i64_vector.clone(),
441 null_i64_vector.clone(),
442 null_i64_vector.clone(),
443 null_i64_vector.clone(),
444 Arc::new(self.create_times.finish()),
445 null_timestamp_second_vector.clone(),
447 null_timestamp_second_vector,
448 null_i64_vector,
449 null_string_vector.clone(),
450 null_string_vector.clone(),
451 null_string_vector,
452 Arc::new(self.partition_ids.finish()),
453 ];
454 RecordBatch::new(self.schema.clone(), columns).context(CreateRecordBatchSnafu)
455 }
456}
457
458impl DfPartitionStream for InformationSchemaPartitions {
459 fn schema(&self) -> &ArrowSchemaRef {
460 self.schema.arrow_schema()
461 }
462
463 fn execute(&self, _: Arc<TaskContext>) -> DfSendableRecordBatchStream {
464 let schema = self.schema.arrow_schema().clone();
465 let mut builder = self.builder();
466 Box::pin(DfRecordBatchStreamAdapter::new(
467 schema,
468 futures::stream::once(async move {
469 builder
470 .make_partitions(None)
471 .await
472 .map(|x| x.into_df_record_batch())
473 .map_err(Into::into)
474 }),
475 ))
476 }
477}
478
479#[cfg(test)]
480mod tests {
481 use api::v1::meta::Peer;
482 use cache::{build_fundamental_cache_registry, with_default_composite_cache_registry};
483 use common_catalog::consts::{DEFAULT_CATALOG_NAME, INFORMATION_SCHEMA_NAME};
484 use common_meta::cache::{CacheRegistryBuilder, LayeredCacheRegistryBuilder};
485 use common_meta::key::schema_name::SchemaNameKey;
486 use common_meta::key::table_route::TableRouteValue;
487 use common_meta::kv_backend::memory::MemoryKvBackend;
488 use common_meta::rpc::router::{Region, RegionRoute};
489 use common_meta::wal_provider::RegionWalOptions;
490 use datafusion::logical_expr::{BinaryExpr, Expr, Operator, col, lit};
491 use futures::TryStreamExt;
492 use store_api::storage::RegionId;
493
494 use super::*;
495 use crate::information_schema::NoopInformationExtension;
496 use crate::kvbackend::KvBackendCatalogManagerBuilder;
497
498 const SCHEMA_1: &str = "partitions_schema_1";
499 const SCHEMA_2: &str = "partitions_schema_2";
500 const PARTITIONED_TABLE: &str = "partitioned_table";
501 const MISSING_TABLE: &str = "missing_table";
502 const TABLE_ID_1: TableId = 1;
503 const TABLE_ID_2: TableId = 2;
504 const REGION_NUMBER_1: u32 = 1;
505 const REGION_NUMBER_2: u32 = 2;
506 const REGION_NUMBER_7: u32 = 7;
507
508 fn kv_catalog_manager() -> Arc<KvBackendCatalogManager> {
509 let backend = Arc::new(MemoryKvBackend::default());
510 let layered_cache_builder = LayeredCacheRegistryBuilder::default()
511 .add_cache_registry(CacheRegistryBuilder::default().build());
512 let fundamental_cache_registry = build_fundamental_cache_registry(backend.clone());
513 let layered_cache_registry = Arc::new(
514 with_default_composite_cache_registry(
515 layered_cache_builder.add_cache_registry(fundamental_cache_registry),
516 )
517 .unwrap()
518 .build(),
519 );
520
521 KvBackendCatalogManagerBuilder::new(
522 Arc::new(NoopInformationExtension),
523 backend,
524 layered_cache_registry,
525 )
526 .build()
527 }
528
529 fn region_route(table_id: TableId, region_number: u32) -> RegionRoute {
530 RegionRoute {
531 region: Region {
532 id: RegionId::new(table_id, region_number),
533 ..Default::default()
534 },
535 leader_peer: Some(Peer {
536 id: 1,
537 addr: "127.0.0.1:3001".to_string(),
538 }),
539 ..Default::default()
540 }
541 }
542
543 fn table_info(schema: &str, table: &str, table_id: TableId) -> TableInfo {
544 let mut table_info =
546 common_meta::key::test_utils::new_test_table_info_with_name(table_id, table);
547 table_info.schema_name = schema.to_string();
548 table_info
549 }
550
551 async fn setup_catalog() -> Arc<KvBackendCatalogManager> {
557 let manager = kv_catalog_manager();
558 let metadata = manager.table_metadata_manager_ref().clone();
559
560 for schema in [SCHEMA_1, SCHEMA_2] {
561 metadata
562 .schema_manager()
563 .create(SchemaNameKey::new(DEFAULT_CATALOG_NAME, schema), None, true)
564 .await
565 .unwrap();
566 }
567
568 for (table_id, schema, region_numbers) in [
569 (TABLE_ID_1, SCHEMA_1, vec![REGION_NUMBER_1, REGION_NUMBER_2]),
570 (TABLE_ID_2, SCHEMA_2, vec![REGION_NUMBER_7]),
571 ] {
572 let routes = region_numbers
573 .into_iter()
574 .map(|region_number| region_route(table_id, region_number))
575 .collect();
576 metadata
577 .create_table_metadata(
578 table_info(schema, PARTITIONED_TABLE, table_id),
579 TableRouteValue::physical(routes),
580 RegionWalOptions::default(),
581 )
582 .await
583 .unwrap();
584 }
585
586 manager
587 }
588
589 async fn scan_partitions(
592 manager: &Arc<dyn CatalogManager>,
593 filters: Vec<Expr>,
594 ) -> (usize, String) {
595 let table = manager
596 .table(
597 DEFAULT_CATALOG_NAME,
598 INFORMATION_SCHEMA_NAME,
599 PARTITIONS,
600 None,
601 )
602 .await
603 .unwrap()
604 .unwrap();
605 let stream = table
606 .scan_to_stream(ScanRequest {
607 filters,
608 ..Default::default()
609 })
610 .await
611 .unwrap();
612 let batches = stream.try_collect::<Vec<_>>().await.unwrap();
613 let rows = batches.iter().map(|batch| batch.num_rows()).sum();
614 let output = batches
615 .iter()
616 .map(|batch| batch.pretty_print())
617 .collect::<Vec<_>>()
618 .join("\n");
619 (rows, output)
620 }
621
622 fn eq_filter(column: &str, value: &str) -> Expr {
623 Expr::BinaryExpr(BinaryExpr::new(
624 Box::new(col(column)),
625 Operator::Eq,
626 Box::new(lit(value)),
627 ))
628 }
629
630 fn not_eq_filter(column: &str, value: &str) -> Expr {
631 Expr::BinaryExpr(BinaryExpr::new(
632 Box::new(col(column)),
633 Operator::NotEq,
634 Box::new(lit(value)),
635 ))
636 }
637
638 #[tokio::test]
639 async fn test_scan_pinned_table_skips_catalog_enumeration() {
640 let manager: Arc<dyn CatalogManager> = setup_catalog().await;
641
642 let (rows, output) = scan_partitions(&manager, vec![]).await;
644 assert_eq!(3, rows, "{output}");
645
646 let (rows, output) = scan_partitions(
649 &manager,
650 vec![
651 eq_filter(TABLE_SCHEMA, SCHEMA_1),
652 eq_filter(TABLE_NAME, PARTITIONED_TABLE),
653 ],
654 )
655 .await;
656 assert_eq!(2, rows, "{output}");
657 assert!(output.contains(SCHEMA_1), "{output}");
658 assert!(!output.contains(SCHEMA_2), "{output}");
659 assert!(
660 output.contains(
661 &RegionId::new(TABLE_ID_1, REGION_NUMBER_1)
662 .as_u64()
663 .to_string()
664 ),
665 "{output}"
666 );
667 assert!(
668 output.contains(
669 &RegionId::new(TABLE_ID_1, REGION_NUMBER_2)
670 .as_u64()
671 .to_string()
672 ),
673 "{output}"
674 );
675
676 let (fallback_rows, fallback_output) = scan_partitions(
680 &manager,
681 vec![
682 eq_filter(TABLE_SCHEMA, SCHEMA_1),
683 not_eq_filter(TABLE_NAME, MISSING_TABLE),
684 ],
685 )
686 .await;
687 assert_eq!(rows, fallback_rows, "{fallback_output}");
688 assert_eq!(output, fallback_output);
689
690 let (rows, output) =
693 scan_partitions(&manager, vec![eq_filter(TABLE_NAME, PARTITIONED_TABLE)]).await;
694 assert_eq!(3, rows, "{output}");
695 assert!(output.contains(SCHEMA_1), "{output}");
696 assert!(output.contains(SCHEMA_2), "{output}");
697
698 let (rows, output) = scan_partitions(
700 &manager,
701 vec![
702 eq_filter(TABLE_SCHEMA, SCHEMA_1),
703 eq_filter(TABLE_NAME, MISSING_TABLE),
704 ],
705 )
706 .await;
707 assert_eq!(0, rows, "{output}");
708 }
709
710 #[tokio::test]
711 async fn test_scan_equality_under_disjunction_falls_back_to_enumeration() {
712 let manager: Arc<dyn CatalogManager> = setup_catalog().await;
713
714 let filter = Expr::BinaryExpr(BinaryExpr::new(
718 Box::new(eq_filter(TABLE_SCHEMA, SCHEMA_1)),
719 Operator::Or,
720 Box::new(eq_filter(TABLE_NAME, PARTITIONED_TABLE)),
721 ));
722 let (rows, output) = scan_partitions(&manager, vec![filter]).await;
723 assert_eq!(3, rows, "{output}");
724 assert!(output.contains(SCHEMA_1), "{output}");
725 assert!(output.contains(SCHEMA_2), "{output}");
726 }
727}