query/optimizer/
sst_manifest_aggregate.rs1#[cfg(test)]
18mod tests;
19
20use std::fmt;
21use std::sync::Arc;
22
23use arrow_schema::DataType;
24use catalog::information_schema::{InformationExtensionRef, SSTS_MANIFEST};
25use common_catalog::consts;
26use common_function::aggrs::aggr_wrapper::{StateMergeHelper, get_aggr_func};
27use common_query::logical_plan::is_lossless_substrait_type;
28use datafusion::catalog::Session;
29use datafusion::datasource::DefaultTableSource;
30use datafusion::functions_aggregate::{count::count_udaf, sum::sum_udaf};
31use datafusion::logical_expr::physical_planning_context::PhysicalPlanningContext;
32use datafusion::physical_plan::ExecutionPlan;
33use datafusion::physical_planner::{ExtensionPlanner, PhysicalPlanner};
34use datafusion_common::config::ConfigOptions;
35use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion};
36use datafusion_common::{DFSchema, DFSchemaRef, DataFusionError, Result, ScalarValue};
37use datafusion_expr::{
38 Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder, UserDefinedLogicalNode,
39 UserDefinedLogicalNodeCore,
40};
41use datafusion_optimizer::AnalyzerRule;
42use datatypes::schema::Schema;
43use store_api::sst_entry::ManifestSstEntry;
44use store_api::storage::ScanRequest;
45use table::table::adapter::DfTableProviderAdapter;
46
47#[derive(Debug)]
49pub(crate) struct SstManifestAggregateRule;
50
51impl AnalyzerRule for SstManifestAggregateRule {
52 fn name(&self) -> &str {
53 "sst_manifest_aggregate"
54 }
55
56 fn analyze(&self, plan: LogicalPlan, _config: &ConfigOptions) -> Result<LogicalPlan> {
57 Ok(plan
58 .transform_up_with_subqueries(|plan| {
59 let LogicalPlan::Aggregate(aggregate) = &plan else {
60 return Ok(Transformed::no(plan));
61 };
62 if !aggregate.group_expr.is_empty()
63 || aggregate.aggr_expr.is_empty()
64 || !aggregate
65 .aggr_expr
66 .iter()
67 .all(|expr| is_supported_aggregate(expr, aggregate.input.schema()))
68 || !is_supported_input(&aggregate.input)
69 {
70 return Ok(Transformed::no(plan));
71 }
72 let split = StateMergeHelper::split_aggr_node(aggregate.clone())?;
73 if Schema::try_from(split.lower_state.schema().clone()).is_err() {
75 return Ok(Transformed::no(plan));
76 }
77 let remote = split
78 .lower_state
79 .transform_up(|node| {
80 if let LogicalPlan::TableScan(scan) = node {
81 let reserved = ManifestSstEntry::build_plan(ScanRequest::default())?;
83 let replacement = LogicalPlanBuilder::from(reserved)
84 .alias(scan.table_name)?
85 .build()?;
86 Ok(Transformed::yes(replacement))
87 } else {
88 Ok(Transformed::no(node))
89 }
90 })?
91 .data;
92 let leaf = LogicalPlan::Extension(Extension {
93 node: Arc::new(SstManifestAggregate { remote }),
94 });
95 let upper = split
96 .upper_merge
97 .with_new_exprs(split.upper_merge.expressions(), vec![leaf])?;
98 Ok(Transformed::yes(upper))
99 })?
100 .data)
101 }
102}
103
104pub(crate) struct SstManifestAggregatePlanner {
106 extension: InformationExtensionRef,
107}
108
109impl SstManifestAggregatePlanner {
110 pub(crate) fn new(extension: InformationExtensionRef) -> Self {
112 Self { extension }
113 }
114}
115
116#[async_trait::async_trait]
117impl ExtensionPlanner for SstManifestAggregatePlanner {
118 async fn plan_extension(
119 &self,
120 _planner: &dyn PhysicalPlanner,
121 node: &dyn UserDefinedLogicalNode,
122 _logical_inputs: &[&LogicalPlan],
123 _physical_inputs: &[Arc<dyn ExecutionPlan>],
124 _session: &dyn Session,
125 _planning_ctx: &PhysicalPlanningContext,
126 ) -> Result<Option<Arc<dyn ExecutionPlan>>> {
127 let Some(node) = node.as_any().downcast_ref::<SstManifestAggregate>() else {
128 return Ok(None);
129 };
130 self.extension
131 .inspect_datanode_query_plan(node.remote.clone())
132 .map_err(|error| DataFusionError::External(Box::new(error)))?
133 .map(Some)
134 .ok_or_else(|| {
135 DataFusionError::Internal(
136 "Manifest aggregation requires inspect query support".into(),
137 )
138 })
139 }
140}
141
142#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Hash)]
144struct SstManifestAggregate {
145 remote: LogicalPlan,
146}
147
148impl UserDefinedLogicalNodeCore for SstManifestAggregate {
149 fn name(&self) -> &str {
150 "SstManifestAggregate"
151 }
152 fn inputs(&self) -> Vec<&LogicalPlan> {
153 vec![]
154 }
155 fn schema(&self) -> &DFSchemaRef {
156 self.remote.schema()
157 }
158 fn expressions(&self) -> Vec<Expr> {
159 vec![]
160 }
161 fn fmt_for_explain(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
162 write!(f, "SstManifestAggregate: {}", self.remote.display_indent())
163 }
164 fn with_exprs_and_inputs(&self, exprs: Vec<Expr>, inputs: Vec<LogicalPlan>) -> Result<Self> {
165 if !exprs.is_empty() || !inputs.is_empty() {
166 return Err(DataFusionError::Internal(
167 "SstManifestAggregate is a leaf".into(),
168 ));
169 }
170 Ok(self.clone())
171 }
172}
173
174fn is_supported_aggregate(expr: &Expr, schema: &DFSchema) -> bool {
175 let Some(function) = get_aggr_func(expr) else {
176 return false;
177 };
178 let params = &function.params;
179 if params.distinct || !params.order_by.is_empty() || params.null_treatment.is_some() {
180 return false;
181 }
182 if params
183 .filter
184 .as_deref()
185 .is_some_and(|filter| !is_supported_predicate(filter, schema))
186 {
187 return false;
188 }
189 if function.func == count_udaf() {
190 matches!(
191 params.args.as_slice(),
192 [Expr::Column(_)]
193 | [Expr::Literal(
194 ScalarValue::Int64(Some(1))
195 | ScalarValue::UInt64(Some(1))
196 | ScalarValue::Int32(Some(1))
197 | ScalarValue::UInt8(Some(1)),
198 _
199 )]
200 )
201 } else if function.func == sum_udaf() {
202 matches!(params.args.as_slice(), [arg] if is_numeric_column(arg, schema))
203 } else {
204 false
205 }
206}
207
208fn is_numeric_column(expr: &Expr, schema: &DFSchema) -> bool {
211 match expr {
212 Expr::Column(_) => expr
213 .get_type(schema)
214 .is_ok_and(|data_type| data_type.is_numeric()),
215 Expr::Cast(cast) if cast.field.data_type().is_numeric() => {
216 is_numeric_column(&cast.expr, schema)
217 }
218 _ => false,
219 }
220}
221
222fn is_supported_predicate(expr: &Expr, schema: &DFSchema) -> bool {
223 let mut supported = true;
224 let visited = expr.apply(|expr| {
225 let Ok(data_type) = expr.get_type(schema) else {
228 supported = false;
229 return Ok(TreeNodeRecursion::Stop);
230 };
231 if !is_lossless_substrait_type(&data_type) {
232 supported = false;
233 return Ok(TreeNodeRecursion::Stop);
234 }
235 if let Expr::ScalarFunction(function) = expr
238 && function.func == datafusion::functions::core::arrow_cast()
239 && matches!(data_type, DataType::Timestamp(_, None))
240 && function
241 .args
242 .iter()
243 .all(|arg| matches!(arg, Expr::Literal(_, _)))
244 {
245 return Ok(TreeNodeRecursion::Continue);
246 }
247 if !matches!(
248 expr,
249 Expr::Column(_)
250 | Expr::Literal(_, _)
251 | Expr::BinaryExpr(_)
252 | Expr::Cast(_)
253 | Expr::TryCast(_)
254 | Expr::Not(_)
255 | Expr::IsNull(_)
256 | Expr::IsNotNull(_)
257 | Expr::Between(_)
258 | Expr::InList(_)
259 ) {
260 supported = false;
261 return Ok(TreeNodeRecursion::Stop);
262 }
263 Ok(TreeNodeRecursion::Continue)
264 });
265 visited.is_ok() && supported
266}
267
268fn is_supported_input(plan: &LogicalPlan) -> bool {
269 match plan {
270 LogicalPlan::Filter(filter) => {
271 is_supported_predicate(&filter.predicate, filter.input.schema())
272 && is_supported_input(&filter.input)
273 }
274 LogicalPlan::SubqueryAlias(alias) => is_supported_input(&alias.input),
275 LogicalPlan::TableScan(scan) => {
276 if scan.fetch.is_some() || scan.projection.is_some() || !scan.filters.is_empty() {
277 return false;
278 }
279 let Some(source) = scan.source.downcast_ref::<DefaultTableSource>() else {
280 return false;
281 };
282 let Some(adapter) = source
283 .table_provider
284 .downcast_ref::<DfTableProviderAdapter>()
285 else {
286 return false;
287 };
288 let info = adapter.table().table_info();
289 info.ident.table_id == consts::INFORMATION_SCHEMA_SSTS_MANIFEST_TABLE_ID
290 && info.schema_name == consts::INFORMATION_SCHEMA_NAME
291 && info.name == SSTS_MANIFEST
292 && info.meta.schema.as_ref() == ManifestSstEntry::schema().as_ref()
293 }
294 _ => false,
295 }
296}