1use std::collections::HashSet;
16use std::fmt::{Debug, Display, Formatter};
17use std::sync::Arc;
18
19use api::v1::meta::Partition;
20use datafusion_common::{ScalarValue, ToDFSchema};
21use datafusion_expr::Expr;
22use datafusion_expr::execution_props::ExecutionProps;
23use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
24use datafusion_expr::simplify::SimplifyContext;
25use datafusion_optimizer::simplify_expressions::ExprSimplifier;
26use datafusion_physical_expr::{PhysicalExpr, create_physical_expr};
27use datatypes::arrow;
28use datatypes::value::{
29 Value, duration_to_scalar_value, time_to_scalar_value, timestamp_to_scalar_value,
30};
31use serde::{Deserialize, Serialize};
32use snafu::ResultExt;
33use sql::statements::value_to_sql_value;
34use sqlparser::ast::{
35 BinaryOperator as ParserBinaryOperator, Expr as ParserExpr, Function, FunctionArg,
36 FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName,
37};
38
39use crate::error;
40use crate::function::IcebergHash;
41use crate::partition::PartitionBound;
42
43#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
48pub struct PartitionExpr {
49 pub lhs: Box<Operand>,
50 pub op: RestrictedOp,
51 pub rhs: Box<Operand>,
52}
53
54#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
55pub enum Operand {
56 Column(String),
57 Value(Value),
58 Expr(PartitionExpr),
59 IcebergHash(IcebergHash),
60}
61
62pub fn col(column_name: impl Into<String>) -> Operand {
63 Operand::Column(column_name.into())
64}
65
66impl From<Value> for Operand {
67 fn from(value: Value) -> Self {
68 Operand::Value(value)
69 }
70}
71
72impl Operand {
73 pub fn try_as_logical_expr(&self) -> error::Result<Expr> {
74 match self {
75 Self::Column(c) => Ok(Expr::Column(datafusion_common::Column::from_name(c))),
76 Self::Value(v) => {
77 let scalar_value = match v {
78 Value::Boolean(v) => ScalarValue::Boolean(Some(*v)),
79 Value::UInt8(v) => ScalarValue::UInt8(Some(*v)),
80 Value::UInt16(v) => ScalarValue::UInt16(Some(*v)),
81 Value::UInt32(v) => ScalarValue::UInt32(Some(*v)),
82 Value::UInt64(v) => ScalarValue::UInt64(Some(*v)),
83 Value::Int8(v) => ScalarValue::Int8(Some(*v)),
84 Value::Int16(v) => ScalarValue::Int16(Some(*v)),
85 Value::Int32(v) => ScalarValue::Int32(Some(*v)),
86 Value::Int64(v) => ScalarValue::Int64(Some(*v)),
87 Value::Float32(v) => ScalarValue::Float32(Some(v.0)),
88 Value::Float64(v) => ScalarValue::Float64(Some(v.0)),
89 Value::String(v) => ScalarValue::Utf8(Some(v.as_utf8().to_string())),
90 Value::Binary(v) => ScalarValue::Binary(Some(v.to_vec())),
91 Value::Date(v) => ScalarValue::Date32(Some(v.val())),
92 Value::Null => ScalarValue::Null,
93 Value::Timestamp(t) => timestamp_to_scalar_value(t.unit(), Some(t.value())),
94 Value::Time(t) => time_to_scalar_value(*t.unit(), Some(t.value()))
95 .context(error::ConvertPartitionExprValueSnafu { value: v.clone() })?,
96 Value::IntervalYearMonth(v) => ScalarValue::IntervalYearMonth(Some(v.to_i32())),
97 Value::IntervalDayTime(v) => ScalarValue::IntervalDayTime(Some((*v).into())),
98 Value::IntervalMonthDayNano(v) => {
99 ScalarValue::IntervalMonthDayNano(Some((*v).into()))
100 }
101 Value::Duration(d) => duration_to_scalar_value(d.unit(), Some(d.value())),
102 Value::Decimal128(d) => {
103 let (v, p, s) = d.to_scalar_value();
104 ScalarValue::Decimal128(v, p, s)
105 }
106 other => {
107 return error::UnsupportedPartitionExprValueSnafu {
108 value: other.clone(),
109 }
110 .fail();
111 }
112 };
113 Ok(datafusion_expr::lit(scalar_value))
114 }
115 Self::Expr(e) => e.try_as_logical_expr(),
116 Self::IcebergHash(f) => Ok(f.to_logical_expr()),
117 }
118 }
119
120 fn is_dimension(&self) -> bool {
122 matches!(self, Self::Column(_) | Self::IcebergHash(_))
123 }
124
125 fn to_parser_expr(&self) -> ParserExpr {
126 match self {
129 Self::Column(c) => ParserExpr::Identifier(Ident::new(c.clone())),
130 Self::Value(v) => ParserExpr::Value(value_to_sql_value(v).unwrap().into()),
131 Self::Expr(e) => e.to_parser_expr(),
132 Self::IcebergHash(f) => ParserExpr::Function(Function {
133 name: ObjectName::from(vec![Ident::new(IcebergHash::NAME)]),
134 uses_odbc_syntax: false,
135 parameters: FunctionArguments::None,
136 args: FunctionArguments::List(FunctionArgumentList {
137 duplicate_treatment: None,
138 args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(
139 ParserExpr::Identifier(Ident::new(f.column())),
140 ))],
141 clauses: vec![],
142 }),
143 filter: None,
144 null_treatment: None,
145 over: None,
146 within_group: vec![],
147 }),
148 }
149 }
150
151 pub fn lt(self, rhs: impl Into<Self>) -> PartitionExpr {
152 PartitionExpr::new(self, RestrictedOp::Lt, rhs.into())
153 }
154
155 pub fn gt_eq(self, rhs: impl Into<Self>) -> PartitionExpr {
156 PartitionExpr::new(self, RestrictedOp::GtEq, rhs.into())
157 }
158
159 pub fn eq(self, rhs: impl Into<Self>) -> PartitionExpr {
160 PartitionExpr::new(self, RestrictedOp::Eq, rhs.into())
161 }
162
163 pub fn not_eq(self, rhs: impl Into<Self>) -> PartitionExpr {
164 PartitionExpr::new(self, RestrictedOp::NotEq, rhs.into())
165 }
166
167 pub fn gt(self, rhs: impl Into<Self>) -> PartitionExpr {
168 PartitionExpr::new(self, RestrictedOp::Gt, rhs.into())
169 }
170
171 pub fn lt_eq(self, rhs: impl Into<Self>) -> PartitionExpr {
172 PartitionExpr::new(self, RestrictedOp::LtEq, rhs.into())
173 }
174}
175
176impl Display for Operand {
177 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
178 match self {
179 Self::Column(v) => write!(f, "{v}"),
180 Self::Value(v) => write!(f, "{v}"),
181 Self::Expr(v) => write!(f, "{v}"),
182 Self::IcebergHash(v) => write!(f, "{v}"),
183 }
184 }
185}
186
187#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
190pub enum RestrictedOp {
191 Eq,
193 NotEq,
194 Lt,
195 LtEq,
196 Gt,
197 GtEq,
198
199 And,
201 Or,
202}
203
204impl RestrictedOp {
205 pub fn try_from_parser(op: &ParserBinaryOperator) -> Option<Self> {
206 match op {
207 ParserBinaryOperator::Eq => Some(Self::Eq),
208 ParserBinaryOperator::NotEq => Some(Self::NotEq),
209 ParserBinaryOperator::Lt => Some(Self::Lt),
210 ParserBinaryOperator::LtEq => Some(Self::LtEq),
211 ParserBinaryOperator::Gt => Some(Self::Gt),
212 ParserBinaryOperator::GtEq => Some(Self::GtEq),
213 ParserBinaryOperator::And => Some(Self::And),
214 ParserBinaryOperator::Or => Some(Self::Or),
215 _ => None,
216 }
217 }
218
219 pub fn to_parser_op(&self) -> ParserBinaryOperator {
220 match self {
221 Self::Eq => ParserBinaryOperator::Eq,
222 Self::NotEq => ParserBinaryOperator::NotEq,
223 Self::Lt => ParserBinaryOperator::Lt,
224 Self::LtEq => ParserBinaryOperator::LtEq,
225 Self::Gt => ParserBinaryOperator::Gt,
226 Self::GtEq => ParserBinaryOperator::GtEq,
227 Self::And => ParserBinaryOperator::And,
228 Self::Or => ParserBinaryOperator::Or,
229 }
230 }
231
232 fn invert_for_swap(&self) -> Self {
233 match self {
234 Self::Eq => Self::Eq,
235 Self::NotEq => Self::NotEq,
236 Self::Lt => Self::Gt,
237 Self::LtEq => Self::GtEq,
238 Self::Gt => Self::Lt,
239 Self::GtEq => Self::LtEq,
240 Self::And => Self::And,
241 Self::Or => Self::Or,
242 }
243 }
244}
245impl Display for RestrictedOp {
246 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
247 match self {
248 Self::Eq => write!(f, "="),
249 Self::NotEq => write!(f, "<>"),
250 Self::Lt => write!(f, "<"),
251 Self::LtEq => write!(f, "<="),
252 Self::Gt => write!(f, ">"),
253 Self::GtEq => write!(f, ">="),
254 Self::And => write!(f, "AND"),
255 Self::Or => write!(f, "OR"),
256 }
257 }
258}
259
260impl PartitionExpr {
261 pub fn new(lhs: Operand, op: RestrictedOp, rhs: Operand) -> Self {
262 Self {
263 lhs: Box::new(lhs),
264 op,
265 rhs: Box::new(rhs),
266 }
267 .canonicalize()
268 }
269
270 pub fn canonicalize(self) -> Self {
272 let lhs = Self::canonicalize_operand(*self.lhs);
273 let rhs = Self::canonicalize_operand(*self.rhs);
274 let mut expr = Self {
275 lhs: Box::new(lhs),
276 op: self.op,
277 rhs: Box::new(rhs),
278 };
279
280 if matches!(&*expr.lhs, Operand::Value(_)) && expr.rhs.is_dimension() {
281 std::mem::swap(&mut expr.lhs, &mut expr.rhs);
282 expr.op = expr.op.invert_for_swap();
283 }
284
285 expr
286 }
287
288 fn canonicalize_operand(operand: Operand) -> Operand {
289 match operand {
290 Operand::Expr(expr) => Operand::Expr(expr.canonicalize()),
291 other => other,
292 }
293 }
294
295 pub fn to_parser_expr(&self) -> ParserExpr {
299 ParserExpr::BinaryOp {
300 left: Box::new(self.lhs.to_parser_expr()),
301 op: self.op.to_parser_op(),
302 right: Box::new(self.rhs.to_parser_expr()),
303 }
304 }
305
306 pub fn try_as_logical_expr(&self) -> error::Result<Expr> {
307 let comparison = match (self.lhs.as_ref(), self.rhs.as_ref()) {
311 (column, Operand::Value(value)) if column.is_dimension() => {
312 Some((column, value, self.op.clone()))
313 }
314 (Operand::Value(value), column) if column.is_dimension() => {
315 Some((column, value, self.op.invert_for_swap()))
316 }
317 _ => None,
318 };
319 if let Some((column, value, op)) = comparison {
320 let column = column.try_as_logical_expr()?;
321 if matches!(value, Value::Null) {
322 return match op {
323 RestrictedOp::Eq | RestrictedOp::LtEq => Ok(column.is_null()),
324 RestrictedOp::NotEq | RestrictedOp::Gt => Ok(column.is_not_null()),
325 RestrictedOp::Lt => Ok(datafusion_expr::lit(false)),
326 RestrictedOp::GtEq => Ok(datafusion_expr::lit(true)),
327 RestrictedOp::And | RestrictedOp::Or => {
328 error::InvalidExprSnafu { expr: self.clone() }.fail()
329 }
330 };
331 }
332 let bound = Operand::Value(value.clone()).try_as_logical_expr()?;
333 return match op {
334 RestrictedOp::Eq => Ok(column.eq(bound)),
335 RestrictedOp::NotEq => Ok(column.clone().not_eq(bound).or(column.is_null())),
336 RestrictedOp::Lt => Ok(column.clone().lt(bound).or(column.is_null())),
337 RestrictedOp::LtEq => Ok(column.clone().lt_eq(bound).or(column.is_null())),
338 RestrictedOp::Gt => Ok(column.gt(bound)),
339 RestrictedOp::GtEq => Ok(column.gt_eq(bound)),
340 RestrictedOp::And | RestrictedOp::Or => {
341 error::InvalidExprSnafu { expr: self.clone() }.fail()
342 }
343 };
344 }
345
346 let lhs = self.lhs.try_as_logical_expr()?;
348 let rhs = self.rhs.try_as_logical_expr()?;
349
350 let expr = match &self.op {
351 RestrictedOp::And => datafusion_expr::and(lhs, rhs),
352 RestrictedOp::Or => datafusion_expr::or(lhs, rhs),
353 RestrictedOp::Gt => lhs.gt(rhs),
354 RestrictedOp::GtEq => lhs.gt_eq(rhs),
355 RestrictedOp::Lt => lhs.lt(rhs),
356 RestrictedOp::LtEq => lhs.lt_eq(rhs),
357 RestrictedOp::Eq => lhs.eq(rhs),
358 RestrictedOp::NotEq => lhs.not_eq(rhs),
359 };
360 Ok(expr)
361 }
362
363 pub fn lhs(&self) -> &Operand {
365 &self.lhs
366 }
367
368 pub fn rhs(&self) -> &Operand {
370 &self.rhs
371 }
372
373 pub fn op(&self) -> &RestrictedOp {
375 &self.op
376 }
377
378 pub fn try_as_physical_expr(
379 &self,
380 schema: &arrow::datatypes::SchemaRef,
381 ) -> error::Result<Arc<dyn PhysicalExpr>> {
382 plan_physical_expr(self.try_as_logical_expr()?, schema)
383 }
384
385 pub fn and(self, rhs: PartitionExpr) -> PartitionExpr {
386 PartitionExpr::new(Operand::Expr(self), RestrictedOp::And, Operand::Expr(rhs))
387 }
388
389 pub fn as_json_str(&self) -> error::Result<String> {
393 serde_json::to_string(&PartitionBound::Expr(self.clone()))
394 .context(error::SerializeJsonSnafu)
395 }
396
397 pub fn from_json_str(s: &str) -> error::Result<Option<Self>> {
401 if s.is_empty() {
402 return Ok(None);
403 }
404
405 let bound: PartitionBound = serde_json::from_str(s).context(error::DeserializeJsonSnafu)?;
406 match bound {
407 PartitionBound::Expr(expr) => Ok(Some(expr.canonicalize())),
408 _ => Ok(None),
409 }
410 }
411
412 pub fn as_pb_partition(&self) -> error::Result<Partition> {
414 Ok(Partition {
415 expression: self.as_json_str()?,
416 ..Default::default()
417 })
418 }
419
420 pub fn collect_column_names(&self, columns: &mut HashSet<String>) {
422 Self::collect_operand_columns(&self.lhs, columns);
423 Self::collect_operand_columns(&self.rhs, columns);
424 }
425
426 fn collect_operand_columns(operand: &Operand, columns: &mut HashSet<String>) {
427 match operand {
428 Operand::Column(c) => {
429 columns.insert(c.clone());
430 }
431 Operand::Expr(e) => {
432 e.collect_column_names(columns);
433 }
434 Operand::IcebergHash(f) => {
435 columns.insert(f.column().to_string());
436 }
437 Operand::Value(_) => {}
438 }
439 }
440}
441
442pub(crate) fn plan_physical_expr(
444 expr: Expr,
445 schema: &arrow::datatypes::SchemaRef,
446) -> error::Result<Arc<dyn PhysicalExpr>> {
447 let df_schema = schema
448 .clone()
449 .to_dfschema_ref()
450 .context(error::ToDFSchemaSnafu)?;
451 let simplifier = ExprSimplifier::new(
457 SimplifyContext::builder()
458 .with_schema(df_schema.clone())
459 .build(),
460 );
461 let expr = simplifier
462 .coerce(expr, &df_schema)
463 .context(error::CoercePartitionExprSnafu)?;
464 create_physical_expr(
465 &expr,
466 &df_schema,
467 &ExecutionProps::default(),
468 &PhysicalPlanningContext::default(),
469 )
470 .context(error::CreatePhysicalExprSnafu)
471}
472
473impl Display for PartitionExpr {
474 fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
475 write!(f, "{} {} {}", self.lhs, self.op, self.rhs)
476 }
477}
478
479#[cfg(test)]
480mod tests {
481 use std::collections::HashMap;
482
483 use common_meta::ddl::alter_table::validate_region_partition_bounds;
484 use common_meta::error::Error;
485 use common_meta::rpc::router::{Region, RegionRoute};
486 use datatypes::data_type::ConcreteDataType;
487 use store_api::storage::RegionId;
488
489 use super::*;
490
491 #[test]
492 fn test_expression_hash_respects_value_equality() {
493 use std::collections::hash_map::RandomState;
494 use std::hash::BuildHasher;
495
496 use common_time::Timestamp;
497
498 let seconds = col("ts").lt(Value::Timestamp(Timestamp::new_second(1)));
499 let millis = col("ts").lt(Value::Timestamp(Timestamp::new_millisecond(1000)));
500 assert_eq!(seconds, millis);
501 let hasher = RandomState::new();
502 assert_eq!(hasher.hash_one(&seconds), hasher.hash_one(&millis));
503 for value in [Value::Int32(10), Value::Int64(10)] {
504 let expr = col("tag").lt(value);
505 let decoded: PartitionExpr =
506 serde_json::from_slice(&serde_json::to_vec(&expr).unwrap()).unwrap();
507 assert_eq!(expr, decoded);
508 assert_eq!(hasher.hash_one(&expr), hasher.hash_one(&decoded));
509 }
510 }
511
512 #[test]
513 fn test_partition_expr() {
514 let cases = [
515 (
516 Operand::Column("a".to_string()),
517 RestrictedOp::Eq,
518 Operand::Value(Value::UInt32(10)),
519 "a = 10",
520 ),
521 (
522 Operand::Column("a".to_string()),
523 RestrictedOp::NotEq,
524 Operand::Value(Value::UInt32(10)),
525 "a <> 10",
526 ),
527 (
528 Operand::Column("a".to_string()),
529 RestrictedOp::Lt,
530 Operand::Value(Value::UInt32(10)),
531 "a < 10",
532 ),
533 (
534 Operand::Column("a".to_string()),
535 RestrictedOp::LtEq,
536 Operand::Value(Value::UInt32(10)),
537 "a <= 10",
538 ),
539 (
540 Operand::Column("a".to_string()),
541 RestrictedOp::Gt,
542 Operand::Value(Value::UInt32(10)),
543 "a > 10",
544 ),
545 (
546 Operand::Column("a".to_string()),
547 RestrictedOp::GtEq,
548 Operand::Value(Value::UInt32(10)),
549 "a >= 10",
550 ),
551 (
552 Operand::Column("a".to_string()),
553 RestrictedOp::And,
554 Operand::Column("b".to_string()),
555 "a AND b",
556 ),
557 (
558 Operand::Column("a".to_string()),
559 RestrictedOp::Or,
560 Operand::Column("b".to_string()),
561 "a OR b",
562 ),
563 (
564 Operand::Column("a".to_string()),
565 RestrictedOp::Or,
566 Operand::Expr(PartitionExpr::new(
567 Operand::Column("c".to_string()),
568 RestrictedOp::And,
569 Operand::Column("d".to_string()),
570 )),
571 "a OR c AND d",
572 ),
573 ];
574
575 for case in cases {
576 let expr = PartitionExpr::new(case.0, case.1.clone(), case.2);
577 assert_eq!(case.3, expr.to_string());
578 }
579 }
580
581 #[test]
582 fn test_try_as_logical_expr_null_equality() {
583 let eq_expr = PartitionExpr::new(
584 Operand::Column("a".to_string()),
585 RestrictedOp::Eq,
586 Operand::Value(Value::Null),
587 );
588 assert_eq!(
589 eq_expr.try_as_logical_expr().unwrap().to_string(),
590 "a IS NULL"
591 );
592
593 let neq_expr = PartitionExpr::new(
594 Operand::Column("a".to_string()),
595 RestrictedOp::NotEq,
596 Operand::Value(Value::Null),
597 );
598 assert_eq!(
599 neq_expr.try_as_logical_expr().unwrap().to_string(),
600 "a IS NOT NULL"
601 );
602 }
603
604 #[test]
605 fn test_try_as_logical_expr_null_range_comparison() {
606 let lt_expr = PartitionExpr::new(
608 Operand::Column("a".to_string()),
609 RestrictedOp::Lt,
610 Operand::Value(Value::Int64(10)),
611 );
612 assert_eq!(
613 lt_expr.try_as_logical_expr().unwrap().to_string(),
614 "a < Int64(10) OR a IS NULL"
615 );
616
617 let lt_expr_rhs_column = PartitionExpr::new(
619 Operand::Value(Value::Int64(10)),
620 RestrictedOp::Lt,
621 Operand::Column("a".to_string()),
622 );
623 assert_eq!(
624 lt_expr_rhs_column
625 .try_as_logical_expr()
626 .unwrap()
627 .to_string(),
628 "a > Int64(10)"
629 );
630
631 let gt_expr = PartitionExpr::new(
633 Operand::Column("a".to_string()),
634 RestrictedOp::Gt,
635 Operand::Value(Value::Int64(10)),
636 );
637 assert_eq!(
638 gt_expr.try_as_logical_expr().unwrap().to_string(),
639 "a > Int64(10)"
640 );
641
642 let gt_expr_rhs_column = PartitionExpr::new(
644 Operand::Value(Value::Int64(10)),
645 RestrictedOp::Gt,
646 Operand::Column("a".to_string()),
647 );
648 assert_eq!(
649 gt_expr_rhs_column
650 .try_as_logical_expr()
651 .unwrap()
652 .to_string(),
653 "a < Int64(10) OR a IS NULL"
654 );
655
656 let gteq_expr = PartitionExpr::new(
658 Operand::Column("a".to_string()),
659 RestrictedOp::GtEq,
660 Operand::Value(Value::Int64(10)),
661 );
662 assert_eq!(
663 gteq_expr.try_as_logical_expr().unwrap().to_string(),
664 "a >= Int64(10)"
665 );
666
667 let lteq_expr = PartitionExpr::new(
669 Operand::Column("a".to_string()),
670 RestrictedOp::LtEq,
671 Operand::Value(Value::Int64(10)),
672 );
673 assert_eq!(
674 lteq_expr.try_as_logical_expr().unwrap().to_string(),
675 "a <= Int64(10) OR a IS NULL"
676 );
677
678 let gteq_expr_rhs_column = PartitionExpr::new(
679 Operand::Value(Value::Int64(10)),
680 RestrictedOp::GtEq,
681 Operand::Column("a".to_string()),
682 );
683 assert_eq!(
684 gteq_expr_rhs_column
685 .try_as_logical_expr()
686 .unwrap()
687 .to_string(),
688 "a <= Int64(10) OR a IS NULL"
689 );
690
691 let lteq_expr_rhs_column = PartitionExpr::new(
692 Operand::Value(Value::Int64(10)),
693 RestrictedOp::LtEq,
694 Operand::Column("a".to_string()),
695 );
696 assert_eq!(
697 lteq_expr_rhs_column
698 .try_as_logical_expr()
699 .unwrap()
700 .to_string(),
701 "a >= Int64(10)"
702 );
703
704 let and_expr = PartitionExpr::new(
705 Operand::Expr(PartitionExpr::new(
706 Operand::Column("a".to_string()),
707 RestrictedOp::LtEq,
708 Operand::Value(Value::Int64(10)),
709 )),
710 RestrictedOp::And,
711 Operand::Expr(PartitionExpr::new(
712 Operand::Column("b".to_string()),
713 RestrictedOp::Gt,
714 Operand::Value(Value::Int64(5)),
715 )),
716 );
717 assert_eq!(
718 and_expr.try_as_logical_expr().unwrap().to_string(),
719 "(a <= Int64(10) OR a IS NULL) AND b > Int64(5)"
720 );
721
722 let and_expr = PartitionExpr::new(
723 Operand::Expr(PartitionExpr::new(
724 Operand::Column("a".to_string()),
725 RestrictedOp::LtEq,
726 Operand::Value(Value::Int64(10)),
727 )),
728 RestrictedOp::And,
729 Operand::Expr(PartitionExpr::new(
730 Operand::Column("a".to_string()),
731 RestrictedOp::Gt,
732 Operand::Value(Value::Int64(5)),
733 )),
734 );
735 assert_eq!(
736 and_expr.try_as_logical_expr().unwrap().to_string(),
737 "(a <= Int64(10) OR a IS NULL) AND a > Int64(5)"
738 );
739
740 let and_expr_strict_lower = PartitionExpr::new(
741 Operand::Expr(PartitionExpr::new(
742 Operand::Column("a".to_string()),
743 RestrictedOp::Lt,
744 Operand::Value(Value::Int64(10)),
745 )),
746 RestrictedOp::And,
747 Operand::Expr(PartitionExpr::new(
748 Operand::Column("a".to_string()),
749 RestrictedOp::GtEq,
750 Operand::Value(Value::Int64(5)),
751 )),
752 );
753 assert_eq!(
754 and_expr_strict_lower
755 .try_as_logical_expr()
756 .unwrap()
757 .to_string(),
758 "(a < Int64(10) OR a IS NULL) AND a >= Int64(5)"
759 );
760
761 let and_expr_rhs_column = PartitionExpr::new(
762 Operand::Expr(PartitionExpr::new(
763 Operand::Value(Value::Int64(10)),
764 RestrictedOp::GtEq,
765 Operand::Column("a".to_string()),
766 )),
767 RestrictedOp::And,
768 Operand::Expr(PartitionExpr::new(
769 Operand::Value(Value::Int64(5)),
770 RestrictedOp::Lt,
771 Operand::Column("a".to_string()),
772 )),
773 );
774 assert_eq!(
775 and_expr_rhs_column
776 .try_as_logical_expr()
777 .unwrap()
778 .to_string(),
779 "(a <= Int64(10) OR a IS NULL) AND a > Int64(5)"
780 );
781
782 let or_expr_same_column = PartitionExpr::new(
783 Operand::Expr(PartitionExpr::new(
784 Operand::Column("a".to_string()),
785 RestrictedOp::LtEq,
786 Operand::Value(Value::Int64(10)),
787 )),
788 RestrictedOp::Or,
789 Operand::Expr(PartitionExpr::new(
790 Operand::Column("a".to_string()),
791 RestrictedOp::Gt,
792 Operand::Value(Value::Int64(5)),
793 )),
794 );
795 assert_eq!(
796 or_expr_same_column
797 .try_as_logical_expr()
798 .unwrap()
799 .to_string(),
800 "a <= Int64(10) OR a IS NULL OR a > Int64(5)"
801 );
802 }
803
804 #[test]
805 fn test_try_as_logical_expr_rhs_column_without_canonicalize() {
806 let gt_expr_rhs_column = PartitionExpr {
807 lhs: Box::new(Operand::Value(Value::Int64(10))),
808 op: RestrictedOp::Gt,
809 rhs: Box::new(Operand::Column("a".to_string())),
810 };
811 assert_eq!(
812 gt_expr_rhs_column
813 .try_as_logical_expr()
814 .unwrap()
815 .to_string(),
816 "a < Int64(10) OR a IS NULL"
817 );
818
819 let gteq_expr_rhs_column = PartitionExpr {
820 lhs: Box::new(Operand::Value(Value::Int64(10))),
821 op: RestrictedOp::GtEq,
822 rhs: Box::new(Operand::Column("a".to_string())),
823 };
824 assert_eq!(
825 gteq_expr_rhs_column
826 .try_as_logical_expr()
827 .unwrap()
828 .to_string(),
829 "a <= Int64(10) OR a IS NULL"
830 );
831 }
832
833 #[test]
834 fn test_serde_partition_expr() {
835 let expr = PartitionExpr::new(
836 Operand::Column("a".to_string()),
837 RestrictedOp::Eq,
838 Operand::Value(Value::UInt32(10)),
839 );
840 let json = expr.as_json_str().unwrap();
841 assert_eq!(
842 json,
843 "{\"Expr\":{\"lhs\":{\"Column\":\"a\"},\"op\":\"Eq\",\"rhs\":{\"Value\":{\"UInt32\":10}}}}"
844 );
845
846 let json = r#"{"Expr":{"lhs":{"Column":"a"},"op":"GtEq","rhs":{"Value":{"UInt32":10}}}}"#;
847 let expr2 = PartitionExpr::from_json_str(json).unwrap().unwrap();
848 let expected = PartitionExpr::new(
849 Operand::Column("a".to_string()),
850 RestrictedOp::GtEq,
851 Operand::Value(Value::UInt32(10)),
852 );
853 assert_eq!(expr2, expected);
854
855 let json = "";
857 let expr3 = PartitionExpr::from_json_str(json).unwrap();
858 assert!(expr3.is_none());
859
860 let json = r#""MaxValue""#;
862 let expr4 = PartitionExpr::from_json_str(json).unwrap();
863 assert!(expr4.is_none());
864
865 let json = r#"{"Value":{"UInt32":10}}"#;
866 let expr5 = PartitionExpr::from_json_str(json).unwrap();
867 assert!(expr5.is_none());
868 }
869
870 #[test]
871 fn test_serde_iceberg_hash_partition_expr() {
872 let hash = Operand::IcebergHash(IcebergHash::new("host"));
873 let expr = hash.clone().lt(Value::Int32(0));
874 let json = expr.as_json_str().unwrap();
875 assert_eq!(
876 json,
877 r#"{"Expr":{"lhs":{"IcebergHash":"host"},"op":"Lt","rhs":{"Value":{"Int32":0}}}}"#
878 );
879 assert_eq!(PartitionExpr::from_json_str(&json).unwrap().unwrap(), expr);
880
881 let swapped =
882 r#"{"Expr":{"lhs":{"Value":{"Int32":0}},"op":"Gt","rhs":{"IcebergHash":"host"}}}"#;
883 assert_eq!(
884 PartitionExpr::from_json_str(swapped).unwrap().unwrap(),
885 expr
886 );
887 assert_eq!(expr.to_string(), "iceberg_hash(host) < 0");
888 }
889
890 #[test]
891 fn test_logical_expr_keeps_quotes_in_column_names() {
892 use datatypes::arrow::datatypes::{DataType, Field, Schema};
893
894 let name = r#"a"b"#;
896 let schema = Arc::new(Schema::new(vec![Field::new(name, DataType::Utf8, true)]));
897 for expr in [
898 col(name).lt(Value::from("m")),
899 Operand::IcebergHash(IcebergHash::new(name)).lt(Value::Int32(0)),
900 ] {
901 expr.try_as_physical_expr(&schema)
902 .unwrap_or_else(|e| panic!("{expr}: {e}"));
903 }
904 }
905
906 #[test]
907 fn test_collect_column_names() {
908 let expr = col("a").eq(Value::Int64(1));
910 let mut columns = HashSet::new();
911 expr.collect_column_names(&mut columns);
912 assert_eq!(columns.len(), 1);
913 assert!(columns.contains("a"));
914
915 let expr = col("a")
917 .gt_eq(Value::Int64(0))
918 .and(col("a").lt(Value::Int64(10)));
919 let mut columns = HashSet::new();
920 expr.collect_column_names(&mut columns);
921 assert_eq!(columns.len(), 1);
922 assert!(columns.contains("a"));
923
924 let expr = col("a")
926 .gt_eq(Value::Int64(0))
927 .and(col("b").lt(Value::Int64(10)));
928 let mut columns = HashSet::new();
929 expr.collect_column_names(&mut columns);
930 assert_eq!(columns.len(), 2);
931 assert!(columns.contains("a"));
932 assert!(columns.contains("b"));
933
934 let expr = col("a")
936 .gt_eq(Value::Int64(0))
937 .and(col("b").lt(Value::Int64(10)))
938 .and(col("c").eq(Value::Int64(5)));
939 let mut columns = HashSet::new();
940 expr.collect_column_names(&mut columns);
941 assert_eq!(columns.len(), 3);
942 assert!(columns.contains("a"));
943 assert!(columns.contains("b"));
944 assert!(columns.contains("c"));
945 }
946
947 #[test]
950 fn test_serialized_expressions_validate_against_type_changes() {
951 let overflow_ms = serde_json::from_str::<Value>(
952 r#"{"Timestamp":{"value":32503680000000,"unit":"Millisecond"}}"#,
953 )
954 .unwrap();
955 let start_ms =
956 serde_json::from_str::<Value>(r#"{"Timestamp":{"value":0,"unit":"Millisecond"}}"#)
957 .unwrap();
958 let ts_ms = ConcreteDataType::timestamp_millisecond_datatype();
959 let ts_ns = ConcreteDataType::timestamp_nanosecond_datatype();
960
961 let routes = |expr: &PartitionExpr| {
962 vec![RegionRoute {
963 region: Region {
964 id: RegionId::new(1024, 1),
965 partition_expr: expr.as_json_str().unwrap(),
966 ..Default::default()
967 },
968 ..Default::default()
969 }]
970 };
971 let changed = |column: &str, old_type: ConcreteDataType, new_type: ConcreteDataType| {
972 HashMap::from([(column.to_string(), (old_type, new_type))])
973 };
974
975 fn walk(expr: &PartitionExpr) {
977 walk_operand(&expr.lhs);
978 walk_operand(&expr.rhs);
979 match expr.op {
980 RestrictedOp::Eq
981 | RestrictedOp::NotEq
982 | RestrictedOp::Lt
983 | RestrictedOp::LtEq
984 | RestrictedOp::Gt
985 | RestrictedOp::GtEq
986 | RestrictedOp::And
987 | RestrictedOp::Or => {}
988 }
989 }
990 fn walk_operand(operand: &Operand) {
991 match operand {
992 Operand::Column(_) | Operand::Value(_) | Operand::IcebergHash(_) => {}
993 Operand::Expr(expr) => walk(expr),
994 }
995 }
996
997 let simple = col("ts").lt(overflow_ms.clone());
998 let nested_and = col("ts")
999 .gt_eq(start_ms.clone())
1000 .and(col("ts").lt(overflow_ms.clone()));
1001 let nested_or = PartitionExpr::new(
1002 Operand::Expr(col("host").eq(Value::String("a".into()))),
1003 RestrictedOp::Or,
1004 Operand::Expr(col("ts").lt(overflow_ms.clone())),
1005 );
1006 let with_hash = col("ts")
1007 .lt(overflow_ms.clone())
1008 .and(Operand::IcebergHash(IcebergHash::new("host")).lt(Value::Int32(0)));
1009 for expr in [&simple, &nested_and, &nested_or, &with_hash] {
1010 walk(expr);
1011 let err = validate_region_partition_bounds(
1012 &changed("ts", ts_ms.clone(), ts_ns.clone()),
1013 &routes(expr),
1014 )
1015 .unwrap_err();
1016 assert!(
1017 matches!(
1018 err,
1019 Error::PartitionExprIncompatible { ref column, .. } if column == "ts"
1020 ),
1021 "unexpected error: {err}"
1022 );
1023 }
1024
1025 let compatible = col("ts").lt(start_ms);
1027 validate_region_partition_bounds(&changed("ts", ts_ms, ts_ns), &routes(&compatible))
1028 .unwrap();
1029
1030 let narrowing = col("v").lt(Value::Int64(3_000_000_000));
1031 let err = validate_region_partition_bounds(
1032 &changed(
1033 "v",
1034 ConcreteDataType::int64_datatype(),
1035 ConcreteDataType::int32_datatype(),
1036 ),
1037 &routes(&narrowing),
1038 )
1039 .unwrap_err();
1040 assert!(
1041 matches!(
1042 err,
1043 Error::PartitionExprIncompatible { ref column, .. } if column == "v"
1044 ),
1045 "unexpected error: {err}"
1046 );
1047
1048 let hashed = Operand::IcebergHash(IcebergHash::new("v")).lt(Value::Int32(0));
1050 let err = validate_region_partition_bounds(
1051 &changed(
1052 "v",
1053 ConcreteDataType::int32_datatype(),
1054 ConcreteDataType::int64_datatype(),
1055 ),
1056 &routes(&hashed),
1057 )
1058 .unwrap_err();
1059 assert!(
1060 matches!(
1061 err,
1062 Error::PartitionExprIncompatible { ref column, .. } if column == "v"
1063 ),
1064 "unexpected error: {err}"
1065 );
1066 }
1067}