Skip to main content

partition/
expr.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashSet;
16use std::fmt::{Debug, Display, Formatter};
17use std::sync::Arc;
18
19use api::v1::meta::Partition;
20use datafusion_common::{ScalarValue, ToDFSchema};
21use datafusion_expr::Expr;
22use datafusion_expr::execution_props::ExecutionProps;
23use datafusion_expr::physical_planning_context::PhysicalPlanningContext;
24use datafusion_expr::simplify::SimplifyContext;
25use datafusion_optimizer::simplify_expressions::ExprSimplifier;
26use datafusion_physical_expr::{PhysicalExpr, create_physical_expr};
27use datatypes::arrow;
28use datatypes::value::{
29    Value, duration_to_scalar_value, time_to_scalar_value, timestamp_to_scalar_value,
30};
31use serde::{Deserialize, Serialize};
32use snafu::ResultExt;
33use sql::statements::value_to_sql_value;
34use sqlparser::ast::{
35    BinaryOperator as ParserBinaryOperator, Expr as ParserExpr, Function, FunctionArg,
36    FunctionArgExpr, FunctionArgumentList, FunctionArguments, Ident, ObjectName,
37};
38
39use crate::error;
40use crate::function::IcebergHash;
41use crate::partition::PartitionBound;
42
43/// Struct for partition expression. This can be converted back to sqlparser's [Expr].
44/// by [`Self::to_parser_expr`].
45///
46/// [Expr]: sqlparser::ast::Expr
47#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
48pub struct PartitionExpr {
49    pub lhs: Box<Operand>,
50    pub op: RestrictedOp,
51    pub rhs: Box<Operand>,
52}
53
54#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
55pub enum Operand {
56    Column(String),
57    Value(Value),
58    Expr(PartitionExpr),
59    IcebergHash(IcebergHash),
60}
61
62pub fn col(column_name: impl Into<String>) -> Operand {
63    Operand::Column(column_name.into())
64}
65
66impl From<Value> for Operand {
67    fn from(value: Value) -> Self {
68        Operand::Value(value)
69    }
70}
71
72impl Operand {
73    pub fn try_as_logical_expr(&self) -> error::Result<Expr> {
74        match self {
75            Self::Column(c) => Ok(Expr::Column(datafusion_common::Column::from_name(c))),
76            Self::Value(v) => {
77                let scalar_value = match v {
78                    Value::Boolean(v) => ScalarValue::Boolean(Some(*v)),
79                    Value::UInt8(v) => ScalarValue::UInt8(Some(*v)),
80                    Value::UInt16(v) => ScalarValue::UInt16(Some(*v)),
81                    Value::UInt32(v) => ScalarValue::UInt32(Some(*v)),
82                    Value::UInt64(v) => ScalarValue::UInt64(Some(*v)),
83                    Value::Int8(v) => ScalarValue::Int8(Some(*v)),
84                    Value::Int16(v) => ScalarValue::Int16(Some(*v)),
85                    Value::Int32(v) => ScalarValue::Int32(Some(*v)),
86                    Value::Int64(v) => ScalarValue::Int64(Some(*v)),
87                    Value::Float32(v) => ScalarValue::Float32(Some(v.0)),
88                    Value::Float64(v) => ScalarValue::Float64(Some(v.0)),
89                    Value::String(v) => ScalarValue::Utf8(Some(v.as_utf8().to_string())),
90                    Value::Binary(v) => ScalarValue::Binary(Some(v.to_vec())),
91                    Value::Date(v) => ScalarValue::Date32(Some(v.val())),
92                    Value::Null => ScalarValue::Null,
93                    Value::Timestamp(t) => timestamp_to_scalar_value(t.unit(), Some(t.value())),
94                    Value::Time(t) => time_to_scalar_value(*t.unit(), Some(t.value()))
95                        .context(error::ConvertPartitionExprValueSnafu { value: v.clone() })?,
96                    Value::IntervalYearMonth(v) => ScalarValue::IntervalYearMonth(Some(v.to_i32())),
97                    Value::IntervalDayTime(v) => ScalarValue::IntervalDayTime(Some((*v).into())),
98                    Value::IntervalMonthDayNano(v) => {
99                        ScalarValue::IntervalMonthDayNano(Some((*v).into()))
100                    }
101                    Value::Duration(d) => duration_to_scalar_value(d.unit(), Some(d.value())),
102                    Value::Decimal128(d) => {
103                        let (v, p, s) = d.to_scalar_value();
104                        ScalarValue::Decimal128(v, p, s)
105                    }
106                    other => {
107                        return error::UnsupportedPartitionExprValueSnafu {
108                            value: other.clone(),
109                        }
110                        .fail();
111                    }
112                };
113                Ok(datafusion_expr::lit(scalar_value))
114            }
115            Self::Expr(e) => e.try_as_logical_expr(),
116            Self::IcebergHash(f) => Ok(f.to_logical_expr()),
117        }
118    }
119
120    /// Whether this operand is a partition dimension that values are compared against.
121    fn is_dimension(&self) -> bool {
122        matches!(self, Self::Column(_) | Self::IcebergHash(_))
123    }
124
125    fn to_parser_expr(&self) -> ParserExpr {
126        // Safety: Partition rule won't contains unsupported value type.
127        // Otherwise it will be rejected by the parser.
128        match self {
129            Self::Column(c) => ParserExpr::Identifier(Ident::new(c.clone())),
130            Self::Value(v) => ParserExpr::Value(value_to_sql_value(v).unwrap().into()),
131            Self::Expr(e) => e.to_parser_expr(),
132            Self::IcebergHash(f) => ParserExpr::Function(Function {
133                name: ObjectName::from(vec![Ident::new(IcebergHash::NAME)]),
134                uses_odbc_syntax: false,
135                parameters: FunctionArguments::None,
136                args: FunctionArguments::List(FunctionArgumentList {
137                    duplicate_treatment: None,
138                    args: vec![FunctionArg::Unnamed(FunctionArgExpr::Expr(
139                        ParserExpr::Identifier(Ident::new(f.column())),
140                    ))],
141                    clauses: vec![],
142                }),
143                filter: None,
144                null_treatment: None,
145                over: None,
146                within_group: vec![],
147            }),
148        }
149    }
150
151    pub fn lt(self, rhs: impl Into<Self>) -> PartitionExpr {
152        PartitionExpr::new(self, RestrictedOp::Lt, rhs.into())
153    }
154
155    pub fn gt_eq(self, rhs: impl Into<Self>) -> PartitionExpr {
156        PartitionExpr::new(self, RestrictedOp::GtEq, rhs.into())
157    }
158
159    pub fn eq(self, rhs: impl Into<Self>) -> PartitionExpr {
160        PartitionExpr::new(self, RestrictedOp::Eq, rhs.into())
161    }
162
163    pub fn not_eq(self, rhs: impl Into<Self>) -> PartitionExpr {
164        PartitionExpr::new(self, RestrictedOp::NotEq, rhs.into())
165    }
166
167    pub fn gt(self, rhs: impl Into<Self>) -> PartitionExpr {
168        PartitionExpr::new(self, RestrictedOp::Gt, rhs.into())
169    }
170
171    pub fn lt_eq(self, rhs: impl Into<Self>) -> PartitionExpr {
172        PartitionExpr::new(self, RestrictedOp::LtEq, rhs.into())
173    }
174}
175
176impl Display for Operand {
177    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
178        match self {
179            Self::Column(v) => write!(f, "{v}"),
180            Self::Value(v) => write!(f, "{v}"),
181            Self::Expr(v) => write!(f, "{v}"),
182            Self::IcebergHash(v) => write!(f, "{v}"),
183        }
184    }
185}
186
187/// A restricted set of [Operator](datafusion_expr::Operator) that can be used in
188/// partition expressions.
189#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
190pub enum RestrictedOp {
191    // Evaluate to binary
192    Eq,
193    NotEq,
194    Lt,
195    LtEq,
196    Gt,
197    GtEq,
198
199    // Conjunction
200    And,
201    Or,
202}
203
204impl RestrictedOp {
205    pub fn try_from_parser(op: &ParserBinaryOperator) -> Option<Self> {
206        match op {
207            ParserBinaryOperator::Eq => Some(Self::Eq),
208            ParserBinaryOperator::NotEq => Some(Self::NotEq),
209            ParserBinaryOperator::Lt => Some(Self::Lt),
210            ParserBinaryOperator::LtEq => Some(Self::LtEq),
211            ParserBinaryOperator::Gt => Some(Self::Gt),
212            ParserBinaryOperator::GtEq => Some(Self::GtEq),
213            ParserBinaryOperator::And => Some(Self::And),
214            ParserBinaryOperator::Or => Some(Self::Or),
215            _ => None,
216        }
217    }
218
219    pub fn to_parser_op(&self) -> ParserBinaryOperator {
220        match self {
221            Self::Eq => ParserBinaryOperator::Eq,
222            Self::NotEq => ParserBinaryOperator::NotEq,
223            Self::Lt => ParserBinaryOperator::Lt,
224            Self::LtEq => ParserBinaryOperator::LtEq,
225            Self::Gt => ParserBinaryOperator::Gt,
226            Self::GtEq => ParserBinaryOperator::GtEq,
227            Self::And => ParserBinaryOperator::And,
228            Self::Or => ParserBinaryOperator::Or,
229        }
230    }
231
232    fn invert_for_swap(&self) -> Self {
233        match self {
234            Self::Eq => Self::Eq,
235            Self::NotEq => Self::NotEq,
236            Self::Lt => Self::Gt,
237            Self::LtEq => Self::GtEq,
238            Self::Gt => Self::Lt,
239            Self::GtEq => Self::LtEq,
240            Self::And => Self::And,
241            Self::Or => Self::Or,
242        }
243    }
244}
245impl Display for RestrictedOp {
246    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
247        match self {
248            Self::Eq => write!(f, "="),
249            Self::NotEq => write!(f, "<>"),
250            Self::Lt => write!(f, "<"),
251            Self::LtEq => write!(f, "<="),
252            Self::Gt => write!(f, ">"),
253            Self::GtEq => write!(f, ">="),
254            Self::And => write!(f, "AND"),
255            Self::Or => write!(f, "OR"),
256        }
257    }
258}
259
260impl PartitionExpr {
261    pub fn new(lhs: Operand, op: RestrictedOp, rhs: Operand) -> Self {
262        Self {
263            lhs: Box::new(lhs),
264            op,
265            rhs: Box::new(rhs),
266        }
267        .canonicalize()
268    }
269
270    /// Canonicalize to `Column op Value` form when possible for consistent equality checks.
271    pub fn canonicalize(self) -> Self {
272        let lhs = Self::canonicalize_operand(*self.lhs);
273        let rhs = Self::canonicalize_operand(*self.rhs);
274        let mut expr = Self {
275            lhs: Box::new(lhs),
276            op: self.op,
277            rhs: Box::new(rhs),
278        };
279
280        if matches!(&*expr.lhs, Operand::Value(_)) && expr.rhs.is_dimension() {
281            std::mem::swap(&mut expr.lhs, &mut expr.rhs);
282            expr.op = expr.op.invert_for_swap();
283        }
284
285        expr
286    }
287
288    fn canonicalize_operand(operand: Operand) -> Operand {
289        match operand {
290            Operand::Expr(expr) => Operand::Expr(expr.canonicalize()),
291            other => other,
292        }
293    }
294
295    /// Convert [Self] back to sqlparser's [Expr]
296    ///
297    /// [Expr]: ParserExpr
298    pub fn to_parser_expr(&self) -> ParserExpr {
299        ParserExpr::BinaryOp {
300            left: Box::new(self.lhs.to_parser_expr()),
301            op: self.op.to_parser_op(),
302            right: Box::new(self.rhs.to_parser_expr()),
303        }
304    }
305
306    pub fn try_as_logical_expr(&self) -> error::Result<Expr> {
307        // Row routing compares `Value`s, which order NULL before every other value.
308        // Batch routing and region filters treat a NULL result as false, so only the
309        // comparisons that hold for NULL need an explicit `IS NULL`.
310        let comparison = match (self.lhs.as_ref(), self.rhs.as_ref()) {
311            (column, Operand::Value(value)) if column.is_dimension() => {
312                Some((column, value, self.op.clone()))
313            }
314            (Operand::Value(value), column) if column.is_dimension() => {
315                Some((column, value, self.op.invert_for_swap()))
316            }
317            _ => None,
318        };
319        if let Some((column, value, op)) = comparison {
320            let column = column.try_as_logical_expr()?;
321            if matches!(value, Value::Null) {
322                return match op {
323                    RestrictedOp::Eq | RestrictedOp::LtEq => Ok(column.is_null()),
324                    RestrictedOp::NotEq | RestrictedOp::Gt => Ok(column.is_not_null()),
325                    RestrictedOp::Lt => Ok(datafusion_expr::lit(false)),
326                    RestrictedOp::GtEq => Ok(datafusion_expr::lit(true)),
327                    RestrictedOp::And | RestrictedOp::Or => {
328                        error::InvalidExprSnafu { expr: self.clone() }.fail()
329                    }
330                };
331            }
332            let bound = Operand::Value(value.clone()).try_as_logical_expr()?;
333            return match op {
334                RestrictedOp::Eq => Ok(column.eq(bound)),
335                RestrictedOp::NotEq => Ok(column.clone().not_eq(bound).or(column.is_null())),
336                RestrictedOp::Lt => Ok(column.clone().lt(bound).or(column.is_null())),
337                RestrictedOp::LtEq => Ok(column.clone().lt_eq(bound).or(column.is_null())),
338                RestrictedOp::Gt => Ok(column.gt(bound)),
339                RestrictedOp::GtEq => Ok(column.gt_eq(bound)),
340                RestrictedOp::And | RestrictedOp::Or => {
341                    error::InvalidExprSnafu { expr: self.clone() }.fail()
342                }
343            };
344        }
345
346        // Normal cases handling, without NULL
347        let lhs = self.lhs.try_as_logical_expr()?;
348        let rhs = self.rhs.try_as_logical_expr()?;
349
350        let expr = match &self.op {
351            RestrictedOp::And => datafusion_expr::and(lhs, rhs),
352            RestrictedOp::Or => datafusion_expr::or(lhs, rhs),
353            RestrictedOp::Gt => lhs.gt(rhs),
354            RestrictedOp::GtEq => lhs.gt_eq(rhs),
355            RestrictedOp::Lt => lhs.lt(rhs),
356            RestrictedOp::LtEq => lhs.lt_eq(rhs),
357            RestrictedOp::Eq => lhs.eq(rhs),
358            RestrictedOp::NotEq => lhs.not_eq(rhs),
359        };
360        Ok(expr)
361    }
362
363    /// Get the left-hand side operand
364    pub fn lhs(&self) -> &Operand {
365        &self.lhs
366    }
367
368    /// Get the right-hand side operand
369    pub fn rhs(&self) -> &Operand {
370        &self.rhs
371    }
372
373    /// Get the operation
374    pub fn op(&self) -> &RestrictedOp {
375        &self.op
376    }
377
378    pub fn try_as_physical_expr(
379        &self,
380        schema: &arrow::datatypes::SchemaRef,
381    ) -> error::Result<Arc<dyn PhysicalExpr>> {
382        plan_physical_expr(self.try_as_logical_expr()?, schema)
383    }
384
385    pub fn and(self, rhs: PartitionExpr) -> PartitionExpr {
386        PartitionExpr::new(Operand::Expr(self), RestrictedOp::And, Operand::Expr(rhs))
387    }
388
389    /// Serializes `PartitionExpr` to json string.
390    ///
391    /// Wraps `PartitionBound::Expr` for compatibility.
392    pub fn as_json_str(&self) -> error::Result<String> {
393        serde_json::to_string(&PartitionBound::Expr(self.clone()))
394            .context(error::SerializeJsonSnafu)
395    }
396
397    /// Deserializes `PartitionExpr` from json string.
398    ///
399    /// Deserializes to `PartitionBound` for compatibility.
400    pub fn from_json_str(s: &str) -> error::Result<Option<Self>> {
401        if s.is_empty() {
402            return Ok(None);
403        }
404
405        let bound: PartitionBound = serde_json::from_str(s).context(error::DeserializeJsonSnafu)?;
406        match bound {
407            PartitionBound::Expr(expr) => Ok(Some(expr.canonicalize())),
408            _ => Ok(None),
409        }
410    }
411
412    /// Converts [Self] to [Partition].
413    pub fn as_pb_partition(&self) -> error::Result<Partition> {
414        Ok(Partition {
415            expression: self.as_json_str()?,
416            ..Default::default()
417        })
418    }
419
420    /// Collects all column names referenced by this expression.
421    pub fn collect_column_names(&self, columns: &mut HashSet<String>) {
422        Self::collect_operand_columns(&self.lhs, columns);
423        Self::collect_operand_columns(&self.rhs, columns);
424    }
425
426    fn collect_operand_columns(operand: &Operand, columns: &mut HashSet<String>) {
427        match operand {
428            Operand::Column(c) => {
429                columns.insert(c.clone());
430            }
431            Operand::Expr(e) => {
432                e.collect_column_names(columns);
433            }
434            Operand::IcebergHash(f) => {
435                columns.insert(f.column().to_string());
436            }
437            Operand::Value(_) => {}
438        }
439    }
440}
441
442/// Plans a logical partition expression for evaluation against `schema`.
443pub(crate) fn plan_physical_expr(
444    expr: Expr,
445    schema: &arrow::datatypes::SchemaRef,
446) -> error::Result<Arc<dyn PhysicalExpr>> {
447    let df_schema = schema
448        .clone()
449        .to_dfschema_ref()
450        .context(error::ToDFSchemaSnafu)?;
451    // Persisted partition bounds keep the unit (and type) they were created
452    // with, so after a schema change like widening the time index unit the
453    // expression may mix types with the columns it is evaluated against.
454    // Coerce it against the schema so comparisons share a common type, the
455    // same way `PredicateGroup` does for region partition filters.
456    let simplifier = ExprSimplifier::new(
457        SimplifyContext::builder()
458            .with_schema(df_schema.clone())
459            .build(),
460    );
461    let expr = simplifier
462        .coerce(expr, &df_schema)
463        .context(error::CoercePartitionExprSnafu)?;
464    create_physical_expr(
465        &expr,
466        &df_schema,
467        &ExecutionProps::default(),
468        &PhysicalPlanningContext::default(),
469    )
470    .context(error::CreatePhysicalExprSnafu)
471}
472
473impl Display for PartitionExpr {
474    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
475        write!(f, "{} {} {}", self.lhs, self.op, self.rhs)
476    }
477}
478
479#[cfg(test)]
480mod tests {
481    use std::collections::HashMap;
482
483    use common_meta::ddl::alter_table::validate_region_partition_bounds;
484    use common_meta::error::Error;
485    use common_meta::rpc::router::{Region, RegionRoute};
486    use datatypes::data_type::ConcreteDataType;
487    use store_api::storage::RegionId;
488
489    use super::*;
490
491    #[test]
492    fn test_expression_hash_respects_value_equality() {
493        use std::collections::hash_map::RandomState;
494        use std::hash::BuildHasher;
495
496        use common_time::Timestamp;
497
498        let seconds = col("ts").lt(Value::Timestamp(Timestamp::new_second(1)));
499        let millis = col("ts").lt(Value::Timestamp(Timestamp::new_millisecond(1000)));
500        assert_eq!(seconds, millis);
501        let hasher = RandomState::new();
502        assert_eq!(hasher.hash_one(&seconds), hasher.hash_one(&millis));
503        for value in [Value::Int32(10), Value::Int64(10)] {
504            let expr = col("tag").lt(value);
505            let decoded: PartitionExpr =
506                serde_json::from_slice(&serde_json::to_vec(&expr).unwrap()).unwrap();
507            assert_eq!(expr, decoded);
508            assert_eq!(hasher.hash_one(&expr), hasher.hash_one(&decoded));
509        }
510    }
511
512    #[test]
513    fn test_partition_expr() {
514        let cases = [
515            (
516                Operand::Column("a".to_string()),
517                RestrictedOp::Eq,
518                Operand::Value(Value::UInt32(10)),
519                "a = 10",
520            ),
521            (
522                Operand::Column("a".to_string()),
523                RestrictedOp::NotEq,
524                Operand::Value(Value::UInt32(10)),
525                "a <> 10",
526            ),
527            (
528                Operand::Column("a".to_string()),
529                RestrictedOp::Lt,
530                Operand::Value(Value::UInt32(10)),
531                "a < 10",
532            ),
533            (
534                Operand::Column("a".to_string()),
535                RestrictedOp::LtEq,
536                Operand::Value(Value::UInt32(10)),
537                "a <= 10",
538            ),
539            (
540                Operand::Column("a".to_string()),
541                RestrictedOp::Gt,
542                Operand::Value(Value::UInt32(10)),
543                "a > 10",
544            ),
545            (
546                Operand::Column("a".to_string()),
547                RestrictedOp::GtEq,
548                Operand::Value(Value::UInt32(10)),
549                "a >= 10",
550            ),
551            (
552                Operand::Column("a".to_string()),
553                RestrictedOp::And,
554                Operand::Column("b".to_string()),
555                "a AND b",
556            ),
557            (
558                Operand::Column("a".to_string()),
559                RestrictedOp::Or,
560                Operand::Column("b".to_string()),
561                "a OR b",
562            ),
563            (
564                Operand::Column("a".to_string()),
565                RestrictedOp::Or,
566                Operand::Expr(PartitionExpr::new(
567                    Operand::Column("c".to_string()),
568                    RestrictedOp::And,
569                    Operand::Column("d".to_string()),
570                )),
571                "a OR c AND d",
572            ),
573        ];
574
575        for case in cases {
576            let expr = PartitionExpr::new(case.0, case.1.clone(), case.2);
577            assert_eq!(case.3, expr.to_string());
578        }
579    }
580
581    #[test]
582    fn test_try_as_logical_expr_null_equality() {
583        let eq_expr = PartitionExpr::new(
584            Operand::Column("a".to_string()),
585            RestrictedOp::Eq,
586            Operand::Value(Value::Null),
587        );
588        assert_eq!(
589            eq_expr.try_as_logical_expr().unwrap().to_string(),
590            "a IS NULL"
591        );
592
593        let neq_expr = PartitionExpr::new(
594            Operand::Column("a".to_string()),
595            RestrictedOp::NotEq,
596            Operand::Value(Value::Null),
597        );
598        assert_eq!(
599            neq_expr.try_as_logical_expr().unwrap().to_string(),
600            "a IS NOT NULL"
601        );
602    }
603
604    #[test]
605    fn test_try_as_logical_expr_null_range_comparison() {
606        // Test Lt with column on LHS
607        let lt_expr = PartitionExpr::new(
608            Operand::Column("a".to_string()),
609            RestrictedOp::Lt,
610            Operand::Value(Value::Int64(10)),
611        );
612        assert_eq!(
613            lt_expr.try_as_logical_expr().unwrap().to_string(),
614            "a < Int64(10) OR a IS NULL"
615        );
616
617        // Test Lt with column on RHS
618        let lt_expr_rhs_column = PartitionExpr::new(
619            Operand::Value(Value::Int64(10)),
620            RestrictedOp::Lt,
621            Operand::Column("a".to_string()),
622        );
623        assert_eq!(
624            lt_expr_rhs_column
625                .try_as_logical_expr()
626                .unwrap()
627                .to_string(),
628            "a > Int64(10)"
629        );
630
631        // Test Gt with column on LHS
632        let gt_expr = PartitionExpr::new(
633            Operand::Column("a".to_string()),
634            RestrictedOp::Gt,
635            Operand::Value(Value::Int64(10)),
636        );
637        assert_eq!(
638            gt_expr.try_as_logical_expr().unwrap().to_string(),
639            "a > Int64(10)"
640        );
641
642        // Test Gt with column on RHS
643        let gt_expr_rhs_column = PartitionExpr::new(
644            Operand::Value(Value::Int64(10)),
645            RestrictedOp::Gt,
646            Operand::Column("a".to_string()),
647        );
648        assert_eq!(
649            gt_expr_rhs_column
650                .try_as_logical_expr()
651                .unwrap()
652                .to_string(),
653            "a < Int64(10) OR a IS NULL"
654        );
655
656        // Test GtEq with column on LHS
657        let gteq_expr = PartitionExpr::new(
658            Operand::Column("a".to_string()),
659            RestrictedOp::GtEq,
660            Operand::Value(Value::Int64(10)),
661        );
662        assert_eq!(
663            gteq_expr.try_as_logical_expr().unwrap().to_string(),
664            "a >= Int64(10)"
665        );
666
667        // Test LtEq with column on LHS
668        let lteq_expr = PartitionExpr::new(
669            Operand::Column("a".to_string()),
670            RestrictedOp::LtEq,
671            Operand::Value(Value::Int64(10)),
672        );
673        assert_eq!(
674            lteq_expr.try_as_logical_expr().unwrap().to_string(),
675            "a <= Int64(10) OR a IS NULL"
676        );
677
678        let gteq_expr_rhs_column = PartitionExpr::new(
679            Operand::Value(Value::Int64(10)),
680            RestrictedOp::GtEq,
681            Operand::Column("a".to_string()),
682        );
683        assert_eq!(
684            gteq_expr_rhs_column
685                .try_as_logical_expr()
686                .unwrap()
687                .to_string(),
688            "a <= Int64(10) OR a IS NULL"
689        );
690
691        let lteq_expr_rhs_column = PartitionExpr::new(
692            Operand::Value(Value::Int64(10)),
693            RestrictedOp::LtEq,
694            Operand::Column("a".to_string()),
695        );
696        assert_eq!(
697            lteq_expr_rhs_column
698                .try_as_logical_expr()
699                .unwrap()
700                .to_string(),
701            "a >= Int64(10)"
702        );
703
704        let and_expr = PartitionExpr::new(
705            Operand::Expr(PartitionExpr::new(
706                Operand::Column("a".to_string()),
707                RestrictedOp::LtEq,
708                Operand::Value(Value::Int64(10)),
709            )),
710            RestrictedOp::And,
711            Operand::Expr(PartitionExpr::new(
712                Operand::Column("b".to_string()),
713                RestrictedOp::Gt,
714                Operand::Value(Value::Int64(5)),
715            )),
716        );
717        assert_eq!(
718            and_expr.try_as_logical_expr().unwrap().to_string(),
719            "(a <= Int64(10) OR a IS NULL) AND b > Int64(5)"
720        );
721
722        let and_expr = PartitionExpr::new(
723            Operand::Expr(PartitionExpr::new(
724                Operand::Column("a".to_string()),
725                RestrictedOp::LtEq,
726                Operand::Value(Value::Int64(10)),
727            )),
728            RestrictedOp::And,
729            Operand::Expr(PartitionExpr::new(
730                Operand::Column("a".to_string()),
731                RestrictedOp::Gt,
732                Operand::Value(Value::Int64(5)),
733            )),
734        );
735        assert_eq!(
736            and_expr.try_as_logical_expr().unwrap().to_string(),
737            "(a <= Int64(10) OR a IS NULL) AND a > Int64(5)"
738        );
739
740        let and_expr_strict_lower = PartitionExpr::new(
741            Operand::Expr(PartitionExpr::new(
742                Operand::Column("a".to_string()),
743                RestrictedOp::Lt,
744                Operand::Value(Value::Int64(10)),
745            )),
746            RestrictedOp::And,
747            Operand::Expr(PartitionExpr::new(
748                Operand::Column("a".to_string()),
749                RestrictedOp::GtEq,
750                Operand::Value(Value::Int64(5)),
751            )),
752        );
753        assert_eq!(
754            and_expr_strict_lower
755                .try_as_logical_expr()
756                .unwrap()
757                .to_string(),
758            "(a < Int64(10) OR a IS NULL) AND a >= Int64(5)"
759        );
760
761        let and_expr_rhs_column = PartitionExpr::new(
762            Operand::Expr(PartitionExpr::new(
763                Operand::Value(Value::Int64(10)),
764                RestrictedOp::GtEq,
765                Operand::Column("a".to_string()),
766            )),
767            RestrictedOp::And,
768            Operand::Expr(PartitionExpr::new(
769                Operand::Value(Value::Int64(5)),
770                RestrictedOp::Lt,
771                Operand::Column("a".to_string()),
772            )),
773        );
774        assert_eq!(
775            and_expr_rhs_column
776                .try_as_logical_expr()
777                .unwrap()
778                .to_string(),
779            "(a <= Int64(10) OR a IS NULL) AND a > Int64(5)"
780        );
781
782        let or_expr_same_column = PartitionExpr::new(
783            Operand::Expr(PartitionExpr::new(
784                Operand::Column("a".to_string()),
785                RestrictedOp::LtEq,
786                Operand::Value(Value::Int64(10)),
787            )),
788            RestrictedOp::Or,
789            Operand::Expr(PartitionExpr::new(
790                Operand::Column("a".to_string()),
791                RestrictedOp::Gt,
792                Operand::Value(Value::Int64(5)),
793            )),
794        );
795        assert_eq!(
796            or_expr_same_column
797                .try_as_logical_expr()
798                .unwrap()
799                .to_string(),
800            "a <= Int64(10) OR a IS NULL OR a > Int64(5)"
801        );
802    }
803
804    #[test]
805    fn test_try_as_logical_expr_rhs_column_without_canonicalize() {
806        let gt_expr_rhs_column = PartitionExpr {
807            lhs: Box::new(Operand::Value(Value::Int64(10))),
808            op: RestrictedOp::Gt,
809            rhs: Box::new(Operand::Column("a".to_string())),
810        };
811        assert_eq!(
812            gt_expr_rhs_column
813                .try_as_logical_expr()
814                .unwrap()
815                .to_string(),
816            "a < Int64(10) OR a IS NULL"
817        );
818
819        let gteq_expr_rhs_column = PartitionExpr {
820            lhs: Box::new(Operand::Value(Value::Int64(10))),
821            op: RestrictedOp::GtEq,
822            rhs: Box::new(Operand::Column("a".to_string())),
823        };
824        assert_eq!(
825            gteq_expr_rhs_column
826                .try_as_logical_expr()
827                .unwrap()
828                .to_string(),
829            "a <= Int64(10) OR a IS NULL"
830        );
831    }
832
833    #[test]
834    fn test_serde_partition_expr() {
835        let expr = PartitionExpr::new(
836            Operand::Column("a".to_string()),
837            RestrictedOp::Eq,
838            Operand::Value(Value::UInt32(10)),
839        );
840        let json = expr.as_json_str().unwrap();
841        assert_eq!(
842            json,
843            "{\"Expr\":{\"lhs\":{\"Column\":\"a\"},\"op\":\"Eq\",\"rhs\":{\"Value\":{\"UInt32\":10}}}}"
844        );
845
846        let json = r#"{"Expr":{"lhs":{"Column":"a"},"op":"GtEq","rhs":{"Value":{"UInt32":10}}}}"#;
847        let expr2 = PartitionExpr::from_json_str(json).unwrap().unwrap();
848        let expected = PartitionExpr::new(
849            Operand::Column("a".to_string()),
850            RestrictedOp::GtEq,
851            Operand::Value(Value::UInt32(10)),
852        );
853        assert_eq!(expr2, expected);
854
855        // empty string
856        let json = "";
857        let expr3 = PartitionExpr::from_json_str(json).unwrap();
858        assert!(expr3.is_none());
859
860        // variants other than Expr
861        let json = r#""MaxValue""#;
862        let expr4 = PartitionExpr::from_json_str(json).unwrap();
863        assert!(expr4.is_none());
864
865        let json = r#"{"Value":{"UInt32":10}}"#;
866        let expr5 = PartitionExpr::from_json_str(json).unwrap();
867        assert!(expr5.is_none());
868    }
869
870    #[test]
871    fn test_serde_iceberg_hash_partition_expr() {
872        let hash = Operand::IcebergHash(IcebergHash::new("host"));
873        let expr = hash.clone().lt(Value::Int32(0));
874        let json = expr.as_json_str().unwrap();
875        assert_eq!(
876            json,
877            r#"{"Expr":{"lhs":{"IcebergHash":"host"},"op":"Lt","rhs":{"Value":{"Int32":0}}}}"#
878        );
879        assert_eq!(PartitionExpr::from_json_str(&json).unwrap().unwrap(), expr);
880
881        let swapped =
882            r#"{"Expr":{"lhs":{"Value":{"Int32":0}},"op":"Gt","rhs":{"IcebergHash":"host"}}}"#;
883        assert_eq!(
884            PartitionExpr::from_json_str(swapped).unwrap().unwrap(),
885            expr
886        );
887        assert_eq!(expr.to_string(), "iceberg_hash(host) < 0");
888    }
889
890    #[test]
891    fn test_logical_expr_keeps_quotes_in_column_names() {
892        use datatypes::arrow::datatypes::{DataType, Field, Schema};
893
894        // Declared as `"a""b"`; the stored name contains the quote.
895        let name = r#"a"b"#;
896        let schema = Arc::new(Schema::new(vec![Field::new(name, DataType::Utf8, true)]));
897        for expr in [
898            col(name).lt(Value::from("m")),
899            Operand::IcebergHash(IcebergHash::new(name)).lt(Value::Int32(0)),
900        ] {
901            expr.try_as_physical_expr(&schema)
902                .unwrap_or_else(|e| panic!("{expr}: {e}"));
903        }
904    }
905
906    #[test]
907    fn test_collect_column_names() {
908        // Simple expression: col_a = 1 should give {col_a}
909        let expr = col("a").eq(Value::Int64(1));
910        let mut columns = HashSet::new();
911        expr.collect_column_names(&mut columns);
912        assert_eq!(columns.len(), 1);
913        assert!(columns.contains("a"));
914
915        // Compound AND with same column: col_a >= 0 AND col_a < 10 should give {col_a}
916        let expr = col("a")
917            .gt_eq(Value::Int64(0))
918            .and(col("a").lt(Value::Int64(10)));
919        let mut columns = HashSet::new();
920        expr.collect_column_names(&mut columns);
921        assert_eq!(columns.len(), 1);
922        assert!(columns.contains("a"));
923
924        // Multiple columns: col_a >= 0 AND col_b < 10 should give {col_a, col_b}
925        let expr = col("a")
926            .gt_eq(Value::Int64(0))
927            .and(col("b").lt(Value::Int64(10)));
928        let mut columns = HashSet::new();
929        expr.collect_column_names(&mut columns);
930        assert_eq!(columns.len(), 2);
931        assert!(columns.contains("a"));
932        assert!(columns.contains("b"));
933
934        // Nested expression: (col_a >= 0 AND col_b < 10) AND col_c = 5
935        let expr = col("a")
936            .gt_eq(Value::Int64(0))
937            .and(col("b").lt(Value::Int64(10)))
938            .and(col("c").eq(Value::Int64(5)));
939        let mut columns = HashSet::new();
940        expr.collect_column_names(&mut columns);
941        assert_eq!(columns.len(), 3);
942        assert!(columns.contains("a"));
943        assert!(columns.contains("b"));
944        assert!(columns.contains("c"));
945    }
946
947    /// `common-meta` mirrors these types to validate ALTERs. If the mirror drifts,
948    /// it can't parse the expressions and skips validation, which fails these cases.
949    #[test]
950    fn test_serialized_expressions_validate_against_type_changes() {
951        let overflow_ms = serde_json::from_str::<Value>(
952            r#"{"Timestamp":{"value":32503680000000,"unit":"Millisecond"}}"#,
953        )
954        .unwrap();
955        let start_ms =
956            serde_json::from_str::<Value>(r#"{"Timestamp":{"value":0,"unit":"Millisecond"}}"#)
957                .unwrap();
958        let ts_ms = ConcreteDataType::timestamp_millisecond_datatype();
959        let ts_ns = ConcreteDataType::timestamp_nanosecond_datatype();
960
961        let routes = |expr: &PartitionExpr| {
962            vec![RegionRoute {
963                region: Region {
964                    id: RegionId::new(1024, 1),
965                    partition_expr: expr.as_json_str().unwrap(),
966                    ..Default::default()
967                },
968                ..Default::default()
969            }]
970        };
971        let changed = |column: &str, old_type: ConcreteDataType, new_type: ConcreteDataType| {
972            HashMap::from([(column.to_string(), (old_type, new_type))])
973        };
974
975        // Fails to compile on a new variant, which the mirror must learn too.
976        fn walk(expr: &PartitionExpr) {
977            walk_operand(&expr.lhs);
978            walk_operand(&expr.rhs);
979            match expr.op {
980                RestrictedOp::Eq
981                | RestrictedOp::NotEq
982                | RestrictedOp::Lt
983                | RestrictedOp::LtEq
984                | RestrictedOp::Gt
985                | RestrictedOp::GtEq
986                | RestrictedOp::And
987                | RestrictedOp::Or => {}
988            }
989        }
990        fn walk_operand(operand: &Operand) {
991            match operand {
992                Operand::Column(_) | Operand::Value(_) | Operand::IcebergHash(_) => {}
993                Operand::Expr(expr) => walk(expr),
994            }
995        }
996
997        let simple = col("ts").lt(overflow_ms.clone());
998        let nested_and = col("ts")
999            .gt_eq(start_ms.clone())
1000            .and(col("ts").lt(overflow_ms.clone()));
1001        let nested_or = PartitionExpr::new(
1002            Operand::Expr(col("host").eq(Value::String("a".into()))),
1003            RestrictedOp::Or,
1004            Operand::Expr(col("ts").lt(overflow_ms.clone())),
1005        );
1006        let with_hash = col("ts")
1007            .lt(overflow_ms.clone())
1008            .and(Operand::IcebergHash(IcebergHash::new("host")).lt(Value::Int32(0)));
1009        for expr in [&simple, &nested_and, &nested_or, &with_hash] {
1010            walk(expr);
1011            let err = validate_region_partition_bounds(
1012                &changed("ts", ts_ms.clone(), ts_ns.clone()),
1013                &routes(expr),
1014            )
1015            .unwrap_err();
1016            assert!(
1017                matches!(
1018                    err,
1019                    Error::PartitionExprIncompatible { ref column, .. } if column == "ts"
1020                ),
1021                "unexpected error: {err}"
1022            );
1023        }
1024
1025        // A representable bound passes.
1026        let compatible = col("ts").lt(start_ms);
1027        validate_region_partition_bounds(&changed("ts", ts_ms, ts_ns), &routes(&compatible))
1028            .unwrap();
1029
1030        let narrowing = col("v").lt(Value::Int64(3_000_000_000));
1031        let err = validate_region_partition_bounds(
1032            &changed(
1033                "v",
1034                ConcreteDataType::int64_datatype(),
1035                ConcreteDataType::int32_datatype(),
1036            ),
1037            &routes(&narrowing),
1038        )
1039        .unwrap_err();
1040        assert!(
1041            matches!(
1042                err,
1043                Error::PartitionExprIncompatible { ref column, .. } if column == "v"
1044            ),
1045            "unexpected error: {err}"
1046        );
1047
1048        // The hash of a column depends on its type.
1049        let hashed = Operand::IcebergHash(IcebergHash::new("v")).lt(Value::Int32(0));
1050        let err = validate_region_partition_bounds(
1051            &changed(
1052                "v",
1053                ConcreteDataType::int32_datatype(),
1054                ConcreteDataType::int64_datatype(),
1055            ),
1056            &routes(&hashed),
1057        )
1058        .unwrap_err();
1059        assert!(
1060            matches!(
1061                err,
1062                Error::PartitionExprIncompatible { ref column, .. } if column == "v"
1063            ),
1064            "unexpected error: {err}"
1065        );
1066    }
1067}