Skip to main content

query/promql/planner/
set_operator.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Planning of the PromQL set operators (`and`, `unless`, and `or`).
16
17use std::collections::{BTreeSet, HashMap, HashSet};
18use std::sync::Arc;
19
20use datafusion::logical_expr::{Cast, Extension, LogicalPlan, LogicalPlanBuilder};
21use datafusion::prelude::{Column, Expr as DfExpr, JoinType};
22use datafusion::scalar::ScalarValue;
23use datafusion_common::{DFSchema, NullEquality, TableReference};
24use datatypes::arrow::datatypes::DataType as ArrowDataType;
25use promql::extension_plan::UnionDistinctOn;
26use promql_parser::parser::token::{self, TokenType};
27use promql_parser::parser::{BinModifier, LabelModifier, VectorMatchCardinality};
28use snafu::{OptionExt, ResultExt, ensure};
29use store_api::metric_engine_consts::DATA_SCHEMA_TSID_COLUMN_NAME;
30
31use crate::promql::error::{
32    ColumnNotFoundSnafu, CombineTableColumnMismatchSnafu, DataFusionPlanningSnafu,
33    MultiFieldsNotSupportedSnafu, Result, TimeIndexNotFoundSnafu, UnexpectedPlanExprSnafu,
34    UnexpectedTokenSnafu, UnsupportedVectorMatchSnafu,
35};
36use crate::promql::planner::{
37    OR_FLOAT_FIELD_PREFIX, OR_HISTOGRAM_FIELD_PREFIX, PromPlanner, PromPlannerContext,
38};
39
40impl PromPlanner {
41    // TODO(ruihang): change function name
42    #[allow(clippy::too_many_arguments)]
43    pub(super) fn or_operator(
44        &mut self,
45        left: LogicalPlan,
46        right: LogicalPlan,
47        left_tag_cols_set: HashSet<String>,
48        right_tag_cols_set: HashSet<String>,
49        left_context: PromPlannerContext,
50        right_context: PromPlannerContext,
51        modifier: &Option<BinModifier>,
52    ) -> Result<LogicalPlan> {
53        let left_is_empty = Self::is_zero_row_empty_relation(&left);
54        let right_is_empty = Self::is_zero_row_empty_relation(&right);
55        match (left_is_empty, right_is_empty) {
56            (true, false) => {
57                self.ctx = right_context;
58                return Ok(right);
59            }
60            (false, true) => {
61                self.ctx = left_context;
62                return Ok(left);
63            }
64            (true, true) => {
65                self.ctx = left_context;
66                return Ok(left);
67            }
68            (false, false) => {}
69        }
70
71        ensure!(
72            !left.schema().fields().is_empty() && !right.schema().fields().is_empty(),
73            UnexpectedPlanExprSnafu {
74                desc: "OR operator input has zero columns",
75            }
76        );
77        let left_has_alternative_samples =
78            Self::field_columns_are_alternative_samples(left.schema(), &left_context.field_columns);
79        let right_has_alternative_samples = Self::field_columns_are_alternative_samples(
80            right.schema(),
81            &right_context.field_columns,
82        );
83        ensure!(
84            left_context.field_columns.len() == 1 || left_has_alternative_samples,
85            MultiFieldsNotSupportedSnafu {
86                operator: "OR operator"
87            }
88        );
89        ensure!(
90            right_context.field_columns.len() == 1 || right_has_alternative_samples,
91            MultiFieldsNotSupportedSnafu {
92                operator: "OR operator"
93            }
94        );
95
96        // prepare hash sets
97        let all_tags = left_tag_cols_set
98            .union(&right_tag_cols_set)
99            .cloned()
100            .collect::<HashSet<_>>();
101        let left_qualifier = left.schema().qualified_field(0).0.cloned();
102        let right_qualifier = right.schema().qualified_field(0).0.cloned();
103        let left_qualifier_string = left_qualifier
104            .as_ref()
105            .map(|l| l.to_string())
106            .unwrap_or_default();
107        let right_qualifier_string = right_qualifier
108            .as_ref()
109            .map(|r| r.to_string())
110            .unwrap_or_default();
111        let left_time_index_column =
112            left_context
113                .time_index_column
114                .clone()
115                .with_context(|| TimeIndexNotFoundSnafu {
116                    table: left_qualifier_string.clone(),
117                })?;
118        let right_time_index_column =
119            right_context
120                .time_index_column
121                .clone()
122                .with_context(|| TimeIndexNotFoundSnafu {
123                    table: right_qualifier_string.clone(),
124                })?;
125        let native_histogram_type = Self::native_histogram_arrow_type();
126        let is_numeric = |data_type: &ArrowDataType| {
127            matches!(
128                data_type,
129                ArrowDataType::Int8
130                    | ArrowDataType::Int16
131                    | ArrowDataType::Int32
132                    | ArrowDataType::Int64
133                    | ArrowDataType::UInt8
134                    | ArrowDataType::UInt16
135                    | ArrowDataType::UInt32
136                    | ArrowDataType::UInt64
137                    | ArrowDataType::Float32
138                    | ArrowDataType::Float64
139            )
140        };
141        let left_fields = left_context
142            .field_columns
143            .iter()
144            .map(|name| {
145                left.schema()
146                    .iter()
147                    .find(|(_, field)| field.name() == name)
148                    .map(|(qualifier, field)| {
149                        (name.clone(), qualifier.cloned(), field.data_type().clone())
150                    })
151                    .with_context(|| ColumnNotFoundSnafu { col: name.clone() })
152            })
153            .collect::<Result<Vec<_>>>()?;
154        let right_fields = right_context
155            .field_columns
156            .iter()
157            .map(|name| {
158                right
159                    .schema()
160                    .iter()
161                    .find(|(_, field)| field.name() == name)
162                    .map(|(qualifier, field)| {
163                        (name.clone(), qualifier.cloned(), field.data_type().clone())
164                    })
165                    .with_context(|| ColumnNotFoundSnafu { col: name.clone() })
166            })
167            .collect::<Result<Vec<_>>>()?;
168        let left_field = &left_fields[0];
169        let right_field = &right_fields[0];
170        let left_field_col = &left_field.0;
171        let right_field_col = &right_field.0;
172        let fields_are_samples = |fields: &[(String, Option<TableReference>, ArrowDataType)]| {
173            fields.iter().all(|(_, _, data_type)| {
174                is_numeric(data_type) || data_type == &native_histogram_type
175            })
176        };
177        let mixed_sample_types = if left_has_alternative_samples || right_has_alternative_samples {
178            if !fields_are_samples(&left_fields) || !fields_are_samples(&right_fields) {
179                return UnexpectedPlanExprSnafu {
180                    desc: format!(
181                        "OR value fields have incompatible types: {:?} and {:?}",
182                        left_fields
183                            .iter()
184                            .map(|(_, _, data_type)| data_type)
185                            .collect::<Vec<_>>(),
186                        right_fields
187                            .iter()
188                            .map(|(_, _, data_type)| data_type)
189                            .collect::<Vec<_>>()
190                    ),
191                }
192                .fail();
193            }
194            true
195        } else {
196            (left_field.2 == native_histogram_type && is_numeric(&right_field.2))
197                || (right_field.2 == native_histogram_type && is_numeric(&left_field.2))
198        };
199        let target_field_type = if mixed_sample_types {
200            // Mixed vectors use the existing response representation: one nullable float column
201            // and one nullable native-histogram column.
202            ArrowDataType::Float64
203        } else if left_field.2 == right_field.2 {
204            left_field.2.clone()
205        } else if is_numeric(&left_field.2) && is_numeric(&right_field.2) {
206            ArrowDataType::Float64
207        } else {
208            return UnexpectedPlanExprSnafu {
209                desc: format!(
210                    "OR value fields have incompatible types: {:?} and {:?}",
211                    left_field.2, right_field.2
212                ),
213            }
214            .fail();
215        };
216        let (mixed_float_field_col, mixed_histogram_field_col) = if mixed_sample_types {
217            let mut reserved_names = left
218                .schema()
219                .fields()
220                .iter()
221                .chain(right.schema().fields().iter())
222                .map(|field| field.name().clone())
223                .collect::<HashSet<_>>();
224            for (name, _, _) in left_fields.iter().chain(&right_fields) {
225                reserved_names.remove(name);
226            }
227            reserved_names.extend(all_tags.iter().cloned());
228            let unique_name = |prefix: &str, reserved_names: &mut HashSet<String>| {
229                let mut index = 0;
230                loop {
231                    let name = format!("{prefix}{index}");
232                    index += 1;
233                    if reserved_names.insert(name.clone()) {
234                        break name;
235                    }
236                }
237            };
238            let float_field = unique_name(OR_FLOAT_FIELD_PREFIX, &mut reserved_names);
239            let histogram_field = unique_name(OR_HISTOGRAM_FIELD_PREFIX, &mut reserved_names);
240            (float_field, histogram_field)
241        } else {
242            (left_field_col.clone(), String::new())
243        };
244        let left_tag_types = left_tag_cols_set
245            .iter()
246            .map(|label| {
247                left.schema()
248                    .fields()
249                    .iter()
250                    .find(|field| field.name() == label)
251                    .map(|field| (label.clone(), field.data_type().clone()))
252                    .with_context(|| ColumnNotFoundSnafu { col: label.clone() })
253            })
254            .collect::<Result<HashMap<_, _>>>()?;
255        let right_tag_types = right_tag_cols_set
256            .iter()
257            .map(|label| {
258                right
259                    .schema()
260                    .fields()
261                    .iter()
262                    .find(|field| field.name() == label)
263                    .map(|field| (label.clone(), field.data_type().clone()))
264                    .with_context(|| ColumnNotFoundSnafu { col: label.clone() })
265            })
266            .collect::<Result<HashMap<_, _>>>()?;
267        let mut target_tag_types = HashMap::with_capacity(all_tags.len());
268        for label in &all_tags {
269            let Some(data_type) =
270                Self::common_label_data_type(left_tag_types.get(label), right_tag_types.get(label))
271            else {
272                return UnexpectedPlanExprSnafu {
273                    desc: format!(
274                        "OR label {label} has incompatible types: {:?} and {:?}",
275                        left_tag_types.get(label),
276                        right_tag_types.get(label)
277                    ),
278                }
279                .fail();
280            };
281            target_tag_types.insert(label.clone(), data_type);
282        }
283        let left_has_tsid = left
284            .schema()
285            .fields()
286            .iter()
287            .any(|field| field.name() == DATA_SCHEMA_TSID_COLUMN_NAME);
288        let right_has_tsid = right
289            .schema()
290            .fields()
291            .iter()
292            .any(|field| field.name() == DATA_SCHEMA_TSID_COLUMN_NAME);
293
294        // step 0: fill all columns in output schema
295        let mut all_columns_set = left
296            .schema()
297            .fields()
298            .iter()
299            .chain(right.schema().fields().iter())
300            .map(|field| field.name().clone())
301            .collect::<HashSet<_>>();
302        // Keep `__tsid` only when both sides contain it, otherwise it may break schema alignment
303        // (e.g. `unknown_metric or some_metric`).
304        if !(left_has_tsid && right_has_tsid) {
305            all_columns_set.remove(DATA_SCHEMA_TSID_COLUMN_NAME);
306        }
307        // remove time index column
308        all_columns_set.remove(&left_time_index_column);
309        all_columns_set.remove(&right_time_index_column);
310        if mixed_sample_types {
311            for (name, _, _) in left_fields.iter().chain(&right_fields) {
312                all_columns_set.remove(name);
313            }
314            all_columns_set.extend(all_tags.iter().cloned());
315            all_columns_set.insert(mixed_float_field_col.clone());
316            all_columns_set.insert(mixed_histogram_field_col.clone());
317        } else if left_field_col != right_field_col {
318            // remove field column in the right
319            all_columns_set.remove(right_field_col);
320        }
321        let mut all_columns = all_columns_set.into_iter().collect::<Vec<_>>();
322        // sort to ensure the generated schema is not volatile
323        all_columns.sort_unstable();
324        // use left time index column name as the result time index column name
325        all_columns.insert(0, left_time_index_column.clone());
326        let mut occupied_column_names = left
327            .schema()
328            .fields()
329            .iter()
330            .chain(right.schema().fields().iter())
331            .map(|field| field.name().clone())
332            .collect::<HashSet<_>>();
333
334        // step 1: align schema using project, fill non-exist columns with null
335        let aligned_label_expr = |col: &String, source_types: &HashMap<String, ArrowDataType>| {
336            let target_type = &target_tag_types[col];
337            if let Some(source_type) = source_types.get(col) {
338                let expr = DfExpr::Column(Column::new(None::<String>, col));
339                if source_type == target_type {
340                    expr
341                } else {
342                    DfExpr::Cast(Cast::new(Box::new(expr), target_type.clone())).alias(col.clone())
343                }
344            } else {
345                DfExpr::Literal(
346                    Self::string_scalar_value(target_type, None)
347                        .expect("target label type is a string"),
348                    None,
349                )
350                .alias(col.clone())
351            }
352        };
353        let null_histogram =
354            ScalarValue::try_new_null(&native_histogram_type).context(DataFusionPlanningSnafu)?;
355        let mixed_value_expr = |fields: &[(String, Option<TableReference>, ArrowDataType)],
356                                output_col: &String| {
357            if output_col == &mixed_float_field_col {
358                if let Some((name, qualifier, data_type)) = fields
359                    .iter()
360                    .find(|(_, _, data_type)| is_numeric(data_type))
361                {
362                    let expr = DfExpr::Column(Column::new(qualifier.clone(), name));
363                    if data_type == &ArrowDataType::Float64 {
364                        expr.alias(output_col)
365                    } else {
366                        DfExpr::Cast(Cast::new(Box::new(expr), ArrowDataType::Float64))
367                            .alias(output_col)
368                    }
369                } else {
370                    DfExpr::Literal(ScalarValue::Float64(None), None).alias(output_col)
371                }
372            } else {
373                fields
374                    .iter()
375                    .find(|(_, _, data_type)| data_type == &native_histogram_type)
376                    .map(|(name, qualifier, _)| {
377                        DfExpr::Column(Column::new(qualifier.clone(), name)).alias(output_col)
378                    })
379                    .unwrap_or_else(|| {
380                        DfExpr::Literal(null_histogram.clone(), None).alias(output_col)
381                    })
382            }
383        };
384        let left_proj_exprs = all_columns.iter().map(|col| {
385            if mixed_sample_types
386                && (col == &mixed_float_field_col || col == &mixed_histogram_field_col)
387            {
388                mixed_value_expr(&left_fields, col)
389            } else if !mixed_sample_types
390                && col == left_field_col
391                && left_field.2 != target_field_type
392            {
393                DfExpr::Cast(Cast::new(
394                    Box::new(DfExpr::Column(Column::new(
395                        left_field.1.clone(),
396                        left_field_col,
397                    ))),
398                    target_field_type.clone(),
399                ))
400                .alias(left_field_col.clone())
401            } else if target_tag_types.contains_key(col) {
402                aligned_label_expr(col, &left_tag_types)
403            } else {
404                DfExpr::Column(Column::new(None::<String>, col))
405            }
406        });
407        // The right operand does not necessarily qualify every column with the qualifier of its
408        // first field: a binary operation with a matching modifier (`on`, `ignoring`, `group_left`,
409        // `group_right`) derives its result labels from the left operand and projects them under
410        // that side's qualifier (`lhs.<label>`), while its time index keeps the qualifier of the
411        // context's table. Resolve the time index against the actual schema so the reference
412        // cannot miss.
413        let right_time_index_expr = DfExpr::Column(Self::resolved_column(
414            right.schema(),
415            right_qualifier.clone(),
416            &right_time_index_column,
417        )?)
418        .alias(left_time_index_column.clone());
419        // The field column in right side may not have qualifier (it may be removed by join operation),
420        // so we need to find it from the schema.
421        // `skip(1)` to skip the time index column
422        let right_proj_exprs_without_time_index = all_columns.iter().skip(1).map(|col| {
423            // expr
424            if mixed_sample_types
425                && (col == &mixed_float_field_col || col == &mixed_histogram_field_col)
426            {
427                mixed_value_expr(&right_fields, col)
428            } else if !mixed_sample_types && col == left_field_col {
429                let expr = DfExpr::Column(Column::new(right_field.1.clone(), right_field_col));
430                if right_field.2 != target_field_type {
431                    DfExpr::Cast(Cast::new(Box::new(expr), target_field_type.clone()))
432                        .alias(left_field_col.clone())
433                } else if left_field_col != right_field_col {
434                    expr.alias(left_field_col.clone())
435                } else {
436                    expr
437                }
438            } else if target_tag_types.contains_key(col) {
439                aligned_label_expr(col, &right_tag_types)
440            } else {
441                DfExpr::Column(Column::new(None::<String>, col))
442            }
443        });
444        let right_proj_exprs = [right_time_index_expr]
445            .into_iter()
446            .chain(right_proj_exprs_without_time_index);
447
448        let left_projected = LogicalPlanBuilder::from(left)
449            .project(left_proj_exprs)
450            .context(DataFusionPlanningSnafu)?
451            .alias(left_qualifier_string.clone())
452            .context(DataFusionPlanningSnafu)?
453            .build()
454            .context(DataFusionPlanningSnafu)?;
455        let right_projected = LogicalPlanBuilder::from(right)
456            .project(right_proj_exprs)
457            .context(DataFusionPlanningSnafu)?
458            .alias(right_qualifier_string.clone())
459            .context(DataFusionPlanningSnafu)?
460            .build()
461            .context(DataFusionPlanningSnafu)?;
462
463        // step 2: compute match columns
464        let mut match_columns = if let Some(modifier) = modifier
465            && let Some(matching) = &modifier.matching
466        {
467            match matching {
468                // keeps columns mentioned in `on`
469                LabelModifier::Include(on) => on.labels.clone(),
470                // removes columns memtioned in `ignoring`
471                LabelModifier::Exclude(ignoring) => {
472                    let ignoring = ignoring.labels.iter().cloned().collect::<HashSet<_>>();
473                    all_tags.difference(&ignoring).cloned().collect()
474                }
475            }
476        } else {
477            all_tags.iter().cloned().collect()
478        };
479        // sort to ensure the generated plan is not volatile
480        match_columns.sort_unstable();
481        match_columns.dedup();
482        occupied_column_names.extend(
483            left_projected
484                .schema()
485                .fields()
486                .iter()
487                .chain(right_projected.schema().fields().iter())
488                .map(|field| field.name().clone()),
489        );
490
491        let visible_schema = left_projected.schema().clone();
492        let visible_left_exprs = left_projected
493            .schema()
494            .iter()
495            .map(|(qualifier, field)| {
496                DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone()))
497            })
498            .collect::<Vec<_>>();
499        let visible_right_exprs = right_projected
500            .schema()
501            .iter()
502            .map(|(qualifier, field)| {
503                DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone()))
504            })
505            .collect::<Vec<_>>();
506        let mut left_match_exprs = Vec::with_capacity(match_columns.len());
507        let mut right_match_exprs = Vec::with_capacity(match_columns.len());
508        let mut next_internal_column = 0;
509
510        for label in &match_columns {
511            let left_field = if left_tag_cols_set.contains(label) {
512                Some(
513                    left_projected
514                        .schema()
515                        .iter()
516                        .find(|(_, field)| field.name() == label)
517                        .map(|(qualifier, field)| (qualifier.cloned(), field.data_type().clone()))
518                        .with_context(|| ColumnNotFoundSnafu { col: label.clone() })?,
519                )
520            } else {
521                None
522            };
523            let right_field = if right_tag_cols_set.contains(label) {
524                Some(
525                    right_projected
526                        .schema()
527                        .iter()
528                        .find(|(_, field)| field.name() == label)
529                        .map(|(qualifier, field)| (qualifier.cloned(), field.data_type().clone()))
530                        .with_context(|| ColumnNotFoundSnafu { col: label.clone() })?,
531                )
532            } else {
533                None
534            };
535            let data_type = match (left_field.as_ref(), right_field.as_ref()) {
536                (Some((_, left_type)), Some((_, right_type))) if left_type == right_type => {
537                    left_type.clone()
538                }
539                (Some((_, left_type)), Some((_, right_type))) => {
540                    return UnexpectedPlanExprSnafu {
541                        desc: format!(
542                            "OR match label {label} has incompatible types: {left_type:?} and {right_type:?}"
543                        ),
544                    }
545                    .fail();
546                }
547                (Some((_, data_type)), None) | (None, Some((_, data_type))) => data_type.clone(),
548                (None, None) => ArrowDataType::Utf8,
549            };
550            let Some(value_type) = Self::string_value_data_type(&data_type).cloned() else {
551                return UnexpectedPlanExprSnafu {
552                    desc: format!("OR match label {label} must be a string"),
553                }
554                .fail();
555            };
556            let internal_name = loop {
557                let name = format!("__promql_or_match_{next_internal_column}");
558                next_internal_column += 1;
559                if occupied_column_names.insert(name.clone()) {
560                    break name;
561                }
562            };
563            left_match_exprs.push(Self::normalized_match_key_expr(
564                label,
565                left_field,
566                &value_type,
567                &internal_name,
568            ));
569            right_match_exprs.push(Self::normalized_match_key_expr(
570                label,
571                right_field,
572                &value_type,
573                &internal_name,
574            ));
575        }
576
577        let left_augmented = LogicalPlanBuilder::from(left_projected)
578            .project(visible_left_exprs.into_iter().chain(left_match_exprs))
579            .context(DataFusionPlanningSnafu)?
580            .build()
581            .context(DataFusionPlanningSnafu)?;
582        let right_augmented = LogicalPlanBuilder::from(right_projected)
583            .project(visible_right_exprs.into_iter().chain(right_match_exprs))
584            .context(DataFusionPlanningSnafu)?
585            .build()
586            .context(DataFusionPlanningSnafu)?;
587
588        // step 3: build `UnionDistinctOn` with normalized internal match keys.
589        let visible_field_count = visible_schema.fields().len();
590        let compare_key_indices =
591            (visible_field_count..visible_field_count + match_columns.len()).collect::<Vec<_>>();
592        let (time_qualifier, _) = visible_schema
593            .iter()
594            .find(|(_, field)| field.name() == &left_time_index_column)
595            .with_context(|| TimeIndexNotFoundSnafu {
596                table: left_qualifier_string.clone(),
597            })?;
598        let ts_col_idx = left_augmented
599            .schema()
600            .iter()
601            .position(|(qualifier, field)| {
602                qualifier == time_qualifier && field.name() == &left_time_index_column
603            })
604            .with_context(|| TimeIndexNotFoundSnafu {
605                table: left_qualifier_string.clone(),
606            })?;
607        let union_distinct_on = UnionDistinctOn::try_new(
608            left_augmented,
609            right_augmented,
610            compare_key_indices,
611            ts_col_idx,
612        )
613        .context(DataFusionPlanningSnafu)?;
614        let augmented_result = LogicalPlan::Extension(Extension {
615            node: Arc::new(union_distinct_on),
616        });
617        let result = LogicalPlanBuilder::from(augmented_result)
618            .project(visible_schema.iter().map(|(qualifier, field)| {
619                DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone()))
620            }))
621            .context(DataFusionPlanningSnafu)?
622            .build()
623            .context(DataFusionPlanningSnafu)?;
624
625        // step 4: update context
626        let output_field_col = left_field_col.clone();
627        let mut output_context = left_context;
628        let mut visible_tags = all_tags.into_iter().collect::<Vec<_>>();
629        visible_tags.sort_unstable();
630        output_context.time_index_column = Some(left_time_index_column);
631        output_context.tag_columns = visible_tags;
632        output_context.field_columns = if mixed_sample_types {
633            vec![mixed_float_field_col, mixed_histogram_field_col]
634        } else {
635            vec![output_field_col]
636        };
637        output_context.use_tsid = left_has_tsid && right_has_tsid;
638        self.ctx = output_context;
639
640        Ok(result)
641    }
642
643    /// Resolves the column `name` of `schema`, preferring `preferred_qualifier`.
644    ///
645    /// A single operand plan can mix qualifiers, so the qualifier of its first field is not a
646    /// reliable prefix for every column of that plan. When `preferred_qualifier` does not hold the
647    /// column, the fallback is `qualified_field_with_unqualified_name`: it resolves a unique name
648    /// match, or a single unqualified candidate next to qualified ones, and reports
649    /// `AmbiguousReference` otherwise. This mirrors how the planner already resolves derived
650    /// columns against a join output.
651    fn resolved_column(
652        schema: &DFSchema,
653        preferred_qualifier: Option<TableReference>,
654        name: &str,
655    ) -> Result<Column> {
656        let qualifier = match &preferred_qualifier {
657            Some(qualifier) if schema.field_with_name(Some(qualifier), name).is_ok() => {
658                preferred_qualifier.clone()
659            }
660            _ => schema
661                .qualified_field_with_unqualified_name(name)
662                .context(DataFusionPlanningSnafu)?
663                .0
664                .cloned(),
665        };
666        Ok(Column::new(qualifier, name))
667    }
668
669    /// Build a set operator (AND/OR/UNLESS)
670    pub(super) fn set_op_on_non_field_columns(
671        &mut self,
672        mut left: LogicalPlan,
673        mut right: LogicalPlan,
674        left_context: PromPlannerContext,
675        right_context: PromPlannerContext,
676        op: TokenType,
677        modifier: &Option<BinModifier>,
678    ) -> Result<LogicalPlan> {
679        let left_tag_col_set = left_context
680            .tag_columns
681            .iter()
682            .cloned()
683            .collect::<HashSet<_>>();
684        let right_tag_col_set = right_context
685            .tag_columns
686            .iter()
687            .cloned()
688            .collect::<HashSet<_>>();
689
690        if matches!(op.id(), token::T_LOR) {
691            return self.or_operator(
692                left,
693                right,
694                left_tag_col_set,
695                right_tag_col_set,
696                left_context,
697                right_context,
698                modifier,
699            );
700        }
701
702        if let Some(modifier) = modifier {
703            ensure!(
704                matches!(
705                    modifier.card,
706                    VectorMatchCardinality::OneToOne | VectorMatchCardinality::ManyToMany
707                ),
708                UnsupportedVectorMatchSnafu {
709                    name: modifier.card.clone(),
710                },
711            );
712        }
713
714        let output_context = left_context.clone();
715        let visible_left_schema = left.schema().clone();
716        let mut left_context = left_context;
717        let mut right_context = right_context;
718        let added_marker_to_left = if Self::only_temporality_match_label_mismatches(
719            &left_context,
720            &right_context,
721            modifier,
722        ) {
723            let aligned = Self::align_temporality_match_column(
724                left,
725                right,
726                &mut left_context,
727                &mut right_context,
728            )?;
729            left = aligned.0;
730            right = aligned.1;
731            aligned.2
732        } else {
733            false
734        };
735
736        let mut left_tag_col_set = left_context
737            .tag_columns
738            .iter()
739            .cloned()
740            .collect::<BTreeSet<_>>();
741        let mut right_tag_col_set = right_context
742            .tag_columns
743            .iter()
744            .cloned()
745            .collect::<BTreeSet<_>>();
746        if let Some(matching) = modifier
747            .as_ref()
748            .and_then(|modifier| modifier.matching.as_ref())
749        {
750            match matching {
751                LabelModifier::Include(on) => {
752                    let mask = on.labels.iter().cloned().collect::<BTreeSet<_>>();
753                    left_tag_col_set = left_tag_col_set.intersection(&mask).cloned().collect();
754                    right_tag_col_set = right_tag_col_set.intersection(&mask).cloned().collect();
755                }
756                LabelModifier::Exclude(ignoring) => {
757                    for label in &ignoring.labels {
758                        let _ = left_tag_col_set.remove(label);
759                        let _ = right_tag_col_set.remove(label);
760                    }
761                }
762            }
763        }
764        ensure!(
765            left_tag_col_set == right_tag_col_set,
766            CombineTableColumnMismatchSnafu {
767                left: left_tag_col_set.iter().cloned().collect::<Vec<_>>(),
768                right: right_tag_col_set.iter().cloned().collect::<Vec<_>>(),
769            }
770        );
771
772        let left_time_index = left_context.time_index_column.clone().unwrap();
773        let right_time_index = right_context.time_index_column.clone().unwrap();
774
775        // alias right time index column if necessary
776        if left_context.time_index_column != right_context.time_index_column {
777            let right_project_exprs = right
778                .schema()
779                .fields()
780                .iter()
781                .map(|field| {
782                    if field.name() == &right_time_index {
783                        DfExpr::Column(Column::from_name(&right_time_index)).alias(&left_time_index)
784                    } else {
785                        DfExpr::Column(Column::from_name(field.name()))
786                    }
787                })
788                .collect::<Vec<_>>();
789
790            right = LogicalPlanBuilder::from(right)
791                .project(right_project_exprs)
792                .context(DataFusionPlanningSnafu)?
793                .build()
794                .context(DataFusionPlanningSnafu)?;
795        }
796
797        let join_keys = left_tag_col_set
798            .into_iter()
799            .chain([left_time_index])
800            .map(Column::from_name)
801            .collect::<Vec<_>>();
802
803        ensure!(
804            left_context.field_columns.len() == 1
805                || Self::field_columns_are_alternative_samples(
806                    left.schema(),
807                    &left_context.field_columns,
808                ),
809            MultiFieldsNotSupportedSnafu {
810                operator: "AND/UNLESS operator"
811            }
812        );
813        // Generate join plan.
814        // All set operations in PromQL are "distinct"
815        let result = match op.id() {
816            token::T_LAND => LogicalPlanBuilder::from(left)
817                .distinct()
818                .context(DataFusionPlanningSnafu)?
819                .join_detailed(
820                    right,
821                    JoinType::LeftSemi,
822                    (join_keys.clone(), join_keys),
823                    None,
824                    NullEquality::NullEqualsNull,
825                )
826                .context(DataFusionPlanningSnafu)?
827                .build()
828                .context(DataFusionPlanningSnafu),
829            token::T_LUNLESS => LogicalPlanBuilder::from(left)
830                .distinct()
831                .context(DataFusionPlanningSnafu)?
832                .join_detailed(
833                    right,
834                    JoinType::LeftAnti,
835                    (join_keys.clone(), join_keys),
836                    None,
837                    NullEquality::NullEqualsNull,
838                )
839                .context(DataFusionPlanningSnafu)?
840                .build()
841                .context(DataFusionPlanningSnafu),
842            token::T_LOR => {
843                // OR is handled at the beginning of this function, as it cannot
844                // be expressed using JOIN like AND and UNLESS.
845                unreachable!()
846            }
847            _ => UnexpectedTokenSnafu { token: op }.fail(),
848        }?;
849        let result = if added_marker_to_left {
850            LogicalPlanBuilder::from(result)
851                .project(visible_left_schema.iter().map(|(qualifier, field)| {
852                    DfExpr::Column(Column::new(qualifier.cloned(), field.name().clone()))
853                }))
854                .context(DataFusionPlanningSnafu)?
855                .build()
856                .context(DataFusionPlanningSnafu)?
857        } else {
858            result
859        };
860
861        // AND/UNLESS preserve the complete left operand's visible columns and values; encoded
862        // markers are decoded.
863        self.ctx = output_context;
864        Ok(result)
865    }
866}