common_function/scalars/
uddsketch_calc.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Implementation of the scalar function `uddsketch_calc`.
16
17use std::fmt;
18use std::fmt::Display;
19use std::sync::Arc;
20
21use datafusion_common::DataFusionError;
22use datafusion_common::arrow::array::{Array, AsArray, Float64Builder};
23use datafusion_common::arrow::datatypes::{DataType, Float64Type};
24use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, Signature, Volatility};
25
26use crate::function::{Function, extract_args};
27use crate::function_registry::FunctionRegistry;
28use crate::uddsketch_compat;
29
30const NAME: &str = "uddsketch_calc";
31
32/// UddSketchCalcFunction implements the scalar function `uddsketch_calc`.
33///
34/// It accepts two arguments:
35/// 1. A percentile (as f64) for which to compute the estimated quantile (e.g. 0.95 for p95).
36/// 2. The serialized UDDSketch state, as produced by the aggregator (binary).
37///
38/// For each row, it deserializes the sketch and returns the computed quantile value.
39#[derive(Debug)]
40pub(crate) struct UddSketchCalcFunction {
41    signature: Signature,
42}
43
44impl UddSketchCalcFunction {
45    pub fn register(registry: &FunctionRegistry) {
46        registry.register_scalar(UddSketchCalcFunction::default());
47    }
48}
49
50impl Default for UddSketchCalcFunction {
51    fn default() -> Self {
52        Self {
53            // First argument: percentile (float64)
54            // Second argument: UDDSketch state (binary)
55            signature: Signature::exact(
56                vec![DataType::Float64, DataType::Binary],
57                Volatility::Immutable,
58            ),
59        }
60    }
61}
62
63impl Display for UddSketchCalcFunction {
64    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
65        write!(f, "{}", NAME.to_ascii_uppercase())
66    }
67}
68
69impl Function for UddSketchCalcFunction {
70    fn name(&self) -> &str {
71        NAME
72    }
73
74    fn return_type(&self, _: &[DataType]) -> datafusion_common::Result<DataType> {
75        Ok(DataType::Float64)
76    }
77
78    fn signature(&self) -> &Signature {
79        &self.signature
80    }
81
82    fn invoke_with_args(
83        &self,
84        args: ScalarFunctionArgs,
85    ) -> datafusion_common::Result<ColumnarValue> {
86        let [arg0, arg1] = extract_args(self.name(), &args)?;
87
88        let Some(percentages) = arg0.as_primitive_opt::<Float64Type>() else {
89            return Err(DataFusionError::Execution(format!(
90                "'{}' expects 1st argument to be Float64 datatype, got {}",
91                self.name(),
92                arg0.data_type()
93            )));
94        };
95        let Some(sketch_vec) = arg1.as_binary_opt::<i32>() else {
96            return Err(DataFusionError::Execution(format!(
97                "'{}' expects 2nd argument to be Binary datatype, got {}",
98                self.name(),
99                arg1.data_type()
100            )));
101        };
102        let len = sketch_vec.len();
103        let mut builder = Float64Builder::with_capacity(len);
104
105        for i in 0..len {
106            let perc_opt = percentages.is_valid(i).then(|| percentages.value(i));
107            let sketch_opt = sketch_vec.is_valid(i).then(|| sketch_vec.value(i));
108
109            if sketch_opt.is_none() || perc_opt.is_none() {
110                builder.append_null();
111                continue;
112            }
113
114            let sketch_bytes = sketch_opt.unwrap();
115            let perc = perc_opt.unwrap();
116
117            let value = match uddsketch_compat::quantile(sketch_bytes, perc) {
118                Ok(value) => value,
119                Err(e) => {
120                    common_telemetry::trace!("Failed to parse UDDSketch: {}", e);
121                    builder.append_null();
122                    continue;
123                }
124            };
125
126            match value {
127                Some(value) => builder.append_value(value),
128                None => builder.append_null(),
129            }
130        }
131
132        Ok(ColumnarValue::Array(Arc::new(builder.finish())))
133    }
134}
135
136#[cfg(test)]
137mod tests {
138    use std::sync::Arc;
139
140    use arrow_schema::Field;
141    use datafusion_common::arrow::array::{BinaryArray, Float64Array};
142    use uddsketch::UddSketch;
143
144    use super::*;
145
146    #[test]
147    fn test_uddsketch_calc_function() {
148        let function = UddSketchCalcFunction::default();
149        assert_eq!("uddsketch_calc", function.name());
150        assert_eq!(
151            DataType::Float64,
152            function.return_type(&[DataType::Float64]).unwrap()
153        );
154
155        // Create a test sketch
156        let mut sketch = UddSketch::new(128, 0.01).unwrap();
157        sketch
158            .add_batch(&[10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0, 80.0, 90.0, 100.0])
159            .unwrap();
160
161        // Get expected values directly from the sketch
162        let expected_p50 = sketch.quantile(0.5).unwrap().unwrap();
163        let expected_p90 = sketch.quantile(0.9).unwrap().unwrap();
164        let expected_p95 = sketch.quantile(0.95).unwrap().unwrap();
165
166        let serialized = sketch.encode().unwrap();
167        let percentiles = vec![0.5, 0.9, 0.95];
168
169        let args = vec![
170            ColumnarValue::Array(Arc::new(Float64Array::from(percentiles.clone()))),
171            ColumnarValue::Array(Arc::new(BinaryArray::from_iter_values(vec![serialized; 3]))),
172        ];
173
174        let result = function
175            .invoke_with_args(ScalarFunctionArgs {
176                args,
177                arg_fields: vec![],
178                number_rows: 3,
179                return_field: Arc::new(Field::new("x", DataType::Float64, false)),
180                config_options: Arc::new(Default::default()),
181            })
182            .unwrap();
183        let ColumnarValue::Array(result) = result else {
184            unreachable!()
185        };
186        let result = result.as_primitive::<Float64Type>();
187        assert_eq!(result.len(), 3);
188
189        // Test median (p50)
190        assert!((result.value(0) - expected_p50).abs() < 1e-10);
191        // Test p90
192        assert!((result.value(1) - expected_p90).abs() < 1e-10);
193        // Test p95
194        assert!((result.value(2) - expected_p95).abs() < 1e-10);
195    }
196
197    #[test]
198    fn test_uddsketch_calc_function_reads_legacy_state() {
199        let function = UddSketchCalcFunction::default();
200        let args = vec![
201            ColumnarValue::Array(Arc::new(Float64Array::from(vec![0.5]))),
202            ColumnarValue::Array(Arc::new(BinaryArray::from_iter_values(vec![
203                uddsketch_compat::LEGACY_STATE,
204            ]))),
205        ];
206
207        let result = function
208            .invoke_with_args(ScalarFunctionArgs {
209                args,
210                arg_fields: vec![],
211                number_rows: 1,
212                return_field: Arc::new(Field::new("x", DataType::Float64, false)),
213                config_options: Arc::new(Default::default()),
214            })
215            .unwrap();
216        let ColumnarValue::Array(result) = result else {
217            unreachable!()
218        };
219        let result = result.as_primitive::<Float64Type>();
220        assert_eq!(result.len(), 1);
221        assert!(!result.is_null(0));
222        assert_eq!(result.value(0), 0.9900000000000001);
223    }
224
225    #[test]
226    fn test_uddsketch_calc_function_errors() {
227        let function = UddSketchCalcFunction::default();
228
229        // Test with invalid number of arguments
230        let result = function.invoke_with_args(ScalarFunctionArgs {
231            args: vec![ColumnarValue::Array(Arc::new(Float64Array::from(vec![
232                0.95,
233            ])))],
234            arg_fields: vec![],
235            number_rows: 0,
236            return_field: Arc::new(Field::new("x", DataType::Float64, false)),
237            config_options: Arc::new(Default::default()),
238        });
239        assert!(result.is_err());
240        assert!(
241            result
242                .unwrap_err()
243                .to_string()
244                .contains("Execution error: uddsketch_calc function requires 2 arguments, got 1")
245        );
246
247        // Test with invalid binary data
248        let args = vec![
249            ColumnarValue::Array(Arc::new(Float64Array::from(vec![0.95]))),
250            ColumnarValue::Array(Arc::new(BinaryArray::from_iter(vec![Some(vec![1, 2, 3])]))),
251        ];
252        let result = function
253            .invoke_with_args(ScalarFunctionArgs {
254                args,
255                arg_fields: vec![],
256                number_rows: 0,
257                return_field: Arc::new(Field::new("x", DataType::Float64, false)),
258                config_options: Arc::new(Default::default()),
259            })
260            .unwrap();
261        let ColumnarValue::Array(result) = result else {
262            unreachable!()
263        };
264        let result = result.as_primitive::<Float64Type>();
265        assert_eq!(result.len(), 1);
266        assert!(result.is_null(0));
267
268        let empty = UddSketch::new(128, 0.01).unwrap().encode().unwrap();
269        let mut populated = UddSketch::new(128, 0.01).unwrap();
270        populated.add(1.0).unwrap();
271        let populated = populated.encode().unwrap();
272        let args = vec![
273            ColumnarValue::Array(Arc::new(Float64Array::from(vec![0.5, -0.1, f64::NAN]))),
274            ColumnarValue::Array(Arc::new(BinaryArray::from_iter_values(vec![
275                empty,
276                populated.clone(),
277                populated,
278            ]))),
279        ];
280        let result = function
281            .invoke_with_args(ScalarFunctionArgs {
282                args,
283                arg_fields: vec![],
284                number_rows: 3,
285                return_field: Arc::new(Field::new("x", DataType::Float64, false)),
286                config_options: Arc::new(Default::default()),
287            })
288            .unwrap();
289        let ColumnarValue::Array(result) = result else {
290            unreachable!()
291        };
292        assert_eq!(result.as_primitive::<Float64Type>().null_count(), 3);
293    }
294}
common_function/scalars/uddsketch_calc.rs

common_function/scalars/
uddsketch_calc.rs
