promql/functions/
idelta.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::fmt::Display;
16use std::sync::Arc;
17
18use datafusion::arrow::array::{Float64Array, Float64Builder, TimestampMillisecondArray};
19use datafusion::arrow::datatypes::TimeUnit;
20use datafusion::common::DataFusionError;
21use datafusion::logical_expr::{ScalarUDF, Volatility};
22use datafusion::physical_plan::ColumnarValue;
23use datafusion_expr::create_udf;
24use datatypes::arrow::array::Array;
25use datatypes::arrow::datatypes::DataType;
26
27use crate::error;
28use crate::functions::extract_array;
29use crate::range_array::RangeArray;
30
31/// The `funcIdelta` in Promql,
32/// from <https://github.com/prometheus/prometheus/blob/6bdecf377cea8e856509914f35234e948c4fcb80/promql/functions.go#L235>
33#[derive(Debug)]
34pub struct IDelta<const IS_RATE: bool> {}
35
36impl<const IS_RATE: bool> IDelta<IS_RATE> {
37    pub const fn name() -> &'static str {
38        if IS_RATE { "prom_irate" } else { "prom_idelta" }
39    }
40
41    pub fn scalar_udf() -> ScalarUDF {
42        create_udf(
43            Self::name(),
44            Self::input_type(),
45            Self::return_type(),
46            Volatility::Volatile,
47            Arc::new(Self::calc) as _,
48        )
49    }
50
51    // time index column and value column
52    fn input_type() -> Vec<DataType> {
53        vec![
54            RangeArray::convert_data_type(DataType::Timestamp(TimeUnit::Millisecond, None)),
55            RangeArray::convert_data_type(DataType::Float64),
56        ]
57    }
58
59    fn return_type() -> DataType {
60        DataType::Float64
61    }
62
63    fn calc(input: &[ColumnarValue]) -> Result<ColumnarValue, DataFusionError> {
64        // construct matrix from input
65        assert_eq!(input.len(), 2);
66        let ts_array = extract_array(&input[0])?;
67        let value_array = extract_array(&input[1])?;
68
69        let ts_range: RangeArray = RangeArray::try_new(ts_array.to_data().into())?;
70        let value_range: RangeArray = RangeArray::try_new(value_array.to_data().into())?;
71        error::ensure(
72            ts_range.len() == value_range.len(),
73            DataFusionError::Execution(format!(
74                "{}: input arrays should have the same length, found {} and {}",
75                Self::name(),
76                ts_range.len(),
77                value_range.len()
78            )),
79        )?;
80        error::ensure(
81            ts_range.value_type() == DataType::Timestamp(TimeUnit::Millisecond, None),
82            DataFusionError::Execution(format!(
83                "{}: expect TimestampMillisecond as time index array's type, found {}",
84                Self::name(),
85                ts_range.value_type()
86            )),
87        )?;
88        error::ensure(
89            value_range.value_type() == DataType::Float64,
90            DataFusionError::Execution(format!(
91                "{}: expect Float64 as value array's type, found {}",
92                Self::name(),
93                value_range.value_type()
94            )),
95        )?;
96
97        let ts_values = ts_range.values();
98        let ts_values = ts_values
99            .as_any()
100            .downcast_ref::<TimestampMillisecondArray>()
101            .unwrap()
102            .values();
103
104        let value_array = value_range.values();
105        let value_array = value_array.as_any().downcast_ref::<Float64Array>().unwrap();
106        // A NULL field value means the series has no sample at that timestamp, so the last two
107        // samples are not necessarily the last two slots.
108        let has_nulls = value_array.null_count() > 0;
109        let value_values = value_array.values();
110
111        let mut result_builder = Float64Builder::with_capacity(ts_range.len());
112
113        for index in 0..ts_range.len() {
114            let (ts_offset, len) = ts_range.get_offset_length(index).unwrap();
115            let (value_offset, value_len) = value_range.get_offset_length(index).unwrap();
116            error::ensure(
117                len == value_len,
118                DataFusionError::Execution(format!(
119                    "{}: input arrays should have the same length, found {} and {}",
120                    Self::name(),
121                    len,
122                    value_len
123                )),
124            )?;
125            let (last_position, prev_position) = if has_nulls {
126                match last_two_samples(value_array, value_offset, len) {
127                    Some(positions) => positions,
128                    None => {
129                        result_builder.append_null();
130                        continue;
131                    }
132                }
133            } else {
134                if len < 2 {
135                    result_builder.append_null();
136                    continue;
137                }
138                (len - 1, len - 2)
139            };
140
141            let last_offset = ts_offset + last_position;
142            let prev_offset = ts_offset + prev_position;
143            let sampled_interval =
144                (ts_values[last_offset] - ts_values[prev_offset]) as f64 / 1000.0;
145
146            let last_value = value_values[value_offset + last_position];
147            let prev_value = value_values[value_offset + prev_position];
148
149            if !IS_RATE {
150                result_builder.append_value(last_value - prev_value);
151                continue;
152            }
153
154            let result_value = if last_value < prev_value {
155                // counter reset
156                last_value
157            } else {
158                last_value - prev_value
159            };
160
161            result_builder.append_value(result_value / sampled_interval);
162        }
163
164        let result = ColumnarValue::Array(Arc::new(result_builder.finish()));
165        Ok(result)
166    }
167}
168
169/// Locates the last two samples inside `[offset, offset + len)`, returning their positions
170/// relative to `offset`. Returns `None` when the window holds fewer than two samples.
171fn last_two_samples(values: &Float64Array, offset: usize, len: usize) -> Option<(usize, usize)> {
172    let mut last = None;
173    for position in (0..len).rev() {
174        if values.is_null(offset + position) {
175            continue;
176        }
177        match last {
178            None => last = Some(position),
179            Some(last) => return Some((last, position)),
180        }
181    }
182    None
183}
184
185impl<const IS_RATE: bool> Display for IDelta<IS_RATE> {
186    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
187        write!(f, "PromQL Idelta Function (is_rate: {IS_RATE})",)
188    }
189}
190
191#[cfg(test)]
192mod test {
193
194    use datafusion::arrow::buffer::NullBuffer;
195
196    use super::*;
197    use crate::functions::test_util::simple_range_udf_runner;
198
199    #[test]
200    fn basic_idelta_and_irate() {
201        let ts_array = Arc::new(TimestampMillisecondArray::from_iter(
202            [1000i64, 3000, 5000, 7000, 9000, 11000, 13000, 15000, 17000]
203                .into_iter()
204                .map(Some),
205        ));
206        let ts_ranges = [(0, 2), (0, 5), (1, 1), (3, 3), (8, 1), (9, 0)];
207
208        let values_array = Arc::new(Float64Array::from_iter([
209            1.0, 2.0, 3.0, 5.0, 0.0, 6.0, 7.0, 8.0, 9.0,
210        ]));
211        let values_ranges = [(0, 2), (0, 5), (1, 1), (3, 3), (8, 1), (9, 0)];
212
213        // test idelta
214        let ts_range_array = RangeArray::from_ranges(ts_array.clone(), ts_ranges).unwrap();
215        let value_range_array =
216            RangeArray::from_ranges(values_array.clone(), values_ranges).unwrap();
217        simple_range_udf_runner(
218            IDelta::<false>::scalar_udf(),
219            ts_range_array,
220            value_range_array,
221            vec![],
222            vec![Some(1.0), Some(-5.0), None, Some(6.0), None, None],
223        );
224
225        // test irate
226        let ts_range_array = RangeArray::from_ranges(ts_array, ts_ranges).unwrap();
227        let value_range_array = RangeArray::from_ranges(values_array, values_ranges).unwrap();
228        simple_range_udf_runner(
229            IDelta::<true>::scalar_udf(),
230            ts_range_array,
231            value_range_array,
232            vec![],
233            // the second point represent counter reset
234            vec![Some(0.5), Some(0.0), None, Some(3.0), None, None],
235        );
236    }
237
238    #[test]
239    fn idelta_uses_last_two_samples_not_last_two_slots() {
240        let ts_array = Arc::new(TimestampMillisecondArray::from_iter_values([
241            0i64, 1000, 2000, 3000,
242        ]));
243        // Samples are 1.0@0 and 4.0@1000; the trailing slots only carry padding.
244        let values_array = Arc::new(Float64Array::new(
245            vec![1.0, 4.0, 100.0, 200.0].into(),
246            Some(NullBuffer::from_iter([true, true, false, false])),
247        ));
248        let ranges = [(0, 4), (2, 2), (3, 1)];
249
250        simple_range_udf_runner(
251            IDelta::<false>::scalar_udf(),
252            RangeArray::from_ranges(ts_array.clone(), ranges).unwrap(),
253            RangeArray::from_ranges(values_array.clone(), ranges).unwrap(),
254            vec![],
255            vec![Some(3.0), None, None],
256        );
257
258        simple_range_udf_runner(
259            IDelta::<true>::scalar_udf(),
260            RangeArray::from_ranges(ts_array, ranges).unwrap(),
261            RangeArray::from_ranges(values_array, ranges).unwrap(),
262            vec![],
263            vec![Some(3.0), None, None],
264        );
265    }
266}
promql/functions/idelta.rs

promql/functions/
idelta.rs
