sql/parsers/
tql_parser.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datafusion_common::ScalarValue;
16use snafu::{OptionExt, ResultExt};
17use sqlparser::keywords::Keyword;
18use sqlparser::parser::ParserError;
19use sqlparser::tokenizer::Token;
20
21use crate::error::{self, Result};
22use crate::parser::ParserContext;
23use crate::parsers::utils;
24use crate::statements::statement::Statement;
25use crate::statements::tql::{Tql, TqlAnalyze, TqlEval, TqlExplain, TqlParameters};
26
27pub const TQL: &str = "TQL";
28const EVAL: &str = "EVAL";
29const EVALUATE: &str = "EVALUATE";
30const VERBOSE: &str = "VERBOSE";
31
32use sqlparser::parser::Parser;
33
34use crate::dialect::GreptimeDbDialect;
35use crate::parsers::error::{
36    ConvertToLogicalExpressionSnafu, EvaluationSnafu, ParserSnafu, TQLError,
37};
38
39/// TQL extension parser, including:
40/// - `TQL EVAL <query>`
41/// - `TQL EXPLAIN [VERBOSE] <query>`
42/// - `TQL ANALYZE [VERBOSE] <query>`
43impl ParserContext<'_> {
44    pub(crate) fn parse_tql(&mut self) -> Result<Statement> {
45        let _ = self.parser.next_token();
46
47        match self.parser.peek_token().token {
48            Token::Word(w) => {
49                let uppercase = w.value.to_uppercase();
50                let _consume_tql_keyword_token = self.parser.next_token();
51                match w.keyword {
52                    Keyword::NoKeyword
53                        if (uppercase == EVAL || uppercase == EVALUATE)
54                            && w.quote_style.is_none() =>
55                    {
56                        self.parse_tql_params()
57                            .map(|params| Statement::Tql(Tql::Eval(TqlEval::from(params))))
58                            .context(error::TQLSyntaxSnafu)
59                    }
60
61                    Keyword::EXPLAIN => {
62                        let is_verbose = self.has_verbose_keyword();
63                        if is_verbose {
64                            let _consume_verbose_token = self.parser.next_token();
65                        }
66                        self.parse_tql_params()
67                            .map(|mut params| {
68                                params.is_verbose = is_verbose;
69                                Statement::Tql(Tql::Explain(TqlExplain::from(params)))
70                            })
71                            .context(error::TQLSyntaxSnafu)
72                    }
73
74                    Keyword::ANALYZE => {
75                        let is_verbose = self.has_verbose_keyword();
76                        if is_verbose {
77                            let _consume_verbose_token = self.parser.next_token();
78                        }
79                        self.parse_tql_params()
80                            .map(|mut params| {
81                                params.is_verbose = is_verbose;
82                                Statement::Tql(Tql::Analyze(TqlAnalyze::from(params)))
83                            })
84                            .context(error::TQLSyntaxSnafu)
85                    }
86                    _ => self.unsupported(self.peek_token_as_string()),
87                }
88            }
89            unexpected => self.unsupported(unexpected.to_string()),
90        }
91    }
92
93    fn parse_tql_params(&mut self) -> std::result::Result<TqlParameters, TQLError> {
94        let parser = &mut self.parser;
95        let (start, end, step, lookback) = match parser.peek_token().token {
96            Token::LParen => {
97                let _consume_lparen_token = parser.next_token();
98                let start = Self::parse_string_or_number_or_word(parser, &[Token::Comma])?.0;
99                let end = Self::parse_string_or_number_or_word(parser, &[Token::Comma])?.0;
100
101                let (step, delimiter) =
102                    Self::parse_string_or_number_or_word(parser, &[Token::Comma, Token::RParen])?;
103                let lookback = if delimiter == Token::Comma {
104                    Self::parse_string_or_number_or_word(parser, &[Token::RParen])
105                        .ok()
106                        .map(|t| t.0)
107                } else {
108                    None
109                };
110
111                (start, end, step, lookback)
112            }
113            _ => ("0".to_string(), "0".to_string(), "5m".to_string(), None),
114        };
115        let query = Self::parse_tql_query(parser, self.sql).context(ParserSnafu)?;
116        Ok(TqlParameters::new(start, end, step, lookback, query))
117    }
118
119    pub fn comma_or_rparen(token: &Token) -> bool {
120        Self::is_comma(token) || Self::is_rparen(token)
121    }
122
123    #[inline]
124    fn is_comma(token: &Token) -> bool {
125        matches!(token, Token::Comma)
126    }
127
128    #[inline]
129    fn is_rparen(token: &Token) -> bool {
130        matches!(token, Token::RParen)
131    }
132
133    fn has_verbose_keyword(&mut self) -> bool {
134        self.peek_token_as_string().eq_ignore_ascii_case(VERBOSE)
135    }
136
137    /// Try to parse and consume a string, number or word token.
138    /// Return `Ok` if it's parsed and one of the given delimiter tokens is consumed.
139    /// The string and matched delimiter will be returned as a tuple.
140    fn parse_string_or_number_or_word(
141        parser: &mut Parser,
142        delimiter_tokens: &[Token],
143    ) -> std::result::Result<(String, Token), TQLError> {
144        let mut tokens = vec![];
145
146        while !delimiter_tokens.contains(&parser.peek_token().token) {
147            let token = parser.next_token().token;
148            if matches!(token, Token::EOF) {
149                break;
150            }
151            tokens.push(token);
152        }
153        let result = match tokens.len() {
154            0 => Err(ParserError::ParserError(
155                "Expected at least one token".to_string(),
156            ))
157            .context(ParserSnafu),
158            1 => {
159                let value = match tokens[0].clone() {
160                    Token::Number(n, _) => n,
161                    Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s,
162                    Token::Word(_) => Self::parse_tokens(tokens)?,
163                    unexpected => {
164                        return Err(ParserError::ParserError(format!(
165                            "Expected number, string or word, but have {unexpected:?}"
166                        )))
167                        .context(ParserSnafu);
168                    }
169                };
170                Ok(value)
171            }
172            _ => Self::parse_tokens(tokens),
173        };
174        for token in delimiter_tokens {
175            if parser.consume_token(token) {
176                return result.map(|v| (v, token.clone()));
177            }
178        }
179        Err(ParserError::ParserError(format!(
180            "Delimiters not match {delimiter_tokens:?}"
181        )))
182        .context(ParserSnafu)
183    }
184
185    fn parse_tokens(tokens: Vec<Token>) -> std::result::Result<String, TQLError> {
186        let parser_expr = Self::parse_to_expr(tokens)?;
187        let lit = utils::parser_expr_to_scalar_value(parser_expr)
188            .map_err(Box::new)
189            .context(ConvertToLogicalExpressionSnafu)?;
190
191        let second = match lit {
192            ScalarValue::TimestampNanosecond(ts_nanos, _)
193            | ScalarValue::DurationNanosecond(ts_nanos) => ts_nanos.map(|v| v / 1_000_000_000),
194            ScalarValue::TimestampMicrosecond(ts_micros, _)
195            | ScalarValue::DurationMicrosecond(ts_micros) => ts_micros.map(|v| v / 1_000_000),
196            ScalarValue::TimestampMillisecond(ts_millis, _)
197            | ScalarValue::DurationMillisecond(ts_millis) => ts_millis.map(|v| v / 1_000),
198            ScalarValue::TimestampSecond(ts_secs, _) | ScalarValue::DurationSecond(ts_secs) => {
199                ts_secs
200            }
201            _ => None,
202        };
203
204        second.map(|ts| ts.to_string()).context(EvaluationSnafu {
205            msg: format!("Failed to extract a timestamp value {lit:?}"),
206        })
207    }
208
209    fn parse_to_expr(tokens: Vec<Token>) -> std::result::Result<sqlparser::ast::Expr, TQLError> {
210        Parser::new(&GreptimeDbDialect {})
211            .with_tokens(tokens)
212            .parse_expr()
213            .context(ParserSnafu)
214    }
215
216    fn parse_tql_query(parser: &mut Parser, sql: &str) -> std::result::Result<String, ParserError> {
217        while matches!(parser.peek_token().token, Token::Comma) {
218            let _skip_token = parser.next_token();
219        }
220        let index = parser.next_token().span.start.column as usize;
221        if index == 0 {
222            return Err(ParserError::ParserError("empty TQL query".to_string()));
223        }
224
225        let query = &sql[index - 1..];
226        while parser.next_token() != Token::EOF {
227            // consume all tokens
228            // TODO(dennis): supports multi TQL statements separated by ';'?
229        }
230        // remove the last ';' or tailing space if exists
231        Ok(query.trim().trim_end_matches(';').to_string())
232    }
233}
234
235#[cfg(test)]
236mod tests {
237    use common_error::ext::ErrorExt;
238
239    use super::*;
240    use crate::dialect::GreptimeDbDialect;
241    use crate::parser::ParseOptions;
242
243    fn parse_into_statement(sql: &str) -> Statement {
244        let mut result =
245            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
246                .unwrap();
247        assert_eq!(1, result.len());
248        result.remove(0)
249    }
250
251    #[test]
252    fn test_parse_tql_eval_with_functions() {
253        let sql = "TQL EVAL (now() - now(), now() -  (now() - '10 seconds'::interval), '1s') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
254        let statement = parse_into_statement(sql);
255        match statement {
256            Statement::Tql(Tql::Eval(eval)) => {
257                assert_eq!(eval.start, "0");
258                assert_eq!(eval.end, "10");
259                assert_eq!(eval.step, "1s");
260                assert_eq!(eval.lookback, None);
261                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
262            }
263            _ => unreachable!(),
264        }
265
266        let sql = "TQL EVAL ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
267        match parse_into_statement(sql) {
268            Statement::Tql(Tql::Eval(eval)) => {
269                assert_eq!(eval.start, "300");
270                assert_eq!(eval.end, "1200");
271                assert_eq!(eval.step, "1m");
272                assert_eq!(eval.lookback, None);
273                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
274            }
275            _ => unreachable!(),
276        }
277
278        let sql = "TQL EVAL (now(), now()-'5m', '30s') http_requests_total";
279        let result =
280            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
281        assert!(result.is_err());
282    }
283
284    #[test]
285    fn test_parse_tql_eval() {
286        let sql = "TQL EVAL (1676887657, 1676887659, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
287        match parse_into_statement(sql) {
288            Statement::Tql(Tql::Eval(eval)) => {
289                assert_eq!(eval.start, "1676887657");
290                assert_eq!(eval.end, "1676887659");
291                assert_eq!(eval.step, "1m");
292                assert_eq!(eval.lookback, None);
293                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
294            }
295            _ => unreachable!(),
296        }
297
298        let sql = "TQL EVAL (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
299        let statement = parse_into_statement(sql);
300
301        match &statement {
302            Statement::Tql(Tql::Eval(eval)) => {
303                assert_eq!(eval.start, "1676887657.1");
304                assert_eq!(eval.end, "1676887659.5");
305                assert_eq!(eval.step, "30.3");
306                assert_eq!(eval.lookback, None);
307                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
308            }
309            _ => unreachable!(),
310        }
311
312        let sql2 = "TQL EVALUATE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
313        let statement2 = parse_into_statement(sql2);
314        assert_eq!(statement, statement2);
315
316        let sql = "tql eval ('2015-07-01T20:10:30.781Z', '2015-07-01T20:11:00.781Z', '30s') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
317        match parse_into_statement(sql) {
318            Statement::Tql(Tql::Eval(eval)) => {
319                assert_eq!(eval.start, "2015-07-01T20:10:30.781Z");
320                assert_eq!(eval.end, "2015-07-01T20:11:00.781Z");
321                assert_eq!(eval.step, "30s");
322                assert_eq!(eval.lookback, None);
323                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
324            }
325            _ => unreachable!(),
326        }
327    }
328
329    #[test]
330    fn test_parse_tql_with_lookback_values() {
331        let sql = "TQL EVAL (1676887657, 1676887659, '1m', '5m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
332        match parse_into_statement(sql) {
333            Statement::Tql(Tql::Eval(eval)) => {
334                assert_eq!(eval.start, "1676887657");
335                assert_eq!(eval.end, "1676887659");
336                assert_eq!(eval.step, "1m".to_string());
337                assert_eq!(eval.lookback, Some("5m".to_string()));
338                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
339            }
340            _ => unreachable!(),
341        }
342
343        let sql = "TQL EVAL ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, '1m', '7m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
344        match parse_into_statement(sql) {
345            Statement::Tql(Tql::Eval(eval)) => {
346                assert_eq!(eval.start, "300");
347                assert_eq!(eval.end, "1200");
348                assert_eq!(eval.step, "1m");
349                assert_eq!(eval.lookback, Some("7m".to_string()));
350                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
351            }
352            _ => unreachable!(),
353        }
354
355        let sql = "TQL EXPLAIN (20, 100, 10, '3m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
356        match parse_into_statement(sql) {
357            Statement::Tql(Tql::Explain(explain)) => {
358                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
359                assert_eq!(explain.start, "20");
360                assert_eq!(explain.end, "100");
361                assert_eq!(explain.step, "10");
362                assert_eq!(explain.lookback, Some("3m".to_string()));
363                assert!(!explain.is_verbose);
364            }
365            _ => unreachable!(),
366        }
367
368        let sql = "TQL EXPLAIN VERBOSE (20, 100, 10, '3m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
369        match parse_into_statement(sql) {
370            Statement::Tql(Tql::Explain(explain)) => {
371                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
372                assert_eq!(explain.start, "20");
373                assert_eq!(explain.end, "100");
374                assert_eq!(explain.step, "10");
375                assert_eq!(explain.lookback, Some("3m".to_string()));
376                assert!(explain.is_verbose);
377            }
378            _ => unreachable!(),
379        }
380
381        let sql = "TQL ANALYZE (1676887657, 1676887659, '1m', '9m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
382        match parse_into_statement(sql) {
383            Statement::Tql(Tql::Analyze(analyze)) => {
384                assert_eq!(analyze.start, "1676887657");
385                assert_eq!(analyze.end, "1676887659");
386                assert_eq!(analyze.step, "1m");
387                assert_eq!(analyze.lookback, Some("9m".to_string()));
388                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
389                assert!(!analyze.is_verbose);
390            }
391            _ => unreachable!(),
392        }
393
394        let sql = "TQL ANALYZE VERBOSE (1676887657, 1676887659, '1m', '9m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
395        match parse_into_statement(sql) {
396            Statement::Tql(Tql::Analyze(analyze)) => {
397                assert_eq!(analyze.start, "1676887657");
398                assert_eq!(analyze.end, "1676887659");
399                assert_eq!(analyze.step, "1m");
400                assert_eq!(analyze.lookback, Some("9m".to_string()));
401                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
402                assert!(analyze.is_verbose);
403            }
404            _ => unreachable!(),
405        }
406    }
407
408    #[test]
409    fn test_parse_tql_explain() {
410        let sql = "TQL EXPLAIN http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
411        match parse_into_statement(sql) {
412            Statement::Tql(Tql::Explain(explain)) => {
413                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
414                assert_eq!(explain.start, "0");
415                assert_eq!(explain.end, "0");
416                assert_eq!(explain.step, "5m");
417                assert_eq!(explain.lookback, None);
418                assert!(!explain.is_verbose);
419            }
420            _ => unreachable!(),
421        }
422
423        let sql = "TQL EXPLAIN VERBOSE http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
424        match parse_into_statement(sql) {
425            Statement::Tql(Tql::Explain(explain)) => {
426                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
427                assert_eq!(explain.start, "0");
428                assert_eq!(explain.end, "0");
429                assert_eq!(explain.step, "5m");
430                assert_eq!(explain.lookback, None);
431                assert!(explain.is_verbose);
432            }
433            _ => unreachable!(),
434        }
435
436        let sql = "TQL EXPLAIN (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
437        match parse_into_statement(sql) {
438            Statement::Tql(Tql::Explain(explain)) => {
439                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
440                assert_eq!(explain.start, "20");
441                assert_eq!(explain.end, "100");
442                assert_eq!(explain.step, "10");
443                assert_eq!(explain.lookback, None);
444                assert!(!explain.is_verbose);
445            }
446            _ => unreachable!(),
447        }
448
449        let sql = "TQL EXPLAIN ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
450        match parse_into_statement(sql) {
451            Statement::Tql(Tql::Explain(explain)) => {
452                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
453                assert_eq!(explain.start, "300");
454                assert_eq!(explain.end, "1200");
455                assert_eq!(explain.step, "10");
456                assert_eq!(explain.lookback, None);
457                assert!(!explain.is_verbose);
458            }
459            _ => unreachable!(),
460        }
461
462        let sql = "TQL EXPLAIN VERBOSE (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
463        match parse_into_statement(sql) {
464            Statement::Tql(Tql::Explain(explain)) => {
465                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
466                assert_eq!(explain.start, "20");
467                assert_eq!(explain.end, "100");
468                assert_eq!(explain.step, "10");
469                assert_eq!(explain.lookback, None);
470                assert!(explain.is_verbose);
471            }
472            _ => unreachable!(),
473        }
474
475        let sql = "TQL EXPLAIN verbose (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
476        match parse_into_statement(sql) {
477            Statement::Tql(Tql::Explain(explain)) => {
478                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
479                assert_eq!(explain.start, "20");
480                assert_eq!(explain.end, "100");
481                assert_eq!(explain.step, "10");
482                assert_eq!(explain.lookback, None);
483                assert!(explain.is_verbose);
484            }
485            _ => unreachable!(),
486        }
487
488        let sql = "TQL EXPLAIN VERBOSE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
489        match parse_into_statement(sql) {
490            Statement::Tql(Tql::Explain(explain)) => {
491                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
492                assert_eq!(explain.start, "300");
493                assert_eq!(explain.end, "1200");
494                assert_eq!(explain.step, "10");
495                assert_eq!(explain.lookback, None);
496                assert!(explain.is_verbose);
497            }
498            _ => unreachable!(),
499        }
500    }
501
502    #[test]
503    fn test_parse_tql_analyze() {
504        let sql = "TQL ANALYZE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
505        match parse_into_statement(sql) {
506            Statement::Tql(Tql::Analyze(analyze)) => {
507                assert_eq!(analyze.start, "1676887657.1");
508                assert_eq!(analyze.end, "1676887659.5");
509                assert_eq!(analyze.step, "30.3");
510                assert_eq!(analyze.lookback, None);
511                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
512                assert!(!analyze.is_verbose);
513            }
514            _ => unreachable!(),
515        }
516
517        let sql = "TQL ANALYZE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
518        match parse_into_statement(sql) {
519            Statement::Tql(Tql::Analyze(analyze)) => {
520                assert_eq!(analyze.start, "300");
521                assert_eq!(analyze.end, "1200");
522                assert_eq!(analyze.step, "10");
523                assert_eq!(analyze.lookback, None);
524                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
525                assert!(!analyze.is_verbose);
526            }
527            _ => unreachable!(),
528        }
529
530        let sql = "TQL ANALYZE VERBOSE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
531        match parse_into_statement(sql) {
532            Statement::Tql(Tql::Analyze(analyze)) => {
533                assert_eq!(analyze.start, "1676887657.1");
534                assert_eq!(analyze.end, "1676887659.5");
535                assert_eq!(analyze.step, "30.3");
536                assert_eq!(analyze.lookback, None);
537                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
538                assert!(analyze.is_verbose);
539            }
540            _ => unreachable!(),
541        }
542
543        let sql = "TQL ANALYZE verbose (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
544        match parse_into_statement(sql) {
545            Statement::Tql(Tql::Analyze(analyze)) => {
546                assert_eq!(analyze.start, "1676887657.1");
547                assert_eq!(analyze.end, "1676887659.5");
548                assert_eq!(analyze.step, "30.3");
549                assert_eq!(analyze.lookback, None);
550                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
551                assert!(analyze.is_verbose);
552            }
553            _ => unreachable!(),
554        }
555
556        let sql = "TQL ANALYZE VERBOSE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
557        match parse_into_statement(sql) {
558            Statement::Tql(Tql::Analyze(analyze)) => {
559                assert_eq!(analyze.start, "300");
560                assert_eq!(analyze.end, "1200");
561                assert_eq!(analyze.step, "10");
562                assert_eq!(analyze.lookback, None);
563                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
564                assert!(analyze.is_verbose);
565            }
566            _ => unreachable!(),
567        }
568    }
569
570    #[test]
571    fn test_parse_tql_with_various_queries() {
572        // query has whitespaces and comma
573        match parse_into_statement("TQL EVAL (0, 30, '10s')           ,       data + (1 < bool 2);")
574        {
575            Statement::Tql(Tql::Eval(eval)) => {
576                assert_eq!(eval.start, "0");
577                assert_eq!(eval.end, "30");
578                assert_eq!(eval.step, "10s");
579                assert_eq!(eval.lookback, None);
580                assert_eq!(eval.query, "data + (1 < bool 2)");
581            }
582            _ => unreachable!(),
583        }
584        // query starts with a quote
585        match parse_into_statement("TQL EVAL (0, 10, '5s') '1+1';") {
586            Statement::Tql(Tql::Eval(eval)) => {
587                assert_eq!(eval.start, "0");
588                assert_eq!(eval.end, "10");
589                assert_eq!(eval.step, "5s");
590                assert_eq!(eval.lookback, None);
591                assert_eq!(eval.query, "'1+1'");
592            }
593            _ => unreachable!(),
594        }
595
596        // query starts with number
597        match parse_into_statement("TQL EVAL (300, 300, '1s') 10 atan2 20;") {
598            Statement::Tql(Tql::Eval(eval)) => {
599                assert_eq!(eval.start, "300");
600                assert_eq!(eval.end, "300");
601                assert_eq!(eval.step, "1s");
602                assert_eq!(eval.lookback, None);
603                assert_eq!(eval.query, "10 atan2 20");
604            }
605            _ => unreachable!(),
606        }
607
608        // query starts with a bracket
609        let sql = "TQL EVAL (0, 30, '10s') (sum by(host) (irate(host_cpu_seconds_total{mode!='idle'}[1m0s])) / sum by (host)((irate(host_cpu_seconds_total[1m0s])))) * 100;";
610        match parse_into_statement(sql) {
611            Statement::Tql(Tql::Eval(eval)) => {
612                assert_eq!(eval.start, "0");
613                assert_eq!(eval.end, "30");
614                assert_eq!(eval.step, "10s");
615                assert_eq!(eval.lookback, None);
616                assert_eq!(eval.query, "(sum by(host) (irate(host_cpu_seconds_total{mode!='idle'}[1m0s])) / sum by (host)((irate(host_cpu_seconds_total[1m0s])))) * 100");
617            }
618            _ => unreachable!(),
619        }
620
621        // query starts with a curly bracket
622        match parse_into_statement("TQL EVAL (0, 10, '5s') {__name__=\"test\"}") {
623            Statement::Tql(Tql::Eval(eval)) => {
624                assert_eq!(eval.start, "0");
625                assert_eq!(eval.end, "10");
626                assert_eq!(eval.step, "5s");
627                assert_eq!(eval.lookback, None);
628                assert_eq!(eval.query, "{__name__=\"test\"}");
629            }
630            _ => unreachable!(),
631        }
632    }
633
634    #[test]
635    fn test_parse_tql_error() {
636        let dialect = &GreptimeDbDialect {};
637        let parse_options = ParseOptions::default();
638
639        // invalid duration
640        let sql = "TQL EVAL (1676887657, 1676887659, 1m) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
641        let result =
642            ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err();
643        assert!(result
644            .output_msg()
645            .contains("Failed to extract a timestamp value"));
646
647        // missing end
648        let sql = "TQL EVAL (1676887657, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
649        let result =
650            ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err();
651        assert!(result
652            .output_msg()
653            .contains("Failed to extract a timestamp value"));
654
655        // empty TQL query
656        let sql = "TQL EVAL (0, 30, '10s')";
657        let result =
658            ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err();
659        assert!(result.output_msg().contains("empty TQL query"));
660
661        // invalid token
662        let sql = "tql eval (0, 0, '1s) t;;';";
663        let result =
664            ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err();
665        assert!(result.output_msg().contains("Delimiters not match"));
666    }
667}
sql/parsers/tql_parser.rs

sql/parsers/
tql_parser.rs
