sql/parsers/
tql_parser.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datafusion_common::ScalarValue;
16use snafu::{OptionExt, ResultExt};
17use sqlparser::ast::AnalyzeFormat;
18use sqlparser::keywords::Keyword;
19use sqlparser::parser::ParserError;
20use sqlparser::tokenizer::Token;
21
22use crate::error::{self, Result};
23use crate::parser::ParserContext;
24use crate::parsers::utils;
25use crate::statements::statement::Statement;
26use crate::statements::tql::{Tql, TqlAnalyze, TqlEval, TqlExplain, TqlParameters};
27use crate::util::location_to_index;
28
29pub const TQL: &str = "TQL";
30const EVAL: &str = "EVAL";
31const EVALUATE: &str = "EVALUATE";
32const VERBOSE: &str = "VERBOSE";
33const FORMAT: &str = "FORMAT";
34
35use sqlparser::parser::Parser;
36
37use crate::dialect::GreptimeDbDialect;
38use crate::parsers::error::{
39    ConvertToLogicalExpressionSnafu, EvaluationSnafu, ParserSnafu, TQLError,
40};
41
42/// TQL extension parser, including:
43/// - `TQL EVAL <query>`
44/// - `TQL EXPLAIN [VERBOSE] [FORMAT format] <query>`
45/// - `TQL ANALYZE [VERBOSE] [FORMAT format] <query>`
46impl ParserContext<'_> {
47    pub(crate) fn parse_tql(&mut self) -> Result<Statement> {
48        let _ = self.parser.next_token();
49
50        match self.parser.peek_token().token {
51            Token::Word(w) => {
52                let uppercase = w.value.to_uppercase();
53                let _consume_tql_keyword_token = self.parser.next_token();
54                match w.keyword {
55                    Keyword::NoKeyword
56                        if (uppercase == EVAL || uppercase == EVALUATE)
57                            && w.quote_style.is_none() =>
58                    {
59                        self.parse_tql_params()
60                            .map(|params| Statement::Tql(Tql::Eval(TqlEval::from(params))))
61                            .context(error::TQLSyntaxSnafu)
62                    }
63
64                    Keyword::EXPLAIN => {
65                        let is_verbose = self.has_verbose_keyword();
66                        if is_verbose {
67                            let _consume_verbose_token = self.parser.next_token();
68                        }
69                        let format = self.parse_format_option();
70                        self.parse_tql_params()
71                            .map(|mut params| {
72                                params.is_verbose = is_verbose;
73                                params.format = format;
74                                Statement::Tql(Tql::Explain(TqlExplain::from(params)))
75                            })
76                            .context(error::TQLSyntaxSnafu)
77                    }
78
79                    Keyword::ANALYZE => {
80                        let is_verbose = self.has_verbose_keyword();
81                        if is_verbose {
82                            let _consume_verbose_token = self.parser.next_token();
83                        }
84                        let format = self.parse_format_option();
85                        self.parse_tql_params()
86                            .map(|mut params| {
87                                params.is_verbose = is_verbose;
88                                params.format = format;
89                                Statement::Tql(Tql::Analyze(TqlAnalyze::from(params)))
90                            })
91                            .context(error::TQLSyntaxSnafu)
92                    }
93                    _ => self.unsupported(self.peek_token_as_string()),
94                }
95            }
96            unexpected => self.unsupported(unexpected.to_string()),
97        }
98    }
99
100    fn parse_tql_params(&mut self) -> std::result::Result<TqlParameters, TQLError> {
101        let parser = &mut self.parser;
102        let (start, end, step, lookback) = match parser.peek_token().token {
103            Token::LParen => {
104                let _consume_lparen_token = parser.next_token();
105                let start = Self::parse_string_or_number_or_word(parser, &[Token::Comma])?.0;
106                let end = Self::parse_string_or_number_or_word(parser, &[Token::Comma])?.0;
107
108                let (step, delimiter) =
109                    Self::parse_string_or_number_or_word(parser, &[Token::Comma, Token::RParen])?;
110                let lookback = if delimiter == Token::Comma {
111                    Self::parse_string_or_number_or_word(parser, &[Token::RParen])
112                        .ok()
113                        .map(|t| t.0)
114                } else {
115                    None
116                };
117
118                (start, end, step, lookback)
119            }
120            _ => ("0".to_string(), "0".to_string(), "5m".to_string(), None),
121        };
122        let query = Self::parse_tql_query(parser, self.sql).context(ParserSnafu)?;
123        Ok(TqlParameters::new(start, end, step, lookback, query))
124    }
125
126    pub fn comma_or_rparen(token: &Token) -> bool {
127        Self::is_comma(token) || Self::is_rparen(token)
128    }
129
130    #[inline]
131    fn is_comma(token: &Token) -> bool {
132        matches!(token, Token::Comma)
133    }
134
135    #[inline]
136    fn is_rparen(token: &Token) -> bool {
137        matches!(token, Token::RParen)
138    }
139
140    fn has_verbose_keyword(&mut self) -> bool {
141        self.peek_token_as_string().eq_ignore_ascii_case(VERBOSE)
142    }
143
144    fn parse_format_option(&mut self) -> Option<AnalyzeFormat> {
145        if self.peek_token_as_string().eq_ignore_ascii_case(FORMAT) {
146            let _consume_format_token = self.parser.next_token();
147            // Parse format type
148            if let Token::Word(w) = &self.parser.peek_token().token {
149                let format_type = w.value.to_uppercase();
150                let _consume_format_type_token = self.parser.next_token();
151                match format_type.as_str() {
152                    "JSON" => Some(AnalyzeFormat::JSON),
153                    "TEXT" => Some(AnalyzeFormat::TEXT),
154                    "GRAPHVIZ" => Some(AnalyzeFormat::GRAPHVIZ),
155                    _ => None, // Invalid format, ignore silently
156                }
157            } else {
158                None
159            }
160        } else {
161            None
162        }
163    }
164
165    /// Try to parse and consume a string, number or word token.
166    /// Return `Ok` if it's parsed and one of the given delimiter tokens is consumed.
167    /// The string and matched delimiter will be returned as a tuple.
168    fn parse_string_or_number_or_word(
169        parser: &mut Parser,
170        delimiter_tokens: &[Token],
171    ) -> std::result::Result<(String, Token), TQLError> {
172        let mut tokens = vec![];
173
174        while !delimiter_tokens.contains(&parser.peek_token().token) {
175            let token = parser.next_token().token;
176            if matches!(token, Token::EOF) {
177                break;
178            }
179            tokens.push(token);
180        }
181        let result = match tokens.len() {
182            0 => Err(ParserError::ParserError(
183                "Expected at least one token".to_string(),
184            ))
185            .context(ParserSnafu),
186            1 => {
187                let value = match tokens[0].clone() {
188                    Token::Number(n, _) => n,
189                    Token::DoubleQuotedString(s) | Token::SingleQuotedString(s) => s,
190                    Token::Word(_) => Self::parse_tokens_to_ts(tokens)?,
191                    unexpected => {
192                        return Err(ParserError::ParserError(format!(
193                            "Expected number, string or word, but have {unexpected:?}"
194                        )))
195                        .context(ParserSnafu);
196                    }
197                };
198                Ok(value)
199            }
200            _ => Self::parse_tokens_to_ts(tokens),
201        };
202        for token in delimiter_tokens {
203            if parser.consume_token(token) {
204                return result.map(|v| (v, token.clone()));
205            }
206        }
207        Err(ParserError::ParserError(format!(
208            "Delimiters not match {delimiter_tokens:?}"
209        )))
210        .context(ParserSnafu)
211    }
212
213    /// Parse the tokens to seconds and convert to string.
214    fn parse_tokens_to_ts(tokens: Vec<Token>) -> std::result::Result<String, TQLError> {
215        let parser_expr = Self::parse_to_expr(tokens)?;
216        let lit = utils::parser_expr_to_scalar_value_literal(parser_expr)
217            .map_err(Box::new)
218            .context(ConvertToLogicalExpressionSnafu)?;
219
220        let second = match lit {
221            ScalarValue::TimestampNanosecond(ts_nanos, _)
222            | ScalarValue::DurationNanosecond(ts_nanos) => ts_nanos.map(|v| v / 1_000_000_000),
223            ScalarValue::TimestampMicrosecond(ts_micros, _)
224            | ScalarValue::DurationMicrosecond(ts_micros) => ts_micros.map(|v| v / 1_000_000),
225            ScalarValue::TimestampMillisecond(ts_millis, _)
226            | ScalarValue::DurationMillisecond(ts_millis) => ts_millis.map(|v| v / 1_000),
227            ScalarValue::TimestampSecond(ts_secs, _) | ScalarValue::DurationSecond(ts_secs) => {
228                ts_secs
229            }
230            _ => None,
231        };
232
233        second.map(|ts| ts.to_string()).context(EvaluationSnafu {
234            msg: format!("Failed to extract a timestamp value {lit:?}"),
235        })
236    }
237
238    fn parse_to_expr(tokens: Vec<Token>) -> std::result::Result<sqlparser::ast::Expr, TQLError> {
239        Parser::new(&GreptimeDbDialect {})
240            .with_tokens(tokens)
241            .parse_expr()
242            .context(ParserSnafu)
243    }
244
245    fn parse_tql_query(parser: &mut Parser, sql: &str) -> std::result::Result<String, ParserError> {
246        while matches!(parser.peek_token().token, Token::Comma) {
247            let _skip_token = parser.next_token();
248        }
249        let start_tql = parser.next_token();
250        if start_tql == Token::EOF {
251            return Err(ParserError::ParserError("empty TQL query".to_string()));
252        }
253
254        let start_location = start_tql.span.start;
255        // translate the start location to the index in the sql string
256        let index = location_to_index(sql, &start_location);
257
258        let query = &sql[index - 1..];
259        while parser.next_token() != Token::EOF {
260            // consume all tokens
261            // TODO(dennis): supports multi TQL statements separated by ';'?
262        }
263        // remove the last ';' or tailing space if exists
264        Ok(query.trim().trim_end_matches(';').to_string())
265    }
266}
267
268#[cfg(test)]
269mod tests {
270    use common_error::ext::ErrorExt;
271
272    use super::*;
273    use crate::dialect::GreptimeDbDialect;
274    use crate::parser::ParseOptions;
275
276    fn parse_into_statement(sql: &str) -> Statement {
277        let mut result =
278            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
279                .unwrap();
280        assert_eq!(1, result.len());
281        result.remove(0)
282    }
283
284    #[test]
285    fn test_parse_tql_eval_with_functions() {
286        let sql = "TQL EVAL (now() - now(), now() -  (now() - '10 seconds'::interval), '1s') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
287        let statement = parse_into_statement(sql);
288        match statement {
289            Statement::Tql(Tql::Eval(eval)) => {
290                assert_eq!(eval.start, "0");
291                assert_eq!(eval.end, "10");
292                assert_eq!(eval.step, "1s");
293                assert_eq!(eval.lookback, None);
294                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
295            }
296            _ => unreachable!(),
297        }
298
299        let sql = "TQL EVAL ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
300        match parse_into_statement(sql) {
301            Statement::Tql(Tql::Eval(eval)) => {
302                assert_eq!(eval.start, "300");
303                assert_eq!(eval.end, "1200");
304                assert_eq!(eval.step, "1m");
305                assert_eq!(eval.lookback, None);
306                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
307            }
308            _ => unreachable!(),
309        }
310
311        let sql = "TQL EVAL (now(), now()-'5m', '30s') http_requests_total";
312        let result =
313            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default());
314        assert!(result.is_err());
315    }
316
317    #[test]
318    fn test_parse_tql_eval() {
319        let sql = "TQL EVAL (1676887657, 1676887659, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
320        match parse_into_statement(sql) {
321            Statement::Tql(Tql::Eval(eval)) => {
322                assert_eq!(eval.start, "1676887657");
323                assert_eq!(eval.end, "1676887659");
324                assert_eq!(eval.step, "1m");
325                assert_eq!(eval.lookback, None);
326                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
327            }
328            _ => unreachable!(),
329        }
330
331        let sql = "TQL EVAL (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
332        let statement = parse_into_statement(sql);
333
334        match &statement {
335            Statement::Tql(Tql::Eval(eval)) => {
336                assert_eq!(eval.start, "1676887657.1");
337                assert_eq!(eval.end, "1676887659.5");
338                assert_eq!(eval.step, "30.3");
339                assert_eq!(eval.lookback, None);
340                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
341            }
342            _ => unreachable!(),
343        }
344
345        let sql2 = "TQL EVALUATE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
346        let statement2 = parse_into_statement(sql2);
347        assert_eq!(statement, statement2);
348
349        let sql = "tql eval ('2015-07-01T20:10:30.781Z', '2015-07-01T20:11:00.781Z', '30s') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
350        match parse_into_statement(sql) {
351            Statement::Tql(Tql::Eval(eval)) => {
352                assert_eq!(eval.start, "2015-07-01T20:10:30.781Z");
353                assert_eq!(eval.end, "2015-07-01T20:11:00.781Z");
354                assert_eq!(eval.step, "30s");
355                assert_eq!(eval.lookback, None);
356                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
357            }
358            _ => unreachable!(),
359        }
360    }
361
362    #[test]
363    fn test_parse_tql_with_lookback_values() {
364        let sql = "TQL EVAL (1676887657, 1676887659, '1m', '5m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
365        match parse_into_statement(sql) {
366            Statement::Tql(Tql::Eval(eval)) => {
367                assert_eq!(eval.start, "1676887657");
368                assert_eq!(eval.end, "1676887659");
369                assert_eq!(eval.step, "1m".to_string());
370                assert_eq!(eval.lookback, Some("5m".to_string()));
371                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
372            }
373            _ => unreachable!(),
374        }
375
376        let sql = "TQL EVAL ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, '1m', '7m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
377        match parse_into_statement(sql) {
378            Statement::Tql(Tql::Eval(eval)) => {
379                assert_eq!(eval.start, "300");
380                assert_eq!(eval.end, "1200");
381                assert_eq!(eval.step, "1m");
382                assert_eq!(eval.lookback, Some("7m".to_string()));
383                assert_eq!(eval.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
384            }
385            _ => unreachable!(),
386        }
387
388        let sql = "TQL EXPLAIN (20, 100, 10, '3m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
389        match parse_into_statement(sql) {
390            Statement::Tql(Tql::Explain(explain)) => {
391                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
392                assert_eq!(explain.start, "20");
393                assert_eq!(explain.end, "100");
394                assert_eq!(explain.step, "10");
395                assert_eq!(explain.lookback, Some("3m".to_string()));
396                assert!(!explain.is_verbose);
397            }
398            _ => unreachable!(),
399        }
400
401        let sql = "TQL EXPLAIN VERBOSE (20, 100, 10, '3m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
402        match parse_into_statement(sql) {
403            Statement::Tql(Tql::Explain(explain)) => {
404                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
405                assert_eq!(explain.start, "20");
406                assert_eq!(explain.end, "100");
407                assert_eq!(explain.step, "10");
408                assert_eq!(explain.lookback, Some("3m".to_string()));
409                assert!(explain.is_verbose);
410            }
411            _ => unreachable!(),
412        }
413
414        let sql = "TQL ANALYZE (1676887657, 1676887659, '1m', '9m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
415        match parse_into_statement(sql) {
416            Statement::Tql(Tql::Analyze(analyze)) => {
417                assert_eq!(analyze.start, "1676887657");
418                assert_eq!(analyze.end, "1676887659");
419                assert_eq!(analyze.step, "1m");
420                assert_eq!(analyze.lookback, Some("9m".to_string()));
421                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
422                assert!(!analyze.is_verbose);
423            }
424            _ => unreachable!(),
425        }
426
427        let sql = "TQL ANALYZE VERBOSE (1676887657, 1676887659, '1m', '9m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
428        match parse_into_statement(sql) {
429            Statement::Tql(Tql::Analyze(analyze)) => {
430                assert_eq!(analyze.start, "1676887657");
431                assert_eq!(analyze.end, "1676887659");
432                assert_eq!(analyze.step, "1m");
433                assert_eq!(analyze.lookback, Some("9m".to_string()));
434                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
435                assert!(analyze.is_verbose);
436            }
437            _ => unreachable!(),
438        }
439    }
440
441    #[test]
442    fn test_parse_tql_explain() {
443        let sql = "TQL EXPLAIN http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
444        match parse_into_statement(sql) {
445            Statement::Tql(Tql::Explain(explain)) => {
446                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
447                assert_eq!(explain.start, "0");
448                assert_eq!(explain.end, "0");
449                assert_eq!(explain.step, "5m");
450                assert_eq!(explain.lookback, None);
451                assert!(!explain.is_verbose);
452                assert_eq!(explain.format, None);
453            }
454            _ => unreachable!(),
455        }
456
457        let sql = "TQL EXPLAIN VERBOSE http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
458        match parse_into_statement(sql) {
459            Statement::Tql(Tql::Explain(explain)) => {
460                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
461                assert_eq!(explain.start, "0");
462                assert_eq!(explain.end, "0");
463                assert_eq!(explain.step, "5m");
464                assert_eq!(explain.lookback, None);
465                assert!(explain.is_verbose);
466                assert_eq!(explain.format, None);
467            }
468            _ => unreachable!(),
469        }
470
471        let sql = "TQL EXPLAIN FORMAT JSON http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
472        match parse_into_statement(sql) {
473            Statement::Tql(Tql::Explain(explain)) => {
474                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
475                assert_eq!(explain.start, "0");
476                assert_eq!(explain.end, "0");
477                assert_eq!(explain.step, "5m");
478                assert_eq!(explain.lookback, None);
479                assert!(!explain.is_verbose);
480                assert_eq!(explain.format, Some(AnalyzeFormat::JSON));
481            }
482            _ => unreachable!(),
483        }
484
485        let sql = "TQL EXPLAIN VERBOSE FORMAT JSON http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
486        match parse_into_statement(sql) {
487            Statement::Tql(Tql::Explain(explain)) => {
488                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
489                assert_eq!(explain.start, "0");
490                assert_eq!(explain.end, "0");
491                assert_eq!(explain.step, "5m");
492                assert_eq!(explain.lookback, None);
493                assert!(explain.is_verbose);
494                assert_eq!(explain.format, Some(AnalyzeFormat::JSON));
495            }
496            _ => unreachable!(),
497        }
498
499        let sql = "TQL EXPLAIN FORMAT TEXT (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
500        match parse_into_statement(sql) {
501            Statement::Tql(Tql::Explain(explain)) => {
502                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
503                assert_eq!(explain.start, "20");
504                assert_eq!(explain.end, "100");
505                assert_eq!(explain.step, "10");
506                assert_eq!(explain.lookback, None);
507                assert!(!explain.is_verbose);
508                assert_eq!(explain.format, Some(AnalyzeFormat::TEXT));
509            }
510            _ => unreachable!(),
511        }
512
513        let sql = "TQL EXPLAIN (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
514        match parse_into_statement(sql) {
515            Statement::Tql(Tql::Explain(explain)) => {
516                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
517                assert_eq!(explain.start, "20");
518                assert_eq!(explain.end, "100");
519                assert_eq!(explain.step, "10");
520                assert_eq!(explain.lookback, None);
521                assert!(!explain.is_verbose);
522                assert_eq!(explain.format, None);
523            }
524            _ => unreachable!(),
525        }
526
527        let sql = "TQL EXPLAIN ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
528        match parse_into_statement(sql) {
529            Statement::Tql(Tql::Explain(explain)) => {
530                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
531                assert_eq!(explain.start, "300");
532                assert_eq!(explain.end, "1200");
533                assert_eq!(explain.step, "10");
534                assert_eq!(explain.lookback, None);
535                assert!(!explain.is_verbose);
536                assert_eq!(explain.format, None);
537            }
538            _ => unreachable!(),
539        }
540
541        let sql = "TQL EXPLAIN VERBOSE (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
542        match parse_into_statement(sql) {
543            Statement::Tql(Tql::Explain(explain)) => {
544                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
545                assert_eq!(explain.start, "20");
546                assert_eq!(explain.end, "100");
547                assert_eq!(explain.step, "10");
548                assert_eq!(explain.lookback, None);
549                assert!(explain.is_verbose);
550                assert_eq!(explain.format, None);
551            }
552            _ => unreachable!(),
553        }
554
555        let sql = "TQL EXPLAIN verbose (20,100,10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
556        match parse_into_statement(sql) {
557            Statement::Tql(Tql::Explain(explain)) => {
558                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
559                assert_eq!(explain.start, "20");
560                assert_eq!(explain.end, "100");
561                assert_eq!(explain.step, "10");
562                assert_eq!(explain.lookback, None);
563                assert!(explain.is_verbose);
564                assert_eq!(explain.format, None);
565            }
566            _ => unreachable!(),
567        }
568
569        let sql = "TQL EXPLAIN VERBOSE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
570        match parse_into_statement(sql) {
571            Statement::Tql(Tql::Explain(explain)) => {
572                assert_eq!(explain.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
573                assert_eq!(explain.start, "300");
574                assert_eq!(explain.end, "1200");
575                assert_eq!(explain.step, "10");
576                assert_eq!(explain.lookback, None);
577                assert!(explain.is_verbose);
578                assert_eq!(explain.format, None);
579            }
580            _ => unreachable!(),
581        }
582    }
583
584    #[test]
585    fn test_parse_tql_analyze() {
586        let sql = "TQL ANALYZE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
587        match parse_into_statement(sql) {
588            Statement::Tql(Tql::Analyze(analyze)) => {
589                assert_eq!(analyze.start, "1676887657.1");
590                assert_eq!(analyze.end, "1676887659.5");
591                assert_eq!(analyze.step, "30.3");
592                assert_eq!(analyze.lookback, None);
593                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
594                assert!(!analyze.is_verbose);
595                assert_eq!(analyze.format, None);
596            }
597            _ => unreachable!(),
598        }
599
600        let sql = "TQL ANALYZE FORMAT JSON (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
601        match parse_into_statement(sql) {
602            Statement::Tql(Tql::Analyze(analyze)) => {
603                assert_eq!(analyze.start, "1676887657.1");
604                assert_eq!(analyze.end, "1676887659.5");
605                assert_eq!(analyze.step, "30.3");
606                assert_eq!(analyze.lookback, None);
607                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
608                assert!(!analyze.is_verbose);
609                assert_eq!(analyze.format, Some(AnalyzeFormat::JSON));
610            }
611            _ => unreachable!(),
612        }
613
614        let sql = "TQL ANALYZE VERBOSE FORMAT JSON (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
615        match parse_into_statement(sql) {
616            Statement::Tql(Tql::Analyze(analyze)) => {
617                assert_eq!(analyze.start, "1676887657.1");
618                assert_eq!(analyze.end, "1676887659.5");
619                assert_eq!(analyze.step, "30.3");
620                assert_eq!(analyze.lookback, None);
621                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
622                assert!(analyze.is_verbose);
623                assert_eq!(analyze.format, Some(AnalyzeFormat::JSON));
624            }
625            _ => unreachable!(),
626        }
627
628        let sql = "TQL ANALYZE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
629        match parse_into_statement(sql) {
630            Statement::Tql(Tql::Analyze(analyze)) => {
631                assert_eq!(analyze.start, "300");
632                assert_eq!(analyze.end, "1200");
633                assert_eq!(analyze.step, "10");
634                assert_eq!(analyze.lookback, None);
635                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
636                assert!(!analyze.is_verbose);
637                assert_eq!(analyze.format, None);
638            }
639            _ => unreachable!(),
640        }
641
642        let sql = "TQL ANALYZE VERBOSE (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
643        match parse_into_statement(sql) {
644            Statement::Tql(Tql::Analyze(analyze)) => {
645                assert_eq!(analyze.start, "1676887657.1");
646                assert_eq!(analyze.end, "1676887659.5");
647                assert_eq!(analyze.step, "30.3");
648                assert_eq!(analyze.lookback, None);
649                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
650                assert!(analyze.is_verbose);
651                assert_eq!(analyze.format, None);
652            }
653            _ => unreachable!(),
654        }
655
656        let sql = "TQL ANALYZE verbose (1676887657.1, 1676887659.5, 30.3) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
657        match parse_into_statement(sql) {
658            Statement::Tql(Tql::Analyze(analyze)) => {
659                assert_eq!(analyze.start, "1676887657.1");
660                assert_eq!(analyze.end, "1676887659.5");
661                assert_eq!(analyze.step, "30.3");
662                assert_eq!(analyze.lookback, None);
663                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
664                assert!(analyze.is_verbose);
665                assert_eq!(analyze.format, None);
666            }
667            _ => unreachable!(),
668        }
669
670        let sql = "TQL ANALYZE VERBOSE ('1970-01-01T00:05:00'::timestamp, '1970-01-01T00:10:00'::timestamp + '10 minutes'::interval, 10) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
671        match parse_into_statement(sql) {
672            Statement::Tql(Tql::Analyze(analyze)) => {
673                assert_eq!(analyze.start, "300");
674                assert_eq!(analyze.end, "1200");
675                assert_eq!(analyze.step, "10");
676                assert_eq!(analyze.lookback, None);
677                assert_eq!(analyze.query, "http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m");
678                assert!(analyze.is_verbose);
679                assert_eq!(analyze.format, None);
680            }
681            _ => unreachable!(),
682        }
683    }
684
685    #[test]
686    fn test_parse_tql_format() {
687        // Test FORMAT JSON for EXPLAIN
688        let sql = "TQL EXPLAIN FORMAT JSON http_requests_total";
689        match parse_into_statement(sql) {
690            Statement::Tql(Tql::Explain(explain)) => {
691                assert_eq!(explain.format, Some(AnalyzeFormat::JSON));
692                assert!(!explain.is_verbose);
693            }
694            _ => unreachable!(),
695        }
696
697        // Test FORMAT TEXT for EXPLAIN
698        let sql = "TQL EXPLAIN FORMAT TEXT http_requests_total";
699        match parse_into_statement(sql) {
700            Statement::Tql(Tql::Explain(explain)) => {
701                assert_eq!(explain.format, Some(AnalyzeFormat::TEXT));
702                assert!(!explain.is_verbose);
703            }
704            _ => unreachable!(),
705        }
706
707        // Test FORMAT GRAPHVIZ for EXPLAIN
708        let sql = "TQL EXPLAIN FORMAT GRAPHVIZ http_requests_total";
709        match parse_into_statement(sql) {
710            Statement::Tql(Tql::Explain(explain)) => {
711                assert_eq!(explain.format, Some(AnalyzeFormat::GRAPHVIZ));
712                assert!(!explain.is_verbose);
713            }
714            _ => unreachable!(),
715        }
716
717        // Test VERBOSE FORMAT JSON for ANALYZE
718        let sql = "TQL ANALYZE VERBOSE FORMAT JSON (0,10,'5s') http_requests_total";
719        match parse_into_statement(sql) {
720            Statement::Tql(Tql::Analyze(analyze)) => {
721                assert_eq!(analyze.format, Some(AnalyzeFormat::JSON));
722                assert!(analyze.is_verbose);
723            }
724            _ => unreachable!(),
725        }
726
727        // Test FORMAT before parameters
728        let sql = "TQL EXPLAIN FORMAT JSON (0,10,'5s') http_requests_total";
729        match parse_into_statement(sql) {
730            Statement::Tql(Tql::Explain(explain)) => {
731                assert_eq!(explain.format, Some(AnalyzeFormat::JSON));
732                assert_eq!(explain.start, "0");
733                assert_eq!(explain.end, "10");
734                assert_eq!(explain.step, "5s");
735            }
736            _ => unreachable!(),
737        }
738    }
739
740    #[test]
741    fn test_parse_tql_with_various_queries() {
742        // query has whitespaces and comma
743        match parse_into_statement("TQL EVAL (0, 30, '10s')           ,       data + (1 < bool 2);")
744        {
745            Statement::Tql(Tql::Eval(eval)) => {
746                assert_eq!(eval.start, "0");
747                assert_eq!(eval.end, "30");
748                assert_eq!(eval.step, "10s");
749                assert_eq!(eval.lookback, None);
750                assert_eq!(eval.query, "data + (1 < bool 2)");
751            }
752            _ => unreachable!(),
753        }
754        // query starts with a quote
755        match parse_into_statement("TQL EVAL (0, 10, '5s') '1+1';") {
756            Statement::Tql(Tql::Eval(eval)) => {
757                assert_eq!(eval.start, "0");
758                assert_eq!(eval.end, "10");
759                assert_eq!(eval.step, "5s");
760                assert_eq!(eval.lookback, None);
761                assert_eq!(eval.query, "'1+1'");
762            }
763            _ => unreachable!(),
764        }
765
766        // query starts with number
767        match parse_into_statement("TQL EVAL (300, 300, '1s') 10 atan2 20;") {
768            Statement::Tql(Tql::Eval(eval)) => {
769                assert_eq!(eval.start, "300");
770                assert_eq!(eval.end, "300");
771                assert_eq!(eval.step, "1s");
772                assert_eq!(eval.lookback, None);
773                assert_eq!(eval.query, "10 atan2 20");
774            }
775            _ => unreachable!(),
776        }
777
778        // query starts with a bracket
779        let sql = "TQL EVAL (0, 30, '10s') (sum by(host) (irate(host_cpu_seconds_total{mode!='idle'}[1m0s])) / sum by (host)((irate(host_cpu_seconds_total[1m0s])))) * 100;";
780        match parse_into_statement(sql) {
781            Statement::Tql(Tql::Eval(eval)) => {
782                assert_eq!(eval.start, "0");
783                assert_eq!(eval.end, "30");
784                assert_eq!(eval.step, "10s");
785                assert_eq!(eval.lookback, None);
786                assert_eq!(eval.query, "(sum by(host) (irate(host_cpu_seconds_total{mode!='idle'}[1m0s])) / sum by (host)((irate(host_cpu_seconds_total[1m0s])))) * 100");
787            }
788            _ => unreachable!(),
789        }
790
791        // query starts with a curly bracket
792        match parse_into_statement("TQL EVAL (0, 10, '5s') {__name__=\"test\"}") {
793            Statement::Tql(Tql::Eval(eval)) => {
794                assert_eq!(eval.start, "0");
795                assert_eq!(eval.end, "10");
796                assert_eq!(eval.step, "5s");
797                assert_eq!(eval.lookback, None);
798                assert_eq!(eval.query, "{__name__=\"test\"}");
799            }
800            _ => unreachable!(),
801        }
802    }
803
804    #[test]
805    fn test_parse_tql_error() {
806        let dialect = &GreptimeDbDialect {};
807        let parse_options = ParseOptions::default();
808
809        // invalid duration
810        let sql = "TQL EVAL (1676887657, 1676887659, 1m) http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
811        let result =
812            ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err();
813        assert!(result
814            .output_msg()
815            .contains("Failed to extract a timestamp value"));
816
817        // missing end
818        let sql = "TQL EVAL (1676887657, '1m') http_requests_total{environment=~'staging|testing|development',method!='GET'} @ 1609746000 offset 5m";
819        let result =
820            ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err();
821        assert!(result
822            .output_msg()
823            .contains("Failed to extract a timestamp value"));
824
825        // empty TQL query
826        let sql = "TQL EVAL (0, 30, '10s')";
827        let result =
828            ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err();
829        assert!(result.output_msg().contains("empty TQL query"));
830
831        // invalid token
832        let sql = "tql eval (0, 0, '1s) t;;';";
833        let result =
834            ParserContext::create_with_dialect(sql, dialect, parse_options.clone()).unwrap_err();
835        assert!(result.output_msg().contains("Delimiters not match"));
836    }
837}
sql/parsers/tql_parser.rs

sql/parsers/
tql_parser.rs
