sql/parsers/create_parser/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
16use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
17use snafu::{ResultExt, ensure};
18use sqlparser::ast::{DataType, ExactNumberInfo, Expr, ObjectName, UnaryOperator};
19use sqlparser::dialect::keywords::Keyword;
20use sqlparser::parser::Parser;
21use sqlparser::tokenizer::Token;
22
23use crate::ast::Ident;
24use crate::dialect::GreptimeDbDialect;
25use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
26use crate::parsers::create_parser::{INVERTED, SKIPPING};
27use crate::statements::create::{Json2Options, JsonTypeHint};
28use crate::statements::transform::type_alias::get_type_by_alias;
29
30const JSON2_TYPE_NAME: &str = "JSON2";
31const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
32
33/// Parses a JSON2 type hint path with the same grammar used by `CREATE TABLE`.
34pub fn parse_json2_type_hint_path(path: &str) -> Result<Vec<String>> {
35    let dialect = GreptimeDbDialect {};
36    let mut parser = Parser::new(&dialect)
37        .try_with_sql(path)
38        .context(SyntaxSnafu)?;
39    let path = parse_json2_path(&mut parser)?;
40    ensure!(
41        parser.peek_token().token == Token::EOF,
42        InvalidSqlSnafu {
43            msg: format!(
44                "unexpected token '{}' in JSON2 type hint path",
45                parser.peek_token()
46            )
47        }
48    );
49    Ok(path)
50}
51
52pub(super) fn parse_json2_type_and_options(
53    parser: &mut Parser<'_>,
54) -> Result<Option<(DataType, Option<Json2Options>)>> {
55    let token = parser.peek_token();
56    let Token::Word(word) = &token.token else {
57        return Ok(None);
58    };
59
60    if !word.value.eq_ignore_ascii_case(JSON2_TYPE_NAME) || word.quote_style.is_some() {
61        return Ok(None);
62    }
63
64    parser.next_token();
65    let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
66    let options = if parser.consume_token(&Token::LParen) {
67        parse_json2_options(parser)?
68    } else {
69        None
70    };
71
72    Ok(Some((data_type, options)))
73}
74
75fn parse_json2_options(parser: &mut Parser<'_>) -> Result<Option<Json2Options>> {
76    if parser.consume_token(&Token::RParen) {
77        return Ok(None);
78    }
79
80    let mut max_auto_expanded_paths = None;
81    let mut type_hints = Vec::new();
82    loop {
83        let token = parser.peek_token();
84        let is_max_auto_expanded_paths = matches!(
85            &token.token,
86            Token::Word(word)
87                if word.quote_style.is_none()
88                    && word.value.eq_ignore_ascii_case(MAX_AUTO_EXPANDED_PATHS)
89        );
90        if is_max_auto_expanded_paths {
91            parser.next_token();
92            ensure!(
93                max_auto_expanded_paths.is_none(),
94                InvalidSqlSnafu {
95                    msg: format!("duplicated JSON2 option '{MAX_AUTO_EXPANDED_PATHS}'")
96                }
97            );
98            parser.expect_token(&Token::Eq).context(SyntaxSnafu)?;
99
100            let token = parser.next_token();
101            let Token::Number(value, _) = token.token else {
102                return InvalidSqlSnafu {
103                    msg: format!(
104                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
105                    ),
106                }
107                .fail();
108            };
109            max_auto_expanded_paths = Some(value.parse::<u32>().map_err(|_| {
110                InvalidSqlSnafu {
111                    msg: format!(
112                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
113                    ),
114                }
115                .build()
116            })?);
117        } else {
118            let hint = parse_json2_type_hint(parser)?;
119            ensure_no_path_conflict(&type_hints, &hint.path)?;
120            type_hints.push(hint);
121        }
122
123        if parser.consume_token(&Token::Comma) {
124            if parser.consume_token(&Token::RParen) {
125                break;
126            }
127        } else {
128            parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
129            break;
130        }
131    }
132
133    Ok(Some(Json2Options {
134        max_auto_expanded_paths,
135        type_hints,
136    }))
137}
138
139fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
140    let path = parse_json2_path(parser)?;
141    ensure!(
142        path.first().is_none_or(|x| x != JSON2_REMAINDER_FIELD_NAME),
143        InvalidSqlSnafu {
144            msg: format!(
145                "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
146            )
147        }
148    );
149    ensure!(
150        path.len() <= JSON2_MAX_STRUCTURED_DEPTH,
151        InvalidSqlSnafu {
152            msg: format!(
153                "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
154            ),
155        }
156    );
157    let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
158    let data_type = normalize_json2_type_hint_type(data_type)?;
159
160    let mut nullable = true;
161    let mut nullable_set = false;
162    let mut default = None;
163    let mut inverted_index = false;
164
165    loop {
166        if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL]) {
167            ensure!(
168                !nullable_set,
169                InvalidSqlSnafu {
170                    msg: format!(
171                        "NULL/NOT NULL option already specified for JSON2 type hint '{}'",
172                        path.join(".")
173                    )
174                }
175            );
176            nullable = false;
177            nullable_set = true;
178        } else if parser.parse_keyword(Keyword::NULL) {
179            ensure!(
180                !nullable_set,
181                InvalidSqlSnafu {
182                    msg: format!(
183                        "NULL/NOT NULL option already specified for JSON2 type hint '{}'",
184                        path.join(".")
185                    )
186                }
187            );
188            nullable = true;
189            nullable_set = true;
190        } else if parser.parse_keyword(Keyword::DEFAULT) {
191            ensure!(
192                default.is_none(),
193                InvalidSqlSnafu {
194                    msg: format!(
195                        "duplicated DEFAULT option for JSON2 type hint '{}'",
196                        path.join(".")
197                    )
198                }
199            );
200            let expr = parser.parse_expr().context(SyntaxSnafu)?;
201            ensure_json2_default_expr_is_literal(&expr)?;
202            default = Some(expr);
203        } else if let Token::Word(word) = parser.peek_token().token
204            && word.value.eq_ignore_ascii_case(INVERTED)
205        {
206            parser.next_token();
207            ensure!(
208                parser.parse_keyword(Keyword::INDEX),
209                InvalidSqlSnafu {
210                    msg: format!(
211                        "expect INDEX after INVERTED keyword for JSON2 type hint '{}'",
212                        path.join(".")
213                    )
214                }
215            );
216            ensure!(
217                !inverted_index,
218                InvalidSqlSnafu {
219                    msg: format!(
220                        "duplicated INVERTED INDEX option for JSON2 type hint '{}'",
221                        path.join(".")
222                    )
223                }
224            );
225            inverted_index = true;
226        } else if let Token::Word(word) = parser.peek_token().token
227            && word.value.eq_ignore_ascii_case(SKIPPING)
228        {
229            return InvalidSqlSnafu {
230                msg: "JSON2 type hint SKIPPING INDEX is not supported yet".to_string(),
231            }
232            .fail();
233        } else if matches!(parser.peek_token().token, Token::Comma | Token::RParen) {
234            break;
235        } else {
236            return parser
237                .expected("JSON2 type hint option", parser.peek_token())
238                .context(SyntaxSnafu);
239        }
240    }
241
242    Ok(JsonTypeHint {
243        path,
244        data_type,
245        nullable,
246        default,
247        inverted_index,
248    })
249}
250
251fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
252    let first = parser.parse_identifier().context(SyntaxSnafu)?;
253    let mut path = vec![first.value];
254
255    while parser.consume_token(&Token::Period) {
256        let segment = parser.parse_identifier().context(SyntaxSnafu)?;
257        path.push(segment.value);
258    }
259
260    ensure!(
261        !path.iter().any(|segment| segment.is_empty()),
262        InvalidSqlSnafu {
263            msg: "JSON2 type hint path segment cannot be empty".to_string(),
264        }
265    );
266
267    Ok(path)
268}
269
270fn normalize_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
271    let data_type = get_type_by_alias(&data_type).unwrap_or(data_type);
272    let normalized = match data_type {
273        DataType::String(_) | DataType::Text | DataType::Varchar(_) | DataType::Char(_) => {
274            DataType::String(None)
275        }
276        DataType::TinyInt(_)
277        | DataType::SmallInt(_)
278        | DataType::Int(_)
279        | DataType::Integer(_)
280        | DataType::BigInt(_) => DataType::BigInt(None),
281        DataType::TinyIntUnsigned(_)
282        | DataType::SmallIntUnsigned(_)
283        | DataType::IntUnsigned(_)
284        | DataType::UnsignedInteger
285        | DataType::BigIntUnsigned(_) => DataType::BigIntUnsigned(None),
286        DataType::Float(_) | DataType::Real | DataType::Double(_) => {
287            DataType::Double(ExactNumberInfo::None)
288        }
289        DataType::Boolean => DataType::Boolean,
290        _ => {
291            return InvalidSqlSnafu {
292                msg: format!("unsupported JSON2 type hint data type: {data_type}"),
293            }
294            .fail();
295        }
296    };
297
298    Ok(normalized)
299}
300
301fn ensure_json2_default_expr_is_literal(expr: &Expr) -> Result<()> {
302    let is_literal = match expr {
303        Expr::Value(_) => true,
304        Expr::UnaryOp { op, expr } => {
305            matches!(op, UnaryOperator::Plus | UnaryOperator::Minus)
306                && matches!(expr.as_ref(), Expr::Value(_))
307        }
308        _ => false,
309    };
310    ensure!(
311        is_literal,
312        InvalidSqlSnafu {
313            msg: "JSON2 type hint DEFAULT only supports literal values",
314        }
315    );
316    Ok(())
317}
318
319fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
320    for hint in hints {
321        ensure!(
322            hint.path != path,
323            InvalidSqlSnafu {
324                msg: format!("duplicated JSON2 type hint path '{}'", path.join("."))
325            }
326        );
327        ensure!(
328            !hint.path.starts_with(path) && !path.starts_with(&hint.path),
329            InvalidSqlSnafu {
330                msg: format!(
331                    "JSON2 type hint path '{}' conflicts with '{}'",
332                    path.join("."),
333                    hint.path.join(".")
334                )
335            }
336        );
337    }
338    Ok(())
339}
340
341#[cfg(test)]
342mod tests {
343    use sqlparser::ast::{DataType, ExactNumberInfo};
344
345    use super::parse_json2_type_hint_path;
346    use crate::dialect::GreptimeDbDialect;
347    use crate::parser::{ParseOptions, ParserContext};
348    use crate::statements::create::Column;
349    use crate::statements::statement::Statement;
350
351    fn parse_json2_column(sql: &str) -> Column {
352        let Statement::CreateTable(mut create_table) =
353            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
354                .unwrap()
355                .remove(0)
356        else {
357            unreachable!()
358        };
359
360        create_table.columns.remove(0)
361    }
362
363    #[test]
364    fn test_parse_json2_type_hint_path() {
365        assert_eq!(
366            parse_json2_type_hint_path(r#"attrs."http.status_code""#).unwrap(),
367            vec!["attrs", "http.status_code"]
368        );
369        assert!(parse_json2_type_hint_path("user.id trailing").is_err());
370    }
371
372    #[test]
373    fn test_parse_json2_type_hints() {
374        let column = parse_json2_column(
375            r#"
376CREATE TABLE traces (
377    log_json_data JSON2 (
378        "service.name" STRING NOT NULL DEFAULT 'null' INVERTED INDEX,
379        http.method STRING NOT NULL,
380        status_code INT64 NOT NULL,
381        comment STRING NULL,
382    ),
383    ts TIMESTAMP TIME INDEX,
384)"#,
385        );
386
387        assert!(matches!(
388            column.column_def.data_type,
389            DataType::Custom(_, _)
390        ));
391        let hints = column.extensions.json2_options.unwrap().type_hints;
392        assert_eq!(hints.len(), 4);
393
394        assert_eq!(hints[0].path, vec!["service.name"]);
395        assert_eq!(hints[0].data_type, DataType::String(None));
396        assert!(!hints[0].nullable);
397        assert_eq!(
398            hints[0]
399                .default
400                .as_ref()
401                .map(|expr| expr.to_string())
402                .as_deref(),
403            Some("'null'")
404        );
405        assert!(hints[0].inverted_index);
406
407        assert_eq!(hints[1].path, vec!["http", "method"]);
408        assert_eq!(hints[1].data_type, DataType::String(None));
409        assert!(!hints[1].nullable);
410        assert!(!hints[1].inverted_index);
411
412        assert_eq!(hints[2].path, vec!["status_code"]);
413        assert_eq!(hints[2].data_type, DataType::BigInt(None));
414        assert!(!hints[2].nullable);
415
416        assert_eq!(hints[3].path, vec!["comment"]);
417        assert_eq!(hints[3].data_type, DataType::String(None));
418        assert!(hints[3].nullable);
419    }
420
421    #[test]
422    fn test_parse_json2_max_auto_expanded_paths() {
423        let column = parse_json2_column(
424            r#"
425CREATE TABLE traces (
426    log_json_data JSON2 (
427        http.method STRING,
428        max_auto_expanded_paths = 0
429    ),
430    ts TIMESTAMP TIME INDEX,
431)"#,
432        );
433
434        let options = column.extensions.json2_options.unwrap();
435        assert_eq!(options.max_auto_expanded_paths, Some(0));
436        assert_eq!(options.type_hints.len(), 1);
437
438        let empty = parse_json2_column(
439            r#"
440CREATE TABLE traces (
441    log_json_data JSON2 (),
442    ts TIMESTAMP TIME INDEX,
443)"#,
444        );
445        assert!(empty.extensions.json2_options.is_none());
446
447        let quoted = parse_json2_column(
448            r#"
449CREATE TABLE traces (
450    log_json_data JSON2 (
451        "max_auto_expanded_paths" STRING,
452        nested."!__remainder__!" STRING
453    ),
454    ts TIMESTAMP TIME INDEX,
455)"#,
456        );
457        let options = quoted.extensions.json2_options.unwrap();
458        assert_eq!(options.max_auto_expanded_paths, None);
459        assert_eq!(options.type_hints.len(), 2);
460    }
461
462    #[test]
463    fn test_parse_json2_max_auto_expanded_paths_rejects_invalid_options() {
464        for options in [
465            "max_auto_expanded_paths = 0, max_auto_expanded_paths = 1",
466            "max_auto_expanded_paths = -1",
467            "max_auto_expanded_paths = 1.5",
468            "max_auto_expanded_paths = 4294967296",
469            r#""!__remainder__!".value STRING"#,
470        ] {
471            let sql = format!(
472                "CREATE TABLE traces (log_json_data JSON2 ({options}), ts TIMESTAMP TIME INDEX)"
473            );
474            assert!(
475                ParserContext::create_with_dialect(
476                    &sql,
477                    &GreptimeDbDialect {},
478                    ParseOptions::default()
479                )
480                .is_err(),
481                "{options}"
482            );
483        }
484    }
485
486    #[test]
487    fn test_parse_json2_type_hint_default_nullable() {
488        let column = parse_json2_column(
489            r#"
490CREATE TABLE traces (
491    log_json_data JSON2 (http.method STRING),
492    ts TIMESTAMP TIME INDEX,
493)"#,
494        );
495
496        let hints = column.extensions.json2_options.unwrap().type_hints;
497        assert_eq!(hints.len(), 1);
498        assert!(hints[0].nullable);
499    }
500
501    #[test]
502    fn test_parse_json2_type_hint_quoted_path_segments() {
503        let column = parse_json2_column(
504            r#"
505CREATE TABLE traces (
506    log_json_data JSON2 (
507        "a".b STRING,
508        "x"."y" STRING,
509        "a.b"."c" STRING,
510        a."b.c" STRING
511    ),
512    ts TIMESTAMP TIME INDEX,
513)"#,
514        );
515
516        let hints = column.extensions.json2_options.unwrap().type_hints;
517        assert_eq!(hints.len(), 4);
518        assert_eq!(hints[0].path, vec!["a", "b"]);
519        assert_eq!(hints[1].path, vec!["x", "y"]);
520        assert_eq!(hints[2].path, vec!["a.b", "c"]);
521        assert_eq!(hints[3].path, vec!["a", "b.c"]);
522    }
523
524    #[test]
525    fn test_parse_json2_type_hint_normalizes_numeric_types() {
526        let column = parse_json2_column(
527            r#"
528CREATE TABLE traces (
529    log_json_data JSON2 (
530        tinyint_value TINYINT,
531        smallint_value SMALLINT,
532        int_value INT,
533        integer_value INTEGER,
534        bigint_value BIGINT,
535        int64_value INT64,
536        tinyuint_value TINYINT UNSIGNED,
537        smalluint_value SMALLINT UNSIGNED,
538        uint_value INT UNSIGNED,
539        uint64_value UINT64,
540        float_value FLOAT,
541        real_value REAL,
542        double_value DOUBLE,
543        float64_value FLOAT64
544    ),
545    ts TIMESTAMP TIME INDEX,
546)"#,
547        );
548
549        let hints = column.extensions.json2_options.unwrap().type_hints;
550        assert_eq!(hints.len(), 14);
551        for hint in hints.iter().take(6) {
552            assert_eq!(hint.data_type, DataType::BigInt(None));
553        }
554        for hint in hints.iter().skip(6).take(4) {
555            assert_eq!(hint.data_type, DataType::BigIntUnsigned(None));
556        }
557        for hint in hints.iter().skip(10) {
558            assert_eq!(hint.data_type, DataType::Double(ExactNumberInfo::None));
559        }
560    }
561
562    #[test]
563    fn test_parse_json2_type_hint_default_accepts_signed_literals() {
564        let column = parse_json2_column(
565            r#"
566CREATE TABLE traces (
567    log_json_data JSON2 (
568        negative_int INT64 DEFAULT -5,
569        positive_float FLOAT64 DEFAULT +1.5
570    ),
571    ts TIMESTAMP TIME INDEX,
572)"#,
573        );
574
575        let hints = column.extensions.json2_options.unwrap().type_hints;
576        assert_eq!(hints.len(), 2);
577        assert_eq!(
578            hints[0]
579                .default
580                .as_ref()
581                .map(|expr| expr.to_string())
582                .as_deref(),
583            Some("-5")
584        );
585        assert_eq!(
586            hints[1]
587                .default
588                .as_ref()
589                .map(|expr| expr.to_string())
590                .as_deref(),
591            Some("+1.5")
592        );
593    }
594
595    #[test]
596    fn test_parse_json2_type_hint_default_rejects_function() {
597        let result = ParserContext::create_with_dialect(
598            r#"
599CREATE TABLE traces (
600    log_json_data JSON2 (status_code INT64 DEFAULT abs(-1)),
601    ts TIMESTAMP TIME INDEX,
602)"#,
603            &GreptimeDbDialect {},
604            ParseOptions::default(),
605        );
606
607        assert!(result.is_err());
608        assert!(
609            result
610                .unwrap_err()
611                .to_string()
612                .contains("DEFAULT only supports literal values")
613        );
614    }
615
616    #[test]
617    fn test_parse_json2_type_hint_rejects_duplicate_path() {
618        let result = ParserContext::create_with_dialect(
619            r#"
620CREATE TABLE traces (
621    log_json_data JSON2 (a.b STRING, a.b INT64),
622    ts TIMESTAMP TIME INDEX,
623)"#,
624            &GreptimeDbDialect {},
625            ParseOptions::default(),
626        );
627
628        assert!(result.is_err());
629        assert!(result.unwrap_err().to_string().contains("duplicated"));
630    }
631
632    #[test]
633    fn test_parse_json2_type_hint_rejects_parent_child_path() {
634        let result = ParserContext::create_with_dialect(
635            r#"
636CREATE TABLE traces (
637    log_json_data JSON2 (a STRING, a.b INT64),
638    ts TIMESTAMP TIME INDEX,
639)"#,
640            &GreptimeDbDialect {},
641            ParseOptions::default(),
642        );
643
644        assert!(result.is_err());
645        assert!(result.unwrap_err().to_string().contains("conflicts"));
646    }
647
648    #[test]
649    fn test_parse_json2_type_hint_rejects_duplicated_nullability() {
650        for sql in [
651            r#"
652CREATE TABLE traces (
653    log_json_data JSON2 (a STRING NULL NULL),
654    ts TIMESTAMP TIME INDEX,
655)"#,
656            r#"
657CREATE TABLE traces (
658    log_json_data JSON2 (a STRING NOT NULL NOT NULL),
659    ts TIMESTAMP TIME INDEX,
660)"#,
661            r#"
662CREATE TABLE traces (
663    log_json_data JSON2 (a STRING NOT NULL NULL),
664    ts TIMESTAMP TIME INDEX,
665)"#,
666            r#"
667CREATE TABLE traces (
668    log_json_data JSON2 (a STRING NULL NOT NULL),
669    ts TIMESTAMP TIME INDEX,
670)"#,
671        ] {
672            let result = ParserContext::create_with_dialect(
673                sql,
674                &GreptimeDbDialect {},
675                ParseOptions::default(),
676            );
677
678            assert!(result.is_err());
679            assert!(
680                result
681                    .unwrap_err()
682                    .to_string()
683                    .contains("NULL/NOT NULL option already specified")
684            );
685        }
686    }
687}
sql/parsers/create_parser/json.rs

sql/parsers/create_parser/
json.rs
