sql/parsers/create_parser/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datatypes::extension::json::JSON2_REMAINDER_FIELD_NAME;
16use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
17use snafu::{ResultExt, ensure};
18use sqlparser::ast::{DataType, ExactNumberInfo, ObjectName};
19use sqlparser::dialect::keywords::Keyword;
20use sqlparser::parser::Parser;
21use sqlparser::tokenizer::Token;
22
23use crate::ast::Ident;
24use crate::dialect::GreptimeDbDialect;
25use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
26use crate::parsers::create_parser::{INVERTED, SKIPPING};
27use crate::statements::create::{Json2Options, JsonTypeHint};
28use crate::statements::transform::type_alias::get_type_by_alias;
29
30const JSON2_TYPE_NAME: &str = "JSON2";
31const MAX_AUTO_EXPANDED_PATHS: &str = "max_auto_expanded_paths";
32
33/// Parses a JSON2 type hint path with the same grammar used by `CREATE TABLE`.
34pub fn parse_json2_type_hint_path(path: &str) -> Result<Vec<String>> {
35    let dialect = GreptimeDbDialect {};
36    let mut parser = Parser::new(&dialect)
37        .try_with_sql(path)
38        .context(SyntaxSnafu)?;
39    let path = parse_json2_path(&mut parser)?;
40    ensure!(
41        parser.peek_token().token == Token::EOF,
42        InvalidSqlSnafu {
43            msg: format!(
44                "unexpected token '{}' in JSON2 type hint path",
45                parser.peek_token()
46            )
47        }
48    );
49    Ok(path)
50}
51
52pub(crate) fn parse_json2_type_and_options(
53    parser: &mut Parser<'_>,
54) -> Result<Option<(DataType, Option<Json2Options>)>> {
55    let token = parser.peek_token();
56    let Token::Word(word) = &token.token else {
57        return Ok(None);
58    };
59
60    if !word.value.eq_ignore_ascii_case(JSON2_TYPE_NAME) || word.quote_style.is_some() {
61        return Ok(None);
62    }
63
64    parser.next_token();
65    let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
66    let options = if parser.consume_token(&Token::LParen) {
67        parse_json2_options(parser)?
68    } else {
69        None
70    };
71
72    Ok(Some((data_type, options)))
73}
74
75fn parse_json2_options(parser: &mut Parser<'_>) -> Result<Option<Json2Options>> {
76    if parser.consume_token(&Token::RParen) {
77        return Ok(None);
78    }
79
80    let mut max_auto_expanded_paths = None;
81    let mut type_hints = Vec::new();
82    loop {
83        let token = parser.peek_token();
84        let is_max_auto_expanded_paths = matches!(
85            &token.token,
86            Token::Word(word)
87                if word.quote_style.is_none()
88                    && word.value.eq_ignore_ascii_case(MAX_AUTO_EXPANDED_PATHS)
89        );
90        if is_max_auto_expanded_paths {
91            parser.next_token();
92            ensure!(
93                max_auto_expanded_paths.is_none(),
94                InvalidSqlSnafu {
95                    msg: format!("duplicated JSON2 option '{MAX_AUTO_EXPANDED_PATHS}'")
96                }
97            );
98            parser.expect_token(&Token::Eq).context(SyntaxSnafu)?;
99
100            let token = parser.next_token();
101            let Token::Number(value, _) = token.token else {
102                return InvalidSqlSnafu {
103                    msg: format!(
104                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
105                    ),
106                }
107                .fail();
108            };
109            max_auto_expanded_paths = Some(value.parse::<u32>().map_err(|_| {
110                InvalidSqlSnafu {
111                    msg: format!(
112                        "JSON2 option '{MAX_AUTO_EXPANDED_PATHS}' expects a non-negative integer"
113                    ),
114                }
115                .build()
116            })?);
117        } else {
118            let hint = parse_json2_type_hint(parser)?;
119            ensure_no_path_conflict(&type_hints, &hint.path)?;
120            type_hints.push(hint);
121        }
122
123        if parser.consume_token(&Token::Comma) {
124            if parser.consume_token(&Token::RParen) {
125                break;
126            }
127        } else {
128            parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
129            break;
130        }
131    }
132
133    Ok(Some(Json2Options {
134        max_auto_expanded_paths,
135        type_hints,
136    }))
137}
138
139fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
140    let path = parse_json2_path(parser)?;
141    ensure!(
142        path.first().is_none_or(|x| x != JSON2_REMAINDER_FIELD_NAME),
143        InvalidSqlSnafu {
144            msg: format!(
145                "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
146            )
147        }
148    );
149    ensure!(
150        path.len() <= JSON2_MAX_STRUCTURED_DEPTH,
151        InvalidSqlSnafu {
152            msg: format!(
153                "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
154            ),
155        }
156    );
157    let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
158    let data_type = normalize_json2_type_hint_type(data_type)?;
159
160    let mut inverted_index = false;
161
162    loop {
163        if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL])
164            || parser.parse_keyword(Keyword::NULL)
165        {
166            return InvalidSqlSnafu {
167                msg: "JSON2 type hint NULL/NOT NULL is not supported; hinted fields are always nullable"
168                    .to_string(),
169            }
170            .fail();
171        } else if parser.parse_keyword(Keyword::DEFAULT) {
172            return InvalidSqlSnafu {
173                msg: "JSON2 type hint DEFAULT is not supported".to_string(),
174            }
175            .fail();
176        } else if let Token::Word(word) = parser.peek_token().token
177            && word.value.eq_ignore_ascii_case(INVERTED)
178        {
179            parser.next_token();
180            ensure!(
181                parser.parse_keyword(Keyword::INDEX),
182                InvalidSqlSnafu {
183                    msg: format!(
184                        "expect INDEX after INVERTED keyword for JSON2 type hint '{}'",
185                        path.join(".")
186                    )
187                }
188            );
189            ensure!(
190                !inverted_index,
191                InvalidSqlSnafu {
192                    msg: format!(
193                        "duplicated INVERTED INDEX option for JSON2 type hint '{}'",
194                        path.join(".")
195                    )
196                }
197            );
198            inverted_index = true;
199        } else if let Token::Word(word) = parser.peek_token().token
200            && word.value.eq_ignore_ascii_case(SKIPPING)
201        {
202            return InvalidSqlSnafu {
203                msg: "JSON2 type hint SKIPPING INDEX is not supported yet".to_string(),
204            }
205            .fail();
206        } else if matches!(parser.peek_token().token, Token::Comma | Token::RParen) {
207            break;
208        } else {
209            return parser
210                .expected("JSON2 type hint option", parser.peek_token())
211                .context(SyntaxSnafu);
212        }
213    }
214
215    Ok(JsonTypeHint {
216        path,
217        data_type,
218        inverted_index,
219    })
220}
221
222fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
223    let first = parser.parse_identifier().context(SyntaxSnafu)?;
224    let mut path = vec![first.value];
225
226    while parser.consume_token(&Token::Period) {
227        let segment = parser.parse_identifier().context(SyntaxSnafu)?;
228        path.push(segment.value);
229    }
230
231    ensure!(
232        !path.iter().any(|segment| segment.is_empty()),
233        InvalidSqlSnafu {
234            msg: "JSON2 type hint path segment cannot be empty".to_string(),
235        }
236    );
237
238    Ok(path)
239}
240
241fn normalize_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
242    let data_type = get_type_by_alias(&data_type).unwrap_or(data_type);
243    let normalized = match data_type {
244        DataType::String(_) | DataType::Text | DataType::Varchar(_) | DataType::Char(_) => {
245            DataType::String(None)
246        }
247        DataType::TinyInt(_)
248        | DataType::SmallInt(_)
249        | DataType::Int(_)
250        | DataType::Integer(_)
251        | DataType::BigInt(_) => DataType::BigInt(None),
252        DataType::TinyIntUnsigned(_)
253        | DataType::SmallIntUnsigned(_)
254        | DataType::IntUnsigned(_)
255        | DataType::UnsignedInteger
256        | DataType::BigIntUnsigned(_) => DataType::BigIntUnsigned(None),
257        DataType::Float(_) | DataType::Real | DataType::Double(_) => {
258            DataType::Double(ExactNumberInfo::None)
259        }
260        DataType::Boolean => DataType::Boolean,
261        _ => {
262            return InvalidSqlSnafu {
263                msg: format!("unsupported JSON2 type hint data type: {data_type}"),
264            }
265            .fail();
266        }
267    };
268
269    Ok(normalized)
270}
271
272fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
273    for hint in hints {
274        ensure!(
275            hint.path != path,
276            InvalidSqlSnafu {
277                msg: format!("duplicated JSON2 type hint path '{}'", path.join("."))
278            }
279        );
280        ensure!(
281            !hint.path.starts_with(path) && !path.starts_with(&hint.path),
282            InvalidSqlSnafu {
283                msg: format!(
284                    "JSON2 type hint path '{}' conflicts with '{}'",
285                    path.join("."),
286                    hint.path.join(".")
287                )
288            }
289        );
290    }
291    Ok(())
292}
293
294#[cfg(test)]
295mod tests {
296    use sqlparser::ast::{DataType, ExactNumberInfo};
297
298    use super::parse_json2_type_hint_path;
299    use crate::dialect::GreptimeDbDialect;
300    use crate::parser::{ParseOptions, ParserContext};
301    use crate::statements::create::Column;
302    use crate::statements::statement::Statement;
303
304    fn parse_json2_column(sql: &str) -> Column {
305        let Statement::CreateTable(mut create_table) =
306            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
307                .unwrap()
308                .remove(0)
309        else {
310            unreachable!()
311        };
312
313        create_table.columns.remove(0)
314    }
315
316    #[test]
317    fn test_parse_json2_type_hint_path() {
318        assert_eq!(
319            parse_json2_type_hint_path(r#"attrs."http.status_code""#).unwrap(),
320            vec!["attrs", "http.status_code"]
321        );
322        assert!(parse_json2_type_hint_path("user.id trailing").is_err());
323    }
324
325    #[test]
326    fn test_parse_json2_type_hints() {
327        let column = parse_json2_column(
328            r#"
329CREATE TABLE traces (
330    log_json_data JSON2 (
331        "service.name" STRING INVERTED INDEX,
332        http.method STRING,
333        status_code INT64,
334        comment STRING,
335    ),
336    ts TIMESTAMP TIME INDEX,
337)"#,
338        );
339
340        assert!(matches!(
341            column.column_def.data_type,
342            DataType::Custom(_, _)
343        ));
344        let hints = column.extensions.json2_options.unwrap().type_hints;
345        assert_eq!(hints.len(), 4);
346
347        assert_eq!(hints[0].path, vec!["service.name"]);
348        assert_eq!(hints[0].data_type, DataType::String(None));
349        assert!(hints[0].inverted_index);
350
351        assert_eq!(hints[1].path, vec!["http", "method"]);
352        assert_eq!(hints[1].data_type, DataType::String(None));
353        assert!(!hints[1].inverted_index);
354
355        assert_eq!(hints[2].path, vec!["status_code"]);
356        assert_eq!(hints[2].data_type, DataType::BigInt(None));
357
358        assert_eq!(hints[3].path, vec!["comment"]);
359        assert_eq!(hints[3].data_type, DataType::String(None));
360    }
361
362    #[test]
363    fn test_parse_json2_max_auto_expanded_paths() {
364        let column = parse_json2_column(
365            r#"
366CREATE TABLE traces (
367    log_json_data JSON2 (
368        http.method STRING,
369        max_auto_expanded_paths = 0
370    ),
371    ts TIMESTAMP TIME INDEX,
372)"#,
373        );
374
375        let options = column.extensions.json2_options.unwrap();
376        assert_eq!(options.max_auto_expanded_paths, Some(0));
377        assert_eq!(options.type_hints.len(), 1);
378
379        let empty = parse_json2_column(
380            r#"
381CREATE TABLE traces (
382    log_json_data JSON2 (),
383    ts TIMESTAMP TIME INDEX,
384)"#,
385        );
386        assert!(empty.extensions.json2_options.is_none());
387
388        let quoted = parse_json2_column(
389            r#"
390CREATE TABLE traces (
391    log_json_data JSON2 (
392        "max_auto_expanded_paths" STRING,
393        nested."!__remainder__!" STRING
394    ),
395    ts TIMESTAMP TIME INDEX,
396)"#,
397        );
398        let options = quoted.extensions.json2_options.unwrap();
399        assert_eq!(options.max_auto_expanded_paths, None);
400        assert_eq!(options.type_hints.len(), 2);
401    }
402
403    #[test]
404    fn test_parse_json2_max_auto_expanded_paths_rejects_invalid_options() {
405        for options in [
406            "max_auto_expanded_paths = 0, max_auto_expanded_paths = 1",
407            "max_auto_expanded_paths = -1",
408            "max_auto_expanded_paths = 1.5",
409            "max_auto_expanded_paths = 4294967296",
410            r#""!__remainder__!".value STRING"#,
411        ] {
412            let sql = format!(
413                "CREATE TABLE traces (log_json_data JSON2 ({options}), ts TIMESTAMP TIME INDEX)"
414            );
415            assert!(
416                ParserContext::create_with_dialect(
417                    &sql,
418                    &GreptimeDbDialect {},
419                    ParseOptions::default()
420                )
421                .is_err(),
422                "{options}"
423            );
424        }
425    }
426
427    #[test]
428    fn test_parse_json2_type_hint_defaults_to_nullable() {
429        let column = parse_json2_column(
430            r#"
431CREATE TABLE traces (
432    log_json_data JSON2 (http.method STRING),
433    ts TIMESTAMP TIME INDEX,
434)"#,
435        );
436
437        let hints = column.extensions.json2_options.unwrap().type_hints;
438        assert_eq!(hints.len(), 1);
439        assert_eq!(hints[0].data_type, DataType::String(None));
440    }
441
442    #[test]
443    fn test_parse_json2_type_hint_quoted_path_segments() {
444        let column = parse_json2_column(
445            r#"
446CREATE TABLE traces (
447    log_json_data JSON2 (
448        "a".b STRING,
449        "x"."y" STRING,
450        "a.b"."c" STRING,
451        a."b.c" STRING
452    ),
453    ts TIMESTAMP TIME INDEX,
454)"#,
455        );
456
457        let hints = column.extensions.json2_options.unwrap().type_hints;
458        assert_eq!(hints.len(), 4);
459        assert_eq!(hints[0].path, vec!["a", "b"]);
460        assert_eq!(hints[1].path, vec!["x", "y"]);
461        assert_eq!(hints[2].path, vec!["a.b", "c"]);
462        assert_eq!(hints[3].path, vec!["a", "b.c"]);
463    }
464
465    #[test]
466    fn test_parse_json2_type_hint_normalizes_numeric_types() {
467        let column = parse_json2_column(
468            r#"
469CREATE TABLE traces (
470    log_json_data JSON2 (
471        tinyint_value TINYINT,
472        smallint_value SMALLINT,
473        int_value INT,
474        integer_value INTEGER,
475        bigint_value BIGINT,
476        int64_value INT64,
477        tinyuint_value TINYINT UNSIGNED,
478        smalluint_value SMALLINT UNSIGNED,
479        uint_value INT UNSIGNED,
480        uint64_value UINT64,
481        float_value FLOAT,
482        real_value REAL,
483        double_value DOUBLE,
484        float64_value FLOAT64
485    ),
486    ts TIMESTAMP TIME INDEX,
487)"#,
488        );
489
490        let hints = column.extensions.json2_options.unwrap().type_hints;
491        assert_eq!(hints.len(), 14);
492        for hint in hints.iter().take(6) {
493            assert_eq!(hint.data_type, DataType::BigInt(None));
494        }
495        for hint in hints.iter().skip(6).take(4) {
496            assert_eq!(hint.data_type, DataType::BigIntUnsigned(None));
497        }
498        for hint in hints.iter().skip(10) {
499            assert_eq!(hint.data_type, DataType::Double(ExactNumberInfo::None));
500        }
501    }
502
503    #[test]
504    fn test_parse_json2_type_hint_rejects_default() {
505        for default in ["-5", "abs(-1)"] {
506            let sql = format!(
507                "CREATE TABLE traces (log_json_data JSON2 (status_code INT64 DEFAULT {default}), ts TIMESTAMP TIME INDEX)"
508            );
509            let err = ParserContext::create_with_dialect(
510                &sql,
511                &GreptimeDbDialect {},
512                ParseOptions::default(),
513            )
514            .unwrap_err();
515            assert!(err.to_string().contains("DEFAULT is not supported"));
516        }
517    }
518
519    #[test]
520    fn test_parse_json2_type_hint_rejects_duplicate_path() {
521        let result = ParserContext::create_with_dialect(
522            r#"
523CREATE TABLE traces (
524    log_json_data JSON2 (a.b STRING, a.b INT64),
525    ts TIMESTAMP TIME INDEX,
526)"#,
527            &GreptimeDbDialect {},
528            ParseOptions::default(),
529        );
530
531        assert!(result.is_err());
532        assert!(result.unwrap_err().to_string().contains("duplicated"));
533    }
534
535    #[test]
536    fn test_parse_json2_type_hint_rejects_parent_child_path() {
537        let result = ParserContext::create_with_dialect(
538            r#"
539CREATE TABLE traces (
540    log_json_data JSON2 (a STRING, a.b INT64),
541    ts TIMESTAMP TIME INDEX,
542)"#,
543            &GreptimeDbDialect {},
544            ParseOptions::default(),
545        );
546
547        assert!(result.is_err());
548        assert!(result.unwrap_err().to_string().contains("conflicts"));
549    }
550
551    #[test]
552    fn test_parse_json2_type_hint_rejects_nullability() {
553        for sql in [
554            r#"
555CREATE TABLE traces (
556    log_json_data JSON2 (a STRING NULL NULL),
557    ts TIMESTAMP TIME INDEX,
558)"#,
559            r#"
560CREATE TABLE traces (
561    log_json_data JSON2 (a STRING NOT NULL NOT NULL),
562    ts TIMESTAMP TIME INDEX,
563)"#,
564            r#"
565CREATE TABLE traces (
566    log_json_data JSON2 (a STRING NOT NULL NULL),
567    ts TIMESTAMP TIME INDEX,
568)"#,
569            r#"
570CREATE TABLE traces (
571    log_json_data JSON2 (a STRING NULL NOT NULL),
572    ts TIMESTAMP TIME INDEX,
573)"#,
574        ] {
575            let result = ParserContext::create_with_dialect(
576                sql,
577                &GreptimeDbDialect {},
578                ParseOptions::default(),
579            );
580
581            assert!(result.is_err());
582            assert!(
583                result
584                    .unwrap_err()
585                    .to_string()
586                    .contains("NULL/NOT NULL is not supported")
587            );
588        }
589    }
590}
sql/parsers/create_parser/json.rs

sql/parsers/create_parser/
json.rs
