sql/parsers/create_parser/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use datatypes::json::JSON2_MAX_STRUCTURED_DEPTH;
16use snafu::{ResultExt, ensure};
17use sqlparser::ast::{DataType, ExactNumberInfo, Expr, ObjectName, UnaryOperator};
18use sqlparser::dialect::keywords::Keyword;
19use sqlparser::parser::Parser;
20use sqlparser::tokenizer::Token;
21
22use crate::ast::Ident;
23use crate::error::{InvalidSqlSnafu, Result, SyntaxSnafu};
24use crate::parsers::create_parser::{INVERTED, SKIPPING};
25use crate::statements::create::JsonTypeHint;
26use crate::statements::transform::type_alias::get_type_by_alias;
27
28const JSON2_TYPE_NAME: &str = "JSON2";
29
30pub(super) fn parse_json2_type_and_hints(
31    parser: &mut Parser<'_>,
32) -> Result<Option<(DataType, Vec<JsonTypeHint>)>> {
33    let token = parser.peek_token();
34    let Token::Word(word) = &token.token else {
35        return Ok(None);
36    };
37
38    if !word.value.eq_ignore_ascii_case(JSON2_TYPE_NAME) || word.quote_style.is_some() {
39        return Ok(None);
40    }
41
42    parser.next_token();
43    let data_type = DataType::Custom(ObjectName::from(vec![Ident::new(JSON2_TYPE_NAME)]), vec![]);
44    let type_hints = if parser.consume_token(&Token::LParen) {
45        parse_json2_type_hints(parser)?
46    } else {
47        vec![]
48    };
49
50    Ok(Some((data_type, type_hints)))
51}
52
53fn parse_json2_type_hints(parser: &mut Parser<'_>) -> Result<Vec<JsonTypeHint>> {
54    let mut hints = Vec::new();
55
56    if parser.consume_token(&Token::RParen) {
57        return Ok(hints);
58    }
59
60    loop {
61        let hint = parse_json2_type_hint(parser)?;
62        ensure_no_path_conflict(&hints, &hint.path)?;
63        hints.push(hint);
64
65        if parser.consume_token(&Token::Comma) {
66            if parser.consume_token(&Token::RParen) {
67                break;
68            }
69        } else {
70            parser.expect_token(&Token::RParen).context(SyntaxSnafu)?;
71            break;
72        }
73    }
74
75    Ok(hints)
76}
77
78fn parse_json2_type_hint(parser: &mut Parser<'_>) -> Result<JsonTypeHint> {
79    let path = parse_json2_path(parser)?;
80    ensure!(
81        path.len() <= JSON2_MAX_STRUCTURED_DEPTH,
82        InvalidSqlSnafu {
83            msg: format!(
84                "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
85            ),
86        }
87    );
88    let data_type = parser.parse_data_type().context(SyntaxSnafu)?;
89    let data_type = normalize_json2_type_hint_type(data_type)?;
90
91    let mut nullable = true;
92    let mut nullable_set = false;
93    let mut default = None;
94    let mut inverted_index = false;
95
96    loop {
97        if parser.parse_keywords(&[Keyword::NOT, Keyword::NULL]) {
98            ensure!(
99                !nullable_set,
100                InvalidSqlSnafu {
101                    msg: format!(
102                        "NULL/NOT NULL option already specified for JSON2 type hint '{}'",
103                        path.join(".")
104                    )
105                }
106            );
107            nullable = false;
108            nullable_set = true;
109        } else if parser.parse_keyword(Keyword::NULL) {
110            ensure!(
111                !nullable_set,
112                InvalidSqlSnafu {
113                    msg: format!(
114                        "NULL/NOT NULL option already specified for JSON2 type hint '{}'",
115                        path.join(".")
116                    )
117                }
118            );
119            nullable = true;
120            nullable_set = true;
121        } else if parser.parse_keyword(Keyword::DEFAULT) {
122            ensure!(
123                default.is_none(),
124                InvalidSqlSnafu {
125                    msg: format!(
126                        "duplicated DEFAULT option for JSON2 type hint '{}'",
127                        path.join(".")
128                    )
129                }
130            );
131            let expr = parser.parse_expr().context(SyntaxSnafu)?;
132            ensure_json2_default_expr_is_literal(&expr)?;
133            default = Some(expr);
134        } else if let Token::Word(word) = parser.peek_token().token
135            && word.value.eq_ignore_ascii_case(INVERTED)
136        {
137            parser.next_token();
138            ensure!(
139                parser.parse_keyword(Keyword::INDEX),
140                InvalidSqlSnafu {
141                    msg: format!(
142                        "expect INDEX after INVERTED keyword for JSON2 type hint '{}'",
143                        path.join(".")
144                    )
145                }
146            );
147            ensure!(
148                !inverted_index,
149                InvalidSqlSnafu {
150                    msg: format!(
151                        "duplicated INVERTED INDEX option for JSON2 type hint '{}'",
152                        path.join(".")
153                    )
154                }
155            );
156            inverted_index = true;
157        } else if let Token::Word(word) = parser.peek_token().token
158            && word.value.eq_ignore_ascii_case(SKIPPING)
159        {
160            return InvalidSqlSnafu {
161                msg: "JSON2 type hint SKIPPING INDEX is not supported yet".to_string(),
162            }
163            .fail();
164        } else if matches!(parser.peek_token().token, Token::Comma | Token::RParen) {
165            break;
166        } else {
167            return parser
168                .expected("JSON2 type hint option", parser.peek_token())
169                .context(SyntaxSnafu);
170        }
171    }
172
173    Ok(JsonTypeHint {
174        path,
175        data_type,
176        nullable,
177        default,
178        inverted_index,
179    })
180}
181
182fn parse_json2_path(parser: &mut Parser<'_>) -> Result<Vec<String>> {
183    let first = parser.parse_identifier().context(SyntaxSnafu)?;
184    let mut path = vec![first.value];
185
186    while parser.consume_token(&Token::Period) {
187        let segment = parser.parse_identifier().context(SyntaxSnafu)?;
188        path.push(segment.value);
189    }
190
191    ensure!(
192        !path.iter().any(|segment| segment.is_empty()),
193        InvalidSqlSnafu {
194            msg: "JSON2 type hint path segment cannot be empty".to_string(),
195        }
196    );
197
198    Ok(path)
199}
200
201fn normalize_json2_type_hint_type(data_type: DataType) -> Result<DataType> {
202    let data_type = get_type_by_alias(&data_type).unwrap_or(data_type);
203    let normalized = match data_type {
204        DataType::String(_) | DataType::Text | DataType::Varchar(_) | DataType::Char(_) => {
205            DataType::String(None)
206        }
207        DataType::TinyInt(_)
208        | DataType::SmallInt(_)
209        | DataType::Int(_)
210        | DataType::Integer(_)
211        | DataType::BigInt(_) => DataType::BigInt(None),
212        DataType::TinyIntUnsigned(_)
213        | DataType::SmallIntUnsigned(_)
214        | DataType::IntUnsigned(_)
215        | DataType::UnsignedInteger
216        | DataType::BigIntUnsigned(_) => DataType::BigIntUnsigned(None),
217        DataType::Float(_) | DataType::Real | DataType::Double(_) => {
218            DataType::Double(ExactNumberInfo::None)
219        }
220        DataType::Boolean => DataType::Boolean,
221        _ => {
222            return InvalidSqlSnafu {
223                msg: format!("unsupported JSON2 type hint data type: {data_type}"),
224            }
225            .fail();
226        }
227    };
228
229    Ok(normalized)
230}
231
232fn ensure_json2_default_expr_is_literal(expr: &Expr) -> Result<()> {
233    let is_literal = match expr {
234        Expr::Value(_) => true,
235        Expr::UnaryOp { op, expr } => {
236            matches!(op, UnaryOperator::Plus | UnaryOperator::Minus)
237                && matches!(expr.as_ref(), Expr::Value(_))
238        }
239        _ => false,
240    };
241    ensure!(
242        is_literal,
243        InvalidSqlSnafu {
244            msg: "JSON2 type hint DEFAULT only supports literal values",
245        }
246    );
247    Ok(())
248}
249
250fn ensure_no_path_conflict(hints: &[JsonTypeHint], path: &[String]) -> Result<()> {
251    for hint in hints {
252        ensure!(
253            hint.path != path,
254            InvalidSqlSnafu {
255                msg: format!("duplicated JSON2 type hint path '{}'", path.join("."))
256            }
257        );
258        ensure!(
259            !hint.path.starts_with(path) && !path.starts_with(&hint.path),
260            InvalidSqlSnafu {
261                msg: format!(
262                    "JSON2 type hint path '{}' conflicts with '{}'",
263                    path.join("."),
264                    hint.path.join(".")
265                )
266            }
267        );
268    }
269    Ok(())
270}
271
272#[cfg(test)]
273mod tests {
274    use sqlparser::ast::{DataType, ExactNumberInfo};
275
276    use crate::dialect::GreptimeDbDialect;
277    use crate::parser::{ParseOptions, ParserContext};
278    use crate::statements::create::Column;
279    use crate::statements::statement::Statement;
280
281    fn parse_json2_column(sql: &str) -> Column {
282        let Statement::CreateTable(mut create_table) =
283            ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default())
284                .unwrap()
285                .remove(0)
286        else {
287            unreachable!()
288        };
289
290        create_table.columns.remove(0)
291    }
292
293    #[test]
294    fn test_parse_json2_type_hints() {
295        let column = parse_json2_column(
296            r#"
297CREATE TABLE traces (
298    log_json_data JSON2 (
299        "service.name" STRING NOT NULL DEFAULT 'null' INVERTED INDEX,
300        http.method STRING NOT NULL,
301        status_code INT64 NOT NULL,
302        comment STRING NULL,
303    ),
304    ts TIMESTAMP TIME INDEX,
305)"#,
306        );
307
308        assert!(matches!(
309            column.column_def.data_type,
310            DataType::Custom(_, _)
311        ));
312        let hints = column.extensions.json_type_hints;
313        assert_eq!(hints.len(), 4);
314
315        assert_eq!(hints[0].path, vec!["service.name"]);
316        assert_eq!(hints[0].data_type, DataType::String(None));
317        assert!(!hints[0].nullable);
318        assert_eq!(
319            hints[0]
320                .default
321                .as_ref()
322                .map(|expr| expr.to_string())
323                .as_deref(),
324            Some("'null'")
325        );
326        assert!(hints[0].inverted_index);
327
328        assert_eq!(hints[1].path, vec!["http", "method"]);
329        assert_eq!(hints[1].data_type, DataType::String(None));
330        assert!(!hints[1].nullable);
331        assert!(!hints[1].inverted_index);
332
333        assert_eq!(hints[2].path, vec!["status_code"]);
334        assert_eq!(hints[2].data_type, DataType::BigInt(None));
335        assert!(!hints[2].nullable);
336
337        assert_eq!(hints[3].path, vec!["comment"]);
338        assert_eq!(hints[3].data_type, DataType::String(None));
339        assert!(hints[3].nullable);
340    }
341
342    #[test]
343    fn test_parse_json2_type_hint_default_nullable() {
344        let column = parse_json2_column(
345            r#"
346CREATE TABLE traces (
347    log_json_data JSON2 (http.method STRING),
348    ts TIMESTAMP TIME INDEX,
349)"#,
350        );
351
352        let hints = column.extensions.json_type_hints;
353        assert_eq!(hints.len(), 1);
354        assert!(hints[0].nullable);
355    }
356
357    #[test]
358    fn test_parse_json2_type_hint_quoted_path_segments() {
359        let column = parse_json2_column(
360            r#"
361CREATE TABLE traces (
362    log_json_data JSON2 (
363        "a".b STRING,
364        "x"."y" STRING,
365        "a.b"."c" STRING,
366        a."b.c" STRING
367    ),
368    ts TIMESTAMP TIME INDEX,
369)"#,
370        );
371
372        let hints = column.extensions.json_type_hints;
373        assert_eq!(hints.len(), 4);
374        assert_eq!(hints[0].path, vec!["a", "b"]);
375        assert_eq!(hints[1].path, vec!["x", "y"]);
376        assert_eq!(hints[2].path, vec!["a.b", "c"]);
377        assert_eq!(hints[3].path, vec!["a", "b.c"]);
378    }
379
380    #[test]
381    fn test_parse_json2_type_hint_normalizes_numeric_types() {
382        let column = parse_json2_column(
383            r#"
384CREATE TABLE traces (
385    log_json_data JSON2 (
386        tinyint_value TINYINT,
387        smallint_value SMALLINT,
388        int_value INT,
389        integer_value INTEGER,
390        bigint_value BIGINT,
391        int64_value INT64,
392        tinyuint_value TINYINT UNSIGNED,
393        smalluint_value SMALLINT UNSIGNED,
394        uint_value INT UNSIGNED,
395        uint64_value UINT64,
396        float_value FLOAT,
397        real_value REAL,
398        double_value DOUBLE,
399        float64_value FLOAT64
400    ),
401    ts TIMESTAMP TIME INDEX,
402)"#,
403        );
404
405        let hints = column.extensions.json_type_hints;
406        assert_eq!(hints.len(), 14);
407        for hint in hints.iter().take(6) {
408            assert_eq!(hint.data_type, DataType::BigInt(None));
409        }
410        for hint in hints.iter().skip(6).take(4) {
411            assert_eq!(hint.data_type, DataType::BigIntUnsigned(None));
412        }
413        for hint in hints.iter().skip(10) {
414            assert_eq!(hint.data_type, DataType::Double(ExactNumberInfo::None));
415        }
416    }
417
418    #[test]
419    fn test_parse_json2_type_hint_default_accepts_signed_literals() {
420        let column = parse_json2_column(
421            r#"
422CREATE TABLE traces (
423    log_json_data JSON2 (
424        negative_int INT64 DEFAULT -5,
425        positive_float FLOAT64 DEFAULT +1.5
426    ),
427    ts TIMESTAMP TIME INDEX,
428)"#,
429        );
430
431        let hints = column.extensions.json_type_hints;
432        assert_eq!(hints.len(), 2);
433        assert_eq!(
434            hints[0]
435                .default
436                .as_ref()
437                .map(|expr| expr.to_string())
438                .as_deref(),
439            Some("-5")
440        );
441        assert_eq!(
442            hints[1]
443                .default
444                .as_ref()
445                .map(|expr| expr.to_string())
446                .as_deref(),
447            Some("+1.5")
448        );
449    }
450
451    #[test]
452    fn test_parse_json2_type_hint_default_rejects_function() {
453        let result = ParserContext::create_with_dialect(
454            r#"
455CREATE TABLE traces (
456    log_json_data JSON2 (status_code INT64 DEFAULT abs(-1)),
457    ts TIMESTAMP TIME INDEX,
458)"#,
459            &GreptimeDbDialect {},
460            ParseOptions::default(),
461        );
462
463        assert!(result.is_err());
464        assert!(
465            result
466                .unwrap_err()
467                .to_string()
468                .contains("DEFAULT only supports literal values")
469        );
470    }
471
472    #[test]
473    fn test_parse_json2_type_hint_rejects_duplicate_path() {
474        let result = ParserContext::create_with_dialect(
475            r#"
476CREATE TABLE traces (
477    log_json_data JSON2 (a.b STRING, a.b INT64),
478    ts TIMESTAMP TIME INDEX,
479)"#,
480            &GreptimeDbDialect {},
481            ParseOptions::default(),
482        );
483
484        assert!(result.is_err());
485        assert!(result.unwrap_err().to_string().contains("duplicated"));
486    }
487
488    #[test]
489    fn test_parse_json2_type_hint_rejects_parent_child_path() {
490        let result = ParserContext::create_with_dialect(
491            r#"
492CREATE TABLE traces (
493    log_json_data JSON2 (a STRING, a.b INT64),
494    ts TIMESTAMP TIME INDEX,
495)"#,
496            &GreptimeDbDialect {},
497            ParseOptions::default(),
498        );
499
500        assert!(result.is_err());
501        assert!(result.unwrap_err().to_string().contains("conflicts"));
502    }
503
504    #[test]
505    fn test_parse_json2_type_hint_rejects_duplicated_nullability() {
506        for sql in [
507            r#"
508CREATE TABLE traces (
509    log_json_data JSON2 (a STRING NULL NULL),
510    ts TIMESTAMP TIME INDEX,
511)"#,
512            r#"
513CREATE TABLE traces (
514    log_json_data JSON2 (a STRING NOT NULL NOT NULL),
515    ts TIMESTAMP TIME INDEX,
516)"#,
517            r#"
518CREATE TABLE traces (
519    log_json_data JSON2 (a STRING NOT NULL NULL),
520    ts TIMESTAMP TIME INDEX,
521)"#,
522            r#"
523CREATE TABLE traces (
524    log_json_data JSON2 (a STRING NULL NOT NULL),
525    ts TIMESTAMP TIME INDEX,
526)"#,
527        ] {
528            let result = ParserContext::create_with_dialect(
529                sql,
530                &GreptimeDbDialect {},
531                ParseOptions::default(),
532            );
533
534            assert!(result.is_err());
535            assert!(
536                result
537                    .unwrap_err()
538                    .to_string()
539                    .contains("NULL/NOT NULL option already specified")
540            );
541        }
542    }
543}
sql/parsers/create_parser/json.rs

sql/parsers/create_parser/
json.rs
