datatypes/types/
json_type.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::BTreeMap;
16use std::fmt::{Debug, Display, Formatter};
17use std::sync::{Arc, LazyLock};
18
19use arrow::datatypes::DataType as ArrowDataType;
20use arrow_schema::{Field, Fields};
21use common_base::bytes::Bytes;
22use regex::{Captures, Regex};
23use serde::{Deserialize, Serialize};
24
25use crate::Error;
26use crate::data_type::DataType;
27use crate::error::{InvalidJsonSnafu, InvalidJsonbSnafu, Result, UnsupportedArrowTypeSnafu};
28use crate::prelude::ConcreteDataType;
29use crate::scalars::ScalarVectorBuilder;
30use crate::type_id::LogicalTypeId;
31use crate::value::Value;
32use crate::vectors::json::builder::JsonVectorBuilder;
33use crate::vectors::{BinaryVectorBuilder, MutableVector};
34
35pub const JSON_TYPE_NAME: &str = "Json";
36const JSON2_TYPE_NAME: &str = "Json2";
37
38pub type JsonObjectType = BTreeMap<String, JsonNativeType>;
39
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
41pub enum JsonNumberType {
42    U64,
43    I64,
44    F64,
45}
46
47#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, Default)]
48pub enum JsonNativeType {
49    /// JSON null value type.
50    ///
51    /// This variant may also appear as the initial state while merging inferred
52    /// types, but it does not represent an empty object. Empty objects are
53    /// represented as `Object({})`.
54    #[default]
55    Null,
56    Bool,
57    Number(JsonNumberType),
58    String,
59    Array(Box<JsonNativeType>),
60    Object(JsonObjectType),
61    /// A special (not in the JSON official specification) JSON type to indicate the "resolved" or
62    /// "lifted" type of two conflicting JSON types. For example, when merging JSON types of "Bool"
63    /// and "Number".
64    Variant,
65}
66
67impl JsonNativeType {
68    pub fn is_null(&self) -> bool {
69        matches!(self, JsonNativeType::Null)
70    }
71
72    pub fn u64() -> Self {
73        Self::Number(JsonNumberType::U64)
74    }
75
76    pub fn i64() -> Self {
77        Self::Number(JsonNumberType::I64)
78    }
79
80    pub fn f64() -> Self {
81        Self::Number(JsonNumberType::F64)
82    }
83
84    pub fn object() -> Self {
85        Self::Object(JsonObjectType::new())
86    }
87
88    /// Merge other [JsonNativeType] into this.
89    /// Conflicting fields will be resolved to the "Variant" type.
90    pub fn merge(&mut self, other: &JsonNativeType) {
91        if self == other {
92            return;
93        }
94
95        fn merge_object(this: &mut JsonObjectType, that: &JsonObjectType) {
96            // merge "that" into "this" directly:
97            for (type_name, that_type) in that {
98                if let Some(this_type) = this.get_mut(type_name) {
99                    this_type.merge(that_type);
100                } else {
101                    this.insert(type_name.clone(), that_type.clone());
102                }
103            }
104        }
105
106        let zelf = std::mem::take(self);
107        *self = match (zelf, other) {
108            (JsonNativeType::Object(mut this), JsonNativeType::Object(that)) => {
109                merge_object(&mut this, that);
110                JsonNativeType::Object(this)
111            }
112            (JsonNativeType::Array(mut this), JsonNativeType::Array(that)) => {
113                this.merge(that);
114                JsonNativeType::Array(this)
115            }
116            (JsonNativeType::Null, that) => that.clone(),
117            (this, JsonNativeType::Null) => this,
118            (this, that) if this == *that => this,
119
120            _ => JsonNativeType::Variant,
121        };
122    }
123
124    pub fn as_arrow_type(&self) -> ArrowDataType {
125        match self {
126            JsonNativeType::Null => ArrowDataType::Null,
127            JsonNativeType::Bool => ArrowDataType::Boolean,
128            JsonNativeType::Number(n) => match n {
129                JsonNumberType::U64 => ArrowDataType::UInt64,
130                JsonNumberType::I64 => ArrowDataType::Int64,
131                JsonNumberType::F64 => ArrowDataType::Float64,
132            },
133            JsonNativeType::String => ArrowDataType::Utf8View,
134            JsonNativeType::Array(array) => {
135                ArrowDataType::List(Arc::new(Field::new("item", array.as_arrow_type(), true)))
136            }
137            JsonNativeType::Object(object) => {
138                let fields = object
139                    .iter()
140                    .map(|(k, v)| Arc::new(Field::new(k, v.as_arrow_type(), true)))
141                    .collect::<Vec<_>>();
142                ArrowDataType::Struct(Fields::from(fields))
143            }
144            JsonNativeType::Variant => ArrowDataType::Binary,
145        }
146    }
147
148    /// Returns whether this type is a boolean, number, or string scalar.
149    pub fn is_primitive(&self) -> bool {
150        matches!(
151            self,
152            JsonNativeType::Bool | JsonNativeType::Number(_) | JsonNativeType::String
153        )
154    }
155}
156
157impl From<&ConcreteDataType> for JsonNativeType {
158    fn from(value: &ConcreteDataType) -> Self {
159        match value {
160            ConcreteDataType::Null(_) => JsonNativeType::Null,
161            ConcreteDataType::Boolean(_) => JsonNativeType::Bool,
162            ConcreteDataType::UInt64(_)
163            | ConcreteDataType::UInt32(_)
164            | ConcreteDataType::UInt16(_)
165            | ConcreteDataType::UInt8(_) => JsonNativeType::u64(),
166            ConcreteDataType::Int64(_)
167            | ConcreteDataType::Int32(_)
168            | ConcreteDataType::Int16(_)
169            | ConcreteDataType::Int8(_) => JsonNativeType::i64(),
170            ConcreteDataType::Float64(_) | ConcreteDataType::Float32(_) => JsonNativeType::f64(),
171            ConcreteDataType::String(_) => JsonNativeType::String,
172            ConcreteDataType::List(list_type) => {
173                JsonNativeType::Array(Box::new(JsonNativeType::from(list_type.item_type())))
174            }
175            ConcreteDataType::Struct(struct_type) => JsonNativeType::Object(
176                struct_type
177                    .fields()
178                    .iter()
179                    .map(|field| (field.name().to_string(), field.data_type().into()))
180                    .collect(),
181            ),
182            ConcreteDataType::Json(json_type) => json_type.native_type().clone(),
183            ConcreteDataType::Binary(_) => JsonNativeType::Variant,
184            _ => unreachable!(),
185        }
186    }
187}
188
189impl TryFrom<&ArrowDataType> for JsonNativeType {
190    type Error = Error;
191
192    fn try_from(t: &ArrowDataType) -> Result<Self> {
193        let t = match t {
194            ArrowDataType::Null => JsonNativeType::Null,
195            ArrowDataType::Boolean => JsonNativeType::Bool,
196            ArrowDataType::Int8
197            | ArrowDataType::Int16
198            | ArrowDataType::Int32
199            | ArrowDataType::Int64 => JsonNativeType::i64(),
200            ArrowDataType::UInt8
201            | ArrowDataType::UInt16
202            | ArrowDataType::UInt32
203            | ArrowDataType::UInt64 => JsonNativeType::u64(),
204            ArrowDataType::Float16 | ArrowDataType::Float32 | ArrowDataType::Float64 => {
205                JsonNativeType::f64()
206            }
207            ArrowDataType::Binary
208            | ArrowDataType::FixedSizeBinary(_)
209            | ArrowDataType::LargeBinary
210            | ArrowDataType::BinaryView => JsonNativeType::Variant,
211            ArrowDataType::Utf8 | ArrowDataType::LargeUtf8 | ArrowDataType::Utf8View => {
212                JsonNativeType::String
213            }
214            ArrowDataType::List(field)
215            | ArrowDataType::ListView(field)
216            | ArrowDataType::FixedSizeList(field, _)
217            | ArrowDataType::LargeList(field)
218            | ArrowDataType::LargeListView(field) => {
219                JsonNativeType::Array(Box::new(Self::try_from(field.data_type())?))
220            }
221            ArrowDataType::Struct(fields) => {
222                let mut object = JsonObjectType::new();
223                for field in fields {
224                    object.insert(field.name().clone(), Self::try_from(field.data_type())?);
225                }
226                JsonNativeType::Object(object)
227            }
228            t => {
229                return UnsupportedArrowTypeSnafu {
230                    arrow_type: t.clone(),
231                }
232                .fail();
233            }
234        };
235        Ok(t)
236    }
237}
238
239impl Display for JsonNativeType {
240    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
241        match self {
242            JsonNativeType::Null => write!(f, r#""<Null>""#),
243            JsonNativeType::Bool => write!(f, r#""<Bool>""#),
244            JsonNativeType::Number(_) => {
245                write!(f, r#""<Number>""#)
246            }
247            JsonNativeType::String => write!(f, r#""<String>""#),
248            JsonNativeType::Array(item_type) => write!(f, "[{}]", item_type),
249            JsonNativeType::Object(object) => {
250                write!(
251                    f,
252                    "{{{}}}",
253                    object
254                        .iter()
255                        .map(|(k, v)| format!(r#""{k}":{v}"#))
256                        .collect::<Vec<_>>()
257                        .join(",")
258                )
259            }
260            JsonNativeType::Variant => write!(f, r#""<Variant>""#),
261        }
262    }
263}
264
265#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize, Default)]
266pub enum JsonFormat {
267    #[default]
268    Jsonb,
269    Json2(Arc<JsonNativeType>),
270}
271
272/// JsonType is a data type for JSON data. It is stored as binary data of jsonb format.
273/// It utilizes current binary value and vector implementation.
274#[derive(Debug, Default, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)]
275pub struct JsonType {
276    pub format: JsonFormat,
277}
278
279impl JsonType {
280    pub fn new(format: JsonFormat) -> Self {
281        Self { format }
282    }
283
284    pub(crate) fn json2(json_type: Arc<JsonNativeType>) -> Self {
285        Self {
286            format: JsonFormat::Json2(json_type),
287        }
288    }
289
290    pub fn is_json2(&self) -> bool {
291        matches!(self.format, JsonFormat::Json2(_))
292    }
293
294    /// Returns the native JSON type represented by this data type.
295    pub fn native_type(&self) -> &JsonNativeType {
296        match &self.format {
297            JsonFormat::Jsonb => &JsonNativeType::String,
298            JsonFormat::Json2(x) => x.as_ref(),
299        }
300    }
301
302    pub fn null() -> Self {
303        Self::json2(Arc::new(JsonNativeType::Null))
304    }
305
306    /// Check if it includes all fields in `other` json type.
307    pub fn is_include(&self, other: &JsonType) -> bool {
308        match (&self.format, &other.format) {
309            (JsonFormat::Jsonb, JsonFormat::Jsonb) => true,
310            (JsonFormat::Json2(this), JsonFormat::Json2(that)) => is_include(this, that),
311            _ => false,
312        }
313    }
314}
315
316pub(crate) fn is_include(this: &JsonNativeType, that: &JsonNativeType) -> bool {
317    fn is_include_object(this: &JsonObjectType, that: &JsonObjectType) -> bool {
318        for (type_name, that_type) in that {
319            let Some(this_type) = this.get(type_name) else {
320                return false;
321            };
322            if !is_include(this_type, that_type) {
323                return false;
324            }
325        }
326        true
327    }
328
329    match (this, that) {
330        (this, that) if this == that => true,
331        (JsonNativeType::Array(this), JsonNativeType::Array(that)) => is_include(this, that),
332        (JsonNativeType::Object(this), JsonNativeType::Object(that)) => {
333            is_include_object(this, that)
334        }
335        (_, JsonNativeType::Null) => true,
336        _ => false,
337    }
338}
339
340impl DataType for JsonType {
341    fn name(&self) -> String {
342        match &self.format {
343            JsonFormat::Jsonb => JSON_TYPE_NAME.to_string(),
344            JsonFormat::Json2(ty) => {
345                format!("{JSON2_TYPE_NAME}{}", ty)
346            }
347        }
348    }
349
350    fn logical_type_id(&self) -> LogicalTypeId {
351        LogicalTypeId::Json
352    }
353
354    fn default_value(&self) -> Value {
355        Bytes::default().into()
356    }
357
358    fn as_arrow_type(&self) -> ArrowDataType {
359        match &self.format {
360            JsonFormat::Jsonb => ArrowDataType::Binary,
361            JsonFormat::Json2(x) => {
362                let mut object = JsonNativeType::object();
363                object.merge(x.as_ref());
364                object.as_arrow_type()
365            }
366        }
367    }
368
369    fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector> {
370        match &self.format {
371            JsonFormat::Jsonb => Box::new(BinaryVectorBuilder::with_capacity(capacity)),
372            // TODO(LFC): Carry JsonSettings in JsonFormat::Json2 and use with_settings here.
373            JsonFormat::Json2(x) => Box::new(JsonVectorBuilder::new(x.as_ref().clone(), capacity)),
374        }
375    }
376
377    fn try_cast(&self, from: Value) -> Option<Value> {
378        match from {
379            Value::Binary(v) => Some(Value::Binary(v)),
380            _ => None,
381        }
382    }
383}
384
385impl Display for JsonType {
386    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
387        write!(f, "{}", self.name())
388    }
389}
390
391/// Converts a json type value to string
392pub fn jsonb_to_string(val: &[u8]) -> Result<String> {
393    if val.is_empty() {
394        return Ok("".to_string());
395    }
396    match jsonb::from_slice(val) {
397        Ok(jsonb_value) => {
398            let serialized = jsonb_value.to_string();
399            fix_unicode_point(&serialized)
400        }
401        Err(e) => InvalidJsonbSnafu { error: e }.fail(),
402    }
403}
404
405/// Converts a json type value to serde_json::Value
406pub fn jsonb_to_serde_json(val: &[u8]) -> Result<serde_json::Value> {
407    jsonb::from_slice(val)
408        .map(Into::into)
409        .map_err(|error| InvalidJsonbSnafu { error }.build())
410}
411
412/// Normalizes a JSON string by converting Rust-style Unicode escape sequences to JSON-compatible format.
413///
414/// The input is scanned for Rust-style Unicode code
415/// point escapes of the form `\\u{H...}` (a backslash, `u`, an opening brace,
416/// followed by 1–6 hexadecimal digits, and a closing brace). Each such escape is
417/// converted into JSON-compatible UTF‑16 escape sequences:
418///
419/// - For code points in the Basic Multilingual Plane (≤ `0xFFFF`), the escape is
420///   converted to a single JSON `\\uXXXX` sequence with four uppercase hex digits.
421/// - For code points above `0xFFFF` and less than Unicode max code point `0x10FFFF`,
422///   the code point is encoded as a UTF‑16 surrogate pair and emitted as two consecutive
423///   `\\uXXXX` sequences (as JSON format required).
424///
425/// After this normalization, the function returns the normalized string
426fn fix_unicode_point(json: &str) -> Result<String> {
427    static UNICODE_CODE_POINT_PATTERN: LazyLock<Regex> = LazyLock::new(|| {
428        // Match literal "\u{...}" sequences, capturing 1–6 (code point range) hex digits
429        // inside braces.
430        Regex::new(r"\\u\{([0-9a-fA-F]{1,6})}").unwrap_or_else(|e| panic!("{}", e))
431    });
432
433    let v = UNICODE_CODE_POINT_PATTERN.replace_all(json, |caps: &Captures| {
434        // Extract the hex payload (without braces) and parse to a code point.
435        let hex = &caps[1];
436        let Ok(code) = u32::from_str_radix(hex, 16) else {
437            // On parse failure, leave the original escape sequence unchanged.
438            return caps[0].to_string();
439        };
440
441        if code <= 0xFFFF {
442            // Basic Multilingual Plane: JSON can represent this directly as \uXXXX.
443            format!("\\u{:04X}", code)
444        } else if code > 0x10FFFF {
445            // Beyond max Unicode code point
446            caps[0].to_string()
447        } else {
448            // Supplementary planes: JSON needs UTF-16 surrogate pairs.
449            // Convert the code point to a 20-bit value.
450            let code = code - 0x10000;
451
452            // High surrogate: top 10 bits, offset by 0xD800.
453            let high = 0xD800 + ((code >> 10) & 0x3FF);
454
455            // Low surrogate: bottom 10 bits, offset by 0xDC00.
456            let low = 0xDC00 + (code & 0x3FF);
457
458            // Emit two \uXXXX escapes in sequence.
459            format!("\\u{:04X}\\u{:04X}", high, low)
460        }
461    });
462    Ok(v.to_string())
463}
464
465/// Parses a string to a json type value
466pub fn parse_string_to_jsonb(s: &str) -> Result<Vec<u8>> {
467    jsonb::parse_value_standard_mode(s.as_bytes())
468        .map_err(|_| InvalidJsonSnafu { value: s }.build())
469        .map(|json| json.to_vec())
470}
471
472#[cfg(test)]
473mod tests {
474    #[test]
475    fn test_parse_json_rejects_extended_syntax() {
476        for input in ["+1", "01", ".5", "[1,,2]", "{'a':1}"] {
477            assert!(parse_string_to_jsonb(input).is_err(), "{input}");
478        }
479    }
480
481    #[test]
482    fn test_v2_jsonb_to_serde_json_escaped_keys()
483    -> std::result::Result<(), Box<dyn std::error::Error>> {
484        let expected = serde_json::json!({"": 9, "a\"b": {"a\\b": "\u{1f600}\n"}});
485        let input = expected.to_string();
486        let value = jsonb::parse_value(input.as_bytes()).map_err(|e| e.to_string())?;
487        assert_eq!(jsonb_to_serde_json(&value.to_vec())?, expected);
488        Ok(())
489    }
490
491    use super::*;
492
493    #[test]
494    fn test_fix_unicode_point() -> Result<()> {
495        let valid_cases = vec![
496            (r#"{"data": "simple ascii"}"#, r#"{"data": "simple ascii"}"#),
497            (
498                r#"{"data":"Greek sigma: \u{03a3}"}"#,
499                r#"{"data":"Greek sigma: \u03A3"}"#,
500            ),
501            (
502                r#"{"data":"Joker card: \u{1f0df}"}"#,
503                r#"{"data":"Joker card: \uD83C\uDCDF"}"#,
504            ),
505            (
506                r#"{"data":"BMP boundary: \u{ffff}"}"#,
507                r#"{"data":"BMP boundary: \uFFFF"}"#,
508            ),
509            (
510                r#"{"data":"Supplementary min: \u{10000}"}"#,
511                r#"{"data":"Supplementary min: \uD800\uDC00"}"#,
512            ),
513            (
514                r#"{"data":"Supplementary max: \u{10ffff}"}"#,
515                r#"{"data":"Supplementary max: \uDBFF\uDFFF"}"#,
516            ),
517        ];
518        for (input, expect) in valid_cases {
519            let v = fix_unicode_point(input)?;
520            assert_eq!(v, expect);
521        }
522
523        let invalid_escape_cases = vec![
524            (
525                r#"{"data": "Invalid hex: \u{gggg}"}"#,
526                r#"{"data": "Invalid hex: \u{gggg}"}"#,
527            ),
528            (
529                r#"{"data": "Empty braces: \u{}"}"#,
530                r#"{"data": "Empty braces: \u{}"}"#,
531            ),
532            (
533                r#"{"data": "Out of range: \u{1100000}"}"#,
534                r#"{"data": "Out of range: \u{1100000}"}"#,
535            ),
536        ];
537        for (input, expect) in invalid_escape_cases {
538            let v = fix_unicode_point(input)?;
539            assert_eq!(v, expect);
540        }
541
542        Ok(())
543    }
544
545    #[test]
546    fn test_json_type_include() {
547        fn test(this: &JsonNativeType, that: &JsonNativeType, expected: bool) {
548            assert_eq!(is_include(this, that), expected, "this={this}, that={that}");
549        }
550
551        test(&JsonNativeType::Null, &JsonNativeType::Null, true);
552        test(&JsonNativeType::Null, &JsonNativeType::Bool, false);
553        test(&JsonNativeType::Bool, &JsonNativeType::Null, true);
554
555        test(&JsonNativeType::Bool, &JsonNativeType::Bool, true);
556        test(&JsonNativeType::Bool, &JsonNativeType::u64(), false);
557
558        test(&JsonNativeType::u64(), &JsonNativeType::Null, true);
559        test(&JsonNativeType::u64(), &JsonNativeType::u64(), true);
560        test(&JsonNativeType::u64(), &JsonNativeType::String, false);
561
562        test(&JsonNativeType::String, &JsonNativeType::Null, true);
563        test(&JsonNativeType::String, &JsonNativeType::String, true);
564        test(
565            &JsonNativeType::String,
566            &JsonNativeType::Array(Box::new(JsonNativeType::f64())),
567            false,
568        );
569
570        test(
571            &JsonNativeType::Array(Box::new(JsonNativeType::f64())),
572            &JsonNativeType::Null,
573            true,
574        );
575        test(
576            &JsonNativeType::Array(Box::new(JsonNativeType::f64())),
577            &JsonNativeType::Array(Box::new(JsonNativeType::Null)),
578            true,
579        );
580        test(
581            &JsonNativeType::Array(Box::new(JsonNativeType::f64())),
582            &JsonNativeType::Array(Box::new(JsonNativeType::f64())),
583            true,
584        );
585        test(
586            &JsonNativeType::Array(Box::new(JsonNativeType::f64())),
587            &JsonNativeType::String,
588            false,
589        );
590        test(
591            &JsonNativeType::Array(Box::new(JsonNativeType::f64())),
592            &JsonNativeType::Object(JsonObjectType::new()),
593            false,
594        );
595
596        let simple_json_object = &JsonNativeType::Object(JsonObjectType::from([(
597            "foo".to_string(),
598            JsonNativeType::String,
599        )]));
600        test(simple_json_object, simple_json_object, true);
601        test(simple_json_object, &JsonNativeType::i64(), false);
602        test(
603            simple_json_object,
604            &JsonNativeType::Object(JsonObjectType::from([(
605                "bar".to_string(),
606                JsonNativeType::i64(),
607            )])),
608            false,
609        );
610
611        let complex_json_object = &JsonNativeType::Object(JsonObjectType::from([
612            (
613                "nested".to_string(),
614                JsonNativeType::Object(JsonObjectType::from([(
615                    "a".to_string(),
616                    JsonNativeType::Object(JsonObjectType::from([(
617                        "b".to_string(),
618                        JsonNativeType::Object(JsonObjectType::from([(
619                            "c".to_string(),
620                            JsonNativeType::String,
621                        )])),
622                    )])),
623                )])),
624            ),
625            ("bar".to_string(), JsonNativeType::i64()),
626        ]));
627        test(simple_json_object, &JsonNativeType::Null, true);
628        test(complex_json_object, &JsonNativeType::String, false);
629        test(complex_json_object, complex_json_object, true);
630        test(
631            complex_json_object,
632            &JsonNativeType::Object(JsonObjectType::from([(
633                "bar".to_string(),
634                JsonNativeType::i64(),
635            )])),
636            true,
637        );
638        test(
639            complex_json_object,
640            &JsonNativeType::Object(JsonObjectType::from([
641                (
642                    "nested".to_string(),
643                    JsonNativeType::Object(JsonObjectType::from([(
644                        "a".to_string(),
645                        JsonNativeType::Null,
646                    )])),
647                ),
648                ("bar".to_string(), JsonNativeType::i64()),
649            ])),
650            true,
651        );
652        test(
653            complex_json_object,
654            &JsonNativeType::Object(JsonObjectType::from([
655                (
656                    "nested".to_string(),
657                    JsonNativeType::Object(JsonObjectType::from([(
658                        "a".to_string(),
659                        JsonNativeType::String,
660                    )])),
661                ),
662                ("bar".to_string(), JsonNativeType::i64()),
663            ])),
664            false,
665        );
666        test(
667            complex_json_object,
668            &JsonNativeType::Object(JsonObjectType::from([
669                (
670                    "nested".to_string(),
671                    JsonNativeType::Object(JsonObjectType::from([(
672                        "a".to_string(),
673                        JsonNativeType::Object(JsonObjectType::from([(
674                            "b".to_string(),
675                            JsonNativeType::String,
676                        )])),
677                    )])),
678                ),
679                ("bar".to_string(), JsonNativeType::i64()),
680            ])),
681            false,
682        );
683        test(
684            complex_json_object,
685            &JsonNativeType::Object(JsonObjectType::from([
686                (
687                    "nested".to_string(),
688                    JsonNativeType::Object(JsonObjectType::from([(
689                        "a".to_string(),
690                        JsonNativeType::Object(JsonObjectType::from([(
691                            "b".to_string(),
692                            JsonNativeType::Object(JsonObjectType::from([(
693                                "c".to_string(),
694                                JsonNativeType::Null,
695                            )])),
696                        )])),
697                    )])),
698                ),
699                ("bar".to_string(), JsonNativeType::i64()),
700            ])),
701            true,
702        );
703        test(
704            complex_json_object,
705            &JsonNativeType::Object(JsonObjectType::from([
706                (
707                    "nested".to_string(),
708                    JsonNativeType::Object(JsonObjectType::from([(
709                        "a".to_string(),
710                        JsonNativeType::Object(JsonObjectType::from([(
711                            "b".to_string(),
712                            JsonNativeType::Object(JsonObjectType::from([(
713                                "c".to_string(),
714                                JsonNativeType::Bool,
715                            )])),
716                        )])),
717                    )])),
718                ),
719                ("bar".to_string(), JsonNativeType::i64()),
720            ])),
721            false,
722        );
723        test(
724            complex_json_object,
725            &JsonNativeType::Object(JsonObjectType::from([(
726                "nested".to_string(),
727                JsonNativeType::Object(JsonObjectType::from([(
728                    "a".to_string(),
729                    JsonNativeType::Object(JsonObjectType::from([(
730                        "b".to_string(),
731                        JsonNativeType::Object(JsonObjectType::from([(
732                            "c".to_string(),
733                            JsonNativeType::String,
734                        )])),
735                    )])),
736                )])),
737            )])),
738            true,
739        );
740    }
741
742    #[test]
743    fn test_merge_json_type() {
744        fn test(other: JsonNativeType, json_type: &mut JsonNativeType, expected: &str) {
745            json_type.merge(&other);
746            assert_eq!(json_type.to_string(), expected);
747        }
748
749        // Null should be absorbed by a concrete scalar type.
750        test(
751            JsonNativeType::Bool,
752            &mut JsonNativeType::Null,
753            r#""<Bool>""#,
754        );
755
756        // Merging a null value into an existing concrete type should keep the type unchanged.
757        test(
758            JsonNativeType::Null,
759            &mut JsonNativeType::Bool,
760            r#""<Bool>""#,
761        );
762
763        // Identical number categories should stay as Number.
764        test(
765            JsonNativeType::i64(),
766            &mut JsonNativeType::i64(),
767            r#""<Number>""#,
768        );
769
770        // Conflicting number categories should be lifted to Variant.
771        for (mut this, other) in [
772            (JsonNativeType::u64(), JsonNativeType::i64()),
773            (JsonNativeType::u64(), JsonNativeType::f64()),
774            (JsonNativeType::i64(), JsonNativeType::f64()),
775        ] {
776            test(other, &mut this, r#""<Variant>""#);
777        }
778
779        // Object merge should preserve existing fields and append missing fields.
780        test(
781            JsonNativeType::Object(JsonObjectType::from([(
782                "foo".to_string(),
783                JsonNativeType::String,
784            )])),
785            &mut JsonNativeType::Object(JsonObjectType::from([(
786                "bar".to_string(),
787                JsonNativeType::i64(),
788            )])),
789            r#"{"bar":"<Number>","foo":"<String>"}"#,
790        );
791
792        // Conflicting object field types should only lift that field to Variant.
793        test(
794            JsonNativeType::Object(JsonObjectType::from([(
795                "foo".to_string(),
796                JsonNativeType::i64(),
797            )])),
798            &mut JsonNativeType::Object(JsonObjectType::from([(
799                "foo".to_string(),
800                JsonNativeType::Bool,
801            )])),
802            r#"{"foo":"<Variant>"}"#,
803        );
804
805        // Nested objects should merge recursively.
806        test(
807            JsonNativeType::Object(JsonObjectType::from([(
808                "nested".to_string(),
809                JsonNativeType::Object(JsonObjectType::from([(
810                    "foo".to_string(),
811                    JsonNativeType::String,
812                )])),
813            )])),
814            &mut JsonNativeType::Object(JsonObjectType::from([(
815                "nested".to_string(),
816                JsonNativeType::Object(JsonObjectType::from([(
817                    "bar".to_string(),
818                    JsonNativeType::Bool,
819                )])),
820            )])),
821            r#"{"nested":{"bar":"<Bool>","foo":"<String>"}}"#,
822        );
823
824        // Arrays should merge their element types recursively.
825        test(
826            JsonNativeType::Array(Box::new(JsonNativeType::String)),
827            &mut JsonNativeType::Array(Box::new(JsonNativeType::u64())),
828            r#"["<Variant>"]"#,
829        );
830
831        // Root-level incompatible types should be lifted to Variant.
832        test(
833            JsonNativeType::Object(JsonObjectType::from([(
834                "foo".to_string(),
835                JsonNativeType::String,
836            )])),
837            &mut JsonNativeType::Bool,
838            r#""<Variant>""#,
839        );
840    }
841}
datatypes/types/json_type.rs

datatypes/types/
json_type.rs
