datatypes/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Data conversion between greptime's StructType and Json
16//!
17//! The idea of this module is to provide utilities to convert serde_json::Value to greptime's StructType and vice versa.
18//!
19//! The struct will carry all the fields of the Json object. We will not flatten any json object in this implementation.
20//!
21
22pub mod value;
23
24use std::collections::BTreeMap;
25use std::collections::btree_map::Entry;
26
27use serde::{Deserialize, Serialize};
28use serde_json::{Map, Value as Json};
29use snafu::ResultExt;
30
31use crate::data_type::ConcreteDataType;
32use crate::error::{self, Result, UnsupportedJsonTypeSnafu};
33use crate::json::value::{JsonValue, JsonVariant};
34use crate::schema::ColumnDefaultConstraint;
35use crate::types::json_type::JsonNativeType;
36use crate::value::{ListValue, StructValue, Value};
37
38/// JSON2 settings stored in column schema metadata and represented through
39/// Arrow extension metadata.
40#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
41pub struct JsonSettings {
42    #[serde(default)]
43    pub type_hints: Vec<JsonTypeHint>,
44}
45
46/// Declares selected JSON2 subpaths as typed fields.
47///
48/// These hints let JSON2 encode frequently used subpaths in a typed layout, so
49/// queries over those subpaths can get behavior and performance closer to
50/// ordinary columns.
51#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
52pub struct JsonTypeHint {
53    /// JSON2 subpath for a typed field.
54    ///
55    /// Each item is one JSON object key. For example, `["user", "age"]`
56    /// represents `user.age`.
57    ///
58    /// Array traversal is not currently supported. For example, a hint cannot
59    /// describe `events[0].name` or fields shared by all items in `events[*]`.
60    pub path: Vec<String>,
61    #[serde(rename = "type")]
62    pub data_type: ConcreteDataType,
63    pub nullable: bool,
64    #[serde(skip_serializing_if = "Option::is_none")]
65    pub default_constraint: Option<ColumnDefaultConstraint>,
66    pub inverted_index: bool,
67}
68
69/// Context for JSON encoding/decoding that tracks the current key path.
70#[derive(Clone, Debug)]
71pub struct JsonContext<'a> {
72    /// Current key path from the JSON2 root.
73    pub path: Vec<String>,
74    /// Settings for JSON encoding/decoding.
75    pub settings: &'a JsonSettings,
76}
77
78impl JsonSettings {
79    pub fn new(type_hints: Vec<JsonTypeHint>) -> Self {
80        Self { type_hints }
81    }
82
83    /// Decode an encoded StructValue back into a serde_json::Value.
84    pub fn decode(&self, value: Value) -> Result<Json> {
85        let context = JsonContext {
86            path: Vec::new(),
87            settings: self,
88        };
89        decode_value_with_context(value, &context)
90    }
91
92    /// Encode a serde_json::Value into a Value::Json using current settings.
93    pub fn encode(&self, json: Json) -> Result<Value> {
94        let context = JsonContext {
95            path: Vec::new(),
96            settings: self,
97        };
98        encode_json_with_context(json, &context).map(|v| Value::Json(Box::new(v)))
99    }
100}
101
102impl<'a> JsonContext<'a> {
103    /// Create a new context with an updated key path
104    pub fn with_key(&self, key: &str) -> JsonContext<'a> {
105        let mut path = self.path.clone();
106        path.push(key.to_string());
107        JsonContext {
108            path,
109            settings: self.settings,
110        }
111    }
112
113    fn type_hint(&self) -> Option<&'a JsonTypeHint> {
114        self.settings
115            .type_hints
116            .iter()
117            .find(|hint| hint.path == self.path)
118    }
119}
120
121/// Main encoding function with key path tracking
122pub fn encode_json_with_context<'a>(json: Json, context: &JsonContext<'a>) -> Result<JsonValue> {
123    if context.path.is_empty() && !matches!(json, Json::Object(_)) {
124        return UnsupportedJsonTypeSnafu.fail();
125    }
126
127    match json {
128        Json::Object(json_object) => encode_json_object_with_context(json_object, context),
129        Json::Array(json_array) => encode_json_array_with_context(json_array, context),
130        _ => encode_json_value_with_context(json, context),
131    }
132}
133
134fn encode_json_object_with_context<'a>(
135    json_object: Map<String, Json>,
136    context: &JsonContext<'a>,
137) -> Result<JsonValue> {
138    let mut object = BTreeMap::new();
139    for (key, value) in json_object {
140        let field_context = context.with_key(&key);
141
142        let value = if let Some(hint) = field_context.type_hint() {
143            encode_json_value_with_hint(value, hint, &field_context)?
144        } else {
145            encode_json_value_with_context(value, &field_context)?
146        };
147
148        object.insert(key, value.into_variant());
149    }
150
151    apply_missing_type_hints(&mut object, context)?;
152
153    Ok(JsonValue::new(JsonVariant::Object(object)))
154}
155
156fn apply_missing_type_hints(
157    object: &mut BTreeMap<String, JsonVariant>,
158    context: &JsonContext,
159) -> Result<()> {
160    for hint in &context.settings.type_hints {
161        if hint.path.len() > context.path.len() && hint.path.starts_with(&context.path) {
162            insert_missing_type_hint(object, context, hint, context.path.len())?;
163        }
164    }
165    Ok(())
166}
167
168fn insert_missing_type_hint(
169    object: &mut BTreeMap<String, JsonVariant>,
170    context: &JsonContext,
171    hint: &JsonTypeHint,
172    depth: usize,
173) -> Result<()> {
174    let key = &hint.path[depth];
175    let field_context = context.with_key(key);
176    let is_leaf = depth + 1 == hint.path.len();
177
178    if is_leaf {
179        if !object.contains_key(key) {
180            let value = encode_missing_type_hint_value(hint, &field_context)?;
181            object.insert(key.clone(), value.into_variant());
182        }
183        return Ok(());
184    }
185
186    match object.entry(key.clone()) {
187        Entry::Occupied(mut entry) => match entry.get_mut() {
188            JsonVariant::Object(child) => {
189                insert_missing_type_hint(child, &field_context, hint, depth + 1)
190            }
191            _ => error::InvalidJsonSnafu {
192                value: format!(
193                    "JSON2 type hint path {} expects object at {}",
194                    hint.path.join("."),
195                    field_context.path.join(".")
196                ),
197            }
198            .fail(),
199        },
200        Entry::Vacant(entry) => {
201            let mut child = BTreeMap::new();
202            insert_missing_type_hint(&mut child, &field_context, hint, depth + 1)?;
203            entry.insert(JsonVariant::Object(child));
204            Ok(())
205        }
206    }
207}
208
209fn encode_missing_type_hint_value(hint: &JsonTypeHint, context: &JsonContext) -> Result<JsonValue> {
210    if let Some(default_constraint) = &hint.default_constraint {
211        let value = default_constraint.create_default(&hint.data_type, hint.nullable)?;
212        let json = decode_primitive_value(value)?;
213        return encode_json_value_with_hint(json, hint, context);
214    }
215
216    if hint.nullable {
217        Ok(JsonValue::null())
218    } else {
219        error::InvalidJsonSnafu {
220            value: format!(
221                "missing non-null JSON2 type hint path {}",
222                hint.path.join(".")
223            ),
224        }
225        .fail()
226    }
227}
228
229fn encode_json_value_with_hint(
230    json: Json,
231    hint: &JsonTypeHint,
232    context: &JsonContext,
233) -> Result<JsonValue> {
234    if json.is_null() {
235        return if hint.nullable {
236            Ok(JsonValue::null())
237        } else {
238            error::InvalidJsonSnafu {
239                value: format!(
240                    "JSON2 type hint path {} is not nullable",
241                    context.path.join(".")
242                ),
243            }
244            .fail()
245        };
246    }
247
248    let invalid_type = || {
249        error::InvalidJsonSnafu {
250            value: format!(
251                "JSON value at {} does not match JSON2 type hint {}",
252                context.path.join("."),
253                hint.data_type
254            ),
255        }
256        .fail()
257    };
258
259    match (&hint.data_type, json) {
260        (ConcreteDataType::String(_), Json::String(v)) => Ok(v.into()),
261        (
262            ConcreteDataType::Int8(_)
263            | ConcreteDataType::Int16(_)
264            | ConcreteDataType::Int32(_)
265            | ConcreteDataType::Int64(_),
266            Json::Number(v),
267        ) => match v.as_i64() {
268            Some(v) => Ok(v.into()),
269            None => invalid_type(),
270        },
271        (
272            ConcreteDataType::UInt8(_)
273            | ConcreteDataType::UInt16(_)
274            | ConcreteDataType::UInt32(_)
275            | ConcreteDataType::UInt64(_),
276            Json::Number(v),
277        ) => match v.as_u64() {
278            Some(v) => Ok(v.into()),
279            None => invalid_type(),
280        },
281        (ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(v)) => {
282            match v.as_f64() {
283                Some(v) => Ok(v.into()),
284                None => invalid_type(),
285            }
286        }
287        (ConcreteDataType::Boolean(_), Json::Bool(v)) => Ok(v.into()),
288        _ => invalid_type(),
289    }
290}
291
292fn encode_json_array_with_context<'a>(
293    json_array: Vec<Json>,
294    context: &JsonContext<'a>,
295) -> Result<JsonValue> {
296    let json_array_len = json_array.len();
297    let mut items = Vec::with_capacity(json_array_len);
298
299    for (index, value) in json_array.into_iter().enumerate() {
300        let array_context = context.with_key(&index.to_string());
301        let item_value = encode_json_value_with_context(value, &array_context)?;
302        items.push(item_value);
303    }
304
305    // In specification, it's valid for a JSON array to have different types of items, for example,
306    // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
307    // array, which requires all items have exactly the same type. So we merge out the maybe
308    // different item types to a unified type, and align all the item values to it.
309
310    let merged_item_type = if let Some((first, rests)) = items.split_first() {
311        let mut merged = first.json_type().clone();
312        for rest in rests.iter().map(|x| x.json_type()) {
313            if matches!(merged.native_type(), JsonNativeType::Variant) {
314                break;
315            }
316            merged.merge(rest)?;
317        }
318        Some(merged)
319    } else {
320        None
321    };
322    if let Some(unified_item_type) = merged_item_type {
323        for item in &mut items {
324            item.try_align(&unified_item_type)?;
325        }
326    }
327    let items = items
328        .into_iter()
329        .map(|x| x.into_variant())
330        .collect::<Vec<_>>();
331    Ok(JsonValue::new(JsonVariant::Array(items)))
332}
333
334/// Helper function to encode a JSON value to a Value and determine its ConcreteDataType with context
335fn encode_json_value_with_context<'a>(json: Json, context: &JsonContext<'a>) -> Result<JsonValue> {
336    match json {
337        Json::Null => Ok(JsonValue::null()),
338        Json::Bool(b) => Ok(b.into()),
339        Json::Number(n) => {
340            if let Some(i) = n.as_i64() {
341                Ok(i.into())
342            } else if let Some(u) = n.as_u64() {
343                if u <= i64::MAX as u64 {
344                    Ok((u as i64).into())
345                } else {
346                    Ok(u.into())
347                }
348            } else if let Some(f) = n.as_f64() {
349                Ok(f.into())
350            } else {
351                // Fallback to string representation
352                Ok(n.to_string().into())
353            }
354        }
355        Json::String(s) => Ok(s.into()),
356        Json::Array(arr) => encode_json_array_with_context(arr, context),
357        Json::Object(obj) => encode_json_object_with_context(obj, context),
358    }
359}
360
361/// Main decoding function with key path tracking
362pub fn decode_value_with_context(value: Value, context: &JsonContext) -> Result<Json> {
363    match value {
364        Value::Struct(struct_value) => decode_struct_with_context(struct_value, context),
365        Value::List(list_value) => decode_list_with_context(list_value, context),
366        _ => decode_primitive_value(value),
367    }
368}
369
370/// Decode a structured value to JSON object
371fn decode_struct_with_context<'a>(
372    struct_value: StructValue,
373    context: &JsonContext<'a>,
374) -> Result<Json> {
375    let mut json_object = Map::with_capacity(struct_value.len());
376
377    let (items, fields) = struct_value.into_parts();
378
379    for (field, field_value) in fields.fields().iter().zip(items) {
380        let field_context = context.with_key(field.name());
381        let json_value = decode_value_with_context(field_value, &field_context)?;
382        json_object.insert(field.name().to_string(), json_value);
383    }
384
385    Ok(Json::Object(json_object))
386}
387
388/// Decode a list value to JSON array
389fn decode_list_with_context(list_value: ListValue, context: &JsonContext) -> Result<Json> {
390    let mut json_array = Vec::with_capacity(list_value.len());
391
392    let data_items = list_value.take_items();
393
394    for (index, item) in data_items.into_iter().enumerate() {
395        let array_context = context.with_key(&index.to_string());
396        let json_value = decode_value_with_context(item, &array_context)?;
397        json_array.push(json_value);
398    }
399
400    Ok(Json::Array(json_array))
401}
402
403/// Decode primitive value to JSON
404fn decode_primitive_value(value: Value) -> Result<Json> {
405    match value {
406        Value::Null => Ok(Json::Null),
407        Value::Boolean(b) => Ok(Json::Bool(b)),
408        Value::UInt8(v) => Ok(Json::from(v)),
409        Value::UInt16(v) => Ok(Json::from(v)),
410        Value::UInt32(v) => Ok(Json::from(v)),
411        Value::UInt64(v) => Ok(Json::from(v)),
412        Value::Int8(v) => Ok(Json::from(v)),
413        Value::Int16(v) => Ok(Json::from(v)),
414        Value::Int32(v) => Ok(Json::from(v)),
415        Value::Int64(v) => Ok(Json::from(v)),
416        Value::Float32(v) => Ok(Json::from(v.0)),
417        Value::Float64(v) => Ok(Json::from(v.0)),
418        Value::String(s) => Ok(Json::String(s.as_utf8().to_string())),
419        Value::Binary(b) => serde_json::to_value(b.as_ref()).context(error::SerializeSnafu),
420        Value::Date(v) => Ok(Json::from(v.val())),
421        Value::Timestamp(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
422        Value::Time(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
423        Value::IntervalYearMonth(v) => {
424            serde_json::to_value(v.to_i32()).context(error::SerializeSnafu)
425        }
426        Value::IntervalDayTime(v) => {
427            serde_json::to_value(v.to_i64()).context(error::SerializeSnafu)
428        }
429        Value::IntervalMonthDayNano(v) => {
430            serde_json::to_value(v.to_i128()).context(error::SerializeSnafu)
431        }
432        Value::Duration(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
433        Value::Decimal128(v) => serde_json::to_value(v.to_string()).context(error::SerializeSnafu),
434        Value::Struct(_) | Value::List(_) | Value::Json(_) => {
435            // These should be handled by the context-aware functions
436            Err(error::InvalidJsonSnafu {
437                value: "Structured values should be handled by context-aware decoding".to_string(),
438            }
439            .build())
440        }
441    }
442}
443
444#[cfg(test)]
445mod tests {
446    use std::sync::Arc;
447
448    use serde_json::json;
449
450    use super::*;
451    use crate::data_type::ConcreteDataType;
452    use crate::types::{ListType, StructField, StructType};
453
454    fn struct_field_value<'a>(struct_value: &'a StructValue, field_name: &str) -> &'a Value {
455        let index = struct_value
456            .struct_type()
457            .fields()
458            .iter()
459            .position(|field| field.name() == field_name)
460            .expect("field exists");
461        &struct_value.items()[index]
462    }
463
464    #[test]
465    fn test_json_settings_forward_compatibility() {
466        let json_str = r#"{
467            "type_hints": [
468                {
469                    "path": ["user", "age"],
470                    "type": {
471                        "Int64": {}
472                    },
473                    "nullable": false,
474                    "default_constraint": {
475                        "Value": {
476                            "Int64": 18
477                        }
478                    },
479                    "inverted_index": true
480                },
481                {
482                    "path": ["user", "name"],
483                    "type": {
484                        "String": {
485                            "size_type": "Utf8"
486                        }
487                    },
488                    "nullable": true,
489                    "inverted_index": false
490                }
491            ]
492        }"#;
493
494        let deserialized = serde_json::from_str::<JsonSettings>(json_str).unwrap();
495
496        assert_eq!(
497            deserialized,
498            JsonSettings::new(vec![
499                JsonTypeHint {
500                    path: vec!["user".to_string(), "age".to_string()],
501                    data_type: ConcreteDataType::int64_datatype(),
502                    nullable: false,
503                    default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
504                    inverted_index: true,
505                },
506                JsonTypeHint {
507                    path: vec!["user".to_string(), "name".to_string()],
508                    data_type: ConcreteDataType::string_datatype(),
509                    nullable: true,
510                    default_constraint: None,
511                    inverted_index: false,
512                },
513            ])
514        );
515    }
516
517    #[test]
518    fn test_json_settings_ser_de() {
519        let settings = JsonSettings::new(vec![
520            JsonTypeHint {
521                path: vec!["user".to_string(), "age".to_string()],
522                data_type: ConcreteDataType::int64_datatype(),
523                nullable: false,
524                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
525                inverted_index: true,
526            },
527            JsonTypeHint {
528                path: vec!["user".to_string(), "name".to_string()],
529                data_type: ConcreteDataType::string_datatype(),
530                nullable: true,
531                default_constraint: None,
532                inverted_index: false,
533            },
534        ]);
535
536        let serialized = serde_json::to_string(&settings).unwrap();
537        let deserialized = serde_json::from_str::<JsonSettings>(&serialized).unwrap();
538
539        assert_eq!(settings, deserialized);
540    }
541
542    #[test]
543    fn test_encode_root_non_object_json() {
544        let settings = JsonSettings::default();
545        let cases = [
546            ("null", Json::Null),
547            ("boolean", Json::Bool(true)),
548            ("integer", Json::from(42)),
549            ("float", Json::from(3.15)),
550            ("string", Json::String("hello".to_string())),
551            ("array", json!([1, 2, 3])),
552            ("mixed array", json!([1, "hello", true, 3.15])),
553            ("empty array", json!([])),
554        ];
555
556        for (name, json) in cases {
557            let err = settings.encode(json).unwrap_err();
558            assert!(
559                matches!(err, crate::error::Error::UnsupportedJsonType { .. }),
560                "{name}: {err:?}"
561            );
562        }
563    }
564
565    #[test]
566    fn test_encode_json_object() {
567        let json = json!({
568            "name": "John",
569            "age": 30,
570            "active": true
571        });
572
573        let settings = JsonSettings::default();
574        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
575        let Value::Struct(result) = result else {
576            panic!("Expected Struct value");
577        };
578        assert_eq!(result.items().len(), 3);
579
580        let items = result.items();
581        let struct_type = result.struct_type();
582
583        // Check that we have the expected fields
584        let fields = struct_type.fields();
585        let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
586        assert!(field_names.contains(&"name"));
587        assert!(field_names.contains(&"age"));
588        assert!(field_names.contains(&"active"));
589
590        // Find and check each field
591        for (i, field) in struct_type.fields().iter().enumerate() {
592            match field.name() {
593                "name" => {
594                    assert_eq!(items[i], Value::String("John".into()));
595                    assert_eq!(field.data_type(), &ConcreteDataType::string_datatype());
596                }
597                "age" => {
598                    assert_eq!(items[i], Value::Int64(30));
599                    assert_eq!(field.data_type(), &ConcreteDataType::int64_datatype());
600                }
601                "active" => {
602                    assert_eq!(items[i], Value::Boolean(true));
603                    assert_eq!(field.data_type(), &ConcreteDataType::boolean_datatype());
604                }
605                _ => panic!("Unexpected field: {}", field.name()),
606            }
607        }
608    }
609
610    #[test]
611    fn test_encode_json_nested_object() {
612        let json = json!({
613            "person": {
614                "name": "Alice",
615                "age": 25
616            },
617            "scores": [95, 87, 92]
618        });
619
620        let settings = JsonSettings::default();
621        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
622        let Value::Struct(result) = result else {
623            panic!("Expected Struct value");
624        };
625        assert_eq!(result.items().len(), 2);
626
627        let items = result.items();
628        let struct_type = result.struct_type();
629
630        // Check person field (nested struct)
631        let person_index = struct_type
632            .fields()
633            .iter()
634            .position(|f| f.name() == "person")
635            .unwrap();
636        if let Value::Struct(person_struct) = &items[person_index] {
637            assert_eq!(person_struct.items().len(), 2);
638            let fields = person_struct.struct_type().fields();
639            let person_fields: Vec<&str> = fields.iter().map(|f| f.name()).collect();
640            assert!(person_fields.contains(&"name"));
641            assert!(person_fields.contains(&"age"));
642        } else {
643            panic!("Expected Struct value for person field");
644        }
645
646        // Check scores field (list)
647        let scores_index = struct_type
648            .fields()
649            .iter()
650            .position(|f| f.name() == "scores")
651            .unwrap();
652        if let Value::List(scores_list) = &items[scores_index] {
653            assert_eq!(scores_list.items().len(), 3);
654            assert_eq!(scores_list.items()[0], Value::Int64(95));
655            assert_eq!(scores_list.items()[1], Value::Int64(87));
656            assert_eq!(scores_list.items()[2], Value::Int64(92));
657        } else {
658            panic!("Expected List value for scores field");
659        }
660    }
661
662    #[test]
663    fn test_encode_json_structured() {
664        let json = json!({
665            "name": "Bob",
666            "age": 35
667        });
668
669        let settings = JsonSettings::default();
670        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
671
672        if let Value::Struct(struct_value) = result {
673            assert_eq!(struct_value.items().len(), 2);
674            let fields = struct_value.struct_type().fields();
675            let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
676            assert!(field_names.contains(&"name"));
677            assert!(field_names.contains(&"age"));
678        } else {
679            panic!("Expected Struct value");
680        }
681    }
682
683    #[test]
684    fn test_encode_json_respects_type_hint() {
685        let settings = JsonSettings::new(vec![JsonTypeHint {
686            path: vec!["age".to_string()],
687            data_type: ConcreteDataType::int64_datatype(),
688            nullable: false,
689            default_constraint: None,
690            inverted_index: false,
691        }]);
692
693        let result = settings
694            .encode(json!({
695                "name": "Alice",
696                "age": 42
697            }))
698            .unwrap()
699            .into_json_inner()
700            .unwrap();
701
702        let Value::Struct(struct_value) = result else {
703            panic!("Expected Struct value");
704        };
705        assert_eq!(struct_field_value(&struct_value, "age"), &Value::Int64(42));
706
707        let err = settings
708            .encode(json!({
709                "age": "42"
710            }))
711            .unwrap_err();
712        assert!(err.to_string().contains("does not match JSON2 type hint"));
713    }
714
715    #[test]
716    fn test_encode_json_respects_unsigned_type_hint() {
717        let settings = JsonSettings::new(vec![JsonTypeHint {
718            path: vec!["count".to_string()],
719            data_type: ConcreteDataType::uint64_datatype(),
720            nullable: false,
721            default_constraint: None,
722            inverted_index: false,
723        }]);
724
725        let result = settings
726            .encode(json!({
727                "count": u64::MAX
728            }))
729            .unwrap()
730            .into_json_inner()
731            .unwrap();
732
733        let Value::Struct(struct_value) = result else {
734            panic!("Expected Struct value");
735        };
736        assert_eq!(
737            struct_field_value(&struct_value, "count"),
738            &Value::UInt64(u64::MAX)
739        );
740
741        let err = settings
742            .encode(json!({
743                "count": -1
744            }))
745            .unwrap_err();
746        assert!(err.to_string().contains("does not match JSON2 type hint"));
747    }
748
749    #[test]
750    fn test_encode_json_fills_missing_type_hint_with_default() {
751        let settings = JsonSettings::new(vec![JsonTypeHint {
752            path: vec!["user".to_string(), "age".to_string()],
753            data_type: ConcreteDataType::int64_datatype(),
754            nullable: false,
755            default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
756            inverted_index: false,
757        }]);
758
759        let result = settings
760            .encode(json!({}))
761            .unwrap()
762            .into_json_inner()
763            .unwrap();
764
765        let Value::Struct(root) = result else {
766            panic!("Expected Struct value");
767        };
768        let Value::Struct(user) = struct_field_value(&root, "user") else {
769            panic!("Expected user Struct value");
770        };
771        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
772    }
773
774    #[test]
775    fn test_encode_json_fills_missing_nullable_type_hint_with_null() {
776        let settings = JsonSettings::new(vec![JsonTypeHint {
777            path: vec!["user".to_string(), "name".to_string()],
778            data_type: ConcreteDataType::string_datatype(),
779            nullable: true,
780            default_constraint: None,
781            inverted_index: false,
782        }]);
783
784        let result = settings
785            .encode(json!({ "user": {} }))
786            .unwrap()
787            .into_json_inner()
788            .unwrap();
789
790        let Value::Struct(root) = result else {
791            panic!("Expected Struct value");
792        };
793        let Value::Struct(user) = struct_field_value(&root, "user") else {
794            panic!("Expected user Struct value");
795        };
796        assert_eq!(struct_field_value(user, "name"), &Value::Null);
797    }
798
799    #[test]
800    fn test_encode_json_rejects_missing_non_null_type_hint() {
801        let settings = JsonSettings::new(vec![JsonTypeHint {
802            path: vec!["user".to_string(), "age".to_string()],
803            data_type: ConcreteDataType::int64_datatype(),
804            nullable: false,
805            default_constraint: None,
806            inverted_index: false,
807        }]);
808
809        let err = settings.encode(json!({})).unwrap_err();
810        assert!(
811            err.to_string()
812                .contains("missing non-null JSON2 type hint path user.age")
813        );
814    }
815
816    #[test]
817    fn test_encode_json_merges_missing_type_hint_prefix() {
818        let settings = JsonSettings::new(vec![
819            JsonTypeHint {
820                path: vec!["user".to_string(), "age".to_string()],
821                data_type: ConcreteDataType::int64_datatype(),
822                nullable: false,
823                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
824                inverted_index: false,
825            },
826            JsonTypeHint {
827                path: vec!["user".to_string(), "name".to_string()],
828                data_type: ConcreteDataType::string_datatype(),
829                nullable: false,
830                default_constraint: Some(ColumnDefaultConstraint::Value(Value::String(
831                    "unknown".into(),
832                ))),
833                inverted_index: false,
834            },
835        ]);
836
837        let result = settings
838            .encode(json!({}))
839            .unwrap()
840            .into_json_inner()
841            .unwrap();
842
843        let Value::Struct(root) = result else {
844            panic!("Expected Struct value");
845        };
846        let Value::Struct(user) = struct_field_value(&root, "user") else {
847            panic!("Expected user Struct value");
848        };
849        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
850        assert_eq!(
851            struct_field_value(user, "name"),
852            &Value::String("unknown".into())
853        );
854    }
855
856    #[test]
857    fn test_json_settings_structured() {
858        let json = json!({
859            "name": "Eve",
860            "score": 95
861        });
862
863        let settings = JsonSettings::default();
864        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
865
866        if let Value::Struct(struct_value) = result {
867            assert_eq!(struct_value.items().len(), 2);
868        } else {
869            panic!("Expected Struct value");
870        }
871    }
872
873    #[cfg(test)]
874    mod decode_tests {
875        use ordered_float::OrderedFloat;
876        use serde_json::json;
877
878        use super::*;
879
880        #[test]
881        fn test_decode_primitive_values() {
882            let settings = JsonSettings::default();
883
884            // Test null
885            let result = settings.decode(Value::Null).unwrap();
886            assert_eq!(result, Json::Null);
887
888            // Test boolean
889            let result = settings.decode(Value::Boolean(true)).unwrap();
890            assert_eq!(result, Json::Bool(true));
891
892            // Test integer
893            let result = settings.decode(Value::Int64(42)).unwrap();
894            assert_eq!(result, Json::from(42));
895
896            // Test float
897            let result = settings.decode(Value::Float64(OrderedFloat(3.16))).unwrap();
898            assert_eq!(result, Json::from(3.16));
899
900            // Test string
901            let result = settings.decode(Value::String("hello".into())).unwrap();
902            assert_eq!(result, Json::String("hello".to_string()));
903        }
904
905        #[test]
906        fn test_decode_struct() {
907            let settings = JsonSettings::default();
908
909            let struct_value = StructValue::new(
910                vec![
911                    Value::String("Alice".into()),
912                    Value::Int64(25),
913                    Value::Boolean(true),
914                ],
915                StructType::new(Arc::new(vec![
916                    StructField::new(
917                        "name".to_string(),
918                        ConcreteDataType::string_datatype(),
919                        true,
920                    ),
921                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
922                    StructField::new(
923                        "active".to_string(),
924                        ConcreteDataType::boolean_datatype(),
925                        true,
926                    ),
927                ])),
928            );
929
930            let result = settings.decode(Value::Struct(struct_value)).unwrap();
931            let expected = json!({
932                "name": "Alice",
933                "age": 25,
934                "active": true
935            });
936            assert_eq!(result, expected);
937        }
938
939        #[test]
940        fn test_decode_list() {
941            let settings = JsonSettings::default();
942
943            let list_value = ListValue::new(
944                vec![Value::Int64(1), Value::Int64(2), Value::Int64(3)],
945                Arc::new(ConcreteDataType::int64_datatype()),
946            );
947
948            let result = settings.decode(Value::List(list_value)).unwrap();
949            let expected = json!([1, 2, 3]);
950            assert_eq!(result, expected);
951        }
952
953        #[test]
954        fn test_decode_nested_structure() {
955            let settings = JsonSettings::default();
956
957            let inner_struct = StructValue::new(
958                vec![Value::String("Alice".into()), Value::Int64(25)],
959                StructType::new(Arc::new(vec![
960                    StructField::new(
961                        "name".to_string(),
962                        ConcreteDataType::string_datatype(),
963                        true,
964                    ),
965                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
966                ])),
967            );
968
969            let score_list_item_type = Arc::new(ConcreteDataType::int64_datatype());
970            let outer_struct = StructValue::new(
971                vec![
972                    Value::Struct(inner_struct),
973                    Value::List(ListValue::new(
974                        vec![Value::Int64(95), Value::Int64(87)],
975                        score_list_item_type.clone(),
976                    )),
977                ],
978                StructType::new(Arc::new(vec![
979                    StructField::new(
980                        "user".to_string(),
981                        ConcreteDataType::Struct(StructType::new(Arc::new(vec![
982                            StructField::new(
983                                "name".to_string(),
984                                ConcreteDataType::string_datatype(),
985                                true,
986                            ),
987                            StructField::new(
988                                "age".to_string(),
989                                ConcreteDataType::int64_datatype(),
990                                true,
991                            ),
992                        ]))),
993                        true,
994                    ),
995                    StructField::new(
996                        "scores".to_string(),
997                        ConcreteDataType::List(ListType::new(score_list_item_type.clone())),
998                        true,
999                    ),
1000                ])),
1001            );
1002
1003            let result = settings.decode(Value::Struct(outer_struct)).unwrap();
1004            let expected = json!({
1005                "user": {
1006                    "name": "Alice",
1007                    "age": 25
1008                },
1009                "scores": [95, 87]
1010            });
1011            assert_eq!(result, expected);
1012        }
1013
1014        #[test]
1015        fn test_decode_missing_fields() {
1016            let settings = JsonSettings::default();
1017
1018            // Struct with missing field (null value)
1019            let struct_value = StructValue::new(
1020                vec![
1021                    Value::String("Bob".into()),
1022                    Value::Null, // missing age field
1023                ],
1024                StructType::new(Arc::new(vec![
1025                    StructField::new(
1026                        "name".to_string(),
1027                        ConcreteDataType::string_datatype(),
1028                        true,
1029                    ),
1030                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1031                ])),
1032            );
1033
1034            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1035            let expected = json!({
1036                "name": "Bob",
1037                "age": null
1038            });
1039            assert_eq!(result, expected);
1040        }
1041    }
1042}
datatypes/json.rs

datatypes/
json.rs
