datatypes/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Data conversion between greptime's StructType and Json
16//!
17//! The idea of this module is to provide utilities to convert serde_json::Value to greptime's StructType and vice versa.
18//!
19//! The struct will carry all the fields of the Json object. We will not flatten any json object in this implementation.
20//!
21
22pub mod value;
23
24use std::collections::BTreeMap;
25use std::collections::btree_map::Entry;
26
27use serde::{Deserialize, Serialize};
28use serde_json::{Map, Value as Json};
29use snafu::ResultExt;
30
31use crate::data_type::ConcreteDataType;
32use crate::error::{self, Result, UnsupportedJsonTypeSnafu};
33use crate::json::value::{JsonValue, JsonVariant, encode_serde_json_as_jsonb};
34use crate::schema::ColumnDefaultConstraint;
35use crate::types::json_type::JsonNativeType;
36use crate::value::{ListValue, StructValue, Value};
37
38/// Maximum number of JSON container levels represented as nested Arrow types.
39pub const JSON2_MAX_STRUCTURED_DEPTH: usize = 50;
40
41/// JSON2 settings stored in column schema metadata and represented through
42/// Arrow extension metadata.
43#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
44pub struct JsonSettings {
45    #[serde(default)]
46    pub type_hints: Vec<JsonTypeHint>,
47}
48
49/// Declares selected JSON2 subpaths as typed fields.
50///
51/// These hints let JSON2 encode frequently used subpaths in a typed layout, so
52/// queries over those subpaths can get behavior and performance closer to
53/// ordinary columns.
54#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
55pub struct JsonTypeHint {
56    /// JSON2 subpath for a typed field.
57    ///
58    /// Each item is one JSON object key. For example, `["user", "age"]`
59    /// represents `user.age`.
60    ///
61    /// Array traversal is not currently supported. For example, a hint cannot
62    /// describe `events[0].name` or fields shared by all items in `events[*]`.
63    pub path: Vec<String>,
64    #[serde(rename = "type")]
65    pub data_type: ConcreteDataType,
66    pub nullable: bool,
67    #[serde(skip_serializing_if = "Option::is_none")]
68    pub default_constraint: Option<ColumnDefaultConstraint>,
69    pub inverted_index: bool,
70}
71
72/// Context for JSON encoding/decoding that tracks the current key path.
73#[derive(Clone, Debug)]
74pub struct JsonContext<'a> {
75    /// Current key path from the JSON2 root.
76    pub path: Vec<String>,
77    /// Settings for JSON encoding/decoding.
78    pub settings: &'a JsonSettings,
79}
80
81impl JsonSettings {
82    pub fn new(type_hints: Vec<JsonTypeHint>) -> Self {
83        Self { type_hints }
84    }
85
86    /// Decode an encoded StructValue back into a serde_json::Value.
87    pub fn decode(&self, value: Value) -> Result<Json> {
88        let mut context = JsonContext {
89            path: Vec::new(),
90            settings: self,
91        };
92        decode_value_with_context(value, &mut context)
93    }
94
95    /// Encode a serde_json::Value into a Value::Json using current settings.
96    pub fn encode(&self, json: Json) -> Result<Value> {
97        let mut context = JsonContext {
98            path: Vec::new(),
99            settings: self,
100        };
101        encode_json_with_context(json, &mut context).map(|v| Value::Json(Box::new(v)))
102    }
103}
104
105impl<'a> JsonContext<'a> {
106    fn type_hint(&self) -> Option<&'a JsonTypeHint> {
107        self.settings
108            .type_hints
109            .iter()
110            .find(|hint| hint.path == self.path)
111    }
112}
113
114fn with_key_context<T>(
115    context: &mut JsonContext,
116    key: &str,
117    f: impl FnOnce(&mut JsonContext) -> Result<T>,
118) -> Result<T> {
119    context.path.push(key.to_string());
120    let result = f(context);
121    context.path.pop();
122    result
123}
124
125/// Main encoding function with key path tracking
126fn encode_json_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
127    if context.path.is_empty() && !matches!(json, Json::Object(_)) {
128        return UnsupportedJsonTypeSnafu.fail();
129    }
130
131    match json {
132        Json::Object(json_object) => encode_json_object_with_context(json_object, context),
133        Json::Array(json_array) => encode_json_array_with_context(json_array, context),
134        _ => encode_json_value_with_context(json, context),
135    }
136}
137
138fn encode_json_object_with_context<'a>(
139    json_object: Map<String, Json>,
140    context: &mut JsonContext<'a>,
141) -> Result<JsonValue> {
142    let mut object = BTreeMap::new();
143    for (key, value) in json_object {
144        let value = with_key_context(context, &key, |context| {
145            if let Some(hint) = context.type_hint() {
146                encode_json_value_with_hint(value, hint, context)
147            } else {
148                encode_json_value_with_context(value, context)
149            }
150        })?;
151
152        object.insert(key, value.into_variant());
153    }
154
155    apply_missing_type_hints(&mut object, context)?;
156
157    Ok(JsonValue::new(JsonVariant::Object(object)))
158}
159
160fn apply_missing_type_hints(
161    object: &mut BTreeMap<String, JsonVariant>,
162    context: &mut JsonContext,
163) -> Result<()> {
164    for hint in &context.settings.type_hints {
165        if hint.path.len() > context.path.len() && hint.path.starts_with(&context.path) {
166            let depth = context.path.len();
167            let key = &hint.path[depth];
168            with_key_context(context, key, |context| {
169                insert_missing_type_hint(object, context, hint, depth)
170            })?;
171        }
172    }
173    Ok(())
174}
175
176fn insert_missing_type_hint(
177    object: &mut BTreeMap<String, JsonVariant>,
178    field_context: &mut JsonContext,
179    hint: &JsonTypeHint,
180    depth: usize,
181) -> Result<()> {
182    let key = &hint.path[depth];
183    let is_leaf = depth + 1 == hint.path.len();
184
185    if is_leaf {
186        if !object.contains_key(key) {
187            let value = encode_missing_type_hint_value(hint, field_context)?;
188            object.insert(key.clone(), value.into_variant());
189        }
190        return Ok(());
191    }
192
193    match object.entry(key.clone()) {
194        Entry::Occupied(mut entry) => match entry.get_mut() {
195            JsonVariant::Object(child) => {
196                insert_missing_type_hint(child, field_context, hint, depth + 1)
197            }
198            _ => error::InvalidJsonSnafu {
199                value: format!(
200                    "JSON2 type hint path {} expects object at {}",
201                    hint.path.join("."),
202                    field_context.path.join(".")
203                ),
204            }
205            .fail(),
206        },
207        Entry::Vacant(entry) => {
208            let mut child = BTreeMap::new();
209            insert_missing_type_hint(&mut child, field_context, hint, depth + 1)?;
210            entry.insert(JsonVariant::Object(child));
211            Ok(())
212        }
213    }
214}
215
216fn encode_missing_type_hint_value(
217    hint: &JsonTypeHint,
218    context: &mut JsonContext,
219) -> Result<JsonValue> {
220    if let Some(default_constraint) = &hint.default_constraint {
221        let value = default_constraint.create_default(&hint.data_type, hint.nullable)?;
222        let json = decode_primitive_value(value)?;
223        return encode_json_value_with_hint(json, hint, context);
224    }
225
226    if hint.nullable {
227        Ok(JsonValue::null())
228    } else {
229        error::InvalidJsonSnafu {
230            value: format!(
231                "missing non-null JSON2 type hint path {}",
232                hint.path.join(".")
233            ),
234        }
235        .fail()
236    }
237}
238
239fn encode_json_value_with_hint(
240    json: Json,
241    hint: &JsonTypeHint,
242    context: &mut JsonContext,
243) -> Result<JsonValue> {
244    if json.is_null() {
245        return if hint.nullable {
246            Ok(JsonValue::null())
247        } else {
248            error::InvalidJsonSnafu {
249                value: format!(
250                    "JSON2 type hint path {} is not nullable",
251                    context.path.join(".")
252                ),
253            }
254            .fail()
255        };
256    }
257
258    let invalid_type = || {
259        error::InvalidJsonSnafu {
260            value: format!(
261                "JSON value at {} does not match JSON2 type hint {}",
262                context.path.join("."),
263                hint.data_type
264            ),
265        }
266        .fail()
267    };
268
269    match (&hint.data_type, json) {
270        (ConcreteDataType::String(_), Json::String(v)) => Ok(v.into()),
271        (
272            ConcreteDataType::Int8(_)
273            | ConcreteDataType::Int16(_)
274            | ConcreteDataType::Int32(_)
275            | ConcreteDataType::Int64(_),
276            Json::Number(v),
277        ) => match v.as_i64() {
278            Some(v) => Ok(v.into()),
279            None => invalid_type(),
280        },
281        (
282            ConcreteDataType::UInt8(_)
283            | ConcreteDataType::UInt16(_)
284            | ConcreteDataType::UInt32(_)
285            | ConcreteDataType::UInt64(_),
286            Json::Number(v),
287        ) => match v.as_u64() {
288            Some(v) => Ok(v.into()),
289            None => invalid_type(),
290        },
291        (ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(v)) => {
292            match v.as_f64() {
293                Some(v) => Ok(v.into()),
294                None => invalid_type(),
295            }
296        }
297        (ConcreteDataType::Boolean(_), Json::Bool(v)) => Ok(v.into()),
298        _ => invalid_type(),
299    }
300}
301
302fn encode_json_array_with_context<'a>(
303    json_array: Vec<Json>,
304    context: &mut JsonContext<'a>,
305) -> Result<JsonValue> {
306    let json_array_len = json_array.len();
307    let mut items = Vec::with_capacity(json_array_len);
308
309    for (index, value) in json_array.into_iter().enumerate() {
310        let item_value = with_key_context(context, &index.to_string(), |context| {
311            encode_json_value_with_context(value, context)
312        })?;
313        items.push(item_value);
314    }
315
316    // In specification, it's valid for a JSON array to have different types of items, for example,
317    // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
318    // array, which requires all items have exactly the same type. So we merge out the maybe
319    // different item types to a unified type, and align all the item values to it.
320
321    let merged_item_type = if let Some((first, rests)) = items.split_first() {
322        let mut merged = first.json_type().clone();
323        for rest in rests.iter().map(|x| x.json_type()) {
324            if matches!(merged, JsonNativeType::Variant) {
325                break;
326            }
327            merged.merge(rest);
328        }
329        Some(merged)
330    } else {
331        None
332    };
333    if let Some(unified_item_type) = merged_item_type {
334        for item in &mut items {
335            item.try_align(&unified_item_type)?;
336        }
337    }
338    let items = items
339        .into_iter()
340        .map(|x| x.into_variant())
341        .collect::<Vec<_>>();
342    Ok(JsonValue::new(JsonVariant::Array(items)))
343}
344
345/// Helper function to encode a JSON value to a Value and determine its ConcreteDataType with context
346fn encode_json_value_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
347    if context.path.len() >= JSON2_MAX_STRUCTURED_DEPTH
348        && matches!(&json, Json::Object(_) | Json::Array(_))
349    {
350        return Ok(JsonValue::new(JsonVariant::Variant(
351            encode_serde_json_as_jsonb(json),
352        )));
353    }
354
355    match json {
356        Json::Null => Ok(JsonValue::null()),
357        Json::Bool(b) => Ok(b.into()),
358        Json::Number(n) => {
359            if let Some(i) = n.as_i64() {
360                Ok(i.into())
361            } else if let Some(u) = n.as_u64() {
362                if u <= i64::MAX as u64 {
363                    Ok((u as i64).into())
364                } else {
365                    Ok(u.into())
366                }
367            } else if let Some(f) = n.as_f64() {
368                Ok(f.into())
369            } else {
370                // Fallback to string representation
371                Ok(n.to_string().into())
372            }
373        }
374        Json::String(s) => Ok(s.into()),
375        Json::Array(arr) => encode_json_array_with_context(arr, context),
376        Json::Object(obj) => encode_json_object_with_context(obj, context),
377    }
378}
379
380/// Main decoding function with key path tracking
381fn decode_value_with_context(value: Value, context: &mut JsonContext) -> Result<Json> {
382    match value {
383        Value::Struct(struct_value) => decode_struct_with_context(struct_value, context),
384        Value::List(list_value) => decode_list_with_context(list_value, context),
385        _ => decode_primitive_value(value),
386    }
387}
388
389/// Decode a structured value to JSON object
390fn decode_struct_with_context<'a>(
391    struct_value: StructValue,
392    context: &mut JsonContext<'a>,
393) -> Result<Json> {
394    let mut json_object = Map::with_capacity(struct_value.len());
395
396    let (items, fields) = struct_value.into_parts();
397
398    for (field, field_value) in fields.fields().iter().zip(items) {
399        let json_value = with_key_context(context, field.name(), |context| {
400            decode_value_with_context(field_value, context)
401        })?;
402        json_object.insert(field.name().to_string(), json_value);
403    }
404
405    Ok(Json::Object(json_object))
406}
407
408/// Decode a list value to JSON array
409fn decode_list_with_context(list_value: ListValue, context: &mut JsonContext) -> Result<Json> {
410    let mut json_array = Vec::with_capacity(list_value.len());
411
412    let data_items = list_value.take_items();
413
414    for (index, item) in data_items.into_iter().enumerate() {
415        let json_value = with_key_context(context, &index.to_string(), |context| {
416            decode_value_with_context(item, context)
417        })?;
418        json_array.push(json_value);
419    }
420
421    Ok(Json::Array(json_array))
422}
423
424/// Decode primitive value to JSON
425fn decode_primitive_value(value: Value) -> Result<Json> {
426    match value {
427        Value::Null => Ok(Json::Null),
428        Value::Boolean(b) => Ok(Json::Bool(b)),
429        Value::UInt8(v) => Ok(Json::from(v)),
430        Value::UInt16(v) => Ok(Json::from(v)),
431        Value::UInt32(v) => Ok(Json::from(v)),
432        Value::UInt64(v) => Ok(Json::from(v)),
433        Value::Int8(v) => Ok(Json::from(v)),
434        Value::Int16(v) => Ok(Json::from(v)),
435        Value::Int32(v) => Ok(Json::from(v)),
436        Value::Int64(v) => Ok(Json::from(v)),
437        Value::Float32(v) => Ok(Json::from(v.0)),
438        Value::Float64(v) => Ok(Json::from(v.0)),
439        Value::String(s) => Ok(Json::String(s.as_utf8().to_string())),
440        Value::Binary(b) => serde_json::to_value(b.as_ref()).context(error::SerializeSnafu),
441        Value::Date(v) => Ok(Json::from(v.val())),
442        Value::Timestamp(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
443        Value::Time(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
444        Value::IntervalYearMonth(v) => {
445            serde_json::to_value(v.to_i32()).context(error::SerializeSnafu)
446        }
447        Value::IntervalDayTime(v) => {
448            serde_json::to_value(v.to_i64()).context(error::SerializeSnafu)
449        }
450        Value::IntervalMonthDayNano(v) => {
451            serde_json::to_value(v.to_i128()).context(error::SerializeSnafu)
452        }
453        Value::Duration(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
454        Value::Decimal128(v) => serde_json::to_value(v.to_string()).context(error::SerializeSnafu),
455        Value::Struct(_) | Value::List(_) | Value::Json(_) => {
456            // These should be handled by the context-aware functions
457            Err(error::InvalidJsonSnafu {
458                value: "Structured values should be handled by context-aware decoding".to_string(),
459            }
460            .build())
461        }
462    }
463}
464
465#[cfg(test)]
466mod tests {
467    use std::sync::Arc;
468
469    use serde_json::json;
470
471    use super::*;
472    use crate::data_type::ConcreteDataType;
473    use crate::types::{ListType, StructField, StructType};
474
475    fn struct_field_value<'a>(struct_value: &'a StructValue, field_name: &str) -> &'a Value {
476        let index = struct_value
477            .struct_type()
478            .fields()
479            .iter()
480            .position(|field| field.name() == field_name)
481            .expect("field exists");
482        &struct_value.items()[index]
483    }
484
485    #[test]
486    fn test_json_settings_forward_compatibility() {
487        let json_str = r#"{
488            "type_hints": [
489                {
490                    "path": ["user", "age"],
491                    "type": {
492                        "Int64": {}
493                    },
494                    "nullable": false,
495                    "default_constraint": {
496                        "Value": {
497                            "Int64": 18
498                        }
499                    },
500                    "inverted_index": true
501                },
502                {
503                    "path": ["user", "name"],
504                    "type": {
505                        "String": {
506                            "size_type": "Utf8"
507                        }
508                    },
509                    "nullable": true,
510                    "inverted_index": false
511                }
512            ]
513        }"#;
514
515        let deserialized = serde_json::from_str::<JsonSettings>(json_str).unwrap();
516
517        assert_eq!(
518            deserialized,
519            JsonSettings::new(vec![
520                JsonTypeHint {
521                    path: vec!["user".to_string(), "age".to_string()],
522                    data_type: ConcreteDataType::int64_datatype(),
523                    nullable: false,
524                    default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
525                    inverted_index: true,
526                },
527                JsonTypeHint {
528                    path: vec!["user".to_string(), "name".to_string()],
529                    data_type: ConcreteDataType::string_datatype(),
530                    nullable: true,
531                    default_constraint: None,
532                    inverted_index: false,
533                },
534            ])
535        );
536    }
537
538    #[test]
539    fn test_json_settings_ser_de() {
540        let settings = JsonSettings::new(vec![
541            JsonTypeHint {
542                path: vec!["user".to_string(), "age".to_string()],
543                data_type: ConcreteDataType::int64_datatype(),
544                nullable: false,
545                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
546                inverted_index: true,
547            },
548            JsonTypeHint {
549                path: vec!["user".to_string(), "name".to_string()],
550                data_type: ConcreteDataType::string_datatype(),
551                nullable: true,
552                default_constraint: None,
553                inverted_index: false,
554            },
555        ]);
556
557        let serialized = serde_json::to_string(&settings).unwrap();
558        let deserialized = serde_json::from_str::<JsonSettings>(&serialized).unwrap();
559
560        assert_eq!(settings, deserialized);
561    }
562
563    #[test]
564    fn test_encode_root_non_object_json() {
565        let settings = JsonSettings::default();
566        let cases = [
567            ("null", Json::Null),
568            ("boolean", Json::Bool(true)),
569            ("integer", Json::from(42)),
570            ("float", Json::from(3.15)),
571            ("string", Json::String("hello".to_string())),
572            ("array", json!([1, 2, 3])),
573            ("mixed array", json!([1, "hello", true, 3.15])),
574            ("empty array", json!([])),
575        ];
576
577        for (name, json) in cases {
578            let err = settings.encode(json).unwrap_err();
579            assert!(
580                matches!(err, crate::error::Error::UnsupportedJsonType { .. }),
581                "{name}: {err:?}"
582            );
583        }
584    }
585
586    #[test]
587    fn test_encode_json_object() {
588        let json = json!({
589            "name": "John",
590            "age": 30,
591            "active": true
592        });
593
594        let settings = JsonSettings::default();
595        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
596        let Value::Struct(result) = result else {
597            panic!("Expected Struct value");
598        };
599        assert_eq!(result.items().len(), 3);
600
601        let items = result.items();
602        let struct_type = result.struct_type();
603
604        // Check that we have the expected fields
605        let fields = struct_type.fields();
606        let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
607        assert!(field_names.contains(&"name"));
608        assert!(field_names.contains(&"age"));
609        assert!(field_names.contains(&"active"));
610
611        // Find and check each field
612        for (i, field) in struct_type.fields().iter().enumerate() {
613            match field.name() {
614                "name" => {
615                    assert_eq!(items[i], Value::String("John".into()));
616                    assert_eq!(field.data_type(), &ConcreteDataType::string_datatype());
617                }
618                "age" => {
619                    assert_eq!(items[i], Value::Int64(30));
620                    assert_eq!(field.data_type(), &ConcreteDataType::int64_datatype());
621                }
622                "active" => {
623                    assert_eq!(items[i], Value::Boolean(true));
624                    assert_eq!(field.data_type(), &ConcreteDataType::boolean_datatype());
625                }
626                _ => panic!("Unexpected field: {}", field.name()),
627            }
628        }
629    }
630
631    #[test]
632    fn test_encode_json_nested_object() {
633        let json = json!({
634            "person": {
635                "name": "Alice",
636                "age": 25
637            },
638            "scores": [95, 87, 92]
639        });
640
641        let settings = JsonSettings::default();
642        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
643        let Value::Struct(result) = result else {
644            panic!("Expected Struct value");
645        };
646        assert_eq!(result.items().len(), 2);
647
648        let items = result.items();
649        let struct_type = result.struct_type();
650
651        // Check person field (nested struct)
652        let person_index = struct_type
653            .fields()
654            .iter()
655            .position(|f| f.name() == "person")
656            .unwrap();
657        if let Value::Struct(person_struct) = &items[person_index] {
658            assert_eq!(person_struct.items().len(), 2);
659            let fields = person_struct.struct_type().fields();
660            let person_fields: Vec<&str> = fields.iter().map(|f| f.name()).collect();
661            assert!(person_fields.contains(&"name"));
662            assert!(person_fields.contains(&"age"));
663        } else {
664            panic!("Expected Struct value for person field");
665        }
666
667        // Check scores field (list)
668        let scores_index = struct_type
669            .fields()
670            .iter()
671            .position(|f| f.name() == "scores")
672            .unwrap();
673        if let Value::List(scores_list) = &items[scores_index] {
674            assert_eq!(scores_list.items().len(), 3);
675            assert_eq!(scores_list.items()[0], Value::Int64(95));
676            assert_eq!(scores_list.items()[1], Value::Int64(87));
677            assert_eq!(scores_list.items()[2], Value::Int64(92));
678        } else {
679            panic!("Expected List value for scores field");
680        }
681    }
682
683    #[test]
684    fn test_encode_json_structured() {
685        let json = json!({
686            "name": "Bob",
687            "age": 35
688        });
689
690        let settings = JsonSettings::default();
691        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
692
693        if let Value::Struct(struct_value) = result {
694            assert_eq!(struct_value.items().len(), 2);
695            let fields = struct_value.struct_type().fields();
696            let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
697            assert!(field_names.contains(&"name"));
698            assert!(field_names.contains(&"age"));
699        } else {
700            panic!("Expected Struct value");
701        }
702    }
703
704    #[test]
705    fn test_encode_json_respects_type_hint() {
706        let settings = JsonSettings::new(vec![JsonTypeHint {
707            path: vec!["age".to_string()],
708            data_type: ConcreteDataType::int64_datatype(),
709            nullable: false,
710            default_constraint: None,
711            inverted_index: false,
712        }]);
713
714        let result = settings
715            .encode(json!({
716                "name": "Alice",
717                "age": 42
718            }))
719            .unwrap()
720            .into_json_inner()
721            .unwrap();
722
723        let Value::Struct(struct_value) = result else {
724            panic!("Expected Struct value");
725        };
726        assert_eq!(struct_field_value(&struct_value, "age"), &Value::Int64(42));
727
728        let err = settings
729            .encode(json!({
730                "age": "42"
731            }))
732            .unwrap_err();
733        assert!(err.to_string().contains("does not match JSON2 type hint"));
734    }
735
736    #[test]
737    fn test_encode_json_respects_unsigned_type_hint() {
738        let settings = JsonSettings::new(vec![JsonTypeHint {
739            path: vec!["count".to_string()],
740            data_type: ConcreteDataType::uint64_datatype(),
741            nullable: false,
742            default_constraint: None,
743            inverted_index: false,
744        }]);
745
746        let result = settings
747            .encode(json!({
748                "count": u64::MAX
749            }))
750            .unwrap()
751            .into_json_inner()
752            .unwrap();
753
754        let Value::Struct(struct_value) = result else {
755            panic!("Expected Struct value");
756        };
757        assert_eq!(
758            struct_field_value(&struct_value, "count"),
759            &Value::UInt64(u64::MAX)
760        );
761
762        let err = settings
763            .encode(json!({
764                "count": -1
765            }))
766            .unwrap_err();
767        assert!(err.to_string().contains("does not match JSON2 type hint"));
768    }
769
770    #[test]
771    fn test_encode_json_fills_missing_type_hint_with_default() {
772        let settings = JsonSettings::new(vec![JsonTypeHint {
773            path: vec!["user".to_string(), "age".to_string()],
774            data_type: ConcreteDataType::int64_datatype(),
775            nullable: false,
776            default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
777            inverted_index: false,
778        }]);
779
780        let result = settings
781            .encode(json!({}))
782            .unwrap()
783            .into_json_inner()
784            .unwrap();
785
786        let Value::Struct(root) = result else {
787            panic!("Expected Struct value");
788        };
789        let Value::Struct(user) = struct_field_value(&root, "user") else {
790            panic!("Expected user Struct value");
791        };
792        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
793    }
794
795    #[test]
796    fn test_encode_json_fills_missing_nullable_type_hint_with_null() {
797        let settings = JsonSettings::new(vec![JsonTypeHint {
798            path: vec!["user".to_string(), "name".to_string()],
799            data_type: ConcreteDataType::string_datatype(),
800            nullable: true,
801            default_constraint: None,
802            inverted_index: false,
803        }]);
804
805        let result = settings
806            .encode(json!({ "user": {} }))
807            .unwrap()
808            .into_json_inner()
809            .unwrap();
810
811        let Value::Struct(root) = result else {
812            panic!("Expected Struct value");
813        };
814        let Value::Struct(user) = struct_field_value(&root, "user") else {
815            panic!("Expected user Struct value");
816        };
817        assert_eq!(struct_field_value(user, "name"), &Value::Null);
818    }
819
820    #[test]
821    fn test_encode_json_rejects_missing_non_null_type_hint() {
822        let settings = JsonSettings::new(vec![JsonTypeHint {
823            path: vec!["user".to_string(), "age".to_string()],
824            data_type: ConcreteDataType::int64_datatype(),
825            nullable: false,
826            default_constraint: None,
827            inverted_index: false,
828        }]);
829
830        let err = settings.encode(json!({})).unwrap_err();
831        assert!(
832            err.to_string()
833                .contains("missing non-null JSON2 type hint path user.age")
834        );
835    }
836
837    #[test]
838    fn test_encode_json_merges_missing_type_hint_prefix() {
839        let settings = JsonSettings::new(vec![
840            JsonTypeHint {
841                path: vec!["user".to_string(), "age".to_string()],
842                data_type: ConcreteDataType::int64_datatype(),
843                nullable: false,
844                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
845                inverted_index: false,
846            },
847            JsonTypeHint {
848                path: vec!["user".to_string(), "name".to_string()],
849                data_type: ConcreteDataType::string_datatype(),
850                nullable: false,
851                default_constraint: Some(ColumnDefaultConstraint::Value(Value::String(
852                    "unknown".into(),
853                ))),
854                inverted_index: false,
855            },
856        ]);
857
858        let result = settings
859            .encode(json!({}))
860            .unwrap()
861            .into_json_inner()
862            .unwrap();
863
864        let Value::Struct(root) = result else {
865            panic!("Expected Struct value");
866        };
867        let Value::Struct(user) = struct_field_value(&root, "user") else {
868            panic!("Expected user Struct value");
869        };
870        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
871        assert_eq!(
872            struct_field_value(user, "name"),
873            &Value::String("unknown".into())
874        );
875    }
876
877    #[test]
878    fn test_json_settings_structured() {
879        let json = json!({
880            "name": "Eve",
881            "score": 95
882        });
883
884        let settings = JsonSettings::default();
885        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
886
887        if let Value::Struct(struct_value) = result {
888            assert_eq!(struct_value.items().len(), 2);
889        } else {
890            panic!("Expected Struct value");
891        }
892    }
893
894    #[cfg(test)]
895    mod decode_tests {
896        use ordered_float::OrderedFloat;
897        use serde_json::json;
898
899        use super::*;
900
901        #[test]
902        fn test_decode_primitive_values() {
903            let settings = JsonSettings::default();
904
905            // Test null
906            let result = settings.decode(Value::Null).unwrap();
907            assert_eq!(result, Json::Null);
908
909            // Test boolean
910            let result = settings.decode(Value::Boolean(true)).unwrap();
911            assert_eq!(result, Json::Bool(true));
912
913            // Test integer
914            let result = settings.decode(Value::Int64(42)).unwrap();
915            assert_eq!(result, Json::from(42));
916
917            // Test float
918            let result = settings.decode(Value::Float64(OrderedFloat(3.16))).unwrap();
919            assert_eq!(result, Json::from(3.16));
920
921            // Test string
922            let result = settings.decode(Value::String("hello".into())).unwrap();
923            assert_eq!(result, Json::String("hello".to_string()));
924        }
925
926        #[test]
927        fn test_decode_struct() {
928            let settings = JsonSettings::default();
929
930            let struct_value = StructValue::new(
931                vec![
932                    Value::String("Alice".into()),
933                    Value::Int64(25),
934                    Value::Boolean(true),
935                ],
936                StructType::new(Arc::new(vec![
937                    StructField::new(
938                        "name".to_string(),
939                        ConcreteDataType::string_datatype(),
940                        true,
941                    ),
942                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
943                    StructField::new(
944                        "active".to_string(),
945                        ConcreteDataType::boolean_datatype(),
946                        true,
947                    ),
948                ])),
949            );
950
951            let result = settings.decode(Value::Struct(struct_value)).unwrap();
952            let expected = json!({
953                "name": "Alice",
954                "age": 25,
955                "active": true
956            });
957            assert_eq!(result, expected);
958        }
959
960        #[test]
961        fn test_decode_list() {
962            let settings = JsonSettings::default();
963
964            let list_value = ListValue::new(
965                vec![Value::Int64(1), Value::Int64(2), Value::Int64(3)],
966                Arc::new(ConcreteDataType::int64_datatype()),
967            );
968
969            let result = settings.decode(Value::List(list_value)).unwrap();
970            let expected = json!([1, 2, 3]);
971            assert_eq!(result, expected);
972        }
973
974        #[test]
975        fn test_decode_nested_structure() {
976            let settings = JsonSettings::default();
977
978            let inner_struct = StructValue::new(
979                vec![Value::String("Alice".into()), Value::Int64(25)],
980                StructType::new(Arc::new(vec![
981                    StructField::new(
982                        "name".to_string(),
983                        ConcreteDataType::string_datatype(),
984                        true,
985                    ),
986                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
987                ])),
988            );
989
990            let score_list_item_type = Arc::new(ConcreteDataType::int64_datatype());
991            let outer_struct = StructValue::new(
992                vec![
993                    Value::Struct(inner_struct),
994                    Value::List(ListValue::new(
995                        vec![Value::Int64(95), Value::Int64(87)],
996                        score_list_item_type.clone(),
997                    )),
998                ],
999                StructType::new(Arc::new(vec![
1000                    StructField::new(
1001                        "user".to_string(),
1002                        ConcreteDataType::Struct(StructType::new(Arc::new(vec![
1003                            StructField::new(
1004                                "name".to_string(),
1005                                ConcreteDataType::string_datatype(),
1006                                true,
1007                            ),
1008                            StructField::new(
1009                                "age".to_string(),
1010                                ConcreteDataType::int64_datatype(),
1011                                true,
1012                            ),
1013                        ]))),
1014                        true,
1015                    ),
1016                    StructField::new(
1017                        "scores".to_string(),
1018                        ConcreteDataType::List(ListType::new(score_list_item_type.clone())),
1019                        true,
1020                    ),
1021                ])),
1022            );
1023
1024            let result = settings.decode(Value::Struct(outer_struct)).unwrap();
1025            let expected = json!({
1026                "user": {
1027                    "name": "Alice",
1028                    "age": 25
1029                },
1030                "scores": [95, 87]
1031            });
1032            assert_eq!(result, expected);
1033        }
1034
1035        #[test]
1036        fn test_decode_missing_fields() {
1037            let settings = JsonSettings::default();
1038
1039            // Struct with missing field (null value)
1040            let struct_value = StructValue::new(
1041                vec![
1042                    Value::String("Bob".into()),
1043                    Value::Null, // missing age field
1044                ],
1045                StructType::new(Arc::new(vec![
1046                    StructField::new(
1047                        "name".to_string(),
1048                        ConcreteDataType::string_datatype(),
1049                        true,
1050                    ),
1051                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1052                ])),
1053            );
1054
1055            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1056            let expected = json!({
1057                "name": "Bob",
1058                "age": null
1059            });
1060            assert_eq!(result, expected);
1061        }
1062    }
1063}
datatypes/json.rs

datatypes/
json.rs
