datatypes/
json.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Data conversion between greptime's StructType and Json
16//!
17//! The idea of this module is to provide utilities to convert serde_json::Value to greptime's StructType and vice versa.
18//!
19//! The struct will carry all the fields of the Json object. We will not flatten any json object in this implementation.
20//!
21
22pub mod value;
23
24use std::collections::BTreeMap;
25use std::collections::btree_map::Entry;
26
27use serde::{Deserialize, Serialize};
28use serde_json::{Map, Value as Json};
29use snafu::ResultExt;
30
31use crate::data_type::ConcreteDataType;
32use crate::error::{self, InvalidJson2SettingsSnafu, Result, UnsupportedJsonTypeSnafu};
33use crate::json::value::{JsonValue, JsonVariant, encode_serde_json_as_jsonb};
34use crate::schema::ColumnDefaultConstraint;
35use crate::types::json_type::{JsonNativeType, JsonObjectType};
36use crate::value::{ListValue, StructValue, Value};
37
38/// Maximum number of JSON container levels represented as nested Arrow types.
39pub const JSON2_MAX_STRUCTURED_DEPTH: usize = 50;
40/// Reserved physical field containing unexpanded JSON2 paths.
41pub const JSON2_REMAINDER_FIELD_NAME: &str = "!__remainder__!";
42/// Default maximum number of unhinted JSON leaf paths expanded into Arrow fields.
43pub const JSON2_DEFAULT_MAX_AUTO_EXPANDED_PATHS: u32 = 100;
44
45/// JSON2 settings stored in column schema metadata and represented through
46/// Arrow extension metadata.
47#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
48pub struct JsonSettings {
49    #[serde(default)]
50    type_hints: Vec<JsonTypeHint>,
51    /// Maximum number of unhinted JSON paths expanded into Arrow fields.
52    ///
53    /// `None` preserves the legacy unlimited layout.
54    #[serde(default, skip_serializing_if = "Option::is_none")]
55    max_auto_expanded_paths: Option<u32>,
56}
57
58#[derive(Deserialize)]
59struct JsonSettingsSerde {
60    #[serde(default)]
61    type_hints: Vec<JsonTypeHint>,
62    #[serde(default)]
63    max_auto_expanded_paths: Option<u32>,
64}
65
66impl<'de> Deserialize<'de> for JsonSettings {
67    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
68    where
69        D: serde::Deserializer<'de>,
70    {
71        let settings = JsonSettingsSerde::deserialize(deserializer)?;
72        Self::try_new(settings.type_hints, settings.max_auto_expanded_paths)
73            .map_err(serde::de::Error::custom)
74    }
75}
76
77/// Declares selected JSON2 subpaths as typed fields.
78///
79/// These hints let JSON2 encode frequently used subpaths in a typed layout, so
80/// queries over those subpaths can get behavior and performance closer to
81/// ordinary columns.
82#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
83pub struct JsonTypeHint {
84    /// JSON2 subpath for a typed field.
85    ///
86    /// Each item is one JSON object key. For example, `["user", "age"]`
87    /// represents `user.age`.
88    ///
89    /// Array traversal is not currently supported. For example, a hint cannot
90    /// describe `events[0].name` or fields shared by all items in `events[*]`.
91    pub path: Vec<String>,
92    #[serde(rename = "type")]
93    pub data_type: ConcreteDataType,
94    pub nullable: bool,
95    #[serde(skip_serializing_if = "Option::is_none")]
96    pub default_constraint: Option<ColumnDefaultConstraint>,
97    pub inverted_index: bool,
98}
99
100/// Context for JSON encoding/decoding that tracks the current key path.
101#[derive(Clone, Debug)]
102pub struct JsonContext<'a> {
103    /// Current key path from the JSON2 root.
104    pub path: Vec<String>,
105    /// Settings for JSON encoding/decoding.
106    pub settings: &'a JsonSettings,
107}
108
109impl JsonSettings {
110    /// Creates default v2 settings for newly created JSON2 columns.
111    pub fn new_v2() -> Self {
112        Self {
113            type_hints: vec![],
114            max_auto_expanded_paths: Some(JSON2_DEFAULT_MAX_AUTO_EXPANDED_PATHS),
115        }
116    }
117
118    /// Creates and validates JSON2 settings.
119    pub fn try_new(
120        type_hints: Vec<JsonTypeHint>,
121        max_auto_expanded_paths: Option<u32>,
122    ) -> Result<Self> {
123        validate_type_hints(&type_hints)?;
124        Ok(Self {
125            type_hints,
126            max_auto_expanded_paths,
127        })
128    }
129
130    pub fn type_hints(&self) -> &[JsonTypeHint] {
131        &self.type_hints
132    }
133
134    pub fn max_auto_expanded_paths(&self) -> Option<u32> {
135        self.max_auto_expanded_paths
136    }
137
138    pub fn into_parts(self) -> (Vec<JsonTypeHint>, Option<u32>) {
139        (self.type_hints, self.max_auto_expanded_paths)
140    }
141
142    /// Decode an encoded StructValue back into a serde_json::Value.
143    pub fn decode(&self, value: Value) -> Result<Json> {
144        let mut context = JsonContext {
145            path: Vec::new(),
146            settings: self,
147        };
148        decode_value_with_context(value, &mut context)
149    }
150
151    /// Encode a serde_json::Value into a Value::Json using current settings.
152    pub fn encode(&self, json: Json) -> Result<Value> {
153        if let Json::Object(object) = &json
154            && object.contains_key(JSON2_REMAINDER_FIELD_NAME)
155        {
156            return error::InvalidJsonSnafu {
157                value: format!(
158                    "root object cannot contain reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
159                ),
160            }
161            .fail();
162        }
163        let mut context = JsonContext {
164            path: Vec::new(),
165            settings: self,
166        };
167        encode_json_with_context(json, &mut context).map(|v| Value::Json(Box::new(v)))
168    }
169}
170
171fn validate_type_hints(type_hints: &[JsonTypeHint]) -> Result<()> {
172    if type_hints.is_empty() {
173        return Ok(());
174    }
175
176    let mut object = JsonObjectType::new();
177    for hint in type_hints {
178        if hint.path.len() > JSON2_MAX_STRUCTURED_DEPTH {
179            return InvalidJson2SettingsSnafu {
180                reason: format!(
181                    "JSON2 type hint path cannot exceed {JSON2_MAX_STRUCTURED_DEPTH} segments"
182                ),
183            }
184            .fail();
185        }
186        if hint
187            .path
188            .first()
189            .is_some_and(|x| x == JSON2_REMAINDER_FIELD_NAME)
190        {
191            return InvalidJson2SettingsSnafu {
192                reason: format!(
193                    "JSON2 type hint path cannot be rooted at reserved field '{JSON2_REMAINDER_FIELD_NAME}'"
194                ),
195            }
196            .fail();
197        }
198        let data_type = match &hint.data_type {
199            ConcreteDataType::Boolean(_)
200            | ConcreteDataType::UInt8(_)
201            | ConcreteDataType::UInt16(_)
202            | ConcreteDataType::UInt32(_)
203            | ConcreteDataType::UInt64(_)
204            | ConcreteDataType::Int8(_)
205            | ConcreteDataType::Int16(_)
206            | ConcreteDataType::Int32(_)
207            | ConcreteDataType::Int64(_)
208            | ConcreteDataType::Float32(_)
209            | ConcreteDataType::Float64(_)
210            | ConcreteDataType::String(_)
211            | ConcreteDataType::List(_)
212            | ConcreteDataType::Struct(_) => (&hint.data_type).into(),
213            data_type => {
214                return InvalidJson2SettingsSnafu {
215                    reason: format!("unsupported JSON2 type hint data type: {data_type}"),
216                }
217                .fail();
218            }
219        };
220        let non_finite_default = match &hint.default_constraint {
221            Some(ColumnDefaultConstraint::Value(Value::Float32(value))) => !value.0.is_finite(),
222            Some(ColumnDefaultConstraint::Value(Value::Float64(value))) => !value.0.is_finite(),
223            _ => false,
224        };
225        if non_finite_default {
226            return InvalidJson2SettingsSnafu {
227                reason: format!(
228                    "JSON2 type hint default for '{}' must be finite",
229                    hint.path.join(".")
230                ),
231            }
232            .fail();
233        }
234        validate_type_hint(&mut object, &hint.path, data_type)?;
235    }
236    Ok(())
237}
238
239fn validate_type_hint(
240    object: &mut JsonObjectType,
241    path: &[String],
242    data_type: JsonNativeType,
243) -> Result<()> {
244    let Some((name, path)) = path.split_first() else {
245        return InvalidJson2SettingsSnafu {
246            reason: "JSON2 type hint path must not be empty".to_string(),
247        }
248        .fail();
249    };
250    if path.is_empty() {
251        if object.insert(name.clone(), data_type).is_some() {
252            return InvalidJson2SettingsSnafu {
253                reason: format!("duplicate JSON2 type hint path '{name}'"),
254            }
255            .fail();
256        }
257        return Ok(());
258    }
259
260    let child = object
261        .entry(name.clone())
262        .or_insert_with(|| JsonNativeType::Object(JsonObjectType::new()));
263    let JsonNativeType::Object(child) = child else {
264        return InvalidJson2SettingsSnafu {
265            reason: format!("conflicting JSON2 type hint path at '{name}'"),
266        }
267        .fail();
268    };
269    validate_type_hint(child, path, data_type)
270}
271
272impl<'a> JsonContext<'a> {
273    fn type_hint(&self) -> Option<&'a JsonTypeHint> {
274        self.settings
275            .type_hints
276            .iter()
277            .find(|hint| hint.path == self.path)
278    }
279}
280
281fn with_key_context<T>(
282    context: &mut JsonContext,
283    key: &str,
284    f: impl FnOnce(&mut JsonContext) -> Result<T>,
285) -> Result<T> {
286    context.path.push(key.to_string());
287    let result = f(context);
288    context.path.pop();
289    result
290}
291
292/// Main encoding function with key path tracking
293fn encode_json_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
294    if context.path.is_empty() && !matches!(json, Json::Object(_)) {
295        return UnsupportedJsonTypeSnafu.fail();
296    }
297
298    match json {
299        Json::Object(json_object) => encode_json_object_with_context(json_object, context),
300        Json::Array(json_array) => encode_json_array_with_context(json_array, context),
301        _ => encode_json_value_with_context(json, context),
302    }
303}
304
305fn encode_json_object_with_context<'a>(
306    json_object: Map<String, Json>,
307    context: &mut JsonContext<'a>,
308) -> Result<JsonValue> {
309    let mut object = BTreeMap::new();
310    for (key, value) in json_object {
311        let value = with_key_context(context, &key, |context| {
312            if let Some(hint) = context.type_hint() {
313                encode_json_value_with_hint(value, hint, context)
314            } else {
315                encode_json_value_with_context(value, context)
316            }
317        })?;
318
319        object.insert(key, value.into_variant());
320    }
321
322    apply_missing_type_hints(&mut object, context)?;
323
324    Ok(JsonValue::new(JsonVariant::Object(object)))
325}
326
327fn apply_missing_type_hints(
328    object: &mut BTreeMap<String, JsonVariant>,
329    context: &mut JsonContext,
330) -> Result<()> {
331    for hint in &context.settings.type_hints {
332        if hint.path.len() > context.path.len() && hint.path.starts_with(&context.path) {
333            let depth = context.path.len();
334            let key = &hint.path[depth];
335            with_key_context(context, key, |context| {
336                insert_missing_type_hint(object, context, hint, depth)
337            })?;
338        }
339    }
340    Ok(())
341}
342
343fn insert_missing_type_hint(
344    object: &mut BTreeMap<String, JsonVariant>,
345    field_context: &mut JsonContext,
346    hint: &JsonTypeHint,
347    depth: usize,
348) -> Result<()> {
349    let key = &hint.path[depth];
350    let is_leaf = depth + 1 == hint.path.len();
351
352    if is_leaf {
353        if !object.contains_key(key) {
354            let value = encode_missing_type_hint_value(hint, field_context)?;
355            object.insert(key.clone(), value.into_variant());
356        }
357        return Ok(());
358    }
359
360    match object.entry(key.clone()) {
361        Entry::Occupied(mut entry) => match entry.get_mut() {
362            JsonVariant::Object(child) => {
363                insert_missing_type_hint(child, field_context, hint, depth + 1)
364            }
365            _ => error::InvalidJsonSnafu {
366                value: format!(
367                    "JSON2 type hint path {} expects object at {}",
368                    hint.path.join("."),
369                    field_context.path.join(".")
370                ),
371            }
372            .fail(),
373        },
374        Entry::Vacant(entry) => {
375            let mut child = BTreeMap::new();
376            insert_missing_type_hint(&mut child, field_context, hint, depth + 1)?;
377            entry.insert(JsonVariant::Object(child));
378            Ok(())
379        }
380    }
381}
382
383fn encode_missing_type_hint_value(
384    hint: &JsonTypeHint,
385    context: &mut JsonContext,
386) -> Result<JsonValue> {
387    if let Some(default_constraint) = &hint.default_constraint {
388        let value = default_constraint.create_default(&hint.data_type, hint.nullable)?;
389        let json = decode_primitive_value(value)?;
390        return encode_json_value_with_hint(json, hint, context);
391    }
392
393    if hint.nullable {
394        Ok(JsonValue::null())
395    } else {
396        error::InvalidJsonSnafu {
397            value: format!(
398                "missing non-null JSON2 type hint path {}",
399                hint.path.join(".")
400            ),
401        }
402        .fail()
403    }
404}
405
406fn encode_json_value_with_hint(
407    json: Json,
408    hint: &JsonTypeHint,
409    context: &mut JsonContext,
410) -> Result<JsonValue> {
411    if json.is_null() {
412        return if hint.nullable {
413            Ok(JsonValue::null())
414        } else {
415            error::InvalidJsonSnafu {
416                value: format!(
417                    "JSON2 type hint path {} is not nullable",
418                    context.path.join(".")
419                ),
420            }
421            .fail()
422        };
423    }
424
425    let invalid_type = || {
426        error::InvalidJsonSnafu {
427            value: format!(
428                "JSON value at {} does not match JSON2 type hint {}",
429                context.path.join("."),
430                hint.data_type
431            ),
432        }
433        .fail()
434    };
435
436    match (&hint.data_type, json) {
437        (ConcreteDataType::String(_), Json::String(v)) => Ok(v.into()),
438        (
439            ConcreteDataType::Int8(_)
440            | ConcreteDataType::Int16(_)
441            | ConcreteDataType::Int32(_)
442            | ConcreteDataType::Int64(_),
443            Json::Number(v),
444        ) => match v.as_i64() {
445            Some(v) => Ok(v.into()),
446            None => invalid_type(),
447        },
448        (
449            ConcreteDataType::UInt8(_)
450            | ConcreteDataType::UInt16(_)
451            | ConcreteDataType::UInt32(_)
452            | ConcreteDataType::UInt64(_),
453            Json::Number(v),
454        ) => match v.as_u64() {
455            Some(v) => Ok(v.into()),
456            None => invalid_type(),
457        },
458        (ConcreteDataType::Float32(_) | ConcreteDataType::Float64(_), Json::Number(v)) => {
459            match v.as_f64() {
460                Some(v) => Ok(v.into()),
461                None => invalid_type(),
462            }
463        }
464        (ConcreteDataType::Boolean(_), Json::Bool(v)) => Ok(v.into()),
465        _ => invalid_type(),
466    }
467}
468
469fn encode_json_array_with_context<'a>(
470    json_array: Vec<Json>,
471    context: &mut JsonContext<'a>,
472) -> Result<JsonValue> {
473    let json_array_len = json_array.len();
474    let mut items = Vec::with_capacity(json_array_len);
475
476    for (index, value) in json_array.into_iter().enumerate() {
477        let item_value = with_key_context(context, &index.to_string(), |context| {
478            encode_json_value_with_context(value, context)
479        })?;
480        items.push(item_value);
481    }
482
483    // In specification, it's valid for a JSON array to have different types of items, for example,
484    // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list
485    // array, which requires all items have exactly the same type. So we merge out the maybe
486    // different item types to a unified type, and align all the item values to it.
487
488    let merged_item_type = if let Some((first, rests)) = items.split_first() {
489        let mut merged = first.json_type().clone();
490        for rest in rests.iter().map(|x| x.json_type()) {
491            if matches!(merged, JsonNativeType::Variant) {
492                break;
493            }
494            merged.merge(rest);
495        }
496        Some(merged)
497    } else {
498        None
499    };
500    if let Some(unified_item_type) = merged_item_type {
501        for item in &mut items {
502            item.try_align(&unified_item_type)?;
503        }
504    }
505    let items = items
506        .into_iter()
507        .map(|x| x.into_variant())
508        .collect::<Vec<_>>();
509    Ok(JsonValue::new(JsonVariant::Array(items)))
510}
511
512/// Helper function to encode a JSON value to a Value and determine its ConcreteDataType with context
513fn encode_json_value_with_context(json: Json, context: &mut JsonContext) -> Result<JsonValue> {
514    if context.path.len() >= JSON2_MAX_STRUCTURED_DEPTH
515        && matches!(&json, Json::Object(_) | Json::Array(_))
516    {
517        return Ok(JsonValue::new(JsonVariant::Variant(
518            encode_serde_json_as_jsonb(json),
519        )));
520    }
521
522    match json {
523        Json::Null => Ok(JsonValue::null()),
524        Json::Bool(b) => Ok(b.into()),
525        Json::Number(n) => {
526            if let Some(i) = n.as_i64() {
527                Ok(i.into())
528            } else if let Some(u) = n.as_u64() {
529                if u <= i64::MAX as u64 {
530                    Ok((u as i64).into())
531                } else {
532                    Ok(u.into())
533                }
534            } else if let Some(f) = n.as_f64() {
535                Ok(f.into())
536            } else {
537                // Fallback to string representation
538                Ok(n.to_string().into())
539            }
540        }
541        Json::String(s) => Ok(s.into()),
542        Json::Array(arr) => encode_json_array_with_context(arr, context),
543        Json::Object(obj) => encode_json_object_with_context(obj, context),
544    }
545}
546
547/// Main decoding function with key path tracking
548fn decode_value_with_context(value: Value, context: &mut JsonContext) -> Result<Json> {
549    match value {
550        Value::Struct(struct_value) => decode_struct_with_context(struct_value, context),
551        Value::List(list_value) => decode_list_with_context(list_value, context),
552        _ => decode_primitive_value(value),
553    }
554}
555
556/// Decode a structured value to JSON object
557fn decode_struct_with_context<'a>(
558    struct_value: StructValue,
559    context: &mut JsonContext<'a>,
560) -> Result<Json> {
561    let mut json_object = Map::with_capacity(struct_value.len());
562
563    let (items, fields) = struct_value.into_parts();
564
565    for (field, field_value) in fields.fields().iter().zip(items) {
566        let json_value = with_key_context(context, field.name(), |context| {
567            decode_value_with_context(field_value, context)
568        })?;
569        json_object.insert(field.name().to_string(), json_value);
570    }
571
572    Ok(Json::Object(json_object))
573}
574
575/// Decode a list value to JSON array
576fn decode_list_with_context(list_value: ListValue, context: &mut JsonContext) -> Result<Json> {
577    let mut json_array = Vec::with_capacity(list_value.len());
578
579    let data_items = list_value.take_items();
580
581    for (index, item) in data_items.into_iter().enumerate() {
582        let json_value = with_key_context(context, &index.to_string(), |context| {
583            decode_value_with_context(item, context)
584        })?;
585        json_array.push(json_value);
586    }
587
588    Ok(Json::Array(json_array))
589}
590
591/// Decode primitive value to JSON
592fn decode_primitive_value(value: Value) -> Result<Json> {
593    match value {
594        Value::Null => Ok(Json::Null),
595        Value::Boolean(b) => Ok(Json::Bool(b)),
596        Value::UInt8(v) => Ok(Json::from(v)),
597        Value::UInt16(v) => Ok(Json::from(v)),
598        Value::UInt32(v) => Ok(Json::from(v)),
599        Value::UInt64(v) => Ok(Json::from(v)),
600        Value::Int8(v) => Ok(Json::from(v)),
601        Value::Int16(v) => Ok(Json::from(v)),
602        Value::Int32(v) => Ok(Json::from(v)),
603        Value::Int64(v) => Ok(Json::from(v)),
604        Value::Float32(v) => Ok(Json::from(v.0)),
605        Value::Float64(v) => Ok(Json::from(v.0)),
606        Value::String(s) => Ok(Json::String(s.as_utf8().to_string())),
607        Value::Binary(b) => serde_json::to_value(b.as_ref()).context(error::SerializeSnafu),
608        Value::Date(v) => Ok(Json::from(v.val())),
609        Value::Timestamp(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
610        Value::Time(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
611        Value::IntervalYearMonth(v) => {
612            serde_json::to_value(v.to_i32()).context(error::SerializeSnafu)
613        }
614        Value::IntervalDayTime(v) => {
615            serde_json::to_value(v.to_i64()).context(error::SerializeSnafu)
616        }
617        Value::IntervalMonthDayNano(v) => {
618            serde_json::to_value(v.to_i128()).context(error::SerializeSnafu)
619        }
620        Value::Duration(v) => serde_json::to_value(v.value()).context(error::SerializeSnafu),
621        Value::Decimal128(v) => serde_json::to_value(v.to_string()).context(error::SerializeSnafu),
622        Value::Struct(_) | Value::List(_) | Value::Json(_) => {
623            // These should be handled by the context-aware functions
624            Err(error::InvalidJsonSnafu {
625                value: "Structured values should be handled by context-aware decoding".to_string(),
626            }
627            .build())
628        }
629    }
630}
631
632#[cfg(test)]
633mod tests {
634    use std::sync::Arc;
635
636    use serde_json::json;
637
638    use super::*;
639    use crate::data_type::ConcreteDataType;
640    use crate::types::{ListType, StructField, StructType};
641
642    fn struct_field_value<'a>(struct_value: &'a StructValue, field_name: &str) -> &'a Value {
643        let index = struct_value
644            .struct_type()
645            .fields()
646            .iter()
647            .position(|field| field.name() == field_name)
648            .expect("field exists");
649        &struct_value.items()[index]
650    }
651
652    #[test]
653    fn test_json_settings_forward_compatibility()
654    -> std::result::Result<(), Box<dyn std::error::Error>> {
655        let json_str = r#"{
656            "type_hints": [
657                {
658                    "path": ["user", "age"],
659                    "type": {
660                        "Int64": {}
661                    },
662                    "nullable": false,
663                    "default_constraint": {
664                        "Value": {
665                            "Int64": 18
666                        }
667                    },
668                    "inverted_index": true
669                },
670                {
671                    "path": ["user", "name"],
672                    "type": {
673                        "String": {
674                            "size_type": "Utf8"
675                        }
676                    },
677                    "nullable": true,
678                    "inverted_index": false
679                }
680            ]
681        }"#;
682
683        let deserialized = serde_json::from_str::<JsonSettings>(json_str)?;
684
685        let expected = JsonSettings::try_new(
686            vec![
687                JsonTypeHint {
688                    path: vec!["user".to_string(), "age".to_string()],
689                    data_type: ConcreteDataType::int64_datatype(),
690                    nullable: false,
691                    default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
692                    inverted_index: true,
693                },
694                JsonTypeHint {
695                    path: vec!["user".to_string(), "name".to_string()],
696                    data_type: ConcreteDataType::string_datatype(),
697                    nullable: true,
698                    default_constraint: None,
699                    inverted_index: false,
700                },
701            ],
702            None,
703        )?;
704        assert_eq!(deserialized, expected);
705        Ok(())
706    }
707
708    #[test]
709    fn test_json_settings_ser_de() -> std::result::Result<(), Box<dyn std::error::Error>> {
710        let type_hints = vec![
711            JsonTypeHint {
712                path: vec!["user".to_string(), "age".to_string()],
713                data_type: ConcreteDataType::int64_datatype(),
714                nullable: false,
715                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(18))),
716                inverted_index: true,
717            },
718            JsonTypeHint {
719                path: vec!["user".to_string(), "name".to_string()],
720                data_type: ConcreteDataType::string_datatype(),
721                nullable: true,
722                default_constraint: None,
723                inverted_index: false,
724            },
725        ];
726
727        for settings in [
728            JsonSettings {
729                type_hints: type_hints.clone(),
730                max_auto_expanded_paths: None,
731            },
732            JsonSettings {
733                type_hints,
734                max_auto_expanded_paths: Some(1),
735            },
736        ] {
737            let serialized = serde_json::to_string(&settings)?;
738            let deserialized = serde_json::from_str::<JsonSettings>(&serialized)?;
739            assert_eq!(settings, deserialized);
740        }
741        Ok(())
742    }
743
744    #[test]
745    fn test_json_settings_reject_invalid_type_hints() {
746        for type_hints in [
747            json!([{"path": [], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}]),
748            json!([
749                {"path": ["a"], "type": {"Int64": {}}, "nullable": true, "inverted_index": false},
750                {"path": ["a", "b"], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}
751            ]),
752            json!([{"path": [JSON2_REMAINDER_FIELD_NAME], "type": {"Int64": {}}, "nullable": true, "inverted_index": false}]),
753        ] {
754            let settings = json!({"type_hints": type_hints});
755            assert!(serde_json::from_value::<JsonSettings>(settings).is_err());
756        }
757
758        let hint = |path, data_type| JsonTypeHint {
759            path,
760            data_type,
761            nullable: true,
762            default_constraint: None,
763            inverted_index: false,
764        };
765        assert!(
766            JsonSettings::try_new(
767                vec![hint(
768                    vec!["nested".to_string(), JSON2_REMAINDER_FIELD_NAME.to_string()],
769                    ConcreteDataType::string_datatype(),
770                )],
771                None,
772            )
773            .is_ok()
774        );
775        assert!(
776            JsonSettings::try_new(
777                vec![hint(
778                    vec!["nested".to_string(); JSON2_MAX_STRUCTURED_DEPTH + 1],
779                    ConcreteDataType::string_datatype(),
780                )],
781                None,
782            )
783            .is_err()
784        );
785        assert!(
786            JsonSettings::try_new(
787                vec![hint(
788                    vec!["date".to_string()],
789                    ConcreteDataType::date_datatype(),
790                )],
791                None,
792            )
793            .is_err()
794        );
795    }
796
797    #[test]
798    fn test_encode_root_non_object_json() {
799        let settings = JsonSettings::default();
800        let cases = [
801            ("null", Json::Null),
802            ("boolean", Json::Bool(true)),
803            ("integer", Json::from(42)),
804            ("float", Json::from(3.15)),
805            ("string", Json::String("hello".to_string())),
806            ("array", json!([1, 2, 3])),
807            ("mixed array", json!([1, "hello", true, 3.15])),
808            ("empty array", json!([])),
809        ];
810
811        for (name, json) in cases {
812            let err = settings.encode(json).unwrap_err();
813            assert!(
814                matches!(err, crate::error::Error::UnsupportedJsonType { .. }),
815                "{name}: {err:?}"
816            );
817        }
818    }
819
820    #[test]
821    fn test_encode_rejects_reserved_remainder_field() -> Result<()> {
822        let settings = JsonSettings::default();
823        let err = settings
824            .encode(json!({"!__remainder__!": "user-value"}))
825            .unwrap_err();
826        assert!(
827            err.to_string()
828                .contains("root object cannot contain reserved field")
829        );
830
831        settings.encode(json!({"nested": {"!__remainder__!": "user-value"}}))?;
832        Ok(())
833    }
834
835    #[test]
836    fn test_encode_json_object() {
837        let json = json!({
838            "name": "John",
839            "age": 30,
840            "active": true
841        });
842
843        let settings = JsonSettings::default();
844        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
845        let Value::Struct(result) = result else {
846            panic!("Expected Struct value");
847        };
848        assert_eq!(result.items().len(), 3);
849
850        let items = result.items();
851        let struct_type = result.struct_type();
852
853        // Check that we have the expected fields
854        let fields = struct_type.fields();
855        let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
856        assert!(field_names.contains(&"name"));
857        assert!(field_names.contains(&"age"));
858        assert!(field_names.contains(&"active"));
859
860        // Find and check each field
861        for (i, field) in struct_type.fields().iter().enumerate() {
862            match field.name() {
863                "name" => {
864                    assert_eq!(items[i], Value::String("John".into()));
865                    assert_eq!(field.data_type(), &ConcreteDataType::string_datatype());
866                }
867                "age" => {
868                    assert_eq!(items[i], Value::Int64(30));
869                    assert_eq!(field.data_type(), &ConcreteDataType::int64_datatype());
870                }
871                "active" => {
872                    assert_eq!(items[i], Value::Boolean(true));
873                    assert_eq!(field.data_type(), &ConcreteDataType::boolean_datatype());
874                }
875                _ => panic!("Unexpected field: {}", field.name()),
876            }
877        }
878    }
879
880    #[test]
881    fn test_encode_json_nested_object() {
882        let json = json!({
883            "person": {
884                "name": "Alice",
885                "age": 25
886            },
887            "scores": [95, 87, 92]
888        });
889
890        let settings = JsonSettings::default();
891        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
892        let Value::Struct(result) = result else {
893            panic!("Expected Struct value");
894        };
895        assert_eq!(result.items().len(), 2);
896
897        let items = result.items();
898        let struct_type = result.struct_type();
899
900        // Check person field (nested struct)
901        let person_index = struct_type
902            .fields()
903            .iter()
904            .position(|f| f.name() == "person")
905            .unwrap();
906        if let Value::Struct(person_struct) = &items[person_index] {
907            assert_eq!(person_struct.items().len(), 2);
908            let fields = person_struct.struct_type().fields();
909            let person_fields: Vec<&str> = fields.iter().map(|f| f.name()).collect();
910            assert!(person_fields.contains(&"name"));
911            assert!(person_fields.contains(&"age"));
912        } else {
913            panic!("Expected Struct value for person field");
914        }
915
916        // Check scores field (list)
917        let scores_index = struct_type
918            .fields()
919            .iter()
920            .position(|f| f.name() == "scores")
921            .unwrap();
922        if let Value::List(scores_list) = &items[scores_index] {
923            assert_eq!(scores_list.items().len(), 3);
924            assert_eq!(scores_list.items()[0], Value::Int64(95));
925            assert_eq!(scores_list.items()[1], Value::Int64(87));
926            assert_eq!(scores_list.items()[2], Value::Int64(92));
927        } else {
928            panic!("Expected List value for scores field");
929        }
930    }
931
932    #[test]
933    fn test_encode_json_structured() {
934        let json = json!({
935            "name": "Bob",
936            "age": 35
937        });
938
939        let settings = JsonSettings::default();
940        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
941
942        if let Value::Struct(struct_value) = result {
943            assert_eq!(struct_value.items().len(), 2);
944            let fields = struct_value.struct_type().fields();
945            let field_names: Vec<&str> = fields.iter().map(|f| f.name()).collect();
946            assert!(field_names.contains(&"name"));
947            assert!(field_names.contains(&"age"));
948        } else {
949            panic!("Expected Struct value");
950        }
951    }
952
953    #[test]
954    fn test_encode_json_respects_type_hint() -> Result<()> {
955        let type_hints = vec![JsonTypeHint {
956            path: vec!["age".to_string()],
957            data_type: ConcreteDataType::int64_datatype(),
958            nullable: false,
959            default_constraint: None,
960            inverted_index: false,
961        }];
962        let settings = JsonSettings::try_new(type_hints, None)?;
963
964        let result = settings
965            .encode(json!({
966                "name": "Alice",
967                "age": 42
968            }))
969            .unwrap()
970            .into_json_inner()
971            .unwrap();
972
973        let Value::Struct(struct_value) = result else {
974            panic!("Expected Struct value");
975        };
976        assert_eq!(struct_field_value(&struct_value, "age"), &Value::Int64(42));
977
978        let err = settings
979            .encode(json!({
980                "age": "42"
981            }))
982            .unwrap_err();
983        assert!(err.to_string().contains("does not match JSON2 type hint"));
984        Ok(())
985    }
986
987    #[test]
988    fn test_encode_json_respects_unsigned_type_hint() -> Result<()> {
989        let type_hints = vec![JsonTypeHint {
990            path: vec!["count".to_string()],
991            data_type: ConcreteDataType::uint64_datatype(),
992            nullable: false,
993            default_constraint: None,
994            inverted_index: false,
995        }];
996        let settings = JsonSettings::try_new(type_hints, None)?;
997
998        let result = settings
999            .encode(json!({
1000                "count": u64::MAX
1001            }))
1002            .unwrap()
1003            .into_json_inner()
1004            .unwrap();
1005
1006        let Value::Struct(struct_value) = result else {
1007            panic!("Expected Struct value");
1008        };
1009        assert_eq!(
1010            struct_field_value(&struct_value, "count"),
1011            &Value::UInt64(u64::MAX)
1012        );
1013
1014        let err = settings
1015            .encode(json!({
1016                "count": -1
1017            }))
1018            .unwrap_err();
1019        assert!(err.to_string().contains("does not match JSON2 type hint"));
1020        Ok(())
1021    }
1022
1023    #[test]
1024    fn test_encode_json_fills_missing_type_hint_with_default() -> Result<()> {
1025        let type_hints = vec![JsonTypeHint {
1026            path: vec!["user".to_string(), "age".to_string()],
1027            data_type: ConcreteDataType::int64_datatype(),
1028            nullable: false,
1029            default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
1030            inverted_index: false,
1031        }];
1032        let settings = JsonSettings::try_new(type_hints, None)?;
1033
1034        let result = settings
1035            .encode(json!({}))
1036            .unwrap()
1037            .into_json_inner()
1038            .unwrap();
1039
1040        let Value::Struct(root) = result else {
1041            panic!("Expected Struct value");
1042        };
1043        let Value::Struct(user) = struct_field_value(&root, "user") else {
1044            panic!("Expected user Struct value");
1045        };
1046        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
1047        Ok(())
1048    }
1049
1050    #[test]
1051    fn test_encode_json_fills_missing_nullable_type_hint_with_null() -> Result<()> {
1052        let type_hints = vec![JsonTypeHint {
1053            path: vec!["user".to_string(), "name".to_string()],
1054            data_type: ConcreteDataType::string_datatype(),
1055            nullable: true,
1056            default_constraint: None,
1057            inverted_index: false,
1058        }];
1059        let settings = JsonSettings::try_new(type_hints, None)?;
1060
1061        let result = settings
1062            .encode(json!({ "user": {} }))
1063            .unwrap()
1064            .into_json_inner()
1065            .unwrap();
1066
1067        let Value::Struct(root) = result else {
1068            panic!("Expected Struct value");
1069        };
1070        let Value::Struct(user) = struct_field_value(&root, "user") else {
1071            panic!("Expected user Struct value");
1072        };
1073        assert_eq!(struct_field_value(user, "name"), &Value::Null);
1074        Ok(())
1075    }
1076
1077    #[test]
1078    fn test_encode_json_rejects_missing_non_null_type_hint() -> Result<()> {
1079        let type_hints = vec![JsonTypeHint {
1080            path: vec!["user".to_string(), "age".to_string()],
1081            data_type: ConcreteDataType::int64_datatype(),
1082            nullable: false,
1083            default_constraint: None,
1084            inverted_index: false,
1085        }];
1086        let settings = JsonSettings::try_new(type_hints, None)?;
1087
1088        let err = settings.encode(json!({})).unwrap_err();
1089        assert!(
1090            err.to_string()
1091                .contains("missing non-null JSON2 type hint path user.age")
1092        );
1093        Ok(())
1094    }
1095
1096    #[test]
1097    fn test_encode_json_merges_missing_type_hint_prefix() -> Result<()> {
1098        let type_hints = vec![
1099            JsonTypeHint {
1100                path: vec!["user".to_string(), "age".to_string()],
1101                data_type: ConcreteDataType::int64_datatype(),
1102                nullable: false,
1103                default_constraint: Some(ColumnDefaultConstraint::Value(Value::Int64(7))),
1104                inverted_index: false,
1105            },
1106            JsonTypeHint {
1107                path: vec!["user".to_string(), "name".to_string()],
1108                data_type: ConcreteDataType::string_datatype(),
1109                nullable: false,
1110                default_constraint: Some(ColumnDefaultConstraint::Value(Value::String(
1111                    "unknown".into(),
1112                ))),
1113                inverted_index: false,
1114            },
1115        ];
1116        let settings = JsonSettings::try_new(type_hints, None)?;
1117
1118        let result = settings.encode(json!({}))?.into_json_inner().unwrap();
1119
1120        let Value::Struct(root) = result else {
1121            panic!("Expected Struct value");
1122        };
1123        let Value::Struct(user) = struct_field_value(&root, "user") else {
1124            panic!("Expected user Struct value");
1125        };
1126        assert_eq!(struct_field_value(user, "age"), &Value::Int64(7));
1127        assert_eq!(
1128            struct_field_value(user, "name"),
1129            &Value::String("unknown".into())
1130        );
1131        Ok(())
1132    }
1133
1134    #[test]
1135    fn test_json_settings_structured() {
1136        let json = json!({
1137            "name": "Eve",
1138            "score": 95
1139        });
1140
1141        let settings = JsonSettings::default();
1142        let result = settings.encode(json).unwrap().into_json_inner().unwrap();
1143
1144        if let Value::Struct(struct_value) = result {
1145            assert_eq!(struct_value.items().len(), 2);
1146        } else {
1147            panic!("Expected Struct value");
1148        }
1149    }
1150
1151    #[cfg(test)]
1152    mod decode_tests {
1153        use ordered_float::OrderedFloat;
1154        use serde_json::json;
1155
1156        use super::*;
1157
1158        #[test]
1159        fn test_decode_primitive_values() {
1160            let settings = JsonSettings::default();
1161
1162            // Test null
1163            let result = settings.decode(Value::Null).unwrap();
1164            assert_eq!(result, Json::Null);
1165
1166            // Test boolean
1167            let result = settings.decode(Value::Boolean(true)).unwrap();
1168            assert_eq!(result, Json::Bool(true));
1169
1170            // Test integer
1171            let result = settings.decode(Value::Int64(42)).unwrap();
1172            assert_eq!(result, Json::from(42));
1173
1174            // Test float
1175            let result = settings.decode(Value::Float64(OrderedFloat(3.16))).unwrap();
1176            assert_eq!(result, Json::from(3.16));
1177
1178            // Test string
1179            let result = settings.decode(Value::String("hello".into())).unwrap();
1180            assert_eq!(result, Json::String("hello".to_string()));
1181        }
1182
1183        #[test]
1184        fn test_decode_struct() {
1185            let settings = JsonSettings::default();
1186
1187            let struct_value = StructValue::new(
1188                vec![
1189                    Value::String("Alice".into()),
1190                    Value::Int64(25),
1191                    Value::Boolean(true),
1192                ],
1193                StructType::new(Arc::new(vec![
1194                    StructField::new(
1195                        "name".to_string(),
1196                        ConcreteDataType::string_datatype(),
1197                        true,
1198                    ),
1199                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1200                    StructField::new(
1201                        "active".to_string(),
1202                        ConcreteDataType::boolean_datatype(),
1203                        true,
1204                    ),
1205                ])),
1206            );
1207
1208            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1209            let expected = json!({
1210                "name": "Alice",
1211                "age": 25,
1212                "active": true
1213            });
1214            assert_eq!(result, expected);
1215        }
1216
1217        #[test]
1218        fn test_decode_list() {
1219            let settings = JsonSettings::default();
1220
1221            let list_value = ListValue::new(
1222                vec![Value::Int64(1), Value::Int64(2), Value::Int64(3)],
1223                Arc::new(ConcreteDataType::int64_datatype()),
1224            );
1225
1226            let result = settings.decode(Value::List(list_value)).unwrap();
1227            let expected = json!([1, 2, 3]);
1228            assert_eq!(result, expected);
1229        }
1230
1231        #[test]
1232        fn test_decode_nested_structure() {
1233            let settings = JsonSettings::default();
1234
1235            let inner_struct = StructValue::new(
1236                vec![Value::String("Alice".into()), Value::Int64(25)],
1237                StructType::new(Arc::new(vec![
1238                    StructField::new(
1239                        "name".to_string(),
1240                        ConcreteDataType::string_datatype(),
1241                        true,
1242                    ),
1243                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1244                ])),
1245            );
1246
1247            let score_list_item_type = Arc::new(ConcreteDataType::int64_datatype());
1248            let outer_struct = StructValue::new(
1249                vec![
1250                    Value::Struct(inner_struct),
1251                    Value::List(ListValue::new(
1252                        vec![Value::Int64(95), Value::Int64(87)],
1253                        score_list_item_type.clone(),
1254                    )),
1255                ],
1256                StructType::new(Arc::new(vec![
1257                    StructField::new(
1258                        "user".to_string(),
1259                        ConcreteDataType::Struct(StructType::new(Arc::new(vec![
1260                            StructField::new(
1261                                "name".to_string(),
1262                                ConcreteDataType::string_datatype(),
1263                                true,
1264                            ),
1265                            StructField::new(
1266                                "age".to_string(),
1267                                ConcreteDataType::int64_datatype(),
1268                                true,
1269                            ),
1270                        ]))),
1271                        true,
1272                    ),
1273                    StructField::new(
1274                        "scores".to_string(),
1275                        ConcreteDataType::List(ListType::new(score_list_item_type.clone())),
1276                        true,
1277                    ),
1278                ])),
1279            );
1280
1281            let result = settings.decode(Value::Struct(outer_struct)).unwrap();
1282            let expected = json!({
1283                "user": {
1284                    "name": "Alice",
1285                    "age": 25
1286                },
1287                "scores": [95, 87]
1288            });
1289            assert_eq!(result, expected);
1290        }
1291
1292        #[test]
1293        fn test_decode_missing_fields() {
1294            let settings = JsonSettings::default();
1295
1296            // Struct with missing field (null value)
1297            let struct_value = StructValue::new(
1298                vec![
1299                    Value::String("Bob".into()),
1300                    Value::Null, // missing age field
1301                ],
1302                StructType::new(Arc::new(vec![
1303                    StructField::new(
1304                        "name".to_string(),
1305                        ConcreteDataType::string_datatype(),
1306                        true,
1307                    ),
1308                    StructField::new("age".to_string(), ConcreteDataType::int64_datatype(), true),
1309                ])),
1310            );
1311
1312            let result = settings.decode(Value::Struct(struct_value)).unwrap();
1313            let expected = json!({
1314                "name": "Bob",
1315                "age": null
1316            });
1317            assert_eq!(result, expected);
1318        }
1319    }
1320}
datatypes/json.rs

datatypes/
json.rs
