datatypes/vectors/json/
builder.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16use std::collections::{BTreeMap, HashMap};
17use std::sync::Arc;
18
19use arrow_array::cast::AsArray;
20use arrow_array::{Array, ArrayRef, StructArray};
21use arrow_schema::DataType;
22use parquet_variant_compute::VariantArrayBuilder;
23use snafu::{ResultExt, ensure};
24
25use crate::data_type::ConcreteDataType;
26use crate::error::{
27    ArrowComputeSnafu, Result, TryFromValueSnafu, UnexpectedSnafu, UnimplementedSnafu,
28    UnsupportedOperationSnafu,
29};
30use crate::extension::json::JSON2_REMAINDER_FIELD_NAME;
31use crate::json::value::{JsonNumber, JsonVariant, JsonVariantRef, encode_json_variant};
32use crate::json::{
33    JSON2_DEFAULT_MAX_AUTO_EXPANDED_PATHS, JSON2_MAX_STRUCTURED_DEPTH, JsonSettings,
34};
35use crate::prelude::{ValueRef, Vector, VectorRef};
36use crate::types::StructType;
37use crate::types::json_type::{JsonNativeType, is_include};
38use crate::value::{ListValue, ListValueRef, StructValue, StructValueRef, Value};
39use crate::vectors::json::variant::{append_json_variant, append_json_variant_ref, variant_field};
40use crate::vectors::{Helper, MutableVector, NullVector, StructVectorBuilder};
41
42type JsonObjectValue = BTreeMap<String, JsonVariant>;
43
44/// Builds JSON2 vectors from object values.
45///
46/// Legacy mode merges all observed paths into the explicit Struct schema.
47/// Auto-expanding mode always materializes type-hinted paths, selects up to
48/// `max_auto_expanded_paths` compatible unhinted leaf paths by frequency, and stores
49/// conflicting or unselected paths in the Variant remainder field.
50pub(crate) struct JsonVectorBuilder {
51    state: JsonVectorBuilderState,
52}
53
54enum JsonVectorBuilderState {
55    Legacy {
56        merged_type: JsonNativeType,
57        values: Vec<JsonVariant>,
58    },
59    ExplicitOnly {
60        /// Paths declared by type hints and stored as dedicated Struct fields.
61        explicit_type: JsonNativeType,
62        /// Concrete Struct type used to append explicit values without buffering rows.
63        struct_type: StructType,
64        /// Builder for values selected by the explicit type hints.
65        explicit: StructVectorBuilder,
66        /// Builder for all values outside the explicit type hints.
67        remainder: VariantArrayBuilder,
68    },
69    AutoExpanding {
70        /// Paths declared by type hints and always stored as dedicated Struct fields.
71        explicit_type: JsonNativeType,
72        /// Maximum number of additional paths selected from buffered values.
73        max_auto_expanded_paths: u32,
74        /// Buffered values used to infer auto-expanded paths before building the vector.
75        values: Vec<JsonVariant>,
76    },
77}
78
79impl JsonVectorBuilderState {
80    fn native_type(&self) -> JsonNativeType {
81        match self {
82            Self::Legacy { merged_type, .. } => merged_type.clone(),
83            Self::ExplicitOnly { explicit_type, .. } => explicit_type.clone(),
84            Self::AutoExpanding {
85                explicit_type,
86                max_auto_expanded_paths,
87                values,
88            } => infer_expanded_type(explicit_type, *max_auto_expanded_paths, values),
89        }
90    }
91
92    fn len(&self) -> usize {
93        match self {
94            Self::Legacy { values, .. } | Self::AutoExpanding { values, .. } => values.len(),
95            Self::ExplicitOnly { explicit, .. } => explicit.len(),
96        }
97    }
98
99    fn try_build(&mut self) -> Result<VectorRef> {
100        match self {
101            Self::Legacy {
102                merged_type,
103                values,
104            } => build_legacy(values, merged_type),
105            Self::ExplicitOnly {
106                explicit,
107                remainder,
108                ..
109            } => {
110                let remainder = std::mem::replace(remainder, VariantArrayBuilder::new(0)).build();
111                finish_vector(explicit.to_vector(), ArrayRef::from(remainder))
112            }
113            Self::AutoExpanding {
114                explicit_type,
115                max_auto_expanded_paths,
116                values,
117            } => {
118                let expanded_type =
119                    infer_expanded_type(explicit_type, *max_auto_expanded_paths, values);
120                build_with_remainder(values, &expanded_type)
121            }
122        }
123    }
124
125    fn try_build_cloned(&self) -> Result<VectorRef> {
126        let mut state = match self {
127            Self::Legacy {
128                merged_type,
129                values,
130            } => Self::Legacy {
131                merged_type: merged_type.clone(),
132                values: values.clone(),
133            },
134            Self::AutoExpanding {
135                explicit_type,
136                max_auto_expanded_paths,
137                values,
138            } => Self::AutoExpanding {
139                explicit_type: explicit_type.clone(),
140                max_auto_expanded_paths: *max_auto_expanded_paths,
141                values: values.clone(),
142            },
143            // Only TimeSeriesMemtable requires a non-consuming snapshot, while JSON2 targets
144            // BulkMemtable. We've tried our best to support it above, but if this match arm does
145            // not, it's OK. The only reason it doesn't is because of `VariantArrayBuilder`. We'll
146            // track the upstream and see.
147            Self::ExplicitOnly { .. } => {
148                return UnimplementedSnafu {
149                    feat: "no auto expanded JSON2 array builder",
150                }
151                .fail();
152            }
153        };
154        state.try_build()
155    }
156
157    fn try_push_value_ref(&mut self, value: &ValueRef) -> Result<()> {
158        let ValueRef::Json(value) = value else {
159            return TryFromValueSnafu {
160                reason: format!("expected JSON value, got {value:?}"),
161            }
162            .fail();
163        };
164        ensure!(
165            value.is_object() || value.is_null(),
166            TryFromValueSnafu {
167                reason: format!("expected JSON object value, got {value:?}"),
168            }
169        );
170        match self {
171            Self::Legacy {
172                merged_type,
173                values,
174            } => {
175                let json_type = value.json_type();
176                if !is_include(merged_type, json_type.as_ref()) {
177                    merged_type.merge(json_type.as_ref());
178                }
179                values.push(JsonVariant::from(value.variant()));
180            }
181            Self::ExplicitOnly {
182                explicit_type,
183                struct_type,
184                explicit,
185                remainder,
186            } => {
187                if value.is_null() {
188                    explicit.push_null();
189                    remainder.append_null();
190                } else {
191                    let (value, rest) =
192                        split_to_explicit_ref(value.variant(), explicit_type, struct_type)?;
193                    explicit.push_struct_value_ref(value)?;
194                    append_json_variant_ref(remainder, &rest).context(ArrowComputeSnafu)?;
195                }
196            }
197            Self::AutoExpanding { values, .. } => {
198                values.push(JsonVariant::from(value.variant()));
199            }
200        }
201        Ok(())
202    }
203
204    fn push_null(&mut self) {
205        match self {
206            Self::Legacy { values, .. } | Self::AutoExpanding { values, .. } => {
207                values.push(JsonVariant::Null)
208            }
209            Self::ExplicitOnly {
210                explicit,
211                remainder,
212                ..
213            } => {
214                explicit.push_null();
215                remainder.append_null();
216            }
217        }
218    }
219}
220
221/// Returns the fixed v2 Arrow physical type produced from `settings`.
222pub fn json2_physical_data_type(settings: &JsonSettings) -> DataType {
223    let DataType::Struct(fields) = explicit_type(settings).as_arrow_type() else {
224        unreachable!("JSON2 explicit type must map to Arrow Struct")
225    };
226    let mut fields = fields
227        .iter()
228        .cloned()
229        .chain(std::iter::once(Arc::new(variant_field(
230            JSON2_REMAINDER_FIELD_NAME,
231            true,
232        ))))
233        .collect::<Vec<_>>();
234    fields.sort_unstable_by(|x, y| x.name().cmp(y.name()));
235    DataType::Struct(fields.into())
236}
237
238fn explicit_type(settings: &JsonSettings) -> JsonNativeType {
239    let mut explicit_type = JsonNativeType::Object(Default::default());
240    for hint in settings.type_hints() {
241        insert_dynamic_type(&mut explicit_type, &hint.path, (&hint.data_type).into());
242    }
243    explicit_type
244}
245
246impl JsonVectorBuilder {
247    /// Creates a builder that merges all observed paths into the explicit schema.
248    pub(crate) fn new(initial_native_type: JsonNativeType, capacity: usize) -> Self {
249        debug_assert!(matches!(
250            initial_native_type,
251            JsonNativeType::Object(_) | JsonNativeType::Null
252        ));
253        Self {
254            state: JsonVectorBuilderState::Legacy {
255                merged_type: initial_native_type,
256                values: Vec::with_capacity(capacity),
257            },
258        }
259    }
260
261    /// Creates a builder bounded by the JSON settings and their type hints.
262    pub(crate) fn with_settings(settings: &JsonSettings, capacity: usize) -> Self {
263        let explicit_type = explicit_type(settings);
264        let state = if settings.max_auto_expanded_paths() == Some(0) {
265            let DataType::Struct(fields) = explicit_type.as_arrow_type() else {
266                unreachable!("JSON2 explicit type must map to Arrow Struct")
267            };
268            let struct_type = StructType::from(&fields);
269            JsonVectorBuilderState::ExplicitOnly {
270                explicit_type,
271                explicit: StructVectorBuilder::with_type_and_capacity(
272                    struct_type.clone(),
273                    capacity,
274                ),
275                struct_type,
276                remainder: VariantArrayBuilder::new(capacity),
277            }
278        } else {
279            JsonVectorBuilderState::AutoExpanding {
280                explicit_type,
281                max_auto_expanded_paths: settings
282                    .max_auto_expanded_paths()
283                    .unwrap_or(JSON2_DEFAULT_MAX_AUTO_EXPANDED_PATHS),
284                values: Vec::with_capacity(capacity),
285            }
286        };
287        Self { state }
288    }
289
290    fn try_build(&mut self) -> Result<VectorRef> {
291        self.state.try_build()
292    }
293}
294
295fn build_legacy(values: &mut Vec<JsonVariant>, merged_type: &JsonNativeType) -> Result<VectorRef> {
296    build_explicit(values, merged_type, false, |value| match value {
297        JsonVariant::Null => Ok(None),
298        JsonVariant::Object(value) => Ok(Some(value)),
299        _ => TryFromValueSnafu {
300            reason: "expected json object value".to_string(),
301        }
302        .fail(),
303    })
304}
305
306fn build_with_remainder(
307    values: &mut Vec<JsonVariant>,
308    expanded_type: &JsonNativeType,
309) -> Result<VectorRef> {
310    let mut remainder = VariantArrayBuilder::new(values.len());
311    let explicit = build_explicit(values, expanded_type, true, |value| {
312        if matches!(value, JsonVariant::Null) {
313            remainder.append_null();
314            return Ok(None);
315        }
316        let (value, rest) = split_to_explicit(value, expanded_type)?;
317        append_json_variant(&mut remainder, &JsonVariant::Object(rest))
318            .context(ArrowComputeSnafu)?;
319        Ok(Some(value))
320    })?;
321    finish_vector(explicit, ArrayRef::from(remainder.build()))
322}
323
324fn build_explicit(
325    values: &mut Vec<JsonVariant>,
326    explicit_type: &JsonNativeType,
327    // Temporary compatibility switch for the legacy storage layout. Once JSON2 fully switches to
328    // the v2 storage layout, empty objects should always be preserved instead of treated as null.
329    preserve_empty_structs: bool,
330    mut project: impl FnMut(JsonVariant) -> Result<Option<JsonObjectValue>>,
331) -> Result<VectorRef> {
332    let DataType::Struct(fields) = explicit_type.as_arrow_type() else {
333        return UnexpectedSnafu {
334            reason: "merged JSON2 type must map to Arrow Struct in JsonVectorBuilder",
335        }
336        .fail();
337    };
338    // TODO(LFC): Direct use Arrow's Struct datatype here.
339    let struct_type = StructType::from(&fields);
340
341    let mut builder =
342        StructVectorBuilder::with_type_and_capacity(struct_type.clone(), values.len());
343    for value in std::mem::take(values) {
344        let Some(value) = project(value)? else {
345            builder.push_null();
346            continue;
347        };
348        let value =
349            json_variant_into_struct_value(value, struct_type.clone(), preserve_empty_structs)?;
350        builder.push_struct_value_ref(StructValueRef::Ref(&value))?;
351    }
352    Ok(builder.to_vector())
353}
354
355fn finish_vector(explicit: VectorRef, remainder: ArrayRef) -> Result<VectorRef> {
356    let explicit = explicit.to_arrow_array();
357    let explicit = explicit.as_struct();
358    let mut children = explicit
359        .fields()
360        .iter()
361        .cloned()
362        .zip(explicit.columns().iter().cloned())
363        .chain(std::iter::once((
364            Arc::new(variant_field(JSON2_REMAINDER_FIELD_NAME, true)),
365            remainder,
366        )))
367        .collect::<Vec<_>>();
368    children.sort_unstable_by(|(x, _), (y, _)| x.name().cmp(y.name()));
369    let (fields, columns): (Vec<_>, Vec<ArrayRef>) = children.into_iter().unzip();
370    let array: ArrayRef = Arc::new(StructArray::new(
371        fields.into(),
372        columns,
373        explicit.nulls().cloned(),
374    ));
375    Helper::try_into_vector(array)
376}
377
378fn infer_expanded_type(
379    explicit_type: &JsonNativeType,
380    max_auto_expanded_paths: u32,
381    values: &[JsonVariant],
382) -> JsonNativeType {
383    if max_auto_expanded_paths == 0 {
384        return explicit_type.clone();
385    }
386
387    let mut stats = HashMap::new();
388    let mut path = Vec::new();
389    init_explicit_path_stats(explicit_type, &mut path, &mut stats);
390    for value in values {
391        count_dynamic_paths(value, &mut path, &mut stats);
392    }
393    let mut candidates = stats
394        .iter()
395        // Explicit paths are already in the output schema and do not consume the dynamic
396        // expansion budget.
397        .filter(|(_, stat)| !stat.is_explicit && stat.is_leaf)
398        // A leaf is eligible only when both itself and every object prefix have one stable
399        // role and type across all observed values.
400        .filter(|(path, _)| {
401            !(1..=path.len())
402                .any(|len| stats.get(&path[..len]).is_some_and(|stats| stats.conflicts))
403        })
404        .collect::<Vec<_>>();
405    candidates.sort_unstable_by(|(x_path, x), (y_path, y)| {
406        y.seen_count
407            .cmp(&x.seen_count)
408            .then_with(|| x_path.cmp(y_path))
409    });
410
411    let mut expanded_type = explicit_type.clone();
412    for (path, candidate) in candidates
413        .into_iter()
414        .take(max_auto_expanded_paths as usize)
415    {
416        insert_dynamic_type(
417            &mut expanded_type,
418            path,
419            candidate.expected_leaf_type.clone(),
420        );
421    }
422    expanded_type
423}
424
425/// Aggregated observations for one JSON path.
426///
427/// Schema inference first seeds the map with explicit paths, then walks all input values once.
428/// Objects, including empty objects, are non-leaf paths; every other non-null value is a leaf.
429/// The first dynamic observation fixes the path role and exact leaf type. A later role or type
430/// mismatch sets [`PathStats::conflicts`] permanently. Missing paths and null values do not affect
431/// the statistics.
432///
433/// After collection, dynamic leaves are ranked by [`PathStats::seen_count`]. A candidate is
434/// rejected when it or any parent path conflicts, so candidate selection never rescans the input
435/// values.
436struct PathStats {
437    /// Whether the path came from a type hint and is already part of the output schema.
438    is_explicit: bool,
439    /// Whether the path is a non-object value rather than an object prefix.
440    is_leaf: bool,
441    /// Exact type required for a leaf; unused non-leaf paths keep [`JsonNativeType::Null`].
442    expected_leaf_type: JsonNativeType,
443    /// Number of compatible observations used to rank dynamic leaves.
444    seen_count: usize,
445    /// Whether the path has ever had inconsistent roles or leaf types.
446    conflicts: bool,
447}
448
449/// Seeds path statistics from the configured explicit JSON shape.
450fn init_explicit_path_stats<'a>(
451    explicit_type: &'a JsonNativeType,
452    path: &mut Vec<&'a str>,
453    stats: &mut HashMap<Vec<&'a str>, PathStats>,
454) {
455    let JsonNativeType::Object(fields) = explicit_type else {
456        return;
457    };
458    for (name, data_type) in fields {
459        path.push(name);
460        let is_leaf = !matches!(data_type, JsonNativeType::Object(_));
461        let expected_leaf_type = if is_leaf {
462            data_type.clone()
463        } else {
464            JsonNativeType::default()
465        };
466        stats.insert(
467            path.clone(),
468            PathStats {
469                is_explicit: true,
470                is_leaf,
471                expected_leaf_type,
472                seen_count: 0,
473                conflicts: false,
474            },
475        );
476        init_explicit_path_stats(data_type, path, stats);
477        path.pop();
478    }
479}
480
481/// Collects dynamic path statistics while traversing each input value once.
482fn count_dynamic_paths<'a>(
483    value: &'a JsonVariant,
484    path: &mut Vec<&'a str>,
485    stats: &mut HashMap<Vec<&'a str>, PathStats>,
486) {
487    if matches!(value, JsonVariant::Null) || path.len() > JSON2_MAX_STRUCTURED_DEPTH {
488        return;
489    }
490
491    if !path.is_empty() {
492        let is_leaf = !matches!(value, JsonVariant::Object(_));
493        let conflicts = if let Some(stats) = stats.get_mut(path.as_slice()) {
494            if !stats.conflicts {
495                let role_conflict = stats.is_leaf != is_leaf;
496                let type_conflict = || match (&stats.expected_leaf_type, value) {
497                    // If both objects, they are compatible.
498                    (JsonNativeType::Null | JsonNativeType::Object(_), JsonVariant::Object(_)) => {
499                        false
500                    }
501                    _ => stats.expected_leaf_type != value.native_type(),
502                };
503                if role_conflict || type_conflict() {
504                    stats.conflicts = true;
505                } else {
506                    stats.seen_count += 1;
507                }
508            }
509            stats.conflicts
510        } else {
511            let expected_leaf_type = if is_leaf {
512                value.native_type()
513            } else {
514                JsonNativeType::default()
515            };
516            stats.insert(
517                path.clone(),
518                PathStats {
519                    is_explicit: false,
520                    is_leaf,
521                    expected_leaf_type,
522                    seen_count: 1,
523                    conflicts: false,
524                },
525            );
526            false
527        };
528        if conflicts {
529            return;
530        }
531    }
532
533    if let JsonVariant::Object(object) = value
534        && !object.is_empty()
535    {
536        for (name, value) in object {
537            path.push(name);
538            count_dynamic_paths(value, path, stats);
539            path.pop();
540        }
541    }
542}
543
544fn insert_dynamic_type<S: AsRef<str>>(
545    explicit_type: &mut JsonNativeType,
546    path: &[S],
547    data_type: JsonNativeType,
548) {
549    let JsonNativeType::Object(fields) = explicit_type else {
550        return;
551    };
552    let Some((name, path)) = path.split_first() else {
553        return;
554    };
555    let name = name.as_ref().to_string();
556    if path.is_empty() {
557        fields.insert(name, data_type);
558        return;
559    }
560    insert_dynamic_type(
561        fields
562            .entry(name)
563            .or_insert_with(|| JsonNativeType::Object(Default::default())),
564        path,
565        data_type,
566    )
567}
568
569fn split_to_explicit_ref<'a>(
570    value: &JsonVariantRef<'a>,
571    explicit_type: &JsonNativeType,
572    struct_type: &StructType,
573) -> Result<(StructValueRef<'a>, JsonVariantRef<'a>)> {
574    let JsonVariantRef::Object(object) = value else {
575        return TryFromValueSnafu {
576            reason: "expected json object value".to_string(),
577        }
578        .fail();
579    };
580    let explicit = json_object_ref_into_struct_value_ref(object, struct_type)?;
581    let remainder = remainder_ref(object, explicit_type)?;
582    Ok((explicit, JsonVariantRef::Object(remainder)))
583}
584
585fn json_object_ref_into_struct_value_ref<'a>(
586    object: &BTreeMap<&'a str, JsonVariantRef<'a>>,
587    struct_type: &StructType,
588) -> Result<StructValueRef<'a>> {
589    let mut values = Vec::with_capacity(struct_type.fields().len());
590    for field in struct_type.fields().iter() {
591        let value = match object.get(field.name()) {
592            Some(value) => json_variant_ref_into_value_ref(value, field.data_type())?,
593            None => ValueRef::Null,
594        };
595        values.push(value);
596    }
597    Ok(StructValueRef::RefList {
598        val: values,
599        fields: struct_type.clone(),
600    })
601}
602
603fn json_variant_ref_into_value_ref<'a>(
604    value: &JsonVariantRef<'a>,
605    expected_type: &ConcreteDataType,
606) -> Result<ValueRef<'a>> {
607    let value = match (value, expected_type) {
608        (JsonVariantRef::Null, _) | (_, ConcreteDataType::Null(_)) => ValueRef::Null,
609        (JsonVariantRef::Object(object), ConcreteDataType::Struct(struct_type)) => {
610            ValueRef::Struct(json_object_ref_into_struct_value_ref(object, struct_type)?)
611        }
612        (JsonVariantRef::Bool(x), ConcreteDataType::Boolean(_)) => ValueRef::Boolean(*x),
613        (JsonVariantRef::Number(x), ConcreteDataType::UInt64(_)) => {
614            let Some(x) = x.as_u64() else {
615                return TryFromValueSnafu {
616                    reason: format!("unable to convert {x:?} to UInt64"),
617                }
618                .fail();
619            };
620            ValueRef::UInt64(x)
621        }
622        (JsonVariantRef::Number(x), ConcreteDataType::Int64(_)) => {
623            let x = match x {
624                JsonNumber::PosInt(x) => i64::try_from(*x).ok(),
625                JsonNumber::NegInt(x) => Some(*x),
626                JsonNumber::Float(_) => None,
627            };
628            let Some(x) = x else {
629                return TryFromValueSnafu {
630                    reason: format!("unable to convert {x:?} to Int64"),
631                }
632                .fail();
633            };
634            ValueRef::Int64(x)
635        }
636        (JsonVariantRef::Number(JsonNumber::PosInt(x)), ConcreteDataType::Float64(_)) => {
637            ValueRef::Float64((*x as f64).into())
638        }
639        (JsonVariantRef::Number(JsonNumber::NegInt(x)), ConcreteDataType::Float64(_)) => {
640            ValueRef::Float64((*x as f64).into())
641        }
642        (JsonVariantRef::Number(JsonNumber::Float(x)), ConcreteDataType::Float64(_)) => {
643            ValueRef::Float64(*x)
644        }
645        (JsonVariantRef::String(x), ConcreteDataType::String(_)) => ValueRef::String(x),
646        (JsonVariantRef::Array(array), ConcreteDataType::List(list_type)) => {
647            let item_type = list_type.item_type().clone();
648            let values = array
649                .iter()
650                .map(|x| json_variant_ref_into_value_ref(x, &item_type))
651                .collect::<Result<Vec<_>>>()?;
652            ValueRef::List(ListValueRef::RefList {
653                val: values,
654                item_datatype: Arc::new(item_type),
655            })
656        }
657        (value, expected_type) => {
658            return TryFromValueSnafu {
659                reason: format!("unable to convert json value {value:?} to {expected_type}"),
660            }
661            .fail();
662        }
663    };
664    Ok(value)
665}
666
667fn remainder_ref<'a>(
668    object: &BTreeMap<&'a str, JsonVariantRef<'a>>,
669    explicit_type: &JsonNativeType,
670) -> Result<BTreeMap<&'a str, JsonVariantRef<'a>>> {
671    let JsonNativeType::Object(fields) = explicit_type else {
672        return UnexpectedSnafu {
673            reason: "JSON2 explicit type must be an object",
674        }
675        .fail();
676    };
677    let mut remainder = BTreeMap::new();
678    for (&name, value) in object {
679        // Preserve explicit JSON nulls in the remainder because Arrow child nulls cannot
680        // distinguish a present JSON null from a missing path.
681        if *value == JsonVariantRef::Null {
682            remainder.insert(name, JsonVariantRef::Null);
683            continue;
684        }
685
686        match fields.get(name) {
687            Some(data_type @ JsonNativeType::Object(_)) => match value {
688                JsonVariantRef::Object(object) => {
689                    let child = remainder_ref(object, data_type)?;
690                    if !child.is_empty() {
691                        remainder.insert(name, JsonVariantRef::Object(child));
692                    }
693                }
694                _ => {
695                    return TryFromValueSnafu {
696                        reason: "expected json object value".to_string(),
697                    }
698                    .fail();
699                }
700            },
701            // A non-object entry in the explicit type tree is an explicit leaf and is
702            // already written to the Struct builder. So here does nothing.
703            Some(_) => {}
704            None => {
705                remainder.insert(name, value.clone());
706            }
707        }
708    }
709    Ok(remainder)
710}
711
712fn split_to_explicit(
713    value: JsonVariant,
714    explicit_type: &JsonNativeType,
715) -> Result<(JsonObjectValue, JsonObjectValue)> {
716    let JsonVariant::Object(mut remainder) = value else {
717        return TryFromValueSnafu {
718            reason: "expected json object value".to_string(),
719        }
720        .fail();
721    };
722    let JsonNativeType::Object(fields) = explicit_type else {
723        return UnexpectedSnafu {
724            reason: "JSON2 explicit type must be an object",
725        }
726        .fail();
727    };
728    let mut explicit = JsonObjectValue::new();
729
730    for (name, data_type) in fields {
731        let Some(value) = remainder.remove(name) else {
732            continue;
733        };
734        if value == JsonVariant::Null {
735            explicit.insert(name.clone(), JsonVariant::Null);
736            // Preserve explicit JSON nulls in the remainder because Arrow child nulls cannot
737            // distinguish a present JSON null from a missing path.
738            remainder.insert(name.clone(), JsonVariant::Null);
739            continue;
740        }
741        if matches!(data_type, JsonNativeType::Object(_)) {
742            let (child_explicit, child_remainder) = split_to_explicit(value, data_type)?;
743            explicit.insert(name.clone(), JsonVariant::Object(child_explicit));
744            if !child_remainder.is_empty() {
745                remainder.insert(name.clone(), JsonVariant::Object(child_remainder));
746            }
747        } else {
748            explicit.insert(name.clone(), value);
749        }
750    }
751    Ok((explicit, remainder))
752}
753
754fn json_variant_into_struct_value(
755    object: JsonObjectValue,
756    struct_type: StructType,
757    preserve_empty_structs: bool,
758) -> Result<StructValue> {
759    let mut entries = object.into_iter();
760    let mut entry = entries.next();
761    let mut values = Vec::with_capacity(struct_type.fields().len());
762    for field in struct_type.fields().iter() {
763        let value = match entry.take() {
764            Some((name, value)) if name == field.name() => {
765                entry = entries.next();
766                json_variant_into_value(value, field.data_type(), preserve_empty_structs)?
767            }
768            Some((name, _)) if name.as_str() < field.name() => {
769                return TryFromValueSnafu {
770                    reason: format!("field {name} is missing from merged JSON type"),
771                }
772                .fail();
773            }
774            next => {
775                entry = next;
776                Value::Null
777            }
778        };
779        values.push(value);
780    }
781    if let Some((name, _)) = entry {
782        return TryFromValueSnafu {
783            reason: format!("field {name} is missing from merged JSON type"),
784        }
785        .fail();
786    }
787
788    Ok(StructValue::new(values, struct_type))
789}
790
791fn json_variant_into_value(
792    value: JsonVariant,
793    expected_type: &ConcreteDataType,
794    preserve_empty_structs: bool,
795) -> Result<Value> {
796    let value = match (value, expected_type) {
797        (JsonVariant::Null, _) | (_, ConcreteDataType::Null(_)) => Value::Null,
798        (JsonVariant::Object(object), _) if object.is_empty() && !preserve_empty_structs => {
799            Value::Null
800        }
801        (JsonVariant::Object(object), ConcreteDataType::Struct(struct_type)) => Value::Struct(
802            json_variant_into_struct_value(object, struct_type.clone(), preserve_empty_structs)?,
803        ),
804        (JsonVariant::Bool(x), ConcreteDataType::Boolean(_)) => Value::Boolean(x),
805        (JsonVariant::Number(x), ConcreteDataType::UInt64(_)) => {
806            let Some(x) = x.as_u64() else {
807                return TryFromValueSnafu {
808                    reason: format!("unable to convert {x:?} to UInt64"),
809                }
810                .fail();
811            };
812            Value::UInt64(x)
813        }
814        (JsonVariant::Number(x), ConcreteDataType::Int64(_)) => {
815            let x = match x {
816                JsonNumber::PosInt(x) => i64::try_from(x).ok(),
817                JsonNumber::NegInt(x) => Some(x),
818                JsonNumber::Float(_) => None,
819            };
820            let Some(x) = x else {
821                return TryFromValueSnafu {
822                    reason: format!("unable to convert {x:?} to Int64"),
823                }
824                .fail();
825            };
826            Value::Int64(x)
827        }
828        (JsonVariant::Number(JsonNumber::PosInt(x)), ConcreteDataType::Float64(_)) => {
829            Value::Float64((x as f64).into())
830        }
831        (JsonVariant::Number(JsonNumber::NegInt(x)), ConcreteDataType::Float64(_)) => {
832            Value::Float64((x as f64).into())
833        }
834        (JsonVariant::Number(JsonNumber::Float(x)), ConcreteDataType::Float64(_)) => {
835            Value::Float64(x)
836        }
837        (JsonVariant::String(x), ConcreteDataType::String(_)) => Value::String(x.into()),
838        (JsonVariant::Array(array), ConcreteDataType::List(list_type)) => {
839            let item_type = list_type.item_type().clone();
840            let values = array
841                .into_iter()
842                .map(|v| json_variant_into_value(v, &item_type, preserve_empty_structs))
843                .collect::<Result<Vec<_>>>()?;
844            Value::List(ListValue::new(values, Arc::new(item_type)))
845        }
846        (value, ConcreteDataType::Binary(_)) => Value::from(encode_json_variant(value)?),
847        (value, expected_type) => {
848            return TryFromValueSnafu {
849                reason: format!("unable to convert json value {value:?} to {expected_type}"),
850            }
851            .fail();
852        }
853    };
854    Ok(value)
855}
856
857impl MutableVector for JsonVectorBuilder {
858    fn data_type(&self) -> ConcreteDataType {
859        ConcreteDataType::json2(self.state.native_type())
860    }
861
862    fn len(&self) -> usize {
863        self.state.len()
864    }
865
866    fn as_any(&self) -> &dyn Any {
867        self
868    }
869
870    fn as_mut_any(&mut self) -> &mut dyn Any {
871        self
872    }
873
874    fn to_vector(&mut self) -> VectorRef {
875        self.try_build().unwrap_or_else(|e| {
876            // Just try to avoid panicking here.
877            common_telemetry::error!(e; "Unable to build JSON2 vector");
878            Arc::new(NullVector::new(self.len()))
879        })
880    }
881
882    fn to_vector_cloned(&self) -> VectorRef {
883        self.state.try_build_cloned().unwrap_or_else(|e| {
884            // Just try to avoid panicking here.
885            common_telemetry::error!(e; "Unable to build JSON2 vector");
886            Arc::new(NullVector::new(self.len()))
887        })
888    }
889
890    fn try_push_value_ref(&mut self, value: &ValueRef) -> Result<()> {
891        self.state.try_push_value_ref(value)
892    }
893
894    fn push_null(&mut self) {
895        self.state.push_null()
896    }
897
898    fn extend_slice_of(&mut self, _: &dyn Vector, _: usize, _: usize) -> Result<()> {
899        UnsupportedOperationSnafu {
900            op: "extend_slice_of",
901            vector_type: "JsonVector",
902        }
903        .fail()
904    }
905}
906
907#[cfg(test)]
908mod tests {
909    use std::sync::Arc;
910
911    use arrow_array::cast::AsArray;
912    use arrow_schema::Field;
913    use common_base::bytes::Bytes;
914    use serde_json::json;
915
916    use super::*;
917    use crate::data_type::ConcreteDataType;
918    use crate::extension::json::{Json2ExtensionType, JsonMetadata};
919    use crate::json::JsonTypeHint;
920    use crate::json::value::decode_json_variant;
921    use crate::types::StructField;
922    use crate::types::json_type::JsonObjectType;
923    use crate::value::{ListValue, StructValue, Value, ValueRef};
924    use crate::vectors::json::array::JsonArray;
925    use crate::vectors::json::variant::variant_to_json_values;
926
927    #[test]
928    fn test_json_vector_builder() -> Result<()> {
929        fn parse_json_value(json: &str) -> Value {
930            let value: serde_json::Value = serde_json::from_str(json).unwrap();
931            Value::Json(Box::new(value.into()))
932        }
933
934        fn jsonb_bytes(json: &str) -> Bytes {
935            Bytes::from(jsonb::parse_value(json.as_bytes()).unwrap().to_vec())
936        }
937
938        // Object inputs should merge into a superset schema, preserve null rows,
939        // and project conflicting nested values into Variant payloads.
940        let mut builder = JsonVectorBuilder::new(JsonNativeType::Object(Default::default()), 3);
941        let first = parse_json_value(r#"{"id":1,"payload":{"name":"foo"}}"#);
942        let second = parse_json_value(r#"{"id":2,"extra":true,"payload":"raw"}"#);
943        builder.try_push_value_ref(&first.as_value_ref())?;
944        builder.push_null();
945        builder.try_push_value_ref(&second.as_value_ref())?;
946
947        let merged_type = JsonNativeType::Object(JsonObjectType::from([
948            ("extra".to_string(), JsonNativeType::Bool),
949            ("id".to_string(), JsonNativeType::i64()),
950            ("payload".to_string(), JsonNativeType::Variant),
951        ]));
952        assert_eq!(
953            builder.data_type(),
954            ConcreteDataType::json2(merged_type.clone())
955        );
956
957        let DataType::Struct(fields) = merged_type.as_arrow_type() else {
958            unreachable!()
959        };
960        let merged_struct_type = StructType::from(&fields);
961        let vector = builder.to_vector();
962        assert_eq!(vector.len(), 3);
963        assert_eq!(
964            vector.get(0),
965            Value::Struct(StructValue::new(
966                vec![
967                    Value::Null,
968                    Value::Int64(1),
969                    Value::Binary(jsonb_bytes(r#"{"name":"foo"}"#)),
970                ],
971                merged_struct_type.clone(),
972            ))
973        );
974        assert_eq!(vector.get(1), Value::Null);
975        assert_eq!(
976            vector.get(2),
977            Value::Struct(StructValue::new(
978                vec![
979                    Value::Boolean(true),
980                    Value::Int64(2),
981                    Value::Binary(jsonb_bytes(r#""raw""#)),
982                ],
983                merged_struct_type,
984            ))
985        );
986
987        // A Null initial type represents an unknown JSON2 runtime type. The first
988        // non-null value should set the concrete type instead of aligning all rows to Null.
989        let mut inferred_builder = JsonVectorBuilder::new(JsonNativeType::Null, 2);
990        let inferred_value = parse_json_value(r#"{"id":3}"#);
991        inferred_builder.push_null();
992        inferred_builder.try_push_value_ref(&inferred_value.as_value_ref())?;
993
994        let inferred_type = JsonNativeType::Object(JsonObjectType::from([(
995            "id".to_string(),
996            JsonNativeType::i64(),
997        )]));
998        assert_eq!(
999            inferred_builder.data_type(),
1000            ConcreteDataType::json2(inferred_type.clone())
1001        );
1002
1003        let DataType::Struct(fields) = inferred_type.as_arrow_type() else {
1004            unreachable!()
1005        };
1006        let inferred_struct_type = StructType::from(&fields);
1007        let vector = inferred_builder.to_vector();
1008        assert_eq!(vector.get(0), Value::Null);
1009        assert_eq!(
1010            vector.get(1),
1011            Value::Struct(StructValue::new(
1012                vec![Value::Int64(3)],
1013                inferred_struct_type,
1014            ))
1015        );
1016
1017        // Non-object initial types are rejected by the builder invariant.
1018        let result = std::panic::catch_unwind(|| JsonVectorBuilder::new(JsonNativeType::Bool, 2));
1019        assert!(result.is_err());
1020
1021        // Non-object root values should be rejected at push time.
1022        let mut object_builder =
1023            JsonVectorBuilder::new(JsonNativeType::Object(Default::default()), 2);
1024        let object = parse_json_value(r#"{"k":1}"#);
1025        let boolean = parse_json_value("true");
1026        let err = object_builder
1027            .try_push_value_ref(&boolean.as_value_ref())
1028            .unwrap_err();
1029        assert!(err.to_string().contains("expected JSON object value"));
1030        object_builder.try_push_value_ref(&object.as_value_ref())?;
1031
1032        // Non-JSON values should be rejected at push time.
1033        let mut invalid_builder =
1034            JsonVectorBuilder::new(JsonNativeType::Object(Default::default()), 1);
1035        let err = invalid_builder
1036            .try_push_value_ref(&ValueRef::Boolean(true))
1037            .unwrap_err();
1038        assert!(err.to_string().contains("expected JSON value"));
1039
1040        Ok(())
1041    }
1042
1043    #[test]
1044    fn test_zero_budget_builder_uses_explicit_only_schema_and_remainder() -> Result<()> {
1045        let settings = JsonSettings::try_new(
1046            vec![
1047                JsonTypeHint {
1048                    path: vec!["kind".to_string()],
1049                    data_type: ConcreteDataType::string_datatype(),
1050                    nullable: true,
1051                    default_constraint: None,
1052                    inverted_index: false,
1053                },
1054                JsonTypeHint {
1055                    path: vec!["commit".to_string(), "operation".to_string()],
1056                    data_type: ConcreteDataType::string_datatype(),
1057                    nullable: true,
1058                    default_constraint: None,
1059                    inverted_index: false,
1060                },
1061                JsonTypeHint {
1062                    path: vec!["time_us".to_string()],
1063                    data_type: ConcreteDataType::int64_datatype(),
1064                    nullable: true,
1065                    default_constraint: None,
1066                    inverted_index: false,
1067                },
1068            ],
1069            Some(0),
1070        )?;
1071        let mut builder = JsonVectorBuilder::with_settings(&settings, 2);
1072        assert!(matches!(
1073            &builder.state,
1074            JsonVectorBuilderState::ExplicitOnly { .. }
1075        ));
1076        let values = [
1077            json!({
1078                "kind": "record",
1079                "commit": {"operation": "create", "collection": "post"},
1080                "extra": 1,
1081                "time_us": 1
1082            }),
1083            json!({"kind": "other", "dynamic": true, "time_us": 2}),
1084        ];
1085        for value in values.clone() {
1086            let value = settings.encode(value)?;
1087            builder.try_push_value_ref(&value.as_value_ref())?;
1088        }
1089        let array = builder.to_vector().to_arrow_array();
1090        assert_eq!(&json2_physical_data_type(&settings), array.data_type());
1091        assert_eq!(0, builder.len());
1092        let structs = array.as_struct();
1093        assert_eq!(
1094            vec![JSON2_REMAINDER_FIELD_NAME, "commit", "kind", "time_us"],
1095            structs
1096                .fields()
1097                .iter()
1098                .map(|x| x.name().as_str())
1099                .collect::<Vec<_>>()
1100        );
1101        assert_eq!(
1102            vec![
1103                Some(json!({"commit": {"collection": "post"}, "extra": 1})),
1104                Some(json!({"commit": {"operation": null}, "dynamic": true})),
1105            ],
1106            variant_to_json_values(structs.column_by_name(JSON2_REMAINDER_FIELD_NAME).unwrap())?
1107        );
1108
1109        let field = Field::new("data", array.data_type().clone(), true).with_extension_type(
1110            Json2ExtensionType::new(Arc::new(JsonMetadata::new(settings))),
1111        );
1112        let reconstructed = JsonArray::from(&array).project_to_v2(&field, &DataType::Binary)?;
1113        let reconstructed = reconstructed.as_binary::<i32>();
1114        assert_eq!(
1115            values[0],
1116            decode_json_variant(reconstructed.value(0)).unwrap()
1117        );
1118        assert_eq!(
1119            json!({
1120                "kind": "other",
1121                "commit": {"operation": null},
1122                "dynamic": true,
1123                "time_us": 2
1124            }),
1125            decode_json_variant(reconstructed.value(1)).unwrap()
1126        );
1127
1128        let settings = JsonSettings::try_new(vec![], Some(0))?;
1129        let mut builder = JsonVectorBuilder::with_settings(&settings, 3);
1130        for value in [json!({}), json!({"x": 1})] {
1131            let value = settings.encode(value)?;
1132            builder.try_push_value_ref(&value.as_value_ref())?;
1133        }
1134        builder.push_null();
1135        let array = builder.to_vector().to_arrow_array();
1136        let structs = array.as_struct();
1137        assert_eq!(1, structs.num_columns());
1138        assert_eq!(
1139            vec![Some(json!({})), Some(json!({"x": 1})), None],
1140            variant_to_json_values(structs.column(0))?
1141        );
1142
1143        Ok(())
1144    }
1145
1146    #[test]
1147    fn test_finite_budget_selects_dynamic_paths() -> Result<()> {
1148        let builder = JsonVectorBuilder::with_settings(&JsonSettings::default(), 0);
1149        assert!(matches!(
1150            builder.state,
1151            JsonVectorBuilderState::AutoExpanding {
1152                max_auto_expanded_paths: JSON2_DEFAULT_MAX_AUTO_EXPANDED_PATHS,
1153                ..
1154            }
1155        ));
1156
1157        let settings = JsonSettings::try_new(
1158            vec![JsonTypeHint {
1159                path: vec!["hint".to_string()],
1160                data_type: ConcreteDataType::string_datatype(),
1161                nullable: true,
1162                default_constraint: None,
1163                inverted_index: false,
1164            }],
1165            Some(2),
1166        )?;
1167        let values = [
1168            json!({
1169                "hint": "first",
1170                "conflict": 1,
1171                "popular": {"nested": 1},
1172                "tie_a": "a",
1173                "tie_b": true,
1174                "rare": 1
1175            }),
1176            json!({
1177                "hint": "second",
1178                "conflict": "string",
1179                "popular": {"nested": 2},
1180                "tie_a": "b",
1181                "tie_b": false
1182            }),
1183            json!({"hint": "third", "popular": "scalar"}),
1184        ];
1185        let mut builder = JsonVectorBuilder::with_settings(&settings, values.len());
1186        for value in values.clone() {
1187            let value = settings.encode(value)?;
1188            builder.try_push_value_ref(&value.as_value_ref())?;
1189        }
1190
1191        let array = builder.to_vector().to_arrow_array();
1192        let structs = array.as_struct();
1193        assert_eq!(
1194            vec![JSON2_REMAINDER_FIELD_NAME, "hint", "tie_a", "tie_b"],
1195            structs
1196                .fields()
1197                .iter()
1198                .map(|x| x.name().as_str())
1199                .collect::<Vec<_>>()
1200        );
1201        assert!(structs.column_by_name("popular").is_none());
1202        assert_eq!(
1203            vec![
1204                Some(json!({"conflict": 1, "popular": {"nested": 1}, "rare": 1})),
1205                Some(json!({"conflict": "string", "popular": {"nested": 2}})),
1206                Some(json!({"popular": "scalar"})),
1207            ],
1208            variant_to_json_values(structs.column_by_name(JSON2_REMAINDER_FIELD_NAME).unwrap())?
1209        );
1210
1211        let field = Field::new("data", array.data_type().clone(), true).with_extension_type(
1212            Json2ExtensionType::new(Arc::new(JsonMetadata::new(settings))),
1213        );
1214        let reconstructed = JsonArray::from(&array).project_to_v2(&field, &DataType::Binary)?;
1215        let reconstructed = reconstructed.as_binary::<i32>();
1216        assert_eq!(
1217            values[0],
1218            decode_json_variant(reconstructed.value(0)).unwrap()
1219        );
1220        assert_eq!(
1221            values[1],
1222            decode_json_variant(reconstructed.value(1)).unwrap()
1223        );
1224        assert_eq!(
1225            json!({
1226                "hint": "third",
1227                "popular": "scalar"
1228            }),
1229            decode_json_variant(reconstructed.value(2)).unwrap()
1230        );
1231
1232        Ok(())
1233    }
1234
1235    #[test]
1236    fn test_v2_builder_preserves_explicit_null_presence() -> Result<()> {
1237        let settings = JsonSettings::try_new(vec![], Some(1))?;
1238        let values = [json!({"value": 1}), json!({"value": null}), json!({})];
1239        let mut builder = JsonVectorBuilder::with_settings(&settings, values.len());
1240        for value in values.clone() {
1241            let value = settings.encode(value)?;
1242            builder.try_push_value_ref(&value.as_value_ref())?;
1243        }
1244        let array = builder.to_vector().to_arrow_array();
1245        let structs = array.as_struct();
1246        assert_eq!(
1247            vec![
1248                Some(json!({})),
1249                Some(json!({"value": null})),
1250                Some(json!({}))
1251            ],
1252            variant_to_json_values(structs.column_by_name(JSON2_REMAINDER_FIELD_NAME).unwrap())?
1253        );
1254        let field = Field::new("data", array.data_type().clone(), true).with_extension_type(
1255            Json2ExtensionType::new(Arc::new(JsonMetadata::new(settings))),
1256        );
1257        let reconstructed = JsonArray::from(&array).project_to_v2(&field, &DataType::Binary)?;
1258        let reconstructed = reconstructed.as_binary::<i32>();
1259
1260        assert_eq!(
1261            values[0],
1262            decode_json_variant(reconstructed.value(0)).unwrap()
1263        );
1264        assert_eq!(
1265            values[1],
1266            decode_json_variant(reconstructed.value(1)).unwrap()
1267        );
1268        assert_eq!(
1269            values[2],
1270            decode_json_variant(reconstructed.value(2)).unwrap()
1271        );
1272        Ok(())
1273    }
1274
1275    #[test]
1276    fn test_reconstruct_nested_remainder_only_value() -> Result<()> {
1277        let settings = JsonSettings::try_new(vec![], Some(1))?;
1278        let values = [
1279            json!({"a": {"hot": 1}}),
1280            json!({"a": {"hot": 2}}),
1281            json!({"a": {"cold": 3}}),
1282        ];
1283        let mut builder = JsonVectorBuilder::with_settings(&settings, values.len());
1284        for value in values.clone() {
1285            let value = settings.encode(value)?;
1286            builder.try_push_value_ref(&value.as_value_ref())?;
1287        }
1288
1289        let array = builder.to_vector().to_arrow_array();
1290        let field = Field::new("data", array.data_type().clone(), true).with_extension_type(
1291            Json2ExtensionType::new(Arc::new(JsonMetadata::new(settings))),
1292        );
1293        let reconstructed = JsonArray::from(&array).project_to_v2(&field, &DataType::Binary)?;
1294        let reconstructed = reconstructed.as_binary::<i32>();
1295        assert_eq!(
1296            vec![values[0].clone(), values[1].clone(), values[2].clone()],
1297            (0..reconstructed.len())
1298                .map(|i| decode_json_variant(reconstructed.value(i)).unwrap())
1299                .collect::<Vec<_>>()
1300        );
1301
1302        Ok(())
1303    }
1304
1305    #[test]
1306    fn test_dynamic_paths_require_the_same_leaf_type() -> Result<()> {
1307        let settings = JsonSettings::try_new(
1308            vec![JsonTypeHint {
1309                path: vec!["nested".to_string(), "hinted".to_string()],
1310                data_type: ConcreteDataType::string_datatype(),
1311                nullable: true,
1312                default_constraint: None,
1313                inverted_index: false,
1314            }],
1315            Some(8),
1316        )?;
1317        let values = [
1318            json!({
1319                "branch": {},
1320                "different": [1],
1321                "empty": {},
1322                "nested": {},
1323                "reverse": "scalar",
1324                "same": [1]
1325            }),
1326            json!({
1327                "branch": {"leaf": 1},
1328                "different": ["x"],
1329                "empty": {},
1330                "nested": {"hinted": "x", "leaf": 1},
1331                "same": [2]
1332            }),
1333            json!({
1334                "branch": {},
1335                "different": [2],
1336                "empty": {},
1337                "nested": {},
1338                "reverse": {"leaf": 1},
1339                "same": [3]
1340            }),
1341        ];
1342        let mut builder = JsonVectorBuilder::with_settings(&settings, values.len());
1343        for value in values {
1344            let value = settings.encode(value)?;
1345            builder.try_push_value_ref(&value.as_value_ref())?;
1346        }
1347
1348        let JsonNativeType::Object(fields) = builder.state.native_type() else {
1349            unreachable!();
1350        };
1351        assert!(!fields.contains_key("different"));
1352        assert!(!fields.contains_key("empty"));
1353        assert!(!fields.contains_key("reverse"));
1354        assert!(matches!(fields.get("same"), Some(JsonNativeType::Array(_))));
1355        assert!(matches!(
1356            fields.get("branch"),
1357            Some(JsonNativeType::Object(fields)) if fields.contains_key("leaf")
1358        ));
1359        assert!(matches!(
1360            fields.get("nested"),
1361            Some(JsonNativeType::Object(fields))
1362                if fields.contains_key("hinted") && fields.contains_key("leaf")
1363        ));
1364
1365        Ok(())
1366    }
1367
1368    #[test]
1369    fn test_json_variant_into_struct_value() -> Result<()> {
1370        let struct_type = StructType::new(Arc::new(vec![StructField::new(
1371            "value".to_string(),
1372            ConcreteDataType::string_datatype(),
1373            true,
1374        )]));
1375        assert_eq!(
1376            json_variant_into_value(
1377                JsonVariant::Object(Default::default()),
1378                &ConcreteDataType::struct_datatype(struct_type.clone()),
1379                false,
1380            )?,
1381            Value::Null
1382        );
1383        assert_eq!(
1384            json_variant_into_value(
1385                JsonVariant::Object(Default::default()),
1386                &ConcreteDataType::struct_datatype(struct_type.clone()),
1387                true,
1388            )?,
1389            Value::Struct(StructValue::new(vec![Value::Null], struct_type))
1390        );
1391
1392        let item_type =
1393            ConcreteDataType::struct_datatype(StructType::new(Arc::new(vec![StructField::new(
1394                "id".to_string(),
1395                ConcreteDataType::int64_datatype(),
1396                true,
1397            )])));
1398        let struct_type = StructType::new(Arc::new(vec![
1399            StructField::new(
1400                "items".to_string(),
1401                ConcreteDataType::list_datatype(Arc::new(item_type.clone())),
1402                true,
1403            ),
1404            StructField::new(
1405                "meta".to_string(),
1406                ConcreteDataType::struct_datatype(StructType::new(Arc::new(vec![
1407                    StructField::new(
1408                        "name".to_string(),
1409                        ConcreteDataType::string_datatype(),
1410                        true,
1411                    ),
1412                ]))),
1413                true,
1414            ),
1415        ]));
1416        let variant = JsonObjectValue::from([
1417            (
1418                "items".to_string(),
1419                JsonVariant::Array(vec![
1420                    JsonVariant::from([("id", JsonVariant::from(1i64))]),
1421                    JsonVariant::from([("id", JsonVariant::from(2i64))]),
1422                ]),
1423            ),
1424            (
1425                "meta".to_string(),
1426                JsonVariant::from([("name", JsonVariant::from("foo"))]),
1427            ),
1428        ]);
1429        let value = Value::Struct(json_variant_into_struct_value(
1430            variant,
1431            struct_type.clone(),
1432            true,
1433        )?);
1434
1435        assert_eq!(
1436            value,
1437            Value::Struct(StructValue::new(
1438                vec![
1439                    Value::List(ListValue::new(
1440                        vec![
1441                            Value::Struct(StructValue::new(
1442                                vec![Value::Int64(1)],
1443                                StructType::new(Arc::new(vec![StructField::new(
1444                                    "id".to_string(),
1445                                    ConcreteDataType::int64_datatype(),
1446                                    true,
1447                                )]))
1448                            )),
1449                            Value::Struct(StructValue::new(
1450                                vec![Value::Int64(2)],
1451                                StructType::new(Arc::new(vec![StructField::new(
1452                                    "id".to_string(),
1453                                    ConcreteDataType::int64_datatype(),
1454                                    true,
1455                                )]))
1456                            )),
1457                        ],
1458                        Arc::new(item_type),
1459                    )),
1460                    Value::Struct(StructValue::new(
1461                        vec![Value::String("foo".into())],
1462                        StructType::new(Arc::new(vec![StructField::new(
1463                            "name".to_string(),
1464                            ConcreteDataType::string_datatype(),
1465                            true,
1466                        )])),
1467                    )),
1468                ],
1469                struct_type,
1470            ))
1471        );
1472        Ok(())
1473    }
1474}
datatypes/vectors/json/builder.rs

datatypes/vectors/json/
builder.rs
