datatypes/schema/
column_schema.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::str::FromStr;
17use std::{fmt, mem};
18
19use arrow::datatypes::Field;
20use arrow_schema::extension::{
21    EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY, ExtensionType,
22};
23use serde::{Deserialize, Serialize};
24use snafu::{ResultExt, ensure};
25use sqlparser_derive::{Visit, VisitMut};
26
27use crate::data_type::{ConcreteDataType, DataType};
28use crate::error::{
29    self, ArrowMetadataSnafu, Error, InvalidFulltextOptionSnafu, ParseExtendedTypeSnafu, Result,
30};
31use crate::extension::json::Json2ExtensionType;
32use crate::schema::TYPE_KEY;
33use crate::schema::constraint::ColumnDefaultConstraint;
34use crate::value::Value;
35use crate::vectors::json::builder::JsonVectorBuilder;
36use crate::vectors::{MutableVector, VectorRef};
37
38pub type Metadata = HashMap<String, String>;
39
40/// Key used to store whether the column is time index in arrow field's metadata.
41pub const TIME_INDEX_KEY: &str = "greptime:time_index";
42pub const COMMENT_KEY: &str = "greptime:storage:comment";
43/// Key used to store default constraint in arrow field's metadata.
44const DEFAULT_CONSTRAINT_KEY: &str = "greptime:default_constraint";
45/// Key used to store fulltext options in arrow field's metadata.
46pub const FULLTEXT_KEY: &str = "greptime:fulltext";
47/// Key used to store whether the column has inverted index in arrow field's metadata.
48pub const INVERTED_INDEX_KEY: &str = "greptime:inverted_index";
49/// Key used to store skip options in arrow field's metadata.
50pub const SKIPPING_INDEX_KEY: &str = "greptime:skipping_index";
51/// Key used to store vector index options in arrow field's metadata.
52pub const VECTOR_INDEX_KEY: &str = "greptime:vector_index";
53
54/// Keys used in fulltext options
55pub const COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE: &str = "enable";
56pub const COLUMN_FULLTEXT_OPT_KEY_ANALYZER: &str = "analyzer";
57pub const COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE: &str = "case_sensitive";
58pub const COLUMN_FULLTEXT_OPT_KEY_BACKEND: &str = "backend";
59pub const COLUMN_FULLTEXT_OPT_KEY_GRANULARITY: &str = "granularity";
60pub const COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE: &str = "false_positive_rate";
61
62/// Keys used in SKIPPING index options
63pub const COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY: &str = "granularity";
64pub const COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE: &str = "false_positive_rate";
65pub const COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE: &str = "type";
66
67/// Keys used in VECTOR index options
68pub const COLUMN_VECTOR_INDEX_OPT_KEY_ENGINE: &str = "engine";
69pub const COLUMN_VECTOR_INDEX_OPT_KEY_METRIC: &str = "metric";
70pub const COLUMN_VECTOR_INDEX_OPT_KEY_CONNECTIVITY: &str = "connectivity";
71pub const COLUMN_VECTOR_INDEX_OPT_KEY_EXPANSION_ADD: &str = "expansion_add";
72pub const COLUMN_VECTOR_INDEX_OPT_KEY_EXPANSION_SEARCH: &str = "expansion_search";
73
74pub const DEFAULT_GRANULARITY: u32 = 10240;
75
76pub const DEFAULT_FALSE_POSITIVE_RATE: f64 = 0.01;
77
78/// Schema of a column, used as an immutable struct.
79#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
80pub struct ColumnSchema {
81    pub name: String,
82    pub data_type: ConcreteDataType,
83    is_nullable: bool,
84    is_time_index: bool,
85    default_constraint: Option<ColumnDefaultConstraint>,
86    metadata: Metadata,
87}
88
89impl fmt::Debug for ColumnSchema {
90    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
91        write!(
92            f,
93            "{} {} {}",
94            self.name,
95            self.data_type,
96            if self.is_nullable { "null" } else { "not null" },
97        )?;
98
99        if self.is_time_index {
100            write!(f, " time_index")?;
101        }
102
103        // Add default constraint if present
104        if let Some(default_constraint) = &self.default_constraint {
105            write!(f, " default={:?}", default_constraint)?;
106        }
107
108        // Add metadata if present
109        if !self.metadata.is_empty() {
110            write!(f, " metadata={:?}", self.metadata)?;
111        }
112
113        Ok(())
114    }
115}
116
117impl ColumnSchema {
118    pub fn new<T: Into<String>>(
119        name: T,
120        data_type: ConcreteDataType,
121        is_nullable: bool,
122    ) -> ColumnSchema {
123        ColumnSchema {
124            name: name.into(),
125            data_type,
126            is_nullable,
127            is_time_index: false,
128            default_constraint: None,
129            metadata: Metadata::new(),
130        }
131    }
132
133    /// Creates a mutable vector using this column's extension metadata.
134    pub fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector> {
135        if self.data_type.is_json2()
136            && let Some(extension) = self.extension_type::<Json2ExtensionType>().ok().flatten()
137            && extension.metadata().is_version_2()
138        {
139            Box::new(JsonVectorBuilder::with_settings(
140                extension.metadata().json_settings(),
141                capacity,
142            ))
143        } else {
144            self.data_type.create_mutable_vector(capacity)
145        }
146    }
147
148    #[inline]
149    pub fn is_time_index(&self) -> bool {
150        self.is_time_index
151    }
152
153    #[inline]
154    pub fn is_nullable(&self) -> bool {
155        self.is_nullable
156    }
157
158    #[inline]
159    pub fn default_constraint(&self) -> Option<&ColumnDefaultConstraint> {
160        self.default_constraint.as_ref()
161    }
162
163    /// Check if the default constraint is a impure function.
164    pub fn is_default_impure(&self) -> bool {
165        self.default_constraint
166            .as_ref()
167            .map(|c| c.is_function())
168            .unwrap_or(false)
169    }
170
171    #[inline]
172    pub fn metadata(&self) -> &Metadata {
173        &self.metadata
174    }
175
176    #[inline]
177    pub fn mut_metadata(&mut self) -> &mut Metadata {
178        &mut self.metadata
179    }
180
181    /// Retrieve the column comment
182    pub fn column_comment(&self) -> Option<&String> {
183        self.metadata.get(COMMENT_KEY)
184    }
185
186    pub fn with_time_index(mut self, is_time_index: bool) -> Self {
187        self.is_time_index = is_time_index;
188        if is_time_index {
189            let _ = self
190                .metadata
191                .insert(TIME_INDEX_KEY.to_string(), "true".to_string());
192        } else {
193            let _ = self.metadata.remove(TIME_INDEX_KEY);
194        }
195        self
196    }
197
198    /// Returns the estimated memory footprint of this schema.
199    pub fn estimated_size(&self) -> usize {
200        mem::size_of_val(self) - mem::size_of_val(&self.data_type)
201            + self.data_type.as_arrow_type().size()
202            + self.name.capacity()
203            + self
204                .default_constraint
205                .as_ref()
206                .map(column_default_constraint_size)
207                .unwrap_or_default()
208            + metadata_size(&self.metadata)
209    }
210
211    /// Set the inverted index for the column.
212    /// Similar to [with_inverted_index] but don't take the ownership.
213    ///
214    /// [with_inverted_index]: Self::with_inverted_index
215    pub fn set_inverted_index(&mut self, value: bool) {
216        match value {
217            true => {
218                self.metadata
219                    .insert(INVERTED_INDEX_KEY.to_string(), value.to_string());
220            }
221            false => {
222                self.metadata.remove(INVERTED_INDEX_KEY);
223            }
224        }
225    }
226
227    /// Set the inverted index for the column.
228    /// Similar to [set_inverted_index] but take the ownership and return a owned value.
229    ///
230    /// [set_inverted_index]: Self::set_inverted_index
231    pub fn with_inverted_index(mut self, value: bool) -> Self {
232        self.set_inverted_index(value);
233        self
234    }
235
236    pub fn is_inverted_indexed(&self) -> bool {
237        self.metadata
238            .get(INVERTED_INDEX_KEY)
239            .map(|v| v.eq_ignore_ascii_case("true"))
240            .unwrap_or(false)
241    }
242
243    pub fn is_fulltext_indexed(&self) -> bool {
244        self.fulltext_options()
245            .unwrap_or_default()
246            .map(|option| option.enable)
247            .unwrap_or_default()
248    }
249
250    pub fn is_skipping_indexed(&self) -> bool {
251        self.skipping_index_options().unwrap_or_default().is_some()
252    }
253
254    pub fn has_inverted_index_key(&self) -> bool {
255        self.metadata.contains_key(INVERTED_INDEX_KEY)
256    }
257
258    /// Checks if this column has a vector index.
259    pub fn is_vector_indexed(&self) -> bool {
260        match self.vector_index_options() {
261            Ok(opts) => opts.is_some(),
262            Err(e) => {
263                common_telemetry::warn!(
264                    "Failed to deserialize vector_index_options for column '{}': {}",
265                    self.name,
266                    e
267                );
268                false
269            }
270        }
271    }
272
273    /// Gets the vector index options.
274    pub fn vector_index_options(&self) -> Result<Option<VectorIndexOptions>> {
275        match self.metadata.get(VECTOR_INDEX_KEY) {
276            None => Ok(None),
277            Some(json) => {
278                let options =
279                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
280                Ok(Some(options))
281            }
282        }
283    }
284
285    /// Sets the vector index options.
286    pub fn set_vector_index_options(&mut self, options: &VectorIndexOptions) -> Result<()> {
287        self.metadata.insert(
288            VECTOR_INDEX_KEY.to_string(),
289            serde_json::to_string(options).context(error::SerializeSnafu)?,
290        );
291        Ok(())
292    }
293
294    /// Removes the vector index options.
295    pub fn unset_vector_index_options(&mut self) {
296        self.metadata.remove(VECTOR_INDEX_KEY);
297    }
298
299    /// Sets vector index options and returns self for chaining.
300    pub fn with_vector_index_options(mut self, options: &VectorIndexOptions) -> Result<Self> {
301        self.set_vector_index_options(options)?;
302        Ok(self)
303    }
304
305    /// Set default constraint.
306    ///
307    /// If a default constraint exists for the column, this method will
308    /// validate it against the column's data type and nullability.
309    pub fn with_default_constraint(
310        mut self,
311        default_constraint: Option<ColumnDefaultConstraint>,
312    ) -> Result<Self> {
313        if let Some(constraint) = &default_constraint {
314            constraint.validate(&self.data_type, self.is_nullable)?;
315        }
316
317        self.default_constraint = default_constraint;
318        Ok(self)
319    }
320
321    /// Set the nullablity to `true` of the column.
322    /// Similar to [set_nullable] but take the ownership and return a owned value.
323    ///
324    /// [set_nullable]: Self::set_nullable
325    pub fn with_nullable_set(mut self) -> Self {
326        self.is_nullable = true;
327        self
328    }
329
330    /// Set the nullability to `true` of the column.
331    /// Similar to [with_nullable_set] but don't take the ownership
332    ///
333    /// [with_nullable_set]: Self::with_nullable_set
334    pub fn set_nullable(&mut self) {
335        self.is_nullable = true;
336    }
337
338    /// Set the `is_time_index` to `true` of the column.
339    /// Similar to [with_time_index] but don't take the ownership.
340    ///
341    /// [with_time_index]: Self::with_time_index
342    pub fn set_time_index(&mut self) {
343        self.is_time_index = true;
344    }
345
346    /// Creates a new [`ColumnSchema`] with given metadata.
347    pub fn with_metadata(mut self, metadata: Metadata) -> Self {
348        self.metadata = metadata;
349        self
350    }
351
352    /// Creates a vector with default value for this column.
353    ///
354    /// If the column is `NOT NULL` but doesn't has `DEFAULT` value supplied, returns `Ok(None)`.
355    pub fn create_default_vector(&self, num_rows: usize) -> Result<Option<VectorRef>> {
356        match &self.default_constraint {
357            Some(c) => c
358                .create_default_vector(&self.data_type, self.is_nullable, num_rows)
359                .map(Some),
360            None => {
361                if self.is_nullable {
362                    // No default constraint, use null as default value.
363                    // TODO(yingwen): Use NullVector once it supports setting logical type.
364                    ColumnDefaultConstraint::null_value()
365                        .create_default_vector(&self.data_type, self.is_nullable, num_rows)
366                        .map(Some)
367                } else {
368                    Ok(None)
369                }
370            }
371        }
372    }
373
374    /// Creates a vector for padding.
375    ///
376    /// This method always returns a vector since it uses [DataType::default_value]
377    /// to fill the vector. Callers should only use the created vector for padding
378    /// and never read its content.
379    pub fn create_default_vector_for_padding(&self, num_rows: usize) -> VectorRef {
380        let padding_value = if self.is_nullable {
381            Value::Null
382        } else {
383            // If the column is not null, use the data type's default value as it is
384            // more efficient to acquire.
385            self.data_type.default_value()
386        };
387        let value_ref = padding_value.as_value_ref();
388        let mut mutable_vector = self.data_type.create_mutable_vector(num_rows);
389        for _ in 0..num_rows {
390            mutable_vector.push_value_ref(&value_ref);
391        }
392        mutable_vector.to_vector()
393    }
394
395    /// Creates a default value for this column.
396    ///
397    /// If the column is `NOT NULL` but doesn't has `DEFAULT` value supplied, returns `Ok(None)`.
398    pub fn create_default(&self) -> Result<Option<Value>> {
399        match &self.default_constraint {
400            Some(c) => c
401                .create_default(&self.data_type, self.is_nullable)
402                .map(Some),
403            None => {
404                if self.is_nullable {
405                    // No default constraint, use null as default value.
406                    ColumnDefaultConstraint::null_value()
407                        .create_default(&self.data_type, self.is_nullable)
408                        .map(Some)
409                } else {
410                    Ok(None)
411                }
412            }
413        }
414    }
415
416    /// Creates an impure default value for this column, only if it have a impure default constraint.
417    /// Otherwise, returns `Ok(None)`.
418    pub fn create_impure_default(&self) -> Result<Option<Value>> {
419        match &self.default_constraint {
420            Some(c) => c.create_impure_default(&self.data_type),
421            None => Ok(None),
422        }
423    }
424
425    /// Retrieves the fulltext options for the column.
426    pub fn fulltext_options(&self) -> Result<Option<FulltextOptions>> {
427        match self.metadata.get(FULLTEXT_KEY) {
428            None => Ok(None),
429            Some(json) => {
430                let options =
431                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
432                Ok(Some(options))
433            }
434        }
435    }
436
437    pub fn with_fulltext_options(mut self, options: FulltextOptions) -> Result<Self> {
438        self.metadata.insert(
439            FULLTEXT_KEY.to_string(),
440            serde_json::to_string(&options).context(error::SerializeSnafu)?,
441        );
442        Ok(self)
443    }
444
445    pub fn set_fulltext_options(&mut self, options: &FulltextOptions) -> Result<()> {
446        self.metadata.insert(
447            FULLTEXT_KEY.to_string(),
448            serde_json::to_string(options).context(error::SerializeSnafu)?,
449        );
450        Ok(())
451    }
452
453    /// Retrieves the skipping index options for the column.
454    pub fn skipping_index_options(&self) -> Result<Option<SkippingIndexOptions>> {
455        match self.metadata.get(SKIPPING_INDEX_KEY) {
456            None => Ok(None),
457            Some(json) => {
458                let options =
459                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
460                Ok(Some(options))
461            }
462        }
463    }
464
465    pub fn with_skipping_options(mut self, options: SkippingIndexOptions) -> Result<Self> {
466        self.metadata.insert(
467            SKIPPING_INDEX_KEY.to_string(),
468            serde_json::to_string(&options).context(error::SerializeSnafu)?,
469        );
470        Ok(self)
471    }
472
473    pub fn set_skipping_options(&mut self, options: &SkippingIndexOptions) -> Result<()> {
474        self.metadata.insert(
475            SKIPPING_INDEX_KEY.to_string(),
476            serde_json::to_string(options).context(error::SerializeSnafu)?,
477        );
478        Ok(())
479    }
480
481    pub fn unset_skipping_options(&mut self) -> Result<()> {
482        self.metadata.remove(SKIPPING_INDEX_KEY);
483        Ok(())
484    }
485
486    pub fn extension_type<E>(&self) -> Result<Option<E>>
487    where
488        E: ExtensionType,
489    {
490        let extension_type_name = self.metadata.get(EXTENSION_TYPE_NAME_KEY);
491
492        if extension_type_name.map(|s| s.as_str()) == Some(E::NAME) {
493            let extension_metadata = self.metadata.get(EXTENSION_TYPE_METADATA_KEY);
494            let extension_metadata =
495                E::deserialize_metadata(extension_metadata.map(|s| s.as_str()))
496                    .context(ArrowMetadataSnafu)?;
497
498            let extension = E::try_new(&self.data_type.as_arrow_type(), extension_metadata)
499                .context(ArrowMetadataSnafu)?;
500            Ok(Some(extension))
501        } else {
502            Ok(None)
503        }
504    }
505
506    /// Sets the Arrow extension type metadata for this column.
507    pub fn with_extension_type<E>(&mut self, extension_type: &E)
508    where
509        E: ExtensionType,
510    {
511        self.metadata
512            .insert(EXTENSION_TYPE_NAME_KEY.to_string(), E::NAME.to_string());
513
514        if let Some(extension_metadata) = extension_type.serialize_metadata() {
515            self.metadata
516                .insert(EXTENSION_TYPE_METADATA_KEY.to_string(), extension_metadata);
517        } else {
518            // Replacing an extension must not retain metadata owned by the previous type.
519            self.metadata.remove(EXTENSION_TYPE_METADATA_KEY);
520        }
521    }
522
523    pub fn is_indexed(&self) -> bool {
524        self.is_inverted_indexed() || self.is_fulltext_indexed() || self.is_skipping_indexed()
525    }
526}
527
528fn metadata_size(metadata: &Metadata) -> usize {
529    mem::size_of::<(String, String)>() * metadata.capacity()
530        + metadata
531            .iter()
532            .map(|(key, value)| key.capacity() + value.capacity())
533            .sum::<usize>()
534}
535
536fn column_default_constraint_size(default_constraint: &ColumnDefaultConstraint) -> usize {
537    match default_constraint {
538        ColumnDefaultConstraint::Function(expr) => expr.capacity(),
539        ColumnDefaultConstraint::Value(value) => value.as_value_ref().data_size(),
540    }
541}
542
543/// Column extended type set in column schema's metadata.
544#[derive(Debug, Clone, PartialEq, Eq)]
545pub enum ColumnExtType {
546    /// Json type.
547    Json,
548
549    /// Vector type with dimension.
550    Vector(u32),
551}
552
553impl fmt::Display for ColumnExtType {
554    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
555        match self {
556            ColumnExtType::Json => write!(f, "Json"),
557            ColumnExtType::Vector(dim) => write!(f, "Vector({})", dim),
558        }
559    }
560}
561
562impl FromStr for ColumnExtType {
563    type Err = String;
564
565    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
566        match s {
567            "Json" => Ok(ColumnExtType::Json),
568            _ if s.starts_with("Vector(") && s.ends_with(')') => s[7..s.len() - 1]
569                .parse::<u32>()
570                .map(ColumnExtType::Vector)
571                .map_err(|_| "Invalid dimension for Vector".to_string()),
572            _ => Err("Unknown variant".to_string()),
573        }
574    }
575}
576
577impl TryFrom<&Field> for ColumnSchema {
578    type Error = Error;
579
580    fn try_from(field: &Field) -> Result<ColumnSchema> {
581        let mut data_type = ConcreteDataType::try_from(field.data_type())?;
582        // Override the data type if it is specified in the metadata.
583        if let Some(s) = field.metadata().get(TYPE_KEY) {
584            let extype = ColumnExtType::from_str(s)
585                .map_err(|_| ParseExtendedTypeSnafu { value: s }.build())?;
586            match extype {
587                ColumnExtType::Json => {
588                    data_type = ConcreteDataType::json_datatype();
589                }
590                ColumnExtType::Vector(dim) => {
591                    data_type = ConcreteDataType::vector_datatype(dim);
592                }
593            }
594        }
595        let mut metadata = field.metadata().clone();
596        let default_constraint = match metadata.remove(DEFAULT_CONSTRAINT_KEY) {
597            Some(json) => {
598                Some(serde_json::from_str(&json).context(error::DeserializeSnafu { json })?)
599            }
600            None => None,
601        };
602        let mut is_time_index = metadata.contains_key(TIME_INDEX_KEY);
603        if is_time_index && !data_type.is_timestamp() {
604            // If the column is time index but the data type is not timestamp, it is invalid.
605            // We set the time index to false and remove the metadata.
606            // This is possible if we cast the time index column to another type. DataFusion will
607            // keep the metadata:
608            // https://github.com/apache/datafusion/pull/12951
609            is_time_index = false;
610            metadata.remove(TIME_INDEX_KEY);
611            common_telemetry::debug!(
612                "Column {} is not timestamp ({:?}) but has time index metadata",
613                data_type,
614                field.name(),
615            );
616        }
617
618        Ok(ColumnSchema {
619            name: field.name().clone(),
620            data_type,
621            is_nullable: field.is_nullable(),
622            is_time_index,
623            default_constraint,
624            metadata,
625        })
626    }
627}
628
629impl TryFrom<&ColumnSchema> for Field {
630    type Error = Error;
631
632    fn try_from(column_schema: &ColumnSchema) -> Result<Field> {
633        let mut metadata = column_schema.metadata.clone();
634        if let Some(value) = &column_schema.default_constraint {
635            // Adds an additional metadata to store the default constraint.
636            let old = metadata.insert(
637                DEFAULT_CONSTRAINT_KEY.to_string(),
638                serde_json::to_string(&value).context(error::SerializeSnafu)?,
639            );
640
641            ensure!(
642                old.is_none(),
643                error::DuplicateMetaSnafu {
644                    key: DEFAULT_CONSTRAINT_KEY,
645                }
646            );
647        }
648
649        Ok(Field::new(
650            &column_schema.name,
651            column_schema.data_type.as_arrow_type(),
652            column_schema.is_nullable(),
653        )
654        .with_metadata(metadata))
655    }
656}
657
658/// Fulltext options for a column.
659#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
660#[serde(rename_all = "kebab-case")]
661pub struct FulltextOptions {
662    /// Whether the fulltext index is enabled.
663    pub enable: bool,
664    /// The fulltext analyzer to use.
665    #[serde(default)]
666    pub analyzer: FulltextAnalyzer,
667    /// Whether the fulltext index is case-sensitive.
668    #[serde(default)]
669    pub case_sensitive: bool,
670    /// The fulltext backend to use.
671    #[serde(default)]
672    pub backend: FulltextBackend,
673    /// The granularity of the fulltext index (for bloom backend only)
674    #[serde(default = "fulltext_options_default_granularity")]
675    pub granularity: u32,
676    /// The false positive rate of the fulltext index (for bloom backend only)
677    #[serde(default = "index_options_default_false_positive_rate_in_10000")]
678    pub false_positive_rate_in_10000: u32,
679}
680
681fn fulltext_options_default_granularity() -> u32 {
682    DEFAULT_GRANULARITY
683}
684
685fn index_options_default_false_positive_rate_in_10000() -> u32 {
686    (DEFAULT_FALSE_POSITIVE_RATE * 10000.0) as u32
687}
688
689impl FulltextOptions {
690    /// Creates a new fulltext options.
691    pub fn new(
692        enable: bool,
693        analyzer: FulltextAnalyzer,
694        case_sensitive: bool,
695        backend: FulltextBackend,
696        granularity: u32,
697        false_positive_rate: f64,
698    ) -> Result<Self> {
699        ensure!(
700            0.0 < false_positive_rate && false_positive_rate <= 1.0,
701            error::InvalidFulltextOptionSnafu {
702                msg: format!(
703                    "Invalid false positive rate: {false_positive_rate}, expected: 0.0 < rate <= 1.0"
704                ),
705            }
706        );
707        ensure!(
708            granularity > 0,
709            error::InvalidFulltextOptionSnafu {
710                msg: format!("Invalid granularity: {granularity}, expected: positive integer"),
711            }
712        );
713        Ok(Self::new_unchecked(
714            enable,
715            analyzer,
716            case_sensitive,
717            backend,
718            granularity,
719            false_positive_rate,
720        ))
721    }
722
723    /// Creates a new fulltext options without checking `false_positive_rate` and `granularity`.
724    pub fn new_unchecked(
725        enable: bool,
726        analyzer: FulltextAnalyzer,
727        case_sensitive: bool,
728        backend: FulltextBackend,
729        granularity: u32,
730        false_positive_rate: f64,
731    ) -> Self {
732        Self {
733            enable,
734            analyzer,
735            case_sensitive,
736            backend,
737            granularity,
738            false_positive_rate_in_10000: (false_positive_rate * 10000.0) as u32,
739        }
740    }
741
742    /// Gets the false positive rate.
743    pub fn false_positive_rate(&self) -> f64 {
744        self.false_positive_rate_in_10000 as f64 / 10000.0
745    }
746}
747
748impl Default for FulltextOptions {
749    fn default() -> Self {
750        Self::new_unchecked(
751            false,
752            FulltextAnalyzer::default(),
753            false,
754            FulltextBackend::default(),
755            DEFAULT_GRANULARITY,
756            DEFAULT_FALSE_POSITIVE_RATE,
757        )
758    }
759}
760
761impl fmt::Display for FulltextOptions {
762    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
763        write!(f, "enable={}", self.enable)?;
764        if self.enable {
765            write!(f, ", analyzer={}", self.analyzer)?;
766            write!(f, ", case_sensitive={}", self.case_sensitive)?;
767            write!(f, ", backend={}", self.backend)?;
768            if self.backend == FulltextBackend::Bloom {
769                write!(f, ", granularity={}", self.granularity)?;
770                write!(f, ", false_positive_rate={}", self.false_positive_rate())?;
771            }
772        }
773        Ok(())
774    }
775}
776
777/// The backend of the fulltext index.
778#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
779#[serde(rename_all = "kebab-case")]
780pub enum FulltextBackend {
781    #[default]
782    Bloom,
783    Tantivy,
784}
785
786impl fmt::Display for FulltextBackend {
787    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
788        match self {
789            FulltextBackend::Tantivy => write!(f, "tantivy"),
790            FulltextBackend::Bloom => write!(f, "bloom"),
791        }
792    }
793}
794
795impl TryFrom<HashMap<String, String>> for FulltextOptions {
796    type Error = Error;
797
798    fn try_from(options: HashMap<String, String>) -> Result<Self> {
799        let mut fulltext_options = FulltextOptions {
800            enable: true,
801            ..Default::default()
802        };
803
804        if let Some(enable) = options.get(COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE) {
805            match enable.to_ascii_lowercase().as_str() {
806                "true" => fulltext_options.enable = true,
807                "false" => fulltext_options.enable = false,
808                _ => {
809                    return InvalidFulltextOptionSnafu {
810                        msg: format!("{enable}, expected: 'true' | 'false'"),
811                    }
812                    .fail();
813                }
814            }
815        };
816
817        if let Some(analyzer) = options.get(COLUMN_FULLTEXT_OPT_KEY_ANALYZER) {
818            match analyzer.to_ascii_lowercase().as_str() {
819                "english" => fulltext_options.analyzer = FulltextAnalyzer::English,
820                "chinese" => fulltext_options.analyzer = FulltextAnalyzer::Chinese,
821                _ => {
822                    return InvalidFulltextOptionSnafu {
823                        msg: format!("{analyzer}, expected: 'English' | 'Chinese'"),
824                    }
825                    .fail();
826                }
827            }
828        };
829
830        if let Some(case_sensitive) = options.get(COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE) {
831            match case_sensitive.to_ascii_lowercase().as_str() {
832                "true" => fulltext_options.case_sensitive = true,
833                "false" => fulltext_options.case_sensitive = false,
834                _ => {
835                    return InvalidFulltextOptionSnafu {
836                        msg: format!("{case_sensitive}, expected: 'true' | 'false'"),
837                    }
838                    .fail();
839                }
840            }
841        }
842
843        if let Some(backend) = options.get(COLUMN_FULLTEXT_OPT_KEY_BACKEND) {
844            match backend.to_ascii_lowercase().as_str() {
845                "bloom" => fulltext_options.backend = FulltextBackend::Bloom,
846                "tantivy" => fulltext_options.backend = FulltextBackend::Tantivy,
847                _ => {
848                    return InvalidFulltextOptionSnafu {
849                        msg: format!("{backend}, expected: 'bloom' | 'tantivy'"),
850                    }
851                    .fail();
852                }
853            }
854        }
855
856        if fulltext_options.backend == FulltextBackend::Bloom {
857            // Parse granularity with default value 10240
858            let granularity = match options.get(COLUMN_FULLTEXT_OPT_KEY_GRANULARITY) {
859                Some(value) => value
860                    .parse::<u32>()
861                    .ok()
862                    .filter(|&v| v > 0)
863                    .ok_or_else(|| {
864                        error::InvalidFulltextOptionSnafu {
865                            msg: format!(
866                                "Invalid granularity: {value}, expected: positive integer"
867                            ),
868                        }
869                        .build()
870                    })?,
871                None => DEFAULT_GRANULARITY,
872            };
873            fulltext_options.granularity = granularity;
874
875            // Parse false positive rate with default value 0.01
876            let false_positive_rate = match options.get(COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE)
877            {
878                Some(value) => value
879                    .parse::<f64>()
880                    .ok()
881                    .filter(|&v| v > 0.0 && v <= 1.0)
882                    .ok_or_else(|| {
883                        error::InvalidFulltextOptionSnafu {
884                            msg: format!(
885                                "Invalid false positive rate: {value}, expected: 0.0 < rate <= 1.0"
886                            ),
887                        }
888                        .build()
889                    })?,
890                None => DEFAULT_FALSE_POSITIVE_RATE,
891            };
892            fulltext_options.false_positive_rate_in_10000 = (false_positive_rate * 10000.0) as u32;
893        }
894
895        Ok(fulltext_options)
896    }
897}
898
899/// Fulltext analyzer.
900#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
901pub enum FulltextAnalyzer {
902    #[default]
903    English,
904    Chinese,
905}
906
907impl fmt::Display for FulltextAnalyzer {
908    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
909        match self {
910            FulltextAnalyzer::English => write!(f, "English"),
911            FulltextAnalyzer::Chinese => write!(f, "Chinese"),
912        }
913    }
914}
915
916/// Skipping options for a column.
917#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
918#[serde(rename_all = "kebab-case")]
919pub struct SkippingIndexOptions {
920    /// The granularity of the skip index.
921    pub granularity: u32,
922    /// The false positive rate of the skip index (in ten-thousandths, e.g., 100 = 1%).
923    #[serde(default = "index_options_default_false_positive_rate_in_10000")]
924    pub false_positive_rate_in_10000: u32,
925    /// The type of the skip index.
926    #[serde(default)]
927    pub index_type: SkippingIndexType,
928}
929
930impl SkippingIndexOptions {
931    /// Creates a new skipping index options without checking `false_positive_rate` and `granularity`.
932    pub fn new_unchecked(
933        granularity: u32,
934        false_positive_rate: f64,
935        index_type: SkippingIndexType,
936    ) -> Self {
937        Self {
938            granularity,
939            false_positive_rate_in_10000: (false_positive_rate * 10000.0) as u32,
940            index_type,
941        }
942    }
943
944    /// Creates a new skipping index options.
945    pub fn new(
946        granularity: u32,
947        false_positive_rate: f64,
948        index_type: SkippingIndexType,
949    ) -> Result<Self> {
950        ensure!(
951            0.0 < false_positive_rate && false_positive_rate <= 1.0,
952            error::InvalidSkippingIndexOptionSnafu {
953                msg: format!(
954                    "Invalid false positive rate: {false_positive_rate}, expected: 0.0 < rate <= 1.0"
955                ),
956            }
957        );
958        ensure!(
959            granularity > 0,
960            error::InvalidSkippingIndexOptionSnafu {
961                msg: format!("Invalid granularity: {granularity}, expected: positive integer"),
962            }
963        );
964        Ok(Self::new_unchecked(
965            granularity,
966            false_positive_rate,
967            index_type,
968        ))
969    }
970
971    /// Gets the false positive rate.
972    pub fn false_positive_rate(&self) -> f64 {
973        self.false_positive_rate_in_10000 as f64 / 10000.0
974    }
975}
976
977impl Default for SkippingIndexOptions {
978    fn default() -> Self {
979        Self::new_unchecked(
980            DEFAULT_GRANULARITY,
981            DEFAULT_FALSE_POSITIVE_RATE,
982            SkippingIndexType::default(),
983        )
984    }
985}
986
987impl fmt::Display for SkippingIndexOptions {
988    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
989        write!(f, "granularity={}", self.granularity)?;
990        write!(f, ", false_positive_rate={}", self.false_positive_rate())?;
991        write!(f, ", index_type={}", self.index_type)?;
992        Ok(())
993    }
994}
995
996/// Skip index types.
997#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
998pub enum SkippingIndexType {
999    #[default]
1000    BloomFilter,
1001}
1002
1003impl fmt::Display for SkippingIndexType {
1004    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1005        match self {
1006            SkippingIndexType::BloomFilter => write!(f, "BLOOM"),
1007        }
1008    }
1009}
1010
1011impl TryFrom<HashMap<String, String>> for SkippingIndexOptions {
1012    type Error = Error;
1013
1014    fn try_from(options: HashMap<String, String>) -> Result<Self> {
1015        // Parse granularity with default value 1
1016        let granularity = match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY) {
1017            Some(value) => value
1018                .parse::<u32>()
1019                .ok()
1020                .filter(|&v| v > 0)
1021                .ok_or_else(|| {
1022                    error::InvalidSkippingIndexOptionSnafu {
1023                        msg: format!("Invalid granularity: {value}, expected: positive integer"),
1024                    }
1025                    .build()
1026                })?,
1027            None => DEFAULT_GRANULARITY,
1028        };
1029
1030        // Parse false positive rate with default value 100
1031        let false_positive_rate =
1032            match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE) {
1033                Some(value) => value
1034                    .parse::<f64>()
1035                    .ok()
1036                    .filter(|&v| v > 0.0 && v <= 1.0)
1037                    .ok_or_else(|| {
1038                        error::InvalidSkippingIndexOptionSnafu {
1039                            msg: format!(
1040                                "Invalid false positive rate: {value}, expected: 0.0 < rate <= 1.0"
1041                            ),
1042                        }
1043                        .build()
1044                    })?,
1045                None => DEFAULT_FALSE_POSITIVE_RATE,
1046            };
1047
1048        // Parse index type with default value BloomFilter
1049        let index_type = match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE) {
1050            Some(typ) => match typ.to_ascii_uppercase().as_str() {
1051                "BLOOM" => SkippingIndexType::BloomFilter,
1052                _ => {
1053                    return error::InvalidSkippingIndexOptionSnafu {
1054                        msg: format!("Invalid index type: {typ}, expected: 'BLOOM'"),
1055                    }
1056                    .fail();
1057                }
1058            },
1059            None => SkippingIndexType::default(),
1060        };
1061
1062        Ok(SkippingIndexOptions::new_unchecked(
1063            granularity,
1064            false_positive_rate,
1065            index_type,
1066        ))
1067    }
1068}
1069
1070/// Distance metric for vector similarity search.
1071#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
1072#[serde(rename_all = "lowercase")]
1073pub enum VectorDistanceMetric {
1074    /// Squared Euclidean distance (L2^2).
1075    #[default]
1076    L2sq,
1077    /// Cosine distance (1 - cosine similarity).
1078    Cosine,
1079    /// Inner product (negative, for maximum inner product search).
1080    #[serde(alias = "ip")]
1081    InnerProduct,
1082}
1083
1084impl fmt::Display for VectorDistanceMetric {
1085    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1086        match self {
1087            VectorDistanceMetric::L2sq => write!(f, "l2sq"),
1088            VectorDistanceMetric::Cosine => write!(f, "cosine"),
1089            VectorDistanceMetric::InnerProduct => write!(f, "ip"),
1090        }
1091    }
1092}
1093
1094impl std::str::FromStr for VectorDistanceMetric {
1095    type Err = String;
1096
1097    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
1098        match s.to_lowercase().as_str() {
1099            "l2sq" | "l2" | "euclidean" => Ok(VectorDistanceMetric::L2sq),
1100            "cosine" | "cos" => Ok(VectorDistanceMetric::Cosine),
1101            "inner_product" | "ip" | "dot" => Ok(VectorDistanceMetric::InnerProduct),
1102            _ => Err(format!(
1103                "Unknown distance metric: {}. Expected: l2sq, cosine, or ip",
1104                s
1105            )),
1106        }
1107    }
1108}
1109
1110impl VectorDistanceMetric {
1111    /// Returns the metric as u8 for blob serialization.
1112    pub fn as_u8(&self) -> u8 {
1113        match self {
1114            Self::L2sq => 0,
1115            Self::Cosine => 1,
1116            Self::InnerProduct => 2,
1117        }
1118    }
1119
1120    /// Parses metric from u8 (used when reading blob).
1121    pub fn try_from_u8(v: u8) -> Option<Self> {
1122        match v {
1123            0 => Some(Self::L2sq),
1124            1 => Some(Self::Cosine),
1125            2 => Some(Self::InnerProduct),
1126            _ => None,
1127        }
1128    }
1129}
1130
1131/// Default HNSW connectivity parameter.
1132const DEFAULT_VECTOR_INDEX_CONNECTIVITY: u32 = 16;
1133/// Default expansion factor during index construction.
1134const DEFAULT_VECTOR_INDEX_EXPANSION_ADD: u32 = 128;
1135/// Default expansion factor during search.
1136const DEFAULT_VECTOR_INDEX_EXPANSION_SEARCH: u32 = 64;
1137
1138fn default_vector_index_connectivity() -> u32 {
1139    DEFAULT_VECTOR_INDEX_CONNECTIVITY
1140}
1141
1142fn default_vector_index_expansion_add() -> u32 {
1143    DEFAULT_VECTOR_INDEX_EXPANSION_ADD
1144}
1145
1146fn default_vector_index_expansion_search() -> u32 {
1147    DEFAULT_VECTOR_INDEX_EXPANSION_SEARCH
1148}
1149
1150/// Supported vector index engine types.
1151#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, Visit, VisitMut)]
1152#[serde(rename_all = "lowercase")]
1153pub enum VectorIndexEngineType {
1154    /// USearch HNSW implementation.
1155    #[default]
1156    Usearch,
1157    // Future: Vsag,
1158}
1159
1160impl VectorIndexEngineType {
1161    /// Returns the engine type as u8 for blob serialization.
1162    pub fn as_u8(&self) -> u8 {
1163        match self {
1164            Self::Usearch => 0,
1165        }
1166    }
1167
1168    /// Parses engine type from u8 (used when reading blob).
1169    pub fn try_from_u8(v: u8) -> Option<Self> {
1170        match v {
1171            0 => Some(Self::Usearch),
1172            _ => None,
1173        }
1174    }
1175}
1176
1177impl fmt::Display for VectorIndexEngineType {
1178    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1179        match self {
1180            Self::Usearch => write!(f, "usearch"),
1181        }
1182    }
1183}
1184
1185impl std::str::FromStr for VectorIndexEngineType {
1186    type Err = String;
1187
1188    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
1189        match s.to_lowercase().as_str() {
1190            "usearch" => Ok(Self::Usearch),
1191            _ => Err(format!(
1192                "Unknown vector index engine: {}. Expected: usearch",
1193                s
1194            )),
1195        }
1196    }
1197}
1198
1199/// Options for vector index (HNSW).
1200#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
1201#[serde(rename_all = "kebab-case")]
1202pub struct VectorIndexOptions {
1203    /// Vector index engine type (default: usearch).
1204    #[serde(default)]
1205    pub engine: VectorIndexEngineType,
1206    /// Distance metric for similarity search.
1207    #[serde(default)]
1208    pub metric: VectorDistanceMetric,
1209    /// HNSW connectivity parameter (M in the paper).
1210    /// Higher values improve recall but increase memory usage.
1211    #[serde(default = "default_vector_index_connectivity")]
1212    pub connectivity: u32,
1213    /// Expansion factor during index construction (ef_construction).
1214    /// Higher values improve index quality but slow down construction.
1215    #[serde(default = "default_vector_index_expansion_add")]
1216    pub expansion_add: u32,
1217    /// Expansion factor during search (ef_search).
1218    /// Higher values improve recall but slow down search.
1219    #[serde(default = "default_vector_index_expansion_search")]
1220    pub expansion_search: u32,
1221}
1222
1223impl Default for VectorIndexOptions {
1224    fn default() -> Self {
1225        Self {
1226            engine: VectorIndexEngineType::default(),
1227            metric: VectorDistanceMetric::default(),
1228            connectivity: DEFAULT_VECTOR_INDEX_CONNECTIVITY,
1229            expansion_add: DEFAULT_VECTOR_INDEX_EXPANSION_ADD,
1230            expansion_search: DEFAULT_VECTOR_INDEX_EXPANSION_SEARCH,
1231        }
1232    }
1233}
1234
1235impl fmt::Display for VectorIndexOptions {
1236    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1237        write!(
1238            f,
1239            "engine={}, metric={}, connectivity={}, expansion_add={}, expansion_search={}",
1240            self.engine, self.metric, self.connectivity, self.expansion_add, self.expansion_search
1241        )
1242    }
1243}
1244
1245#[cfg(test)]
1246mod tests {
1247    use std::sync::Arc;
1248
1249    use arrow::datatypes::{DataType as ArrowDataType, TimeUnit};
1250
1251    use super::*;
1252    use crate::extension::json::{Json2ExtensionType, JsonExtensionType};
1253    use crate::types::{StructField, StructType};
1254    use crate::value::Value;
1255    use crate::vectors::Int32Vector;
1256
1257    #[test]
1258    fn test_column_schema() {
1259        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1260        let field = Field::try_from(&column_schema).unwrap();
1261        assert_eq!("test", field.name());
1262        assert_eq!(ArrowDataType::Int32, *field.data_type());
1263        assert!(field.is_nullable());
1264
1265        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1266        assert_eq!(column_schema, new_column_schema);
1267    }
1268
1269    #[test]
1270    fn test_with_extension_type_replaces_metadata() {
1271        let mut schema = ColumnSchema::new("j", ConcreteDataType::json_datatype(), true);
1272
1273        schema.with_extension_type(&Json2ExtensionType::default());
1274        assert_eq!(
1275            Some(Json2ExtensionType::NAME),
1276            schema
1277                .metadata()
1278                .get(EXTENSION_TYPE_NAME_KEY)
1279                .map(String::as_str)
1280        );
1281        assert!(schema.metadata().contains_key(EXTENSION_TYPE_METADATA_KEY));
1282
1283        schema.with_extension_type(&JsonExtensionType);
1284        assert_eq!(
1285            Some(JsonExtensionType::NAME),
1286            schema
1287                .metadata()
1288                .get(EXTENSION_TYPE_NAME_KEY)
1289                .map(String::as_str)
1290        );
1291        assert!(!schema.metadata().contains_key(EXTENSION_TYPE_METADATA_KEY));
1292    }
1293
1294    #[test]
1295    fn test_column_schema_with_default_constraint() {
1296        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1297            .with_default_constraint(Some(ColumnDefaultConstraint::Value(Value::from(99))))
1298            .unwrap();
1299        assert!(
1300            column_schema
1301                .metadata()
1302                .get(DEFAULT_CONSTRAINT_KEY)
1303                .is_none()
1304        );
1305
1306        let field = Field::try_from(&column_schema).unwrap();
1307        assert_eq!("test", field.name());
1308        assert_eq!(ArrowDataType::Int32, *field.data_type());
1309        assert!(field.is_nullable());
1310        assert_eq!(
1311            "{\"Value\":{\"Int32\":99}}",
1312            field.metadata().get(DEFAULT_CONSTRAINT_KEY).unwrap()
1313        );
1314
1315        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1316        assert_eq!(column_schema, new_column_schema);
1317    }
1318
1319    #[test]
1320    fn test_column_schema_with_metadata() {
1321        let metadata = Metadata::from([
1322            ("k1".to_string(), "v1".to_string()),
1323            (COMMENT_KEY.to_string(), "test comment".to_string()),
1324        ]);
1325        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1326            .with_metadata(metadata)
1327            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1328            .unwrap();
1329        assert_eq!("v1", column_schema.metadata().get("k1").unwrap());
1330        assert_eq!("test comment", column_schema.column_comment().unwrap());
1331        assert!(
1332            column_schema
1333                .metadata()
1334                .get(DEFAULT_CONSTRAINT_KEY)
1335                .is_none()
1336        );
1337
1338        let field = Field::try_from(&column_schema).unwrap();
1339        assert_eq!("v1", field.metadata().get("k1").unwrap());
1340        let _ = field.metadata().get(DEFAULT_CONSTRAINT_KEY).unwrap();
1341
1342        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1343        assert_eq!(column_schema, new_column_schema);
1344    }
1345
1346    #[test]
1347    fn test_column_schema_with_duplicate_metadata() {
1348        let metadata = Metadata::from([(DEFAULT_CONSTRAINT_KEY.to_string(), "v1".to_string())]);
1349        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1350            .with_metadata(metadata)
1351            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1352            .unwrap();
1353        assert!(Field::try_from(&column_schema).is_err());
1354    }
1355
1356    #[test]
1357    fn test_column_schema_invalid_default_constraint() {
1358        assert!(
1359            ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false)
1360                .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1361                .is_err()
1362        );
1363    }
1364
1365    #[test]
1366    fn test_column_default_constraint_try_into_from() {
1367        let default_constraint = ColumnDefaultConstraint::Value(Value::from(42i64));
1368
1369        let bytes: Vec<u8> = default_constraint.clone().try_into().unwrap();
1370        let from_value = ColumnDefaultConstraint::try_from(&bytes[..]).unwrap();
1371
1372        assert_eq!(default_constraint, from_value);
1373    }
1374
1375    #[test]
1376    fn test_column_schema_create_default_null() {
1377        // Implicit default null.
1378        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1379        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1380        assert_eq!(5, v.len());
1381        assert!(v.only_null());
1382
1383        // Explicit default null.
1384        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1385            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1386            .unwrap();
1387        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1388        assert_eq!(5, v.len());
1389        assert!(v.only_null());
1390    }
1391
1392    #[test]
1393    fn test_column_schema_create_default_null_struct_with_list() {
1394        let list_type =
1395            ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype()));
1396        let struct_type =
1397            ConcreteDataType::struct_datatype(StructType::new(Arc::new(vec![StructField::new(
1398                "values".to_string(),
1399                list_type,
1400                true,
1401            )])));
1402        let column_schema = ColumnSchema::new("test", struct_type, true);
1403
1404        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1405
1406        assert_eq!(5, v.len());
1407        assert!(v.only_null());
1408    }
1409
1410    #[test]
1411    fn test_column_schema_no_default() {
1412        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1413        assert!(column_schema.create_default_vector(5).unwrap().is_none());
1414    }
1415
1416    #[test]
1417    fn test_create_default_vector_for_padding() {
1418        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1419        let vector = column_schema.create_default_vector_for_padding(4);
1420        assert!(vector.only_null());
1421        assert_eq!(4, vector.len());
1422
1423        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1424        let vector = column_schema.create_default_vector_for_padding(4);
1425        assert_eq!(4, vector.len());
1426        let expect: VectorRef = Arc::new(Int32Vector::from_slice([0, 0, 0, 0]));
1427        assert_eq!(expect, vector);
1428    }
1429
1430    #[test]
1431    fn test_column_schema_single_create_default_null() {
1432        // Implicit default null.
1433        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1434        let v = column_schema.create_default().unwrap().unwrap();
1435        assert!(v.is_null());
1436
1437        // Explicit default null.
1438        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1439            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1440            .unwrap();
1441        let v = column_schema.create_default().unwrap().unwrap();
1442        assert!(v.is_null());
1443    }
1444
1445    #[test]
1446    fn test_column_schema_single_create_default_not_null() {
1447        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1448            .with_default_constraint(Some(ColumnDefaultConstraint::Value(Value::Int32(6))))
1449            .unwrap();
1450        let v = column_schema.create_default().unwrap().unwrap();
1451        assert_eq!(v, Value::Int32(6));
1452    }
1453
1454    #[test]
1455    fn test_column_schema_single_no_default() {
1456        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1457        assert!(column_schema.create_default().unwrap().is_none());
1458    }
1459
1460    #[test]
1461    fn test_debug_for_column_schema() {
1462        let column_schema_int8 =
1463            ColumnSchema::new("test_column_1", ConcreteDataType::int8_datatype(), true);
1464
1465        let column_schema_int32 =
1466            ColumnSchema::new("test_column_2", ConcreteDataType::int32_datatype(), false);
1467
1468        let formatted_int8 = format!("{:?}", column_schema_int8);
1469        let formatted_int32 = format!("{:?}", column_schema_int32);
1470        assert_eq!(formatted_int8, "test_column_1 Int8 null");
1471        assert_eq!(formatted_int32, "test_column_2 Int32 not null");
1472    }
1473
1474    #[test]
1475    fn test_from_field_to_column_schema() {
1476        let field = Field::new("test", ArrowDataType::Int32, true);
1477        let column_schema = ColumnSchema::try_from(&field).unwrap();
1478        assert_eq!("test", column_schema.name);
1479        assert_eq!(ConcreteDataType::int32_datatype(), column_schema.data_type);
1480        assert!(column_schema.is_nullable);
1481        assert!(!column_schema.is_time_index);
1482        assert!(column_schema.default_constraint.is_none());
1483        assert!(column_schema.metadata.is_empty());
1484
1485        let field = Field::new("test", ArrowDataType::Binary, true);
1486        let field = field.with_metadata(Metadata::from([(
1487            TYPE_KEY.to_string(),
1488            ConcreteDataType::json_datatype().name(),
1489        )]));
1490        let column_schema = ColumnSchema::try_from(&field).unwrap();
1491        assert_eq!("test", column_schema.name);
1492        assert_eq!(ConcreteDataType::json_datatype(), column_schema.data_type);
1493        assert!(column_schema.is_nullable);
1494        assert!(!column_schema.is_time_index);
1495        assert!(column_schema.default_constraint.is_none());
1496        assert_eq!(
1497            column_schema.metadata.get(TYPE_KEY).unwrap(),
1498            &ConcreteDataType::json_datatype().name()
1499        );
1500
1501        let field = Field::new("test", ArrowDataType::Binary, true);
1502        let field = field.with_metadata(Metadata::from([(
1503            TYPE_KEY.to_string(),
1504            ConcreteDataType::vector_datatype(3).name(),
1505        )]));
1506        let column_schema = ColumnSchema::try_from(&field).unwrap();
1507        assert_eq!("test", column_schema.name);
1508        assert_eq!(
1509            ConcreteDataType::vector_datatype(3),
1510            column_schema.data_type
1511        );
1512        assert!(column_schema.is_nullable);
1513        assert!(!column_schema.is_time_index);
1514        assert!(column_schema.default_constraint.is_none());
1515        assert_eq!(
1516            column_schema.metadata.get(TYPE_KEY).unwrap(),
1517            &ConcreteDataType::vector_datatype(3).name()
1518        );
1519    }
1520
1521    #[test]
1522    fn test_column_schema_fix_time_index() {
1523        let field = Field::new(
1524            "test",
1525            ArrowDataType::Timestamp(TimeUnit::Second, None),
1526            false,
1527        );
1528        let field = field.with_metadata(Metadata::from([(
1529            TIME_INDEX_KEY.to_string(),
1530            "true".to_string(),
1531        )]));
1532        let column_schema = ColumnSchema::try_from(&field).unwrap();
1533        assert_eq!("test", column_schema.name);
1534        assert_eq!(
1535            ConcreteDataType::timestamp_second_datatype(),
1536            column_schema.data_type
1537        );
1538        assert!(!column_schema.is_nullable);
1539        assert!(column_schema.is_time_index);
1540        assert!(column_schema.default_constraint.is_none());
1541        assert_eq!(1, column_schema.metadata().len());
1542
1543        let field = Field::new("test", ArrowDataType::Int32, false);
1544        let field = field.with_metadata(Metadata::from([(
1545            TIME_INDEX_KEY.to_string(),
1546            "true".to_string(),
1547        )]));
1548        let column_schema = ColumnSchema::try_from(&field).unwrap();
1549        assert_eq!("test", column_schema.name);
1550        assert_eq!(ConcreteDataType::int32_datatype(), column_schema.data_type);
1551        assert!(!column_schema.is_nullable);
1552        assert!(!column_schema.is_time_index);
1553        assert!(column_schema.default_constraint.is_none());
1554        assert!(column_schema.metadata.is_empty());
1555    }
1556
1557    #[test]
1558    fn test_skipping_index_options_deserialization() {
1559        let original_options = "{\"granularity\":1024,\"false-positive-rate-in-10000\":10,\"index-type\":\"BloomFilter\"}";
1560        let options = serde_json::from_str::<SkippingIndexOptions>(original_options).unwrap();
1561        assert_eq!(1024, options.granularity);
1562        assert_eq!(SkippingIndexType::BloomFilter, options.index_type);
1563        assert_eq!(0.001, options.false_positive_rate());
1564
1565        let options_str = serde_json::to_string(&options).unwrap();
1566        assert_eq!(options_str, original_options);
1567    }
1568
1569    #[test]
1570    fn test_skipping_index_options_deserialization_v0_14_to_v0_15() {
1571        let options = "{\"granularity\":10240,\"index-type\":\"BloomFilter\"}";
1572        let options = serde_json::from_str::<SkippingIndexOptions>(options).unwrap();
1573        assert_eq!(10240, options.granularity);
1574        assert_eq!(SkippingIndexType::BloomFilter, options.index_type);
1575        assert_eq!(DEFAULT_FALSE_POSITIVE_RATE, options.false_positive_rate());
1576
1577        let options_str = serde_json::to_string(&options).unwrap();
1578        assert_eq!(
1579            options_str,
1580            "{\"granularity\":10240,\"false-positive-rate-in-10000\":100,\"index-type\":\"BloomFilter\"}"
1581        );
1582    }
1583
1584    #[test]
1585    fn test_fulltext_options_deserialization() {
1586        let original_options = "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":1024,\"false-positive-rate-in-10000\":10}";
1587        let options = serde_json::from_str::<FulltextOptions>(original_options).unwrap();
1588        assert!(!options.case_sensitive);
1589        assert!(options.enable);
1590        assert_eq!(FulltextBackend::Bloom, options.backend);
1591        assert_eq!(FulltextAnalyzer::default(), options.analyzer);
1592        assert_eq!(1024, options.granularity);
1593        assert_eq!(0.001, options.false_positive_rate());
1594
1595        let options_str = serde_json::to_string(&options).unwrap();
1596        assert_eq!(options_str, original_options);
1597    }
1598
1599    #[test]
1600    fn test_fulltext_options_deserialization_v0_14_to_v0_15() {
1601        // 0.14 to 0.15
1602        let options = "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}";
1603        let options = serde_json::from_str::<FulltextOptions>(options).unwrap();
1604        assert!(!options.case_sensitive);
1605        assert!(options.enable);
1606        assert_eq!(FulltextBackend::Bloom, options.backend);
1607        assert_eq!(FulltextAnalyzer::default(), options.analyzer);
1608        assert_eq!(DEFAULT_GRANULARITY, options.granularity);
1609        assert_eq!(DEFAULT_FALSE_POSITIVE_RATE, options.false_positive_rate());
1610
1611        let options_str = serde_json::to_string(&options).unwrap();
1612        assert_eq!(
1613            options_str,
1614            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
1615        );
1616    }
1617}
datatypes/schema/column_schema.rs

datatypes/schema/
column_schema.rs
