datatypes/schema/
column_schema.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::str::FromStr;
17use std::{fmt, mem};
18
19use arrow::datatypes::Field;
20use arrow_schema::extension::{
21    EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY, ExtensionType,
22};
23use serde::{Deserialize, Serialize};
24use snafu::{ResultExt, ensure};
25use sqlparser_derive::{Visit, VisitMut};
26
27use crate::data_type::{ConcreteDataType, DataType};
28use crate::error::{
29    self, ArrowMetadataSnafu, Error, InvalidFulltextOptionSnafu, ParseExtendedTypeSnafu, Result,
30};
31use crate::schema::TYPE_KEY;
32use crate::schema::constraint::ColumnDefaultConstraint;
33use crate::value::Value;
34use crate::vectors::VectorRef;
35
36pub type Metadata = HashMap<String, String>;
37
38/// Key used to store whether the column is time index in arrow field's metadata.
39pub const TIME_INDEX_KEY: &str = "greptime:time_index";
40pub const COMMENT_KEY: &str = "greptime:storage:comment";
41/// Key used to store default constraint in arrow field's metadata.
42const DEFAULT_CONSTRAINT_KEY: &str = "greptime:default_constraint";
43/// Key used to store fulltext options in arrow field's metadata.
44pub const FULLTEXT_KEY: &str = "greptime:fulltext";
45/// Key used to store whether the column has inverted index in arrow field's metadata.
46pub const INVERTED_INDEX_KEY: &str = "greptime:inverted_index";
47/// Key used to store skip options in arrow field's metadata.
48pub const SKIPPING_INDEX_KEY: &str = "greptime:skipping_index";
49/// Key used to store vector index options in arrow field's metadata.
50pub const VECTOR_INDEX_KEY: &str = "greptime:vector_index";
51
52/// Keys used in fulltext options
53pub const COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE: &str = "enable";
54pub const COLUMN_FULLTEXT_OPT_KEY_ANALYZER: &str = "analyzer";
55pub const COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE: &str = "case_sensitive";
56pub const COLUMN_FULLTEXT_OPT_KEY_BACKEND: &str = "backend";
57pub const COLUMN_FULLTEXT_OPT_KEY_GRANULARITY: &str = "granularity";
58pub const COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE: &str = "false_positive_rate";
59
60/// Keys used in SKIPPING index options
61pub const COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY: &str = "granularity";
62pub const COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE: &str = "false_positive_rate";
63pub const COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE: &str = "type";
64
65/// Keys used in VECTOR index options
66pub const COLUMN_VECTOR_INDEX_OPT_KEY_ENGINE: &str = "engine";
67pub const COLUMN_VECTOR_INDEX_OPT_KEY_METRIC: &str = "metric";
68pub const COLUMN_VECTOR_INDEX_OPT_KEY_CONNECTIVITY: &str = "connectivity";
69pub const COLUMN_VECTOR_INDEX_OPT_KEY_EXPANSION_ADD: &str = "expansion_add";
70pub const COLUMN_VECTOR_INDEX_OPT_KEY_EXPANSION_SEARCH: &str = "expansion_search";
71
72pub const DEFAULT_GRANULARITY: u32 = 10240;
73
74pub const DEFAULT_FALSE_POSITIVE_RATE: f64 = 0.01;
75
76/// Schema of a column, used as an immutable struct.
77#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
78pub struct ColumnSchema {
79    pub name: String,
80    pub data_type: ConcreteDataType,
81    is_nullable: bool,
82    is_time_index: bool,
83    default_constraint: Option<ColumnDefaultConstraint>,
84    metadata: Metadata,
85}
86
87impl fmt::Debug for ColumnSchema {
88    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
89        write!(
90            f,
91            "{} {} {}",
92            self.name,
93            self.data_type,
94            if self.is_nullable { "null" } else { "not null" },
95        )?;
96
97        if self.is_time_index {
98            write!(f, " time_index")?;
99        }
100
101        // Add default constraint if present
102        if let Some(default_constraint) = &self.default_constraint {
103            write!(f, " default={:?}", default_constraint)?;
104        }
105
106        // Add metadata if present
107        if !self.metadata.is_empty() {
108            write!(f, " metadata={:?}", self.metadata)?;
109        }
110
111        Ok(())
112    }
113}
114
115impl ColumnSchema {
116    pub fn new<T: Into<String>>(
117        name: T,
118        data_type: ConcreteDataType,
119        is_nullable: bool,
120    ) -> ColumnSchema {
121        ColumnSchema {
122            name: name.into(),
123            data_type,
124            is_nullable,
125            is_time_index: false,
126            default_constraint: None,
127            metadata: Metadata::new(),
128        }
129    }
130
131    #[inline]
132    pub fn is_time_index(&self) -> bool {
133        self.is_time_index
134    }
135
136    #[inline]
137    pub fn is_nullable(&self) -> bool {
138        self.is_nullable
139    }
140
141    #[inline]
142    pub fn default_constraint(&self) -> Option<&ColumnDefaultConstraint> {
143        self.default_constraint.as_ref()
144    }
145
146    /// Check if the default constraint is a impure function.
147    pub fn is_default_impure(&self) -> bool {
148        self.default_constraint
149            .as_ref()
150            .map(|c| c.is_function())
151            .unwrap_or(false)
152    }
153
154    #[inline]
155    pub fn metadata(&self) -> &Metadata {
156        &self.metadata
157    }
158
159    #[inline]
160    pub fn mut_metadata(&mut self) -> &mut Metadata {
161        &mut self.metadata
162    }
163
164    /// Retrieve the column comment
165    pub fn column_comment(&self) -> Option<&String> {
166        self.metadata.get(COMMENT_KEY)
167    }
168
169    pub fn with_time_index(mut self, is_time_index: bool) -> Self {
170        self.is_time_index = is_time_index;
171        if is_time_index {
172            let _ = self
173                .metadata
174                .insert(TIME_INDEX_KEY.to_string(), "true".to_string());
175        } else {
176            let _ = self.metadata.remove(TIME_INDEX_KEY);
177        }
178        self
179    }
180
181    /// Returns the estimated memory footprint of this schema.
182    pub fn estimated_size(&self) -> usize {
183        mem::size_of_val(self) - mem::size_of_val(&self.data_type)
184            + self.data_type.as_arrow_type().size()
185            + self.name.capacity()
186            + self
187                .default_constraint
188                .as_ref()
189                .map(column_default_constraint_size)
190                .unwrap_or_default()
191            + metadata_size(&self.metadata)
192    }
193
194    /// Set the inverted index for the column.
195    /// Similar to [with_inverted_index] but don't take the ownership.
196    ///
197    /// [with_inverted_index]: Self::with_inverted_index
198    pub fn set_inverted_index(&mut self, value: bool) {
199        match value {
200            true => {
201                self.metadata
202                    .insert(INVERTED_INDEX_KEY.to_string(), value.to_string());
203            }
204            false => {
205                self.metadata.remove(INVERTED_INDEX_KEY);
206            }
207        }
208    }
209
210    /// Set the inverted index for the column.
211    /// Similar to [set_inverted_index] but take the ownership and return a owned value.
212    ///
213    /// [set_inverted_index]: Self::set_inverted_index
214    pub fn with_inverted_index(mut self, value: bool) -> Self {
215        self.set_inverted_index(value);
216        self
217    }
218
219    pub fn is_inverted_indexed(&self) -> bool {
220        self.metadata
221            .get(INVERTED_INDEX_KEY)
222            .map(|v| v.eq_ignore_ascii_case("true"))
223            .unwrap_or(false)
224    }
225
226    pub fn is_fulltext_indexed(&self) -> bool {
227        self.fulltext_options()
228            .unwrap_or_default()
229            .map(|option| option.enable)
230            .unwrap_or_default()
231    }
232
233    pub fn is_skipping_indexed(&self) -> bool {
234        self.skipping_index_options().unwrap_or_default().is_some()
235    }
236
237    pub fn has_inverted_index_key(&self) -> bool {
238        self.metadata.contains_key(INVERTED_INDEX_KEY)
239    }
240
241    /// Checks if this column has a vector index.
242    pub fn is_vector_indexed(&self) -> bool {
243        match self.vector_index_options() {
244            Ok(opts) => opts.is_some(),
245            Err(e) => {
246                common_telemetry::warn!(
247                    "Failed to deserialize vector_index_options for column '{}': {}",
248                    self.name,
249                    e
250                );
251                false
252            }
253        }
254    }
255
256    /// Gets the vector index options.
257    pub fn vector_index_options(&self) -> Result<Option<VectorIndexOptions>> {
258        match self.metadata.get(VECTOR_INDEX_KEY) {
259            None => Ok(None),
260            Some(json) => {
261                let options =
262                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
263                Ok(Some(options))
264            }
265        }
266    }
267
268    /// Sets the vector index options.
269    pub fn set_vector_index_options(&mut self, options: &VectorIndexOptions) -> Result<()> {
270        self.metadata.insert(
271            VECTOR_INDEX_KEY.to_string(),
272            serde_json::to_string(options).context(error::SerializeSnafu)?,
273        );
274        Ok(())
275    }
276
277    /// Removes the vector index options.
278    pub fn unset_vector_index_options(&mut self) {
279        self.metadata.remove(VECTOR_INDEX_KEY);
280    }
281
282    /// Sets vector index options and returns self for chaining.
283    pub fn with_vector_index_options(mut self, options: &VectorIndexOptions) -> Result<Self> {
284        self.set_vector_index_options(options)?;
285        Ok(self)
286    }
287
288    /// Set default constraint.
289    ///
290    /// If a default constraint exists for the column, this method will
291    /// validate it against the column's data type and nullability.
292    pub fn with_default_constraint(
293        mut self,
294        default_constraint: Option<ColumnDefaultConstraint>,
295    ) -> Result<Self> {
296        if let Some(constraint) = &default_constraint {
297            constraint.validate(&self.data_type, self.is_nullable)?;
298        }
299
300        self.default_constraint = default_constraint;
301        Ok(self)
302    }
303
304    /// Set the nullablity to `true` of the column.
305    /// Similar to [set_nullable] but take the ownership and return a owned value.
306    ///
307    /// [set_nullable]: Self::set_nullable
308    pub fn with_nullable_set(mut self) -> Self {
309        self.is_nullable = true;
310        self
311    }
312
313    /// Set the nullability to `true` of the column.
314    /// Similar to [with_nullable_set] but don't take the ownership
315    ///
316    /// [with_nullable_set]: Self::with_nullable_set
317    pub fn set_nullable(&mut self) {
318        self.is_nullable = true;
319    }
320
321    /// Set the `is_time_index` to `true` of the column.
322    /// Similar to [with_time_index] but don't take the ownership.
323    ///
324    /// [with_time_index]: Self::with_time_index
325    pub fn set_time_index(&mut self) {
326        self.is_time_index = true;
327    }
328
329    /// Creates a new [`ColumnSchema`] with given metadata.
330    pub fn with_metadata(mut self, metadata: Metadata) -> Self {
331        self.metadata = metadata;
332        self
333    }
334
335    /// Creates a vector with default value for this column.
336    ///
337    /// If the column is `NOT NULL` but doesn't has `DEFAULT` value supplied, returns `Ok(None)`.
338    pub fn create_default_vector(&self, num_rows: usize) -> Result<Option<VectorRef>> {
339        match &self.default_constraint {
340            Some(c) => c
341                .create_default_vector(&self.data_type, self.is_nullable, num_rows)
342                .map(Some),
343            None => {
344                if self.is_nullable {
345                    // No default constraint, use null as default value.
346                    // TODO(yingwen): Use NullVector once it supports setting logical type.
347                    ColumnDefaultConstraint::null_value()
348                        .create_default_vector(&self.data_type, self.is_nullable, num_rows)
349                        .map(Some)
350                } else {
351                    Ok(None)
352                }
353            }
354        }
355    }
356
357    /// Creates a vector for padding.
358    ///
359    /// This method always returns a vector since it uses [DataType::default_value]
360    /// to fill the vector. Callers should only use the created vector for padding
361    /// and never read its content.
362    pub fn create_default_vector_for_padding(&self, num_rows: usize) -> VectorRef {
363        let padding_value = if self.is_nullable {
364            Value::Null
365        } else {
366            // If the column is not null, use the data type's default value as it is
367            // more efficient to acquire.
368            self.data_type.default_value()
369        };
370        let value_ref = padding_value.as_value_ref();
371        let mut mutable_vector = self.data_type.create_mutable_vector(num_rows);
372        for _ in 0..num_rows {
373            mutable_vector.push_value_ref(&value_ref);
374        }
375        mutable_vector.to_vector()
376    }
377
378    /// Creates a default value for this column.
379    ///
380    /// If the column is `NOT NULL` but doesn't has `DEFAULT` value supplied, returns `Ok(None)`.
381    pub fn create_default(&self) -> Result<Option<Value>> {
382        match &self.default_constraint {
383            Some(c) => c
384                .create_default(&self.data_type, self.is_nullable)
385                .map(Some),
386            None => {
387                if self.is_nullable {
388                    // No default constraint, use null as default value.
389                    ColumnDefaultConstraint::null_value()
390                        .create_default(&self.data_type, self.is_nullable)
391                        .map(Some)
392                } else {
393                    Ok(None)
394                }
395            }
396        }
397    }
398
399    /// Creates an impure default value for this column, only if it have a impure default constraint.
400    /// Otherwise, returns `Ok(None)`.
401    pub fn create_impure_default(&self) -> Result<Option<Value>> {
402        match &self.default_constraint {
403            Some(c) => c.create_impure_default(&self.data_type),
404            None => Ok(None),
405        }
406    }
407
408    /// Retrieves the fulltext options for the column.
409    pub fn fulltext_options(&self) -> Result<Option<FulltextOptions>> {
410        match self.metadata.get(FULLTEXT_KEY) {
411            None => Ok(None),
412            Some(json) => {
413                let options =
414                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
415                Ok(Some(options))
416            }
417        }
418    }
419
420    pub fn with_fulltext_options(mut self, options: FulltextOptions) -> Result<Self> {
421        self.metadata.insert(
422            FULLTEXT_KEY.to_string(),
423            serde_json::to_string(&options).context(error::SerializeSnafu)?,
424        );
425        Ok(self)
426    }
427
428    pub fn set_fulltext_options(&mut self, options: &FulltextOptions) -> Result<()> {
429        self.metadata.insert(
430            FULLTEXT_KEY.to_string(),
431            serde_json::to_string(options).context(error::SerializeSnafu)?,
432        );
433        Ok(())
434    }
435
436    /// Retrieves the skipping index options for the column.
437    pub fn skipping_index_options(&self) -> Result<Option<SkippingIndexOptions>> {
438        match self.metadata.get(SKIPPING_INDEX_KEY) {
439            None => Ok(None),
440            Some(json) => {
441                let options =
442                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
443                Ok(Some(options))
444            }
445        }
446    }
447
448    pub fn with_skipping_options(mut self, options: SkippingIndexOptions) -> Result<Self> {
449        self.metadata.insert(
450            SKIPPING_INDEX_KEY.to_string(),
451            serde_json::to_string(&options).context(error::SerializeSnafu)?,
452        );
453        Ok(self)
454    }
455
456    pub fn set_skipping_options(&mut self, options: &SkippingIndexOptions) -> Result<()> {
457        self.metadata.insert(
458            SKIPPING_INDEX_KEY.to_string(),
459            serde_json::to_string(options).context(error::SerializeSnafu)?,
460        );
461        Ok(())
462    }
463
464    pub fn unset_skipping_options(&mut self) -> Result<()> {
465        self.metadata.remove(SKIPPING_INDEX_KEY);
466        Ok(())
467    }
468
469    pub fn extension_type<E>(&self) -> Result<Option<E>>
470    where
471        E: ExtensionType,
472    {
473        let extension_type_name = self.metadata.get(EXTENSION_TYPE_NAME_KEY);
474
475        if extension_type_name.map(|s| s.as_str()) == Some(E::NAME) {
476            let extension_metadata = self.metadata.get(EXTENSION_TYPE_METADATA_KEY);
477            let extension_metadata =
478                E::deserialize_metadata(extension_metadata.map(|s| s.as_str()))
479                    .context(ArrowMetadataSnafu)?;
480
481            let extension = E::try_new(&self.data_type.as_arrow_type(), extension_metadata)
482                .context(ArrowMetadataSnafu)?;
483            Ok(Some(extension))
484        } else {
485            Ok(None)
486        }
487    }
488
489    pub fn with_extension_type<E>(&mut self, extension_type: &E) -> Result<()>
490    where
491        E: ExtensionType,
492    {
493        self.metadata
494            .insert(EXTENSION_TYPE_NAME_KEY.to_string(), E::NAME.to_string());
495
496        if let Some(extension_metadata) = extension_type.serialize_metadata() {
497            self.metadata
498                .insert(EXTENSION_TYPE_METADATA_KEY.to_string(), extension_metadata);
499        }
500
501        Ok(())
502    }
503
504    pub fn is_indexed(&self) -> bool {
505        self.is_inverted_indexed() || self.is_fulltext_indexed() || self.is_skipping_indexed()
506    }
507}
508
509fn metadata_size(metadata: &Metadata) -> usize {
510    mem::size_of::<(String, String)>() * metadata.capacity()
511        + metadata
512            .iter()
513            .map(|(key, value)| key.capacity() + value.capacity())
514            .sum::<usize>()
515}
516
517fn column_default_constraint_size(default_constraint: &ColumnDefaultConstraint) -> usize {
518    match default_constraint {
519        ColumnDefaultConstraint::Function(expr) => expr.capacity(),
520        ColumnDefaultConstraint::Value(value) => value.as_value_ref().data_size(),
521    }
522}
523
524/// Column extended type set in column schema's metadata.
525#[derive(Debug, Clone, PartialEq, Eq)]
526pub enum ColumnExtType {
527    /// Json type.
528    Json,
529
530    /// Vector type with dimension.
531    Vector(u32),
532}
533
534impl fmt::Display for ColumnExtType {
535    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
536        match self {
537            ColumnExtType::Json => write!(f, "Json"),
538            ColumnExtType::Vector(dim) => write!(f, "Vector({})", dim),
539        }
540    }
541}
542
543impl FromStr for ColumnExtType {
544    type Err = String;
545
546    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
547        match s {
548            "Json" => Ok(ColumnExtType::Json),
549            _ if s.starts_with("Vector(") && s.ends_with(')') => s[7..s.len() - 1]
550                .parse::<u32>()
551                .map(ColumnExtType::Vector)
552                .map_err(|_| "Invalid dimension for Vector".to_string()),
553            _ => Err("Unknown variant".to_string()),
554        }
555    }
556}
557
558impl TryFrom<&Field> for ColumnSchema {
559    type Error = Error;
560
561    fn try_from(field: &Field) -> Result<ColumnSchema> {
562        let mut data_type = ConcreteDataType::try_from(field.data_type())?;
563        // Override the data type if it is specified in the metadata.
564        if let Some(s) = field.metadata().get(TYPE_KEY) {
565            let extype = ColumnExtType::from_str(s)
566                .map_err(|_| ParseExtendedTypeSnafu { value: s }.build())?;
567            match extype {
568                ColumnExtType::Json => {
569                    data_type = ConcreteDataType::json_datatype();
570                }
571                ColumnExtType::Vector(dim) => {
572                    data_type = ConcreteDataType::vector_datatype(dim);
573                }
574            }
575        }
576        let mut metadata = field.metadata().clone();
577        let default_constraint = match metadata.remove(DEFAULT_CONSTRAINT_KEY) {
578            Some(json) => {
579                Some(serde_json::from_str(&json).context(error::DeserializeSnafu { json })?)
580            }
581            None => None,
582        };
583        let mut is_time_index = metadata.contains_key(TIME_INDEX_KEY);
584        if is_time_index && !data_type.is_timestamp() {
585            // If the column is time index but the data type is not timestamp, it is invalid.
586            // We set the time index to false and remove the metadata.
587            // This is possible if we cast the time index column to another type. DataFusion will
588            // keep the metadata:
589            // https://github.com/apache/datafusion/pull/12951
590            is_time_index = false;
591            metadata.remove(TIME_INDEX_KEY);
592            common_telemetry::debug!(
593                "Column {} is not timestamp ({:?}) but has time index metadata",
594                data_type,
595                field.name(),
596            );
597        }
598
599        Ok(ColumnSchema {
600            name: field.name().clone(),
601            data_type,
602            is_nullable: field.is_nullable(),
603            is_time_index,
604            default_constraint,
605            metadata,
606        })
607    }
608}
609
610impl TryFrom<&ColumnSchema> for Field {
611    type Error = Error;
612
613    fn try_from(column_schema: &ColumnSchema) -> Result<Field> {
614        let mut metadata = column_schema.metadata.clone();
615        if let Some(value) = &column_schema.default_constraint {
616            // Adds an additional metadata to store the default constraint.
617            let old = metadata.insert(
618                DEFAULT_CONSTRAINT_KEY.to_string(),
619                serde_json::to_string(&value).context(error::SerializeSnafu)?,
620            );
621
622            ensure!(
623                old.is_none(),
624                error::DuplicateMetaSnafu {
625                    key: DEFAULT_CONSTRAINT_KEY,
626                }
627            );
628        }
629
630        Ok(Field::new(
631            &column_schema.name,
632            column_schema.data_type.as_arrow_type(),
633            column_schema.is_nullable(),
634        )
635        .with_metadata(metadata))
636    }
637}
638
639/// Fulltext options for a column.
640#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
641#[serde(rename_all = "kebab-case")]
642pub struct FulltextOptions {
643    /// Whether the fulltext index is enabled.
644    pub enable: bool,
645    /// The fulltext analyzer to use.
646    #[serde(default)]
647    pub analyzer: FulltextAnalyzer,
648    /// Whether the fulltext index is case-sensitive.
649    #[serde(default)]
650    pub case_sensitive: bool,
651    /// The fulltext backend to use.
652    #[serde(default)]
653    pub backend: FulltextBackend,
654    /// The granularity of the fulltext index (for bloom backend only)
655    #[serde(default = "fulltext_options_default_granularity")]
656    pub granularity: u32,
657    /// The false positive rate of the fulltext index (for bloom backend only)
658    #[serde(default = "index_options_default_false_positive_rate_in_10000")]
659    pub false_positive_rate_in_10000: u32,
660}
661
662fn fulltext_options_default_granularity() -> u32 {
663    DEFAULT_GRANULARITY
664}
665
666fn index_options_default_false_positive_rate_in_10000() -> u32 {
667    (DEFAULT_FALSE_POSITIVE_RATE * 10000.0) as u32
668}
669
670impl FulltextOptions {
671    /// Creates a new fulltext options.
672    pub fn new(
673        enable: bool,
674        analyzer: FulltextAnalyzer,
675        case_sensitive: bool,
676        backend: FulltextBackend,
677        granularity: u32,
678        false_positive_rate: f64,
679    ) -> Result<Self> {
680        ensure!(
681            0.0 < false_positive_rate && false_positive_rate <= 1.0,
682            error::InvalidFulltextOptionSnafu {
683                msg: format!(
684                    "Invalid false positive rate: {false_positive_rate}, expected: 0.0 < rate <= 1.0"
685                ),
686            }
687        );
688        ensure!(
689            granularity > 0,
690            error::InvalidFulltextOptionSnafu {
691                msg: format!("Invalid granularity: {granularity}, expected: positive integer"),
692            }
693        );
694        Ok(Self::new_unchecked(
695            enable,
696            analyzer,
697            case_sensitive,
698            backend,
699            granularity,
700            false_positive_rate,
701        ))
702    }
703
704    /// Creates a new fulltext options without checking `false_positive_rate` and `granularity`.
705    pub fn new_unchecked(
706        enable: bool,
707        analyzer: FulltextAnalyzer,
708        case_sensitive: bool,
709        backend: FulltextBackend,
710        granularity: u32,
711        false_positive_rate: f64,
712    ) -> Self {
713        Self {
714            enable,
715            analyzer,
716            case_sensitive,
717            backend,
718            granularity,
719            false_positive_rate_in_10000: (false_positive_rate * 10000.0) as u32,
720        }
721    }
722
723    /// Gets the false positive rate.
724    pub fn false_positive_rate(&self) -> f64 {
725        self.false_positive_rate_in_10000 as f64 / 10000.0
726    }
727}
728
729impl Default for FulltextOptions {
730    fn default() -> Self {
731        Self::new_unchecked(
732            false,
733            FulltextAnalyzer::default(),
734            false,
735            FulltextBackend::default(),
736            DEFAULT_GRANULARITY,
737            DEFAULT_FALSE_POSITIVE_RATE,
738        )
739    }
740}
741
742impl fmt::Display for FulltextOptions {
743    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
744        write!(f, "enable={}", self.enable)?;
745        if self.enable {
746            write!(f, ", analyzer={}", self.analyzer)?;
747            write!(f, ", case_sensitive={}", self.case_sensitive)?;
748            write!(f, ", backend={}", self.backend)?;
749            if self.backend == FulltextBackend::Bloom {
750                write!(f, ", granularity={}", self.granularity)?;
751                write!(f, ", false_positive_rate={}", self.false_positive_rate())?;
752            }
753        }
754        Ok(())
755    }
756}
757
758/// The backend of the fulltext index.
759#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
760#[serde(rename_all = "kebab-case")]
761pub enum FulltextBackend {
762    #[default]
763    Bloom,
764    Tantivy,
765}
766
767impl fmt::Display for FulltextBackend {
768    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
769        match self {
770            FulltextBackend::Tantivy => write!(f, "tantivy"),
771            FulltextBackend::Bloom => write!(f, "bloom"),
772        }
773    }
774}
775
776impl TryFrom<HashMap<String, String>> for FulltextOptions {
777    type Error = Error;
778
779    fn try_from(options: HashMap<String, String>) -> Result<Self> {
780        let mut fulltext_options = FulltextOptions {
781            enable: true,
782            ..Default::default()
783        };
784
785        if let Some(enable) = options.get(COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE) {
786            match enable.to_ascii_lowercase().as_str() {
787                "true" => fulltext_options.enable = true,
788                "false" => fulltext_options.enable = false,
789                _ => {
790                    return InvalidFulltextOptionSnafu {
791                        msg: format!("{enable}, expected: 'true' | 'false'"),
792                    }
793                    .fail();
794                }
795            }
796        };
797
798        if let Some(analyzer) = options.get(COLUMN_FULLTEXT_OPT_KEY_ANALYZER) {
799            match analyzer.to_ascii_lowercase().as_str() {
800                "english" => fulltext_options.analyzer = FulltextAnalyzer::English,
801                "chinese" => fulltext_options.analyzer = FulltextAnalyzer::Chinese,
802                _ => {
803                    return InvalidFulltextOptionSnafu {
804                        msg: format!("{analyzer}, expected: 'English' | 'Chinese'"),
805                    }
806                    .fail();
807                }
808            }
809        };
810
811        if let Some(case_sensitive) = options.get(COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE) {
812            match case_sensitive.to_ascii_lowercase().as_str() {
813                "true" => fulltext_options.case_sensitive = true,
814                "false" => fulltext_options.case_sensitive = false,
815                _ => {
816                    return InvalidFulltextOptionSnafu {
817                        msg: format!("{case_sensitive}, expected: 'true' | 'false'"),
818                    }
819                    .fail();
820                }
821            }
822        }
823
824        if let Some(backend) = options.get(COLUMN_FULLTEXT_OPT_KEY_BACKEND) {
825            match backend.to_ascii_lowercase().as_str() {
826                "bloom" => fulltext_options.backend = FulltextBackend::Bloom,
827                "tantivy" => fulltext_options.backend = FulltextBackend::Tantivy,
828                _ => {
829                    return InvalidFulltextOptionSnafu {
830                        msg: format!("{backend}, expected: 'bloom' | 'tantivy'"),
831                    }
832                    .fail();
833                }
834            }
835        }
836
837        if fulltext_options.backend == FulltextBackend::Bloom {
838            // Parse granularity with default value 10240
839            let granularity = match options.get(COLUMN_FULLTEXT_OPT_KEY_GRANULARITY) {
840                Some(value) => value
841                    .parse::<u32>()
842                    .ok()
843                    .filter(|&v| v > 0)
844                    .ok_or_else(|| {
845                        error::InvalidFulltextOptionSnafu {
846                            msg: format!(
847                                "Invalid granularity: {value}, expected: positive integer"
848                            ),
849                        }
850                        .build()
851                    })?,
852                None => DEFAULT_GRANULARITY,
853            };
854            fulltext_options.granularity = granularity;
855
856            // Parse false positive rate with default value 0.01
857            let false_positive_rate = match options.get(COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE)
858            {
859                Some(value) => value
860                    .parse::<f64>()
861                    .ok()
862                    .filter(|&v| v > 0.0 && v <= 1.0)
863                    .ok_or_else(|| {
864                        error::InvalidFulltextOptionSnafu {
865                            msg: format!(
866                                "Invalid false positive rate: {value}, expected: 0.0 < rate <= 1.0"
867                            ),
868                        }
869                        .build()
870                    })?,
871                None => DEFAULT_FALSE_POSITIVE_RATE,
872            };
873            fulltext_options.false_positive_rate_in_10000 = (false_positive_rate * 10000.0) as u32;
874        }
875
876        Ok(fulltext_options)
877    }
878}
879
880/// Fulltext analyzer.
881#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
882pub enum FulltextAnalyzer {
883    #[default]
884    English,
885    Chinese,
886}
887
888impl fmt::Display for FulltextAnalyzer {
889    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
890        match self {
891            FulltextAnalyzer::English => write!(f, "English"),
892            FulltextAnalyzer::Chinese => write!(f, "Chinese"),
893        }
894    }
895}
896
897/// Skipping options for a column.
898#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
899#[serde(rename_all = "kebab-case")]
900pub struct SkippingIndexOptions {
901    /// The granularity of the skip index.
902    pub granularity: u32,
903    /// The false positive rate of the skip index (in ten-thousandths, e.g., 100 = 1%).
904    #[serde(default = "index_options_default_false_positive_rate_in_10000")]
905    pub false_positive_rate_in_10000: u32,
906    /// The type of the skip index.
907    #[serde(default)]
908    pub index_type: SkippingIndexType,
909}
910
911impl SkippingIndexOptions {
912    /// Creates a new skipping index options without checking `false_positive_rate` and `granularity`.
913    pub fn new_unchecked(
914        granularity: u32,
915        false_positive_rate: f64,
916        index_type: SkippingIndexType,
917    ) -> Self {
918        Self {
919            granularity,
920            false_positive_rate_in_10000: (false_positive_rate * 10000.0) as u32,
921            index_type,
922        }
923    }
924
925    /// Creates a new skipping index options.
926    pub fn new(
927        granularity: u32,
928        false_positive_rate: f64,
929        index_type: SkippingIndexType,
930    ) -> Result<Self> {
931        ensure!(
932            0.0 < false_positive_rate && false_positive_rate <= 1.0,
933            error::InvalidSkippingIndexOptionSnafu {
934                msg: format!(
935                    "Invalid false positive rate: {false_positive_rate}, expected: 0.0 < rate <= 1.0"
936                ),
937            }
938        );
939        ensure!(
940            granularity > 0,
941            error::InvalidSkippingIndexOptionSnafu {
942                msg: format!("Invalid granularity: {granularity}, expected: positive integer"),
943            }
944        );
945        Ok(Self::new_unchecked(
946            granularity,
947            false_positive_rate,
948            index_type,
949        ))
950    }
951
952    /// Gets the false positive rate.
953    pub fn false_positive_rate(&self) -> f64 {
954        self.false_positive_rate_in_10000 as f64 / 10000.0
955    }
956}
957
958impl Default for SkippingIndexOptions {
959    fn default() -> Self {
960        Self::new_unchecked(
961            DEFAULT_GRANULARITY,
962            DEFAULT_FALSE_POSITIVE_RATE,
963            SkippingIndexType::default(),
964        )
965    }
966}
967
968impl fmt::Display for SkippingIndexOptions {
969    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
970        write!(f, "granularity={}", self.granularity)?;
971        write!(f, ", false_positive_rate={}", self.false_positive_rate())?;
972        write!(f, ", index_type={}", self.index_type)?;
973        Ok(())
974    }
975}
976
977/// Skip index types.
978#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
979pub enum SkippingIndexType {
980    #[default]
981    BloomFilter,
982}
983
984impl fmt::Display for SkippingIndexType {
985    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
986        match self {
987            SkippingIndexType::BloomFilter => write!(f, "BLOOM"),
988        }
989    }
990}
991
992impl TryFrom<HashMap<String, String>> for SkippingIndexOptions {
993    type Error = Error;
994
995    fn try_from(options: HashMap<String, String>) -> Result<Self> {
996        // Parse granularity with default value 1
997        let granularity = match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY) {
998            Some(value) => value
999                .parse::<u32>()
1000                .ok()
1001                .filter(|&v| v > 0)
1002                .ok_or_else(|| {
1003                    error::InvalidSkippingIndexOptionSnafu {
1004                        msg: format!("Invalid granularity: {value}, expected: positive integer"),
1005                    }
1006                    .build()
1007                })?,
1008            None => DEFAULT_GRANULARITY,
1009        };
1010
1011        // Parse false positive rate with default value 100
1012        let false_positive_rate =
1013            match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE) {
1014                Some(value) => value
1015                    .parse::<f64>()
1016                    .ok()
1017                    .filter(|&v| v > 0.0 && v <= 1.0)
1018                    .ok_or_else(|| {
1019                        error::InvalidSkippingIndexOptionSnafu {
1020                            msg: format!(
1021                                "Invalid false positive rate: {value}, expected: 0.0 < rate <= 1.0"
1022                            ),
1023                        }
1024                        .build()
1025                    })?,
1026                None => DEFAULT_FALSE_POSITIVE_RATE,
1027            };
1028
1029        // Parse index type with default value BloomFilter
1030        let index_type = match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE) {
1031            Some(typ) => match typ.to_ascii_uppercase().as_str() {
1032                "BLOOM" => SkippingIndexType::BloomFilter,
1033                _ => {
1034                    return error::InvalidSkippingIndexOptionSnafu {
1035                        msg: format!("Invalid index type: {typ}, expected: 'BLOOM'"),
1036                    }
1037                    .fail();
1038                }
1039            },
1040            None => SkippingIndexType::default(),
1041        };
1042
1043        Ok(SkippingIndexOptions::new_unchecked(
1044            granularity,
1045            false_positive_rate,
1046            index_type,
1047        ))
1048    }
1049}
1050
1051/// Distance metric for vector similarity search.
1052#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
1053#[serde(rename_all = "lowercase")]
1054pub enum VectorDistanceMetric {
1055    /// Squared Euclidean distance (L2^2).
1056    #[default]
1057    L2sq,
1058    /// Cosine distance (1 - cosine similarity).
1059    Cosine,
1060    /// Inner product (negative, for maximum inner product search).
1061    #[serde(alias = "ip")]
1062    InnerProduct,
1063}
1064
1065impl fmt::Display for VectorDistanceMetric {
1066    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1067        match self {
1068            VectorDistanceMetric::L2sq => write!(f, "l2sq"),
1069            VectorDistanceMetric::Cosine => write!(f, "cosine"),
1070            VectorDistanceMetric::InnerProduct => write!(f, "ip"),
1071        }
1072    }
1073}
1074
1075impl std::str::FromStr for VectorDistanceMetric {
1076    type Err = String;
1077
1078    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
1079        match s.to_lowercase().as_str() {
1080            "l2sq" | "l2" | "euclidean" => Ok(VectorDistanceMetric::L2sq),
1081            "cosine" | "cos" => Ok(VectorDistanceMetric::Cosine),
1082            "inner_product" | "ip" | "dot" => Ok(VectorDistanceMetric::InnerProduct),
1083            _ => Err(format!(
1084                "Unknown distance metric: {}. Expected: l2sq, cosine, or ip",
1085                s
1086            )),
1087        }
1088    }
1089}
1090
1091impl VectorDistanceMetric {
1092    /// Returns the metric as u8 for blob serialization.
1093    pub fn as_u8(&self) -> u8 {
1094        match self {
1095            Self::L2sq => 0,
1096            Self::Cosine => 1,
1097            Self::InnerProduct => 2,
1098        }
1099    }
1100
1101    /// Parses metric from u8 (used when reading blob).
1102    pub fn try_from_u8(v: u8) -> Option<Self> {
1103        match v {
1104            0 => Some(Self::L2sq),
1105            1 => Some(Self::Cosine),
1106            2 => Some(Self::InnerProduct),
1107            _ => None,
1108        }
1109    }
1110}
1111
1112/// Default HNSW connectivity parameter.
1113const DEFAULT_VECTOR_INDEX_CONNECTIVITY: u32 = 16;
1114/// Default expansion factor during index construction.
1115const DEFAULT_VECTOR_INDEX_EXPANSION_ADD: u32 = 128;
1116/// Default expansion factor during search.
1117const DEFAULT_VECTOR_INDEX_EXPANSION_SEARCH: u32 = 64;
1118
1119fn default_vector_index_connectivity() -> u32 {
1120    DEFAULT_VECTOR_INDEX_CONNECTIVITY
1121}
1122
1123fn default_vector_index_expansion_add() -> u32 {
1124    DEFAULT_VECTOR_INDEX_EXPANSION_ADD
1125}
1126
1127fn default_vector_index_expansion_search() -> u32 {
1128    DEFAULT_VECTOR_INDEX_EXPANSION_SEARCH
1129}
1130
1131/// Supported vector index engine types.
1132#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, Visit, VisitMut)]
1133#[serde(rename_all = "lowercase")]
1134pub enum VectorIndexEngineType {
1135    /// USearch HNSW implementation.
1136    #[default]
1137    Usearch,
1138    // Future: Vsag,
1139}
1140
1141impl VectorIndexEngineType {
1142    /// Returns the engine type as u8 for blob serialization.
1143    pub fn as_u8(&self) -> u8 {
1144        match self {
1145            Self::Usearch => 0,
1146        }
1147    }
1148
1149    /// Parses engine type from u8 (used when reading blob).
1150    pub fn try_from_u8(v: u8) -> Option<Self> {
1151        match v {
1152            0 => Some(Self::Usearch),
1153            _ => None,
1154        }
1155    }
1156}
1157
1158impl fmt::Display for VectorIndexEngineType {
1159    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1160        match self {
1161            Self::Usearch => write!(f, "usearch"),
1162        }
1163    }
1164}
1165
1166impl std::str::FromStr for VectorIndexEngineType {
1167    type Err = String;
1168
1169    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
1170        match s.to_lowercase().as_str() {
1171            "usearch" => Ok(Self::Usearch),
1172            _ => Err(format!(
1173                "Unknown vector index engine: {}. Expected: usearch",
1174                s
1175            )),
1176        }
1177    }
1178}
1179
1180/// Options for vector index (HNSW).
1181#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
1182#[serde(rename_all = "kebab-case")]
1183pub struct VectorIndexOptions {
1184    /// Vector index engine type (default: usearch).
1185    #[serde(default)]
1186    pub engine: VectorIndexEngineType,
1187    /// Distance metric for similarity search.
1188    #[serde(default)]
1189    pub metric: VectorDistanceMetric,
1190    /// HNSW connectivity parameter (M in the paper).
1191    /// Higher values improve recall but increase memory usage.
1192    #[serde(default = "default_vector_index_connectivity")]
1193    pub connectivity: u32,
1194    /// Expansion factor during index construction (ef_construction).
1195    /// Higher values improve index quality but slow down construction.
1196    #[serde(default = "default_vector_index_expansion_add")]
1197    pub expansion_add: u32,
1198    /// Expansion factor during search (ef_search).
1199    /// Higher values improve recall but slow down search.
1200    #[serde(default = "default_vector_index_expansion_search")]
1201    pub expansion_search: u32,
1202}
1203
1204impl Default for VectorIndexOptions {
1205    fn default() -> Self {
1206        Self {
1207            engine: VectorIndexEngineType::default(),
1208            metric: VectorDistanceMetric::default(),
1209            connectivity: DEFAULT_VECTOR_INDEX_CONNECTIVITY,
1210            expansion_add: DEFAULT_VECTOR_INDEX_EXPANSION_ADD,
1211            expansion_search: DEFAULT_VECTOR_INDEX_EXPANSION_SEARCH,
1212        }
1213    }
1214}
1215
1216impl fmt::Display for VectorIndexOptions {
1217    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1218        write!(
1219            f,
1220            "engine={}, metric={}, connectivity={}, expansion_add={}, expansion_search={}",
1221            self.engine, self.metric, self.connectivity, self.expansion_add, self.expansion_search
1222        )
1223    }
1224}
1225
1226#[cfg(test)]
1227mod tests {
1228    use std::sync::Arc;
1229
1230    use arrow::datatypes::{DataType as ArrowDataType, TimeUnit};
1231
1232    use super::*;
1233    use crate::types::{StructField, StructType};
1234    use crate::value::Value;
1235    use crate::vectors::Int32Vector;
1236
1237    #[test]
1238    fn test_column_schema() {
1239        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1240        let field = Field::try_from(&column_schema).unwrap();
1241        assert_eq!("test", field.name());
1242        assert_eq!(ArrowDataType::Int32, *field.data_type());
1243        assert!(field.is_nullable());
1244
1245        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1246        assert_eq!(column_schema, new_column_schema);
1247    }
1248
1249    #[test]
1250    fn test_column_schema_with_default_constraint() {
1251        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1252            .with_default_constraint(Some(ColumnDefaultConstraint::Value(Value::from(99))))
1253            .unwrap();
1254        assert!(
1255            column_schema
1256                .metadata()
1257                .get(DEFAULT_CONSTRAINT_KEY)
1258                .is_none()
1259        );
1260
1261        let field = Field::try_from(&column_schema).unwrap();
1262        assert_eq!("test", field.name());
1263        assert_eq!(ArrowDataType::Int32, *field.data_type());
1264        assert!(field.is_nullable());
1265        assert_eq!(
1266            "{\"Value\":{\"Int32\":99}}",
1267            field.metadata().get(DEFAULT_CONSTRAINT_KEY).unwrap()
1268        );
1269
1270        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1271        assert_eq!(column_schema, new_column_schema);
1272    }
1273
1274    #[test]
1275    fn test_column_schema_with_metadata() {
1276        let metadata = Metadata::from([
1277            ("k1".to_string(), "v1".to_string()),
1278            (COMMENT_KEY.to_string(), "test comment".to_string()),
1279        ]);
1280        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1281            .with_metadata(metadata)
1282            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1283            .unwrap();
1284        assert_eq!("v1", column_schema.metadata().get("k1").unwrap());
1285        assert_eq!("test comment", column_schema.column_comment().unwrap());
1286        assert!(
1287            column_schema
1288                .metadata()
1289                .get(DEFAULT_CONSTRAINT_KEY)
1290                .is_none()
1291        );
1292
1293        let field = Field::try_from(&column_schema).unwrap();
1294        assert_eq!("v1", field.metadata().get("k1").unwrap());
1295        let _ = field.metadata().get(DEFAULT_CONSTRAINT_KEY).unwrap();
1296
1297        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1298        assert_eq!(column_schema, new_column_schema);
1299    }
1300
1301    #[test]
1302    fn test_column_schema_with_duplicate_metadata() {
1303        let metadata = Metadata::from([(DEFAULT_CONSTRAINT_KEY.to_string(), "v1".to_string())]);
1304        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1305            .with_metadata(metadata)
1306            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1307            .unwrap();
1308        assert!(Field::try_from(&column_schema).is_err());
1309    }
1310
1311    #[test]
1312    fn test_column_schema_invalid_default_constraint() {
1313        assert!(
1314            ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false)
1315                .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1316                .is_err()
1317        );
1318    }
1319
1320    #[test]
1321    fn test_column_default_constraint_try_into_from() {
1322        let default_constraint = ColumnDefaultConstraint::Value(Value::from(42i64));
1323
1324        let bytes: Vec<u8> = default_constraint.clone().try_into().unwrap();
1325        let from_value = ColumnDefaultConstraint::try_from(&bytes[..]).unwrap();
1326
1327        assert_eq!(default_constraint, from_value);
1328    }
1329
1330    #[test]
1331    fn test_column_schema_create_default_null() {
1332        // Implicit default null.
1333        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1334        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1335        assert_eq!(5, v.len());
1336        assert!(v.only_null());
1337
1338        // Explicit default null.
1339        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1340            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1341            .unwrap();
1342        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1343        assert_eq!(5, v.len());
1344        assert!(v.only_null());
1345    }
1346
1347    #[test]
1348    fn test_column_schema_create_default_null_struct_with_list() {
1349        let list_type =
1350            ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype()));
1351        let struct_type =
1352            ConcreteDataType::struct_datatype(StructType::new(Arc::new(vec![StructField::new(
1353                "values".to_string(),
1354                list_type,
1355                true,
1356            )])));
1357        let column_schema = ColumnSchema::new("test", struct_type, true);
1358
1359        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1360
1361        assert_eq!(5, v.len());
1362        assert!(v.only_null());
1363    }
1364
1365    #[test]
1366    fn test_column_schema_no_default() {
1367        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1368        assert!(column_schema.create_default_vector(5).unwrap().is_none());
1369    }
1370
1371    #[test]
1372    fn test_create_default_vector_for_padding() {
1373        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1374        let vector = column_schema.create_default_vector_for_padding(4);
1375        assert!(vector.only_null());
1376        assert_eq!(4, vector.len());
1377
1378        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1379        let vector = column_schema.create_default_vector_for_padding(4);
1380        assert_eq!(4, vector.len());
1381        let expect: VectorRef = Arc::new(Int32Vector::from_slice([0, 0, 0, 0]));
1382        assert_eq!(expect, vector);
1383    }
1384
1385    #[test]
1386    fn test_column_schema_single_create_default_null() {
1387        // Implicit default null.
1388        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1389        let v = column_schema.create_default().unwrap().unwrap();
1390        assert!(v.is_null());
1391
1392        // Explicit default null.
1393        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1394            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1395            .unwrap();
1396        let v = column_schema.create_default().unwrap().unwrap();
1397        assert!(v.is_null());
1398    }
1399
1400    #[test]
1401    fn test_column_schema_single_create_default_not_null() {
1402        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1403            .with_default_constraint(Some(ColumnDefaultConstraint::Value(Value::Int32(6))))
1404            .unwrap();
1405        let v = column_schema.create_default().unwrap().unwrap();
1406        assert_eq!(v, Value::Int32(6));
1407    }
1408
1409    #[test]
1410    fn test_column_schema_single_no_default() {
1411        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1412        assert!(column_schema.create_default().unwrap().is_none());
1413    }
1414
1415    #[test]
1416    fn test_debug_for_column_schema() {
1417        let column_schema_int8 =
1418            ColumnSchema::new("test_column_1", ConcreteDataType::int8_datatype(), true);
1419
1420        let column_schema_int32 =
1421            ColumnSchema::new("test_column_2", ConcreteDataType::int32_datatype(), false);
1422
1423        let formatted_int8 = format!("{:?}", column_schema_int8);
1424        let formatted_int32 = format!("{:?}", column_schema_int32);
1425        assert_eq!(formatted_int8, "test_column_1 Int8 null");
1426        assert_eq!(formatted_int32, "test_column_2 Int32 not null");
1427    }
1428
1429    #[test]
1430    fn test_from_field_to_column_schema() {
1431        let field = Field::new("test", ArrowDataType::Int32, true);
1432        let column_schema = ColumnSchema::try_from(&field).unwrap();
1433        assert_eq!("test", column_schema.name);
1434        assert_eq!(ConcreteDataType::int32_datatype(), column_schema.data_type);
1435        assert!(column_schema.is_nullable);
1436        assert!(!column_schema.is_time_index);
1437        assert!(column_schema.default_constraint.is_none());
1438        assert!(column_schema.metadata.is_empty());
1439
1440        let field = Field::new("test", ArrowDataType::Binary, true);
1441        let field = field.with_metadata(Metadata::from([(
1442            TYPE_KEY.to_string(),
1443            ConcreteDataType::json_datatype().name(),
1444        )]));
1445        let column_schema = ColumnSchema::try_from(&field).unwrap();
1446        assert_eq!("test", column_schema.name);
1447        assert_eq!(ConcreteDataType::json_datatype(), column_schema.data_type);
1448        assert!(column_schema.is_nullable);
1449        assert!(!column_schema.is_time_index);
1450        assert!(column_schema.default_constraint.is_none());
1451        assert_eq!(
1452            column_schema.metadata.get(TYPE_KEY).unwrap(),
1453            &ConcreteDataType::json_datatype().name()
1454        );
1455
1456        let field = Field::new("test", ArrowDataType::Binary, true);
1457        let field = field.with_metadata(Metadata::from([(
1458            TYPE_KEY.to_string(),
1459            ConcreteDataType::vector_datatype(3).name(),
1460        )]));
1461        let column_schema = ColumnSchema::try_from(&field).unwrap();
1462        assert_eq!("test", column_schema.name);
1463        assert_eq!(
1464            ConcreteDataType::vector_datatype(3),
1465            column_schema.data_type
1466        );
1467        assert!(column_schema.is_nullable);
1468        assert!(!column_schema.is_time_index);
1469        assert!(column_schema.default_constraint.is_none());
1470        assert_eq!(
1471            column_schema.metadata.get(TYPE_KEY).unwrap(),
1472            &ConcreteDataType::vector_datatype(3).name()
1473        );
1474    }
1475
1476    #[test]
1477    fn test_column_schema_fix_time_index() {
1478        let field = Field::new(
1479            "test",
1480            ArrowDataType::Timestamp(TimeUnit::Second, None),
1481            false,
1482        );
1483        let field = field.with_metadata(Metadata::from([(
1484            TIME_INDEX_KEY.to_string(),
1485            "true".to_string(),
1486        )]));
1487        let column_schema = ColumnSchema::try_from(&field).unwrap();
1488        assert_eq!("test", column_schema.name);
1489        assert_eq!(
1490            ConcreteDataType::timestamp_second_datatype(),
1491            column_schema.data_type
1492        );
1493        assert!(!column_schema.is_nullable);
1494        assert!(column_schema.is_time_index);
1495        assert!(column_schema.default_constraint.is_none());
1496        assert_eq!(1, column_schema.metadata().len());
1497
1498        let field = Field::new("test", ArrowDataType::Int32, false);
1499        let field = field.with_metadata(Metadata::from([(
1500            TIME_INDEX_KEY.to_string(),
1501            "true".to_string(),
1502        )]));
1503        let column_schema = ColumnSchema::try_from(&field).unwrap();
1504        assert_eq!("test", column_schema.name);
1505        assert_eq!(ConcreteDataType::int32_datatype(), column_schema.data_type);
1506        assert!(!column_schema.is_nullable);
1507        assert!(!column_schema.is_time_index);
1508        assert!(column_schema.default_constraint.is_none());
1509        assert!(column_schema.metadata.is_empty());
1510    }
1511
1512    #[test]
1513    fn test_skipping_index_options_deserialization() {
1514        let original_options = "{\"granularity\":1024,\"false-positive-rate-in-10000\":10,\"index-type\":\"BloomFilter\"}";
1515        let options = serde_json::from_str::<SkippingIndexOptions>(original_options).unwrap();
1516        assert_eq!(1024, options.granularity);
1517        assert_eq!(SkippingIndexType::BloomFilter, options.index_type);
1518        assert_eq!(0.001, options.false_positive_rate());
1519
1520        let options_str = serde_json::to_string(&options).unwrap();
1521        assert_eq!(options_str, original_options);
1522    }
1523
1524    #[test]
1525    fn test_skipping_index_options_deserialization_v0_14_to_v0_15() {
1526        let options = "{\"granularity\":10240,\"index-type\":\"BloomFilter\"}";
1527        let options = serde_json::from_str::<SkippingIndexOptions>(options).unwrap();
1528        assert_eq!(10240, options.granularity);
1529        assert_eq!(SkippingIndexType::BloomFilter, options.index_type);
1530        assert_eq!(DEFAULT_FALSE_POSITIVE_RATE, options.false_positive_rate());
1531
1532        let options_str = serde_json::to_string(&options).unwrap();
1533        assert_eq!(
1534            options_str,
1535            "{\"granularity\":10240,\"false-positive-rate-in-10000\":100,\"index-type\":\"BloomFilter\"}"
1536        );
1537    }
1538
1539    #[test]
1540    fn test_fulltext_options_deserialization() {
1541        let original_options = "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":1024,\"false-positive-rate-in-10000\":10}";
1542        let options = serde_json::from_str::<FulltextOptions>(original_options).unwrap();
1543        assert!(!options.case_sensitive);
1544        assert!(options.enable);
1545        assert_eq!(FulltextBackend::Bloom, options.backend);
1546        assert_eq!(FulltextAnalyzer::default(), options.analyzer);
1547        assert_eq!(1024, options.granularity);
1548        assert_eq!(0.001, options.false_positive_rate());
1549
1550        let options_str = serde_json::to_string(&options).unwrap();
1551        assert_eq!(options_str, original_options);
1552    }
1553
1554    #[test]
1555    fn test_fulltext_options_deserialization_v0_14_to_v0_15() {
1556        // 0.14 to 0.15
1557        let options = "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}";
1558        let options = serde_json::from_str::<FulltextOptions>(options).unwrap();
1559        assert!(!options.case_sensitive);
1560        assert!(options.enable);
1561        assert_eq!(FulltextBackend::Bloom, options.backend);
1562        assert_eq!(FulltextAnalyzer::default(), options.analyzer);
1563        assert_eq!(DEFAULT_GRANULARITY, options.granularity);
1564        assert_eq!(DEFAULT_FALSE_POSITIVE_RATE, options.false_positive_rate());
1565
1566        let options_str = serde_json::to_string(&options).unwrap();
1567        assert_eq!(
1568            options_str,
1569            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
1570        );
1571    }
1572}
datatypes/schema/column_schema.rs

datatypes/schema/
column_schema.rs
