datatypes/schema/
column_schema.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::str::FromStr;
17use std::{fmt, mem};
18
19use arrow::datatypes::Field;
20use arrow_schema::extension::{
21    EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY, ExtensionType,
22};
23use serde::{Deserialize, Serialize};
24use snafu::{ResultExt, ensure};
25use sqlparser_derive::{Visit, VisitMut};
26
27use crate::data_type::{ConcreteDataType, DataType};
28use crate::error::{
29    self, ArrowMetadataSnafu, Error, InvalidFulltextOptionSnafu, ParseExtendedTypeSnafu, Result,
30};
31use crate::extension::json::Json2ExtensionType;
32use crate::schema::TYPE_KEY;
33use crate::schema::constraint::ColumnDefaultConstraint;
34use crate::value::Value;
35use crate::vectors::json::builder::JsonVectorBuilder;
36use crate::vectors::{MutableVector, VectorRef};
37
38pub type Metadata = HashMap<String, String>;
39
40/// Key used to store whether the column is time index in arrow field's metadata.
41pub const TIME_INDEX_KEY: &str = "greptime:time_index";
42pub const COMMENT_KEY: &str = "greptime:storage:comment";
43/// Key used to store default constraint in arrow field's metadata.
44const DEFAULT_CONSTRAINT_KEY: &str = "greptime:default_constraint";
45/// Key used to store fulltext options in arrow field's metadata.
46pub const FULLTEXT_KEY: &str = "greptime:fulltext";
47/// Key used to store whether the column has inverted index in arrow field's metadata.
48pub const INVERTED_INDEX_KEY: &str = "greptime:inverted_index";
49/// Key used to store skip options in arrow field's metadata.
50pub const SKIPPING_INDEX_KEY: &str = "greptime:skipping_index";
51
52/// Keys used in fulltext options
53pub const COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE: &str = "enable";
54pub const COLUMN_FULLTEXT_OPT_KEY_ANALYZER: &str = "analyzer";
55pub const COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE: &str = "case_sensitive";
56pub const COLUMN_FULLTEXT_OPT_KEY_BACKEND: &str = "backend";
57pub const COLUMN_FULLTEXT_OPT_KEY_GRANULARITY: &str = "granularity";
58pub const COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE: &str = "false_positive_rate";
59
60/// Keys used in SKIPPING index options
61pub const COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY: &str = "granularity";
62pub const COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE: &str = "false_positive_rate";
63pub const COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE: &str = "type";
64
65pub const DEFAULT_GRANULARITY: u32 = 10240;
66
67pub const DEFAULT_FALSE_POSITIVE_RATE: f64 = 0.01;
68
69/// Schema of a column, used as an immutable struct.
70#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
71pub struct ColumnSchema {
72    pub name: String,
73    pub data_type: ConcreteDataType,
74    is_nullable: bool,
75    is_time_index: bool,
76    default_constraint: Option<ColumnDefaultConstraint>,
77    metadata: Metadata,
78}
79
80impl fmt::Debug for ColumnSchema {
81    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
82        write!(
83            f,
84            "{} {} {}",
85            self.name,
86            self.data_type,
87            if self.is_nullable { "null" } else { "not null" },
88        )?;
89
90        if self.is_time_index {
91            write!(f, " time_index")?;
92        }
93
94        // Add default constraint if present
95        if let Some(default_constraint) = &self.default_constraint {
96            write!(f, " default={:?}", default_constraint)?;
97        }
98
99        // Add metadata if present
100        if !self.metadata.is_empty() {
101            write!(f, " metadata={:?}", self.metadata)?;
102        }
103
104        Ok(())
105    }
106}
107
108impl ColumnSchema {
109    pub fn new<T: Into<String>>(
110        name: T,
111        data_type: ConcreteDataType,
112        is_nullable: bool,
113    ) -> ColumnSchema {
114        ColumnSchema {
115            name: name.into(),
116            data_type,
117            is_nullable,
118            is_time_index: false,
119            default_constraint: None,
120            metadata: Metadata::new(),
121        }
122    }
123
124    /// Creates a mutable vector using this column's extension metadata.
125    pub fn create_mutable_vector(&self, capacity: usize) -> Box<dyn MutableVector> {
126        if self.data_type.is_json2()
127            && let Some(extension) = self.extension_type::<Json2ExtensionType>().ok().flatten()
128            && extension.metadata().is_version_2()
129        {
130            Box::new(JsonVectorBuilder::with_settings(
131                extension.metadata().json_settings(),
132                capacity,
133            ))
134        } else {
135            self.data_type.create_mutable_vector(capacity)
136        }
137    }
138
139    #[inline]
140    pub fn is_time_index(&self) -> bool {
141        self.is_time_index
142    }
143
144    #[inline]
145    pub fn is_nullable(&self) -> bool {
146        self.is_nullable
147    }
148
149    #[inline]
150    pub fn default_constraint(&self) -> Option<&ColumnDefaultConstraint> {
151        self.default_constraint.as_ref()
152    }
153
154    /// Check if the default constraint is a impure function.
155    pub fn is_default_impure(&self) -> bool {
156        self.default_constraint
157            .as_ref()
158            .map(|c| c.is_function())
159            .unwrap_or(false)
160    }
161
162    #[inline]
163    pub fn metadata(&self) -> &Metadata {
164        &self.metadata
165    }
166
167    #[inline]
168    pub fn mut_metadata(&mut self) -> &mut Metadata {
169        &mut self.metadata
170    }
171
172    /// Retrieve the column comment
173    pub fn column_comment(&self) -> Option<&String> {
174        self.metadata.get(COMMENT_KEY)
175    }
176
177    pub fn with_time_index(mut self, is_time_index: bool) -> Self {
178        self.is_time_index = is_time_index;
179        if is_time_index {
180            let _ = self
181                .metadata
182                .insert(TIME_INDEX_KEY.to_string(), "true".to_string());
183        } else {
184            let _ = self.metadata.remove(TIME_INDEX_KEY);
185        }
186        self
187    }
188
189    /// Returns the estimated memory footprint of this schema.
190    pub fn estimated_size(&self) -> usize {
191        mem::size_of_val(self) - mem::size_of_val(&self.data_type)
192            + self.data_type.as_arrow_type().size()
193            + self.name.capacity()
194            + self
195                .default_constraint
196                .as_ref()
197                .map(column_default_constraint_size)
198                .unwrap_or_default()
199            + metadata_size(&self.metadata)
200    }
201
202    /// Set the inverted index for the column.
203    /// Similar to [with_inverted_index] but don't take the ownership.
204    ///
205    /// [with_inverted_index]: Self::with_inverted_index
206    pub fn set_inverted_index(&mut self, value: bool) {
207        match value {
208            true => {
209                self.metadata
210                    .insert(INVERTED_INDEX_KEY.to_string(), value.to_string());
211            }
212            false => {
213                self.metadata.remove(INVERTED_INDEX_KEY);
214            }
215        }
216    }
217
218    /// Set the inverted index for the column.
219    /// Similar to [set_inverted_index] but take the ownership and return a owned value.
220    ///
221    /// [set_inverted_index]: Self::set_inverted_index
222    pub fn with_inverted_index(mut self, value: bool) -> Self {
223        self.set_inverted_index(value);
224        self
225    }
226
227    pub fn is_inverted_indexed(&self) -> bool {
228        self.metadata
229            .get(INVERTED_INDEX_KEY)
230            .map(|v| v.eq_ignore_ascii_case("true"))
231            .unwrap_or(false)
232    }
233
234    pub fn is_fulltext_indexed(&self) -> bool {
235        self.fulltext_options()
236            .unwrap_or_default()
237            .map(|option| option.enable)
238            .unwrap_or_default()
239    }
240
241    pub fn is_skipping_indexed(&self) -> bool {
242        self.skipping_index_options().unwrap_or_default().is_some()
243    }
244
245    pub fn has_inverted_index_key(&self) -> bool {
246        self.metadata.contains_key(INVERTED_INDEX_KEY)
247    }
248
249    /// Set default constraint.
250    ///
251    /// If a default constraint exists for the column, this method will
252    /// validate it against the column's data type and nullability.
253    pub fn with_default_constraint(
254        mut self,
255        default_constraint: Option<ColumnDefaultConstraint>,
256    ) -> Result<Self> {
257        if let Some(constraint) = &default_constraint {
258            constraint.validate(&self.data_type, self.is_nullable)?;
259        }
260
261        self.default_constraint = default_constraint;
262        Ok(self)
263    }
264
265    /// Set the nullablity to `true` of the column.
266    /// Similar to [set_nullable] but take the ownership and return a owned value.
267    ///
268    /// [set_nullable]: Self::set_nullable
269    pub fn with_nullable_set(mut self) -> Self {
270        self.is_nullable = true;
271        self
272    }
273
274    /// Set the nullability to `true` of the column.
275    /// Similar to [with_nullable_set] but don't take the ownership
276    ///
277    /// [with_nullable_set]: Self::with_nullable_set
278    pub fn set_nullable(&mut self) {
279        self.is_nullable = true;
280    }
281
282    /// Set the `is_time_index` to `true` of the column.
283    /// Similar to [with_time_index] but don't take the ownership.
284    ///
285    /// [with_time_index]: Self::with_time_index
286    pub fn set_time_index(&mut self) {
287        self.is_time_index = true;
288    }
289
290    /// Creates a new [`ColumnSchema`] with given metadata.
291    pub fn with_metadata(mut self, metadata: Metadata) -> Self {
292        self.metadata = metadata;
293        self
294    }
295
296    /// Creates a vector with default value for this column.
297    ///
298    /// If the column is `NOT NULL` but doesn't has `DEFAULT` value supplied, returns `Ok(None)`.
299    pub fn create_default_vector(&self, num_rows: usize) -> Result<Option<VectorRef>> {
300        match &self.default_constraint {
301            Some(c) => c
302                .create_default_vector(&self.data_type, self.is_nullable, num_rows)
303                .map(Some),
304            None => {
305                if self.is_nullable {
306                    // No default constraint, use null as default value.
307                    // TODO(yingwen): Use NullVector once it supports setting logical type.
308                    ColumnDefaultConstraint::null_value()
309                        .create_default_vector(&self.data_type, self.is_nullable, num_rows)
310                        .map(Some)
311                } else {
312                    Ok(None)
313                }
314            }
315        }
316    }
317
318    /// Creates a vector for padding.
319    ///
320    /// This method always returns a vector since it uses [DataType::default_value]
321    /// to fill the vector. Callers should only use the created vector for padding
322    /// and never read its content.
323    pub fn create_default_vector_for_padding(&self, num_rows: usize) -> VectorRef {
324        let padding_value = if self.is_nullable {
325            Value::Null
326        } else {
327            // If the column is not null, use the data type's default value as it is
328            // more efficient to acquire.
329            self.data_type.default_value()
330        };
331        let value_ref = padding_value.as_value_ref();
332        let mut mutable_vector = self.data_type.create_mutable_vector(num_rows);
333        for _ in 0..num_rows {
334            mutable_vector.push_value_ref(&value_ref);
335        }
336        mutable_vector.to_vector()
337    }
338
339    /// Creates a default value for this column.
340    ///
341    /// If the column is `NOT NULL` but doesn't has `DEFAULT` value supplied, returns `Ok(None)`.
342    pub fn create_default(&self) -> Result<Option<Value>> {
343        match &self.default_constraint {
344            Some(c) => c
345                .create_default(&self.data_type, self.is_nullable)
346                .map(Some),
347            None => {
348                if self.is_nullable {
349                    // No default constraint, use null as default value.
350                    ColumnDefaultConstraint::null_value()
351                        .create_default(&self.data_type, self.is_nullable)
352                        .map(Some)
353                } else {
354                    Ok(None)
355                }
356            }
357        }
358    }
359
360    /// Creates an impure default value for this column, only if it have a impure default constraint.
361    /// Otherwise, returns `Ok(None)`.
362    pub fn create_impure_default(&self) -> Result<Option<Value>> {
363        match &self.default_constraint {
364            Some(c) => c.create_impure_default(&self.data_type),
365            None => Ok(None),
366        }
367    }
368
369    /// Retrieves the fulltext options for the column.
370    pub fn fulltext_options(&self) -> Result<Option<FulltextOptions>> {
371        match self.metadata.get(FULLTEXT_KEY) {
372            None => Ok(None),
373            Some(json) => {
374                let options =
375                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
376                Ok(Some(options))
377            }
378        }
379    }
380
381    pub fn with_fulltext_options(mut self, options: FulltextOptions) -> Result<Self> {
382        self.metadata.insert(
383            FULLTEXT_KEY.to_string(),
384            serde_json::to_string(&options).context(error::SerializeSnafu)?,
385        );
386        Ok(self)
387    }
388
389    pub fn set_fulltext_options(&mut self, options: &FulltextOptions) -> Result<()> {
390        self.metadata.insert(
391            FULLTEXT_KEY.to_string(),
392            serde_json::to_string(options).context(error::SerializeSnafu)?,
393        );
394        Ok(())
395    }
396
397    /// Retrieves the skipping index options for the column.
398    pub fn skipping_index_options(&self) -> Result<Option<SkippingIndexOptions>> {
399        match self.metadata.get(SKIPPING_INDEX_KEY) {
400            None => Ok(None),
401            Some(json) => {
402                let options =
403                    serde_json::from_str(json).context(error::DeserializeSnafu { json })?;
404                Ok(Some(options))
405            }
406        }
407    }
408
409    pub fn with_skipping_options(mut self, options: SkippingIndexOptions) -> Result<Self> {
410        self.metadata.insert(
411            SKIPPING_INDEX_KEY.to_string(),
412            serde_json::to_string(&options).context(error::SerializeSnafu)?,
413        );
414        Ok(self)
415    }
416
417    pub fn set_skipping_options(&mut self, options: &SkippingIndexOptions) -> Result<()> {
418        self.metadata.insert(
419            SKIPPING_INDEX_KEY.to_string(),
420            serde_json::to_string(options).context(error::SerializeSnafu)?,
421        );
422        Ok(())
423    }
424
425    pub fn unset_skipping_options(&mut self) -> Result<()> {
426        self.metadata.remove(SKIPPING_INDEX_KEY);
427        Ok(())
428    }
429
430    pub fn extension_type<E>(&self) -> Result<Option<E>>
431    where
432        E: ExtensionType,
433    {
434        let extension_type_name = self.metadata.get(EXTENSION_TYPE_NAME_KEY);
435
436        if extension_type_name.map(|s| s.as_str()) == Some(E::NAME) {
437            let extension_metadata = self.metadata.get(EXTENSION_TYPE_METADATA_KEY);
438            let extension_metadata =
439                E::deserialize_metadata(extension_metadata.map(|s| s.as_str()))
440                    .context(ArrowMetadataSnafu)?;
441
442            let extension = E::try_new(&self.data_type.as_arrow_type(), extension_metadata)
443                .context(ArrowMetadataSnafu)?;
444            Ok(Some(extension))
445        } else {
446            Ok(None)
447        }
448    }
449
450    /// Sets the Arrow extension type metadata for this column.
451    pub fn with_extension_type<E>(&mut self, extension_type: &E)
452    where
453        E: ExtensionType,
454    {
455        self.metadata
456            .insert(EXTENSION_TYPE_NAME_KEY.to_string(), E::NAME.to_string());
457
458        if let Some(extension_metadata) = extension_type.serialize_metadata() {
459            self.metadata
460                .insert(EXTENSION_TYPE_METADATA_KEY.to_string(), extension_metadata);
461        } else {
462            // Replacing an extension must not retain metadata owned by the previous type.
463            self.metadata.remove(EXTENSION_TYPE_METADATA_KEY);
464        }
465    }
466
467    pub fn is_indexed(&self) -> bool {
468        self.is_inverted_indexed() || self.is_fulltext_indexed() || self.is_skipping_indexed()
469    }
470}
471
472fn metadata_size(metadata: &Metadata) -> usize {
473    mem::size_of::<(String, String)>() * metadata.capacity()
474        + metadata
475            .iter()
476            .map(|(key, value)| key.capacity() + value.capacity())
477            .sum::<usize>()
478}
479
480fn column_default_constraint_size(default_constraint: &ColumnDefaultConstraint) -> usize {
481    match default_constraint {
482        ColumnDefaultConstraint::Function(expr) => expr.capacity(),
483        ColumnDefaultConstraint::Value(value) => value.as_value_ref().data_size(),
484    }
485}
486
487/// Column extended type set in column schema's metadata.
488#[derive(Debug, Clone, PartialEq, Eq)]
489pub enum ColumnExtType {
490    /// Json type.
491    Json,
492
493    /// Vector type with dimension.
494    Vector(u32),
495}
496
497impl fmt::Display for ColumnExtType {
498    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
499        match self {
500            ColumnExtType::Json => write!(f, "Json"),
501            ColumnExtType::Vector(dim) => write!(f, "Vector({})", dim),
502        }
503    }
504}
505
506impl FromStr for ColumnExtType {
507    type Err = String;
508
509    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
510        match s {
511            "Json" => Ok(ColumnExtType::Json),
512            _ if s.starts_with("Vector(") && s.ends_with(')') => s[7..s.len() - 1]
513                .parse::<u32>()
514                .map(ColumnExtType::Vector)
515                .map_err(|_| "Invalid dimension for Vector".to_string()),
516            _ => Err("Unknown variant".to_string()),
517        }
518    }
519}
520
521impl TryFrom<&Field> for ColumnSchema {
522    type Error = Error;
523
524    fn try_from(field: &Field) -> Result<ColumnSchema> {
525        let mut data_type = ConcreteDataType::try_from(field.data_type())?;
526        // Override the data type if it is specified in the metadata.
527        if let Some(s) = field.metadata().get(TYPE_KEY) {
528            let extype = ColumnExtType::from_str(s)
529                .map_err(|_| ParseExtendedTypeSnafu { value: s }.build())?;
530            match extype {
531                ColumnExtType::Json => {
532                    data_type = ConcreteDataType::json_datatype();
533                }
534                ColumnExtType::Vector(dim) => {
535                    data_type = ConcreteDataType::vector_datatype(dim);
536                }
537            }
538        }
539        let mut metadata = field.metadata().clone();
540        let default_constraint = match metadata.remove(DEFAULT_CONSTRAINT_KEY) {
541            Some(json) => {
542                Some(serde_json::from_str(&json).context(error::DeserializeSnafu { json })?)
543            }
544            None => None,
545        };
546        let mut is_time_index = metadata.contains_key(TIME_INDEX_KEY);
547        if is_time_index && !data_type.is_timestamp() {
548            // If the column is time index but the data type is not timestamp, it is invalid.
549            // We set the time index to false and remove the metadata.
550            // This is possible if we cast the time index column to another type. DataFusion will
551            // keep the metadata:
552            // https://github.com/apache/datafusion/pull/12951
553            is_time_index = false;
554            metadata.remove(TIME_INDEX_KEY);
555            common_telemetry::debug!(
556                "Column {} is not timestamp ({:?}) but has time index metadata",
557                data_type,
558                field.name(),
559            );
560        }
561
562        Ok(ColumnSchema {
563            name: field.name().clone(),
564            data_type,
565            is_nullable: field.is_nullable(),
566            is_time_index,
567            default_constraint,
568            metadata,
569        })
570    }
571}
572
573impl TryFrom<&ColumnSchema> for Field {
574    type Error = Error;
575
576    fn try_from(column_schema: &ColumnSchema) -> Result<Field> {
577        let mut metadata = column_schema.metadata.clone();
578        if let Some(value) = &column_schema.default_constraint {
579            // Adds an additional metadata to store the default constraint.
580            let old = metadata.insert(
581                DEFAULT_CONSTRAINT_KEY.to_string(),
582                serde_json::to_string(&value).context(error::SerializeSnafu)?,
583            );
584
585            ensure!(
586                old.is_none(),
587                error::DuplicateMetaSnafu {
588                    key: DEFAULT_CONSTRAINT_KEY,
589                }
590            );
591        }
592
593        Ok(Field::new(
594            &column_schema.name,
595            column_schema.data_type.as_arrow_type(),
596            column_schema.is_nullable(),
597        )
598        .with_metadata(metadata))
599    }
600}
601
602/// Fulltext options for a column.
603#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
604#[serde(rename_all = "kebab-case")]
605pub struct FulltextOptions {
606    /// Whether the fulltext index is enabled.
607    pub enable: bool,
608    /// The fulltext analyzer to use.
609    #[serde(default)]
610    pub analyzer: FulltextAnalyzer,
611    /// Whether the fulltext index is case-sensitive.
612    #[serde(default)]
613    pub case_sensitive: bool,
614    /// The fulltext backend to use.
615    #[serde(default)]
616    pub backend: FulltextBackend,
617    /// The granularity of the fulltext index (for bloom backend only)
618    #[serde(default = "fulltext_options_default_granularity")]
619    pub granularity: u32,
620    /// The false positive rate of the fulltext index (for bloom backend only)
621    #[serde(default = "index_options_default_false_positive_rate_in_10000")]
622    pub false_positive_rate_in_10000: u32,
623}
624
625fn fulltext_options_default_granularity() -> u32 {
626    DEFAULT_GRANULARITY
627}
628
629fn index_options_default_false_positive_rate_in_10000() -> u32 {
630    (DEFAULT_FALSE_POSITIVE_RATE * 10000.0) as u32
631}
632
633impl FulltextOptions {
634    /// Creates a new fulltext options.
635    pub fn new(
636        enable: bool,
637        analyzer: FulltextAnalyzer,
638        case_sensitive: bool,
639        backend: FulltextBackend,
640        granularity: u32,
641        false_positive_rate: f64,
642    ) -> Result<Self> {
643        ensure!(
644            0.0 < false_positive_rate && false_positive_rate <= 1.0,
645            error::InvalidFulltextOptionSnafu {
646                msg: format!(
647                    "Invalid false positive rate: {false_positive_rate}, expected: 0.0 < rate <= 1.0"
648                ),
649            }
650        );
651        ensure!(
652            granularity > 0,
653            error::InvalidFulltextOptionSnafu {
654                msg: format!("Invalid granularity: {granularity}, expected: positive integer"),
655            }
656        );
657        Ok(Self::new_unchecked(
658            enable,
659            analyzer,
660            case_sensitive,
661            backend,
662            granularity,
663            false_positive_rate,
664        ))
665    }
666
667    /// Creates a new fulltext options without checking `false_positive_rate` and `granularity`.
668    pub fn new_unchecked(
669        enable: bool,
670        analyzer: FulltextAnalyzer,
671        case_sensitive: bool,
672        backend: FulltextBackend,
673        granularity: u32,
674        false_positive_rate: f64,
675    ) -> Self {
676        Self {
677            enable,
678            analyzer,
679            case_sensitive,
680            backend,
681            granularity,
682            false_positive_rate_in_10000: (false_positive_rate * 10000.0) as u32,
683        }
684    }
685
686    /// Gets the false positive rate.
687    pub fn false_positive_rate(&self) -> f64 {
688        self.false_positive_rate_in_10000 as f64 / 10000.0
689    }
690}
691
692impl Default for FulltextOptions {
693    fn default() -> Self {
694        Self::new_unchecked(
695            false,
696            FulltextAnalyzer::default(),
697            false,
698            FulltextBackend::default(),
699            DEFAULT_GRANULARITY,
700            DEFAULT_FALSE_POSITIVE_RATE,
701        )
702    }
703}
704
705impl fmt::Display for FulltextOptions {
706    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
707        write!(f, "enable={}", self.enable)?;
708        if self.enable {
709            write!(f, ", analyzer={}", self.analyzer)?;
710            write!(f, ", case_sensitive={}", self.case_sensitive)?;
711            write!(f, ", backend={}", self.backend)?;
712            if self.backend == FulltextBackend::Bloom {
713                write!(f, ", granularity={}", self.granularity)?;
714                write!(f, ", false_positive_rate={}", self.false_positive_rate())?;
715            }
716        }
717        Ok(())
718    }
719}
720
721/// The backend of the fulltext index.
722#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
723#[serde(rename_all = "kebab-case")]
724pub enum FulltextBackend {
725    #[default]
726    Bloom,
727    Tantivy,
728}
729
730impl fmt::Display for FulltextBackend {
731    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
732        match self {
733            FulltextBackend::Tantivy => write!(f, "tantivy"),
734            FulltextBackend::Bloom => write!(f, "bloom"),
735        }
736    }
737}
738
739impl TryFrom<HashMap<String, String>> for FulltextOptions {
740    type Error = Error;
741
742    fn try_from(options: HashMap<String, String>) -> Result<Self> {
743        let mut fulltext_options = FulltextOptions {
744            enable: true,
745            ..Default::default()
746        };
747
748        if let Some(enable) = options.get(COLUMN_FULLTEXT_CHANGE_OPT_KEY_ENABLE) {
749            match enable.to_ascii_lowercase().as_str() {
750                "true" => fulltext_options.enable = true,
751                "false" => fulltext_options.enable = false,
752                _ => {
753                    return InvalidFulltextOptionSnafu {
754                        msg: format!("{enable}, expected: 'true' | 'false'"),
755                    }
756                    .fail();
757                }
758            }
759        };
760
761        if let Some(analyzer) = options.get(COLUMN_FULLTEXT_OPT_KEY_ANALYZER) {
762            match analyzer.to_ascii_lowercase().as_str() {
763                "english" => fulltext_options.analyzer = FulltextAnalyzer::English,
764                "chinese" => fulltext_options.analyzer = FulltextAnalyzer::Chinese,
765                _ => {
766                    return InvalidFulltextOptionSnafu {
767                        msg: format!("{analyzer}, expected: 'English' | 'Chinese'"),
768                    }
769                    .fail();
770                }
771            }
772        };
773
774        if let Some(case_sensitive) = options.get(COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE) {
775            match case_sensitive.to_ascii_lowercase().as_str() {
776                "true" => fulltext_options.case_sensitive = true,
777                "false" => fulltext_options.case_sensitive = false,
778                _ => {
779                    return InvalidFulltextOptionSnafu {
780                        msg: format!("{case_sensitive}, expected: 'true' | 'false'"),
781                    }
782                    .fail();
783                }
784            }
785        }
786
787        if let Some(backend) = options.get(COLUMN_FULLTEXT_OPT_KEY_BACKEND) {
788            match backend.to_ascii_lowercase().as_str() {
789                "bloom" => fulltext_options.backend = FulltextBackend::Bloom,
790                "tantivy" => fulltext_options.backend = FulltextBackend::Tantivy,
791                _ => {
792                    return InvalidFulltextOptionSnafu {
793                        msg: format!("{backend}, expected: 'bloom' | 'tantivy'"),
794                    }
795                    .fail();
796                }
797            }
798        }
799
800        if fulltext_options.backend == FulltextBackend::Bloom {
801            // Parse granularity with default value 10240
802            let granularity = match options.get(COLUMN_FULLTEXT_OPT_KEY_GRANULARITY) {
803                Some(value) => value
804                    .parse::<u32>()
805                    .ok()
806                    .filter(|&v| v > 0)
807                    .ok_or_else(|| {
808                        error::InvalidFulltextOptionSnafu {
809                            msg: format!(
810                                "Invalid granularity: {value}, expected: positive integer"
811                            ),
812                        }
813                        .build()
814                    })?,
815                None => DEFAULT_GRANULARITY,
816            };
817            fulltext_options.granularity = granularity;
818
819            // Parse false positive rate with default value 0.01
820            let false_positive_rate = match options.get(COLUMN_FULLTEXT_OPT_KEY_FALSE_POSITIVE_RATE)
821            {
822                Some(value) => value
823                    .parse::<f64>()
824                    .ok()
825                    .filter(|&v| v > 0.0 && v <= 1.0)
826                    .ok_or_else(|| {
827                        error::InvalidFulltextOptionSnafu {
828                            msg: format!(
829                                "Invalid false positive rate: {value}, expected: 0.0 < rate <= 1.0"
830                            ),
831                        }
832                        .build()
833                    })?,
834                None => DEFAULT_FALSE_POSITIVE_RATE,
835            };
836            fulltext_options.false_positive_rate_in_10000 = (false_positive_rate * 10000.0) as u32;
837        }
838
839        Ok(fulltext_options)
840    }
841}
842
843/// Fulltext analyzer.
844#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Visit, VisitMut)]
845pub enum FulltextAnalyzer {
846    #[default]
847    English,
848    Chinese,
849}
850
851impl fmt::Display for FulltextAnalyzer {
852    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
853        match self {
854            FulltextAnalyzer::English => write!(f, "English"),
855            FulltextAnalyzer::Chinese => write!(f, "Chinese"),
856        }
857    }
858}
859
860/// Skipping options for a column.
861#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
862#[serde(rename_all = "kebab-case")]
863pub struct SkippingIndexOptions {
864    /// The granularity of the skip index.
865    pub granularity: u32,
866    /// The false positive rate of the skip index (in ten-thousandths, e.g., 100 = 1%).
867    #[serde(default = "index_options_default_false_positive_rate_in_10000")]
868    pub false_positive_rate_in_10000: u32,
869    /// The type of the skip index.
870    #[serde(default)]
871    pub index_type: SkippingIndexType,
872}
873
874impl SkippingIndexOptions {
875    /// Creates a new skipping index options without checking `false_positive_rate` and `granularity`.
876    pub fn new_unchecked(
877        granularity: u32,
878        false_positive_rate: f64,
879        index_type: SkippingIndexType,
880    ) -> Self {
881        Self {
882            granularity,
883            false_positive_rate_in_10000: (false_positive_rate * 10000.0) as u32,
884            index_type,
885        }
886    }
887
888    /// Creates a new skipping index options.
889    pub fn new(
890        granularity: u32,
891        false_positive_rate: f64,
892        index_type: SkippingIndexType,
893    ) -> Result<Self> {
894        ensure!(
895            0.0 < false_positive_rate && false_positive_rate <= 1.0,
896            error::InvalidSkippingIndexOptionSnafu {
897                msg: format!(
898                    "Invalid false positive rate: {false_positive_rate}, expected: 0.0 < rate <= 1.0"
899                ),
900            }
901        );
902        ensure!(
903            granularity > 0,
904            error::InvalidSkippingIndexOptionSnafu {
905                msg: format!("Invalid granularity: {granularity}, expected: positive integer"),
906            }
907        );
908        Ok(Self::new_unchecked(
909            granularity,
910            false_positive_rate,
911            index_type,
912        ))
913    }
914
915    /// Gets the false positive rate.
916    pub fn false_positive_rate(&self) -> f64 {
917        self.false_positive_rate_in_10000 as f64 / 10000.0
918    }
919}
920
921impl Default for SkippingIndexOptions {
922    fn default() -> Self {
923        Self::new_unchecked(
924            DEFAULT_GRANULARITY,
925            DEFAULT_FALSE_POSITIVE_RATE,
926            SkippingIndexType::default(),
927        )
928    }
929}
930
931impl fmt::Display for SkippingIndexOptions {
932    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
933        write!(f, "granularity={}", self.granularity)?;
934        write!(f, ", false_positive_rate={}", self.false_positive_rate())?;
935        write!(f, ", index_type={}", self.index_type)?;
936        Ok(())
937    }
938}
939
940/// Skip index types.
941#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, Visit, VisitMut)]
942pub enum SkippingIndexType {
943    #[default]
944    BloomFilter,
945}
946
947impl fmt::Display for SkippingIndexType {
948    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
949        match self {
950            SkippingIndexType::BloomFilter => write!(f, "BLOOM"),
951        }
952    }
953}
954
955impl TryFrom<HashMap<String, String>> for SkippingIndexOptions {
956    type Error = Error;
957
958    fn try_from(options: HashMap<String, String>) -> Result<Self> {
959        // Parse granularity with default value 1
960        let granularity = match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_GRANULARITY) {
961            Some(value) => value
962                .parse::<u32>()
963                .ok()
964                .filter(|&v| v > 0)
965                .ok_or_else(|| {
966                    error::InvalidSkippingIndexOptionSnafu {
967                        msg: format!("Invalid granularity: {value}, expected: positive integer"),
968                    }
969                    .build()
970                })?,
971            None => DEFAULT_GRANULARITY,
972        };
973
974        // Parse false positive rate with default value 100
975        let false_positive_rate =
976            match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_FALSE_POSITIVE_RATE) {
977                Some(value) => value
978                    .parse::<f64>()
979                    .ok()
980                    .filter(|&v| v > 0.0 && v <= 1.0)
981                    .ok_or_else(|| {
982                        error::InvalidSkippingIndexOptionSnafu {
983                            msg: format!(
984                                "Invalid false positive rate: {value}, expected: 0.0 < rate <= 1.0"
985                            ),
986                        }
987                        .build()
988                    })?,
989                None => DEFAULT_FALSE_POSITIVE_RATE,
990            };
991
992        // Parse index type with default value BloomFilter
993        let index_type = match options.get(COLUMN_SKIPPING_INDEX_OPT_KEY_TYPE) {
994            Some(typ) => match typ.to_ascii_uppercase().as_str() {
995                "BLOOM" => SkippingIndexType::BloomFilter,
996                _ => {
997                    return error::InvalidSkippingIndexOptionSnafu {
998                        msg: format!("Invalid index type: {typ}, expected: 'BLOOM'"),
999                    }
1000                    .fail();
1001                }
1002            },
1003            None => SkippingIndexType::default(),
1004        };
1005
1006        Ok(SkippingIndexOptions::new_unchecked(
1007            granularity,
1008            false_positive_rate,
1009            index_type,
1010        ))
1011    }
1012}
1013
1014#[cfg(test)]
1015mod tests {
1016    use std::sync::Arc;
1017
1018    use arrow::datatypes::{DataType as ArrowDataType, TimeUnit};
1019
1020    use super::*;
1021    use crate::extension::json::{Json2ExtensionType, JsonExtensionType};
1022    use crate::types::{StructField, StructType};
1023    use crate::value::Value;
1024    use crate::vectors::Int32Vector;
1025
1026    #[test]
1027    fn test_column_schema() {
1028        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1029        let field = Field::try_from(&column_schema).unwrap();
1030        assert_eq!("test", field.name());
1031        assert_eq!(ArrowDataType::Int32, *field.data_type());
1032        assert!(field.is_nullable());
1033
1034        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1035        assert_eq!(column_schema, new_column_schema);
1036    }
1037
1038    #[test]
1039    fn test_with_extension_type_replaces_metadata() {
1040        let mut schema = ColumnSchema::new("j", ConcreteDataType::json_datatype(), true);
1041
1042        schema.with_extension_type(&Json2ExtensionType::default());
1043        assert_eq!(
1044            Some(Json2ExtensionType::NAME),
1045            schema
1046                .metadata()
1047                .get(EXTENSION_TYPE_NAME_KEY)
1048                .map(String::as_str)
1049        );
1050        assert!(schema.metadata().contains_key(EXTENSION_TYPE_METADATA_KEY));
1051
1052        schema.with_extension_type(&JsonExtensionType);
1053        assert_eq!(
1054            Some(JsonExtensionType::NAME),
1055            schema
1056                .metadata()
1057                .get(EXTENSION_TYPE_NAME_KEY)
1058                .map(String::as_str)
1059        );
1060        assert!(!schema.metadata().contains_key(EXTENSION_TYPE_METADATA_KEY));
1061    }
1062
1063    #[test]
1064    fn test_column_schema_with_default_constraint() {
1065        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1066            .with_default_constraint(Some(ColumnDefaultConstraint::Value(Value::from(99))))
1067            .unwrap();
1068        assert!(
1069            column_schema
1070                .metadata()
1071                .get(DEFAULT_CONSTRAINT_KEY)
1072                .is_none()
1073        );
1074
1075        let field = Field::try_from(&column_schema).unwrap();
1076        assert_eq!("test", field.name());
1077        assert_eq!(ArrowDataType::Int32, *field.data_type());
1078        assert!(field.is_nullable());
1079        assert_eq!(
1080            "{\"Value\":{\"Int32\":99}}",
1081            field.metadata().get(DEFAULT_CONSTRAINT_KEY).unwrap()
1082        );
1083
1084        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1085        assert_eq!(column_schema, new_column_schema);
1086    }
1087
1088    #[test]
1089    fn test_column_schema_with_metadata() {
1090        let metadata = Metadata::from([
1091            ("k1".to_string(), "v1".to_string()),
1092            (COMMENT_KEY.to_string(), "test comment".to_string()),
1093        ]);
1094        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1095            .with_metadata(metadata)
1096            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1097            .unwrap();
1098        assert_eq!("v1", column_schema.metadata().get("k1").unwrap());
1099        assert_eq!("test comment", column_schema.column_comment().unwrap());
1100        assert!(
1101            column_schema
1102                .metadata()
1103                .get(DEFAULT_CONSTRAINT_KEY)
1104                .is_none()
1105        );
1106
1107        let field = Field::try_from(&column_schema).unwrap();
1108        assert_eq!("v1", field.metadata().get("k1").unwrap());
1109        let _ = field.metadata().get(DEFAULT_CONSTRAINT_KEY).unwrap();
1110
1111        let new_column_schema = ColumnSchema::try_from(&field).unwrap();
1112        assert_eq!(column_schema, new_column_schema);
1113    }
1114
1115    #[test]
1116    fn test_column_schema_with_duplicate_metadata() {
1117        let metadata = Metadata::from([(DEFAULT_CONSTRAINT_KEY.to_string(), "v1".to_string())]);
1118        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1119            .with_metadata(metadata)
1120            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1121            .unwrap();
1122        assert!(Field::try_from(&column_schema).is_err());
1123    }
1124
1125    #[test]
1126    fn test_column_schema_invalid_default_constraint() {
1127        assert!(
1128            ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false)
1129                .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1130                .is_err()
1131        );
1132    }
1133
1134    #[test]
1135    fn test_column_default_constraint_try_into_from() {
1136        let default_constraint = ColumnDefaultConstraint::Value(Value::from(42i64));
1137
1138        let bytes: Vec<u8> = default_constraint.clone().try_into().unwrap();
1139        let from_value = ColumnDefaultConstraint::try_from(&bytes[..]).unwrap();
1140
1141        assert_eq!(default_constraint, from_value);
1142    }
1143
1144    #[test]
1145    fn test_column_schema_create_default_null() {
1146        // Implicit default null.
1147        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1148        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1149        assert_eq!(5, v.len());
1150        assert!(v.only_null());
1151
1152        // Explicit default null.
1153        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1154            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1155            .unwrap();
1156        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1157        assert_eq!(5, v.len());
1158        assert!(v.only_null());
1159    }
1160
1161    #[test]
1162    fn test_column_schema_create_default_null_struct_with_list() {
1163        let list_type =
1164            ConcreteDataType::list_datatype(Arc::new(ConcreteDataType::int32_datatype()));
1165        let struct_type =
1166            ConcreteDataType::struct_datatype(StructType::new(Arc::new(vec![StructField::new(
1167                "values".to_string(),
1168                list_type,
1169                true,
1170            )])));
1171        let column_schema = ColumnSchema::new("test", struct_type, true);
1172
1173        let v = column_schema.create_default_vector(5).unwrap().unwrap();
1174
1175        assert_eq!(5, v.len());
1176        assert!(v.only_null());
1177    }
1178
1179    #[test]
1180    fn test_column_schema_no_default() {
1181        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1182        assert!(column_schema.create_default_vector(5).unwrap().is_none());
1183    }
1184
1185    #[test]
1186    fn test_create_default_vector_for_padding() {
1187        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1188        let vector = column_schema.create_default_vector_for_padding(4);
1189        assert!(vector.only_null());
1190        assert_eq!(4, vector.len());
1191
1192        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1193        let vector = column_schema.create_default_vector_for_padding(4);
1194        assert_eq!(4, vector.len());
1195        let expect: VectorRef = Arc::new(Int32Vector::from_slice([0, 0, 0, 0]));
1196        assert_eq!(expect, vector);
1197    }
1198
1199    #[test]
1200    fn test_column_schema_single_create_default_null() {
1201        // Implicit default null.
1202        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true);
1203        let v = column_schema.create_default().unwrap().unwrap();
1204        assert!(v.is_null());
1205
1206        // Explicit default null.
1207        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1208            .with_default_constraint(Some(ColumnDefaultConstraint::null_value()))
1209            .unwrap();
1210        let v = column_schema.create_default().unwrap().unwrap();
1211        assert!(v.is_null());
1212    }
1213
1214    #[test]
1215    fn test_column_schema_single_create_default_not_null() {
1216        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), true)
1217            .with_default_constraint(Some(ColumnDefaultConstraint::Value(Value::Int32(6))))
1218            .unwrap();
1219        let v = column_schema.create_default().unwrap().unwrap();
1220        assert_eq!(v, Value::Int32(6));
1221    }
1222
1223    #[test]
1224    fn test_column_schema_single_no_default() {
1225        let column_schema = ColumnSchema::new("test", ConcreteDataType::int32_datatype(), false);
1226        assert!(column_schema.create_default().unwrap().is_none());
1227    }
1228
1229    #[test]
1230    fn test_debug_for_column_schema() {
1231        let column_schema_int8 =
1232            ColumnSchema::new("test_column_1", ConcreteDataType::int8_datatype(), true);
1233
1234        let column_schema_int32 =
1235            ColumnSchema::new("test_column_2", ConcreteDataType::int32_datatype(), false);
1236
1237        let formatted_int8 = format!("{:?}", column_schema_int8);
1238        let formatted_int32 = format!("{:?}", column_schema_int32);
1239        assert_eq!(formatted_int8, "test_column_1 Int8 null");
1240        assert_eq!(formatted_int32, "test_column_2 Int32 not null");
1241    }
1242
1243    #[test]
1244    fn test_from_field_to_column_schema() {
1245        let field = Field::new("test", ArrowDataType::Int32, true);
1246        let column_schema = ColumnSchema::try_from(&field).unwrap();
1247        assert_eq!("test", column_schema.name);
1248        assert_eq!(ConcreteDataType::int32_datatype(), column_schema.data_type);
1249        assert!(column_schema.is_nullable);
1250        assert!(!column_schema.is_time_index);
1251        assert!(column_schema.default_constraint.is_none());
1252        assert!(column_schema.metadata.is_empty());
1253
1254        let field = Field::new("test", ArrowDataType::Binary, true);
1255        let field = field.with_metadata(Metadata::from([(
1256            TYPE_KEY.to_string(),
1257            ConcreteDataType::json_datatype().name(),
1258        )]));
1259        let column_schema = ColumnSchema::try_from(&field).unwrap();
1260        assert_eq!("test", column_schema.name);
1261        assert_eq!(ConcreteDataType::json_datatype(), column_schema.data_type);
1262        assert!(column_schema.is_nullable);
1263        assert!(!column_schema.is_time_index);
1264        assert!(column_schema.default_constraint.is_none());
1265        assert_eq!(
1266            column_schema.metadata.get(TYPE_KEY).unwrap(),
1267            &ConcreteDataType::json_datatype().name()
1268        );
1269
1270        let field = Field::new("test", ArrowDataType::Binary, true);
1271        let field = field.with_metadata(Metadata::from([(
1272            TYPE_KEY.to_string(),
1273            ConcreteDataType::vector_datatype(3).name(),
1274        )]));
1275        let column_schema = ColumnSchema::try_from(&field).unwrap();
1276        assert_eq!("test", column_schema.name);
1277        assert_eq!(
1278            ConcreteDataType::vector_datatype(3),
1279            column_schema.data_type
1280        );
1281        assert!(column_schema.is_nullable);
1282        assert!(!column_schema.is_time_index);
1283        assert!(column_schema.default_constraint.is_none());
1284        assert_eq!(
1285            column_schema.metadata.get(TYPE_KEY).unwrap(),
1286            &ConcreteDataType::vector_datatype(3).name()
1287        );
1288    }
1289
1290    #[test]
1291    fn test_column_schema_fix_time_index() {
1292        let field = Field::new(
1293            "test",
1294            ArrowDataType::Timestamp(TimeUnit::Second, None),
1295            false,
1296        );
1297        let field = field.with_metadata(Metadata::from([(
1298            TIME_INDEX_KEY.to_string(),
1299            "true".to_string(),
1300        )]));
1301        let column_schema = ColumnSchema::try_from(&field).unwrap();
1302        assert_eq!("test", column_schema.name);
1303        assert_eq!(
1304            ConcreteDataType::timestamp_second_datatype(),
1305            column_schema.data_type
1306        );
1307        assert!(!column_schema.is_nullable);
1308        assert!(column_schema.is_time_index);
1309        assert!(column_schema.default_constraint.is_none());
1310        assert_eq!(1, column_schema.metadata().len());
1311
1312        let field = Field::new("test", ArrowDataType::Int32, false);
1313        let field = field.with_metadata(Metadata::from([(
1314            TIME_INDEX_KEY.to_string(),
1315            "true".to_string(),
1316        )]));
1317        let column_schema = ColumnSchema::try_from(&field).unwrap();
1318        assert_eq!("test", column_schema.name);
1319        assert_eq!(ConcreteDataType::int32_datatype(), column_schema.data_type);
1320        assert!(!column_schema.is_nullable);
1321        assert!(!column_schema.is_time_index);
1322        assert!(column_schema.default_constraint.is_none());
1323        assert!(column_schema.metadata.is_empty());
1324    }
1325
1326    #[test]
1327    fn test_skipping_index_options_deserialization() {
1328        let original_options = "{\"granularity\":1024,\"false-positive-rate-in-10000\":10,\"index-type\":\"BloomFilter\"}";
1329        let options = serde_json::from_str::<SkippingIndexOptions>(original_options).unwrap();
1330        assert_eq!(1024, options.granularity);
1331        assert_eq!(SkippingIndexType::BloomFilter, options.index_type);
1332        assert_eq!(0.001, options.false_positive_rate());
1333
1334        let options_str = serde_json::to_string(&options).unwrap();
1335        assert_eq!(options_str, original_options);
1336    }
1337
1338    #[test]
1339    fn test_skipping_index_options_deserialization_v0_14_to_v0_15() {
1340        let options = "{\"granularity\":10240,\"index-type\":\"BloomFilter\"}";
1341        let options = serde_json::from_str::<SkippingIndexOptions>(options).unwrap();
1342        assert_eq!(10240, options.granularity);
1343        assert_eq!(SkippingIndexType::BloomFilter, options.index_type);
1344        assert_eq!(DEFAULT_FALSE_POSITIVE_RATE, options.false_positive_rate());
1345
1346        let options_str = serde_json::to_string(&options).unwrap();
1347        assert_eq!(
1348            options_str,
1349            "{\"granularity\":10240,\"false-positive-rate-in-10000\":100,\"index-type\":\"BloomFilter\"}"
1350        );
1351    }
1352
1353    #[test]
1354    fn test_fulltext_options_deserialization() {
1355        let original_options = "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":1024,\"false-positive-rate-in-10000\":10}";
1356        let options = serde_json::from_str::<FulltextOptions>(original_options).unwrap();
1357        assert!(!options.case_sensitive);
1358        assert!(options.enable);
1359        assert_eq!(FulltextBackend::Bloom, options.backend);
1360        assert_eq!(FulltextAnalyzer::default(), options.analyzer);
1361        assert_eq!(1024, options.granularity);
1362        assert_eq!(0.001, options.false_positive_rate());
1363
1364        let options_str = serde_json::to_string(&options).unwrap();
1365        assert_eq!(options_str, original_options);
1366    }
1367
1368    #[test]
1369    fn test_fulltext_options_deserialization_v0_14_to_v0_15() {
1370        // 0.14 to 0.15
1371        let options = "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}";
1372        let options = serde_json::from_str::<FulltextOptions>(options).unwrap();
1373        assert!(!options.case_sensitive);
1374        assert!(options.enable);
1375        assert_eq!(FulltextBackend::Bloom, options.backend);
1376        assert_eq!(FulltextAnalyzer::default(), options.analyzer);
1377        assert_eq!(DEFAULT_GRANULARITY, options.granularity);
1378        assert_eq!(DEFAULT_FALSE_POSITIVE_RATE, options.false_positive_rate());
1379
1380        let options_str = serde_json::to_string(&options).unwrap();
1381        assert_eq!(
1382            options_str,
1383            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
1384        );
1385    }
1386}
datatypes/schema/column_schema.rs

datatypes/schema/
column_schema.rs
