api/v1/
column_def.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16
17use datatypes::schema::{
18    ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextBackend, FulltextOptions,
19    SkippingIndexOptions, SkippingIndexType, COMMENT_KEY, FULLTEXT_KEY, INVERTED_INDEX_KEY,
20    SKIPPING_INDEX_KEY,
21};
22use greptime_proto::v1::{
23    Analyzer, FulltextBackend as PbFulltextBackend, SkippingIndexType as PbSkippingIndexType,
24};
25use snafu::ResultExt;
26
27use crate::error::{self, Result};
28use crate::helper::ColumnDataTypeWrapper;
29use crate::v1::{ColumnDef, ColumnOptions, SemanticType};
30
31/// Key used to store fulltext options in gRPC column options.
32const FULLTEXT_GRPC_KEY: &str = "fulltext";
33/// Key used to store inverted index options in gRPC column options.
34const INVERTED_INDEX_GRPC_KEY: &str = "inverted_index";
35/// Key used to store skip index options in gRPC column options.
36const SKIPPING_INDEX_GRPC_KEY: &str = "skipping_index";
37
38/// Tries to construct a `ColumnSchema` from the given  `ColumnDef`.
39pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
40    let data_type =
41        ColumnDataTypeWrapper::try_new(column_def.data_type, column_def.datatype_extension)?;
42
43    let constraint = if column_def.default_constraint.is_empty() {
44        None
45    } else {
46        Some(
47            ColumnDefaultConstraint::try_from(column_def.default_constraint.as_slice()).context(
48                error::ConvertColumnDefaultConstraintSnafu {
49                    column: &column_def.name,
50                },
51            )?,
52        )
53    };
54
55    let mut metadata = HashMap::new();
56    if !column_def.comment.is_empty() {
57        metadata.insert(COMMENT_KEY.to_string(), column_def.comment.clone());
58    }
59    if let Some(options) = column_def.options.as_ref() {
60        if let Some(fulltext) = options.options.get(FULLTEXT_GRPC_KEY) {
61            metadata.insert(FULLTEXT_KEY.to_string(), fulltext.to_owned());
62        }
63        if let Some(inverted_index) = options.options.get(INVERTED_INDEX_GRPC_KEY) {
64            metadata.insert(INVERTED_INDEX_KEY.to_string(), inverted_index.to_owned());
65        }
66        if let Some(skipping_index) = options.options.get(SKIPPING_INDEX_GRPC_KEY) {
67            metadata.insert(SKIPPING_INDEX_KEY.to_string(), skipping_index.to_owned());
68        }
69    }
70
71    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
72        .with_metadata(metadata)
73        .with_time_index(column_def.semantic_type() == SemanticType::Timestamp)
74        .with_default_constraint(constraint)
75        .context(error::InvalidColumnDefaultConstraintSnafu {
76            column: &column_def.name,
77        })
78}
79
80/// Constructs a `ColumnOptions` from the given `ColumnSchema`.
81pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option<ColumnOptions> {
82    let mut options = ColumnOptions::default();
83    if let Some(fulltext) = column_schema.metadata().get(FULLTEXT_KEY) {
84        options
85            .options
86            .insert(FULLTEXT_GRPC_KEY.to_string(), fulltext.to_owned());
87    }
88    if let Some(inverted_index) = column_schema.metadata().get(INVERTED_INDEX_KEY) {
89        options
90            .options
91            .insert(INVERTED_INDEX_GRPC_KEY.to_string(), inverted_index.clone());
92    }
93    if let Some(skipping_index) = column_schema.metadata().get(SKIPPING_INDEX_KEY) {
94        options
95            .options
96            .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), skipping_index.clone());
97    }
98
99    (!options.options.is_empty()).then_some(options)
100}
101
102/// Checks if the `ColumnOptions` contains fulltext options.
103pub fn contains_fulltext(options: &Option<ColumnOptions>) -> bool {
104    options
105        .as_ref()
106        .is_some_and(|o| o.options.contains_key(FULLTEXT_GRPC_KEY))
107}
108
109/// Checks if the `ColumnOptions` contains skipping index options.
110pub fn contains_skipping(options: &Option<ColumnOptions>) -> bool {
111    options
112        .as_ref()
113        .is_some_and(|o| o.options.contains_key(SKIPPING_INDEX_GRPC_KEY))
114}
115
116/// Tries to construct a `ColumnOptions` from the given `FulltextOptions`.
117pub fn options_from_fulltext(fulltext: &FulltextOptions) -> Result<Option<ColumnOptions>> {
118    let mut options = ColumnOptions::default();
119
120    let v = serde_json::to_string(fulltext).context(error::SerializeJsonSnafu)?;
121    options.options.insert(FULLTEXT_GRPC_KEY.to_string(), v);
122
123    Ok((!options.options.is_empty()).then_some(options))
124}
125
126/// Tries to construct a `ColumnOptions` from the given `SkippingIndexOptions`.
127pub fn options_from_skipping(skipping: &SkippingIndexOptions) -> Result<Option<ColumnOptions>> {
128    let mut options = ColumnOptions::default();
129
130    let v = serde_json::to_string(skipping).context(error::SerializeJsonSnafu)?;
131    options
132        .options
133        .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), v);
134
135    Ok((!options.options.is_empty()).then_some(options))
136}
137
138/// Tries to construct a `ColumnOptions` for inverted index.
139pub fn options_from_inverted() -> ColumnOptions {
140    let mut options = ColumnOptions::default();
141    options
142        .options
143        .insert(INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string());
144    options
145}
146
147/// Tries to construct a `FulltextAnalyzer` from the given analyzer.
148pub fn as_fulltext_option_analyzer(analyzer: Analyzer) -> FulltextAnalyzer {
149    match analyzer {
150        Analyzer::English => FulltextAnalyzer::English,
151        Analyzer::Chinese => FulltextAnalyzer::Chinese,
152    }
153}
154
155/// Tries to construct a `FulltextBackend` from the given backend.
156pub fn as_fulltext_option_backend(backend: PbFulltextBackend) -> FulltextBackend {
157    match backend {
158        PbFulltextBackend::Bloom => FulltextBackend::Bloom,
159        PbFulltextBackend::Tantivy => FulltextBackend::Tantivy,
160    }
161}
162
163/// Tries to construct a `SkippingIndexType` from the given skipping index type.
164pub fn as_skipping_index_type(skipping_index_type: PbSkippingIndexType) -> SkippingIndexType {
165    match skipping_index_type {
166        PbSkippingIndexType::BloomFilter => SkippingIndexType::BloomFilter,
167    }
168}
169
170#[cfg(test)]
171mod tests {
172
173    use datatypes::data_type::ConcreteDataType;
174    use datatypes::schema::{FulltextAnalyzer, FulltextBackend};
175
176    use super::*;
177    use crate::v1::ColumnDataType;
178
179    #[test]
180    fn test_try_as_column_schema() {
181        let column_def = ColumnDef {
182            name: "test".to_string(),
183            data_type: ColumnDataType::String as i32,
184            is_nullable: true,
185            default_constraint: ColumnDefaultConstraint::Value("test_default".into())
186                .try_into()
187                .unwrap(),
188            semantic_type: SemanticType::Field as i32,
189            comment: "test_comment".to_string(),
190            datatype_extension: None,
191            options: Some(ColumnOptions {
192                options: HashMap::from([
193                    (
194                        FULLTEXT_GRPC_KEY.to_string(),
195                        "{\"enable\":true}".to_string(),
196                    ),
197                    (INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string()),
198                ]),
199            }),
200        };
201
202        let schema = try_as_column_schema(&column_def).unwrap();
203        assert_eq!(schema.name, "test");
204        assert_eq!(schema.data_type, ConcreteDataType::string_datatype());
205        assert!(!schema.is_time_index());
206        assert!(schema.is_nullable());
207        assert_eq!(
208            schema.default_constraint().unwrap(),
209            &ColumnDefaultConstraint::Value("test_default".into())
210        );
211        assert_eq!(schema.metadata().get(COMMENT_KEY).unwrap(), "test_comment");
212        assert_eq!(
213            schema.fulltext_options().unwrap().unwrap(),
214            FulltextOptions {
215                enable: true,
216                ..Default::default()
217            }
218        );
219        assert!(schema.is_inverted_indexed());
220    }
221
222    #[test]
223    fn test_options_from_column_schema() {
224        let schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true);
225        let options = options_from_column_schema(&schema);
226        assert!(options.is_none());
227
228        let mut schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true)
229            .with_fulltext_options(FulltextOptions {
230                enable: true,
231                analyzer: FulltextAnalyzer::English,
232                case_sensitive: false,
233                backend: FulltextBackend::Bloom,
234            })
235            .unwrap();
236        schema.set_inverted_index(true);
237        let options = options_from_column_schema(&schema).unwrap();
238        assert_eq!(
239            options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
240            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}"
241        );
242        assert_eq!(
243            options.options.get(INVERTED_INDEX_GRPC_KEY).unwrap(),
244            "true"
245        );
246    }
247
248    #[test]
249    fn test_options_with_fulltext() {
250        let fulltext = FulltextOptions {
251            enable: true,
252            analyzer: FulltextAnalyzer::English,
253            case_sensitive: false,
254            backend: FulltextBackend::Bloom,
255        };
256        let options = options_from_fulltext(&fulltext).unwrap().unwrap();
257        assert_eq!(
258            options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
259            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\"}"
260        );
261    }
262
263    #[test]
264    fn test_contains_fulltext() {
265        let options = ColumnOptions {
266            options: HashMap::from([(
267                FULLTEXT_GRPC_KEY.to_string(),
268                "{\"enable\":true}".to_string(),
269            )]),
270        };
271        assert!(contains_fulltext(&Some(options)));
272
273        let options = ColumnOptions {
274            options: HashMap::new(),
275        };
276        assert!(!contains_fulltext(&Some(options)));
277
278        assert!(!contains_fulltext(&None));
279    }
280}
api/v1/column_def.rs

api/v1/
column_def.rs
