api/v1/
column_def.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16
17use datatypes::schema::{
18    ColumnDefaultConstraint, ColumnSchema, FulltextAnalyzer, FulltextBackend, FulltextOptions,
19    SkippingIndexOptions, SkippingIndexType, COMMENT_KEY, FULLTEXT_KEY, INVERTED_INDEX_KEY,
20    SKIPPING_INDEX_KEY,
21};
22use greptime_proto::v1::{
23    Analyzer, FulltextBackend as PbFulltextBackend, SkippingIndexType as PbSkippingIndexType,
24};
25use snafu::ResultExt;
26
27use crate::error::{self, ConvertColumnDefaultConstraintSnafu, Result};
28use crate::helper::ColumnDataTypeWrapper;
29use crate::v1::{ColumnDef, ColumnOptions, SemanticType};
30
31/// Key used to store fulltext options in gRPC column options.
32const FULLTEXT_GRPC_KEY: &str = "fulltext";
33/// Key used to store inverted index options in gRPC column options.
34const INVERTED_INDEX_GRPC_KEY: &str = "inverted_index";
35/// Key used to store skip index options in gRPC column options.
36const SKIPPING_INDEX_GRPC_KEY: &str = "skipping_index";
37
38/// Tries to construct a `ColumnSchema` from the given  `ColumnDef`.
39pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
40    let data_type =
41        ColumnDataTypeWrapper::try_new(column_def.data_type, column_def.datatype_extension)?;
42
43    let constraint = if column_def.default_constraint.is_empty() {
44        None
45    } else {
46        Some(
47            ColumnDefaultConstraint::try_from(column_def.default_constraint.as_slice()).context(
48                error::ConvertColumnDefaultConstraintSnafu {
49                    column: &column_def.name,
50                },
51            )?,
52        )
53    };
54
55    let mut metadata = HashMap::new();
56    if !column_def.comment.is_empty() {
57        metadata.insert(COMMENT_KEY.to_string(), column_def.comment.clone());
58    }
59    if let Some(options) = column_def.options.as_ref() {
60        if let Some(fulltext) = options.options.get(FULLTEXT_GRPC_KEY) {
61            metadata.insert(FULLTEXT_KEY.to_string(), fulltext.to_owned());
62        }
63        if let Some(inverted_index) = options.options.get(INVERTED_INDEX_GRPC_KEY) {
64            metadata.insert(INVERTED_INDEX_KEY.to_string(), inverted_index.to_owned());
65        }
66        if let Some(skipping_index) = options.options.get(SKIPPING_INDEX_GRPC_KEY) {
67            metadata.insert(SKIPPING_INDEX_KEY.to_string(), skipping_index.to_owned());
68        }
69    }
70
71    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
72        .with_metadata(metadata)
73        .with_time_index(column_def.semantic_type() == SemanticType::Timestamp)
74        .with_default_constraint(constraint)
75        .context(error::InvalidColumnDefaultConstraintSnafu {
76            column: &column_def.name,
77        })
78}
79
80/// Tries to construct a `ColumnDef` from the given `ColumnSchema`.
81///
82/// TODO(weny): Add tests for this function.
83pub fn try_as_column_def(column_schema: &ColumnSchema, is_primary_key: bool) -> Result<ColumnDef> {
84    let column_datatype =
85        ColumnDataTypeWrapper::try_from(column_schema.data_type.clone()).map(|w| w.to_parts())?;
86
87    let semantic_type = if column_schema.is_time_index() {
88        SemanticType::Timestamp
89    } else if is_primary_key {
90        SemanticType::Tag
91    } else {
92        SemanticType::Field
93    } as i32;
94    let comment = column_schema
95        .metadata()
96        .get(COMMENT_KEY)
97        .cloned()
98        .unwrap_or_default();
99
100    let default_constraint = match column_schema.default_constraint() {
101        None => vec![],
102        Some(v) => v
103            .clone()
104            .try_into()
105            .context(ConvertColumnDefaultConstraintSnafu {
106                column: &column_schema.name,
107            })?,
108    };
109    let options = options_from_column_schema(column_schema);
110    Ok(ColumnDef {
111        name: column_schema.name.clone(),
112        data_type: column_datatype.0 as i32,
113        is_nullable: column_schema.is_nullable(),
114        default_constraint,
115        semantic_type,
116        comment,
117        datatype_extension: column_datatype.1,
118        options,
119    })
120}
121
122/// Constructs a `ColumnOptions` from the given `ColumnSchema`.
123pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option<ColumnOptions> {
124    let mut options = ColumnOptions::default();
125    if let Some(fulltext) = column_schema.metadata().get(FULLTEXT_KEY) {
126        options
127            .options
128            .insert(FULLTEXT_GRPC_KEY.to_string(), fulltext.to_owned());
129    }
130    if let Some(inverted_index) = column_schema.metadata().get(INVERTED_INDEX_KEY) {
131        options
132            .options
133            .insert(INVERTED_INDEX_GRPC_KEY.to_string(), inverted_index.clone());
134    }
135    if let Some(skipping_index) = column_schema.metadata().get(SKIPPING_INDEX_KEY) {
136        options
137            .options
138            .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), skipping_index.clone());
139    }
140
141    (!options.options.is_empty()).then_some(options)
142}
143
144/// Checks if the `ColumnOptions` contains fulltext options.
145pub fn contains_fulltext(options: &Option<ColumnOptions>) -> bool {
146    options
147        .as_ref()
148        .is_some_and(|o| o.options.contains_key(FULLTEXT_GRPC_KEY))
149}
150
151/// Checks if the `ColumnOptions` contains skipping index options.
152pub fn contains_skipping(options: &Option<ColumnOptions>) -> bool {
153    options
154        .as_ref()
155        .is_some_and(|o| o.options.contains_key(SKIPPING_INDEX_GRPC_KEY))
156}
157
158/// Tries to construct a `ColumnOptions` from the given `FulltextOptions`.
159pub fn options_from_fulltext(fulltext: &FulltextOptions) -> Result<Option<ColumnOptions>> {
160    let mut options = ColumnOptions::default();
161
162    let v = serde_json::to_string(fulltext).context(error::SerializeJsonSnafu)?;
163    options.options.insert(FULLTEXT_GRPC_KEY.to_string(), v);
164
165    Ok((!options.options.is_empty()).then_some(options))
166}
167
168/// Tries to construct a `ColumnOptions` from the given `SkippingIndexOptions`.
169pub fn options_from_skipping(skipping: &SkippingIndexOptions) -> Result<Option<ColumnOptions>> {
170    let mut options = ColumnOptions::default();
171
172    let v = serde_json::to_string(skipping).context(error::SerializeJsonSnafu)?;
173    options
174        .options
175        .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), v);
176
177    Ok((!options.options.is_empty()).then_some(options))
178}
179
180/// Tries to construct a `ColumnOptions` for inverted index.
181pub fn options_from_inverted() -> ColumnOptions {
182    let mut options = ColumnOptions::default();
183    options
184        .options
185        .insert(INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string());
186    options
187}
188
189/// Tries to construct a `FulltextAnalyzer` from the given analyzer.
190pub fn as_fulltext_option_analyzer(analyzer: Analyzer) -> FulltextAnalyzer {
191    match analyzer {
192        Analyzer::English => FulltextAnalyzer::English,
193        Analyzer::Chinese => FulltextAnalyzer::Chinese,
194    }
195}
196
197/// Tries to construct a `FulltextBackend` from the given backend.
198pub fn as_fulltext_option_backend(backend: PbFulltextBackend) -> FulltextBackend {
199    match backend {
200        PbFulltextBackend::Bloom => FulltextBackend::Bloom,
201        PbFulltextBackend::Tantivy => FulltextBackend::Tantivy,
202    }
203}
204
205/// Tries to construct a `SkippingIndexType` from the given skipping index type.
206pub fn as_skipping_index_type(skipping_index_type: PbSkippingIndexType) -> SkippingIndexType {
207    match skipping_index_type {
208        PbSkippingIndexType::BloomFilter => SkippingIndexType::BloomFilter,
209    }
210}
211
212#[cfg(test)]
213mod tests {
214
215    use datatypes::data_type::ConcreteDataType;
216    use datatypes::schema::{FulltextAnalyzer, FulltextBackend};
217
218    use super::*;
219    use crate::v1::ColumnDataType;
220
221    #[test]
222    fn test_try_as_column_schema() {
223        let column_def = ColumnDef {
224            name: "test".to_string(),
225            data_type: ColumnDataType::String as i32,
226            is_nullable: true,
227            default_constraint: ColumnDefaultConstraint::Value("test_default".into())
228                .try_into()
229                .unwrap(),
230            semantic_type: SemanticType::Field as i32,
231            comment: "test_comment".to_string(),
232            datatype_extension: None,
233            options: Some(ColumnOptions {
234                options: HashMap::from([
235                    (
236                        FULLTEXT_GRPC_KEY.to_string(),
237                        "{\"enable\":true}".to_string(),
238                    ),
239                    (INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string()),
240                ]),
241            }),
242        };
243
244        let schema = try_as_column_schema(&column_def).unwrap();
245        assert_eq!(schema.name, "test");
246        assert_eq!(schema.data_type, ConcreteDataType::string_datatype());
247        assert!(!schema.is_time_index());
248        assert!(schema.is_nullable());
249        assert_eq!(
250            schema.default_constraint().unwrap(),
251            &ColumnDefaultConstraint::Value("test_default".into())
252        );
253        assert_eq!(schema.metadata().get(COMMENT_KEY).unwrap(), "test_comment");
254        assert_eq!(
255            schema.fulltext_options().unwrap().unwrap(),
256            FulltextOptions {
257                enable: true,
258                ..Default::default()
259            }
260        );
261        assert!(schema.is_inverted_indexed());
262    }
263
264    #[test]
265    fn test_options_from_column_schema() {
266        let schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true);
267        let options = options_from_column_schema(&schema);
268        assert!(options.is_none());
269
270        let mut schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true)
271            .with_fulltext_options(FulltextOptions::new_unchecked(
272                true,
273                FulltextAnalyzer::English,
274                false,
275                FulltextBackend::Bloom,
276                10240,
277                0.01,
278            ))
279            .unwrap();
280        schema.set_inverted_index(true);
281        let options = options_from_column_schema(&schema).unwrap();
282        assert_eq!(
283            options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
284            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
285        );
286        assert_eq!(
287            options.options.get(INVERTED_INDEX_GRPC_KEY).unwrap(),
288            "true"
289        );
290    }
291
292    #[test]
293    fn test_options_with_fulltext() {
294        let fulltext = FulltextOptions::new_unchecked(
295            true,
296            FulltextAnalyzer::English,
297            false,
298            FulltextBackend::Bloom,
299            10240,
300            0.01,
301        );
302        let options = options_from_fulltext(&fulltext).unwrap().unwrap();
303        assert_eq!(
304            options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
305            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
306        );
307    }
308
309    #[test]
310    fn test_contains_fulltext() {
311        let options = ColumnOptions {
312            options: HashMap::from([(
313                FULLTEXT_GRPC_KEY.to_string(),
314                "{\"enable\":true}".to_string(),
315            )]),
316        };
317        assert!(contains_fulltext(&Some(options)));
318
319        let options = ColumnOptions {
320            options: HashMap::new(),
321        };
322        assert!(!contains_fulltext(&Some(options)));
323
324        assert!(!contains_fulltext(&None));
325    }
326}
api/v1/column_def.rs

api/v1/
column_def.rs
