api/v1/
column_def.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16
17use datatypes::schema::{
18    COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema, FULLTEXT_KEY, FulltextAnalyzer,
19    FulltextBackend, FulltextOptions, INVERTED_INDEX_KEY, JSON_STRUCTURE_SETTINGS_KEY,
20    SKIPPING_INDEX_KEY, SkippingIndexOptions, SkippingIndexType,
21};
22use greptime_proto::v1::{
23    Analyzer, FulltextBackend as PbFulltextBackend, SkippingIndexType as PbSkippingIndexType,
24};
25use snafu::ResultExt;
26
27use crate::error::{self, ConvertColumnDefaultConstraintSnafu, Result};
28use crate::helper::ColumnDataTypeWrapper;
29use crate::v1::{ColumnDef, ColumnOptions, SemanticType};
30
31/// Key used to store fulltext options in gRPC column options.
32const FULLTEXT_GRPC_KEY: &str = "fulltext";
33/// Key used to store inverted index options in gRPC column options.
34const INVERTED_INDEX_GRPC_KEY: &str = "inverted_index";
35/// Key used to store skip index options in gRPC column options.
36const SKIPPING_INDEX_GRPC_KEY: &str = "skipping_index";
37
38/// Tries to construct a `ColumnSchema` from the given  `ColumnDef`.
39pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
40    let data_type = ColumnDataTypeWrapper::try_new(
41        column_def.data_type,
42        column_def.datatype_extension.clone(),
43    )?;
44
45    let constraint = if column_def.default_constraint.is_empty() {
46        None
47    } else {
48        Some(
49            ColumnDefaultConstraint::try_from(column_def.default_constraint.as_slice()).context(
50                error::ConvertColumnDefaultConstraintSnafu {
51                    column: &column_def.name,
52                },
53            )?,
54        )
55    };
56
57    let mut metadata = HashMap::new();
58    if !column_def.comment.is_empty() {
59        metadata.insert(COMMENT_KEY.to_string(), column_def.comment.clone());
60    }
61    if let Some(options) = column_def.options.as_ref() {
62        if let Some(fulltext) = options.options.get(FULLTEXT_GRPC_KEY) {
63            metadata.insert(FULLTEXT_KEY.to_string(), fulltext.to_owned());
64        }
65        if let Some(inverted_index) = options.options.get(INVERTED_INDEX_GRPC_KEY) {
66            metadata.insert(INVERTED_INDEX_KEY.to_string(), inverted_index.to_owned());
67        }
68        if let Some(skipping_index) = options.options.get(SKIPPING_INDEX_GRPC_KEY) {
69            metadata.insert(SKIPPING_INDEX_KEY.to_string(), skipping_index.to_owned());
70        }
71        if let Some(settings) = options.options.get(JSON_STRUCTURE_SETTINGS_KEY) {
72            metadata.insert(JSON_STRUCTURE_SETTINGS_KEY.to_string(), settings.clone());
73        }
74    }
75
76    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
77        .with_metadata(metadata)
78        .with_time_index(column_def.semantic_type() == SemanticType::Timestamp)
79        .with_default_constraint(constraint)
80        .context(error::InvalidColumnDefaultConstraintSnafu {
81            column: &column_def.name,
82        })
83}
84
85/// Tries to construct a `ColumnDef` from the given `ColumnSchema`.
86///
87/// TODO(weny): Add tests for this function.
88pub fn try_as_column_def(column_schema: &ColumnSchema, is_primary_key: bool) -> Result<ColumnDef> {
89    let column_datatype =
90        ColumnDataTypeWrapper::try_from(column_schema.data_type.clone()).map(|w| w.to_parts())?;
91
92    let semantic_type = if column_schema.is_time_index() {
93        SemanticType::Timestamp
94    } else if is_primary_key {
95        SemanticType::Tag
96    } else {
97        SemanticType::Field
98    } as i32;
99    let comment = column_schema
100        .metadata()
101        .get(COMMENT_KEY)
102        .cloned()
103        .unwrap_or_default();
104
105    let default_constraint = match column_schema.default_constraint() {
106        None => vec![],
107        Some(v) => v
108            .clone()
109            .try_into()
110            .context(ConvertColumnDefaultConstraintSnafu {
111                column: &column_schema.name,
112            })?,
113    };
114    let options = options_from_column_schema(column_schema);
115    Ok(ColumnDef {
116        name: column_schema.name.clone(),
117        data_type: column_datatype.0 as i32,
118        is_nullable: column_schema.is_nullable(),
119        default_constraint,
120        semantic_type,
121        comment,
122        datatype_extension: column_datatype.1,
123        options,
124    })
125}
126
127/// Constructs a `ColumnOptions` from the given `ColumnSchema`.
128pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option<ColumnOptions> {
129    let mut options = ColumnOptions::default();
130    if let Some(fulltext) = column_schema.metadata().get(FULLTEXT_KEY) {
131        options
132            .options
133            .insert(FULLTEXT_GRPC_KEY.to_string(), fulltext.to_owned());
134    }
135    if let Some(inverted_index) = column_schema.metadata().get(INVERTED_INDEX_KEY) {
136        options
137            .options
138            .insert(INVERTED_INDEX_GRPC_KEY.to_string(), inverted_index.clone());
139    }
140    if let Some(skipping_index) = column_schema.metadata().get(SKIPPING_INDEX_KEY) {
141        options
142            .options
143            .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), skipping_index.clone());
144    }
145    if let Some(settings) = column_schema.metadata().get(JSON_STRUCTURE_SETTINGS_KEY) {
146        options
147            .options
148            .insert(JSON_STRUCTURE_SETTINGS_KEY.to_string(), settings.clone());
149    }
150
151    (!options.options.is_empty()).then_some(options)
152}
153
154/// Checks if the `ColumnOptions` contains fulltext options.
155pub fn contains_fulltext(options: &Option<ColumnOptions>) -> bool {
156    options
157        .as_ref()
158        .is_some_and(|o| o.options.contains_key(FULLTEXT_GRPC_KEY))
159}
160
161/// Checks if the `ColumnOptions` contains skipping index options.
162pub fn contains_skipping(options: &Option<ColumnOptions>) -> bool {
163    options
164        .as_ref()
165        .is_some_and(|o| o.options.contains_key(SKIPPING_INDEX_GRPC_KEY))
166}
167
168/// Tries to construct a `ColumnOptions` from the given `FulltextOptions`.
169pub fn options_from_fulltext(fulltext: &FulltextOptions) -> Result<Option<ColumnOptions>> {
170    let mut options = ColumnOptions::default();
171
172    let v = serde_json::to_string(fulltext).context(error::SerializeJsonSnafu)?;
173    options.options.insert(FULLTEXT_GRPC_KEY.to_string(), v);
174
175    Ok((!options.options.is_empty()).then_some(options))
176}
177
178/// Tries to construct a `ColumnOptions` from the given `SkippingIndexOptions`.
179pub fn options_from_skipping(skipping: &SkippingIndexOptions) -> Result<Option<ColumnOptions>> {
180    let mut options = ColumnOptions::default();
181
182    let v = serde_json::to_string(skipping).context(error::SerializeJsonSnafu)?;
183    options
184        .options
185        .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), v);
186
187    Ok((!options.options.is_empty()).then_some(options))
188}
189
190/// Tries to construct a `ColumnOptions` for inverted index.
191pub fn options_from_inverted() -> ColumnOptions {
192    let mut options = ColumnOptions::default();
193    options
194        .options
195        .insert(INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string());
196    options
197}
198
199/// Tries to construct a `FulltextAnalyzer` from the given analyzer.
200pub fn as_fulltext_option_analyzer(analyzer: Analyzer) -> FulltextAnalyzer {
201    match analyzer {
202        Analyzer::English => FulltextAnalyzer::English,
203        Analyzer::Chinese => FulltextAnalyzer::Chinese,
204    }
205}
206
207/// Tries to construct a `FulltextBackend` from the given backend.
208pub fn as_fulltext_option_backend(backend: PbFulltextBackend) -> FulltextBackend {
209    match backend {
210        PbFulltextBackend::Bloom => FulltextBackend::Bloom,
211        PbFulltextBackend::Tantivy => FulltextBackend::Tantivy,
212    }
213}
214
215/// Tries to construct a `SkippingIndexType` from the given skipping index type.
216pub fn as_skipping_index_type(skipping_index_type: PbSkippingIndexType) -> SkippingIndexType {
217    match skipping_index_type {
218        PbSkippingIndexType::BloomFilter => SkippingIndexType::BloomFilter,
219    }
220}
221
222#[cfg(test)]
223mod tests {
224
225    use datatypes::data_type::ConcreteDataType;
226    use datatypes::schema::{FulltextAnalyzer, FulltextBackend};
227
228    use super::*;
229    use crate::v1::ColumnDataType;
230
231    #[test]
232    fn test_try_as_column_schema() {
233        let column_def = ColumnDef {
234            name: "test".to_string(),
235            data_type: ColumnDataType::String as i32,
236            is_nullable: true,
237            default_constraint: ColumnDefaultConstraint::Value("test_default".into())
238                .try_into()
239                .unwrap(),
240            semantic_type: SemanticType::Field as i32,
241            comment: "test_comment".to_string(),
242            datatype_extension: None,
243            options: Some(ColumnOptions {
244                options: HashMap::from([
245                    (
246                        FULLTEXT_GRPC_KEY.to_string(),
247                        "{\"enable\":true}".to_string(),
248                    ),
249                    (INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string()),
250                ]),
251            }),
252        };
253
254        let schema = try_as_column_schema(&column_def).unwrap();
255        assert_eq!(schema.name, "test");
256        assert_eq!(schema.data_type, ConcreteDataType::string_datatype());
257        assert!(!schema.is_time_index());
258        assert!(schema.is_nullable());
259        assert_eq!(
260            schema.default_constraint().unwrap(),
261            &ColumnDefaultConstraint::Value("test_default".into())
262        );
263        assert_eq!(schema.metadata().get(COMMENT_KEY).unwrap(), "test_comment");
264        assert_eq!(
265            schema.fulltext_options().unwrap().unwrap(),
266            FulltextOptions {
267                enable: true,
268                ..Default::default()
269            }
270        );
271        assert!(schema.is_inverted_indexed());
272    }
273
274    #[test]
275    fn test_options_from_column_schema() {
276        let schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true);
277        let options = options_from_column_schema(&schema);
278        assert!(options.is_none());
279
280        let mut schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true)
281            .with_fulltext_options(FulltextOptions::new_unchecked(
282                true,
283                FulltextAnalyzer::English,
284                false,
285                FulltextBackend::Bloom,
286                10240,
287                0.01,
288            ))
289            .unwrap();
290        schema.set_inverted_index(true);
291        let options = options_from_column_schema(&schema).unwrap();
292        assert_eq!(
293            options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
294            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
295        );
296        assert_eq!(
297            options.options.get(INVERTED_INDEX_GRPC_KEY).unwrap(),
298            "true"
299        );
300    }
301
302    #[test]
303    fn test_options_with_fulltext() {
304        let fulltext = FulltextOptions::new_unchecked(
305            true,
306            FulltextAnalyzer::English,
307            false,
308            FulltextBackend::Bloom,
309            10240,
310            0.01,
311        );
312        let options = options_from_fulltext(&fulltext).unwrap().unwrap();
313        assert_eq!(
314            options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
315            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
316        );
317    }
318
319    #[test]
320    fn test_contains_fulltext() {
321        let options = ColumnOptions {
322            options: HashMap::from([(
323                FULLTEXT_GRPC_KEY.to_string(),
324                "{\"enable\":true}".to_string(),
325            )]),
326        };
327        assert!(contains_fulltext(&Some(options)));
328
329        let options = ColumnOptions {
330            options: HashMap::new(),
331        };
332        assert!(!contains_fulltext(&Some(options)));
333
334        assert!(!contains_fulltext(&None));
335    }
336}
api/v1/column_def.rs

api/v1/
column_def.rs
