api/v1/
column_def.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16
17use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY};
18use datatypes::schema::{
19    COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema, FULLTEXT_KEY, FulltextAnalyzer,
20    FulltextBackend, FulltextOptions, INVERTED_INDEX_KEY, SKIPPING_INDEX_KEY, SkippingIndexOptions,
21    SkippingIndexType,
22};
23use greptime_proto::v1::{
24    Analyzer, FulltextBackend as PbFulltextBackend, SkippingIndexType as PbSkippingIndexType,
25};
26use snafu::ResultExt;
27
28use crate::error::{self, ConvertColumnDefaultConstraintSnafu, Result};
29use crate::helper::ColumnDataTypeWrapper;
30use crate::v1::{ColumnDef, ColumnOptions, SemanticType};
31
32/// Key used to store fulltext options in gRPC column options.
33const FULLTEXT_GRPC_KEY: &str = "fulltext";
34/// Key used to store inverted index options in gRPC column options.
35const INVERTED_INDEX_GRPC_KEY: &str = "inverted_index";
36/// Key used to store skip index options in gRPC column options.
37const SKIPPING_INDEX_GRPC_KEY: &str = "skipping_index";
38
39/// Tries to construct a `ColumnSchema` from the given  `ColumnDef`.
40pub fn try_as_column_schema(column_def: &ColumnDef) -> Result<ColumnSchema> {
41    let data_type = ColumnDataTypeWrapper::try_new(
42        column_def.data_type,
43        column_def.datatype_extension.clone(),
44    )?;
45
46    let constraint = if column_def.default_constraint.is_empty() {
47        None
48    } else {
49        Some(
50            ColumnDefaultConstraint::try_from(column_def.default_constraint.as_slice()).context(
51                error::ConvertColumnDefaultConstraintSnafu {
52                    column: &column_def.name,
53                },
54            )?,
55        )
56    };
57
58    let mut metadata = HashMap::new();
59    if !column_def.comment.is_empty() {
60        metadata.insert(COMMENT_KEY.to_string(), column_def.comment.clone());
61    }
62    if let Some(options) = column_def.options.as_ref() {
63        if let Some(fulltext) = options.options.get(FULLTEXT_GRPC_KEY) {
64            metadata.insert(FULLTEXT_KEY.to_string(), fulltext.to_owned());
65        }
66        if let Some(inverted_index) = options.options.get(INVERTED_INDEX_GRPC_KEY) {
67            metadata.insert(INVERTED_INDEX_KEY.to_string(), inverted_index.to_owned());
68        }
69        if let Some(skipping_index) = options.options.get(SKIPPING_INDEX_GRPC_KEY) {
70            metadata.insert(SKIPPING_INDEX_KEY.to_string(), skipping_index.to_owned());
71        }
72        if let Some(extension_name) = options.options.get(EXTENSION_TYPE_NAME_KEY) {
73            metadata.insert(EXTENSION_TYPE_NAME_KEY.to_string(), extension_name.clone());
74        }
75        if let Some(extension_metadata) = options.options.get(EXTENSION_TYPE_METADATA_KEY) {
76            metadata.insert(
77                EXTENSION_TYPE_METADATA_KEY.to_string(),
78                extension_metadata.clone(),
79            );
80        }
81    }
82
83    ColumnSchema::new(&column_def.name, data_type.into(), column_def.is_nullable)
84        .with_metadata(metadata)
85        .with_time_index(column_def.semantic_type() == SemanticType::Timestamp)
86        .with_default_constraint(constraint)
87        .context(error::InvalidColumnDefaultConstraintSnafu {
88            column: &column_def.name,
89        })
90}
91
92/// Tries to construct a `ColumnDef` from the given `ColumnSchema`.
93///
94/// TODO(weny): Add tests for this function.
95pub fn try_as_column_def(column_schema: &ColumnSchema, is_primary_key: bool) -> Result<ColumnDef> {
96    let column_datatype =
97        ColumnDataTypeWrapper::try_from(column_schema.data_type.clone()).map(|w| w.to_parts())?;
98
99    let semantic_type = if column_schema.is_time_index() {
100        SemanticType::Timestamp
101    } else if is_primary_key {
102        SemanticType::Tag
103    } else {
104        SemanticType::Field
105    } as i32;
106    let comment = column_schema
107        .metadata()
108        .get(COMMENT_KEY)
109        .cloned()
110        .unwrap_or_default();
111
112    let default_constraint = match column_schema.default_constraint() {
113        None => vec![],
114        Some(v) => v
115            .clone()
116            .try_into()
117            .context(ConvertColumnDefaultConstraintSnafu {
118                column: &column_schema.name,
119            })?,
120    };
121    let options = options_from_column_schema(column_schema);
122    Ok(ColumnDef {
123        name: column_schema.name.clone(),
124        data_type: column_datatype.0 as i32,
125        is_nullable: column_schema.is_nullable(),
126        default_constraint,
127        semantic_type,
128        comment,
129        datatype_extension: column_datatype.1,
130        options,
131    })
132}
133
134/// Constructs a `ColumnOptions` from the given `ColumnSchema`.
135pub fn options_from_column_schema(column_schema: &ColumnSchema) -> Option<ColumnOptions> {
136    let mut options = ColumnOptions::default();
137    if let Some(fulltext) = column_schema.metadata().get(FULLTEXT_KEY) {
138        options
139            .options
140            .insert(FULLTEXT_GRPC_KEY.to_string(), fulltext.to_owned());
141    }
142    if let Some(inverted_index) = column_schema.metadata().get(INVERTED_INDEX_KEY) {
143        options
144            .options
145            .insert(INVERTED_INDEX_GRPC_KEY.to_string(), inverted_index.clone());
146    }
147    if let Some(skipping_index) = column_schema.metadata().get(SKIPPING_INDEX_KEY) {
148        options
149            .options
150            .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), skipping_index.clone());
151    }
152    if let Some(extension_name) = column_schema.metadata().get(EXTENSION_TYPE_NAME_KEY) {
153        options
154            .options
155            .insert(EXTENSION_TYPE_NAME_KEY.to_string(), extension_name.clone());
156    }
157    if let Some(extension_metadata) = column_schema.metadata().get(EXTENSION_TYPE_METADATA_KEY) {
158        options.options.insert(
159            EXTENSION_TYPE_METADATA_KEY.to_string(),
160            extension_metadata.clone(),
161        );
162    }
163
164    (!options.options.is_empty()).then_some(options)
165}
166
167/// Checks if the `ColumnOptions` contains fulltext options.
168pub fn contains_fulltext(options: &Option<ColumnOptions>) -> bool {
169    options
170        .as_ref()
171        .is_some_and(|o| o.options.contains_key(FULLTEXT_GRPC_KEY))
172}
173
174/// Checks if the `ColumnOptions` contains skipping index options.
175pub fn contains_skipping(options: &Option<ColumnOptions>) -> bool {
176    options
177        .as_ref()
178        .is_some_and(|o| o.options.contains_key(SKIPPING_INDEX_GRPC_KEY))
179}
180
181/// Tries to construct a `ColumnOptions` from the given `FulltextOptions`.
182pub fn options_from_fulltext(fulltext: &FulltextOptions) -> Result<Option<ColumnOptions>> {
183    let mut options = ColumnOptions::default();
184
185    let v = serde_json::to_string(fulltext).context(error::SerializeJsonSnafu)?;
186    options.options.insert(FULLTEXT_GRPC_KEY.to_string(), v);
187
188    Ok((!options.options.is_empty()).then_some(options))
189}
190
191/// Tries to construct a `ColumnOptions` from the given `SkippingIndexOptions`.
192pub fn options_from_skipping(skipping: &SkippingIndexOptions) -> Result<Option<ColumnOptions>> {
193    let mut options = ColumnOptions::default();
194
195    let v = serde_json::to_string(skipping).context(error::SerializeJsonSnafu)?;
196    options
197        .options
198        .insert(SKIPPING_INDEX_GRPC_KEY.to_string(), v);
199
200    Ok((!options.options.is_empty()).then_some(options))
201}
202
203/// Tries to construct a `ColumnOptions` for inverted index.
204pub fn options_from_inverted() -> ColumnOptions {
205    let mut options = ColumnOptions::default();
206    options
207        .options
208        .insert(INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string());
209    options
210}
211
212/// Tries to construct a `FulltextAnalyzer` from the given analyzer.
213pub fn as_fulltext_option_analyzer(analyzer: Analyzer) -> FulltextAnalyzer {
214    match analyzer {
215        Analyzer::English => FulltextAnalyzer::English,
216        Analyzer::Chinese => FulltextAnalyzer::Chinese,
217    }
218}
219
220/// Tries to construct a `FulltextBackend` from the given backend.
221pub fn as_fulltext_option_backend(backend: PbFulltextBackend) -> FulltextBackend {
222    match backend {
223        PbFulltextBackend::Bloom => FulltextBackend::Bloom,
224        PbFulltextBackend::Tantivy => FulltextBackend::Tantivy,
225    }
226}
227
228/// Tries to construct a `SkippingIndexType` from the given skipping index type.
229pub fn as_skipping_index_type(skipping_index_type: PbSkippingIndexType) -> SkippingIndexType {
230    match skipping_index_type {
231        PbSkippingIndexType::BloomFilter => SkippingIndexType::BloomFilter,
232    }
233}
234
235#[cfg(test)]
236mod tests {
237
238    use datatypes::data_type::ConcreteDataType;
239    use datatypes::schema::{FulltextAnalyzer, FulltextBackend};
240
241    use super::*;
242    use crate::v1::ColumnDataType;
243
244    #[test]
245    fn test_try_as_column_schema() {
246        let column_def = ColumnDef {
247            name: "test".to_string(),
248            data_type: ColumnDataType::String as i32,
249            is_nullable: true,
250            default_constraint: ColumnDefaultConstraint::Value("test_default".into())
251                .try_into()
252                .unwrap(),
253            semantic_type: SemanticType::Field as i32,
254            comment: "test_comment".to_string(),
255            datatype_extension: None,
256            options: Some(ColumnOptions {
257                options: HashMap::from([
258                    (
259                        FULLTEXT_GRPC_KEY.to_string(),
260                        "{\"enable\":true}".to_string(),
261                    ),
262                    (INVERTED_INDEX_GRPC_KEY.to_string(), "true".to_string()),
263                ]),
264            }),
265        };
266
267        let schema = try_as_column_schema(&column_def).unwrap();
268        assert_eq!(schema.name, "test");
269        assert_eq!(schema.data_type, ConcreteDataType::string_datatype());
270        assert!(!schema.is_time_index());
271        assert!(schema.is_nullable());
272        assert_eq!(
273            schema.default_constraint().unwrap(),
274            &ColumnDefaultConstraint::Value("test_default".into())
275        );
276        assert_eq!(schema.metadata().get(COMMENT_KEY).unwrap(), "test_comment");
277        assert_eq!(
278            schema.fulltext_options().unwrap().unwrap(),
279            FulltextOptions {
280                enable: true,
281                ..Default::default()
282            }
283        );
284        assert!(schema.is_inverted_indexed());
285    }
286
287    #[test]
288    fn test_options_from_column_schema() {
289        let schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true);
290        let options = options_from_column_schema(&schema);
291        assert!(options.is_none());
292
293        let mut schema = ColumnSchema::new("test", ConcreteDataType::string_datatype(), true)
294            .with_fulltext_options(FulltextOptions::new_unchecked(
295                true,
296                FulltextAnalyzer::English,
297                false,
298                FulltextBackend::Bloom,
299                10240,
300                0.01,
301            ))
302            .unwrap();
303        schema.set_inverted_index(true);
304        let options = options_from_column_schema(&schema).unwrap();
305        assert_eq!(
306            options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
307            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
308        );
309        assert_eq!(
310            options.options.get(INVERTED_INDEX_GRPC_KEY).unwrap(),
311            "true"
312        );
313    }
314
315    #[test]
316    fn test_options_with_fulltext() {
317        let fulltext = FulltextOptions::new_unchecked(
318            true,
319            FulltextAnalyzer::English,
320            false,
321            FulltextBackend::Bloom,
322            10240,
323            0.01,
324        );
325        let options = options_from_fulltext(&fulltext).unwrap().unwrap();
326        assert_eq!(
327            options.options.get(FULLTEXT_GRPC_KEY).unwrap(),
328            "{\"enable\":true,\"analyzer\":\"English\",\"case-sensitive\":false,\"backend\":\"bloom\",\"granularity\":10240,\"false-positive-rate-in-10000\":100}"
329        );
330    }
331
332    #[test]
333    fn test_contains_fulltext() {
334        let options = ColumnOptions {
335            options: HashMap::from([(
336                FULLTEXT_GRPC_KEY.to_string(),
337                "{\"enable\":true}".to_string(),
338            )]),
339        };
340        assert!(contains_fulltext(&Some(options)));
341
342        let options = ColumnOptions {
343            options: HashMap::new(),
344        };
345        assert!(!contains_fulltext(&Some(options)));
346
347        assert!(!contains_fulltext(&None));
348    }
349}
api/v1/column_def.rs

api/v1/
column_def.rs
