sql/
statements.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod admin;
16pub mod alter;
17pub mod comment;
18pub mod copy;
19pub mod create;
20pub mod cursor;
21pub mod delete;
22pub mod describe;
23pub mod drop;
24pub mod explain;
25pub mod insert;
26pub mod kill;
27mod option_map;
28pub mod query;
29pub mod set_variables;
30pub mod show;
31pub mod statement;
32pub mod tql;
33pub(crate) mod transform;
34pub mod truncate;
35
36use std::sync::Arc;
37
38use api::helper::ColumnDataTypeWrapper;
39use api::v1::SemanticType;
40use common_sql::default_constraint::parse_column_default_constraint;
41use common_time::timezone::Timezone;
42use datatypes::extension::json::{Json2ExtensionType, JsonMetadata};
43use datatypes::json::JsonSettings;
44use datatypes::prelude::ConcreteDataType;
45use datatypes::schema::{COMMENT_KEY, ColumnDefaultConstraint, ColumnSchema};
46use datatypes::types::json_type::JsonNativeType;
47use datatypes::types::{JsonFormat, JsonType, TimestampType};
48use datatypes::value::Value;
49use snafu::ResultExt;
50use sqlparser::ast::{ExactNumberInfo, Ident};
51
52use crate::ast::{
53    ColumnDef, ColumnOption, DataType as SqlDataType, ObjectNamePartExt, TimezoneInfo,
54    Value as SqlValue,
55};
56use crate::error::{
57    self, ConvertToGrpcDataTypeSnafu, ConvertValueSnafu, Result,
58    SerializeColumnDefaultConstraintSnafu, SetFulltextOptionSnafu, SetSkippingIndexOptionSnafu,
59    SetVectorIndexOptionSnafu, SqlCommonSnafu,
60};
61use crate::statements::create::Column;
62pub use crate::statements::option_map::OptionMap;
63pub(crate) use crate::statements::transform::transform_statements;
64
65const VECTOR_TYPE_NAME: &str = "VECTOR";
66const JSON2_TYPE_NAME: &str = "JSON2";
67
68pub fn value_to_sql_value(val: &Value) -> Result<SqlValue> {
69    Ok(match val {
70        Value::Int8(v) => SqlValue::Number(v.to_string(), false),
71        Value::UInt8(v) => SqlValue::Number(v.to_string(), false),
72        Value::Int16(v) => SqlValue::Number(v.to_string(), false),
73        Value::UInt16(v) => SqlValue::Number(v.to_string(), false),
74        Value::Int32(v) => SqlValue::Number(v.to_string(), false),
75        Value::UInt32(v) => SqlValue::Number(v.to_string(), false),
76        Value::Int64(v) => SqlValue::Number(v.to_string(), false),
77        Value::UInt64(v) => SqlValue::Number(v.to_string(), false),
78        Value::Float32(v) => SqlValue::Number(v.to_string(), false),
79        Value::Float64(v) => SqlValue::Number(v.to_string(), false),
80        Value::Boolean(b) => SqlValue::Boolean(*b),
81        Value::Date(d) => SqlValue::SingleQuotedString(d.to_string()),
82        Value::Timestamp(ts) => SqlValue::SingleQuotedString(ts.to_iso8601_string()),
83        Value::String(s) => SqlValue::SingleQuotedString(s.as_utf8().to_string()),
84        Value::Null => SqlValue::Null,
85        // TODO(dennis): supports binary
86        _ => return ConvertValueSnafu { value: val.clone() }.fail(),
87    })
88}
89
90/// Return true when the `ColumnDef` options contain primary key
91pub fn has_primary_key_option(column_def: &ColumnDef) -> bool {
92    column_def
93        .options
94        .iter()
95        .any(|options| matches!(options.option, ColumnOption::PrimaryKey(..)))
96}
97
98/// Create a `ColumnSchema` from `Column`.
99pub fn column_to_schema(
100    column: &Column,
101    time_index: &str,
102    timezone: Option<&Timezone>,
103) -> Result<ColumnSchema> {
104    let is_time_index = column.name().value == time_index;
105
106    let is_nullable = column
107        .options()
108        .iter()
109        .all(|o| !matches!(o.option, ColumnOption::NotNull))
110        && !is_time_index;
111
112    let name = column.name().value.clone();
113    let data_type = sql_data_type_to_concrete_data_type(column.data_type())?;
114    let default_constraint =
115        parse_column_default_constraint(&name, &data_type, column.options(), timezone)
116            .context(SqlCommonSnafu)?;
117
118    let mut column_schema = ColumnSchema::new(name, data_type, is_nullable)
119        .with_time_index(is_time_index)
120        .with_default_constraint(default_constraint)
121        .context(error::InvalidDefaultSnafu {
122            column: &column.name().value,
123        })?;
124
125    if let Some(ColumnOption::Comment(c)) = column.options().iter().find_map(|o| {
126        if matches!(o.option, ColumnOption::Comment(_)) {
127            Some(&o.option)
128        } else {
129            None
130        }
131    }) {
132        let _ = column_schema
133            .mut_metadata()
134            .insert(COMMENT_KEY.to_string(), c.clone());
135    }
136
137    if let Some(options) = column.extensions.build_fulltext_options()? {
138        column_schema = column_schema
139            .with_fulltext_options(options)
140            .context(SetFulltextOptionSnafu)?;
141    }
142
143    if let Some(options) = column.extensions.build_skipping_index_options()? {
144        column_schema = column_schema
145            .with_skipping_options(options)
146            .context(SetSkippingIndexOptionSnafu)?;
147    }
148
149    if let Some(options) = column.extensions.build_vector_index_options()? {
150        column_schema = column_schema
151            .with_vector_index_options(&options)
152            .context(SetVectorIndexOptionSnafu)?;
153    }
154
155    column_schema.set_inverted_index(column.extensions.inverted_index_options.is_some());
156
157    let is_json2_column = if let SqlDataType::Custom(object_name, _) = column.data_type() {
158        object_name
159            .0
160            .first()
161            .map(|x| x.to_string_unquoted().eq_ignore_ascii_case(JSON2_TYPE_NAME))
162            .unwrap_or_default()
163    } else {
164        false
165    };
166    if is_json2_column {
167        let settings = column
168            .extensions
169            .build_json_settings()?
170            .unwrap_or_else(JsonSettings::new_v2);
171        let extension = Json2ExtensionType::new(Arc::new(JsonMetadata::new(settings)));
172        column_schema.with_extension_type(&extension);
173    }
174
175    Ok(column_schema)
176}
177
178/// Convert `ColumnDef` in sqlparser to `ColumnDef` in gRPC proto.
179pub fn sql_column_def_to_grpc_column_def(
180    col: &ColumnDef,
181    timezone: Option<&Timezone>,
182) -> Result<api::v1::ColumnDef> {
183    let name = col.name.value.clone();
184    let data_type = sql_data_type_to_concrete_data_type(&col.data_type)?;
185
186    let is_nullable = col
187        .options
188        .iter()
189        .all(|o| !matches!(o.option, ColumnOption::NotNull));
190
191    let default_constraint =
192        parse_column_default_constraint(&name, &data_type, &col.options, timezone)
193            .context(SqlCommonSnafu)?
194            .map(ColumnDefaultConstraint::try_into) // serialize default constraint to bytes
195            .transpose()
196            .context(SerializeColumnDefaultConstraintSnafu)?;
197    // convert ConcreteDataType to grpc ColumnDataTypeWrapper
198    let (datatype, datatype_ext) = ColumnDataTypeWrapper::try_from(data_type.clone())
199        .context(ConvertToGrpcDataTypeSnafu)?
200        .to_parts();
201
202    let is_primary_key = col
203        .options
204        .iter()
205        .any(|o| matches!(o.option, ColumnOption::PrimaryKey(..)));
206
207    let semantic_type = if is_primary_key {
208        SemanticType::Tag
209    } else {
210        SemanticType::Field
211    };
212
213    Ok(api::v1::ColumnDef {
214        name,
215        data_type: datatype as i32,
216        is_nullable,
217        default_constraint: default_constraint.unwrap_or_default(),
218        semantic_type: semantic_type as _,
219        comment: String::new(),
220        datatype_extension: datatype_ext,
221        options: None,
222    })
223}
224
225pub fn sql_data_type_to_concrete_data_type(data_type: &SqlDataType) -> Result<ConcreteDataType> {
226    match data_type {
227        SqlDataType::BigInt(_) | SqlDataType::Int64 => Ok(ConcreteDataType::int64_datatype()),
228        SqlDataType::BigIntUnsigned(_) => Ok(ConcreteDataType::uint64_datatype()),
229        SqlDataType::Int(_) | SqlDataType::Integer(_) => Ok(ConcreteDataType::int32_datatype()),
230        SqlDataType::IntUnsigned(_) | SqlDataType::UnsignedInteger => {
231            Ok(ConcreteDataType::uint32_datatype())
232        }
233        SqlDataType::SmallInt(_) => Ok(ConcreteDataType::int16_datatype()),
234        SqlDataType::SmallIntUnsigned(_) => Ok(ConcreteDataType::uint16_datatype()),
235        SqlDataType::TinyInt(_) | SqlDataType::Int8(_) => Ok(ConcreteDataType::int8_datatype()),
236        SqlDataType::TinyIntUnsigned(_) | SqlDataType::Int8Unsigned(_) => {
237            Ok(ConcreteDataType::uint8_datatype())
238        }
239        SqlDataType::Char(_)
240        | SqlDataType::Varchar(_)
241        | SqlDataType::Text
242        | SqlDataType::TinyText
243        | SqlDataType::MediumText
244        | SqlDataType::LongText
245        | SqlDataType::String(_) => Ok(ConcreteDataType::string_datatype()),
246        SqlDataType::Float(_) => Ok(ConcreteDataType::float32_datatype()),
247        SqlDataType::Double(_) | SqlDataType::Float64 => Ok(ConcreteDataType::float64_datatype()),
248        SqlDataType::Boolean => Ok(ConcreteDataType::boolean_datatype()),
249        SqlDataType::Date => Ok(ConcreteDataType::date_datatype()),
250        SqlDataType::Binary(_)
251        | SqlDataType::Blob(_)
252        | SqlDataType::Bytea
253        | SqlDataType::Varbinary(_) => Ok(ConcreteDataType::binary_datatype()),
254        SqlDataType::Datetime(_) => Ok(ConcreteDataType::timestamp_microsecond_datatype()),
255        SqlDataType::Timestamp(precision, _) => Ok(precision
256            .as_ref()
257            .map(|v| TimestampType::try_from(*v))
258            .transpose()
259            .map_err(|_| {
260                error::SqlTypeNotSupportedSnafu {
261                    t: data_type.clone(),
262                }
263                .build()
264            })?
265            .map(|t| ConcreteDataType::timestamp_datatype(t.unit()))
266            .unwrap_or(ConcreteDataType::timestamp_millisecond_datatype())),
267        SqlDataType::Interval { .. } => Ok(ConcreteDataType::interval_month_day_nano_datatype()),
268        SqlDataType::Decimal(exact_info) => match exact_info {
269            ExactNumberInfo::None => Ok(ConcreteDataType::decimal128_default_datatype()),
270            // refer to https://dev.mysql.com/doc/refman/8.0/en/fixed-point-types.html
271            // In standard SQL, the syntax DECIMAL(M) is equivalent to DECIMAL(M,0).
272            ExactNumberInfo::Precision(p) => Ok(ConcreteDataType::decimal128_datatype(*p as u8, 0)),
273            ExactNumberInfo::PrecisionAndScale(p, s) => {
274                Ok(ConcreteDataType::decimal128_datatype(*p as u8, *s as i8))
275            }
276        },
277        SqlDataType::JSON => Ok(ConcreteDataType::Json(JsonType::new(JsonFormat::Jsonb))),
278        // Vector type and JSON2 type
279        SqlDataType::Custom(name, args) if name.0.len() == 1 => {
280            let name = name.0[0].to_string_unquoted().to_ascii_uppercase();
281            match name.as_str() {
282                VECTOR_TYPE_NAME if args.len() == 1 => {
283                    let dim = &args[0];
284                    let dim = dim.parse().map_err(|e| {
285                        error::ParseSqlValueSnafu {
286                            msg: format!("Failed to parse vector dimension '{}': {}", dim, e),
287                        }
288                        .build()
289                    })?;
290                    Ok(ConcreteDataType::vector_datatype(dim))
291                }
292                JSON2_TYPE_NAME if args.is_empty() => {
293                    // Currently, JSON2 is not inferred as any native type initially.
294                    // TODO(fys): infer it later from type hints.
295                    let format = JsonFormat::Json2(Arc::new(JsonNativeType::Null));
296                    Ok(ConcreteDataType::Json(JsonType::new(format)))
297                }
298                _ => error::SqlTypeNotSupportedSnafu {
299                    t: data_type.clone(),
300                }
301                .fail(),
302            }
303        }
304        _ => error::SqlTypeNotSupportedSnafu {
305            t: data_type.clone(),
306        }
307        .fail(),
308    }
309}
310
311pub fn concrete_data_type_to_sql_data_type(data_type: &ConcreteDataType) -> Result<SqlDataType> {
312    match data_type {
313        ConcreteDataType::Int64(_) => Ok(SqlDataType::BigInt(None)),
314        ConcreteDataType::UInt64(_) => Ok(SqlDataType::BigIntUnsigned(None)),
315        ConcreteDataType::Int32(_) => Ok(SqlDataType::Int(None)),
316        ConcreteDataType::UInt32(_) => Ok(SqlDataType::IntUnsigned(None)),
317        ConcreteDataType::Int16(_) => Ok(SqlDataType::SmallInt(None)),
318        ConcreteDataType::UInt16(_) => Ok(SqlDataType::SmallIntUnsigned(None)),
319        ConcreteDataType::Int8(_) => Ok(SqlDataType::TinyInt(None)),
320        ConcreteDataType::UInt8(_) => Ok(SqlDataType::TinyIntUnsigned(None)),
321        ConcreteDataType::String(_) => Ok(SqlDataType::String(None)),
322        ConcreteDataType::Float32(_) => Ok(SqlDataType::Float(ExactNumberInfo::None)),
323        ConcreteDataType::Float64(_) => Ok(SqlDataType::Double(ExactNumberInfo::None)),
324        ConcreteDataType::Boolean(_) => Ok(SqlDataType::Boolean),
325        ConcreteDataType::Date(_) => Ok(SqlDataType::Date),
326        ConcreteDataType::Timestamp(ts_type) => Ok(SqlDataType::Timestamp(
327            Some(ts_type.precision()),
328            TimezoneInfo::None,
329        )),
330        ConcreteDataType::Time(time_type) => Ok(SqlDataType::Time(
331            Some(time_type.precision()),
332            TimezoneInfo::None,
333        )),
334        ConcreteDataType::Interval(_) => Ok(SqlDataType::Interval {
335            fields: None,
336            precision: None,
337        }),
338        ConcreteDataType::Binary(_) => Ok(SqlDataType::Varbinary(None)),
339        ConcreteDataType::Decimal128(d) => Ok(SqlDataType::Decimal(
340            ExactNumberInfo::PrecisionAndScale(d.precision() as u64, d.scale() as i64),
341        )),
342        ConcreteDataType::Json(_) => Ok(SqlDataType::JSON),
343        ConcreteDataType::Vector(v) => Ok(SqlDataType::Custom(
344            vec![Ident::new(VECTOR_TYPE_NAME)].into(),
345            vec![v.dim.to_string()],
346        )),
347        ConcreteDataType::Duration(_)
348        | ConcreteDataType::Null(_)
349        | ConcreteDataType::List(_)
350        | ConcreteDataType::Struct(_)
351        | ConcreteDataType::Dictionary(_) => error::ConcreteTypeNotSupportedSnafu {
352            t: data_type.clone(),
353        }
354        .fail(),
355    }
356}
357
358#[cfg(test)]
359mod tests {
360    use api::v1::ColumnDataType;
361    use datatypes::schema::{
362        COLUMN_FULLTEXT_OPT_KEY_ANALYZER, COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE, FulltextAnalyzer,
363    };
364    use sqlparser::ast::{ColumnOptionDef, Expr, PrimaryKeyConstraint};
365
366    use super::*;
367    use crate::ast::TimezoneInfo;
368    use crate::statements::ColumnOption;
369    use crate::statements::create::ColumnExtensions;
370
371    fn check_type(sql_type: SqlDataType, data_type: ConcreteDataType) {
372        assert_eq!(
373            data_type,
374            sql_data_type_to_concrete_data_type(&sql_type).unwrap()
375        );
376    }
377
378    #[test]
379    pub fn test_sql_data_type_to_concrete_data_type() {
380        check_type(
381            SqlDataType::BigInt(None),
382            ConcreteDataType::int64_datatype(),
383        );
384        check_type(SqlDataType::Int(None), ConcreteDataType::int32_datatype());
385        check_type(
386            SqlDataType::Integer(None),
387            ConcreteDataType::int32_datatype(),
388        );
389        check_type(
390            SqlDataType::SmallInt(None),
391            ConcreteDataType::int16_datatype(),
392        );
393        check_type(SqlDataType::Char(None), ConcreteDataType::string_datatype());
394        check_type(
395            SqlDataType::Varchar(None),
396            ConcreteDataType::string_datatype(),
397        );
398        check_type(SqlDataType::Text, ConcreteDataType::string_datatype());
399        check_type(
400            SqlDataType::String(None),
401            ConcreteDataType::string_datatype(),
402        );
403        check_type(
404            SqlDataType::Float(ExactNumberInfo::None),
405            ConcreteDataType::float32_datatype(),
406        );
407        check_type(
408            SqlDataType::Double(ExactNumberInfo::None),
409            ConcreteDataType::float64_datatype(),
410        );
411        check_type(SqlDataType::Boolean, ConcreteDataType::boolean_datatype());
412        check_type(SqlDataType::Date, ConcreteDataType::date_datatype());
413        check_type(
414            SqlDataType::Timestamp(None, TimezoneInfo::None),
415            ConcreteDataType::timestamp_millisecond_datatype(),
416        );
417        check_type(
418            SqlDataType::Varbinary(None),
419            ConcreteDataType::binary_datatype(),
420        );
421        check_type(
422            SqlDataType::BigIntUnsigned(None),
423            ConcreteDataType::uint64_datatype(),
424        );
425        check_type(
426            SqlDataType::IntUnsigned(None),
427            ConcreteDataType::uint32_datatype(),
428        );
429        check_type(
430            SqlDataType::SmallIntUnsigned(None),
431            ConcreteDataType::uint16_datatype(),
432        );
433        check_type(
434            SqlDataType::TinyIntUnsigned(None),
435            ConcreteDataType::uint8_datatype(),
436        );
437        check_type(
438            SqlDataType::Datetime(None),
439            ConcreteDataType::timestamp_microsecond_datatype(),
440        );
441        check_type(
442            SqlDataType::Interval {
443                fields: None,
444                precision: None,
445            },
446            ConcreteDataType::interval_month_day_nano_datatype(),
447        );
448        check_type(SqlDataType::JSON, ConcreteDataType::json_datatype());
449        check_type(
450            SqlDataType::Custom(
451                vec![Ident::new(VECTOR_TYPE_NAME)].into(),
452                vec!["3".to_string()],
453            ),
454            ConcreteDataType::vector_datatype(3),
455        );
456    }
457
458    #[test]
459    pub fn test_sql_column_def_to_grpc_column_def() {
460        // test basic
461        let column_def = ColumnDef {
462            name: "col".into(),
463            data_type: SqlDataType::Double(ExactNumberInfo::None),
464            options: vec![],
465        };
466
467        let grpc_column_def = sql_column_def_to_grpc_column_def(&column_def, None).unwrap();
468
469        assert_eq!("col", grpc_column_def.name);
470        assert!(grpc_column_def.is_nullable); // nullable when options are empty
471        assert_eq!(ColumnDataType::Float64 as i32, grpc_column_def.data_type);
472        assert!(grpc_column_def.default_constraint.is_empty());
473        assert_eq!(grpc_column_def.semantic_type, SemanticType::Field as i32);
474
475        // test not null
476        let column_def = ColumnDef {
477            name: "col".into(),
478            data_type: SqlDataType::Double(ExactNumberInfo::None),
479            options: vec![ColumnOptionDef {
480                name: None,
481                option: ColumnOption::NotNull,
482            }],
483        };
484
485        let grpc_column_def = sql_column_def_to_grpc_column_def(&column_def, None).unwrap();
486        assert!(!grpc_column_def.is_nullable);
487
488        // test primary key
489        let column_def = ColumnDef {
490            name: "col".into(),
491            data_type: SqlDataType::Double(ExactNumberInfo::None),
492            options: vec![ColumnOptionDef {
493                name: None,
494                option: ColumnOption::PrimaryKey(PrimaryKeyConstraint {
495                    name: None,
496                    index_name: None,
497                    index_type: None,
498                    columns: vec![],
499                    index_options: vec![],
500                    characteristics: None,
501                }),
502            }],
503        };
504
505        let grpc_column_def = sql_column_def_to_grpc_column_def(&column_def, None).unwrap();
506        assert_eq!(grpc_column_def.semantic_type, SemanticType::Tag as i32);
507    }
508
509    #[test]
510    pub fn test_sql_column_def_to_grpc_column_def_with_timezone() {
511        let column_def = ColumnDef {
512            name: "col".into(),
513            // MILLISECOND
514            data_type: SqlDataType::Timestamp(Some(3), TimezoneInfo::None),
515            options: vec![ColumnOptionDef {
516                name: None,
517                option: ColumnOption::Default(Expr::Value(
518                    SqlValue::SingleQuotedString("2024-01-30T00:01:01".to_string()).into(),
519                )),
520            }],
521        };
522
523        // with timezone "Asia/Shanghai"
524        let grpc_column_def = sql_column_def_to_grpc_column_def(
525            &column_def,
526            Some(&Timezone::from_tz_string("Asia/Shanghai").unwrap()),
527        )
528        .unwrap();
529        assert_eq!("col", grpc_column_def.name);
530        assert!(grpc_column_def.is_nullable); // nullable when options are empty
531        assert_eq!(
532            ColumnDataType::TimestampMillisecond as i32,
533            grpc_column_def.data_type
534        );
535        assert!(!grpc_column_def.default_constraint.is_empty());
536
537        let constraint =
538            ColumnDefaultConstraint::try_from(&grpc_column_def.default_constraint[..]).unwrap();
539        assert!(
540            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
541                         if ts.to_iso8601_string() == "2024-01-29 16:01:01+0000")
542        );
543
544        // without timezone
545        let grpc_column_def = sql_column_def_to_grpc_column_def(&column_def, None).unwrap();
546        assert_eq!("col", grpc_column_def.name);
547        assert!(grpc_column_def.is_nullable); // nullable when options are empty
548        assert_eq!(
549            ColumnDataType::TimestampMillisecond as i32,
550            grpc_column_def.data_type
551        );
552        assert!(!grpc_column_def.default_constraint.is_empty());
553
554        let constraint =
555            ColumnDefaultConstraint::try_from(&grpc_column_def.default_constraint[..]).unwrap();
556        assert!(
557            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
558                         if ts.to_iso8601_string() == "2024-01-30 00:01:01+0000")
559        );
560    }
561
562    #[test]
563    pub fn test_has_primary_key_option() {
564        let column_def = ColumnDef {
565            name: "col".into(),
566            data_type: SqlDataType::Double(ExactNumberInfo::None),
567            options: vec![],
568        };
569        assert!(!has_primary_key_option(&column_def));
570
571        let column_def = ColumnDef {
572            name: "col".into(),
573            data_type: SqlDataType::Double(ExactNumberInfo::None),
574            options: vec![ColumnOptionDef {
575                name: None,
576                option: ColumnOption::PrimaryKey(PrimaryKeyConstraint {
577                    name: None,
578                    index_name: None,
579                    index_type: None,
580                    columns: vec![],
581                    index_options: vec![],
582                    characteristics: None,
583                }),
584            }],
585        };
586        assert!(has_primary_key_option(&column_def));
587    }
588
589    #[test]
590    pub fn test_column_to_schema() {
591        let column_def = Column {
592            column_def: ColumnDef {
593                name: "col".into(),
594                data_type: SqlDataType::Double(ExactNumberInfo::None),
595                options: vec![],
596            },
597            extensions: ColumnExtensions::default(),
598        };
599
600        let column_schema = column_to_schema(&column_def, "ts", None).unwrap();
601
602        assert_eq!("col", column_schema.name);
603        assert_eq!(
604            ConcreteDataType::float64_datatype(),
605            column_schema.data_type
606        );
607        assert!(column_schema.is_nullable());
608        assert!(!column_schema.is_time_index());
609
610        let column_schema = column_to_schema(&column_def, "col", None).unwrap();
611
612        assert_eq!("col", column_schema.name);
613        assert_eq!(
614            ConcreteDataType::float64_datatype(),
615            column_schema.data_type
616        );
617        assert!(!column_schema.is_nullable());
618        assert!(column_schema.is_time_index());
619
620        let column_def = Column {
621            column_def: ColumnDef {
622                name: "col2".into(),
623                data_type: SqlDataType::String(None),
624                options: vec![
625                    ColumnOptionDef {
626                        name: None,
627                        option: ColumnOption::NotNull,
628                    },
629                    ColumnOptionDef {
630                        name: None,
631                        option: ColumnOption::Comment("test comment".to_string()),
632                    },
633                ],
634            },
635            extensions: ColumnExtensions::default(),
636        };
637
638        let column_schema = column_to_schema(&column_def, "ts", None).unwrap();
639
640        assert_eq!("col2", column_schema.name);
641        assert_eq!(ConcreteDataType::string_datatype(), column_schema.data_type);
642        assert!(!column_schema.is_nullable());
643        assert!(!column_schema.is_time_index());
644        assert_eq!(
645            column_schema.metadata().get(COMMENT_KEY),
646            Some(&"test comment".to_string())
647        );
648    }
649
650    #[test]
651    fn test_new_json2_column_uses_v2_layout() -> std::result::Result<(), Box<dyn std::error::Error>>
652    {
653        let column = Column {
654            column_def: ColumnDef {
655                name: "data".into(),
656                data_type: SqlDataType::Custom(
657                    sqlparser::ast::ObjectName::from(vec!["JSON2".into()]),
658                    vec![],
659                ),
660                options: vec![],
661            },
662            extensions: ColumnExtensions::default(),
663        };
664
665        let schema = column_to_schema(&column, "ts", None)?;
666        let metadata: serde_json::Value =
667            serde_json::from_str(schema.metadata().get("ARROW:extension:metadata").unwrap())?;
668        assert_eq!(Some(2), metadata["layout_version"].as_u64());
669        assert_eq!(
670            Some(100),
671            metadata["json_settings"]["max_auto_expanded_paths"].as_u64()
672        );
673
674        let mut hinted = column;
675        hinted
676            .extensions
677            .set_json_settings(datatypes::json::JsonSettings::try_new(
678                vec![datatypes::json::JsonTypeHint {
679                    path: vec!["kind".to_string()],
680                    data_type: ConcreteDataType::string_datatype(),
681                    inverted_index: false,
682                }],
683                None,
684            )?)?;
685        let schema = column_to_schema(&hinted, "ts", None)?;
686        let metadata: serde_json::Value =
687            serde_json::from_str(schema.metadata().get("ARROW:extension:metadata").unwrap())?;
688        assert_eq!(
689            Some(100),
690            metadata["json_settings"]["max_auto_expanded_paths"].as_u64()
691        );
692        Ok(())
693    }
694
695    #[test]
696    pub fn test_column_to_schema_timestamp_with_timezone() {
697        let column = Column {
698            column_def: ColumnDef {
699                name: "col".into(),
700                // MILLISECOND
701                data_type: SqlDataType::Timestamp(Some(3), TimezoneInfo::None),
702                options: vec![ColumnOptionDef {
703                    name: None,
704                    option: ColumnOption::Default(Expr::Value(
705                        SqlValue::SingleQuotedString("2024-01-30T00:01:01".to_string()).into(),
706                    )),
707                }],
708            },
709            extensions: ColumnExtensions::default(),
710        };
711
712        // with timezone "Asia/Shanghai"
713
714        let column_schema = column_to_schema(
715            &column,
716            "ts",
717            Some(&Timezone::from_tz_string("Asia/Shanghai").unwrap()),
718        )
719        .unwrap();
720
721        assert_eq!("col", column_schema.name);
722        assert_eq!(
723            ConcreteDataType::timestamp_millisecond_datatype(),
724            column_schema.data_type
725        );
726        assert!(column_schema.is_nullable());
727
728        let constraint = column_schema.default_constraint().unwrap();
729        assert!(
730            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
731                         if ts.to_iso8601_string() == "2024-01-29 16:01:01+0000")
732        );
733
734        // without timezone
735        let column_schema = column_to_schema(&column, "ts", None).unwrap();
736
737        assert_eq!("col", column_schema.name);
738        assert_eq!(
739            ConcreteDataType::timestamp_millisecond_datatype(),
740            column_schema.data_type
741        );
742        assert!(column_schema.is_nullable());
743
744        let constraint = column_schema.default_constraint().unwrap();
745        assert!(
746            matches!(constraint, ColumnDefaultConstraint::Value(Value::Timestamp(ts))
747                         if ts.to_iso8601_string() == "2024-01-30 00:01:01+0000")
748        );
749    }
750
751    #[test]
752    fn test_column_to_schema_with_fulltext() {
753        let column = Column {
754            column_def: ColumnDef {
755                name: "col".into(),
756                data_type: SqlDataType::Text,
757                options: vec![],
758            },
759            extensions: ColumnExtensions {
760                fulltext_index_options: Some(OptionMap::from([
761                    (
762                        COLUMN_FULLTEXT_OPT_KEY_ANALYZER.to_string(),
763                        "English".to_string(),
764                    ),
765                    (
766                        COLUMN_FULLTEXT_OPT_KEY_CASE_SENSITIVE.to_string(),
767                        "true".to_string(),
768                    ),
769                ])),
770                ..Default::default()
771            },
772        };
773
774        let column_schema = column_to_schema(&column, "ts", None).unwrap();
775        assert_eq!("col", column_schema.name);
776        assert_eq!(ConcreteDataType::string_datatype(), column_schema.data_type);
777        let fulltext_options = column_schema.fulltext_options().unwrap().unwrap();
778        assert_eq!(fulltext_options.analyzer, FulltextAnalyzer::English);
779        assert!(fulltext_options.case_sensitive);
780    }
781
782    #[test]
783    fn test_column_to_schema_with_vector_index() {
784        use datatypes::schema::{VectorDistanceMetric, VectorIndexEngineType};
785
786        // Test with custom metric and parameters
787        let column = Column {
788            column_def: ColumnDef {
789                name: "embedding".into(),
790                data_type: SqlDataType::Custom(
791                    vec![Ident::new(VECTOR_TYPE_NAME)].into(),
792                    vec!["128".to_string()],
793                ),
794                options: vec![],
795            },
796            extensions: ColumnExtensions {
797                vector_index_options: Some(OptionMap::from([
798                    ("metric".to_string(), "cosine".to_string()),
799                    ("connectivity".to_string(), "32".to_string()),
800                    ("expansion_add".to_string(), "200".to_string()),
801                    ("expansion_search".to_string(), "100".to_string()),
802                ])),
803                ..Default::default()
804            },
805        };
806
807        let column_schema = column_to_schema(&column, "ts", None).unwrap();
808        assert_eq!("embedding", column_schema.name);
809        assert!(column_schema.is_vector_indexed());
810
811        let vector_options = column_schema.vector_index_options().unwrap().unwrap();
812        assert_eq!(vector_options.engine, VectorIndexEngineType::Usearch);
813        assert_eq!(vector_options.metric, VectorDistanceMetric::Cosine);
814        assert_eq!(vector_options.connectivity, 32);
815        assert_eq!(vector_options.expansion_add, 200);
816        assert_eq!(vector_options.expansion_search, 100);
817    }
818
819    #[test]
820    fn test_column_to_schema_with_vector_index_defaults() {
821        use datatypes::schema::{VectorDistanceMetric, VectorIndexEngineType};
822
823        // Test with default values (empty options map)
824        let column = Column {
825            column_def: ColumnDef {
826                name: "vec".into(),
827                data_type: SqlDataType::Custom(
828                    vec![Ident::new(VECTOR_TYPE_NAME)].into(),
829                    vec!["64".to_string()],
830                ),
831                options: vec![],
832            },
833            extensions: ColumnExtensions {
834                vector_index_options: Some(OptionMap::default()),
835                ..Default::default()
836            },
837        };
838
839        let column_schema = column_to_schema(&column, "ts", None).unwrap();
840        assert_eq!("vec", column_schema.name);
841        assert!(column_schema.is_vector_indexed());
842
843        let vector_options = column_schema.vector_index_options().unwrap().unwrap();
844        // Verify defaults
845        assert_eq!(vector_options.engine, VectorIndexEngineType::Usearch);
846        assert_eq!(vector_options.metric, VectorDistanceMetric::L2sq);
847        assert_eq!(vector_options.connectivity, 16);
848        assert_eq!(vector_options.expansion_add, 128);
849        assert_eq!(vector_options.expansion_search, 64);
850    }
851}
sql/statements.rs

sql/
statements.rs
