operator/req_convert/insert/
stmt_to_region.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use api::helper::{ColumnDataTypeWrapper, to_grpc_value};
16use api::v1::column_def::options_from_column_schema;
17use api::v1::region::InsertRequests as RegionInsertRequests;
18use api::v1::{ColumnSchema as GrpcColumnSchema, Row, Rows};
19use catalog::CatalogManager;
20use common_time::Timezone;
21use datatypes::schema::{ColumnSchema, SchemaRef};
22use datatypes::value::Value;
23use partition::manager::PartitionRuleManager;
24use session::context::{QueryContext, QueryContextRef};
25use snafu::{OptionExt, ResultExt, ensure};
26use sql::ast::ObjectNamePartExt;
27use sql::statements::insert::Insert;
28use sqlparser::ast::{ObjectName, Value as SqlValue};
29use table::TableRef;
30use table::metadata::TableInfoRef;
31
32use crate::error::{
33    CatalogSnafu, ColumnDataTypeSnafu, ColumnDefaultValueSnafu, ColumnNoneDefaultValueSnafu,
34    ColumnNotFoundSnafu, InvalidInsertRequestSnafu, InvalidSqlSnafu, MissingInsertBodySnafu,
35    ParseSqlSnafu, Result, SchemaReadOnlySnafu, TableNotFoundSnafu, TableReadOnlySnafu,
36};
37use crate::insert::InstantAndNormalInsertRequests;
38use crate::req_convert::common::partitioner::Partitioner;
39use crate::req_convert::insert::semantic_type;
40
41const DEFAULT_PLACEHOLDER_VALUE: &str = "default";
42
43pub struct StatementToRegion<'a> {
44    catalog_manager: &'a dyn CatalogManager,
45    partition_manager: &'a PartitionRuleManager,
46    ctx: &'a QueryContext,
47}
48
49impl<'a> StatementToRegion<'a> {
50    pub fn new(
51        catalog_manager: &'a dyn CatalogManager,
52        partition_manager: &'a PartitionRuleManager,
53        ctx: &'a QueryContext,
54    ) -> Self {
55        Self {
56            catalog_manager,
57            partition_manager,
58            ctx,
59        }
60    }
61
62    pub async fn convert(
63        &self,
64        stmt: &Insert,
65        query_ctx: &QueryContextRef,
66    ) -> Result<(InstantAndNormalInsertRequests, TableInfoRef)> {
67        let name = stmt.table_name().context(ParseSqlSnafu)?;
68        let (catalog, schema, table_name) = self.get_full_name(name)?;
69        let table = self.get_table(&catalog, &schema, &table_name).await?;
70        let table_schema = table.schema();
71
72        ensure!(
73            !common_catalog::consts::is_readonly_schema(&schema),
74            SchemaReadOnlySnafu { name: schema }
75        );
76        // The computed entity-graph tables overlay a writable schema, so the
77        // schema-level check above does not cover them.
78        ensure!(
79            !common_catalog::consts::is_readonly_table(&schema, &table_name),
80            TableReadOnlySnafu { name: table_name }
81        );
82
83        let column_names = column_names(stmt, &table_schema);
84        let column_count = column_names.len();
85
86        let sql_rows = stmt.values_body().context(MissingInsertBodySnafu)?;
87        let row_count = sql_rows.len();
88
89        sql_rows.iter().try_for_each(|r| {
90            ensure!(
91                r.len() == column_count,
92                InvalidSqlSnafu {
93                    err_msg: format!(
94                        "column count mismatch, columns: {}, values: {}",
95                        column_count,
96                        r.len()
97                    )
98                }
99            );
100            Ok(())
101        })?;
102
103        let mut rows = vec![
104            Row {
105                values: Vec::with_capacity(column_count)
106            };
107            row_count
108        ];
109
110        fn find_insert_columns<'a>(
111            table: &'a TableRef,
112            column_names: &[&String],
113        ) -> Result<Vec<&'a ColumnSchema>> {
114            let schema = table.schema_ref();
115            column_names
116                .iter()
117                .map(|name| {
118                    schema
119                        .column_schema_by_name(name)
120                        .context(ColumnNotFoundSnafu { msg: *name })
121                })
122                .collect::<Result<Vec<_>>>()
123        }
124
125        let insert_columns = find_insert_columns(&table, &column_names)?;
126        let converter = SqlRowConverter::new(&insert_columns, query_ctx);
127        let value_rows = converter.convert(&sql_rows)?;
128        for (i, row) in value_rows.into_iter().enumerate() {
129            for value in row {
130                let grpc_value = to_grpc_value(value);
131                rows[i].values.push(grpc_value);
132            }
133        }
134
135        let table_info = table.table_info();
136        let mut schema = Vec::with_capacity(column_count);
137        for column_schema in insert_columns {
138            let (datatype, datatype_extension) =
139                ColumnDataTypeWrapper::try_from(column_schema.data_type.clone())
140                    .context(ColumnDataTypeSnafu)?
141                    .to_parts();
142
143            let column_name = &column_schema.name;
144            let semantic_type = semantic_type(&table_info, column_name)?;
145
146            let grpc_column_schema = GrpcColumnSchema {
147                column_name: column_name.clone(),
148                datatype: datatype.into(),
149                semantic_type: semantic_type.into(),
150                datatype_extension,
151                options: options_from_column_schema(column_schema),
152            };
153            schema.push(grpc_column_schema);
154        }
155
156        let requests = Partitioner::new(self.partition_manager)
157            .partition_insert_requests(&table_info, Rows { schema, rows })
158            .await?;
159        let requests = RegionInsertRequests { requests };
160        if table_info.is_ttl_instant_table() {
161            Ok((
162                InstantAndNormalInsertRequests {
163                    normal_requests: Default::default(),
164                    instant_requests: requests,
165                },
166                table_info,
167            ))
168        } else {
169            Ok((
170                InstantAndNormalInsertRequests {
171                    normal_requests: requests,
172                    instant_requests: Default::default(),
173                },
174                table_info,
175            ))
176        }
177    }
178
179    async fn get_table(&self, catalog: &str, schema: &str, table: &str) -> Result<TableRef> {
180        self.catalog_manager
181            .table(catalog, schema, table, None)
182            .await
183            .context(CatalogSnafu)?
184            .with_context(|| TableNotFoundSnafu {
185                table_name: common_catalog::format_full_table_name(catalog, schema, table),
186            })
187    }
188
189    fn get_full_name(&self, obj_name: &ObjectName) -> Result<(String, String, String)> {
190        match &obj_name.0[..] {
191            [table] => Ok((
192                self.ctx.current_catalog().to_owned(),
193                self.ctx.current_schema(),
194                table.to_string_unquoted(),
195            )),
196            [schema, table] => Ok((
197                self.ctx.current_catalog().to_owned(),
198                schema.to_string_unquoted(),
199                table.to_string_unquoted(),
200            )),
201            [catalog, schema, table] => Ok((
202                catalog.to_string_unquoted(),
203                schema.to_string_unquoted(),
204                table.to_string_unquoted(),
205            )),
206            _ => InvalidSqlSnafu {
207                err_msg: format!(
208                    "expect table name to be <catalog>.<schema>.<table>, <schema>.<table> or <table>, actual: {obj_name}",
209                ),
210            }.fail(),
211        }
212    }
213}
214
215struct SqlRowConverter<'a, 'b> {
216    insert_columns: &'a [&'a ColumnSchema],
217    query_context: &'b QueryContextRef,
218}
219
220impl<'a, 'b> SqlRowConverter<'a, 'b> {
221    fn new(insert_columns: &'a [&'a ColumnSchema], query_context: &'b QueryContextRef) -> Self {
222        Self {
223            insert_columns,
224            query_context,
225        }
226    }
227
228    fn convert(&self, sql_rows: &[Vec<SqlValue>]) -> Result<Vec<Vec<Value>>> {
229        let timezone = Some(&self.query_context.timezone());
230        let auto_string_to_numeric = self.query_context.auto_string_to_numeric();
231
232        let mut value_rows = Vec::with_capacity(sql_rows.len());
233        for sql_row in sql_rows {
234            let mut value_row = Vec::with_capacity(self.insert_columns.len());
235
236            for (insert_column, sql_value) in self.insert_columns.iter().zip(sql_row) {
237                let value =
238                    sql_value_to_value(insert_column, sql_value, timezone, auto_string_to_numeric)?;
239                value_row.push(value);
240            }
241            value_rows.push(value_row);
242        }
243        Ok(value_rows)
244    }
245}
246
247fn column_names<'a>(stmt: &'a Insert, table_schema: &'a SchemaRef) -> Vec<&'a String> {
248    if !stmt.columns().is_empty() {
249        stmt.columns()
250    } else {
251        table_schema
252            .column_schemas()
253            .iter()
254            .map(|column| &column.name)
255            .collect()
256    }
257}
258
259/// Converts SQL value to gRPC value according to the column schema.
260/// If `auto_string_to_numeric` is true, tries to cast the string value to numeric values,
261/// and fills the default value if the cast fails.
262fn sql_value_to_value(
263    column_schema: &ColumnSchema,
264    sql_val: &SqlValue,
265    timezone: Option<&Timezone>,
266    auto_string_to_numeric: bool,
267) -> Result<Value> {
268    let column = &column_schema.name;
269    let value = if replace_default(sql_val) {
270        let default_value = column_schema
271            .create_default()
272            .context(ColumnDefaultValueSnafu {
273                column: column.clone(),
274            })?;
275
276        default_value.context(ColumnNoneDefaultValueSnafu {
277            column: column.clone(),
278        })?
279    } else {
280        common_sql::convert::sql_value_to_value(
281            column_schema,
282            sql_val,
283            timezone,
284            None,
285            auto_string_to_numeric,
286        )
287        .context(crate::error::SqlCommonSnafu)?
288    };
289    validate(&value)?;
290    Ok(value)
291}
292
293fn validate(value: &Value) -> Result<()> {
294    match value {
295        Value::Json(value) => {
296            // Json object will be stored as Arrow struct in parquet, and it has the restriction:
297            // "Parquet does not support writing empty structs".
298            ensure!(
299                !value.is_empty_object(),
300                InvalidInsertRequestSnafu {
301                    reason: "empty json object is not supported, consider adding a dummy field"
302                }
303            );
304            Ok(())
305        }
306        _ => Ok(()),
307    }
308}
309
310fn replace_default(sql_val: &SqlValue) -> bool {
311    matches!(sql_val, SqlValue::Placeholder(s) if s.to_lowercase() == DEFAULT_PLACEHOLDER_VALUE)
312}
operator/req_convert/insert/stmt_to_region.rs

operator/req_convert/insert/
stmt_to_region.rs
