pipeline/etl/
ctx_req.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::hash_map::IntoIter;
16use std::sync::Arc;
17
18use ahash::{HashMap, HashMapExt};
19use api::v1::{RowInsertRequest, RowInsertRequests, Rows};
20use session::context::{QueryContext, QueryContextRef};
21use snafu::OptionExt;
22use vrl::value::Value as VrlValue;
23
24use crate::error::{Result, ValueMustBeMapSnafu};
25use crate::tablesuffix::TableSuffixTemplate;
26
27const GREPTIME_AUTO_CREATE_TABLE: &str = "greptime_auto_create_table";
28const GREPTIME_TTL: &str = "greptime_ttl";
29const GREPTIME_APPEND_MODE: &str = "greptime_append_mode";
30const GREPTIME_MERGE_MODE: &str = "greptime_merge_mode";
31const GREPTIME_PHYSICAL_TABLE: &str = "greptime_physical_table";
32const GREPTIME_SKIP_WAL: &str = "greptime_skip_wal";
33const GREPTIME_TABLE_SUFFIX: &str = "greptime_table_suffix";
34
35pub(crate) const AUTO_CREATE_TABLE_KEY: &str = "auto_create_table";
36pub(crate) const TTL_KEY: &str = "ttl";
37pub(crate) const APPEND_MODE_KEY: &str = "append_mode";
38pub(crate) const MERGE_MODE_KEY: &str = "merge_mode";
39pub(crate) const PHYSICAL_TABLE_KEY: &str = "physical_table";
40pub(crate) const SKIP_WAL_KEY: &str = "skip_wal";
41pub(crate) const TABLE_SUFFIX_KEY: &str = "table_suffix";
42
43pub const PIPELINE_HINT_KEYS: [&str; 7] = [
44    GREPTIME_AUTO_CREATE_TABLE,
45    GREPTIME_TTL,
46    GREPTIME_APPEND_MODE,
47    GREPTIME_MERGE_MODE,
48    GREPTIME_PHYSICAL_TABLE,
49    GREPTIME_SKIP_WAL,
50    GREPTIME_TABLE_SUFFIX,
51];
52
53const PIPELINE_HINT_PREFIX: &str = "greptime_";
54
55/// ContextOpt is a collection of options(including table options and pipeline options)
56/// that should be extracted during the pipeline execution.
57///
58/// The options are set in the format of hint keys. See [`PIPELINE_HINT_KEYS`].
59/// It's is used as the key in [`ContextReq`] for grouping the row insert requests.
60#[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)]
61pub struct ContextOpt {
62    // table options, that need to be set in the query context before making row insert requests
63    auto_create_table: Option<String>,
64    ttl: Option<String>,
65    append_mode: Option<String>,
66    merge_mode: Option<String>,
67    physical_table: Option<String>,
68    skip_wal: Option<String>,
69
70    // reset the schema in query context
71    schema: Option<String>,
72}
73
74impl ContextOpt {
75    pub fn set_physical_table(&mut self, physical_table: String) {
76        self.physical_table = Some(physical_table);
77    }
78
79    pub fn set_schema(&mut self, schema: String) {
80        self.schema = Some(schema);
81    }
82}
83
84impl ContextOpt {
85    pub fn from_pipeline_map_to_opt(value: &mut VrlValue) -> Result<Self> {
86        let map = value.as_object_mut().context(ValueMustBeMapSnafu)?;
87
88        let mut opt = Self::default();
89        for k in PIPELINE_HINT_KEYS {
90            if let Some(v) = map.remove(k) {
91                let v = v.to_string_lossy().to_string();
92                match k {
93                    GREPTIME_AUTO_CREATE_TABLE => {
94                        opt.auto_create_table = Some(v);
95                    }
96                    GREPTIME_TTL => {
97                        opt.ttl = Some(v);
98                    }
99                    GREPTIME_APPEND_MODE => {
100                        opt.append_mode = Some(v);
101                    }
102                    GREPTIME_MERGE_MODE => {
103                        opt.merge_mode = Some(v);
104                    }
105                    GREPTIME_PHYSICAL_TABLE => {
106                        opt.physical_table = Some(v);
107                    }
108                    GREPTIME_SKIP_WAL => {
109                        opt.skip_wal = Some(v);
110                    }
111                    GREPTIME_TABLE_SUFFIX => {}
112                    _ => {}
113                }
114            }
115        }
116        Ok(opt)
117    }
118
119    pub(crate) fn resolve_table_suffix(
120        table_suffix: Option<&TableSuffixTemplate>,
121        pipeline_map: &VrlValue,
122    ) -> Option<String> {
123        pipeline_map
124            .as_object()
125            .and_then(|map| map.get(GREPTIME_TABLE_SUFFIX))
126            .map(|value| value.to_string_lossy().to_string())
127            .or_else(|| table_suffix.and_then(|s| s.apply(pipeline_map)))
128    }
129
130    pub fn set_query_context(self, ctx: &mut QueryContext) {
131        if let Some(auto_create_table) = &self.auto_create_table {
132            ctx.set_extension(AUTO_CREATE_TABLE_KEY, auto_create_table);
133        }
134        if let Some(ttl) = &self.ttl {
135            ctx.set_extension(TTL_KEY, ttl);
136        }
137        if let Some(append_mode) = &self.append_mode {
138            ctx.set_extension(APPEND_MODE_KEY, append_mode);
139        }
140        if let Some(merge_mode) = &self.merge_mode {
141            ctx.set_extension(MERGE_MODE_KEY, merge_mode);
142        }
143        if let Some(physical_table) = &self.physical_table {
144            ctx.set_extension(PHYSICAL_TABLE_KEY, physical_table);
145        }
146        if let Some(skip_wal) = &self.skip_wal {
147            ctx.set_extension(SKIP_WAL_KEY, skip_wal);
148        }
149    }
150}
151
152/// ContextReq is a collection of row insert requests with different options.
153/// The default option is all empty.
154/// Because options are set in query context, we have to split them into sequential calls
155/// The key is a [`ContextOpt`] struct for strong type.
156/// e.g:
157/// {
158///     "skip_wal=true,ttl=1d": [RowInsertRequest],
159///     "ttl=1d": [RowInsertRequest],
160/// }
161#[derive(Debug, Default)]
162pub struct ContextReq {
163    req: HashMap<ContextOpt, Vec<RowInsertRequest>>,
164}
165
166impl ContextReq {
167    pub fn from_opt_map(opt_map: HashMap<ContextOpt, Rows>, table_name: String) -> Self {
168        Self {
169            req: opt_map
170                .into_iter()
171                .map(|(opt, rows)| {
172                    (
173                        opt,
174                        vec![RowInsertRequest {
175                            table_name: table_name.clone(),
176                            rows: Some(rows),
177                        }],
178                    )
179                })
180                .collect::<HashMap<ContextOpt, Vec<RowInsertRequest>>>(),
181        }
182    }
183
184    pub fn default_opt_with_reqs(reqs: Vec<RowInsertRequest>) -> Self {
185        let mut req_map = HashMap::new();
186        req_map.insert(ContextOpt::default(), reqs);
187        Self { req: req_map }
188    }
189
190    pub fn add_row(&mut self, opt: &ContextOpt, req: RowInsertRequest) {
191        match self.req.get_mut(opt) {
192            None => {
193                self.req.insert(opt.clone(), vec![req]);
194            }
195            Some(e) => {
196                e.push(req);
197            }
198        }
199    }
200
201    pub fn add_rows(&mut self, opt: ContextOpt, reqs: impl IntoIterator<Item = RowInsertRequest>) {
202        self.req.entry(opt).or_default().extend(reqs);
203    }
204
205    pub fn merge(&mut self, other: Self) {
206        for (opt, req) in other.req {
207            self.req.entry(opt).or_default().extend(req);
208        }
209    }
210
211    pub fn as_req_iter(self, ctx: QueryContextRef) -> ContextReqIter {
212        ContextReqIter {
213            opt_req: self.req.into_iter(),
214            ctx_template: ctx.fork(),
215        }
216    }
217
218    pub fn all_req(self) -> impl Iterator<Item = RowInsertRequest> {
219        self.req.into_values().flatten()
220    }
221
222    pub fn ref_all_req(&self) -> impl Iterator<Item = &RowInsertRequest> {
223        self.req.values().flatten()
224    }
225
226    pub fn map_len(&self) -> usize {
227        self.req.len()
228    }
229}
230
231// ContextReqIter is an iterator that iterates over the ContextReq.
232// The context template is cloned from the original query context.
233// It will clone the query context for each option and set the options to the context.
234// Then it will return the context and the row insert requests for actual insert.
235pub struct ContextReqIter {
236    opt_req: IntoIter<ContextOpt, Vec<RowInsertRequest>>,
237    ctx_template: QueryContext,
238}
239
240impl Iterator for ContextReqIter {
241    type Item = (QueryContextRef, RowInsertRequests);
242
243    fn next(&mut self) -> Option<Self::Item> {
244        let (mut opt, req_vec) = self.opt_req.next()?;
245        let mut ctx = self.ctx_template.fork();
246        if let Some(schema) = opt.schema.take() {
247            ctx.set_current_schema(&schema);
248        }
249        opt.set_query_context(&mut ctx);
250
251        Some((Arc::new(ctx), RowInsertRequests { inserts: req_vec }))
252    }
253}
254
255#[cfg(test)]
256mod tests {
257    use super::*;
258
259    #[test]
260    fn test_collected_contexts_keep_independent_schemas() {
261        let mut req = ContextReq::default();
262        for schema in ["schema_a", "schema_b"] {
263            let mut opt = ContextOpt::default();
264            opt.set_schema(schema.to_string());
265            req.add_row(
266                &opt,
267                RowInsertRequest {
268                    table_name: "metrics".to_string(),
269                    rows: None,
270                },
271            );
272        }
273
274        let original = Arc::new(QueryContext::with("greptime", "public"));
275        let batches = req.as_req_iter(original.clone()).collect::<Vec<_>>();
276        let mut schemas = batches
277            .iter()
278            .map(|(ctx, _)| ctx.current_schema())
279            .collect::<Vec<_>>();
280        schemas.sort_unstable();
281
282        assert_eq!(vec!["schema_a", "schema_b"], schemas);
283        assert_eq!("public", original.current_schema());
284
285        let other_schema = batches[1].0.current_schema();
286        batches[0].0.set_current_schema("changed");
287        assert_eq!(other_schema, batches[1].0.current_schema());
288        assert_eq!("public", original.current_schema());
289    }
290}
pipeline/etl/ctx_req.rs

pipeline/etl/
ctx_req.rs
