flow/
engine.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Define a trait for flow engine, which is used by both streaming engine and batch engine
16
17use std::collections::HashMap;
18
19use common_meta::key::flow::flow_state::FlowStat;
20use session::context::QueryContext;
21use table::metadata::TableId;
22
23use crate::Error;
24// TODO(discord9): refactor common types for flow to a separate module
25/// FlowId is a unique identifier for a flow task
26pub type FlowId = u64;
27pub type TableName = [String; 3];
28
29#[derive(Clone)]
30pub struct FlowAuthHeader {
31    auth_schema: api::v1::auth_header::AuthScheme,
32}
33
34impl std::fmt::Debug for FlowAuthHeader {
35    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
36        match self.auth() {
37            api::v1::auth_header::AuthScheme::Basic(basic) => f
38                .debug_struct("Basic")
39                .field("username", &basic.username)
40                .field("password", &"<RETRACTED>")
41                .finish(),
42            api::v1::auth_header::AuthScheme::Token(_) => f
43                .debug_struct("Token")
44                .field("token", &"<RETRACTED>")
45                .finish(),
46        }
47    }
48}
49
50impl FlowAuthHeader {
51    pub fn from_user_pwd(username: &str, pwd: &str) -> Self {
52        Self {
53            auth_schema: api::v1::auth_header::AuthScheme::Basic(api::v1::Basic {
54                username: username.to_string(),
55                password: pwd.to_string(),
56            }),
57        }
58    }
59
60    pub fn auth(&self) -> &api::v1::auth_header::AuthScheme {
61        &self.auth_schema
62    }
63}
64
65/// The arguments to create a flow
66#[derive(Debug, Clone)]
67pub struct CreateFlowArgs {
68    pub flow_id: FlowId,
69    pub sink_table_name: TableName,
70    pub source_table_ids: Vec<TableId>,
71    pub create_if_not_exists: bool,
72    pub or_replace: bool,
73    pub expire_after: Option<i64>,
74    pub eval_interval: Option<i64>,
75    pub comment: Option<String>,
76    pub sql: String,
77    pub flow_options: HashMap<String, String>,
78    pub query_ctx: Option<QueryContext>,
79}
80
81pub trait FlowEngine {
82    /// Create a flow using the provided arguments, return previous flow id if exists and is replaced
83    async fn create_flow(&self, args: CreateFlowArgs) -> Result<Option<FlowId>, Error>;
84    /// Remove a flow by its ID
85    async fn remove_flow(&self, flow_id: FlowId) -> Result<(), Error>;
86    /// Flush the flow, return the number of rows flushed
87    async fn flush_flow(&self, flow_id: FlowId) -> Result<usize, Error>;
88    /// Check if the flow exists
89    async fn flow_exist(&self, flow_id: FlowId) -> Result<bool, Error>;
90    /// List all flows
91    async fn list_flows(&self) -> Result<impl IntoIterator<Item = FlowId>, Error>;
92    /// Handle the insert requests for the flow
93    async fn handle_flow_inserts(
94        &self,
95        request: api::v1::region::InsertRequests,
96    ) -> Result<(), Error>;
97
98    async fn handle_mark_window_dirty(
99        &self,
100        req: api::v1::flow::DirtyWindowRequests,
101    ) -> Result<(), Error>;
102}
103
104/// Provides flow runtime statistics for information schema and heartbeat reporting.
105pub trait FlowStatProvider {
106    /// Returns current runtime stats of an engine.
107    async fn flow_stat(&self) -> FlowStat;
108}
flow/engine.rs

flow/
engine.rs
