log_store/kafka/worker/
flush.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use common_telemetry::warn;
16use snafu::ResultExt;
17
18use crate::error;
19use crate::kafka::log_store::TopicStat;
20use crate::kafka::worker::{BackgroundProducerWorker, PendingRequest};
21
22impl BackgroundProducerWorker {
23    async fn do_flush(
24        &mut self,
25        PendingRequest {
26            batch,
27            sender,
28            size: _size,
29        }: PendingRequest,
30    ) {
31        let record_num = batch.len() as u64;
32        let result = self
33            .client
34            .produce(batch, self.compression)
35            .await
36            .context(error::BatchProduceSnafu);
37
38        if let Ok(result) = &result {
39            let total_record_size = result.encoded_request_size as u64;
40            let max_offset = result.offsets.iter().max().cloned().unwrap_or_default() as u64;
41            self.topic_stats
42                .entry(self.provider.clone())
43                .and_modify(|stat| {
44                    stat.latest_offset = stat.latest_offset.max(max_offset);
45                    stat.record_size += total_record_size;
46                    stat.record_num += record_num;
47                })
48                .or_insert(TopicStat {
49                    latest_offset: max_offset,
50                    record_size: total_record_size,
51                    record_num,
52                });
53        }
54
55        if let Err(err) = sender.send(result.map(|r| r.offsets)) {
56            warn!(err; "BatchFlushState Receiver is dropped");
57        }
58    }
59
60    pub(crate) async fn try_flush_pending_requests(
61        &mut self,
62        pending_requests: Vec<PendingRequest>,
63    ) {
64        // TODO(weny): Considering merge `PendingRequest`s.
65        for req in pending_requests {
66            self.do_flush(req).await
67        }
68    }
69}
log_store/kafka/worker/flush.rs

log_store/kafka/worker/
flush.rs
