mito_codec/row_converter/
sparse.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{HashMap, HashSet};
16use std::sync::Arc;
17
18use bytes::BufMut;
19use common_recordbatch::filter::SimpleFilterEvaluator;
20use datatypes::prelude::ConcreteDataType;
21use datatypes::value::{Value, ValueRef};
22use memcomparable::{Deserializer, Serializer};
23use serde::{Deserialize, Serialize};
24use snafu::ResultExt;
25use store_api::codec::PrimaryKeyEncoding;
26use store_api::metadata::RegionMetadataRef;
27use store_api::storage::ColumnId;
28use store_api::storage::consts::ReservedColumnId;
29
30use crate::error::{DeserializeFieldSnafu, Result, SerializeFieldSnafu, UnsupportedOperationSnafu};
31use crate::key_values::KeyValue;
32use crate::primary_key_filter::SparsePrimaryKeyFilter;
33use crate::row_converter::dense::SortField;
34use crate::row_converter::{CompositeValues, PrimaryKeyCodec, PrimaryKeyFilter};
35
36/// A codec for sparse key of metrics.
37/// It requires the input primary key columns are sorted by the column name in lexicographical order.
38/// It encodes the column id of the physical region.
39#[derive(Clone, Debug)]
40pub struct SparsePrimaryKeyCodec {
41    inner: Arc<SparsePrimaryKeyCodecInner>,
42}
43
44#[derive(Debug)]
45struct SparsePrimaryKeyCodecInner {
46    // Internal fields
47    table_id_field: SortField,
48    // Internal fields
49    tsid_field: SortField,
50    // User defined label field
51    label_field: SortField,
52    // Columns in primary key
53    //
54    // None means all unknown columns is primary key(`Self::label_field`).
55    columns: Option<HashSet<ColumnId>>,
56}
57
58/// Sparse values representation.
59///
60/// A map of [`ColumnId`] to [`Value`].
61#[derive(Debug, Clone, PartialEq, Eq)]
62pub struct SparseValues {
63    values: HashMap<ColumnId, Value>,
64}
65
66impl SparseValues {
67    /// Creates a new [`SparseValues`] instance.
68    pub fn new(values: HashMap<ColumnId, Value>) -> Self {
69        Self { values }
70    }
71
72    /// Returns the value of the given column, or [`Value::Null`] if the column is not present.
73    pub fn get_or_null(&self, column_id: ColumnId) -> &Value {
74        self.values.get(&column_id).unwrap_or(&Value::Null)
75    }
76
77    /// Returns the value of the given column, or [`None`] if the column is not present.
78    pub fn get(&self, column_id: &ColumnId) -> Option<&Value> {
79        self.values.get(column_id)
80    }
81
82    /// Inserts a new value into the [`SparseValues`].
83    pub fn insert(&mut self, column_id: ColumnId, value: Value) {
84        self.values.insert(column_id, value);
85    }
86
87    /// Returns an iterator over all stored column id/value pairs.
88    pub fn iter(&self) -> impl Iterator<Item = (&ColumnId, &Value)> {
89        self.values.iter()
90    }
91}
92
93/// The column id of the tsid.
94pub const RESERVED_COLUMN_ID_TSID: ColumnId = ReservedColumnId::tsid();
95/// The column id of the table id.
96pub const RESERVED_COLUMN_ID_TABLE_ID: ColumnId = ReservedColumnId::table_id();
97/// The size of the column id in the encoded sparse row.
98pub const COLUMN_ID_ENCODE_SIZE: usize = 4;
99
100impl SparsePrimaryKeyCodec {
101    /// Creates a new [`SparsePrimaryKeyCodec`] instance.
102    pub fn from_columns(columns_ids: impl Iterator<Item = ColumnId>) -> Self {
103        let columns = columns_ids.collect();
104        Self {
105            inner: Arc::new(SparsePrimaryKeyCodecInner {
106                table_id_field: SortField::new(ConcreteDataType::uint32_datatype()),
107                tsid_field: SortField::new(ConcreteDataType::uint64_datatype()),
108                label_field: SortField::new(ConcreteDataType::string_datatype()),
109                columns: Some(columns),
110            }),
111        }
112    }
113
114    /// Creates a new [`SparsePrimaryKeyCodec`] instance.
115    pub fn new(region_metadata: &RegionMetadataRef) -> Self {
116        Self::from_columns(region_metadata.primary_key_columns().map(|c| c.column_id))
117    }
118
119    /// Returns a new [`SparsePrimaryKeyCodec`] instance.
120    ///
121    /// It treats all unknown columns as primary key(label field).
122    pub fn schemaless() -> Self {
123        Self {
124            inner: Arc::new(SparsePrimaryKeyCodecInner {
125                table_id_field: SortField::new(ConcreteDataType::uint32_datatype()),
126                tsid_field: SortField::new(ConcreteDataType::uint64_datatype()),
127                label_field: SortField::new(ConcreteDataType::string_datatype()),
128                columns: None,
129            }),
130        }
131    }
132
133    /// Creates a new [`SparsePrimaryKeyCodec`] instance with additional label `fields`.
134    pub fn with_fields(fields: Vec<(ColumnId, SortField)>) -> Self {
135        Self {
136            inner: Arc::new(SparsePrimaryKeyCodecInner {
137                columns: Some(fields.iter().map(|f| f.0).collect()),
138                table_id_field: SortField::new(ConcreteDataType::uint32_datatype()),
139                tsid_field: SortField::new(ConcreteDataType::uint64_datatype()),
140                label_field: SortField::new(ConcreteDataType::string_datatype()),
141            }),
142        }
143    }
144
145    /// Returns the field of the given column id.
146    fn get_field(&self, column_id: ColumnId) -> Option<&SortField> {
147        // if the `columns` is not specified, all unknown columns is primary key(label field).
148        if let Some(columns) = &self.inner.columns
149            && !columns.contains(&column_id)
150        {
151            return None;
152        }
153
154        match column_id {
155            RESERVED_COLUMN_ID_TABLE_ID => Some(&self.inner.table_id_field),
156            RESERVED_COLUMN_ID_TSID => Some(&self.inner.tsid_field),
157            _ => Some(&self.inner.label_field),
158        }
159    }
160
161    /// Encodes the given bytes into a [`SparseValues`].
162    pub fn encode_to_vec<'a, I>(&self, row: I, buffer: &mut Vec<u8>) -> Result<()>
163    where
164        I: Iterator<Item = (ColumnId, ValueRef<'a>)>,
165    {
166        let mut serializer = Serializer::new(buffer);
167        for (column_id, value) in row {
168            if value.is_null() {
169                continue;
170            }
171
172            if let Some(field) = self.get_field(column_id) {
173                column_id
174                    .serialize(&mut serializer)
175                    .context(SerializeFieldSnafu)?;
176                field.serialize(&mut serializer, &value)?;
177            } else {
178                // TODO(weny): handle the error.
179                common_telemetry::warn!("Column {} is not in primary key, skipping", column_id);
180            }
181        }
182        Ok(())
183    }
184
185    pub fn encode_raw_tag_value<'a, I>(&self, row: I, buffer: &mut Vec<u8>) -> Result<()>
186    where
187        I: Iterator<Item = (ColumnId, &'a [u8])>,
188    {
189        for (tag_column_id, tag_value) in row {
190            let value_len = tag_value.len();
191            buffer.reserve(6 + value_len / 8 * 9);
192            buffer.put_u32(tag_column_id);
193            buffer.put_u8(1);
194            buffer.put_u8(!tag_value.is_empty() as u8);
195
196            // Manual implementation of memcomparable::ser::Serializer::serialize_bytes
197            // to avoid byte-by-byte put.
198            let mut len = 0;
199            let num_chucks = value_len / 8;
200            let remainder = value_len % 8;
201
202            for idx in 0..num_chucks {
203                buffer.extend_from_slice(&tag_value[idx * 8..idx * 8 + 8]);
204                len += 8;
205                // append an extra byte that signals the number of significant bytes in this chunk
206                // 1-8: many bytes were significant and this group is the last group
207                // 9: all 8 bytes were significant and there is more data to come
208                let extra = if len == value_len { 8 } else { 9 };
209                buffer.put_u8(extra);
210            }
211
212            if remainder != 0 {
213                buffer.extend_from_slice(&tag_value[len..value_len]);
214                buffer.put_bytes(0, 8 - remainder);
215                buffer.put_u8(remainder as u8);
216            }
217        }
218        Ok(())
219    }
220
221    /// Encodes the given bytes into a [`SparseValues`].
222    pub fn encode_internal(&self, table_id: u32, tsid: u64, buffer: &mut Vec<u8>) -> Result<()> {
223        buffer.reserve_exact(22);
224        buffer.put_u32(RESERVED_COLUMN_ID_TABLE_ID);
225        buffer.put_u8(1);
226        buffer.put_u32(table_id);
227        buffer.put_u32(RESERVED_COLUMN_ID_TSID);
228        buffer.put_u8(1);
229        buffer.put_u64(tsid);
230        Ok(())
231    }
232
233    /// Decodes the given bytes into a [`SparseValues`].
234    fn decode_sparse(&self, bytes: &[u8]) -> Result<SparseValues> {
235        let mut deserializer = Deserializer::new(bytes);
236        let mut values = SparseValues::new(HashMap::new());
237
238        let column_id = u32::deserialize(&mut deserializer).context(DeserializeFieldSnafu)?;
239        let value = self.inner.table_id_field.deserialize(&mut deserializer)?;
240        values.insert(column_id, value);
241
242        let column_id = u32::deserialize(&mut deserializer).context(DeserializeFieldSnafu)?;
243        let value = self.inner.tsid_field.deserialize(&mut deserializer)?;
244        values.insert(column_id, value);
245        while deserializer.has_remaining() {
246            let column_id = u32::deserialize(&mut deserializer).context(DeserializeFieldSnafu)?;
247            let value = self.inner.label_field.deserialize(&mut deserializer)?;
248            values.insert(column_id, value);
249        }
250
251        Ok(values)
252    }
253
254    /// Decodes the given bytes into a [`Value`].
255    fn decode_leftmost(&self, bytes: &[u8]) -> Result<Option<Value>> {
256        let mut deserializer = Deserializer::new(bytes);
257        // Skip the column id.
258        deserializer.advance(COLUMN_ID_ENCODE_SIZE);
259        let value = self.inner.table_id_field.deserialize(&mut deserializer)?;
260        Ok(Some(value))
261    }
262
263    /// Returns the offset of the given column id in the given primary key.
264    pub fn has_column(
265        &self,
266        pk: &[u8],
267        offsets_map: &mut HashMap<u32, usize>,
268        column_id: ColumnId,
269    ) -> Option<usize> {
270        if offsets_map.is_empty() {
271            let mut deserializer = Deserializer::new(pk);
272            let mut offset = 0;
273            while deserializer.has_remaining() {
274                let column_id = u32::deserialize(&mut deserializer).unwrap();
275                offset += 4;
276                offsets_map.insert(column_id, offset);
277                let Some(field) = self.get_field(column_id) else {
278                    break;
279                };
280
281                let skip = field.skip_deserialize(pk, &mut deserializer).unwrap();
282                offset += skip;
283            }
284
285            offsets_map.get(&column_id).copied()
286        } else {
287            offsets_map.get(&column_id).copied()
288        }
289    }
290
291    /// Decode value at `offset` in `pk`.
292    pub fn decode_value_at(&self, pk: &[u8], offset: usize, column_id: ColumnId) -> Result<Value> {
293        let mut deserializer = Deserializer::new(pk);
294        deserializer.advance(offset);
295        // Safety: checked by `has_column`
296        let field = self.get_field(column_id).unwrap();
297        field.deserialize(&mut deserializer)
298    }
299
300    /// Returns the encoded bytes of the given `column_id` in `pk`.
301    ///
302    /// Returns `Ok(None)` if the `column_id` is missing in `pk`.
303    pub fn encoded_value_for_column<'a>(
304        &self,
305        pk: &'a [u8],
306        offsets_map: &mut HashMap<u32, usize>,
307        column_id: ColumnId,
308    ) -> Result<Option<&'a [u8]>> {
309        let Some(offset) = self.has_column(pk, offsets_map, column_id) else {
310            return Ok(None);
311        };
312
313        let Some(field) = self.get_field(column_id) else {
314            return Ok(None);
315        };
316
317        let mut deserializer = Deserializer::new(pk);
318        deserializer.advance(offset);
319        let len = field.skip_deserialize(pk, &mut deserializer)?;
320        Ok(Some(&pk[offset..offset + len]))
321    }
322}
323
324impl PrimaryKeyCodec for SparsePrimaryKeyCodec {
325    fn encode_key_value(&self, _key_value: &KeyValue, _buffer: &mut Vec<u8>) -> Result<()> {
326        UnsupportedOperationSnafu {
327            err_msg: "The encode_key_value method is not supported in SparsePrimaryKeyCodec.",
328        }
329        .fail()
330    }
331
332    fn encode_values(&self, values: &[(ColumnId, Value)], buffer: &mut Vec<u8>) -> Result<()> {
333        self.encode_to_vec(values.iter().map(|v| (v.0, v.1.as_value_ref())), buffer)
334    }
335
336    fn encode_value_refs(
337        &self,
338        values: &[(ColumnId, ValueRef)],
339        buffer: &mut Vec<u8>,
340    ) -> Result<()> {
341        self.encode_to_vec(values.iter().map(|v| (v.0, v.1.clone())), buffer)
342    }
343
344    fn estimated_size(&self) -> Option<usize> {
345        None
346    }
347
348    fn num_fields(&self) -> Option<usize> {
349        None
350    }
351
352    fn encoding(&self) -> PrimaryKeyEncoding {
353        PrimaryKeyEncoding::Sparse
354    }
355
356    fn primary_key_filter(
357        &self,
358        metadata: &RegionMetadataRef,
359        filters: Arc<Vec<SimpleFilterEvaluator>>,
360    ) -> Box<dyn PrimaryKeyFilter> {
361        Box::new(SparsePrimaryKeyFilter::new(
362            metadata.clone(),
363            filters,
364            self.clone(),
365        ))
366    }
367
368    fn decode(&self, bytes: &[u8]) -> Result<CompositeValues> {
369        Ok(CompositeValues::Sparse(self.decode_sparse(bytes)?))
370    }
371
372    fn decode_leftmost(&self, bytes: &[u8]) -> Result<Option<Value>> {
373        self.decode_leftmost(bytes)
374    }
375}
376
377/// Field with column id.
378pub struct FieldWithId {
379    pub field: SortField,
380    pub column_id: ColumnId,
381}
382
383/// A special encoder for memtable.
384pub struct SparseEncoder {
385    fields: Vec<FieldWithId>,
386}
387
388impl SparseEncoder {
389    pub fn new(fields: Vec<FieldWithId>) -> Self {
390        Self { fields }
391    }
392
393    pub fn encode_to_vec<'a, I>(&self, row: I, buffer: &mut Vec<u8>) -> Result<()>
394    where
395        I: Iterator<Item = ValueRef<'a>>,
396    {
397        let mut serializer = Serializer::new(buffer);
398        for (value, field) in row.zip(self.fields.iter()) {
399            if !value.is_null() {
400                field
401                    .column_id
402                    .serialize(&mut serializer)
403                    .context(SerializeFieldSnafu)?;
404                field.field.serialize(&mut serializer, &value)?;
405            }
406        }
407        Ok(())
408    }
409}
410
411#[cfg(test)]
412mod tests {
413    use std::sync::Arc;
414
415    use api::v1::SemanticType;
416    use common_query::prelude::{greptime_timestamp, greptime_value};
417    use common_time::Timestamp;
418    use common_time::timestamp::TimeUnit;
419    use datatypes::schema::ColumnSchema;
420    use datatypes::value::{OrderedFloat, Value};
421    use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
422    use store_api::metric_engine_consts::{
423        DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME,
424    };
425    use store_api::storage::{ColumnId, RegionId};
426
427    use super::*;
428
429    fn test_region_metadata() -> RegionMetadataRef {
430        let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 1));
431        builder
432            .push_column_metadata(ColumnMetadata {
433                column_schema: ColumnSchema::new(
434                    DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
435                    ConcreteDataType::uint32_datatype(),
436                    false,
437                ),
438                semantic_type: SemanticType::Tag,
439                column_id: ReservedColumnId::table_id(),
440            })
441            .push_column_metadata(ColumnMetadata {
442                column_schema: ColumnSchema::new(
443                    DATA_SCHEMA_TSID_COLUMN_NAME,
444                    ConcreteDataType::uint64_datatype(),
445                    false,
446                ),
447                semantic_type: SemanticType::Tag,
448                column_id: ReservedColumnId::tsid(),
449            })
450            .push_column_metadata(ColumnMetadata {
451                column_schema: ColumnSchema::new("pod", ConcreteDataType::string_datatype(), true),
452                semantic_type: SemanticType::Tag,
453                column_id: 1,
454            })
455            .push_column_metadata(ColumnMetadata {
456                column_schema: ColumnSchema::new(
457                    "namespace",
458                    ConcreteDataType::string_datatype(),
459                    true,
460                ),
461                semantic_type: SemanticType::Tag,
462                column_id: 2,
463            })
464            .push_column_metadata(ColumnMetadata {
465                column_schema: ColumnSchema::new(
466                    "container",
467                    ConcreteDataType::string_datatype(),
468                    true,
469                ),
470                semantic_type: SemanticType::Tag,
471                column_id: 3,
472            })
473            .push_column_metadata(ColumnMetadata {
474                column_schema: ColumnSchema::new(
475                    "pod_name",
476                    ConcreteDataType::string_datatype(),
477                    true,
478                ),
479                semantic_type: SemanticType::Tag,
480                column_id: 4,
481            })
482            .push_column_metadata(ColumnMetadata {
483                column_schema: ColumnSchema::new(
484                    "pod_ip",
485                    ConcreteDataType::string_datatype(),
486                    true,
487                ),
488                semantic_type: SemanticType::Tag,
489                column_id: 5,
490            })
491            .push_column_metadata(ColumnMetadata {
492                column_schema: ColumnSchema::new(
493                    greptime_value(),
494                    ConcreteDataType::float64_datatype(),
495                    false,
496                ),
497                semantic_type: SemanticType::Field,
498                column_id: 6,
499            })
500            .push_column_metadata(ColumnMetadata {
501                column_schema: ColumnSchema::new(
502                    greptime_timestamp(),
503                    ConcreteDataType::timestamp_nanosecond_datatype(),
504                    false,
505                ),
506                semantic_type: SemanticType::Timestamp,
507                column_id: 7,
508            })
509            .primary_key(vec![
510                ReservedColumnId::table_id(),
511                ReservedColumnId::tsid(),
512                1,
513                2,
514                3,
515                4,
516                5,
517            ]);
518        let metadata = builder.build().unwrap();
519        Arc::new(metadata)
520    }
521
522    #[test]
523    fn test_sparse_value_new_and_get_or_null() {
524        let mut values = HashMap::new();
525        values.insert(1, Value::Int32(42));
526        let sparse_value = SparseValues::new(values);
527
528        assert_eq!(sparse_value.get_or_null(1), &Value::Int32(42));
529        assert_eq!(sparse_value.get_or_null(2), &Value::Null);
530    }
531
532    #[test]
533    fn test_sparse_value_insert() {
534        let mut sparse_value = SparseValues::new(HashMap::new());
535        sparse_value.insert(1, Value::Int32(42));
536
537        assert_eq!(sparse_value.get_or_null(1), &Value::Int32(42));
538    }
539
540    fn test_row() -> Vec<(ColumnId, ValueRef<'static>)> {
541        vec![
542            (RESERVED_COLUMN_ID_TABLE_ID, ValueRef::UInt32(42)),
543            (
544                RESERVED_COLUMN_ID_TSID,
545                ValueRef::UInt64(123843349035232323),
546            ),
547            // label: pod
548            (1, ValueRef::String("greptime-frontend-6989d9899-22222")),
549            // label: namespace
550            (2, ValueRef::String("greptime-cluster")),
551            // label: container
552            (3, ValueRef::String("greptime-frontend-6989d9899-22222")),
553            // label: pod_name
554            (4, ValueRef::String("greptime-frontend-6989d9899-22222")),
555            // label: pod_ip
556            (5, ValueRef::String("10.10.10.10")),
557            // field: greptime_value
558            (6, ValueRef::Float64(OrderedFloat(1.0))),
559            // field: greptime_timestamp
560            (
561                7,
562                ValueRef::Timestamp(Timestamp::new(1618876800000000000, TimeUnit::Nanosecond)),
563            ),
564        ]
565    }
566
567    #[test]
568    fn test_encode_by_short_cuts() {
569        let region_metadata = test_region_metadata();
570        let codec = SparsePrimaryKeyCodec::new(&region_metadata);
571        let mut buffer = Vec::new();
572        let internal_columns = [
573            (RESERVED_COLUMN_ID_TABLE_ID, ValueRef::UInt32(1024)),
574            (RESERVED_COLUMN_ID_TSID, ValueRef::UInt64(42)),
575        ];
576        let tags = [
577            (1, "greptime-frontend-6989d9899-22222"),
578            (2, "greptime-cluster"),
579            (3, "greptime-frontend-6989d9899-22222"),
580            (4, "greptime-frontend-6989d9899-22222"),
581            (5, "10.10.10.10"),
582        ];
583        codec
584            .encode_to_vec(internal_columns.into_iter(), &mut buffer)
585            .unwrap();
586        codec
587            .encode_to_vec(
588                tags.iter()
589                    .map(|(col_id, tag_value)| (*col_id, ValueRef::String(tag_value))),
590                &mut buffer,
591            )
592            .unwrap();
593
594        let mut buffer_by_raw_encoding = Vec::new();
595        codec
596            .encode_internal(1024, 42, &mut buffer_by_raw_encoding)
597            .unwrap();
598        let tags: Vec<_> = tags
599            .into_iter()
600            .map(|(col_id, tag_value)| (col_id, tag_value.as_bytes()))
601            .collect();
602        codec
603            .encode_raw_tag_value(
604                tags.iter().map(|(c, b)| (*c, *b)),
605                &mut buffer_by_raw_encoding,
606            )
607            .unwrap();
608        assert_eq!(buffer, buffer_by_raw_encoding);
609    }
610
611    #[test]
612    fn test_encode_to_vec() {
613        let region_metadata = test_region_metadata();
614        let codec = SparsePrimaryKeyCodec::new(&region_metadata);
615        let mut buffer = Vec::new();
616
617        let row = test_row();
618        codec.encode_to_vec(row.into_iter(), &mut buffer).unwrap();
619        assert!(!buffer.is_empty());
620        let sparse_value = codec.decode_sparse(&buffer).unwrap();
621        assert_eq!(
622            sparse_value.get_or_null(RESERVED_COLUMN_ID_TABLE_ID),
623            &Value::UInt32(42)
624        );
625        assert_eq!(
626            sparse_value.get_or_null(1),
627            &Value::String("greptime-frontend-6989d9899-22222".into())
628        );
629        assert_eq!(
630            sparse_value.get_or_null(2),
631            &Value::String("greptime-cluster".into())
632        );
633        assert_eq!(
634            sparse_value.get_or_null(3),
635            &Value::String("greptime-frontend-6989d9899-22222".into())
636        );
637        assert_eq!(
638            sparse_value.get_or_null(4),
639            &Value::String("greptime-frontend-6989d9899-22222".into())
640        );
641        assert_eq!(
642            sparse_value.get_or_null(5),
643            &Value::String("10.10.10.10".into())
644        );
645    }
646
647    #[test]
648    fn test_decode_leftmost() {
649        let region_metadata = test_region_metadata();
650        let codec = SparsePrimaryKeyCodec::new(&region_metadata);
651        let mut buffer = Vec::new();
652        let row = test_row();
653        codec.encode_to_vec(row.into_iter(), &mut buffer).unwrap();
654        assert!(!buffer.is_empty());
655        let result = codec.decode_leftmost(&buffer).unwrap().unwrap();
656        assert_eq!(result, Value::UInt32(42));
657    }
658
659    #[test]
660    fn test_has_column() {
661        let region_metadata = test_region_metadata();
662        let codec = SparsePrimaryKeyCodec::new(&region_metadata);
663        let mut buffer = Vec::new();
664        let row = test_row();
665        codec.encode_to_vec(row.into_iter(), &mut buffer).unwrap();
666        assert!(!buffer.is_empty());
667
668        let mut offsets_map = HashMap::new();
669        for column_id in [
670            RESERVED_COLUMN_ID_TABLE_ID,
671            RESERVED_COLUMN_ID_TSID,
672            1,
673            2,
674            3,
675            4,
676            5,
677        ] {
678            let offset = codec.has_column(&buffer, &mut offsets_map, column_id);
679            assert!(offset.is_some());
680        }
681
682        let offset = codec.has_column(&buffer, &mut offsets_map, 6);
683        assert!(offset.is_none());
684    }
685
686    #[test]
687    fn test_decode_value_at() {
688        let region_metadata = test_region_metadata();
689        let codec = SparsePrimaryKeyCodec::new(&region_metadata);
690        let mut buffer = Vec::new();
691        let row = test_row();
692        codec.encode_to_vec(row.into_iter(), &mut buffer).unwrap();
693        assert!(!buffer.is_empty());
694
695        let row = test_row();
696        let mut offsets_map = HashMap::new();
697        for column_id in [
698            RESERVED_COLUMN_ID_TABLE_ID,
699            RESERVED_COLUMN_ID_TSID,
700            1,
701            2,
702            3,
703            4,
704            5,
705        ] {
706            let offset = codec
707                .has_column(&buffer, &mut offsets_map, column_id)
708                .unwrap();
709            let value = codec.decode_value_at(&buffer, offset, column_id).unwrap();
710            let expected_value = row
711                .iter()
712                .find(|(id, _)| *id == column_id)
713                .unwrap()
714                .1
715                .clone();
716            assert_eq!(value.as_value_ref(), expected_value);
717        }
718    }
719
720    #[test]
721    fn test_encoded_value_for_column() {
722        let region_metadata = test_region_metadata();
723        let codec = SparsePrimaryKeyCodec::new(&region_metadata);
724        let mut buffer = Vec::new();
725        let row = test_row();
726        codec
727            .encode_to_vec(row.clone().into_iter(), &mut buffer)
728            .unwrap();
729        assert!(!buffer.is_empty());
730
731        let mut offsets_map = HashMap::new();
732        for column_id in [
733            RESERVED_COLUMN_ID_TABLE_ID,
734            RESERVED_COLUMN_ID_TSID,
735            1,
736            2,
737            3,
738            4,
739            5,
740        ] {
741            let encoded_value = codec
742                .encoded_value_for_column(&buffer, &mut offsets_map, column_id)
743                .unwrap()
744                .unwrap();
745            let expected_value = row
746                .iter()
747                .find(|(id, _)| *id == column_id)
748                .unwrap()
749                .1
750                .clone();
751            let data_type = match column_id {
752                RESERVED_COLUMN_ID_TABLE_ID => ConcreteDataType::uint32_datatype(),
753                RESERVED_COLUMN_ID_TSID => ConcreteDataType::uint64_datatype(),
754                _ => ConcreteDataType::string_datatype(),
755            };
756            let field = SortField::new(data_type);
757            let mut expected_encoded = Vec::new();
758            let mut serializer = Serializer::new(&mut expected_encoded);
759            field.serialize(&mut serializer, &expected_value).unwrap();
760            assert_eq!(encoded_value, expected_encoded.as_slice());
761        }
762
763        for column_id in [6_u32, 7_u32, 999_u32] {
764            let encoded_value = codec
765                .encoded_value_for_column(&buffer, &mut offsets_map, column_id)
766                .unwrap();
767            assert!(encoded_value.is_none());
768        }
769    }
770}
mito_codec/row_converter/sparse.rs

mito_codec/row_converter/
sparse.rs
