1pub mod index;
16pub mod transformer;
17
18use std::collections::HashMap;
19
20use api::helper::ColumnDataTypeWrapper;
21use api::v1::ColumnDataType;
22use api::v1::value::ValueData;
23use chrono::Utc;
24use datatypes::json::{JsonSettings, JsonTypeHint};
25use datatypes::schema::{FulltextOptions, SkippingIndexOptions};
26use snafu::{OptionExt, ResultExt, ensure};
27use sql::parsers::utils::{
28 validate_column_fulltext_create_option, validate_column_skipping_index_create_option,
29};
30
31use crate::error::{
32 Error, FieldMustBeTypeSnafu, InvalidJson2TypeHintSnafu, KeyMustBeStringSnafu,
33 ParseJson2TypeHintPathSnafu, Result, TransformElementMustBeMapSnafu,
34 TransformFieldMustBeSetSnafu, TransformIndexOptionMustBeScalarSnafu, TransformIndexOptionSnafu,
35 TransformIndexOptionUnsupportedSnafu, TransformIndexOptionsUnsupportedSnafu,
36 TransformIndexTypeMismatchSnafu, TransformIndexTypeMustBeSetSnafu,
37 TransformIndexUnsupportedFieldSnafu, TransformOnFailureInvalidValueSnafu,
38 TransformTypeMustBeSetSnafu, UnsupportedTypeInPipelineSnafu,
39};
40use crate::etl::field::Fields;
41use crate::etl::processor::{yaml_bool, yaml_new_field, yaml_new_fields, yaml_string};
42use crate::etl::transform::index::Index;
43use crate::etl::value::{parse_str_type, parse_str_value};
44
45const TRANSFORM_FIELD: &str = "field";
46const TRANSFORM_FIELDS: &str = "fields";
47const TRANSFORM_TYPE: &str = "type";
48const TRANSFORM_INDEX: &str = "index";
49const TRANSFORM_INDEX_TYPE_FIELD: &str = "index.type";
50const TRANSFORM_INDEX_OPTIONS: &str = "options";
51const TRANSFORM_INDEX_OPTIONS_FIELD: &str = "index.options";
52const TRANSFORM_TAG: &str = "tag";
53const TRANSFORM_DEFAULT: &str = "default";
54const TRANSFORM_ON_FAILURE: &str = "on_failure";
55const JSON2_TYPE: &str = "json2";
56const JSON2_TYPE_HINT: &str = "type.json2[]";
57const JSON2_TYPE_HINT_PATH: &str = "path";
58
59pub use transformer::greptime::GreptimeTransformer;
60
61#[derive(Debug, Clone, Default, Copy)]
63pub enum OnFailure {
64 #[default]
66 Ignore,
67 Default,
70}
71
72impl std::str::FromStr for OnFailure {
73 type Err = Error;
74
75 fn from_str(s: &str) -> Result<Self> {
76 match s {
77 "ignore" => Ok(OnFailure::Ignore),
78 "default" => Ok(OnFailure::Default),
79 _ => TransformOnFailureInvalidValueSnafu { value: s }.fail(),
80 }
81 }
82}
83
84#[derive(Debug, Default, Clone)]
85pub struct Transforms {
86 pub(crate) transforms: Vec<Transform>,
87}
88
89impl Transforms {
90 pub fn transforms(&self) -> &Vec<Transform> {
91 &self.transforms
92 }
93}
94
95impl std::ops::Deref for Transforms {
96 type Target = Vec<Transform>;
97
98 fn deref(&self) -> &Self::Target {
99 &self.transforms
100 }
101}
102
103impl std::ops::DerefMut for Transforms {
104 fn deref_mut(&mut self) -> &mut Self::Target {
105 &mut self.transforms
106 }
107}
108
109impl TryFrom<&Vec<yaml_rust::Yaml>> for Transforms {
110 type Error = Error;
111
112 fn try_from(docs: &Vec<yaml_rust::Yaml>) -> Result<Self> {
113 let mut transforms = Vec::with_capacity(32);
114 let mut all_output_keys: Vec<String> = Vec::with_capacity(32);
115 let mut all_required_keys = Vec::with_capacity(32);
116
117 for doc in docs {
118 let transform_builder: Transform = doc
119 .as_hash()
120 .context(TransformElementMustBeMapSnafu)?
121 .try_into()?;
122 let mut transform_output_keys = transform_builder
123 .fields
124 .iter()
125 .map(|f| f.target_or_input_field().to_string())
126 .collect();
127 all_output_keys.append(&mut transform_output_keys);
128
129 let mut transform_required_keys = transform_builder
130 .fields
131 .iter()
132 .map(|f| f.input_field().to_string())
133 .collect();
134 all_required_keys.append(&mut transform_required_keys);
135
136 transforms.push(transform_builder);
137 }
138
139 all_required_keys.sort();
140
141 Ok(Transforms { transforms })
142 }
143}
144
145#[derive(Debug, Clone)]
147pub struct Transform {
148 pub fields: Fields,
149 pub type_: ColumnDataType,
150 pub(crate) json_settings: Option<JsonSettings>,
151 pub default: Option<ValueData>,
152 pub index: Option<Index>,
153 pub index_options: Option<TransformIndexOptions>,
154 pub tag: bool,
155 pub on_failure: Option<OnFailure>,
156}
157
158#[derive(Debug, Clone, PartialEq, Eq)]
159pub enum TransformIndexOptions {
160 Fulltext(FulltextOptions),
161 Skipping(SkippingIndexOptions),
162}
163
164impl TransformIndexOptions {
165 pub(crate) fn index(&self) -> Index {
166 match self {
167 TransformIndexOptions::Fulltext(_) => Index::Fulltext,
168 TransformIndexOptions::Skipping(_) => Index::Skipping,
169 }
170 }
171
172 #[cfg(test)]
173 pub(crate) fn as_fulltext(&self) -> Option<&FulltextOptions> {
174 match self {
175 TransformIndexOptions::Fulltext(options) => Some(options),
176 TransformIndexOptions::Skipping(_) => None,
177 }
178 }
179
180 #[cfg(test)]
181 pub(crate) fn as_skipping(&self) -> Option<&SkippingIndexOptions> {
182 match self {
183 TransformIndexOptions::Skipping(options) => Some(options),
184 TransformIndexOptions::Fulltext(_) => None,
185 }
186 }
187}
188
189impl Transform {
210 pub(crate) fn get_default(&self) -> Option<&ValueData> {
211 self.default.as_ref()
212 }
213
214 pub(crate) fn get_type_matched_default_val(&self) -> Result<ValueData> {
215 get_default_for_type(&self.type_)
216 }
217
218 pub(crate) fn get_default_value_when_data_is_none(&self) -> Option<ValueData> {
219 if is_timestamp_type(&self.type_) && self.index.is_some_and(|i| i == Index::Time) {
220 let now = Utc::now();
221 match self.type_ {
222 ColumnDataType::TimestampSecond => {
223 return Some(ValueData::TimestampSecondValue(now.timestamp()));
224 }
225 ColumnDataType::TimestampMillisecond => {
226 return Some(ValueData::TimestampMillisecondValue(now.timestamp_millis()));
227 }
228 ColumnDataType::TimestampMicrosecond => {
229 return Some(ValueData::TimestampMicrosecondValue(now.timestamp_micros()));
230 }
231 ColumnDataType::TimestampNanosecond => {
232 return Some(ValueData::TimestampNanosecondValue(
233 now.timestamp_nanos_opt()?,
234 ));
235 }
236 _ => {}
237 }
238 }
239 None
240 }
241
242 pub(crate) fn is_timeindex(&self) -> bool {
243 self.index.is_some_and(|i| i == Index::Time)
244 }
245}
246
247fn is_timestamp_type(ty: &ColumnDataType) -> bool {
248 matches!(
249 ty,
250 ColumnDataType::TimestampSecond
251 | ColumnDataType::TimestampMillisecond
252 | ColumnDataType::TimestampMicrosecond
253 | ColumnDataType::TimestampNanosecond
254 )
255}
256
257fn get_default_for_type(ty: &ColumnDataType) -> Result<ValueData> {
258 let v = match ty {
259 ColumnDataType::Boolean => ValueData::BoolValue(false),
260 ColumnDataType::Int8 => ValueData::I8Value(0),
261 ColumnDataType::Int16 => ValueData::I16Value(0),
262 ColumnDataType::Int32 => ValueData::I32Value(0),
263 ColumnDataType::Int64 => ValueData::I64Value(0),
264 ColumnDataType::Uint8 => ValueData::U8Value(0),
265 ColumnDataType::Uint16 => ValueData::U16Value(0),
266 ColumnDataType::Uint32 => ValueData::U32Value(0),
267 ColumnDataType::Uint64 => ValueData::U64Value(0),
268 ColumnDataType::Float32 => ValueData::F32Value(0.0),
269 ColumnDataType::Float64 => ValueData::F64Value(0.0),
270 ColumnDataType::Binary => ValueData::BinaryValue(jsonb::Value::Null.to_vec()),
271 ColumnDataType::Json => ValueData::JsonValue(Default::default()),
272 ColumnDataType::String => ValueData::StringValue(String::new()),
273
274 ColumnDataType::TimestampSecond => ValueData::TimestampSecondValue(0),
275 ColumnDataType::TimestampMillisecond => ValueData::TimestampMillisecondValue(0),
276 ColumnDataType::TimestampMicrosecond => ValueData::TimestampMicrosecondValue(0),
277 ColumnDataType::TimestampNanosecond => ValueData::TimestampNanosecondValue(0),
278
279 _ => UnsupportedTypeInPipelineSnafu {
280 ty: ty.as_str_name(),
281 }
282 .fail()?,
283 };
284 Ok(v)
285}
286
287fn parse_transform_index(
288 value: &yaml_rust::Yaml,
289) -> Result<(Index, Option<HashMap<String, String>>)> {
290 match value {
291 yaml_rust::Yaml::String(_) => {
292 let index_str = yaml_string(value, TRANSFORM_INDEX)?;
293 Ok((index_str.try_into()?, None))
294 }
295 yaml_rust::Yaml::Hash(hash) => {
296 let mut index = None;
297 let mut index_options = None;
298
299 for (k, v) in hash {
300 let key = k
301 .as_str()
302 .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
303 match key {
304 TRANSFORM_TYPE => {
305 let index_str = yaml_string(v, TRANSFORM_INDEX_TYPE_FIELD)?;
306 index = Some(index_str.try_into()?);
307 }
308 TRANSFORM_INDEX_OPTIONS => {
309 index_options = Some(parse_transform_index_options(v)?);
310 }
311 _ => {
312 return TransformIndexUnsupportedFieldSnafu {
313 field: key.to_string(),
314 }
315 .fail();
316 }
317 }
318 }
319
320 Ok((
321 index.context(TransformIndexTypeMustBeSetSnafu)?,
322 index_options,
323 ))
324 }
325 _ => FieldMustBeTypeSnafu {
326 field: TRANSFORM_INDEX,
327 ty: "string or map",
328 }
329 .fail(),
330 }
331}
332
333fn parse_transform_index_options(value: &yaml_rust::Yaml) -> Result<HashMap<String, String>> {
334 let hash = value.as_hash().context(FieldMustBeTypeSnafu {
335 field: TRANSFORM_INDEX_OPTIONS_FIELD,
336 ty: "map",
337 })?;
338 let mut options = HashMap::with_capacity(hash.len());
339
340 for (k, v) in hash {
341 let key = k
342 .as_str()
343 .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
344
345 let field = format!("{TRANSFORM_INDEX_OPTIONS_FIELD}.{key}");
346 let value = match v {
347 yaml_rust::Yaml::String(v) => v.clone(),
348 yaml_rust::Yaml::Boolean(v) => v.to_string(),
349 yaml_rust::Yaml::Integer(v) => v.to_string(),
350 yaml_rust::Yaml::Real(v) => v.clone(),
351 _ => {
352 return TransformIndexOptionMustBeScalarSnafu { field }.fail();
353 }
354 };
355 options.insert(key.to_string(), value);
356 }
357
358 Ok(options)
359}
360
361fn lower_typed_transform_index_options<T>(
362 index: Index,
363 index_options: Option<HashMap<String, String>>,
364 validate: fn(&str) -> bool,
365 wrap: fn(T) -> TransformIndexOptions,
366) -> Result<Option<TransformIndexOptions>>
367where
368 T: TryFrom<HashMap<String, String>, Error = datatypes::error::Error>,
369{
370 index_options
371 .map(|opts| {
372 for key in opts.keys() {
373 ensure!(
374 validate(key),
375 TransformIndexOptionUnsupportedSnafu {
376 index: index.to_string(),
377 key: key.clone(),
378 }
379 );
380 }
381
382 let options = opts.try_into().context(TransformIndexOptionSnafu {
383 index: index.to_string(),
384 })?;
385
386 Ok(wrap(options))
387 })
388 .transpose()
389}
390
391fn lower_transform_index_options(
392 index: Index,
393 column_type: &ColumnDataType,
394 index_options: Option<HashMap<String, String>>,
395) -> Result<Option<TransformIndexOptions>> {
396 match index {
397 Index::Fulltext => {
398 ensure!(
399 *column_type == ColumnDataType::String,
400 TransformIndexTypeMismatchSnafu {
401 index: index.to_string(),
402 expected: ColumnDataType::String.as_str_name().to_string(),
403 actual: column_type.as_str_name().to_string(),
404 }
405 );
406
407 lower_typed_transform_index_options(
408 index,
409 index_options,
410 validate_column_fulltext_create_option,
411 TransformIndexOptions::Fulltext,
412 )
413 }
414 Index::Skipping => lower_typed_transform_index_options(
415 index,
416 index_options,
417 validate_column_skipping_index_create_option,
418 TransformIndexOptions::Skipping,
419 ),
420 Index::Inverted | Index::Time | Index::Tag => {
421 ensure!(
422 index_options.is_none(),
423 TransformIndexOptionsUnsupportedSnafu {
424 index: index.to_string(),
425 }
426 );
427 Ok(None)
428 }
429 }
430}
431
432fn parse_transform_type(value: &yaml_rust::Yaml) -> Result<(ColumnDataType, Option<JsonSettings>)> {
433 if let Some(type_name) = value.as_str() {
434 return Ok((parse_str_type(type_name)?, None));
435 }
436
437 let config = value.as_hash().context(FieldMustBeTypeSnafu {
438 field: TRANSFORM_TYPE,
439 ty: "string or map",
440 })?;
441 ensure!(
442 config.len() == 1,
443 InvalidJson2TypeHintSnafu {
444 reason: "transform type map must contain exactly one `json2` field".to_string()
445 }
446 );
447 let (type_name, hints) = config.iter().next().context(InvalidJson2TypeHintSnafu {
448 reason: "transform type map must contain a `json2` field".to_string(),
449 })?;
450 let type_name = type_name.as_str().with_context(|| KeyMustBeStringSnafu {
451 k: type_name.clone(),
452 })?;
453 ensure!(
454 type_name.eq_ignore_ascii_case(JSON2_TYPE),
455 InvalidJson2TypeHintSnafu {
456 reason: format!("unsupported transform type map `{type_name}`")
457 }
458 );
459
460 let hints = hints.as_vec().context(FieldMustBeTypeSnafu {
461 field: JSON2_TYPE,
462 ty: "list",
463 })?;
464 let hints = hints
465 .iter()
466 .map(parse_json2_type_hint)
467 .collect::<Result<Vec<_>>>()?;
468 Ok((
469 ColumnDataType::Json,
470 Some(JsonSettings::try_new(hints, None)?),
471 ))
472}
473
474fn parse_json2_type_hint(value: &yaml_rust::Yaml) -> Result<JsonTypeHint> {
475 let config = value.as_hash().context(FieldMustBeTypeSnafu {
476 field: JSON2_TYPE_HINT,
477 ty: "map",
478 })?;
479 let mut path = None;
480 let mut type_name = None;
481 let mut index = None;
482
483 for (key, value) in config {
484 let key = key
485 .as_str()
486 .with_context(|| KeyMustBeStringSnafu { k: key.clone() })?;
487 match key {
488 JSON2_TYPE_HINT_PATH => path = Some(yaml_string(value, JSON2_TYPE_HINT_PATH)?),
489 TRANSFORM_TYPE => type_name = Some(yaml_string(value, TRANSFORM_TYPE)?),
490 TRANSFORM_INDEX => index = Some(value),
491 _ => {
492 return InvalidJson2TypeHintSnafu {
493 reason: format!("unsupported field `{key}`"),
494 }
495 .fail();
496 }
497 }
498 }
499
500 let path = path.context(InvalidJson2TypeHintSnafu {
501 reason: "`path` must be set".to_string(),
502 })?;
503 let path = sql::parse_json2_type_hint_path(&path)
504 .with_context(|_| ParseJson2TypeHintPathSnafu { path: path.clone() })?;
505 let type_name = type_name.context(InvalidJson2TypeHintSnafu {
506 reason: "`type` must be set".to_string(),
507 })?;
508 let type_ = parse_str_type(&type_name)?;
509 ensure!(
510 matches!(
511 type_,
512 ColumnDataType::String
513 | ColumnDataType::Int64
514 | ColumnDataType::Uint64
515 | ColumnDataType::Float64
516 | ColumnDataType::Boolean
517 ),
518 InvalidJson2TypeHintSnafu {
519 reason: format!("unsupported type `{type_name}`")
520 }
521 );
522 let data_type = ColumnDataTypeWrapper::new(type_, None).into();
523 let inverted_index = if let Some(value) = index {
524 let (index, options) = parse_transform_index(value)?;
525 ensure!(
526 index == Index::Inverted,
527 InvalidJson2TypeHintSnafu {
528 reason: format!("unsupported index `{index}`")
529 }
530 );
531 lower_transform_index_options(index, &ColumnDataType::Json, options)?;
532 true
533 } else {
534 false
535 };
536
537 Ok(JsonTypeHint {
538 path,
539 data_type,
540 inverted_index,
541 skipping_index: None,
542 fulltext_index: None,
543 })
544}
545
546impl TryFrom<&yaml_rust::yaml::Hash> for Transform {
547 type Error = Error;
548
549 fn try_from(hash: &yaml_rust::yaml::Hash) -> Result<Self> {
550 let mut fields = Fields::default();
551 let mut default = None;
552 let mut index_value = None;
553 let mut tag = false;
554 let mut on_failure = None;
555
556 let mut type_ = None;
557 let mut json_settings = None;
558
559 for (k, v) in hash {
560 let key = k
561 .as_str()
562 .with_context(|| KeyMustBeStringSnafu { k: k.clone() })?;
563 match key {
564 TRANSFORM_FIELD => {
565 fields = Fields::one(yaml_new_field(v, TRANSFORM_FIELD)?);
566 }
567
568 TRANSFORM_FIELDS => {
569 fields = yaml_new_fields(v, TRANSFORM_FIELDS)?;
570 }
571
572 TRANSFORM_TYPE => {
573 let (parsed_type, parsed_json_settings) = parse_transform_type(v)?;
574 type_ = Some(parsed_type);
575 json_settings = parsed_json_settings;
576 }
577
578 TRANSFORM_INDEX => {
579 index_value = Some(v);
580 }
581
582 TRANSFORM_TAG => {
583 tag = yaml_bool(v, TRANSFORM_TAG)?;
584 }
585
586 TRANSFORM_DEFAULT => {
587 default = match v {
588 yaml_rust::Yaml::Real(r) => Some(r.clone()),
589 yaml_rust::Yaml::Integer(i) => Some(i.to_string()),
590 yaml_rust::Yaml::String(s) => Some(s.clone()),
591 yaml_rust::Yaml::Boolean(b) => Some(b.to_string()),
592 yaml_rust::Yaml::Array(_)
593 | yaml_rust::Yaml::Hash(_)
594 | yaml_rust::Yaml::Alias(_)
595 | yaml_rust::Yaml::Null
596 | yaml_rust::Yaml::BadValue => None,
597 };
598 }
599
600 TRANSFORM_ON_FAILURE => {
601 let on_failure_str = yaml_string(v, TRANSFORM_ON_FAILURE)?;
602 on_failure = Some(on_failure_str.parse()?);
603 }
604
605 _ => {}
606 }
607 }
608
609 ensure!(!fields.is_empty(), TransformFieldMustBeSetSnafu);
611 let type_ = type_.context(TransformTypeMustBeSetSnafu {
612 fields: format!("{:?}", fields),
613 })?;
614
615 let (index, index_options) = match index_value {
616 Some(value) => {
617 let (index, raw_index_options) = parse_transform_index(value)?;
618 let index_options =
619 lower_transform_index_options(index, &type_, raw_index_options)?;
620 (Some(index), index_options)
621 }
622 None => (None, None),
623 };
624
625 let final_default = if let Some(default_value) = default {
626 let target = parse_str_value(&type_, &default_value)?;
627 on_failure = Some(OnFailure::Default);
628 Some(target)
629 } else {
630 None
631 };
632
633 let builder = Transform {
634 fields,
635 type_,
636 json_settings,
637 default: final_default,
638 index,
639 index_options,
640 on_failure,
641 tag,
642 };
643
644 Ok(builder)
645 }
646}
647
648#[cfg(test)]
649mod tests {
650 use yaml_rust::YamlLoader;
651
652 use super::*;
653
654 fn parse_transform(yaml: &str) -> Result<Transform> {
655 let docs = YamlLoader::load_from_str(yaml).unwrap();
656 docs[0].as_hash().unwrap().try_into()
657 }
658
659 #[test]
660 fn test_transform_parses_json2_type_hints() {
661 let transform = parse_transform(
662 r#"
663field: payload
664type:
665 json2:
666 - path: "user.id"
667 type: int64
668 index:
669 type: inverted
670 - path: 'attrs."http.status_code"'
671 type: string
672"#,
673 )
674 .unwrap();
675
676 assert_eq!(transform.type_, ColumnDataType::Json);
677 let hints = transform.json_settings.as_ref().unwrap().type_hints();
678 assert_eq!(hints.len(), 2);
679 assert_eq!(hints[0].path, ["user", "id"]);
680 assert_eq!(
681 hints[0].data_type,
682 datatypes::prelude::ConcreteDataType::int64_datatype()
683 );
684 assert!(hints[0].inverted_index);
685 assert_eq!(hints[1].path, ["attrs", "http.status_code"]);
686 }
687
688 #[test]
689 fn test_transform_rejects_json2_nullable_and_default() {
690 for option in ["nullable: false", "default: 7"] {
691 let err = parse_transform(&format!(
692 r#"
693field: payload
694type:
695 json2:
696 - path: score
697 type: float64
698 {option}
699"#,
700 ))
701 .unwrap_err();
702
703 assert!(err.to_string().contains("unsupported field"), "{err}");
704 }
705 }
706
707 #[test]
708 fn test_transform_parses_legacy_string_index() {
709 let transform = parse_transform(
710 r#"
711field: message
712type: string
713index: fulltext
714"#,
715 )
716 .unwrap();
717
718 assert_eq!(transform.index, Some(Index::Fulltext));
719 assert!(transform.index_options.is_none());
720 }
721
722 #[test]
723 fn test_transform_parses_index_object_without_options() {
724 let transform = parse_transform(
725 r#"
726field: message
727type: string
728index:
729 type: inverted
730"#,
731 )
732 .unwrap();
733
734 assert_eq!(transform.index, Some(Index::Inverted));
735 assert!(transform.index_options.is_none());
736 }
737
738 #[test]
739 fn test_transform_parses_index_object_with_scalar_options() {
740 let transform = parse_transform(
741 r#"
742field: message
743type: string
744index:
745 type: fulltext
746 options:
747 analyzer: English
748 case_sensitive: false
749 granularity: 2048
750 false_positive_rate: 0.02
751"#,
752 )
753 .unwrap();
754
755 assert_eq!(transform.index, Some(Index::Fulltext));
756 let options = transform.index_options.as_ref().unwrap();
757 let fulltext = options.as_fulltext().unwrap();
758 assert!(fulltext.enable);
759 assert_eq!(fulltext.analyzer.to_string(), "English");
760 assert!(!fulltext.case_sensitive);
761 assert_eq!(fulltext.granularity, 2048);
762 assert_eq!(fulltext.false_positive_rate(), 0.02);
763 }
764
765 #[test]
766 fn test_transform_rejects_invalid_index_options_type() {
767 let result = parse_transform(
768 r#"
769field: message
770type: string
771index:
772 type: fulltext
773 options: invalid
774"#,
775 );
776
777 assert!(result.is_err());
778 }
779
780 #[test]
781 fn test_transform_rejects_non_scalar_index_option_value() {
782 let result = parse_transform(
783 r#"
784field: message
785type: string
786index:
787 type: fulltext
788 options:
789 analyzer:
790 kind: English
791"#,
792 );
793
794 assert!(result.is_err());
795 }
796
797 #[test]
798 fn test_transform_rejects_unknown_index_field() {
799 let result = parse_transform(
800 r#"
801field: message
802type: string
803index:
804 type: fulltext
805 config: {}
806"#,
807 );
808
809 assert!(result.is_err());
810 }
811
812 #[test]
813 fn test_transform_rejects_unsupported_fulltext_option_key() {
814 let result = parse_transform(
815 r#"
816field: message
817type: string
818index:
819 type: fulltext
820 options:
821 tokenizer: english
822"#,
823 );
824
825 assert!(result.is_err());
826 }
827
828 #[test]
829 fn test_transform_rejects_options_for_inverted_index() {
830 let result = parse_transform(
831 r#"
832field: message
833type: string
834index:
835 type: inverted
836 options:
837 backend: bloom
838"#,
839 );
840
841 assert!(result.is_err());
842 }
843
844 #[test]
845 fn test_transform_rejects_empty_options_for_unsupported_indexes() {
846 for index in ["inverted", "time", "tag"] {
847 let yaml = format!(
848 r#"
849field: message
850type: string
851index:
852 type: {index}
853 options: {{}}
854"#
855 );
856
857 let result = parse_transform(&yaml);
858 assert!(
859 result.is_err(),
860 "expected `{index}` to reject empty options"
861 );
862 }
863 }
864
865 #[test]
866 fn test_transform_rejects_fulltext_index_on_non_string_column() {
867 let result = parse_transform(
868 r#"
869field: count
870type: int64
871index: fulltext
872"#,
873 );
874
875 assert!(result.is_err());
876 }
877
878 #[test]
879 fn test_transform_allows_skipping_index_on_numeric_column() {
880 let transform = parse_transform(
881 r#"
882field: count
883type: int64
884index:
885 type: skipping
886 options:
887 granularity: 2048
888 false_positive_rate: 0.02
889 type: BLOOM
890"#,
891 )
892 .unwrap();
893
894 assert_eq!(transform.index, Some(Index::Skipping));
895 let skipping = transform
896 .index_options
897 .as_ref()
898 .unwrap()
899 .as_skipping()
900 .unwrap();
901 assert_eq!(skipping.granularity, 2048);
902 assert_eq!(skipping.false_positive_rate(), 0.02);
903 assert_eq!(skipping.index_type.to_string(), "BLOOM");
904 }
905}