Skip to main content

mito2/sst/
index.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub(crate) mod bloom_filter;
16pub mod build;
17mod column;
18#[cfg(test)]
19mod column_test;
20pub(crate) mod fulltext_index;
21pub mod handler;
22mod indexer;
23pub mod intermediate;
24pub(crate) mod inverted_index;
25pub mod kind;
26mod primary_key;
27pub mod puffin_manager;
28pub mod registry;
29#[cfg(test)]
30mod sparse_test;
31mod statistics;
32pub(crate) mod store;
33
34use std::cmp::Ordering;
35use std::collections::{BTreeMap, HashMap, HashSet};
36use std::ops::ControlFlow;
37use std::sync::Arc;
38
39use bytes::Bytes;
40use common_error::ext::BoxedError;
41use common_telemetry::{debug, error, info, warn};
42use datatypes::arrow::record_batch::RecordBatch;
43use index::target::IndexTarget;
44use mito_codec::row_converter::DensePrimaryKeyCodec;
45use object_store::ObjectStore;
46use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
47use puffin_manager::SstPuffinManager;
48use smallvec::SmallVec;
49use snafu::{IntoError, ResultExt};
50#[cfg(feature = "enterprise")]
51pub use statistics::{Statistics, TimerGuard};
52use store_api::codec::PrimaryKeyEncoding;
53use store_api::metadata::RegionMetadataRef;
54use store_api::storage::{ColumnId, FileId, RegionId};
55use strum::IntoStaticStr;
56use tokio::sync::mpsc::Sender;
57
58use crate::access_layer::{AccessLayerRef, FilePathProvider, OperationType, RegionFilePathFactory};
59use crate::batch_size::{estimate_batch_size, largest_row_group_statistic};
60use crate::cache::file_cache::{FileCacheRef, FileType, IndexKey};
61use crate::cache::write_cache::{UploadOptions, UploadTracker, WriteCacheRef};
62use crate::cache::{CacheManagerRef, CacheStrategy};
63#[cfg(test)]
64use crate::config::{BloomFilterConfig, InvertedIndexConfig};
65use crate::error::{
66    BuildIndexAsyncSnafu, Error, ExternalSnafu, InvalidConfigSnafu, JoinSnafu,
67    OperateAbortedIndexSnafu, RegionClosedSnafu, RegionDroppedSnafu, RegionTruncatedSnafu, Result,
68};
69use crate::metrics::{
70    INDEX_ARTIFACT_CLEANUP_FAILURE_TOTAL, INDEX_CREATE_DISCARDED_TOTAL, INDEX_CREATE_MEMORY_USAGE,
71    INDEX_DISCARD_MEMORY_BUDGET_EXCEEDED, INDEX_PUBLICATION_STALE_TOTAL,
72};
73use crate::read::Batch;
74use crate::region::options::IndexOptions;
75use crate::region::version::VersionControlRef;
76use crate::region::{
77    IndexBuildSource, IndexPublication, IndexPublicationStale, ManifestContextRef,
78};
79use crate::request::{
80    BackgroundNotify, BuildIndexRequest, IndexBuildFinished, IndexBuildStopped, WorkerRequest,
81    WorkerRequestWithTime,
82};
83use crate::schedule::scheduler::{Job, SchedulerRef};
84use crate::sst::file::{
85    ColumnIndexMetadata, FileHandle, IndexType, IndexTypes, RegionFileId, RegionIndexId,
86};
87use crate::sst::file_purger::FilePurgerRef;
88#[cfg(feature = "testing")]
89use crate::sst::index::bloom_filter::creator::BloomFilterIndexer;
90#[cfg(test)]
91use crate::sst::index::fulltext_index::FULLTEXT_BLOOM_VERSION;
92use crate::sst::index::intermediate::IntermediateManager;
93#[cfg(feature = "testing")]
94use crate::sst::index::inverted_index::creator::InvertedIndexer;
95use crate::sst::parquet::SstInfo;
96use crate::worker::WorkerListener;
97
98pub(crate) const TYPE_EXTENSION_INDEX: &str = "extension";
99pub(crate) const TYPE_INVERTED_INDEX: &str = kind::IndexKind::Inverted.as_str();
100pub(crate) const TYPE_FULLTEXT_INDEX: &str = kind::IndexKind::Fulltext.as_str();
101pub(crate) const TYPE_BLOOM_FILTER_INDEX: &str = kind::IndexKind::Bloom.as_str();
102
103/// Evicts local state for a stale index artifact.
104///
105/// The remote artifact may already be referenced by another region manifest
106/// because repartitioned regions share physical SST and index paths. With GC
107/// enabled, remote deletion therefore stays in the reference-aware GC lifecycle.
108pub(crate) async fn cleanup_stale_index_caches(
109    index_id: RegionIndexId,
110    access_layer: &AccessLayerRef,
111    cache_manager: Option<&CacheManagerRef>,
112    write_cache: Option<&WriteCacheRef>,
113) {
114    if let Some(cache_manager) = cache_manager {
115        CacheStrategy::EnableAll(cache_manager.clone())
116            .evict_puffin_cache(index_id)
117            .await;
118    }
119    if let Some(write_cache) = write_cache {
120        write_cache
121            .remove(IndexKey::new(
122                index_id.region_id(),
123                index_id.file_id(),
124                FileType::Puffin(index_id.version),
125            ))
126            .await;
127    }
128    if let Err(e) = access_layer
129        .puffin_manager_factory()
130        .purge_stager(index_id)
131        .await
132    {
133        INDEX_ARTIFACT_CLEANUP_FAILURE_TOTAL.inc();
134        warn!(
135            e;
136            "Failed to purge stale index artifact from stager, region: {}, file_id: {}, index_version: {}",
137            index_id.region_id(),
138            index_id.file_id(),
139            index_id.version
140        );
141    }
142}
143
144/// Triggers background download of an index file to the local cache.
145pub(crate) fn trigger_index_background_download(
146    file_cache: Option<&FileCacheRef>,
147    file_id: &RegionIndexId,
148    file_size_hint: Option<u64>,
149    path_factory: &RegionFilePathFactory,
150    object_store: &ObjectStore,
151) {
152    if let (Some(file_cache), Some(file_size)) = (file_cache, file_size_hint) {
153        let index_key = IndexKey::new(
154            file_id.region_id(),
155            file_id.file_id(),
156            FileType::Puffin(file_id.version),
157        );
158        let remote_path = path_factory.build_index_file_path(file_id.file_id);
159        file_cache.maybe_download_background(
160            index_key,
161            remote_path,
162            object_store.clone(),
163            file_size,
164        );
165    }
166}
167
168/// One index successfully written into the enclosing Puffin file.
169#[derive(Debug, Clone, PartialEq, Eq)]
170pub struct CompletedEntry {
171    /// Indexed column. `None` is reserved for file-level built-in indexes, such as
172    /// JSON-path-only indexes; these appear only in `available_indexes`.
173    pub column_id: Option<ColumnId>,
174    /// Registered index identity.
175    pub kind: crate::sst::index::kind::IndexKind,
176    /// Fulltext bloom version published in column metadata. Extension format
177    /// versions belong to their Puffin blob properties and are not published here.
178    pub format_version: u32,
179}
180
181impl CompletedEntry {
182    fn fulltext_bloom_version(&self) -> Result<u8> {
183        u8::try_from(self.format_version).map_err(|_| {
184            InvalidConfigSnafu {
185                reason: format!("fulltext bloom version {} exceeds u8", self.format_version),
186            }
187            .build()
188        })
189    }
190
191    fn validate(&self) -> Result<()> {
192        if self.column_id.is_none() && !self.kind.is_builtin() {
193            return InvalidConfigSnafu {
194                reason: "extension index completion requires a column id".to_string(),
195            }
196            .fail();
197        }
198        if self.kind.index_type() == IndexType::FulltextIndex {
199            self.fulltext_bloom_version()?;
200        }
201        Ok(())
202    }
203}
204
205/// Output published only after the enclosing Puffin file finishes.
206#[derive(Debug, Clone, Default)]
207pub struct IndexOutput {
208    /// Size of the file.
209    pub file_size: u64,
210    /// Index version.
211    pub version: u64,
212    /// Successfully written column indexes.
213    pub entries: Vec<CompletedEntry>,
214}
215
216impl IndexOutput {
217    pub fn build_available_indexes(&self) -> IndexTypes {
218        let mut kinds = SmallVec::<[kind::IndexKind; 6]>::new();
219        for entry in &self.entries {
220            if !kinds.contains(&entry.kind) {
221                kinds.push(entry.kind);
222            }
223        }
224        kinds.sort_unstable();
225        kinds.into_iter().map(kind::IndexKind::index_type).collect()
226    }
227
228    /// Builds column metadata, rejecting values that cannot be represented on disk.
229    pub fn build_indexes(&self) -> Result<Vec<ColumnIndexMetadata>> {
230        let mut map: BTreeMap<ColumnId, ColumnIndexMetadata> = BTreeMap::new();
231        for kind in self.build_available_indexes() {
232            for entry in self
233                .entries
234                .iter()
235                .filter(|entry| entry.kind.index_type() == kind)
236            {
237                let Some(column_id) = entry.column_id else {
238                    continue;
239                };
240                let metadata = map.entry(column_id).or_default();
241                metadata.column_id = column_id;
242                metadata.created_indexes.push(kind);
243                if kind == IndexType::FulltextIndex {
244                    metadata.fulltext_bloom_version = entry.fulltext_bloom_version()?;
245                }
246            }
247        }
248        Ok(map.into_values().collect())
249    }
250}
251
252/// Executes planned creators through the existing Puffin lifecycle.
253#[derive(Default)]
254pub struct Indexer {
255    file_id: FileId,
256    region_id: RegionId,
257    physical_region_id: RegionId,
258    index_version: u64,
259    puffin_manager: Option<SstPuffinManager>,
260    write_cache_enabled: bool,
261    creators: Vec<ActiveIndex>,
262    /// Shared retained-memory ceiling for all injected creators.
263    extension_memory_limit: Option<usize>,
264    dense_pk_decoder: Option<DensePrimaryKeyCodec>,
265    intermediate_manager: Option<IntermediateManager>,
266    /// First failure retained even if best-effort execution discards a creator.
267    failure: Option<Arc<Error>>,
268    /// Optional index kinds dropped by the operation's resource policy.
269    discarded: Vec<kind::IndexKind>,
270}
271
272impl Indexer {
273    /// Reports failures even when surviving creators produced a partial result.
274    pub fn check_success(&self) -> Result<(), Arc<Error>> {
275        self.failure.clone().map_or(Ok(()), Err)
276    }
277
278    /// Index kinds intentionally discarded because their memory budget was exceeded.
279    pub fn discarded(&self) -> &[kind::IndexKind] {
280        &self.discarded
281    }
282
283    /// Whether any active creator requires decoded rows.
284    pub fn requires_row_scan(&self) -> bool {
285        self.creators
286            .iter()
287            .any(|active| active.creator.requires_row_scan())
288    }
289
290    /// Returns true if no index creator is active.
291    pub(crate) fn is_empty(&self) -> bool {
292        self.creators.is_empty()
293    }
294
295    #[cfg(test)]
296    fn has_kind(&self, kind: kind::IndexKind) -> bool {
297        self.creators
298            .iter()
299            .any(|active| active.kinds.contains(&kind))
300    }
301
302    fn push_creator(
303        &mut self,
304        kinds: Vec<kind::IndexKind>,
305        creator: Box<dyn handler::IndexCreator>,
306        memory_limit: Option<usize>,
307        label: &'static str,
308    ) {
309        self.creators.push(ActiveIndex {
310            kinds,
311            creator,
312            memory_limit,
313            memory: IndexMemoryUsage { label, bytes: 0 },
314        });
315    }
316
317    /// Wraps real creators for update-only benchmarks without a Puffin output.
318    #[cfg(feature = "testing")]
319    pub fn for_bench(
320        metadata: &RegionMetadataRef,
321        inverted_indexer: Option<InvertedIndexer>,
322        bloom_filter_indexer: Option<BloomFilterIndexer>,
323    ) -> Self {
324        let mut indexer = Self {
325            region_id: metadata.region_id,
326            ..Default::default()
327        };
328        if let Some(creator) = inverted_indexer {
329            indexer.push_creator(
330                vec![kind::IndexKind::Inverted],
331                Box::new(creator),
332                None,
333                TYPE_INVERTED_INDEX,
334            );
335        }
336        if let Some(creator) = bloom_filter_indexer {
337            indexer.push_creator(
338                vec![kind::IndexKind::Bloom],
339                Box::new(creator),
340                None,
341                TYPE_BLOOM_FILTER_INDEX,
342            );
343        }
344        indexer.prepare_dense_pk_decoder(metadata);
345        indexer
346    }
347
348    fn prepare_dense_pk_decoder(&mut self, metadata: &RegionMetadataRef) {
349        if metadata.primary_key_encoding != PrimaryKeyEncoding::Dense
350            || metadata.primary_key.is_empty()
351        {
352            return;
353        }
354        let requested: HashSet<_> = self
355            .creators
356            .iter()
357            .filter(|active| active.creator.needs_dense_pk())
358            .flat_map(|active| active.creator.indexed_targets())
359            .filter_map(|target| match target {
360                IndexTarget::ColumnId(id) => Some(id),
361                IndexTarget::JsonPath(_) => None,
362            })
363            .collect();
364        if metadata.primary_key.iter().all(|id| requested.contains(id)) {
365            self.dense_pk_decoder = Some(DensePrimaryKeyCodec::new(metadata));
366        }
367    }
368
369    fn prepare_primary_key(&self, batch: &mut Batch) -> Result<()> {
370        if let Some(codec) = &self.dense_pk_decoder {
371            batch.ensure_dense_pk_decoded(codec)?;
372        }
373        Ok(())
374    }
375
376    /// Updates creators that require rows using the borrowed batch.
377    pub async fn update(&mut self, batch: &mut Batch) {
378        self.do_update(batch).await;
379        self.enforce_memory_limits().await;
380        self.flush_mem_metrics();
381    }
382
383    /// Updates creators that require rows using the borrowed flat batch.
384    pub async fn update_flat(&mut self, batch: &RecordBatch) {
385        self.do_update_flat(batch).await;
386        self.enforce_memory_limits().await;
387        self.flush_mem_metrics();
388    }
389
390    /// Finalizes the index creation.
391    pub async fn finish(&mut self) -> IndexOutput {
392        let output = self.do_finish().await;
393        self.flush_mem_metrics();
394        output
395    }
396
397    /// Aborts the index creation.
398    pub async fn abort(&mut self) {
399        self.do_abort().await;
400        self.flush_mem_metrics();
401    }
402
403    fn record_failure(&mut self, source: BoxedError) {
404        self.failure.get_or_insert_with(|| {
405            Arc::new(
406                ExternalSnafu {
407                    context: "index creation".to_string(),
408                }
409                .into_error(source),
410            )
411        });
412    }
413
414    async fn enforce_memory_limits(&mut self) {
415        let mut position = 0;
416        let mut extension_bytes = 0usize;
417        while position < self.creators.len() {
418            let active = &self.creators[position];
419            if active.memory_limit.is_none()
420                && (active.is_builtin() || self.extension_memory_limit.is_none())
421            {
422                position += 1;
423                continue;
424            }
425            let bytes = active.creator.memory_usage();
426            let retained = extension_bytes.saturating_add(bytes);
427            let exceeds_shared_limit = !active.is_builtin()
428                && self
429                    .extension_memory_limit
430                    .is_some_and(|limit| retained > limit);
431            if exceeds_shared_limit || active.memory_limit.is_some_and(|limit| bytes > limit) {
432                warn!(
433                    "Discarding index exceeding memory budget, region_id: {}, file_id: {}, kinds: {:?}",
434                    self.region_id, self.file_id, active.kinds
435                );
436                for kind in &active.kinds {
437                    INDEX_CREATE_DISCARDED_TOTAL
438                        .with_label_values(&[kind.as_str(), INDEX_DISCARD_MEMORY_BUDGET_EXCEEDED])
439                        .inc();
440                    if !self.discarded.contains(kind) {
441                        self.discarded.push(*kind);
442                    }
443                }
444                self.abort_creator(position).await;
445            } else {
446                if !active.is_builtin() {
447                    extension_bytes = retained;
448                }
449                position += 1;
450            }
451        }
452    }
453
454    fn flush_mem_metrics(&mut self) {
455        for active in &mut self.creators {
456            let bytes = active.creator.memory_usage();
457            INDEX_CREATE_MEMORY_USAGE
458                .with_label_values(&[active.memory.label])
459                .add(bytes as i64 - active.memory.bytes as i64);
460            active.memory.bytes = bytes;
461        }
462    }
463}
464
465/// One handler's active creator and its shared memory accounting.
466struct ActiveIndex {
467    kinds: Vec<kind::IndexKind>,
468    creator: Box<dyn handler::IndexCreator>,
469    memory_limit: Option<usize>,
470    memory: IndexMemoryUsage,
471}
472
473impl ActiveIndex {
474    fn is_builtin(&self) -> bool {
475        self.kinds.iter().all(|kind| kind.is_builtin())
476    }
477}
478
479/// Releases each creator's gauge contribution on completion or cancellation.
480struct IndexMemoryUsage {
481    label: &'static str,
482    bytes: usize,
483}
484
485impl Drop for IndexMemoryUsage {
486    fn drop(&mut self) {
487        INDEX_CREATE_MEMORY_USAGE
488            .with_label_values(&[self.label])
489            .sub(self.bytes as i64);
490    }
491}
492
493#[async_trait::async_trait]
494pub trait IndexerBuilder {
495    /// Builds an indexer for the physical SST file.
496    async fn build(
497        &self,
498        region_file_id: RegionFileId,
499        index_version: u64,
500        row_group_size: Option<usize>,
501    ) -> Indexer;
502}
503#[derive(Clone)]
504pub(crate) struct IndexerBuilderImpl {
505    pub(crate) index_build_inputs: Option<build::IndexBuildInputs>,
506    pub(crate) registry: registry::IndexRegistryRef,
507    pub(crate) build_type: IndexBuildType,
508    pub(crate) metadata: RegionMetadataRef,
509    pub(crate) puffin_manager: SstPuffinManager,
510    pub(crate) write_cache_enabled: bool,
511    pub(crate) intermediate_manager: IntermediateManager,
512    pub(crate) index_options: IndexOptions,
513}
514
515impl IndexerBuilderImpl {
516    fn plan(&self) -> Arc<registry::IndexBuildPlan> {
517        if let Some(inputs) = &self.index_build_inputs {
518            return inputs.plan.clone();
519        }
520        Arc::new(self.registry.plan(&handler::PlanContext {
521            metadata: &self.metadata,
522            index_options: &self.index_options,
523            build_type: self.build_type.clone(),
524            existing_indexes: &[],
525        }))
526    }
527}
528
529#[async_trait::async_trait]
530impl IndexerBuilder for IndexerBuilderImpl {
531    /// Sanity check for arguments and create a new [Indexer] if arguments are valid.
532    async fn build(
533        &self,
534        region_file_id: RegionFileId,
535        index_version: u64,
536        row_group_size: Option<usize>,
537    ) -> Indexer {
538        let mut indexer = Indexer {
539            file_id: region_file_id.file_id(),
540            region_id: self.metadata.region_id,
541            physical_region_id: region_file_id.region_id(),
542            index_version,
543            write_cache_enabled: self.write_cache_enabled,
544            ..Default::default()
545        };
546
547        let plan = self.plan();
548        indexer.failure = plan.check_success().err();
549        let mut context = handler::BuildContext {
550            metadata: &self.metadata,
551            output: RegionIndexId::new(region_file_id, index_version),
552            puffin_manager: &self.puffin_manager,
553            intermediate_manager: &self.intermediate_manager,
554            inputs: self
555                .index_build_inputs
556                .as_ref()
557                .map_or(&[], |inputs| &inputs.inputs),
558            input_puffin_manager: self
559                .index_build_inputs
560                .as_ref()
561                .map_or(&self.puffin_manager, |inputs| &inputs.input_puffin_manager),
562            can_merge_inputs: self
563                .index_build_inputs
564                .as_ref()
565                .is_some_and(|inputs| inputs.can_merge_inputs),
566            memory_limit: self
567                .index_build_inputs
568                .as_ref()
569                .and_then(|inputs| inputs.memory_limit),
570            row_group_size,
571            output_rows: self
572                .index_build_inputs
573                .as_ref()
574                .and_then(|inputs| inputs.output_rows),
575        };
576        indexer.extension_memory_limit = context.memory_limit;
577        for planned in plan.planned() {
578            let Some(first_kind) = planned.kinds().first() else {
579                let err = BoxedError::new(
580                    InvalidConfigSnafu {
581                        reason: "index plan must own at least one kind",
582                    }
583                    .build(),
584                );
585                warn!(err; "Failed to create index, region_id: {}", self.metadata.region_id);
586                indexer.record_failure(err);
587                continue;
588            };
589            let builtin = planned.kinds().iter().all(|kind| kind.is_builtin());
590            // Optional creators share the operation allocation. Pass only the
591            // remaining retained budget before allowing another creator to build.
592            context.memory_limit = if builtin {
593                indexer.extension_memory_limit
594            } else {
595                let remaining = indexer.extension_memory_limit.map(|limit| {
596                    let retained = indexer
597                        .creators
598                        .iter()
599                        .filter(|active| !active.is_builtin())
600                        .fold(0usize, |bytes, active| {
601                            bytes.saturating_add(active.creator.memory_usage())
602                        });
603                    limit.saturating_sub(retained)
604                });
605                remaining.into_iter().chain(planned.memory_limit()).min()
606            };
607            match planned.build(&context).await {
608                Ok(Some(creator)) => {
609                    let label = if builtin {
610                        first_kind.as_str()
611                    } else {
612                        TYPE_EXTENSION_INDEX
613                    };
614                    let limit = if builtin { None } else { context.memory_limit };
615                    indexer.push_creator(planned.kinds().to_vec(), creator, limit, label);
616                    if !builtin {
617                        indexer.enforce_memory_limits().await;
618                    }
619                }
620                Ok(None) => {}
621                Err(err) => {
622                    if planned.kinds().iter().all(|kind| kind.is_builtin())
623                        && cfg!(any(test, feature = "test"))
624                    {
625                        panic!("Failed to create index: {err:?}");
626                    }
627                    warn!(err; "Failed to create index, region_id: {}, kinds: {:?}", self.metadata.region_id, planned.kinds());
628                    indexer.record_failure(err);
629                }
630            }
631        }
632        indexer.prepare_dense_pk_decoder(&self.metadata);
633        indexer.intermediate_manager = Some(self.intermediate_manager.clone());
634
635        if indexer.is_empty() {
636            indexer.abort().await;
637            return indexer;
638        }
639
640        indexer.puffin_manager = Some(self.puffin_manager.clone());
641        indexer
642    }
643}
644
645/// Builds the indexes of an SST from its encoded parquet bytes.
646///
647/// Encoded bulk parts are flushed without passing through [ParquetWriter], so their indexes
648/// are built from the bytes still in memory instead of reading the uploaded SST back. Like
649/// the writer, a failure only drops the index and leaves the SST unindexed.
650///
651/// [ParquetWriter]: crate::sst::parquet::writer::ParquetWriter
652pub(crate) async fn build_index_from_parquet_bytes(
653    builder: &IndexerBuilderImpl,
654    region_file_id: RegionFileId,
655    data: &Bytes,
656) -> IndexOutput {
657    let reader_builder = match ParquetRecordBatchReaderBuilder::try_new(data.clone()) {
658        Ok(reader_builder) => reader_builder,
659        Err(e) => {
660            warn!(e; "Failed to read encoded SST {:?} for index creation", region_file_id);
661            return IndexOutput::default();
662        }
663    };
664    let row_group_size = reader_builder
665        .metadata()
666        .row_groups()
667        .first()
668        .map(|row_group| row_group.num_rows() as usize)
669        .filter(|size| *size > 0);
670    let mut indexer = builder.build(region_file_id, 0, row_group_size).await;
671    if indexer.is_empty() {
672        return IndexOutput::default();
673    }
674    // Follow the batch size the flush reads use, so wide rows are not decoded 8192 at a time.
675    let batch_size = estimate_batch_size(largest_row_group_statistic(reader_builder.metadata()));
676
677    // TODO(dennis): decode only the columns the enabled indexers need, including the encoded
678    // primary key for tag indexes. This needs the indexer to stop locating columns by position.
679    let mut reader = match reader_builder.with_batch_size(batch_size).build() {
680        Ok(reader) => reader,
681        Err(e) => {
682            warn!(e; "Failed to read encoded SST {:?} for index creation", region_file_id);
683            indexer.abort().await;
684            return IndexOutput::default();
685        }
686    };
687    loop {
688        // Decoding is CPU-bound, so each batch is decoded on the blocking pool. Only one batch
689        // is in flight, and an aborted flush stops after the batch being decoded.
690        let decoded = common_runtime::spawn_blocking_global(move || {
691            let batch = reader.next();
692            (reader, batch)
693        })
694        .await;
695        let batch = match decoded {
696            Ok((next_reader, batch)) => {
697                reader = next_reader;
698                batch
699            }
700            Err(e) => {
701                warn!(e; "Failed to decode encoded SST {:?} for index creation", region_file_id);
702                indexer.abort().await;
703                return IndexOutput::default();
704            }
705        };
706        match batch {
707            Some(Ok(batch)) => indexer.update_flat(&batch).await,
708            Some(Err(e)) => {
709                warn!(e; "Failed to decode encoded SST {:?} for index creation", region_file_id);
710                indexer.abort().await;
711                return IndexOutput::default();
712            }
713            None => break,
714        }
715    }
716    indexer.finish().await
717}
718
719/// Type of an index build task.
720#[derive(Debug, Clone, IntoStaticStr, PartialEq)]
721pub enum IndexBuildType {
722    /// Build index when schema change.
723    SchemaChange,
724    /// Create or update index after flush.
725    Flush,
726    /// Create or update index after compact.
727    Compact,
728    /// Manually build index.
729    Manual,
730}
731
732impl IndexBuildType {
733    fn as_str(&self) -> &'static str {
734        self.into()
735    }
736
737    // Higher value means higher priority.
738    fn priority(&self) -> u8 {
739        match self {
740            IndexBuildType::Manual => 3,
741            IndexBuildType::SchemaChange => 2,
742            IndexBuildType::Flush => 1,
743            IndexBuildType::Compact => 0,
744        }
745    }
746}
747
748impl From<OperationType> for IndexBuildType {
749    fn from(op_type: OperationType) -> Self {
750        match op_type {
751            OperationType::Flush => IndexBuildType::Flush,
752            OperationType::Compact => IndexBuildType::Compact,
753        }
754    }
755}
756
757/// Outcome of an index build task.
758#[derive(Debug, Clone, PartialEq, Eq, Hash)]
759pub enum IndexBuildOutcome {
760    Finished,
761    Aborted(String),
762}
763
764/// How an index build task reports its result.
765enum IndexBuildReport {
766    /// The task sends the outcome to its waiters.
767    Done(IndexBuildOutcome),
768    /// The worker or a retry task owns the waiters, so the task must not
769    /// report to them.
770    Deferred,
771}
772
773/// Mpsc output result sender.
774pub type ResultMpscSender = Sender<Result<IndexBuildOutcome>>;
775
776#[derive(Clone)]
777pub struct IndexBuildTask {
778    /// The logical region whose manifest and in-memory version this task updates.
779    pub region_id: RegionId,
780    /// The SST file handle to build index for.
781    pub file: FileHandle,
782    /// The target region metadata used to decode rows from the SST.
783    ///
784    /// An SST may originate in another region while being visible in the target
785    /// manifest. This metadata defines the target schema and sequence domain;
786    /// applying the staging manifest only makes imported files visible. Index
787    /// rebuild happens later when a flush, compaction, schema change, or manual
788    /// index build request schedules it.
789    pub(crate) target_region_metadata: RegionMetadataRef,
790    /// The manifest state this build is based on.
791    pub(crate) source: IndexBuildSource,
792    /// Index options of the target region when the task was created.
793    pub(crate) index_options: IndexOptions,
794    pub reason: IndexBuildType,
795    pub access_layer: AccessLayerRef,
796    pub(crate) listener: WorkerListener,
797    pub(crate) manifest_ctx: ManifestContextRef,
798    pub write_cache: Option<WriteCacheRef>,
799    pub cache_manager: Option<CacheManagerRef>,
800    pub file_purger: FilePurgerRef,
801    /// When write cache is enabled, the indexer builder should be built from the write cache.
802    /// Otherwise, it should be built from the access layer.
803    /// Rebuild owns plan assembly; external implementations enter through the registry.
804    pub(crate) indexer_builder: Arc<IndexerBuilderImpl>,
805    /// Request sender to notify the region worker.
806    pub(crate) request_sender: Sender<WorkerRequestWithTime>,
807    /// Senders of callers waiting for this build. Coalesced tasks and
808    /// schema-change retries hand their senders over to the surviving task.
809    pub(crate) result_senders: Vec<ResultMpscSender>,
810}
811
812impl std::fmt::Debug for IndexBuildTask {
813    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
814        f.debug_struct("IndexBuildTask")
815            .field("region_id", &self.region_id)
816            .field("origin_region_id", &self.source.file_meta.region_id)
817            .field("file_id", &self.source.file_meta.file_id)
818            .field("schema_version", &self.source.schema_version)
819            .field("reason", &self.reason)
820            .finish()
821    }
822}
823
824impl IndexBuildTask {
825    /// Notify the caller the job is success.
826    pub async fn on_success(&self, outcome: IndexBuildOutcome) {
827        for sender in &self.result_senders {
828            let _ = sender.send(Ok(outcome.clone())).await;
829        }
830    }
831
832    /// Send index build error to waiter.
833    pub async fn on_failure(&self, err: Arc<Error>) {
834        for sender in &self.result_senders {
835            let _ = sender
836                .send(Err(err.clone()).context(BuildIndexAsyncSnafu {
837                    region_id: self.region_id,
838                }))
839                .await;
840        }
841    }
842
843    fn into_index_build_job(mut self, version_control: VersionControlRef) -> Job {
844        Box::pin(async move {
845            self.do_index_build(version_control).await;
846        })
847    }
848
849    async fn do_index_build(&mut self, version_control: VersionControlRef) {
850        self.listener
851            .on_index_build_begin(self.region_file_id())
852            .await;
853        let result = if self.reason == IndexBuildType::Compact {
854            let mut task = self.clone();
855            // Keep the scheduler slot occupied until the compact runtime finishes the build.
856            match common_runtime::spawn_compact(
857                async move { task.index_build(version_control).await },
858            )
859            .await
860            {
861                Ok(result) => result,
862                Err(err) => Err(err).context(JoinSnafu),
863            }
864        } else {
865            self.index_build(version_control).await
866        };
867        match result {
868            Ok(IndexBuildReport::Done(outcome)) => {
869                if let IndexBuildOutcome::Aborted(reason) = &outcome {
870                    debug!("Index build aborted: {}", reason);
871                }
872                self.on_success(outcome).await
873            }
874            Ok(IndexBuildReport::Deferred) => {}
875            Err(e) => {
876                warn!(
877                    e; "Index build task failed, region: {}, file_id: {}",
878                    self.region_id, self.source.file_meta.file_id,
879                );
880                self.on_failure(e.into()).await
881            }
882        }
883        self.notify_worker(BackgroundNotify::IndexBuildStopped(IndexBuildStopped {
884            file_id: self.source.file_meta.file_id,
885        }))
886        .await;
887    }
888
889    fn region_file_id(&self) -> RegionFileId {
890        RegionFileId::new(
891            self.source.file_meta.region_id,
892            self.source.file_meta.file_id,
893        )
894    }
895
896    async fn notify_worker(&self, notify: BackgroundNotify) {
897        let worker_request = WorkerRequest::Background {
898            region_id: self.region_id,
899            notify,
900        };
901        let _ = self
902            .request_sender
903            .send(WorkerRequestWithTime::new(worker_request))
904            .await;
905    }
906
907    /// Builds an aborted outcome without a listener event. Builds that create
908    /// no index stop here, and the listener observes them as neither finished
909    /// nor aborted.
910    fn aborted(&self, reason: &str) -> IndexBuildReport {
911        IndexBuildReport::Done(IndexBuildOutcome::Aborted(format!(
912            "{reason}, region: {}, file_id: {}",
913            self.region_id, self.source.file_meta.file_id
914        )))
915    }
916
917    async fn abort_and_notify(
918        &self,
919        region_file_id: RegionFileId,
920        reason: &str,
921    ) -> IndexBuildReport {
922        self.listener.on_index_build_abort(region_file_id).await;
923        self.aborted(reason)
924    }
925
926    // Checks if the SST file still exists in object store and version to avoid conflict with compaction.
927    async fn check_sst_file_exists(&self, version_control: &VersionControlRef) -> bool {
928        let file_id = self.source.file_meta.file_id;
929        let level = self.source.file_meta.level;
930        // We should check current version instead of the version when the job is created.
931        let version = version_control.current().version;
932
933        let Some(level_files) = version.ssts.levels().get(level as usize) else {
934            warn!(
935                "File id {} not found in level {} for index build, region: {}",
936                file_id, level, self.region_id
937            );
938            return false;
939        };
940
941        match level_files.files.get(&file_id) {
942            Some(handle) if !handle.is_deleted() && !handle.compacting() => {
943                // If the file's metadata is present in the current version, the physical SST file
944                // is guaranteed to exist on object store. The file purger removes the physical
945                // file only after its metadata is removed from the version.
946                true
947            }
948            _ => {
949                warn!(
950                    "File id {} not found in region version for index build, region: {}",
951                    file_id, self.region_id
952                );
953                false
954            }
955        }
956    }
957
958    async fn index_build(
959        &mut self,
960        version_control: VersionControlRef,
961    ) -> Result<IndexBuildReport> {
962        // Taken before `prepare_source` replaces the source, and used for the
963        // index files and listener events of this build.
964        let region_file_id = self.region_file_id();
965        if let ControlFlow::Break(report) = self
966            .prepare_source(region_file_id, &version_control)
967            .await?
968        {
969            return Ok(report);
970        }
971        // Metadata-only removal leaves no current index, but its version still reserves
972        // every older path until GC finishes. Never reuse those paths on a later build.
973        let file_meta = &self.source.file_meta;
974        let new_index_version = if file_meta.index_version > 0 || file_meta.exists_index() {
975            file_meta.index_version + 1
976        } else {
977            0
978        };
979        let (mut indexer, index_output) =
980            match self.create_index(region_file_id, new_index_version).await? {
981                ControlFlow::Continue(created) => created,
982                ControlFlow::Break(report) => return Ok(report),
983            };
984
985        // Check SST file existence again after building index.
986        if !self.check_sst_file_exists(&version_control).await {
987            // Calls abort to clean up index files.
988            indexer.abort().await;
989            return Ok(self
990                .abort_and_notify(region_file_id, "SST file not found during index build")
991                .await);
992        }
993
994        self.publish(region_file_id, index_output, new_index_version)
995            .await
996    }
997
998    /// Checks that the SST still exists and moves the build source to the
999    /// file's current manifest state.
1000    async fn prepare_source(
1001        &mut self,
1002        region_file_id: RegionFileId,
1003        version_control: &VersionControlRef,
1004    ) -> Result<ControlFlow<IndexBuildReport>> {
1005        // Check SST file existence before building index to avoid failure of parquet reader.
1006        if !self.check_sst_file_exists(version_control).await {
1007            return Ok(ControlFlow::Break(
1008                self.abort_and_notify(region_file_id, "SST file not found during index build")
1009                    .await,
1010            ));
1011        }
1012
1013        // A task may have waited behind another build of the same SST. Build on
1014        // the index that build published, otherwise this publication is stale.
1015        let current_file_meta = self
1016            .manifest_ctx
1017            .manifest()
1018            .await
1019            .files
1020            .get(&self.source.file_meta.file_id)
1021            .cloned();
1022        let Some(current_file_meta) = current_file_meta else {
1023            return Ok(ControlFlow::Break(
1024                self.abort_and_notify(
1025                    region_file_id,
1026                    "SST file not found in manifest during index build",
1027                )
1028                .await,
1029            ));
1030        };
1031        if matches!(
1032            self.reason,
1033            IndexBuildType::Manual | IndexBuildType::SchemaChange
1034        ) {
1035            self.indexer_builder
1036                .registry
1037                .validate_schema_kinds(&self.target_region_metadata)
1038                .with_context(|_| ExternalSnafu {
1039                    context: "validating index handlers for rebuild".to_string(),
1040                })?;
1041        }
1042        let context = handler::PlanContext {
1043            metadata: &self.target_region_metadata,
1044            index_options: &self.index_options,
1045            build_type: self.reason.clone(),
1046            existing_indexes: &current_file_meta.indexes,
1047        };
1048        let plan = Arc::new(self.indexer_builder.registry.plan(&context));
1049        plan.check_success().context(BuildIndexAsyncSnafu {
1050            region_id: self.region_id,
1051        })?;
1052        // Manual requests skip current indexes; schema changes must run their maintenance plans.
1053        if self.reason == IndexBuildType::Manual
1054            && current_file_meta.is_index_consistent_with_region(
1055                &self.target_region_metadata.column_metadatas,
1056                &self.index_options,
1057            )
1058            && plan.is_current(&current_file_meta, &context)
1059        {
1060            return Ok(ControlFlow::Break(
1061                self.abort_and_notify(region_file_id, "Index is already up to date")
1062                    .await,
1063            ));
1064        }
1065        let current_file = self.file.with_updated_index(current_file_meta.clone());
1066        let builder = Arc::make_mut(&mut self.indexer_builder);
1067        let output_rows = (current_file_meta.num_rows != 0).then_some(current_file_meta.num_rows);
1068        if let Some(inputs) = &mut builder.index_build_inputs {
1069            inputs.plan = plan;
1070            inputs.inputs = vec![current_file.clone()].into();
1071            inputs.output_rows = output_rows;
1072        } else {
1073            builder.index_build_inputs = Some(build::IndexBuildInputs {
1074                plan,
1075                inputs: vec![current_file.clone()].into(),
1076                input_puffin_manager: self.access_layer.build_puffin_manager(),
1077                can_merge_inputs: self
1078                    .target_region_metadata
1079                    .partition_expr
1080                    .as_deref()
1081                    .is_none_or(str::is_empty)
1082                    && current_file_meta.region_id == self.region_id,
1083                output_rows,
1084                memory_limit: None,
1085            });
1086        }
1087        self.file = current_file;
1088        self.source.file_meta = current_file_meta.as_ref().clone();
1089        Ok(ControlFlow::Continue(()))
1090    }
1091
1092    /// Builds a replacement index file, scanning data only when a creator needs rows.
1093    async fn create_index(
1094        &self,
1095        region_file_id: RegionFileId,
1096        new_index_version: u64,
1097    ) -> Result<ControlFlow<IndexBuildReport, (Indexer, IndexOutput)>> {
1098        let reader_builder = self
1099            .access_layer
1100            .read_sst(self.file.clone())
1101            .expected_metadata(Some(self.target_region_metadata.clone()));
1102        let plan = self.indexer_builder.plan();
1103        // Inverted segments must align with the actual SST row groups. Other
1104        // creators can decide input reuse without opening the Parquet file.
1105        let needs_row_group_size = plan
1106            .entries()
1107            .any(|entry| entry.kind.index_type() == IndexType::InvertedIndex);
1108        let mut parquet_reader = if needs_row_group_size {
1109            reader_builder.build().await?
1110        } else {
1111            None
1112        };
1113
1114        let row_group_size = parquet_reader.as_ref().and_then(|reader| {
1115            reader
1116                .parquet_metadata()
1117                .row_groups()
1118                .first()
1119                .map(|row_group| row_group.num_rows() as usize)
1120                .filter(|size| *size > 0)
1121        });
1122        let mut indexer = self
1123            .indexer_builder
1124            .build(region_file_id, new_index_version, row_group_size)
1125            .await;
1126        self.check_rebuild_indexer_or_abort(&mut indexer).await?;
1127        if indexer.is_empty() {
1128            if self.can_remove_all_indexes() {
1129                return Ok(ControlFlow::Continue((indexer, IndexOutput::default())));
1130            }
1131            return Ok(ControlFlow::Break(self.aborted("No index to build")));
1132        }
1133
1134        if indexer.requires_row_scan() && !needs_row_group_size {
1135            parquet_reader = match reader_builder.build().await {
1136                Ok(reader) => reader,
1137                Err(err) => {
1138                    indexer.abort().await;
1139                    return Err(err);
1140                }
1141            };
1142        }
1143        if indexer.requires_row_scan()
1144            && let Some(mut reader) = parquet_reader.take()
1145        {
1146            while indexer.requires_row_scan() {
1147                match reader.next_record_batch().await {
1148                    Ok(Some(batch)) => {
1149                        indexer.update_flat(&batch).await;
1150                        self.check_rebuild_indexer_or_abort(&mut indexer).await?;
1151                    }
1152                    Ok(None) => break,
1153                    Err(err) => {
1154                        indexer.abort().await;
1155                        return Err(err);
1156                    }
1157                }
1158            }
1159        }
1160        let output = indexer.finish().await;
1161        self.check_rebuild_indexer_or_abort(&mut indexer).await?;
1162        if let Err(err) = self.check_rebuild_output(&output) {
1163            indexer.abort().await;
1164            return Err(err).context(BuildIndexAsyncSnafu {
1165                region_id: self.region_id,
1166            });
1167        }
1168        if output.file_size == 0 {
1169            return Ok(ControlFlow::Break(self.aborted("No index was created")));
1170        }
1171        Ok(ControlFlow::Continue((indexer, output)))
1172    }
1173
1174    /// Aborts a failed replacement before propagating the rebuild error.
1175    async fn check_rebuild_indexer_or_abort(&self, indexer: &mut Indexer) -> Result<()> {
1176        if let Err(err) = self.check_rebuild_indexer(indexer) {
1177            indexer.abort().await;
1178            return Err(err).context(BuildIndexAsyncSnafu {
1179                region_id: self.region_id,
1180            });
1181        }
1182        Ok(())
1183    }
1184
1185    /// A budget skip may omit a new kind, but must not erase an existing one.
1186    fn check_rebuild_indexer(&self, indexer: &Indexer) -> Result<(), Arc<Error>> {
1187        indexer.check_success()?;
1188        let source = &self.source.file_meta;
1189        if indexer.discarded().iter().any(|kind| {
1190            let index_type = kind.index_type();
1191            source.available_indexes.contains(&index_type)
1192                || source
1193                    .indexes
1194                    .iter()
1195                    .any(|column| column.created_indexes.contains(&index_type))
1196        }) {
1197            warn!(
1198                "Preserving previous index generation after budget discard, region_id: {}, file_id: {}, discarded: {:?}",
1199                self.region_id,
1200                source.file_id,
1201                indexer.discarded()
1202            );
1203            return Err(Arc::new(OperateAbortedIndexSnafu.build()));
1204        }
1205        Ok(())
1206    }
1207
1208    /// A skipped creator or partial completion must not erase a requested existing extension.
1209    fn check_rebuild_output(&self, output: &IndexOutput) -> Result<(), Arc<Error>> {
1210        let source = &self.source.file_meta;
1211        for entry in self.indexer_builder.plan().entries() {
1212            if entry.kind.is_builtin() {
1213                continue;
1214            }
1215            let index_type = entry.kind.index_type();
1216            let existing_column = source.indexes.iter().any(|column| {
1217                column.column_id == entry.column_id && column.created_indexes.contains(&index_type)
1218            });
1219            // Legacy file-level descriptors do not identify individual indexed columns.
1220            let legacy_kind = source.available_indexes.contains(&index_type)
1221                && !source
1222                    .indexes
1223                    .iter()
1224                    .any(|column| column.created_indexes.contains(&index_type));
1225            if (existing_column || legacy_kind)
1226                && !output.entries.iter().any(|completed| {
1227                    completed.kind == entry.kind
1228                        && (legacy_kind || completed.column_id == Some(entry.column_id))
1229                })
1230            {
1231                warn!(
1232                    "Preserving previous index generation after incomplete rebuild, region_id: {}, file_id: {}, column_id: {}, kind: {}",
1233                    self.region_id,
1234                    source.file_id,
1235                    entry.column_id,
1236                    entry.kind.as_str()
1237                );
1238                return Err(Arc::new(OperateAbortedIndexSnafu.build()));
1239            }
1240        }
1241        Ok(())
1242    }
1243
1244    /// Disabled options may remove old indexes after handler validation.
1245    /// A failed plan is not an empty removal plan and must retain the old file.
1246    fn can_remove_all_indexes(&self) -> bool {
1247        self.source.file_meta.exists_index()
1248            && self
1249                .indexer_builder
1250                .index_build_inputs
1251                .as_ref()
1252                .is_some_and(|inputs| {
1253                    inputs.plan.check_success().is_ok() && inputs.plan.entries().next().is_none()
1254                })
1255    }
1256
1257    /// Uploads the index files and commits them to the manifest.
1258    async fn publish(
1259        &self,
1260        region_file_id: RegionFileId,
1261        index_output: IndexOutput,
1262        new_index_version: u64,
1263    ) -> Result<IndexBuildReport> {
1264        // Use the same file_id but with new version for index file.
1265        self.maybe_upload_index_file(
1266            index_output.clone(),
1267            region_file_id.file_id(),
1268            new_index_version,
1269        )
1270        .await?;
1271
1272        self.listener
1273            .on_index_build_before_manifest_commit(region_file_id)
1274            .await;
1275
1276        // The commit may have reached storage even on error, so the uploaded
1277        // index file is left for GC instead of being deleted here.
1278        match self
1279            .update_manifest(index_output, new_index_version)
1280            .await?
1281        {
1282            IndexPublication::Committed {
1283                manifest_version,
1284                file_meta,
1285            } => {
1286                self.listener
1287                    .on_index_build_manifest_committed(region_file_id)
1288                    .await;
1289                self.notify_worker(BackgroundNotify::IndexBuildFinished(IndexBuildFinished {
1290                    manifest_version,
1291                    file_meta,
1292                    result_senders: self.result_senders.clone(),
1293                }))
1294                .await;
1295                Ok(IndexBuildReport::Deferred)
1296            }
1297            IndexPublication::Stale(stale) => Ok(self
1298                .handle_stale_publication(region_file_id, stale, new_index_version)
1299                .await),
1300        }
1301    }
1302
1303    async fn handle_stale_publication(
1304        &self,
1305        region_file_id: RegionFileId,
1306        stale: IndexPublicationStale,
1307        new_index_version: u64,
1308    ) -> IndexBuildReport {
1309        INDEX_PUBLICATION_STALE_TOTAL
1310            .with_label_values(&["manifest_commit"])
1311            .inc();
1312        // If no successor publishes the same version, GC collects the
1313        // remote artifact. Repartition requires GC, while local mode
1314        // accepts this narrow orphan window when GC is disabled.
1315        cleanup_stale_index_caches(
1316            RegionIndexId::new(region_file_id, new_index_version),
1317            &self.access_layer,
1318            self.cache_manager.as_ref(),
1319            self.write_cache.as_ref(),
1320        )
1321        .await;
1322        if stale != IndexPublicationStale::SchemaChanged {
1323            return self
1324                .abort_and_notify(
1325                    region_file_id,
1326                    "Index build source changed before publication",
1327                )
1328                .await;
1329        }
1330
1331        self.listener.on_index_build_abort(region_file_id).await;
1332        // An index-unrelated schema change also invalidates the generation
1333        // fence. Retry to avoid leaving the SST unindexed; per-SST coalescing
1334        // limits this to one active and one pending task, so repeated changes
1335        // cannot storm.
1336        self.notify_worker(BackgroundNotify::IndexBuildRetry(BuildIndexRequest {
1337            region_id: self.region_id,
1338            build_type: self.reason.clone(),
1339            file_metas: vec![self.source.file_meta.clone()],
1340            result_senders: self.result_senders.clone(),
1341        }))
1342        .await;
1343        IndexBuildReport::Deferred
1344    }
1345
1346    async fn maybe_upload_index_file(
1347        &self,
1348        output: IndexOutput,
1349        index_file_id: FileId,
1350        index_version: u64,
1351    ) -> Result<()> {
1352        if output.file_size == 0 {
1353            return Ok(());
1354        }
1355        if let Some(write_cache) = &self.write_cache {
1356            let file_id = self.source.file_meta.file_id;
1357            let region_id = self.source.file_meta.region_id;
1358            let remote_store = self.access_layer.object_store();
1359            let mut upload_tracker = UploadTracker::new(region_id);
1360            let mut err = None;
1361            let puffin_key =
1362                IndexKey::new(region_id, index_file_id, FileType::Puffin(output.version));
1363            let index_id = RegionIndexId::new(RegionFileId::new(region_id, file_id), index_version);
1364            let puffin_path = RegionFilePathFactory::new(
1365                self.access_layer.table_dir().to_string(),
1366                self.access_layer.path_type(),
1367            )
1368            .build_index_file_path_with_version(index_id);
1369            if let Err(e) = write_cache
1370                // Index rebuild is background maintenance work, so its uploads
1371                // are reported as compaction uploads.
1372                .upload(
1373                    puffin_key,
1374                    &puffin_path,
1375                    remote_store,
1376                    UploadOptions {
1377                        op_type: OperationType::Compact,
1378                        write_buffer_size: crate::sst::DEFAULT_WRITE_BUFFER_SIZE,
1379                    },
1380                )
1381                .await
1382            {
1383                err = Some(e);
1384            }
1385            upload_tracker.push_uploaded_file(puffin_path);
1386            if let Some(err) = err {
1387                // Cleans index files on failure.
1388                upload_tracker
1389                    .clean(
1390                        &[SstInfo {
1391                            file_id,
1392                            index_metadata: output,
1393                            ..Default::default()
1394                        }],
1395                        &write_cache.file_cache(),
1396                        remote_store,
1397                    )
1398                    .await;
1399                return Err(err);
1400            }
1401        } else {
1402            debug!("write cache is not available, skip uploading index file");
1403        }
1404        Ok(())
1405    }
1406
1407    async fn update_manifest(
1408        &self,
1409        output: IndexOutput,
1410        new_index_version: u64,
1411    ) -> Result<IndexPublication> {
1412        let mut updated = self.source.file_meta.clone();
1413        updated.available_indexes = output.build_available_indexes();
1414        updated.indexes = output.build_indexes()?;
1415        updated.index_file_size = output.file_size;
1416        updated.index_version = new_index_version;
1417        let publication = self
1418            .manifest_ctx
1419            .update_manifest_for_index(&self.source, updated)
1420            .await?;
1421        if let IndexPublication::Committed {
1422            manifest_version, ..
1423        } = &publication
1424        {
1425            info!(
1426                "Successfully update manifest version to {}, region: {}, reason: {}",
1427                manifest_version,
1428                self.region_id,
1429                self.reason.as_str()
1430            );
1431        }
1432        Ok(publication)
1433    }
1434}
1435
1436impl PartialEq for IndexBuildTask {
1437    fn eq(&self, other: &Self) -> bool {
1438        self.reason.priority() == other.reason.priority()
1439    }
1440}
1441
1442impl Eq for IndexBuildTask {}
1443
1444impl PartialOrd for IndexBuildTask {
1445    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1446        Some(self.cmp(other))
1447    }
1448}
1449
1450impl Ord for IndexBuildTask {
1451    fn cmp(&self, other: &Self) -> Ordering {
1452        self.reason.priority().cmp(&other.reason.priority())
1453    }
1454}
1455
1456#[derive(Clone)]
1457struct PendingIndexBuild {
1458    task: IndexBuildTask,
1459    version_control: VersionControlRef,
1460}
1461
1462impl PartialEq for PendingIndexBuild {
1463    fn eq(&self, other: &Self) -> bool {
1464        self.task == other.task
1465    }
1466}
1467
1468impl Eq for PendingIndexBuild {}
1469
1470impl PartialOrd for PendingIndexBuild {
1471    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1472        Some(self.cmp(other))
1473    }
1474}
1475
1476impl Ord for PendingIndexBuild {
1477    fn cmp(&self, other: &Self) -> Ordering {
1478        self.task.cmp(&other.task)
1479    }
1480}
1481
1482impl PendingIndexBuild {
1483    /// Returns whether this task should replace another pending task for the
1484    /// same SST.
1485    fn supersedes(&self, other: &Self) -> bool {
1486        match self
1487            .task
1488            .source
1489            .schema_version
1490            .cmp(&other.task.source.schema_version)
1491        {
1492            Ordering::Greater => true,
1493            Ordering::Less => false,
1494            Ordering::Equal => match self
1495                .task
1496                .source
1497                .file_meta
1498                .index_version()
1499                .cmp(&other.task.source.file_meta.index_version())
1500            {
1501                Ordering::Greater => true,
1502                Ordering::Less => false,
1503                Ordering::Equal => self.task.reason.priority() > other.task.reason.priority(),
1504            },
1505        }
1506    }
1507}
1508
1509#[derive(Default)]
1510struct PendingIndexBuilds {
1511    tasks: HashMap<FileId, PendingIndexBuild>,
1512}
1513
1514impl PendingIndexBuilds {
1515    /// Inserts a pending task and returns the task it replaces or that is dropped.
1516    ///
1517    /// The surviving task takes over the result senders of the returned task, so
1518    /// callers waiting on either task wait for the build that actually runs.
1519    fn insert(&mut self, mut pending: PendingIndexBuild) -> Option<IndexBuildTask> {
1520        let file_id = pending.task.source.file_meta.file_id;
1521        match self.tasks.entry(file_id) {
1522            std::collections::hash_map::Entry::Vacant(entry) => {
1523                entry.insert(pending);
1524                None
1525            }
1526            std::collections::hash_map::Entry::Occupied(mut entry) => {
1527                let survivor = entry.get_mut();
1528                if pending.supersedes(survivor) {
1529                    std::mem::swap(survivor, &mut pending);
1530                }
1531                let senders = std::mem::take(&mut pending.task.result_senders);
1532                survivor.task.result_senders.extend(senders);
1533                Some(pending.task)
1534            }
1535        }
1536    }
1537
1538    fn highest_ready(&self, building_files: &HashSet<FileId>) -> Option<PendingIndexBuild> {
1539        self.tasks
1540            .values()
1541            .filter(|pending| !building_files.contains(&pending.task.source.file_meta.file_id))
1542            .max()
1543            .cloned()
1544    }
1545
1546    fn remove(&mut self, file_id: FileId) {
1547        self.tasks.remove(&file_id);
1548    }
1549
1550    fn drain(&mut self) -> impl Iterator<Item = PendingIndexBuild> {
1551        std::mem::take(&mut self.tasks).into_values()
1552    }
1553
1554    #[cfg(test)]
1555    fn len(&self) -> usize {
1556        self.tasks.len()
1557    }
1558
1559    fn is_empty(&self) -> bool {
1560        self.tasks.is_empty()
1561    }
1562}
1563
1564/// Tracks the index build status of a region scheduled by the [IndexBuildScheduler].
1565struct IndexBuildStatus {
1566    building_files: HashSet<FileId>,
1567    /// At most one coalesced pending task is kept for each SST.
1568    pending_tasks: PendingIndexBuilds,
1569    /// Whether the active builds belong to a region incarnation that has
1570    /// stopped accepting index publications.
1571    retiring: bool,
1572}
1573
1574impl IndexBuildStatus {
1575    fn new() -> Self {
1576        IndexBuildStatus {
1577            building_files: HashSet::new(),
1578            pending_tasks: PendingIndexBuilds::default(),
1579            retiring: false,
1580        }
1581    }
1582
1583    async fn fail_pending(&mut self, err: Arc<Error>) {
1584        for pending in self.pending_tasks.drain() {
1585            pending.task.on_failure(err.clone()).await;
1586        }
1587    }
1588
1589    async fn retire(&mut self, err: Arc<Error>) {
1590        self.retiring = !self.building_files.is_empty();
1591        self.fail_pending(err).await;
1592    }
1593}
1594
1595pub struct IndexBuildScheduler {
1596    /// Background job scheduler.
1597    scheduler: SchedulerRef,
1598    /// Tracks regions need to build index.
1599    region_status: HashMap<RegionId, IndexBuildStatus>,
1600    /// Limit of files allowed to build index concurrently on this worker.
1601    files_limit: usize,
1602}
1603
1604/// Manager background index build tasks of a worker.
1605impl IndexBuildScheduler {
1606    pub fn new(scheduler: SchedulerRef, files_limit: usize) -> Self {
1607        IndexBuildScheduler {
1608            scheduler,
1609            region_status: HashMap::new(),
1610            files_limit,
1611        }
1612    }
1613
1614    pub(crate) async fn schedule_build(
1615        &mut self,
1616        version_control: &VersionControlRef,
1617        task: IndexBuildTask,
1618    ) {
1619        self.schedule_build_inner(version_control, task, false)
1620            .await;
1621    }
1622
1623    /// A retry is queued before its previous task releases the active slot.
1624    pub(crate) async fn schedule_retry(
1625        &mut self,
1626        version_control: &VersionControlRef,
1627        task: IndexBuildTask,
1628    ) {
1629        self.schedule_build_inner(version_control, task, true).await;
1630    }
1631
1632    async fn schedule_build_inner(
1633        &mut self,
1634        version_control: &VersionControlRef,
1635        task: IndexBuildTask,
1636        is_retry: bool,
1637    ) {
1638        let status = self
1639            .region_status
1640            .entry(task.region_id)
1641            .or_insert_with(IndexBuildStatus::new);
1642
1643        let file_id = task.source.file_meta.file_id;
1644        let region_file_id = RegionFileId::new(
1645            task.source.file_meta.region_id,
1646            task.source.file_meta.file_id,
1647        );
1648        // A manual build must not report success while an active build that may
1649        // not create the requested indexes is still running, so it waits as well.
1650        let can_wait_for_active = is_retry
1651            || status.retiring
1652            || matches!(
1653                task.reason,
1654                IndexBuildType::SchemaChange | IndexBuildType::Manual
1655            );
1656        let (rejected, coalesced) =
1657            if status.building_files.contains(&file_id) && !can_wait_for_active {
1658                (Some(task), None)
1659            } else {
1660                let pending = PendingIndexBuild {
1661                    task,
1662                    version_control: version_control.clone(),
1663                };
1664                (None, status.pending_tasks.insert(pending))
1665            };
1666        let should_schedule = !status.retiring;
1667
1668        if let Some(rejected) = rejected {
1669            debug!(
1670                "Rejecting index build because region file {:?} is already being built",
1671                region_file_id
1672            );
1673            rejected
1674                .on_success(IndexBuildOutcome::Aborted(format!(
1675                    "Index is already being built for region file {:?}",
1676                    region_file_id
1677                )))
1678                .await;
1679            rejected.listener.on_index_build_abort(region_file_id).await;
1680        }
1681
1682        if let Some(coalesced) = coalesced {
1683            debug!(
1684                "Coalescing redundant index build task for region file {:?}",
1685                region_file_id
1686            );
1687            coalesced
1688                .on_success(IndexBuildOutcome::Aborted(format!(
1689                    "Index build was coalesced for region file {:?}",
1690                    region_file_id
1691                )))
1692                .await;
1693            coalesced
1694                .listener
1695                .on_index_build_abort(region_file_id)
1696                .await;
1697        }
1698
1699        if should_schedule {
1700            self.schedule_next_build_batch();
1701        }
1702    }
1703
1704    /// Schedule tasks until reaching the files limit or no more tasks.
1705    fn schedule_next_build_batch(&mut self) {
1706        let mut building_count = self
1707            .region_status
1708            .values()
1709            .map(|status| status.building_files.len())
1710            .sum::<usize>();
1711
1712        while building_count < self.files_limit {
1713            let Some(pending) = self.find_next_task() else {
1714                break;
1715            };
1716
1717            let task = pending.task;
1718            let region_id = task.region_id;
1719            let file_id = task.source.file_meta.file_id;
1720            let job = task.clone().into_index_build_job(pending.version_control);
1721            match self.scheduler.schedule(job) {
1722                Ok(()) => {
1723                    if let Some(status) = self.region_status.get_mut(&region_id) {
1724                        status.pending_tasks.remove(file_id);
1725                        status.building_files.insert(file_id);
1726                        building_count += 1;
1727                    } else {
1728                        error!(
1729                            "Region status not found when scheduling index build task, region: {}",
1730                            region_id
1731                        );
1732                    }
1733                }
1734                Err(err) => {
1735                    error!(
1736                        err;
1737                        "Failed to schedule index build job, region: {}, file_id: {}",
1738                        region_id,
1739                        file_id
1740                    );
1741                    if let Some(status) = self.region_status.get_mut(&region_id) {
1742                        status.pending_tasks.remove(file_id);
1743                    }
1744                    common_runtime::spawn_global(async move {
1745                        task.on_failure(Arc::new(err)).await;
1746                    });
1747                }
1748            }
1749        }
1750
1751        self.region_status.retain(|_, status| {
1752            !status.building_files.is_empty() || !status.pending_tasks.is_empty()
1753        });
1754    }
1755
1756    /// Find the ready task with the highest priority.
1757    fn find_next_task(&self) -> Option<PendingIndexBuild> {
1758        self.region_status
1759            .values()
1760            .filter(|status| !status.retiring)
1761            .filter_map(|status| status.pending_tasks.highest_ready(&status.building_files))
1762            .max()
1763    }
1764
1765    pub(crate) fn on_task_stopped(&mut self, region_id: RegionId, file_id: FileId) {
1766        if let Some(status) = self.region_status.get_mut(&region_id) {
1767            if !status.building_files.remove(&file_id) {
1768                debug!(
1769                    "Index build task is not tracked as building, region: {}, file: {}",
1770                    region_id, file_id
1771                );
1772                return;
1773            }
1774            if status.building_files.is_empty() && status.pending_tasks.is_empty() {
1775                // No more tasks for this region, remove it.
1776                self.region_status.remove(&region_id);
1777            } else if status.building_files.is_empty() {
1778                // All builds from the previous region incarnation have stopped.
1779                status.retiring = false;
1780            }
1781        }
1782
1783        self.schedule_next_build_batch();
1784    }
1785
1786    /// Notifies the scheduler that the region is dropped.
1787    pub(crate) async fn on_region_dropped(&mut self, region_id: RegionId) {
1788        self.retire_region(
1789            region_id,
1790            Arc::new(RegionDroppedSnafu { region_id }.build()),
1791        )
1792        .await;
1793    }
1794
1795    /// Notifies the scheduler that the region is closed.
1796    pub(crate) async fn on_region_closed(&mut self, region_id: RegionId) {
1797        self.retire_region(region_id, Arc::new(RegionClosedSnafu { region_id }.build()))
1798            .await;
1799    }
1800
1801    /// Notifies the scheduler that the region is truncated.
1802    pub(crate) async fn on_region_truncated(&mut self, region_id: RegionId) {
1803        self.retire_region(
1804            region_id,
1805            Arc::new(RegionTruncatedSnafu { region_id }.build()),
1806        )
1807        .await;
1808    }
1809
1810    async fn retire_region(&mut self, region_id: RegionId, err: Arc<Error>) {
1811        let Some(status) = self.region_status.get_mut(&region_id) else {
1812            return;
1813        };
1814        status.retire(err).await;
1815        if status.building_files.is_empty() {
1816            self.region_status.remove(&region_id);
1817        }
1818    }
1819}
1820
1821#[cfg(test)]
1822mod tests {
1823    use std::sync::{Arc, Mutex};
1824
1825    use api::v1::SemanticType;
1826    use common_base::readable_size::ReadableSize;
1827    use datafusion_common::HashMap;
1828    use datatypes::data_type::ConcreteDataType;
1829    use datatypes::schema::{
1830        ColumnSchema, FulltextOptions, SkippingIndexOptions, SkippingIndexType,
1831    };
1832    use datatypes::value::Value;
1833    use index::inverted_index::format::reader::InvertedIndexReader;
1834    use object_store::ObjectStore;
1835    use object_store::services::Memory;
1836    use partition::expr::col;
1837    use puffin::puffin_manager::{PuffinManager, PuffinReader};
1838    use puffin_manager::PuffinManagerFactory;
1839    use smallvec::smallvec;
1840    use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
1841    use tokio::sync::mpsc;
1842
1843    use super::*;
1844    use crate::access_layer::{FilePathProvider, Metrics, SstWriteRequest, WriteType};
1845    use crate::cache::write_cache::WriteCache;
1846    use crate::config::{FulltextIndexConfig, IndexBuildMode, MitoConfig, Mode};
1847    use crate::manifest::action::{RegionEdit, RegionMetaAction, RegionMetaActionList};
1848    use crate::memtable::time_partition::TimePartitions;
1849    use crate::region::RegionLeaderState;
1850    use crate::region::version::{VersionBuilder, VersionControl};
1851    use crate::schedule::scheduler::{LocalScheduler, Scheduler};
1852    use crate::sst::file::{FileMeta, RegionFileId};
1853    use crate::sst::file_purger::{FilePurger, NoopFilePurger};
1854    use crate::sst::location;
1855    use crate::sst::parquet::WriteOptions;
1856    use crate::test_util::memtable_util::EmptyMemtableBuilder;
1857    use crate::test_util::scheduler_util::{SchedulerEnv, VecScheduler};
1858    use crate::test_util::sst_util::{
1859        new_flat_source_from_record_batches, new_record_batch_by_range, sst_region_metadata,
1860    };
1861
1862    struct MetaConfig {
1863        with_inverted: bool,
1864        with_fulltext: bool,
1865        with_skipping_bloom: bool,
1866    }
1867
1868    /// Receives the published index that the worker applies before notifying waiters.
1869    async fn recv_index_build_finished(
1870        rx: &mut mpsc::Receiver<WorkerRequestWithTime>,
1871    ) -> IndexBuildFinished {
1872        match rx.recv().await.unwrap().request {
1873            WorkerRequest::Background {
1874                notify: BackgroundNotify::IndexBuildFinished(finished),
1875                ..
1876            } => finished,
1877            other => panic!("Expect index build finished, got: {other:?}"),
1878        }
1879    }
1880
1881    async fn seed_manifest_file(manifest_ctx: &ManifestContextRef, file_meta: &FileMeta) {
1882        manifest_ctx
1883            .update_manifest(
1884                RegionLeaderState::Writable,
1885                RegionMetaActionList::with_action(RegionMetaAction::Edit(RegionEdit {
1886                    files_to_add: vec![file_meta.clone()],
1887                    files_to_remove: Vec::new(),
1888                    timestamp_ms: None,
1889                    flushed_sequence: None,
1890                    flushed_entry_id: None,
1891                    committed_sequence: None,
1892                    compaction_time_window: None,
1893                    wal_recovery: Default::default(),
1894                })),
1895                false,
1896            )
1897            .await
1898            .unwrap();
1899    }
1900
1901    fn mock_region_metadata(
1902        MetaConfig {
1903            with_inverted,
1904            with_fulltext,
1905            with_skipping_bloom,
1906        }: MetaConfig,
1907    ) -> RegionMetadataRef {
1908        let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 2));
1909        let mut column_schema = ColumnSchema::new("a", ConcreteDataType::int64_datatype(), false);
1910        if with_inverted {
1911            column_schema = column_schema.with_inverted_index(true);
1912        }
1913        builder
1914            .push_column_metadata(ColumnMetadata {
1915                column_schema,
1916                semantic_type: SemanticType::Field,
1917                column_id: 1,
1918            })
1919            .push_column_metadata(ColumnMetadata {
1920                column_schema: ColumnSchema::new("b", ConcreteDataType::float64_datatype(), false),
1921                semantic_type: SemanticType::Field,
1922                column_id: 2,
1923            })
1924            .push_column_metadata(ColumnMetadata {
1925                column_schema: ColumnSchema::new(
1926                    "c",
1927                    ConcreteDataType::timestamp_millisecond_datatype(),
1928                    false,
1929                ),
1930                semantic_type: SemanticType::Timestamp,
1931                column_id: 3,
1932            });
1933
1934        if with_fulltext {
1935            let column_schema =
1936                ColumnSchema::new("text", ConcreteDataType::string_datatype(), true)
1937                    .with_fulltext_options(FulltextOptions {
1938                        enable: true,
1939                        ..Default::default()
1940                    })
1941                    .unwrap();
1942
1943            let column = ColumnMetadata {
1944                column_schema,
1945                semantic_type: SemanticType::Field,
1946                column_id: 4,
1947            };
1948
1949            builder.push_column_metadata(column);
1950        }
1951
1952        if with_skipping_bloom {
1953            let column_schema =
1954                ColumnSchema::new("bloom", ConcreteDataType::string_datatype(), false)
1955                    .with_skipping_options(SkippingIndexOptions::new_unchecked(
1956                        42,
1957                        0.01,
1958                        SkippingIndexType::BloomFilter,
1959                    ))
1960                    .unwrap();
1961
1962            let column = ColumnMetadata {
1963                column_schema,
1964                semantic_type: SemanticType::Field,
1965                column_id: 5,
1966            };
1967
1968            builder.push_column_metadata(column);
1969        }
1970
1971        Arc::new(builder.build().unwrap())
1972    }
1973
1974    fn mock_object_store() -> ObjectStore {
1975        ObjectStore::new(Memory::default()).unwrap()
1976    }
1977
1978    async fn mock_intm_mgr(path: impl AsRef<str>) -> IntermediateManager {
1979        IntermediateManager::init_fs(path).await.unwrap()
1980    }
1981
1982    fn random_region_file_id() -> RegionFileId {
1983        RegionFileId::new(RegionId::new(1, 2), FileId::random())
1984    }
1985
1986    struct NoopPathProvider;
1987
1988    impl FilePathProvider for NoopPathProvider {
1989        fn build_index_file_path(&self, _file_id: RegionFileId) -> String {
1990            unreachable!()
1991        }
1992
1993        fn build_index_file_path_with_version(&self, _index_id: RegionIndexId) -> String {
1994            unreachable!()
1995        }
1996
1997        fn build_sst_file_path(&self, _file_id: RegionFileId) -> String {
1998            unreachable!()
1999        }
2000    }
2001
2002    async fn mock_sst_file(
2003        metadata: RegionMetadataRef,
2004        env: &SchedulerEnv,
2005        build_mode: IndexBuildMode,
2006    ) -> SstInfo {
2007        let source = new_flat_source_from_record_batches(vec![
2008            new_record_batch_by_range(&["a", "d"], 0, 60),
2009            new_record_batch_by_range(&["b", "f"], 0, 40),
2010            new_record_batch_by_range(&["b", "h"], 100, 200),
2011        ]);
2012        let mut index_config = MitoConfig::default().index;
2013        index_config.build_mode = build_mode;
2014        let write_request = SstWriteRequest {
2015            index_build_inputs: None,
2016            op_type: OperationType::Flush,
2017            metadata: metadata.clone(),
2018            source,
2019            max_sequence: None,
2020            sst_write_format: Default::default(),
2021            cache_manager: Default::default(),
2022            preserve_row_sequence: false,
2023            index_options: IndexOptions::default(),
2024            index_config,
2025            inverted_index_config: Default::default(),
2026            fulltext_index_config: Default::default(),
2027            bloom_filter_index_config: Default::default(),
2028        };
2029        let mut metrics = Metrics::new(WriteType::Flush);
2030        env.access_layer
2031            .write_sst(write_request, &WriteOptions::default(), &mut metrics)
2032            .await
2033            .unwrap()
2034            .remove(0)
2035    }
2036
2037    async fn mock_version_control(
2038        metadata: RegionMetadataRef,
2039        file_purger: FilePurgerRef,
2040        files: HashMap<FileId, FileMeta>,
2041    ) -> VersionControlRef {
2042        let mutable = TimePartitions::new(
2043            metadata.clone(),
2044            Arc::new(EmptyMemtableBuilder::default()),
2045            0,
2046            None,
2047        );
2048        let version_builder = VersionBuilder::new(metadata, mutable)
2049            .add_files(file_purger, files.values().cloned())
2050            .build();
2051        Arc::new(VersionControl::new(version_builder))
2052    }
2053
2054    async fn mock_indexer_builder(
2055        metadata: RegionMetadataRef,
2056        env: &SchedulerEnv,
2057    ) -> Arc<IndexerBuilderImpl> {
2058        let (dir, factory) = PuffinManagerFactory::new_for_test_async("mock_indexer_builder").await;
2059        let intm_manager = mock_intm_mgr(dir.path().to_string_lossy()).await;
2060        let puffin_manager = factory.build(
2061            env.access_layer.object_store().clone(),
2062            RegionFilePathFactory::new(
2063                env.access_layer.table_dir().to_string(),
2064                env.access_layer.path_type(),
2065            ),
2066        );
2067        Arc::new(IndexerBuilderImpl {
2068            index_build_inputs: None,
2069            build_type: IndexBuildType::Flush,
2070            metadata,
2071            puffin_manager,
2072            write_cache_enabled: false,
2073            intermediate_manager: intm_manager,
2074            index_options: IndexOptions::default(),
2075            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
2076                InvertedIndexConfig::default(),
2077                FulltextIndexConfig::default(),
2078                BloomFilterConfig::default(),
2079            )),
2080        })
2081    }
2082
2083    #[tokio::test]
2084    async fn test_builtin_output_with_missing_or_failing_provider() {
2085        use datatypes::arrow::array::{
2086            Float64Array, Int64Array, StringArray, TimestampMillisecondArray, UInt8Array,
2087            UInt64Array,
2088        };
2089
2090        for failure in [
2091            None,
2092            Some(ProviderFailure::Plan),
2093            Some(ProviderFailure::Finish),
2094        ] {
2095            let (dir, factory) =
2096                PuffinManagerFactory::new_for_test_async("builtin_provider_failure").await;
2097            let mut metadata = mock_region_metadata(MetaConfig {
2098                with_inverted: true,
2099                with_fulltext: true,
2100                with_skipping_bloom: true,
2101            });
2102            Arc::make_mut(&mut metadata).column_metadatas[0]
2103                .column_schema
2104                .mut_metadata()
2105                .insert(
2106                    datatypes::schema::HLL_KEY.into(),
2107                    "{\"precision\":10}".into(),
2108                );
2109            let registry =
2110                registry::IndexRegistry::new(
2111                    Default::default(),
2112                    Default::default(),
2113                    Default::default(),
2114                )
2115                .with_provider(failure.map(|failure| {
2116                    Arc::new(FailingProvider(failure)) as handler::IndexKindProviderRef
2117                }))
2118                .unwrap();
2119            let builder = IndexerBuilderImpl {
2120                index_build_inputs: None,
2121                build_type: IndexBuildType::Compact,
2122                metadata,
2123                puffin_manager: factory.build(
2124                    mock_object_store(),
2125                    RegionFilePathFactory::new(
2126                        "builtin-provider-test/".into(),
2127                        store_api::region_request::PathType::Bare,
2128                    ),
2129                ),
2130                write_cache_enabled: false,
2131                intermediate_manager: mock_intm_mgr(dir.path().to_string_lossy()).await,
2132                index_options: IndexOptions::default(),
2133                registry: Arc::new(registry),
2134            };
2135            let mut indexer = builder.build(random_region_file_id(), 0, Some(1024)).await;
2136            let mut batch = crate::read::BatchBuilder::new(Vec::new());
2137            batch
2138                .push_field_array(1, Arc::new(Int64Array::from(vec![42])))
2139                .unwrap();
2140            batch
2141                .push_field_array(2, Arc::new(Float64Array::from(vec![1.0])))
2142                .unwrap();
2143            batch
2144                .push_field_array(4, Arc::new(StringArray::from(vec!["hello world"])))
2145                .unwrap();
2146            batch
2147                .push_field_array(5, Arc::new(StringArray::from(vec!["hello"])))
2148                .unwrap();
2149            batch
2150                .timestamps_array(Arc::new(TimestampMillisecondArray::from(vec![0])))
2151                .unwrap();
2152            batch
2153                .sequences_array(Arc::new(UInt64Array::from(vec![0])))
2154                .unwrap();
2155            batch
2156                .op_types_array(Arc::new(UInt8Array::from(vec![0])))
2157                .unwrap();
2158            indexer.update(&mut batch.build().unwrap()).await;
2159            let output = indexer.finish().await;
2160            assert_eq!(
2161                indexer.check_success().is_err(),
2162                failure.is_some(),
2163                "{failure:?}"
2164            );
2165            if failure == Some(ProviderFailure::Finish) {
2166                assert_eq!(output.file_size, 0);
2167                assert!(output.entries.is_empty());
2168            } else {
2169                assert!(output.file_size > 0, "{failure:?}");
2170                assert_eq!(
2171                    output.build_available_indexes().as_slice(),
2172                    &[
2173                        crate::sst::file::IndexType::InvertedIndex,
2174                        crate::sst::file::IndexType::FulltextIndex,
2175                        crate::sst::file::IndexType::BloomFilterIndex,
2176                    ],
2177                    "{failure:?}"
2178                );
2179                assert!(!output.build_indexes().unwrap().is_empty());
2180            }
2181        }
2182    }
2183
2184    #[rstest::rstest]
2185    #[case::batch(false)]
2186    #[case::flat(true)]
2187    #[tokio::test]
2188    async fn test_mixed_reused_and_scanning_creators(#[case] flat: bool) {
2189        use datatypes::arrow::array::{ArrayRef, Int64Array};
2190        use datatypes::vectors::{TimestampMillisecondVector, UInt8Vector, UInt64Vector};
2191        use index::inverted_index::format::reader::InvertedIndexBlobReader;
2192
2193        let (dir, factory) = PuffinManagerFactory::new_for_test_async("mixed_reuse").await;
2194        let manager = factory.build(
2195            mock_object_store(),
2196            RegionFilePathFactory::new(
2197                "mixed-reuse/".into(),
2198                store_api::region_request::PathType::Bare,
2199            ),
2200        );
2201        let builder = IndexerBuilderImpl {
2202            index_build_inputs: None,
2203            build_type: IndexBuildType::Compact,
2204            metadata: mock_region_metadata(MetaConfig {
2205                with_inverted: true,
2206                with_fulltext: false,
2207                with_skipping_bloom: false,
2208            }),
2209            puffin_manager: manager.clone(),
2210            write_cache_enabled: false,
2211            intermediate_manager: mock_intm_mgr(dir.path().to_string_lossy()).await,
2212            index_options: IndexOptions::default(),
2213            registry: Arc::new(
2214                registry::IndexRegistry::new(
2215                    Default::default(),
2216                    Default::default(),
2217                    Default::default(),
2218                )
2219                .with_provider(Some(Arc::new(FailingProvider(ProviderFailure::Reused))))
2220                .unwrap(),
2221            ),
2222        };
2223        let file = random_region_file_id();
2224        let mut indexer = builder.build(file, 0, Some(1024)).await;
2225        assert!(indexer.requires_row_scan());
2226        let values: ArrayRef = Arc::new(Int64Array::from(vec![1, 2, 3]));
2227        if flat {
2228            let batch = RecordBatch::try_from_iter([("a", values)]).unwrap();
2229            indexer.update_flat(&batch).await;
2230        } else {
2231            let mut batch = crate::read::BatchBuilder::with_required_columns(
2232                Vec::new(),
2233                Arc::new(TimestampMillisecondVector::from_vec(vec![0, 1, 2])),
2234                Arc::new(UInt64Vector::from_vec(vec![0; 3])),
2235                Arc::new(UInt8Vector::from_vec(vec![0; 3])),
2236            );
2237            batch.push_field_array(1, values).unwrap();
2238            indexer.update(&mut batch.build().unwrap()).await;
2239        }
2240        indexer.check_success().unwrap();
2241        let output = indexer.finish().await;
2242        indexer.check_success().unwrap();
2243        assert_eq!(
2244            output.build_available_indexes().as_slice(),
2245            &[IndexType::InvertedIndex, IndexType::HllIndex]
2246        );
2247        let reader = manager.reader(&RegionIndexId::new(file, 0)).await.unwrap();
2248        reader.blob("test-sketch").await.unwrap();
2249        let blob = reader.blob("greptime-inverted-index-v1").await.unwrap();
2250        let inverted = InvertedIndexBlobReader::new(blob.reader().await.unwrap());
2251        let metadata = inverted.metadata(None).await.unwrap();
2252        assert_eq!(metadata.total_row_count, 3);
2253    }
2254
2255    #[derive(Debug, Default, Clone)]
2256    struct TestProvider {
2257        build_calls: Arc<std::sync::atomic::AtomicUsize>,
2258        kind: Option<kind::ExtensionIndexKind>,
2259        received_limits: Arc<std::sync::Mutex<Vec<Option<usize>>>>,
2260        hard_limit: Option<usize>,
2261    }
2262
2263    impl handler::IndexKindProvider for TestProvider {
2264        fn handlers(&self) -> Vec<Arc<dyn handler::IndexKindHandler>> {
2265            vec![Arc::new(self.clone())]
2266        }
2267    }
2268
2269    impl handler::IndexKindHandler for TestProvider {
2270        fn validate(
2271            &self,
2272            _context: &handler::PlanContext<'_>,
2273        ) -> std::result::Result<(), common_error::ext::BoxedError> {
2274            Ok(())
2275        }
2276
2277        fn kinds(&self) -> &[kind::IndexKind] {
2278            match self.kind {
2279                Some(kind::ExtensionIndexKind::Kll) => {
2280                    &[kind::IndexKind::Extension(kind::ExtensionIndexKind::Kll)]
2281                }
2282                _ => &[kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll)],
2283            }
2284        }
2285        fn plan(
2286            &self,
2287            _context: &handler::PlanContext<'_>,
2288        ) -> std::result::Result<
2289            Option<Arc<dyn handler::PlannedIndex>>,
2290            common_error::ext::BoxedError,
2291        > {
2292            Ok(Some(Arc::new(self.clone())))
2293        }
2294    }
2295
2296    #[async_trait::async_trait]
2297    impl handler::PlannedIndex for TestProvider {
2298        fn kinds(&self) -> &[kind::IndexKind] {
2299            handler::IndexKindHandler::kinds(self)
2300        }
2301        fn entries(&self) -> &[handler::PlanEntry] {
2302            match self.kind {
2303                Some(kind::ExtensionIndexKind::Kll) => &[handler::PlanEntry {
2304                    column_id: 0,
2305                    kind: kind::IndexKind::Extension(kind::ExtensionIndexKind::Kll),
2306                    format_version: 2,
2307                }],
2308                _ => &[handler::PlanEntry {
2309                    column_id: 0,
2310                    kind: kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll),
2311                    format_version: 2,
2312                }],
2313            }
2314        }
2315        fn is_current(&self, _: &crate::sst::file::FileMeta, _: &handler::PlanContext<'_>) -> bool {
2316            false
2317        }
2318
2319        fn memory_limit(&self) -> Option<usize> {
2320            self.hard_limit
2321        }
2322
2323        async fn build(
2324            &self,
2325            context: &handler::BuildContext<'_>,
2326        ) -> std::result::Result<
2327            Option<Box<dyn handler::IndexCreator>>,
2328            common_error::ext::BoxedError,
2329        > {
2330            self.build_calls
2331                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2332            self.received_limits
2333                .lock()
2334                .unwrap()
2335                .push(context.memory_limit);
2336            Ok(Some(Box::new(TestCreator)))
2337        }
2338    }
2339
2340    #[derive(Debug)]
2341    struct TestCreator;
2342
2343    #[async_trait::async_trait]
2344    impl handler::IndexCreator for TestCreator {
2345        async fn update(
2346            &mut self,
2347            _batch: &mut Batch,
2348        ) -> std::result::Result<(), common_error::ext::BoxedError> {
2349            Ok(())
2350        }
2351
2352        async fn update_flat(
2353            &mut self,
2354            _batch: &RecordBatch,
2355        ) -> std::result::Result<(), common_error::ext::BoxedError> {
2356            Ok(())
2357        }
2358
2359        async fn finish(
2360            &mut self,
2361            _writer: &mut puffin_manager::SstPuffinWriter,
2362        ) -> std::result::Result<Vec<CompletedEntry>, common_error::ext::BoxedError> {
2363            Ok(Vec::new())
2364        }
2365
2366        async fn abort(&mut self) -> std::result::Result<(), common_error::ext::BoxedError> {
2367            Ok(())
2368        }
2369
2370        fn memory_usage(&self) -> usize {
2371            17
2372        }
2373    }
2374
2375    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
2376    enum ProviderFailure {
2377        None,
2378        Plan,
2379        EmptyKinds,
2380        Reused,
2381        Budget,
2382        Create,
2383        SkipCreate,
2384        EmptyFinish,
2385        CompletePair,
2386        PartialFinish,
2387        Update,
2388        UpdateFlat,
2389        Finish,
2390        Close,
2391    }
2392
2393    #[derive(Debug, Clone)]
2394    struct FailingProvider(ProviderFailure);
2395
2396    fn provider_test_error() -> common_error::ext::BoxedError {
2397        common_error::ext::BoxedError::new(common_error::ext::PlainError::new(
2398            "injected provider failure".into(),
2399            common_error::status_code::StatusCode::Internal,
2400        ))
2401    }
2402
2403    impl handler::IndexKindProvider for FailingProvider {
2404        fn handlers(&self) -> Vec<Arc<dyn handler::IndexKindHandler>> {
2405            vec![Arc::new(self.clone())]
2406        }
2407    }
2408
2409    impl handler::IndexKindHandler for FailingProvider {
2410        fn validate(
2411            &self,
2412            _context: &handler::PlanContext<'_>,
2413        ) -> std::result::Result<(), common_error::ext::BoxedError> {
2414            Ok(())
2415        }
2416
2417        fn kinds(&self) -> &[kind::IndexKind] {
2418            &[kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll)]
2419        }
2420        fn plan(
2421            &self,
2422            context: &handler::PlanContext<'_>,
2423        ) -> std::result::Result<
2424            Option<Arc<dyn handler::PlannedIndex>>,
2425            common_error::ext::BoxedError,
2426        > {
2427            if self.0 == ProviderFailure::Plan {
2428                return Err(provider_test_error());
2429            }
2430            if context.build_type == IndexBuildType::Flush {
2431                return Ok(None);
2432            }
2433            Ok(Some(Arc::new(self.clone())))
2434        }
2435    }
2436
2437    #[async_trait::async_trait]
2438    impl handler::PlannedIndex for FailingProvider {
2439        fn kinds(&self) -> &[kind::IndexKind] {
2440            if self.0 == ProviderFailure::EmptyKinds {
2441                return &[];
2442            }
2443            handler::IndexKindHandler::kinds(self)
2444        }
2445        fn entries(&self) -> &[handler::PlanEntry] {
2446            if matches!(
2447                self.0,
2448                ProviderFailure::CompletePair | ProviderFailure::PartialFinish
2449            ) {
2450                return &[
2451                    handler::PlanEntry {
2452                        column_id: 0,
2453                        kind: kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll),
2454                        format_version: 2,
2455                    },
2456                    handler::PlanEntry {
2457                        column_id: 1,
2458                        kind: kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll),
2459                        format_version: 2,
2460                    },
2461                ];
2462            }
2463            &[handler::PlanEntry {
2464                column_id: 0,
2465                kind: kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll),
2466                format_version: 2,
2467            }]
2468        }
2469        fn is_current(&self, _: &crate::sst::file::FileMeta, _: &handler::PlanContext<'_>) -> bool {
2470            false
2471        }
2472
2473        fn memory_limit(&self) -> Option<usize> {
2474            (self.0 == ProviderFailure::Budget).then_some(16)
2475        }
2476
2477        async fn build(
2478            &self,
2479            _context: &handler::BuildContext<'_>,
2480        ) -> std::result::Result<
2481            Option<Box<dyn handler::IndexCreator>>,
2482            common_error::ext::BoxedError,
2483        > {
2484            if self.0 == ProviderFailure::Create {
2485                return Err(provider_test_error());
2486            }
2487            if self.0 == ProviderFailure::SkipCreate {
2488                return Ok(None);
2489            }
2490            Ok(Some(Box::new(FailingCreator(self.0))))
2491        }
2492    }
2493
2494    #[derive(Debug)]
2495    struct FailingCreator(ProviderFailure);
2496
2497    #[async_trait::async_trait]
2498    impl handler::IndexCreator for FailingCreator {
2499        fn requires_row_scan(&self) -> bool {
2500            self.0 != ProviderFailure::Reused
2501        }
2502
2503        async fn update(
2504            &mut self,
2505            _batch: &mut Batch,
2506        ) -> std::result::Result<(), common_error::ext::BoxedError> {
2507            if matches!(self.0, ProviderFailure::Update | ProviderFailure::Reused) {
2508                Err(provider_test_error())
2509            } else {
2510                Ok(())
2511            }
2512        }
2513
2514        async fn update_flat(
2515            &mut self,
2516            _batch: &RecordBatch,
2517        ) -> std::result::Result<(), common_error::ext::BoxedError> {
2518            if matches!(
2519                self.0,
2520                ProviderFailure::UpdateFlat | ProviderFailure::Reused
2521            ) {
2522                Err(provider_test_error())
2523            } else {
2524                Ok(())
2525            }
2526        }
2527
2528        async fn finish(
2529            &mut self,
2530            writer: &mut puffin_manager::SstPuffinWriter,
2531        ) -> std::result::Result<Vec<CompletedEntry>, common_error::ext::BoxedError> {
2532            use puffin::puffin_manager::PuffinWriter;
2533
2534            if self.0 == ProviderFailure::EmptyFinish {
2535                return Ok(Vec::new());
2536            }
2537            writer
2538                .put_blob(
2539                    "test-sketch",
2540                    futures::io::Cursor::new(vec![1_u8, 2, 3]),
2541                    puffin::puffin_manager::PutOptions::default(),
2542                    Default::default(),
2543                )
2544                .await
2545                .map_err(common_error::ext::BoxedError::new)?;
2546            if self.0 == ProviderFailure::Finish {
2547                // Fail after writing to exercise partially written artifact handling.
2548                return Err(provider_test_error());
2549            }
2550            let mut entries = vec![CompletedEntry {
2551                column_id: Some(0),
2552                kind: kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll),
2553                format_version: 2,
2554            }];
2555            if self.0 == ProviderFailure::CompletePair {
2556                entries.push(CompletedEntry {
2557                    column_id: Some(1),
2558                    ..entries[0].clone()
2559                });
2560            }
2561            Ok(entries)
2562        }
2563
2564        async fn abort(&mut self) -> std::result::Result<(), common_error::ext::BoxedError> {
2565            Ok(())
2566        }
2567
2568        fn memory_usage(&self) -> usize {
2569            17
2570        }
2571    }
2572
2573    struct FailingIndexClose {
2574        inner: object_store::layers::mock::Writer,
2575        attempts: Arc<std::sync::atomic::AtomicUsize>,
2576    }
2577
2578    impl object_store::layers::mock::Write for FailingIndexClose {
2579        async fn write(
2580            &mut self,
2581            bytes: object_store::layers::mock::Buffer,
2582        ) -> object_store::layers::mock::Result<()> {
2583            self.inner.write(bytes).await
2584        }
2585
2586        async fn close(
2587            &mut self,
2588        ) -> object_store::layers::mock::Result<object_store::layers::mock::Metadata> {
2589            self.attempts
2590                .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
2591            Err(object_store::layers::mock::Error::new(
2592                object_store::layers::mock::ErrorKind::Unexpected,
2593                "injected Puffin close failure",
2594            ))
2595        }
2596
2597        async fn abort(&mut self) -> object_store::layers::mock::Result<()> {
2598            self.inner.abort().await
2599        }
2600    }
2601
2602    #[tokio::test]
2603    async fn test_provider_failures_do_not_publish_sketch_descriptors() {
2604        for failure in [
2605            ProviderFailure::None,
2606            ProviderFailure::Plan,
2607            ProviderFailure::EmptyKinds,
2608            ProviderFailure::Create,
2609            ProviderFailure::Update,
2610            ProviderFailure::UpdateFlat,
2611            ProviderFailure::Finish,
2612            ProviderFailure::Close,
2613        ] {
2614            let (dir, factory) = PuffinManagerFactory::new_for_test_async("provider_failure").await;
2615            let close_attempts = Arc::new(std::sync::atomic::AtomicUsize::new(0));
2616            let store = if failure == ProviderFailure::Close {
2617                let attempts = close_attempts.clone();
2618                let layer = object_store::layers::mock::MockLayerBuilder::default()
2619                    .writer_factory(Arc::new(move |_, _, inner| {
2620                        Box::new(FailingIndexClose {
2621                            inner,
2622                            attempts: attempts.clone(),
2623                        })
2624                    }))
2625                    .build()
2626                    .unwrap();
2627                mock_object_store().layer(layer)
2628            } else {
2629                mock_object_store()
2630            };
2631            let manager = factory.build(
2632                store,
2633                crate::access_layer::RegionFilePathFactory::new(
2634                    "provider-test/".into(),
2635                    store_api::region_request::PathType::Bare,
2636                ),
2637            );
2638            let builder = IndexerBuilderImpl {
2639                index_build_inputs: None,
2640                build_type: IndexBuildType::Compact,
2641                metadata: mock_region_metadata(MetaConfig {
2642                    with_inverted: false,
2643                    with_fulltext: false,
2644                    with_skipping_bloom: false,
2645                }),
2646                puffin_manager: manager,
2647                write_cache_enabled: false,
2648                intermediate_manager: mock_intm_mgr(dir.path().to_string_lossy()).await,
2649                index_options: IndexOptions::default(),
2650                registry: Arc::new(
2651                    registry::IndexRegistry::new(
2652                        InvertedIndexConfig::default(),
2653                        FulltextIndexConfig::default(),
2654                        BloomFilterConfig::default(),
2655                    )
2656                    .with_provider(Some(Arc::new(FailingProvider(failure))))
2657                    .unwrap(),
2658                ),
2659            };
2660            let mut indexer = builder.build(random_region_file_id(), 0, Some(1024)).await;
2661            if failure == ProviderFailure::Update {
2662                let mut batch = crate::read::BatchBuilder::with_required_columns(
2663                    Vec::new(),
2664                    Arc::new(datatypes::vectors::TimestampMillisecondVector::from_vec(
2665                        vec![0],
2666                    )),
2667                    Arc::new(datatypes::vectors::UInt64Vector::from_vec(vec![0])),
2668                    Arc::new(datatypes::vectors::UInt8Vector::from_vec(vec![0])),
2669                )
2670                .build()
2671                .unwrap();
2672                indexer.update(&mut batch).await;
2673            } else {
2674                indexer
2675                    .update_flat(&new_record_batch_by_range(&["a", "b"], 0, 1))
2676                    .await;
2677            }
2678            let output = indexer.finish().await;
2679            if failure == ProviderFailure::None {
2680                assert_eq!(
2681                    output
2682                        .entries
2683                        .iter()
2684                        .filter(|entry| !entry.kind.is_builtin())
2685                        .count(),
2686                    1
2687                );
2688                assert!(output.file_size > 0);
2689                assert!(output.build_indexes().unwrap().iter().any(|index| {
2690                    index.created_indexes.iter().any(|kind| {
2691                        matches!(
2692                            kind,
2693                            crate::sst::file::IndexType::HllIndex
2694                                | crate::sst::file::IndexType::KllIndex
2695                                | crate::sst::file::IndexType::MgIndex
2696                        )
2697                    })
2698                }));
2699            } else {
2700                assert!(
2701                    output.entries.iter().all(|entry| entry.kind.is_builtin()),
2702                    "{failure:?}"
2703                );
2704                assert!(output.build_indexes().unwrap().is_empty(), "{failure:?}");
2705                assert_eq!(output.file_size, 0, "{failure:?}");
2706            }
2707            assert!(indexer.creators.is_empty());
2708            assert_eq!(
2709                indexer.check_success().is_err(),
2710                failure != ProviderFailure::None
2711            );
2712            assert_eq!(
2713                close_attempts.load(std::sync::atomic::Ordering::Relaxed),
2714                usize::from(failure == ProviderFailure::Close)
2715            );
2716        }
2717    }
2718
2719    #[tokio::test]
2720    async fn test_extension_finish_failure_discards_completed_builtin_output() {
2721        for failure in [ProviderFailure::None, ProviderFailure::Finish] {
2722            let env = SchedulerEnv::new().await;
2723            let mut builder = mock_indexer_builder(Arc::new(sst_region_metadata()), &env).await;
2724            let builder = Arc::make_mut(&mut builder);
2725            builder.build_type = IndexBuildType::Compact;
2726            Arc::make_mut(&mut builder.registry)
2727                .register(Arc::new(FailingProvider(failure)))
2728                .unwrap();
2729            let mut indexer = builder.build(random_region_file_id(), 0, Some(1024)).await;
2730            // Registry order puts real built-in creators before the failing extension.
2731            assert!(indexer.creators.first().unwrap().is_builtin());
2732            assert!(!indexer.creators.last().unwrap().is_builtin());
2733            indexer
2734                .update_flat(&new_record_batch_by_range(&["a", "b"], 0, 10))
2735                .await;
2736            assert!(indexer.check_success().is_ok());
2737            let output = indexer.finish().await;
2738            if failure == ProviderFailure::None {
2739                assert!(output.entries.iter().any(|entry| entry.kind.is_builtin()));
2740                assert!(output.entries.iter().any(|entry| !entry.kind.is_builtin()));
2741                assert!(output.file_size > 0);
2742                assert!(indexer.check_success().is_ok());
2743            } else {
2744                // A shared writer cannot publish the preceding successful index alone.
2745                assert!(output.entries.is_empty());
2746                assert!(output.build_indexes().unwrap().is_empty());
2747                assert_eq!(output.file_size, 0);
2748                assert!(indexer.check_success().is_err());
2749            }
2750            assert!(indexer.creators.is_empty());
2751        }
2752    }
2753
2754    #[tokio::test]
2755    async fn test_provider_compaction_only_and_per_output() {
2756        let (dir, factory) = PuffinManagerFactory::new_for_test_async("provider_scope").await;
2757        let provider = Arc::new(TestProvider {
2758            hard_limit: Some(17),
2759            ..Default::default()
2760        });
2761        let mut builder = IndexerBuilderImpl {
2762            index_build_inputs: None,
2763            build_type: IndexBuildType::Flush,
2764            metadata: mock_region_metadata(MetaConfig {
2765                with_inverted: false,
2766                with_fulltext: false,
2767                with_skipping_bloom: false,
2768            }),
2769            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
2770            write_cache_enabled: false,
2771            intermediate_manager: mock_intm_mgr(dir.path().to_string_lossy()).await,
2772            index_options: IndexOptions::default(),
2773            registry: Arc::new(
2774                registry::IndexRegistry::new(
2775                    InvertedIndexConfig::default(),
2776                    FulltextIndexConfig::default(),
2777                    BloomFilterConfig::default(),
2778                )
2779                .with_provider(Some(provider.clone()))
2780                .unwrap(),
2781            ),
2782        };
2783        let flush = builder.build(random_region_file_id(), 0, Some(1024)).await;
2784        assert!(flush.creators.is_empty());
2785        assert_eq!(
2786            provider
2787                .build_calls
2788                .load(std::sync::atomic::Ordering::Relaxed),
2789            0
2790        );
2791
2792        builder.build_type = IndexBuildType::Compact;
2793        let mut first = builder.build(random_region_file_id(), 0, Some(1024)).await;
2794        let mut second = builder.build(random_region_file_id(), 0, Some(1024)).await;
2795        assert_eq!(
2796            provider
2797                .build_calls
2798                .load(std::sync::atomic::Ordering::Relaxed),
2799            2
2800        );
2801        assert!(!first.is_empty());
2802        assert!(!second.is_empty());
2803        assert_eq!(first.creators[0].creator.memory_usage(), 17);
2804        first.abort().await;
2805        assert!(first.creators.is_empty());
2806        assert!(!second.creators.is_empty());
2807        second.abort().await;
2808
2809        // Reject a creator at preparation when its retained state already exceeds
2810        // the caller's allocation; equality is allowed.
2811        let plan = builder.registry.plan(&handler::PlanContext {
2812            metadata: &builder.metadata,
2813            index_options: &builder.index_options,
2814            build_type: builder.build_type.clone(),
2815            existing_indexes: &[],
2816        });
2817        builder.index_build_inputs = Some(build::IndexBuildInputs {
2818            output_rows: None,
2819            plan: Arc::new(plan),
2820            inputs: Arc::from([]),
2821            input_puffin_manager: builder.puffin_manager.clone(),
2822            can_merge_inputs: false,
2823            memory_limit: Some(16),
2824        });
2825        let rejected = builder.build(random_region_file_id(), 0, Some(1024)).await;
2826        assert!(rejected.is_empty());
2827        builder.index_build_inputs.as_mut().unwrap().memory_limit = Some(17);
2828        let mut bounded = builder.build(random_region_file_id(), 0, Some(1024)).await;
2829        assert!(!bounded.creators.is_empty());
2830        bounded.creators[0].memory_limit = Some(16);
2831        bounded.enforce_memory_limits().await;
2832        bounded.flush_mem_metrics();
2833        assert!(bounded.creators.is_empty());
2834
2835        // A larger operation allocation must not relax the handler's ceiling.
2836        builder.index_build_inputs.as_mut().unwrap().memory_limit = Some(34);
2837        let mut bounded = builder.build(random_region_file_id(), 0, Some(1024)).await;
2838        assert_eq!(bounded.creators[0].memory_limit, Some(17));
2839        bounded.abort().await;
2840    }
2841
2842    #[tokio::test]
2843    async fn test_builder_passes_remaining_extension_budget() {
2844        let (dir, factory) = PuffinManagerFactory::new_for_test_async("build_budget").await;
2845        for (operation_limit, ceiling, expected, retained) in [
2846            (Some(30), None, vec![Some(30), Some(13)], 1),
2847            (Some(17), None, vec![Some(17), Some(0)], 1),
2848            (Some(30), Some(20), vec![Some(20), Some(13)], 1),
2849            (Some(40), Some(20), vec![Some(20), Some(20)], 2),
2850            (Some(16), None, vec![Some(16), Some(16)], 0),
2851            (None, None, vec![None, None], 2),
2852            (None, Some(17), vec![Some(17), Some(17)], 2),
2853        ] {
2854            let observed = Arc::new(std::sync::Mutex::new(Vec::new()));
2855            let mut registry = registry::IndexRegistry::new(
2856                InvertedIndexConfig::default(),
2857                FulltextIndexConfig::default(),
2858                BloomFilterConfig::default(),
2859            );
2860            for kind in [kind::ExtensionIndexKind::Hll, kind::ExtensionIndexKind::Kll] {
2861                registry
2862                    .register(Arc::new(TestProvider {
2863                        kind: Some(kind),
2864                        received_limits: observed.clone(),
2865                        hard_limit: ceiling,
2866                        ..Default::default()
2867                    }))
2868                    .unwrap();
2869            }
2870            let mut builder = IndexerBuilderImpl {
2871                index_build_inputs: None,
2872                build_type: IndexBuildType::Compact,
2873                metadata: mock_region_metadata(MetaConfig {
2874                    with_inverted: false,
2875                    with_fulltext: false,
2876                    with_skipping_bloom: false,
2877                }),
2878                puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
2879                write_cache_enabled: false,
2880                intermediate_manager: mock_intm_mgr(dir.path().to_string_lossy()).await,
2881                index_options: IndexOptions::default(),
2882                registry: Arc::new(registry),
2883            };
2884            let plan = builder.registry.plan(&handler::PlanContext {
2885                metadata: &builder.metadata,
2886                index_options: &builder.index_options,
2887                build_type: builder.build_type.clone(),
2888                existing_indexes: &[],
2889            });
2890            builder.index_build_inputs = Some(build::IndexBuildInputs {
2891                output_rows: None,
2892                plan: Arc::new(plan),
2893                inputs: Arc::from([]),
2894                input_puffin_manager: builder.puffin_manager.clone(),
2895                can_merge_inputs: false,
2896                memory_limit: operation_limit,
2897            });
2898            let mut indexer = builder.build(random_region_file_id(), 0, Some(1024)).await;
2899            assert_eq!(*observed.lock().unwrap(), expected);
2900            assert_eq!(indexer.creators.len(), retained);
2901            indexer.abort().await;
2902        }
2903    }
2904
2905    #[tokio::test]
2906    async fn test_extension_creators_share_memory_budget() {
2907        for (limit, expected_extensions) in [(34, 2), (33, 1), (16, 0)] {
2908            let mut indexer = Indexer {
2909                extension_memory_limit: Some(limit),
2910                ..Default::default()
2911            };
2912            // Built-in state uses its own spill budget and does not consume the
2913            // optional extension allocation. Each test creator retains 17 bytes.
2914            indexer.push_creator(
2915                vec![kind::IndexKind::Inverted],
2916                Box::new(TestCreator),
2917                None,
2918                TYPE_INVERTED_INDEX,
2919            );
2920            for kind in [kind::ExtensionIndexKind::Hll, kind::ExtensionIndexKind::Kll] {
2921                indexer.push_creator(
2922                    vec![kind::IndexKind::Extension(kind)],
2923                    Box::new(TestCreator),
2924                    Some(34),
2925                    TYPE_EXTENSION_INDEX,
2926                );
2927            }
2928            let before = INDEX_CREATE_DISCARDED_TOTAL
2929                .with_label_values(&["kll_index", "memory_budget_exceeded"])
2930                .get();
2931            indexer.enforce_memory_limits().await;
2932            assert_eq!(indexer.creators.len(), 1 + expected_extensions);
2933            assert!(indexer.check_success().is_ok());
2934            assert_eq!(indexer.discarded().len(), 2 - expected_extensions);
2935            let kinds = [
2936                kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll),
2937                kind::IndexKind::Extension(kind::ExtensionIndexKind::Kll),
2938            ];
2939            assert_eq!(indexer.discarded(), &kinds[expected_extensions..]);
2940            if expected_extensions < 2 {
2941                assert!(
2942                    INDEX_CREATE_DISCARDED_TOTAL
2943                        .with_label_values(&["kll_index", "memory_budget_exceeded"])
2944                        .get()
2945                        > before
2946                );
2947            }
2948            assert!(indexer.creators[0].is_builtin());
2949        }
2950    }
2951
2952    #[test]
2953    fn test_file_level_completion_preserves_json_only_index() {
2954        let output = IndexOutput {
2955            entries: vec![CompletedEntry {
2956                column_id: None,
2957                kind: kind::IndexKind::Inverted,
2958                format_version: 0,
2959            }],
2960            ..Default::default()
2961        };
2962        assert_eq!(
2963            output.build_available_indexes().as_slice(),
2964            &[IndexType::InvertedIndex]
2965        );
2966        assert!(output.build_indexes().unwrap().is_empty());
2967    }
2968
2969    #[test]
2970    fn test_completed_entries_preserve_all_kinds_in_order() {
2971        use crate::sst::index::kind::{ExtensionIndexKind, IndexKind};
2972
2973        let output = IndexOutput {
2974            entries: [
2975                IndexKind::Extension(ExtensionIndexKind::Mg),
2976                IndexKind::Fulltext,
2977                IndexKind::Extension(ExtensionIndexKind::Kll),
2978                IndexKind::Bloom,
2979                IndexKind::Extension(ExtensionIndexKind::Hll),
2980                IndexKind::Inverted,
2981                IndexKind::Extension(ExtensionIndexKind::Hll),
2982            ]
2983            .into_iter()
2984            .enumerate()
2985            .map(|(position, kind)| CompletedEntry {
2986                column_id: Some(if position == 6 { 2 } else { 1 }),
2987                kind,
2988                format_version: 0,
2989            })
2990            .collect(),
2991            ..Default::default()
2992        };
2993        let expected = [
2994            IndexType::InvertedIndex,
2995            IndexType::FulltextIndex,
2996            IndexType::BloomFilterIndex,
2997            IndexType::HllIndex,
2998            IndexType::KllIndex,
2999            IndexType::MgIndex,
3000        ];
3001        let available = output.build_available_indexes();
3002        assert_eq!(available.as_slice(), &expected);
3003        assert!(!available.spilled());
3004        let columns = output.build_indexes().unwrap();
3005        assert_eq!(columns.len(), 2);
3006        assert_eq!(columns[0].created_indexes.as_slice(), &expected);
3007        assert_eq!(
3008            columns[1].created_indexes.as_slice(),
3009            &[IndexType::HllIndex]
3010        );
3011    }
3012
3013    #[test]
3014    fn test_completed_entries_preserve_builtin_metadata() {
3015        use crate::sst::index::kind::IndexKind;
3016
3017        // Deliberately reverse creator order: the published builtin order is contractual.
3018        let output = IndexOutput {
3019            entries: vec![
3020                CompletedEntry {
3021                    column_id: Some(2),
3022                    kind: IndexKind::Bloom,
3023                    format_version: 0,
3024                },
3025                CompletedEntry {
3026                    column_id: Some(2),
3027                    kind: IndexKind::Fulltext,
3028                    format_version: u32::from(FULLTEXT_BLOOM_VERSION),
3029                },
3030                CompletedEntry {
3031                    column_id: Some(2),
3032                    kind: IndexKind::Inverted,
3033                    format_version: 0,
3034                },
3035                CompletedEntry {
3036                    column_id: Some(3),
3037                    kind: IndexKind::Fulltext,
3038                    format_version: 0,
3039                },
3040            ],
3041            ..Default::default()
3042        };
3043        let expected_kinds = vec![
3044            IndexType::InvertedIndex,
3045            IndexType::FulltextIndex,
3046            IndexType::BloomFilterIndex,
3047        ];
3048        assert_eq!(
3049            output.build_available_indexes().as_slice(),
3050            expected_kinds.as_slice()
3051        );
3052        let actual = output.build_indexes().unwrap();
3053        assert_eq!(
3054            actual,
3055            vec![
3056                ColumnIndexMetadata {
3057                    column_id: 2,
3058                    created_indexes: expected_kinds.into(),
3059                    fulltext_bloom_version: FULLTEXT_BLOOM_VERSION,
3060                },
3061                ColumnIndexMetadata {
3062                    column_id: 3,
3063                    created_indexes: vec![IndexType::FulltextIndex].into(),
3064                    fulltext_bloom_version: 0,
3065                },
3066            ]
3067        );
3068    }
3069
3070    #[test]
3071    fn test_build_indexes_rejects_unrepresentable_fulltext_version() {
3072        let mut output = IndexOutput {
3073            entries: vec![CompletedEntry {
3074                column_id: Some(7),
3075                kind: kind::IndexKind::Fulltext,
3076                format_version: u8::MAX.into(),
3077            }],
3078            ..Default::default()
3079        };
3080        assert_eq!(
3081            output.build_indexes().unwrap()[0].fulltext_bloom_version,
3082            u8::MAX
3083        );
3084        output.entries[0].format_version += 1;
3085        let error = output.build_indexes().unwrap_err();
3086        assert!(
3087            error
3088                .to_string()
3089                .contains("fulltext bloom version 256 exceeds u8")
3090        );
3091    }
3092
3093    #[test]
3094    fn test_completed_entry_validation() {
3095        let mut entry = CompletedEntry {
3096            column_id: Some(1),
3097            kind: kind::IndexKind::Fulltext,
3098            format_version: u8::MAX.into(),
3099        };
3100        assert!(entry.validate().is_ok());
3101        entry.format_version += 1;
3102        assert!(entry.validate().is_err());
3103        entry.kind = kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll);
3104        assert!(entry.validate().is_ok()); // Extension versions belong to Puffin.
3105        entry.column_id = None;
3106        assert!(entry.validate().is_err());
3107    }
3108
3109    #[tokio::test]
3110    async fn test_build_indexer_basic() {
3111        let (dir, factory) =
3112            PuffinManagerFactory::new_for_test_async("test_build_indexer_basic_").await;
3113        let intm_manager = mock_intm_mgr(dir.path().to_string_lossy()).await;
3114
3115        let metadata = mock_region_metadata(MetaConfig {
3116            with_inverted: true,
3117            with_fulltext: true,
3118            with_skipping_bloom: true,
3119        });
3120        let indexer = IndexerBuilderImpl {
3121            index_build_inputs: None,
3122            build_type: IndexBuildType::Flush,
3123            metadata,
3124            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
3125            write_cache_enabled: false,
3126            intermediate_manager: intm_manager,
3127            index_options: IndexOptions::default(),
3128            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
3129                InvertedIndexConfig::default(),
3130                FulltextIndexConfig::default(),
3131                BloomFilterConfig::default(),
3132            )),
3133        }
3134        .build(random_region_file_id(), 0, Some(1024))
3135        .await;
3136
3137        assert!(indexer.has_kind(kind::IndexKind::Inverted));
3138        assert!(indexer.has_kind(kind::IndexKind::Fulltext));
3139        assert!(indexer.has_kind(kind::IndexKind::Bloom));
3140    }
3141
3142    #[tokio::test]
3143    async fn test_build_indexer_disable_create() {
3144        let (dir, factory) =
3145            PuffinManagerFactory::new_for_test_async("test_build_indexer_disable_create_").await;
3146        let intm_manager = mock_intm_mgr(dir.path().to_string_lossy()).await;
3147
3148        let metadata = mock_region_metadata(MetaConfig {
3149            with_inverted: true,
3150            with_fulltext: true,
3151            with_skipping_bloom: true,
3152        });
3153        let indexer = IndexerBuilderImpl {
3154            index_build_inputs: None,
3155            build_type: IndexBuildType::Flush,
3156            metadata: metadata.clone(),
3157            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
3158            write_cache_enabled: false,
3159            intermediate_manager: intm_manager.clone(),
3160            index_options: IndexOptions::default(),
3161            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
3162                InvertedIndexConfig {
3163                    create_on_flush: Mode::Disable,
3164                    ..Default::default()
3165                },
3166                FulltextIndexConfig::default(),
3167                BloomFilterConfig::default(),
3168            )),
3169        }
3170        .build(random_region_file_id(), 0, Some(1024))
3171        .await;
3172
3173        assert!(!indexer.has_kind(kind::IndexKind::Inverted));
3174        assert!(indexer.has_kind(kind::IndexKind::Fulltext));
3175        assert!(indexer.has_kind(kind::IndexKind::Bloom));
3176
3177        let indexer = IndexerBuilderImpl {
3178            index_build_inputs: None,
3179            build_type: IndexBuildType::Compact,
3180            metadata: metadata.clone(),
3181            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
3182            write_cache_enabled: false,
3183            intermediate_manager: intm_manager.clone(),
3184            index_options: IndexOptions::default(),
3185            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
3186                InvertedIndexConfig::default(),
3187                FulltextIndexConfig {
3188                    create_on_compaction: Mode::Disable,
3189                    ..Default::default()
3190                },
3191                BloomFilterConfig::default(),
3192            )),
3193        }
3194        .build(random_region_file_id(), 0, Some(1024))
3195        .await;
3196
3197        assert!(indexer.has_kind(kind::IndexKind::Inverted));
3198        assert!(!indexer.has_kind(kind::IndexKind::Fulltext));
3199        assert!(indexer.has_kind(kind::IndexKind::Bloom));
3200
3201        let indexer = IndexerBuilderImpl {
3202            index_build_inputs: None,
3203            build_type: IndexBuildType::Compact,
3204            metadata,
3205            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
3206            write_cache_enabled: false,
3207            intermediate_manager: intm_manager,
3208            index_options: IndexOptions::default(),
3209            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
3210                InvertedIndexConfig::default(),
3211                FulltextIndexConfig::default(),
3212                BloomFilterConfig {
3213                    create_on_compaction: Mode::Disable,
3214                    ..Default::default()
3215                },
3216            )),
3217        }
3218        .build(random_region_file_id(), 0, Some(1024))
3219        .await;
3220
3221        assert!(indexer.has_kind(kind::IndexKind::Inverted));
3222        assert!(indexer.has_kind(kind::IndexKind::Fulltext));
3223        assert!(!indexer.has_kind(kind::IndexKind::Bloom));
3224    }
3225
3226    #[tokio::test]
3227    async fn test_build_indexer_no_required() {
3228        let (dir, factory) =
3229            PuffinManagerFactory::new_for_test_async("test_build_indexer_no_required_").await;
3230        let intm_manager = mock_intm_mgr(dir.path().to_string_lossy()).await;
3231
3232        let metadata = mock_region_metadata(MetaConfig {
3233            with_inverted: false,
3234            with_fulltext: true,
3235            with_skipping_bloom: true,
3236        });
3237        let indexer = IndexerBuilderImpl {
3238            index_build_inputs: None,
3239            build_type: IndexBuildType::Flush,
3240            metadata: metadata.clone(),
3241            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
3242            write_cache_enabled: false,
3243            intermediate_manager: intm_manager.clone(),
3244            index_options: IndexOptions::default(),
3245            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
3246                InvertedIndexConfig::default(),
3247                FulltextIndexConfig::default(),
3248                BloomFilterConfig::default(),
3249            )),
3250        }
3251        .build(random_region_file_id(), 0, Some(1024))
3252        .await;
3253
3254        assert!(!indexer.has_kind(kind::IndexKind::Inverted));
3255        assert!(indexer.has_kind(kind::IndexKind::Fulltext));
3256        assert!(indexer.has_kind(kind::IndexKind::Bloom));
3257
3258        let metadata = mock_region_metadata(MetaConfig {
3259            with_inverted: true,
3260            with_fulltext: false,
3261            with_skipping_bloom: true,
3262        });
3263        let indexer = IndexerBuilderImpl {
3264            index_build_inputs: None,
3265            build_type: IndexBuildType::Flush,
3266            metadata: metadata.clone(),
3267            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
3268            write_cache_enabled: false,
3269            intermediate_manager: intm_manager.clone(),
3270            index_options: IndexOptions::default(),
3271            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
3272                InvertedIndexConfig::default(),
3273                FulltextIndexConfig::default(),
3274                BloomFilterConfig::default(),
3275            )),
3276        }
3277        .build(random_region_file_id(), 0, Some(1024))
3278        .await;
3279
3280        assert!(indexer.has_kind(kind::IndexKind::Inverted));
3281        assert!(!indexer.has_kind(kind::IndexKind::Fulltext));
3282        assert!(indexer.has_kind(kind::IndexKind::Bloom));
3283
3284        let metadata = mock_region_metadata(MetaConfig {
3285            with_inverted: true,
3286            with_fulltext: true,
3287            with_skipping_bloom: false,
3288        });
3289        let indexer = IndexerBuilderImpl {
3290            index_build_inputs: None,
3291            build_type: IndexBuildType::Flush,
3292            metadata: metadata.clone(),
3293            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
3294            write_cache_enabled: false,
3295            intermediate_manager: intm_manager,
3296            index_options: IndexOptions::default(),
3297            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
3298                InvertedIndexConfig::default(),
3299                FulltextIndexConfig::default(),
3300                BloomFilterConfig::default(),
3301            )),
3302        }
3303        .build(random_region_file_id(), 0, Some(1024))
3304        .await;
3305
3306        assert!(indexer.has_kind(kind::IndexKind::Inverted));
3307        assert!(indexer.has_kind(kind::IndexKind::Fulltext));
3308        assert!(!indexer.has_kind(kind::IndexKind::Bloom));
3309    }
3310
3311    #[tokio::test]
3312    async fn test_build_indexer_zero_row_group_hint() {
3313        let (dir, factory) =
3314            PuffinManagerFactory::new_for_test_async("test_build_indexer_zero_row_group_").await;
3315        let intm_manager = mock_intm_mgr(dir.path().to_string_lossy()).await;
3316
3317        let metadata = mock_region_metadata(MetaConfig {
3318            with_inverted: true,
3319            with_fulltext: true,
3320            with_skipping_bloom: true,
3321        });
3322        let indexer = IndexerBuilderImpl {
3323            index_build_inputs: None,
3324            build_type: IndexBuildType::Flush,
3325            metadata,
3326            puffin_manager: factory.build(mock_object_store(), NoopPathProvider),
3327            write_cache_enabled: false,
3328            intermediate_manager: intm_manager,
3329            index_options: IndexOptions::default(),
3330            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
3331                InvertedIndexConfig::default(),
3332                FulltextIndexConfig::default(),
3333                BloomFilterConfig::default(),
3334            )),
3335        }
3336        .build(random_region_file_id(), 0, Some(0))
3337        .await;
3338
3339        assert!(indexer.has_kind(kind::IndexKind::Inverted));
3340    }
3341
3342    #[tokio::test]
3343    async fn test_index_build_task_sst_not_exist() {
3344        let env = SchedulerEnv::new().await;
3345        let (tx, _rx) = mpsc::channel(4);
3346        let (result_tx, mut result_rx) = mpsc::channel::<Result<IndexBuildOutcome>>(4);
3347        let mut scheduler = env.mock_index_build_scheduler(4);
3348        let metadata = Arc::new(sst_region_metadata());
3349        let manifest_ctx = env.mock_manifest_context(metadata.clone()).await;
3350        let file_purger = Arc::new(NoopFilePurger {});
3351        let files = HashMap::new();
3352        let version_control =
3353            mock_version_control(metadata.clone(), file_purger.clone(), files).await;
3354        let region_id = metadata.region_id;
3355        let indexer_builder = mock_indexer_builder(metadata, &env).await;
3356
3357        let file_meta = FileMeta {
3358            region_id,
3359            file_id: FileId::random(),
3360            file_size: 100,
3361            ..Default::default()
3362        };
3363
3364        let file = FileHandle::new(file_meta.clone(), file_purger.clone());
3365
3366        // Create mock task.
3367        let task = IndexBuildTask {
3368            region_id,
3369            file,
3370            target_region_metadata: version_control.current().version.metadata.clone(),
3371            source: IndexBuildSource::new(
3372                file_meta,
3373                version_control.current().version.metadata.schema_version,
3374            ),
3375            reason: IndexBuildType::Flush,
3376            access_layer: env.access_layer.clone(),
3377            listener: WorkerListener::default(),
3378            manifest_ctx,
3379            write_cache: None,
3380            cache_manager: None,
3381            file_purger,
3382            indexer_builder,
3383            request_sender: tx,
3384            index_options: IndexOptions::default(),
3385            result_senders: vec![result_tx],
3386        };
3387
3388        // Schedule the build task and check result.
3389        scheduler.schedule_build(&version_control, task).await;
3390        match result_rx.recv().await.unwrap() {
3391            Ok(outcome) => {
3392                if outcome == IndexBuildOutcome::Finished {
3393                    panic!("Expect aborted result due to missing SST file")
3394                }
3395            }
3396            _ => panic!("Expect aborted result due to missing SST file"),
3397        }
3398    }
3399
3400    #[tokio::test]
3401    async fn test_index_build_task_foreign_file_uses_target_metadata() {
3402        let env = SchedulerEnv::new().await;
3403        let mut scheduler = env.mock_index_build_scheduler(4);
3404        let source_metadata = Arc::new(sst_region_metadata());
3405        let mut target_metadata = (*source_metadata).clone();
3406        target_metadata.region_id = RegionId::new(1, 3);
3407        let mut target_builder = RegionMetadataBuilder::new(target_metadata.region_id);
3408        for mut column_metadata in target_metadata.column_metadatas.clone() {
3409            if column_metadata.column_id == 2 {
3410                column_metadata.column_schema =
3411                    column_metadata.column_schema.with_inverted_index(true);
3412            }
3413            target_builder.push_column_metadata(column_metadata);
3414        }
3415        let partition_expr = col("field_0")
3416            .gt_eq(Value::UInt64(100))
3417            .and(col("field_0").lt(Value::UInt64(200)));
3418        target_builder
3419            .primary_key(target_metadata.primary_key.clone())
3420            .partition_expr_json(Some(partition_expr.as_json_str().unwrap()))
3421            .bump_version();
3422        let target_metadata = Arc::new(target_builder.build().unwrap());
3423        let manifest_ctx = env.mock_manifest_context(target_metadata.clone()).await;
3424        let region_id = target_metadata.region_id;
3425        let file_purger = Arc::new(NoopFilePurger {});
3426        let sst_info = mock_sst_file(source_metadata.clone(), &env, IndexBuildMode::Async).await;
3427        let file_meta = FileMeta {
3428            region_id: source_metadata.region_id,
3429            file_id: sst_info.file_id,
3430            file_size: sst_info.file_size,
3431            max_row_group_uncompressed_size: sst_info.max_row_group_uncompressed_size,
3432            available_indexes: smallvec![IndexType::InvertedIndex],
3433            indexes: vec![ColumnIndexMetadata {
3434                column_id: 2,
3435                created_indexes: smallvec::smallvec![IndexType::InvertedIndex],
3436                ..Default::default()
3437            }],
3438            // Old manifests may publish an index without recording its size.
3439            index_file_size: 0,
3440            index_version: 0,
3441            num_rows: sst_info.num_rows as u64,
3442            num_row_groups: sst_info.num_row_groups,
3443            ..Default::default()
3444        };
3445        seed_manifest_file(&manifest_ctx, &file_meta).await;
3446        let files = HashMap::from([(file_meta.file_id, file_meta.clone())]);
3447        let version_control =
3448            mock_version_control(target_metadata.clone(), file_purger.clone(), files).await;
3449        let indexer_builder = mock_indexer_builder(target_metadata.clone(), &env).await;
3450
3451        let file = FileHandle::new(file_meta.clone(), file_purger.clone());
3452
3453        // Create mock task.
3454        let (tx, mut rx) = mpsc::channel(4);
3455        let (result_tx, _result_rx) = mpsc::channel::<Result<IndexBuildOutcome>>(4);
3456        let task = IndexBuildTask {
3457            region_id,
3458            file,
3459            target_region_metadata: version_control.current().version.metadata.clone(),
3460            source: IndexBuildSource::new(
3461                file_meta.clone(),
3462                version_control.current().version.metadata.schema_version,
3463            ),
3464            reason: IndexBuildType::Flush,
3465            access_layer: env.access_layer.clone(),
3466            listener: WorkerListener::default(),
3467            manifest_ctx,
3468            write_cache: None,
3469            cache_manager: None,
3470            file_purger,
3471            indexer_builder,
3472            request_sender: tx,
3473            index_options: IndexOptions::default(),
3474            result_senders: vec![result_tx],
3475        };
3476
3477        scheduler.schedule_build(&version_control, task).await;
3478
3479        // A notification should be sent to the worker to update the manifest.
3480        let worker_req = rx.recv().await.unwrap().request;
3481        match worker_req {
3482            WorkerRequest::Background {
3483                region_id: req_region_id,
3484                notify: BackgroundNotify::IndexBuildFinished(finished),
3485            } => {
3486                assert_eq!(req_region_id, region_id);
3487                let updated_meta = &finished.file_meta;
3488
3489                // The mock indexer builder creates all index types.
3490                assert!(!updated_meta.available_indexes.is_empty());
3491                assert!(updated_meta.index_file_size > 0);
3492                assert_eq!(updated_meta.file_id, file_meta.file_id);
3493                assert_eq!(updated_meta.index_version, 1);
3494                // This built-in-only rebuild must not advertise extension indexes.
3495                assert!(
3496                    updated_meta
3497                        .indexes
3498                        .iter()
3499                        .all(|index| index.created_indexes.iter().all(|kind| !matches!(
3500                            kind,
3501                            crate::sst::file::IndexType::HllIndex
3502                                | crate::sst::file::IndexType::KllIndex
3503                                | crate::sst::file::IndexType::MgIndex
3504                        )))
3505                );
3506                let field_0_index = updated_meta
3507                    .indexes
3508                    .iter()
3509                    .find(|index| index.column_id == 2)
3510                    .expect("field_0 should have an inverted index");
3511                assert_eq!(
3512                    field_0_index.created_indexes.as_slice(),
3513                    [IndexType::InvertedIndex]
3514                );
3515            }
3516            _ => panic!("Unexpected worker request: {:?}", worker_req),
3517        }
3518
3519        let puffin_reader = env
3520            .access_layer
3521            .build_puffin_manager()
3522            .reader(&RegionIndexId::new(
3523                RegionFileId::new(source_metadata.region_id, file_meta.file_id),
3524                1,
3525            ))
3526            .await
3527            .unwrap();
3528        let blob = puffin_reader
3529            .blob(inverted_index::INDEX_BLOB_TYPE)
3530            .await
3531            .unwrap();
3532        let blob_reader = blob.reader().await.unwrap();
3533        let index_metadata =
3534            index::inverted_index::format::reader::InvertedIndexBlobReader::new(blob_reader)
3535                .metadata(None)
3536                .await
3537                .unwrap();
3538        assert!(index_metadata.metas.contains_key("2"));
3539        assert_eq!(index_metadata.total_row_count, 100);
3540    }
3541
3542    async fn schedule_index_build_task_with_mode(build_mode: IndexBuildMode) {
3543        let env = SchedulerEnv::new().await;
3544        let mut scheduler = env.mock_index_build_scheduler(4);
3545        let metadata = Arc::new(sst_region_metadata());
3546        let manifest_ctx = env.mock_manifest_context(metadata.clone()).await;
3547        let file_purger = Arc::new(NoopFilePurger {});
3548        let region_id = metadata.region_id;
3549        let sst_info = mock_sst_file(metadata.clone(), &env, build_mode.clone()).await;
3550        let file_meta = FileMeta {
3551            region_id,
3552            file_id: sst_info.file_id,
3553            file_size: sst_info.file_size,
3554            max_row_group_uncompressed_size: sst_info.max_row_group_uncompressed_size,
3555            index_file_size: sst_info.index_metadata.file_size,
3556            num_rows: sst_info.num_rows as u64,
3557            num_row_groups: sst_info.num_row_groups,
3558            ..Default::default()
3559        };
3560        seed_manifest_file(&manifest_ctx, &file_meta).await;
3561        let files = HashMap::from([(file_meta.file_id, file_meta.clone())]);
3562        let version_control =
3563            mock_version_control(metadata.clone(), file_purger.clone(), files).await;
3564        let indexer_builder = mock_indexer_builder(metadata.clone(), &env).await;
3565
3566        let file = FileHandle::new(file_meta.clone(), file_purger.clone());
3567
3568        // Create mock task.
3569        let (tx, mut rx) = mpsc::channel(4);
3570        let (result_tx, _result_rx) = mpsc::channel::<Result<IndexBuildOutcome>>(4);
3571        let task = IndexBuildTask {
3572            region_id,
3573            file,
3574            target_region_metadata: version_control.current().version.metadata.clone(),
3575            source: IndexBuildSource::new(
3576                file_meta.clone(),
3577                version_control.current().version.metadata.schema_version,
3578            ),
3579            reason: IndexBuildType::Flush,
3580            access_layer: env.access_layer.clone(),
3581            listener: WorkerListener::default(),
3582            manifest_ctx,
3583            write_cache: None,
3584            cache_manager: None,
3585            file_purger,
3586            indexer_builder,
3587            request_sender: tx,
3588            index_options: IndexOptions::default(),
3589            result_senders: vec![result_tx],
3590        };
3591
3592        scheduler.schedule_build(&version_control, task).await;
3593
3594        let puffin_path = location::index_file_path(
3595            env.access_layer.table_dir(),
3596            RegionIndexId::new(RegionFileId::new(region_id, file_meta.file_id), 0),
3597            env.access_layer.path_type(),
3598        );
3599
3600        if build_mode == IndexBuildMode::Async {
3601            // The index file should not exist before the task finishes.
3602            assert!(
3603                !env.access_layer
3604                    .object_store()
3605                    .exists(&puffin_path)
3606                    .await
3607                    .unwrap()
3608            );
3609        } else {
3610            // The index file should exist before the task finishes.
3611            assert!(
3612                env.access_layer
3613                    .object_store()
3614                    .exists(&puffin_path)
3615                    .await
3616                    .unwrap()
3617            );
3618        }
3619
3620        // The worker is notified to apply the published index.
3621        recv_index_build_finished(&mut rx).await;
3622
3623        // The index file should exist after the task finishes.
3624        assert!(
3625            env.access_layer
3626                .object_store()
3627                .exists(&puffin_path)
3628                .await
3629                .unwrap()
3630        );
3631    }
3632
3633    #[tokio::test]
3634    async fn test_index_build_task_build_mode() {
3635        schedule_index_build_task_with_mode(IndexBuildMode::Async).await;
3636        schedule_index_build_task_with_mode(IndexBuildMode::Sync).await;
3637    }
3638
3639    #[derive(Debug, Default)]
3640    struct RecordingRebuildPurger(Mutex<Vec<FileMeta>>);
3641
3642    impl FilePurger for RecordingRebuildPurger {
3643        fn purge_file(&self, meta: FileMeta) {
3644            self.0.lock().unwrap().push(meta);
3645        }
3646    }
3647
3648    #[rstest::rstest]
3649    #[case(false)]
3650    #[case(true)]
3651    #[tokio::test]
3652    async fn test_rebuild_metadata_refresh_retains_sst_lifetime(#[case] unknown_rows: bool) {
3653        let env = SchedulerEnv::new().await;
3654        let metadata = Arc::new(sst_region_metadata());
3655        let manifest_ctx = env.mock_manifest_context(metadata.clone()).await;
3656        let purger = Arc::new(RecordingRebuildPurger::default());
3657        let info = mock_sst_file(metadata.clone(), &env, IndexBuildMode::Async).await;
3658        let meta = FileMeta {
3659            region_id: metadata.region_id,
3660            file_id: info.file_id,
3661            file_size: info.file_size,
3662            num_rows: if unknown_rows {
3663                0
3664            } else {
3665                info.num_rows as u64
3666            },
3667            num_row_groups: info.num_row_groups,
3668            ..Default::default()
3669        };
3670        let mut refreshed = meta.clone();
3671        refreshed.index_version = 1;
3672        seed_manifest_file(&manifest_ctx, &refreshed).await;
3673        let version = mock_version_control(
3674            metadata.clone(),
3675            Arc::new(NoopFilePurger),
3676            HashMap::from([(meta.file_id, refreshed.clone())]),
3677        )
3678        .await;
3679        let original = FileHandle::new(meta.clone(), purger.clone());
3680        let builder = mock_indexer_builder(metadata.clone(), &env).await;
3681        let (tx, _rx) = mpsc::channel(4);
3682        let mut task = IndexBuildTask {
3683            region_id: metadata.region_id,
3684            file: original.clone(),
3685            target_region_metadata: metadata.clone(),
3686            source: IndexBuildSource::new(meta.clone(), metadata.schema_version),
3687            reason: IndexBuildType::SchemaChange,
3688            access_layer: env.access_layer.clone(),
3689            listener: WorkerListener::default(),
3690            manifest_ctx,
3691            write_cache: None,
3692            cache_manager: None,
3693            file_purger: purger.clone(),
3694            indexer_builder: builder,
3695            request_sender: tx,
3696            index_options: IndexOptions::default(),
3697            result_senders: Vec::new(),
3698        };
3699        assert!(matches!(
3700            task.prepare_source(meta.file_id(), &version).await.unwrap(),
3701            ControlFlow::Continue(())
3702        ));
3703        assert_eq!(task.file.meta_ref().index_version, 1);
3704        assert_eq!(
3705            task.indexer_builder
3706                .index_build_inputs
3707                .as_ref()
3708                .unwrap()
3709                .output_rows,
3710            (!unknown_rows).then_some(info.num_rows as u64),
3711        );
3712        let reader = task
3713            .indexer_builder
3714            .index_build_inputs
3715            .as_ref()
3716            .unwrap()
3717            .inputs[0]
3718            .clone();
3719        // Removal after preparation must wait for the task and its plugin readers.
3720        original.mark_deleted();
3721        drop(original);
3722        assert!(purger.0.lock().unwrap().is_empty());
3723        drop(task);
3724        assert!(purger.0.lock().unwrap().is_empty());
3725        drop(reader);
3726        assert_eq!(*purger.0.lock().unwrap(), vec![refreshed]);
3727    }
3728
3729    #[tokio::test]
3730    async fn test_rebuild_scans_only_when_a_creator_requires_rows() {
3731        for with_builtin in [false, true] {
3732            let env = SchedulerEnv::new().await;
3733            let mut metadata = sst_region_metadata();
3734            if !with_builtin {
3735                for column in &mut metadata.column_metadatas {
3736                    column.column_schema.set_inverted_index(false);
3737                    column.column_schema.unset_skipping_options().unwrap();
3738                }
3739            }
3740            let metadata = Arc::new(metadata);
3741            let manifest_ctx = env.mock_manifest_context(metadata.clone()).await;
3742            let file_purger = Arc::new(NoopFilePurger {});
3743            let info = mock_sst_file(metadata.clone(), &env, IndexBuildMode::Async).await;
3744            let file_meta = FileMeta {
3745                region_id: metadata.region_id,
3746                file_id: info.file_id,
3747                file_size: info.file_size,
3748                max_row_group_uncompressed_size: info.max_row_group_uncompressed_size,
3749                num_rows: info.num_rows as u64,
3750                num_row_groups: info.num_row_groups,
3751                ..Default::default()
3752            };
3753            seed_manifest_file(&manifest_ctx, &file_meta).await;
3754            let version = mock_version_control(
3755                metadata.clone(),
3756                file_purger.clone(),
3757                HashMap::from([(file_meta.file_id, file_meta.clone())]),
3758            )
3759            .await;
3760            let mut builder = mock_indexer_builder(metadata.clone(), &env).await;
3761            let mutable = Arc::make_mut(&mut builder);
3762            mutable.build_type = IndexBuildType::Manual;
3763            Arc::make_mut(&mut mutable.registry)
3764                .register(Arc::new(FailingProvider(ProviderFailure::Reused)))
3765                .unwrap();
3766            let (tx, _rx) = mpsc::channel(4);
3767            let mut task = IndexBuildTask {
3768                region_id: metadata.region_id,
3769                file: FileHandle::new(file_meta.clone(), file_purger.clone()),
3770                target_region_metadata: metadata.clone(),
3771                source: IndexBuildSource::new(file_meta.clone(), metadata.schema_version),
3772                reason: IndexBuildType::Manual,
3773                access_layer: env.access_layer.clone(),
3774                listener: WorkerListener::default(),
3775                manifest_ctx: manifest_ctx.clone(),
3776                write_cache: None,
3777                cache_manager: None,
3778                file_purger,
3779                indexer_builder: builder,
3780                request_sender: tx,
3781                index_options: IndexOptions::default(),
3782                result_senders: Vec::new(),
3783            };
3784            if !with_builtin {
3785                // A reuse-only build must not open even the Parquet footer.
3786                // Keep the file identity published, but make footer reads fail.
3787                let path = task
3788                    .file
3789                    .file_path(env.access_layer.table_dir(), env.access_layer.path_type());
3790                env.access_layer
3791                    .object_store()
3792                    .write(&path, Bytes::from_static(b"not a parquet file"))
3793                    .await
3794                    .unwrap();
3795            }
3796            // The reused creator rejects row updates, even when built-in creators need a scan.
3797            task.index_build(version).await.unwrap();
3798            let published = manifest_ctx.manifest().await.files[&file_meta.file_id].clone();
3799            assert!(published.available_indexes.contains(&IndexType::HllIndex));
3800            assert!(published.index_file_size > 0);
3801            let reader = env
3802                .access_layer
3803                .build_puffin_manager()
3804                .reader(&RegionIndexId::new(
3805                    RegionFileId::new(metadata.region_id, file_meta.file_id),
3806                    published.index_version,
3807                ))
3808                .await
3809                .unwrap();
3810            reader.blob("test-sketch").await.unwrap();
3811            if with_builtin {
3812                assert!(
3813                    published
3814                        .available_indexes
3815                        .contains(&IndexType::InvertedIndex)
3816                );
3817                let blob = reader.blob(inverted_index::INDEX_BLOB_TYPE).await.unwrap();
3818                let index_metadata =
3819                    index::inverted_index::format::reader::InvertedIndexBlobReader::new(
3820                        blob.reader().await.unwrap(),
3821                    )
3822                    .metadata(None)
3823                    .await
3824                    .unwrap();
3825                assert_eq!(index_metadata.total_row_count, file_meta.num_rows);
3826            } else {
3827                assert_eq!(
3828                    published.available_indexes.as_slice(),
3829                    &[IndexType::HllIndex]
3830                );
3831            }
3832        }
3833    }
3834
3835    #[tokio::test]
3836    async fn test_rebuild_failure_preserves_published_indexes() {
3837        for (failure, old_hll, legacy, with_builtin) in [
3838            (Some(ProviderFailure::Plan), true, false, true),
3839            (Some(ProviderFailure::Create), true, false, true),
3840            (Some(ProviderFailure::UpdateFlat), true, false, true),
3841            (Some(ProviderFailure::Finish), true, false, true),
3842            (Some(ProviderFailure::Budget), true, false, true),
3843            (Some(ProviderFailure::Budget), false, false, true),
3844            (Some(ProviderFailure::SkipCreate), true, false, true),
3845            (Some(ProviderFailure::SkipCreate), true, false, false),
3846            (Some(ProviderFailure::SkipCreate), true, true, true),
3847            (Some(ProviderFailure::SkipCreate), false, false, true),
3848            (Some(ProviderFailure::EmptyFinish), true, false, true),
3849            (Some(ProviderFailure::PartialFinish), true, false, true),
3850            (Some(ProviderFailure::None), true, false, true),
3851            (None, true, false, true),
3852            (None, false, false, true),
3853        ] {
3854            let env = SchedulerEnv::new().await;
3855            let mut metadata = sst_region_metadata();
3856            if !with_builtin {
3857                for column in &mut metadata.column_metadatas {
3858                    column.column_schema.set_inverted_index(false);
3859                    column.column_schema.unset_skipping_options().unwrap();
3860                }
3861            }
3862            let metadata = Arc::new(metadata);
3863            let region_id = metadata.region_id;
3864            let manifest_ctx = env.mock_manifest_context(metadata.clone()).await;
3865            let file_purger = Arc::new(NoopFilePurger {});
3866            let info = mock_sst_file(metadata.clone(), &env, IndexBuildMode::Async).await;
3867            let initial = FileMeta {
3868                region_id,
3869                file_id: info.file_id,
3870                file_size: info.file_size,
3871                max_row_group_uncompressed_size: info.max_row_group_uncompressed_size,
3872                num_rows: info.num_rows as u64,
3873                num_row_groups: info.num_row_groups,
3874                ..Default::default()
3875            };
3876            seed_manifest_file(&manifest_ctx, &initial).await;
3877            let version = mock_version_control(
3878                metadata.clone(),
3879                file_purger.clone(),
3880                HashMap::from([(initial.file_id, initial.clone())]),
3881            )
3882            .await;
3883            let mut builder = mock_indexer_builder(metadata.clone(), &env).await;
3884            Arc::make_mut(&mut builder).build_type = IndexBuildType::Manual;
3885            if old_hll {
3886                Arc::make_mut(&mut Arc::make_mut(&mut builder).registry)
3887                    .register(Arc::new(FailingProvider(
3888                        if matches!(
3889                            failure,
3890                            Some(ProviderFailure::PartialFinish | ProviderFailure::None)
3891                        ) {
3892                            ProviderFailure::CompletePair
3893                        } else {
3894                            ProviderFailure::None
3895                        },
3896                    )))
3897                    .unwrap();
3898            }
3899            let (tx, _rx) = mpsc::channel(4);
3900            let mut task = IndexBuildTask {
3901                region_id,
3902                file: FileHandle::new(initial.clone(), file_purger.clone()),
3903                target_region_metadata: metadata.clone(),
3904                source: IndexBuildSource::new(initial.clone(), metadata.schema_version),
3905                reason: IndexBuildType::Manual,
3906                access_layer: env.access_layer.clone(),
3907                listener: WorkerListener::default(),
3908                manifest_ctx: manifest_ctx.clone(),
3909                write_cache: None,
3910                cache_manager: None,
3911                file_purger: file_purger.clone(),
3912                indexer_builder: builder,
3913                request_sender: tx,
3914                index_options: IndexOptions::default(),
3915                result_senders: Vec::new(),
3916            };
3917            // First publish real built-in blobs and the test provider's HLL blob.
3918            task.index_build(version).await.unwrap();
3919            let mut old = manifest_ctx.manifest().await.files[&initial.file_id].clone();
3920            if legacy {
3921                let mut legacy_meta = old.as_ref().clone();
3922                for column in &mut legacy_meta.indexes {
3923                    column
3924                        .created_indexes
3925                        .retain(|kind| *kind != IndexType::HllIndex);
3926                }
3927                old = legacy_meta.into();
3928                seed_manifest_file(&manifest_ctx, old.as_ref()).await;
3929            }
3930            assert_eq!(
3931                old.available_indexes.contains(&IndexType::HllIndex),
3932                old_hll
3933            );
3934            assert_eq!(
3935                old.available_indexes
3936                    .iter()
3937                    .any(|kind| *kind != IndexType::HllIndex),
3938                with_builtin,
3939            );
3940            let old_path = location::index_file_path(
3941                env.access_layer.table_dir(),
3942                RegionIndexId::new(RegionFileId::new(region_id, old.file_id), old.index_version),
3943                env.access_layer.path_type(),
3944            );
3945            let old_bytes = env
3946                .access_layer
3947                .object_store()
3948                .read(&old_path)
3949                .await
3950                .unwrap()
3951                .to_bytes();
3952            assert!(!old_bytes.is_empty());
3953            let version = mock_version_control(
3954                metadata.clone(),
3955                file_purger.clone(),
3956                HashMap::from([(old.file_id, old.as_ref().clone())]),
3957            )
3958            .await;
3959            let mut builder = mock_indexer_builder(metadata.clone(), &env).await;
3960            Arc::make_mut(&mut builder).build_type = IndexBuildType::Manual;
3961            if let Some(failure) = failure {
3962                Arc::make_mut(&mut Arc::make_mut(&mut builder).registry)
3963                    .register(Arc::new(FailingProvider(failure)))
3964                    .unwrap();
3965            }
3966            if failure.is_none() && !old_hll {
3967                // Newly configured sketches must also fail before replacing the
3968                // built-in indexes when their provider is unavailable.
3969                Arc::make_mut(&mut task.target_region_metadata).column_metadatas[0]
3970                    .column_schema
3971                    .mut_metadata()
3972                    .insert(
3973                        datatypes::schema::HLL_KEY.into(),
3974                        r#"{"precision":10}"#.into(),
3975                    );
3976            }
3977            if failure == Some(ProviderFailure::None) {
3978                // Keep only the first column configured; the second old HLL is obsolete.
3979                Arc::make_mut(&mut task.target_region_metadata).column_metadatas[0]
3980                    .column_schema
3981                    .mut_metadata()
3982                    .insert(
3983                        datatypes::schema::HLL_KEY.into(),
3984                        r#"{"precision":10}"#.into(),
3985                    );
3986                Arc::make_mut(&mut builder).metadata = task.target_region_metadata.clone();
3987            }
3988            task.file = FileHandle::new(old.clone(), file_purger);
3989            task.source = IndexBuildSource::new(old.as_ref().clone(), metadata.schema_version);
3990            task.indexer_builder = builder;
3991            let result = task.index_build(version).await;
3992            let published = manifest_ctx.manifest().await.files[&old.file_id].clone();
3993            if !with_builtin {
3994                // No creator means no publication; the requested plan is not a removal plan.
3995                assert!(matches!(
3996                    result.unwrap(),
3997                    IndexBuildReport::Done(IndexBuildOutcome::Aborted(_))
3998                ));
3999                assert_eq!(old, published);
4000            } else if failure == Some(ProviderFailure::None) {
4001                result.unwrap();
4002                assert_eq!(published.index_version, old.index_version + 1);
4003                let hll_columns = published
4004                    .indexes
4005                    .iter()
4006                    .filter(|column| column.created_indexes.contains(&IndexType::HllIndex))
4007                    .map(|column| column.column_id)
4008                    .collect::<Vec<_>>();
4009                assert_eq!(hll_columns, vec![0]);
4010            } else if (old_hll && failure.is_some()) || (failure.is_none() && !old_hll) {
4011                assert!(result.is_err(), "maintenance must fail for {failure:?}");
4012                assert_eq!(
4013                    old, published,
4014                    "failed {failure:?} must not change manifest"
4015                );
4016            } else {
4017                result.unwrap();
4018                assert_eq!(published.index_version, old.index_version + 1);
4019                assert!(!published.available_indexes.contains(&IndexType::HllIndex));
4020                assert!(
4021                    published
4022                        .indexes
4023                        .iter()
4024                        .all(|column| !column.created_indexes.contains(&IndexType::HllIndex))
4025                );
4026                let expected: Vec<_> = old
4027                    .available_indexes
4028                    .iter()
4029                    .copied()
4030                    .filter(|kind| *kind != IndexType::HllIndex)
4031                    .collect();
4032                assert_eq!(published.available_indexes.as_slice(), expected.as_slice());
4033                assert!(published.index_file_size > 0);
4034            }
4035            // NoopFilePurger keeps the historical generation available for byte comparison.
4036            assert_eq!(
4037                old_bytes,
4038                env.access_layer
4039                    .object_store()
4040                    .read(&old_path)
4041                    .await
4042                    .unwrap()
4043                    .to_bytes()
4044            );
4045        }
4046    }
4047
4048    #[tokio::test]
4049    async fn test_index_build_task_no_index() {
4050        for remove_sketches in [false, true] {
4051            check_empty_index_build(remove_sketches, true).await;
4052            if remove_sketches {
4053                check_empty_index_build(true, false).await;
4054            }
4055        }
4056    }
4057
4058    #[derive(Debug)]
4059    struct RemoveTestIndex;
4060
4061    impl handler::IndexKindHandler for RemoveTestIndex {
4062        fn validate(
4063            &self,
4064            _context: &handler::PlanContext<'_>,
4065        ) -> std::result::Result<(), common_error::ext::BoxedError> {
4066            Ok(())
4067        }
4068
4069        fn kinds(&self) -> &[kind::IndexKind] {
4070            &[kind::IndexKind::Extension(kind::ExtensionIndexKind::Hll)]
4071        }
4072
4073        fn plan(
4074            &self,
4075            context: &handler::PlanContext<'_>,
4076        ) -> std::result::Result<
4077            Option<Arc<dyn handler::PlannedIndex>>,
4078            common_error::ext::BoxedError,
4079        > {
4080            Ok(context
4081                .existing_indexes
4082                .iter()
4083                .any(|column| {
4084                    column
4085                        .created_indexes
4086                        .contains(&crate::sst::file::IndexType::HllIndex)
4087                })
4088                .then(|| Arc::new(Self) as Arc<dyn handler::PlannedIndex>))
4089        }
4090    }
4091
4092    #[async_trait::async_trait]
4093    impl handler::PlannedIndex for RemoveTestIndex {
4094        fn kinds(&self) -> &[kind::IndexKind] {
4095            handler::IndexKindHandler::kinds(self)
4096        }
4097
4098        fn entries(&self) -> &[handler::PlanEntry] {
4099            &[]
4100        }
4101
4102        async fn build(
4103            &self,
4104            _: &handler::BuildContext<'_>,
4105        ) -> std::result::Result<
4106            Option<Box<dyn handler::IndexCreator>>,
4107            common_error::ext::BoxedError,
4108        > {
4109            Ok(None)
4110        }
4111
4112        fn is_current(&self, file: &FileMeta, _: &handler::PlanContext<'_>) -> bool {
4113            !file.indexes.iter().any(|column| {
4114                column
4115                    .created_indexes
4116                    .contains(&crate::sst::file::IndexType::HllIndex)
4117            })
4118        }
4119    }
4120
4121    async fn check_empty_index_build(remove_sketches: bool, registered: bool) {
4122        let env = SchedulerEnv::new().await;
4123        let mut scheduler = env.mock_index_build_scheduler(4);
4124        let mut metadata = sst_region_metadata();
4125        // Unset indexes in metadata to simulate no index scenario.
4126        metadata.column_metadatas.iter_mut().for_each(|col| {
4127            col.column_schema.set_inverted_index(false);
4128            let _ = col.column_schema.unset_skipping_options();
4129        });
4130        let region_id = metadata.region_id;
4131        let metadata = Arc::new(metadata);
4132        let manifest_ctx = env.mock_manifest_context(metadata.clone()).await;
4133        let file_purger = Arc::new(NoopFilePurger {});
4134        let sst_info = mock_sst_file(metadata.clone(), &env, IndexBuildMode::Async).await;
4135        let mut file_meta = FileMeta {
4136            region_id,
4137            file_id: sst_info.file_id,
4138            file_size: sst_info.file_size,
4139            max_row_group_uncompressed_size: sst_info.max_row_group_uncompressed_size,
4140            index_file_size: sst_info.index_metadata.file_size,
4141            num_rows: sst_info.num_rows as u64,
4142            num_row_groups: sst_info.num_row_groups,
4143            ..Default::default()
4144        };
4145        if remove_sketches {
4146            file_meta
4147                .indexes
4148                .push(crate::sst::file::ColumnIndexMetadata {
4149                    column_id: metadata.column_metadatas[0].column_id,
4150                    created_indexes: smallvec::smallvec![crate::sst::file::IndexType::HllIndex],
4151                    fulltext_bloom_version: 0,
4152                });
4153            file_meta.index_file_size = 123;
4154        }
4155        seed_manifest_file(&manifest_ctx, &file_meta).await;
4156        let files = HashMap::from([(file_meta.file_id, file_meta.clone())]);
4157        let version_control =
4158            mock_version_control(metadata.clone(), file_purger.clone(), files).await;
4159        let mut indexer_builder = mock_indexer_builder(metadata.clone(), &env).await;
4160        let file = FileHandle::new(file_meta.clone(), file_purger.clone());
4161        if remove_sketches {
4162            // A registered owner can remove its index after options are disabled.
4163            let builder = Arc::make_mut(&mut indexer_builder);
4164            if registered {
4165                Arc::make_mut(&mut builder.registry)
4166                    .register(Arc::new(RemoveTestIndex))
4167                    .unwrap();
4168            }
4169            let plan = builder.registry.plan(&handler::PlanContext {
4170                metadata: &metadata,
4171                index_options: &builder.index_options,
4172                build_type: IndexBuildType::SchemaChange,
4173                existing_indexes: &file_meta.indexes,
4174            });
4175            assert_eq!(!plan.is_empty(), registered);
4176            assert!(plan.entries().next().is_none());
4177            builder.index_build_inputs = Some(build::IndexBuildInputs {
4178                output_rows: None,
4179                plan: Arc::new(plan),
4180                inputs: vec![file.clone()].into(),
4181                input_puffin_manager: builder.puffin_manager.clone(),
4182                can_merge_inputs: false,
4183                memory_limit: None,
4184            });
4185        }
4186
4187        // Create mock task.
4188        let (tx, mut rx) = mpsc::channel(4);
4189        let (result_tx, mut result_rx) = mpsc::channel::<Result<IndexBuildOutcome>>(4);
4190        let task = IndexBuildTask {
4191            region_id,
4192            file,
4193            target_region_metadata: version_control.current().version.metadata.clone(),
4194            source: IndexBuildSource::new(
4195                file_meta.clone(),
4196                version_control.current().version.metadata.schema_version,
4197            ),
4198            reason: if registered {
4199                IndexBuildType::SchemaChange
4200            } else {
4201                IndexBuildType::Manual
4202            },
4203            access_layer: env.access_layer.clone(),
4204            listener: WorkerListener::default(),
4205            manifest_ctx: manifest_ctx.clone(),
4206            write_cache: None,
4207            cache_manager: None,
4208            file_purger,
4209            indexer_builder,
4210            request_sender: tx,
4211            index_options: IndexOptions::default(),
4212            result_senders: vec![result_tx],
4213        };
4214
4215        if remove_sketches {
4216            assert!(task.can_remove_all_indexes());
4217            for column_level in [false, true] {
4218                let mut mixed = task.clone();
4219                if column_level {
4220                    mixed.source.file_meta.indexes[0]
4221                        .created_indexes
4222                        .push(IndexType::InvertedIndex);
4223                } else {
4224                    mixed
4225                        .source
4226                        .file_meta
4227                        .available_indexes
4228                        .push(IndexType::InvertedIndex);
4229                }
4230                assert!(mixed.can_remove_all_indexes());
4231            }
4232        }
4233
4234        scheduler
4235            .schedule_build(&version_control, task.clone())
4236            .await;
4237
4238        if remove_sketches {
4239            let finished = recv_index_build_finished(&mut rx).await;
4240            assert!(finished.file_meta.indexes.is_empty());
4241            assert!(finished.file_meta.available_indexes.is_empty());
4242            assert_eq!(finished.file_meta.index_file_size, 0);
4243            assert_eq!(finished.file_meta.index_version, 1);
4244            assert!(
4245                manifest_ctx.manifest().await.files[&file_meta.file_id]
4246                    .indexes
4247                    .is_empty()
4248            );
4249            // Reconstruct the file from persisted metadata as reopening does, then enable
4250            // an ordinary index. Its new Puffin must not reuse the historical version 0.
4251            let persisted = serde_json::to_vec(&finished.file_meta).unwrap();
4252            let reopened: FileMeta = serde_json::from_slice(&persisted).unwrap();
4253            assert!(!reopened.exists_index());
4254            let target_metadata = Arc::new(sst_region_metadata());
4255            let reopened_version = mock_version_control(
4256                target_metadata.clone(),
4257                task.file_purger.clone(),
4258                HashMap::from([(reopened.file_id, reopened.clone())]),
4259            )
4260            .await;
4261            let (tx, mut rx) = mpsc::channel(4);
4262            let mut rebuild = task;
4263            rebuild.file = FileHandle::new(reopened.clone(), rebuild.file_purger.clone());
4264            rebuild.source.file_meta = reopened;
4265            rebuild.target_region_metadata = target_metadata.clone();
4266            rebuild.indexer_builder = mock_indexer_builder(target_metadata, &env).await;
4267            rebuild.request_sender = tx;
4268            let mut reopened_scheduler = env.mock_index_build_scheduler(4);
4269            reopened_scheduler
4270                .schedule_build(&reopened_version, rebuild)
4271                .await;
4272            let rebuilt = recv_index_build_finished(&mut rx).await;
4273            assert!(rebuilt.file_meta.exists_index());
4274            assert_eq!(rebuilt.file_meta.index_version, 2);
4275            let path = location::index_file_path(
4276                env.access_layer.table_dir(),
4277                RegionIndexId::new(RegionFileId::new(region_id, file_meta.file_id), 2),
4278                env.access_layer.path_type(),
4279            );
4280            assert!(env.access_layer.object_store().exists(&path).await.unwrap());
4281            return;
4282        }
4283
4284        // No index needs to be built, so the SST is not read.
4285        assert!(matches!(
4286            result_rx.recv().await.unwrap(),
4287            Ok(IndexBuildOutcome::Aborted(reason)) if reason.contains("No index to build")
4288        ));
4289
4290        // Only the stop notification reaches the worker.
4291        assert!(matches!(
4292            rx.recv().await.unwrap().request,
4293            WorkerRequest::Background {
4294                notify: BackgroundNotify::IndexBuildStopped(_),
4295                ..
4296            }
4297        ));
4298    }
4299
4300    #[tokio::test]
4301    async fn test_index_build_task_with_write_cache() {
4302        let env = SchedulerEnv::new().await;
4303        let mut scheduler = env.mock_index_build_scheduler(4);
4304        let metadata = Arc::new(sst_region_metadata());
4305        let manifest_ctx = env.mock_manifest_context(metadata.clone()).await;
4306        let file_purger = Arc::new(NoopFilePurger {});
4307        let region_id = metadata.region_id;
4308
4309        let (dir, factory) = PuffinManagerFactory::new_for_test_async("test_write_cache").await;
4310        let intm_manager = mock_intm_mgr(dir.path().to_string_lossy()).await;
4311
4312        // Create mock write cache
4313        let write_cache = Arc::new(
4314            WriteCache::new_fs(
4315                dir.path().to_str().unwrap(),
4316                ReadableSize::mb(10),
4317                None,
4318                None,
4319                true, // enable_background_worker
4320                factory,
4321                intm_manager,
4322                ReadableSize::mb(10),
4323            )
4324            .await
4325            .unwrap(),
4326        );
4327        // Indexer builder built from write cache.
4328        let indexer_builder = Arc::new(IndexerBuilderImpl {
4329            index_build_inputs: None,
4330            build_type: IndexBuildType::Flush,
4331            metadata: metadata.clone(),
4332            puffin_manager: write_cache.build_puffin_manager().clone(),
4333            write_cache_enabled: true,
4334            intermediate_manager: write_cache.intermediate_manager().clone(),
4335            index_options: IndexOptions::default(),
4336            registry: Arc::new(crate::sst::index::registry::IndexRegistry::new(
4337                InvertedIndexConfig::default(),
4338                FulltextIndexConfig::default(),
4339                BloomFilterConfig::default(),
4340            )),
4341        });
4342
4343        let sst_info = mock_sst_file(metadata.clone(), &env, IndexBuildMode::Async).await;
4344        let file_meta = FileMeta {
4345            region_id,
4346            file_id: sst_info.file_id,
4347            file_size: sst_info.file_size,
4348            index_file_size: sst_info.index_metadata.file_size,
4349            num_rows: sst_info.num_rows as u64,
4350            num_row_groups: sst_info.num_row_groups,
4351            ..Default::default()
4352        };
4353        seed_manifest_file(&manifest_ctx, &file_meta).await;
4354        let files = HashMap::from([(file_meta.file_id, file_meta.clone())]);
4355        let version_control =
4356            mock_version_control(metadata.clone(), file_purger.clone(), files).await;
4357
4358        let file = FileHandle::new(file_meta.clone(), file_purger.clone());
4359
4360        // Create mock task.
4361        let (tx, mut rx) = mpsc::channel(4);
4362        let (result_tx, _result_rx) = mpsc::channel::<Result<IndexBuildOutcome>>(4);
4363        let task = IndexBuildTask {
4364            region_id,
4365            file,
4366            target_region_metadata: version_control.current().version.metadata.clone(),
4367            source: IndexBuildSource::new(
4368                file_meta.clone(),
4369                version_control.current().version.metadata.schema_version,
4370            ),
4371            reason: IndexBuildType::Flush,
4372            access_layer: env.access_layer.clone(),
4373            listener: WorkerListener::default(),
4374            manifest_ctx,
4375            write_cache: Some(write_cache.clone()),
4376            cache_manager: None,
4377            file_purger,
4378            indexer_builder,
4379            request_sender: tx,
4380            index_options: IndexOptions::default(),
4381            result_senders: vec![result_tx],
4382        };
4383
4384        scheduler.schedule_build(&version_control, task).await;
4385
4386        recv_index_build_finished(&mut rx).await;
4387
4388        // The write cache should contain the uploaded index file.
4389        let index_key = IndexKey::new(
4390            region_id,
4391            file_meta.file_id,
4392            FileType::Puffin(sst_info.index_metadata.version),
4393        );
4394        assert!(write_cache.file_cache().contains_key(&index_key));
4395    }
4396
4397    async fn create_mock_task_for_schedule(
4398        env: &SchedulerEnv,
4399        file_id: FileId,
4400        region_id: RegionId,
4401        reason: IndexBuildType,
4402    ) -> IndexBuildTask {
4403        create_mock_task_for_schedule_with_result(env, file_id, region_id, reason)
4404            .await
4405            .0
4406    }
4407
4408    /// Like [`create_mock_task_for_schedule`] but also returns the result receiver
4409    /// so tests can verify pending task cancellation errors.
4410    async fn create_mock_task_for_schedule_with_result(
4411        env: &SchedulerEnv,
4412        file_id: FileId,
4413        region_id: RegionId,
4414        reason: IndexBuildType,
4415    ) -> (IndexBuildTask, mpsc::Receiver<Result<IndexBuildOutcome>>) {
4416        let metadata = Arc::new(sst_region_metadata());
4417        let schema_version = metadata.schema_version;
4418        let manifest_ctx = env.mock_manifest_context(metadata.clone()).await;
4419        let file_purger = Arc::new(NoopFilePurger {});
4420        let indexer_builder = mock_indexer_builder(metadata.clone(), env).await;
4421        let (tx, _rx) = mpsc::channel(4);
4422        let (result_tx, result_rx) = mpsc::channel::<Result<IndexBuildOutcome>>(4);
4423
4424        let file_meta = FileMeta {
4425            region_id,
4426            file_id,
4427            file_size: 100,
4428            ..Default::default()
4429        };
4430
4431        let file = FileHandle::new(file_meta.clone(), file_purger.clone());
4432
4433        let task = IndexBuildTask {
4434            region_id,
4435            file,
4436            target_region_metadata: metadata,
4437            source: IndexBuildSource::new(file_meta, schema_version),
4438            reason,
4439            access_layer: env.access_layer.clone(),
4440            listener: WorkerListener::default(),
4441            manifest_ctx,
4442            write_cache: None,
4443            cache_manager: None,
4444            file_purger,
4445            indexer_builder,
4446            request_sender: tx,
4447            index_options: IndexOptions::default(),
4448            result_senders: vec![result_tx],
4449        };
4450        (task, result_rx)
4451    }
4452
4453    #[tokio::test]
4454    async fn test_scheduler_coalesces_latest_schema_generation_per_sst() {
4455        let job_scheduler = Arc::new(VecScheduler::default());
4456        let env = SchedulerEnv::new().await.scheduler(job_scheduler.clone());
4457        let mut scheduler = env.mock_index_build_scheduler(2);
4458        let metadata = Arc::new(sst_region_metadata());
4459        let region_id = metadata.region_id;
4460        let file_id = FileId::random();
4461        let file_purger = Arc::new(NoopFilePurger {});
4462        let files = HashMap::from([(
4463            file_id,
4464            FileMeta {
4465                region_id,
4466                file_id,
4467                file_size: 100,
4468                ..Default::default()
4469            },
4470        )]);
4471        let version_control = mock_version_control(metadata, file_purger, files).await;
4472
4473        let (active, _active_rx) = create_mock_task_for_schedule_with_result(
4474            &env,
4475            file_id,
4476            region_id,
4477            IndexBuildType::Manual,
4478        )
4479        .await;
4480        scheduler.schedule_build(&version_control, active).await;
4481        assert_eq!(job_scheduler.num_jobs(), 1);
4482
4483        let (mut older, mut older_rx) = create_mock_task_for_schedule_with_result(
4484            &env,
4485            file_id,
4486            region_id,
4487            IndexBuildType::SchemaChange,
4488        )
4489        .await;
4490        older.source.schema_version = 1;
4491        scheduler.schedule_build(&version_control, older).await;
4492
4493        let (mut latest, mut latest_rx) = create_mock_task_for_schedule_with_result(
4494            &env,
4495            file_id,
4496            region_id,
4497            IndexBuildType::SchemaChange,
4498        )
4499        .await;
4500        latest.source.schema_version = 2;
4501        scheduler.schedule_build(&version_control, latest).await;
4502
4503        // The replaced task's caller waits for the surviving build instead of
4504        // being told the replaced build has completed.
4505        for rx in [&mut older_rx, &mut latest_rx] {
4506            assert!(matches!(
4507                rx.try_recv(),
4508                Err(mpsc::error::TryRecvError::Empty)
4509            ));
4510        }
4511
4512        let status = &scheduler.region_status[&region_id];
4513        assert_eq!(status.building_files.len(), 1);
4514        assert_eq!(status.pending_tasks.len(), 1);
4515        let survivor = &status.pending_tasks.tasks[&file_id].task;
4516        assert_eq!(survivor.source.schema_version, 2);
4517        assert_eq!(survivor.result_senders.len(), 2);
4518        assert_eq!(job_scheduler.num_jobs(), 1);
4519
4520        scheduler.on_task_stopped(region_id, file_id);
4521        let status = &scheduler.region_status[&region_id];
4522        assert_eq!(status.building_files.len(), 1);
4523        assert!(status.pending_tasks.is_empty());
4524        assert_eq!(job_scheduler.num_jobs(), 2);
4525    }
4526
4527    #[tokio::test]
4528    async fn test_scheduler_completes_sender_when_job_is_rejected() {
4529        let job_scheduler = Arc::new(LocalScheduler::new(1));
4530        job_scheduler.stop(false).await.unwrap();
4531        let env = SchedulerEnv::new().await.scheduler(job_scheduler);
4532        let mut scheduler = env.mock_index_build_scheduler(1);
4533        let metadata = Arc::new(sst_region_metadata());
4534        let region_id = metadata.region_id;
4535        let file_id = FileId::random();
4536        let file_purger = Arc::new(NoopFilePurger {});
4537        let files = HashMap::from([(
4538            file_id,
4539            FileMeta {
4540                region_id,
4541                file_id,
4542                file_size: 100,
4543                ..Default::default()
4544            },
4545        )]);
4546        let version_control = mock_version_control(metadata, file_purger, files).await;
4547        let (task, mut result_rx) = create_mock_task_for_schedule_with_result(
4548            &env,
4549            file_id,
4550            region_id,
4551            IndexBuildType::Flush,
4552        )
4553        .await;
4554
4555        scheduler.schedule_build(&version_control, task).await;
4556
4557        let result = tokio::time::timeout(std::time::Duration::from_secs(5), result_rx.recv())
4558            .await
4559            .expect("scheduler rejection did not complete the result sender")
4560            .expect("result channel closed without a result");
4561        assert!(result.is_err());
4562        assert!(!scheduler.region_status.contains_key(&region_id));
4563    }
4564
4565    #[tokio::test]
4566    async fn test_scheduler_comprehensive() {
4567        let env = SchedulerEnv::new().await;
4568        let mut scheduler = env.mock_index_build_scheduler(2);
4569        let metadata = Arc::new(sst_region_metadata());
4570        let region_id = metadata.region_id;
4571        let file_purger = Arc::new(NoopFilePurger {});
4572
4573        // Prepare multiple files for testing
4574        let file_id1 = FileId::random();
4575        let file_id2 = FileId::random();
4576        let file_id3 = FileId::random();
4577        let file_id4 = FileId::random();
4578        let file_id5 = FileId::random();
4579
4580        let mut files = HashMap::new();
4581        for file_id in [file_id1, file_id2, file_id3, file_id4, file_id5] {
4582            files.insert(
4583                file_id,
4584                FileMeta {
4585                    region_id,
4586                    file_id,
4587                    file_size: 100,
4588                    ..Default::default()
4589                },
4590            );
4591        }
4592
4593        let version_control = mock_version_control(metadata, file_purger, files).await;
4594
4595        // Test 1: Basic scheduling
4596        let task1 =
4597            create_mock_task_for_schedule(&env, file_id1, region_id, IndexBuildType::Flush).await;
4598        scheduler.schedule_build(&version_control, task1).await;
4599        assert!(scheduler.region_status.contains_key(&region_id));
4600        let status = scheduler.region_status.get(&region_id).unwrap();
4601        assert_eq!(status.building_files.len(), 1);
4602        assert!(status.building_files.contains(&file_id1));
4603
4604        // Test 2: Duplicate file scheduling (should be skipped)
4605        let task1_dup =
4606            create_mock_task_for_schedule(&env, file_id1, region_id, IndexBuildType::Flush).await;
4607        scheduler.schedule_build(&version_control, task1_dup).await;
4608        let status = scheduler.region_status.get(&region_id).unwrap();
4609        assert_eq!(status.building_files.len(), 1); // Still only one
4610
4611        // Test 3: Fill up to limit (2 building tasks)
4612        let task2 =
4613            create_mock_task_for_schedule(&env, file_id2, region_id, IndexBuildType::Flush).await;
4614        scheduler.schedule_build(&version_control, task2).await;
4615        let status = scheduler.region_status.get(&region_id).unwrap();
4616        assert_eq!(status.building_files.len(), 2); // Reached limit
4617        assert_eq!(status.pending_tasks.len(), 0);
4618
4619        // Test 4: Add tasks with different priorities to pending queue
4620        // Now all new tasks will be pending since we reached the limit
4621        let task3 =
4622            create_mock_task_for_schedule(&env, file_id3, region_id, IndexBuildType::Compact).await;
4623        let task4 =
4624            create_mock_task_for_schedule(&env, file_id4, region_id, IndexBuildType::SchemaChange)
4625                .await;
4626        let task5 =
4627            create_mock_task_for_schedule(&env, file_id5, region_id, IndexBuildType::Manual).await;
4628
4629        scheduler.schedule_build(&version_control, task3).await;
4630        scheduler.schedule_build(&version_control, task4).await;
4631        scheduler.schedule_build(&version_control, task5).await;
4632
4633        let status = scheduler.region_status.get(&region_id).unwrap();
4634        assert_eq!(status.building_files.len(), 2); // Still at limit
4635        assert_eq!(status.pending_tasks.len(), 3); // Three pending
4636
4637        // Test 5: Task completion triggers scheduling next highest priority task (Manual)
4638        scheduler.on_task_stopped(region_id, file_id1);
4639        let status = scheduler.region_status.get(&region_id).unwrap();
4640        assert!(!status.building_files.contains(&file_id1));
4641        assert_eq!(status.building_files.len(), 2); // Should schedule next task
4642        assert_eq!(status.pending_tasks.len(), 2); // One less pending
4643        // The highest priority task (Manual) should now be building
4644        assert!(status.building_files.contains(&file_id5));
4645
4646        // Test 6: Complete another task, should schedule SchemaChange (second highest priority)
4647        scheduler.on_task_stopped(region_id, file_id2);
4648        let status = scheduler.region_status.get(&region_id).unwrap();
4649        assert_eq!(status.building_files.len(), 2);
4650        assert_eq!(status.pending_tasks.len(), 1); // One less pending
4651        assert!(status.building_files.contains(&file_id4)); // SchemaChange should be building
4652
4653        // Test 7: Complete remaining tasks and cleanup
4654        scheduler.on_task_stopped(region_id, file_id5);
4655        scheduler.on_task_stopped(region_id, file_id4);
4656
4657        let status = scheduler.region_status.get(&region_id).unwrap();
4658        assert_eq!(status.building_files.len(), 1); // Last task (Compact) should be building
4659        assert_eq!(status.pending_tasks.len(), 0);
4660        assert!(status.building_files.contains(&file_id3));
4661
4662        scheduler.on_task_stopped(region_id, file_id3);
4663
4664        // Region should be removed when all tasks complete
4665        assert!(!scheduler.region_status.contains_key(&region_id));
4666    }
4667
4668    /// Helper to set up a scheduler with files_limit=1 and 3 scheduled tasks,
4669    /// returning the scheduler, the two pending-task result receivers, and the
4670    /// version control.
4671    async fn setup_scheduler_with_pending_tasks(
4672        env: &SchedulerEnv,
4673    ) -> (
4674        IndexBuildScheduler,
4675        mpsc::Receiver<Result<IndexBuildOutcome>>,
4676        mpsc::Receiver<Result<IndexBuildOutcome>>,
4677        VersionControlRef,
4678        RegionId,
4679        FileId, // building file_id for no-op assertion
4680    ) {
4681        let metadata = Arc::new(sst_region_metadata());
4682        let region_id = metadata.region_id;
4683        let file_purger = Arc::new(NoopFilePurger {});
4684
4685        let file_id1 = FileId::random();
4686        let file_id2 = FileId::random();
4687        let file_id3 = FileId::random();
4688
4689        let files = HashMap::from([
4690            (
4691                file_id1,
4692                FileMeta {
4693                    region_id,
4694                    file_id: file_id1,
4695                    file_size: 100,
4696                    ..Default::default()
4697                },
4698            ),
4699            (
4700                file_id2,
4701                FileMeta {
4702                    region_id,
4703                    file_id: file_id2,
4704                    file_size: 100,
4705                    ..Default::default()
4706                },
4707            ),
4708            (
4709                file_id3,
4710                FileMeta {
4711                    region_id,
4712                    file_id: file_id3,
4713                    file_size: 100,
4714                    ..Default::default()
4715                },
4716            ),
4717        ]);
4718        let version_control =
4719            mock_version_control(metadata.clone(), file_purger.clone(), files).await;
4720
4721        let mut scheduler = env.mock_index_build_scheduler(1);
4722
4723        // task1 becomes the "building" task (files_limit=1).
4724        // We intentionally drop its result receiver: the building task's late-stop
4725        // behavior is covered by the manual on_task_stopped no-op assertion below.
4726        let (task1, _rx1) = create_mock_task_for_schedule_with_result(
4727            env,
4728            file_id1,
4729            region_id,
4730            IndexBuildType::Flush,
4731        )
4732        .await;
4733        let (task2, rx2) = create_mock_task_for_schedule_with_result(
4734            env,
4735            file_id2,
4736            region_id,
4737            IndexBuildType::Flush,
4738        )
4739        .await;
4740        let (task3, rx3) = create_mock_task_for_schedule_with_result(
4741            env,
4742            file_id3,
4743            region_id,
4744            IndexBuildType::Flush,
4745        )
4746        .await;
4747
4748        scheduler.schedule_build(&version_control, task1).await;
4749        scheduler.schedule_build(&version_control, task2).await;
4750        scheduler.schedule_build(&version_control, task3).await;
4751
4752        // Verify: 1 building + 2 pending.
4753        assert!(scheduler.region_status.contains_key(&region_id));
4754        let status = scheduler.region_status.get(&region_id).unwrap();
4755        assert_eq!(status.building_files.len(), 1);
4756        assert_eq!(status.pending_tasks.len(), 2);
4757
4758        (scheduler, rx2, rx3, version_control, region_id, file_id1)
4759    }
4760
4761    /// Pattern‑matches a pending‑task cancellation error: outer **must** be
4762    /// [`crate::error::Error::BuildIndexAsync`] and the inner source **must** be
4763    /// the lifecycle variant named by `expected_source` (`"dropped"`, `"closed"`,
4764    /// or `"truncated"`).  Panics with a descriptive message on mismatch.
4765    fn assert_lifecycle_error(
4766        err: crate::error::Error,
4767        expected_source: &str,
4768        lifecycle_name: &str,
4769    ) {
4770        let crate::error::Error::BuildIndexAsync { source, .. } = err else {
4771            panic!("[{lifecycle_name}] Expected BuildIndexAsync outer error, got: {err:?}");
4772        };
4773        let actual = match &*source {
4774            crate::error::Error::RegionDropped { .. } => "dropped",
4775            crate::error::Error::RegionClosed { .. } => "closed",
4776            crate::error::Error::RegionTruncated { .. } => "truncated",
4777            other => panic!(
4778                "[{lifecycle_name}] Expected lifecycle source variant (RegionDropped/RegionClosed/RegionTruncated), got: {other:?}"
4779            ),
4780        };
4781        assert_eq!(
4782            actual, expected_source,
4783            "[{lifecycle_name}] Source variant mismatch"
4784        );
4785    }
4786
4787    /// Receives a pending‑task error from `rx` with a 5‑second timeout, then
4788    /// delegates to [`assert_lifecycle_error`].
4789    async fn recv_lifecycle_error(
4790        rx: &mut mpsc::Receiver<Result<IndexBuildOutcome>>,
4791        expected_source: &str,
4792        lifecycle_name: &str,
4793    ) {
4794        let result = tokio::time::timeout(std::time::Duration::from_secs(5), rx.recv())
4795            .await
4796            .unwrap_or_else(|_| {
4797                panic!(
4798                    "[{lifecycle_name}] Timeout (5s) waiting for lifecycle error from pending task"
4799                )
4800            });
4801        let result = result.unwrap_or_else(|| {
4802            panic!("[{lifecycle_name}] Channel closed without receiving lifecycle error")
4803        });
4804        let err = result.unwrap_err();
4805        assert_lifecycle_error(err, expected_source, lifecycle_name);
4806    }
4807
4808    #[tokio::test]
4809    async fn test_scheduler_lifecycle_cleanup() {
4810        let env = SchedulerEnv::new().await;
4811
4812        // --- on_region_dropped ---
4813        {
4814            let (mut scheduler, mut rx2, mut rx3, _version_control, region_id, building_file_id) =
4815                setup_scheduler_with_pending_tasks(&env).await;
4816
4817            scheduler.on_region_dropped(region_id).await;
4818
4819            let status = scheduler.region_status.get(&region_id).unwrap();
4820            assert!(status.retiring);
4821            assert_eq!(status.building_files.len(), 1);
4822            assert!(status.pending_tasks.is_empty());
4823
4824            // Pending-task receivers get lifecycle errors (with timeout).
4825            recv_lifecycle_error(&mut rx2, "dropped", "on_region_dropped").await;
4826            recv_lifecycle_error(&mut rx3, "dropped", "on_region_dropped").await;
4827
4828            // The active build keeps its lease until it stops.
4829            scheduler.on_task_stopped(region_id, building_file_id);
4830            assert!(!scheduler.region_status.contains_key(&region_id));
4831        }
4832
4833        // --- on_region_closed ---
4834        {
4835            let (mut scheduler, mut rx2, mut rx3, version_control, region_id, building_file_id) =
4836                setup_scheduler_with_pending_tasks(&env).await;
4837
4838            scheduler.on_region_closed(region_id).await;
4839
4840            let status = scheduler.region_status.get(&region_id).unwrap();
4841            assert!(status.retiring);
4842            assert_eq!(status.building_files.len(), 1);
4843            assert!(status.pending_tasks.is_empty());
4844
4845            recv_lifecycle_error(&mut rx2, "closed", "on_region_closed").await;
4846            recv_lifecycle_error(&mut rx3, "closed", "on_region_closed").await;
4847
4848            // A build scheduled by the reopened region waits behind the old
4849            // incarnation's active lease.
4850            let (reopened_task, _reopened_rx) = create_mock_task_for_schedule_with_result(
4851                &env,
4852                building_file_id,
4853                region_id,
4854                IndexBuildType::Manual,
4855            )
4856            .await;
4857            scheduler
4858                .schedule_build(&version_control, reopened_task)
4859                .await;
4860            let status = scheduler.region_status.get(&region_id).unwrap();
4861            assert!(status.retiring);
4862            assert_eq!(status.building_files.len(), 1);
4863            assert_eq!(status.pending_tasks.len(), 1);
4864
4865            scheduler.on_task_stopped(region_id, building_file_id);
4866            let status = scheduler.region_status.get(&region_id).unwrap();
4867            assert!(!status.retiring);
4868            assert_eq!(status.building_files.len(), 1);
4869            assert!(status.pending_tasks.is_empty());
4870
4871            scheduler.on_task_stopped(region_id, building_file_id);
4872            assert!(!scheduler.region_status.contains_key(&region_id));
4873        }
4874
4875        // --- on_region_truncated ---
4876        {
4877            let (mut scheduler, mut rx2, mut rx3, _version_control, region_id, building_file_id) =
4878                setup_scheduler_with_pending_tasks(&env).await;
4879
4880            scheduler.on_region_truncated(region_id).await;
4881
4882            let status = scheduler.region_status.get(&region_id).unwrap();
4883            assert!(status.retiring);
4884            assert_eq!(status.building_files.len(), 1);
4885            assert!(status.pending_tasks.is_empty());
4886
4887            recv_lifecycle_error(&mut rx2, "truncated", "on_region_truncated").await;
4888            recv_lifecycle_error(&mut rx3, "truncated", "on_region_truncated").await;
4889
4890            scheduler.on_task_stopped(region_id, building_file_id);
4891            assert!(!scheduler.region_status.contains_key(&region_id));
4892        }
4893    }
4894}