Skip to main content

common_meta/key/
table_route.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::{HashMap, HashSet};
16use std::fmt::Display;
17use std::sync::Arc;
18
19use serde::{Deserialize, Deserializer, Serialize};
20use snafu::{OptionExt, ResultExt, ensure};
21use store_api::storage::{MAX_REGION_SEQ, RegionId, RegionNumber};
22use table::metadata::TableId;
23
24use crate::error::{
25    InvalidMetadataSnafu, MetadataCorruptionSnafu, RegionNotFoundSnafu, Result, SerdeJsonSnafu,
26    TableRouteNotFoundSnafu, UnexpectedLogicalRouteTableSnafu,
27};
28use crate::key::node_address::{NodeAddressKey, NodeAddressValue};
29use crate::key::txn_helper::TxnOpGetResponseSet;
30use crate::key::{
31    DeserializedValueWithBytes, MetadataKey, MetadataValue, RegionDistribution,
32    TABLE_ROUTE_KEY_PATTERN, TABLE_ROUTE_PREFIX,
33};
34use crate::kv_backend::KvBackendRef;
35use crate::kv_backend::txn::Txn;
36use crate::rpc::router::{RegionRoute, region_distribution};
37use crate::rpc::store::BatchGetRequest;
38
39/// The key stores table routes
40///
41/// The layout: `__table_route/{table_id}`.
42#[derive(Debug, PartialEq)]
43pub struct TableRouteKey {
44    pub table_id: TableId,
45}
46
47impl TableRouteKey {
48    pub fn new(table_id: TableId) -> Self {
49        Self { table_id }
50    }
51
52    /// Returns the range prefix of the table route key.
53    pub fn range_prefix() -> Vec<u8> {
54        format!("{}/", TABLE_ROUTE_PREFIX).into_bytes()
55    }
56}
57
58#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
59#[serde(tag = "type", rename_all = "snake_case")]
60pub enum TableRouteValue {
61    Physical(PhysicalTableRouteValue),
62    Logical(LogicalTableRouteValue),
63}
64
65/// A table route whose payload can be shared by multiple callers.
66#[derive(Clone)]
67pub enum TableRoute {
68    Physical(Arc<PhysicalTableRouteValue>),
69    Logical(Arc<LogicalTableRouteValue>),
70}
71
72impl From<TableRouteValue> for TableRoute {
73    fn from(value: TableRouteValue) -> Self {
74        match value {
75            TableRouteValue::Physical(route) => Self::Physical(Arc::new(route)),
76            TableRouteValue::Logical(route) => Self::Logical(Arc::new(route)),
77        }
78    }
79}
80
81impl From<Arc<TableRoute>> for TableRoute {
82    fn from(route: Arc<TableRoute>) -> Self {
83        Arc::unwrap_or_clone(route)
84    }
85}
86
87impl TableRoute {
88    /// Returns true if it's physical table.
89    pub fn is_physical(&self) -> bool {
90        matches!(self, TableRoute::Physical(_))
91    }
92
93    /// Returns [PhysicalTableRouteValue] reference if it's [TableRoute::Physical]; Otherwise it returns [None].
94    pub fn as_physical_table_route_ref(&self) -> Option<&Arc<PhysicalTableRouteValue>> {
95        match self {
96            TableRoute::Physical(table_route) => Some(table_route),
97            TableRoute::Logical(_) => None,
98        }
99    }
100}
101
102#[derive(Debug, PartialEq, Serialize, Clone, Default)]
103pub struct PhysicalTableRouteValue {
104    // The region routes of the table.
105    pub region_routes: Vec<RegionRoute>,
106    // Tracks the highest region number ever allocated for the table.
107    // This value only increases: adding a region updates it if needed,
108    // and dropping regions does not decrease it.
109    pub max_region_number: RegionNumber,
110    // The version of the table route.
111    version: u64,
112}
113
114impl<'de> Deserialize<'de> for PhysicalTableRouteValue {
115    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
116    where
117        D: Deserializer<'de>,
118    {
119        #[derive(Deserialize)]
120        struct Helper {
121            region_routes: Vec<RegionRoute>,
122            #[serde(default)]
123            max_region_number: Option<RegionNumber>,
124            version: u64,
125        }
126
127        let mut helper = Helper::deserialize(deserializer)?;
128        // If the max region number is not provided, we will calculate it from the region routes.
129        if helper.max_region_number.is_none() {
130            let max_region = helper
131                .region_routes
132                .iter()
133                .map(|r| r.region.id.region_number())
134                .max()
135                .unwrap_or_default();
136            helper.max_region_number = Some(max_region);
137        }
138
139        Ok(PhysicalTableRouteValue {
140            region_routes: helper.region_routes,
141            max_region_number: helper.max_region_number.unwrap_or_default(),
142            version: helper.version,
143        })
144    }
145}
146
147#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
148pub struct LogicalTableRouteValue {
149    physical_table_id: TableId,
150}
151
152impl TableRouteValue {
153    /// Returns a [TableRouteValue::Physical] if `table_id` equals `physical_table_id`.
154    /// Otherwise returns a [TableRouteValue::Logical].
155    pub(crate) fn new(
156        table_id: TableId,
157        physical_table_id: TableId,
158        region_routes: Vec<RegionRoute>,
159    ) -> Self {
160        if table_id == physical_table_id {
161            TableRouteValue::physical(region_routes)
162        } else {
163            TableRouteValue::logical(physical_table_id)
164        }
165    }
166
167    pub fn physical(region_routes: Vec<RegionRoute>) -> Self {
168        Self::Physical(PhysicalTableRouteValue::new(region_routes))
169    }
170
171    pub fn logical(physical_table_id: TableId) -> Self {
172        Self::Logical(LogicalTableRouteValue::new(physical_table_id))
173    }
174
175    /// Returns a new version [TableRouteValue] with `region_routes`.
176    pub fn update(&self, region_routes: Vec<RegionRoute>) -> Result<Self> {
177        ensure!(
178            self.is_physical(),
179            UnexpectedLogicalRouteTableSnafu {
180                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
181            }
182        );
183        let physical_table_route = self.as_physical_table_route_ref();
184        let original_max_region_number = physical_table_route.max_region_number;
185        let new_max_region_number = region_routes
186            .iter()
187            .map(|r| r.region.id.region_number())
188            .max()
189            .unwrap_or_default();
190        let version = physical_table_route.version;
191        Ok(Self::Physical(PhysicalTableRouteValue {
192            region_routes,
193            // If region routes are added, we will update the max region number.
194            // If region routes are removed, we will keep the original max region number.
195            max_region_number: original_max_region_number.max(new_max_region_number),
196            version: version + 1,
197        }))
198    }
199
200    /// Returns the version.
201    ///
202    /// For test purpose.
203    #[cfg(any(test, feature = "testing"))]
204    pub fn version(&self) -> Result<u64> {
205        ensure!(
206            self.is_physical(),
207            UnexpectedLogicalRouteTableSnafu {
208                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
209            }
210        );
211        Ok(self.as_physical_table_route_ref().version)
212    }
213
214    /// Returns the corresponding [RegionRoute], returns `None` if it's the specific region is not found.
215    ///
216    /// Note: It throws an error if it's a logical table
217    pub fn region_route(&self, region_id: RegionId) -> Result<Option<RegionRoute>> {
218        ensure!(
219            self.is_physical(),
220            UnexpectedLogicalRouteTableSnafu {
221                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
222            }
223        );
224        Ok(self
225            .as_physical_table_route_ref()
226            .region_routes
227            .iter()
228            .find(|route| route.region.id == region_id)
229            .cloned())
230    }
231
232    /// Returns true if it's [TableRouteValue::Physical].
233    pub fn is_physical(&self) -> bool {
234        matches!(self, TableRouteValue::Physical(_))
235    }
236
237    /// Gets the [RegionRoute]s of this [TableRouteValue::Physical].
238    pub fn region_routes(&self) -> Result<&Vec<RegionRoute>> {
239        ensure!(
240            self.is_physical(),
241            UnexpectedLogicalRouteTableSnafu {
242                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
243            }
244        );
245        Ok(&self.as_physical_table_route_ref().region_routes)
246    }
247
248    /// Returns the max region number of this [TableRouteValue::Physical].
249    ///
250    /// # Panic
251    /// If it is not the [`PhysicalTableRouteValue`].
252    pub fn max_region_number(&self) -> Result<RegionNumber> {
253        ensure!(
254            self.is_physical(),
255            UnexpectedLogicalRouteTableSnafu {
256                err_msg: format!("{self:?} is a non-physical TableRouteValue."),
257            }
258        );
259        Ok(self.as_physical_table_route_ref().max_region_number)
260    }
261
262    /// Returns the reference of [`PhysicalTableRouteValue`].
263    ///
264    /// # Panic
265    /// If it is not the [`PhysicalTableRouteValue`].
266    fn as_physical_table_route_ref(&self) -> &PhysicalTableRouteValue {
267        match self {
268            TableRouteValue::Physical(x) => x,
269            _ => unreachable!("Mistakenly been treated as a Physical TableRoute: {self:?}"),
270        }
271    }
272
273    /// Converts to [`PhysicalTableRouteValue`].
274    ///
275    /// # Panic
276    /// If it is not the [`PhysicalTableRouteValue`].
277    pub fn into_physical_table_route(self) -> PhysicalTableRouteValue {
278        match self {
279            TableRouteValue::Physical(x) => x,
280            _ => unreachable!("Mistakenly been treated as a Physical TableRoute: {self:?}"),
281        }
282    }
283
284    /// Converts to [`LogicalTableRouteValue`].
285    ///
286    /// # Panic
287    /// If it is not the [`LogicalTableRouteValue`].
288    pub fn into_logical_table_route(self) -> LogicalTableRouteValue {
289        match self {
290            TableRouteValue::Logical(x) => x,
291            _ => unreachable!("Mistakenly been treated as a Logical TableRoute: {self:?}"),
292        }
293    }
294
295    pub fn region_numbers(&self) -> Vec<RegionNumber> {
296        match self {
297            TableRouteValue::Physical(x) => x
298                .region_routes
299                .iter()
300                .map(|region_route| region_route.region.id.region_number())
301                .collect(),
302            TableRouteValue::Logical(_) => {
303                vec![]
304            }
305        }
306    }
307}
308
309impl MetadataValue for TableRouteValue {
310    fn try_from_raw_value(raw_value: &[u8]) -> Result<Self> {
311        let r = serde_json::from_slice::<TableRouteValue>(raw_value);
312        match r {
313            // Compatible with old TableRouteValue.
314            Err(e) if e.is_data() => Ok(Self::Physical(
315                serde_json::from_slice::<PhysicalTableRouteValue>(raw_value)
316                    .context(SerdeJsonSnafu)?,
317            )),
318            Ok(x) => Ok(x),
319            Err(e) => Err(e).context(SerdeJsonSnafu),
320        }
321    }
322
323    fn try_as_raw_value(&self) -> Result<Vec<u8>> {
324        serde_json::to_vec(self).context(SerdeJsonSnafu)
325    }
326}
327
328impl PhysicalTableRouteValue {
329    /// Returns the version of the table route.
330    pub fn version(&self) -> u64 {
331        self.version
332    }
333    pub fn new(region_routes: Vec<RegionRoute>) -> Self {
334        let max_region_number = region_routes
335            .iter()
336            .map(|r| r.region.id.region_number())
337            .max()
338            .unwrap_or_default();
339        Self {
340            region_routes,
341            max_region_number,
342            version: 0,
343        }
344    }
345}
346
347impl LogicalTableRouteValue {
348    pub fn new(physical_table_id: TableId) -> Self {
349        Self { physical_table_id }
350    }
351
352    pub fn physical_table_id(&self) -> TableId {
353        self.physical_table_id
354    }
355}
356
357impl MetadataKey<'_, TableRouteKey> for TableRouteKey {
358    fn to_bytes(&self) -> Vec<u8> {
359        self.to_string().into_bytes()
360    }
361
362    fn from_bytes(bytes: &[u8]) -> Result<TableRouteKey> {
363        let key = std::str::from_utf8(bytes).map_err(|e| {
364            InvalidMetadataSnafu {
365                err_msg: format!(
366                    "TableRouteKey '{}' is not a valid UTF8 string: {e}",
367                    String::from_utf8_lossy(bytes)
368                ),
369            }
370            .build()
371        })?;
372        let captures = TABLE_ROUTE_KEY_PATTERN
373            .captures(key)
374            .context(InvalidMetadataSnafu {
375                err_msg: format!("Invalid TableRouteKey '{key}'"),
376            })?;
377        // Safety: pass the regex check above
378        let table_id = captures[1].parse::<TableId>().unwrap();
379        Ok(TableRouteKey { table_id })
380    }
381}
382
383impl Display for TableRouteKey {
384    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
385        write!(f, "{}/{}", TABLE_ROUTE_PREFIX, self.table_id)
386    }
387}
388
389pub type TableRouteManagerRef = Arc<TableRouteManager>;
390
391pub struct TableRouteManager {
392    storage: TableRouteStorage,
393}
394
395impl TableRouteManager {
396    pub fn new(kv_backend: KvBackendRef) -> Self {
397        Self {
398            storage: TableRouteStorage::new(kv_backend),
399        }
400    }
401
402    /// Returns the [TableId] recursively.
403    ///
404    /// Returns a [TableRouteNotFound](crate::error::Error::TableRouteNotFound) Error if:
405    /// - the table(`logical_or_physical_table_id`) does not exist.
406    pub async fn get_physical_table_id(
407        &self,
408        logical_or_physical_table_id: TableId,
409    ) -> Result<TableId> {
410        let table_route = self
411            .storage
412            .get_inner(logical_or_physical_table_id)
413            .await?
414            .context(TableRouteNotFoundSnafu {
415                table_id: logical_or_physical_table_id,
416            })?;
417
418        match table_route {
419            TableRouteValue::Physical(_) => Ok(logical_or_physical_table_id),
420            TableRouteValue::Logical(x) => Ok(x.physical_table_id()),
421        }
422    }
423
424    /// Returns the [TableRouteValue::Physical] recursively.
425    ///
426    /// Returns a [TableRouteNotFound](error::Error::TableRouteNotFound) Error if:
427    /// - the physical table(`logical_or_physical_table_id`) does not exist
428    /// - the corresponding physical table of the logical table(`logical_or_physical_table_id`) does not exist.
429    pub async fn get_physical_table_route(
430        &self,
431        logical_or_physical_table_id: TableId,
432    ) -> Result<(TableId, PhysicalTableRouteValue)> {
433        let table_route = self
434            .storage
435            .get(logical_or_physical_table_id)
436            .await?
437            .context(TableRouteNotFoundSnafu {
438                table_id: logical_or_physical_table_id,
439            })?;
440
441        match table_route {
442            TableRouteValue::Physical(x) => Ok((logical_or_physical_table_id, x)),
443            TableRouteValue::Logical(x) => {
444                let physical_table_id = x.physical_table_id();
445                let physical_table_route = self.storage.get(physical_table_id).await?.context(
446                    TableRouteNotFoundSnafu {
447                        table_id: physical_table_id,
448                    },
449                )?;
450                let physical_table_route = physical_table_route.into_physical_table_route();
451                Ok((physical_table_id, physical_table_route))
452            }
453        }
454    }
455
456    /// Returns the [TableRouteValue::Physical] recursively.
457    ///
458    /// Returns a [TableRouteNotFound](crate::error::Error::TableRouteNotFound) Error if:
459    /// - one of the logical tables corresponding to the physical table does not exist.
460    ///
461    /// **Notes**: it may return a subset of `logical_or_physical_table_ids`.
462    pub async fn batch_get_physical_table_routes(
463        &self,
464        logical_or_physical_table_ids: &[TableId],
465    ) -> Result<HashMap<TableId, PhysicalTableRouteValue>> {
466        let routes =
467            Self::resolve_physical_table_routes(logical_or_physical_table_ids, |ids| async move {
468                self.storage.batch_get(&ids).await
469            })
470            .await?;
471        Ok(routes
472            .into_iter()
473            .map(|(id, route)| (id, Arc::unwrap_or_clone(route)))
474            .collect())
475    }
476
477    /// Resolves logical references using the caller's ordered route loader.
478    /// Logical tables backed by the same physical table share its route.
479    pub async fn resolve_physical_table_routes<R, Fut>(
480        ids: &[TableId],
481        load: impl Fn(Vec<TableId>) -> Fut,
482    ) -> Result<HashMap<TableId, Arc<PhysicalTableRouteValue>>>
483    where
484        R: Into<TableRoute>,
485        Fut: Future<Output = Result<Vec<Option<R>>>>,
486    {
487        let requested = Self::index_routes(ids, load(ids.to_vec()).await?);
488        let missing_targets = requested
489            .values()
490            .filter_map(|route| match route {
491                TableRoute::Logical(logical)
492                    if !requested.contains_key(&logical.physical_table_id()) =>
493                {
494                    Some(logical.physical_table_id())
495                }
496                _ => None,
497            })
498            .collect::<HashSet<_>>();
499
500        let targets = if missing_targets.is_empty() {
501            HashMap::new()
502        } else {
503            let ids = missing_targets.into_iter().collect::<Vec<_>>();
504            let routes = load(ids.clone()).await?;
505            Self::index_routes(&ids, routes)
506        };
507
508        let mut resolved = HashMap::with_capacity(requested.len());
509        for (&id, route) in &requested {
510            let physical = match route {
511                TableRoute::Physical(physical) => physical,
512                TableRoute::Logical(logical) => {
513                    let table_id = logical.physical_table_id();
514                    let target = requested
515                        .get(&table_id)
516                        .or_else(|| targets.get(&table_id))
517                        .context(TableRouteNotFoundSnafu { table_id })?;
518                    target.as_physical_table_route_ref().with_context(|| {
519                        MetadataCorruptionSnafu {
520                            err_msg: format!(
521                                "logical table {id} points to a logical route {table_id}."
522                            ),
523                        }
524                    })?
525                }
526            };
527            resolved.insert(id, physical.clone());
528        }
529        Ok(resolved)
530    }
531
532    fn index_routes<R: Into<TableRoute>>(
533        ids: &[TableId],
534        routes: Vec<Option<R>>,
535    ) -> HashMap<TableId, TableRoute> {
536        ids.iter()
537            .copied()
538            .zip(routes)
539            .filter_map(|(id, route)| route.map(|route| (id, route.into())))
540            .collect()
541    }
542
543    /// Returns [`RegionDistribution`] of the table(`table_id`).
544    pub async fn get_region_distribution(
545        &self,
546        table_id: TableId,
547    ) -> Result<Option<RegionDistribution>> {
548        self.storage
549            .get(table_id)
550            .await?
551            .map(|table_route| Ok(region_distribution(table_route.region_routes()?)))
552            .transpose()
553    }
554
555    /// Reserves a positive number of consecutive region numbers without publishing their routes.
556    /// On success, advances the high-water mark and route version and returns the first reserved number.
557    /// If the table route changed, returns `None`. Reload the route before retrying.
558    pub async fn try_reserve_region_numbers(
559        &self,
560        table_id: TableId,
561        current: &DeserializedValueWithBytes<TableRouteValue>,
562        count: usize,
563    ) -> Result<Option<RegionNumber>> {
564        // The read path remaps peer addresses. Preserve the stored routes verbatim.
565        let TableRouteValue::Physical(mut route) =
566            TableRouteValue::try_from_raw_value(&current.bytes)?
567        else {
568            return UnexpectedLogicalRouteTableSnafu {
569                err_msg: format!("Cannot reserve region numbers for logical table {table_id}"),
570            }
571            .fail();
572        };
573        let previous = route.max_region_number;
574        let last = RegionNumber::try_from(count)
575            .ok()
576            .and_then(|count| previous.checked_add(count))
577            .filter(|last| previous < *last && *last <= MAX_REGION_SEQ)
578            .with_context(|| InvalidMetadataSnafu {
579                err_msg: format!(
580                    "Cannot reserve {count} region numbers after {previous} for table {table_id}"
581                ),
582            })?;
583        route.max_region_number = last;
584        route.version += 1;
585        let (txn, _) =
586            self.storage
587                .build_update_txn(table_id, current, &TableRouteValue::Physical(route))?;
588        let response = self.storage.kv_backend.txn(txn).await?;
589        Ok(response.succeeded.then_some(previous + 1))
590    }
591
592    /// Sets the staging state for a specific region.
593    ///
594    /// Returns a [TableRouteNotFound](crate::error::Error::TableRouteNotFound) Error if:
595    /// - the table does not exist
596    /// - the region is not found in the table
597    pub async fn set_region_staging_state(
598        &self,
599        region_id: store_api::storage::RegionId,
600        staging: bool,
601    ) -> Result<()> {
602        let table_id = region_id.table_id();
603
604        // Get current table route with raw bytes for CAS operation
605        let current_table_route = self
606            .storage
607            .get_with_raw_bytes(table_id)
608            .await?
609            .context(TableRouteNotFoundSnafu { table_id })?;
610
611        // Clone the current route value and update the specific region
612        let new_table_route = current_table_route.inner.clone();
613
614        // Only physical tables have region routes
615        ensure!(
616            new_table_route.is_physical(),
617            UnexpectedLogicalRouteTableSnafu {
618                err_msg: format!("Cannot set staging state for logical table {table_id}"),
619            }
620        );
621
622        let region_routes = new_table_route.region_routes()?.clone();
623        let mut updated_routes = region_routes.clone();
624
625        // Find and update the specific region
626        // TODO(ruihang): maybe update them in one transaction
627        let mut region_found = false;
628        for route in &mut updated_routes {
629            if route.region.id == region_id {
630                if staging {
631                    route.set_leader_staging();
632                } else {
633                    route.clear_leader_staging();
634                }
635                region_found = true;
636                break;
637            }
638        }
639
640        ensure!(region_found, RegionNotFoundSnafu { region_id });
641
642        // Create new table route with updated region routes
643        let updated_table_route = new_table_route.update(updated_routes)?;
644
645        // Execute atomic update
646        let (txn, _) =
647            self.storage
648                .build_update_txn(table_id, &current_table_route, &updated_table_route)?;
649
650        let result = self.storage.kv_backend.txn(txn).await?;
651
652        ensure!(
653            result.succeeded,
654            MetadataCorruptionSnafu {
655                err_msg: format!(
656                    "Failed to update staging state for region {}: CAS operation failed",
657                    region_id
658                ),
659            }
660        );
661
662        Ok(())
663    }
664
665    /// Checks if a specific region is in staging state.
666    ///
667    /// Returns false if the table/region doesn't exist.
668    pub async fn is_region_staging(&self, region_id: store_api::storage::RegionId) -> Result<bool> {
669        let table_id = region_id.table_id();
670
671        let table_route = self.storage.get(table_id).await?;
672
673        match table_route {
674            Some(route) if route.is_physical() => {
675                let region_routes = route.region_routes()?;
676                for route in region_routes {
677                    if route.region.id == region_id {
678                        return Ok(route.is_leader_staging());
679                    }
680                }
681                Ok(false)
682            }
683            _ => Ok(false),
684        }
685    }
686
687    /// Returns low-level APIs.
688    pub fn table_route_storage(&self) -> &TableRouteStorage {
689        &self.storage
690    }
691}
692
693/// Low-level operations of [TableRouteValue].
694pub struct TableRouteStorage {
695    kv_backend: KvBackendRef,
696}
697
698pub type TableRouteValueDecodeResult = Result<Option<DeserializedValueWithBytes<TableRouteValue>>>;
699
700impl TableRouteStorage {
701    pub fn new(kv_backend: KvBackendRef) -> Self {
702        Self { kv_backend }
703    }
704
705    /// Builds a create table route transaction,
706    /// it expected the `__table_route/{table_id}` wasn't occupied.
707    pub fn build_create_txn(
708        &self,
709        table_id: TableId,
710        table_route_value: &TableRouteValue,
711    ) -> Result<(
712        Txn,
713        impl FnOnce(&mut TxnOpGetResponseSet) -> TableRouteValueDecodeResult + use<>,
714    )> {
715        let key = TableRouteKey::new(table_id);
716        let raw_key = key.to_bytes();
717
718        let txn = Txn::put_if_not_exists(raw_key.clone(), table_route_value.try_as_raw_value()?);
719
720        Ok((
721            txn,
722            TxnOpGetResponseSet::decode_with(TxnOpGetResponseSet::filter(raw_key)),
723        ))
724    }
725
726    // TODO(LFC): restore its original visibility after some test utility codes are refined
727    /// Builds a update table route transaction,
728    /// it expected the remote value equals the `current_table_route_value`.
729    /// It retrieves the latest value if the comparing failed.
730    pub fn build_update_txn(
731        &self,
732        table_id: TableId,
733        current_table_route_value: &DeserializedValueWithBytes<TableRouteValue>,
734        new_table_route_value: &TableRouteValue,
735    ) -> Result<(
736        Txn,
737        impl FnOnce(&mut TxnOpGetResponseSet) -> TableRouteValueDecodeResult + use<>,
738    )> {
739        let key = TableRouteKey::new(table_id);
740        let raw_key = key.to_bytes();
741        let raw_value = current_table_route_value.get_raw_bytes();
742        let new_raw_value: Vec<u8> = new_table_route_value.try_as_raw_value()?;
743
744        let txn = Txn::compare_and_put(raw_key.clone(), raw_value, new_raw_value);
745
746        Ok((
747            txn,
748            TxnOpGetResponseSet::decode_with(TxnOpGetResponseSet::filter(raw_key)),
749        ))
750    }
751
752    /// Returns the [`TableRouteValue`].
753    pub async fn get(&self, table_id: TableId) -> Result<Option<TableRouteValue>> {
754        let mut table_route = self.get_inner(table_id).await?;
755        if let Some(table_route) = &mut table_route {
756            self.remap_table_route(table_route).await?;
757        };
758
759        Ok(table_route)
760    }
761
762    async fn get_inner(&self, table_id: TableId) -> Result<Option<TableRouteValue>> {
763        let key = TableRouteKey::new(table_id);
764        self.kv_backend
765            .get(&key.to_bytes())
766            .await?
767            .map(|kv| TableRouteValue::try_from_raw_value(&kv.value))
768            .transpose()
769    }
770
771    /// Returns the [`TableRouteValue`] wrapped with [`DeserializedValueWithBytes`].
772    pub async fn get_with_raw_bytes(
773        &self,
774        table_id: TableId,
775    ) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>> {
776        let mut table_route = self.get_with_raw_bytes_inner(table_id).await?;
777        if let Some(table_route) = &mut table_route {
778            self.remap_table_route(table_route).await?;
779        };
780
781        Ok(table_route)
782    }
783
784    async fn get_with_raw_bytes_inner(
785        &self,
786        table_id: TableId,
787    ) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>> {
788        let key = TableRouteKey::new(table_id);
789        self.kv_backend
790            .get(&key.to_bytes())
791            .await?
792            .map(|kv| DeserializedValueWithBytes::from_inner_slice(&kv.value))
793            .transpose()
794    }
795
796    /// Returns batch of [`TableRouteValue`] that respects the order of `table_ids`.
797    pub async fn batch_get(&self, table_ids: &[TableId]) -> Result<Vec<Option<TableRouteValue>>> {
798        let raw_table_routes = self.batch_get_with_raw_bytes(table_ids).await?;
799
800        Ok(raw_table_routes
801            .into_iter()
802            .map(|v| v.map(|x| x.inner))
803            .collect())
804    }
805
806    /// Returns batch of [`TableRouteValue`] wrapped with [`DeserializedValueWithBytes`].
807    ///
808    /// The return value is a vector of [`Option<DeserializedValueWithBytes<TableRouteValue>>`].
809    /// Note: This method remaps the addresses of the table routes, but does not update their raw byte representations.
810    pub async fn batch_get_with_raw_bytes(
811        &self,
812        table_ids: &[TableId],
813    ) -> Result<Vec<Option<DeserializedValueWithBytes<TableRouteValue>>>> {
814        let mut raw_table_routes = self.batch_get_inner(table_ids).await?;
815        self.remap_routes_addresses(&mut raw_table_routes).await?;
816
817        Ok(raw_table_routes)
818    }
819
820    async fn batch_get_inner(
821        &self,
822        table_ids: &[TableId],
823    ) -> Result<Vec<Option<DeserializedValueWithBytes<TableRouteValue>>>> {
824        let keys = table_ids
825            .iter()
826            .map(|id| TableRouteKey::new(*id).to_bytes())
827            .collect::<Vec<_>>();
828        let resp = self
829            .kv_backend
830            .batch_get(BatchGetRequest { keys: keys.clone() })
831            .await?;
832
833        let kvs = resp
834            .kvs
835            .into_iter()
836            .map(|kv| (kv.key, kv.value))
837            .collect::<HashMap<_, _>>();
838        keys.into_iter()
839            .map(|key| {
840                if let Some(value) = kvs.get(&key) {
841                    Ok(Some(DeserializedValueWithBytes::from_inner_slice(value)?))
842                } else {
843                    Ok(None)
844                }
845            })
846            .collect()
847    }
848
849    async fn remap_routes_addresses(
850        &self,
851        table_routes: &mut [Option<DeserializedValueWithBytes<TableRouteValue>>],
852    ) -> Result<()> {
853        let keys = table_routes
854            .iter()
855            .flat_map(|table_route| {
856                table_route
857                    .as_ref()
858                    .map(|x| extract_address_keys(&x.inner))
859                    .unwrap_or_default()
860            })
861            .collect::<HashSet<_>>()
862            .into_iter()
863            .collect();
864        let node_addrs = self.get_node_addresses(keys).await?;
865        for table_route in table_routes.iter_mut().flatten() {
866            set_addresses(&node_addrs, table_route)?;
867        }
868
869        Ok(())
870    }
871
872    pub(crate) async fn remap_table_route(&self, table_route: &mut TableRouteValue) -> Result<()> {
873        let keys = extract_address_keys(table_route).into_iter().collect();
874        let node_addrs = self.get_node_addresses(keys).await?;
875        set_addresses(&node_addrs, table_route)?;
876
877        Ok(())
878    }
879
880    pub(crate) async fn remap_region_routes(
881        &self,
882        region_routes: &mut [RegionRoute],
883    ) -> Result<()> {
884        let keys = extract_address_keys_from_region_routes(region_routes)
885            .into_iter()
886            .collect();
887        let node_addrs = self.get_node_addresses(keys).await?;
888        set_addresses_for_region_routes(&node_addrs, region_routes)
889    }
890
891    async fn get_node_addresses(
892        &self,
893        keys: Vec<Vec<u8>>,
894    ) -> Result<HashMap<u64, NodeAddressValue>> {
895        if keys.is_empty() {
896            return Ok(HashMap::default());
897        }
898
899        self.kv_backend
900            .batch_get(BatchGetRequest { keys })
901            .await?
902            .kvs
903            .into_iter()
904            .map(|kv| {
905                let node_id = NodeAddressKey::from_bytes(&kv.key)?.node_id;
906                let node_addr = NodeAddressValue::try_from_raw_value(&kv.value)?;
907                Ok((node_id, node_addr))
908            })
909            .collect()
910    }
911}
912
913fn set_addresses_for_region_routes(
914    node_addrs: &HashMap<u64, NodeAddressValue>,
915    region_routes: &mut [RegionRoute],
916) -> Result<()> {
917    for region_route in region_routes {
918        if let Some(leader) = &mut region_route.leader_peer
919            && let Some(node_addr) = node_addrs.get(&leader.id)
920        {
921            leader.addr = node_addr.peer.addr.clone();
922        }
923        for follower in &mut region_route.follower_peers {
924            if let Some(node_addr) = node_addrs.get(&follower.id) {
925                follower.addr = node_addr.peer.addr.clone();
926            }
927        }
928    }
929
930    Ok(())
931}
932
933fn set_addresses(
934    node_addrs: &HashMap<u64, NodeAddressValue>,
935    table_route: &mut TableRouteValue,
936) -> Result<()> {
937    let TableRouteValue::Physical(physical_table_route) = table_route else {
938        return Ok(());
939    };
940    set_addresses_for_region_routes(node_addrs, &mut physical_table_route.region_routes)
941}
942
943fn extract_address_keys_from_region_routes(region_routes: &[RegionRoute]) -> HashSet<Vec<u8>> {
944    region_routes
945        .iter()
946        .flat_map(|region_route| {
947            region_route
948                .follower_peers
949                .iter()
950                .map(|peer| NodeAddressKey::with_datanode(peer.id).to_bytes())
951                .chain(
952                    region_route
953                        .leader_peer
954                        .as_ref()
955                        .map(|leader| NodeAddressKey::with_datanode(leader.id).to_bytes()),
956                )
957        })
958        .collect()
959}
960
961fn extract_address_keys(table_route: &TableRouteValue) -> HashSet<Vec<u8>> {
962    let TableRouteValue::Physical(physical_table_route) = table_route else {
963        return HashSet::default();
964    };
965
966    extract_address_keys_from_region_routes(&physical_table_route.region_routes)
967}
968
969#[cfg(test)]
970mod tests {
971    use std::sync::Arc;
972
973    use super::*;
974    use crate::kv_backend::memory::MemoryKvBackend;
975    use crate::kv_backend::{KvBackend, TxnService};
976    use crate::peer::Peer;
977    use crate::rpc::router::Region;
978    use crate::rpc::store::PutRequest;
979
980    #[test]
981    fn test_update_table_route_max_region_number() {
982        let table_route = PhysicalTableRouteValue::new(vec![
983            RegionRoute {
984                region: Region {
985                    id: RegionId::new(0, 1),
986                    ..Default::default()
987                },
988                ..Default::default()
989            },
990            RegionRoute {
991                region: Region {
992                    id: RegionId::new(0, 2),
993                    ..Default::default()
994                },
995                ..Default::default()
996            },
997        ]);
998        assert_eq!(table_route.max_region_number, 2);
999
1000        // Shouldn't change the max region number.
1001        let new_table_route = TableRouteValue::Physical(table_route)
1002            .update(vec![RegionRoute {
1003                region: Region {
1004                    id: RegionId::new(0, 1),
1005                    ..Default::default()
1006                },
1007                ..Default::default()
1008            }])
1009            .unwrap();
1010        assert_eq!(
1011            new_table_route
1012                .as_physical_table_route_ref()
1013                .max_region_number,
1014            2
1015        );
1016
1017        // Should increase the max region number.
1018        let new_table_route = new_table_route
1019            .update(vec![RegionRoute {
1020                region: Region {
1021                    id: RegionId::new(0, 3),
1022                    ..Default::default()
1023                },
1024                ..Default::default()
1025            }])
1026            .unwrap()
1027            .into_physical_table_route();
1028        assert_eq!(new_table_route.max_region_number, 3);
1029    }
1030
1031    #[tokio::test]
1032    async fn test_reserve_region_numbers() {
1033        let table_id = 1024;
1034        let kv = Arc::new(MemoryKvBackend::default());
1035        let manager = TableRouteManager::new(kv.clone());
1036        let storage = manager.table_route_storage();
1037        let routes = vec![RegionRoute {
1038            region: Region {
1039                id: RegionId::new(table_id, MAX_REGION_SEQ - 3),
1040                ..Default::default()
1041            },
1042            ..Default::default()
1043        }];
1044        let initial = TableRouteValue::physical(routes.clone());
1045        let (txn, _) = storage.build_create_txn(table_id, &initial).unwrap();
1046        assert!(kv.txn(txn).await.unwrap().succeeded);
1047        let current = storage.get_with_raw_bytes(table_id).await.unwrap().unwrap();
1048        assert!(
1049            manager
1050                .try_reserve_region_numbers(table_id, &current, 0)
1051                .await
1052                .is_err()
1053        );
1054        assert_eq!(
1055            manager
1056                .try_reserve_region_numbers(table_id, &current, 2)
1057                .await
1058                .unwrap(),
1059            Some(MAX_REGION_SEQ - 2)
1060        );
1061        // Another reservation based on the same snapshot must not claim this range.
1062        assert_eq!(
1063            manager
1064                .try_reserve_region_numbers(table_id, &current, 2)
1065                .await
1066                .unwrap(),
1067            None
1068        );
1069        let current = storage.get_with_raw_bytes(table_id).await.unwrap().unwrap();
1070        assert_eq!(current.region_routes().unwrap(), &routes);
1071        assert_eq!(current.max_region_number().unwrap(), MAX_REGION_SEQ - 1);
1072        assert_eq!(current.version().unwrap(), 1);
1073        assert!(
1074            manager
1075                .try_reserve_region_numbers(table_id, &current, 2)
1076                .await
1077                .is_err()
1078        );
1079        assert_eq!(
1080            manager
1081                .try_reserve_region_numbers(table_id, &current, 1)
1082                .await
1083                .unwrap(),
1084            Some(MAX_REGION_SEQ)
1085        );
1086        let current = storage.get_with_raw_bytes(table_id).await.unwrap().unwrap();
1087        assert!(
1088            manager
1089                .try_reserve_region_numbers(table_id, &current, 1)
1090                .await
1091                .is_err()
1092        );
1093    }
1094
1095    #[test]
1096    fn test_table_route_compatibility() {
1097        let old_raw_v = r#"{"region_routes":[{"region":{"id":1,"name":"r1","partition":null,"attrs":{}},"leader_peer":{"id":2,"addr":"a2"},"follower_peers":[]},{"region":{"id":1,"name":"r1","partition":null,"attrs":{}},"leader_peer":{"id":2,"addr":"a2"},"follower_peers":[]}],"version":0}"#;
1098        let v = TableRouteValue::try_from_raw_value(old_raw_v.as_bytes()).unwrap();
1099
1100        let expected_table_route = TableRouteValue::Physical(PhysicalTableRouteValue {
1101            region_routes: vec![
1102                RegionRoute {
1103                    region: Region {
1104                        id: RegionId::new(0, 1),
1105                        name: "r1".to_string(),
1106                        attrs: Default::default(),
1107                        partition_expr: Default::default(),
1108                    },
1109                    leader_peer: Some(Peer {
1110                        id: 2,
1111                        addr: "a2".to_string(),
1112                    }),
1113                    follower_peers: vec![],
1114                    leader_state: None,
1115                    leader_down_since: None,
1116                    write_route_policy: None,
1117                },
1118                RegionRoute {
1119                    region: Region {
1120                        id: RegionId::new(0, 1),
1121                        name: "r1".to_string(),
1122                        attrs: Default::default(),
1123                        partition_expr: Default::default(),
1124                    },
1125                    leader_peer: Some(Peer {
1126                        id: 2,
1127                        addr: "a2".to_string(),
1128                    }),
1129                    follower_peers: vec![],
1130                    leader_state: None,
1131                    leader_down_since: None,
1132                    write_route_policy: None,
1133                },
1134            ],
1135            max_region_number: 1,
1136            version: 0,
1137        });
1138
1139        assert_eq!(v, expected_table_route);
1140    }
1141
1142    #[test]
1143    fn test_key_serialization() {
1144        let key = TableRouteKey::new(42);
1145        let raw_key = key.to_bytes();
1146        assert_eq!(raw_key, b"__table_route/42");
1147    }
1148
1149    #[test]
1150    fn test_key_deserialization() {
1151        let expected = TableRouteKey::new(42);
1152        let key = TableRouteKey::from_bytes(b"__table_route/42").unwrap();
1153        assert_eq!(key, expected);
1154    }
1155
1156    #[tokio::test]
1157    async fn test_table_route_storage_get_with_raw_bytes_empty() {
1158        let kv = Arc::new(MemoryKvBackend::default());
1159        let table_route_storage = TableRouteStorage::new(kv);
1160        let table_route = table_route_storage.get_with_raw_bytes(1024).await.unwrap();
1161        assert!(table_route.is_none());
1162    }
1163
1164    #[tokio::test]
1165    async fn test_table_route_storage_get_with_raw_bytes() {
1166        let kv = Arc::new(MemoryKvBackend::default());
1167        let table_route_storage = TableRouteStorage::new(kv.clone());
1168        let table_route = table_route_storage.get_with_raw_bytes(1024).await.unwrap();
1169        assert!(table_route.is_none());
1170        let table_route_manager = TableRouteManager::new(kv.clone());
1171        let table_route_value = TableRouteValue::Logical(LogicalTableRouteValue {
1172            physical_table_id: 1023,
1173        });
1174        let (txn, _) = table_route_manager
1175            .table_route_storage()
1176            .build_create_txn(1024, &table_route_value)
1177            .unwrap();
1178        let r = kv.txn(txn).await.unwrap();
1179        assert!(r.succeeded);
1180        let table_route = table_route_storage.get_with_raw_bytes(1024).await.unwrap();
1181        assert!(table_route.is_some());
1182        let got = table_route.unwrap().inner;
1183        assert_eq!(got, table_route_value);
1184    }
1185
1186    #[tokio::test]
1187    async fn test_table_route_batch_get() {
1188        let kv = Arc::new(MemoryKvBackend::default());
1189        let table_route_storage = TableRouteStorage::new(kv.clone());
1190        let routes = table_route_storage
1191            .batch_get(&[1023, 1024, 1025])
1192            .await
1193            .unwrap();
1194
1195        assert!(routes.iter().all(Option::is_none));
1196        let table_route_manager = TableRouteManager::new(kv.clone());
1197        let routes = [
1198            (
1199                1024,
1200                TableRouteValue::Logical(LogicalTableRouteValue {
1201                    physical_table_id: 1023,
1202                }),
1203            ),
1204            (
1205                1025,
1206                TableRouteValue::Logical(LogicalTableRouteValue {
1207                    physical_table_id: 1023,
1208                }),
1209            ),
1210        ];
1211        for (table_id, route) in &routes {
1212            let (txn, _) = table_route_manager
1213                .table_route_storage()
1214                .build_create_txn(*table_id, route)
1215                .unwrap();
1216            let r = kv.txn(txn).await.unwrap();
1217            assert!(r.succeeded);
1218        }
1219
1220        let results = table_route_storage
1221            .batch_get(&[9999, 1025, 8888, 1024])
1222            .await
1223            .unwrap();
1224        assert!(results[0].is_none());
1225        assert_eq!(results[1].as_ref().unwrap(), &routes[1].1);
1226        assert!(results[2].is_none());
1227        assert_eq!(results[3].as_ref().unwrap(), &routes[0].1);
1228    }
1229
1230    #[tokio::test]
1231    async fn test_batch_resolve_logical_routes() {
1232        let routes = HashMap::from([
1233            (1, TableRouteValue::logical(3)),
1234            (2, TableRouteValue::logical(3)),
1235            (3, TableRouteValue::physical(vec![])),
1236            (4, TableRouteValue::logical(5)),
1237        ]);
1238        let load = |ids: Vec<TableId>| {
1239            std::future::ready(Ok(ids.iter().map(|id| routes.get(id).cloned()).collect()))
1240        };
1241        let resolved = TableRouteManager::resolve_physical_table_routes(&[1, 2, 3, 9], load)
1242            .await
1243            .unwrap();
1244        assert_eq!(
1245            resolved.keys().copied().collect::<HashSet<_>>(),
1246            HashSet::from([1, 2, 3])
1247        );
1248        assert!(Arc::ptr_eq(&resolved[&1], &resolved[&3]));
1249        assert!(Arc::ptr_eq(&resolved[&2], &resolved[&3]));
1250        let logical_only = TableRouteManager::resolve_physical_table_routes(&[1, 2], load)
1251            .await
1252            .unwrap();
1253        assert_eq!(logical_only.len(), 2);
1254        assert!(Arc::ptr_eq(&logical_only[&1], &logical_only[&2]));
1255        let missing = TableRouteManager::resolve_physical_table_routes(&[4], load)
1256            .await
1257            .unwrap_err();
1258        assert!(matches!(
1259            missing,
1260            crate::error::Error::TableRouteNotFound { table_id: 5, .. }
1261        ));
1262    }
1263
1264    #[tokio::test]
1265    async fn test_table_route_cache_batches_backend_reads() {
1266        use crate::cache::new_table_route_cache;
1267        use crate::cache::test_util::assert_batch_cache_reads;
1268
1269        assert_batch_cache_reads(new_table_route_cache, |id| {
1270            let value = TableRouteValue::logical(id + 100);
1271            (
1272                TableRouteKey::new(id).to_bytes(),
1273                value.try_as_raw_value().unwrap(),
1274            )
1275        })
1276        .await;
1277    }
1278
1279    #[tokio::test]
1280    async fn test_table_route_batch_get_remaps_addresses() {
1281        let kv = Arc::new(MemoryKvBackend::default());
1282        let table_route_storage = TableRouteStorage::new(kv.clone());
1283        let table_route_manager = TableRouteManager::new(kv.clone());
1284        let table_route_value = TableRouteValue::Physical(PhysicalTableRouteValue {
1285            region_routes: vec![RegionRoute {
1286                leader_peer: Some(Peer {
1287                    id: 1,
1288                    addr: "old-leader".to_string(),
1289                }),
1290                follower_peers: vec![Peer {
1291                    id: 2,
1292                    addr: "old-follower".to_string(),
1293                }],
1294                ..Default::default()
1295            }],
1296            max_region_number: 0,
1297            version: 0,
1298        });
1299        let (txn, _) = table_route_manager
1300            .table_route_storage()
1301            .build_create_txn(1024, &table_route_value)
1302            .unwrap();
1303        let r = kv.txn(txn).await.unwrap();
1304        assert!(r.succeeded);
1305
1306        // Information schema can warm the route cache before a peer changes its address.
1307        let cache = crate::cache::new_table_route_cache(
1308            "routes".into(),
1309            moka::future::Cache::new(8),
1310            kv.clone(),
1311        );
1312        let cached = cache.get_many(&[1024]).await.unwrap();
1313        let physical = cached[0]
1314            .as_ref()
1315            .unwrap()
1316            .as_physical_table_route_ref()
1317            .unwrap();
1318        assert_eq!(
1319            physical.region_routes[0].leader_peer.as_ref().unwrap().addr,
1320            "old-leader"
1321        );
1322
1323        for (node_id, addr) in [(1, "new-leader"), (2, "new-follower")] {
1324            kv.put(PutRequest {
1325                key: NodeAddressKey::with_datanode(node_id).to_bytes(),
1326                value: NodeAddressValue {
1327                    peer: Peer {
1328                        id: node_id,
1329                        addr: addr.to_string(),
1330                    },
1331                }
1332                .try_as_raw_value()
1333                .unwrap(),
1334                ..Default::default()
1335            })
1336            .await
1337            .unwrap();
1338        }
1339
1340        let results = table_route_storage.batch_get(&[1024]).await.unwrap();
1341        let TableRouteValue::Physical(physical_table_route) = results[0].as_ref().unwrap() else {
1342            panic!("Expected PhysicalTableRouteValue");
1343        };
1344
1345        let region_route = &physical_table_route.region_routes[0];
1346        assert_eq!(
1347            region_route.leader_peer.as_ref().unwrap().addr,
1348            "new-leader"
1349        );
1350        assert_eq!(region_route.follower_peers[0].addr, "new-follower");
1351    }
1352
1353    #[tokio::test]
1354    async fn remap_route_address_updates_addresses() {
1355        let kv = Arc::new(MemoryKvBackend::default());
1356        let table_route_storage = TableRouteStorage::new(kv.clone());
1357        let mut table_route = TableRouteValue::Physical(PhysicalTableRouteValue {
1358            region_routes: vec![RegionRoute {
1359                leader_peer: Some(Peer {
1360                    id: 1,
1361                    ..Default::default()
1362                }),
1363                follower_peers: vec![Peer {
1364                    id: 2,
1365                    ..Default::default()
1366                }],
1367                ..Default::default()
1368            }],
1369            max_region_number: 0,
1370            version: 0,
1371        });
1372
1373        kv.put(PutRequest {
1374            key: NodeAddressKey::with_datanode(1).to_bytes(),
1375            value: NodeAddressValue {
1376                peer: Peer {
1377                    addr: "addr1".to_string(),
1378                    ..Default::default()
1379                },
1380            }
1381            .try_as_raw_value()
1382            .unwrap(),
1383            ..Default::default()
1384        })
1385        .await
1386        .unwrap();
1387
1388        table_route_storage
1389            .remap_table_route(&mut table_route)
1390            .await
1391            .unwrap();
1392
1393        if let TableRouteValue::Physical(physical_table_route) = table_route {
1394            assert_eq!(
1395                physical_table_route.region_routes[0]
1396                    .leader_peer
1397                    .as_ref()
1398                    .unwrap()
1399                    .addr,
1400                "addr1"
1401            );
1402            assert_eq!(
1403                physical_table_route.region_routes[0].follower_peers[0].addr,
1404                ""
1405            );
1406        } else {
1407            panic!("Expected PhysicalTableRouteValue");
1408        }
1409    }
1410}